Improve training agent progress reporting

tools/windows_training_agent.py

@@ -128,12 +128,15 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
         reader = threading.Thread(target=read_output, name="training-output-reader", daemon=True)
         reader.start()
         last_report_at = 0.0
+        started_at = time.monotonic()
+        last_output_at = started_at
         last_message = "PyTorch retrain выполняется"
         while True:
             got_line = False
             try:
                 message = output_queue.get(timeout=5)
                 got_line = True
+                last_output_at = time.monotonic()
                 log(log_path, message)
                 line_count += 1
                 if message:
@@ -144,7 +147,10 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
             progress = min(70, 8 + line_count // 3)
             now = time.monotonic()
             if got_line or now - last_report_at >= 30:
-                safe_report_progress(args, job_id, "running", "training", progress, last_message, log_path)
+                report_message = last_message
+                if not got_line:
+                    report_message = training_heartbeat_message(now, started_at, last_output_at, last_message)
+                safe_report_progress(args, job_id, "running", "training", progress, report_message, log_path)
                 last_report_at = now
 
             if process.poll() is not None and output_queue.empty():
@@ -157,6 +163,28 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
     report_progress(args, job_id, "running", "guard", 70, "Guard завершён, подготавливаю артефакты")
 
 
+def training_heartbeat_message(now: float, started_at: float, last_output_at: float, last_message: str) -> str:
+    elapsed = format_duration(now - started_at)
+    idle_seconds = max(0.0, now - last_output_at)
+    if idle_seconds >= 45:
+        return (
+            f"PyTorch обучает модель: процесс активен {elapsed}; "
+            f"последний лог {format_duration(idle_seconds)} назад: {last_message[:140]}"
+        )
+    return last_message or f"PyTorch обучает модель: процесс активен {elapsed}"
+
+
+def format_duration(seconds: float) -> str:
+    total_seconds = max(0, int(seconds))
+    minutes, seconds_part = divmod(total_seconds, 60)
+    hours, minutes_part = divmod(minutes, 60)
+    if hours:
+        return f"{hours}ч {minutes_part}м"
+    if minutes:
+        return f"{minutes}м {seconds_part}с"
+    return f"{seconds_part}с"
+
+
 def upload_artifact(args: argparse.Namespace, job_id: str, path: Path, log_path: Path) -> None:
     digest = hashlib.sha256(path.read_bytes()).hexdigest()
     size = path.stat().st_size