Fix training agent heartbeat status

tools/windows_training_agent.py

@@ -6,8 +6,10 @@ import hashlib
 import json
 import os
 import platform
+import queue
 import subprocess
 import sys
+import threading
 import time
 from datetime import datetime
 from pathlib import Path
@@ -107,6 +109,13 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
     log(log_path, "Running retrain: " + " ".join(quote_for_log(part) for part in cmd))
     report_progress(args, job_id, "running", "training", 8, "PyTorch retrain запущен")
     line_count = 0
+    output_queue: queue.Queue[str] = queue.Queue()
+
+    def read_output() -> None:
+        assert process.stdout is not None
+        for raw_line in process.stdout:
+            output_queue.put(raw_line.rstrip())
+
     with subprocess.Popen(
         cmd,
         cwd=str(repo_root),
@@ -116,14 +125,32 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
         encoding="utf-8",
         errors="replace",
     ) as process:
-        assert process.stdout is not None
-        for line in process.stdout:
-            message = line.rstrip()
-            log(log_path, message)
-            line_count += 1
-            if line_count == 1 or line_count % 12 == 0:
-                progress = min(70, 8 + line_count // 3)
-                report_progress(args, job_id, "running", "training", progress, message[-220:])
+        reader = threading.Thread(target=read_output, name="training-output-reader", daemon=True)
+        reader.start()
+        last_report_at = 0.0
+        last_message = "PyTorch retrain выполняется"
+        while True:
+            got_line = False
+            try:
+                message = output_queue.get(timeout=5)
+                got_line = True
+                log(log_path, message)
+                line_count += 1
+                if message:
+                    last_message = message[-220:]
+            except queue.Empty:
+                pass
+
+            progress = min(70, 8 + line_count // 3)
+            now = time.monotonic()
+            if got_line or now - last_report_at >= 30:
+                safe_report_progress(args, job_id, "running", "training", progress, last_message, log_path)
+                last_report_at = now
+
+            if process.poll() is not None and output_queue.empty():
+                break
+
+        reader.join(timeout=2)
         code = process.wait()
     if code != 0:
         raise RuntimeError(f"retrain failed with exit code {code}")
@@ -173,6 +200,21 @@ def report_progress(
     )
 
 
+def safe_report_progress(
+    args: argparse.Namespace,
+    job_id: str,
+    status: str,
+    phase: str,
+    progress_percent: int,
+    message: str,
+    log_path: Path,
+) -> None:
+    try:
+        report_progress(args, job_id, status, phase, progress_percent, message)
+    except Exception as exc:  # noqa: BLE001 - keep the local training process alive.
+        log(log_path, f"Progress report failed: {exc}")
+
+
 def api_json(args: argparse.Namespace, path: str, payload: dict[str, Any], timeout: int = 30) -> dict[str, Any]:
     url = args.api_base_url.rstrip("/") + path
     body = json.dumps(payload, ensure_ascii=False).encode("utf-8")