feat: production paper trading platform

tools/windows_training_agent.py

@@ -54,23 +54,34 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
         return
     job = claim.get("job") if isinstance(claim.get("job"), dict) else {}
     job_id = str(job.get("id") or "")
+    lease_token = str(claim.get("lease_token") or "")
     if not job_id:
         return
+    if not lease_token:
+        raise RuntimeError("training server did not issue a job lease")
     log(log_path, f"Claimed retrain job {job_id}")
-    report_progress(args, job_id, "running", "claimed", 2, "Задание получено Windows-agent")
+    report_progress(args, job_id, lease_token, "running", "claimed", 2, "Задание получено Windows-agent")
     success = False
     message = ""
     summary: dict[str, Any] = {}
     try:
-        run_retrain(args, job_id, job, repo_root, log_path)
+        run_retrain(args, job_id, lease_token, job, repo_root, log_path)
         summary = read_json(runtime_dir / "torch_retrain_guard.json")
         accepted = summary.get("accepted") is True
         if accepted:
-            report_progress(args, job_id, "running", "uploading", 72, "Обучение завершено, загружаю артефакты")
+            report_progress(
+                args,
+                job_id,
+                lease_token,
+                "running",
+                "uploading",
+                72,
+                "Обучение завершено, загружаю артефакты",
+            )
             for name in ARTIFACT_NAMES:
                 path = runtime_dir / name
                 if path.is_file():
-                    upload_artifact(args, job_id, path, log_path)
+                    upload_artifact(args, job_id, lease_token, path, log_path)
             message = "training completed; candidate accepted"
             log(log_path, f"Completed retrain job {job_id}; candidate accepted")
         else:
@@ -82,11 +93,23 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
         message = str(exc)
         log(log_path, f"Job {job_id} failed: {message}")
     finally:
-        payload = {"success": success, "message": message, "summary": summary}
+        payload = {
+            "success": success,
+            "message": message,
+            "summary": summary,
+            "lease_token": lease_token,
+        }
         api_json(args, f"/api/training/jobs/{job_id}/complete", payload)
 
 
-def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo_root: Path, log_path: Path) -> None:
+def run_retrain(
+    args: argparse.Namespace,
+    job_id: str,
+    lease_token: str,
+    job: dict[str, Any],
+    repo_root: Path,
+    log_path: Path,
+) -> None:
     script = repo_root / "tools" / "run_torch_retrain.ps1"
     if not script.is_file():
         raise RuntimeError(f"retrain script not found: {script}")
@@ -126,12 +149,20 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
         value = parameters.get(key)
         if value not in (None, ""):
             cmd.extend([ps_arg, str(value)])
-    if parameters.get("pooled") is True:
+    if parameters.get("pooled", True) is True:
         cmd.append("-Pooled")
     if parameters.get("resume_candidate") is True:
         cmd.append("-ResumeCandidate")
     log(log_path, "Running retrain: " + " ".join(quote_for_log(part) for part in cmd))
-    report_progress(args, job_id, "running", "training", 8, "PyTorch retrain запущен")
+    report_progress(
+        args,
+        job_id,
+        lease_token,
+        "running",
+        "training",
+        8,
+        "PyTorch retrain запущен",
+    )
     line_count = 0
     output_queue: queue.Queue[str] = queue.Queue()
 
@@ -175,7 +206,16 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
                 report_message = last_message
                 if not got_line:
                     report_message = training_heartbeat_message(now, started_at, last_output_at, last_message)
-                safe_report_progress(args, job_id, "running", "training", progress, report_message, log_path)
+                safe_report_progress(
+                    args,
+                    job_id,
+                    lease_token,
+                    "running",
+                    "training",
+                    progress,
+                    report_message,
+                    log_path,
+                )
                 last_report_at = now
 
             if process.poll() is not None and output_queue.empty():
@@ -185,7 +225,15 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
         code = process.wait()
     if code != 0:
         raise RuntimeError(f"retrain failed with exit code {code}")
-    report_progress(args, job_id, "running", "guard", 70, "Guard завершён, подготавливаю артефакты")
+    report_progress(
+        args,
+        job_id,
+        lease_token,
+        "running",
+        "guard",
+        70,
+        "Guard завершён, подготавливаю артефакты",
+    )
 
 
 def friendly_training_message(message: str) -> str:
@@ -282,7 +330,13 @@ def format_duration(seconds: float) -> str:
     return f"{seconds_part}с"
 
 
-def upload_artifact(args: argparse.Namespace, job_id: str, path: Path, log_path: Path) -> None:
+def upload_artifact(
+    args: argparse.Namespace,
+    job_id: str,
+    lease_token: str,
+    path: Path,
+    log_path: Path,
+) -> None:
     digest = hashlib.sha256(path.read_bytes()).hexdigest()
     size = path.stat().st_size
     chunk_size = max(64 * 1024, args.chunk_size)
@@ -297,16 +351,26 @@ def upload_artifact(args: argparse.Namespace, job_id: str, path: Path, log_path:
                 "total": total,
                 "sha256": digest,
                 "data_base64": base64.b64encode(data).decode("ascii"),
+                "lease_token": lease_token,
             }
             api_json(args, f"/api/training/jobs/{job_id}/artifacts/chunk", payload, timeout=120)
             if index == 0 or index == total - 1 or index % 10 == 0:
                 progress = 72 + int(((index + 1) / total) * 23)
-                report_progress(args, job_id, "running", "uploading", progress, f"Загружаю {path.name}: {index + 1}/{total}")
+                report_progress(
+                    args,
+                    job_id,
+                    lease_token,
+                    "running",
+                    "uploading",
+                    progress,
+                    f"Загружаю {path.name}: {index + 1}/{total}",
+                )
 
 
 def report_progress(
     args: argparse.Namespace,
     job_id: str,
+    lease_token: str,
     status: str,
     phase: str,
     progress_percent: int,
@@ -321,6 +385,7 @@ def report_progress(
             "progress_percent": progress_percent,
             "message": message,
             "worker": worker_payload(args, Path(args.repo_root).resolve()),
+            "lease_token": lease_token,
         },
     )
 
@@ -328,6 +393,7 @@ def report_progress(
 def safe_report_progress(
     args: argparse.Namespace,
     job_id: str,
+    lease_token: str,
     status: str,
     phase: str,
     progress_percent: int,
@@ -337,7 +403,15 @@ def safe_report_progress(
     last_error: Exception | None = None
     for attempt in range(1, 4):
         try:
-            report_progress(args, job_id, status, phase, progress_percent, message)
+            report_progress(
+                args,
+                job_id,
+                lease_token,
+                status,
+                phase,
+                progress_percent,
+                message,
+            )
             return
         except Exception as exc:  # noqa: BLE001 - keep the local training process alive.
             last_error = exc
@@ -385,7 +459,7 @@ def worker_payload(args: argparse.Namespace, repo_root: Path) -> dict[str, Any]:
         "worker_id": args.worker_id or f"{name}:{repo_root}",
         "name": name,
         "path": str(repo_root),
-        "version": "1",
+        "version": "2",
     }