feat: production paper trading platform

tests/test_training_coordination.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import base64
 import hashlib
 import json
+from datetime import UTC, datetime, timedelta
 
 import pytest
 
@@ -16,6 +17,7 @@ def test_training_coordinator_claims_and_completes_job(tmp_path) -> None:
     job_id = requested["job"]["id"]
     heartbeat = coordinator.heartbeat({"worker_id": "win-1", "name": "DESKTOP-TMFDL0H"})
     claimed = coordinator.claim({"worker_id": "win-1", "name": "DESKTOP-TMFDL0H"})
+    lease_token = claimed["lease_token"]
 
     assert requested["queued"] is True
     assert heartbeat["status"]["agent_online"] is True
@@ -25,14 +27,23 @@ def test_training_coordinator_claims_and_completes_job(tmp_path) -> None:
 
     progress = coordinator.progress(
         job_id,
-        {"status": "running", "phase": "training", "progress_percent": 42, "message": "epoch 1"},
+        {
+            "status": "running",
+            "phase": "training",
+            "progress_percent": 42,
+            "message": "epoch 1",
+            "lease_token": lease_token,
+        },
     )
 
     assert progress["job"]["phase"] == "training"
     assert progress["job"]["progress_percent"] == 42
     assert coordinator.status()["active_job"]["message"] == "epoch 1"
 
-    completed = coordinator.complete(job_id, {"success": True, "message": "ok"})
+    completed = coordinator.complete(
+        job_id,
+        {"success": True, "message": "ok", "lease_token": lease_token},
+    )
 
     assert completed["job"]["status"] == "completed"
     assert coordinator.status()["active_job"] is None
@@ -104,7 +115,7 @@ def test_training_coordinator_reports_worker_identity_from_heartbeat(tmp_path) -
 def test_training_coordinator_records_rejected_candidate_as_completed_training(tmp_path) -> None:
     coordinator = TrainingCoordinator(tmp_path)
     job = coordinator.request_retrain({"source": "android"})["job"]
-    coordinator.claim({"worker_id": "worker-1"})
+    lease_token = coordinator.claim({"worker_id": "worker-1"})["lease_token"]
 
     completed = coordinator.complete(
         job["id"],
@@ -112,6 +123,7 @@ def test_training_coordinator_records_rejected_candidate_as_completed_training(t
             "success": True,
             "message": "training completed; candidate rejected by quality gate",
             "summary": {"accepted": False, "reason": "candidate_failed_honest_validation"},
+            "lease_token": lease_token,
         },
     )
 
@@ -124,7 +136,7 @@ def test_training_coordinator_records_rejected_candidate_as_completed_training(t
 def test_training_coordinator_accepts_chunked_artifact_upload(tmp_path) -> None:
     coordinator = TrainingCoordinator(tmp_path)
     job = coordinator.request_retrain({"source": "test"})["job"]
-    coordinator.claim({"worker_id": "test-worker"})
+    lease_token = coordinator.claim({"worker_id": "test-worker"})["lease_token"]
     payload = b'{"type":"pytorch_recurrent_forecaster","symbols":{}}\n'
     sha256 = hashlib.sha256(payload).hexdigest()
     first = payload[:20]
@@ -138,6 +150,7 @@ def test_training_coordinator_accepts_chunked_artifact_upload(tmp_path) -> None:
             "total": 2,
             "sha256": sha256,
             "data_base64": base64.b64encode(first).decode("ascii"),
+            "lease_token": lease_token,
         },
     )
     part_2 = coordinator.save_artifact_chunk(
@@ -148,6 +161,7 @@ def test_training_coordinator_accepts_chunked_artifact_upload(tmp_path) -> None:
             "total": 2,
             "sha256": sha256,
             "data_base64": base64.b64encode(second).decode("ascii"),
+            "lease_token": lease_token,
         },
     )
 
@@ -199,6 +213,35 @@ def test_running_claimed_job_keeps_agent_online_when_heartbeat_is_stale(tmp_path
     assert status["agent_online"] is True
 
 
+def test_stale_training_lease_is_requeued_and_old_lease_is_rejected(tmp_path) -> None:
+    coordinator = TrainingCoordinator(tmp_path)
+    job = coordinator.request_retrain({"source": "android"})["job"]
+    first_claim = coordinator.claim({"worker_id": "worker-1"})
+
+    state_path = tmp_path / "training_coordination.json"
+    state = json.loads(state_path.read_text(encoding="utf-8"))
+    state["jobs"][0]["updated_at"] = (
+        datetime.now(UTC) - timedelta(minutes=11)
+    ).isoformat()
+    state_path.write_text(json.dumps(state), encoding="utf-8")
+
+    second_claim = coordinator.claim({"worker_id": "worker-2"})
+
+    assert second_claim["claimed"] is True
+    assert second_claim["job"]["id"] == job["id"]
+    assert second_claim["job"]["attempts"] == 2
+    assert second_claim["lease_token"] != first_claim["lease_token"]
+    with pytest.raises(ValueError, match="lease"):
+        coordinator.progress(
+            job["id"],
+            {
+                "phase": "training",
+                "progress_percent": 10,
+                "lease_token": first_claim["lease_token"],
+            },
+        )
+
+
 def test_training_upload_rejects_unknown_job(tmp_path) -> None:
     coordinator = TrainingCoordinator(tmp_path)
     payload = b"{}"
@@ -219,7 +262,7 @@ def test_training_upload_rejects_unknown_job(tmp_path) -> None:
 def test_training_bundle_promotes_only_after_successful_guard(tmp_path) -> None:
     coordinator = TrainingCoordinator(tmp_path)
     job = coordinator.request_retrain({"source": "test"})["job"]
-    coordinator.claim({"worker_id": "worker-1"})
+    lease_token = coordinator.claim({"worker_id": "worker-1"})["lease_token"]
     model = {
         "type": "pytorch_recurrent_forecaster",
         "symbols": {
@@ -260,10 +303,14 @@ def test_training_bundle_promotes_only_after_successful_guard(tmp_path) -> None:
                 "total": 1,
                 "sha256": hashlib.sha256(payload).hexdigest(),
                 "data_base64": base64.b64encode(payload).decode("ascii"),
+                "lease_token": lease_token,
             },
         )
 
-    completed = coordinator.complete(job["id"], {"success": True})
+    completed = coordinator.complete(
+        job["id"],
+        {"success": True, "lease_token": lease_token},
+    )
 
     assert completed["job"]["status"] == "completed"
     assert json.loads((tmp_path / "lstm_forecaster.json").read_text())["symbols"]["BTCUSDT"]