sevenhill/TradeBot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add orderbook shadow training pipeline

Browse Source
This commit is contained in:
Курнат Андрей
2026-07-15 09:44:29 +03:00
parent f7a625586e
commit 5d8ad1437e
19 changed files with 1486 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crypto_spot_bot/training_coordination.py
+141 -5
View File
@@ -17,11 +17,17 @@ from threading import Lock
from typing import Any
ALLOWED_TRAINING_ARTIFACTS = {
ACTIVE_TRAINING_ARTIFACTS = {
"lstm_forecaster.json",
"torch_retrain_guard.json",
"torch_threshold_calibration.json",
}
SHADOW_TRAINING_ARTIFACTS = {
"lstm_forecaster.shadow.json",
"torch_shadow_guard.json",
"torch_shadow_calibration.json",
}
ALLOWED_TRAINING_ARTIFACTS = ACTIVE_TRAINING_ARTIFACTS | SHADOW_TRAINING_ARTIFACTS
RUNNING_LEASE_TIMEOUT = timedelta(minutes=10)
ONLINE_WINDOW = timedelta(minutes=3)
MAX_JOB_ATTEMPTS = 3
@@ -29,7 +35,8 @@ MAX_ARTIFACT_CHUNK_BYTES = 1024 * 1024
# Keep uploads bounded while leaving room for explicitly requested per-symbol bundles.
MAX_ARTIFACT_BYTES = 256 * 1024 * 1024
MAX_ARTIFACT_CHUNKS = 1024
REQUIRED_MODEL_BUNDLE = set(ALLOWED_TRAINING_ARTIFACTS)
REQUIRED_MODEL_BUNDLE = set(ACTIVE_TRAINING_ARTIFACTS)
REQUIRED_SHADOW_BUNDLE = set(SHADOW_TRAINING_ARTIFACTS)
class TrainingCoordinator:
@@ -46,6 +53,61 @@ class TrainingCoordinator:
self._save_state(state)
return self._public_status(state)
def promote_shadow(self, forward_gate: dict[str, Any]) -> dict[str, Any]:
with self._lock:
if not bool(forward_gate.get("passed")):
raise ValueError("shadow forward gate has not passed")
shadow_model = self.runtime_dir / "lstm_forecaster.shadow.json"
shadow_calibration = self.runtime_dir / "torch_shadow_calibration.json"
shadow_guard = self.runtime_dir / "torch_shadow_guard.json"
missing = [
path.name
for path in (shadow_model, shadow_calibration, shadow_guard)
if not path.is_file()
]
if missing:
raise ValueError("shadow bundle is incomplete: " + ", ".join(missing))
model_sha256 = hashlib.sha256(shadow_model.read_bytes()).hexdigest()
if str(forward_gate.get("model_sha256") or "") != model_sha256:
raise ValueError("shadow forward gate is bound to another model")
calibration = _read_json(shadow_calibration)
guard = _read_json(shadow_guard)
if calibration.get("artifact_sha256") != model_sha256:
raise ValueError("shadow calibration is not bound to the model")
if not bool(guard.get("accepted")) or guard.get("candidate_artifact_sha256") != model_sha256:
raise ValueError("shadow offline guard is invalid")
promotion_id = str(uuid.uuid4())
backup_dir = self.runtime_dir / ".model_backups" / f"{_compact_now()}-shadow-{promotion_id}"
backup_dir.mkdir(parents=True, exist_ok=True)
targets = {
"lstm_forecaster.json": shadow_model,
"torch_threshold_calibration.json": shadow_calibration,
"torch_retrain_guard.json": shadow_guard,
}
for target_name in targets:
current = self.runtime_dir / target_name
if current.is_file():
shutil.copy2(current, backup_dir / target_name)
for target_name, source in targets.items():
target_tmp = self.runtime_dir / f".{target_name}.{promotion_id}.promote"
shutil.copy2(source, target_tmp)
os.replace(target_tmp, self.runtime_dir / target_name)
gate_path = self.runtime_dir / "torch_shadow_forward_gate.json"
gate_tmp = gate_path.with_suffix(".tmp")
gate_tmp.write_text(
json.dumps(forward_gate, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
os.replace(gate_tmp, gate_path)
return {
"promoted": True,
"model_sha256": model_sha256,
"promotion_id": promotion_id,
"backup_dir": str(backup_dir),
"promoted_at": _now(),
}
def request_retrain(self, payload: dict[str, Any] | None = None) -> dict[str, Any]:
payload = payload or {}
with self._lock:
@@ -238,8 +300,17 @@ class TrainingCoordinator:
self._require_lease(job, payload)
success = bool(payload.get("success", payload.get("status") == "completed"))
if success and job.get("artifacts"):
promoted = self._validate_and_promote(job_id, job)
job["promoted_artifacts"] = promoted
artifact_names = {
str(item.get("name"))
for item in job.get("artifacts", [])
if isinstance(item, dict)
}
if artifact_names & REQUIRED_SHADOW_BUNDLE:
staged = self._validate_and_stage_shadow(job_id, job)
job["shadow_artifacts"] = staged
else:
promoted = self._validate_and_promote(job_id, job)
job["promoted_artifacts"] = promoted
job["status"] = "completed" if success else "failed"
job["phase"] = "completed" if success else "failed"
job["progress_percent"] = 100 if success else _coerce_percent(payload.get("progress_percent"), job.get("progress_percent", 0))
@@ -247,7 +318,9 @@ class TrainingCoordinator:
job["message"] = str(payload.get("message") or "")
if isinstance(payload.get("summary"), dict):
job["summary"] = payload["summary"]
if isinstance(payload["summary"].get("accepted"), bool):
if str(payload["summary"].get("state") or "").startswith("collecting"):
job["model_decision"] = "collecting"
elif isinstance(payload["summary"].get("accepted"), bool):
job["model_decision"] = (
"accepted" if payload["summary"]["accepted"] else "rejected"
)
@@ -318,6 +391,60 @@ class TrainingCoordinator:
_remove_tree(self.upload_root / job_id)
return promoted
def _validate_and_stage_shadow(self, job_id: str, job: dict[str, Any]) -> list[dict[str, Any]]:
ready_dir = self.upload_root / job_id / "ready"
staged = {path.name for path in ready_dir.iterdir() if path.is_file()} if ready_dir.is_dir() else set()
missing = REQUIRED_SHADOW_BUNDLE - staged
if missing:
raise ValueError("shadow training bundle is incomplete: " + ", ".join(sorted(missing)))
model_path = ready_dir / "lstm_forecaster.shadow.json"
calibration_path = ready_dir / "torch_shadow_calibration.json"
guard_path = ready_dir / "torch_shadow_guard.json"
model = _read_json(model_path)
calibration = _read_json(calibration_path)
guard = _read_json(guard_path)
if model.get("type") != "pytorch_recurrent_forecaster":
raise ValueError("shadow candidate model type is invalid")
symbols = model.get("symbols")
if not isinstance(symbols, dict) or not symbols:
raise ValueError("shadow candidate model has no symbol models")
_validate_symbol_models(symbols)
model_sha256 = hashlib.sha256(model_path.read_bytes()).hexdigest()
if calibration.get("artifact_sha256") != model_sha256:
raise ValueError("shadow calibration is not bound to the uploaded model")
if not bool(guard.get("accepted")):
raise ValueError("shadow candidate did not pass the offline guard")
if guard.get("candidate_artifact_sha256") != model_sha256:
raise ValueError("shadow guard is not bound to the uploaded model")
validation = calibration.get("validation")
if not isinstance(validation, dict) or not _validation_passed(validation):
raise ValueError("shadow candidate offline quality gate did not pass")
if validation.get("protocol") != "untouched_model_holdout_with_threshold_walk_forward":
raise ValueError("shadow candidate validation protocol is not an untouched holdout")
self.runtime_dir.mkdir(parents=True, exist_ok=True)
artifact_rows = {
str(item.get("name")): item
for item in job.get("artifacts", [])
if isinstance(item, dict)
}
installed: list[dict[str, Any]] = []
for name in sorted(REQUIRED_SHADOW_BUNDLE):
target_tmp = self.runtime_dir / f".{name}.{job_id}.stage"
shutil.copy2(ready_dir / name, target_tmp)
os.replace(target_tmp, self.runtime_dir / name)
row = artifact_rows.get(name, {})
installed.append(
{
"name": name,
"sha256": row.get("sha256", ""),
"staged_at": _now(),
}
)
_remove_tree(self.upload_root / job_id)
return installed
def _load_state(self) -> dict[str, Any]:
try:
data = json.loads(self.state_path.read_text(encoding="utf-8"))
@@ -464,6 +591,10 @@ def _safe_parameters(value: Any) -> dict[str, Any]:
"interval",
"pooled",
"resume_candidate",
"use_orderbook",
"orderbook_min_samples_per_bucket",
"orderbook_min_covered_buckets",
"orderbook_min_symbols",
}
result = {key: value[key] for key in allowed if key in value}
for key, low, high in (
@@ -475,6 +606,9 @@ def _safe_parameters(value: Any) -> dict[str, Any]:
("horizon", 1, 96),
("patience", 1, 50),
("seed", 1, 2_147_483_647),
("orderbook_min_samples_per_bucket", 1, 5000),
("orderbook_min_covered_buckets", 96, 20000),
("orderbook_min_symbols", 1, 30),
):
if key not in result:
continue
@@ -523,6 +657,8 @@ def _safe_parameters(value: Any) -> dict[str, Any]:
result["pooled"] = result["pooled"] is True
if "resume_candidate" in result:
result["resume_candidate"] = result["resume_candidate"] is True
if "use_orderbook" in result:
result["use_orderbook"] = result["use_orderbook"] is True
return result