feat: auto-queue orderbook retrain at coverage gate

This commit is contained in:
Курнат Андрей
2026-07-15 09:50:57 +03:00
parent 0992da0ece
commit 5082be2e5a
5 changed files with 55 additions and 1 deletions
+41
View File
@@ -38,6 +38,7 @@ SHADOW_ARTIFACT_NAMES = (
"torch_shadow_guard.json",
"torch_shadow_calibration.json",
)
_LAST_ORDERBOOK_AUTO_CHECK = 0.0
def main() -> None:
@@ -63,6 +64,7 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
api_json(args, "/api/training/heartbeat", worker)
claim = api_json(args, "/api/training/claim", worker)
if not claim.get("claimed"):
maybe_auto_queue_orderbook(args, repo_root, runtime_dir, log_path)
return
job = claim.get("job") if isinstance(claim.get("job"), dict) else {}
job_id = str(job.get("id") or "")
@@ -352,6 +354,45 @@ def prepare_orderbook_data(
return result
def maybe_auto_queue_orderbook(
args: argparse.Namespace,
repo_root: Path,
runtime_dir: Path,
log_path: Path,
) -> None:
global _LAST_ORDERBOOK_AUTO_CHECK
try:
interval_seconds = max(
300,
int(os.environ.get("TORCH_ORDERBOOK_AUTO_CHECK_SECONDS", "3600") or 3600),
)
except ValueError:
interval_seconds = 3600
now = time.monotonic()
if _LAST_ORDERBOOK_AUTO_CHECK and now - _LAST_ORDERBOOK_AUTO_CHECK < interval_seconds:
return
_LAST_ORDERBOOK_AUTO_CHECK = now
marker_path = runtime_dir / "orderbook_auto_queue.json"
if marker_path.is_file() or (runtime_dir / "lstm_forecaster.shadow.json").is_file():
return
status = prepare_orderbook_data(args, repo_root, {}, log_path)
if status.get("state") != "ready":
return
response = api_json(args, "/api/training/retrain/auto", {})
if not response.get("queued"):
log(log_path, f"Automatic orderbook retrain was not queued: {response.get('reason', 'unknown')}")
return
marker = {
"queued_at": datetime.now().astimezone().isoformat(timespec="seconds"),
"job_id": (response.get("job") or {}).get("id"),
"coverage": status,
}
marker_tmp = marker_path.with_suffix(".tmp")
marker_tmp.write_text(json.dumps(marker, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
marker_tmp.replace(marker_path)
log(log_path, f"Automatically queued orderbook retrain job {marker['job_id']}")
def friendly_training_message(message: str) -> str:
cleaned = message.strip()
if not cleaned: