feat: auto-queue orderbook retrain at coverage gate
This commit is contained in:
@@ -38,6 +38,7 @@ SHADOW_ARTIFACT_NAMES = (
|
||||
"torch_shadow_guard.json",
|
||||
"torch_shadow_calibration.json",
|
||||
)
|
||||
_LAST_ORDERBOOK_AUTO_CHECK = 0.0
|
||||
|
||||
|
||||
def main() -> None:
|
||||
@@ -63,6 +64,7 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
|
||||
api_json(args, "/api/training/heartbeat", worker)
|
||||
claim = api_json(args, "/api/training/claim", worker)
|
||||
if not claim.get("claimed"):
|
||||
maybe_auto_queue_orderbook(args, repo_root, runtime_dir, log_path)
|
||||
return
|
||||
job = claim.get("job") if isinstance(claim.get("job"), dict) else {}
|
||||
job_id = str(job.get("id") or "")
|
||||
@@ -352,6 +354,45 @@ def prepare_orderbook_data(
|
||||
return result
|
||||
|
||||
|
||||
def maybe_auto_queue_orderbook(
|
||||
args: argparse.Namespace,
|
||||
repo_root: Path,
|
||||
runtime_dir: Path,
|
||||
log_path: Path,
|
||||
) -> None:
|
||||
global _LAST_ORDERBOOK_AUTO_CHECK
|
||||
try:
|
||||
interval_seconds = max(
|
||||
300,
|
||||
int(os.environ.get("TORCH_ORDERBOOK_AUTO_CHECK_SECONDS", "3600") or 3600),
|
||||
)
|
||||
except ValueError:
|
||||
interval_seconds = 3600
|
||||
now = time.monotonic()
|
||||
if _LAST_ORDERBOOK_AUTO_CHECK and now - _LAST_ORDERBOOK_AUTO_CHECK < interval_seconds:
|
||||
return
|
||||
_LAST_ORDERBOOK_AUTO_CHECK = now
|
||||
marker_path = runtime_dir / "orderbook_auto_queue.json"
|
||||
if marker_path.is_file() or (runtime_dir / "lstm_forecaster.shadow.json").is_file():
|
||||
return
|
||||
status = prepare_orderbook_data(args, repo_root, {}, log_path)
|
||||
if status.get("state") != "ready":
|
||||
return
|
||||
response = api_json(args, "/api/training/retrain/auto", {})
|
||||
if not response.get("queued"):
|
||||
log(log_path, f"Automatic orderbook retrain was not queued: {response.get('reason', 'unknown')}")
|
||||
return
|
||||
marker = {
|
||||
"queued_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
"job_id": (response.get("job") or {}).get("id"),
|
||||
"coverage": status,
|
||||
}
|
||||
marker_tmp = marker_path.with_suffix(".tmp")
|
||||
marker_tmp.write_text(json.dumps(marker, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
marker_tmp.replace(marker_path)
|
||||
log(log_path, f"Automatically queued orderbook retrain job {marker['job_id']}")
|
||||
|
||||
|
||||
def friendly_training_message(message: str) -> str:
|
||||
cleaned = message.strip()
|
||||
if not cleaned:
|
||||
|
||||
Reference in New Issue
Block a user