Harden trading, training, and monitoring

This commit is contained in:
Codex
2026-07-10 15:51:53 +03:00
parent 6fb79ee2a9
commit 069d75d2f2
55 changed files with 2658 additions and 2332049 deletions
+8
View File
@@ -64,6 +64,11 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
try:
run_retrain(args, job_id, job, repo_root, log_path)
summary = read_json(runtime_dir / "torch_retrain_guard.json")
if summary.get("accepted") is not True:
raise RuntimeError(
"candidate rejected by untouched-holdout guard: "
+ str(summary.get("reason") or "validation failed")
)
report_progress(args, job_id, "running", "uploading", 72, "Обучение завершено, загружаю артефакты")
for name in ARTIFACT_NAMES:
path = runtime_dir / name
@@ -102,11 +107,14 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
"layers": "-Layers",
"dropouts": "-Dropouts",
"epochs": "-Epochs",
"holdout_window": "-HoldoutWindow",
}
for key, ps_arg in arg_map.items():
value = parameters.get(key)
if value not in (None, ""):
cmd.extend([ps_arg, str(value)])
if parameters.get("resume_candidate") is True:
cmd.append("-ResumeCandidate")
log(log_path, "Running retrain: " + " ".join(quote_for_log(part) for part in cmd))
report_progress(args, job_id, "running", "training", 8, "PyTorch retrain запущен")
line_count = 0