Harden trading, training, and monitoring
This commit is contained in:
@@ -64,6 +64,11 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
|
||||
try:
|
||||
run_retrain(args, job_id, job, repo_root, log_path)
|
||||
summary = read_json(runtime_dir / "torch_retrain_guard.json")
|
||||
if summary.get("accepted") is not True:
|
||||
raise RuntimeError(
|
||||
"candidate rejected by untouched-holdout guard: "
|
||||
+ str(summary.get("reason") or "validation failed")
|
||||
)
|
||||
report_progress(args, job_id, "running", "uploading", 72, "Обучение завершено, загружаю артефакты")
|
||||
for name in ARTIFACT_NAMES:
|
||||
path = runtime_dir / name
|
||||
@@ -102,11 +107,14 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
|
||||
"layers": "-Layers",
|
||||
"dropouts": "-Dropouts",
|
||||
"epochs": "-Epochs",
|
||||
"holdout_window": "-HoldoutWindow",
|
||||
}
|
||||
for key, ps_arg in arg_map.items():
|
||||
value = parameters.get(key)
|
||||
if value not in (None, ""):
|
||||
cmd.extend([ps_arg, str(value)])
|
||||
if parameters.get("resume_candidate") is True:
|
||||
cmd.append("-ResumeCandidate")
|
||||
log(log_path, "Running retrain: " + " ".join(quote_for_log(part) for part in cmd))
|
||||
report_progress(args, job_id, "running", "training", 8, "PyTorch retrain запущен")
|
||||
line_count = 0
|
||||
|
||||
Reference in New Issue
Block a user