Fix remote training and model validation pipeline

This commit is contained in:
Курнат Андрей
2026-07-12 22:56:49 +03:00
parent 18936cf8b1
commit da53483164
22 changed files with 1691 additions and 246 deletions
+17 -12
View File
@@ -64,19 +64,20 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
try:
run_retrain(args, job_id, job, repo_root, log_path)
summary = read_json(runtime_dir / "torch_retrain_guard.json")
if summary.get("accepted") is not True:
raise RuntimeError(
"candidate rejected by untouched-holdout guard: "
+ str(summary.get("reason") or "validation failed")
)
report_progress(args, job_id, "running", "uploading", 72, "Обучение завершено, загружаю артефакты")
for name in ARTIFACT_NAMES:
path = runtime_dir / name
if path.is_file():
upload_artifact(args, job_id, path, log_path)
accepted = summary.get("accepted") is True
if accepted:
report_progress(args, job_id, "running", "uploading", 72, "Обучение завершено, загружаю артефакты")
for name in ARTIFACT_NAMES:
path = runtime_dir / name
if path.is_file():
upload_artifact(args, job_id, path, log_path)
message = "training completed; candidate accepted"
log(log_path, f"Completed retrain job {job_id}; candidate accepted")
else:
reason = str(summary.get("reason") or "validation failed")
message = f"training completed; candidate rejected by quality gate: {reason}"
log(log_path, f"Completed retrain job {job_id}; candidate rejected: {reason}")
success = True
message = "training completed"
log(log_path, f"Completed retrain job {job_id}")
except Exception as exc: # noqa: BLE001 - report failure to the bot.
message = str(exc)
log(log_path, f"Job {job_id} failed: {message}")
@@ -108,6 +109,10 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
"dropouts": "-Dropouts",
"epochs": "-Epochs",
"holdout_window": "-HoldoutWindow",
"ensemble_seeds": "-EnsembleSeeds",
"selection_folds": "-SelectionFolds",
"learning_rate": "-LearningRate",
"weight_decay": "-WeightDecay",
}
for key, ps_arg in arg_map.items():
value = parameters.get(key)