Fix remote training and model validation pipeline
This commit is contained in:
@@ -64,19 +64,20 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
|
||||
try:
|
||||
run_retrain(args, job_id, job, repo_root, log_path)
|
||||
summary = read_json(runtime_dir / "torch_retrain_guard.json")
|
||||
if summary.get("accepted") is not True:
|
||||
raise RuntimeError(
|
||||
"candidate rejected by untouched-holdout guard: "
|
||||
+ str(summary.get("reason") or "validation failed")
|
||||
)
|
||||
report_progress(args, job_id, "running", "uploading", 72, "Обучение завершено, загружаю артефакты")
|
||||
for name in ARTIFACT_NAMES:
|
||||
path = runtime_dir / name
|
||||
if path.is_file():
|
||||
upload_artifact(args, job_id, path, log_path)
|
||||
accepted = summary.get("accepted") is True
|
||||
if accepted:
|
||||
report_progress(args, job_id, "running", "uploading", 72, "Обучение завершено, загружаю артефакты")
|
||||
for name in ARTIFACT_NAMES:
|
||||
path = runtime_dir / name
|
||||
if path.is_file():
|
||||
upload_artifact(args, job_id, path, log_path)
|
||||
message = "training completed; candidate accepted"
|
||||
log(log_path, f"Completed retrain job {job_id}; candidate accepted")
|
||||
else:
|
||||
reason = str(summary.get("reason") or "validation failed")
|
||||
message = f"training completed; candidate rejected by quality gate: {reason}"
|
||||
log(log_path, f"Completed retrain job {job_id}; candidate rejected: {reason}")
|
||||
success = True
|
||||
message = "training completed"
|
||||
log(log_path, f"Completed retrain job {job_id}")
|
||||
except Exception as exc: # noqa: BLE001 - report failure to the bot.
|
||||
message = str(exc)
|
||||
log(log_path, f"Job {job_id} failed: {message}")
|
||||
@@ -108,6 +109,10 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
|
||||
"dropouts": "-Dropouts",
|
||||
"epochs": "-Epochs",
|
||||
"holdout_window": "-HoldoutWindow",
|
||||
"ensemble_seeds": "-EnsembleSeeds",
|
||||
"selection_folds": "-SelectionFolds",
|
||||
"learning_rate": "-LearningRate",
|
||||
"weight_decay": "-WeightDecay",
|
||||
}
|
||||
for key, ps_arg in arg_map.items():
|
||||
value = parameters.get(key)
|
||||
|
||||
Reference in New Issue
Block a user