Clarify transient training progress upload errors

This commit is contained in:
Codex
2026-06-29 21:21:09 +03:00
parent 9f47c04235
commit 0cfd89980e
+7 -1
View File
@@ -237,10 +237,16 @@ def safe_report_progress(
message: str,
log_path: Path,
) -> None:
last_error: Exception | None = None
for attempt in range(1, 4):
try:
report_progress(args, job_id, status, phase, progress_percent, message)
return
except Exception as exc: # noqa: BLE001 - keep the local training process alive.
log(log_path, f"Progress report failed: {exc}")
last_error = exc
if attempt < 3:
time.sleep(attempt * 2)
log(log_path, f"Temporary progress upload error; training continues: {last_error}")
def api_json(args: argparse.Namespace, path: str, payload: dict[str, Any], timeout: int = 30) -> dict[str, Any]: