Clarify transient training progress upload errors
This commit is contained in:
@@ -237,10 +237,16 @@ def safe_report_progress(
|
||||
message: str,
|
||||
log_path: Path,
|
||||
) -> None:
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(1, 4):
|
||||
try:
|
||||
report_progress(args, job_id, status, phase, progress_percent, message)
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001 - keep the local training process alive.
|
||||
log(log_path, f"Progress report failed: {exc}")
|
||||
last_error = exc
|
||||
if attempt < 3:
|
||||
time.sleep(attempt * 2)
|
||||
log(log_path, f"Temporary progress upload error; training continues: {last_error}")
|
||||
|
||||
|
||||
def api_json(args: argparse.Namespace, path: str, payload: dict[str, Any], timeout: int = 30) -> dict[str, Any]:
|
||||
|
||||
Reference in New Issue
Block a user