Clarify transient training progress upload errors
This commit is contained in:
@@ -237,10 +237,16 @@ def safe_report_progress(
|
|||||||
message: str,
|
message: str,
|
||||||
log_path: Path,
|
log_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
try:
|
last_error: Exception | None = None
|
||||||
report_progress(args, job_id, status, phase, progress_percent, message)
|
for attempt in range(1, 4):
|
||||||
except Exception as exc: # noqa: BLE001 - keep the local training process alive.
|
try:
|
||||||
log(log_path, f"Progress report failed: {exc}")
|
report_progress(args, job_id, status, phase, progress_percent, message)
|
||||||
|
return
|
||||||
|
except Exception as exc: # noqa: BLE001 - keep the local training process alive.
|
||||||
|
last_error = exc
|
||||||
|
if attempt < 3:
|
||||||
|
time.sleep(attempt * 2)
|
||||||
|
log(log_path, f"Temporary progress upload error; training continues: {last_error}")
|
||||||
|
|
||||||
|
|
||||||
def api_json(args: argparse.Namespace, path: str, payload: dict[str, Any], timeout: int = 30) -> dict[str, Any]:
|
def api_json(args: argparse.Namespace, path: str, payload: dict[str, Any], timeout: int = 30) -> dict[str, Any]:
|
||||||
|
|||||||
Reference in New Issue
Block a user