diff --git a/tools/windows_training_agent.py b/tools/windows_training_agent.py index 16147be..6290c88 100644 --- a/tools/windows_training_agent.py +++ b/tools/windows_training_agent.py @@ -237,10 +237,16 @@ def safe_report_progress( message: str, log_path: Path, ) -> None: - try: - report_progress(args, job_id, status, phase, progress_percent, message) - except Exception as exc: # noqa: BLE001 - keep the local training process alive. - log(log_path, f"Progress report failed: {exc}") + last_error: Exception | None = None + for attempt in range(1, 4): + try: + report_progress(args, job_id, status, phase, progress_percent, message) + return + except Exception as exc: # noqa: BLE001 - keep the local training process alive. + last_error = exc + if attempt < 3: + time.sleep(attempt * 2) + log(log_path, f"Temporary progress upload error; training continues: {last_error}") def api_json(args: argparse.Namespace, path: str, payload: dict[str, Any], timeout: int = 30) -> dict[str, Any]: