Improve Windows training agent progress

This commit is contained in:
Codex
2026-07-03 20:44:09 +03:00
parent 33c3831bf2
commit 7f1ac694e4
+87 -1
View File
@@ -7,6 +7,7 @@ import json
import os
import platform
import queue
import re
import subprocess
import sys
import threading
@@ -124,6 +125,7 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
text=True,
encoding="utf-8",
errors="replace",
**hidden_subprocess_kwargs(),
) as process:
reader = threading.Thread(target=read_output, name="training-output-reader", daemon=True)
reader.start()
@@ -140,7 +142,7 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
log(log_path, message)
line_count += 1
if message:
last_message = message[-220:]
last_message = friendly_training_message(message)
except queue.Empty:
pass
@@ -163,6 +165,78 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
report_progress(args, job_id, "running", "guard", 70, "Guard завершён, подготавливаю артефакты")
def friendly_training_message(message: str) -> str:
cleaned = message.strip()
if not cleaned:
return "PyTorch обучает модель"
if "Starting PyTorch recurrent retrain:" in cleaned:
return "PyTorch LSTM/GRU запущен: готовлю данные и варианты модели"
started = re.search(
r"training started: symbols=(?P<symbols>\d+) interval=(?P<interval>\d+) "
r"limit=(?P<limit>\d+) epochs=(?P<epochs>\d+)",
cleaned,
)
if started:
interval = started.group("interval")
timeframe = "1h" if interval == "60" else f"{interval}m"
return (
f"Старт обучения: {started.group('symbols')} пар, таймфрейм {timeframe}, "
f"история {started.group('limit')} свечей, до {started.group('epochs')} эпох"
)
pair_started = re.search(r"^(?P<symbol>[A-Z0-9]+): training started \((?P<index>\d+)/(?P<total>\d+)\)", cleaned)
if pair_started:
return (
f"{pair_started.group('symbol')}: обучение пары "
f"{pair_started.group('index')}/{pair_started.group('total')}"
)
preparing = re.search(r"^(?P<symbol>[A-Z0-9]+): preparing lookback=(?P<lookback>\d+)", cleaned)
if preparing:
return f"{preparing.group('symbol')}: готовлю окно {preparing.group('lookback')} свечей"
fitting = re.search(
r"^(?P<symbol>[A-Z0-9]+): fitting (?P<arch>lstm|gru) "
r"lookback=(?P<lookback>\d+) hidden=(?P<hidden>\d+) "
r"layers=(?P<layers>\d+) dropout=(?P<dropout>[0-9.]+)",
cleaned,
)
if fitting:
return (
f"{fitting.group('symbol')}: обучаю {fitting.group('arch').upper()}, "
f"окно {fitting.group('lookback')}, нейронов {fitting.group('hidden')}, "
f"слоёв {fitting.group('layers')}, dropout {fitting.group('dropout')}"
)
model = re.search(
r"^(?P<symbol>[A-Z0-9]+): model=torch_(?P<arch>lstm|gru).*?"
r"mae=(?P<mae>[0-9.]+)%.*?skill=(?P<skill>-?[0-9.]+).*?dir=(?P<direction>[0-9.]+)",
cleaned,
)
if model:
direction = float(model.group("direction")) * 100
skill = float(model.group("skill")) * 100
return (
f"{model.group('symbol')}: выбран {model.group('arch').upper()}, "
f"ошибка {model.group('mae')}%, skill {skill:.1f}%, направление {direction:.1f}%"
)
if "Calibrating current artifact" in cleaned:
return "Проверяю текущую модель на replay"
if "Calibrating candidate artifact" in cleaned:
return "Проверяю новую модель на replay"
if "Running retrain guard" in cleaned:
return "Gate сравнивает новую модель с текущей"
if "Candidate rejected by guard" in cleaned:
return "Новая модель обучилась, но gate не дал ей ходу"
if "Candidate accepted by guard" in cleaned:
return "Новая модель прошла gate и стала активной"
return cleaned[-220:]
def training_heartbeat_message(now: float, started_at: float, last_output_at: float, last_message: str) -> str:
elapsed = format_duration(now - started_at)
idle_seconds = max(0.0, now - last_output_at)
@@ -313,6 +387,18 @@ def read_json(path: Path) -> dict[str, Any]:
return data if isinstance(data, dict) else {}
def hidden_subprocess_kwargs() -> dict[str, Any]:
if os.name != "nt":
return {}
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = 0
return {
"creationflags": getattr(subprocess, "CREATE_NO_WINDOW", 0),
"startupinfo": startupinfo,
}
def quote_for_log(value: str) -> str:
return f'"{value}"' if " " in value else value