Improve Windows training agent progress
This commit is contained in:
@@ -7,6 +7,7 @@ import json
|
||||
import os
|
||||
import platform
|
||||
import queue
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
@@ -124,6 +125,7 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
**hidden_subprocess_kwargs(),
|
||||
) as process:
|
||||
reader = threading.Thread(target=read_output, name="training-output-reader", daemon=True)
|
||||
reader.start()
|
||||
@@ -140,7 +142,7 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
|
||||
log(log_path, message)
|
||||
line_count += 1
|
||||
if message:
|
||||
last_message = message[-220:]
|
||||
last_message = friendly_training_message(message)
|
||||
except queue.Empty:
|
||||
pass
|
||||
|
||||
@@ -163,6 +165,78 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
|
||||
report_progress(args, job_id, "running", "guard", 70, "Guard завершён, подготавливаю артефакты")
|
||||
|
||||
|
||||
def friendly_training_message(message: str) -> str:
|
||||
cleaned = message.strip()
|
||||
if not cleaned:
|
||||
return "PyTorch обучает модель"
|
||||
|
||||
if "Starting PyTorch recurrent retrain:" in cleaned:
|
||||
return "PyTorch LSTM/GRU запущен: готовлю данные и варианты модели"
|
||||
|
||||
started = re.search(
|
||||
r"training started: symbols=(?P<symbols>\d+) interval=(?P<interval>\d+) "
|
||||
r"limit=(?P<limit>\d+) epochs=(?P<epochs>\d+)",
|
||||
cleaned,
|
||||
)
|
||||
if started:
|
||||
interval = started.group("interval")
|
||||
timeframe = "1h" if interval == "60" else f"{interval}m"
|
||||
return (
|
||||
f"Старт обучения: {started.group('symbols')} пар, таймфрейм {timeframe}, "
|
||||
f"история {started.group('limit')} свечей, до {started.group('epochs')} эпох"
|
||||
)
|
||||
|
||||
pair_started = re.search(r"^(?P<symbol>[A-Z0-9]+): training started \((?P<index>\d+)/(?P<total>\d+)\)", cleaned)
|
||||
if pair_started:
|
||||
return (
|
||||
f"{pair_started.group('symbol')}: обучение пары "
|
||||
f"{pair_started.group('index')}/{pair_started.group('total')}"
|
||||
)
|
||||
|
||||
preparing = re.search(r"^(?P<symbol>[A-Z0-9]+): preparing lookback=(?P<lookback>\d+)", cleaned)
|
||||
if preparing:
|
||||
return f"{preparing.group('symbol')}: готовлю окно {preparing.group('lookback')} свечей"
|
||||
|
||||
fitting = re.search(
|
||||
r"^(?P<symbol>[A-Z0-9]+): fitting (?P<arch>lstm|gru) "
|
||||
r"lookback=(?P<lookback>\d+) hidden=(?P<hidden>\d+) "
|
||||
r"layers=(?P<layers>\d+) dropout=(?P<dropout>[0-9.]+)",
|
||||
cleaned,
|
||||
)
|
||||
if fitting:
|
||||
return (
|
||||
f"{fitting.group('symbol')}: обучаю {fitting.group('arch').upper()}, "
|
||||
f"окно {fitting.group('lookback')}, нейронов {fitting.group('hidden')}, "
|
||||
f"слоёв {fitting.group('layers')}, dropout {fitting.group('dropout')}"
|
||||
)
|
||||
|
||||
model = re.search(
|
||||
r"^(?P<symbol>[A-Z0-9]+): model=torch_(?P<arch>lstm|gru).*?"
|
||||
r"mae=(?P<mae>[0-9.]+)%.*?skill=(?P<skill>-?[0-9.]+).*?dir=(?P<direction>[0-9.]+)",
|
||||
cleaned,
|
||||
)
|
||||
if model:
|
||||
direction = float(model.group("direction")) * 100
|
||||
skill = float(model.group("skill")) * 100
|
||||
return (
|
||||
f"{model.group('symbol')}: выбран {model.group('arch').upper()}, "
|
||||
f"ошибка {model.group('mae')}%, skill {skill:.1f}%, направление {direction:.1f}%"
|
||||
)
|
||||
|
||||
if "Calibrating current artifact" in cleaned:
|
||||
return "Проверяю текущую модель на replay"
|
||||
if "Calibrating candidate artifact" in cleaned:
|
||||
return "Проверяю новую модель на replay"
|
||||
if "Running retrain guard" in cleaned:
|
||||
return "Gate сравнивает новую модель с текущей"
|
||||
if "Candidate rejected by guard" in cleaned:
|
||||
return "Новая модель обучилась, но gate не дал ей ходу"
|
||||
if "Candidate accepted by guard" in cleaned:
|
||||
return "Новая модель прошла gate и стала активной"
|
||||
|
||||
return cleaned[-220:]
|
||||
|
||||
|
||||
def training_heartbeat_message(now: float, started_at: float, last_output_at: float, last_message: str) -> str:
|
||||
elapsed = format_duration(now - started_at)
|
||||
idle_seconds = max(0.0, now - last_output_at)
|
||||
@@ -313,6 +387,18 @@ def read_json(path: Path) -> dict[str, Any]:
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def hidden_subprocess_kwargs() -> dict[str, Any]:
|
||||
if os.name != "nt":
|
||||
return {}
|
||||
startupinfo = subprocess.STARTUPINFO()
|
||||
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
|
||||
startupinfo.wShowWindow = 0
|
||||
return {
|
||||
"creationflags": getattr(subprocess, "CREATE_NO_WINDOW", 0),
|
||||
"startupinfo": startupinfo,
|
||||
}
|
||||
|
||||
|
||||
def quote_for_log(value: str) -> str:
|
||||
return f'"{value}"' if " " in value else value
|
||||
|
||||
|
||||
Reference in New Issue
Block a user