From 7f1ac694e4b0a1dba64da3cd263b1f2154404097 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 3 Jul 2026 20:44:09 +0300 Subject: [PATCH] Improve Windows training agent progress --- tools/windows_training_agent.py | 88 ++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/tools/windows_training_agent.py b/tools/windows_training_agent.py index f8301d1..337c316 100644 --- a/tools/windows_training_agent.py +++ b/tools/windows_training_agent.py @@ -7,6 +7,7 @@ import json import os import platform import queue +import re import subprocess import sys import threading @@ -124,6 +125,7 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo text=True, encoding="utf-8", errors="replace", + **hidden_subprocess_kwargs(), ) as process: reader = threading.Thread(target=read_output, name="training-output-reader", daemon=True) reader.start() @@ -140,7 +142,7 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo log(log_path, message) line_count += 1 if message: - last_message = message[-220:] + last_message = friendly_training_message(message) except queue.Empty: pass @@ -163,6 +165,78 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo report_progress(args, job_id, "running", "guard", 70, "Guard завершён, подготавливаю артефакты") +def friendly_training_message(message: str) -> str: + cleaned = message.strip() + if not cleaned: + return "PyTorch обучает модель" + + if "Starting PyTorch recurrent retrain:" in cleaned: + return "PyTorch LSTM/GRU запущен: готовлю данные и варианты модели" + + started = re.search( + r"training started: symbols=(?P\d+) interval=(?P\d+) " + r"limit=(?P\d+) epochs=(?P\d+)", + cleaned, + ) + if started: + interval = started.group("interval") + timeframe = "1h" if interval == "60" else f"{interval}m" + return ( + f"Старт обучения: {started.group('symbols')} пар, таймфрейм {timeframe}, " + f"история {started.group('limit')} свечей, до {started.group('epochs')} эпох" + ) + + pair_started = re.search(r"^(?P[A-Z0-9]+): training started \((?P\d+)/(?P\d+)\)", cleaned) + if pair_started: + return ( + f"{pair_started.group('symbol')}: обучение пары " + f"{pair_started.group('index')}/{pair_started.group('total')}" + ) + + preparing = re.search(r"^(?P[A-Z0-9]+): preparing lookback=(?P\d+)", cleaned) + if preparing: + return f"{preparing.group('symbol')}: готовлю окно {preparing.group('lookback')} свечей" + + fitting = re.search( + r"^(?P[A-Z0-9]+): fitting (?Plstm|gru) " + r"lookback=(?P\d+) hidden=(?P\d+) " + r"layers=(?P\d+) dropout=(?P[0-9.]+)", + cleaned, + ) + if fitting: + return ( + f"{fitting.group('symbol')}: обучаю {fitting.group('arch').upper()}, " + f"окно {fitting.group('lookback')}, нейронов {fitting.group('hidden')}, " + f"слоёв {fitting.group('layers')}, dropout {fitting.group('dropout')}" + ) + + model = re.search( + r"^(?P[A-Z0-9]+): model=torch_(?Plstm|gru).*?" + r"mae=(?P[0-9.]+)%.*?skill=(?P-?[0-9.]+).*?dir=(?P[0-9.]+)", + cleaned, + ) + if model: + direction = float(model.group("direction")) * 100 + skill = float(model.group("skill")) * 100 + return ( + f"{model.group('symbol')}: выбран {model.group('arch').upper()}, " + f"ошибка {model.group('mae')}%, skill {skill:.1f}%, направление {direction:.1f}%" + ) + + if "Calibrating current artifact" in cleaned: + return "Проверяю текущую модель на replay" + if "Calibrating candidate artifact" in cleaned: + return "Проверяю новую модель на replay" + if "Running retrain guard" in cleaned: + return "Gate сравнивает новую модель с текущей" + if "Candidate rejected by guard" in cleaned: + return "Новая модель обучилась, но gate не дал ей ходу" + if "Candidate accepted by guard" in cleaned: + return "Новая модель прошла gate и стала активной" + + return cleaned[-220:] + + def training_heartbeat_message(now: float, started_at: float, last_output_at: float, last_message: str) -> str: elapsed = format_duration(now - started_at) idle_seconds = max(0.0, now - last_output_at) @@ -313,6 +387,18 @@ def read_json(path: Path) -> dict[str, Any]: return data if isinstance(data, dict) else {} +def hidden_subprocess_kwargs() -> dict[str, Any]: + if os.name != "nt": + return {} + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + startupinfo.wShowWindow = 0 + return { + "creationflags": getattr(subprocess, "CREATE_NO_WINDOW", 0), + "startupinfo": startupinfo, + } + + def quote_for_log(value: str) -> str: return f'"{value}"' if " " in value else value