Improve Windows training agent progress
This commit is contained in:
@@ -7,6 +7,7 @@ import json
|
|||||||
import os
|
import os
|
||||||
import platform
|
import platform
|
||||||
import queue
|
import queue
|
||||||
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import threading
|
import threading
|
||||||
@@ -124,6 +125,7 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
|
|||||||
text=True,
|
text=True,
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
errors="replace",
|
errors="replace",
|
||||||
|
**hidden_subprocess_kwargs(),
|
||||||
) as process:
|
) as process:
|
||||||
reader = threading.Thread(target=read_output, name="training-output-reader", daemon=True)
|
reader = threading.Thread(target=read_output, name="training-output-reader", daemon=True)
|
||||||
reader.start()
|
reader.start()
|
||||||
@@ -140,7 +142,7 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
|
|||||||
log(log_path, message)
|
log(log_path, message)
|
||||||
line_count += 1
|
line_count += 1
|
||||||
if message:
|
if message:
|
||||||
last_message = message[-220:]
|
last_message = friendly_training_message(message)
|
||||||
except queue.Empty:
|
except queue.Empty:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -163,6 +165,78 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
|
|||||||
report_progress(args, job_id, "running", "guard", 70, "Guard завершён, подготавливаю артефакты")
|
report_progress(args, job_id, "running", "guard", 70, "Guard завершён, подготавливаю артефакты")
|
||||||
|
|
||||||
|
|
||||||
|
def friendly_training_message(message: str) -> str:
|
||||||
|
cleaned = message.strip()
|
||||||
|
if not cleaned:
|
||||||
|
return "PyTorch обучает модель"
|
||||||
|
|
||||||
|
if "Starting PyTorch recurrent retrain:" in cleaned:
|
||||||
|
return "PyTorch LSTM/GRU запущен: готовлю данные и варианты модели"
|
||||||
|
|
||||||
|
started = re.search(
|
||||||
|
r"training started: symbols=(?P<symbols>\d+) interval=(?P<interval>\d+) "
|
||||||
|
r"limit=(?P<limit>\d+) epochs=(?P<epochs>\d+)",
|
||||||
|
cleaned,
|
||||||
|
)
|
||||||
|
if started:
|
||||||
|
interval = started.group("interval")
|
||||||
|
timeframe = "1h" if interval == "60" else f"{interval}m"
|
||||||
|
return (
|
||||||
|
f"Старт обучения: {started.group('symbols')} пар, таймфрейм {timeframe}, "
|
||||||
|
f"история {started.group('limit')} свечей, до {started.group('epochs')} эпох"
|
||||||
|
)
|
||||||
|
|
||||||
|
pair_started = re.search(r"^(?P<symbol>[A-Z0-9]+): training started \((?P<index>\d+)/(?P<total>\d+)\)", cleaned)
|
||||||
|
if pair_started:
|
||||||
|
return (
|
||||||
|
f"{pair_started.group('symbol')}: обучение пары "
|
||||||
|
f"{pair_started.group('index')}/{pair_started.group('total')}"
|
||||||
|
)
|
||||||
|
|
||||||
|
preparing = re.search(r"^(?P<symbol>[A-Z0-9]+): preparing lookback=(?P<lookback>\d+)", cleaned)
|
||||||
|
if preparing:
|
||||||
|
return f"{preparing.group('symbol')}: готовлю окно {preparing.group('lookback')} свечей"
|
||||||
|
|
||||||
|
fitting = re.search(
|
||||||
|
r"^(?P<symbol>[A-Z0-9]+): fitting (?P<arch>lstm|gru) "
|
||||||
|
r"lookback=(?P<lookback>\d+) hidden=(?P<hidden>\d+) "
|
||||||
|
r"layers=(?P<layers>\d+) dropout=(?P<dropout>[0-9.]+)",
|
||||||
|
cleaned,
|
||||||
|
)
|
||||||
|
if fitting:
|
||||||
|
return (
|
||||||
|
f"{fitting.group('symbol')}: обучаю {fitting.group('arch').upper()}, "
|
||||||
|
f"окно {fitting.group('lookback')}, нейронов {fitting.group('hidden')}, "
|
||||||
|
f"слоёв {fitting.group('layers')}, dropout {fitting.group('dropout')}"
|
||||||
|
)
|
||||||
|
|
||||||
|
model = re.search(
|
||||||
|
r"^(?P<symbol>[A-Z0-9]+): model=torch_(?P<arch>lstm|gru).*?"
|
||||||
|
r"mae=(?P<mae>[0-9.]+)%.*?skill=(?P<skill>-?[0-9.]+).*?dir=(?P<direction>[0-9.]+)",
|
||||||
|
cleaned,
|
||||||
|
)
|
||||||
|
if model:
|
||||||
|
direction = float(model.group("direction")) * 100
|
||||||
|
skill = float(model.group("skill")) * 100
|
||||||
|
return (
|
||||||
|
f"{model.group('symbol')}: выбран {model.group('arch').upper()}, "
|
||||||
|
f"ошибка {model.group('mae')}%, skill {skill:.1f}%, направление {direction:.1f}%"
|
||||||
|
)
|
||||||
|
|
||||||
|
if "Calibrating current artifact" in cleaned:
|
||||||
|
return "Проверяю текущую модель на replay"
|
||||||
|
if "Calibrating candidate artifact" in cleaned:
|
||||||
|
return "Проверяю новую модель на replay"
|
||||||
|
if "Running retrain guard" in cleaned:
|
||||||
|
return "Gate сравнивает новую модель с текущей"
|
||||||
|
if "Candidate rejected by guard" in cleaned:
|
||||||
|
return "Новая модель обучилась, но gate не дал ей ходу"
|
||||||
|
if "Candidate accepted by guard" in cleaned:
|
||||||
|
return "Новая модель прошла gate и стала активной"
|
||||||
|
|
||||||
|
return cleaned[-220:]
|
||||||
|
|
||||||
|
|
||||||
def training_heartbeat_message(now: float, started_at: float, last_output_at: float, last_message: str) -> str:
|
def training_heartbeat_message(now: float, started_at: float, last_output_at: float, last_message: str) -> str:
|
||||||
elapsed = format_duration(now - started_at)
|
elapsed = format_duration(now - started_at)
|
||||||
idle_seconds = max(0.0, now - last_output_at)
|
idle_seconds = max(0.0, now - last_output_at)
|
||||||
@@ -313,6 +387,18 @@ def read_json(path: Path) -> dict[str, Any]:
|
|||||||
return data if isinstance(data, dict) else {}
|
return data if isinstance(data, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def hidden_subprocess_kwargs() -> dict[str, Any]:
|
||||||
|
if os.name != "nt":
|
||||||
|
return {}
|
||||||
|
startupinfo = subprocess.STARTUPINFO()
|
||||||
|
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
|
||||||
|
startupinfo.wShowWindow = 0
|
||||||
|
return {
|
||||||
|
"creationflags": getattr(subprocess, "CREATE_NO_WINDOW", 0),
|
||||||
|
"startupinfo": startupinfo,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def quote_for_log(value: str) -> str:
|
def quote_for_log(value: str) -> str:
|
||||||
return f'"{value}"' if " " in value else value
|
return f'"{value}"' if " " in value else value
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user