Add automated LSTM retraining

README.md

@@ -72,6 +72,22 @@ python tools\train_lstm_forecaster.py --symbols BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,
 
 Файл из `TIME_SERIES_LSTM_MODEL_PATH` читается ботом автоматически. Даже если LSTM-параметры сохранены, сделка меняется только тогда, когда текущая walk-forward проверка в `crypto_spot_bot/time_series.py` показывает качество лучше baseline.
 
+Автопереобучение запускает тот же train-скрипт, пишет лог в `runtime/lstm_retrain.log` и защищается от параллельных запусков:
+
+```powershell
+powershell -ExecutionPolicy Bypass -File tools\run_lstm_retrain.ps1
+powershell -ExecutionPolicy Bypass -File tools\install_windows_lstm_retrainer.ps1
+```
+
+На Linux/Raspberry Pi можно включить user systemd timer:
+
+```bash
+bash tools/run_lstm_retrain.sh
+bash tools/install_lstm_retrainer_systemd.sh
+```
+
+По умолчанию расписание переобучает каждые 6 часов с `--limit 1000`; Windows-установщик фиксирует пары `BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,LTCUSDT`, чтобы первый scheduled run был предсказуемым. Параметры можно переопределить через env: `LSTM_RETRAIN_SYMBOLS`, `LSTM_RETRAIN_LIMIT`, `LSTM_RETRAIN_LOOKBACKS`, `LSTM_RETRAIN_UNITS`, `LSTM_RETRAIN_RIDGES`, `LSTM_RETRAIN_INTERVAL`, `LSTM_RETRAIN_ENV`.
+
 ## Docker
 
 ```bash

tools/install_lstm_retrainer_systemd.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+SYSTEMD_DIR="$HOME/.config/systemd/user"
+SERVICE_NAME="tradebot-lstm-retrainer.service"
+TIMER_NAME="tradebot-lstm-retrainer.timer"
+
+mkdir -p "$SYSTEMD_DIR"
+
+cat > "$SYSTEMD_DIR/$SERVICE_NAME" <<EOF
+[Unit]
+Description=Retrain TradeBot LSTM forecast parameters
+
+[Service]
+Type=oneshot
+WorkingDirectory=$REPO_ROOT
+ExecStart=$REPO_ROOT/tools/run_lstm_retrain.sh
+EOF
+
+cat > "$SYSTEMD_DIR/$TIMER_NAME" <<EOF
+[Unit]
+Description=Retrain TradeBot LSTM forecast parameters every 6 hours
+
+[Timer]
+OnBootSec=5min
+OnUnitActiveSec=6h
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+EOF
+
+systemctl --user daemon-reload
+systemctl --user enable --now "$TIMER_NAME"
+
+echo "Enabled user timer $TIMER_NAME. Check with: systemctl --user list-timers $TIMER_NAME"

tools/install_windows_lstm_retrainer.ps1

@@ -0,0 +1,49 @@
+[CmdletBinding()]
+param(
+    [string]$TaskName = "TradeBot LSTM Retrainer",
+    [int]$EveryHours = 6,
+    [string]$Symbols = "BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,LTCUSDT",
+    [int]$Limit = 1000
+)
+
+$ErrorActionPreference = "Stop"
+
+$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
+$Runner = Join-Path $RepoRoot "tools\run_lstm_retrain.ps1"
+if (-not (Test-Path $Runner)) {
+    throw "Runner not found: $Runner"
+}
+
+$actionArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$Runner`""
+if ($Symbols) {
+    $actionArgs += " -Symbols `"$Symbols`""
+}
+if ($Limit -gt 0) {
+    $actionArgs += " -Limit $Limit"
+}
+$action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $actionArgs -WorkingDirectory $RepoRoot
+$trigger = New-ScheduledTaskTrigger `
+    -Once `
+    -At (Get-Date).AddMinutes(5) `
+    -RepetitionInterval (New-TimeSpan -Hours $EveryHours) `
+    -RepetitionDuration (New-TimeSpan -Days 3650)
+$principal = New-ScheduledTaskPrincipal `
+    -UserId ([System.Security.Principal.WindowsIdentity]::GetCurrent().Name) `
+    -LogonType Interactive `
+    -RunLevel Limited
+$settings = New-ScheduledTaskSettingsSet `
+    -StartWhenAvailable `
+    -MultipleInstances IgnoreNew `
+    -AllowStartIfOnBatteries `
+    -DontStopIfGoingOnBatteries
+
+Register-ScheduledTask `
+    -TaskName $TaskName `
+    -Action $action `
+    -Trigger $trigger `
+    -Principal $principal `
+    -Settings $settings `
+    -Description "Retrains TradeBot LSTM forecast parameters every $EveryHours hours." `
+    -Force | Out-Null
+
+Write-Host "Registered scheduled task '$TaskName' every $EveryHours hours."

tools/run_lstm_retrain.ps1

@@ -0,0 +1,102 @@
+[CmdletBinding()]
+param(
+    [string]$Symbols = "",
+    [int]$Limit = 0,
+    [string]$Lookbacks = "",
+    [string]$Units = "",
+    [string]$Ridges = "",
+    [string]$Interval = "",
+    [string]$EnvFile = ""
+)
+
+$ErrorActionPreference = "Stop"
+
+$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
+$RuntimeDir = Join-Path $RepoRoot "runtime"
+$LogFile = Join-Path $RuntimeDir "lstm_retrain.log"
+New-Item -ItemType Directory -Force -Path $RuntimeDir | Out-Null
+
+function Write-RetrainLog {
+    param([string]$Message)
+    $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ssK"
+    "[$timestamp] $Message" | Tee-Object -FilePath $LogFile -Append
+}
+
+function Resolve-Python {
+    $venvPython = Join-Path $RepoRoot ".venv\Scripts\python.exe"
+    if (Test-Path $venvPython) {
+        return $venvPython
+    }
+
+    $userPython = Join-Path $env:LOCALAPPDATA "Programs\TradeBotPython312\python.exe"
+    if (Test-Path $userPython) {
+        return $userPython
+    }
+
+    foreach ($candidate in @("python.exe", "python")) {
+        $command = Get-Command $candidate -ErrorAction SilentlyContinue
+        if (-not $command) {
+            continue
+        }
+        return $command.Source
+    }
+    throw "Python was not found. Create .venv or install Python 3.12."
+}
+
+if (-not $Symbols -and $env:LSTM_RETRAIN_SYMBOLS) { $Symbols = $env:LSTM_RETRAIN_SYMBOLS }
+if ($Limit -le 0) {
+    $Limit = if ($env:LSTM_RETRAIN_LIMIT) { [int]$env:LSTM_RETRAIN_LIMIT } else { 1000 }
+}
+if (-not $Lookbacks) { $Lookbacks = if ($env:LSTM_RETRAIN_LOOKBACKS) { $env:LSTM_RETRAIN_LOOKBACKS } else { "16,32" } }
+if (-not $Units) { $Units = if ($env:LSTM_RETRAIN_UNITS) { $env:LSTM_RETRAIN_UNITS } else { "4,6" } }
+if (-not $Ridges) { $Ridges = if ($env:LSTM_RETRAIN_RIDGES) { $env:LSTM_RETRAIN_RIDGES } else { "0.001" } }
+if (-not $Interval -and $env:LSTM_RETRAIN_INTERVAL) { $Interval = $env:LSTM_RETRAIN_INTERVAL }
+if (-not $EnvFile -and $env:LSTM_RETRAIN_ENV) { $EnvFile = $env:LSTM_RETRAIN_ENV }
+if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" }
+
+$mutex = New-Object System.Threading.Mutex($false, "TradeBotLstmRetrainer")
+$hasLock = $false
+$pushedLocation = $false
+
+try {
+    $hasLock = $mutex.WaitOne(0)
+    if (-not $hasLock) {
+        Write-RetrainLog "Another LSTM retrain is already running; skipping."
+        exit 0
+    }
+
+    $python = Resolve-Python
+    $trainerArgs = @(
+        "-u",
+        "tools\train_lstm_forecaster.py",
+        "--limit", $Limit.ToString(),
+        "--lookbacks", $Lookbacks,
+        "--units", $Units,
+        "--ridges", $Ridges
+    )
+    if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) }
+    if ($Interval) { $trainerArgs += @("--interval", $Interval) }
+    if ($EnvFile) { $trainerArgs += @("--env", $EnvFile) }
+
+    Push-Location $RepoRoot
+    $pushedLocation = $true
+    Write-RetrainLog "Starting LSTM retrain: $python $($trainerArgs -join ' ')"
+    & $python @trainerArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
+    if ($LASTEXITCODE -ne 0) {
+        throw "Trainer failed with exit code $LASTEXITCODE."
+    }
+    Write-RetrainLog "Finished LSTM retrain."
+}
+catch {
+    Write-RetrainLog "ERROR: $($_.Exception.Message)"
+    exit 1
+}
+finally {
+    if ($pushedLocation) {
+        Pop-Location -ErrorAction SilentlyContinue
+    }
+    if ($hasLock) {
+        $mutex.ReleaseMutex()
+    }
+    $mutex.Dispose()
+}

tools/run_lstm_retrain.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+RUNTIME_DIR="$REPO_ROOT/runtime"
+LOG_FILE="$RUNTIME_DIR/lstm_retrain.log"
+LOCK_FILE="$RUNTIME_DIR/lstm_retrain.lock"
+
+mkdir -p "$RUNTIME_DIR"
+
+log() {
+  printf '[%s] %s\n' "$(date -Is)" "$*" | tee -a "$LOG_FILE"
+}
+
+if command -v flock >/dev/null 2>&1; then
+  exec 9>"$LOCK_FILE"
+  if ! flock -n 9; then
+    log "Another LSTM retrain is already running; skipping."
+    exit 0
+  fi
+else
+  LOCK_DIR="$LOCK_FILE.d"
+  if ! mkdir "$LOCK_DIR" 2>/dev/null; then
+    log "Another LSTM retrain is already running; skipping."
+    exit 0
+  fi
+  trap 'rmdir "$LOCK_DIR"' EXIT
+fi
+
+if [[ -x "$REPO_ROOT/.venv/bin/python" ]]; then
+  PYTHON="$REPO_ROOT/.venv/bin/python"
+elif command -v python3 >/dev/null 2>&1; then
+  PYTHON="$(command -v python3)"
+elif command -v python >/dev/null 2>&1; then
+  PYTHON="$(command -v python)"
+else
+  log "ERROR: Python was not found."
+  exit 1
+fi
+
+SYMBOLS="${LSTM_RETRAIN_SYMBOLS:-}"
+LIMIT="${LSTM_RETRAIN_LIMIT:-1000}"
+LOOKBACKS="${LSTM_RETRAIN_LOOKBACKS:-16,32}"
+UNITS="${LSTM_RETRAIN_UNITS:-4,6}"
+RIDGES="${LSTM_RETRAIN_RIDGES:-0.001}"
+INTERVAL="${LSTM_RETRAIN_INTERVAL:-}"
+ENV_FILE="${LSTM_RETRAIN_ENV:-}"
+
+if [[ -z "$ENV_FILE" && -f "$REPO_ROOT/.env" ]]; then
+  ENV_FILE="$REPO_ROOT/.env"
+fi
+
+args=(
+  "tools/train_lstm_forecaster.py"
+  "--limit" "$LIMIT"
+  "--lookbacks" "$LOOKBACKS"
+  "--units" "$UNITS"
+  "--ridges" "$RIDGES"
+)
+
+if [[ -n "$SYMBOLS" ]]; then args+=("--symbols" "$SYMBOLS"); fi
+if [[ -n "$INTERVAL" ]]; then args+=("--interval" "$INTERVAL"); fi
+if [[ -n "$ENV_FILE" ]]; then args+=("--env" "$ENV_FILE"); fi
+
+cd "$REPO_ROOT"
+log "Starting LSTM retrain: $PYTHON -u ${args[*]}"
+"$PYTHON" -u "${args[@]}" 2>&1 | tee -a "$LOG_FILE"
+log "Finished LSTM retrain."

tools/train_lstm_forecaster.py

@@ -56,7 +56,9 @@ def main() -> None:
         )
 
     output.parent.mkdir(parents=True, exist_ok=True)
-    output.write_text(json.dumps(artifact, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    tmp_output = output.with_name(f"{output.name}.tmp")
+    tmp_output.write_text(json.dumps(artifact, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    tmp_output.replace(output)
     print(f"saved {output}")