sevenhill/TradeBot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove legacy LSTM retraining

Browse Source
This commit is contained in:
Codex
2026-06-20 22:07:21 +03:00
parent d9c0317675
commit ccf457481b
14 changed files with 179 additions and 619 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/install_lstm_retrainer_systemd.sh
-38
View File
@@ -1,38 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
SYSTEMD_DIR="$HOME/.config/systemd/user"
SERVICE_NAME="tradebot-lstm-retrainer.service"
TIMER_NAME="tradebot-lstm-retrainer.timer"
mkdir -p "$SYSTEMD_DIR"
cat > "$SYSTEMD_DIR/$SERVICE_NAME" <<EOF
[Unit]
Description=Retrain TradeBot LSTM forecast parameters
[Service]
Type=oneshot
WorkingDirectory=$REPO_ROOT
ExecStart=$REPO_ROOT/tools/run_lstm_retrain.sh
EOF
cat > "$SYSTEMD_DIR/$TIMER_NAME" <<EOF
[Unit]
Description=Retrain TradeBot LSTM forecast parameters every 6 hours
[Timer]
OnBootSec=5min
OnUnitActiveSec=6h
Persistent=true
[Install]
WantedBy=timers.target
EOF
systemctl --user daemon-reload
systemctl --user enable --now "$TIMER_NAME"
echo "Enabled user timer $TIMER_NAME. Check with: systemctl --user list-timers $TIMER_NAME"
tools/install_windows_lstm_retrainer.ps1 → tools/install_windows_torch_retrainer.ps1
+13 -7
View File
@@ -1,23 +1,29 @@
[CmdletBinding()]
param(
[string]$TaskName = "TradeBot LSTM Retrainer",
[string]$TaskName = "TradeBot PyTorch Forecaster Retrainer",
[int]$EveryHours = 6,
[string]$Symbols = "BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,LTCUSDT",
[string]$Symbols = "BTCUSDT,ETHUSDT,HYPEUSDT,SOLUSDT,LTCUSDT,XRPUSDT",
[int]$Limit = 1000,
[ValidateSet("torch", "reservoir")]
[string]$Trainer = "torch",
[int]$FirstRunMinutes = 0
)
$ErrorActionPreference = "Stop"
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
$Runner = Join-Path $RepoRoot "tools\run_lstm_retrain.ps1"
$Runner = Join-Path $RepoRoot "tools\run_torch_retrain.ps1"
if (-not (Test-Path $Runner)) {
throw "Runner not found: $Runner"
}
$actionArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$Runner`" -Trainer $Trainer"
$LegacyTaskName = "TradeBot LSTM Retrainer"
if ($TaskName -ne $LegacyTaskName) {
$legacyTask = Get-ScheduledTask -TaskName $LegacyTaskName -ErrorAction SilentlyContinue
if ($legacyTask) {
Unregister-ScheduledTask -TaskName $LegacyTaskName -Confirm:$false
}
}
$actionArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$Runner`""
if ($Symbols) {
$actionArgs += " -Symbols `"$Symbols`""
}
@@ -46,7 +52,7 @@ Register-ScheduledTask `
-Trigger $trigger `
-Principal $principal `
-Settings $settings `
-Description "Retrains TradeBot LSTM forecast parameters every $EveryHours hours." `
-Description "Retrains TradeBot PyTorch recurrent forecast parameters every $EveryHours hours." `
-Force | Out-Null
Write-Host "Registered scheduled task '$TaskName' every $EveryHours hours."
tools/run_lstm_retrain.ps1
-131
View File
@@ -1,131 +0,0 @@
[CmdletBinding()]
param(
[ValidateSet("torch", "reservoir")]
[string]$Trainer = "torch",
[string]$Symbols = "",
[int]$Limit = 0,
[string]$Lookbacks = "",
[string]$Units = "",
[string]$Ridges = "",
[string]$Architectures = "",
[string]$HiddenSizes = "",
[string]$Layers = "",
[string]$Dropouts = "",
[int]$Epochs = 0,
[int]$Patience = 0,
[string]$Interval = "",
[string]$EnvFile = ""
)
$ErrorActionPreference = "Stop"
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
$RuntimeDir = Join-Path $RepoRoot "runtime"
$LogFile = Join-Path $RuntimeDir "lstm_retrain.log"
New-Item -ItemType Directory -Force -Path $RuntimeDir | Out-Null
function Write-RetrainLog {
param([string]$Message)
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ssK"
"[$timestamp] $Message" | Tee-Object -FilePath $LogFile -Append
}
function Resolve-Python {
$venvPython = Join-Path $RepoRoot ".venv\Scripts\python.exe"
if (Test-Path $venvPython) {
return $venvPython
}
$userPython = Join-Path $env:LOCALAPPDATA "Programs\TradeBotPython312\python.exe"
if (Test-Path $userPython) {
return $userPython
}
foreach ($candidate in @("python.exe", "python")) {
$command = Get-Command $candidate -ErrorAction SilentlyContinue
if (-not $command) {
continue
}
return $command.Source
}
throw "Python was not found. Create .venv or install Python 3.12."
}
if (-not $Symbols -and $env:LSTM_RETRAIN_SYMBOLS) { $Symbols = $env:LSTM_RETRAIN_SYMBOLS }
if ($Limit -le 0) {
$Limit = if ($env:LSTM_RETRAIN_LIMIT) { [int]$env:LSTM_RETRAIN_LIMIT } else { 1000 }
}
if (-not $Lookbacks) { $Lookbacks = if ($env:LSTM_RETRAIN_LOOKBACKS) { $env:LSTM_RETRAIN_LOOKBACKS } else { "32,64" } }
if (-not $Units) { $Units = if ($env:LSTM_RETRAIN_UNITS) { $env:LSTM_RETRAIN_UNITS } else { "4,6" } }
if (-not $Ridges) { $Ridges = if ($env:LSTM_RETRAIN_RIDGES) { $env:LSTM_RETRAIN_RIDGES } else { "0.001" } }
if (-not $Architectures) { $Architectures = if ($env:LSTM_RETRAIN_ARCHITECTURES) { $env:LSTM_RETRAIN_ARCHITECTURES } else { "lstm,gru" } }
if (-not $HiddenSizes) { $HiddenSizes = if ($env:LSTM_RETRAIN_HIDDEN_SIZES) { $env:LSTM_RETRAIN_HIDDEN_SIZES } else { "16,32" } }
if (-not $Layers) { $Layers = if ($env:LSTM_RETRAIN_LAYERS) { $env:LSTM_RETRAIN_LAYERS } else { "1" } }
if (-not $Dropouts) { $Dropouts = if ($env:LSTM_RETRAIN_DROPOUTS) { $env:LSTM_RETRAIN_DROPOUTS } else { "0.0" } }
if ($Epochs -le 0) { $Epochs = if ($env:LSTM_RETRAIN_EPOCHS) { [int]$env:LSTM_RETRAIN_EPOCHS } else { 60 } }
if ($Patience -le 0) { $Patience = if ($env:LSTM_RETRAIN_PATIENCE) { [int]$env:LSTM_RETRAIN_PATIENCE } else { 10 } }
if (-not $Interval -and $env:LSTM_RETRAIN_INTERVAL) { $Interval = $env:LSTM_RETRAIN_INTERVAL }
if (-not $EnvFile -and $env:LSTM_RETRAIN_ENV) { $EnvFile = $env:LSTM_RETRAIN_ENV }
if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" }
$mutex = New-Object System.Threading.Mutex($false, "TradeBotLstmRetrainer")
$hasLock = $false
$pushedLocation = $false
try {
$hasLock = $mutex.WaitOne(0)
if (-not $hasLock) {
Write-RetrainLog "Another LSTM retrain is already running; skipping."
exit 0
}
$python = Resolve-Python
if ($Trainer -eq "torch") {
$trainerArgs = @(
"-u",
"tools\train_torch_recurrent_forecaster.py",
"--limit", $Limit.ToString(),
"--lookbacks", $Lookbacks,
"--architectures", $Architectures,
"--hidden-sizes", $HiddenSizes,
"--layers", $Layers,
"--dropouts", $Dropouts,
"--epochs", $Epochs.ToString(),
"--patience", $Patience.ToString()
)
} else {
$trainerArgs = @(
"-u",
"tools\train_lstm_forecaster.py",
"--limit", $Limit.ToString(),
"--lookbacks", $Lookbacks,
"--units", $Units,
"--ridges", $Ridges
)
}
if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) }
if ($Interval) { $trainerArgs += @("--interval", $Interval) }
if ($EnvFile) { $trainerArgs += @("--env", $EnvFile) }
Push-Location $RepoRoot
$pushedLocation = $true
Write-RetrainLog "Starting LSTM retrain: $python $($trainerArgs -join ' ')"
& $python @trainerArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
if ($LASTEXITCODE -ne 0) {
throw "Trainer failed with exit code $LASTEXITCODE."
}
Write-RetrainLog "Finished LSTM retrain."
}
catch {
Write-RetrainLog "ERROR: $($_.Exception.Message)"
exit 1
}
finally {
if ($pushedLocation) {
Pop-Location -ErrorAction SilentlyContinue
}
if ($hasLock) {
$mutex.ReleaseMutex()
}
$mutex.Dispose()
}
tools/run_lstm_retrain.sh
-69
View File
@@ -1,69 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
RUNTIME_DIR="$REPO_ROOT/runtime"
LOG_FILE="$RUNTIME_DIR/lstm_retrain.log"
LOCK_FILE="$RUNTIME_DIR/lstm_retrain.lock"
mkdir -p "$RUNTIME_DIR"
log() {
printf '[%s] %s\n' "$(date -Is)" "$*" | tee -a "$LOG_FILE"
}
if command -v flock >/dev/null 2>&1; then
exec 9>"$LOCK_FILE"
if ! flock -n 9; then
log "Another LSTM retrain is already running; skipping."
exit 0
fi
else
LOCK_DIR="$LOCK_FILE.d"
if ! mkdir "$LOCK_DIR" 2>/dev/null; then
log "Another LSTM retrain is already running; skipping."
exit 0
fi
trap 'rmdir "$LOCK_DIR"' EXIT
fi
if [[ -x "$REPO_ROOT/.venv/bin/python" ]]; then
PYTHON="$REPO_ROOT/.venv/bin/python"
elif command -v python3 >/dev/null 2>&1; then
PYTHON="$(command -v python3)"
elif command -v python >/dev/null 2>&1; then
PYTHON="$(command -v python)"
else
log "ERROR: Python was not found."
exit 1
fi
SYMBOLS="${LSTM_RETRAIN_SYMBOLS:-}"
LIMIT="${LSTM_RETRAIN_LIMIT:-1000}"
LOOKBACKS="${LSTM_RETRAIN_LOOKBACKS:-16,32}"
UNITS="${LSTM_RETRAIN_UNITS:-4,6}"
RIDGES="${LSTM_RETRAIN_RIDGES:-0.001}"
INTERVAL="${LSTM_RETRAIN_INTERVAL:-}"
ENV_FILE="${LSTM_RETRAIN_ENV:-}"
if [[ -z "$ENV_FILE" && -f "$REPO_ROOT/.env" ]]; then
ENV_FILE="$REPO_ROOT/.env"
fi
args=(
"tools/train_lstm_forecaster.py"
"--limit" "$LIMIT"
"--lookbacks" "$LOOKBACKS"
"--units" "$UNITS"
"--ridges" "$RIDGES"
)
if [[ -n "$SYMBOLS" ]]; then args+=("--symbols" "$SYMBOLS"); fi
if [[ -n "$INTERVAL" ]]; then args+=("--interval" "$INTERVAL"); fi
if [[ -n "$ENV_FILE" ]]; then args+=("--env" "$ENV_FILE"); fi
cd "$REPO_ROOT"
log "Starting LSTM retrain: $PYTHON -u ${args[*]}"
"$PYTHON" -u "${args[@]}" 2>&1 | tee -a "$LOG_FILE"
log "Finished LSTM retrain."
tools/run_torch_retrain.ps1
+114
View File
@@ -0,0 +1,114 @@
[CmdletBinding()]
param(
[string]$Symbols = "",
[int]$Limit = 0,
[string]$Lookbacks = "",
[string]$Architectures = "",
[string]$HiddenSizes = "",
[string]$Layers = "",
[string]$Dropouts = "",
[int]$Epochs = 0,
[int]$Patience = 0,
[string]$Interval = "",
[string]$EnvFile = ""
)
$ErrorActionPreference = "Stop"
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
$RuntimeDir = Join-Path $RepoRoot "runtime"
$LogFile = Join-Path $RuntimeDir "torch_retrain.log"
New-Item -ItemType Directory -Force -Path $RuntimeDir | Out-Null
function Write-RetrainLog {
param([string]$Message)
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ssK"
"[$timestamp] $Message" | Tee-Object -FilePath $LogFile -Append
}
function Resolve-Python {
$venvPython = Join-Path $RepoRoot ".venv\Scripts\python.exe"
if (Test-Path $venvPython) {
return $venvPython
}
$userPython = Join-Path $env:LOCALAPPDATA "Programs\TradeBotPython312\python.exe"
if (Test-Path $userPython) {
return $userPython
}
foreach ($candidate in @("python.exe", "python")) {
$command = Get-Command $candidate -ErrorAction SilentlyContinue
if (-not $command) {
continue
}
return $command.Source
}
throw "Python was not found. Create .venv or install Python 3.12."
}
if (-not $Symbols -and $env:TORCH_RETRAIN_SYMBOLS) { $Symbols = $env:TORCH_RETRAIN_SYMBOLS }
if ($Limit -le 0) {
$Limit = if ($env:TORCH_RETRAIN_LIMIT) { [int]$env:TORCH_RETRAIN_LIMIT } else { 1000 }
}
if (-not $Lookbacks) { $Lookbacks = if ($env:TORCH_RETRAIN_LOOKBACKS) { $env:TORCH_RETRAIN_LOOKBACKS } else { "32,64" } }
if (-not $Architectures) { $Architectures = if ($env:TORCH_RETRAIN_ARCHITECTURES) { $env:TORCH_RETRAIN_ARCHITECTURES } else { "lstm,gru" } }
if (-not $HiddenSizes) { $HiddenSizes = if ($env:TORCH_RETRAIN_HIDDEN_SIZES) { $env:TORCH_RETRAIN_HIDDEN_SIZES } else { "16,32" } }
if (-not $Layers) { $Layers = if ($env:TORCH_RETRAIN_LAYERS) { $env:TORCH_RETRAIN_LAYERS } else { "1" } }
if (-not $Dropouts) { $Dropouts = if ($env:TORCH_RETRAIN_DROPOUTS) { $env:TORCH_RETRAIN_DROPOUTS } else { "0.0" } }
if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 60 } }
if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 10 } }
if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RETRAIN_INTERVAL }
if (-not $EnvFile -and $env:TORCH_RETRAIN_ENV) { $EnvFile = $env:TORCH_RETRAIN_ENV }
if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" }
$mutex = New-Object System.Threading.Mutex($false, "TradeBotTorchRecurrentRetrainer")
$hasLock = $false
$pushedLocation = $false
try {
$hasLock = $mutex.WaitOne(0)
if (-not $hasLock) {
Write-RetrainLog "Another PyTorch recurrent retrain is already running; skipping."
exit 0
}
$python = Resolve-Python
$trainerArgs = @(
"-u",
"tools\train_torch_recurrent_forecaster.py",
"--limit", $Limit.ToString(),
"--lookbacks", $Lookbacks,
"--architectures", $Architectures,
"--hidden-sizes", $HiddenSizes,
"--layers", $Layers,
"--dropouts", $Dropouts,
"--epochs", $Epochs.ToString(),
"--patience", $Patience.ToString()
)
if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) }
if ($Interval) { $trainerArgs += @("--interval", $Interval) }
if ($EnvFile) { $trainerArgs += @("--env", $EnvFile) }
Push-Location $RepoRoot
$pushedLocation = $true
Write-RetrainLog "Starting PyTorch recurrent retrain: $python $($trainerArgs -join ' ')"
& $python @trainerArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
if ($LASTEXITCODE -ne 0) {
throw "Trainer failed with exit code $LASTEXITCODE."
}
Write-RetrainLog "Finished PyTorch recurrent retrain."
}
catch {
Write-RetrainLog "ERROR: $($_.Exception.Message)"
exit 1
}
finally {
if ($pushedLocation) {
Pop-Location -ErrorAction SilentlyContinue
}
if ($hasLock) {
$mutex.ReleaseMutex()
}
$mutex.Dispose()
}
tools/train_lstm_forecaster.py
-146
View File
@@ -1,146 +0,0 @@
from __future__ import annotations
import argparse
import json
import sys
from dataclasses import replace
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from crypto_spot_bot.bybit import BybitClient
from crypto_spot_bot.config import Settings, load_settings
from crypto_spot_bot.time_series import _log_returns, _validate_candidates
def main() -> None:
args = _parse_args()
settings = load_settings(args.env)
client = BybitClient(settings)
symbols = _symbols(args.symbols, settings, client)
interval = args.interval or settings.base_interval
output = Path(args.output) if args.output else settings.time_series_lstm_model_path
artifact: dict[str, Any] = {
"version": 1,
"type": "lstm_reservoir_ridge_params",
"created_at": datetime.now(timezone.utc).isoformat(),
"interval": interval,
"limit": args.limit,
"symbols": {},
}
for symbol in symbols:
result = _train_symbol(
client=client,
settings=settings,
symbol=symbol,
interval=interval,
limit=args.limit,
lookbacks=_ints(args.lookbacks),
units_values=_ints(args.units),
ridges=_floats(args.ridges),
)
if result is None:
print(f"{symbol}: skipped, not enough candles or returns")
continue
artifact["symbols"][symbol] = result
print(
f"{symbol}: lookback={result['lookback']} units={result['units']} "
f"ridge={result['ridge']} mae={result['validation_mae_percent']:.5f}% "
f"baseline={result['baseline_mae_percent']:.5f}% skill={result['skill']:.4f}"
)
output.parent.mkdir(parents=True, exist_ok=True)
tmp_output = output.with_name(f"{output.name}.tmp")
tmp_output.write_text(json.dumps(artifact, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
tmp_output.replace(output)
print(f"saved {output}")
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Train lightweight LSTM forecast params on Bybit spot candles.")
parser.add_argument("--env", default=None, help="Path to .env file.")
parser.add_argument("--symbols", default="", help="Comma-separated symbols. Defaults to configured or popular spot pairs.")
parser.add_argument("--interval", default="", help="Bybit kline interval. Defaults to BASE_INTERVAL.")
parser.add_argument("--limit", type=int, default=1000, help="Kline limit per symbol.")
parser.add_argument("--lookbacks", default="16,32", help="Comma-separated LSTM lookback candidates.")
parser.add_argument("--units", default="4,6", help="Comma-separated LSTM unit candidates.")
parser.add_argument("--ridges", default="0.001", help="Comma-separated ridge candidates.")
parser.add_argument("--output", default="", help="Output JSON path. Defaults to TIME_SERIES_LSTM_MODEL_PATH.")
return parser.parse_args()
def _symbols(raw: str, settings: Settings, client: BybitClient) -> list[str]:
if raw.strip():
return [item.strip().upper() for item in raw.split(",") if item.strip()]
if settings.symbols:
return list(settings.symbols)
return client.popular_spot_symbols(settings.top_symbols_count)
def _train_symbol(
*,
client: BybitClient,
settings: Settings,
symbol: str,
interval: str,
limit: int,
lookbacks: list[int],
units_values: list[int],
ridges: list[float],
) -> dict[str, Any] | None:
candles = client.klines(symbol, interval, limit)
closes = [float(candle.close) for candle in candles if candle.close > 0]
returns = _log_returns(closes)
if len(returns) < 80:
return None
validation_window = min(max(8, settings.time_series_validation_window), max(8, len(returns) // 3))
best: dict[str, Any] | None = None
for lookback in lookbacks:
for units in units_values:
for ridge in ridges:
candidate_settings = replace(
settings,
time_series_lstm_enabled=True,
time_series_lstm_lookback=lookback,
time_series_lstm_units=units,
time_series_lstm_ridge=ridge,
)
candidates = _validate_candidates(returns, validation_window, candidate_settings, symbol, {})
baseline = next((item for item in candidates if item["model"] == "naive"), None)
lstm = next((item for item in candidates if item["model"] == "lstm"), None)
if baseline is None or lstm is None:
continue
baseline_mae = float(baseline["mae"])
lstm_mae = float(lstm["mae"])
skill = (baseline_mae - lstm_mae) / baseline_mae if baseline_mae > 0 else 0.0
row = {
"lookback": lookback,
"units": units,
"ridge": ridge,
"validation_mae_percent": lstm_mae * 100,
"baseline_mae_percent": baseline_mae * 100,
"skill": skill,
"candles": len(candles),
"returns": len(returns),
}
if best is None or lstm_mae < best["validation_mae_percent"] / 100:
best = row
return best
def _ints(raw: str) -> list[int]:
return [int(item.strip()) for item in raw.split(",") if item.strip()]
def _floats(raw: str) -> list[float]:
return [float(item.strip()) for item in raw.split(",") if item.strip()]
if __name__ == "__main__":
main()