Add automated LSTM retraining
This commit is contained in:
Codex
@@ -72,6 +72,22 @@ python tools\train_lstm_forecaster.py --symbols BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,
|
||||
|
||||
Файл из `TIME_SERIES_LSTM_MODEL_PATH` читается ботом автоматически. Даже если LSTM-параметры сохранены, сделка меняется только тогда, когда текущая walk-forward проверка в `crypto_spot_bot/time_series.py` показывает качество лучше baseline.
|
||||
|
||||
Автопереобучение запускает тот же train-скрипт, пишет лог в `runtime/lstm_retrain.log` и защищается от параллельных запусков:
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File tools\run_lstm_retrain.ps1
|
||||
powershell -ExecutionPolicy Bypass -File tools\install_windows_lstm_retrainer.ps1
|
||||
```
|
||||
|
||||
На Linux/Raspberry Pi можно включить user systemd timer:
|
||||
|
||||
```bash
|
||||
bash tools/run_lstm_retrain.sh
|
||||
bash tools/install_lstm_retrainer_systemd.sh
|
||||
```
|
||||
|
||||
По умолчанию расписание переобучает каждые 6 часов с `--limit 1000`; Windows-установщик фиксирует пары `BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,LTCUSDT`, чтобы первый scheduled run был предсказуемым. Параметры можно переопределить через env: `LSTM_RETRAIN_SYMBOLS`, `LSTM_RETRAIN_LIMIT`, `LSTM_RETRAIN_LOOKBACKS`, `LSTM_RETRAIN_UNITS`, `LSTM_RETRAIN_RIDGES`, `LSTM_RETRAIN_INTERVAL`, `LSTM_RETRAIN_ENV`.
|
||||
|
||||
## Docker
|
||||
|
||||
```bash
|
||||
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
SYSTEMD_DIR="$HOME/.config/systemd/user"
|
||||
SERVICE_NAME="tradebot-lstm-retrainer.service"
|
||||
TIMER_NAME="tradebot-lstm-retrainer.timer"
|
||||
|
||||
mkdir -p "$SYSTEMD_DIR"
|
||||
|
||||
cat > "$SYSTEMD_DIR/$SERVICE_NAME" <<EOF
|
||||
[Unit]
|
||||
Description=Retrain TradeBot LSTM forecast parameters
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=$REPO_ROOT
|
||||
ExecStart=$REPO_ROOT/tools/run_lstm_retrain.sh
|
||||
EOF
|
||||
|
||||
cat > "$SYSTEMD_DIR/$TIMER_NAME" <<EOF
|
||||
[Unit]
|
||||
Description=Retrain TradeBot LSTM forecast parameters every 6 hours
|
||||
|
||||
[Timer]
|
||||
OnBootSec=5min
|
||||
OnUnitActiveSec=6h
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
EOF
|
||||
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable --now "$TIMER_NAME"
|
||||
|
||||
echo "Enabled user timer $TIMER_NAME. Check with: systemctl --user list-timers $TIMER_NAME"
|
||||
@@ -0,0 +1,49 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$TaskName = "TradeBot LSTM Retrainer",
|
||||
[int]$EveryHours = 6,
|
||||
[string]$Symbols = "BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,LTCUSDT",
|
||||
[int]$Limit = 1000
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
||||
$Runner = Join-Path $RepoRoot "tools\run_lstm_retrain.ps1"
|
||||
if (-not (Test-Path $Runner)) {
|
||||
throw "Runner not found: $Runner"
|
||||
}
|
||||
|
||||
$actionArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$Runner`""
|
||||
if ($Symbols) {
|
||||
$actionArgs += " -Symbols `"$Symbols`""
|
||||
}
|
||||
if ($Limit -gt 0) {
|
||||
$actionArgs += " -Limit $Limit"
|
||||
}
|
||||
$action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $actionArgs -WorkingDirectory $RepoRoot
|
||||
$trigger = New-ScheduledTaskTrigger `
|
||||
-Once `
|
||||
-At (Get-Date).AddMinutes(5) `
|
||||
-RepetitionInterval (New-TimeSpan -Hours $EveryHours) `
|
||||
-RepetitionDuration (New-TimeSpan -Days 3650)
|
||||
$principal = New-ScheduledTaskPrincipal `
|
||||
-UserId ([System.Security.Principal.WindowsIdentity]::GetCurrent().Name) `
|
||||
-LogonType Interactive `
|
||||
-RunLevel Limited
|
||||
$settings = New-ScheduledTaskSettingsSet `
|
||||
-StartWhenAvailable `
|
||||
-MultipleInstances IgnoreNew `
|
||||
-AllowStartIfOnBatteries `
|
||||
-DontStopIfGoingOnBatteries
|
||||
|
||||
Register-ScheduledTask `
|
||||
-TaskName $TaskName `
|
||||
-Action $action `
|
||||
-Trigger $trigger `
|
||||
-Principal $principal `
|
||||
-Settings $settings `
|
||||
-Description "Retrains TradeBot LSTM forecast parameters every $EveryHours hours." `
|
||||
-Force | Out-Null
|
||||
|
||||
Write-Host "Registered scheduled task '$TaskName' every $EveryHours hours."
|
||||
@@ -0,0 +1,102 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$Symbols = "",
|
||||
[int]$Limit = 0,
|
||||
[string]$Lookbacks = "",
|
||||
[string]$Units = "",
|
||||
[string]$Ridges = "",
|
||||
[string]$Interval = "",
|
||||
[string]$EnvFile = ""
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
||||
$RuntimeDir = Join-Path $RepoRoot "runtime"
|
||||
$LogFile = Join-Path $RuntimeDir "lstm_retrain.log"
|
||||
New-Item -ItemType Directory -Force -Path $RuntimeDir | Out-Null
|
||||
|
||||
function Write-RetrainLog {
|
||||
param([string]$Message)
|
||||
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ssK"
|
||||
"[$timestamp] $Message" | Tee-Object -FilePath $LogFile -Append
|
||||
}
|
||||
|
||||
function Resolve-Python {
|
||||
$venvPython = Join-Path $RepoRoot ".venv\Scripts\python.exe"
|
||||
if (Test-Path $venvPython) {
|
||||
return $venvPython
|
||||
}
|
||||
|
||||
$userPython = Join-Path $env:LOCALAPPDATA "Programs\TradeBotPython312\python.exe"
|
||||
if (Test-Path $userPython) {
|
||||
return $userPython
|
||||
}
|
||||
|
||||
foreach ($candidate in @("python.exe", "python")) {
|
||||
$command = Get-Command $candidate -ErrorAction SilentlyContinue
|
||||
if (-not $command) {
|
||||
continue
|
||||
}
|
||||
return $command.Source
|
||||
}
|
||||
throw "Python was not found. Create .venv or install Python 3.12."
|
||||
}
|
||||
|
||||
if (-not $Symbols -and $env:LSTM_RETRAIN_SYMBOLS) { $Symbols = $env:LSTM_RETRAIN_SYMBOLS }
|
||||
if ($Limit -le 0) {
|
||||
$Limit = if ($env:LSTM_RETRAIN_LIMIT) { [int]$env:LSTM_RETRAIN_LIMIT } else { 1000 }
|
||||
}
|
||||
if (-not $Lookbacks) { $Lookbacks = if ($env:LSTM_RETRAIN_LOOKBACKS) { $env:LSTM_RETRAIN_LOOKBACKS } else { "16,32" } }
|
||||
if (-not $Units) { $Units = if ($env:LSTM_RETRAIN_UNITS) { $env:LSTM_RETRAIN_UNITS } else { "4,6" } }
|
||||
if (-not $Ridges) { $Ridges = if ($env:LSTM_RETRAIN_RIDGES) { $env:LSTM_RETRAIN_RIDGES } else { "0.001" } }
|
||||
if (-not $Interval -and $env:LSTM_RETRAIN_INTERVAL) { $Interval = $env:LSTM_RETRAIN_INTERVAL }
|
||||
if (-not $EnvFile -and $env:LSTM_RETRAIN_ENV) { $EnvFile = $env:LSTM_RETRAIN_ENV }
|
||||
if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" }
|
||||
|
||||
$mutex = New-Object System.Threading.Mutex($false, "TradeBotLstmRetrainer")
|
||||
$hasLock = $false
|
||||
$pushedLocation = $false
|
||||
|
||||
try {
|
||||
$hasLock = $mutex.WaitOne(0)
|
||||
if (-not $hasLock) {
|
||||
Write-RetrainLog "Another LSTM retrain is already running; skipping."
|
||||
exit 0
|
||||
}
|
||||
|
||||
$python = Resolve-Python
|
||||
$trainerArgs = @(
|
||||
"-u",
|
||||
"tools\train_lstm_forecaster.py",
|
||||
"--limit", $Limit.ToString(),
|
||||
"--lookbacks", $Lookbacks,
|
||||
"--units", $Units,
|
||||
"--ridges", $Ridges
|
||||
)
|
||||
if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) }
|
||||
if ($Interval) { $trainerArgs += @("--interval", $Interval) }
|
||||
if ($EnvFile) { $trainerArgs += @("--env", $EnvFile) }
|
||||
|
||||
Push-Location $RepoRoot
|
||||
$pushedLocation = $true
|
||||
Write-RetrainLog "Starting LSTM retrain: $python $($trainerArgs -join ' ')"
|
||||
& $python @trainerArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Trainer failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
Write-RetrainLog "Finished LSTM retrain."
|
||||
}
|
||||
catch {
|
||||
Write-RetrainLog "ERROR: $($_.Exception.Message)"
|
||||
exit 1
|
||||
}
|
||||
finally {
|
||||
if ($pushedLocation) {
|
||||
Pop-Location -ErrorAction SilentlyContinue
|
||||
}
|
||||
if ($hasLock) {
|
||||
$mutex.ReleaseMutex()
|
||||
}
|
||||
$mutex.Dispose()
|
||||
}
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
RUNTIME_DIR="$REPO_ROOT/runtime"
|
||||
LOG_FILE="$RUNTIME_DIR/lstm_retrain.log"
|
||||
LOCK_FILE="$RUNTIME_DIR/lstm_retrain.lock"
|
||||
|
||||
mkdir -p "$RUNTIME_DIR"
|
||||
|
||||
log() {
|
||||
printf '[%s] %s\n' "$(date -Is)" "$*" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
if command -v flock >/dev/null 2>&1; then
|
||||
exec 9>"$LOCK_FILE"
|
||||
if ! flock -n 9; then
|
||||
log "Another LSTM retrain is already running; skipping."
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
LOCK_DIR="$LOCK_FILE.d"
|
||||
if ! mkdir "$LOCK_DIR" 2>/dev/null; then
|
||||
log "Another LSTM retrain is already running; skipping."
|
||||
exit 0
|
||||
fi
|
||||
trap 'rmdir "$LOCK_DIR"' EXIT
|
||||
fi
|
||||
|
||||
if [[ -x "$REPO_ROOT/.venv/bin/python" ]]; then
|
||||
PYTHON="$REPO_ROOT/.venv/bin/python"
|
||||
elif command -v python3 >/dev/null 2>&1; then
|
||||
PYTHON="$(command -v python3)"
|
||||
elif command -v python >/dev/null 2>&1; then
|
||||
PYTHON="$(command -v python)"
|
||||
else
|
||||
log "ERROR: Python was not found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SYMBOLS="${LSTM_RETRAIN_SYMBOLS:-}"
|
||||
LIMIT="${LSTM_RETRAIN_LIMIT:-1000}"
|
||||
LOOKBACKS="${LSTM_RETRAIN_LOOKBACKS:-16,32}"
|
||||
UNITS="${LSTM_RETRAIN_UNITS:-4,6}"
|
||||
RIDGES="${LSTM_RETRAIN_RIDGES:-0.001}"
|
||||
INTERVAL="${LSTM_RETRAIN_INTERVAL:-}"
|
||||
ENV_FILE="${LSTM_RETRAIN_ENV:-}"
|
||||
|
||||
if [[ -z "$ENV_FILE" && -f "$REPO_ROOT/.env" ]]; then
|
||||
ENV_FILE="$REPO_ROOT/.env"
|
||||
fi
|
||||
|
||||
args=(
|
||||
"tools/train_lstm_forecaster.py"
|
||||
"--limit" "$LIMIT"
|
||||
"--lookbacks" "$LOOKBACKS"
|
||||
"--units" "$UNITS"
|
||||
"--ridges" "$RIDGES"
|
||||
)
|
||||
|
||||
if [[ -n "$SYMBOLS" ]]; then args+=("--symbols" "$SYMBOLS"); fi
|
||||
if [[ -n "$INTERVAL" ]]; then args+=("--interval" "$INTERVAL"); fi
|
||||
if [[ -n "$ENV_FILE" ]]; then args+=("--env" "$ENV_FILE"); fi
|
||||
|
||||
cd "$REPO_ROOT"
|
||||
log "Starting LSTM retrain: $PYTHON -u ${args[*]}"
|
||||
"$PYTHON" -u "${args[@]}" 2>&1 | tee -a "$LOG_FILE"
|
||||
log "Finished LSTM retrain."
|
||||
@@ -56,7 +56,9 @@ def main() -> None:
|
||||
)
|
||||
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(artifact, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
tmp_output = output.with_name(f"{output.name}.tmp")
|
||||
tmp_output.write_text(json.dumps(artifact, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
tmp_output.replace(output)
|
||||
print(f"saved {output}")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user