[CmdletBinding()] param( [string]$Symbols = "", [int]$Limit = 0, [string]$Lookbacks = "", [string]$Architectures = "", [string]$HiddenSizes = "", [string]$Layers = "", [string]$Dropouts = "", [int]$Horizon = 0, [string]$Horizons = "", [string]$Features = "", [string]$ContextSymbols = "", [int]$Seed = 0, [int]$Epochs = 0, [int]$Patience = 0, [string]$Interval = "", [string]$EnvFile = "", [switch]$DeployToPi, [string]$PiHost = "", [string]$PiUser = "", [string]$PiRoot = "", [string]$PiSshKeyPath = "", [switch]$NoPiRestart, [switch]$SkipGuard ) $ErrorActionPreference = "Stop" $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path $RuntimeDir = Join-Path $RepoRoot "runtime" $LogFile = Join-Path $RuntimeDir "torch_retrain.log" New-Item -ItemType Directory -Force -Path $RuntimeDir | Out-Null function Write-RetrainLog { param([string]$Message) $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ssK" "[$timestamp] $Message" | Tee-Object -FilePath $LogFile -Append } function Resolve-Python { $venvPython = Join-Path $RepoRoot ".venv\Scripts\python.exe" if (Test-Path $venvPython) { return $venvPython } $userPython = Join-Path $env:LOCALAPPDATA "Programs\TradeBotPython312\python.exe" if (Test-Path $userPython) { return $userPython } foreach ($candidate in @("python.exe", "python")) { $command = Get-Command $candidate -ErrorAction SilentlyContinue if (-not $command) { continue } return $command.Source } throw "Python was not found. Create .venv or install Python 3.12." } function Test-TorchArtifactFile { param([string]$Path) if (-not (Test-Path $Path)) { return $false } try { $payload = Get-Content -Raw -LiteralPath $Path | ConvertFrom-Json return $payload.type -eq "pytorch_recurrent_forecaster" -and $null -ne $payload.symbols } catch { return $false } } function Sync-AcceptedArtifactsToPi { if (-not ($DeployToPi -or $env:TORCH_RETRAIN_DEPLOY_TO_PI)) { Write-RetrainLog "Pi artifact sync disabled." return } $syncScript = Join-Path $RepoRoot "tools\sync_torch_artifacts_to_pi.ps1" if (-not (Test-Path $syncScript)) { throw "Pi sync script not found: $syncScript" } $syncArgs = @( "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $syncScript, "-RepoRoot", $RepoRoot ) if ($PiHost) { $syncArgs += @("-RemoteHost", $PiHost) } if ($PiUser) { $syncArgs += @("-RemoteUser", $PiUser) } if ($PiRoot) { $syncArgs += @("-RemoteRoot", $PiRoot) } if ($PiSshKeyPath) { $syncArgs += @("-SshKeyPath", $PiSshKeyPath) } if ($NoPiRestart) { $syncArgs += "-NoRestart" } Write-RetrainLog "Syncing accepted Torch artifacts to Raspberry Pi." & powershell.exe @syncArgs 2>&1 | Tee-Object -FilePath $LogFile -Append if ($LASTEXITCODE -ne 0) { throw "Pi artifact sync failed with exit code $LASTEXITCODE." } Write-RetrainLog "Pi artifact sync completed." } if (-not $Symbols -and $env:TORCH_RETRAIN_SYMBOLS) { $Symbols = $env:TORCH_RETRAIN_SYMBOLS } if ($Limit -le 0) { $Limit = if ($env:TORCH_RETRAIN_LIMIT) { [int]$env:TORCH_RETRAIN_LIMIT } else { 3000 } } if (-not $Lookbacks) { $Lookbacks = if ($env:TORCH_RETRAIN_LOOKBACKS) { $env:TORCH_RETRAIN_LOOKBACKS } else { "64" } } if (-not $Architectures) { $Architectures = if ($env:TORCH_RETRAIN_ARCHITECTURES) { $env:TORCH_RETRAIN_ARCHITECTURES } else { "lstm,gru" } } if (-not $HiddenSizes) { $HiddenSizes = if ($env:TORCH_RETRAIN_HIDDEN_SIZES) { $env:TORCH_RETRAIN_HIDDEN_SIZES } else { "64,96" } } if (-not $Layers) { $Layers = if ($env:TORCH_RETRAIN_LAYERS) { $env:TORCH_RETRAIN_LAYERS } else { "2" } } if (-not $Dropouts) { $Dropouts = if ($env:TORCH_RETRAIN_DROPOUTS) { $env:TORCH_RETRAIN_DROPOUTS } else { "0.15" } } if ($Horizon -le 0 -and $env:TORCH_RETRAIN_HORIZON) { $Horizon = [int]$env:TORCH_RETRAIN_HORIZON } if (-not $Horizons -and $env:TORCH_RETRAIN_HORIZONS) { $Horizons = $env:TORCH_RETRAIN_HORIZONS } if (-not $Features -and $env:TORCH_RETRAIN_FEATURES) { $Features = $env:TORCH_RETRAIN_FEATURES } if (-not $ContextSymbols -and $env:TORCH_RETRAIN_CONTEXT_SYMBOLS) { $ContextSymbols = $env:TORCH_RETRAIN_CONTEXT_SYMBOLS } if ($Seed -le 0 -and $env:TORCH_RETRAIN_SEED) { $Seed = [int]$env:TORCH_RETRAIN_SEED } if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 70 } } if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 8 } } if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RETRAIN_INTERVAL } if (-not $EnvFile -and $env:TORCH_RETRAIN_ENV) { $EnvFile = $env:TORCH_RETRAIN_ENV } if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" } $ModelFile = if ($env:TIME_SERIES_LSTM_MODEL_PATH) { $env:TIME_SERIES_LSTM_MODEL_PATH } else { Join-Path $RuntimeDir "lstm_forecaster.json" } if (-not [System.IO.Path]::IsPathRooted($ModelFile)) { $ModelFile = Join-Path $RepoRoot $ModelFile } $CandidateFile = Join-Path $RuntimeDir "lstm_forecaster.candidate.json" $CurrentCalibration = Join-Path $RuntimeDir "torch_guard_current.json" $CandidateCalibration = Join-Path $RuntimeDir "torch_guard_candidate.json" $GuardReport = Join-Path $RuntimeDir "torch_retrain_guard.json" $mutex = New-Object System.Threading.Mutex($false, "TradeBotTorchRecurrentRetrainer") $hasLock = $false $pushedLocation = $false try { $hasLock = $mutex.WaitOne(0) if (-not $hasLock) { Write-RetrainLog "Another PyTorch recurrent retrain is already running; skipping." exit 0 } $python = Resolve-Python $trainerArgs = @( "-u", "tools\train_torch_recurrent_forecaster.py", "--limit", $Limit.ToString(), "--lookbacks", $Lookbacks, "--architectures", $Architectures, "--hidden-sizes", $HiddenSizes, "--layers", $Layers, "--dropouts", $Dropouts, "--epochs", $Epochs.ToString(), "--patience", $Patience.ToString(), "--output", $CandidateFile ) if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) } if ($Interval) { $trainerArgs += @("--interval", $Interval) } if ($EnvFile) { $trainerArgs += @("--env", $EnvFile) } if ($Horizon -gt 0) { $trainerArgs += @("--horizon", $Horizon.ToString()) } if ($Horizons) { $trainerArgs += @("--horizons", $Horizons) } if ($Features) { $trainerArgs += @("--features", $Features) } if ($ContextSymbols) { $trainerArgs += @("--context-symbols", $ContextSymbols) } if ($Seed -gt 0) { $trainerArgs += @("--seed", $Seed.ToString()) } Push-Location $RepoRoot $pushedLocation = $true Write-RetrainLog "Starting PyTorch recurrent retrain: $python $($trainerArgs -join ' ')" & $python @trainerArgs 2>&1 | Tee-Object -FilePath $LogFile -Append $trainerExitCode = $LASTEXITCODE if ($trainerExitCode -ne 0) { if (Test-TorchArtifactFile $CandidateFile) { Write-RetrainLog "WARNING: Trainer exited with code $trainerExitCode after writing a valid candidate artifact; continuing to guard." } else { throw "Trainer failed with exit code $trainerExitCode." } } Write-RetrainLog "Finished PyTorch recurrent retrain candidate: $CandidateFile" if ($SkipGuard -or -not (Test-Path $ModelFile)) { Move-Item -Force -LiteralPath $CandidateFile -Destination $ModelFile Write-RetrainLog "Accepted candidate without guard. Active artifact: $ModelFile" Sync-AcceptedArtifactsToPi exit 0 } $calibrationBaseArgs = @( "-u", "tools\calibrate_torch_thresholds.py", "--limit", "3000", "--calibration-window", "1200", "--min-trades", "60", "--walk-forward-folds", "8", "--confidence-grid", "0.40" ) if ($Symbols) { $calibrationBaseArgs += @("--symbols", $Symbols) } if ($EnvFile) { $calibrationBaseArgs += @("--env", $EnvFile) } Write-RetrainLog "Calibrating current artifact for guard." & $python @($calibrationBaseArgs + @("--artifact", $ModelFile, "--output", $CurrentCalibration)) 2>&1 | Tee-Object -FilePath $LogFile -Append if ($LASTEXITCODE -ne 0) { throw "Current artifact calibration failed with exit code $LASTEXITCODE." } Write-RetrainLog "Calibrating candidate artifact for guard." & $python @($calibrationBaseArgs + @("--artifact", $CandidateFile, "--output", $CandidateCalibration)) 2>&1 | Tee-Object -FilePath $LogFile -Append if ($LASTEXITCODE -ne 0) { throw "Candidate artifact calibration failed with exit code $LASTEXITCODE." } Write-RetrainLog "Running retrain guard." & $python -u "tools\accept_torch_candidate.py" ` --current-report $CurrentCalibration ` --candidate-report $CandidateCalibration ` --candidate-artifact $CandidateFile ` --target-artifact $ModelFile ` --report $GuardReport 2>&1 | Tee-Object -FilePath $LogFile -Append if ($LASTEXITCODE -eq 2) { Write-RetrainLog "Candidate rejected by guard; keeping active artifact: $ModelFile" exit 0 } if ($LASTEXITCODE -ne 0) { throw "Retrain guard failed with exit code $LASTEXITCODE." } if (Test-Path $CandidateCalibration) { Copy-Item -Force -LiteralPath $CandidateCalibration -Destination (Join-Path $RuntimeDir "torch_threshold_calibration.json") Write-RetrainLog "Updated active threshold calibration: $(Join-Path $RuntimeDir "torch_threshold_calibration.json")" } Write-RetrainLog "Candidate accepted by guard. Active artifact: $ModelFile" Sync-AcceptedArtifactsToPi } catch { Write-RetrainLog "ERROR: $($_.Exception.Message)" exit 1 } finally { if ($pushedLocation) { Pop-Location -ErrorAction SilentlyContinue } if ($hasLock) { $mutex.ReleaseMutex() } $mutex.Dispose() }