[CmdletBinding()] param( [string]$Symbols = "", [int]$Limit = 0, [string]$Lookbacks = "", [string]$Architectures = "", [string]$HiddenSizes = "", [string]$Layers = "", [string]$Dropouts = "", [int]$Horizon = 0, [string]$Horizons = "", [string]$Features = "", [string]$ContextSymbols = "", [int]$Seed = 0, [string]$EnsembleSeeds = "", [int]$SelectionFolds = 0, [double]$LearningRate = 0, [double]$WeightDecay = 0, [int]$Epochs = 0, [int]$Patience = 0, [int]$ValidationWindow = 0, [int]$HoldoutWindow = 0, [string]$Interval = "", [string]$EnvFile = "", [string]$OrderbookDb = "", [int]$OrderbookMinSamplesPerBucket = 0, [int]$OrderbookMinCoveredBuckets = 0, [int]$OrderbookMinSymbols = 0, [switch]$Pooled, [switch]$SkipGuard, [switch]$ResumeCandidate ) $ErrorActionPreference = "Stop" $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path $RuntimeDir = Join-Path $RepoRoot "runtime" $LogFile = Join-Path $RuntimeDir "torch_retrain.log" New-Item -ItemType Directory -Force -Path $RuntimeDir | Out-Null function Write-RetrainLog { param([string]$Message) $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ssK" "[$timestamp] $Message" | Tee-Object -FilePath $LogFile -Append } function Invoke-LoggedNativeCommand { param( [string]$FilePath, [object[]]$ArgumentList, [string]$LogPath ) # Windows PowerShell converts redirected native stderr into PowerShell error # records. With the script-wide Stop preference an expected non-zero exit # would jump to catch before callers can inspect LASTEXITCODE. $previousErrorActionPreference = $ErrorActionPreference try { $ErrorActionPreference = "Continue" & $FilePath @ArgumentList 2>&1 | Tee-Object -FilePath $LogPath -Append | Out-Host $exitCode = $LASTEXITCODE } finally { $ErrorActionPreference = $previousErrorActionPreference } return [int]$exitCode } function Resolve-Python { $venvPython = Join-Path $RepoRoot ".venv\Scripts\python.exe" if (Test-Path $venvPython) { return $venvPython } $userPython = Join-Path $env:LOCALAPPDATA "Programs\TradeBotPython312\python.exe" if (Test-Path $userPython) { return $userPython } foreach ($candidate in @("python.exe", "python")) { $command = Get-Command $candidate -ErrorAction SilentlyContinue if (-not $command) { continue } return $command.Source } throw "Python was not found. Create .venv or install Python 3.12." } function Test-TorchArtifactFile { param([string]$Path) if (-not (Test-Path $Path)) { return $false } try { $payload = Get-Content -Raw -LiteralPath $Path | ConvertFrom-Json return $payload.type -eq "pytorch_recurrent_forecaster" -and $null -ne $payload.symbols } catch { return $false } } if (-not $Symbols -and $env:TORCH_RETRAIN_SYMBOLS) { $Symbols = $env:TORCH_RETRAIN_SYMBOLS } if ($Limit -le 0) { $Limit = if ($env:TORCH_RETRAIN_LIMIT) { [int]$env:TORCH_RETRAIN_LIMIT } else { 4000 } } if (-not $Lookbacks) { $Lookbacks = if ($env:TORCH_RETRAIN_LOOKBACKS) { $env:TORCH_RETRAIN_LOOKBACKS } else { "64" } } if (-not $Architectures) { $Architectures = if ($env:TORCH_RETRAIN_ARCHITECTURES) { $env:TORCH_RETRAIN_ARCHITECTURES } else { "lstm" } } if (-not $HiddenSizes) { $HiddenSizes = if ($env:TORCH_RETRAIN_HIDDEN_SIZES) { $env:TORCH_RETRAIN_HIDDEN_SIZES } else { "64" } } if (-not $Layers) { $Layers = if ($env:TORCH_RETRAIN_LAYERS) { $env:TORCH_RETRAIN_LAYERS } else { "2" } } if (-not $Dropouts) { $Dropouts = if ($env:TORCH_RETRAIN_DROPOUTS) { $env:TORCH_RETRAIN_DROPOUTS } else { "0.20" } } if ($Horizon -le 0) { $Horizon = if ($env:TORCH_RETRAIN_HORIZON) { [int]$env:TORCH_RETRAIN_HORIZON } else { 12 } } if (-not $Horizons) { $Horizons = if ($env:TORCH_RETRAIN_HORIZONS) { $env:TORCH_RETRAIN_HORIZONS } else { "3,6,12,24" } } if (-not $Features -and $env:TORCH_RETRAIN_FEATURES) { $Features = $env:TORCH_RETRAIN_FEATURES } if (-not $ContextSymbols -and $env:TORCH_RETRAIN_CONTEXT_SYMBOLS) { $ContextSymbols = $env:TORCH_RETRAIN_CONTEXT_SYMBOLS } if ($Seed -le 0 -and $env:TORCH_RETRAIN_SEED) { $Seed = [int]$env:TORCH_RETRAIN_SEED } if (-not $EnsembleSeeds) { $EnsembleSeeds = if ($env:TORCH_RETRAIN_ENSEMBLE_SEEDS) { $env:TORCH_RETRAIN_ENSEMBLE_SEEDS } else { "7,19" } } if ($SelectionFolds -le 0) { $SelectionFolds = if ($env:TORCH_RETRAIN_SELECTION_FOLDS) { [int]$env:TORCH_RETRAIN_SELECTION_FOLDS } else { 3 } } if ($LearningRate -le 0) { $LearningRate = if ($env:TORCH_RETRAIN_LEARNING_RATE) { [double]$env:TORCH_RETRAIN_LEARNING_RATE } else { 0.0007 } } if ($WeightDecay -le 0) { $WeightDecay = if ($env:TORCH_RETRAIN_WEIGHT_DECAY) { [double]$env:TORCH_RETRAIN_WEIGHT_DECAY } else { 0.0005 } } if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 50 } } if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 8 } } if ($ValidationWindow -le 0) { $ValidationWindow = if ($env:TORCH_RETRAIN_VALIDATION_WINDOW) { [int]$env:TORCH_RETRAIN_VALIDATION_WINDOW } else { 720 } } if ($HoldoutWindow -le 0) { $HoldoutWindow = if ($env:TORCH_RETRAIN_HOLDOUT_WINDOW) { [int]$env:TORCH_RETRAIN_HOLDOUT_WINDOW } else { 1000 } } if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RETRAIN_INTERVAL } if (-not $EnvFile -and $env:TORCH_RETRAIN_ENV) { $EnvFile = $env:TORCH_RETRAIN_ENV } if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" } if (-not $OrderbookDb -and $env:TORCH_ORDERBOOK_DB) { $OrderbookDb = $env:TORCH_ORDERBOOK_DB } if ($OrderbookMinSamplesPerBucket -le 0) { $OrderbookMinSamplesPerBucket = if ($env:TORCH_ORDERBOOK_MIN_SAMPLES_PER_BUCKET) { [int]$env:TORCH_ORDERBOOK_MIN_SAMPLES_PER_BUCKET } else { 20 } } if ($OrderbookMinCoveredBuckets -le 0) { $OrderbookMinCoveredBuckets = if ($env:TORCH_ORDERBOOK_MIN_COVERED_BUCKETS) { [int]$env:TORCH_ORDERBOOK_MIN_COVERED_BUCKETS } else { 240 } } if ($OrderbookMinSymbols -le 0) { $OrderbookMinSymbols = if ($env:TORCH_ORDERBOOK_MIN_SYMBOLS) { [int]$env:TORCH_ORDERBOOK_MIN_SYMBOLS } else { 2 } } $ModelFile = if ($env:TIME_SERIES_LSTM_MODEL_PATH) { $env:TIME_SERIES_LSTM_MODEL_PATH } else { Join-Path $RuntimeDir "lstm_forecaster.json" } if (-not [System.IO.Path]::IsPathRooted($ModelFile)) { $ModelFile = Join-Path $RepoRoot $ModelFile } $CandidateFile = Join-Path $RuntimeDir "lstm_forecaster.candidate.json" $CurrentCalibration = Join-Path $RuntimeDir "torch_guard_current.json" $CandidateCalibration = Join-Path $RuntimeDir "torch_guard_candidate.json" $GuardReport = Join-Path $RuntimeDir "torch_retrain_guard.json" $ShadowModelFile = Join-Path $RuntimeDir "lstm_forecaster.shadow.json" $ShadowCalibration = Join-Path $RuntimeDir "torch_shadow_calibration.json" $ShadowGuard = Join-Path $RuntimeDir "torch_shadow_guard.json" $ShadowMode = -not [string]::IsNullOrWhiteSpace($OrderbookDb) $mutex = New-Object System.Threading.Mutex($false, "TradeBotTorchRecurrentRetrainer") $hasLock = $false $pushedLocation = $false try { $hasLock = $mutex.WaitOne(0) if (-not $hasLock) { Write-RetrainLog "Another PyTorch recurrent retrain is already running; skipping." exit 0 } $python = Resolve-Python $trainerArgs = @( "-u", "tools\train_torch_recurrent_forecaster.py", "--limit", $Limit.ToString(), "--lookbacks", $Lookbacks, "--architectures", $Architectures, "--hidden-sizes", $HiddenSizes, "--layers", $Layers, "--dropouts", $Dropouts, "--epochs", $Epochs.ToString(), "--patience", $Patience.ToString(), "--validation-window", $ValidationWindow.ToString(), "--holdout-window", $HoldoutWindow.ToString(), "--ensemble-seeds", $EnsembleSeeds, "--selection-folds", $SelectionFolds.ToString(), "--learning-rate", $LearningRate.ToString([Globalization.CultureInfo]::InvariantCulture), "--weight-decay", $WeightDecay.ToString([Globalization.CultureInfo]::InvariantCulture), "--output", $CandidateFile ) if ($Pooled) { $trainerArgs += "--pooled" } else { $trainerArgs += "--no-pooled" } if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) } if ($Interval) { $trainerArgs += @("--interval", $Interval) } if ($EnvFile) { $trainerArgs += @("--env", $EnvFile) } if ($Horizon -gt 0) { $trainerArgs += @("--horizon", $Horizon.ToString()) } if ($Horizons) { $trainerArgs += @("--horizons", $Horizons) } if ($Features) { $trainerArgs += @("--features", $Features) } if ($ContextSymbols) { $trainerArgs += @("--context-symbols", $ContextSymbols) } if ($Seed -gt 0) { $trainerArgs += @("--seed", $Seed.ToString()) } if ($OrderbookDb) { $trainerArgs += @( "--orderbook-db", $OrderbookDb, "--orderbook-min-samples-per-bucket", $OrderbookMinSamplesPerBucket.ToString(), "--orderbook-min-covered-buckets", $OrderbookMinCoveredBuckets.ToString(), "--orderbook-min-symbols", $OrderbookMinSymbols.ToString() ) } Push-Location $RepoRoot $pushedLocation = $true if ($ResumeCandidate) { if (-not (Test-TorchArtifactFile $CandidateFile)) { throw "ResumeCandidate requested, but no valid candidate artifact exists: $CandidateFile" } Write-RetrainLog "Resuming guard from existing candidate artifact: $CandidateFile" } else { Write-RetrainLog "Starting PyTorch recurrent retrain: $python $($trainerArgs -join ' ')" $trainerExitCode = Invoke-LoggedNativeCommand -FilePath $python -ArgumentList $trainerArgs -LogPath $LogFile if ($trainerExitCode -ne 0) { if (Test-TorchArtifactFile $CandidateFile) { Write-RetrainLog "WARNING: Trainer exited with code $trainerExitCode after writing a valid candidate artifact; continuing to guard." } else { throw "Trainer failed with exit code $trainerExitCode." } } Write-RetrainLog "Finished PyTorch recurrent retrain candidate: $CandidateFile" } if ($SkipGuard) { throw "SkipGuard is disabled: every candidate must pass untouched-holdout validation." } $calibrationBaseArgs = @( "-u", "tools\calibrate_torch_thresholds.py", "--limit", $Limit.ToString(), "--horizon", $Horizon.ToString(), "--calibration-window", ([Math]::Min(2400, [Math]::Max(1200, [int]($Limit / 2)))).ToString(), "--min-trades", "24", "--walk-forward-folds", "8", "--confidence-grid", "0.40" ) if ($Symbols) { $calibrationBaseArgs += @("--symbols", $Symbols) } if ($EnvFile) { $calibrationBaseArgs += @("--env", $EnvFile) } if ($OrderbookDb) { $calibrationBaseArgs += @( "--orderbook-db", $OrderbookDb, "--orderbook-min-samples-per-bucket", $OrderbookMinSamplesPerBucket.ToString() ) } if (Test-Path $ModelFile) { Write-RetrainLog "Calibrating current artifact for guard." $currentCalibrationExitCode = Invoke-LoggedNativeCommand ` -FilePath $python ` -ArgumentList ($calibrationBaseArgs + @("--artifact", $ModelFile, "--output", $CurrentCalibration)) ` -LogPath $LogFile if ($currentCalibrationExitCode -ne 0) { Write-RetrainLog "Current artifact has no compatible untouched holdout; comparing candidate against an empty current report." "{}" | Set-Content -LiteralPath $CurrentCalibration -Encoding utf8 } } else { Write-RetrainLog "No active artifact yet; candidate still must pass the full guard." "{}" | Set-Content -LiteralPath $CurrentCalibration -Encoding utf8 } Write-RetrainLog "Calibrating candidate artifact for guard." $candidateCalibrationExitCode = Invoke-LoggedNativeCommand ` -FilePath $python ` -ArgumentList ($calibrationBaseArgs + @("--artifact", $CandidateFile, "--output", $CandidateCalibration)) ` -LogPath $LogFile if ($candidateCalibrationExitCode -ne 0) { throw "Candidate artifact calibration failed with exit code $candidateCalibrationExitCode." } Write-RetrainLog "Running retrain guard." $GuardTarget = if ($ShadowMode) { $ShadowModelFile } else { $ModelFile } $guardArgs = @( "-u", "tools\accept_torch_candidate.py", "--current-report", $CurrentCalibration, "--candidate-report", $CandidateCalibration, "--candidate-artifact", $CandidateFile, "--target-artifact", $GuardTarget, "--report", $GuardReport ) $guardExitCode = Invoke-LoggedNativeCommand -FilePath $python -ArgumentList $guardArgs -LogPath $LogFile if ($guardExitCode -eq 2) { Write-RetrainLog "Candidate rejected by guard; keeping active artifact: $ModelFile" exit 0 } if ($guardExitCode -ne 0) { throw "Retrain guard failed with exit code $guardExitCode." } if (Test-Path $CandidateCalibration) { if ($ShadowMode) { Copy-Item -Force -LiteralPath $CandidateCalibration -Destination $ShadowCalibration Copy-Item -Force -LiteralPath $GuardReport -Destination $ShadowGuard Write-RetrainLog "Candidate passed offline gate and was staged for shadow only: $ShadowModelFile" } else { Copy-Item -Force -LiteralPath $CandidateCalibration -Destination (Join-Path $RuntimeDir "torch_threshold_calibration.json") Write-RetrainLog "Updated active threshold calibration: $(Join-Path $RuntimeDir "torch_threshold_calibration.json")" } } if ($ShadowMode) { Write-RetrainLog "Candidate accepted by offline guard. Active artifact was not changed: $ModelFile" } else { Write-RetrainLog "Candidate accepted by guard. Active artifact: $ModelFile" } } catch { Write-RetrainLog "ERROR: $($_.Exception.Message)" exit 1 } finally { if ($pushedLocation) { Pop-Location -ErrorAction SilentlyContinue } if ($hasLock) { $mutex.ReleaseMutex() } $mutex.Dispose() }