282 lines
12 KiB
PowerShell
282 lines
12 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[string]$Symbols = "",
|
|
[int]$Limit = 0,
|
|
[string]$Lookbacks = "",
|
|
[string]$Architectures = "",
|
|
[string]$HiddenSizes = "",
|
|
[string]$Layers = "",
|
|
[string]$Dropouts = "",
|
|
[int]$Horizon = 0,
|
|
[string]$Horizons = "",
|
|
[string]$Features = "",
|
|
[string]$ContextSymbols = "",
|
|
[int]$Seed = 0,
|
|
[string]$EnsembleSeeds = "",
|
|
[int]$SelectionFolds = 0,
|
|
[double]$LearningRate = 0,
|
|
[double]$WeightDecay = 0,
|
|
[int]$Epochs = 0,
|
|
[int]$Patience = 0,
|
|
[int]$ValidationWindow = 0,
|
|
[int]$HoldoutWindow = 0,
|
|
[string]$Interval = "",
|
|
[string]$EnvFile = "",
|
|
[switch]$Pooled,
|
|
[switch]$SkipGuard,
|
|
[switch]$ResumeCandidate
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
|
$RuntimeDir = Join-Path $RepoRoot "runtime"
|
|
$LogFile = Join-Path $RuntimeDir "torch_retrain.log"
|
|
New-Item -ItemType Directory -Force -Path $RuntimeDir | Out-Null
|
|
|
|
function Write-RetrainLog {
|
|
param([string]$Message)
|
|
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ssK"
|
|
"[$timestamp] $Message" | Tee-Object -FilePath $LogFile -Append
|
|
}
|
|
|
|
function Invoke-LoggedNativeCommand {
|
|
param(
|
|
[string]$FilePath,
|
|
[object[]]$ArgumentList,
|
|
[string]$LogPath
|
|
)
|
|
|
|
# Windows PowerShell converts redirected native stderr into PowerShell error
|
|
# records. With the script-wide Stop preference an expected non-zero exit
|
|
# would jump to catch before callers can inspect LASTEXITCODE.
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
try {
|
|
$ErrorActionPreference = "Continue"
|
|
& $FilePath @ArgumentList 2>&1 |
|
|
Tee-Object -FilePath $LogPath -Append |
|
|
Out-Host
|
|
$exitCode = $LASTEXITCODE
|
|
}
|
|
finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
return [int]$exitCode
|
|
}
|
|
|
|
function Resolve-Python {
|
|
$venvPython = Join-Path $RepoRoot ".venv\Scripts\python.exe"
|
|
if (Test-Path $venvPython) {
|
|
return $venvPython
|
|
}
|
|
|
|
$userPython = Join-Path $env:LOCALAPPDATA "Programs\TradeBotPython312\python.exe"
|
|
if (Test-Path $userPython) {
|
|
return $userPython
|
|
}
|
|
|
|
foreach ($candidate in @("python.exe", "python")) {
|
|
$command = Get-Command $candidate -ErrorAction SilentlyContinue
|
|
if (-not $command) {
|
|
continue
|
|
}
|
|
return $command.Source
|
|
}
|
|
throw "Python was not found. Create .venv or install Python 3.12."
|
|
}
|
|
|
|
function Test-TorchArtifactFile {
|
|
param([string]$Path)
|
|
if (-not (Test-Path $Path)) {
|
|
return $false
|
|
}
|
|
try {
|
|
$payload = Get-Content -Raw -LiteralPath $Path | ConvertFrom-Json
|
|
return $payload.type -eq "pytorch_recurrent_forecaster" -and $null -ne $payload.symbols
|
|
}
|
|
catch {
|
|
return $false
|
|
}
|
|
}
|
|
|
|
if (-not $Symbols -and $env:TORCH_RETRAIN_SYMBOLS) { $Symbols = $env:TORCH_RETRAIN_SYMBOLS }
|
|
if ($Limit -le 0) {
|
|
$Limit = if ($env:TORCH_RETRAIN_LIMIT) { [int]$env:TORCH_RETRAIN_LIMIT } else { 4000 }
|
|
}
|
|
if (-not $Lookbacks) { $Lookbacks = if ($env:TORCH_RETRAIN_LOOKBACKS) { $env:TORCH_RETRAIN_LOOKBACKS } else { "64" } }
|
|
if (-not $Architectures) { $Architectures = if ($env:TORCH_RETRAIN_ARCHITECTURES) { $env:TORCH_RETRAIN_ARCHITECTURES } else { "lstm" } }
|
|
if (-not $HiddenSizes) { $HiddenSizes = if ($env:TORCH_RETRAIN_HIDDEN_SIZES) { $env:TORCH_RETRAIN_HIDDEN_SIZES } else { "64" } }
|
|
if (-not $Layers) { $Layers = if ($env:TORCH_RETRAIN_LAYERS) { $env:TORCH_RETRAIN_LAYERS } else { "2" } }
|
|
if (-not $Dropouts) { $Dropouts = if ($env:TORCH_RETRAIN_DROPOUTS) { $env:TORCH_RETRAIN_DROPOUTS } else { "0.20" } }
|
|
if ($Horizon -le 0) { $Horizon = if ($env:TORCH_RETRAIN_HORIZON) { [int]$env:TORCH_RETRAIN_HORIZON } else { 12 } }
|
|
if (-not $Horizons) { $Horizons = if ($env:TORCH_RETRAIN_HORIZONS) { $env:TORCH_RETRAIN_HORIZONS } else { "3,6,12,24" } }
|
|
if (-not $Features -and $env:TORCH_RETRAIN_FEATURES) { $Features = $env:TORCH_RETRAIN_FEATURES }
|
|
if (-not $ContextSymbols -and $env:TORCH_RETRAIN_CONTEXT_SYMBOLS) { $ContextSymbols = $env:TORCH_RETRAIN_CONTEXT_SYMBOLS }
|
|
if ($Seed -le 0 -and $env:TORCH_RETRAIN_SEED) { $Seed = [int]$env:TORCH_RETRAIN_SEED }
|
|
if (-not $EnsembleSeeds) { $EnsembleSeeds = if ($env:TORCH_RETRAIN_ENSEMBLE_SEEDS) { $env:TORCH_RETRAIN_ENSEMBLE_SEEDS } else { "7,19" } }
|
|
if ($SelectionFolds -le 0) { $SelectionFolds = if ($env:TORCH_RETRAIN_SELECTION_FOLDS) { [int]$env:TORCH_RETRAIN_SELECTION_FOLDS } else { 3 } }
|
|
if ($LearningRate -le 0) { $LearningRate = if ($env:TORCH_RETRAIN_LEARNING_RATE) { [double]$env:TORCH_RETRAIN_LEARNING_RATE } else { 0.0007 } }
|
|
if ($WeightDecay -le 0) { $WeightDecay = if ($env:TORCH_RETRAIN_WEIGHT_DECAY) { [double]$env:TORCH_RETRAIN_WEIGHT_DECAY } else { 0.0005 } }
|
|
if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 50 } }
|
|
if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 8 } }
|
|
if ($ValidationWindow -le 0) { $ValidationWindow = if ($env:TORCH_RETRAIN_VALIDATION_WINDOW) { [int]$env:TORCH_RETRAIN_VALIDATION_WINDOW } else { 720 } }
|
|
if ($HoldoutWindow -le 0) { $HoldoutWindow = if ($env:TORCH_RETRAIN_HOLDOUT_WINDOW) { [int]$env:TORCH_RETRAIN_HOLDOUT_WINDOW } else { 1000 } }
|
|
if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RETRAIN_INTERVAL }
|
|
if (-not $EnvFile -and $env:TORCH_RETRAIN_ENV) { $EnvFile = $env:TORCH_RETRAIN_ENV }
|
|
if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" }
|
|
|
|
$ModelFile = if ($env:TIME_SERIES_LSTM_MODEL_PATH) { $env:TIME_SERIES_LSTM_MODEL_PATH } else { Join-Path $RuntimeDir "lstm_forecaster.json" }
|
|
if (-not [System.IO.Path]::IsPathRooted($ModelFile)) { $ModelFile = Join-Path $RepoRoot $ModelFile }
|
|
$CandidateFile = Join-Path $RuntimeDir "lstm_forecaster.candidate.json"
|
|
$CurrentCalibration = Join-Path $RuntimeDir "torch_guard_current.json"
|
|
$CandidateCalibration = Join-Path $RuntimeDir "torch_guard_candidate.json"
|
|
$GuardReport = Join-Path $RuntimeDir "torch_retrain_guard.json"
|
|
|
|
$mutex = New-Object System.Threading.Mutex($false, "TradeBotTorchRecurrentRetrainer")
|
|
$hasLock = $false
|
|
$pushedLocation = $false
|
|
|
|
try {
|
|
$hasLock = $mutex.WaitOne(0)
|
|
if (-not $hasLock) {
|
|
Write-RetrainLog "Another PyTorch recurrent retrain is already running; skipping."
|
|
exit 0
|
|
}
|
|
|
|
$python = Resolve-Python
|
|
$trainerArgs = @(
|
|
"-u",
|
|
"tools\train_torch_recurrent_forecaster.py",
|
|
"--limit", $Limit.ToString(),
|
|
"--lookbacks", $Lookbacks,
|
|
"--architectures", $Architectures,
|
|
"--hidden-sizes", $HiddenSizes,
|
|
"--layers", $Layers,
|
|
"--dropouts", $Dropouts,
|
|
"--epochs", $Epochs.ToString(),
|
|
"--patience", $Patience.ToString(),
|
|
"--validation-window", $ValidationWindow.ToString(),
|
|
"--holdout-window", $HoldoutWindow.ToString(),
|
|
"--ensemble-seeds", $EnsembleSeeds,
|
|
"--selection-folds", $SelectionFolds.ToString(),
|
|
"--learning-rate", $LearningRate.ToString([Globalization.CultureInfo]::InvariantCulture),
|
|
"--weight-decay", $WeightDecay.ToString([Globalization.CultureInfo]::InvariantCulture),
|
|
"--output", $CandidateFile
|
|
)
|
|
if ($Pooled) {
|
|
$trainerArgs += "--pooled"
|
|
}
|
|
else {
|
|
$trainerArgs += "--no-pooled"
|
|
}
|
|
if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) }
|
|
if ($Interval) { $trainerArgs += @("--interval", $Interval) }
|
|
if ($EnvFile) { $trainerArgs += @("--env", $EnvFile) }
|
|
if ($Horizon -gt 0) { $trainerArgs += @("--horizon", $Horizon.ToString()) }
|
|
if ($Horizons) { $trainerArgs += @("--horizons", $Horizons) }
|
|
if ($Features) { $trainerArgs += @("--features", $Features) }
|
|
if ($ContextSymbols) { $trainerArgs += @("--context-symbols", $ContextSymbols) }
|
|
if ($Seed -gt 0) { $trainerArgs += @("--seed", $Seed.ToString()) }
|
|
|
|
Push-Location $RepoRoot
|
|
$pushedLocation = $true
|
|
if ($ResumeCandidate) {
|
|
if (-not (Test-TorchArtifactFile $CandidateFile)) {
|
|
throw "ResumeCandidate requested, but no valid candidate artifact exists: $CandidateFile"
|
|
}
|
|
Write-RetrainLog "Resuming guard from existing candidate artifact: $CandidateFile"
|
|
}
|
|
else {
|
|
Write-RetrainLog "Starting PyTorch recurrent retrain: $python $($trainerArgs -join ' ')"
|
|
$trainerExitCode = Invoke-LoggedNativeCommand -FilePath $python -ArgumentList $trainerArgs -LogPath $LogFile
|
|
if ($trainerExitCode -ne 0) {
|
|
if (Test-TorchArtifactFile $CandidateFile) {
|
|
Write-RetrainLog "WARNING: Trainer exited with code $trainerExitCode after writing a valid candidate artifact; continuing to guard."
|
|
}
|
|
else {
|
|
throw "Trainer failed with exit code $trainerExitCode."
|
|
}
|
|
}
|
|
Write-RetrainLog "Finished PyTorch recurrent retrain candidate: $CandidateFile"
|
|
}
|
|
|
|
if ($SkipGuard) {
|
|
throw "SkipGuard is disabled: every candidate must pass untouched-holdout validation."
|
|
}
|
|
|
|
$calibrationBaseArgs = @(
|
|
"-u",
|
|
"tools\calibrate_torch_thresholds.py",
|
|
"--limit", $Limit.ToString(),
|
|
"--horizon", $Horizon.ToString(),
|
|
"--calibration-window", ([Math]::Min(2400, [Math]::Max(1200, [int]($Limit / 2)))).ToString(),
|
|
"--min-trades", "24",
|
|
"--walk-forward-folds", "8",
|
|
"--confidence-grid", "0.40"
|
|
)
|
|
if ($Symbols) { $calibrationBaseArgs += @("--symbols", $Symbols) }
|
|
if ($EnvFile) { $calibrationBaseArgs += @("--env", $EnvFile) }
|
|
|
|
if (Test-Path $ModelFile) {
|
|
Write-RetrainLog "Calibrating current artifact for guard."
|
|
$currentCalibrationExitCode = Invoke-LoggedNativeCommand `
|
|
-FilePath $python `
|
|
-ArgumentList ($calibrationBaseArgs + @("--artifact", $ModelFile, "--output", $CurrentCalibration)) `
|
|
-LogPath $LogFile
|
|
if ($currentCalibrationExitCode -ne 0) {
|
|
Write-RetrainLog "Current artifact has no compatible untouched holdout; comparing candidate against an empty current report."
|
|
"{}" | Set-Content -LiteralPath $CurrentCalibration -Encoding utf8
|
|
}
|
|
}
|
|
else {
|
|
Write-RetrainLog "No active artifact yet; candidate still must pass the full guard."
|
|
"{}" | Set-Content -LiteralPath $CurrentCalibration -Encoding utf8
|
|
}
|
|
|
|
Write-RetrainLog "Calibrating candidate artifact for guard."
|
|
$candidateCalibrationExitCode = Invoke-LoggedNativeCommand `
|
|
-FilePath $python `
|
|
-ArgumentList ($calibrationBaseArgs + @("--artifact", $CandidateFile, "--output", $CandidateCalibration)) `
|
|
-LogPath $LogFile
|
|
if ($candidateCalibrationExitCode -ne 0) {
|
|
throw "Candidate artifact calibration failed with exit code $candidateCalibrationExitCode."
|
|
}
|
|
|
|
Write-RetrainLog "Running retrain guard."
|
|
$guardArgs = @(
|
|
"-u",
|
|
"tools\accept_torch_candidate.py",
|
|
"--current-report", $CurrentCalibration,
|
|
"--candidate-report", $CandidateCalibration,
|
|
"--candidate-artifact", $CandidateFile,
|
|
"--target-artifact", $ModelFile,
|
|
"--report", $GuardReport
|
|
)
|
|
$guardExitCode = Invoke-LoggedNativeCommand -FilePath $python -ArgumentList $guardArgs -LogPath $LogFile
|
|
if ($guardExitCode -eq 2) {
|
|
Write-RetrainLog "Candidate rejected by guard; keeping active artifact: $ModelFile"
|
|
exit 0
|
|
}
|
|
if ($guardExitCode -ne 0) {
|
|
throw "Retrain guard failed with exit code $guardExitCode."
|
|
}
|
|
if (Test-Path $CandidateCalibration) {
|
|
Copy-Item -Force -LiteralPath $CandidateCalibration -Destination (Join-Path $RuntimeDir "torch_threshold_calibration.json")
|
|
Write-RetrainLog "Updated active threshold calibration: $(Join-Path $RuntimeDir "torch_threshold_calibration.json")"
|
|
}
|
|
Write-RetrainLog "Candidate accepted by guard. Active artifact: $ModelFile"
|
|
}
|
|
catch {
|
|
Write-RetrainLog "ERROR: $($_.Exception.Message)"
|
|
exit 1
|
|
}
|
|
finally {
|
|
if ($pushedLocation) {
|
|
Pop-Location -ErrorAction SilentlyContinue
|
|
}
|
|
if ($hasLock) {
|
|
$mutex.ReleaseMutex()
|
|
}
|
|
$mutex.Dispose()
|
|
}
|