Codex
248 lines
9.9 KiB
PowerShell
248 lines
9.9 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[string]$Symbols = "",
|
|
[int]$Limit = 0,
|
|
[string]$Lookbacks = "",
|
|
[string]$Architectures = "",
|
|
[string]$HiddenSizes = "",
|
|
[string]$Layers = "",
|
|
[string]$Dropouts = "",
|
|
[int]$Horizon = 0,
|
|
[string]$Horizons = "",
|
|
[string]$Features = "",
|
|
[string]$ContextSymbols = "",
|
|
[int]$Seed = 0,
|
|
[int]$Epochs = 0,
|
|
[int]$Patience = 0,
|
|
[string]$Interval = "",
|
|
[string]$EnvFile = "",
|
|
[switch]$DeployToPi,
|
|
[string]$PiHost = "",
|
|
[string]$PiUser = "",
|
|
[string]$PiRoot = "",
|
|
[string]$PiSshKeyPath = "",
|
|
[switch]$NoPiRestart,
|
|
[switch]$SkipGuard
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
|
$RuntimeDir = Join-Path $RepoRoot "runtime"
|
|
$LogFile = Join-Path $RuntimeDir "torch_retrain.log"
|
|
New-Item -ItemType Directory -Force -Path $RuntimeDir | Out-Null
|
|
|
|
function Write-RetrainLog {
|
|
param([string]$Message)
|
|
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ssK"
|
|
"[$timestamp] $Message" | Tee-Object -FilePath $LogFile -Append
|
|
}
|
|
|
|
function Resolve-Python {
|
|
$venvPython = Join-Path $RepoRoot ".venv\Scripts\python.exe"
|
|
if (Test-Path $venvPython) {
|
|
return $venvPython
|
|
}
|
|
|
|
$userPython = Join-Path $env:LOCALAPPDATA "Programs\TradeBotPython312\python.exe"
|
|
if (Test-Path $userPython) {
|
|
return $userPython
|
|
}
|
|
|
|
foreach ($candidate in @("python.exe", "python")) {
|
|
$command = Get-Command $candidate -ErrorAction SilentlyContinue
|
|
if (-not $command) {
|
|
continue
|
|
}
|
|
return $command.Source
|
|
}
|
|
throw "Python was not found. Create .venv or install Python 3.12."
|
|
}
|
|
|
|
function Test-TorchArtifactFile {
|
|
param([string]$Path)
|
|
if (-not (Test-Path $Path)) {
|
|
return $false
|
|
}
|
|
try {
|
|
$payload = Get-Content -Raw -LiteralPath $Path | ConvertFrom-Json
|
|
return $payload.type -eq "pytorch_recurrent_forecaster" -and $null -ne $payload.symbols
|
|
}
|
|
catch {
|
|
return $false
|
|
}
|
|
}
|
|
|
|
function Sync-AcceptedArtifactsToPi {
|
|
if (-not ($DeployToPi -or $env:TORCH_RETRAIN_DEPLOY_TO_PI)) {
|
|
Write-RetrainLog "Pi artifact sync disabled."
|
|
return
|
|
}
|
|
|
|
$syncScript = Join-Path $RepoRoot "tools\sync_torch_artifacts_to_pi.ps1"
|
|
if (-not (Test-Path $syncScript)) {
|
|
throw "Pi sync script not found: $syncScript"
|
|
}
|
|
|
|
$syncArgs = @(
|
|
"-NoProfile",
|
|
"-ExecutionPolicy", "Bypass",
|
|
"-File", $syncScript,
|
|
"-RepoRoot", $RepoRoot
|
|
)
|
|
if ($PiHost) { $syncArgs += @("-RemoteHost", $PiHost) }
|
|
if ($PiUser) { $syncArgs += @("-RemoteUser", $PiUser) }
|
|
if ($PiRoot) { $syncArgs += @("-RemoteRoot", $PiRoot) }
|
|
if ($PiSshKeyPath) { $syncArgs += @("-SshKeyPath", $PiSshKeyPath) }
|
|
if ($NoPiRestart) { $syncArgs += "-NoRestart" }
|
|
|
|
Write-RetrainLog "Syncing accepted Torch artifacts to Raspberry Pi."
|
|
& powershell.exe @syncArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Pi artifact sync failed with exit code $LASTEXITCODE."
|
|
}
|
|
Write-RetrainLog "Pi artifact sync completed."
|
|
}
|
|
|
|
if (-not $Symbols -and $env:TORCH_RETRAIN_SYMBOLS) { $Symbols = $env:TORCH_RETRAIN_SYMBOLS }
|
|
if ($Limit -le 0) {
|
|
$Limit = if ($env:TORCH_RETRAIN_LIMIT) { [int]$env:TORCH_RETRAIN_LIMIT } else { 3000 }
|
|
}
|
|
if (-not $Lookbacks) { $Lookbacks = if ($env:TORCH_RETRAIN_LOOKBACKS) { $env:TORCH_RETRAIN_LOOKBACKS } else { "64" } }
|
|
if (-not $Architectures) { $Architectures = if ($env:TORCH_RETRAIN_ARCHITECTURES) { $env:TORCH_RETRAIN_ARCHITECTURES } else { "lstm,gru" } }
|
|
if (-not $HiddenSizes) { $HiddenSizes = if ($env:TORCH_RETRAIN_HIDDEN_SIZES) { $env:TORCH_RETRAIN_HIDDEN_SIZES } else { "64,96" } }
|
|
if (-not $Layers) { $Layers = if ($env:TORCH_RETRAIN_LAYERS) { $env:TORCH_RETRAIN_LAYERS } else { "2" } }
|
|
if (-not $Dropouts) { $Dropouts = if ($env:TORCH_RETRAIN_DROPOUTS) { $env:TORCH_RETRAIN_DROPOUTS } else { "0.15" } }
|
|
if ($Horizon -le 0 -and $env:TORCH_RETRAIN_HORIZON) { $Horizon = [int]$env:TORCH_RETRAIN_HORIZON }
|
|
if (-not $Horizons -and $env:TORCH_RETRAIN_HORIZONS) { $Horizons = $env:TORCH_RETRAIN_HORIZONS }
|
|
if (-not $Features -and $env:TORCH_RETRAIN_FEATURES) { $Features = $env:TORCH_RETRAIN_FEATURES }
|
|
if (-not $ContextSymbols -and $env:TORCH_RETRAIN_CONTEXT_SYMBOLS) { $ContextSymbols = $env:TORCH_RETRAIN_CONTEXT_SYMBOLS }
|
|
if ($Seed -le 0 -and $env:TORCH_RETRAIN_SEED) { $Seed = [int]$env:TORCH_RETRAIN_SEED }
|
|
if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 70 } }
|
|
if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 8 } }
|
|
if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RETRAIN_INTERVAL }
|
|
if (-not $EnvFile -and $env:TORCH_RETRAIN_ENV) { $EnvFile = $env:TORCH_RETRAIN_ENV }
|
|
if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" }
|
|
|
|
$ModelFile = if ($env:TIME_SERIES_LSTM_MODEL_PATH) { $env:TIME_SERIES_LSTM_MODEL_PATH } else { Join-Path $RuntimeDir "lstm_forecaster.json" }
|
|
if (-not [System.IO.Path]::IsPathRooted($ModelFile)) { $ModelFile = Join-Path $RepoRoot $ModelFile }
|
|
$CandidateFile = Join-Path $RuntimeDir "lstm_forecaster.candidate.json"
|
|
$CurrentCalibration = Join-Path $RuntimeDir "torch_guard_current.json"
|
|
$CandidateCalibration = Join-Path $RuntimeDir "torch_guard_candidate.json"
|
|
$GuardReport = Join-Path $RuntimeDir "torch_retrain_guard.json"
|
|
|
|
$mutex = New-Object System.Threading.Mutex($false, "TradeBotTorchRecurrentRetrainer")
|
|
$hasLock = $false
|
|
$pushedLocation = $false
|
|
|
|
try {
|
|
$hasLock = $mutex.WaitOne(0)
|
|
if (-not $hasLock) {
|
|
Write-RetrainLog "Another PyTorch recurrent retrain is already running; skipping."
|
|
exit 0
|
|
}
|
|
|
|
$python = Resolve-Python
|
|
$trainerArgs = @(
|
|
"-u",
|
|
"tools\train_torch_recurrent_forecaster.py",
|
|
"--limit", $Limit.ToString(),
|
|
"--lookbacks", $Lookbacks,
|
|
"--architectures", $Architectures,
|
|
"--hidden-sizes", $HiddenSizes,
|
|
"--layers", $Layers,
|
|
"--dropouts", $Dropouts,
|
|
"--epochs", $Epochs.ToString(),
|
|
"--patience", $Patience.ToString(),
|
|
"--output", $CandidateFile
|
|
)
|
|
if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) }
|
|
if ($Interval) { $trainerArgs += @("--interval", $Interval) }
|
|
if ($EnvFile) { $trainerArgs += @("--env", $EnvFile) }
|
|
if ($Horizon -gt 0) { $trainerArgs += @("--horizon", $Horizon.ToString()) }
|
|
if ($Horizons) { $trainerArgs += @("--horizons", $Horizons) }
|
|
if ($Features) { $trainerArgs += @("--features", $Features) }
|
|
if ($ContextSymbols) { $trainerArgs += @("--context-symbols", $ContextSymbols) }
|
|
if ($Seed -gt 0) { $trainerArgs += @("--seed", $Seed.ToString()) }
|
|
|
|
Push-Location $RepoRoot
|
|
$pushedLocation = $true
|
|
Write-RetrainLog "Starting PyTorch recurrent retrain: $python $($trainerArgs -join ' ')"
|
|
& $python @trainerArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
|
|
$trainerExitCode = $LASTEXITCODE
|
|
if ($trainerExitCode -ne 0) {
|
|
if (Test-TorchArtifactFile $CandidateFile) {
|
|
Write-RetrainLog "WARNING: Trainer exited with code $trainerExitCode after writing a valid candidate artifact; continuing to guard."
|
|
}
|
|
else {
|
|
throw "Trainer failed with exit code $trainerExitCode."
|
|
}
|
|
}
|
|
Write-RetrainLog "Finished PyTorch recurrent retrain candidate: $CandidateFile"
|
|
|
|
if ($SkipGuard -or -not (Test-Path $ModelFile)) {
|
|
Move-Item -Force -LiteralPath $CandidateFile -Destination $ModelFile
|
|
Write-RetrainLog "Accepted candidate without guard. Active artifact: $ModelFile"
|
|
Sync-AcceptedArtifactsToPi
|
|
exit 0
|
|
}
|
|
|
|
$calibrationBaseArgs = @(
|
|
"-u",
|
|
"tools\calibrate_torch_thresholds.py",
|
|
"--limit", "3000",
|
|
"--calibration-window", "1200",
|
|
"--min-trades", "60",
|
|
"--walk-forward-folds", "8",
|
|
"--confidence-grid", "0.40"
|
|
)
|
|
if ($Symbols) { $calibrationBaseArgs += @("--symbols", $Symbols) }
|
|
if ($EnvFile) { $calibrationBaseArgs += @("--env", $EnvFile) }
|
|
|
|
Write-RetrainLog "Calibrating current artifact for guard."
|
|
& $python @($calibrationBaseArgs + @("--artifact", $ModelFile, "--output", $CurrentCalibration)) 2>&1 | Tee-Object -FilePath $LogFile -Append
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Current artifact calibration failed with exit code $LASTEXITCODE."
|
|
}
|
|
|
|
Write-RetrainLog "Calibrating candidate artifact for guard."
|
|
& $python @($calibrationBaseArgs + @("--artifact", $CandidateFile, "--output", $CandidateCalibration)) 2>&1 | Tee-Object -FilePath $LogFile -Append
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Candidate artifact calibration failed with exit code $LASTEXITCODE."
|
|
}
|
|
|
|
Write-RetrainLog "Running retrain guard."
|
|
& $python -u "tools\accept_torch_candidate.py" `
|
|
--current-report $CurrentCalibration `
|
|
--candidate-report $CandidateCalibration `
|
|
--candidate-artifact $CandidateFile `
|
|
--target-artifact $ModelFile `
|
|
--report $GuardReport 2>&1 | Tee-Object -FilePath $LogFile -Append
|
|
if ($LASTEXITCODE -eq 2) {
|
|
Write-RetrainLog "Candidate rejected by guard; keeping active artifact: $ModelFile"
|
|
exit 0
|
|
}
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Retrain guard failed with exit code $LASTEXITCODE."
|
|
}
|
|
if (Test-Path $CandidateCalibration) {
|
|
Copy-Item -Force -LiteralPath $CandidateCalibration -Destination (Join-Path $RuntimeDir "torch_threshold_calibration.json")
|
|
Write-RetrainLog "Updated active threshold calibration: $(Join-Path $RuntimeDir "torch_threshold_calibration.json")"
|
|
}
|
|
Write-RetrainLog "Candidate accepted by guard. Active artifact: $ModelFile"
|
|
Sync-AcceptedArtifactsToPi
|
|
}
|
|
catch {
|
|
Write-RetrainLog "ERROR: $($_.Exception.Message)"
|
|
exit 1
|
|
}
|
|
finally {
|
|
if ($pushedLocation) {
|
|
Pop-Location -ErrorAction SilentlyContinue
|
|
}
|
|
if ($hasLock) {
|
|
$mutex.ReleaseMutex()
|
|
}
|
|
$mutex.Dispose()
|
|
}
|