sevenhill/TradeBot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden trading, training, and monitoring

Browse Source
This commit is contained in:
Codex
2026-07-10 15:51:53 +03:00
parent 6fb79ee2a9
commit 069d75d2f2
55 changed files with 2658 additions and 2332049 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/run_torch_retrain.ps1
+81 -32
View File
@@ -14,6 +14,7 @@ param(
[int]$Seed = 0,
[int]$Epochs = 0,
[int]$Patience = 0,
[int]$HoldoutWindow = 0,
[string]$Interval = "",
[string]$EnvFile = "",
[switch]$DeployToPi,
@@ -22,7 +23,8 @@ param(
[string]$PiRoot = "",
[string]$PiSshKeyPath = "",
[switch]$NoPiRestart,
[switch]$SkipGuard
[switch]$SkipGuard,
[switch]$ResumeCandidate
)
$ErrorActionPreference = "Stop"
@@ -38,6 +40,30 @@ function Write-RetrainLog {
"[$timestamp] $Message" | Tee-Object -FilePath $LogFile -Append
}
function Invoke-LoggedNativeCommand {
param(
[string]$FilePath,
[object[]]$ArgumentList,
[string]$LogPath
)
# Windows PowerShell converts redirected native stderr into PowerShell error
# records. With the script-wide Stop preference an expected non-zero exit
# would jump to catch before callers can inspect LASTEXITCODE.
$previousErrorActionPreference = $ErrorActionPreference
try {
$ErrorActionPreference = "Continue"
& $FilePath @ArgumentList 2>&1 |
Tee-Object -FilePath $LogPath -Append |
Out-Host
$exitCode = $LASTEXITCODE
}
finally {
$ErrorActionPreference = $previousErrorActionPreference
}
return [int]$exitCode
}
function Resolve-Python {
$venvPython = Join-Path $RepoRoot ".venv\Scripts\python.exe"
if (Test-Path $venvPython) {
@@ -120,6 +146,7 @@ if (-not $ContextSymbols -and $env:TORCH_RETRAIN_CONTEXT_SYMBOLS) { $ContextSymb
if ($Seed -le 0 -and $env:TORCH_RETRAIN_SEED) { $Seed = [int]$env:TORCH_RETRAIN_SEED }
if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 70 } }
if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 8 } }
if ($HoldoutWindow -le 0) { $HoldoutWindow = if ($env:TORCH_RETRAIN_HOLDOUT_WINDOW) { [int]$env:TORCH_RETRAIN_HOLDOUT_WINDOW } else { 240 } }
if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RETRAIN_INTERVAL }
if (-not $EnvFile -and $env:TORCH_RETRAIN_ENV) { $EnvFile = $env:TORCH_RETRAIN_ENV }
if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" }
@@ -154,6 +181,7 @@ try {
"--dropouts", $Dropouts,
"--epochs", $Epochs.ToString(),
"--patience", $Patience.ToString(),
"--holdout-window", $HoldoutWindow.ToString(),
"--output", $CandidateFile
)
if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) }
@@ -167,24 +195,28 @@ try {
Push-Location $RepoRoot
$pushedLocation = $true
Write-RetrainLog "Starting PyTorch recurrent retrain: $python $($trainerArgs -join ' ')"
& $python @trainerArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
$trainerExitCode = $LASTEXITCODE
if ($trainerExitCode -ne 0) {
if (Test-TorchArtifactFile $CandidateFile) {
Write-RetrainLog "WARNING: Trainer exited with code $trainerExitCode after writing a valid candidate artifact; continuing to guard."
}
else {
throw "Trainer failed with exit code $trainerExitCode."
if ($ResumeCandidate) {
if (-not (Test-TorchArtifactFile $CandidateFile)) {
throw "ResumeCandidate requested, but no valid candidate artifact exists: $CandidateFile"
}
Write-RetrainLog "Resuming guard from existing candidate artifact: $CandidateFile"
}
else {
Write-RetrainLog "Starting PyTorch recurrent retrain: $python $($trainerArgs -join ' ')"
$trainerExitCode = Invoke-LoggedNativeCommand -FilePath $python -ArgumentList $trainerArgs -LogPath $LogFile
if ($trainerExitCode -ne 0) {
if (Test-TorchArtifactFile $CandidateFile) {
Write-RetrainLog "WARNING: Trainer exited with code $trainerExitCode after writing a valid candidate artifact; continuing to guard."
}
else {
throw "Trainer failed with exit code $trainerExitCode."
}
}
Write-RetrainLog "Finished PyTorch recurrent retrain candidate: $CandidateFile"
}
Write-RetrainLog "Finished PyTorch recurrent retrain candidate: $CandidateFile"
if ($SkipGuard -or -not (Test-Path $ModelFile)) {
Move-Item -Force -LiteralPath $CandidateFile -Destination $ModelFile
Write-RetrainLog "Accepted candidate without guard. Active artifact: $ModelFile"
Sync-AcceptedArtifactsToPi
exit 0
if ($SkipGuard) {
throw "SkipGuard is disabled: every candidate must pass untouched-holdout validation."
}
$calibrationBaseArgs = @(
@@ -199,31 +231,48 @@ try {
if ($Symbols) { $calibrationBaseArgs += @("--symbols", $Symbols) }
if ($EnvFile) { $calibrationBaseArgs += @("--env", $EnvFile) }
Write-RetrainLog "Calibrating current artifact for guard."
& $python @($calibrationBaseArgs + @("--artifact", $ModelFile, "--output", $CurrentCalibration)) 2>&1 | Tee-Object -FilePath $LogFile -Append
if ($LASTEXITCODE -ne 0) {
throw "Current artifact calibration failed with exit code $LASTEXITCODE."
if (Test-Path $ModelFile) {
Write-RetrainLog "Calibrating current artifact for guard."
$currentCalibrationExitCode = Invoke-LoggedNativeCommand `
-FilePath $python `
-ArgumentList ($calibrationBaseArgs + @("--artifact", $ModelFile, "--output", $CurrentCalibration)) `
-LogPath $LogFile
if ($currentCalibrationExitCode -ne 0) {
Write-RetrainLog "Current artifact has no compatible untouched holdout; comparing candidate against an empty current report."
"{}" | Set-Content -LiteralPath $CurrentCalibration -Encoding utf8
}
}
else {
Write-RetrainLog "No active artifact yet; candidate still must pass the full guard."
"{}" | Set-Content -LiteralPath $CurrentCalibration -Encoding utf8
}
Write-RetrainLog "Calibrating candidate artifact for guard."
& $python @($calibrationBaseArgs + @("--artifact", $CandidateFile, "--output", $CandidateCalibration)) 2>&1 | Tee-Object -FilePath $LogFile -Append
if ($LASTEXITCODE -ne 0) {
throw "Candidate artifact calibration failed with exit code $LASTEXITCODE."
$candidateCalibrationExitCode = Invoke-LoggedNativeCommand `
-FilePath $python `
-ArgumentList ($calibrationBaseArgs + @("--artifact", $CandidateFile, "--output", $CandidateCalibration)) `
-LogPath $LogFile
if ($candidateCalibrationExitCode -ne 0) {
throw "Candidate artifact calibration failed with exit code $candidateCalibrationExitCode."
}
Write-RetrainLog "Running retrain guard."
& $python -u "tools\accept_torch_candidate.py" `
--current-report $CurrentCalibration `
--candidate-report $CandidateCalibration `
--candidate-artifact $CandidateFile `
--target-artifact $ModelFile `
--report $GuardReport 2>&1 | Tee-Object -FilePath $LogFile -Append
if ($LASTEXITCODE -eq 2) {
$guardArgs = @(
"-u",
"tools\accept_torch_candidate.py",
"--current-report", $CurrentCalibration,
"--candidate-report", $CandidateCalibration,
"--candidate-artifact", $CandidateFile,
"--target-artifact", $ModelFile,
"--report", $GuardReport
)
$guardExitCode = Invoke-LoggedNativeCommand -FilePath $python -ArgumentList $guardArgs -LogPath $LogFile
if ($guardExitCode -eq 2) {
Write-RetrainLog "Candidate rejected by guard; keeping active artifact: $ModelFile"
exit 0
}
if ($LASTEXITCODE -ne 0) {
throw "Retrain guard failed with exit code $LASTEXITCODE."
if ($guardExitCode -ne 0) {
throw "Retrain guard failed with exit code $guardExitCode."
}
if (Test-Path $CandidateCalibration) {
Copy-Item -Force -LiteralPath $CandidateCalibration -Destination (Join-Path $RuntimeDir "torch_threshold_calibration.json")