Add analytics risk guard and redesigned dashboard
This commit is contained in:
@@ -14,7 +14,8 @@ param(
|
||||
[int]$Epochs = 0,
|
||||
[int]$Patience = 0,
|
||||
[string]$Interval = "",
|
||||
[string]$EnvFile = ""
|
||||
[string]$EnvFile = "",
|
||||
[switch]$SkipGuard
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
@@ -70,6 +71,13 @@ if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RE
|
||||
if (-not $EnvFile -and $env:TORCH_RETRAIN_ENV) { $EnvFile = $env:TORCH_RETRAIN_ENV }
|
||||
if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" }
|
||||
|
||||
$ModelFile = if ($env:TIME_SERIES_LSTM_MODEL_PATH) { $env:TIME_SERIES_LSTM_MODEL_PATH } else { Join-Path $RuntimeDir "lstm_forecaster.json" }
|
||||
if (-not [System.IO.Path]::IsPathRooted($ModelFile)) { $ModelFile = Join-Path $RepoRoot $ModelFile }
|
||||
$CandidateFile = Join-Path $RuntimeDir "lstm_forecaster.candidate.json"
|
||||
$CurrentCalibration = Join-Path $RuntimeDir "torch_guard_current.json"
|
||||
$CandidateCalibration = Join-Path $RuntimeDir "torch_guard_candidate.json"
|
||||
$GuardReport = Join-Path $RuntimeDir "torch_retrain_guard.json"
|
||||
|
||||
$mutex = New-Object System.Threading.Mutex($false, "TradeBotTorchRecurrentRetrainer")
|
||||
$hasLock = $false
|
||||
$pushedLocation = $false
|
||||
@@ -92,7 +100,8 @@ try {
|
||||
"--layers", $Layers,
|
||||
"--dropouts", $Dropouts,
|
||||
"--epochs", $Epochs.ToString(),
|
||||
"--patience", $Patience.ToString()
|
||||
"--patience", $Patience.ToString(),
|
||||
"--output", $CandidateFile
|
||||
)
|
||||
if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) }
|
||||
if ($Interval) { $trainerArgs += @("--interval", $Interval) }
|
||||
@@ -109,7 +118,51 @@ try {
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Trainer failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
Write-RetrainLog "Finished PyTorch recurrent retrain."
|
||||
Write-RetrainLog "Finished PyTorch recurrent retrain candidate: $CandidateFile"
|
||||
|
||||
if ($SkipGuard -or -not (Test-Path $ModelFile)) {
|
||||
Move-Item -Force -LiteralPath $CandidateFile -Destination $ModelFile
|
||||
Write-RetrainLog "Accepted candidate without guard. Active artifact: $ModelFile"
|
||||
exit 0
|
||||
}
|
||||
|
||||
$calibrationBaseArgs = @(
|
||||
"-u",
|
||||
"tools\calibrate_torch_thresholds.py",
|
||||
"--limit", "2000",
|
||||
"--calibration-window", "720",
|
||||
"--min-trades", "12"
|
||||
)
|
||||
if ($Symbols) { $calibrationBaseArgs += @("--symbols", $Symbols) }
|
||||
if ($EnvFile) { $calibrationBaseArgs += @("--env", $EnvFile) }
|
||||
|
||||
Write-RetrainLog "Calibrating current artifact for guard."
|
||||
& $python @($calibrationBaseArgs + @("--artifact", $ModelFile, "--output", $CurrentCalibration)) 2>&1 | Tee-Object -FilePath $LogFile -Append
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Current artifact calibration failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
|
||||
Write-RetrainLog "Calibrating candidate artifact for guard."
|
||||
& $python @($calibrationBaseArgs + @("--artifact", $CandidateFile, "--output", $CandidateCalibration)) 2>&1 | Tee-Object -FilePath $LogFile -Append
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Candidate artifact calibration failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
|
||||
Write-RetrainLog "Running retrain guard."
|
||||
& $python -u "tools\accept_torch_candidate.py" `
|
||||
--current-report $CurrentCalibration `
|
||||
--candidate-report $CandidateCalibration `
|
||||
--candidate-artifact $CandidateFile `
|
||||
--target-artifact $ModelFile `
|
||||
--report $GuardReport 2>&1 | Tee-Object -FilePath $LogFile -Append
|
||||
if ($LASTEXITCODE -eq 2) {
|
||||
Write-RetrainLog "Candidate rejected by guard; keeping active artifact: $ModelFile"
|
||||
exit 0
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Retrain guard failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
Write-RetrainLog "Candidate accepted by guard. Active artifact: $ModelFile"
|
||||
}
|
||||
catch {
|
||||
Write-RetrainLog "ERROR: $($_.Exception.Message)"
|
||||
|
||||
Reference in New Issue
Block a user