Add Torch probe entries and Pi artifact sync

tools/run_torch_retrain.ps1

@@ -11,10 +11,17 @@ param(
     [string]$Horizons = "",
     [string]$Features = "",
     [string]$ContextSymbols = "",
+    [int]$Seed = 0,
     [int]$Epochs = 0,
     [int]$Patience = 0,
     [string]$Interval = "",
     [string]$EnvFile = "",
+    [switch]$DeployToPi,
+    [string]$PiHost = "",
+    [string]$PiUser = "",
+    [string]$PiRoot = "",
+    [string]$PiSshKeyPath = "",
+    [switch]$NoPiRestart,
     [switch]$SkipGuard
 )
 
@@ -52,6 +59,51 @@ function Resolve-Python {
     throw "Python was not found. Create .venv or install Python 3.12."
 }
 
+function Test-TorchArtifactFile {
+    param([string]$Path)
+    if (-not (Test-Path $Path)) {
+        return $false
+    }
+    try {
+        $payload = Get-Content -Raw -LiteralPath $Path | ConvertFrom-Json
+        return $payload.type -eq "pytorch_recurrent_forecaster" -and $null -ne $payload.symbols
+    }
+    catch {
+        return $false
+    }
+}
+
+function Sync-AcceptedArtifactsToPi {
+    if (-not ($DeployToPi -or $env:TORCH_RETRAIN_DEPLOY_TO_PI)) {
+        Write-RetrainLog "Pi artifact sync disabled."
+        return
+    }
+
+    $syncScript = Join-Path $RepoRoot "tools\sync_torch_artifacts_to_pi.ps1"
+    if (-not (Test-Path $syncScript)) {
+        throw "Pi sync script not found: $syncScript"
+    }
+
+    $syncArgs = @(
+        "-NoProfile",
+        "-ExecutionPolicy", "Bypass",
+        "-File", $syncScript,
+        "-RepoRoot", $RepoRoot
+    )
+    if ($PiHost) { $syncArgs += @("-RemoteHost", $PiHost) }
+    if ($PiUser) { $syncArgs += @("-RemoteUser", $PiUser) }
+    if ($PiRoot) { $syncArgs += @("-RemoteRoot", $PiRoot) }
+    if ($PiSshKeyPath) { $syncArgs += @("-SshKeyPath", $PiSshKeyPath) }
+    if ($NoPiRestart) { $syncArgs += "-NoRestart" }
+
+    Write-RetrainLog "Syncing accepted Torch artifacts to Raspberry Pi."
+    & powershell.exe @syncArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
+    if ($LASTEXITCODE -ne 0) {
+        throw "Pi artifact sync failed with exit code $LASTEXITCODE."
+    }
+    Write-RetrainLog "Pi artifact sync completed."
+}
+
 if (-not $Symbols -and $env:TORCH_RETRAIN_SYMBOLS) { $Symbols = $env:TORCH_RETRAIN_SYMBOLS }
 if ($Limit -le 0) {
     $Limit = if ($env:TORCH_RETRAIN_LIMIT) { [int]$env:TORCH_RETRAIN_LIMIT } else { 3000 }
@@ -65,6 +117,7 @@ if ($Horizon -le 0 -and $env:TORCH_RETRAIN_HORIZON) { $Horizon = [int]$env:TORCH
 if (-not $Horizons -and $env:TORCH_RETRAIN_HORIZONS) { $Horizons = $env:TORCH_RETRAIN_HORIZONS }
 if (-not $Features -and $env:TORCH_RETRAIN_FEATURES) { $Features = $env:TORCH_RETRAIN_FEATURES }
 if (-not $ContextSymbols -and $env:TORCH_RETRAIN_CONTEXT_SYMBOLS) { $ContextSymbols = $env:TORCH_RETRAIN_CONTEXT_SYMBOLS }
+if ($Seed -le 0 -and $env:TORCH_RETRAIN_SEED) { $Seed = [int]$env:TORCH_RETRAIN_SEED }
 if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 70 } }
 if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 8 } }
 if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RETRAIN_INTERVAL }
@@ -110,19 +163,27 @@ try {
     if ($Horizons) { $trainerArgs += @("--horizons", $Horizons) }
     if ($Features) { $trainerArgs += @("--features", $Features) }
     if ($ContextSymbols) { $trainerArgs += @("--context-symbols", $ContextSymbols) }
+    if ($Seed -gt 0) { $trainerArgs += @("--seed", $Seed.ToString()) }
 
     Push-Location $RepoRoot
     $pushedLocation = $true
     Write-RetrainLog "Starting PyTorch recurrent retrain: $python $($trainerArgs -join ' ')"
     & $python @trainerArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
-    if ($LASTEXITCODE -ne 0) {
-        throw "Trainer failed with exit code $LASTEXITCODE."
+    $trainerExitCode = $LASTEXITCODE
+    if ($trainerExitCode -ne 0) {
+        if (Test-TorchArtifactFile $CandidateFile) {
+            Write-RetrainLog "WARNING: Trainer exited with code $trainerExitCode after writing a valid candidate artifact; continuing to guard."
+        }
+        else {
+            throw "Trainer failed with exit code $trainerExitCode."
+        }
     }
     Write-RetrainLog "Finished PyTorch recurrent retrain candidate: $CandidateFile"
 
     if ($SkipGuard -or -not (Test-Path $ModelFile)) {
         Move-Item -Force -LiteralPath $CandidateFile -Destination $ModelFile
         Write-RetrainLog "Accepted candidate without guard. Active artifact: $ModelFile"
+        Sync-AcceptedArtifactsToPi
         exit 0
     }
 
@@ -162,7 +223,12 @@ try {
     if ($LASTEXITCODE -ne 0) {
         throw "Retrain guard failed with exit code $LASTEXITCODE."
     }
+    if (Test-Path $CandidateCalibration) {
+        Copy-Item -Force -LiteralPath $CandidateCalibration -Destination (Join-Path $RuntimeDir "torch_threshold_calibration.json")
+        Write-RetrainLog "Updated active threshold calibration: $(Join-Path $RuntimeDir "torch_threshold_calibration.json")"
+    }
     Write-RetrainLog "Candidate accepted by guard. Active artifact: $ModelFile"
+    Sync-AcceptedArtifactsToPi
 }
 catch {
     Write-RetrainLog "ERROR: $($_.Exception.Message)"