Harden trading, training, and monitoring

tools/accept_torch_candidate.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import argparse
+import hashlib
 import json
 import shutil
 from pathlib import Path
@@ -11,19 +12,25 @@ def main() -> None:
     args = _parse_args()
     current = _read_json(args.current_report)
     candidate = _read_json(args.candidate_report)
-    decision = _decision(
-        current,
-        candidate,
-        min_trades=args.min_trades,
-        min_profit_factor=args.min_profit_factor,
-        min_avg_net_percent=args.min_avg_net_percent,
-        max_score_regression=args.max_score_regression,
-    )
+    candidate_artifact = Path(args.candidate_artifact)
+    candidate_sha256 = _sha256(candidate_artifact)
+    if candidate.get("artifact_sha256") != candidate_sha256:
+        decision = {"accepted": False, "reason": "candidate_report_artifact_hash_mismatch"}
+    else:
+        decision = _decision(
+            current,
+            candidate,
+            min_trades=args.min_trades,
+            min_profit_factor=args.min_profit_factor,
+            min_avg_net_percent=args.min_avg_net_percent,
+            max_score_regression=args.max_score_regression,
+        )
     payload = {
         "accepted": decision["accepted"],
         "reason": decision["reason"],
         "current": _summary(current),
         "candidate": _summary(candidate),
+        "candidate_artifact_sha256": candidate_sha256,
     }
     if args.report:
         Path(args.report).write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
@@ -31,7 +38,6 @@ def main() -> None:
     if not decision["accepted"]:
         raise SystemExit(2)
     target = Path(args.target_artifact)
-    candidate_artifact = Path(args.candidate_artifact)
     target.parent.mkdir(parents=True, exist_ok=True)
     shutil.copy2(candidate_artifact, target)
 
@@ -83,6 +89,8 @@ def _validation_passed(report: dict[str, Any]) -> bool:
     validation = report.get("validation")
     if not isinstance(validation, dict):
         return False
+    if validation.get("protocol") != "untouched_model_holdout_with_threshold_walk_forward":
+        return False
     if "passed" in validation:
         return bool(validation.get("passed"))
     return str(validation.get("status", "")).strip().lower() in {"pass", "passed", "ok"}
@@ -125,5 +133,13 @@ def _read_json(path: str) -> dict[str, Any]:
     return data if isinstance(data, dict) else {}
 
 
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for chunk in iter(lambda: source.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
 if __name__ == "__main__":
     main()

tools/calibrate_torch_thresholds.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import argparse
+import hashlib
 import json
 import math
 import sys
@@ -49,6 +50,10 @@ class ForecastRecord:
     index: int
     timestamp: int
     close: float
+    high: float
+    low: float
+    next_open: float
+    next_timestamp: int
     atr: float
     expected_percent: float
     probability_up: float
@@ -85,7 +90,9 @@ def main() -> None:
     symbols = _symbols(args.symbols, settings.symbols)
     context_symbols = sorted(set(symbols + _symbols(args.context_symbols, ())))
     artifact_path = Path(args.artifact or settings.time_series_lstm_model_path)
-    artifact = json.loads(artifact_path.read_text(encoding="utf-8"))
+    artifact_bytes = artifact_path.read_bytes()
+    artifact_sha256 = hashlib.sha256(artifact_bytes).hexdigest()
+    artifact = json.loads(artifact_bytes.decode("utf-8"))
     horizon = args.horizon if args.horizon > 0 else settings.time_series_forecast_horizon
     round_trip_cost = _artifact_round_trip_cost(artifact, settings)
 
@@ -196,6 +203,7 @@ def main() -> None:
 
     if args.output:
         payload = {
+            "artifact_sha256": artifact_sha256,
             "artifact": _artifact_summary(artifact),
             "records_by_symbol": per_symbol_counts,
             "recommended": _result_dict(recommended),
@@ -273,6 +281,11 @@ def _forecast_records(
     decision_horizon = _entry_horizon(entry, horizon)
     start = max(min_candles, int(float(entry.get("lookback", 64))))
     end = len(candles) - decision_horizon - 1
+    holdout_start_timestamp = int(float(entry.get("holdout_start_timestamp", 0) or 0))
+    if holdout_start_timestamp <= 0:
+        return []
+    while start < end and candles[start].timestamp < holdout_start_timestamp:
+        start += 1
     if calibration_window > 0:
         start = max(start, end - calibration_window)
     batched_records = _batch_forecast_records(
@@ -313,7 +326,10 @@ def _forecast_records(
         q50 = float(selected.get("q50", expected_return))
         expected_percent = (math.exp(expected_return) - 1.0) * 100.0
         q50_percent = (math.exp(q50) - 1.0) * 100.0
-        future_log_return = math.log(closes[index + decision_horizon] / closes[index]) - round_trip_cost
+        next_open = float(candles[index + 1].open)
+        if next_open <= 0:
+            continue
+        future_log_return = math.log(closes[index + decision_horizon] / next_open) - round_trip_cost
         future_net_percent = (math.exp(future_log_return) - 1.0) * 100.0
         records.append(
             ForecastRecord(
@@ -321,6 +337,10 @@ def _forecast_records(
                 index=index,
                 timestamp=candles[index].timestamp,
                 close=closes[index],
+                high=float(candles[index].high),
+                low=float(candles[index].low),
+                next_open=next_open,
+                next_timestamp=candles[index + 1].timestamp,
                 atr=float(candles[index].atr_14 or 0.0),
                 expected_percent=expected_percent,
                 probability_up=probability_up,
@@ -409,7 +429,10 @@ def _batch_forecast_records(
                 q50 = float(selected["q50"])
                 expected_percent = (math.exp(expected_return) - 1.0) * 100.0
                 q50_percent = (math.exp(q50) - 1.0) * 100.0
-                future_log_return = math.log(closes[index + decision_horizon] / closes[index]) - round_trip_cost
+                next_open = float(candles[index + 1].open)
+                if next_open <= 0:
+                    continue
+                future_log_return = math.log(closes[index + decision_horizon] / next_open) - round_trip_cost
                 future_net_percent = (math.exp(future_log_return) - 1.0) * 100.0
                 records.append(
                     ForecastRecord(
@@ -417,6 +440,10 @@ def _batch_forecast_records(
                         index=index,
                         timestamp=candles[index].timestamp,
                         close=closes[index],
+                        high=float(candles[index].high),
+                        low=float(candles[index].low),
+                        next_open=next_open,
+                        next_timestamp=candles[index + 1].timestamp,
                         atr=float(candles[index].atr_14 or 0.0),
                         expected_percent=expected_percent,
                         probability_up=probability_up,
@@ -569,12 +596,13 @@ def _full_backtest(
     rows: list[dict[str, Any]] = []
     max_hold = max(12, horizon * 8)
     stop_loss_percent = max(0.003, min(0.08, float(settings.stop_loss_percent))) * 100.0
+    take_profit_percent = max(0.003, min(0.20, float(settings.take_profit_percent))) * 100.0
     stop_loss_exit_enabled = bool(getattr(settings, "stop_loss_exit_enabled", True))
     atr_multiplier = max(0.5, min(10.0, float(settings.atr_trailing_multiplier)))
     for record in sorted(records, key=lambda item: (item.timestamp, item.symbol)):
         position = positions.get(record.symbol)
         if position is not None:
-            position["highest"] = max(position["highest"], record.close)
+            position["highest"] = max(position["highest"], record.high)
             net_percent = _net_percent(position["entry_price"], record.close, round_trip_cost)
             held = record.index - int(position["entry_index"])
             atr_stop_level = (
@@ -584,7 +612,7 @@ def _full_backtest(
             )
             atr_stop = bool(
                 atr_stop_level is not None
-                and record.close <= atr_stop_level
+                and record.low <= atr_stop_level
                 and (stop_loss_exit_enabled or atr_stop_level > position["entry_price"])
             )
             weak_forecast = (
@@ -593,10 +621,18 @@ def _full_backtest(
                 or record.skill <= 0.0
             )
             exit_reason = ""
-            if stop_loss_exit_enabled and net_percent <= -stop_loss_percent:
+            exit_price = record.close
+            stop_level = position["entry_price"] * (1.0 - stop_loss_percent / 100.0)
+            take_level = position["entry_price"] * (1.0 + take_profit_percent / 100.0)
+            if stop_loss_exit_enabled and record.low <= stop_level:
                 exit_reason = "stop_loss"
+                exit_price = stop_level
+            elif record.high >= take_level:
+                exit_reason = "take_profit"
+                exit_price = take_level
             elif atr_stop:
                 exit_reason = "atr_trailing_stop"
+                exit_price = float(atr_stop_level)
             elif (record.expected_percent <= 0.0 or record.probability_up <= 0.50 or _candidate_blocks(record, thresholds.edge)):
                 exit_reason = "forecast_negative"
             elif weak_forecast and net_percent >= 0:
@@ -604,6 +640,7 @@ def _full_backtest(
             elif held >= max_hold:
                 exit_reason = "max_hold"
             if exit_reason:
+                net_percent = _net_percent(position["entry_price"], exit_price, round_trip_cost)
                 trades.append(net_percent)
                 rows.append(
                     {
@@ -624,10 +661,10 @@ def _full_backtest(
             continue
         if _candidate_allows(record, thresholds.edge, thresholds.probability, thresholds.confidence):
             positions[record.symbol] = {
-                "entry_price": record.close,
-                "entry_index": record.index,
-                "timestamp": record.timestamp,
-                "highest": record.close,
+                "entry_price": record.next_open,
+                "entry_index": record.index + 1,
+                "timestamp": record.next_timestamp,
+                "highest": record.next_open,
                 "probability_up": record.probability_up,
                 "expected_percent": record.expected_percent,
             }
@@ -669,12 +706,13 @@ def _benchmark_backtest(
     rows: list[dict[str, Any]] = []
     max_hold = max(12, horizon * 8)
     stop_loss_percent = max(0.003, min(0.08, float(settings.stop_loss_percent))) * 100.0
+    take_profit_percent = max(0.003, min(0.20, float(settings.take_profit_percent))) * 100.0
     stop_loss_exit_enabled = bool(getattr(settings, "stop_loss_exit_enabled", True))
     atr_multiplier = max(0.5, min(10.0, float(settings.atr_trailing_multiplier)))
     for record in sorted(records, key=lambda item: (item.timestamp, item.symbol)):
         position = positions.get(record.symbol)
         if position is not None:
-            position["highest"] = max(position["highest"], record.close)
+            position["highest"] = max(position["highest"], record.high)
             net_percent = _net_percent(position["entry_price"], record.close, round_trip_cost)
             held = record.index - int(position["entry_index"])
             atr_stop_level = (
@@ -684,19 +722,28 @@ def _benchmark_backtest(
             )
             atr_stop = bool(
                 atr_stop_level is not None
-                and record.close <= atr_stop_level
+                and record.low <= atr_stop_level
                 and (stop_loss_exit_enabled or atr_stop_level > position["entry_price"])
             )
             exit_reason = ""
-            if stop_loss_exit_enabled and net_percent <= -stop_loss_percent:
+            exit_price = record.close
+            stop_level = position["entry_price"] * (1.0 - stop_loss_percent / 100.0)
+            take_level = position["entry_price"] * (1.0 + take_profit_percent / 100.0)
+            if stop_loss_exit_enabled and record.low <= stop_level:
                 exit_reason = "stop_loss"
+                exit_price = stop_level
+            elif record.high >= take_level:
+                exit_reason = "take_profit"
+                exit_price = take_level
             elif atr_stop:
                 exit_reason = "atr_trailing_stop"
+                exit_price = float(atr_stop_level)
             elif record.benchmark_exit:
                 exit_reason = "benchmark_exit"
             elif held >= max_hold:
                 exit_reason = "max_hold"
             if exit_reason:
+                net_percent = _net_percent(position["entry_price"], exit_price, round_trip_cost)
                 trades.append(net_percent)
                 rows.append(
                     {
@@ -715,10 +762,10 @@ def _benchmark_backtest(
             continue
         if record.benchmark_entry:
             positions[record.symbol] = {
-                "entry_price": record.close,
-                "entry_index": record.index,
-                "timestamp": record.timestamp,
-                "highest": record.close,
+                "entry_price": record.next_open,
+                "entry_index": record.index + 1,
+                "timestamp": record.next_timestamp,
+                "highest": record.next_open,
             }
     for symbol, position in list(positions.items()):
         tail = next((record for record in reversed(records) if record.symbol == symbol), None)
@@ -888,6 +935,7 @@ def _quality_gate(
     return {
         "status": "pass" if passed else "fail",
         "passed": passed,
+        "protocol": "untouched_model_holdout_with_threshold_walk_forward",
         "checks": checks,
         "oos_summary": summary,
         "benchmark_summary": benchmark_summary,
@@ -1216,6 +1264,9 @@ def _artifact_summary(artifact: dict[str, Any]) -> dict[str, Any]:
                 "lookback": row.get("lookback"),
                 "hidden_size": row.get("hidden_size"),
                 "skill": row.get("skill"),
+                "validation_skill": row.get("validation_skill"),
+                "holdout_skill": row.get("holdout_skill"),
+                "holdout_start_timestamp": row.get("holdout_start_timestamp"),
                 "directional_accuracy": row.get("directional_accuracy"),
             }
             for symbol, row in (artifact.get("symbols") or {}).items()

tools/run_torch_retrain.ps1

@@ -14,6 +14,7 @@ param(
     [int]$Seed = 0,
     [int]$Epochs = 0,
     [int]$Patience = 0,
+    [int]$HoldoutWindow = 0,
     [string]$Interval = "",
     [string]$EnvFile = "",
     [switch]$DeployToPi,
@@ -22,7 +23,8 @@ param(
     [string]$PiRoot = "",
     [string]$PiSshKeyPath = "",
     [switch]$NoPiRestart,
-    [switch]$SkipGuard
+    [switch]$SkipGuard,
+    [switch]$ResumeCandidate
 )
 
 $ErrorActionPreference = "Stop"
@@ -38,6 +40,30 @@ function Write-RetrainLog {
     "[$timestamp] $Message" | Tee-Object -FilePath $LogFile -Append
 }
 
+function Invoke-LoggedNativeCommand {
+    param(
+        [string]$FilePath,
+        [object[]]$ArgumentList,
+        [string]$LogPath
+    )
+
+    # Windows PowerShell converts redirected native stderr into PowerShell error
+    # records. With the script-wide Stop preference an expected non-zero exit
+    # would jump to catch before callers can inspect LASTEXITCODE.
+    $previousErrorActionPreference = $ErrorActionPreference
+    try {
+        $ErrorActionPreference = "Continue"
+        & $FilePath @ArgumentList 2>&1 |
+            Tee-Object -FilePath $LogPath -Append |
+            Out-Host
+        $exitCode = $LASTEXITCODE
+    }
+    finally {
+        $ErrorActionPreference = $previousErrorActionPreference
+    }
+    return [int]$exitCode
+}
+
 function Resolve-Python {
     $venvPython = Join-Path $RepoRoot ".venv\Scripts\python.exe"
     if (Test-Path $venvPython) {
@@ -120,6 +146,7 @@ if (-not $ContextSymbols -and $env:TORCH_RETRAIN_CONTEXT_SYMBOLS) { $ContextSymb
 if ($Seed -le 0 -and $env:TORCH_RETRAIN_SEED) { $Seed = [int]$env:TORCH_RETRAIN_SEED }
 if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 70 } }
 if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 8 } }
+if ($HoldoutWindow -le 0) { $HoldoutWindow = if ($env:TORCH_RETRAIN_HOLDOUT_WINDOW) { [int]$env:TORCH_RETRAIN_HOLDOUT_WINDOW } else { 240 } }
 if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RETRAIN_INTERVAL }
 if (-not $EnvFile -and $env:TORCH_RETRAIN_ENV) { $EnvFile = $env:TORCH_RETRAIN_ENV }
 if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" }
@@ -154,6 +181,7 @@ try {
         "--dropouts", $Dropouts,
         "--epochs", $Epochs.ToString(),
         "--patience", $Patience.ToString(),
+        "--holdout-window", $HoldoutWindow.ToString(),
         "--output", $CandidateFile
     )
     if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) }
@@ -167,24 +195,28 @@ try {
 
     Push-Location $RepoRoot
     $pushedLocation = $true
-    Write-RetrainLog "Starting PyTorch recurrent retrain: $python $($trainerArgs -join ' ')"
-    & $python @trainerArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
-    $trainerExitCode = $LASTEXITCODE
-    if ($trainerExitCode -ne 0) {
-        if (Test-TorchArtifactFile $CandidateFile) {
-            Write-RetrainLog "WARNING: Trainer exited with code $trainerExitCode after writing a valid candidate artifact; continuing to guard."
-        }
-        else {
-            throw "Trainer failed with exit code $trainerExitCode."
+    if ($ResumeCandidate) {
+        if (-not (Test-TorchArtifactFile $CandidateFile)) {
+            throw "ResumeCandidate requested, but no valid candidate artifact exists: $CandidateFile"
         }
+        Write-RetrainLog "Resuming guard from existing candidate artifact: $CandidateFile"
+    }
+    else {
+        Write-RetrainLog "Starting PyTorch recurrent retrain: $python $($trainerArgs -join ' ')"
+        $trainerExitCode = Invoke-LoggedNativeCommand -FilePath $python -ArgumentList $trainerArgs -LogPath $LogFile
+        if ($trainerExitCode -ne 0) {
+            if (Test-TorchArtifactFile $CandidateFile) {
+                Write-RetrainLog "WARNING: Trainer exited with code $trainerExitCode after writing a valid candidate artifact; continuing to guard."
+            }
+            else {
+                throw "Trainer failed with exit code $trainerExitCode."
+            }
+        }
+        Write-RetrainLog "Finished PyTorch recurrent retrain candidate: $CandidateFile"
     }
-    Write-RetrainLog "Finished PyTorch recurrent retrain candidate: $CandidateFile"
 
-    if ($SkipGuard -or -not (Test-Path $ModelFile)) {
-        Move-Item -Force -LiteralPath $CandidateFile -Destination $ModelFile
-        Write-RetrainLog "Accepted candidate without guard. Active artifact: $ModelFile"
-        Sync-AcceptedArtifactsToPi
-        exit 0
+    if ($SkipGuard) {
+        throw "SkipGuard is disabled: every candidate must pass untouched-holdout validation."
     }
 
     $calibrationBaseArgs = @(
@@ -199,31 +231,48 @@ try {
     if ($Symbols) { $calibrationBaseArgs += @("--symbols", $Symbols) }
     if ($EnvFile) { $calibrationBaseArgs += @("--env", $EnvFile) }
 
-    Write-RetrainLog "Calibrating current artifact for guard."
-    & $python @($calibrationBaseArgs + @("--artifact", $ModelFile, "--output", $CurrentCalibration)) 2>&1 | Tee-Object -FilePath $LogFile -Append
-    if ($LASTEXITCODE -ne 0) {
-        throw "Current artifact calibration failed with exit code $LASTEXITCODE."
+    if (Test-Path $ModelFile) {
+        Write-RetrainLog "Calibrating current artifact for guard."
+        $currentCalibrationExitCode = Invoke-LoggedNativeCommand `
+            -FilePath $python `
+            -ArgumentList ($calibrationBaseArgs + @("--artifact", $ModelFile, "--output", $CurrentCalibration)) `
+            -LogPath $LogFile
+        if ($currentCalibrationExitCode -ne 0) {
+            Write-RetrainLog "Current artifact has no compatible untouched holdout; comparing candidate against an empty current report."
+            "{}" | Set-Content -LiteralPath $CurrentCalibration -Encoding utf8
+        }
+    }
+    else {
+        Write-RetrainLog "No active artifact yet; candidate still must pass the full guard."
+        "{}" | Set-Content -LiteralPath $CurrentCalibration -Encoding utf8
     }
 
     Write-RetrainLog "Calibrating candidate artifact for guard."
-    & $python @($calibrationBaseArgs + @("--artifact", $CandidateFile, "--output", $CandidateCalibration)) 2>&1 | Tee-Object -FilePath $LogFile -Append
-    if ($LASTEXITCODE -ne 0) {
-        throw "Candidate artifact calibration failed with exit code $LASTEXITCODE."
+    $candidateCalibrationExitCode = Invoke-LoggedNativeCommand `
+        -FilePath $python `
+        -ArgumentList ($calibrationBaseArgs + @("--artifact", $CandidateFile, "--output", $CandidateCalibration)) `
+        -LogPath $LogFile
+    if ($candidateCalibrationExitCode -ne 0) {
+        throw "Candidate artifact calibration failed with exit code $candidateCalibrationExitCode."
     }
 
     Write-RetrainLog "Running retrain guard."
-    & $python -u "tools\accept_torch_candidate.py" `
-        --current-report $CurrentCalibration `
-        --candidate-report $CandidateCalibration `
-        --candidate-artifact $CandidateFile `
-        --target-artifact $ModelFile `
-        --report $GuardReport 2>&1 | Tee-Object -FilePath $LogFile -Append
-    if ($LASTEXITCODE -eq 2) {
+    $guardArgs = @(
+        "-u",
+        "tools\accept_torch_candidate.py",
+        "--current-report", $CurrentCalibration,
+        "--candidate-report", $CandidateCalibration,
+        "--candidate-artifact", $CandidateFile,
+        "--target-artifact", $ModelFile,
+        "--report", $GuardReport
+    )
+    $guardExitCode = Invoke-LoggedNativeCommand -FilePath $python -ArgumentList $guardArgs -LogPath $LogFile
+    if ($guardExitCode -eq 2) {
         Write-RetrainLog "Candidate rejected by guard; keeping active artifact: $ModelFile"
         exit 0
     }
-    if ($LASTEXITCODE -ne 0) {
-        throw "Retrain guard failed with exit code $LASTEXITCODE."
+    if ($guardExitCode -ne 0) {
+        throw "Retrain guard failed with exit code $guardExitCode."
     }
     if (Test-Path $CandidateCalibration) {
         Copy-Item -Force -LiteralPath $CandidateCalibration -Destination (Join-Path $RuntimeDir "torch_threshold_calibration.json")

tools/train_torch_recurrent_forecaster.py

@@ -45,6 +45,12 @@ class PreparedData:
     validation_up: torch.Tensor
     validation_targets: list[list[float]]
     validation_volatility_scales: list[list[float]]
+    holdout_x: torch.Tensor
+    holdout_y: torch.Tensor
+    holdout_up: torch.Tensor
+    holdout_targets: list[list[float]]
+    holdout_volatility_scales: list[list[float]]
+    holdout_start_timestamp: int
     feature_names: list[str]
     feature_means: list[float]
     feature_scales: list[float]
@@ -55,6 +61,7 @@ class PreparedData:
     decision_horizon_index: int
     train_samples: int
     validation_samples: int
+    holdout_samples: int
 
 
 @dataclass(slots=True)
@@ -63,6 +70,7 @@ class TrainingSample:
     normalized_targets: list[float]
     raw_targets: list[float]
     volatility_scales: list[float]
+    timestamp: int
 
 
 class RecurrentReturnModel(nn.Module):
@@ -131,6 +139,7 @@ def main() -> None:
         "interval": interval,
         "limit": args.limit,
         "validation_window": args.validation_window,
+        "holdout_window": args.holdout_window,
         "target_horizon": decision_horizon,
         "target_horizons": target_horizons,
         "direct_horizon": True,
@@ -154,6 +163,7 @@ def main() -> None:
             interval=interval,
             limit=args.limit,
             validation_window=args.validation_window,
+            holdout_window=args.holdout_window,
             target_horizons=target_horizons,
             decision_horizon=decision_horizon,
             feature_names=feature_names,
@@ -207,6 +217,7 @@ def _parse_args() -> argparse.Namespace:
     parser.add_argument("--interval", default="", help="Bybit kline interval. Defaults to BASE_INTERVAL.")
     parser.add_argument("--limit", type=int, default=1000, help="Kline limit per symbol.")
     parser.add_argument("--validation-window", type=int, default=120, help="Held-out tail targets used for validation.")
+    parser.add_argument("--holdout-window", type=int, default=240, help="Final untouched samples reserved for model/threshold evaluation.")
     parser.add_argument("--horizon", type=int, default=0, help="Direct forecast horizon in candles. Defaults to TIME_SERIES_FORECAST_HORIZON.")
     parser.add_argument("--horizons", default="1,3,6,12", help="Comma-separated direct forecast horizons.")
     parser.add_argument("--features", default=",".join(DEFAULT_TORCH_FEATURES), help="Comma-separated feature names.")
@@ -246,6 +257,7 @@ def _train_symbol(
     interval: str,
     limit: int,
     validation_window: int,
+    holdout_window: int,
     target_horizons: list[int],
     decision_horizon: int,
     feature_names: list[str],
@@ -272,7 +284,10 @@ def _train_symbol(
     closes = [float(candle.close) for candle in candles if candle.close > 0]
     returns = _log_returns(closes)
     max_horizon = max(target_horizons)
-    if len(candles) < max(180, validation_window + max(lookbacks) + max_horizon + 16):
+    if len(candles) < max(
+        240,
+        validation_window + holdout_window + max(lookbacks) + max_horizon * 2 + 32,
+    ):
         return None
     market_candles: dict[str, list[Candle]] = {symbol.upper(): candles}
     for context_symbol in context_symbols:
@@ -301,6 +316,7 @@ def _train_symbol(
             market_candles=market_candles,
             trend_candles=trend_candles,
             validation_window=validation_window,
+            holdout_window=holdout_window,
             clip=clip,
             device=device,
         )
@@ -376,17 +392,23 @@ def _train_symbol(
                             "clip": clip,
                             "validation_mae_percent": validation_mae * 100,
                             "baseline_mae_percent": baseline_mae * 100,
+                            "holdout_mae_percent": float(candidate.get("holdout_mae", 0.0)) * 100,
+                            "holdout_baseline_mae_percent": float(candidate.get("holdout_baseline_mae", 0.0)) * 100,
                             "skill": skill,
                             "candles": len(candles),
                             "returns": len(returns),
                             "train_samples": prepared.train_samples,
                             "validation_samples": prepared.validation_samples,
+                            "holdout_samples": prepared.holdout_samples,
+                            "holdout_start_timestamp": prepared.holdout_start_timestamp,
                         }
                         score = _candidate_score(row)
                         if best is None or score < _candidate_score(best):
                             best = row
     if best is None:
         return None
+    best["validation_skill"] = best.get("skill", 0.0)
+    best["skill"] = best.get("holdout_skill", 0.0)
     best.pop("validation_mae", None)
     return best
 
@@ -402,6 +424,7 @@ def _prepare_data(
     market_candles: dict[str, list[Candle]],
     trend_candles: list[Candle],
     validation_window: int,
+    holdout_window: int,
     clip: float,
     device: torch.device,
 ) -> PreparedData | None:
@@ -436,14 +459,29 @@ def _prepare_data(
             volatility_scales.append(volatility_scale)
             normalized_targets.append(net_return / max(volatility_scale, 1e-8))
         if valid:
-            samples.append(TrainingSample(window, normalized_targets, raw_targets, volatility_scales))
+            samples.append(
+                TrainingSample(
+                    window,
+                    normalized_targets,
+                    raw_targets,
+                    volatility_scales,
+                    candles[end_index].timestamp,
+                )
+            )
     if len(samples) < 48:
         return None
 
-    validation_window = min(max(16, validation_window), max(16, len(samples) // 3))
-    train_samples = samples[:-validation_window]
-    validation_samples = samples[-validation_window:]
-    if len(train_samples) < 24 or len(validation_samples) < 8:
+    max_horizon = max(target_horizons)
+    holdout_window = min(max(32, holdout_window), max(32, len(samples) // 4))
+    holdout_start = len(samples) - holdout_window
+    validation_end = holdout_start - max_horizon
+    validation_window = min(max(16, validation_window), max(16, validation_end // 3))
+    validation_start = validation_end - validation_window
+    train_end = validation_start - max_horizon
+    train_samples = samples[:train_end]
+    validation_samples = samples[validation_start:validation_end]
+    holdout_samples = samples[holdout_start:]
+    if len(train_samples) < 24 or len(validation_samples) < 8 or len(holdout_samples) < 16:
         return None
 
     feature_means, feature_scales = _feature_stats(train_samples, len(feature_names))
@@ -470,6 +508,14 @@ def _prepare_data(
         target_scales=target_scales,
         clip=clip,
     )
+    holdout_x, holdout_y, holdout_up = _normalize_samples(
+        holdout_samples,
+        feature_means=feature_means,
+        feature_scales=feature_scales,
+        target_means=target_means,
+        target_scales=target_scales,
+        clip=clip,
+    )
     return PreparedData(
         train_x=torch.tensor(train_x, dtype=torch.float32, device=device),
         train_y=torch.tensor(train_y, dtype=torch.float32, device=device),
@@ -479,6 +525,12 @@ def _prepare_data(
         validation_up=torch.tensor(validation_up, dtype=torch.float32, device=device),
         validation_targets=[sample.raw_targets for sample in validation_samples],
         validation_volatility_scales=[sample.volatility_scales for sample in validation_samples],
+        holdout_x=torch.tensor(holdout_x, dtype=torch.float32, device=device),
+        holdout_y=torch.tensor(holdout_y, dtype=torch.float32, device=device),
+        holdout_up=torch.tensor(holdout_up, dtype=torch.float32, device=device),
+        holdout_targets=[sample.raw_targets for sample in holdout_samples],
+        holdout_volatility_scales=[sample.volatility_scales for sample in holdout_samples],
+        holdout_start_timestamp=holdout_samples[0].timestamp,
         feature_names=feature_names,
         feature_means=feature_means,
         feature_scales=feature_scales,
@@ -489,6 +541,7 @@ def _prepare_data(
         decision_horizon_index=decision_horizon_index,
         train_samples=len(train_x),
         validation_samples=len(validation_x),
+        holdout_samples=len(holdout_x),
     )
 
 
@@ -635,8 +688,10 @@ def _fit_candidate(
 
     if best_state:
         model.load_state_dict(best_state)
+    holdout_metrics = _holdout_metrics(model, prepared, clip)
     return {
         **best_metrics,
+        **holdout_metrics,
         "best_epoch": best_epoch,
         "epochs_trained": best_epoch + stale_epochs,
         "state_dict": _export_recurrent_state(model),
@@ -647,10 +702,57 @@ def _fit_candidate(
 
 
 def _validation_metrics(model: nn.Module, prepared: PreparedData, clip: float) -> dict[str, float]:
+    return _evaluation_metrics(
+        model,
+        values=prepared.validation_x,
+        targets=prepared.validation_targets,
+        volatility_scales=prepared.validation_volatility_scales,
+        prepared=prepared,
+        clip=clip,
+    )
+
+
+def _holdout_metrics(model: nn.Module, prepared: PreparedData, clip: float) -> dict[str, Any]:
+    metrics = _evaluation_metrics(
+        model,
+        values=prepared.holdout_x,
+        targets=prepared.holdout_targets,
+        volatility_scales=prepared.holdout_volatility_scales,
+        prepared=prepared,
+        clip=clip,
+    )
+    baseline_by_horizon = metrics.get("baseline_mae_by_horizon", {})
+    holdout_baseline = float(
+        baseline_by_horizon.get(str(prepared.decision_horizon), metrics["validation_mae"])
+    )
+    holdout_mae = float(metrics["validation_mae"])
+    return {
+        "holdout_mae": holdout_mae,
+        "holdout_baseline_mae": holdout_baseline,
+        "holdout_skill": (
+            (holdout_baseline - holdout_mae) / holdout_baseline
+            if holdout_baseline > 0
+            else 0.0
+        ),
+        "holdout_directional_accuracy": metrics["directional_accuracy"],
+        "holdout_buy_precision": metrics["buy_precision"],
+        "holdout_probability_brier": metrics["probability_brier"],
+    }
+
+
+def _evaluation_metrics(
+    model: nn.Module,
+    *,
+    values: torch.Tensor,
+    targets: list[list[float]],
+    volatility_scales: list[list[float]],
+    prepared: PreparedData,
+    clip: float,
+) -> dict[str, float]:
     model.eval()
     with torch.no_grad():
-        raw_outputs = model(prepared.validation_x).detach().cpu()
-    outputs = raw_outputs.view(len(prepared.validation_targets), len(prepared.target_horizons), len(OUTPUT_LAYOUT))
+        raw_outputs = model(values).detach().cpu()
+    outputs = raw_outputs.view(len(targets), len(prepared.target_horizons), len(OUTPUT_LAYOUT))
     mean_predictions = outputs[:, :, 0].tolist()
     logit_predictions = outputs[:, :, 4].tolist()
     predictions: list[list[float]] = []
@@ -664,13 +766,13 @@ def _validation_metrics(model: nn.Module, prepared: PreparedData, clip: float) -
                 * prepared.target_scales[horizon_index]
                 + prepared.target_means[horizon_index]
             )
-            predicted_row.append(transformed * prepared.validation_volatility_scales[row_index][horizon_index])
+            predicted_row.append(transformed * volatility_scales[row_index][horizon_index])
             probability_row.append(_sigmoid(float(logit_predictions[row_index][horizon_index])))
         predictions.append(predicted_row)
         probabilities.append(probability_row)
     decision = prepared.decision_horizon_index
     decision_predictions = [row[decision] for row in predictions]
-    decision_targets = [row[decision] for row in prepared.validation_targets]
+    decision_targets = [row[decision] for row in targets]
     errors = [abs(prediction - actual) for prediction, actual in zip(decision_predictions, decision_targets)]
     correct = [
         1.0
@@ -693,9 +795,9 @@ def _validation_metrics(model: nn.Module, prepared: PreparedData, clip: float) -
     for horizon_index, horizon in enumerate(prepared.target_horizons):
         horizon_errors = [
             abs(row[horizon_index] - actual[horizon_index])
-            for row, actual in zip(predictions, prepared.validation_targets)
+            for row, actual in zip(predictions, targets)
         ]
-        horizon_baseline = [abs(actual[horizon_index]) for actual in prepared.validation_targets]
+        horizon_baseline = [abs(actual[horizon_index]) for actual in targets]
         by_horizon[str(horizon)] = sum(horizon_errors) / len(horizon_errors) if horizon_errors else math.inf
         baseline_by_horizon[str(horizon)] = (
             sum(horizon_baseline) / len(horizon_baseline)

tools/windows_training_agent.py

@@ -64,6 +64,11 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
     try:
         run_retrain(args, job_id, job, repo_root, log_path)
         summary = read_json(runtime_dir / "torch_retrain_guard.json")
+        if summary.get("accepted") is not True:
+            raise RuntimeError(
+                "candidate rejected by untouched-holdout guard: "
+                + str(summary.get("reason") or "validation failed")
+            )
         report_progress(args, job_id, "running", "uploading", 72, "Обучение завершено, загружаю артефакты")
         for name in ARTIFACT_NAMES:
             path = runtime_dir / name
@@ -102,11 +107,14 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
         "layers": "-Layers",
         "dropouts": "-Dropouts",
         "epochs": "-Epochs",
+        "holdout_window": "-HoldoutWindow",
     }
     for key, ps_arg in arg_map.items():
         value = parameters.get(key)
         if value not in (None, ""):
             cmd.extend([ps_arg, str(value)])
+    if parameters.get("resume_candidate") is True:
+        cmd.append("-ResumeCandidate")
     log(log_path, "Running retrain: " + " ".join(quote_for_log(part) for part in cmd))
     report_progress(args, job_id, "running", "training", 8, "PyTorch retrain запущен")
     line_count = 0