Add honest Torch validation gate

tools/accept_torch_candidate.py

@@ -64,6 +64,8 @@ def _decision(
     candidate_replay = candidate.get("full_replay") if isinstance(candidate.get("full_replay"), dict) else {}
     candidate_walk = candidate.get("walk_forward") if isinstance(candidate.get("walk_forward"), dict) else {}
     walk_summary = candidate_walk.get("summary") if isinstance(candidate_walk.get("summary"), dict) else {}
+    if not _validation_passed(candidate):
+        return {"accepted": False, "reason": "candidate_failed_honest_validation"}
     if int(candidate_replay.get("trades", 0) or 0) < min_trades:
         return {"accepted": False, "reason": "candidate_has_too_few_full_replay_trades"}
     if float(candidate_replay.get("profit_factor", 0.0) or 0.0) < min_profit_factor:
@@ -77,6 +79,15 @@ def _decision(
     return {"accepted": True, "reason": "candidate_passed_guard"}
 
 
+def _validation_passed(report: dict[str, Any]) -> bool:
+    validation = report.get("validation")
+    if not isinstance(validation, dict):
+        return False
+    if "passed" in validation:
+        return bool(validation.get("passed"))
+    return str(validation.get("status", "")).strip().lower() in {"pass", "passed", "ok"}
+
+
 def _score(report: dict[str, Any]) -> float:
     replay = report.get("full_replay") if isinstance(report.get("full_replay"), dict) else {}
     recommended = report.get("recommended") if isinstance(report.get("recommended"), dict) else {}
@@ -97,6 +108,10 @@ def _summary(report: dict[str, Any]) -> dict[str, Any]:
         "walk_forward_summary": (report.get("walk_forward") or {}).get("summary", {})
         if isinstance(report.get("walk_forward"), dict)
         else {},
+        "benchmark_summary": (report.get("benchmark") or {}).get("summary", {})
+        if isinstance(report.get("benchmark"), dict)
+        else {},
+        "validation": report.get("validation", {}),
     }
 
 

tools/calibrate_torch_thresholds.py

@@ -57,6 +57,8 @@ class ForecastRecord:
     q50_percent: float
     block_entry: bool
     future_net_percent: float
+    benchmark_entry: bool
+    benchmark_exit: bool
 
 
 @dataclass(slots=True)
@@ -159,9 +161,32 @@ def main() -> None:
         min_trades=args.min_trades,
         horizon=horizon,
         folds=args.walk_forward_folds,
+        round_trip_cost=round_trip_cost,
+        settings=settings,
+    )
+    benchmark = _benchmark_walk_forward(
+        records,
+        horizon=horizon,
+        folds=args.walk_forward_folds,
+        round_trip_cost=round_trip_cost,
+        settings=settings,
+    )
+    validation = _quality_gate(
+        walk_forward=walk_forward,
+        benchmark=benchmark,
+        min_oos_trades=args.min_oos_trades,
+        min_oos_symbols=args.min_oos_symbols,
+        max_symbol_share=args.max_oos_symbol_share,
+        min_oos_folds=args.min_oos_folds_with_trades,
+        min_profit_factor=args.min_oos_profit_factor,
+        min_benchmark_edge=args.min_benchmark_edge_percent,
     )
     print("\nWALK_FORWARD")
     print(json.dumps(walk_forward["summary"], ensure_ascii=False, sort_keys=True))
+    print("\nBENCHMARK")
+    print(json.dumps(benchmark["summary"], ensure_ascii=False, sort_keys=True))
+    print("\nQUALITY_GATE")
+    print(json.dumps(validation, ensure_ascii=False, sort_keys=True))
     print(
         "env "
         f"TIME_SERIES_MIN_EDGE_PERCENT={recommended.edge:.4f} "
@@ -176,6 +201,8 @@ def main() -> None:
             "recommended": _result_dict(recommended),
             "full_replay": full_backtest,
             "walk_forward": walk_forward,
+            "benchmark": benchmark,
+            "validation": validation,
             "probability_calibration": _probability_calibration(records),
             "top_results": [_result_dict(result) for result in results[: args.top]],
         }
@@ -202,6 +229,12 @@ def _parse_args() -> argparse.Namespace:
     parser.add_argument("--batch-size", type=int, default=256, help="Torch inference batch size.")
     parser.add_argument("--threads", type=int, default=0, help="Torch CPU threads; 0 keeps torch default.")
     parser.add_argument("--walk-forward-folds", type=int, default=4, help="Threshold walk-forward folds.")
+    parser.add_argument("--min-oos-trades", type=int, default=30, help="Minimum out-of-sample walk-forward trades for a valid model.")
+    parser.add_argument("--min-oos-symbols", type=int, default=2, help="Minimum symbols with out-of-sample trades.")
+    parser.add_argument("--max-oos-symbol-share", type=float, default=0.75, help="Reject if one symbol contributes more than this share of out-of-sample trades.")
+    parser.add_argument("--min-oos-folds-with-trades", type=int, default=2, help="Minimum walk-forward folds that must produce trades.")
+    parser.add_argument("--min-oos-profit-factor", type=float, default=1.10, help="Minimum out-of-sample profit factor.")
+    parser.add_argument("--min-benchmark-edge-percent", type=float, default=0.0, help="Required total-net percent advantage over the benchmark.")
     return parser.parse_args()
 
 
@@ -245,6 +278,7 @@ def _forecast_records(
     batched_records = _batch_forecast_records(
         symbol=symbol,
         candles=candles,
+        trend_candles=trend_candles,
         feature_rows=feature_rows,
         closes=closes,
         entry=entry,
@@ -295,6 +329,8 @@ def _forecast_records(
                 q50_percent=q50_percent,
                 block_entry=False,
                 future_net_percent=future_net_percent,
+                benchmark_entry=_benchmark_entry_signal(candles, trend_candles, index),
+                benchmark_exit=_benchmark_exit_signal(candles, index),
             )
         )
     return records
@@ -304,6 +340,7 @@ def _batch_forecast_records(
     *,
     symbol: str,
     candles: list[Candle],
+    trend_candles: list[Candle],
     feature_rows: list[list[float]],
     closes: list[float],
     entry: dict[str, Any],
@@ -388,6 +425,8 @@ def _batch_forecast_records(
                         q50_percent=q50_percent,
                         block_entry=False,
                         future_net_percent=future_net_percent,
+                        benchmark_entry=_benchmark_entry_signal(candles, trend_candles, index),
+                        benchmark_exit=_benchmark_exit_signal(candles, index),
                     )
                 )
     return records
@@ -523,6 +562,7 @@ def _full_backtest(
     horizon: int,
     round_trip_cost: float,
     settings: Any,
+    detail_limit: int = 50,
 ) -> dict[str, Any]:
     positions: dict[str, dict[str, Any]] = {}
     trades: list[float] = []
@@ -603,7 +643,92 @@ def _full_backtest(
                 "entry_expected_percent": round(float(position["expected_percent"]), 4),
             }
         )
-    return {**_stats(trades), "trades_detail": rows[-50:]}
+    return {
+        **_stats(trades),
+        "trades_detail": _limited_rows(rows, detail_limit),
+        "symbol_breakdown": _symbol_breakdown(rows),
+    }
+
+
+def _benchmark_backtest(
+    records: list[ForecastRecord],
+    *,
+    horizon: int,
+    round_trip_cost: float,
+    settings: Any,
+    detail_limit: int = 50,
+) -> dict[str, Any]:
+    positions: dict[str, dict[str, Any]] = {}
+    trades: list[float] = []
+    rows: list[dict[str, Any]] = []
+    max_hold = max(12, horizon * 8)
+    stop_loss_percent = max(0.003, min(0.08, float(settings.stop_loss_percent))) * 100.0
+    atr_multiplier = max(0.5, min(10.0, float(settings.atr_trailing_multiplier)))
+    for record in sorted(records, key=lambda item: (item.timestamp, item.symbol)):
+        position = positions.get(record.symbol)
+        if position is not None:
+            position["highest"] = max(position["highest"], record.close)
+            net_percent = _net_percent(position["entry_price"], record.close, round_trip_cost)
+            held = record.index - int(position["entry_index"])
+            atr_stop = (
+                record.close <= position["highest"] - record.atr * atr_multiplier
+                if record.atr > 0 and position["highest"] > position["entry_price"]
+                else False
+            )
+            exit_reason = ""
+            if net_percent <= -stop_loss_percent:
+                exit_reason = "stop_loss"
+            elif atr_stop:
+                exit_reason = "atr_trailing_stop"
+            elif record.benchmark_exit:
+                exit_reason = "benchmark_exit"
+            elif held >= max_hold:
+                exit_reason = "max_hold"
+            if exit_reason:
+                trades.append(net_percent)
+                rows.append(
+                    {
+                        "symbol": record.symbol,
+                        "entry_timestamp": position["timestamp"],
+                        "exit_timestamp": record.timestamp,
+                        "net_percent": round(net_percent, 4),
+                        "reason": exit_reason,
+                        "held_bars": held,
+                    }
+                )
+                positions.pop(record.symbol, None)
+                continue
+
+        if record.symbol in positions:
+            continue
+        if record.benchmark_entry:
+            positions[record.symbol] = {
+                "entry_price": record.close,
+                "entry_index": record.index,
+                "timestamp": record.timestamp,
+                "highest": record.close,
+            }
+    for symbol, position in list(positions.items()):
+        tail = next((record for record in reversed(records) if record.symbol == symbol), None)
+        if tail is None:
+            continue
+        net_percent = _net_percent(position["entry_price"], tail.close, round_trip_cost)
+        trades.append(net_percent)
+        rows.append(
+            {
+                "symbol": symbol,
+                "entry_timestamp": position["timestamp"],
+                "exit_timestamp": tail.timestamp,
+                "net_percent": round(net_percent, 4),
+                "reason": "end_of_replay",
+                "held_bars": tail.index - int(position["entry_index"]),
+            }
+        )
+    return {
+        **_stats(trades),
+        "trades_detail": _limited_rows(rows, detail_limit),
+        "symbol_breakdown": _symbol_breakdown(rows),
+    }
 
 
 def _walk_forward(
@@ -615,6 +740,8 @@ def _walk_forward(
     min_trades: int,
     horizon: int,
     folds: int,
+    round_trip_cost: float,
+    settings: Any,
 ) -> dict[str, Any]:
     ordered = sorted(records, key=lambda item: item.timestamp)
     if folds < 2 or len(ordered) < folds * 20:
@@ -623,6 +750,7 @@ def _walk_forward(
     fold_size = max(1, len(timestamps) // folds)
     rows = []
     all_test_trades: list[float] = []
+    all_test_rows: list[dict[str, Any]] = []
     for fold in range(1, folds):
         test_start = timestamps[fold * fold_size]
         test_end = timestamps[(fold + 1) * fold_size - 1] if fold < folds - 1 else timestamps[-1]
@@ -639,20 +767,124 @@ def _walk_forward(
         if not train_results:
             continue
         selected = _choose_recommendation(train_results, min_trades=max(4, min_trades // 2))
-        test_trades = _selected_trades(test, selected.edge, selected.probability, selected.confidence, horizon)
+        test_backtest = _full_backtest(
+            test,
+            selected,
+            horizon=horizon,
+            round_trip_cost=round_trip_cost,
+            settings=settings,
+            detail_limit=0,
+        )
+        test_rows = test_backtest.get("trades_detail", [])
+        test_trades = [float(row.get("net_percent", 0.0) or 0.0) for row in test_rows if isinstance(row, dict)]
         all_test_trades.extend(test_trades)
+        all_test_rows.extend(test_rows)
         rows.append(
             {
                 "fold": fold,
                 "train_records": len(train),
                 "test_records": len(test),
                 "thresholds": _result_dict(selected),
-                "test": _stats(test_trades),
+                "test": {key: value for key, value in test_backtest.items() if key != "trades_detail"},
             }
         )
     summary = _stats(all_test_trades)
     summary["status"] = "ok" if summary["trades"] >= min_trades and summary["avg_net_percent"] > 0 else "warn"
-    return {"summary": summary, "folds": rows}
+    return {
+        "summary": summary,
+        "symbol_breakdown": _symbol_breakdown(all_test_rows),
+        "folds_with_trades": sum(1 for row in rows if int((row.get("test") or {}).get("trades", 0) or 0) > 0),
+        "folds": rows,
+    }
+
+
+def _benchmark_walk_forward(
+    records: list[ForecastRecord],
+    *,
+    horizon: int,
+    folds: int,
+    round_trip_cost: float,
+    settings: Any,
+) -> dict[str, Any]:
+    ordered = sorted(records, key=lambda item: item.timestamp)
+    if folds < 2 or len(ordered) < folds * 20:
+        return {"summary": {"status": "insufficient"}, "symbol_breakdown": [], "folds_with_trades": 0, "folds": []}
+    timestamps = sorted({record.timestamp for record in ordered})
+    fold_size = max(1, len(timestamps) // folds)
+    rows = []
+    all_test_rows: list[dict[str, Any]] = []
+    for fold in range(1, folds):
+        test_start = timestamps[fold * fold_size]
+        test_end = timestamps[(fold + 1) * fold_size - 1] if fold < folds - 1 else timestamps[-1]
+        test = [record for record in ordered if test_start <= record.timestamp <= test_end]
+        test_backtest = _benchmark_backtest(
+            test,
+            horizon=horizon,
+            round_trip_cost=round_trip_cost,
+            settings=settings,
+            detail_limit=0,
+        )
+        test_rows = test_backtest.get("trades_detail", [])
+        all_test_rows.extend(test_rows)
+        rows.append(
+            {
+                "fold": fold,
+                "test_records": len(test),
+                "test": {key: value for key, value in test_backtest.items() if key != "trades_detail"},
+            }
+        )
+    trades = [float(row.get("net_percent", 0.0) or 0.0) for row in all_test_rows if isinstance(row, dict)]
+    summary = _stats(trades)
+    summary["status"] = "ok" if summary["trades"] > 0 else "no_trades"
+    return {
+        "name": "trend_macd_baseline",
+        "summary": summary,
+        "symbol_breakdown": _symbol_breakdown(all_test_rows),
+        "folds_with_trades": sum(1 for row in rows if int((row.get("test") or {}).get("trades", 0) or 0) > 0),
+        "folds": rows,
+    }
+
+
+def _quality_gate(
+    *,
+    walk_forward: dict[str, Any],
+    benchmark: dict[str, Any],
+    min_oos_trades: int,
+    min_oos_symbols: int,
+    max_symbol_share: float,
+    min_oos_folds: int,
+    min_profit_factor: float,
+    min_benchmark_edge: float,
+) -> dict[str, Any]:
+    summary = walk_forward.get("summary") if isinstance(walk_forward.get("summary"), dict) else {}
+    benchmark_summary = benchmark.get("summary") if isinstance(benchmark.get("summary"), dict) else {}
+    breakdown = walk_forward.get("symbol_breakdown") if isinstance(walk_forward.get("symbol_breakdown"), list) else []
+    symbols_with_trades = sum(1 for row in breakdown if int(row.get("trades", 0) or 0) > 0)
+    max_share = max((float(row.get("trade_share", 0.0) or 0.0) for row in breakdown), default=0.0)
+    oos_total = float(summary.get("total_net_percent", 0.0) or 0.0)
+    benchmark_total = float(benchmark_summary.get("total_net_percent", 0.0) or 0.0)
+    checks = [
+        _gate_check("oos_trades", int(summary.get("trades", 0) or 0), min_oos_trades, int(summary.get("trades", 0) or 0) >= min_oos_trades),
+        _gate_check("oos_symbols", symbols_with_trades, min_oos_symbols, symbols_with_trades >= min_oos_symbols),
+        _gate_check("max_symbol_share", round(max_share, 4), max_symbol_share, max_share <= max_symbol_share if breakdown else False),
+        _gate_check("folds_with_trades", int(walk_forward.get("folds_with_trades", 0) or 0), min_oos_folds, int(walk_forward.get("folds_with_trades", 0) or 0) >= min_oos_folds),
+        _gate_check("oos_avg_net_positive", float(summary.get("avg_net_percent", 0.0) or 0.0), "> 0", float(summary.get("avg_net_percent", 0.0) or 0.0) > 0),
+        _gate_check("oos_profit_factor", float(summary.get("profit_factor", 0.0) or 0.0), min_profit_factor, float(summary.get("profit_factor", 0.0) or 0.0) >= min_profit_factor),
+        _gate_check("beats_benchmark_total", round(oos_total - benchmark_total, 4), min_benchmark_edge, (oos_total - benchmark_total) > min_benchmark_edge),
+    ]
+    passed = all(bool(row["passed"]) for row in checks)
+    return {
+        "status": "pass" if passed else "fail",
+        "passed": passed,
+        "checks": checks,
+        "oos_summary": summary,
+        "benchmark_summary": benchmark_summary,
+        "symbol_breakdown": breakdown,
+    }
+
+
+def _gate_check(name: str, value: Any, required: Any, passed: bool) -> dict[str, Any]:
+    return {"name": name, "value": value, "required": required, "passed": bool(passed)}
 
 
 def _probability_calibration(records: list[ForecastRecord]) -> dict[str, Any]:
@@ -699,12 +931,89 @@ def _candidate_allows(record: ForecastRecord, edge: float, probability: float, c
     )
 
 
+def _benchmark_entry_signal(candles: list[Candle], trend_candles: list[Candle], index: int) -> bool:
+    if index <= 0 or index >= len(candles):
+        return False
+    previous = candles[index - 1]
+    current = candles[index]
+    rsi = current.rsi_14
+    return bool(
+        _daily_trend_ok(trend_candles, current.timestamp)
+        and _macd_crossed_up(previous, current)
+        and current.ema_50 is not None
+        and current.close > current.ema_50
+        and rsi is not None
+        and 45.0 <= rsi <= 65.0
+    )
+
+
+def _benchmark_exit_signal(candles: list[Candle], index: int) -> bool:
+    if index <= 0 or index >= len(candles):
+        return False
+    previous = candles[index - 1]
+    current = candles[index]
+    return bool(_macd_crossed_down(previous, current) or (current.ema_50 is not None and current.close < current.ema_50))
+
+
+def _daily_trend_ok(trend_candles: list[Candle], timestamp: int) -> bool:
+    for candle in reversed(trend_candles):
+        if candle.timestamp > timestamp:
+            continue
+        return bool(
+            candle.ema_50 is not None
+            and candle.ema_200 is not None
+            and candle.close > candle.ema_200
+            and candle.ema_50 > candle.ema_200
+        )
+    return False
+
+
+def _macd_crossed_up(previous: Candle, current: Candle) -> bool:
+    if None in (previous.macd, previous.macd_signal, current.macd, current.macd_signal):
+        return False
+    return bool(previous.macd <= previous.macd_signal and current.macd > current.macd_signal)
+
+
+def _macd_crossed_down(previous: Candle, current: Candle) -> bool:
+    if None in (previous.macd, previous.macd_signal, current.macd, current.macd_signal):
+        return False
+    return bool(previous.macd >= previous.macd_signal and current.macd < current.macd_signal)
+
+
 def _net_percent(entry_price: float, exit_price: float, round_trip_cost: float) -> float:
     if entry_price <= 0 or exit_price <= 0:
         return 0.0
     return (math.exp(math.log(exit_price / entry_price) - round_trip_cost) - 1.0) * 100.0
 
 
+def _limited_rows(rows: list[dict[str, Any]], detail_limit: int) -> list[dict[str, Any]]:
+    if detail_limit <= 0:
+        return rows
+    return rows[-detail_limit:]
+
+
+def _symbol_breakdown(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    by_symbol: dict[str, list[float]] = {}
+    for row in rows:
+        symbol = str(row.get("symbol", ""))
+        if not symbol:
+            continue
+        by_symbol.setdefault(symbol, []).append(float(row.get("net_percent", 0.0) or 0.0))
+    total_trades = sum(len(values) for values in by_symbol.values())
+    result = []
+    for symbol in sorted(by_symbol):
+        values = by_symbol[symbol]
+        stats = _stats(values)
+        result.append(
+            {
+                "symbol": symbol,
+                **stats,
+                "trade_share": round(len(values) / total_trades, 4) if total_trades else 0.0,
+            }
+        )
+    return result
+
+
 def _stats(values: list[float]) -> dict[str, Any]:
     wins = sum(1 for value in values if value > 0)
     total = sum(values)

tools/run_retrain_until_replay8.ps1

@@ -85,11 +85,24 @@ function Read-ActiveReplayTrades {
     }
 }
 
+function Read-ActiveValidationPassed {
+    if (-not (Test-Path $ActiveCalibration)) {
+        return $false
+    }
+    try {
+        $payload = Get-Content -Raw -LiteralPath $ActiveCalibration | ConvertFrom-Json
+        return [bool]$payload.validation.passed
+    }
+    catch {
+        return $false
+    }
+}
+
 $attempt = 0
 while ($true) {
     $activeReplayTrades = Read-ActiveReplayTrades
-    if ($activeReplayTrades -ge $MinReplayTrades) {
-        Write-LoopLog "Stop condition reached: active calibration full_replay.trades=$activeReplayTrades >= $MinReplayTrades."
+    if (Read-ActiveValidationPassed) {
+        Write-LoopLog "Stop condition reached: active calibration passed honest validation with full_replay.trades=$activeReplayTrades."
         exit 0
     }
 
@@ -123,12 +136,8 @@ while ($true) {
     $summary = Read-GuardSummary
     Write-LoopLog "Attempt $attempt finished; runner_exit=$runnerExit accepted=$($summary.Accepted) reason=$($summary.Reason) candidate_full_replay.trades=$($summary.CandidateReplayTrades) current_full_replay.trades=$($summary.CurrentReplayTrades) walk_forward.trades=$($summary.WalkForwardTrades)."
 
-    if ($summary.Accepted -and $summary.CandidateReplayTrades -ge $MinReplayTrades) {
-        Write-LoopLog "Stop condition reached: accepted candidate full_replay.trades=$($summary.CandidateReplayTrades) >= $MinReplayTrades."
-        exit 0
-    }
-    if ($summary.CurrentReplayTrades -ge $MinReplayTrades) {
-        Write-LoopLog "Stop condition reached: current artifact full_replay.trades=$($summary.CurrentReplayTrades) >= $MinReplayTrades."
+    if ($summary.Accepted -and (Read-ActiveValidationPassed)) {
+        Write-LoopLog "Stop condition reached: accepted candidate passed honest validation with full_replay.trades=$($summary.CandidateReplayTrades)."
         exit 0
     }