Add analytics risk guard and redesigned dashboard

tools/calibrate_torch_thresholds.py

@@ -48,6 +48,8 @@ class ForecastRecord:
     symbol: str
     index: int
     timestamp: int
+    close: float
+    atr: float
     expected_percent: float
     probability_up: float
     confidence: float
@@ -139,6 +141,20 @@ def main() -> None:
     recommended = _choose_recommendation(results, min_trades=args.min_trades)
     print("\nRECOMMENDED")
     print(_result_line(recommended))
+    full_backtest = _full_backtest(records, recommended, horizon=horizon, round_trip_cost=round_trip_cost, settings=settings)
+    print("\nFULL_REPLAY")
+    print(_stats_line(full_backtest))
+    walk_forward = _walk_forward(
+        records,
+        edges=_float_grid(args.edge_grid),
+        probabilities=_float_grid(args.probability_grid),
+        confidences=_float_grid(args.confidence_grid),
+        min_trades=args.min_trades,
+        horizon=horizon,
+        folds=args.walk_forward_folds,
+    )
+    print("\nWALK_FORWARD")
+    print(json.dumps(walk_forward["summary"], ensure_ascii=False, sort_keys=True))
     print(
         "env "
         f"TIME_SERIES_MIN_EDGE_PERCENT={recommended.edge:.4f} "
@@ -151,6 +167,9 @@ def main() -> None:
             "artifact": _artifact_summary(artifact),
             "records_by_symbol": per_symbol_counts,
             "recommended": _result_dict(recommended),
+            "full_replay": full_backtest,
+            "walk_forward": walk_forward,
+            "probability_calibration": _probability_calibration(records),
             "top_results": [_result_dict(result) for result in results[: args.top]],
         }
         Path(args.output).write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
@@ -174,6 +193,7 @@ def _parse_args() -> argparse.Namespace:
     parser.add_argument("--output", default="", help="Optional JSON output path.")
     parser.add_argument("--batch-size", type=int, default=256, help="Torch inference batch size.")
     parser.add_argument("--threads", type=int, default=0, help="Torch CPU threads; 0 keeps torch default.")
+    parser.add_argument("--walk-forward-folds", type=int, default=4, help="Threshold walk-forward folds.")
     return parser.parse_args()
 
 
@@ -258,6 +278,8 @@ def _forecast_records(
                 symbol=symbol,
                 index=index,
                 timestamp=candles[index].timestamp,
+                close=closes[index],
+                atr=float(candles[index].atr_14 or 0.0),
                 expected_percent=expected_percent,
                 probability_up=probability_up,
                 confidence=_forecast_confidence(expected_percent, probability_up, skill, 0.04),
@@ -349,6 +371,8 @@ def _batch_forecast_records(
                         symbol=symbol,
                         index=index,
                         timestamp=candles[index].timestamp,
+                        close=closes[index],
+                        atr=float(candles[index].atr_14 or 0.0),
                         expected_percent=expected_percent,
                         probability_up=probability_up,
                         confidence=_forecast_confidence(expected_percent, probability_up, skill, 0.04),
@@ -484,6 +508,212 @@ def _feature_vector(entry: dict[str, Any], key: str, size: int, default: float)
     return [default for _ in range(size)]
 
 
+def _full_backtest(
+    records: list[ForecastRecord],
+    thresholds: CalibrationResult,
+    *,
+    horizon: int,
+    round_trip_cost: float,
+    settings: Any,
+) -> dict[str, Any]:
+    positions: dict[str, dict[str, Any]] = {}
+    trades: list[float] = []
+    rows: list[dict[str, Any]] = []
+    max_hold = max(12, horizon * 8)
+    stop_loss_percent = max(0.003, min(0.08, float(settings.stop_loss_percent))) * 100.0
+    atr_multiplier = max(0.5, min(10.0, float(settings.atr_trailing_multiplier)))
+    for record in sorted(records, key=lambda item: (item.timestamp, item.symbol)):
+        position = positions.get(record.symbol)
+        if position is not None:
+            position["highest"] = max(position["highest"], record.close)
+            net_percent = _net_percent(position["entry_price"], record.close, round_trip_cost)
+            held = record.index - int(position["entry_index"])
+            atr_stop = (
+                record.close <= position["highest"] - record.atr * atr_multiplier
+                if record.atr > 0 and position["highest"] > position["entry_price"]
+                else False
+            )
+            weak_forecast = (
+                record.expected_percent < thresholds.edge
+                or record.probability_up < thresholds.probability
+                or record.skill <= 0.0
+            )
+            exit_reason = ""
+            if net_percent <= -stop_loss_percent:
+                exit_reason = "stop_loss"
+            elif atr_stop:
+                exit_reason = "atr_trailing_stop"
+            elif (record.expected_percent <= 0.0 or record.probability_up <= 0.50 or _candidate_blocks(record, thresholds.edge)):
+                exit_reason = "forecast_negative"
+            elif weak_forecast and net_percent >= 0:
+                exit_reason = "forecast_weak_profit_lock"
+            elif held >= max_hold:
+                exit_reason = "max_hold"
+            if exit_reason:
+                trades.append(net_percent)
+                rows.append(
+                    {
+                        "symbol": record.symbol,
+                        "entry_timestamp": position["timestamp"],
+                        "exit_timestamp": record.timestamp,
+                        "net_percent": round(net_percent, 4),
+                        "reason": exit_reason,
+                        "held_bars": held,
+                        "entry_probability": round(float(position["probability_up"]), 4),
+                        "entry_expected_percent": round(float(position["expected_percent"]), 4),
+                    }
+                )
+                positions.pop(record.symbol, None)
+                continue
+
+        if record.symbol in positions:
+            continue
+        if _candidate_allows(record, thresholds.edge, thresholds.probability, thresholds.confidence):
+            positions[record.symbol] = {
+                "entry_price": record.close,
+                "entry_index": record.index,
+                "timestamp": record.timestamp,
+                "highest": record.close,
+                "probability_up": record.probability_up,
+                "expected_percent": record.expected_percent,
+            }
+    for symbol, position in list(positions.items()):
+        tail = next((record for record in reversed(records) if record.symbol == symbol), None)
+        if tail is None:
+            continue
+        net_percent = _net_percent(position["entry_price"], tail.close, round_trip_cost)
+        trades.append(net_percent)
+        rows.append(
+            {
+                "symbol": symbol,
+                "entry_timestamp": position["timestamp"],
+                "exit_timestamp": tail.timestamp,
+                "net_percent": round(net_percent, 4),
+                "reason": "end_of_replay",
+                "held_bars": tail.index - int(position["entry_index"]),
+                "entry_probability": round(float(position["probability_up"]), 4),
+                "entry_expected_percent": round(float(position["expected_percent"]), 4),
+            }
+        )
+    return {**_stats(trades), "trades_detail": rows[-50:]}
+
+
+def _walk_forward(
+    records: list[ForecastRecord],
+    *,
+    edges: list[float],
+    probabilities: list[float],
+    confidences: list[float],
+    min_trades: int,
+    horizon: int,
+    folds: int,
+) -> dict[str, Any]:
+    ordered = sorted(records, key=lambda item: item.timestamp)
+    if folds < 2 or len(ordered) < folds * 20:
+        return {"summary": {"status": "insufficient"}, "folds": []}
+    timestamps = sorted({record.timestamp for record in ordered})
+    fold_size = max(1, len(timestamps) // folds)
+    rows = []
+    all_test_trades: list[float] = []
+    for fold in range(1, folds):
+        test_start = timestamps[fold * fold_size]
+        test_end = timestamps[(fold + 1) * fold_size - 1] if fold < folds - 1 else timestamps[-1]
+        train = [record for record in ordered if record.timestamp < test_start]
+        test = [record for record in ordered if test_start <= record.timestamp <= test_end]
+        train_results = _calibrate(
+            train,
+            edges=edges,
+            probabilities=probabilities,
+            confidences=confidences,
+            min_trades=max(4, min_trades // 2),
+            horizon=horizon,
+        )
+        if not train_results:
+            continue
+        selected = _choose_recommendation(train_results, min_trades=max(4, min_trades // 2))
+        test_trades = _selected_trades(test, selected.edge, selected.probability, selected.confidence, horizon)
+        all_test_trades.extend(test_trades)
+        rows.append(
+            {
+                "fold": fold,
+                "train_records": len(train),
+                "test_records": len(test),
+                "thresholds": _result_dict(selected),
+                "test": _stats(test_trades),
+            }
+        )
+    summary = _stats(all_test_trades)
+    summary["status"] = "ok" if summary["trades"] >= min_trades and summary["avg_net_percent"] > 0 else "warn"
+    return {"summary": summary, "folds": rows}
+
+
+def _probability_calibration(records: list[ForecastRecord]) -> dict[str, Any]:
+    buckets: dict[str, list[ForecastRecord]] = {}
+    for record in records:
+        low = max(0.0, min(0.95, int(record.probability_up * 20) / 20))
+        key = f"{low:.2f}-{low + 0.05:.2f}"
+        buckets.setdefault(key, []).append(record)
+    rows = []
+    for key in sorted(buckets):
+        items = buckets[key]
+        wins = sum(1 for item in items if item.future_net_percent > 0)
+        avg_probability = sum(item.probability_up for item in items) / len(items)
+        rows.append(
+            {
+                "bucket": key,
+                "samples": len(items),
+                "avg_probability": round(avg_probability, 4),
+                "actual_win_rate": round(wins / len(items), 4),
+                "avg_future_net_percent": round(sum(item.future_net_percent for item in items) / len(items), 4),
+            }
+        )
+    return {"samples": len(records), "buckets": rows}
+
+
+def _candidate_blocks(record: ForecastRecord, edge: float) -> bool:
+    return (
+        record.expected_percent <= -edge
+        and record.probability_up <= 0.45
+    ) or (
+        record.q50_percent <= -edge
+        and record.probability_up <= 0.48
+    )
+
+
+def _candidate_allows(record: ForecastRecord, edge: float, probability: float, confidence: float) -> bool:
+    dynamic_confidence = _forecast_confidence(record.expected_percent, record.probability_up, record.skill, edge)
+    return (
+        not _candidate_blocks(record, edge)
+        and record.expected_percent >= edge
+        and record.probability_up >= probability
+        and dynamic_confidence >= confidence
+        and record.skill > 0.0
+    )
+
+
+def _net_percent(entry_price: float, exit_price: float, round_trip_cost: float) -> float:
+    if entry_price <= 0 or exit_price <= 0:
+        return 0.0
+    return (math.exp(math.log(exit_price / entry_price) - round_trip_cost) - 1.0) * 100.0
+
+
+def _stats(values: list[float]) -> dict[str, Any]:
+    wins = sum(1 for value in values if value > 0)
+    total = sum(values)
+    gross_profit = sum(value for value in values if value > 0)
+    gross_loss = abs(sum(value for value in values if value < 0))
+    profit_factor = gross_profit / gross_loss if gross_loss > 0 else (999.0 if gross_profit > 0 else 0.0)
+    return {
+        "trades": len(values),
+        "wins": wins,
+        "win_rate": round(wins / len(values), 4) if values else 0.0,
+        "total_net_percent": round(total, 4),
+        "avg_net_percent": round(total / len(values), 4) if values else 0.0,
+        "max_drawdown_percent": round(_max_drawdown(values), 4),
+        "profit_factor": round(profit_factor, 4),
+    }
+
+
 def _calibrate(
     records: list[ForecastRecord],
     *,
@@ -551,26 +781,7 @@ def _selected_trades(
     for record in sorted(records, key=lambda item: (item.timestamp, item.symbol)):
         if record.index < next_allowed_by_symbol.get(record.symbol, -1):
             continue
-        dynamic_confidence = _forecast_confidence(
-            record.expected_percent,
-            record.probability_up,
-            record.skill,
-            edge,
-        )
-        block_entry = (
-            record.expected_percent <= -edge
-            and record.probability_up <= 0.45
-        ) or (
-            record.q50_percent <= -edge
-            and record.probability_up <= 0.48
-        )
-        if (
-            not block_entry
-            and record.expected_percent >= edge
-            and record.probability_up >= probability
-            and dynamic_confidence >= confidence
-            and record.skill > 0.0
-        ):
+        if _candidate_allows(record, edge, probability, confidence):
             trades.append(record.future_net_percent)
             next_allowed_by_symbol[record.symbol] = record.index + max(1, horizon)
     return trades
@@ -648,6 +859,14 @@ def _result_line(result: CalibrationResult) -> str:
     )
 
 
+def _stats_line(stats: dict[str, Any]) -> str:
+    return (
+        f"trades={stats.get('trades', 0)} win={stats.get('win_rate', 0):.3f} "
+        f"avg={stats.get('avg_net_percent', 0):.4f}% total={stats.get('total_net_percent', 0):.4f}% "
+        f"dd={stats.get('max_drawdown_percent', 0):.4f}% pf={stats.get('profit_factor', 0):.3f}"
+    )
+
+
 def _result_dict(result: CalibrationResult) -> dict[str, Any]:
     return {
         "edge": result.edge,