feat: add orderbook shadow training pipeline

tools/train_torch_recurrent_forecaster.py

@@ -28,6 +28,7 @@ from crypto_spot_bot.bybit import BybitClient
 from crypto_spot_bot.config import load_settings
 from crypto_spot_bot.indicators import add_indicators
 from crypto_spot_bot.models import Candle
+from crypto_spot_bot.orderbook_features import ORDERBOOK_FEATURES, load_orderbook_feature_map
 from crypto_spot_bot.time_series import (
     DEFAULT_TORCH_FEATURES,
     _barrier_outcome,
@@ -41,6 +42,8 @@ EVENT_OUTPUT_NAME = "logit_tp_first"
 OUTPUT_LAYOUT = (*RETURN_OUTPUT_LAYOUT, EVENT_OUTPUT_NAME)
 TARGET_TRANSFORM = "barrier_net_return"
 QUANTILES = {"q10": 0.10, "q50": 0.50, "q90": 0.90}
+_ORDERBOOK_FEATURES_BY_SYMBOL: dict[str, dict[int, dict[str, float]]] = {}
+_ORDERBOOK_MANIFEST: dict[str, dict[str, Any]] = {}
 
 
 @dataclass(slots=True)
@@ -161,6 +164,7 @@ class RecurrentReturnModel(nn.Module):
 
 
 def main() -> None:
+    global _ORDERBOOK_FEATURES_BY_SYMBOL, _ORDERBOOK_MANIFEST
     args = _parse_args()
     if args.threads > 0:
         torch.set_num_threads(args.threads)
@@ -175,6 +179,33 @@ def main() -> None:
     decision_horizon = args.horizon if args.horizon > 0 else max(1, settings.time_series_forecast_horizon)
     target_horizons = _horizons(args.horizons, decision_horizon)
     feature_names = _feature_names_arg(args.features)
+    if args.orderbook_db:
+        _ORDERBOOK_FEATURES_BY_SYMBOL, _ORDERBOOK_MANIFEST = load_orderbook_feature_map(
+            args.orderbook_db,
+            interval=interval,
+            symbols=symbols,
+            min_samples_per_bucket=args.orderbook_min_samples_per_bucket,
+        )
+        eligible_symbols = [
+            symbol
+            for symbol in symbols
+            if int(_ORDERBOOK_MANIFEST.get(symbol, {}).get("covered_buckets", 0) or 0)
+            >= args.orderbook_min_covered_buckets
+        ]
+        if len(eligible_symbols) < max(1, args.orderbook_min_symbols):
+            coverage = ", ".join(
+                f"{symbol}={int(_ORDERBOOK_MANIFEST.get(symbol, {}).get('covered_buckets', 0) or 0)}"
+                for symbol in symbols
+            )
+            raise SystemExit(
+                "Orderbook coverage is below the training minimum: "
+                f"need {args.orderbook_min_covered_buckets} buckets for "
+                f"{args.orderbook_min_symbols} symbols; got {coverage or 'no data'}"
+            )
+        symbols = eligible_symbols
+        for feature_name in ORDERBOOK_FEATURES:
+            if feature_name not in feature_names:
+                feature_names.append(feature_name)
     if args.pooled:
         feature_names.extend(f"symbol_is_{symbol}" for symbol in symbols)
     ensemble_seeds = _ints(args.ensemble_seeds) or [args.seed]
@@ -214,6 +245,15 @@ def main() -> None:
         "selection_folds": args.selection_folds,
         "symbols": {},
     }
+    if args.orderbook_db:
+        artifact["orderbook_features"] = {
+            "source": "forward_collected_bybit_l1",
+            "interval": interval,
+            "min_samples_per_bucket": args.orderbook_min_samples_per_bucket,
+            "min_covered_buckets": args.orderbook_min_covered_buckets,
+            "features": list(ORDERBOOK_FEATURES),
+            "coverage": {symbol: _ORDERBOOK_MANIFEST.get(symbol, {}) for symbol in symbols},
+        }
 
     if args.pooled:
         artifact["version"] = 7
@@ -584,6 +624,10 @@ def _parse_args() -> argparse.Namespace:
     parser.add_argument("--threads", type=int, default=0, help="Torch CPU threads; 0 keeps torch default.")
     parser.add_argument("--device", default="auto", help="auto, cpu, cuda, or mps.")
     parser.add_argument("--output", default="", help="Output JSON path. Defaults to TIME_SERIES_LSTM_MODEL_PATH.")
+    parser.add_argument("--orderbook-db", default="", help="SQLite cache containing forward-collected L1 observations.")
+    parser.add_argument("--orderbook-min-samples-per-bucket", type=int, default=20)
+    parser.add_argument("--orderbook-min-covered-buckets", type=int, default=240)
+    parser.add_argument("--orderbook-min-symbols", type=int, default=2)
     return parser.parse_args()
 
 
@@ -802,6 +846,7 @@ def _prepare_data(
         symbol=symbol,
         market_candles=market_candles,
         trend_candles=trend_candles,
+        orderbook_features=_ORDERBOOK_FEATURES_BY_SYMBOL,
     )
     max_horizon = max(target_horizons)
     samples: list[TrainingSample] = []
@@ -812,6 +857,11 @@ def _prepare_data(
         window = feature_rows[end_index - lookback + 1 : end_index + 1]
         if len(window) != lookback:
             continue
+        if any(name in ORDERBOOK_FEATURES for name in feature_names):
+            symbol_orderbook = _ORDERBOOK_FEATURES_BY_SYMBOL.get(symbol.upper(), {})
+            window_candles = candles[end_index - lookback + 1 : end_index + 1]
+            if any(row.timestamp not in symbol_orderbook for row in window_candles):
+                continue
         raw_targets: list[float] = []
         event_targets: list[float] = []
         volatility_scales: list[float] = []
@@ -856,6 +906,8 @@ def _prepare_data(
     validation_window = min(max(16, validation_window), max(16, validation_end // 3))
     validation_start = validation_end - validation_window
     train_end = validation_start - max_horizon
+    if validation_start < 0 or train_end <= 0:
+        return None
     train_samples = samples[:train_end]
     validation_samples = samples[validation_start:validation_end]
     holdout_samples = samples[holdout_start:]