feat: add orderbook shadow training pipeline

tools/calibrate_torch_thresholds.py

@@ -26,6 +26,7 @@ from crypto_spot_bot.bybit import BybitClient
 from crypto_spot_bot.config import load_settings
 from crypto_spot_bot.indicators import add_indicators
 from crypto_spot_bot.models import Candle
+from crypto_spot_bot.orderbook_features import ORDERBOOK_FEATURES, load_orderbook_feature_map
 from crypto_spot_bot.time_series import (
     DEFAULT_TORCH_FEATURES,
     _barrier_outcome,
@@ -97,6 +98,14 @@ def main() -> None:
     context_symbols = sorted(set(symbols + _symbols(args.context_symbols, ())))
     horizon = args.horizon if args.horizon > 0 else settings.time_series_forecast_horizon
     round_trip_cost = _artifact_round_trip_cost(artifact, settings)
+    orderbook_features: dict[str, dict[int, dict[str, float]]] = {}
+    if args.orderbook_db:
+        orderbook_features, _orderbook_manifest = load_orderbook_feature_map(
+            args.orderbook_db,
+            interval=settings.base_interval,
+            symbols=symbols,
+            min_samples_per_bucket=args.orderbook_min_samples_per_bucket,
+        )
 
     market_candles: dict[str, list[Candle]] = {}
     for symbol in context_symbols:
@@ -125,6 +134,7 @@ def main() -> None:
             min_candles=max(30, settings.time_series_min_candles),
             calibration_window=args.calibration_window,
             batch_size=args.batch_size,
+            orderbook_features=orderbook_features,
         )
         records.extend(symbol_records)
         per_symbol_counts[symbol] = len(symbol_records)
@@ -304,6 +314,8 @@ def _parse_args() -> argparse.Namespace:
     parser.add_argument("--min-oos-folds-with-trades", type=int, default=2, help="Minimum walk-forward folds that must produce trades.")
     parser.add_argument("--min-oos-profit-factor", type=float, default=1.10, help="Minimum out-of-sample profit factor.")
     parser.add_argument("--min-benchmark-edge-percent", type=float, default=0.0, help="Required total-net percent advantage over the benchmark.")
+    parser.add_argument("--orderbook-db", default="", help="SQLite cache used by an artifact with L1 features.")
+    parser.add_argument("--orderbook-min-samples-per-bucket", type=int, default=20)
     return parser.parse_args()
 
 
@@ -352,6 +364,7 @@ def _forecast_records(
     min_candles: int,
     calibration_window: int,
     batch_size: int,
+    orderbook_features: dict[str, dict[int, dict[str, float]]] | None = None,
 ) -> list[ForecastRecord]:
     entry = _torch_recurrent_entry(symbol, artifact)
     model = _torch_recurrent_model_name(symbol, artifact)
@@ -364,6 +377,7 @@ def _forecast_records(
         symbol=symbol,
         market_candles=market_candles,
         trend_candles=trend_candles,
+        orderbook_features=orderbook_features,
     )
     closes = [float(candle.close) for candle in candles]
     decision_horizon = _calibration_horizon(entry, horizon, explicit=horizon_is_explicit)
@@ -376,6 +390,18 @@ def _forecast_records(
         start += 1
     if calibration_window > 0:
         start = max(start, end - calibration_window)
+    lookback = max(1, int(float(entry.get("lookback", 64))))
+    requires_orderbook = any(name in ORDERBOOK_FEATURES for name in feature_names)
+    symbol_orderbook = (orderbook_features or {}).get(symbol.upper(), {})
+    valid_indices = {
+        index
+        for index in range(start, max(start, end))
+        if not requires_orderbook
+        or all(
+            candles[position].timestamp in symbol_orderbook
+            for position in range(index - lookback + 1, index + 1)
+        )
+    }
     batched_records = _batch_forecast_records(
         symbol=symbol,
         candles=candles,
@@ -389,6 +415,7 @@ def _forecast_records(
         start=start,
         end=end,
         batch_size=batch_size,
+        valid_indices=valid_indices,
     )
     if batched_records is not None:
         return batched_records
@@ -398,6 +425,8 @@ def _forecast_records(
     # belong exclusively to the final quality gate and cannot influence replay.
     skill = _entry_validation_skill(entry)
     for index in range(start, max(start, end)):
+        if index not in valid_indices:
+            continue
         prediction = _torch_recurrent_predict(
             _log_returns(closes[: index + 1]),
             symbol,
@@ -476,6 +505,7 @@ def _batch_forecast_records(
     start: int,
     end: int,
     batch_size: int,
+    valid_indices: set[int] | None = None,
 ) -> list[ForecastRecord] | None:
     if torch is None or RecurrentReturnModel is None:
         return None
@@ -494,7 +524,9 @@ def _batch_forecast_records(
     indices = [
         index
         for index in range(start, max(start, end))
-        if index - lookback + 1 >= 0 and index + decision_horizon < len(closes)
+        if index - lookback + 1 >= 0
+        and index + decision_horizon < len(closes)
+        and (valid_indices is None or index in valid_indices)
     ]
     if not indices:
         return []

tools/run_torch_retrain.ps1

@@ -22,6 +22,10 @@ param(
     [int]$HoldoutWindow = 0,
     [string]$Interval = "",
     [string]$EnvFile = "",
+    [string]$OrderbookDb = "",
+    [int]$OrderbookMinSamplesPerBucket = 0,
+    [int]$OrderbookMinCoveredBuckets = 0,
+    [int]$OrderbookMinSymbols = 0,
     [switch]$Pooled,
     [switch]$SkipGuard,
     [switch]$ResumeCandidate
@@ -124,6 +128,10 @@ if ($HoldoutWindow -le 0) { $HoldoutWindow = if ($env:TORCH_RETRAIN_HOLDOUT_WIND
 if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RETRAIN_INTERVAL }
 if (-not $EnvFile -and $env:TORCH_RETRAIN_ENV) { $EnvFile = $env:TORCH_RETRAIN_ENV }
 if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" }
+if (-not $OrderbookDb -and $env:TORCH_ORDERBOOK_DB) { $OrderbookDb = $env:TORCH_ORDERBOOK_DB }
+if ($OrderbookMinSamplesPerBucket -le 0) { $OrderbookMinSamplesPerBucket = if ($env:TORCH_ORDERBOOK_MIN_SAMPLES_PER_BUCKET) { [int]$env:TORCH_ORDERBOOK_MIN_SAMPLES_PER_BUCKET } else { 20 } }
+if ($OrderbookMinCoveredBuckets -le 0) { $OrderbookMinCoveredBuckets = if ($env:TORCH_ORDERBOOK_MIN_COVERED_BUCKETS) { [int]$env:TORCH_ORDERBOOK_MIN_COVERED_BUCKETS } else { 240 } }
+if ($OrderbookMinSymbols -le 0) { $OrderbookMinSymbols = if ($env:TORCH_ORDERBOOK_MIN_SYMBOLS) { [int]$env:TORCH_ORDERBOOK_MIN_SYMBOLS } else { 2 } }
 
 $ModelFile = if ($env:TIME_SERIES_LSTM_MODEL_PATH) { $env:TIME_SERIES_LSTM_MODEL_PATH } else { Join-Path $RuntimeDir "lstm_forecaster.json" }
 if (-not [System.IO.Path]::IsPathRooted($ModelFile)) { $ModelFile = Join-Path $RepoRoot $ModelFile }
@@ -131,6 +139,10 @@ $CandidateFile = Join-Path $RuntimeDir "lstm_forecaster.candidate.json"
 $CurrentCalibration = Join-Path $RuntimeDir "torch_guard_current.json"
 $CandidateCalibration = Join-Path $RuntimeDir "torch_guard_candidate.json"
 $GuardReport = Join-Path $RuntimeDir "torch_retrain_guard.json"
+$ShadowModelFile = Join-Path $RuntimeDir "lstm_forecaster.shadow.json"
+$ShadowCalibration = Join-Path $RuntimeDir "torch_shadow_calibration.json"
+$ShadowGuard = Join-Path $RuntimeDir "torch_shadow_guard.json"
+$ShadowMode = -not [string]::IsNullOrWhiteSpace($OrderbookDb)
 
 $mutex = New-Object System.Threading.Mutex($false, "TradeBotTorchRecurrentRetrainer")
 $hasLock = $false
@@ -177,6 +189,14 @@ try {
     if ($Features) { $trainerArgs += @("--features", $Features) }
     if ($ContextSymbols) { $trainerArgs += @("--context-symbols", $ContextSymbols) }
     if ($Seed -gt 0) { $trainerArgs += @("--seed", $Seed.ToString()) }
+    if ($OrderbookDb) {
+        $trainerArgs += @(
+            "--orderbook-db", $OrderbookDb,
+            "--orderbook-min-samples-per-bucket", $OrderbookMinSamplesPerBucket.ToString(),
+            "--orderbook-min-covered-buckets", $OrderbookMinCoveredBuckets.ToString(),
+            "--orderbook-min-symbols", $OrderbookMinSymbols.ToString()
+        )
+    }
 
     Push-Location $RepoRoot
     $pushedLocation = $true
@@ -216,6 +236,12 @@ try {
     )
     if ($Symbols) { $calibrationBaseArgs += @("--symbols", $Symbols) }
     if ($EnvFile) { $calibrationBaseArgs += @("--env", $EnvFile) }
+    if ($OrderbookDb) {
+        $calibrationBaseArgs += @(
+            "--orderbook-db", $OrderbookDb,
+            "--orderbook-min-samples-per-bucket", $OrderbookMinSamplesPerBucket.ToString()
+        )
+    }
 
     if (Test-Path $ModelFile) {
         Write-RetrainLog "Calibrating current artifact for guard."
@@ -243,13 +269,14 @@ try {
     }
 
     Write-RetrainLog "Running retrain guard."
+    $GuardTarget = if ($ShadowMode) { $ShadowModelFile } else { $ModelFile }
     $guardArgs = @(
         "-u",
         "tools\accept_torch_candidate.py",
         "--current-report", $CurrentCalibration,
         "--candidate-report", $CandidateCalibration,
         "--candidate-artifact", $CandidateFile,
-        "--target-artifact", $ModelFile,
+        "--target-artifact", $GuardTarget,
         "--report", $GuardReport
     )
     $guardExitCode = Invoke-LoggedNativeCommand -FilePath $python -ArgumentList $guardArgs -LogPath $LogFile
@@ -261,10 +288,22 @@ try {
         throw "Retrain guard failed with exit code $guardExitCode."
     }
     if (Test-Path $CandidateCalibration) {
-        Copy-Item -Force -LiteralPath $CandidateCalibration -Destination (Join-Path $RuntimeDir "torch_threshold_calibration.json")
-        Write-RetrainLog "Updated active threshold calibration: $(Join-Path $RuntimeDir "torch_threshold_calibration.json")"
+        if ($ShadowMode) {
+            Copy-Item -Force -LiteralPath $CandidateCalibration -Destination $ShadowCalibration
+            Copy-Item -Force -LiteralPath $GuardReport -Destination $ShadowGuard
+            Write-RetrainLog "Candidate passed offline gate and was staged for shadow only: $ShadowModelFile"
+        }
+        else {
+            Copy-Item -Force -LiteralPath $CandidateCalibration -Destination (Join-Path $RuntimeDir "torch_threshold_calibration.json")
+            Write-RetrainLog "Updated active threshold calibration: $(Join-Path $RuntimeDir "torch_threshold_calibration.json")"
+        }
+    }
+    if ($ShadowMode) {
+        Write-RetrainLog "Candidate accepted by offline guard. Active artifact was not changed: $ModelFile"
+    }
+    else {
+        Write-RetrainLog "Candidate accepted by guard. Active artifact: $ModelFile"
     }
-    Write-RetrainLog "Candidate accepted by guard. Active artifact: $ModelFile"
 }
 catch {
     Write-RetrainLog "ERROR: $($_.Exception.Message)"

tools/sync_orderbook_observations.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import os
+import sqlite3
+from pathlib import Path
+from typing import Any
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+
+def sync_orderbook_observations(
+    *,
+    api_base_url: str,
+    token: str,
+    database_path: str | Path,
+    timeout: int = 60,
+    page_limit: int = 5000,
+) -> dict[str, Any]:
+    path = Path(database_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    _init_schema(path)
+    manifest = _get_json(
+        api_base_url,
+        "/api/training/market-observations/manifest",
+        token=token,
+        timeout=timeout,
+    )
+    rows = manifest.get("items") if isinstance(manifest.get("items"), list) else []
+    downloaded = 0
+    symbol_results: list[dict[str, Any]] = []
+    for row in rows:
+        if not isinstance(row, dict):
+            continue
+        symbol = str(row.get("symbol") or "").strip().upper()
+        remote_max_id = int(row.get("max_id", 0) or 0)
+        if not symbol or remote_max_id <= 0:
+            continue
+        after_id = _local_max_id(path, symbol)
+        symbol_downloaded = 0
+        while after_id < remote_max_id:
+            query = urlencode(
+                {
+                    "symbol": symbol,
+                    "after_id": after_id,
+                    "limit": max(1, min(5000, int(page_limit))),
+                }
+            )
+            payload = _get_json(
+                api_base_url,
+                f"/api/training/market-observations?{query}",
+                token=token,
+                timeout=timeout,
+            )
+            items = payload.get("items") if isinstance(payload.get("items"), list) else []
+            if not items:
+                break
+            inserted = _insert_rows(path, items)
+            symbol_downloaded += inserted
+            downloaded += inserted
+            next_after_id = int(payload.get("next_after_id", after_id) or after_id)
+            if next_after_id <= after_id:
+                break
+            after_id = next_after_id
+        symbol_results.append(
+            {
+                "symbol": symbol,
+                "downloaded": symbol_downloaded,
+                "local_max_id": _local_max_id(path, symbol),
+                "remote_max_id": remote_max_id,
+            }
+        )
+    return {
+        "database_path": str(path.resolve()),
+        "downloaded": downloaded,
+        "symbols": symbol_results,
+        "local_samples": _local_count(path),
+    }
+
+
+def _init_schema(path: Path) -> None:
+    with sqlite3.connect(path) as connection:
+        connection.executescript(
+            """
+            PRAGMA journal_mode=WAL;
+            CREATE TABLE IF NOT EXISTS market_observations (
+                id INTEGER PRIMARY KEY,
+                symbol TEXT NOT NULL,
+                bid_price REAL NOT NULL,
+                bid_size REAL NOT NULL,
+                ask_price REAL NOT NULL,
+                ask_size REAL NOT NULL,
+                mid_price REAL NOT NULL,
+                microprice REAL NOT NULL,
+                spread_bps REAL NOT NULL,
+                imbalance REAL NOT NULL,
+                last_price REAL NOT NULL,
+                source_timestamp_ms INTEGER NOT NULL DEFAULT 0,
+                created_at TEXT NOT NULL
+            );
+            CREATE INDEX IF NOT EXISTS idx_local_market_observations_symbol_id
+                ON market_observations(symbol, id);
+            """
+        )
+
+
+def _insert_rows(path: Path, rows: list[Any]) -> int:
+    values = []
+    for row in rows:
+        if not isinstance(row, dict):
+            continue
+        values.append(
+            (
+                int(row.get("id", 0) or 0),
+                str(row.get("symbol") or "").upper(),
+                float(row.get("bid_price", 0.0) or 0.0),
+                float(row.get("bid_size", 0.0) or 0.0),
+                float(row.get("ask_price", 0.0) or 0.0),
+                float(row.get("ask_size", 0.0) or 0.0),
+                float(row.get("mid_price", 0.0) or 0.0),
+                float(row.get("microprice", 0.0) or 0.0),
+                float(row.get("spread_bps", 0.0) or 0.0),
+                float(row.get("imbalance", 0.0) or 0.0),
+                float(row.get("last_price", 0.0) or 0.0),
+                int(row.get("source_timestamp_ms", 0) or 0),
+                str(row.get("created_at") or ""),
+            )
+        )
+    if not values:
+        return 0
+    with sqlite3.connect(path) as connection:
+        before = connection.total_changes
+        connection.executemany(
+            """
+            INSERT OR IGNORE INTO market_observations (
+                id, symbol, bid_price, bid_size, ask_price, ask_size,
+                mid_price, microprice, spread_bps, imbalance, last_price,
+                source_timestamp_ms, created_at
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            values,
+        )
+        return connection.total_changes - before
+
+
+def _local_max_id(path: Path, symbol: str) -> int:
+    with sqlite3.connect(path) as connection:
+        row = connection.execute(
+            "SELECT MAX(id) FROM market_observations WHERE symbol = ?",
+            (symbol,),
+        ).fetchone()
+    return int(row[0] or 0) if row else 0
+
+
+def _local_count(path: Path) -> int:
+    with sqlite3.connect(path) as connection:
+        row = connection.execute("SELECT COUNT(*) FROM market_observations").fetchone()
+    return int(row[0] or 0) if row else 0
+
+
+def _get_json(api_base_url: str, path: str, *, token: str, timeout: int) -> dict[str, Any]:
+    headers = {"Accept": "application/json"}
+    headers.update(_auth_headers(token))
+    request = Request(api_base_url.rstrip("/") + path, headers=headers, method="GET")
+    try:
+        with urlopen(request, timeout=timeout) as response:
+            text = response.read().decode("utf-8")
+    except HTTPError as exc:
+        detail = exc.read().decode("utf-8", errors="replace")
+        raise RuntimeError(f"HTTP {exc.code} {path}: {detail[:300]}") from exc
+    except URLError as exc:
+        raise RuntimeError(f"network error {path}: {exc.reason}") from exc
+    data = json.loads(text) if text.strip() else {}
+    return data if isinstance(data, dict) else {}
+
+
+def _auth_headers(token: str) -> dict[str, str]:
+    value = token.strip()
+    if not value:
+        return {}
+    headers = {"X-TradeBot-Token": value}
+    if value.lower().startswith(("basic ", "bearer ")):
+        headers["Authorization"] = value
+    elif ":" in value:
+        encoded = base64.b64encode(value.encode("utf-8")).decode("ascii")
+        headers["Authorization"] = f"Basic {encoded}"
+    else:
+        headers["Authorization"] = f"Bearer {value}"
+    return headers
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Synchronize TradeBot L1 observations to a local SQLite cache.")
+    parser.add_argument("--api-base-url", default=os.environ.get("TRADEBOT_API_BASE_URL", "https://tb.kusoft.xyz"))
+    parser.add_argument("--api-auth", default=os.environ.get("TRADEBOT_API_AUTH", ""))
+    parser.add_argument("--database", default="runtime/orderbook_observations.sqlite3")
+    args = parser.parse_args()
+    result = sync_orderbook_observations(
+        api_base_url=args.api_base_url,
+        token=args.api_auth,
+        database_path=args.database,
+    )
+    print(json.dumps(result, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

tools/train_torch_recurrent_forecaster.py

@@ -28,6 +28,7 @@ from crypto_spot_bot.bybit import BybitClient
 from crypto_spot_bot.config import load_settings
 from crypto_spot_bot.indicators import add_indicators
 from crypto_spot_bot.models import Candle
+from crypto_spot_bot.orderbook_features import ORDERBOOK_FEATURES, load_orderbook_feature_map
 from crypto_spot_bot.time_series import (
     DEFAULT_TORCH_FEATURES,
     _barrier_outcome,
@@ -41,6 +42,8 @@ EVENT_OUTPUT_NAME = "logit_tp_first"
 OUTPUT_LAYOUT = (*RETURN_OUTPUT_LAYOUT, EVENT_OUTPUT_NAME)
 TARGET_TRANSFORM = "barrier_net_return"
 QUANTILES = {"q10": 0.10, "q50": 0.50, "q90": 0.90}
+_ORDERBOOK_FEATURES_BY_SYMBOL: dict[str, dict[int, dict[str, float]]] = {}
+_ORDERBOOK_MANIFEST: dict[str, dict[str, Any]] = {}
 
 
 @dataclass(slots=True)
@@ -161,6 +164,7 @@ class RecurrentReturnModel(nn.Module):
 
 
 def main() -> None:
+    global _ORDERBOOK_FEATURES_BY_SYMBOL, _ORDERBOOK_MANIFEST
     args = _parse_args()
     if args.threads > 0:
         torch.set_num_threads(args.threads)
@@ -175,6 +179,33 @@ def main() -> None:
     decision_horizon = args.horizon if args.horizon > 0 else max(1, settings.time_series_forecast_horizon)
     target_horizons = _horizons(args.horizons, decision_horizon)
     feature_names = _feature_names_arg(args.features)
+    if args.orderbook_db:
+        _ORDERBOOK_FEATURES_BY_SYMBOL, _ORDERBOOK_MANIFEST = load_orderbook_feature_map(
+            args.orderbook_db,
+            interval=interval,
+            symbols=symbols,
+            min_samples_per_bucket=args.orderbook_min_samples_per_bucket,
+        )
+        eligible_symbols = [
+            symbol
+            for symbol in symbols
+            if int(_ORDERBOOK_MANIFEST.get(symbol, {}).get("covered_buckets", 0) or 0)
+            >= args.orderbook_min_covered_buckets
+        ]
+        if len(eligible_symbols) < max(1, args.orderbook_min_symbols):
+            coverage = ", ".join(
+                f"{symbol}={int(_ORDERBOOK_MANIFEST.get(symbol, {}).get('covered_buckets', 0) or 0)}"
+                for symbol in symbols
+            )
+            raise SystemExit(
+                "Orderbook coverage is below the training minimum: "
+                f"need {args.orderbook_min_covered_buckets} buckets for "
+                f"{args.orderbook_min_symbols} symbols; got {coverage or 'no data'}"
+            )
+        symbols = eligible_symbols
+        for feature_name in ORDERBOOK_FEATURES:
+            if feature_name not in feature_names:
+                feature_names.append(feature_name)
     if args.pooled:
         feature_names.extend(f"symbol_is_{symbol}" for symbol in symbols)
     ensemble_seeds = _ints(args.ensemble_seeds) or [args.seed]
@@ -214,6 +245,15 @@ def main() -> None:
         "selection_folds": args.selection_folds,
         "symbols": {},
     }
+    if args.orderbook_db:
+        artifact["orderbook_features"] = {
+            "source": "forward_collected_bybit_l1",
+            "interval": interval,
+            "min_samples_per_bucket": args.orderbook_min_samples_per_bucket,
+            "min_covered_buckets": args.orderbook_min_covered_buckets,
+            "features": list(ORDERBOOK_FEATURES),
+            "coverage": {symbol: _ORDERBOOK_MANIFEST.get(symbol, {}) for symbol in symbols},
+        }
 
     if args.pooled:
         artifact["version"] = 7
@@ -584,6 +624,10 @@ def _parse_args() -> argparse.Namespace:
     parser.add_argument("--threads", type=int, default=0, help="Torch CPU threads; 0 keeps torch default.")
     parser.add_argument("--device", default="auto", help="auto, cpu, cuda, or mps.")
     parser.add_argument("--output", default="", help="Output JSON path. Defaults to TIME_SERIES_LSTM_MODEL_PATH.")
+    parser.add_argument("--orderbook-db", default="", help="SQLite cache containing forward-collected L1 observations.")
+    parser.add_argument("--orderbook-min-samples-per-bucket", type=int, default=20)
+    parser.add_argument("--orderbook-min-covered-buckets", type=int, default=240)
+    parser.add_argument("--orderbook-min-symbols", type=int, default=2)
     return parser.parse_args()
 
 
@@ -802,6 +846,7 @@ def _prepare_data(
         symbol=symbol,
         market_candles=market_candles,
         trend_candles=trend_candles,
+        orderbook_features=_ORDERBOOK_FEATURES_BY_SYMBOL,
     )
     max_horizon = max(target_horizons)
     samples: list[TrainingSample] = []
@@ -812,6 +857,11 @@ def _prepare_data(
         window = feature_rows[end_index - lookback + 1 : end_index + 1]
         if len(window) != lookback:
             continue
+        if any(name in ORDERBOOK_FEATURES for name in feature_names):
+            symbol_orderbook = _ORDERBOOK_FEATURES_BY_SYMBOL.get(symbol.upper(), {})
+            window_candles = candles[end_index - lookback + 1 : end_index + 1]
+            if any(row.timestamp not in symbol_orderbook for row in window_candles):
+                continue
         raw_targets: list[float] = []
         event_targets: list[float] = []
         volatility_scales: list[float] = []
@@ -856,6 +906,8 @@ def _prepare_data(
     validation_window = min(max(16, validation_window), max(16, validation_end // 3))
     validation_start = validation_end - validation_window
     train_end = validation_start - max_horizon
+    if validation_start < 0 or train_end <= 0:
+        return None
     train_samples = samples[:train_end]
     validation_samples = samples[validation_start:validation_end]
     holdout_samples = samples[holdout_start:]

tools/windows_training_agent.py

@@ -20,12 +20,24 @@ from urllib.error import URLError
 from urllib.request import Request
 from urllib.request import urlopen
 
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from crypto_spot_bot.orderbook_features import load_orderbook_feature_map
+from tools.sync_orderbook_observations import sync_orderbook_observations
+
 
 ARTIFACT_NAMES = (
     "lstm_forecaster.json",
     "torch_retrain_guard.json",
     "torch_threshold_calibration.json",
 )
+SHADOW_ARTIFACT_NAMES = (
+    "lstm_forecaster.shadow.json",
+    "torch_shadow_guard.json",
+    "torch_shadow_calibration.json",
+)
 
 
 def main() -> None:
@@ -65,7 +77,35 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
     message = ""
     summary: dict[str, Any] = {}
     try:
-        run_retrain(args, job_id, lease_token, job, repo_root, log_path)
+        parameters = job.get("parameters") if isinstance(job.get("parameters"), dict) else {}
+        use_orderbook = parameters.get("use_orderbook", True) is not False
+        orderbook_status: dict[str, Any] = {}
+        if use_orderbook:
+            report_progress(
+                args,
+                job_id,
+                lease_token,
+                "running",
+                "orderbook_sync",
+                4,
+                "Синхронизирую forward-наблюдения стакана",
+            )
+            orderbook_status = prepare_orderbook_data(args, repo_root, parameters, log_path)
+            if orderbook_status["state"] != "ready":
+                summary = orderbook_status
+                message = "forward orderbook coverage is still accumulating"
+                success = True
+                log(log_path, f"Job {job_id} remains in collecting state: {orderbook_status}")
+                return
+        run_retrain(
+            args,
+            job_id,
+            lease_token,
+            job,
+            repo_root,
+            log_path,
+            orderbook_db=(repo_root / "runtime" / "orderbook_observations.sqlite3") if use_orderbook else None,
+        )
         summary = read_json(runtime_dir / "torch_retrain_guard.json")
         accepted = summary.get("accepted") is True
         if accepted:
@@ -78,12 +118,20 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
                 72,
                 "Обучение завершено, загружаю артефакты",
             )
-            for name in ARTIFACT_NAMES:
+            artifact_names = SHADOW_ARTIFACT_NAMES if use_orderbook else ARTIFACT_NAMES
+            if use_orderbook:
+                summary["deployment"] = "shadow"
+                summary["orderbook"] = orderbook_status
+            for name in artifact_names:
                 path = runtime_dir / name
                 if path.is_file():
                     upload_artifact(args, job_id, lease_token, path, log_path)
-            message = "training completed; candidate accepted"
-            log(log_path, f"Completed retrain job {job_id}; candidate accepted")
+            message = (
+                "training completed; candidate staged in shadow"
+                if use_orderbook
+                else "training completed; candidate accepted"
+            )
+            log(log_path, f"Completed retrain job {job_id}; {message}")
         else:
             reason = str(summary.get("reason") or "validation failed")
             message = f"training completed; candidate rejected by quality gate: {reason}"
@@ -109,6 +157,7 @@ def run_retrain(
     job: dict[str, Any],
     repo_root: Path,
     log_path: Path,
+    orderbook_db: Path | None = None,
 ) -> None:
     script = repo_root / "tools" / "run_torch_retrain.ps1"
     if not script.is_file():
@@ -153,6 +202,14 @@ def run_retrain(
         cmd.append("-Pooled")
     if parameters.get("resume_candidate") is True:
         cmd.append("-ResumeCandidate")
+    if orderbook_db is not None:
+        cmd.extend(["-OrderbookDb", str(orderbook_db)])
+        for key, ps_arg, default in (
+            ("orderbook_min_samples_per_bucket", "-OrderbookMinSamplesPerBucket", 20),
+            ("orderbook_min_covered_buckets", "-OrderbookMinCoveredBuckets", 240),
+            ("orderbook_min_symbols", "-OrderbookMinSymbols", 2),
+        ):
+            cmd.extend([ps_arg, str(int(parameters.get(key, default) or default))])
     log(log_path, "Running retrain: " + " ".join(quote_for_log(part) for part in cmd))
     report_progress(
         args,
@@ -236,6 +293,65 @@ def run_retrain(
     )
 
 
+def prepare_orderbook_data(
+    args: argparse.Namespace,
+    repo_root: Path,
+    parameters: dict[str, Any],
+    log_path: Path,
+) -> dict[str, Any]:
+    database_path = repo_root / "runtime" / "orderbook_observations.sqlite3"
+    token = args.api_auth or os.environ.get("TRADEBOT_API_AUTH", "")
+    sync_result = sync_orderbook_observations(
+        api_base_url=args.api_base_url,
+        token=token,
+        database_path=database_path,
+    )
+    interval = str(parameters.get("interval") or os.environ.get("TORCH_RETRAIN_INTERVAL") or "60")
+    minimum_samples = int(parameters.get("orderbook_min_samples_per_bucket", 20) or 20)
+    minimum_buckets = int(parameters.get("orderbook_min_covered_buckets", 240) or 240)
+    minimum_symbols = int(parameters.get("orderbook_min_symbols", 2) or 2)
+    requested_symbols = {
+        item.strip().upper()
+        for item in str(parameters.get("symbols") or "").split(",")
+        if item.strip()
+    }
+    _features, manifest = load_orderbook_feature_map(
+        database_path,
+        interval=interval,
+        symbols=sorted(requested_symbols) if requested_symbols else None,
+        min_samples_per_bucket=minimum_samples,
+    )
+    eligible = sorted(
+        symbol
+        for symbol, row in manifest.items()
+        if int(row.get("covered_buckets", 0) or 0) >= minimum_buckets
+    )
+    state = "ready" if len(eligible) >= minimum_symbols else "collecting_orderbook"
+    coverage = {
+        symbol: int(row.get("covered_buckets", 0) or 0)
+        for symbol, row in sorted(manifest.items())
+    }
+    result = {
+        "accepted": False,
+        "state": state,
+        "reason": (
+            "orderbook coverage ready for training"
+            if state == "ready"
+            else "forward orderbook coverage is below the configured minimum"
+        ),
+        "eligible_symbols": eligible,
+        "eligible_symbol_count": len(eligible),
+        "minimum_symbols": minimum_symbols,
+        "minimum_covered_buckets": minimum_buckets,
+        "minimum_samples_per_bucket": minimum_samples,
+        "covered_buckets_by_symbol": coverage,
+        "local_samples": int(sync_result.get("local_samples", 0) or 0),
+        "downloaded_samples": int(sync_result.get("downloaded", 0) or 0),
+    }
+    log(log_path, "Orderbook preparation: " + json.dumps(result, ensure_ascii=False, sort_keys=True))
+    return result
+
+
 def friendly_training_message(message: str) -> str:
     cleaned = message.strip()
     if not cleaned: