Harden trading, training, and monitoring

crypto_spot_bot/analytics.py

@@ -198,9 +198,12 @@ def _group_stats(trades: list[dict[str, Any]], key_fn) -> list[dict[str, Any]]:
 
 def _active_universe_trades(settings: Settings, trades: list[dict[str, Any]]) -> list[dict[str, Any]]:
     symbols = {symbol.upper() for symbol in settings.symbols}
-    if not symbols:
-        return trades
-    return [trade for trade in trades if str(trade.get("symbol", "")).upper() in symbols]
+    return [
+        trade
+        for trade in trades
+        if (not symbols or str(trade.get("symbol", "")).upper() in symbols)
+        and str(trade.get("mode", "paper")) == settings.trading_mode
+    ]
 
 
 def _symbol_guard_stats(settings: Settings, trades: list[dict[str, Any]]) -> list[dict[str, Any]]:

crypto_spot_bot/auth.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import base64
+import binascii
+import hmac
+
+from fastapi import HTTPException, Request, status
+
+from crypto_spot_bot.config import Settings
+
+
+class ApiAuthorizer:
+    """Authenticate API calls either directly or through an authenticated proxy."""
+
+    def __init__(self, settings: Settings):
+        self.settings = settings
+
+    async def require(self, request: Request) -> None:
+        if self._proxy_authenticated(request) or self._token_authenticated(
+            request, self.settings.api_auth_token
+        ):
+            return
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="API authentication required",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    async def require_training(self, request: Request) -> None:
+        expected = self.settings.training_worker_token or self.settings.api_auth_token
+        if self._proxy_authenticated(request) or self._token_authenticated(request, expected):
+            return
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="training worker authentication required",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    def configured(self) -> bool:
+        return bool(
+            self.settings.api_auth_token
+            or self.settings.training_worker_token
+            or self.settings.trusted_proxy_user_header
+        )
+
+    def _proxy_authenticated(self, request: Request) -> bool:
+        header = self.settings.trusted_proxy_user_header
+        if not header:
+            return False
+        return bool(request.headers.get(header, "").strip())
+
+    def _token_authenticated(self, request: Request, expected: str) -> bool:
+        if not expected:
+            return False
+        candidates = [request.headers.get("X-TradeBot-Token", "").strip()]
+        authorization = request.headers.get("Authorization", "").strip()
+        if authorization.lower().startswith("bearer "):
+            candidates.append(authorization[7:].strip())
+        elif authorization.lower().startswith("basic "):
+            decoded = _decode_basic(authorization[6:].strip())
+            if decoded:
+                candidates.append(decoded)
+        return any(candidate and hmac.compare_digest(candidate, expected) for candidate in candidates)
+
+
+def _decode_basic(value: str) -> str:
+    try:
+        return base64.b64decode(value, validate=True).decode("utf-8")
+    except (binascii.Error, UnicodeDecodeError, ValueError):
+        return ""

crypto_spot_bot/bot.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import asyncio
+import logging
+import sqlite3
 from datetime import datetime
 
 from crypto_spot_bot.analytics import risk_guard_snapshot
@@ -15,6 +17,9 @@ from crypto_spot_bot.storage import Storage
 from crypto_spot_bot.time_series import TimeSeriesForecaster
 
 
+logger = logging.getLogger(__name__)
+
+
 class CryptoSpotBot:
     def __init__(
         self,
@@ -44,6 +49,9 @@ class CryptoSpotBot:
         self._entry_cooldown_until: dict[str, datetime] = {}
         self._loop_task: asyncio.Task | None = None
         self._ws_task: asyncio.Task | None = None
+        self._last_reconciliation_at: datetime | None = None
+        self._last_prune_at: datetime | None = None
+        self._consecutive_loop_errors = 0
 
     async def start(self) -> None:
         if self.running:
@@ -51,6 +59,17 @@ class CryptoSpotBot:
         self.market.reset_stop()
         if not self.market.symbols:
             await self.market.bootstrap()
+        if isinstance(self.broker, LiveBroker):
+            try:
+                await asyncio.to_thread(self.broker.reconcile, self.market.instruments)
+                self._last_reconciliation_at = utc_now()
+            except Exception as exc:
+                self.broker.reconciliation_state = {
+                    "status": "error",
+                    "blocking": True,
+                    "discrepancies": [{"code": "initial_reconciliation_failed", "message": str(exc)}],
+                }
+                self.storage.event(f"Initial live reconciliation failed: {exc}", "ERROR")
         self._close_paper_positions_outside_symbol_universe()
         self._update_patterns()
         self._update_forecasts()
@@ -58,7 +77,7 @@ class CryptoSpotBot:
         self.running = True
         self.started_at = utc_now()
         self.message = "бот работает"
-        self.storage.event("Бот запущен")
+        self._safe_event("Бот запущен")
         if self.settings.websocket_enabled:
             self._ws_task = asyncio.create_task(self.market.websocket_loop())
         self._loop_task = asyncio.create_task(self._run_loop())
@@ -72,7 +91,13 @@ class CryptoSpotBot:
             task.cancel()
         if tasks:
             await asyncio.gather(*tasks, return_exceptions=True)
-        self.storage.event("Бот остановлен")
+        self._safe_event("Бот остановлен")
+
+    def _safe_event(self, message: str, level: str = "INFO") -> None:
+        try:
+            self.storage.event(message, level)
+        except sqlite3.Error:
+            logger.exception("Could not persist non-critical bot event: %s", message)
 
     async def _run_loop(self) -> None:
         while self.running:
@@ -80,6 +105,7 @@ class CryptoSpotBot:
                 rest_refresh_seconds = self._rest_refresh_seconds()
                 if self._needs_rest_refresh(rest_refresh_seconds):
                     await asyncio.to_thread(self.market.refresh_rest)
+                await self._maintain_runtime()
                 self.broker.update_highs(self.market.tickers)
                 self._update_patterns()
                 self._update_forecasts()
@@ -88,9 +114,11 @@ class CryptoSpotBot:
                 await self._process_entries()
                 self.broker.mark_equity(self.market.prices())
                 self.last_loop_at = utc_now()
+                self._consecutive_loop_errors = 0
             except asyncio.CancelledError:
                 raise
             except Exception as exc:
+                self._consecutive_loop_errors += 1
                 self.message = f"ошибка цикла: {exc}"
                 self.storage.event(self.message, "ERROR")
             await asyncio.sleep(self.settings.effective_loop_interval_seconds)
@@ -110,6 +138,18 @@ class CryptoSpotBot:
         prices = self.market.prices()
         reduction_candidate_id = self._reduction_candidate_id(prices)
         for position in list(self.broker.open_positions()):
+            freshness = self.market.symbol_freshness(position.symbol)
+            if not freshness["ok"]:
+                self._record_signal(
+                    Signal(
+                        position.symbol,
+                        "HOLD",
+                        0.0,
+                        "market data is stale; exchange protective stop remains authoritative",
+                        {"market_freshness": freshness},
+                    )
+                )
+                continue
             ticker = self.market.tickers.get(position.symbol)
             candles = self.market.candles.get(position.symbol, [])
             forecast = self.market.forecasts.get(position.symbol, {})
@@ -117,25 +157,40 @@ class CryptoSpotBot:
             adaptive_rules["reduce_now"] = position.id is not None and position.id == reduction_candidate_id
             learning = {"adaptive_rules": adaptive_rules}
             signal = self.strategy.exit_signal(position, candles, ticker, learning, forecast)
-            self.storage.insert_signal(signal)
+            self._record_signal(signal)
             if signal.action == "SELL" and ticker is not None:
-                self.broker.sell(position, ticker, signal.reason)
+                await asyncio.to_thread(self.broker.sell, position, ticker, signal.reason)
                 self._entry_cooldown_until[position.symbol] = utc_now()
 
     async def _process_entries(self) -> None:
         prices = self.market.prices()
         risk_guard = risk_guard_snapshot(
             self.settings,
-            self.storage.closed_trades(self.settings.learning_lookback_trades),
-            self.storage.latest_equity(),
+            self.storage.closed_trades(
+                self.settings.learning_lookback_trades,
+                mode=self.settings.trading_mode,
+            ),
+            self.storage.latest_equity(mode=self.settings.trading_mode),
         )
         for symbol in self.market.symbols:
+            freshness = self.market.symbol_freshness(symbol)
+            if not freshness["ok"]:
+                self._record_signal(
+                    Signal(
+                        symbol,
+                        "HOLD",
+                        0.0,
+                        "market data is stale; new entries blocked",
+                        {"market_freshness": freshness, "checks": {"market_fresh": False}},
+                    )
+                )
+                continue
             cooldown_since = self._entry_cooldown_until.get(symbol)
             if cooldown_since:
                 age = (utc_now() - cooldown_since).total_seconds()
                 cooldown_seconds = self.settings.effective_entry_cooldown_seconds
                 if age < cooldown_seconds:
-                    self.storage.insert_signal(
+                    self._record_signal(
                         Signal(
                             symbol,
                             "HOLD",
@@ -162,7 +217,7 @@ class CryptoSpotBot:
             account["open_positions_for_symbol"] = open_count
             account["exchange_min_entry_usdt"] = self.broker.minimum_entry_budget(instrument, ticker)
             if risk_guard.get("block_new_entries"):
-                self.storage.insert_signal(
+                self._record_signal(
                     Signal(
                         symbol,
                         "HOLD",
@@ -178,7 +233,7 @@ class CryptoSpotBot:
                 continue
             symbol_guard = self._risk_guard_for_symbol(risk_guard, symbol)
             if symbol_guard.get("block_new_entries"):
-                self.storage.insert_signal(
+                self._record_signal(
                     Signal(
                         symbol,
                         "HOLD",
@@ -224,9 +279,10 @@ class CryptoSpotBot:
                 account,
                 trend_candles,
             )
-            self.storage.insert_signal(signal)
+            self._record_signal(signal)
             if signal.action == "BUY" and ticker is not None:
-                position = self.broker.buy(
+                position = await asyncio.to_thread(
+                    self.broker.buy,
                     signal,
                     ticker,
                     instrument,
@@ -235,6 +291,41 @@ class CryptoSpotBot:
                 if position is not None:
                     self._entry_cooldown_until[symbol] = utc_now()
 
+    def _record_signal(self, signal: Signal) -> None:
+        self.storage.insert_signal(signal, self.settings.hold_signal_sample_seconds)
+
+    async def _maintain_runtime(self) -> None:
+        now = utc_now()
+        if isinstance(self.broker, LiveBroker):
+            age = (
+                (now - self._last_reconciliation_at).total_seconds()
+                if self._last_reconciliation_at
+                else float("inf")
+            )
+            if age >= self.settings.live_reconciliation_interval_seconds:
+                try:
+                    await asyncio.to_thread(self.broker.reconcile, self.market.instruments)
+                except Exception as exc:
+                    self.broker.reconciliation_state = {
+                        "status": "error",
+                        "blocking": True,
+                        "discrepancies": [
+                            {"code": "periodic_reconciliation_failed", "message": str(exc)}
+                        ],
+                        "checked_at": utc_now().isoformat(),
+                    }
+                    self.storage.event(f"Periodic live reconciliation failed: {exc}", "ERROR")
+                finally:
+                    self._last_reconciliation_at = utc_now()
+        prune_age = (
+            (now - self._last_prune_at).total_seconds()
+            if self._last_prune_at
+            else float("inf")
+        )
+        if prune_age >= self.settings.storage_prune_interval_seconds:
+            await asyncio.to_thread(self.storage.prune, self.settings.storage_retention_days)
+            self._last_prune_at = utc_now()
+
     @staticmethod
     def _risk_guard_for_symbol(risk_guard: dict, symbol: str) -> dict:
         rows = risk_guard.get("symbols")
@@ -334,16 +425,68 @@ class CryptoSpotBot:
         self.market.forecasts = forecasts
 
     def status(self) -> BotStatus:
+        live_ready = self.settings.live_ready
+        if isinstance(self.broker, LiveBroker):
+            live_ready = live_ready and not self.broker.reconciliation_state.get("blocking", True)
         return BotStatus(
             running=self.running,
             mode=self.settings.trading_mode,
-            live_trading_ready=self.settings.live_ready,
+            live_trading_ready=live_ready,
             symbols=self.market.symbols,
             started_at=self.started_at,
             last_loop_at=self.last_loop_at,
             message=self.message,
         )
 
+    def readiness_snapshot(self) -> dict:
+        reasons: list[str] = []
+        now = utc_now()
+        if not self.running:
+            reasons.append("bot_not_running")
+        max_loop_age = max(30.0, self.settings.effective_loop_interval_seconds * 4)
+        loop_age = (now - self.last_loop_at).total_seconds() if self.last_loop_at else None
+        if loop_age is None or loop_age > max_loop_age:
+            reasons.append("decision_loop_stale")
+        stale_symbols = [
+            symbol for symbol in self.market.symbols if not self.market.symbol_freshness(symbol)["ok"]
+        ]
+        if stale_symbols:
+            reasons.append("stale_market_data")
+        if self._consecutive_loop_errors >= 3:
+            reasons.append("repeated_loop_errors")
+        if self.settings.strategy_mode == "torch_forecast":
+            invalid_models = []
+            for symbol in self.market.symbols:
+                forecast = self.market.forecasts.get(symbol, {})
+                if not forecast.get("usable"):
+                    invalid_models.append(symbol)
+                    continue
+                if (
+                    self.settings.time_series_require_quality_gate
+                    and not self.settings.time_series_manual_quality_override
+                    and forecast.get("quality_gate_passed") is not True
+                ):
+                    invalid_models.append(symbol)
+                    continue
+                if self.settings.time_series_require_fresh_model and forecast.get("model_fresh") is not True:
+                    invalid_models.append(symbol)
+            if invalid_models:
+                reasons.append("forecast_model_not_ready")
+        reconciliation: dict = {}
+        if isinstance(self.broker, LiveBroker):
+            reconciliation = dict(self.broker.reconciliation_state)
+            if reconciliation.get("blocking", True):
+                reasons.append("live_reconciliation_blocking")
+        return {
+            "ready": not reasons,
+            "mode": self.settings.trading_mode,
+            "reasons": reasons,
+            "loop_age_seconds": round(loop_age, 3) if loop_age is not None else None,
+            "stale_symbols": stale_symbols,
+            "consecutive_loop_errors": self._consecutive_loop_errors,
+            "reconciliation": reconciliation,
+        }
+
     def account_snapshot(self) -> dict[str, float]:
         prices = self.market.prices()
         state = self.broker.account_state(prices)

crypto_spot_bot/bybit.py

@@ -9,6 +9,8 @@ from typing import Any
 from urllib.parse import urlencode
 
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
 from crypto_spot_bot.config import Settings
 from crypto_spot_bot.models import Candle, Ticker
@@ -41,6 +43,17 @@ class BybitClient:
     def __init__(self, settings: Settings):
         self.settings = settings
         self.session = requests.Session()
+        retry = Retry(
+            total=3,
+            connect=3,
+            read=3,
+            status=3,
+            backoff_factor=0.4,
+            status_forcelist=(429, 500, 502, 503, 504),
+            allowed_methods=frozenset({"GET"}),
+            respect_retry_after_header=True,
+        )
+        self.session.mount("https://", HTTPAdapter(max_retries=retry))
 
     def public_get(self, path: str, params: dict[str, Any]) -> dict[str, Any]:
         response = self.session.get(
@@ -208,27 +221,165 @@ class BybitClient:
             "symbol": symbol,
             "side": side,
             "orderType": "Market",
-            "qty": f"{qty:.8f}".rstrip("0").rstrip("."),
+            "qty": _decimal_text(qty),
             "timeInForce": "IOC",
             "isLeverage": 0,
             "orderFilter": "Order",
             "marketUnit": market_unit,
             "orderLinkId": order_link_id,
         }
+        slippage_percent = max(0.01, min(10.0, self.settings.slippage_rate * 100.0))
+        payload["slippageToleranceType"] = "Percent"
+        payload["slippageTolerance"] = f"{slippage_percent:.2f}"
         return self.private_post("/v5/order/create", payload)
 
+    def place_spot_protective_stop(
+        self,
+        *,
+        symbol: str,
+        qty: float,
+        trigger_price: float,
+        order_link_id: str,
+    ) -> dict[str, Any]:
+        payload = {
+            "category": "spot",
+            "symbol": symbol,
+            "side": "Sell",
+            "orderType": "Market",
+            "qty": _decimal_text(qty),
+            "triggerPrice": _decimal_text(trigger_price),
+            "timeInForce": "IOC",
+            "isLeverage": 0,
+            "orderFilter": "tpslOrder",
+            "marketUnit": "baseCoin",
+            "orderLinkId": order_link_id,
+        }
+        return self.private_post("/v5/order/create", payload)
+
+    def cancel_spot_order(
+        self,
+        *,
+        symbol: str,
+        order_id: str | None = None,
+        order_link_id: str | None = None,
+        order_filter: str = "Order",
+    ) -> dict[str, Any]:
+        if not order_id and not order_link_id:
+            raise ValueError("order_id or order_link_id is required")
+        payload: dict[str, Any] = {
+            "category": "spot",
+            "symbol": symbol,
+            "orderFilter": order_filter,
+        }
+        if order_id:
+            payload["orderId"] = order_id
+        if order_link_id:
+            payload["orderLinkId"] = order_link_id
+        return self.private_post("/v5/order/cancel", payload)
+
     def wallet_balance(self, account_type: str = "UNIFIED", coin: str | None = None) -> dict[str, Any]:
         return self.private_get(
             "/v5/account/wallet-balance",
             {"accountType": account_type, "coin": coin},
         )
 
-    def realtime_orders(self, *, category: str = "spot", open_only: int = 0, limit: int = 50) -> dict[str, Any]:
+    def realtime_orders(
+        self,
+        *,
+        category: str = "spot",
+        open_only: int = 0,
+        limit: int = 50,
+        symbol: str | None = None,
+        order_id: str | None = None,
+        order_link_id: str | None = None,
+        order_filter: str | None = None,
+    ) -> dict[str, Any]:
         return self.private_get(
             "/v5/order/realtime",
-            {"category": category, "openOnly": open_only, "limit": max(1, min(limit, 50))},
+            {
+                "category": category,
+                "openOnly": open_only,
+                "limit": max(1, min(limit, 50)),
+                "symbol": symbol,
+                "orderId": order_id,
+                "orderLinkId": order_link_id,
+                "orderFilter": order_filter,
+            },
         )
 
+    def order_history(
+        self,
+        *,
+        symbol: str | None = None,
+        order_id: str | None = None,
+        order_link_id: str | None = None,
+        limit: int = 50,
+    ) -> dict[str, Any]:
+        return self.private_get(
+            "/v5/order/history",
+            {
+                "category": "spot",
+                "symbol": symbol,
+                "orderId": order_id,
+                "orderLinkId": order_link_id,
+                "limit": max(1, min(limit, 50)),
+            },
+        )
+
+    def executions(
+        self,
+        *,
+        symbol: str | None = None,
+        order_id: str | None = None,
+        order_link_id: str | None = None,
+        limit: int = 100,
+    ) -> dict[str, Any]:
+        return self.private_get(
+            "/v5/execution/list",
+            {
+                "category": "spot",
+                "symbol": symbol,
+                "orderId": order_id,
+                "orderLinkId": order_link_id,
+                "limit": max(1, min(limit, 100)),
+            },
+        )
+
+    def wait_for_spot_order(
+        self,
+        *,
+        order_id: str,
+        symbol: str,
+        timeout_seconds: float,
+        poll_seconds: float = 0.5,
+    ) -> dict[str, Any]:
+        deadline = time.monotonic() + max(1.0, timeout_seconds)
+        latest: dict[str, Any] = {}
+        terminal = {
+            "Filled",
+            "Cancelled",
+            "Rejected",
+            "PartiallyFilledCanceled",
+            "PartillyFilledCancelled",
+            "Deactivated",
+        }
+        while time.monotonic() < deadline:
+            realtime = self.realtime_orders(symbol=symbol, order_id=order_id, open_only=1, limit=1)
+            rows = realtime.get("list") if isinstance(realtime.get("list"), list) else []
+            if rows and isinstance(rows[0], dict):
+                latest = rows[0]
+                if str(latest.get("orderStatus", "")) in terminal:
+                    break
+            time.sleep(max(0.1, poll_seconds))
+        if not latest or str(latest.get("orderStatus", "")) not in terminal:
+            history = self.order_history(symbol=symbol, order_id=order_id, limit=1)
+            rows = history.get("list") if isinstance(history.get("list"), list) else []
+            if rows and isinstance(rows[0], dict):
+                latest = rows[0]
+        execution_result = self.executions(symbol=symbol, order_id=order_id)
+        executions = execution_result.get("list") if isinstance(execution_result.get("list"), list) else []
+        return {"order": latest, "executions": executions}
+
 
 def websocket_subscribe_message(symbols: list[str], interval: str = "1") -> str:
     args: list[str] = []
@@ -254,3 +405,7 @@ def _looks_like_stablecoin(base_coin: str) -> bool:
         "PYUSD",
         "USD1",
     }
+
+
+def _decimal_text(value: float) -> str:
+    return f"{value:.12f}".rstrip("0").rstrip(".")

crypto_spot_bot/config.py

@@ -154,6 +154,20 @@ class Settings:
     database_path: Path
     log_path: Path
     env_file_path: Path
+    api_auth_token: str = ""
+    training_worker_token: str = ""
+    trusted_proxy_user_header: str = ""
+    time_series_require_quality_gate: bool = False
+    time_series_manual_quality_override: bool = False
+    time_series_require_fresh_model: bool = False
+    time_series_model_max_age_hours: float = 48.0
+    market_ticker_max_age_seconds: float = 45.0
+    live_order_fill_timeout_seconds: float = 20.0
+    live_reconciliation_interval_seconds: float = 30.0
+    live_protective_stop_enabled: bool = True
+    hold_signal_sample_seconds: int = 60
+    storage_retention_days: int = 30
+    storage_prune_interval_seconds: int = 3600
 
     @property
     def rest_base_url(self) -> str:
@@ -289,7 +303,7 @@ def load_settings(env_file: str | Path | None = None) -> Settings:
         time_series_probe_min_edge_percent=_float_env("TIME_SERIES_PROBE_MIN_EDGE_PERCENT", 0.02),
         time_series_probe_min_probability_up=_float_env("TIME_SERIES_PROBE_MIN_PROBABILITY_UP", 0.55),
         time_series_probe_size_multiplier=_float_env("TIME_SERIES_PROBE_SIZE_MULTIPLIER", 0.40),
-        time_series_rebound_fallback_enabled=_bool_env("TIME_SERIES_REBOUND_FALLBACK_ENABLED", True),
+        time_series_rebound_fallback_enabled=_bool_env("TIME_SERIES_REBOUND_FALLBACK_ENABLED", False),
         stop_loss_percent=_float_env("STOP_LOSS_PERCENT", 0.04),
         stop_loss_exit_enabled=_bool_env("STOP_LOSS_EXIT_ENABLED", True),
         take_profit_percent=_float_env("TAKE_PROFIT_PERCENT", 0.035),
@@ -307,7 +321,28 @@ def load_settings(env_file: str | Path | None = None) -> Settings:
         database_path=Path(os.getenv("DATABASE_PATH", "runtime/tradebot.sqlite3")),
         log_path=Path(os.getenv("LOG_PATH", "runtime/tradebot.log")),
         env_file_path=env_path,
+        api_auth_token=os.getenv("TRADEBOT_API_TOKEN", "").strip(),
+        training_worker_token=os.getenv("TRADEBOT_TRAINING_TOKEN", "").strip(),
+        trusted_proxy_user_header=os.getenv("TRUSTED_PROXY_USER_HEADER", "").strip(),
+        time_series_require_quality_gate=_bool_env(
+            "TIME_SERIES_REQUIRE_QUALITY_GATE", strategy_mode == "torch_forecast"
+        ),
+        time_series_manual_quality_override=_bool_env(
+            "TIME_SERIES_MANUAL_QUALITY_OVERRIDE", False
+        ),
+        time_series_require_fresh_model=_bool_env(
+            "TIME_SERIES_REQUIRE_FRESH_MODEL", strategy_mode == "torch_forecast"
+        ),
+        time_series_model_max_age_hours=_float_env("TIME_SERIES_MODEL_MAX_AGE_HOURS", 48.0),
+        market_ticker_max_age_seconds=_float_env("MARKET_TICKER_MAX_AGE_SECONDS", 45.0),
+        live_order_fill_timeout_seconds=_float_env("LIVE_ORDER_FILL_TIMEOUT_SECONDS", 20.0),
+        live_reconciliation_interval_seconds=_float_env("LIVE_RECONCILIATION_INTERVAL_SECONDS", 30.0),
+        live_protective_stop_enabled=_bool_env("LIVE_PROTECTIVE_STOP_ENABLED", True),
+        hold_signal_sample_seconds=_int_env("HOLD_SIGNAL_SAMPLE_SECONDS", 60),
+        storage_retention_days=_int_env("STORAGE_RETENTION_DAYS", 30),
+        storage_prune_interval_seconds=_int_env("STORAGE_PRUNE_INTERVAL_SECONDS", 3600),
     )
+    _validate_settings(settings)
     if settings.trading_mode == "live" and not settings.live_ready:
         raise ValueError(
             "Live mode is locked. Set ENABLE_LIVE_TRADING=true, "
@@ -316,6 +351,36 @@ def load_settings(env_file: str | Path | None = None) -> Settings:
     return settings
 
 
+def _validate_settings(settings: Settings) -> None:
+    errors: list[str] = []
+    if not 1 <= settings.port <= 65535:
+        errors.append("PORT must be in range 1..65535")
+    if settings.starting_balance_usdt <= 0:
+        errors.append("STARTING_BALANCE_USDT must be positive")
+    if settings.min_position_usdt < 0:
+        errors.append("MIN_POSITION_USDT must be non-negative")
+    if settings.max_position_usdt < settings.min_position_usdt:
+        errors.append("MAX_POSITION_USDT must be >= MIN_POSITION_USDT")
+    if settings.max_symbol_exposure_usdt < settings.min_position_usdt:
+        errors.append("MAX_SYMBOL_EXPOSURE_USDT must be >= MIN_POSITION_USDT")
+    if settings.max_total_exposure_usdt < settings.max_symbol_exposure_usdt:
+        errors.append("MAX_TOTAL_EXPOSURE_USDT must be >= MAX_SYMBOL_EXPOSURE_USDT")
+    if settings.max_open_positions < 1 or settings.max_positions_per_symbol < 1:
+        errors.append("position count limits must be positive")
+    if settings.taker_fee_rate < 0 or settings.slippage_rate < 0:
+        errors.append("TAKER_FEE_RATE and SLIPPAGE_RATE must be non-negative")
+    if settings.market_ticker_max_age_seconds <= 0:
+        errors.append("MARKET_TICKER_MAX_AGE_SECONDS must be positive")
+    if settings.time_series_model_max_age_hours <= 0:
+        errors.append("TIME_SERIES_MODEL_MAX_AGE_HOURS must be positive")
+    if settings.live_order_fill_timeout_seconds <= 0:
+        errors.append("LIVE_ORDER_FILL_TIMEOUT_SECONDS must be positive")
+    if settings.live_reconciliation_interval_seconds <= 0:
+        errors.append("LIVE_RECONCILIATION_INTERVAL_SECONDS must be positive")
+    if errors:
+        raise ValueError("; ".join(errors))
+
+
 def update_env_value(path: Path, key: str, value: str) -> None:
     lines = path.read_text(encoding="utf-8").splitlines() if path.exists() else []
     output: list[str] = []

crypto_spot_bot/dashboard.py

@@ -1,13 +1,16 @@
 from __future__ import annotations
 
+import asyncio
 import json
+import logging
 from contextlib import asynccontextmanager
 from typing import Any
 
-from fastapi import FastAPI, HTTPException, Response
+from fastapi import Depends, FastAPI, HTTPException, Response
 from fastapi.responses import JSONResponse, PlainTextResponse
 
 from crypto_spot_bot.analytics import analytics_snapshot
+from crypto_spot_bot.auth import ApiAuthorizer
 from crypto_spot_bot.bot import CryptoSpotBot
 from crypto_spot_bot.bybit import BybitClient
 from crypto_spot_bot.config import Settings, load_settings, update_env_value
@@ -23,6 +26,7 @@ from crypto_spot_bot.training_coordination import TrainingCoordinator
 
 
 WEB_UI_REMOVED_MESSAGE = "Web UI removed. Use the Android TradeBot AI app and /api/* endpoints."
+logger = logging.getLogger(__name__)
 
 
 def create_app(settings: Settings | None = None) -> FastAPI:
@@ -44,6 +48,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     forecaster = TimeSeriesForecaster(settings)
     bot = CryptoSpotBot(settings, storage, market, broker, strategy, pattern_analyzer, learner, forecaster)
     training = TrainingCoordinator(settings.time_series_lstm_model_path.parent)
+    authorizer = ApiAuthorizer(settings)
 
     @asynccontextmanager
     async def lifespan(_: FastAPI):
@@ -66,50 +71,62 @@ def create_app(settings: Settings | None = None) -> FastAPI:
 
     @app.get("/api/health")
     async def health() -> dict[str, Any]:
-        return {"ok": True, "running": bot.running, "mode": settings.trading_mode}
+        return {
+            "ok": True,
+            "running": bot.running,
+            "mode": settings.trading_mode,
+            "auth_configured": authorizer.configured(),
+        }
+
+    @app.get("/api/ready")
+    async def ready() -> JSONResponse:
+        payload = bot.readiness_snapshot()
+        return JSONResponse(payload, status_code=200 if payload["ready"] else 503)
 
     @app.get("/api/status")
-    async def status() -> dict[str, Any]:
+    async def status(_: None = Depends(authorizer.require)) -> dict[str, Any]:
         return {
             "status": bot.status().as_dict(),
             "account": bot.account_snapshot(),
             "positions": bot.positions_snapshot(),
             "learning": bot.learning_snapshot(),
-            "latest_equity": storage.latest_equity(),
+            "latest_equity": storage.latest_equity(mode=settings.trading_mode),
+            "readiness": bot.readiness_snapshot(),
         }
 
     @app.get("/api/markets")
-    async def markets() -> dict[str, Any]:
+    async def markets(_: None = Depends(authorizer.require)) -> dict[str, Any]:
         return market.snapshot()
 
     @app.get("/api/trades")
-    async def trades(limit: int = 80) -> dict[str, Any]:
+    async def trades(limit: int = 80, _: None = Depends(authorizer.require)) -> dict[str, Any]:
         row_limit = _limit(limit)
         return {
-            "items": storage.recent_trades(row_limit),
-            "closed_items": storage.closed_trades(row_limit),
-            "closed_summary": storage.closed_trade_summary(),
+            "items": storage.recent_trades(row_limit, mode=settings.trading_mode),
+            "closed_items": storage.closed_trades(row_limit, mode=settings.trading_mode),
+            "closed_summary": storage.closed_trade_summary(mode=settings.trading_mode),
         }
 
     @app.get("/api/signals")
-    async def signals(limit: int = 120) -> dict[str, Any]:
+    async def signals(limit: int = 120, _: None = Depends(authorizer.require)) -> dict[str, Any]:
         return {"items": storage.recent_signals(_limit(limit))}
 
     @app.get("/api/events")
-    async def events(limit: int = 120) -> dict[str, Any]:
+    async def events(limit: int = 120, _: None = Depends(authorizer.require)) -> dict[str, Any]:
         return {"items": storage.recent_events(_limit(limit))}
 
     @app.get("/api/analytics")
-    async def analytics() -> dict[str, Any]:
+    async def analytics(_: None = Depends(authorizer.require)) -> dict[str, Any]:
         return analytics_snapshot(settings, storage)
 
     @app.get("/api/quality")
-    async def quality() -> dict[str, Any]:
+    async def quality(_: None = Depends(authorizer.require)) -> dict[str, Any]:
         return market.snapshot().get("quality", {})
 
     @app.get("/api/reconciliation")
-    async def reconciliation() -> dict[str, Any]:
-        return reconciliation_snapshot(
+    async def reconciliation(_: None = Depends(authorizer.require)) -> dict[str, Any]:
+        return await asyncio.to_thread(
+            reconciliation_snapshot,
             settings=settings,
             storage=storage,
             client=client,
@@ -117,58 +134,113 @@ def create_app(settings: Settings | None = None) -> FastAPI:
         )
 
     @app.get("/api/backtest")
-    async def backtest() -> dict[str, Any]:
+    async def backtest(_: None = Depends(authorizer.require)) -> dict[str, Any]:
         return _runtime_json(settings, "torch_threshold_calibration.json")
 
     @app.get("/api/retrain")
-    async def retrain() -> dict[str, Any]:
+    async def retrain(_: None = Depends(authorizer.require)) -> dict[str, Any]:
         data = _runtime_json(settings, "torch_retrain_guard.json")
         data["coordination"] = training.status()
         return data
 
     @app.get("/api/training/status")
-    async def training_status() -> dict[str, Any]:
+    async def training_status(_: None = Depends(authorizer.require)) -> dict[str, Any]:
         return training.status()
 
     @app.post("/api/training/retrain")
-    async def training_retrain(payload: dict[str, Any] | None = None) -> dict[str, Any]:
+    async def training_retrain(
+        payload: dict[str, Any] | None = None,
+        _: None = Depends(authorizer.require),
+    ) -> dict[str, Any]:
         return training.request_retrain(payload)
 
     @app.post("/api/training/heartbeat")
-    async def training_heartbeat(payload: dict[str, Any] | None = None) -> dict[str, Any]:
+    async def training_heartbeat(
+        payload: dict[str, Any] | None = None,
+        _: None = Depends(authorizer.require_training),
+    ) -> dict[str, Any]:
         return training.heartbeat(payload)
 
     @app.post("/api/training/claim")
-    async def training_claim(payload: dict[str, Any] | None = None) -> dict[str, Any]:
+    async def training_claim(
+        payload: dict[str, Any] | None = None,
+        _: None = Depends(authorizer.require_training),
+    ) -> dict[str, Any]:
         return training.claim(payload)
 
     @app.post("/api/training/jobs/{job_id}/artifacts/chunk")
-    async def training_artifact_chunk(job_id: str, payload: dict[str, Any]) -> dict[str, Any]:
+    async def training_artifact_chunk(
+        job_id: str,
+        payload: dict[str, Any],
+        _: None = Depends(authorizer.require_training),
+    ) -> dict[str, Any]:
         try:
             return training.save_artifact_chunk(job_id, payload)
         except ValueError as exc:
             raise HTTPException(status_code=400, detail=str(exc)) from exc
 
     @app.post("/api/training/jobs/{job_id}/progress")
-    async def training_progress(job_id: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
+    async def training_progress(
+        job_id: str,
+        payload: dict[str, Any] | None = None,
+        _: None = Depends(authorizer.require_training),
+    ) -> dict[str, Any]:
         try:
             return training.progress(job_id, payload)
         except ValueError as exc:
             raise HTTPException(status_code=404, detail=str(exc)) from exc
 
     @app.post("/api/training/jobs/{job_id}/complete")
-    async def training_complete(job_id: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
+    async def training_complete(
+        job_id: str,
+        payload: dict[str, Any] | None = None,
+        _: None = Depends(authorizer.require_training),
+    ) -> dict[str, Any]:
         try:
             return training.complete(job_id, payload)
         except ValueError as exc:
-            raise HTTPException(status_code=404, detail=str(exc)) from exc
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
 
     @app.get("/api/config")
-    async def config() -> dict[str, Any]:
+    async def config(_: None = Depends(authorizer.require)) -> dict[str, Any]:
         return _safe_config(settings)
 
+    @app.get("/api/mobile/snapshot")
+    async def mobile_snapshot(_: None = Depends(authorizer.require)) -> dict[str, Any]:
+        row_limit = 220
+        retrain_data = _runtime_json(settings, "torch_retrain_guard.json")
+        retrain_data["coordination"] = training.status()
+        return {
+            "health": {
+                "ok": True,
+                "running": bot.running,
+                "mode": settings.trading_mode,
+            },
+            "status": {
+                "status": bot.status().as_dict(),
+                "account": bot.account_snapshot(),
+                "positions": bot.positions_snapshot(),
+                "learning": bot.learning_snapshot(),
+                "latest_equity": storage.latest_equity(mode=settings.trading_mode),
+                "readiness": bot.readiness_snapshot(),
+            },
+            "markets": market.snapshot(),
+            "signals": {"items": storage.recent_signals(row_limit)},
+            "config": _safe_config(settings),
+            "trades": {
+                "items": storage.recent_trades(10, mode=settings.trading_mode),
+                "closed_items": storage.closed_trades(10, mode=settings.trading_mode),
+                "closed_summary": storage.closed_trade_summary(mode=settings.trading_mode),
+            },
+            "retrain": retrain_data,
+            "backtest": _runtime_json(settings, "torch_threshold_calibration.json"),
+        }
+
     @app.post("/api/config/fast-trading")
-    async def set_fast_trading(payload: dict[str, Any]) -> dict[str, Any]:
+    async def set_fast_trading(
+        payload: dict[str, Any],
+        _: None = Depends(authorizer.require),
+    ) -> dict[str, Any]:
         enabled = _enabled_from_payload(payload)
         env_persisted = _apply_fast_trading(settings, storage, enabled)
         response = _safe_config(settings)
@@ -176,12 +248,12 @@ def create_app(settings: Settings | None = None) -> FastAPI:
         return response
 
     @app.post("/api/control/start")
-    async def start() -> dict[str, Any]:
+    async def start(_: None = Depends(authorizer.require)) -> dict[str, Any]:
         await bot.start()
         return bot.status().as_dict()
 
     @app.post("/api/control/stop")
-    async def stop() -> dict[str, Any]:
+    async def stop(_: None = Depends(authorizer.require)) -> dict[str, Any]:
         await bot.stop()
         return bot.status().as_dict()
 
@@ -207,13 +279,22 @@ def create_app(settings: Settings | None = None) -> FastAPI:
             "# HELP tradebot_loop_interval_seconds Effective bot decision loop interval.",
             "# TYPE tradebot_loop_interval_seconds gauge",
             f"tradebot_loop_interval_seconds {settings.effective_loop_interval_seconds:.4f}",
+            "# HELP tradebot_ready Whether trading prerequisites are ready.",
+            "# TYPE tradebot_ready gauge",
+            f"tradebot_ready {1 if bot.readiness_snapshot()['ready'] else 0}",
+            "# HELP tradebot_rest_errors_total REST refresh errors observed by market data.",
+            "# TYPE tradebot_rest_errors_total counter",
+            f"tradebot_rest_errors_total {market.rest_error_count}",
         ]
         return PlainTextResponse("\n".join(lines) + "\n")
 
     @app.exception_handler(Exception)
     async def error_handler(_, exc: Exception) -> JSONResponse:
-        storage.event(f"API error: {exc}", "ERROR")
-        return JSONResponse({"error": str(exc)}, status_code=500)
+        try:
+            storage.event(f"API error: {exc}", "ERROR")
+        except Exception:
+            logger.exception("Could not persist API error event")
+        return JSONResponse({"error": "internal server error"}, status_code=500)
 
     return app
 
@@ -317,6 +398,11 @@ def _safe_config(settings: Settings) -> dict[str, Any]:
         "time_series_probe_min_probability_up": settings.time_series_probe_min_probability_up,
         "time_series_probe_size_multiplier": settings.time_series_probe_size_multiplier,
         "time_series_rebound_fallback_enabled": settings.time_series_rebound_fallback_enabled,
+        "time_series_require_quality_gate": settings.time_series_require_quality_gate,
+        "time_series_manual_quality_override": settings.time_series_manual_quality_override,
+        "time_series_require_fresh_model": settings.time_series_require_fresh_model,
+        "time_series_model_max_age_hours": settings.time_series_model_max_age_hours,
+        "market_ticker_max_age_seconds": settings.market_ticker_max_age_seconds,
         "time_series_model_artifact": _time_series_model_artifact(settings),
         "stop_loss_percent": settings.stop_loss_percent,
         "stop_loss_exit_enabled": settings.stop_loss_exit_enabled,
@@ -331,6 +417,14 @@ def _safe_config(settings: Settings) -> dict[str, Any]:
         "slippage_rate": settings.slippage_rate,
         "live_ready": settings.live_ready,
         "live_order_max_usdt": settings.live_order_max_usdt,
+        "live_order_fill_timeout_seconds": settings.live_order_fill_timeout_seconds,
+        "live_reconciliation_interval_seconds": settings.live_reconciliation_interval_seconds,
+        "live_protective_stop_enabled": settings.live_protective_stop_enabled,
+        "api_auth_configured": bool(
+            settings.api_auth_token
+            or settings.training_worker_token
+            or settings.trusted_proxy_user_header
+        ),
     }
 
 

crypto_spot_bot/execution.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from collections import deque
 from datetime import timedelta
 from decimal import Decimal, ROUND_DOWN, ROUND_UP
-from typing import Iterable
+from typing import Any, Iterable
 from uuid import uuid4
 
 from crypto_spot_bot.bybit import BybitClient, Instrument
@@ -38,9 +38,19 @@ class PaperBroker:
     def __init__(self, settings: Settings, storage: Storage):
         self.settings = settings
         self.storage = storage
-        self.positions = storage.open_positions()
+        self.positions = storage.open_positions(settings.trading_mode)
         self.cash = float(storage.get_runtime("paper_cash", settings.starting_balance_usdt))
-        self.peak_equity = float(storage.get_runtime("peak_equity", settings.starting_balance_usdt))
+        today = utc_now().date().isoformat()
+        stored_peak_day = str(storage.get_runtime("paper_peak_equity_day", ""))
+        self.peak_equity_day = today
+        self.peak_equity = float(
+            storage.get_runtime("paper_daily_peak_equity", settings.starting_balance_usdt)
+            if stored_peak_day == today
+            else settings.starting_balance_usdt
+        )
+        self.lifetime_peak_equity = float(
+            storage.get_runtime("paper_lifetime_peak_equity", settings.starting_balance_usdt)
+        )
         self._entry_timestamps = deque()
 
     def open_positions(self) -> list[Position]:
@@ -64,11 +74,24 @@ class PaperBroker:
     def mark_equity(self, prices: dict[str, float]) -> dict[str, float]:
         state = self.account_state(prices)
         equity = state["equity"]
+        today = utc_now().date().isoformat()
+        if today != self.peak_equity_day:
+            self.peak_equity_day = today
+            self.peak_equity = equity
         self.peak_equity = max(self.peak_equity, equity)
+        self.lifetime_peak_equity = max(self.lifetime_peak_equity, equity)
         state["drawdown"] = max(0.0, self.peak_equity - equity)
         self.storage.set_runtime("paper_cash", self.cash)
-        self.storage.set_runtime("peak_equity", self.peak_equity)
-        self.storage.insert_equity(equity, self.cash, self.exposure(), state["drawdown"])
+        self.storage.set_runtime("paper_peak_equity_day", self.peak_equity_day)
+        self.storage.set_runtime("paper_daily_peak_equity", self.peak_equity)
+        self.storage.set_runtime("paper_lifetime_peak_equity", self.lifetime_peak_equity)
+        self.storage.insert_equity(
+            equity,
+            self.cash,
+            self.exposure(),
+            state["drawdown"],
+            mode=self.settings.trading_mode,
+        )
         return state
 
     def account_state(self, prices: dict[str, float]) -> dict[str, float]:
@@ -181,6 +204,7 @@ class PaperBroker:
             entry_confidence=signal.confidence,
             entry_pattern=str(signal.diagnostics.get("pattern", {}).get("label", "")),
             entry_diagnostics=signal.diagnostics,
+            mode=self.settings.trading_mode,
         )
         position.id = self.storage.insert_position(position)
         self.positions.append(position)
@@ -201,6 +225,7 @@ class PaperBroker:
                 entry_confidence=position.entry_confidence,
                 entry_diagnostics=position.entry_diagnostics,
                 opened_at=position.opened_at,
+                mode=self.settings.trading_mode,
             )
         )
         self.storage.event(
@@ -244,6 +269,7 @@ class PaperBroker:
             entry_diagnostics=position.entry_diagnostics,
             opened_at=position.opened_at,
             closed_at=utc_now(),
+            mode=self.settings.trading_mode,
         )
         trade.id = self.storage.insert_trade(trade)
         self.storage.event(
@@ -368,11 +394,139 @@ class PaperBroker:
 
 
 class LiveBroker(PaperBroker):
+    TERMINAL_ORDER_STATUSES = {
+        "Filled",
+        "Cancelled",
+        "Rejected",
+        "PartiallyFilledCanceled",
+        "PartillyFilledCancelled",
+        "Deactivated",
+    }
+
     def __init__(self, settings: Settings, storage: Storage, client: BybitClient):
         super().__init__(settings, storage)
         if not settings.live_ready:
             raise BrokerError("Live mode is not unlocked by settings")
         self.client = client
+        self.reconciliation_state: dict[str, Any] = {
+            "status": "unknown",
+            "blocking": True,
+            "discrepancies": ["live account has not been reconciled"],
+        }
+
+    def can_open(
+        self,
+        symbol: str,
+        prices: dict[str, float],
+        requested_notional: float | None = None,
+    ) -> tuple[bool, str]:
+        if self.reconciliation_state.get("blocking", True):
+            return False, "live reconciliation is not clean"
+        return super().can_open(symbol, prices, requested_notional)
+
+    def reconcile(self, instruments: dict[str, Instrument]) -> dict[str, Any]:
+        coins = {"USDT"}
+        for symbol in self.settings.symbols:
+            instrument = instruments.get(symbol)
+            if instrument and instrument.base_coin:
+                coins.add(instrument.base_coin.upper())
+        wallet = self.client.wallet_balance(coin=",".join(sorted(coins)))
+        balances = _wallet_balances(wallet)
+        usdt = balances.get("USDT", {})
+        self.cash = max(0.0, float(usdt.get("wallet_balance", 0.0)) - float(usdt.get("locked", 0.0)))
+
+        local_by_coin: dict[str, float] = {}
+        discrepancies: list[dict[str, Any]] = []
+        for position in self.positions:
+            instrument = instruments.get(position.symbol)
+            coin = instrument.base_coin.upper() if instrument and instrument.base_coin else position.symbol.removesuffix("USDT")
+            local_by_coin[coin] = local_by_coin.get(coin, 0.0) + position.qty
+        for coin, local_qty in local_by_coin.items():
+            remote_qty = float((balances.get(coin) or {}).get("wallet_balance", 0.0))
+            tolerance = max(1e-8, local_qty * 0.002)
+            if remote_qty + tolerance < local_qty:
+                discrepancies.append(
+                    {
+                        "severity": "error",
+                        "code": "remote_balance_below_local_position",
+                        "coin": coin,
+                        "local_qty": round(local_qty, 12),
+                        "remote_qty": round(remote_qty, 12),
+                    }
+                )
+        for coin, row in balances.items():
+            if coin == "USDT" or coin not in coins:
+                continue
+            remote_qty = float(row.get("wallet_balance", 0.0))
+            local_qty = local_by_coin.get(coin, 0.0)
+            tolerance = max(1e-8, local_qty * 0.002)
+            if remote_qty > local_qty + tolerance:
+                discrepancies.append(
+                    {
+                        "severity": "error",
+                        "code": "remote_asset_without_matching_local_position",
+                        "coin": coin,
+                        "local_qty": round(local_qty, 12),
+                        "remote_qty": round(remote_qty, 12),
+                    }
+                )
+
+        normal_orders = self.client.realtime_orders(
+            category="spot",
+            open_only=0,
+            limit=50,
+            order_filter="Order",
+        )
+        unresolved_orders = [
+            row
+            for row in normal_orders.get("list", [])
+            if isinstance(row, dict)
+            and str(row.get("orderStatus", "")) not in self.TERMINAL_ORDER_STATUSES
+        ]
+        if unresolved_orders:
+            discrepancies.append(
+                {
+                    "severity": "error",
+                    "code": "unresolved_exchange_orders",
+                    "count": len(unresolved_orders),
+                    "order_ids": [str(row.get("orderId", "")) for row in unresolved_orders[:10]],
+                }
+            )
+
+        protection_rows = self.client.realtime_orders(
+            category="spot",
+            open_only=0,
+            limit=50,
+            order_filter="tpslOrder",
+        )
+        active_protection = {
+            str(row.get("orderId", ""))
+            for row in protection_rows.get("list", [])
+            if isinstance(row, dict)
+            and str(row.get("orderStatus", "")) not in self.TERMINAL_ORDER_STATUSES
+        }
+        if self.settings.live_protective_stop_enabled:
+            for position in self.positions:
+                if not position.protective_order_id or position.protective_order_id not in active_protection:
+                    discrepancies.append(
+                        {
+                            "severity": "error",
+                            "code": "missing_exchange_protective_stop",
+                            "position_id": position.id,
+                            "symbol": position.symbol,
+                        }
+                    )
+
+        blocking = any(row.get("severity") == "error" for row in discrepancies)
+        self.reconciliation_state = {
+            "status": "error" if blocking else ("warn" if discrepancies else "ok"),
+            "blocking": blocking,
+            "discrepancies": discrepancies,
+            "cash_usdt": round(self.cash, 8),
+            "checked_at": utc_now().isoformat(),
+        }
+        self.storage.set_runtime("live_reconciliation", self.reconciliation_state)
+        return dict(self.reconciliation_state)
 
     def buy(
         self,
@@ -400,30 +554,356 @@ class LiveBroker(PaperBroker):
         if budget < max(self.settings.min_position_usdt, minimum_budget):
             self.storage.event(f"{ticker.symbol}: live BUY skipped, adjusted budget below minimum", "WARN")
             return None
+
         signal.diagnostics["position_notional_usdt"] = budget
-        notional = budget / (1 + self.settings.taker_fee_rate)
-        response = self.client.place_spot_market_order(
+        requested_quote = budget / (1 + self.settings.taker_fee_rate)
+        client_order_id = f"tb-buy-{uuid4().hex[:18]}"
+        self.storage.upsert_order(
+            client_order_id=client_order_id,
             symbol=ticker.symbol,
             side="Buy",
-            qty=notional,
-            market_unit="quoteCoin",
-            order_link_id=f"tb-buy-{uuid4().hex[:18]}",
+            order_kind="MARKET",
+            status="PENDING_SUBMIT",
+            requested_notional=requested_quote,
+            raw={"signal": signal.as_dict()},
         )
-        self.storage.event(f"{ticker.symbol}: реальная покупка отправлена orderId={response.get('orderId')}")
-        return self._record_buy(signal, ticker, instrument, "реальная покупка, локальная запись")
+        try:
+            response = self.client.place_spot_market_order(
+                symbol=ticker.symbol,
+                side="Buy",
+                qty=requested_quote,
+                market_unit="quoteCoin",
+                order_link_id=client_order_id,
+            )
+            order_id = str(response.get("orderId", ""))
+            if not order_id:
+                raise BrokerError("Bybit did not return orderId for live BUY")
+            self.storage.upsert_order(
+                client_order_id=client_order_id,
+                exchange_order_id=order_id,
+                symbol=ticker.symbol,
+                side="Buy",
+                order_kind="MARKET",
+                status="ACCEPTED",
+                requested_notional=requested_quote,
+                raw=response,
+            )
+            result = self.client.wait_for_spot_order(
+                order_id=order_id,
+                symbol=ticker.symbol,
+                timeout_seconds=self.settings.live_order_fill_timeout_seconds,
+            )
+            fill = _execution_fill(result, side="Buy", instrument=instrument)
+            self._save_order_fill(client_order_id, order_id, ticker.symbol, "Buy", requested_quote, result, fill)
+            if fill["qty"] <= 0 or fill["value"] <= 0:
+                raise BrokerError(f"live BUY was not filled, status={fill['status']}")
+            position = self._record_live_buy(signal, ticker, fill)
+            if self.settings.live_protective_stop_enabled:
+                try:
+                    self._place_protective_stop(position)
+                except Exception as exc:
+                    self.storage.event(
+                        f"{ticker.symbol}: protective stop placement failed, closing position: {exc}",
+                        "ERROR",
+                    )
+                    self.sell(position, ticker, "protective stop placement failed")
+                    raise BrokerError("live BUY was unwound because protective stop failed") from exc
+            return position
+        except Exception as exc:
+            self.reconciliation_state["blocking"] = True
+            self.reconciliation_state["status"] = "error"
+            self.storage.event(f"{ticker.symbol}: live BUY failed: {exc}", "ERROR")
+            raise
 
     def sell(self, position: Position, ticker: Ticker, reason: str) -> Trade:
+        if position.protective_order_id or position.protective_order_link_id:
+            self.client.cancel_spot_order(
+                symbol=position.symbol,
+                order_id=position.protective_order_id or None,
+                order_link_id=position.protective_order_link_id or None,
+                order_filter="tpslOrder",
+            )
+            if position.protective_order_id:
+                cancelled = self.client.wait_for_spot_order(
+                    order_id=position.protective_order_id,
+                    symbol=position.symbol,
+                    timeout_seconds=min(10.0, self.settings.live_order_fill_timeout_seconds),
+                )
+                status = str((cancelled.get("order") or {}).get("orderStatus", ""))
+                if status and status != "Cancelled":
+                    raise BrokerError(f"protective order was not cancelled, status={status}")
+
+        client_order_id = f"tb-sell-{uuid4().hex[:18]}"
+        self.storage.upsert_order(
+            client_order_id=client_order_id,
+            symbol=position.symbol,
+            side="Sell",
+            order_kind="MARKET",
+            status="PENDING_SUBMIT",
+            requested_qty=position.qty,
+            raw={"position_id": position.id, "reason": reason},
+        )
         response = self.client.place_spot_market_order(
             symbol=position.symbol,
             side="Sell",
             qty=position.qty,
             market_unit="baseCoin",
-            order_link_id=f"tb-sell-{uuid4().hex[:18]}",
+            order_link_id=client_order_id,
+        )
+        order_id = str(response.get("orderId", ""))
+        if not order_id:
+            raise BrokerError("Bybit did not return orderId for live SELL")
+        result = self.client.wait_for_spot_order(
+            order_id=order_id,
+            symbol=position.symbol,
+            timeout_seconds=self.settings.live_order_fill_timeout_seconds,
+        )
+        fill = _execution_fill(result, side="Sell", instrument=None)
+        self._save_order_fill(client_order_id, order_id, position.symbol, "Sell", position.qty, result, fill)
+        if fill["qty"] <= 0 or fill["value"] <= 0:
+            self.reconciliation_state["blocking"] = True
+            raise BrokerError(f"live SELL was not filled, status={fill['status']}")
+        return self._record_live_sell(position, reason, fill)
+
+    def _record_live_buy(self, signal: Signal, ticker: Ticker, fill: dict[str, Any]) -> Position:
+        qty = float(fill["net_qty"])
+        value = float(fill["value"])
+        price = value / max(float(fill["qty"]), 1e-12)
+        fee_usdt = float(fill["fee_usdt"])
+        stop_loss_percent = self._signal_percent(
+            signal, "stop_loss_percent", self.settings.stop_loss_percent, 0.003, 0.08
+        )
+        take_profit_percent = self._signal_percent(
+            signal, "take_profit_percent", self.settings.take_profit_percent, 0.003, 0.20
+        )
+        position = Position(
+            id=None,
+            symbol=ticker.symbol,
+            qty=qty,
+            entry_price=price,
+            notional_usdt=value,
+            entry_fee_usdt=fee_usdt,
+            stop_loss=price * (1 - stop_loss_percent),
+            take_profit=price * (1 + take_profit_percent),
+            highest_price=price,
+            entry_reason=signal.reason,
+            entry_confidence=signal.confidence,
+            entry_pattern=str(signal.diagnostics.get("pattern", {}).get("label", "")),
+            entry_diagnostics=signal.diagnostics,
+            mode="live",
+        )
+        position.id = self.storage.insert_position(position)
+        self.positions.append(position)
+        self._record_entry_timestamp()
+        self.cash = max(0.0, self.cash - value - float(fill["quote_fee"]))
+        self.storage.insert_trade(
+            Trade(
+                id=None,
+                symbol=ticker.symbol,
+                side="BUY",
+                qty=qty,
+                entry_price=price,
+                fee_usdt=fee_usdt,
+                net_pnl=-fee_usdt,
+                reason=signal.reason,
+                entry_pattern=position.entry_pattern,
+                entry_confidence=position.entry_confidence,
+                entry_diagnostics=position.entry_diagnostics,
+                opened_at=position.opened_at,
+                mode="live",
+            )
         )
         self.storage.event(
-            f"{position.symbol}: реальная продажа отправлена orderId={response.get('orderId')} причина={reason}"
+            f"{ticker.symbol}: live BUY filled qty={qty:.8f} avg={price:.8f} value={value:.4f}"
         )
-        return self._record_sell(position, ticker, reason, "реальная продажа, локальная запись")
+        return position
+
+    def _record_live_sell(self, position: Position, reason: str, fill: dict[str, Any]) -> Trade:
+        sold_qty = min(position.qty, float(fill["qty"]))
+        value = float(fill["value"])
+        price = value / max(float(fill["qty"]), 1e-12)
+        exit_fee = float(fill["fee_usdt"])
+        ratio = min(1.0, sold_qty / max(position.qty, 1e-12))
+        allocated_entry_fee = position.entry_fee_usdt * ratio
+        gross_pnl = (price - position.entry_price) * sold_qty
+        net_pnl = gross_pnl - allocated_entry_fee - exit_fee
+        self.cash += value - float(fill["quote_fee"])
+        remaining_qty = max(0.0, position.qty - sold_qty)
+        if remaining_qty <= max(1e-12, position.qty * 1e-6):
+            if position.id is not None:
+                self.storage.close_position(position.id)
+            self.positions = [item for item in self.positions if item.id != position.id]
+        else:
+            remaining_ratio = remaining_qty / position.qty
+            position.qty = remaining_qty
+            position.notional_usdt *= remaining_ratio
+            position.entry_fee_usdt *= remaining_ratio
+            position.protective_order_id = ""
+            position.protective_order_link_id = ""
+            if position.id is not None:
+                self.storage.update_position_after_partial_sell(
+                    position.id,
+                    qty=position.qty,
+                    notional_usdt=position.notional_usdt,
+                    entry_fee_usdt=position.entry_fee_usdt,
+                )
+            self.reconciliation_state["blocking"] = True
+        trade = Trade(
+            id=None,
+            symbol=position.symbol,
+            side="SELL",
+            qty=sold_qty,
+            entry_price=position.entry_price,
+            exit_price=price,
+            gross_pnl=gross_pnl,
+            fee_usdt=allocated_entry_fee + exit_fee,
+            net_pnl=net_pnl,
+            reason=reason,
+            entry_pattern=position.entry_pattern,
+            entry_confidence=position.entry_confidence,
+            entry_diagnostics=position.entry_diagnostics,
+            opened_at=position.opened_at,
+            closed_at=utc_now(),
+            mode="live",
+        )
+        trade.id = self.storage.insert_trade(trade)
+        self.storage.event(
+            f"{position.symbol}: live SELL filled qty={sold_qty:.8f} avg={price:.8f} pnl={net_pnl:.4f} reason={reason}"
+        )
+        return trade
+
+    def _place_protective_stop(self, position: Position) -> None:
+        link_id = f"tb-stop-{uuid4().hex[:17]}"
+        response = self.client.place_spot_protective_stop(
+            symbol=position.symbol,
+            qty=position.qty,
+            trigger_price=position.stop_loss,
+            order_link_id=link_id,
+        )
+        order_id = str(response.get("orderId", ""))
+        if not order_id:
+            raise BrokerError("Bybit did not return orderId for protective stop")
+        position.protective_order_id = order_id
+        position.protective_order_link_id = link_id
+        if position.id is not None:
+            self.storage.update_position_protective_order(position.id, order_id, link_id)
+        self.storage.upsert_order(
+            client_order_id=link_id,
+            exchange_order_id=order_id,
+            symbol=position.symbol,
+            side="Sell",
+            order_kind="PROTECTIVE_STOP",
+            status="ACCEPTED",
+            requested_qty=position.qty,
+            raw=response,
+        )
+
+    def _save_order_fill(
+        self,
+        client_order_id: str,
+        order_id: str,
+        symbol: str,
+        side: str,
+        requested: float,
+        result: dict[str, Any],
+        fill: dict[str, Any],
+    ) -> None:
+        self.storage.upsert_order(
+            client_order_id=client_order_id,
+            exchange_order_id=order_id,
+            symbol=symbol,
+            side=side,
+            order_kind="MARKET",
+            status=str(fill["status"]),
+            requested_qty=requested if side == "Sell" else 0.0,
+            requested_notional=requested if side == "Buy" else 0.0,
+            executed_qty=float(fill["qty"]),
+            executed_value=float(fill["value"]),
+            fee_usdt=float(fill["fee_usdt"]),
+            raw=result,
+        )
+
+
+def _wallet_balances(wallet: dict[str, Any]) -> dict[str, dict[str, float]]:
+    accounts = wallet.get("list")
+    if not isinstance(accounts, list) or not accounts:
+        return {}
+    coins = accounts[0].get("coin") if isinstance(accounts[0], dict) else None
+    if not isinstance(coins, list):
+        return {}
+    result: dict[str, dict[str, float]] = {}
+    for row in coins:
+        if not isinstance(row, dict):
+            continue
+        coin = str(row.get("coin", "")).upper()
+        if not coin:
+            continue
+        result[coin] = {
+            "wallet_balance": _safe_float(row.get("walletBalance")),
+            "equity": _safe_float(row.get("equity")),
+            "locked": _safe_float(row.get("locked")),
+        }
+    return result
+
+
+def _execution_fill(
+    result: dict[str, Any],
+    *,
+    side: str,
+    instrument: Instrument | None,
+) -> dict[str, Any]:
+    order = result.get("order") if isinstance(result.get("order"), dict) else {}
+    executions = result.get("executions") if isinstance(result.get("executions"), list) else []
+    qty = 0.0
+    value = 0.0
+    quote_fee = 0.0
+    base_fee = 0.0
+    fee_usdt = 0.0
+    base_coin = instrument.base_coin.upper() if instrument and instrument.base_coin else ""
+    for row in executions:
+        if not isinstance(row, dict):
+            continue
+        exec_qty = _safe_float(row.get("execQty"))
+        exec_value = _safe_float(row.get("execValue"))
+        exec_price = _safe_float(row.get("execPrice"))
+        fee = max(0.0, _safe_float(row.get("execFee")))
+        fee_currency = str(row.get("feeCurrency", "")).upper()
+        if not base_coin:
+            symbol = str(row.get("symbol", ""))
+            base_coin = symbol.removesuffix("USDT") if symbol.endswith("USDT") else ""
+        qty += exec_qty
+        value += exec_value or exec_qty * exec_price
+        if fee_currency == "USDT" or not fee_currency:
+            quote_fee += fee
+            fee_usdt += fee
+        elif fee_currency == base_coin:
+            base_fee += fee
+            fee_usdt += fee * exec_price
+        else:
+            fee_usdt += fee * exec_price
+    if qty <= 0:
+        qty = _safe_float(order.get("cumExecQty"))
+    if value <= 0:
+        value = _safe_float(order.get("cumExecValue"))
+    if value <= 0 and qty > 0:
+        value = qty * _safe_float(order.get("avgPrice"))
+    net_qty = max(0.0, qty - base_fee) if side == "Buy" else qty
+    return {
+        "status": str(order.get("orderStatus", "Unknown")),
+        "qty": qty,
+        "net_qty": net_qty,
+        "value": value,
+        "quote_fee": quote_fee,
+        "base_fee": base_fee,
+        "fee_usdt": fee_usdt,
+    }
+
+
+def _safe_float(value: Any, default: float = 0.0) -> float:
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return default
 
 
 def prices_from_tickers(tickers: Iterable[Ticker]) -> dict[str, float]:

crypto_spot_bot/learning.py

@@ -60,7 +60,10 @@ class TradeLearner:
             self.storage.set_runtime("learning_state", self.state.as_dict())
             return self.state
 
-        trades = self.storage.closed_trades(self.settings.learning_lookback_trades)
+        trades = self.storage.closed_trades(
+            self.settings.learning_lookback_trades,
+            mode=self.settings.trading_mode,
+        )
         total_net = sum(float(trade.get("net_pnl") or 0.0) for trade in trades)
         wins = sum(1 for trade in trades if float(trade.get("net_pnl") or 0.0) > 0)
         symbol_stats = _group_stats(trades, "symbol")

crypto_spot_bot/main.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+from logging.handlers import RotatingFileHandler
 
 import uvicorn
 
@@ -15,7 +16,12 @@ def main() -> None:
         level=logging.INFO,
         format="%(asctime)s %(levelname)s %(name)s %(message)s",
         handlers=[
-            logging.FileHandler(settings.log_path, encoding="utf-8"),
+            RotatingFileHandler(
+                settings.log_path,
+                maxBytes=10 * 1024 * 1024,
+                backupCount=5,
+                encoding="utf-8",
+            ),
             logging.StreamHandler(),
         ],
     )

crypto_spot_bot/market_data.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import json
+import threading
 from dataclasses import asdict
 from datetime import datetime
 from typing import Any
@@ -56,6 +57,9 @@ class MarketData:
         self.last_ws_message_at: datetime | None = None
         self.ws_connected = False
         self._stop_event = asyncio.Event()
+        self._refresh_lock = threading.Lock()
+        self.rest_error_count = 0
+        self.last_rest_error = ""
 
     async def bootstrap(self) -> None:
         self.instruments = await asyncio.to_thread(self.client.instruments)
@@ -76,47 +80,60 @@ class MarketData:
                 if symbol in self.instruments
             ]
         self.storage.event("Торговые пары: " + ", ".join(self.symbols))
-        await asyncio.to_thread(self.refresh_rest)
+        await asyncio.to_thread(self.refresh_rest, True)
 
-    def refresh_rest(self) -> None:
-        ticker_map = {ticker.symbol: ticker for ticker in self.client.spot_tickers()}
-        for symbol in self.symbols:
-            ticker = ticker_map.get(symbol)
-            if ticker:
-                self.tickers[symbol] = ticker
-            try:
-                candles = self.client.klines(
-                    symbol=symbol,
-                    interval=self.settings.base_interval,
-                    limit=self.settings.kline_limit,
-                )
-                candles = _closed_candles(candles, self.settings.base_interval)
-                add_indicators(candles)
-                self.candles[symbol] = candles
-                trend_candles = self.client.klines(
-                    symbol=symbol,
-                    interval=self.settings.trend_interval,
-                    limit=self.settings.trend_kline_limit,
-                )
-                trend_candles = _closed_candles(trend_candles, self.settings.trend_interval)
-                add_indicators(trend_candles)
-                self.trend_candles[symbol] = trend_candles
-                bid, ask = self.client.orderbook_top(symbol)
-                self.orderbook_top[symbol] = (bid, ask)
-                if symbol in self.tickers:
-                    current = self.tickers[symbol]
-                    self.tickers[symbol] = Ticker(
-                        symbol=current.symbol,
-                        last_price=current.last_price,
-                        bid=bid or current.bid,
-                        ask=ask or current.ask,
-                        turnover_24h=current.turnover_24h,
-                        volume_24h=current.volume_24h,
-                        change_24h=current.change_24h,
-                    )
-            except Exception as exc:
-                self.storage.event(f"{symbol}: ошибка обновления REST данных: {exc}", "ERROR")
-        self.last_rest_refresh_at = utc_now()
+    def refresh_rest(self, force_candles: bool = False) -> None:
+        if not self._refresh_lock.acquire(blocking=False):
+            return
+        try:
+            ticker_map = {ticker.symbol: ticker for ticker in self.client.spot_tickers()}
+            for symbol in self.symbols:
+                ticker = ticker_map.get(symbol)
+                if ticker:
+                    self.tickers[symbol] = ticker
+                try:
+                    if force_candles or _candles_due(self.candles.get(symbol, []), self.settings.base_interval):
+                        candles = self.client.klines(
+                            symbol=symbol,
+                            interval=self.settings.base_interval,
+                            limit=self.settings.kline_limit,
+                        )
+                        candles = _closed_candles(candles, self.settings.base_interval)
+                        add_indicators(candles)
+                        self.candles[symbol] = candles
+                    if force_candles or _candles_due(
+                        self.trend_candles.get(symbol, []), self.settings.trend_interval
+                    ):
+                        trend_candles = self.client.klines(
+                            symbol=symbol,
+                            interval=self.settings.trend_interval,
+                            limit=self.settings.trend_kline_limit,
+                        )
+                        trend_candles = _closed_candles(trend_candles, self.settings.trend_interval)
+                        add_indicators(trend_candles)
+                        self.trend_candles[symbol] = trend_candles
+                    bid, ask = self.client.orderbook_top(symbol)
+                    self.orderbook_top[symbol] = (bid, ask)
+                    if symbol in self.tickers:
+                        current = self.tickers[symbol]
+                        self.tickers[symbol] = Ticker(
+                            symbol=current.symbol,
+                            last_price=current.last_price,
+                            bid=bid or current.bid,
+                            ask=ask or current.ask,
+                            turnover_24h=current.turnover_24h,
+                            volume_24h=current.volume_24h,
+                            change_24h=current.change_24h,
+                        )
+                except Exception as exc:
+                    self.rest_error_count += 1
+                    self.last_rest_error = str(exc)
+                    self.storage.event(f"{symbol}: ошибка обновления REST данных: {exc}", "ERROR")
+            self.last_rest_refresh_at = utc_now()
+            if ticker_map:
+                self.last_rest_error = ""
+        finally:
+            self._refresh_lock.release()
 
     async def websocket_loop(self) -> None:
         if not self.settings.websocket_enabled:
@@ -227,10 +244,36 @@ class MarketData:
     def prices(self) -> dict[str, float]:
         return {symbol: ticker.last_price for symbol, ticker in self.tickers.items()}
 
+    def symbol_freshness(self, symbol: str) -> dict[str, Any]:
+        ticker = self.tickers.get(symbol)
+        candles = self.candles.get(symbol, [])
+        ticker_age = (utc_now() - ticker.updated_at).total_seconds() if ticker else None
+        interval_ms = _interval_ms(self.settings.base_interval)
+        candle_age = (
+            max(0.0, (utc_now().timestamp() * 1000 - candles[-1].timestamp) / 1000)
+            if candles
+            else None
+        )
+        ticker_ok = ticker_age is not None and ticker_age <= self.settings.market_ticker_max_age_seconds
+        candle_ok = bool(
+            candle_age is not None
+            and interval_ms > 0
+            and candle_age <= (interval_ms / 1000) * 2.5
+        )
+        return {
+            "ok": bool(ticker_ok and candle_ok),
+            "ticker_ok": ticker_ok,
+            "candle_ok": candle_ok,
+            "ticker_age_seconds": round(ticker_age, 3) if ticker_age is not None else None,
+            "candle_age_seconds": round(candle_age, 3) if candle_age is not None else None,
+        }
+
     def snapshot(self) -> dict[str, Any]:
         return {
             "symbols": self.symbols,
             "ws_connected": self.ws_connected,
+            "rest_error_count": self.rest_error_count,
+            "last_rest_error": self.last_rest_error,
             "quality": market_quality_snapshot(
                 symbols=self.symbols,
                 candles_by_symbol=self.candles,
@@ -291,3 +334,14 @@ def _interval_ms(interval: str) -> int:
     if normalized.isdigit():
         return int(normalized) * 60 * 1000
     return 0
+
+
+def _candles_due(candles: list[Candle], interval: str, now_ms: int | None = None) -> bool:
+    if not candles:
+        return True
+    interval_ms = _interval_ms(interval)
+    if interval_ms <= 0:
+        return True
+    now_ms = now_ms if now_ms is not None else int(utc_now().timestamp() * 1000)
+    expected_latest_start = (now_ms // interval_ms - 1) * interval_ms
+    return candles[-1].timestamp < expected_latest_start

crypto_spot_bot/models.py

@@ -88,6 +88,9 @@ class Position:
     entry_confidence: float = 0.0
     entry_pattern: str = ""
     entry_diagnostics: dict[str, Any] = field(default_factory=dict)
+    protective_order_id: str = ""
+    protective_order_link_id: str = ""
+    mode: str = "paper"
 
     def mark_price(self, price: float) -> float:
         return self.qty * price
@@ -131,6 +134,7 @@ class Trade:
     entry_diagnostics: dict[str, Any] = field(default_factory=dict)
     opened_at: datetime | None = None
     closed_at: datetime | None = None
+    mode: str = "paper"
 
     def as_dict(self) -> dict[str, Any]:
         data = asdict(self)

crypto_spot_bot/reconciliation.py

@@ -15,7 +15,7 @@ def reconciliation_snapshot(
     client: BybitClient,
     instruments: dict[str, Instrument],
 ) -> dict[str, Any]:
-    local_positions = storage.open_positions()
+    local_positions = storage.open_positions(settings.trading_mode)
     local = [
         {
             "id": position.id,
@@ -23,6 +23,7 @@ def reconciliation_snapshot(
             "qty": position.qty,
             "entry_price": position.entry_price,
             "notional_usdt": position.notional_usdt,
+            "protective_order_id": position.protective_order_id,
         }
         for position in local_positions
     ]

crypto_spot_bot/storage.py

@@ -2,23 +2,61 @@ from __future__ import annotations
 
 import json
 import sqlite3
+import time
 from contextlib import contextmanager
+from datetime import timedelta
 from pathlib import Path
 from typing import Any, Iterator
 
 from crypto_spot_bot.models import Position, Signal, Trade, utc_now
 
 
+MAX_SIGNAL_DIAGNOSTICS_BYTES = 16 * 1024
+PRUNE_BATCH_SIZE = 1000
+_STORED_FORECAST_KEYS = {
+    "enabled",
+    "usable",
+    "model",
+    "volatility_model",
+    "expected_return_percent",
+    "expected_price",
+    "volatility_percent",
+    "probability_up",
+    "confidence_adjustment",
+    "block_entry",
+    "validation_mae_percent",
+    "baseline_mae_percent",
+    "skill",
+    "horizon",
+    "reason",
+    "expected_gross_return_percent",
+    "quantile_10_percent",
+    "quantile_50_percent",
+    "quantile_90_percent",
+    "conservative_return_percent",
+    "target_transform",
+    "horizon_forecasts",
+    "candidates",
+    "quality_gate_passed",
+    "model_created_at",
+    "model_age_hours",
+    "model_fresh",
+}
+
+
 class Storage:
     def __init__(self, path: str | Path):
         self.path = Path(path)
         self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._last_hold_signal: dict[tuple[str, str], float] = {}
         self.init_schema()
 
     @contextmanager
     def connect(self) -> Iterator[sqlite3.Connection]:
         conn = sqlite3.connect(self.path)
         conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA busy_timeout=5000")
+        conn.execute("PRAGMA foreign_keys=ON")
         try:
             yield conn
             conn.commit()
@@ -27,6 +65,7 @@ class Storage:
 
     def init_schema(self) -> None:
         with self.connect() as conn:
+            conn.execute("PRAGMA journal_mode=WAL")
             conn.executescript(
                 """
                 CREATE TABLE IF NOT EXISTS positions (
@@ -44,6 +83,9 @@ class Storage:
                     entry_confidence REAL NOT NULL DEFAULT 0,
                     entry_pattern TEXT NOT NULL DEFAULT '',
                     entry_diagnostics_json TEXT NOT NULL DEFAULT '{}',
+                    protective_order_id TEXT NOT NULL DEFAULT '',
+                    protective_order_link_id TEXT NOT NULL DEFAULT '',
+                    mode TEXT NOT NULL DEFAULT 'paper',
                     status TEXT NOT NULL DEFAULT 'OPEN'
                 );
                 CREATE TABLE IF NOT EXISTS trades (
@@ -61,7 +103,8 @@ class Storage:
                     entry_confidence REAL NOT NULL DEFAULT 0,
                     entry_diagnostics_json TEXT NOT NULL DEFAULT '{}',
                     opened_at TEXT,
-                    closed_at TEXT
+                    closed_at TEXT,
+                    mode TEXT NOT NULL DEFAULT 'paper'
                 );
                 CREATE TABLE IF NOT EXISTS signals (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -78,7 +121,8 @@ class Storage:
                     cash REAL NOT NULL,
                     exposure REAL NOT NULL,
                     drawdown REAL NOT NULL,
-                    created_at TEXT NOT NULL
+                    created_at TEXT NOT NULL,
+                    mode TEXT NOT NULL DEFAULT 'paper'
                 );
                 CREATE TABLE IF NOT EXISTS events (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -101,6 +145,35 @@ class Storage:
                     error TEXT NOT NULL DEFAULT '',
                     created_at TEXT NOT NULL
                 );
+                CREATE TABLE IF NOT EXISTS orders (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    client_order_id TEXT NOT NULL UNIQUE,
+                    exchange_order_id TEXT NOT NULL DEFAULT '',
+                    symbol TEXT NOT NULL,
+                    side TEXT NOT NULL,
+                    order_kind TEXT NOT NULL DEFAULT 'MARKET',
+                    status TEXT NOT NULL,
+                    requested_qty REAL NOT NULL DEFAULT 0,
+                    requested_notional REAL NOT NULL DEFAULT 0,
+                    executed_qty REAL NOT NULL DEFAULT 0,
+                    executed_value REAL NOT NULL DEFAULT 0,
+                    fee_usdt REAL NOT NULL DEFAULT 0,
+                    raw_json TEXT NOT NULL DEFAULT '{}',
+                    created_at TEXT NOT NULL,
+                    updated_at TEXT NOT NULL
+                );
+                CREATE INDEX IF NOT EXISTS idx_positions_status_opened
+                    ON positions(status, opened_at);
+                CREATE INDEX IF NOT EXISTS idx_trades_closed
+                    ON trades(side, closed_at, id DESC);
+                CREATE INDEX IF NOT EXISTS idx_signals_symbol_created
+                    ON signals(symbol, created_at DESC);
+                CREATE INDEX IF NOT EXISTS idx_equity_created
+                    ON equity(created_at DESC);
+                CREATE INDEX IF NOT EXISTS idx_events_created
+                    ON events(created_at DESC);
+                CREATE INDEX IF NOT EXISTS idx_orders_status_updated
+                    ON orders(status, updated_at DESC);
                 """
             )
             columns = {
@@ -116,6 +189,9 @@ class Storage:
                 "entry_confidence": "REAL NOT NULL DEFAULT 0",
                 "entry_pattern": "TEXT NOT NULL DEFAULT ''",
                 "entry_diagnostics_json": "TEXT NOT NULL DEFAULT '{}'",
+                "protective_order_id": "TEXT NOT NULL DEFAULT ''",
+                "protective_order_link_id": "TEXT NOT NULL DEFAULT ''",
+                "mode": "TEXT NOT NULL DEFAULT 'paper'",
             }.items():
                 if column not in columns:
                     conn.execute(f"ALTER TABLE positions ADD COLUMN {column} {definition}")
@@ -127,9 +203,19 @@ class Storage:
                 "entry_pattern": "TEXT NOT NULL DEFAULT ''",
                 "entry_confidence": "REAL NOT NULL DEFAULT 0",
                 "entry_diagnostics_json": "TEXT NOT NULL DEFAULT '{}'",
+                "mode": "TEXT NOT NULL DEFAULT 'paper'",
             }.items():
                 if column not in trade_columns:
                     conn.execute(f"ALTER TABLE trades ADD COLUMN {column} {definition}")
+            equity_columns = {
+                row["name"]
+                for row in conn.execute("PRAGMA table_info(equity)").fetchall()
+            }
+            if "mode" not in equity_columns:
+                conn.execute("ALTER TABLE equity ADD COLUMN mode TEXT NOT NULL DEFAULT 'paper'")
+            conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_equity_mode_created ON equity(mode, created_at DESC)"
+            )
 
     def insert_position(self, position: Position) -> int:
         with self.connect() as conn:
@@ -138,8 +224,9 @@ class Storage:
                 INSERT INTO positions (
                     symbol, qty, entry_price, notional_usdt, entry_fee_usdt, stop_loss,
                     take_profit, highest_price, opened_at, entry_reason,
-                    entry_confidence, entry_pattern, entry_diagnostics_json, status
-                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'OPEN')
+                    entry_confidence, entry_pattern, entry_diagnostics_json,
+                    protective_order_id, protective_order_link_id, mode, status
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'OPEN')
                 """,
                 (
                     position.symbol,
@@ -155,6 +242,9 @@ class Storage:
                     position.entry_confidence,
                     position.entry_pattern,
                     json.dumps(position.entry_diagnostics, ensure_ascii=False),
+                    position.protective_order_id,
+                    position.protective_order_link_id,
+                    position.mode,
                 ),
             )
             return int(cur.lastrowid)
@@ -170,11 +260,52 @@ class Storage:
                 (highest_price, position_id),
             )
 
-    def open_positions(self) -> list[Position]:
+    def update_position_protective_order(
+        self,
+        position_id: int,
+        order_id: str,
+        order_link_id: str,
+    ) -> None:
         with self.connect() as conn:
-            rows = conn.execute(
-                "SELECT * FROM positions WHERE status='OPEN' ORDER BY opened_at"
-            ).fetchall()
+            conn.execute(
+                """
+                UPDATE positions
+                SET protective_order_id=?, protective_order_link_id=?
+                WHERE id=? AND status='OPEN'
+                """,
+                (order_id, order_link_id, position_id),
+            )
+
+    def update_position_after_partial_sell(
+        self,
+        position_id: int,
+        *,
+        qty: float,
+        notional_usdt: float,
+        entry_fee_usdt: float,
+    ) -> None:
+        with self.connect() as conn:
+            conn.execute(
+                """
+                UPDATE positions
+                SET qty=?, notional_usdt=?, entry_fee_usdt=?,
+                    protective_order_id='', protective_order_link_id=''
+                WHERE id=? AND status='OPEN'
+                """,
+                (qty, notional_usdt, entry_fee_usdt, position_id),
+            )
+
+    def open_positions(self, mode: str | None = None) -> list[Position]:
+        with self.connect() as conn:
+            if mode:
+                rows = conn.execute(
+                    "SELECT * FROM positions WHERE status='OPEN' AND mode=? ORDER BY opened_at",
+                    (mode,),
+                ).fetchall()
+            else:
+                rows = conn.execute(
+                    "SELECT * FROM positions WHERE status='OPEN' ORDER BY opened_at"
+                ).fetchall()
         return [
             Position(
                 id=int(row["id"]),
@@ -191,6 +322,9 @@ class Storage:
                 entry_confidence=float(row["entry_confidence"]),
                 entry_pattern=row["entry_pattern"],
                 entry_diagnostics=_json_or_default(row["entry_diagnostics_json"], {}),
+                protective_order_id=row["protective_order_id"],
+                protective_order_link_id=row["protective_order_link_id"],
+                mode=row["mode"],
             )
             for row in rows
         ]
@@ -203,7 +337,8 @@ class Storage:
                     symbol, side, qty, entry_price, exit_price, gross_pnl,
                     fee_usdt, net_pnl, reason, entry_pattern, entry_confidence,
                     entry_diagnostics_json, opened_at, closed_at
-                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    , mode
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                 """,
                 (
                     trade.symbol,
@@ -220,32 +355,41 @@ class Storage:
                     json.dumps(trade.entry_diagnostics, ensure_ascii=False),
                     trade.opened_at.isoformat() if trade.opened_at else None,
                     trade.closed_at.isoformat() if trade.closed_at else None,
+                    trade.mode,
                 ),
             )
             return int(cur.lastrowid)
 
-    def recent_trades(self, limit: int = 50) -> list[dict[str, Any]]:
+    def recent_trades(self, limit: int = 50, mode: str | None = None) -> list[dict[str, Any]]:
         with self.connect() as conn:
-            rows = conn.execute("SELECT * FROM trades ORDER BY id DESC LIMIT ?", (limit,)).fetchall()
+            if mode:
+                rows = conn.execute(
+                    "SELECT * FROM trades WHERE mode=? ORDER BY id DESC LIMIT ?",
+                    (mode, limit),
+                ).fetchall()
+            else:
+                rows = conn.execute("SELECT * FROM trades ORDER BY id DESC LIMIT ?", (limit,)).fetchall()
         return [dict(row) for row in rows]
 
-    def closed_trades(self, limit: int = 200) -> list[dict[str, Any]]:
+    def closed_trades(self, limit: int = 200, mode: str | None = None) -> list[dict[str, Any]]:
         with self.connect() as conn:
-            rows = conn.execute(
-                """
+            query = """
                 SELECT * FROM trades
                 WHERE side='SELL' AND closed_at IS NOT NULL
-                ORDER BY id DESC
-                LIMIT ?
-                """,
-                (limit,),
-            ).fetchall()
+            """
+            params: tuple[Any, ...]
+            if mode:
+                query += " AND mode=?"
+                params = (mode, limit)
+            else:
+                params = (limit,)
+            query += " ORDER BY id DESC LIMIT ?"
+            rows = conn.execute(query, params).fetchall()
         return [dict(row) for row in rows]
 
-    def closed_trade_summary(self) -> dict[str, Any]:
+    def closed_trade_summary(self, mode: str | None = None) -> dict[str, Any]:
         with self.connect() as conn:
-            row = conn.execute(
-                """
+            query = """
                 SELECT
                     COUNT(*) AS trades,
                     COALESCE(SUM(net_pnl), 0) AS net_pnl,
@@ -255,8 +399,12 @@ class Storage:
                     COALESCE(SUM(CASE WHEN net_pnl < 0 THEN 1 ELSE 0 END), 0) AS losses
                 FROM trades
                 WHERE side='SELL' AND closed_at IS NOT NULL
-                """
-            ).fetchone()
+            """
+            params: tuple[Any, ...] = ()
+            if mode:
+                query += " AND mode=?"
+                params = (mode,)
+            row = conn.execute(query, params).fetchone()
         trades = int(row["trades"] if row else 0)
         wins = int(row["wins"] if row else 0)
         losses = int(row["losses"] if row else 0)
@@ -270,7 +418,15 @@ class Storage:
             "win_rate": round(wins / trades, 4) if trades else 0.0,
         }
 
-    def insert_signal(self, signal: Signal) -> None:
+    def insert_signal(self, signal: Signal, hold_sample_seconds: int = 0) -> bool:
+        if signal.action == "HOLD" and hold_sample_seconds > 0:
+            fingerprint = f"{signal.action}\0{signal.reason}"
+            now = time.monotonic()
+            sample_key = (signal.symbol, fingerprint)
+            previous = self._last_hold_signal.get(sample_key)
+            if previous is not None and now - previous < hold_sample_seconds:
+                return False
+            self._last_hold_signal[sample_key] = now
         with self.connect() as conn:
             conn.execute(
                 """
@@ -282,26 +438,40 @@ class Storage:
                     signal.action,
                     signal.confidence,
                     signal.reason,
-                    json.dumps(signal.diagnostics, ensure_ascii=False),
+                    _signal_diagnostics_json(signal.diagnostics),
                     signal.created_at.isoformat(),
                 ),
             )
+        return True
 
     def recent_signals(self, limit: int = 80) -> list[dict[str, Any]]:
         with self.connect() as conn:
             rows = conn.execute("SELECT * FROM signals ORDER BY id DESC LIMIT ?", (limit,)).fetchall()
         return [dict(row) for row in rows]
 
-    def insert_equity(self, equity: float, cash: float, exposure: float, drawdown: float) -> None:
+    def insert_equity(
+        self,
+        equity: float,
+        cash: float,
+        exposure: float,
+        drawdown: float,
+        mode: str = "paper",
+    ) -> None:
         with self.connect() as conn:
             conn.execute(
-                "INSERT INTO equity (equity, cash, exposure, drawdown, created_at) VALUES (?, ?, ?, ?, ?)",
-                (equity, cash, exposure, drawdown, utc_now().isoformat()),
+                "INSERT INTO equity (equity, cash, exposure, drawdown, created_at, mode) VALUES (?, ?, ?, ?, ?, ?)",
+                (equity, cash, exposure, drawdown, utc_now().isoformat(), mode),
             )
 
-    def latest_equity(self) -> dict[str, Any] | None:
+    def latest_equity(self, mode: str | None = None) -> dict[str, Any] | None:
         with self.connect() as conn:
-            row = conn.execute("SELECT * FROM equity ORDER BY id DESC LIMIT 1").fetchone()
+            if mode:
+                row = conn.execute(
+                    "SELECT * FROM equity WHERE mode=? ORDER BY id DESC LIMIT 1",
+                    (mode,),
+                ).fetchone()
+            else:
+                row = conn.execute("SELECT * FROM equity ORDER BY id DESC LIMIT 1").fetchone()
         return dict(row) if row else None
 
     def event(self, message: str, level: str = "INFO") -> None:
@@ -376,12 +546,146 @@ class Storage:
         except json.JSONDecodeError:
             return default
 
+    def upsert_order(
+        self,
+        *,
+        client_order_id: str,
+        exchange_order_id: str = "",
+        symbol: str,
+        side: str,
+        order_kind: str,
+        status: str,
+        requested_qty: float = 0.0,
+        requested_notional: float = 0.0,
+        executed_qty: float = 0.0,
+        executed_value: float = 0.0,
+        fee_usdt: float = 0.0,
+        raw: dict[str, Any] | None = None,
+    ) -> None:
+        now = utc_now().isoformat()
+        with self.connect() as conn:
+            conn.execute(
+                """
+                INSERT INTO orders (
+                    client_order_id, exchange_order_id, symbol, side, order_kind,
+                    status, requested_qty, requested_notional, executed_qty,
+                    executed_value, fee_usdt, raw_json, created_at, updated_at
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(client_order_id) DO UPDATE SET
+                    exchange_order_id=excluded.exchange_order_id,
+                    status=excluded.status,
+                    executed_qty=excluded.executed_qty,
+                    executed_value=excluded.executed_value,
+                    fee_usdt=excluded.fee_usdt,
+                    raw_json=excluded.raw_json,
+                    updated_at=excluded.updated_at
+                """,
+                (
+                    client_order_id,
+                    exchange_order_id,
+                    symbol,
+                    side,
+                    order_kind,
+                    status,
+                    requested_qty,
+                    requested_notional,
+                    executed_qty,
+                    executed_value,
+                    fee_usdt,
+                    json.dumps(raw or {}, ensure_ascii=False),
+                    now,
+                    now,
+                ),
+            )
+
+    def recent_orders(self, limit: int = 100) -> list[dict[str, Any]]:
+        with self.connect() as conn:
+            rows = conn.execute(
+                "SELECT * FROM orders ORDER BY id DESC LIMIT ?",
+                (max(1, min(limit, 500)),),
+            ).fetchall()
+        items = []
+        for row in rows:
+            item = dict(row)
+            item["raw"] = _json_or_default(item.pop("raw_json", "{}"), {})
+            items.append(item)
+        return items
+
+    def pending_orders(self) -> list[dict[str, Any]]:
+        terminal = ("Filled", "Cancelled", "Rejected", "PartiallyFilledCanceled", "Deactivated")
+        placeholders = ",".join("?" for _ in terminal)
+        with self.connect() as conn:
+            rows = conn.execute(
+                f"SELECT * FROM orders WHERE status NOT IN ({placeholders}) ORDER BY id",
+                terminal,
+            ).fetchall()
+        return [dict(row) for row in rows]
+
+    def prune(self, retention_days: int) -> dict[str, int]:
+        if retention_days <= 0:
+            return {}
+        cutoff = (utc_now() - timedelta(days=retention_days)).isoformat()
+        deleted: dict[str, int] = {}
+        for table in ("signals", "equity", "events", "llm_advice"):
+            with self.connect() as conn:
+                # Keep write locks short on large runtime databases. Each maintenance
+                # cycle removes at most one bounded batch per table.
+                cursor = conn.execute(
+                    f"""
+                    DELETE FROM {table}
+                    WHERE id IN (
+                        SELECT id FROM {table}
+                        WHERE created_at < ?
+                        ORDER BY id
+                        LIMIT ?
+                    )
+                    """,
+                    (cutoff, PRUNE_BATCH_SIZE),
+                )
+                deleted[table] = max(0, int(cursor.rowcount))
+        return deleted
+
     def clear_all(self) -> None:
         with self.connect() as conn:
-            for table in ("positions", "trades", "signals", "equity", "events", "runtime", "llm_advice"):
+            for table in ("positions", "trades", "signals", "equity", "events", "runtime", "llm_advice", "orders"):
                 conn.execute(f"DELETE FROM {table}")
 
 
+def _signal_diagnostics_json(diagnostics: dict[str, Any]) -> str:
+    compact = dict(diagnostics)
+    forecast = compact.get("forecast")
+    if isinstance(forecast, dict):
+        compact["forecast"] = {
+            key: value for key, value in forecast.items() if key in _STORED_FORECAST_KEYS
+        }
+    encoded = json.dumps(compact, ensure_ascii=False, separators=(",", ":"))
+    size = len(encoded.encode("utf-8"))
+    if size <= MAX_SIGNAL_DIAGNOSTICS_BYTES:
+        return encoded
+
+    fallback = {
+        "truncated": True,
+        "original_size_bytes": size,
+        "strategy_mode": compact.get("strategy_mode"),
+        "trade_mode": compact.get("trade_mode"),
+        "checks": compact.get("checks", {}),
+        "forecast": compact.get("forecast", {}),
+    }
+    encoded = json.dumps(fallback, ensure_ascii=False, separators=(",", ":"))
+    if len(encoded.encode("utf-8")) <= MAX_SIGNAL_DIAGNOSTICS_BYTES:
+        return encoded
+    return json.dumps(
+        {
+            "truncated": True,
+            "original_size_bytes": size,
+            "strategy_mode": compact.get("strategy_mode"),
+            "trade_mode": compact.get("trade_mode"),
+        },
+        ensure_ascii=False,
+        separators=(",", ":"),
+    )
+
+
 def _json_or_default(value: str, default: Any) -> Any:
     try:
         return json.loads(value)

crypto_spot_bot/strategy.py

@@ -678,7 +678,20 @@ def _torch_forecast_entry_signal(
     spread_ok = ticker.spread_percent <= settings.max_spread_percent
     liquidity_ok = ticker.turnover_24h >= settings.min_24h_turnover_usdt
     model_ok = _is_torch_forecast(forecast)
-    quality_gate_ok = forecast.get("quality_gate_passed") is not False
+    manual_quality_override = settings.time_series_manual_quality_override
+    quality_gate_ok = bool(
+        manual_quality_override
+        or (
+            forecast.get("quality_gate_passed") is True
+            if settings.time_series_require_quality_gate
+            else forecast.get("quality_gate_passed") is not False
+        )
+    )
+    model_fresh_ok = (
+        forecast.get("model_fresh") is True
+        if settings.time_series_require_fresh_model
+        else True
+    )
     rebound = _torch_rebound_overlay(
         settings=settings,
         candles=candles or [],
@@ -697,6 +710,7 @@ def _torch_forecast_entry_signal(
         rebound.get("active")
         and model_ok
         and quality_gate_ok
+        and model_fresh_ok
         and bool(forecast.get("usable", False))
         and not bool(forecast.get("block_entry", False))
         and expected_return >= 0.0
@@ -709,6 +723,7 @@ def _torch_forecast_entry_signal(
         and rebound.get("active")
         and missing_torch_model
         and quality_gate_ok
+        and model_fresh_ok
         and not bool(forecast.get("block_entry", False))
         and confidence >= settings.time_series_min_confidence
     )
@@ -733,6 +748,7 @@ def _torch_forecast_entry_signal(
     checks = {
         "torch_model_ok": model_ok,
         "quality_gate_ok": quality_gate_ok,
+        "model_fresh_ok": model_fresh_ok,
         "forecast_usable": bool(forecast.get("usable", False)),
         "forecast_not_blocked": not bool(forecast.get("block_entry", False)),
         "expected_edge_ok": full_edge_ok or probe_edge_ok,
@@ -770,6 +786,10 @@ def _torch_forecast_entry_signal(
         "skill": skill,
         "quality_gate": forecast.get("quality_gate", {}),
         "quality_gate_passed": forecast.get("quality_gate_passed"),
+        "manual_quality_override": manual_quality_override,
+        "model_created_at": forecast.get("model_created_at", ""),
+        "model_age_hours": forecast.get("model_age_hours"),
+        "model_fresh": forecast.get("model_fresh", False),
         "spread_percent": round(ticker.spread_percent, 5),
         "turnover_24h": ticker.turnover_24h,
         "checks": checks,
@@ -926,6 +946,28 @@ def _torch_forecast_exit_signal(
                 diagnostics,
             )
         return Signal(position.symbol, "SELL", 0.94, "torch_forecast: ATR trailing stop hit", diagnostics)
+    if (
+        settings.time_series_require_quality_gate
+        and not settings.time_series_manual_quality_override
+        and forecast.get("quality_gate_passed") is not True
+    ):
+        diagnostics["forecast_exit_blocked_by_quality_gate"] = True
+        return Signal(
+            position.symbol,
+            "HOLD",
+            0.42,
+            "torch_forecast: hold uses only risk exits while quality gate is unavailable",
+            diagnostics,
+        )
+    if settings.time_series_require_fresh_model and forecast.get("model_fresh") is not True:
+        diagnostics["forecast_exit_blocked_by_model_age"] = True
+        return Signal(
+            position.symbol,
+            "HOLD",
+            0.42,
+            "torch_forecast: hold uses only risk exits while model is stale",
+            diagnostics,
+        )
     if not _is_torch_forecast(forecast):
         if rebound_fallback_position:
             hold_seconds = (utc_now() - position.opened_at).total_seconds()

crypto_spot_bot/time_series.py

@@ -4,6 +4,7 @@ import json
 import math
 from bisect import bisect_right
 from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime
 from typing import Any
 
 from crypto_spot_bot.config import Settings
@@ -156,6 +157,9 @@ class TimeSeriesForecast:
     candidates: list[dict[str, Any]] = field(default_factory=list)
     quality_gate_passed: bool | None = None
     quality_gate: dict[str, Any] = field(default_factory=dict)
+    model_created_at: str = ""
+    model_age_hours: float | None = None
+    model_fresh: bool = False
 
     def as_dict(self) -> dict[str, Any]:
         return asdict(self)
@@ -188,6 +192,10 @@ class TimeSeriesForecaster:
             return _empty_forecast(True, "not enough returns for PyTorch forecast")
 
         artifact = self._load_lstm_artifact()
+        model_created_at, model_age_hours, model_fresh = _model_freshness(
+            artifact,
+            self.settings.time_series_model_max_age_hours,
+        )
         quality_gate = self._load_quality_gate()
         quality_gate_passed = _quality_gate_passed(quality_gate)
         entry = _torch_recurrent_entry(symbol, artifact)
@@ -288,6 +296,9 @@ class TimeSeriesForecaster:
                 candidates=[{"model": model, "mae_percent": round(model_mae * 100, 4)}],
                 quality_gate_passed=quality_gate_passed,
                 quality_gate=quality_gate,
+                model_created_at=model_created_at,
+                model_age_hours=model_age_hours,
+                model_fresh=model_fresh,
             )
 
         direct_horizon = _is_direct_horizon(entry)
@@ -350,6 +361,9 @@ class TimeSeriesForecaster:
             candidates=[{"model": model, "mae_percent": round(model_mae * 100, 4)}],
             quality_gate_passed=quality_gate_passed,
             quality_gate=quality_gate,
+            model_created_at=model_created_at,
+            model_age_hours=model_age_hours,
+            model_fresh=model_fresh,
         )
 
     def _load_lstm_artifact(self) -> dict[str, Any]:
@@ -420,6 +434,9 @@ def _empty_forecast(enabled: bool, reason: str) -> TimeSeriesForecast:
         candidates=[],
         quality_gate_passed=None,
         quality_gate={},
+        model_created_at="",
+        model_age_hours=None,
+        model_fresh=False,
     )
 
 
@@ -436,6 +453,20 @@ def _quality_gate_passed(quality_gate: dict[str, Any]) -> bool | None:
     return None
 
 
+def _model_freshness(artifact: dict[str, Any], max_age_hours: float) -> tuple[str, float | None, bool]:
+    raw = str(artifact.get("created_at", "")).strip() if isinstance(artifact, dict) else ""
+    if not raw:
+        return "", None, False
+    try:
+        created_at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except ValueError:
+        return raw, None, False
+    if created_at.tzinfo is None:
+        created_at = created_at.replace(tzinfo=UTC)
+    age_hours = max(0.0, (datetime.now(UTC) - created_at.astimezone(UTC)).total_seconds() / 3600)
+    return raw, round(age_hours, 4), age_hours <= max(0.1, max_age_hours)
+
+
 def _log_returns(closes: list[float]) -> list[float]:
     return [math.log(closes[index] / closes[index - 1]) for index in range(1, len(closes))]
 

crypto_spot_bot/training_coordination.py

@@ -4,6 +4,8 @@ import base64
 import hashlib
 import json
 import os
+import re
+import shutil
 import uuid
 from datetime import UTC
 from datetime import datetime
@@ -20,6 +22,10 @@ ALLOWED_TRAINING_ARTIFACTS = {
 }
 RUNNING_TIMEOUT = timedelta(hours=12)
 ONLINE_WINDOW = timedelta(minutes=3)
+MAX_ARTIFACT_CHUNK_BYTES = 1024 * 1024
+MAX_ARTIFACT_BYTES = 64 * 1024 * 1024
+MAX_ARTIFACT_CHUNKS = 1024
+REQUIRED_MODEL_BUNDLE = set(ALLOWED_TRAINING_ARTIFACTS)
 
 
 class TrainingCoordinator:
@@ -91,6 +97,7 @@ class TrainingCoordinator:
             return {"claimed": True, "job": job, "status": self._public_status(state)}
 
     def save_artifact_chunk(self, job_id: str, payload: dict[str, Any]) -> dict[str, Any]:
+        job_id = _valid_job_id(job_id)
         name = Path(str(payload.get("name") or "")).name
         if name not in ALLOWED_TRAINING_ARTIFACTS:
             raise ValueError(f"artifact is not allowed: {name}")
@@ -99,58 +106,87 @@ class TrainingCoordinator:
         sha256 = str(payload.get("sha256") or "").strip().lower()
         if index < 0 or total <= 0 or index >= total:
             raise ValueError("invalid artifact chunk index")
-        if not sha256:
-            raise ValueError("artifact sha256 is required")
+        if total > MAX_ARTIFACT_CHUNKS:
+            raise ValueError("artifact has too many chunks")
+        if not re.fullmatch(r"[0-9a-f]{64}", sha256):
+            raise ValueError("artifact sha256 is invalid")
         try:
             chunk = base64.b64decode(str(payload.get("data_base64") or ""), validate=True)
         except (ValueError, TypeError) as exc:
             raise ValueError("invalid artifact chunk payload") from exc
+        if not chunk or len(chunk) > MAX_ARTIFACT_CHUNK_BYTES:
+            raise ValueError("artifact chunk size is invalid")
 
-        chunk_dir = self.upload_root / job_id / name
-        chunk_dir.mkdir(parents=True, exist_ok=True)
-        (chunk_dir / f"{index:06d}.part").write_bytes(chunk)
-
-        if not all((chunk_dir / f"{part:06d}.part").is_file() for part in range(total)):
-            return {"complete": False, "received": index + 1, "total": total}
-
-        target_tmp = self.runtime_dir / f".{name}.{job_id}.tmp"
-        digest = hashlib.sha256()
-        with target_tmp.open("wb") as output:
-            for part in range(total):
-                data = (chunk_dir / f"{part:06d}.part").read_bytes()
-                digest.update(data)
-                output.write(data)
-        if digest.hexdigest().lower() != sha256:
-            target_tmp.unlink(missing_ok=True)
-            raise ValueError("artifact sha256 mismatch")
-
-        self.runtime_dir.mkdir(parents=True, exist_ok=True)
-        os.replace(target_tmp, self.runtime_dir / name)
-        _remove_tree(chunk_dir)
-
-        with self._lock:
-            state = self._load_state()
-            job = self._job_by_id(state, job_id)
-            if job is not None:
-                artifacts = job.setdefault("artifacts", [])
-                artifacts = [item for item in artifacts if item.get("name") != name]
-                artifacts.append({"name": name, "sha256": sha256, "uploaded_at": _now()})
-                job["artifacts"] = artifacts
-                self._save_state(state)
-        return {"complete": True, "name": name, "sha256": sha256}
-
-    def progress(self, job_id: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
-        payload = payload or {}
         with self._lock:
             state = self._load_state()
             job = self._job_by_id(state, job_id)
             if job is None:
                 raise ValueError(f"training job not found: {job_id}")
+            if job.get("status") != "running" or not job.get("claimed_by"):
+                raise ValueError("training job is not claimed and running")
+            uploads = job.setdefault("uploads", {})
+            upload = uploads.setdefault(name, {"sha256": sha256, "total": total})
+            if upload.get("sha256") != sha256 or int(upload.get("total", 0)) != total:
+                raise ValueError("artifact upload metadata changed during upload")
+
+            chunk_dir = self.upload_root / job_id / "chunks" / name
+            chunk_dir.mkdir(parents=True, exist_ok=True)
+            (chunk_dir / f"{index:06d}.part").write_bytes(chunk)
+
+            received = sum(1 for part in range(total) if (chunk_dir / f"{part:06d}.part").is_file())
+            if received < total:
+                upload["received"] = received
+                self._save_state(state)
+                return {"complete": False, "received": received, "total": total}
+
+            ready_dir = self.upload_root / job_id / "ready"
+            ready_dir.mkdir(parents=True, exist_ok=True)
+            target_tmp = ready_dir / f".{name}.tmp"
+            digest = hashlib.sha256()
+            size = 0
+            with target_tmp.open("wb") as output:
+                for part in range(total):
+                    data = (chunk_dir / f"{part:06d}.part").read_bytes()
+                    size += len(data)
+                    if size > MAX_ARTIFACT_BYTES:
+                        target_tmp.unlink(missing_ok=True)
+                        raise ValueError("artifact exceeds maximum size")
+                    digest.update(data)
+                    output.write(data)
+            if digest.hexdigest().lower() != sha256:
+                target_tmp.unlink(missing_ok=True)
+                raise ValueError("artifact sha256 mismatch")
+
+            target = ready_dir / name
+            os.replace(target_tmp, target)
+            _remove_tree(chunk_dir)
+
+            artifacts = job.setdefault("artifacts", [])
+            artifacts = [item for item in artifacts if item.get("name") != name]
+            artifacts.append(
+                {"name": name, "sha256": sha256, "size": size, "staged_at": _now()}
+            )
+            job["artifacts"] = artifacts
+            upload["received"] = total
+            upload["complete"] = True
+            self._save_state(state)
+            return {"complete": True, "staged": True, "name": name, "sha256": sha256}
+
+    def progress(self, job_id: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
+        payload = payload or {}
+        job_id = _valid_job_id(job_id)
+        with self._lock:
+            state = self._load_state()
+            job = self._job_by_id(state, job_id)
+            if job is None:
+                raise ValueError(f"training job not found: {job_id}")
+            if job.get("status") != "running" or not job.get("claimed_by"):
+                raise ValueError("training job is not claimed and running")
             if isinstance(payload.get("worker"), dict):
                 state["worker"] = self._worker_from_payload(payload["worker"])
-            job["status"] = str(payload.get("status") or job.get("status") or "running")
-            job["phase"] = str(payload.get("phase") or job.get("phase") or "running")
-            job["message"] = str(payload.get("message") or job.get("message") or "")
+            job["status"] = "running"
+            job["phase"] = str(payload.get("phase") or job.get("phase") or "running")[:80]
+            job["message"] = str(payload.get("message") or job.get("message") or "")[:2000]
             job["progress_percent"] = _coerce_percent(payload.get("progress_percent"), job.get("progress_percent", 0))
             job["updated_at"] = _now()
             if isinstance(payload.get("details"), dict):
@@ -160,12 +196,18 @@ class TrainingCoordinator:
 
     def complete(self, job_id: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
         payload = payload or {}
+        job_id = _valid_job_id(job_id)
         with self._lock:
             state = self._load_state()
             job = self._job_by_id(state, job_id)
             if job is None:
                 raise ValueError(f"training job not found: {job_id}")
+            if job.get("status") != "running" or not job.get("claimed_by"):
+                raise ValueError("training job is not claimed and running")
             success = bool(payload.get("success", payload.get("status") == "completed"))
+            if success and job.get("artifacts"):
+                promoted = self._validate_and_promote(job_id, job)
+                job["promoted_artifacts"] = promoted
             job["status"] = "completed" if success else "failed"
             job["phase"] = "completed" if success else "failed"
             job["progress_percent"] = 100 if success else _coerce_percent(payload.get("progress_percent"), job.get("progress_percent", 0))
@@ -176,6 +218,65 @@ class TrainingCoordinator:
             self._save_state(state)
             return {"ok": True, "job": job, "status": self._public_status(state)}
 
+    def _validate_and_promote(self, job_id: str, job: dict[str, Any]) -> list[dict[str, Any]]:
+        ready_dir = self.upload_root / job_id / "ready"
+        staged = {path.name for path in ready_dir.iterdir() if path.is_file()} if ready_dir.is_dir() else set()
+        missing = REQUIRED_MODEL_BUNDLE - staged
+        if missing:
+            raise ValueError("training bundle is incomplete: " + ", ".join(sorted(missing)))
+
+        model = _read_json(ready_dir / "lstm_forecaster.json")
+        guard = _read_json(ready_dir / "torch_retrain_guard.json")
+        calibration = _read_json(ready_dir / "torch_threshold_calibration.json")
+        if model.get("type") != "pytorch_recurrent_forecaster":
+            raise ValueError("candidate model type is invalid")
+        symbols = model.get("symbols")
+        if not isinstance(symbols, dict) or not symbols:
+            raise ValueError("candidate model has no symbol models")
+        _validate_symbol_models(symbols)
+        model_sha256 = hashlib.sha256((ready_dir / "lstm_forecaster.json").read_bytes()).hexdigest()
+        if calibration.get("artifact_sha256") != model_sha256:
+            raise ValueError("candidate calibration is not bound to the uploaded model")
+        if not bool(guard.get("accepted")):
+            raise ValueError("candidate retrain guard did not accept the model")
+        if guard.get("candidate_artifact_sha256") != model_sha256:
+            raise ValueError("candidate guard is not bound to the uploaded model")
+        validation = calibration.get("validation")
+        if not isinstance(validation, dict) or not _validation_passed(validation):
+            raise ValueError("candidate quality gate did not pass")
+        if validation.get("protocol") != "untouched_model_holdout_with_threshold_walk_forward":
+            raise ValueError("candidate validation protocol is not an untouched holdout")
+
+        self.runtime_dir.mkdir(parents=True, exist_ok=True)
+        backup_dir = self.runtime_dir / ".model_backups" / f"{_compact_now()}-{job_id}"
+        backup_dir.mkdir(parents=True, exist_ok=True)
+        for name in sorted(REQUIRED_MODEL_BUNDLE):
+            current = self.runtime_dir / name
+            if current.is_file():
+                shutil.copy2(current, backup_dir / name)
+
+        promoted: list[dict[str, Any]] = []
+        artifact_rows = {
+            str(item.get("name")): item
+            for item in job.get("artifacts", [])
+            if isinstance(item, dict)
+        }
+        for name in sorted(REQUIRED_MODEL_BUNDLE):
+            staged_path = ready_dir / name
+            target_tmp = self.runtime_dir / f".{name}.{job_id}.promote"
+            shutil.copy2(staged_path, target_tmp)
+            os.replace(target_tmp, self.runtime_dir / name)
+            row = artifact_rows.get(name, {})
+            promoted.append(
+                {
+                    "name": name,
+                    "sha256": row.get("sha256", ""),
+                    "promoted_at": _now(),
+                }
+            )
+        _remove_tree(self.upload_root / job_id)
+        return promoted
+
     def _load_state(self) -> dict[str, Any]:
         try:
             data = json.loads(self.state_path.read_text(encoding="utf-8"))
@@ -262,8 +363,97 @@ class TrainingCoordinator:
 def _safe_parameters(value: Any) -> dict[str, Any]:
     if not isinstance(value, dict):
         return {}
-    allowed = {"symbols", "limit", "lookbacks", "architectures", "hidden_sizes", "layers", "dropouts", "epochs"}
-    return {key: value[key] for key in allowed if key in value}
+    allowed = {
+        "symbols",
+        "limit",
+        "lookbacks",
+        "architectures",
+        "hidden_sizes",
+        "layers",
+        "dropouts",
+        "epochs",
+        "holdout_window",
+        "resume_candidate",
+    }
+    result = {key: value[key] for key in allowed if key in value}
+    for key, low, high in (
+        ("limit", 500, 5000),
+        ("epochs", 1, 200),
+        ("holdout_window", 64, 1000),
+    ):
+        if key not in result:
+            continue
+        try:
+            result[key] = max(low, min(high, int(result[key])))
+        except (TypeError, ValueError):
+            result.pop(key, None)
+    if "symbols" in result:
+        symbols = [
+            item.strip().upper()
+            for item in str(result["symbols"]).split(",")
+            if re.fullmatch(r"[A-Z0-9]{3,20}", item.strip().upper())
+        ]
+        result["symbols"] = ",".join(symbols[:30])
+    if "architectures" in result:
+        architectures = [
+            item.strip().lower()
+            for item in str(result["architectures"]).split(",")
+            if item.strip().lower() in {"lstm", "gru"}
+        ]
+        result["architectures"] = ",".join(architectures) or "lstm,gru"
+    for key in ("lookbacks", "hidden_sizes", "layers", "dropouts"):
+        if key in result:
+            result[key] = str(result[key])[:200]
+    if "resume_candidate" in result:
+        result["resume_candidate"] = result["resume_candidate"] is True
+    return result
+
+
+def _valid_job_id(value: str) -> str:
+    try:
+        return str(uuid.UUID(str(value)))
+    except (ValueError, AttributeError, TypeError) as exc:
+        raise ValueError("invalid training job id") from exc
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise ValueError(f"invalid training artifact: {path.name}") from exc
+    if not isinstance(data, dict):
+        raise ValueError(f"invalid training artifact: {path.name}")
+    return data
+
+
+def _validation_passed(validation: dict[str, Any]) -> bool:
+    if "passed" in validation:
+        return bool(validation.get("passed"))
+    return str(validation.get("status", "")).strip().lower() in {"pass", "passed", "ok"}
+
+
+def _validate_symbol_models(symbols: dict[str, Any]) -> None:
+    for symbol, entry in symbols.items():
+        if not isinstance(entry, dict):
+            raise ValueError(f"candidate model entry is invalid: {symbol}")
+        if entry.get("model") not in {"torch_lstm", "torch_gru"}:
+            raise ValueError(f"candidate model architecture is invalid: {symbol}")
+        try:
+            lookback = int(entry.get("lookback", 0))
+            input_size = int(entry.get("input_size", 0))
+            hidden_size = int(entry.get("hidden_size", 0))
+        except (TypeError, ValueError) as exc:
+            raise ValueError(f"candidate model dimensions are invalid: {symbol}") from exc
+        if not 4 <= lookback <= 512 or not 1 <= input_size <= 256 or not 1 <= hidden_size <= 1024:
+            raise ValueError(f"candidate model dimensions are out of range: {symbol}")
+        if not isinstance(entry.get("state_dict"), dict):
+            raise ValueError(f"candidate recurrent state is missing: {symbol}")
+        if not isinstance(entry.get("head_weight"), list) or not isinstance(entry.get("head_bias"), list):
+            raise ValueError(f"candidate forecast head is missing: {symbol}")
+
+
+def _compact_now() -> str:
+    return datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
 
 
 def _latest_job(state: dict[str, Any]) -> dict[str, Any] | None: