Harden trading, training, and monitoring

crypto_spot_bot/execution.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from collections import deque
 from datetime import timedelta
 from decimal import Decimal, ROUND_DOWN, ROUND_UP
-from typing import Iterable
+from typing import Any, Iterable
 from uuid import uuid4
 
 from crypto_spot_bot.bybit import BybitClient, Instrument
@@ -38,9 +38,19 @@ class PaperBroker:
     def __init__(self, settings: Settings, storage: Storage):
         self.settings = settings
         self.storage = storage
-        self.positions = storage.open_positions()
+        self.positions = storage.open_positions(settings.trading_mode)
         self.cash = float(storage.get_runtime("paper_cash", settings.starting_balance_usdt))
-        self.peak_equity = float(storage.get_runtime("peak_equity", settings.starting_balance_usdt))
+        today = utc_now().date().isoformat()
+        stored_peak_day = str(storage.get_runtime("paper_peak_equity_day", ""))
+        self.peak_equity_day = today
+        self.peak_equity = float(
+            storage.get_runtime("paper_daily_peak_equity", settings.starting_balance_usdt)
+            if stored_peak_day == today
+            else settings.starting_balance_usdt
+        )
+        self.lifetime_peak_equity = float(
+            storage.get_runtime("paper_lifetime_peak_equity", settings.starting_balance_usdt)
+        )
         self._entry_timestamps = deque()
 
     def open_positions(self) -> list[Position]:
@@ -64,11 +74,24 @@ class PaperBroker:
     def mark_equity(self, prices: dict[str, float]) -> dict[str, float]:
         state = self.account_state(prices)
         equity = state["equity"]
+        today = utc_now().date().isoformat()
+        if today != self.peak_equity_day:
+            self.peak_equity_day = today
+            self.peak_equity = equity
         self.peak_equity = max(self.peak_equity, equity)
+        self.lifetime_peak_equity = max(self.lifetime_peak_equity, equity)
         state["drawdown"] = max(0.0, self.peak_equity - equity)
         self.storage.set_runtime("paper_cash", self.cash)
-        self.storage.set_runtime("peak_equity", self.peak_equity)
-        self.storage.insert_equity(equity, self.cash, self.exposure(), state["drawdown"])
+        self.storage.set_runtime("paper_peak_equity_day", self.peak_equity_day)
+        self.storage.set_runtime("paper_daily_peak_equity", self.peak_equity)
+        self.storage.set_runtime("paper_lifetime_peak_equity", self.lifetime_peak_equity)
+        self.storage.insert_equity(
+            equity,
+            self.cash,
+            self.exposure(),
+            state["drawdown"],
+            mode=self.settings.trading_mode,
+        )
         return state
 
     def account_state(self, prices: dict[str, float]) -> dict[str, float]:
@@ -181,6 +204,7 @@ class PaperBroker:
             entry_confidence=signal.confidence,
             entry_pattern=str(signal.diagnostics.get("pattern", {}).get("label", "")),
             entry_diagnostics=signal.diagnostics,
+            mode=self.settings.trading_mode,
         )
         position.id = self.storage.insert_position(position)
         self.positions.append(position)
@@ -201,6 +225,7 @@ class PaperBroker:
                 entry_confidence=position.entry_confidence,
                 entry_diagnostics=position.entry_diagnostics,
                 opened_at=position.opened_at,
+                mode=self.settings.trading_mode,
             )
         )
         self.storage.event(
@@ -244,6 +269,7 @@ class PaperBroker:
             entry_diagnostics=position.entry_diagnostics,
             opened_at=position.opened_at,
             closed_at=utc_now(),
+            mode=self.settings.trading_mode,
         )
         trade.id = self.storage.insert_trade(trade)
         self.storage.event(
@@ -368,11 +394,139 @@ class PaperBroker:
 
 
 class LiveBroker(PaperBroker):
+    TERMINAL_ORDER_STATUSES = {
+        "Filled",
+        "Cancelled",
+        "Rejected",
+        "PartiallyFilledCanceled",
+        "PartillyFilledCancelled",
+        "Deactivated",
+    }
+
     def __init__(self, settings: Settings, storage: Storage, client: BybitClient):
         super().__init__(settings, storage)
         if not settings.live_ready:
             raise BrokerError("Live mode is not unlocked by settings")
         self.client = client
+        self.reconciliation_state: dict[str, Any] = {
+            "status": "unknown",
+            "blocking": True,
+            "discrepancies": ["live account has not been reconciled"],
+        }
+
+    def can_open(
+        self,
+        symbol: str,
+        prices: dict[str, float],
+        requested_notional: float | None = None,
+    ) -> tuple[bool, str]:
+        if self.reconciliation_state.get("blocking", True):
+            return False, "live reconciliation is not clean"
+        return super().can_open(symbol, prices, requested_notional)
+
+    def reconcile(self, instruments: dict[str, Instrument]) -> dict[str, Any]:
+        coins = {"USDT"}
+        for symbol in self.settings.symbols:
+            instrument = instruments.get(symbol)
+            if instrument and instrument.base_coin:
+                coins.add(instrument.base_coin.upper())
+        wallet = self.client.wallet_balance(coin=",".join(sorted(coins)))
+        balances = _wallet_balances(wallet)
+        usdt = balances.get("USDT", {})
+        self.cash = max(0.0, float(usdt.get("wallet_balance", 0.0)) - float(usdt.get("locked", 0.0)))
+
+        local_by_coin: dict[str, float] = {}
+        discrepancies: list[dict[str, Any]] = []
+        for position in self.positions:
+            instrument = instruments.get(position.symbol)
+            coin = instrument.base_coin.upper() if instrument and instrument.base_coin else position.symbol.removesuffix("USDT")
+            local_by_coin[coin] = local_by_coin.get(coin, 0.0) + position.qty
+        for coin, local_qty in local_by_coin.items():
+            remote_qty = float((balances.get(coin) or {}).get("wallet_balance", 0.0))
+            tolerance = max(1e-8, local_qty * 0.002)
+            if remote_qty + tolerance < local_qty:
+                discrepancies.append(
+                    {
+                        "severity": "error",
+                        "code": "remote_balance_below_local_position",
+                        "coin": coin,
+                        "local_qty": round(local_qty, 12),
+                        "remote_qty": round(remote_qty, 12),
+                    }
+                )
+        for coin, row in balances.items():
+            if coin == "USDT" or coin not in coins:
+                continue
+            remote_qty = float(row.get("wallet_balance", 0.0))
+            local_qty = local_by_coin.get(coin, 0.0)
+            tolerance = max(1e-8, local_qty * 0.002)
+            if remote_qty > local_qty + tolerance:
+                discrepancies.append(
+                    {
+                        "severity": "error",
+                        "code": "remote_asset_without_matching_local_position",
+                        "coin": coin,
+                        "local_qty": round(local_qty, 12),
+                        "remote_qty": round(remote_qty, 12),
+                    }
+                )
+
+        normal_orders = self.client.realtime_orders(
+            category="spot",
+            open_only=0,
+            limit=50,
+            order_filter="Order",
+        )
+        unresolved_orders = [
+            row
+            for row in normal_orders.get("list", [])
+            if isinstance(row, dict)
+            and str(row.get("orderStatus", "")) not in self.TERMINAL_ORDER_STATUSES
+        ]
+        if unresolved_orders:
+            discrepancies.append(
+                {
+                    "severity": "error",
+                    "code": "unresolved_exchange_orders",
+                    "count": len(unresolved_orders),
+                    "order_ids": [str(row.get("orderId", "")) for row in unresolved_orders[:10]],
+                }
+            )
+
+        protection_rows = self.client.realtime_orders(
+            category="spot",
+            open_only=0,
+            limit=50,
+            order_filter="tpslOrder",
+        )
+        active_protection = {
+            str(row.get("orderId", ""))
+            for row in protection_rows.get("list", [])
+            if isinstance(row, dict)
+            and str(row.get("orderStatus", "")) not in self.TERMINAL_ORDER_STATUSES
+        }
+        if self.settings.live_protective_stop_enabled:
+            for position in self.positions:
+                if not position.protective_order_id or position.protective_order_id not in active_protection:
+                    discrepancies.append(
+                        {
+                            "severity": "error",
+                            "code": "missing_exchange_protective_stop",
+                            "position_id": position.id,
+                            "symbol": position.symbol,
+                        }
+                    )
+
+        blocking = any(row.get("severity") == "error" for row in discrepancies)
+        self.reconciliation_state = {
+            "status": "error" if blocking else ("warn" if discrepancies else "ok"),
+            "blocking": blocking,
+            "discrepancies": discrepancies,
+            "cash_usdt": round(self.cash, 8),
+            "checked_at": utc_now().isoformat(),
+        }
+        self.storage.set_runtime("live_reconciliation", self.reconciliation_state)
+        return dict(self.reconciliation_state)
 
     def buy(
         self,
@@ -400,30 +554,356 @@ class LiveBroker(PaperBroker):
         if budget < max(self.settings.min_position_usdt, minimum_budget):
             self.storage.event(f"{ticker.symbol}: live BUY skipped, adjusted budget below minimum", "WARN")
             return None
+
         signal.diagnostics["position_notional_usdt"] = budget
-        notional = budget / (1 + self.settings.taker_fee_rate)
-        response = self.client.place_spot_market_order(
+        requested_quote = budget / (1 + self.settings.taker_fee_rate)
+        client_order_id = f"tb-buy-{uuid4().hex[:18]}"
+        self.storage.upsert_order(
+            client_order_id=client_order_id,
             symbol=ticker.symbol,
             side="Buy",
-            qty=notional,
-            market_unit="quoteCoin",
-            order_link_id=f"tb-buy-{uuid4().hex[:18]}",
+            order_kind="MARKET",
+            status="PENDING_SUBMIT",
+            requested_notional=requested_quote,
+            raw={"signal": signal.as_dict()},
         )
-        self.storage.event(f"{ticker.symbol}: реальная покупка отправлена orderId={response.get('orderId')}")
-        return self._record_buy(signal, ticker, instrument, "реальная покупка, локальная запись")
+        try:
+            response = self.client.place_spot_market_order(
+                symbol=ticker.symbol,
+                side="Buy",
+                qty=requested_quote,
+                market_unit="quoteCoin",
+                order_link_id=client_order_id,
+            )
+            order_id = str(response.get("orderId", ""))
+            if not order_id:
+                raise BrokerError("Bybit did not return orderId for live BUY")
+            self.storage.upsert_order(
+                client_order_id=client_order_id,
+                exchange_order_id=order_id,
+                symbol=ticker.symbol,
+                side="Buy",
+                order_kind="MARKET",
+                status="ACCEPTED",
+                requested_notional=requested_quote,
+                raw=response,
+            )
+            result = self.client.wait_for_spot_order(
+                order_id=order_id,
+                symbol=ticker.symbol,
+                timeout_seconds=self.settings.live_order_fill_timeout_seconds,
+            )
+            fill = _execution_fill(result, side="Buy", instrument=instrument)
+            self._save_order_fill(client_order_id, order_id, ticker.symbol, "Buy", requested_quote, result, fill)
+            if fill["qty"] <= 0 or fill["value"] <= 0:
+                raise BrokerError(f"live BUY was not filled, status={fill['status']}")
+            position = self._record_live_buy(signal, ticker, fill)
+            if self.settings.live_protective_stop_enabled:
+                try:
+                    self._place_protective_stop(position)
+                except Exception as exc:
+                    self.storage.event(
+                        f"{ticker.symbol}: protective stop placement failed, closing position: {exc}",
+                        "ERROR",
+                    )
+                    self.sell(position, ticker, "protective stop placement failed")
+                    raise BrokerError("live BUY was unwound because protective stop failed") from exc
+            return position
+        except Exception as exc:
+            self.reconciliation_state["blocking"] = True
+            self.reconciliation_state["status"] = "error"
+            self.storage.event(f"{ticker.symbol}: live BUY failed: {exc}", "ERROR")
+            raise
 
     def sell(self, position: Position, ticker: Ticker, reason: str) -> Trade:
+        if position.protective_order_id or position.protective_order_link_id:
+            self.client.cancel_spot_order(
+                symbol=position.symbol,
+                order_id=position.protective_order_id or None,
+                order_link_id=position.protective_order_link_id or None,
+                order_filter="tpslOrder",
+            )
+            if position.protective_order_id:
+                cancelled = self.client.wait_for_spot_order(
+                    order_id=position.protective_order_id,
+                    symbol=position.symbol,
+                    timeout_seconds=min(10.0, self.settings.live_order_fill_timeout_seconds),
+                )
+                status = str((cancelled.get("order") or {}).get("orderStatus", ""))
+                if status and status != "Cancelled":
+                    raise BrokerError(f"protective order was not cancelled, status={status}")
+
+        client_order_id = f"tb-sell-{uuid4().hex[:18]}"
+        self.storage.upsert_order(
+            client_order_id=client_order_id,
+            symbol=position.symbol,
+            side="Sell",
+            order_kind="MARKET",
+            status="PENDING_SUBMIT",
+            requested_qty=position.qty,
+            raw={"position_id": position.id, "reason": reason},
+        )
         response = self.client.place_spot_market_order(
             symbol=position.symbol,
             side="Sell",
             qty=position.qty,
             market_unit="baseCoin",
-            order_link_id=f"tb-sell-{uuid4().hex[:18]}",
+            order_link_id=client_order_id,
+        )
+        order_id = str(response.get("orderId", ""))
+        if not order_id:
+            raise BrokerError("Bybit did not return orderId for live SELL")
+        result = self.client.wait_for_spot_order(
+            order_id=order_id,
+            symbol=position.symbol,
+            timeout_seconds=self.settings.live_order_fill_timeout_seconds,
+        )
+        fill = _execution_fill(result, side="Sell", instrument=None)
+        self._save_order_fill(client_order_id, order_id, position.symbol, "Sell", position.qty, result, fill)
+        if fill["qty"] <= 0 or fill["value"] <= 0:
+            self.reconciliation_state["blocking"] = True
+            raise BrokerError(f"live SELL was not filled, status={fill['status']}")
+        return self._record_live_sell(position, reason, fill)
+
+    def _record_live_buy(self, signal: Signal, ticker: Ticker, fill: dict[str, Any]) -> Position:
+        qty = float(fill["net_qty"])
+        value = float(fill["value"])
+        price = value / max(float(fill["qty"]), 1e-12)
+        fee_usdt = float(fill["fee_usdt"])
+        stop_loss_percent = self._signal_percent(
+            signal, "stop_loss_percent", self.settings.stop_loss_percent, 0.003, 0.08
+        )
+        take_profit_percent = self._signal_percent(
+            signal, "take_profit_percent", self.settings.take_profit_percent, 0.003, 0.20
+        )
+        position = Position(
+            id=None,
+            symbol=ticker.symbol,
+            qty=qty,
+            entry_price=price,
+            notional_usdt=value,
+            entry_fee_usdt=fee_usdt,
+            stop_loss=price * (1 - stop_loss_percent),
+            take_profit=price * (1 + take_profit_percent),
+            highest_price=price,
+            entry_reason=signal.reason,
+            entry_confidence=signal.confidence,
+            entry_pattern=str(signal.diagnostics.get("pattern", {}).get("label", "")),
+            entry_diagnostics=signal.diagnostics,
+            mode="live",
+        )
+        position.id = self.storage.insert_position(position)
+        self.positions.append(position)
+        self._record_entry_timestamp()
+        self.cash = max(0.0, self.cash - value - float(fill["quote_fee"]))
+        self.storage.insert_trade(
+            Trade(
+                id=None,
+                symbol=ticker.symbol,
+                side="BUY",
+                qty=qty,
+                entry_price=price,
+                fee_usdt=fee_usdt,
+                net_pnl=-fee_usdt,
+                reason=signal.reason,
+                entry_pattern=position.entry_pattern,
+                entry_confidence=position.entry_confidence,
+                entry_diagnostics=position.entry_diagnostics,
+                opened_at=position.opened_at,
+                mode="live",
+            )
         )
         self.storage.event(
-            f"{position.symbol}: реальная продажа отправлена orderId={response.get('orderId')} причина={reason}"
+            f"{ticker.symbol}: live BUY filled qty={qty:.8f} avg={price:.8f} value={value:.4f}"
         )
-        return self._record_sell(position, ticker, reason, "реальная продажа, локальная запись")
+        return position
+
+    def _record_live_sell(self, position: Position, reason: str, fill: dict[str, Any]) -> Trade:
+        sold_qty = min(position.qty, float(fill["qty"]))
+        value = float(fill["value"])
+        price = value / max(float(fill["qty"]), 1e-12)
+        exit_fee = float(fill["fee_usdt"])
+        ratio = min(1.0, sold_qty / max(position.qty, 1e-12))
+        allocated_entry_fee = position.entry_fee_usdt * ratio
+        gross_pnl = (price - position.entry_price) * sold_qty
+        net_pnl = gross_pnl - allocated_entry_fee - exit_fee
+        self.cash += value - float(fill["quote_fee"])
+        remaining_qty = max(0.0, position.qty - sold_qty)
+        if remaining_qty <= max(1e-12, position.qty * 1e-6):
+            if position.id is not None:
+                self.storage.close_position(position.id)
+            self.positions = [item for item in self.positions if item.id != position.id]
+        else:
+            remaining_ratio = remaining_qty / position.qty
+            position.qty = remaining_qty
+            position.notional_usdt *= remaining_ratio
+            position.entry_fee_usdt *= remaining_ratio
+            position.protective_order_id = ""
+            position.protective_order_link_id = ""
+            if position.id is not None:
+                self.storage.update_position_after_partial_sell(
+                    position.id,
+                    qty=position.qty,
+                    notional_usdt=position.notional_usdt,
+                    entry_fee_usdt=position.entry_fee_usdt,
+                )
+            self.reconciliation_state["blocking"] = True
+        trade = Trade(
+            id=None,
+            symbol=position.symbol,
+            side="SELL",
+            qty=sold_qty,
+            entry_price=position.entry_price,
+            exit_price=price,
+            gross_pnl=gross_pnl,
+            fee_usdt=allocated_entry_fee + exit_fee,
+            net_pnl=net_pnl,
+            reason=reason,
+            entry_pattern=position.entry_pattern,
+            entry_confidence=position.entry_confidence,
+            entry_diagnostics=position.entry_diagnostics,
+            opened_at=position.opened_at,
+            closed_at=utc_now(),
+            mode="live",
+        )
+        trade.id = self.storage.insert_trade(trade)
+        self.storage.event(
+            f"{position.symbol}: live SELL filled qty={sold_qty:.8f} avg={price:.8f} pnl={net_pnl:.4f} reason={reason}"
+        )
+        return trade
+
+    def _place_protective_stop(self, position: Position) -> None:
+        link_id = f"tb-stop-{uuid4().hex[:17]}"
+        response = self.client.place_spot_protective_stop(
+            symbol=position.symbol,
+            qty=position.qty,
+            trigger_price=position.stop_loss,
+            order_link_id=link_id,
+        )
+        order_id = str(response.get("orderId", ""))
+        if not order_id:
+            raise BrokerError("Bybit did not return orderId for protective stop")
+        position.protective_order_id = order_id
+        position.protective_order_link_id = link_id
+        if position.id is not None:
+            self.storage.update_position_protective_order(position.id, order_id, link_id)
+        self.storage.upsert_order(
+            client_order_id=link_id,
+            exchange_order_id=order_id,
+            symbol=position.symbol,
+            side="Sell",
+            order_kind="PROTECTIVE_STOP",
+            status="ACCEPTED",
+            requested_qty=position.qty,
+            raw=response,
+        )
+
+    def _save_order_fill(
+        self,
+        client_order_id: str,
+        order_id: str,
+        symbol: str,
+        side: str,
+        requested: float,
+        result: dict[str, Any],
+        fill: dict[str, Any],
+    ) -> None:
+        self.storage.upsert_order(
+            client_order_id=client_order_id,
+            exchange_order_id=order_id,
+            symbol=symbol,
+            side=side,
+            order_kind="MARKET",
+            status=str(fill["status"]),
+            requested_qty=requested if side == "Sell" else 0.0,
+            requested_notional=requested if side == "Buy" else 0.0,
+            executed_qty=float(fill["qty"]),
+            executed_value=float(fill["value"]),
+            fee_usdt=float(fill["fee_usdt"]),
+            raw=result,
+        )
+
+
+def _wallet_balances(wallet: dict[str, Any]) -> dict[str, dict[str, float]]:
+    accounts = wallet.get("list")
+    if not isinstance(accounts, list) or not accounts:
+        return {}
+    coins = accounts[0].get("coin") if isinstance(accounts[0], dict) else None
+    if not isinstance(coins, list):
+        return {}
+    result: dict[str, dict[str, float]] = {}
+    for row in coins:
+        if not isinstance(row, dict):
+            continue
+        coin = str(row.get("coin", "")).upper()
+        if not coin:
+            continue
+        result[coin] = {
+            "wallet_balance": _safe_float(row.get("walletBalance")),
+            "equity": _safe_float(row.get("equity")),
+            "locked": _safe_float(row.get("locked")),
+        }
+    return result
+
+
+def _execution_fill(
+    result: dict[str, Any],
+    *,
+    side: str,
+    instrument: Instrument | None,
+) -> dict[str, Any]:
+    order = result.get("order") if isinstance(result.get("order"), dict) else {}
+    executions = result.get("executions") if isinstance(result.get("executions"), list) else []
+    qty = 0.0
+    value = 0.0
+    quote_fee = 0.0
+    base_fee = 0.0
+    fee_usdt = 0.0
+    base_coin = instrument.base_coin.upper() if instrument and instrument.base_coin else ""
+    for row in executions:
+        if not isinstance(row, dict):
+            continue
+        exec_qty = _safe_float(row.get("execQty"))
+        exec_value = _safe_float(row.get("execValue"))
+        exec_price = _safe_float(row.get("execPrice"))
+        fee = max(0.0, _safe_float(row.get("execFee")))
+        fee_currency = str(row.get("feeCurrency", "")).upper()
+        if not base_coin:
+            symbol = str(row.get("symbol", ""))
+            base_coin = symbol.removesuffix("USDT") if symbol.endswith("USDT") else ""
+        qty += exec_qty
+        value += exec_value or exec_qty * exec_price
+        if fee_currency == "USDT" or not fee_currency:
+            quote_fee += fee
+            fee_usdt += fee
+        elif fee_currency == base_coin:
+            base_fee += fee
+            fee_usdt += fee * exec_price
+        else:
+            fee_usdt += fee * exec_price
+    if qty <= 0:
+        qty = _safe_float(order.get("cumExecQty"))
+    if value <= 0:
+        value = _safe_float(order.get("cumExecValue"))
+    if value <= 0 and qty > 0:
+        value = qty * _safe_float(order.get("avgPrice"))
+    net_qty = max(0.0, qty - base_fee) if side == "Buy" else qty
+    return {
+        "status": str(order.get("orderStatus", "Unknown")),
+        "qty": qty,
+        "net_qty": net_qty,
+        "value": value,
+        "quote_fee": quote_fee,
+        "base_fee": base_fee,
+        "fee_usdt": fee_usdt,
+    }
+
+
+def _safe_float(value: Any, default: float = 0.0) -> float:
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return default
 
 
 def prices_from_tickers(tickers: Iterable[Ticker]) -> dict[str, float]: