sevenhill/TradeBot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: collect Bybit orderbook observations for training

Browse Source
This commit is contained in:
Курнат Андрей
2026-07-15 00:36:56 +03:00
parent d0869b5d29
commit 2967cd607c
14 changed files with 374 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crypto_spot_bot/market_data.py
+116 -15
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import json
import threading
import time
from dataclasses import asdict
from datetime import datetime
from typing import Any
@@ -51,6 +52,7 @@ class MarketData:
self.candles: dict[str, list[Candle]] = {}
self.trend_candles: dict[str, list[Candle]] = {}
self.orderbook_top: dict[str, tuple[float, float]] = {}
self.orderbook_metrics: dict[str, dict[str, Any]] = {}
self.patterns: dict[str, dict[str, Any]] = {}
self.forecasts: dict[str, dict[str, Any]] = {}
self.last_rest_refresh_at: datetime | None = None
@@ -60,6 +62,11 @@ class MarketData:
self._refresh_lock = threading.Lock()
self.rest_error_count = 0
self.last_rest_error = ""
self.observation_samples = 0
self.last_observation_at: datetime | None = None
self.observation_error_count = 0
self.last_observation_error = ""
self._last_observation_monotonic: dict[str, float] = {}
async def bootstrap(self) -> None:
self.instruments = await asyncio.to_thread(self.client.instruments)
@@ -112,19 +119,8 @@ class MarketData:
trend_candles = _closed_candles(trend_candles, self.settings.trend_interval)
add_indicators(trend_candles)
self.trend_candles[symbol] = trend_candles
bid, ask = self.client.orderbook_top(symbol)
self.orderbook_top[symbol] = (bid, ask)
if symbol in self.tickers:
current = self.tickers[symbol]
self.tickers[symbol] = Ticker(
symbol=current.symbol,
last_price=current.last_price,
bid=bid or current.bid,
ask=ask or current.ask,
turnover_24h=current.turnover_24h,
volume_24h=current.volume_24h,
change_24h=current.change_24h,
)
bid, bid_size, ask, ask_size = self.client.orderbook_level_one(symbol)
self._update_orderbook(symbol, bid, bid_size, ask, ask_size)
except Exception as exc:
self.rest_error_count += 1
self.last_rest_error = str(exc)
@@ -180,7 +176,11 @@ class MarketData:
elif topic.startswith("orderbook.") and isinstance(data, dict):
parts = topic.split(".")
if len(parts) >= 3:
self._handle_orderbook(parts[2], data)
self._handle_orderbook(
parts[2],
data,
source_timestamp_ms=int(_float(message.get("ts"))),
)
def _handle_ticker(self, symbol: str, data: dict[str, Any]) -> None:
current = self.tickers.get(symbol)
@@ -222,14 +222,66 @@ class MarketData:
add_indicators(candles)
self.candles[symbol] = candles
def _handle_orderbook(self, symbol: str, data: dict[str, Any]) -> None:
def _handle_orderbook(
self,
symbol: str,
data: dict[str, Any],
source_timestamp_ms: int = 0,
) -> None:
bids = data.get("b") or []
asks = data.get("a") or []
bid = _float(bids[0][0]) if bids else 0.0
bid_size = _float(bids[0][1]) if bids and len(bids[0]) > 1 else 0.0
ask = _float(asks[0][0]) if asks else 0.0
ask_size = _float(asks[0][1]) if asks and len(asks[0]) > 1 else 0.0
self._update_orderbook(
symbol,
bid,
bid_size,
ask,
ask_size,
source_timestamp_ms=source_timestamp_ms,
)
def _update_orderbook(
self,
symbol: str,
bid: float,
bid_size: float,
ask: float,
ask_size: float,
*,
source_timestamp_ms: int = 0,
) -> None:
if bid > 0 and ask > 0:
self.orderbook_top[symbol] = (bid, ask)
current = self.tickers.get(symbol)
size_total = max(0.0, bid_size) + max(0.0, ask_size)
mid_price = (bid + ask) / 2.0
imbalance = (
(max(0.0, bid_size) - max(0.0, ask_size)) / size_total
if size_total > 0
else 0.0
)
microprice = (
(ask * max(0.0, bid_size) + bid * max(0.0, ask_size)) / size_total
if size_total > 0
else mid_price
)
observed_at = utc_now()
metrics = {
"bid_price": bid,
"bid_size": max(0.0, bid_size),
"ask_price": ask,
"ask_size": max(0.0, ask_size),
"mid_price": mid_price,
"microprice": microprice,
"spread_bps": ((ask - bid) / mid_price) * 10_000 if mid_price > 0 else 0.0,
"imbalance": imbalance,
"source_timestamp_ms": max(0, source_timestamp_ms),
"observed_at": observed_at.isoformat(),
}
self.orderbook_metrics[symbol] = metrics
if current:
self.tickers[symbol] = Ticker(
symbol=symbol,
@@ -240,6 +292,44 @@ class MarketData:
volume_24h=current.volume_24h,
change_24h=current.change_24h,
)
self._sample_orderbook(symbol, metrics, current.last_price if current else mid_price, observed_at)
def _sample_orderbook(
self,
symbol: str,
metrics: dict[str, Any],
last_price: float,
observed_at: datetime,
) -> None:
if not self.settings.market_observation_enabled:
return
now = time.monotonic()
previous = self._last_observation_monotonic.get(symbol)
if previous is not None and now - previous < self.settings.market_observation_sample_seconds:
return
try:
self.storage.insert_market_observation(
symbol=symbol,
bid_price=float(metrics["bid_price"]),
bid_size=float(metrics["bid_size"]),
ask_price=float(metrics["ask_price"]),
ask_size=float(metrics["ask_size"]),
mid_price=float(metrics["mid_price"]),
microprice=float(metrics["microprice"]),
spread_bps=float(metrics["spread_bps"]),
imbalance=float(metrics["imbalance"]),
last_price=last_price,
source_timestamp_ms=int(metrics["source_timestamp_ms"]),
created_at=observed_at,
)
except Exception as exc: # Storage errors must not disconnect market data.
self.observation_error_count += 1
self.last_observation_error = str(exc)
return
self._last_observation_monotonic[symbol] = now
self.observation_samples += 1
self.last_observation_at = observed_at
self.last_observation_error = ""
def prices(self) -> dict[str, float]:
return {symbol: ticker.last_price for symbol, ticker in self.tickers.items()}
@@ -286,6 +376,16 @@ class MarketData:
"last_ws_message_at": self.last_ws_message_at.isoformat()
if self.last_ws_message_at
else None,
"observation_collector": {
"enabled": self.settings.market_observation_enabled,
"sample_seconds": self.settings.market_observation_sample_seconds,
"samples_since_start": self.observation_samples,
"last_observation_at": self.last_observation_at.isoformat()
if self.last_observation_at
else None,
"error_count": self.observation_error_count,
"last_error": self.last_observation_error,
},
"markets": [
{
"ticker": self.tickers[symbol].as_dict() if symbol in self.tickers else None,
@@ -293,6 +393,7 @@ class MarketData:
"trend_candles": [candle.as_dict() for candle in self.trend_candles.get(symbol, [])[-5:]],
"pattern": self.patterns.get(symbol),
"forecast": self.forecasts.get(symbol),
"orderbook": self.orderbook_metrics.get(symbol),
"quality": analyze_symbol_quality(
symbol=symbol,
candles=self.candles.get(symbol, []),