sevenhill/TradeBot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: collect Bybit orderbook observations for training

Browse Source
This commit is contained in:
Курнат Андрей
2026-07-15 00:36:56 +03:00
parent d0869b5d29
commit 2967cd607c
14 changed files with 374 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crypto_spot_bot/__init__.py
+1 -1
View File
@@ -1,3 +1,3 @@
"""Crypto spot trading bot package."""
__version__ = "1.0.1"
__version__ = "1.0.2"
crypto_spot_bot/bybit.py
+7 -1
View File
@@ -220,6 +220,10 @@ class BybitClient:
return candles
def orderbook_top(self, symbol: str) -> tuple[float, float]:
bid, _bid_size, ask, _ask_size = self.orderbook_level_one(symbol)
return bid, ask
def orderbook_level_one(self, symbol: str) -> tuple[float, float, float, float]:
result = self.public_get(
"/v5/market/orderbook",
{"category": "spot", "symbol": symbol, "limit": 1},
@@ -227,8 +231,10 @@ class BybitClient:
bids = result.get("b") or []
asks = result.get("a") or []
bid = _float(bids[0][0]) if bids else 0.0
bid_size = _float(bids[0][1]) if bids and len(bids[0]) > 1 else 0.0
ask = _float(asks[0][0]) if asks else 0.0
return bid, ask
ask_size = _float(asks[0][1]) if asks and len(asks[0]) > 1 else 0.0
return bid, bid_size, ask, ask_size
def place_spot_market_order(
self,
crypto_spot_bot/config.py
+8
View File
@@ -172,6 +172,8 @@ class Settings:
bybit_rest_base_url_override: str = ""
bybit_websocket_url_override: str = ""
time_series_fallback_mode: str = "trend_macd"
market_observation_enabled: bool = True
market_observation_sample_seconds: float = 30.0
@property
def rest_base_url(self) -> str:
@@ -358,6 +360,10 @@ def load_settings(env_file: str | Path | None = None) -> Settings:
time_series_fallback_mode=os.getenv(
"TIME_SERIES_FALLBACK_MODE", "trend_macd"
).strip().lower(),
market_observation_enabled=_bool_env("MARKET_OBSERVATION_ENABLED", True),
market_observation_sample_seconds=_float_env(
"MARKET_OBSERVATION_SAMPLE_SECONDS", 30.0
),
)
_validate_settings(settings)
if settings.trading_mode == "live" and not settings.live_ready:
@@ -396,6 +402,8 @@ def _validate_settings(settings: Settings) -> None:
errors.append("LIVE_RECONCILIATION_INTERVAL_SECONDS must be positive")
if settings.time_series_fallback_mode not in {"trend_macd", "legacy"}:
errors.append("TIME_SERIES_FALLBACK_MODE must be trend_macd or legacy")
if settings.market_observation_sample_seconds <= 0:
errors.append("MARKET_OBSERVATION_SAMPLE_SECONDS must be positive")
if errors:
raise ValueError("; ".join(errors))
crypto_spot_bot/dashboard.py
+23
View File
@@ -149,6 +149,27 @@ def create_app(settings: Settings | None = None) -> FastAPI:
async def training_status(_: None = Depends(authorizer.require)) -> dict[str, Any]:
return training.status()
@app.get("/api/training/market-observations")
async def training_market_observations(
symbol: str,
after_id: int = 0,
limit: int = 5000,
_: None = Depends(authorizer.require_training),
) -> dict[str, Any]:
normalized_symbol = symbol.strip().upper()
if not normalized_symbol:
raise HTTPException(status_code=400, detail="symbol is required")
items = storage.market_observations_after(
symbol=normalized_symbol,
after_id=max(0, after_id),
limit=max(1, min(limit, 5000)),
)
return {
"symbol": normalized_symbol,
"items": items,
"next_after_id": int(items[-1]["id"]) if items else max(0, after_id),
}
@app.post("/api/training/retrain")
async def training_retrain(
payload: dict[str, Any] | None = None,
@@ -407,6 +428,8 @@ def _safe_config(settings: Settings) -> dict[str, Any]:
"time_series_require_fresh_model": settings.time_series_require_fresh_model,
"time_series_model_max_age_hours": settings.time_series_model_max_age_hours,
"market_ticker_max_age_seconds": settings.market_ticker_max_age_seconds,
"market_observation_enabled": settings.market_observation_enabled,
"market_observation_sample_seconds": settings.market_observation_sample_seconds,
"time_series_model_artifact": _time_series_model_artifact(settings),
"stop_loss_percent": settings.stop_loss_percent,
"stop_loss_exit_enabled": settings.stop_loss_exit_enabled,
crypto_spot_bot/market_data.py
+116 -15
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import json
import threading
import time
from dataclasses import asdict
from datetime import datetime
from typing import Any
@@ -51,6 +52,7 @@ class MarketData:
self.candles: dict[str, list[Candle]] = {}
self.trend_candles: dict[str, list[Candle]] = {}
self.orderbook_top: dict[str, tuple[float, float]] = {}
self.orderbook_metrics: dict[str, dict[str, Any]] = {}
self.patterns: dict[str, dict[str, Any]] = {}
self.forecasts: dict[str, dict[str, Any]] = {}
self.last_rest_refresh_at: datetime | None = None
@@ -60,6 +62,11 @@ class MarketData:
self._refresh_lock = threading.Lock()
self.rest_error_count = 0
self.last_rest_error = ""
self.observation_samples = 0
self.last_observation_at: datetime | None = None
self.observation_error_count = 0
self.last_observation_error = ""
self._last_observation_monotonic: dict[str, float] = {}
async def bootstrap(self) -> None:
self.instruments = await asyncio.to_thread(self.client.instruments)
@@ -112,19 +119,8 @@ class MarketData:
trend_candles = _closed_candles(trend_candles, self.settings.trend_interval)
add_indicators(trend_candles)
self.trend_candles[symbol] = trend_candles
bid, ask = self.client.orderbook_top(symbol)
self.orderbook_top[symbol] = (bid, ask)
if symbol in self.tickers:
current = self.tickers[symbol]
self.tickers[symbol] = Ticker(
symbol=current.symbol,
last_price=current.last_price,
bid=bid or current.bid,
ask=ask or current.ask,
turnover_24h=current.turnover_24h,
volume_24h=current.volume_24h,
change_24h=current.change_24h,
)
bid, bid_size, ask, ask_size = self.client.orderbook_level_one(symbol)
self._update_orderbook(symbol, bid, bid_size, ask, ask_size)
except Exception as exc:
self.rest_error_count += 1
self.last_rest_error = str(exc)
@@ -180,7 +176,11 @@ class MarketData:
elif topic.startswith("orderbook.") and isinstance(data, dict):
parts = topic.split(".")
if len(parts) >= 3:
self._handle_orderbook(parts[2], data)
self._handle_orderbook(
parts[2],
data,
source_timestamp_ms=int(_float(message.get("ts"))),
)
def _handle_ticker(self, symbol: str, data: dict[str, Any]) -> None:
current = self.tickers.get(symbol)
@@ -222,14 +222,66 @@ class MarketData:
add_indicators(candles)
self.candles[symbol] = candles
def _handle_orderbook(self, symbol: str, data: dict[str, Any]) -> None:
def _handle_orderbook(
self,
symbol: str,
data: dict[str, Any],
source_timestamp_ms: int = 0,
) -> None:
bids = data.get("b") or []
asks = data.get("a") or []
bid = _float(bids[0][0]) if bids else 0.0
bid_size = _float(bids[0][1]) if bids and len(bids[0]) > 1 else 0.0
ask = _float(asks[0][0]) if asks else 0.0
ask_size = _float(asks[0][1]) if asks and len(asks[0]) > 1 else 0.0
self._update_orderbook(
symbol,
bid,
bid_size,
ask,
ask_size,
source_timestamp_ms=source_timestamp_ms,
)
def _update_orderbook(
self,
symbol: str,
bid: float,
bid_size: float,
ask: float,
ask_size: float,
*,
source_timestamp_ms: int = 0,
) -> None:
if bid > 0 and ask > 0:
self.orderbook_top[symbol] = (bid, ask)
current = self.tickers.get(symbol)
size_total = max(0.0, bid_size) + max(0.0, ask_size)
mid_price = (bid + ask) / 2.0
imbalance = (
(max(0.0, bid_size) - max(0.0, ask_size)) / size_total
if size_total > 0
else 0.0
)
microprice = (
(ask * max(0.0, bid_size) + bid * max(0.0, ask_size)) / size_total
if size_total > 0
else mid_price
)
observed_at = utc_now()
metrics = {
"bid_price": bid,
"bid_size": max(0.0, bid_size),
"ask_price": ask,
"ask_size": max(0.0, ask_size),
"mid_price": mid_price,
"microprice": microprice,
"spread_bps": ((ask - bid) / mid_price) * 10_000 if mid_price > 0 else 0.0,
"imbalance": imbalance,
"source_timestamp_ms": max(0, source_timestamp_ms),
"observed_at": observed_at.isoformat(),
}
self.orderbook_metrics[symbol] = metrics
if current:
self.tickers[symbol] = Ticker(
symbol=symbol,
@@ -240,6 +292,44 @@ class MarketData:
volume_24h=current.volume_24h,
change_24h=current.change_24h,
)
self._sample_orderbook(symbol, metrics, current.last_price if current else mid_price, observed_at)
def _sample_orderbook(
self,
symbol: str,
metrics: dict[str, Any],
last_price: float,
observed_at: datetime,
) -> None:
if not self.settings.market_observation_enabled:
return
now = time.monotonic()
previous = self._last_observation_monotonic.get(symbol)
if previous is not None and now - previous < self.settings.market_observation_sample_seconds:
return
try:
self.storage.insert_market_observation(
symbol=symbol,
bid_price=float(metrics["bid_price"]),
bid_size=float(metrics["bid_size"]),
ask_price=float(metrics["ask_price"]),
ask_size=float(metrics["ask_size"]),
mid_price=float(metrics["mid_price"]),
microprice=float(metrics["microprice"]),
spread_bps=float(metrics["spread_bps"]),
imbalance=float(metrics["imbalance"]),
last_price=last_price,
source_timestamp_ms=int(metrics["source_timestamp_ms"]),
created_at=observed_at,
)
except Exception as exc: # Storage errors must not disconnect market data.
self.observation_error_count += 1
self.last_observation_error = str(exc)
return
self._last_observation_monotonic[symbol] = now
self.observation_samples += 1
self.last_observation_at = observed_at
self.last_observation_error = ""
def prices(self) -> dict[str, float]:
return {symbol: ticker.last_price for symbol, ticker in self.tickers.items()}
@@ -286,6 +376,16 @@ class MarketData:
"last_ws_message_at": self.last_ws_message_at.isoformat()
if self.last_ws_message_at
else None,
"observation_collector": {
"enabled": self.settings.market_observation_enabled,
"sample_seconds": self.settings.market_observation_sample_seconds,
"samples_since_start": self.observation_samples,
"last_observation_at": self.last_observation_at.isoformat()
if self.last_observation_at
else None,
"error_count": self.observation_error_count,
"last_error": self.last_observation_error,
},
"markets": [
{
"ticker": self.tickers[symbol].as_dict() if symbol in self.tickers else None,
@@ -293,6 +393,7 @@ class MarketData:
"trend_candles": [candle.as_dict() for candle in self.trend_candles.get(symbol, [])[-5:]],
"pattern": self.patterns.get(symbol),
"forecast": self.forecasts.get(symbol),
"orderbook": self.orderbook_metrics.get(symbol),
"quality": analyze_symbol_quality(
symbol=symbol,
candles=self.candles.get(symbol, []),
crypto_spot_bot/storage.py
+97 -3
View File
@@ -4,7 +4,7 @@ import json
import sqlite3
import time
from contextlib import contextmanager
from datetime import timedelta
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Iterator
@@ -18,6 +18,7 @@ MAX_RUNTIME_ROWS = {
"equity": 100_000,
"events": 20_000,
"llm_advice": 20_000,
"market_observations": 1_200_000,
}
_STORED_FORECAST_KEYS = {
"enabled",
@@ -171,6 +172,21 @@ class Storage:
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS market_observations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
symbol TEXT NOT NULL,
bid_price REAL NOT NULL,
bid_size REAL NOT NULL,
ask_price REAL NOT NULL,
ask_size REAL NOT NULL,
mid_price REAL NOT NULL,
microprice REAL NOT NULL,
spread_bps REAL NOT NULL,
imbalance REAL NOT NULL,
last_price REAL NOT NULL,
source_timestamp_ms INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_positions_status_opened
ON positions(status, opened_at);
CREATE INDEX IF NOT EXISTS idx_trades_closed
@@ -183,6 +199,10 @@ class Storage:
ON events(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_orders_status_updated
ON orders(status, updated_at DESC);
CREATE INDEX IF NOT EXISTS idx_market_observations_symbol_id
ON market_observations(symbol, id);
CREATE INDEX IF NOT EXISTS idx_market_observations_created
ON market_observations(created_at);
"""
)
columns = {
@@ -458,6 +478,70 @@ class Storage:
rows = conn.execute("SELECT * FROM signals ORDER BY id DESC LIMIT ?", (limit,)).fetchall()
return [dict(row) for row in rows]
def insert_market_observation(
self,
*,
symbol: str,
bid_price: float,
bid_size: float,
ask_price: float,
ask_size: float,
mid_price: float,
microprice: float,
spread_bps: float,
imbalance: float,
last_price: float,
source_timestamp_ms: int = 0,
created_at: datetime | None = None,
) -> int:
timestamp = (created_at or utc_now()).isoformat()
with self.connect() as conn:
cursor = conn.execute(
"""
INSERT INTO market_observations (
symbol, bid_price, bid_size, ask_price, ask_size,
mid_price, microprice, spread_bps, imbalance, last_price,
source_timestamp_ms, created_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
symbol.upper(),
bid_price,
bid_size,
ask_price,
ask_size,
mid_price,
microprice,
spread_bps,
imbalance,
last_price,
max(0, source_timestamp_ms),
timestamp,
),
)
return int(cursor.lastrowid)
def market_observations_after(
self,
*,
symbol: str,
after_id: int = 0,
limit: int = 5000,
) -> list[dict[str, Any]]:
row_limit = max(1, min(limit, 5000))
with self.connect() as conn:
rows = conn.execute(
"""
SELECT * FROM market_observations
WHERE symbol = ? AND id > ?
ORDER BY id
LIMIT ?
""",
(symbol.upper(), max(0, after_id), row_limit),
).fetchall()
return [dict(row) for row in rows]
def insert_equity(
self,
equity: float,
@@ -635,7 +719,7 @@ class Storage:
return {}
cutoff = (utc_now() - timedelta(days=retention_days)).isoformat()
deleted: dict[str, int] = {}
for table in ("signals", "equity", "events", "llm_advice"):
for table in ("signals", "equity", "events", "llm_advice", "market_observations"):
with self.connect() as conn:
max_id_row = conn.execute(f"SELECT MAX(id) AS value FROM {table}").fetchone()
max_id = int(max_id_row["value"] or 0) if max_id_row else 0
@@ -676,7 +760,17 @@ class Storage:
def clear_all(self) -> None:
with self.connect() as conn:
for table in ("positions", "trades", "signals", "equity", "events", "runtime", "llm_advice", "orders"):
for table in (
"positions",
"trades",
"signals",
"equity",
"events",
"runtime",
"llm_advice",
"orders",
"market_observations",
):
conn.execute(f"DELETE FROM {table}")