sevenhill/TradeBot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden trading, training, and monitoring

Browse Source
This commit is contained in:
Codex
2026-07-10 15:51:53 +03:00
parent 6fb79ee2a9
commit 069d75d2f2
55 changed files with 2658 additions and 2332049 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crypto_spot_bot/storage.py
+336 -32
View File
@@ -2,23 +2,61 @@ from __future__ import annotations
import json
import sqlite3
import time
from contextlib import contextmanager
from datetime import timedelta
from pathlib import Path
from typing import Any, Iterator
from crypto_spot_bot.models import Position, Signal, Trade, utc_now
MAX_SIGNAL_DIAGNOSTICS_BYTES = 16 * 1024
PRUNE_BATCH_SIZE = 1000
_STORED_FORECAST_KEYS = {
"enabled",
"usable",
"model",
"volatility_model",
"expected_return_percent",
"expected_price",
"volatility_percent",
"probability_up",
"confidence_adjustment",
"block_entry",
"validation_mae_percent",
"baseline_mae_percent",
"skill",
"horizon",
"reason",
"expected_gross_return_percent",
"quantile_10_percent",
"quantile_50_percent",
"quantile_90_percent",
"conservative_return_percent",
"target_transform",
"horizon_forecasts",
"candidates",
"quality_gate_passed",
"model_created_at",
"model_age_hours",
"model_fresh",
}
class Storage:
def __init__(self, path: str | Path):
self.path = Path(path)
self.path.parent.mkdir(parents=True, exist_ok=True)
self._last_hold_signal: dict[tuple[str, str], float] = {}
self.init_schema()
@contextmanager
def connect(self) -> Iterator[sqlite3.Connection]:
conn = sqlite3.connect(self.path)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA busy_timeout=5000")
conn.execute("PRAGMA foreign_keys=ON")
try:
yield conn
conn.commit()
@@ -27,6 +65,7 @@ class Storage:
def init_schema(self) -> None:
with self.connect() as conn:
conn.execute("PRAGMA journal_mode=WAL")
conn.executescript(
"""
CREATE TABLE IF NOT EXISTS positions (
@@ -44,6 +83,9 @@ class Storage:
entry_confidence REAL NOT NULL DEFAULT 0,
entry_pattern TEXT NOT NULL DEFAULT '',
entry_diagnostics_json TEXT NOT NULL DEFAULT '{}',
protective_order_id TEXT NOT NULL DEFAULT '',
protective_order_link_id TEXT NOT NULL DEFAULT '',
mode TEXT NOT NULL DEFAULT 'paper',
status TEXT NOT NULL DEFAULT 'OPEN'
);
CREATE TABLE IF NOT EXISTS trades (
@@ -61,7 +103,8 @@ class Storage:
entry_confidence REAL NOT NULL DEFAULT 0,
entry_diagnostics_json TEXT NOT NULL DEFAULT '{}',
opened_at TEXT,
closed_at TEXT
closed_at TEXT,
mode TEXT NOT NULL DEFAULT 'paper'
);
CREATE TABLE IF NOT EXISTS signals (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -78,7 +121,8 @@ class Storage:
cash REAL NOT NULL,
exposure REAL NOT NULL,
drawdown REAL NOT NULL,
created_at TEXT NOT NULL
created_at TEXT NOT NULL,
mode TEXT NOT NULL DEFAULT 'paper'
);
CREATE TABLE IF NOT EXISTS events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -101,6 +145,35 @@ class Storage:
error TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS orders (
id INTEGER PRIMARY KEY AUTOINCREMENT,
client_order_id TEXT NOT NULL UNIQUE,
exchange_order_id TEXT NOT NULL DEFAULT '',
symbol TEXT NOT NULL,
side TEXT NOT NULL,
order_kind TEXT NOT NULL DEFAULT 'MARKET',
status TEXT NOT NULL,
requested_qty REAL NOT NULL DEFAULT 0,
requested_notional REAL NOT NULL DEFAULT 0,
executed_qty REAL NOT NULL DEFAULT 0,
executed_value REAL NOT NULL DEFAULT 0,
fee_usdt REAL NOT NULL DEFAULT 0,
raw_json TEXT NOT NULL DEFAULT '{}',
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_positions_status_opened
ON positions(status, opened_at);
CREATE INDEX IF NOT EXISTS idx_trades_closed
ON trades(side, closed_at, id DESC);
CREATE INDEX IF NOT EXISTS idx_signals_symbol_created
ON signals(symbol, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_equity_created
ON equity(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_events_created
ON events(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_orders_status_updated
ON orders(status, updated_at DESC);
"""
)
columns = {
@@ -116,6 +189,9 @@ class Storage:
"entry_confidence": "REAL NOT NULL DEFAULT 0",
"entry_pattern": "TEXT NOT NULL DEFAULT ''",
"entry_diagnostics_json": "TEXT NOT NULL DEFAULT '{}'",
"protective_order_id": "TEXT NOT NULL DEFAULT ''",
"protective_order_link_id": "TEXT NOT NULL DEFAULT ''",
"mode": "TEXT NOT NULL DEFAULT 'paper'",
}.items():
if column not in columns:
conn.execute(f"ALTER TABLE positions ADD COLUMN {column} {definition}")
@@ -127,9 +203,19 @@ class Storage:
"entry_pattern": "TEXT NOT NULL DEFAULT ''",
"entry_confidence": "REAL NOT NULL DEFAULT 0",
"entry_diagnostics_json": "TEXT NOT NULL DEFAULT '{}'",
"mode": "TEXT NOT NULL DEFAULT 'paper'",
}.items():
if column not in trade_columns:
conn.execute(f"ALTER TABLE trades ADD COLUMN {column} {definition}")
equity_columns = {
row["name"]
for row in conn.execute("PRAGMA table_info(equity)").fetchall()
}
if "mode" not in equity_columns:
conn.execute("ALTER TABLE equity ADD COLUMN mode TEXT NOT NULL DEFAULT 'paper'")
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_equity_mode_created ON equity(mode, created_at DESC)"
)
def insert_position(self, position: Position) -> int:
with self.connect() as conn:
@@ -138,8 +224,9 @@ class Storage:
INSERT INTO positions (
symbol, qty, entry_price, notional_usdt, entry_fee_usdt, stop_loss,
take_profit, highest_price, opened_at, entry_reason,
entry_confidence, entry_pattern, entry_diagnostics_json, status
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'OPEN')
entry_confidence, entry_pattern, entry_diagnostics_json,
protective_order_id, protective_order_link_id, mode, status
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'OPEN')
""",
(
position.symbol,
@@ -155,6 +242,9 @@ class Storage:
position.entry_confidence,
position.entry_pattern,
json.dumps(position.entry_diagnostics, ensure_ascii=False),
position.protective_order_id,
position.protective_order_link_id,
position.mode,
),
)
return int(cur.lastrowid)
@@ -170,11 +260,52 @@ class Storage:
(highest_price, position_id),
)
def open_positions(self) -> list[Position]:
def update_position_protective_order(
self,
position_id: int,
order_id: str,
order_link_id: str,
) -> None:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM positions WHERE status='OPEN' ORDER BY opened_at"
).fetchall()
conn.execute(
"""
UPDATE positions
SET protective_order_id=?, protective_order_link_id=?
WHERE id=? AND status='OPEN'
""",
(order_id, order_link_id, position_id),
)
def update_position_after_partial_sell(
self,
position_id: int,
*,
qty: float,
notional_usdt: float,
entry_fee_usdt: float,
) -> None:
with self.connect() as conn:
conn.execute(
"""
UPDATE positions
SET qty=?, notional_usdt=?, entry_fee_usdt=?,
protective_order_id='', protective_order_link_id=''
WHERE id=? AND status='OPEN'
""",
(qty, notional_usdt, entry_fee_usdt, position_id),
)
def open_positions(self, mode: str | None = None) -> list[Position]:
with self.connect() as conn:
if mode:
rows = conn.execute(
"SELECT * FROM positions WHERE status='OPEN' AND mode=? ORDER BY opened_at",
(mode,),
).fetchall()
else:
rows = conn.execute(
"SELECT * FROM positions WHERE status='OPEN' ORDER BY opened_at"
).fetchall()
return [
Position(
id=int(row["id"]),
@@ -191,6 +322,9 @@ class Storage:
entry_confidence=float(row["entry_confidence"]),
entry_pattern=row["entry_pattern"],
entry_diagnostics=_json_or_default(row["entry_diagnostics_json"], {}),
protective_order_id=row["protective_order_id"],
protective_order_link_id=row["protective_order_link_id"],
mode=row["mode"],
)
for row in rows
]
@@ -203,7 +337,8 @@ class Storage:
symbol, side, qty, entry_price, exit_price, gross_pnl,
fee_usdt, net_pnl, reason, entry_pattern, entry_confidence,
entry_diagnostics_json, opened_at, closed_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
, mode
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
trade.symbol,
@@ -220,32 +355,41 @@ class Storage:
json.dumps(trade.entry_diagnostics, ensure_ascii=False),
trade.opened_at.isoformat() if trade.opened_at else None,
trade.closed_at.isoformat() if trade.closed_at else None,
trade.mode,
),
)
return int(cur.lastrowid)
def recent_trades(self, limit: int = 50) -> list[dict[str, Any]]:
def recent_trades(self, limit: int = 50, mode: str | None = None) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM trades ORDER BY id DESC LIMIT ?", (limit,)).fetchall()
if mode:
rows = conn.execute(
"SELECT * FROM trades WHERE mode=? ORDER BY id DESC LIMIT ?",
(mode, limit),
).fetchall()
else:
rows = conn.execute("SELECT * FROM trades ORDER BY id DESC LIMIT ?", (limit,)).fetchall()
return [dict(row) for row in rows]
def closed_trades(self, limit: int = 200) -> list[dict[str, Any]]:
def closed_trades(self, limit: int = 200, mode: str | None = None) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"""
query = """
SELECT * FROM trades
WHERE side='SELL' AND closed_at IS NOT NULL
ORDER BY id DESC
LIMIT ?
""",
(limit,),
).fetchall()
"""
params: tuple[Any, ...]
if mode:
query += " AND mode=?"
params = (mode, limit)
else:
params = (limit,)
query += " ORDER BY id DESC LIMIT ?"
rows = conn.execute(query, params).fetchall()
return [dict(row) for row in rows]
def closed_trade_summary(self) -> dict[str, Any]:
def closed_trade_summary(self, mode: str | None = None) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute(
"""
query = """
SELECT
COUNT(*) AS trades,
COALESCE(SUM(net_pnl), 0) AS net_pnl,
@@ -255,8 +399,12 @@ class Storage:
COALESCE(SUM(CASE WHEN net_pnl < 0 THEN 1 ELSE 0 END), 0) AS losses
FROM trades
WHERE side='SELL' AND closed_at IS NOT NULL
"""
).fetchone()
"""
params: tuple[Any, ...] = ()
if mode:
query += " AND mode=?"
params = (mode,)
row = conn.execute(query, params).fetchone()
trades = int(row["trades"] if row else 0)
wins = int(row["wins"] if row else 0)
losses = int(row["losses"] if row else 0)
@@ -270,7 +418,15 @@ class Storage:
"win_rate": round(wins / trades, 4) if trades else 0.0,
}
def insert_signal(self, signal: Signal) -> None:
def insert_signal(self, signal: Signal, hold_sample_seconds: int = 0) -> bool:
if signal.action == "HOLD" and hold_sample_seconds > 0:
fingerprint = f"{signal.action}\0{signal.reason}"
now = time.monotonic()
sample_key = (signal.symbol, fingerprint)
previous = self._last_hold_signal.get(sample_key)
if previous is not None and now - previous < hold_sample_seconds:
return False
self._last_hold_signal[sample_key] = now
with self.connect() as conn:
conn.execute(
"""
@@ -282,26 +438,40 @@ class Storage:
signal.action,
signal.confidence,
signal.reason,
json.dumps(signal.diagnostics, ensure_ascii=False),
_signal_diagnostics_json(signal.diagnostics),
signal.created_at.isoformat(),
),
)
return True
def recent_signals(self, limit: int = 80) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM signals ORDER BY id DESC LIMIT ?", (limit,)).fetchall()
return [dict(row) for row in rows]
def insert_equity(self, equity: float, cash: float, exposure: float, drawdown: float) -> None:
def insert_equity(
self,
equity: float,
cash: float,
exposure: float,
drawdown: float,
mode: str = "paper",
) -> None:
with self.connect() as conn:
conn.execute(
"INSERT INTO equity (equity, cash, exposure, drawdown, created_at) VALUES (?, ?, ?, ?, ?)",
(equity, cash, exposure, drawdown, utc_now().isoformat()),
"INSERT INTO equity (equity, cash, exposure, drawdown, created_at, mode) VALUES (?, ?, ?, ?, ?, ?)",
(equity, cash, exposure, drawdown, utc_now().isoformat(), mode),
)
def latest_equity(self) -> dict[str, Any] | None:
def latest_equity(self, mode: str | None = None) -> dict[str, Any] | None:
with self.connect() as conn:
row = conn.execute("SELECT * FROM equity ORDER BY id DESC LIMIT 1").fetchone()
if mode:
row = conn.execute(
"SELECT * FROM equity WHERE mode=? ORDER BY id DESC LIMIT 1",
(mode,),
).fetchone()
else:
row = conn.execute("SELECT * FROM equity ORDER BY id DESC LIMIT 1").fetchone()
return dict(row) if row else None
def event(self, message: str, level: str = "INFO") -> None:
@@ -376,12 +546,146 @@ class Storage:
except json.JSONDecodeError:
return default
def upsert_order(
self,
*,
client_order_id: str,
exchange_order_id: str = "",
symbol: str,
side: str,
order_kind: str,
status: str,
requested_qty: float = 0.0,
requested_notional: float = 0.0,
executed_qty: float = 0.0,
executed_value: float = 0.0,
fee_usdt: float = 0.0,
raw: dict[str, Any] | None = None,
) -> None:
now = utc_now().isoformat()
with self.connect() as conn:
conn.execute(
"""
INSERT INTO orders (
client_order_id, exchange_order_id, symbol, side, order_kind,
status, requested_qty, requested_notional, executed_qty,
executed_value, fee_usdt, raw_json, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(client_order_id) DO UPDATE SET
exchange_order_id=excluded.exchange_order_id,
status=excluded.status,
executed_qty=excluded.executed_qty,
executed_value=excluded.executed_value,
fee_usdt=excluded.fee_usdt,
raw_json=excluded.raw_json,
updated_at=excluded.updated_at
""",
(
client_order_id,
exchange_order_id,
symbol,
side,
order_kind,
status,
requested_qty,
requested_notional,
executed_qty,
executed_value,
fee_usdt,
json.dumps(raw or {}, ensure_ascii=False),
now,
now,
),
)
def recent_orders(self, limit: int = 100) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM orders ORDER BY id DESC LIMIT ?",
(max(1, min(limit, 500)),),
).fetchall()
items = []
for row in rows:
item = dict(row)
item["raw"] = _json_or_default(item.pop("raw_json", "{}"), {})
items.append(item)
return items
def pending_orders(self) -> list[dict[str, Any]]:
terminal = ("Filled", "Cancelled", "Rejected", "PartiallyFilledCanceled", "Deactivated")
placeholders = ",".join("?" for _ in terminal)
with self.connect() as conn:
rows = conn.execute(
f"SELECT * FROM orders WHERE status NOT IN ({placeholders}) ORDER BY id",
terminal,
).fetchall()
return [dict(row) for row in rows]
def prune(self, retention_days: int) -> dict[str, int]:
if retention_days <= 0:
return {}
cutoff = (utc_now() - timedelta(days=retention_days)).isoformat()
deleted: dict[str, int] = {}
for table in ("signals", "equity", "events", "llm_advice"):
with self.connect() as conn:
# Keep write locks short on large runtime databases. Each maintenance
# cycle removes at most one bounded batch per table.
cursor = conn.execute(
f"""
DELETE FROM {table}
WHERE id IN (
SELECT id FROM {table}
WHERE created_at < ?
ORDER BY id
LIMIT ?
)
""",
(cutoff, PRUNE_BATCH_SIZE),
)
deleted[table] = max(0, int(cursor.rowcount))
return deleted
def clear_all(self) -> None:
with self.connect() as conn:
for table in ("positions", "trades", "signals", "equity", "events", "runtime", "llm_advice"):
for table in ("positions", "trades", "signals", "equity", "events", "runtime", "llm_advice", "orders"):
conn.execute(f"DELETE FROM {table}")
def _signal_diagnostics_json(diagnostics: dict[str, Any]) -> str:
compact = dict(diagnostics)
forecast = compact.get("forecast")
if isinstance(forecast, dict):
compact["forecast"] = {
key: value for key, value in forecast.items() if key in _STORED_FORECAST_KEYS
}
encoded = json.dumps(compact, ensure_ascii=False, separators=(",", ":"))
size = len(encoded.encode("utf-8"))
if size <= MAX_SIGNAL_DIAGNOSTICS_BYTES:
return encoded
fallback = {
"truncated": True,
"original_size_bytes": size,
"strategy_mode": compact.get("strategy_mode"),
"trade_mode": compact.get("trade_mode"),
"checks": compact.get("checks", {}),
"forecast": compact.get("forecast", {}),
}
encoded = json.dumps(fallback, ensure_ascii=False, separators=(",", ":"))
if len(encoded.encode("utf-8")) <= MAX_SIGNAL_DIAGNOSTICS_BYTES:
return encoded
return json.dumps(
{
"truncated": True,
"original_size_bytes": size,
"strategy_mode": compact.get("strategy_mode"),
"trade_mode": compact.get("trade_mode"),
},
ensure_ascii=False,
separators=(",", ":"),
)
def _json_or_default(value: str, default: Any) -> Any:
try:
return json.loads(value)