Harden trading, training, and monitoring
This commit is contained in:
Codex
+155
-12
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
|
||||
from crypto_spot_bot.analytics import risk_guard_snapshot
|
||||
@@ -15,6 +17,9 @@ from crypto_spot_bot.storage import Storage
|
||||
from crypto_spot_bot.time_series import TimeSeriesForecaster
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptoSpotBot:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -44,6 +49,9 @@ class CryptoSpotBot:
|
||||
self._entry_cooldown_until: dict[str, datetime] = {}
|
||||
self._loop_task: asyncio.Task | None = None
|
||||
self._ws_task: asyncio.Task | None = None
|
||||
self._last_reconciliation_at: datetime | None = None
|
||||
self._last_prune_at: datetime | None = None
|
||||
self._consecutive_loop_errors = 0
|
||||
|
||||
async def start(self) -> None:
|
||||
if self.running:
|
||||
@@ -51,6 +59,17 @@ class CryptoSpotBot:
|
||||
self.market.reset_stop()
|
||||
if not self.market.symbols:
|
||||
await self.market.bootstrap()
|
||||
if isinstance(self.broker, LiveBroker):
|
||||
try:
|
||||
await asyncio.to_thread(self.broker.reconcile, self.market.instruments)
|
||||
self._last_reconciliation_at = utc_now()
|
||||
except Exception as exc:
|
||||
self.broker.reconciliation_state = {
|
||||
"status": "error",
|
||||
"blocking": True,
|
||||
"discrepancies": [{"code": "initial_reconciliation_failed", "message": str(exc)}],
|
||||
}
|
||||
self.storage.event(f"Initial live reconciliation failed: {exc}", "ERROR")
|
||||
self._close_paper_positions_outside_symbol_universe()
|
||||
self._update_patterns()
|
||||
self._update_forecasts()
|
||||
@@ -58,7 +77,7 @@ class CryptoSpotBot:
|
||||
self.running = True
|
||||
self.started_at = utc_now()
|
||||
self.message = "бот работает"
|
||||
self.storage.event("Бот запущен")
|
||||
self._safe_event("Бот запущен")
|
||||
if self.settings.websocket_enabled:
|
||||
self._ws_task = asyncio.create_task(self.market.websocket_loop())
|
||||
self._loop_task = asyncio.create_task(self._run_loop())
|
||||
@@ -72,7 +91,13 @@ class CryptoSpotBot:
|
||||
task.cancel()
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
self.storage.event("Бот остановлен")
|
||||
self._safe_event("Бот остановлен")
|
||||
|
||||
def _safe_event(self, message: str, level: str = "INFO") -> None:
|
||||
try:
|
||||
self.storage.event(message, level)
|
||||
except sqlite3.Error:
|
||||
logger.exception("Could not persist non-critical bot event: %s", message)
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
while self.running:
|
||||
@@ -80,6 +105,7 @@ class CryptoSpotBot:
|
||||
rest_refresh_seconds = self._rest_refresh_seconds()
|
||||
if self._needs_rest_refresh(rest_refresh_seconds):
|
||||
await asyncio.to_thread(self.market.refresh_rest)
|
||||
await self._maintain_runtime()
|
||||
self.broker.update_highs(self.market.tickers)
|
||||
self._update_patterns()
|
||||
self._update_forecasts()
|
||||
@@ -88,9 +114,11 @@ class CryptoSpotBot:
|
||||
await self._process_entries()
|
||||
self.broker.mark_equity(self.market.prices())
|
||||
self.last_loop_at = utc_now()
|
||||
self._consecutive_loop_errors = 0
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._consecutive_loop_errors += 1
|
||||
self.message = f"ошибка цикла: {exc}"
|
||||
self.storage.event(self.message, "ERROR")
|
||||
await asyncio.sleep(self.settings.effective_loop_interval_seconds)
|
||||
@@ -110,6 +138,18 @@ class CryptoSpotBot:
|
||||
prices = self.market.prices()
|
||||
reduction_candidate_id = self._reduction_candidate_id(prices)
|
||||
for position in list(self.broker.open_positions()):
|
||||
freshness = self.market.symbol_freshness(position.symbol)
|
||||
if not freshness["ok"]:
|
||||
self._record_signal(
|
||||
Signal(
|
||||
position.symbol,
|
||||
"HOLD",
|
||||
0.0,
|
||||
"market data is stale; exchange protective stop remains authoritative",
|
||||
{"market_freshness": freshness},
|
||||
)
|
||||
)
|
||||
continue
|
||||
ticker = self.market.tickers.get(position.symbol)
|
||||
candles = self.market.candles.get(position.symbol, [])
|
||||
forecast = self.market.forecasts.get(position.symbol, {})
|
||||
@@ -117,25 +157,40 @@ class CryptoSpotBot:
|
||||
adaptive_rules["reduce_now"] = position.id is not None and position.id == reduction_candidate_id
|
||||
learning = {"adaptive_rules": adaptive_rules}
|
||||
signal = self.strategy.exit_signal(position, candles, ticker, learning, forecast)
|
||||
self.storage.insert_signal(signal)
|
||||
self._record_signal(signal)
|
||||
if signal.action == "SELL" and ticker is not None:
|
||||
self.broker.sell(position, ticker, signal.reason)
|
||||
await asyncio.to_thread(self.broker.sell, position, ticker, signal.reason)
|
||||
self._entry_cooldown_until[position.symbol] = utc_now()
|
||||
|
||||
async def _process_entries(self) -> None:
|
||||
prices = self.market.prices()
|
||||
risk_guard = risk_guard_snapshot(
|
||||
self.settings,
|
||||
self.storage.closed_trades(self.settings.learning_lookback_trades),
|
||||
self.storage.latest_equity(),
|
||||
self.storage.closed_trades(
|
||||
self.settings.learning_lookback_trades,
|
||||
mode=self.settings.trading_mode,
|
||||
),
|
||||
self.storage.latest_equity(mode=self.settings.trading_mode),
|
||||
)
|
||||
for symbol in self.market.symbols:
|
||||
freshness = self.market.symbol_freshness(symbol)
|
||||
if not freshness["ok"]:
|
||||
self._record_signal(
|
||||
Signal(
|
||||
symbol,
|
||||
"HOLD",
|
||||
0.0,
|
||||
"market data is stale; new entries blocked",
|
||||
{"market_freshness": freshness, "checks": {"market_fresh": False}},
|
||||
)
|
||||
)
|
||||
continue
|
||||
cooldown_since = self._entry_cooldown_until.get(symbol)
|
||||
if cooldown_since:
|
||||
age = (utc_now() - cooldown_since).total_seconds()
|
||||
cooldown_seconds = self.settings.effective_entry_cooldown_seconds
|
||||
if age < cooldown_seconds:
|
||||
self.storage.insert_signal(
|
||||
self._record_signal(
|
||||
Signal(
|
||||
symbol,
|
||||
"HOLD",
|
||||
@@ -162,7 +217,7 @@ class CryptoSpotBot:
|
||||
account["open_positions_for_symbol"] = open_count
|
||||
account["exchange_min_entry_usdt"] = self.broker.minimum_entry_budget(instrument, ticker)
|
||||
if risk_guard.get("block_new_entries"):
|
||||
self.storage.insert_signal(
|
||||
self._record_signal(
|
||||
Signal(
|
||||
symbol,
|
||||
"HOLD",
|
||||
@@ -178,7 +233,7 @@ class CryptoSpotBot:
|
||||
continue
|
||||
symbol_guard = self._risk_guard_for_symbol(risk_guard, symbol)
|
||||
if symbol_guard.get("block_new_entries"):
|
||||
self.storage.insert_signal(
|
||||
self._record_signal(
|
||||
Signal(
|
||||
symbol,
|
||||
"HOLD",
|
||||
@@ -224,9 +279,10 @@ class CryptoSpotBot:
|
||||
account,
|
||||
trend_candles,
|
||||
)
|
||||
self.storage.insert_signal(signal)
|
||||
self._record_signal(signal)
|
||||
if signal.action == "BUY" and ticker is not None:
|
||||
position = self.broker.buy(
|
||||
position = await asyncio.to_thread(
|
||||
self.broker.buy,
|
||||
signal,
|
||||
ticker,
|
||||
instrument,
|
||||
@@ -235,6 +291,41 @@ class CryptoSpotBot:
|
||||
if position is not None:
|
||||
self._entry_cooldown_until[symbol] = utc_now()
|
||||
|
||||
def _record_signal(self, signal: Signal) -> None:
|
||||
self.storage.insert_signal(signal, self.settings.hold_signal_sample_seconds)
|
||||
|
||||
async def _maintain_runtime(self) -> None:
|
||||
now = utc_now()
|
||||
if isinstance(self.broker, LiveBroker):
|
||||
age = (
|
||||
(now - self._last_reconciliation_at).total_seconds()
|
||||
if self._last_reconciliation_at
|
||||
else float("inf")
|
||||
)
|
||||
if age >= self.settings.live_reconciliation_interval_seconds:
|
||||
try:
|
||||
await asyncio.to_thread(self.broker.reconcile, self.market.instruments)
|
||||
except Exception as exc:
|
||||
self.broker.reconciliation_state = {
|
||||
"status": "error",
|
||||
"blocking": True,
|
||||
"discrepancies": [
|
||||
{"code": "periodic_reconciliation_failed", "message": str(exc)}
|
||||
],
|
||||
"checked_at": utc_now().isoformat(),
|
||||
}
|
||||
self.storage.event(f"Periodic live reconciliation failed: {exc}", "ERROR")
|
||||
finally:
|
||||
self._last_reconciliation_at = utc_now()
|
||||
prune_age = (
|
||||
(now - self._last_prune_at).total_seconds()
|
||||
if self._last_prune_at
|
||||
else float("inf")
|
||||
)
|
||||
if prune_age >= self.settings.storage_prune_interval_seconds:
|
||||
await asyncio.to_thread(self.storage.prune, self.settings.storage_retention_days)
|
||||
self._last_prune_at = utc_now()
|
||||
|
||||
@staticmethod
|
||||
def _risk_guard_for_symbol(risk_guard: dict, symbol: str) -> dict:
|
||||
rows = risk_guard.get("symbols")
|
||||
@@ -334,16 +425,68 @@ class CryptoSpotBot:
|
||||
self.market.forecasts = forecasts
|
||||
|
||||
def status(self) -> BotStatus:
|
||||
live_ready = self.settings.live_ready
|
||||
if isinstance(self.broker, LiveBroker):
|
||||
live_ready = live_ready and not self.broker.reconciliation_state.get("blocking", True)
|
||||
return BotStatus(
|
||||
running=self.running,
|
||||
mode=self.settings.trading_mode,
|
||||
live_trading_ready=self.settings.live_ready,
|
||||
live_trading_ready=live_ready,
|
||||
symbols=self.market.symbols,
|
||||
started_at=self.started_at,
|
||||
last_loop_at=self.last_loop_at,
|
||||
message=self.message,
|
||||
)
|
||||
|
||||
def readiness_snapshot(self) -> dict:
|
||||
reasons: list[str] = []
|
||||
now = utc_now()
|
||||
if not self.running:
|
||||
reasons.append("bot_not_running")
|
||||
max_loop_age = max(30.0, self.settings.effective_loop_interval_seconds * 4)
|
||||
loop_age = (now - self.last_loop_at).total_seconds() if self.last_loop_at else None
|
||||
if loop_age is None or loop_age > max_loop_age:
|
||||
reasons.append("decision_loop_stale")
|
||||
stale_symbols = [
|
||||
symbol for symbol in self.market.symbols if not self.market.symbol_freshness(symbol)["ok"]
|
||||
]
|
||||
if stale_symbols:
|
||||
reasons.append("stale_market_data")
|
||||
if self._consecutive_loop_errors >= 3:
|
||||
reasons.append("repeated_loop_errors")
|
||||
if self.settings.strategy_mode == "torch_forecast":
|
||||
invalid_models = []
|
||||
for symbol in self.market.symbols:
|
||||
forecast = self.market.forecasts.get(symbol, {})
|
||||
if not forecast.get("usable"):
|
||||
invalid_models.append(symbol)
|
||||
continue
|
||||
if (
|
||||
self.settings.time_series_require_quality_gate
|
||||
and not self.settings.time_series_manual_quality_override
|
||||
and forecast.get("quality_gate_passed") is not True
|
||||
):
|
||||
invalid_models.append(symbol)
|
||||
continue
|
||||
if self.settings.time_series_require_fresh_model and forecast.get("model_fresh") is not True:
|
||||
invalid_models.append(symbol)
|
||||
if invalid_models:
|
||||
reasons.append("forecast_model_not_ready")
|
||||
reconciliation: dict = {}
|
||||
if isinstance(self.broker, LiveBroker):
|
||||
reconciliation = dict(self.broker.reconciliation_state)
|
||||
if reconciliation.get("blocking", True):
|
||||
reasons.append("live_reconciliation_blocking")
|
||||
return {
|
||||
"ready": not reasons,
|
||||
"mode": self.settings.trading_mode,
|
||||
"reasons": reasons,
|
||||
"loop_age_seconds": round(loop_age, 3) if loop_age is not None else None,
|
||||
"stale_symbols": stale_symbols,
|
||||
"consecutive_loop_errors": self._consecutive_loop_errors,
|
||||
"reconciliation": reconciliation,
|
||||
}
|
||||
|
||||
def account_snapshot(self) -> dict[str, float]:
|
||||
prices = self.market.prices()
|
||||
state = self.broker.account_state(prices)
|
||||
|
||||
Reference in New Issue
Block a user