Keep bot operational when forecast model is unavailable

crypto_spot_bot/bot.py

@@ -12,7 +12,7 @@ from crypto_spot_bot.learning import TradeLearner
 from crypto_spot_bot.market_data import MarketData
 from crypto_spot_bot.models import BotStatus, Signal, Ticker, utc_now
 from crypto_spot_bot.patterns import PatternAnalyzer
-from crypto_spot_bot.strategy import SpotStrategy
+from crypto_spot_bot.strategy import SpotStrategy, torch_model_readiness_reasons
 from crypto_spot_bot.storage import Storage
 from crypto_spot_bot.time_series import TimeSeriesForecaster
 
@@ -78,6 +78,9 @@ class CryptoSpotBot:
         self.started_at = utc_now()
         self.message = "бот работает"
         self._safe_event("Бот запущен")
+        # Maintenance must never delay the first market decision after startup.
+        # The bounded telemetry prune starts after the configured interval.
+        self._last_prune_at = utc_now()
         if self.settings.websocket_enabled:
             self._ws_task = asyncio.create_task(self.market.websocket_loop())
         self._loop_task = asyncio.create_task(self._run_loop())
@@ -458,20 +461,19 @@ class CryptoSpotBot:
             invalid_models = []
             for symbol in self.market.symbols:
                 forecast = self.market.forecasts.get(symbol, {})
-                if not forecast.get("usable"):
-                    invalid_models.append(symbol)
-                    continue
-                if (
-                    self.settings.time_series_require_quality_gate
-                    and not self.settings.time_series_manual_quality_override
-                    and forecast.get("quality_gate_passed") is not True
-                ):
-                    invalid_models.append(symbol)
-                    continue
-                if self.settings.time_series_require_fresh_model and forecast.get("model_fresh") is not True:
+                if torch_model_readiness_reasons(self.settings, forecast):
                     invalid_models.append(symbol)
             if invalid_models:
-                reasons.append("forecast_model_not_ready")
+                if self.settings.time_series_trend_fallback_enabled:
+                    forecast_fallback_active = True
+                else:
+                    forecast_fallback_active = False
+                    reasons.append("forecast_model_not_ready")
+            else:
+                forecast_fallback_active = False
+        else:
+            invalid_models = []
+            forecast_fallback_active = False
         reconciliation: dict = {}
         if isinstance(self.broker, LiveBroker):
             reconciliation = dict(self.broker.reconciliation_state)
@@ -485,6 +487,9 @@ class CryptoSpotBot:
             "stale_symbols": stale_symbols,
             "consecutive_loop_errors": self._consecutive_loop_errors,
             "reconciliation": reconciliation,
+            "forecast_model_ready": not invalid_models,
+            "forecast_fallback_active": forecast_fallback_active,
+            "forecast_invalid_symbols": invalid_models,
         }
 
     def account_snapshot(self) -> dict[str, float]:

crypto_spot_bot/bybit.py

@@ -42,7 +42,11 @@ class Instrument:
 class BybitClient:
     def __init__(self, settings: Settings):
         self.settings = settings
-        self.session = requests.Session()
+        self.session = self._build_session()
+
+    @staticmethod
+    def _build_session() -> requests.Session:
+        session = requests.Session()
         retry = Retry(
             total=3,
             connect=3,
@@ -53,14 +57,32 @@ class BybitClient:
             allowed_methods=frozenset({"GET"}),
             respect_retry_after_header=True,
         )
-        self.session.mount("https://", HTTPAdapter(max_retries=retry))
+        session.mount("https://", HTTPAdapter(max_retries=retry))
+        return session
+
+    def _reset_session(self) -> None:
+        self.session.close()
+        self.session = self._build_session()
 
     def public_get(self, path: str, params: dict[str, Any]) -> dict[str, Any]:
-        response = self.session.get(
-            f"{self.settings.rest_base_url}{path}",
-            params=params,
-            timeout=12,
-        )
+        response = None
+        for attempt in range(3):
+            try:
+                response = self.session.get(
+                    f"{self.settings.rest_base_url}{path}",
+                    params=params,
+                    timeout=12,
+                )
+                break
+            except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
+                if attempt >= 2:
+                    raise
+                # A failed TLS session can remain poisoned in urllib3's pool.
+                # Recreate the pool before retrying instead of reusing it.
+                self._reset_session()
+                time.sleep(0.5 * (2**attempt))
+        if response is None:  # pragma: no cover - loop either returns or raises.
+            raise BybitError("Bybit public request produced no response")
         response.raise_for_status()
         return self._unwrap(response.json())
 

crypto_spot_bot/config.py

@@ -137,6 +137,7 @@ class Settings:
     time_series_probe_min_probability_up: float
     time_series_probe_size_multiplier: float
     time_series_rebound_fallback_enabled: bool
+    time_series_trend_fallback_enabled: bool
     stop_loss_percent: float
     stop_loss_exit_enabled: bool
     take_profit_percent: float
@@ -304,6 +305,7 @@ def load_settings(env_file: str | Path | None = None) -> Settings:
         time_series_probe_min_probability_up=_float_env("TIME_SERIES_PROBE_MIN_PROBABILITY_UP", 0.55),
         time_series_probe_size_multiplier=_float_env("TIME_SERIES_PROBE_SIZE_MULTIPLIER", 0.40),
         time_series_rebound_fallback_enabled=_bool_env("TIME_SERIES_REBOUND_FALLBACK_ENABLED", False),
+        time_series_trend_fallback_enabled=_bool_env("TIME_SERIES_TREND_FALLBACK_ENABLED", False),
         stop_loss_percent=_float_env("STOP_LOSS_PERCENT", 0.04),
         stop_loss_exit_enabled=_bool_env("STOP_LOSS_EXIT_ENABLED", True),
         take_profit_percent=_float_env("TAKE_PROFIT_PERCENT", 0.035),

crypto_spot_bot/dashboard.py

@@ -398,6 +398,7 @@ def _safe_config(settings: Settings) -> dict[str, Any]:
         "time_series_probe_min_probability_up": settings.time_series_probe_min_probability_up,
         "time_series_probe_size_multiplier": settings.time_series_probe_size_multiplier,
         "time_series_rebound_fallback_enabled": settings.time_series_rebound_fallback_enabled,
+        "time_series_trend_fallback_enabled": settings.time_series_trend_fallback_enabled,
         "time_series_require_quality_gate": settings.time_series_require_quality_gate,
         "time_series_manual_quality_override": settings.time_series_manual_quality_override,
         "time_series_require_fresh_model": settings.time_series_require_fresh_model,

crypto_spot_bot/storage.py

@@ -11,8 +11,14 @@ from typing import Any, Iterator
 from crypto_spot_bot.models import Position, Signal, Trade, utc_now
 
 
-MAX_SIGNAL_DIAGNOSTICS_BYTES = 16 * 1024
-PRUNE_BATCH_SIZE = 1000
+MAX_SIGNAL_DIAGNOSTICS_BYTES = 4 * 1024
+PRUNE_BATCH_SIZE = 5000
+MAX_RUNTIME_ROWS = {
+    "signals": 50_000,
+    "equity": 100_000,
+    "events": 20_000,
+    "llm_advice": 20_000,
+}
 _STORED_FORECAST_KEYS = {
     "enabled",
     "usable",
@@ -65,6 +71,9 @@ class Storage:
 
     def init_schema(self) -> None:
         with self.connect() as conn:
+            # New runtime databases reclaim deleted telemetry pages incrementally.
+            # Existing databases keep their current mode until compacted once.
+            conn.execute("PRAGMA auto_vacuum=INCREMENTAL")
             conn.execute("PRAGMA journal_mode=WAL")
             conn.executescript(
                 """
@@ -628,21 +637,41 @@ class Storage:
         deleted: dict[str, int] = {}
         for table in ("signals", "equity", "events", "llm_advice"):
             with self.connect() as conn:
-                # Keep write locks short on large runtime databases. Each maintenance
-                # cycle removes at most one bounded batch per table.
-                cursor = conn.execute(
-                    f"""
-                    DELETE FROM {table}
-                    WHERE id IN (
-                        SELECT id FROM {table}
-                        WHERE created_at < ?
-                        ORDER BY id
-                        LIMIT ?
+                max_id_row = conn.execute(f"SELECT MAX(id) AS value FROM {table}").fetchone()
+                max_id = int(max_id_row["value"] or 0) if max_id_row else 0
+                cap_boundary = max(0, max_id - MAX_RUNTIME_ROWS[table])
+                removed = 0
+                if cap_boundary > 0:
+                    cursor = conn.execute(
+                        f"""
+                        DELETE FROM {table}
+                        WHERE id IN (
+                            SELECT id FROM {table}
+                            WHERE id <= ?
+                            ORDER BY id
+                            LIMIT ?
+                        )
+                        """,
+                        (cap_boundary, PRUNE_BATCH_SIZE),
                     )
-                    """,
-                    (cutoff, PRUNE_BATCH_SIZE),
-                )
-                deleted[table] = max(0, int(cursor.rowcount))
+                    removed = max(0, int(cursor.rowcount))
+                remaining = max(0, PRUNE_BATCH_SIZE - removed)
+                if remaining:
+                    cursor = conn.execute(
+                        f"""
+                        DELETE FROM {table}
+                        WHERE id IN (
+                            SELECT id FROM {table}
+                            WHERE created_at < ?
+                            ORDER BY id
+                            LIMIT ?
+                        )
+                        """,
+                        (cutoff, remaining),
+                    )
+                    removed += max(0, int(cursor.rowcount))
+                deleted[table] = removed
+                conn.execute("PRAGMA incremental_vacuum(512)")
         return deleted
 
     def clear_all(self) -> None:

crypto_spot_bot/strategy.py

@@ -25,6 +25,35 @@ class SpotStrategy:
         trend_candles: list[Candle] | None = None,
     ) -> Signal:
         if self.settings.strategy_mode == "torch_forecast":
+            fallback_reasons = torch_model_readiness_reasons(self.settings, forecast or {})
+            if self.settings.time_series_trend_fallback_enabled and fallback_reasons:
+                fallback = _trend_macd_entry_signal(
+                    settings=self.settings,
+                    symbol=symbol,
+                    candles=candles,
+                    trend_candles=trend_candles or [],
+                    ticker=ticker,
+                    open_positions_for_symbol=open_positions_for_symbol,
+                    account=account,
+                )
+                diagnostics = dict(fallback.diagnostics)
+                diagnostics.update(
+                    {
+                        "strategy_mode": "torch_forecast",
+                        "trade_mode": "TREND_MACD_FALLBACK",
+                        "entry_path": "trend_macd_fallback",
+                        "forecast_fallback_active": True,
+                        "forecast_fallback_reasons": fallback_reasons,
+                        "forecast": forecast or {},
+                    }
+                )
+                return Signal(
+                    fallback.symbol,
+                    fallback.action,
+                    fallback.confidence,
+                    f"torch_forecast fallback: {fallback.reason}",
+                    diagnostics,
+                )
             return _torch_forecast_entry_signal(
                 settings=self.settings,
                 symbol=symbol,
@@ -368,6 +397,24 @@ class SpotStrategy:
         forecast: dict | None = None,
     ) -> Signal:
         if self.settings.strategy_mode == "torch_forecast":
+            if str(position.entry_diagnostics.get("entry_path", "")) == "trend_macd_fallback":
+                fallback = _trend_macd_exit_signal(self.settings, position, candles, ticker)
+                diagnostics = dict(fallback.diagnostics)
+                diagnostics.update(
+                    {
+                        "strategy_mode": "torch_forecast",
+                        "trade_mode": "TREND_MACD_FALLBACK",
+                        "entry_path": "trend_macd_fallback",
+                        "forecast_fallback_active": True,
+                    }
+                )
+                return Signal(
+                    fallback.symbol,
+                    fallback.action,
+                    fallback.confidence,
+                    f"torch_forecast fallback: {fallback.reason}",
+                    diagnostics,
+                )
             return _torch_forecast_exit_signal(self.settings, position, candles, ticker, forecast or {})
         if self.settings.strategy_mode == "trend_macd":
             return _trend_macd_exit_signal(self.settings, position, candles, ticker)
@@ -1053,6 +1100,21 @@ def _is_torch_forecast(forecast: dict) -> bool:
     return bool(forecast.get("usable", False)) and model in {"torch_lstm", "torch_gru"}
 
 
+def torch_model_readiness_reasons(settings: Settings, forecast: dict) -> list[str]:
+    reasons: list[str] = []
+    if not _is_torch_forecast(forecast):
+        reasons.append("torch_model_unavailable")
+    if (
+        settings.time_series_require_quality_gate
+        and not settings.time_series_manual_quality_override
+        and forecast.get("quality_gate_passed") is not True
+    ):
+        reasons.append("quality_gate_not_passed")
+    if settings.time_series_require_fresh_model and forecast.get("model_fresh") is not True:
+        reasons.append("model_not_fresh")
+    return reasons
+
+
 def _missing_torch_model(forecast: dict) -> bool:
     model = str(forecast.get("model", "")).strip().lower()
     reason = str(forecast.get("reason", "")).lower()

crypto_spot_bot/time_series.py

@@ -213,6 +213,7 @@ class TimeSeriesForecaster:
             probability=self.settings.time_series_min_probability_up,
             confidence=self.settings.time_series_min_confidence,
         )
+        symbol_eligible = _calibration_symbol_eligible(calibration, symbol)
         entry = _torch_recurrent_entry(symbol, artifact)
         model = _torch_recurrent_model_name(symbol, artifact)
         clip = _clamp(_float_entry(entry or {}, "clip", 8.0), 1.0, 50.0)
@@ -274,7 +275,8 @@ class TimeSeriesForecaster:
             )
             conservative_return_percent = min(expected_return_percent, q50_percent)
             block_entry = bool(
-                (expected_return_percent <= -min_edge and probability_up <= 0.45)
+                not symbol_eligible
+                or (expected_return_percent <= -min_edge and probability_up <= 0.45)
                 or (q50_percent <= -min_edge and probability_up <= 0.48)
             )
             reason = _reason(
@@ -284,6 +286,8 @@ class TimeSeriesForecaster:
                 skill=skill,
                 block_entry=block_entry,
             )
+            if not symbol_eligible:
+                reason = "symbol excluded by train-only calibration"
             return TimeSeriesForecast(
                 enabled=True,
                 usable=True,
@@ -344,7 +348,9 @@ class TimeSeriesForecaster:
             min_edge=min_edge,
             max_adjustment=self.settings.time_series_max_adjustment,
         )
-        block_entry = bool(expected_return_percent <= -min_edge and probability_up <= 0.45)
+        block_entry = bool(
+            not symbol_eligible or (expected_return_percent <= -min_edge and probability_up <= 0.45)
+        )
         reason = _reason(
             model=model,
             expected_return_percent=expected_return_percent,
@@ -352,6 +358,8 @@ class TimeSeriesForecaster:
             skill=skill,
             block_entry=block_entry,
         )
+        if not symbol_eligible:
+            reason = "symbol excluded by train-only calibration"
         return TimeSeriesForecast(
             enabled=True,
             usable=True,
@@ -496,6 +504,16 @@ def _calibrated_thresholds(
     }
 
 
+def _calibration_symbol_eligible(calibration: dict[str, Any], symbol: str | None) -> bool:
+    if not isinstance(calibration, dict) or "eligible_symbols" not in calibration:
+        return True
+    eligible = calibration.get("eligible_symbols")
+    if not isinstance(eligible, list) or not symbol:
+        return False
+    allowed = {str(value).strip().upper() for value in eligible if str(value).strip()}
+    return symbol.strip().upper() in allowed
+
+
 def _model_freshness(artifact: dict[str, Any], max_age_hours: float) -> tuple[str, float | None, bool]:
     raw = str(artifact.get("created_at", "")).strip() if isinstance(artifact, dict) else ""
     if not raw:
@@ -998,7 +1016,12 @@ def _torch_recurrent_entry(symbol: str | None, artifact: dict[str, Any]) -> dict
         entry = default if isinstance(default, dict) else None
     if not isinstance(entry, dict):
         return None
-    if not isinstance(entry.get("state_dict"), dict):
+    members = entry.get("ensemble_members")
+    has_member_state = isinstance(members, list) and any(
+        isinstance(member, dict) and isinstance(member.get("state_dict"), dict)
+        for member in members
+    )
+    if not isinstance(entry.get("state_dict"), dict) and not has_member_state:
         return None
     return entry
 

crypto_spot_bot/training_coordination.py

@@ -23,7 +23,9 @@ ALLOWED_TRAINING_ARTIFACTS = {
 RUNNING_TIMEOUT = timedelta(hours=12)
 ONLINE_WINDOW = timedelta(minutes=3)
 MAX_ARTIFACT_CHUNK_BYTES = 1024 * 1024
-MAX_ARTIFACT_BYTES = 64 * 1024 * 1024
+# Independent per-symbol ensembles are intentionally larger than pooled models.
+# Keep a bounded limit, but leave enough room for the supported 12-symbol bundle.
+MAX_ARTIFACT_BYTES = 256 * 1024 * 1024
 MAX_ARTIFACT_CHUNKS = 1024
 REQUIRED_MODEL_BUNDLE = set(ALLOWED_TRAINING_ARTIFACTS)
 
@@ -378,13 +380,19 @@ def _safe_parameters(value: Any) -> dict[str, Any]:
         "dropouts",
         "epochs",
         "holdout_window",
+        "ensemble_seeds",
+        "selection_folds",
+        "learning_rate",
+        "weight_decay",
+        "pooled",
         "resume_candidate",
     }
     result = {key: value[key] for key in allowed if key in value}
     for key, low, high in (
-        ("limit", 500, 5000),
+        ("limit", 500, 20000),
         ("epochs", 1, 200),
         ("holdout_window", 64, 1000),
+        ("selection_folds", 1, 12),
     ):
         if key not in result:
             continue
@@ -406,9 +414,21 @@ def _safe_parameters(value: Any) -> dict[str, Any]:
             if item.strip().lower() in {"lstm", "gru"}
         ]
         result["architectures"] = ",".join(architectures) or "lstm,gru"
-    for key in ("lookbacks", "hidden_sizes", "layers", "dropouts"):
+    for key in ("lookbacks", "hidden_sizes", "layers", "dropouts", "ensemble_seeds"):
         if key in result:
             result[key] = str(result[key])[:200]
+    for key, low, high in (
+        ("learning_rate", 0.00001, 0.1),
+        ("weight_decay", 0.0, 0.1),
+    ):
+        if key not in result:
+            continue
+        try:
+            result[key] = max(low, min(high, float(result[key])))
+        except (TypeError, ValueError):
+            result.pop(key, None)
+    if "pooled" in result:
+        result["pooled"] = result["pooled"] is True
     if "resume_candidate" in result:
         result["resume_candidate"] = result["resume_candidate"] is True
     return result