Harden trading, training, and monitoring

2026-07-10 15:51:53 +03:00
parent 6fb79ee2a9
commit 069d75d2f2
55 changed files with 2658 additions and 2332049 deletions
@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import asyncio
+import base64
+
+import pytest
+from fastapi import HTTPException, Request
+
+from crypto_spot_bot.auth import ApiAuthorizer
+
+
+def _request(**headers: str) -> Request:
+    encoded = [(key.lower().encode(), value.encode()) for key, value in headers.items()]
+    return Request({"type": "http", "method": "GET", "path": "/", "headers": encoded})
+
+
+def test_api_authorizer_accepts_direct_basic_token(make_settings, tmp_path) -> None:
+    settings = make_settings(tmp_path, api_auth_token="user:secret")
+    auth = ApiAuthorizer(settings)
+    basic = base64.b64encode(b"user:secret").decode("ascii")
+
+    asyncio.run(auth.require(_request(Authorization=f"Basic {basic}")))
+
+
+def test_api_authorizer_accepts_trusted_proxy_header(make_settings, tmp_path) -> None:
+    settings = make_settings(tmp_path, trusted_proxy_user_header="X-TradeBot-Proxy-User")
+    auth = ApiAuthorizer(settings)
+
+    asyncio.run(auth.require(_request(**{"X-TradeBot-Proxy-User": "sevenhill"})))
+
+
+def test_api_authorizer_rejects_missing_credentials(make_settings, tmp_path) -> None:
+    auth = ApiAuthorizer(make_settings(tmp_path))
+
+    with pytest.raises(HTTPException) as raised:
+        asyncio.run(auth.require(_request()))
+
+    assert raised.value.status_code == 401
@@ -154,3 +154,20 @@ def test_auto_select_uses_empty_symbol_list(tmp_path, monkeypatch) -> None:
    assert settings.auto_select_symbols is True
    assert settings.top_symbols_count == 12
    assert settings.symbols == ()
+
+
+def test_load_settings_rejects_inconsistent_exposure_limits(tmp_path, monkeypatch) -> None:
+    for key in (
+        "MIN_POSITION_USDT",
+        "MAX_SYMBOL_EXPOSURE_USDT",
+        "MAX_TOTAL_EXPOSURE_USDT",
+    ):
+        monkeypatch.delenv(key, raising=False)
+    env_file = tmp_path / ".env"
+    env_file.write_text(
+        "MIN_POSITION_USDT=10\nMAX_SYMBOL_EXPOSURE_USDT=5\nMAX_TOTAL_EXPOSURE_USDT=20\n",
+        encoding="utf-8",
+    )
+
+    with pytest.raises(ValueError, match="MAX_SYMBOL_EXPOSURE_USDT"):
+        load_settings(env_file)
@@ -4,7 +4,7 @@ from types import SimpleNamespace

 from crypto_spot_bot.bybit import Instrument
 from crypto_spot_bot.bot import CryptoSpotBot
-from crypto_spot_bot.execution import PaperBroker
+from crypto_spot_bot.execution import LiveBroker, PaperBroker
 from crypto_spot_bot.models import Signal, Ticker
 from crypto_spot_bot.storage import Storage
 from crypto_spot_bot.strategy import SpotStrategy
@@ -319,3 +319,108 @@ def test_trend_macd_closes_old_paper_positions_outside_symbol_universe(make_sett
    assert trade["side"] == "SELL"
    assert trade["symbol"] == "HYPEUSDT"
    assert "trend_macd" in trade["reason"]
+
+
+def test_live_broker_records_exchange_fill_and_protective_stop(make_settings, tmp_path) -> None:
+    settings = make_settings(
+        tmp_path,
+        trading_mode="live",
+        enable_live_trading=True,
+        live_trading_confirm="I_ACCEPT_REAL_RISK",
+        bybit_api_key="key",
+        bybit_api_secret="secret",
+        live_protective_stop_enabled=True,
+    )
+    storage = Storage(settings.database_path)
+
+    class Client:
+        def place_spot_market_order(self, **kwargs):
+            return {"orderId": "buy-1"}
+
+        def wait_for_spot_order(self, **kwargs):
+            return {
+                "order": {"orderStatus": "Filled"},
+                "executions": [
+                    {
+                        "symbol": "BTCUSDT",
+                        "execQty": "0.001",
+                        "execValue": "10",
+                        "execPrice": "10000",
+                        "execFee": "0.01",
+                        "feeCurrency": "USDT",
+                    }
+                ],
+            }
+
+        def place_spot_protective_stop(self, **kwargs):
+            return {"orderId": "stop-1"}
+
+    broker = LiveBroker(settings, storage, Client())
+    broker.reconciliation_state = {"status": "ok", "blocking": False, "discrepancies": []}
+    ticker = Ticker("BTCUSDT", 10000, 9999, 10001, 10_000_000, 1000, 0)
+    instrument = Instrument("BTCUSDT", "BTC", "USDT", "Trading", 0.01, 0.000001, 0.000001, 5)
+    signal = Signal("BTCUSDT", "BUY", 0.8, "test", {"position_notional_usdt": 10})
+
+    position = broker.buy(signal, ticker, instrument, {"BTCUSDT": 10000})
+
+    assert position is not None
+    assert position.qty == 0.001
+    assert position.entry_price == 10000
+    assert position.protective_order_id == "stop-1"
+    assert storage.recent_orders()[0]["order_kind"] == "PROTECTIVE_STOP"
+
+
+def test_live_broker_sell_uses_confirmed_exchange_fill(make_settings, tmp_path) -> None:
+    settings = make_settings(
+        tmp_path,
+        trading_mode="live",
+        enable_live_trading=True,
+        live_trading_confirm="I_ACCEPT_REAL_RISK",
+        bybit_api_key="key",
+        bybit_api_secret="secret",
+        live_protective_stop_enabled=False,
+    )
+    storage = Storage(settings.database_path)
+
+    class Client:
+        def place_spot_market_order(self, **kwargs):
+            return {"orderId": "buy-1" if kwargs["side"] == "Buy" else "sell-1"}
+
+        def wait_for_spot_order(self, **kwargs):
+            buying = kwargs["order_id"] == "buy-1"
+            return {
+                "order": {"orderStatus": "Filled"},
+                "executions": [
+                    {
+                        "symbol": "BTCUSDT",
+                        "execQty": "0.001",
+                        "execValue": "10" if buying else "11",
+                        "execPrice": "10000" if buying else "11000",
+                        "execFee": "0.01",
+                        "feeCurrency": "USDT",
+                    }
+                ],
+            }
+
+    broker = LiveBroker(settings, storage, Client())
+    broker.reconciliation_state = {"status": "ok", "blocking": False, "discrepancies": []}
+    instrument = Instrument("BTCUSDT", "BTC", "USDT", "Trading", 0.01, 0.000001, 0.000001, 5)
+    entry_ticker = Ticker("BTCUSDT", 10000, 9999, 10001, 10_000_000, 1000, 0)
+    position = broker.buy(
+        Signal("BTCUSDT", "BUY", 0.8, "test", {"position_notional_usdt": 10}),
+        entry_ticker,
+        instrument,
+        {"BTCUSDT": 10000},
+    )
+    assert position is not None
+
+    trade = broker.sell(
+        position,
+        Ticker("BTCUSDT", 11000, 10999, 11001, 10_000_000, 1000, 0),
+        "test exit",
+    )
+
+    assert trade.exit_price == 11000
+    assert trade.qty == 0.001
+    assert broker.open_positions() == []
+    assert storage.recent_orders()[0]["status"] == "Filled"
@@ -1,6 +1,6 @@
 from __future__ import annotations

-from crypto_spot_bot.market_data import _closed_candles, _is_closed_kline_row
+from crypto_spot_bot.market_data import _candles_due, _closed_candles, _is_closed_kline_row
 from crypto_spot_bot.models import Candle


@@ -19,3 +19,10 @@ def test_closed_candles_excludes_current_open_interval() -> None:
 def test_websocket_kline_requires_confirmed_candle() -> None:
    assert _is_closed_kline_row({"start": 7_200_000, "confirm": False}, "60") is False
    assert _is_closed_kline_row({"start": 7_200_000, "confirm": True}, "60") is True
+
+
+def test_rest_candles_refresh_only_after_next_bar_closes() -> None:
+    candle = Candle(10 * 60_000, 1, 1, 1, 1, 1)
+
+    assert _candles_due([candle], "1", now_ms=11 * 60_000 + 30_000) is False
+    assert _candles_due([candle], "1", now_ms=12 * 60_000) is True
@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import json
+from datetime import timedelta
+
+from crypto_spot_bot.models import Signal, utc_now
+from crypto_spot_bot.storage import MAX_SIGNAL_DIAGNOSTICS_BYTES, PRUNE_BATCH_SIZE, Storage
+
+
+def test_hold_sampling_is_independent_for_each_reason_and_diagnostics_are_bounded(tmp_path) -> None:
+    storage = Storage(tmp_path / "tradebot.sqlite3")
+    diagnostics = {
+        "strategy_mode": "torch_forecast",
+        "checks": {"model_fresh_ok": False},
+        "forecast": {
+            "model": "torch_lstm",
+            "expected_return_percent": 0.42,
+            "model_fresh": False,
+            "feature_snapshot": [
+                {"name": f"feature-{index}", "interpretation": "x" * 1000}
+                for index in range(100)
+            ],
+        },
+    }
+
+    first = Signal("BTCUSDT", "HOLD", 0.2, "entry blocked", diagnostics)
+    second = Signal("BTCUSDT", "HOLD", 0.2, "position held", diagnostics)
+
+    assert storage.insert_signal(first, hold_sample_seconds=60) is True
+    assert storage.insert_signal(second, hold_sample_seconds=60) is True
+    assert storage.insert_signal(first, hold_sample_seconds=60) is False
+
+    rows = storage.recent_signals(10)
+    assert len(rows) == 2
+    stored = json.loads(rows[0]["diagnostics_json"])
+    assert len(rows[0]["diagnostics_json"].encode("utf-8")) <= MAX_SIGNAL_DIAGNOSTICS_BYTES
+    assert stored["forecast"]["model"] == "torch_lstm"
+    assert "feature_snapshot" not in stored["forecast"]
+
+
+def test_prune_deletes_only_one_bounded_batch_per_table(tmp_path) -> None:
+    storage = Storage(tmp_path / "tradebot.sqlite3")
+    old_timestamp = (utc_now() - timedelta(days=90)).isoformat()
+    rows = [
+        ("BTCUSDT", "HOLD", 0.0, "old", "{}", old_timestamp)
+        for _ in range(PRUNE_BATCH_SIZE + 5)
+    ]
+    with storage.connect() as conn:
+        conn.executemany(
+            """
+            INSERT INTO signals (symbol, action, confidence, reason, diagnostics_json, created_at)
+            VALUES (?, ?, ?, ?, ?, ?)
+            """,
+            rows,
+        )
+
+    deleted = storage.prune(30)
+
+    assert deleted["signals"] == PRUNE_BATCH_SIZE
+    assert len(storage.recent_signals(PRUNE_BATCH_SIZE + 10)) == 5
@@ -566,6 +566,43 @@ def test_torch_forecast_blocks_failed_quality_gate(make_settings, tmp_path) -> N
    assert signal.diagnostics["checks"]["quality_gate_ok"] is False


+def test_torch_forecast_allows_explicit_manual_quality_override(make_settings, tmp_path) -> None:
+    settings = make_settings(
+        tmp_path,
+        strategy_mode="torch_forecast",
+        time_series_require_quality_gate=True,
+        time_series_manual_quality_override=True,
+        time_series_min_edge_percent=0.10,
+        time_series_min_probability_up=0.57,
+        max_position_usdt=25,
+        stop_loss_percent=0.04,
+    )
+    strategy = SpotStrategy(settings)
+    ticker = Ticker("BTCUSDT", 105, 104.99, 105.01, 10_000_000, 1000, 1.0)
+
+    signal = strategy.entry_signal(
+        "BTCUSDT",
+        [],
+        ticker,
+        open_positions_for_symbol=0,
+        forecast={
+            "usable": True,
+            "model": "torch_gru",
+            "expected_return_percent": 0.36,
+            "probability_up": 0.66,
+            "skill": 0.22,
+            "block_entry": False,
+            "quality_gate_passed": False,
+            "quality_gate": {"status": "fail"},
+        },
+        account={"equity": 100.0},
+    )
+
+    assert signal.action == "BUY"
+    assert signal.diagnostics["checks"]["quality_gate_ok"] is True
+    assert signal.diagnostics["manual_quality_override"] is True
+
+
 def test_torch_forecast_probe_blocks_when_kelly_size_is_too_small(make_settings, tmp_path) -> None:
    settings = make_settings(
        tmp_path,
@@ -14,7 +14,11 @@ def _report(*, validation_passed: bool = True, trades: int = 30, total: float =
            "max_drawdown_percent": 1.0,
        },
        "walk_forward": {"summary": {"trades": trades, "avg_net_percent": 0.3}},
-        "validation": {"passed": validation_passed, "status": "pass" if validation_passed else "fail"},
+        "validation": {
+            "passed": validation_passed,
+            "status": "pass" if validation_passed else "fail",
+            "protocol": "untouched_model_holdout_with_threshold_walk_forward",
+        },
    }


@@ -4,6 +4,8 @@ import base64
 import hashlib
 import json

+import pytest
+
 from crypto_spot_bot.training_coordination import TrainingCoordinator


@@ -36,9 +38,20 @@ def test_training_coordinator_claims_and_completes_job(tmp_path) -> None:
    assert coordinator.status()["active_job"] is None


+def test_training_coordinator_preserves_boolean_resume_candidate_parameter(tmp_path) -> None:
+    coordinator = TrainingCoordinator(tmp_path)
+
+    requested = coordinator.request_retrain(
+        {"source": "recovery", "parameters": {"resume_candidate": True}}
+    )
+
+    assert requested["job"]["parameters"] == {"resume_candidate": True}
+
+
 def test_training_coordinator_accepts_chunked_artifact_upload(tmp_path) -> None:
    coordinator = TrainingCoordinator(tmp_path)
    job = coordinator.request_retrain({"source": "test"})["job"]
+    coordinator.claim({"worker_id": "test-worker"})
    payload = b'{"type":"pytorch_recurrent_forecaster","symbols":{}}\n'
    sha256 = hashlib.sha256(payload).hexdigest()
    first = payload[:20]
@@ -67,7 +80,8 @@ def test_training_coordinator_accepts_chunked_artifact_upload(tmp_path) -> None:

    assert part_1["complete"] is False
    assert part_2["complete"] is True
-    assert (tmp_path / "lstm_forecaster.json").read_bytes() == payload
+    assert not (tmp_path / "lstm_forecaster.json").exists()
+    assert (tmp_path / ".training_uploads" / job["id"] / "ready" / "lstm_forecaster.json").read_bytes() == payload
    assert coordinator.status()["latest_job"]["artifacts"][0]["sha256"] == sha256


@@ -86,3 +100,73 @@ def test_running_claimed_job_keeps_agent_online_when_heartbeat_is_stale(tmp_path
    assert status["agent_recently_seen"] is False
    assert status["agent_busy"] is True
    assert status["agent_online"] is True
+
+
+def test_training_upload_rejects_unknown_job(tmp_path) -> None:
+    coordinator = TrainingCoordinator(tmp_path)
+    payload = b"{}"
+
+    with pytest.raises(ValueError, match="not found"):
+        coordinator.save_artifact_chunk(
+            "11111111-1111-4111-8111-111111111111",
+            {
+                "name": "lstm_forecaster.json",
+                "index": 0,
+                "total": 1,
+                "sha256": hashlib.sha256(payload).hexdigest(),
+                "data_base64": base64.b64encode(payload).decode("ascii"),
+            },
+        )
+
+
+def test_training_bundle_promotes_only_after_successful_guard(tmp_path) -> None:
+    coordinator = TrainingCoordinator(tmp_path)
+    job = coordinator.request_retrain({"source": "test"})["job"]
+    coordinator.claim({"worker_id": "worker-1"})
+    model = {
+        "type": "pytorch_recurrent_forecaster",
+        "symbols": {
+            "BTCUSDT": {
+                "model": "torch_gru",
+                "lookback": 4,
+                "input_size": 1,
+                "hidden_size": 1,
+                "state_dict": {"weight_ih_l0": [[0.0]]},
+                "head_weight": [[0.0]],
+                "head_bias": [0.0],
+            }
+        },
+    }
+    model_payload = (json.dumps(model) + "\n").encode()
+    model_sha256 = hashlib.sha256(model_payload).hexdigest()
+    artifacts = {
+        "lstm_forecaster.json": model,
+        "torch_retrain_guard.json": {
+            "accepted": True,
+            "candidate_artifact_sha256": model_sha256,
+        },
+        "torch_threshold_calibration.json": {
+            "artifact_sha256": model_sha256,
+            "validation": {
+                "passed": True,
+                "protocol": "untouched_model_holdout_with_threshold_walk_forward",
+            }
+        },
+    }
+    for name, data in artifacts.items():
+        payload = model_payload if name == "lstm_forecaster.json" else (json.dumps(data) + "\n").encode()
+        coordinator.save_artifact_chunk(
+            job["id"],
+            {
+                "name": name,
+                "index": 0,
+                "total": 1,
+                "sha256": hashlib.sha256(payload).hexdigest(),
+                "data_base64": base64.b64encode(payload).decode("ascii"),
+            },
+        )
+
+    completed = coordinator.complete(job["id"], {"success": True})
+
+    assert completed["job"]["status"] == "completed"
+    assert json.loads((tmp_path / "lstm_forecaster.json").read_text())["symbols"]["BTCUSDT"]