diff --git a/.env.example b/.env.example index 2fe2ada..81ed191 100644 --- a/.env.example +++ b/.env.example @@ -7,9 +7,9 @@ BYBIT_API_KEY= BYBIT_API_SECRET= STARTING_BALANCE_USDT=100 -AUTO_SELECT_SYMBOLS=true +AUTO_SELECT_SYMBOLS=false TOP_SYMBOLS_COUNT=6 -SYMBOLS= +SYMBOLS=BTCUSDT,ETHUSDT,HYPEUSDT,SOLUSDT,LTCUSDT,XRPUSDT BASE_INTERVAL=1 KLINE_LIMIT=240 @@ -50,9 +50,6 @@ TIME_SERIES_EWMA_LAMBDA=0.94 TIME_SERIES_MIN_EDGE_PERCENT=0.04 TIME_SERIES_MAX_ADJUSTMENT=0.08 TIME_SERIES_LSTM_ENABLED=true -TIME_SERIES_LSTM_LOOKBACK=32 -TIME_SERIES_LSTM_UNITS=6 -TIME_SERIES_LSTM_RIDGE=0.0001 TIME_SERIES_LSTM_MODEL_PATH=runtime/lstm_forecaster.json STOP_LOSS_PERCENT=0.02 TAKE_PROFIT_PERCENT=0.035 diff --git a/README.md b/README.md index 16a26c6..d462a56 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Spot-бот для демо-торговли криптовалютой на р - Динамический размер позиции: стратегия записывает в сигнал размер входа в пределах `MIN_POSITION_USDT`..`MAX_POSITION_USDT`, а брокер ограничивает суммарную экспозицию по паре через `MAX_SYMBOL_EXPOSURE_USDT`. - Автоматический grid-режим: бот включает grid-входы на боковике, покупает только в нижней части диапазона и выключает grid при падающих/опасных режимах. - Вероятностный rebound-вход: после снижения бот отдельно оценивает стабилизацию, отскок от локального low, RSI, объем и рыночные ограничения; такой вход ограничен меньшим размером позиции. -- Прогнозирование временных рядов: walk-forward выбор между `naive`, `drift`, `EWMA`, `AR(1)`, `AR(3)` и легким `lstm`-кандидатом для ожидаемой доходности плюс EWMA/GARCH-like прогноз волатильности. Прогноз влияет и на новые покупки, и на раннюю продажу при ухудшении ожидаемого движения. +- Прогнозирование временных рядов: walk-forward выбор между `naive`, `drift`, `EWMA`, `AR(1)`, `AR(3)` и экспортированными PyTorch `LSTM/GRU`-моделями для ожидаемой доходности плюс EWMA/GARCH-like прогноз волатильности. Прогноз влияет и на новые покупки, и на раннюю продажу при ухудшении ожидаемого движения. - Защитные блокировки входа: явно отрицательные LONG-шаблоны и setups с сильной отрицательной статистикой обучения запрещают новые покупки. - Быстрый режим торговли: отдельный короткий интервал цикла, короткий cooldown после выхода и лимит новых входов в минуту; выходы по риску этим лимитом не блокируются. - Веб-dashboard на русском: equity, cash, PnL, позиции, сделки, сигналы, события, свечные графики, переключатель быстрой торговли и индикаторы работы обучения. @@ -46,7 +46,7 @@ Live market orders используют `/v5/order/create`; Bybit докумен - Документация `statsmodels` описывает ARIMA как общий интерфейс для AR/MA/ARMA/ARIMA/SARIMA-моделей; в боте используется легкий AR(1)/AR(3) вариант без добавления тяжелой зависимости `statsmodels`: . - Документация `arch` описывает GARCH(p,q) как модель для прогнозирования волатильности; в боте используется фиксированная GARCH(1,1)-подобная рекурсия без MLE-оценки параметров, чтобы сохранить легкий runtime на Raspberry Pi: . - RiskMetrics описывает EWMA-подход к оценке волатильности через коэффициент затухания; в боте `TIME_SERIES_EWMA_LAMBDA=0.94` используется как настраиваемое значение по умолчанию: . -- Hochreiter и Schmidhuber описали LSTM как recurrent neural network architecture для последовательностей; в боте используется легкая LSTM-reservoir рекурсия с ridge-readout, а не полноценное PyTorch/TensorFlow обучение внутри Docker: . +- Hochreiter и Schmidhuber описали LSTM как recurrent neural network architecture для последовательностей; обучение LSTM/GRU в проекте выполняется локально через PyTorch, а Raspberry Pi исполняет только экспортированные JSON-веса без PyTorch runtime: . Я не могу подтвердить, что эта стратегия будет прибыльной. Источники выше описывают технические свойства и риски автоматической торговли, но не гарантируют прибыль. @@ -62,22 +62,13 @@ python -m crypto_spot_bot.main Dashboard: -## Локальное обучение LSTM-кандидата +## Локальное обучение PyTorch LSTM/GRU -Обучение можно запускать на основной машине, а Raspberry Pi оставлять только для исполнения торгового цикла. Команда ниже берет spot-свечи Bybit, перебирает `lookback`, `units` и `ridge`, оценивает LSTM-кандидат через walk-forward MAE и сохраняет параметры в `runtime/lstm_forecaster.json`: - -```powershell -python tools\train_lstm_forecaster.py --symbols BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,LTCUSDT --limit 1000 -``` - -Файл из `TIME_SERIES_LSTM_MODEL_PATH` читается ботом автоматически. Даже если LSTM-параметры сохранены, сделка меняется только тогда, когда текущая walk-forward проверка в `crypto_spot_bot/time_series.py` показывает качество лучше baseline. - -Для более тяжелого локального обучения можно использовать настоящий PyTorch `LSTM/GRU` trainer. PyTorch нужен только на машине обучения; в JSON экспортируются веса, а runtime на Raspberry Pi считает inference обычным Python-кодом: +Обучение запускается на основной Windows-машине, а Raspberry Pi остается только для исполнения торгового цикла. PyTorch нужен только на машине обучения; в JSON экспортируются веса, а runtime на Raspberry Pi считает inference обычным Python-кодом: ```powershell .\.venv\Scripts\python.exe -m pip install torch --index-url https://download.pytorch.org/whl/cpu .\.venv\Scripts\python.exe tools\train_torch_recurrent_forecaster.py ` - --symbols BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,LTCUSDT ` --limit 1000 ` --architectures lstm,gru ` --lookbacks 32,64 ` @@ -86,23 +77,16 @@ python tools\train_lstm_forecaster.py --symbols BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT, --epochs 60 ``` -Экспортированные модели появляются в dashboard как `torch_lstm` или `torch_gru`; легкий `lstm`-кандидат остается доступен как fallback. +Файл из `TIME_SERIES_LSTM_MODEL_PATH` читается ботом автоматически. Экспортированные модели появляются в dashboard как `PyTorch LSTM` или `PyTorch GRU`; старый легкий reservoir LSTM-кандидат удален и больше не участвует в выборе модели. -Автопереобучение запускает тот же train-скрипт, пишет лог в `runtime/lstm_retrain.log` и защищается от параллельных запусков: +Автопереобучение на Windows запускает PyTorch trainer, пишет лог в `runtime/torch_retrain.log` и защищается от параллельных запусков: ```powershell -powershell -ExecutionPolicy Bypass -File tools\run_lstm_retrain.ps1 -powershell -ExecutionPolicy Bypass -File tools\install_windows_lstm_retrainer.ps1 +powershell -ExecutionPolicy Bypass -File tools\run_torch_retrain.ps1 +powershell -ExecutionPolicy Bypass -File tools\install_windows_torch_retrainer.ps1 ``` -На Linux/Raspberry Pi можно включить user systemd timer: - -```bash -bash tools/run_lstm_retrain.sh -bash tools/install_lstm_retrainer_systemd.sh -``` - -По умолчанию Windows-расписание переобучает PyTorch `LSTM/GRU` каждые 6 часов с `--limit 1000`; Windows-установщик фиксирует пары `BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,LTCUSDT`, чтобы первый scheduled run был предсказуемым. Параметры можно переопределить через env: `LSTM_RETRAIN_SYMBOLS`, `LSTM_RETRAIN_LIMIT`, `LSTM_RETRAIN_LOOKBACKS`, `LSTM_RETRAIN_ARCHITECTURES`, `LSTM_RETRAIN_HIDDEN_SIZES`, `LSTM_RETRAIN_LAYERS`, `LSTM_RETRAIN_DROPOUTS`, `LSTM_RETRAIN_EPOCHS`, `LSTM_RETRAIN_PATIENCE`, `LSTM_RETRAIN_INTERVAL`, `LSTM_RETRAIN_ENV`. Для старого легкого trainer можно запустить `tools\run_lstm_retrain.ps1 -Trainer reservoir`. +По умолчанию Windows-расписание переобучает PyTorch `LSTM/GRU` каждые 6 часов с `--limit 1000` на фиксированных парах `BTCUSDT,ETHUSDT,HYPEUSDT,SOLUSDT,LTCUSDT,XRPUSDT`. Параметры можно переопределить через env: `TORCH_RETRAIN_SYMBOLS`, `TORCH_RETRAIN_LIMIT`, `TORCH_RETRAIN_LOOKBACKS`, `TORCH_RETRAIN_ARCHITECTURES`, `TORCH_RETRAIN_HIDDEN_SIZES`, `TORCH_RETRAIN_LAYERS`, `TORCH_RETRAIN_DROPOUTS`, `TORCH_RETRAIN_EPOCHS`, `TORCH_RETRAIN_PATIENCE`, `TORCH_RETRAIN_INTERVAL`, `TORCH_RETRAIN_ENV`. ## Docker @@ -121,8 +105,9 @@ Dashboard: `http://:8787/` ```env TRADING_MODE=paper STARTING_BALANCE_USDT=100 -AUTO_SELECT_SYMBOLS=true +AUTO_SELECT_SYMBOLS=false TOP_SYMBOLS_COUNT=6 +SYMBOLS=BTCUSDT,ETHUSDT,HYPEUSDT,SOLUSDT,LTCUSDT,XRPUSDT BASE_INTERVAL=1 LOOP_INTERVAL_SECONDS=5 FAST_TRADING_ENABLED=false @@ -159,9 +144,6 @@ TIME_SERIES_EWMA_LAMBDA=0.94 TIME_SERIES_MIN_EDGE_PERCENT=0.04 TIME_SERIES_MAX_ADJUSTMENT=0.08 TIME_SERIES_LSTM_ENABLED=true -TIME_SERIES_LSTM_LOOKBACK=32 -TIME_SERIES_LSTM_UNITS=6 -TIME_SERIES_LSTM_RIDGE=0.0001 TIME_SERIES_LSTM_MODEL_PATH=runtime/lstm_forecaster.json STOP_LOSS_PERCENT=0.02 TAKE_PROFIT_PERCENT=0.035 diff --git a/crypto_spot_bot/config.py b/crypto_spot_bot/config.py index 3242a06..6c5fe4e 100644 --- a/crypto_spot_bot/config.py +++ b/crypto_spot_bot/config.py @@ -5,6 +5,9 @@ from dataclasses import dataclass from pathlib import Path +FIXED_SPOT_SYMBOLS = ("BTCUSDT", "ETHUSDT", "HYPEUSDT", "SOLUSDT", "LTCUSDT", "XRPUSDT") + + def _load_dotenv(path: Path) -> None: if not path.exists(): return @@ -97,9 +100,6 @@ class Settings: time_series_min_edge_percent: float time_series_max_adjustment: float time_series_lstm_enabled: bool - time_series_lstm_lookback: int - time_series_lstm_units: int - time_series_lstm_ridge: float time_series_lstm_model_path: Path stop_loss_percent: float take_profit_percent: float @@ -172,9 +172,9 @@ def load_settings(env_file: str | Path | None = None) -> Settings: bybit_api_key=os.getenv("BYBIT_API_KEY", ""), bybit_api_secret=os.getenv("BYBIT_API_SECRET", ""), starting_balance_usdt=_float_env("STARTING_BALANCE_USDT", 100.0), - auto_select_symbols=_bool_env("AUTO_SELECT_SYMBOLS", True), - top_symbols_count=_int_env("TOP_SYMBOLS_COUNT", 6), - symbols=_symbols_env("SYMBOLS"), + auto_select_symbols=_bool_env("AUTO_SELECT_SYMBOLS", False), + top_symbols_count=_int_env("TOP_SYMBOLS_COUNT", len(FIXED_SPOT_SYMBOLS)), + symbols=_symbols_env("SYMBOLS") or FIXED_SPOT_SYMBOLS, base_interval=os.getenv("BASE_INTERVAL", "1"), kline_limit=_int_env("KLINE_LIMIT", 240), loop_interval_seconds=_int_env("LOOP_INTERVAL_SECONDS", 5), @@ -220,9 +220,6 @@ def load_settings(env_file: str | Path | None = None) -> Settings: time_series_min_edge_percent=_float_env("TIME_SERIES_MIN_EDGE_PERCENT", 0.04), time_series_max_adjustment=_float_env("TIME_SERIES_MAX_ADJUSTMENT", 0.08), time_series_lstm_enabled=_bool_env("TIME_SERIES_LSTM_ENABLED", True), - time_series_lstm_lookback=_int_env("TIME_SERIES_LSTM_LOOKBACK", 32), - time_series_lstm_units=_int_env("TIME_SERIES_LSTM_UNITS", 6), - time_series_lstm_ridge=_float_env("TIME_SERIES_LSTM_RIDGE", 0.0001), time_series_lstm_model_path=Path(os.getenv("TIME_SERIES_LSTM_MODEL_PATH", "runtime/lstm_forecaster.json")), stop_loss_percent=_float_env("STOP_LOSS_PERCENT", 0.02), take_profit_percent=_float_env("TAKE_PROFIT_PERCENT", 0.035), diff --git a/crypto_spot_bot/dashboard.py b/crypto_spot_bot/dashboard.py index 415c002..304d1a6 100644 --- a/crypto_spot_bot/dashboard.py +++ b/crypto_spot_bot/dashboard.py @@ -217,9 +217,6 @@ def _safe_config(settings: Settings) -> dict[str, Any]: "time_series_min_edge_percent": settings.time_series_min_edge_percent, "time_series_max_adjustment": settings.time_series_max_adjustment, "time_series_lstm_enabled": settings.time_series_lstm_enabled, - "time_series_lstm_lookback": settings.time_series_lstm_lookback, - "time_series_lstm_units": settings.time_series_lstm_units, - "time_series_lstm_ridge": settings.time_series_lstm_ridge, "time_series_lstm_model_path": str(settings.time_series_lstm_model_path), "time_series_model_artifact": _time_series_model_artifact(settings), "stop_loss_percent": settings.stop_loss_percent, @@ -266,16 +263,19 @@ def _time_series_model_artifact(settings: Settings) -> dict[str, Any]: } ) artifact_type = str(data.get("type", "")).strip() - if artifact_type == "pytorch_recurrent_forecaster": - label = "PyTorch LSTM/GRU" - elif artifact_type == "lstm_reservoir_ridge_params": - label = "легкий LSTM fallback" - else: - label = artifact_type or "настройки прогноза" + if artifact_type != "pytorch_recurrent_forecaster": + return { + "available": False, + "type": artifact_type or "unknown", + "label": "устаревший файл модели не используется", + "created_at": data.get("created_at", ""), + "symbol_count": len(rows), + "models": models, + } return { "available": True, - "type": artifact_type or "unknown", - "label": label, + "type": artifact_type, + "label": "PyTorch LSTM/GRU", "created_at": data.get("created_at", ""), "symbol_count": len(rows), "models": models, @@ -289,7 +289,7 @@ def _forecast_model_label(model: str) -> str: if normalized == "torch_gru": return "PyTorch GRU" if normalized == "lstm": - return "легкий LSTM" + return "устаревший LSTM" if normalized == "gru": return "GRU" return model @@ -742,7 +742,7 @@ HTML = r""" const names = { torch_lstm: 'PyTorch LSTM', torch_gru: 'PyTorch GRU', - lstm: 'Легкий LSTM', + lstm: 'Устаревший LSTM', naive: 'Baseline', drift: 'Drift', ewma: 'EWMA', @@ -757,7 +757,7 @@ HTML = r""" return String(reason || '') .replaceAll('torch_lstm', 'PyTorch LSTM') .replaceAll('torch_gru', 'PyTorch GRU') - .replaceAll('модель lstm', 'модель легкий LSTM'); + .replaceAll('модель lstm', 'модель устаревший LSTM'); } function modelArtifactSummary(config) { diff --git a/crypto_spot_bot/time_series.py b/crypto_spot_bot/time_series.py index ffe0c80..f1040bb 100644 --- a/crypto_spot_bot/time_series.py +++ b/crypto_spot_bot/time_series.py @@ -3,7 +3,6 @@ from __future__ import annotations import json import math from dataclasses import asdict, dataclass, field -from functools import lru_cache from typing import Any from crypto_spot_bot.config import Settings @@ -174,8 +173,6 @@ def _validate_candidates( torch_model = _torch_recurrent_model_name(symbol, lstm_artifact or {}) if torch_model and _can_use_torch_recurrent(returns, symbol, lstm_artifact or {}): models.append(torch_model) - if _can_use_lstm(returns, settings, symbol, lstm_artifact or {}): - models.append("lstm") rows: list[dict[str, float | str]] = [] start = max(8, len(returns) - validation_window) for model in models: @@ -211,8 +208,6 @@ def _predict_next_return( return _ar_predict(returns, 3) if model in {"torch_lstm", "torch_gru"}: return _torch_recurrent_predict(returns, symbol, lstm_artifact or {}) - if model == "lstm" and settings is not None: - return _lstm_predict(returns, settings, symbol, lstm_artifact or {}) return 0.0 @@ -244,52 +239,6 @@ def _ar_predict(returns: list[float], lag_count: int) -> float: return _clamp(prediction, -cap, cap) -def _can_use_lstm( - returns: list[float], - settings: Settings, - symbol: str | None, - lstm_artifact: dict[str, Any], -) -> bool: - if not settings.time_series_lstm_enabled: - return False - params = _lstm_params(settings, symbol, lstm_artifact) - return len(returns) >= params["lookback"] + 16 - - -def _lstm_params(settings: Settings, symbol: str | None, lstm_artifact: dict[str, Any]) -> dict[str, float | int]: - params: dict[str, float | int] = { - "lookback": settings.time_series_lstm_lookback, - "units": settings.time_series_lstm_units, - "ridge": settings.time_series_lstm_ridge, - } - default_params = lstm_artifact.get("default") - if isinstance(default_params, dict): - params.update(_clean_lstm_params(default_params)) - symbols = lstm_artifact.get("symbols") - symbol_params = symbols.get(symbol.upper()) if symbol and isinstance(symbols, dict) else None - if isinstance(symbol_params, dict): - params.update(_clean_lstm_params(symbol_params)) - return { - "lookback": int(_clamp(float(params["lookback"]), 6.0, 128.0)), - "units": int(_clamp(float(params["units"]), 2.0, 16.0)), - "ridge": _clamp(float(params["ridge"]), 1e-8, 0.5), - } - - -def _clean_lstm_params(data: dict[str, Any]) -> dict[str, float | int]: - clean: dict[str, float | int] = {} - for key in ("lookback", "units", "ridge"): - value = data.get(key) - if isinstance(value, (int, float)): - clean[key] = value - elif isinstance(value, str): - try: - clean[key] = float(value) - except ValueError: - continue - return clean - - def _torch_recurrent_model_name(symbol: str | None, lstm_artifact: dict[str, Any]) -> str | None: entry = _torch_recurrent_entry(symbol, lstm_artifact) if not entry: @@ -519,39 +468,6 @@ def _dot(left: list[float], right: list[float]) -> float: return sum(left[index] * right[index] for index in range(min(len(left), len(right)))) -def _lstm_predict( - returns: list[float], - settings: Settings, - symbol: str | None, - lstm_artifact: dict[str, Any], -) -> float: - params = _lstm_params(settings, symbol, lstm_artifact) - lookback = int(params["lookback"]) - units = int(params["units"]) - ridge = float(params["ridge"]) - if len(returns) <= lookback + 8: - return _predict_next_return("drift", returns) - - scale = _return_scale(returns) - normalized = [_clamp(value / scale, -6.0, 6.0) for value in returns] - states = _lstm_states(normalized, units) - rows: list[list[float]] = [] - targets: list[float] = [] - for index in range(lookback, len(returns)): - rows.append([1.0] + states[index - 1]) - targets.append(normalized[index]) - coeffs = _ols(rows, targets, ridge) - if not coeffs: - return _predict_next_return("drift", returns) - - features = [1.0] + states[-1] - prediction = sum(coeff * feature for coeff, feature in zip(coeffs, features)) - prediction = _clamp(prediction, -4.0, 4.0) * scale - recent_abs = sorted(abs(value) for value in returns[-48:]) if len(returns) >= 8 else [0.01] - cap = max(recent_abs[int(len(recent_abs) * 0.9)], 0.0002) - return _clamp(prediction, -cap, cap) - - def _return_scale(returns: list[float]) -> float: recent = returns[-120:] if len(returns) > 120 else returns values = sorted(abs(value) for value in recent if math.isfinite(value)) @@ -562,67 +478,6 @@ def _return_scale(returns: list[float]) -> float: return max(max(median, mean * 0.5), 1e-5) -def _lstm_states(normalized_returns: list[float], units: int) -> list[list[float]]: - weights = _lstm_weights(units) - hidden = [0.0 for _ in range(units)] - cell = [0.0 for _ in range(units)] - states: list[list[float]] = [] - for value in normalized_returns: - hidden, cell = _lstm_step(value, hidden, cell, weights) - states.append(hidden[:]) - return states - - -@lru_cache(maxsize=16) -def _lstm_weights(units: int) -> tuple[list[list[float]], list[list[list[float]]], list[list[float]]]: - input_weights: list[list[float]] = [] - recurrent_weights: list[list[list[float]]] = [] - biases: list[list[float]] = [] - base_biases = (-0.15, 0.70, 0.05, 0.0) - for gate in range(4): - gate_input: list[float] = [] - gate_recurrent: list[list[float]] = [] - gate_bias: list[float] = [] - for unit in range(units): - gate_input.append(0.55 * math.sin((gate + 1) * (unit + 1) * 1.61803398875)) - gate_recurrent.append( - [ - 0.14 * math.sin((gate + 3) * (unit + 1) * (source + 1) * 0.731) - for source in range(units) - ] - ) - gate_bias.append(base_biases[gate] + 0.03 * math.sin((gate + 1) * (unit + 1))) - input_weights.append(gate_input) - recurrent_weights.append(gate_recurrent) - biases.append(gate_bias) - return input_weights, recurrent_weights, biases - - -def _lstm_step( - value: float, - hidden: list[float], - cell: list[float], - weights: tuple[list[list[float]], list[list[list[float]]], list[list[float]]], -) -> tuple[list[float], list[float]]: - input_weights, recurrent_weights, biases = weights - units = len(hidden) - next_hidden = [0.0 for _ in range(units)] - next_cell = [0.0 for _ in range(units)] - for unit in range(units): - gate_values = [] - for gate in range(4): - raw = input_weights[gate][unit] * value + biases[gate][unit] - raw += sum(recurrent_weights[gate][unit][source] * hidden[source] for source in range(units)) - gate_values.append(raw) - input_gate = _sigmoid(gate_values[0]) - forget_gate = _sigmoid(gate_values[1]) - output_gate = _sigmoid(gate_values[2]) - candidate = math.tanh(gate_values[3]) - next_cell[unit] = forget_gate * cell[unit] + input_gate * candidate - next_hidden[unit] = output_gate * math.tanh(next_cell[unit]) - return next_hidden, next_cell - - def _sigmoid(value: float) -> float: if value >= 40: return 1.0 diff --git a/tests/conftest.py b/tests/conftest.py index 46d4a26..a80834c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -66,9 +66,6 @@ def make_settings(): time_series_min_edge_percent=0.04, time_series_max_adjustment=0.08, time_series_lstm_enabled=True, - time_series_lstm_lookback=32, - time_series_lstm_units=6, - time_series_lstm_ridge=0.0001, time_series_lstm_model_path=tmp_path / "lstm_forecaster.json", stop_loss_percent=0.02, take_profit_percent=0.035, diff --git a/tests/test_config.py b/tests/test_config.py index 0e5d3b8..e0d7948 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -2,7 +2,7 @@ from __future__ import annotations import pytest -from crypto_spot_bot.config import load_settings +from crypto_spot_bot.config import FIXED_SPOT_SYMBOLS, load_settings def test_live_mode_requires_explicit_unlock(tmp_path, monkeypatch) -> None: @@ -61,3 +61,18 @@ def test_llm_advisor_is_disabled_by_default(tmp_path, monkeypatch) -> None: settings = load_settings(env_file) assert settings.llm_advisor_enabled is False + + +def test_default_symbols_are_fixed_six_pairs(tmp_path, monkeypatch) -> None: + monkeypatch.delenv("AUTO_SELECT_SYMBOLS", raising=False) + monkeypatch.delenv("TOP_SYMBOLS_COUNT", raising=False) + monkeypatch.delenv("SYMBOLS", raising=False) + monkeypatch.setenv("TRADING_MODE", "paper") + env_file = tmp_path / ".env" + env_file.write_text("TRADING_MODE=paper\nSYMBOLS=\n", encoding="utf-8") + + settings = load_settings(env_file) + + assert settings.auto_select_symbols is False + assert settings.top_symbols_count == 6 + assert settings.symbols == FIXED_SPOT_SYMBOLS diff --git a/tests/test_time_series.py b/tests/test_time_series.py index aee22dd..8b90a8d 100644 --- a/tests/test_time_series.py +++ b/tests/test_time_series.py @@ -76,32 +76,13 @@ def test_time_series_forecaster_blocks_negative_edge(make_settings, tmp_path) -> assert forecast.block_entry is True -def test_time_series_forecaster_includes_lstm_candidate(make_settings, tmp_path) -> None: - settings = make_settings( - tmp_path, - time_series_min_candles=80, - time_series_validation_window=20, - time_series_lstm_enabled=True, - time_series_lstm_lookback=12, - time_series_lstm_units=4, - ) - returns = [] - for index in range(140): - seasonal = 0.00018 if index % 5 in {0, 1, 2} else -0.00011 - returns.append(seasonal + 0.00002 * ((index % 7) - 3)) - - forecast = TimeSeriesForecaster(settings).forecast(_candles_from_returns(returns), symbol="BTCUSDT") - - assert forecast.usable is True - assert any(candidate["model"] == "lstm" for candidate in forecast.candidates) - - -def test_time_series_forecaster_reads_lstm_artifact(make_settings, tmp_path) -> None: +def test_time_series_forecaster_ignores_legacy_lstm_artifact(make_settings, tmp_path) -> None: artifact_path = tmp_path / "lstm_forecaster.json" artifact_path.write_text( json.dumps( { "version": 1, + "type": "lstm_reservoir_ridge_params", "symbols": { "BTCUSDT": {"lookback": 10, "units": 3, "ridge": 0.01}, }, @@ -121,7 +102,7 @@ def test_time_series_forecaster_reads_lstm_artifact(make_settings, tmp_path) -> forecast = TimeSeriesForecaster(settings).forecast(_candles_from_returns(returns), symbol="BTCUSDT") assert forecast.usable is True - assert any(candidate["model"] == "lstm" for candidate in forecast.candidates) + assert all(candidate["model"] != "lstm" for candidate in forecast.candidates) def test_time_series_forecaster_reads_torch_gru_artifact(make_settings, tmp_path) -> None: diff --git a/tools/install_lstm_retrainer_systemd.sh b/tools/install_lstm_retrainer_systemd.sh deleted file mode 100755 index 5ba96ee..0000000 --- a/tools/install_lstm_retrainer_systemd.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -SYSTEMD_DIR="$HOME/.config/systemd/user" -SERVICE_NAME="tradebot-lstm-retrainer.service" -TIMER_NAME="tradebot-lstm-retrainer.timer" - -mkdir -p "$SYSTEMD_DIR" - -cat > "$SYSTEMD_DIR/$SERVICE_NAME" < "$SYSTEMD_DIR/$TIMER_NAME" <&1 | Tee-Object -FilePath $LogFile -Append - if ($LASTEXITCODE -ne 0) { - throw "Trainer failed with exit code $LASTEXITCODE." - } - Write-RetrainLog "Finished LSTM retrain." -} -catch { - Write-RetrainLog "ERROR: $($_.Exception.Message)" - exit 1 -} -finally { - if ($pushedLocation) { - Pop-Location -ErrorAction SilentlyContinue - } - if ($hasLock) { - $mutex.ReleaseMutex() - } - $mutex.Dispose() -} diff --git a/tools/run_lstm_retrain.sh b/tools/run_lstm_retrain.sh deleted file mode 100755 index 962b93e..0000000 --- a/tools/run_lstm_retrain.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -RUNTIME_DIR="$REPO_ROOT/runtime" -LOG_FILE="$RUNTIME_DIR/lstm_retrain.log" -LOCK_FILE="$RUNTIME_DIR/lstm_retrain.lock" - -mkdir -p "$RUNTIME_DIR" - -log() { - printf '[%s] %s\n' "$(date -Is)" "$*" | tee -a "$LOG_FILE" -} - -if command -v flock >/dev/null 2>&1; then - exec 9>"$LOCK_FILE" - if ! flock -n 9; then - log "Another LSTM retrain is already running; skipping." - exit 0 - fi -else - LOCK_DIR="$LOCK_FILE.d" - if ! mkdir "$LOCK_DIR" 2>/dev/null; then - log "Another LSTM retrain is already running; skipping." - exit 0 - fi - trap 'rmdir "$LOCK_DIR"' EXIT -fi - -if [[ -x "$REPO_ROOT/.venv/bin/python" ]]; then - PYTHON="$REPO_ROOT/.venv/bin/python" -elif command -v python3 >/dev/null 2>&1; then - PYTHON="$(command -v python3)" -elif command -v python >/dev/null 2>&1; then - PYTHON="$(command -v python)" -else - log "ERROR: Python was not found." - exit 1 -fi - -SYMBOLS="${LSTM_RETRAIN_SYMBOLS:-}" -LIMIT="${LSTM_RETRAIN_LIMIT:-1000}" -LOOKBACKS="${LSTM_RETRAIN_LOOKBACKS:-16,32}" -UNITS="${LSTM_RETRAIN_UNITS:-4,6}" -RIDGES="${LSTM_RETRAIN_RIDGES:-0.001}" -INTERVAL="${LSTM_RETRAIN_INTERVAL:-}" -ENV_FILE="${LSTM_RETRAIN_ENV:-}" - -if [[ -z "$ENV_FILE" && -f "$REPO_ROOT/.env" ]]; then - ENV_FILE="$REPO_ROOT/.env" -fi - -args=( - "tools/train_lstm_forecaster.py" - "--limit" "$LIMIT" - "--lookbacks" "$LOOKBACKS" - "--units" "$UNITS" - "--ridges" "$RIDGES" -) - -if [[ -n "$SYMBOLS" ]]; then args+=("--symbols" "$SYMBOLS"); fi -if [[ -n "$INTERVAL" ]]; then args+=("--interval" "$INTERVAL"); fi -if [[ -n "$ENV_FILE" ]]; then args+=("--env" "$ENV_FILE"); fi - -cd "$REPO_ROOT" -log "Starting LSTM retrain: $PYTHON -u ${args[*]}" -"$PYTHON" -u "${args[@]}" 2>&1 | tee -a "$LOG_FILE" -log "Finished LSTM retrain." diff --git a/tools/run_torch_retrain.ps1 b/tools/run_torch_retrain.ps1 new file mode 100644 index 0000000..3baaf0a --- /dev/null +++ b/tools/run_torch_retrain.ps1 @@ -0,0 +1,114 @@ +[CmdletBinding()] +param( + [string]$Symbols = "", + [int]$Limit = 0, + [string]$Lookbacks = "", + [string]$Architectures = "", + [string]$HiddenSizes = "", + [string]$Layers = "", + [string]$Dropouts = "", + [int]$Epochs = 0, + [int]$Patience = 0, + [string]$Interval = "", + [string]$EnvFile = "" +) + +$ErrorActionPreference = "Stop" + +$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +$RuntimeDir = Join-Path $RepoRoot "runtime" +$LogFile = Join-Path $RuntimeDir "torch_retrain.log" +New-Item -ItemType Directory -Force -Path $RuntimeDir | Out-Null + +function Write-RetrainLog { + param([string]$Message) + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ssK" + "[$timestamp] $Message" | Tee-Object -FilePath $LogFile -Append +} + +function Resolve-Python { + $venvPython = Join-Path $RepoRoot ".venv\Scripts\python.exe" + if (Test-Path $venvPython) { + return $venvPython + } + + $userPython = Join-Path $env:LOCALAPPDATA "Programs\TradeBotPython312\python.exe" + if (Test-Path $userPython) { + return $userPython + } + + foreach ($candidate in @("python.exe", "python")) { + $command = Get-Command $candidate -ErrorAction SilentlyContinue + if (-not $command) { + continue + } + return $command.Source + } + throw "Python was not found. Create .venv or install Python 3.12." +} + +if (-not $Symbols -and $env:TORCH_RETRAIN_SYMBOLS) { $Symbols = $env:TORCH_RETRAIN_SYMBOLS } +if ($Limit -le 0) { + $Limit = if ($env:TORCH_RETRAIN_LIMIT) { [int]$env:TORCH_RETRAIN_LIMIT } else { 1000 } +} +if (-not $Lookbacks) { $Lookbacks = if ($env:TORCH_RETRAIN_LOOKBACKS) { $env:TORCH_RETRAIN_LOOKBACKS } else { "32,64" } } +if (-not $Architectures) { $Architectures = if ($env:TORCH_RETRAIN_ARCHITECTURES) { $env:TORCH_RETRAIN_ARCHITECTURES } else { "lstm,gru" } } +if (-not $HiddenSizes) { $HiddenSizes = if ($env:TORCH_RETRAIN_HIDDEN_SIZES) { $env:TORCH_RETRAIN_HIDDEN_SIZES } else { "16,32" } } +if (-not $Layers) { $Layers = if ($env:TORCH_RETRAIN_LAYERS) { $env:TORCH_RETRAIN_LAYERS } else { "1" } } +if (-not $Dropouts) { $Dropouts = if ($env:TORCH_RETRAIN_DROPOUTS) { $env:TORCH_RETRAIN_DROPOUTS } else { "0.0" } } +if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 60 } } +if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 10 } } +if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RETRAIN_INTERVAL } +if (-not $EnvFile -and $env:TORCH_RETRAIN_ENV) { $EnvFile = $env:TORCH_RETRAIN_ENV } +if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" } + +$mutex = New-Object System.Threading.Mutex($false, "TradeBotTorchRecurrentRetrainer") +$hasLock = $false +$pushedLocation = $false + +try { + $hasLock = $mutex.WaitOne(0) + if (-not $hasLock) { + Write-RetrainLog "Another PyTorch recurrent retrain is already running; skipping." + exit 0 + } + + $python = Resolve-Python + $trainerArgs = @( + "-u", + "tools\train_torch_recurrent_forecaster.py", + "--limit", $Limit.ToString(), + "--lookbacks", $Lookbacks, + "--architectures", $Architectures, + "--hidden-sizes", $HiddenSizes, + "--layers", $Layers, + "--dropouts", $Dropouts, + "--epochs", $Epochs.ToString(), + "--patience", $Patience.ToString() + ) + if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) } + if ($Interval) { $trainerArgs += @("--interval", $Interval) } + if ($EnvFile) { $trainerArgs += @("--env", $EnvFile) } + + Push-Location $RepoRoot + $pushedLocation = $true + Write-RetrainLog "Starting PyTorch recurrent retrain: $python $($trainerArgs -join ' ')" + & $python @trainerArgs 2>&1 | Tee-Object -FilePath $LogFile -Append + if ($LASTEXITCODE -ne 0) { + throw "Trainer failed with exit code $LASTEXITCODE." + } + Write-RetrainLog "Finished PyTorch recurrent retrain." +} +catch { + Write-RetrainLog "ERROR: $($_.Exception.Message)" + exit 1 +} +finally { + if ($pushedLocation) { + Pop-Location -ErrorAction SilentlyContinue + } + if ($hasLock) { + $mutex.ReleaseMutex() + } + $mutex.Dispose() +} diff --git a/tools/train_lstm_forecaster.py b/tools/train_lstm_forecaster.py deleted file mode 100644 index fd86a2e..0000000 --- a/tools/train_lstm_forecaster.py +++ /dev/null @@ -1,146 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import sys -from dataclasses import replace -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -if str(PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(PROJECT_ROOT)) - -from crypto_spot_bot.bybit import BybitClient -from crypto_spot_bot.config import Settings, load_settings -from crypto_spot_bot.time_series import _log_returns, _validate_candidates - - -def main() -> None: - args = _parse_args() - settings = load_settings(args.env) - client = BybitClient(settings) - symbols = _symbols(args.symbols, settings, client) - interval = args.interval or settings.base_interval - output = Path(args.output) if args.output else settings.time_series_lstm_model_path - - artifact: dict[str, Any] = { - "version": 1, - "type": "lstm_reservoir_ridge_params", - "created_at": datetime.now(timezone.utc).isoformat(), - "interval": interval, - "limit": args.limit, - "symbols": {}, - } - - for symbol in symbols: - result = _train_symbol( - client=client, - settings=settings, - symbol=symbol, - interval=interval, - limit=args.limit, - lookbacks=_ints(args.lookbacks), - units_values=_ints(args.units), - ridges=_floats(args.ridges), - ) - if result is None: - print(f"{symbol}: skipped, not enough candles or returns") - continue - artifact["symbols"][symbol] = result - print( - f"{symbol}: lookback={result['lookback']} units={result['units']} " - f"ridge={result['ridge']} mae={result['validation_mae_percent']:.5f}% " - f"baseline={result['baseline_mae_percent']:.5f}% skill={result['skill']:.4f}" - ) - - output.parent.mkdir(parents=True, exist_ok=True) - tmp_output = output.with_name(f"{output.name}.tmp") - tmp_output.write_text(json.dumps(artifact, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - tmp_output.replace(output) - print(f"saved {output}") - - -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Train lightweight LSTM forecast params on Bybit spot candles.") - parser.add_argument("--env", default=None, help="Path to .env file.") - parser.add_argument("--symbols", default="", help="Comma-separated symbols. Defaults to configured or popular spot pairs.") - parser.add_argument("--interval", default="", help="Bybit kline interval. Defaults to BASE_INTERVAL.") - parser.add_argument("--limit", type=int, default=1000, help="Kline limit per symbol.") - parser.add_argument("--lookbacks", default="16,32", help="Comma-separated LSTM lookback candidates.") - parser.add_argument("--units", default="4,6", help="Comma-separated LSTM unit candidates.") - parser.add_argument("--ridges", default="0.001", help="Comma-separated ridge candidates.") - parser.add_argument("--output", default="", help="Output JSON path. Defaults to TIME_SERIES_LSTM_MODEL_PATH.") - return parser.parse_args() - - -def _symbols(raw: str, settings: Settings, client: BybitClient) -> list[str]: - if raw.strip(): - return [item.strip().upper() for item in raw.split(",") if item.strip()] - if settings.symbols: - return list(settings.symbols) - return client.popular_spot_symbols(settings.top_symbols_count) - - -def _train_symbol( - *, - client: BybitClient, - settings: Settings, - symbol: str, - interval: str, - limit: int, - lookbacks: list[int], - units_values: list[int], - ridges: list[float], -) -> dict[str, Any] | None: - candles = client.klines(symbol, interval, limit) - closes = [float(candle.close) for candle in candles if candle.close > 0] - returns = _log_returns(closes) - if len(returns) < 80: - return None - validation_window = min(max(8, settings.time_series_validation_window), max(8, len(returns) // 3)) - best: dict[str, Any] | None = None - for lookback in lookbacks: - for units in units_values: - for ridge in ridges: - candidate_settings = replace( - settings, - time_series_lstm_enabled=True, - time_series_lstm_lookback=lookback, - time_series_lstm_units=units, - time_series_lstm_ridge=ridge, - ) - candidates = _validate_candidates(returns, validation_window, candidate_settings, symbol, {}) - baseline = next((item for item in candidates if item["model"] == "naive"), None) - lstm = next((item for item in candidates if item["model"] == "lstm"), None) - if baseline is None or lstm is None: - continue - baseline_mae = float(baseline["mae"]) - lstm_mae = float(lstm["mae"]) - skill = (baseline_mae - lstm_mae) / baseline_mae if baseline_mae > 0 else 0.0 - row = { - "lookback": lookback, - "units": units, - "ridge": ridge, - "validation_mae_percent": lstm_mae * 100, - "baseline_mae_percent": baseline_mae * 100, - "skill": skill, - "candles": len(candles), - "returns": len(returns), - } - if best is None or lstm_mae < best["validation_mae_percent"] / 100: - best = row - return best - - -def _ints(raw: str) -> list[int]: - return [int(item.strip()) for item in raw.split(",") if item.strip()] - - -def _floats(raw: str) -> list[float]: - return [float(item.strip()) for item in raw.split(",") if item.strip()] - - -if __name__ == "__main__": - main()