Remove legacy LSTM retraining
This commit is contained in:
+2
-5
@@ -7,9 +7,9 @@ BYBIT_API_KEY=
|
||||
BYBIT_API_SECRET=
|
||||
|
||||
STARTING_BALANCE_USDT=100
|
||||
AUTO_SELECT_SYMBOLS=true
|
||||
AUTO_SELECT_SYMBOLS=false
|
||||
TOP_SYMBOLS_COUNT=6
|
||||
SYMBOLS=
|
||||
SYMBOLS=BTCUSDT,ETHUSDT,HYPEUSDT,SOLUSDT,LTCUSDT,XRPUSDT
|
||||
|
||||
BASE_INTERVAL=1
|
||||
KLINE_LIMIT=240
|
||||
@@ -50,9 +50,6 @@ TIME_SERIES_EWMA_LAMBDA=0.94
|
||||
TIME_SERIES_MIN_EDGE_PERCENT=0.04
|
||||
TIME_SERIES_MAX_ADJUSTMENT=0.08
|
||||
TIME_SERIES_LSTM_ENABLED=true
|
||||
TIME_SERIES_LSTM_LOOKBACK=32
|
||||
TIME_SERIES_LSTM_UNITS=6
|
||||
TIME_SERIES_LSTM_RIDGE=0.0001
|
||||
TIME_SERIES_LSTM_MODEL_PATH=runtime/lstm_forecaster.json
|
||||
STOP_LOSS_PERCENT=0.02
|
||||
TAKE_PROFIT_PERCENT=0.035
|
||||
|
||||
@@ -15,7 +15,7 @@ Spot-бот для демо-торговли криптовалютой на р
|
||||
- Динамический размер позиции: стратегия записывает в сигнал размер входа в пределах `MIN_POSITION_USDT`..`MAX_POSITION_USDT`, а брокер ограничивает суммарную экспозицию по паре через `MAX_SYMBOL_EXPOSURE_USDT`.
|
||||
- Автоматический grid-режим: бот включает grid-входы на боковике, покупает только в нижней части диапазона и выключает grid при падающих/опасных режимах.
|
||||
- Вероятностный rebound-вход: после снижения бот отдельно оценивает стабилизацию, отскок от локального low, RSI, объем и рыночные ограничения; такой вход ограничен меньшим размером позиции.
|
||||
- Прогнозирование временных рядов: walk-forward выбор между `naive`, `drift`, `EWMA`, `AR(1)`, `AR(3)` и легким `lstm`-кандидатом для ожидаемой доходности плюс EWMA/GARCH-like прогноз волатильности. Прогноз влияет и на новые покупки, и на раннюю продажу при ухудшении ожидаемого движения.
|
||||
- Прогнозирование временных рядов: walk-forward выбор между `naive`, `drift`, `EWMA`, `AR(1)`, `AR(3)` и экспортированными PyTorch `LSTM/GRU`-моделями для ожидаемой доходности плюс EWMA/GARCH-like прогноз волатильности. Прогноз влияет и на новые покупки, и на раннюю продажу при ухудшении ожидаемого движения.
|
||||
- Защитные блокировки входа: явно отрицательные LONG-шаблоны и setups с сильной отрицательной статистикой обучения запрещают новые покупки.
|
||||
- Быстрый режим торговли: отдельный короткий интервал цикла, короткий cooldown после выхода и лимит новых входов в минуту; выходы по риску этим лимитом не блокируются.
|
||||
- Веб-dashboard на русском: equity, cash, PnL, позиции, сделки, сигналы, события, свечные графики, переключатель быстрой торговли и индикаторы работы обучения.
|
||||
@@ -46,7 +46,7 @@ Live market orders используют `/v5/order/create`; Bybit докумен
|
||||
- Документация `statsmodels` описывает ARIMA как общий интерфейс для AR/MA/ARMA/ARIMA/SARIMA-моделей; в боте используется легкий AR(1)/AR(3) вариант без добавления тяжелой зависимости `statsmodels`: <https://www.statsmodels.org/stable/generated/statsmodels.tsa.arima.model.ARIMA.html>.
|
||||
- Документация `arch` описывает GARCH(p,q) как модель для прогнозирования волатильности; в боте используется фиксированная GARCH(1,1)-подобная рекурсия без MLE-оценки параметров, чтобы сохранить легкий runtime на Raspberry Pi: <https://arch.readthedocs.io/en/stable/univariate/univariate_volatility_forecasting.html>.
|
||||
- RiskMetrics описывает EWMA-подход к оценке волатильности через коэффициент затухания; в боте `TIME_SERIES_EWMA_LAMBDA=0.94` используется как настраиваемое значение по умолчанию: <https://www.msci.com/documents/10199/d0905614-2771-46dc-b000-1a033146586a>.
|
||||
- Hochreiter и Schmidhuber описали LSTM как recurrent neural network architecture для последовательностей; в боте используется легкая LSTM-reservoir рекурсия с ridge-readout, а не полноценное PyTorch/TensorFlow обучение внутри Docker: <https://direct.mit.edu/neco/article/9/8/1735/6109/Long-Short-Term-Memory>.
|
||||
- Hochreiter и Schmidhuber описали LSTM как recurrent neural network architecture для последовательностей; обучение LSTM/GRU в проекте выполняется локально через PyTorch, а Raspberry Pi исполняет только экспортированные JSON-веса без PyTorch runtime: <https://direct.mit.edu/neco/article/9/8/1735/6109/Long-Short-Term-Memory>.
|
||||
|
||||
Я не могу подтвердить, что эта стратегия будет прибыльной. Источники выше описывают технические свойства и риски автоматической торговли, но не гарантируют прибыль.
|
||||
|
||||
@@ -62,22 +62,13 @@ python -m crypto_spot_bot.main
|
||||
|
||||
Dashboard: <http://127.0.0.1:8787/>
|
||||
|
||||
## Локальное обучение LSTM-кандидата
|
||||
## Локальное обучение PyTorch LSTM/GRU
|
||||
|
||||
Обучение можно запускать на основной машине, а Raspberry Pi оставлять только для исполнения торгового цикла. Команда ниже берет spot-свечи Bybit, перебирает `lookback`, `units` и `ridge`, оценивает LSTM-кандидат через walk-forward MAE и сохраняет параметры в `runtime/lstm_forecaster.json`:
|
||||
|
||||
```powershell
|
||||
python tools\train_lstm_forecaster.py --symbols BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,LTCUSDT --limit 1000
|
||||
```
|
||||
|
||||
Файл из `TIME_SERIES_LSTM_MODEL_PATH` читается ботом автоматически. Даже если LSTM-параметры сохранены, сделка меняется только тогда, когда текущая walk-forward проверка в `crypto_spot_bot/time_series.py` показывает качество лучше baseline.
|
||||
|
||||
Для более тяжелого локального обучения можно использовать настоящий PyTorch `LSTM/GRU` trainer. PyTorch нужен только на машине обучения; в JSON экспортируются веса, а runtime на Raspberry Pi считает inference обычным Python-кодом:
|
||||
Обучение запускается на основной Windows-машине, а Raspberry Pi остается только для исполнения торгового цикла. PyTorch нужен только на машине обучения; в JSON экспортируются веса, а runtime на Raspberry Pi считает inference обычным Python-кодом:
|
||||
|
||||
```powershell
|
||||
.\.venv\Scripts\python.exe -m pip install torch --index-url https://download.pytorch.org/whl/cpu
|
||||
.\.venv\Scripts\python.exe tools\train_torch_recurrent_forecaster.py `
|
||||
--symbols BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,LTCUSDT `
|
||||
--limit 1000 `
|
||||
--architectures lstm,gru `
|
||||
--lookbacks 32,64 `
|
||||
@@ -86,23 +77,16 @@ python tools\train_lstm_forecaster.py --symbols BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,
|
||||
--epochs 60
|
||||
```
|
||||
|
||||
Экспортированные модели появляются в dashboard как `torch_lstm` или `torch_gru`; легкий `lstm`-кандидат остается доступен как fallback.
|
||||
Файл из `TIME_SERIES_LSTM_MODEL_PATH` читается ботом автоматически. Экспортированные модели появляются в dashboard как `PyTorch LSTM` или `PyTorch GRU`; старый легкий reservoir LSTM-кандидат удален и больше не участвует в выборе модели.
|
||||
|
||||
Автопереобучение запускает тот же train-скрипт, пишет лог в `runtime/lstm_retrain.log` и защищается от параллельных запусков:
|
||||
Автопереобучение на Windows запускает PyTorch trainer, пишет лог в `runtime/torch_retrain.log` и защищается от параллельных запусков:
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File tools\run_lstm_retrain.ps1
|
||||
powershell -ExecutionPolicy Bypass -File tools\install_windows_lstm_retrainer.ps1
|
||||
powershell -ExecutionPolicy Bypass -File tools\run_torch_retrain.ps1
|
||||
powershell -ExecutionPolicy Bypass -File tools\install_windows_torch_retrainer.ps1
|
||||
```
|
||||
|
||||
На Linux/Raspberry Pi можно включить user systemd timer:
|
||||
|
||||
```bash
|
||||
bash tools/run_lstm_retrain.sh
|
||||
bash tools/install_lstm_retrainer_systemd.sh
|
||||
```
|
||||
|
||||
По умолчанию Windows-расписание переобучает PyTorch `LSTM/GRU` каждые 6 часов с `--limit 1000`; Windows-установщик фиксирует пары `BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,LTCUSDT`, чтобы первый scheduled run был предсказуемым. Параметры можно переопределить через env: `LSTM_RETRAIN_SYMBOLS`, `LSTM_RETRAIN_LIMIT`, `LSTM_RETRAIN_LOOKBACKS`, `LSTM_RETRAIN_ARCHITECTURES`, `LSTM_RETRAIN_HIDDEN_SIZES`, `LSTM_RETRAIN_LAYERS`, `LSTM_RETRAIN_DROPOUTS`, `LSTM_RETRAIN_EPOCHS`, `LSTM_RETRAIN_PATIENCE`, `LSTM_RETRAIN_INTERVAL`, `LSTM_RETRAIN_ENV`. Для старого легкого trainer можно запустить `tools\run_lstm_retrain.ps1 -Trainer reservoir`.
|
||||
По умолчанию Windows-расписание переобучает PyTorch `LSTM/GRU` каждые 6 часов с `--limit 1000` на фиксированных парах `BTCUSDT,ETHUSDT,HYPEUSDT,SOLUSDT,LTCUSDT,XRPUSDT`. Параметры можно переопределить через env: `TORCH_RETRAIN_SYMBOLS`, `TORCH_RETRAIN_LIMIT`, `TORCH_RETRAIN_LOOKBACKS`, `TORCH_RETRAIN_ARCHITECTURES`, `TORCH_RETRAIN_HIDDEN_SIZES`, `TORCH_RETRAIN_LAYERS`, `TORCH_RETRAIN_DROPOUTS`, `TORCH_RETRAIN_EPOCHS`, `TORCH_RETRAIN_PATIENCE`, `TORCH_RETRAIN_INTERVAL`, `TORCH_RETRAIN_ENV`.
|
||||
|
||||
## Docker
|
||||
|
||||
@@ -121,8 +105,9 @@ Dashboard: `http://<host>:8787/`
|
||||
```env
|
||||
TRADING_MODE=paper
|
||||
STARTING_BALANCE_USDT=100
|
||||
AUTO_SELECT_SYMBOLS=true
|
||||
AUTO_SELECT_SYMBOLS=false
|
||||
TOP_SYMBOLS_COUNT=6
|
||||
SYMBOLS=BTCUSDT,ETHUSDT,HYPEUSDT,SOLUSDT,LTCUSDT,XRPUSDT
|
||||
BASE_INTERVAL=1
|
||||
LOOP_INTERVAL_SECONDS=5
|
||||
FAST_TRADING_ENABLED=false
|
||||
@@ -159,9 +144,6 @@ TIME_SERIES_EWMA_LAMBDA=0.94
|
||||
TIME_SERIES_MIN_EDGE_PERCENT=0.04
|
||||
TIME_SERIES_MAX_ADJUSTMENT=0.08
|
||||
TIME_SERIES_LSTM_ENABLED=true
|
||||
TIME_SERIES_LSTM_LOOKBACK=32
|
||||
TIME_SERIES_LSTM_UNITS=6
|
||||
TIME_SERIES_LSTM_RIDGE=0.0001
|
||||
TIME_SERIES_LSTM_MODEL_PATH=runtime/lstm_forecaster.json
|
||||
STOP_LOSS_PERCENT=0.02
|
||||
TAKE_PROFIT_PERCENT=0.035
|
||||
|
||||
@@ -5,6 +5,9 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
FIXED_SPOT_SYMBOLS = ("BTCUSDT", "ETHUSDT", "HYPEUSDT", "SOLUSDT", "LTCUSDT", "XRPUSDT")
|
||||
|
||||
|
||||
def _load_dotenv(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
@@ -97,9 +100,6 @@ class Settings:
|
||||
time_series_min_edge_percent: float
|
||||
time_series_max_adjustment: float
|
||||
time_series_lstm_enabled: bool
|
||||
time_series_lstm_lookback: int
|
||||
time_series_lstm_units: int
|
||||
time_series_lstm_ridge: float
|
||||
time_series_lstm_model_path: Path
|
||||
stop_loss_percent: float
|
||||
take_profit_percent: float
|
||||
@@ -172,9 +172,9 @@ def load_settings(env_file: str | Path | None = None) -> Settings:
|
||||
bybit_api_key=os.getenv("BYBIT_API_KEY", ""),
|
||||
bybit_api_secret=os.getenv("BYBIT_API_SECRET", ""),
|
||||
starting_balance_usdt=_float_env("STARTING_BALANCE_USDT", 100.0),
|
||||
auto_select_symbols=_bool_env("AUTO_SELECT_SYMBOLS", True),
|
||||
top_symbols_count=_int_env("TOP_SYMBOLS_COUNT", 6),
|
||||
symbols=_symbols_env("SYMBOLS"),
|
||||
auto_select_symbols=_bool_env("AUTO_SELECT_SYMBOLS", False),
|
||||
top_symbols_count=_int_env("TOP_SYMBOLS_COUNT", len(FIXED_SPOT_SYMBOLS)),
|
||||
symbols=_symbols_env("SYMBOLS") or FIXED_SPOT_SYMBOLS,
|
||||
base_interval=os.getenv("BASE_INTERVAL", "1"),
|
||||
kline_limit=_int_env("KLINE_LIMIT", 240),
|
||||
loop_interval_seconds=_int_env("LOOP_INTERVAL_SECONDS", 5),
|
||||
@@ -220,9 +220,6 @@ def load_settings(env_file: str | Path | None = None) -> Settings:
|
||||
time_series_min_edge_percent=_float_env("TIME_SERIES_MIN_EDGE_PERCENT", 0.04),
|
||||
time_series_max_adjustment=_float_env("TIME_SERIES_MAX_ADJUSTMENT", 0.08),
|
||||
time_series_lstm_enabled=_bool_env("TIME_SERIES_LSTM_ENABLED", True),
|
||||
time_series_lstm_lookback=_int_env("TIME_SERIES_LSTM_LOOKBACK", 32),
|
||||
time_series_lstm_units=_int_env("TIME_SERIES_LSTM_UNITS", 6),
|
||||
time_series_lstm_ridge=_float_env("TIME_SERIES_LSTM_RIDGE", 0.0001),
|
||||
time_series_lstm_model_path=Path(os.getenv("TIME_SERIES_LSTM_MODEL_PATH", "runtime/lstm_forecaster.json")),
|
||||
stop_loss_percent=_float_env("STOP_LOSS_PERCENT", 0.02),
|
||||
take_profit_percent=_float_env("TAKE_PROFIT_PERCENT", 0.035),
|
||||
|
||||
@@ -217,9 +217,6 @@ def _safe_config(settings: Settings) -> dict[str, Any]:
|
||||
"time_series_min_edge_percent": settings.time_series_min_edge_percent,
|
||||
"time_series_max_adjustment": settings.time_series_max_adjustment,
|
||||
"time_series_lstm_enabled": settings.time_series_lstm_enabled,
|
||||
"time_series_lstm_lookback": settings.time_series_lstm_lookback,
|
||||
"time_series_lstm_units": settings.time_series_lstm_units,
|
||||
"time_series_lstm_ridge": settings.time_series_lstm_ridge,
|
||||
"time_series_lstm_model_path": str(settings.time_series_lstm_model_path),
|
||||
"time_series_model_artifact": _time_series_model_artifact(settings),
|
||||
"stop_loss_percent": settings.stop_loss_percent,
|
||||
@@ -266,16 +263,19 @@ def _time_series_model_artifact(settings: Settings) -> dict[str, Any]:
|
||||
}
|
||||
)
|
||||
artifact_type = str(data.get("type", "")).strip()
|
||||
if artifact_type == "pytorch_recurrent_forecaster":
|
||||
label = "PyTorch LSTM/GRU"
|
||||
elif artifact_type == "lstm_reservoir_ridge_params":
|
||||
label = "легкий LSTM fallback"
|
||||
else:
|
||||
label = artifact_type or "настройки прогноза"
|
||||
if artifact_type != "pytorch_recurrent_forecaster":
|
||||
return {
|
||||
"available": False,
|
||||
"type": artifact_type or "unknown",
|
||||
"label": "устаревший файл модели не используется",
|
||||
"created_at": data.get("created_at", ""),
|
||||
"symbol_count": len(rows),
|
||||
"models": models,
|
||||
}
|
||||
return {
|
||||
"available": True,
|
||||
"type": artifact_type or "unknown",
|
||||
"label": label,
|
||||
"type": artifact_type,
|
||||
"label": "PyTorch LSTM/GRU",
|
||||
"created_at": data.get("created_at", ""),
|
||||
"symbol_count": len(rows),
|
||||
"models": models,
|
||||
@@ -289,7 +289,7 @@ def _forecast_model_label(model: str) -> str:
|
||||
if normalized == "torch_gru":
|
||||
return "PyTorch GRU"
|
||||
if normalized == "lstm":
|
||||
return "легкий LSTM"
|
||||
return "устаревший LSTM"
|
||||
if normalized == "gru":
|
||||
return "GRU"
|
||||
return model
|
||||
@@ -742,7 +742,7 @@ HTML = r"""
|
||||
const names = {
|
||||
torch_lstm: 'PyTorch LSTM',
|
||||
torch_gru: 'PyTorch GRU',
|
||||
lstm: 'Легкий LSTM',
|
||||
lstm: 'Устаревший LSTM',
|
||||
naive: 'Baseline',
|
||||
drift: 'Drift',
|
||||
ewma: 'EWMA',
|
||||
@@ -757,7 +757,7 @@ HTML = r"""
|
||||
return String(reason || '')
|
||||
.replaceAll('torch_lstm', 'PyTorch LSTM')
|
||||
.replaceAll('torch_gru', 'PyTorch GRU')
|
||||
.replaceAll('модель lstm', 'модель легкий LSTM');
|
||||
.replaceAll('модель lstm', 'модель устаревший LSTM');
|
||||
}
|
||||
|
||||
function modelArtifactSummary(config) {
|
||||
|
||||
@@ -3,7 +3,6 @@ from __future__ import annotations
|
||||
import json
|
||||
import math
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
from crypto_spot_bot.config import Settings
|
||||
@@ -174,8 +173,6 @@ def _validate_candidates(
|
||||
torch_model = _torch_recurrent_model_name(symbol, lstm_artifact or {})
|
||||
if torch_model and _can_use_torch_recurrent(returns, symbol, lstm_artifact or {}):
|
||||
models.append(torch_model)
|
||||
if _can_use_lstm(returns, settings, symbol, lstm_artifact or {}):
|
||||
models.append("lstm")
|
||||
rows: list[dict[str, float | str]] = []
|
||||
start = max(8, len(returns) - validation_window)
|
||||
for model in models:
|
||||
@@ -211,8 +208,6 @@ def _predict_next_return(
|
||||
return _ar_predict(returns, 3)
|
||||
if model in {"torch_lstm", "torch_gru"}:
|
||||
return _torch_recurrent_predict(returns, symbol, lstm_artifact or {})
|
||||
if model == "lstm" and settings is not None:
|
||||
return _lstm_predict(returns, settings, symbol, lstm_artifact or {})
|
||||
return 0.0
|
||||
|
||||
|
||||
@@ -244,52 +239,6 @@ def _ar_predict(returns: list[float], lag_count: int) -> float:
|
||||
return _clamp(prediction, -cap, cap)
|
||||
|
||||
|
||||
def _can_use_lstm(
|
||||
returns: list[float],
|
||||
settings: Settings,
|
||||
symbol: str | None,
|
||||
lstm_artifact: dict[str, Any],
|
||||
) -> bool:
|
||||
if not settings.time_series_lstm_enabled:
|
||||
return False
|
||||
params = _lstm_params(settings, symbol, lstm_artifact)
|
||||
return len(returns) >= params["lookback"] + 16
|
||||
|
||||
|
||||
def _lstm_params(settings: Settings, symbol: str | None, lstm_artifact: dict[str, Any]) -> dict[str, float | int]:
|
||||
params: dict[str, float | int] = {
|
||||
"lookback": settings.time_series_lstm_lookback,
|
||||
"units": settings.time_series_lstm_units,
|
||||
"ridge": settings.time_series_lstm_ridge,
|
||||
}
|
||||
default_params = lstm_artifact.get("default")
|
||||
if isinstance(default_params, dict):
|
||||
params.update(_clean_lstm_params(default_params))
|
||||
symbols = lstm_artifact.get("symbols")
|
||||
symbol_params = symbols.get(symbol.upper()) if symbol and isinstance(symbols, dict) else None
|
||||
if isinstance(symbol_params, dict):
|
||||
params.update(_clean_lstm_params(symbol_params))
|
||||
return {
|
||||
"lookback": int(_clamp(float(params["lookback"]), 6.0, 128.0)),
|
||||
"units": int(_clamp(float(params["units"]), 2.0, 16.0)),
|
||||
"ridge": _clamp(float(params["ridge"]), 1e-8, 0.5),
|
||||
}
|
||||
|
||||
|
||||
def _clean_lstm_params(data: dict[str, Any]) -> dict[str, float | int]:
|
||||
clean: dict[str, float | int] = {}
|
||||
for key in ("lookback", "units", "ridge"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, (int, float)):
|
||||
clean[key] = value
|
||||
elif isinstance(value, str):
|
||||
try:
|
||||
clean[key] = float(value)
|
||||
except ValueError:
|
||||
continue
|
||||
return clean
|
||||
|
||||
|
||||
def _torch_recurrent_model_name(symbol: str | None, lstm_artifact: dict[str, Any]) -> str | None:
|
||||
entry = _torch_recurrent_entry(symbol, lstm_artifact)
|
||||
if not entry:
|
||||
@@ -519,39 +468,6 @@ def _dot(left: list[float], right: list[float]) -> float:
|
||||
return sum(left[index] * right[index] for index in range(min(len(left), len(right))))
|
||||
|
||||
|
||||
def _lstm_predict(
|
||||
returns: list[float],
|
||||
settings: Settings,
|
||||
symbol: str | None,
|
||||
lstm_artifact: dict[str, Any],
|
||||
) -> float:
|
||||
params = _lstm_params(settings, symbol, lstm_artifact)
|
||||
lookback = int(params["lookback"])
|
||||
units = int(params["units"])
|
||||
ridge = float(params["ridge"])
|
||||
if len(returns) <= lookback + 8:
|
||||
return _predict_next_return("drift", returns)
|
||||
|
||||
scale = _return_scale(returns)
|
||||
normalized = [_clamp(value / scale, -6.0, 6.0) for value in returns]
|
||||
states = _lstm_states(normalized, units)
|
||||
rows: list[list[float]] = []
|
||||
targets: list[float] = []
|
||||
for index in range(lookback, len(returns)):
|
||||
rows.append([1.0] + states[index - 1])
|
||||
targets.append(normalized[index])
|
||||
coeffs = _ols(rows, targets, ridge)
|
||||
if not coeffs:
|
||||
return _predict_next_return("drift", returns)
|
||||
|
||||
features = [1.0] + states[-1]
|
||||
prediction = sum(coeff * feature for coeff, feature in zip(coeffs, features))
|
||||
prediction = _clamp(prediction, -4.0, 4.0) * scale
|
||||
recent_abs = sorted(abs(value) for value in returns[-48:]) if len(returns) >= 8 else [0.01]
|
||||
cap = max(recent_abs[int(len(recent_abs) * 0.9)], 0.0002)
|
||||
return _clamp(prediction, -cap, cap)
|
||||
|
||||
|
||||
def _return_scale(returns: list[float]) -> float:
|
||||
recent = returns[-120:] if len(returns) > 120 else returns
|
||||
values = sorted(abs(value) for value in recent if math.isfinite(value))
|
||||
@@ -562,67 +478,6 @@ def _return_scale(returns: list[float]) -> float:
|
||||
return max(max(median, mean * 0.5), 1e-5)
|
||||
|
||||
|
||||
def _lstm_states(normalized_returns: list[float], units: int) -> list[list[float]]:
|
||||
weights = _lstm_weights(units)
|
||||
hidden = [0.0 for _ in range(units)]
|
||||
cell = [0.0 for _ in range(units)]
|
||||
states: list[list[float]] = []
|
||||
for value in normalized_returns:
|
||||
hidden, cell = _lstm_step(value, hidden, cell, weights)
|
||||
states.append(hidden[:])
|
||||
return states
|
||||
|
||||
|
||||
@lru_cache(maxsize=16)
|
||||
def _lstm_weights(units: int) -> tuple[list[list[float]], list[list[list[float]]], list[list[float]]]:
|
||||
input_weights: list[list[float]] = []
|
||||
recurrent_weights: list[list[list[float]]] = []
|
||||
biases: list[list[float]] = []
|
||||
base_biases = (-0.15, 0.70, 0.05, 0.0)
|
||||
for gate in range(4):
|
||||
gate_input: list[float] = []
|
||||
gate_recurrent: list[list[float]] = []
|
||||
gate_bias: list[float] = []
|
||||
for unit in range(units):
|
||||
gate_input.append(0.55 * math.sin((gate + 1) * (unit + 1) * 1.61803398875))
|
||||
gate_recurrent.append(
|
||||
[
|
||||
0.14 * math.sin((gate + 3) * (unit + 1) * (source + 1) * 0.731)
|
||||
for source in range(units)
|
||||
]
|
||||
)
|
||||
gate_bias.append(base_biases[gate] + 0.03 * math.sin((gate + 1) * (unit + 1)))
|
||||
input_weights.append(gate_input)
|
||||
recurrent_weights.append(gate_recurrent)
|
||||
biases.append(gate_bias)
|
||||
return input_weights, recurrent_weights, biases
|
||||
|
||||
|
||||
def _lstm_step(
|
||||
value: float,
|
||||
hidden: list[float],
|
||||
cell: list[float],
|
||||
weights: tuple[list[list[float]], list[list[list[float]]], list[list[float]]],
|
||||
) -> tuple[list[float], list[float]]:
|
||||
input_weights, recurrent_weights, biases = weights
|
||||
units = len(hidden)
|
||||
next_hidden = [0.0 for _ in range(units)]
|
||||
next_cell = [0.0 for _ in range(units)]
|
||||
for unit in range(units):
|
||||
gate_values = []
|
||||
for gate in range(4):
|
||||
raw = input_weights[gate][unit] * value + biases[gate][unit]
|
||||
raw += sum(recurrent_weights[gate][unit][source] * hidden[source] for source in range(units))
|
||||
gate_values.append(raw)
|
||||
input_gate = _sigmoid(gate_values[0])
|
||||
forget_gate = _sigmoid(gate_values[1])
|
||||
output_gate = _sigmoid(gate_values[2])
|
||||
candidate = math.tanh(gate_values[3])
|
||||
next_cell[unit] = forget_gate * cell[unit] + input_gate * candidate
|
||||
next_hidden[unit] = output_gate * math.tanh(next_cell[unit])
|
||||
return next_hidden, next_cell
|
||||
|
||||
|
||||
def _sigmoid(value: float) -> float:
|
||||
if value >= 40:
|
||||
return 1.0
|
||||
|
||||
@@ -66,9 +66,6 @@ def make_settings():
|
||||
time_series_min_edge_percent=0.04,
|
||||
time_series_max_adjustment=0.08,
|
||||
time_series_lstm_enabled=True,
|
||||
time_series_lstm_lookback=32,
|
||||
time_series_lstm_units=6,
|
||||
time_series_lstm_ridge=0.0001,
|
||||
time_series_lstm_model_path=tmp_path / "lstm_forecaster.json",
|
||||
stop_loss_percent=0.02,
|
||||
take_profit_percent=0.035,
|
||||
|
||||
+16
-1
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from crypto_spot_bot.config import load_settings
|
||||
from crypto_spot_bot.config import FIXED_SPOT_SYMBOLS, load_settings
|
||||
|
||||
|
||||
def test_live_mode_requires_explicit_unlock(tmp_path, monkeypatch) -> None:
|
||||
@@ -61,3 +61,18 @@ def test_llm_advisor_is_disabled_by_default(tmp_path, monkeypatch) -> None:
|
||||
settings = load_settings(env_file)
|
||||
|
||||
assert settings.llm_advisor_enabled is False
|
||||
|
||||
|
||||
def test_default_symbols_are_fixed_six_pairs(tmp_path, monkeypatch) -> None:
|
||||
monkeypatch.delenv("AUTO_SELECT_SYMBOLS", raising=False)
|
||||
monkeypatch.delenv("TOP_SYMBOLS_COUNT", raising=False)
|
||||
monkeypatch.delenv("SYMBOLS", raising=False)
|
||||
monkeypatch.setenv("TRADING_MODE", "paper")
|
||||
env_file = tmp_path / ".env"
|
||||
env_file.write_text("TRADING_MODE=paper\nSYMBOLS=\n", encoding="utf-8")
|
||||
|
||||
settings = load_settings(env_file)
|
||||
|
||||
assert settings.auto_select_symbols is False
|
||||
assert settings.top_symbols_count == 6
|
||||
assert settings.symbols == FIXED_SPOT_SYMBOLS
|
||||
|
||||
@@ -76,32 +76,13 @@ def test_time_series_forecaster_blocks_negative_edge(make_settings, tmp_path) ->
|
||||
assert forecast.block_entry is True
|
||||
|
||||
|
||||
def test_time_series_forecaster_includes_lstm_candidate(make_settings, tmp_path) -> None:
|
||||
settings = make_settings(
|
||||
tmp_path,
|
||||
time_series_min_candles=80,
|
||||
time_series_validation_window=20,
|
||||
time_series_lstm_enabled=True,
|
||||
time_series_lstm_lookback=12,
|
||||
time_series_lstm_units=4,
|
||||
)
|
||||
returns = []
|
||||
for index in range(140):
|
||||
seasonal = 0.00018 if index % 5 in {0, 1, 2} else -0.00011
|
||||
returns.append(seasonal + 0.00002 * ((index % 7) - 3))
|
||||
|
||||
forecast = TimeSeriesForecaster(settings).forecast(_candles_from_returns(returns), symbol="BTCUSDT")
|
||||
|
||||
assert forecast.usable is True
|
||||
assert any(candidate["model"] == "lstm" for candidate in forecast.candidates)
|
||||
|
||||
|
||||
def test_time_series_forecaster_reads_lstm_artifact(make_settings, tmp_path) -> None:
|
||||
def test_time_series_forecaster_ignores_legacy_lstm_artifact(make_settings, tmp_path) -> None:
|
||||
artifact_path = tmp_path / "lstm_forecaster.json"
|
||||
artifact_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"version": 1,
|
||||
"type": "lstm_reservoir_ridge_params",
|
||||
"symbols": {
|
||||
"BTCUSDT": {"lookback": 10, "units": 3, "ridge": 0.01},
|
||||
},
|
||||
@@ -121,7 +102,7 @@ def test_time_series_forecaster_reads_lstm_artifact(make_settings, tmp_path) ->
|
||||
forecast = TimeSeriesForecaster(settings).forecast(_candles_from_returns(returns), symbol="BTCUSDT")
|
||||
|
||||
assert forecast.usable is True
|
||||
assert any(candidate["model"] == "lstm" for candidate in forecast.candidates)
|
||||
assert all(candidate["model"] != "lstm" for candidate in forecast.candidates)
|
||||
|
||||
|
||||
def test_time_series_forecaster_reads_torch_gru_artifact(make_settings, tmp_path) -> None:
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
SYSTEMD_DIR="$HOME/.config/systemd/user"
|
||||
SERVICE_NAME="tradebot-lstm-retrainer.service"
|
||||
TIMER_NAME="tradebot-lstm-retrainer.timer"
|
||||
|
||||
mkdir -p "$SYSTEMD_DIR"
|
||||
|
||||
cat > "$SYSTEMD_DIR/$SERVICE_NAME" <<EOF
|
||||
[Unit]
|
||||
Description=Retrain TradeBot LSTM forecast parameters
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=$REPO_ROOT
|
||||
ExecStart=$REPO_ROOT/tools/run_lstm_retrain.sh
|
||||
EOF
|
||||
|
||||
cat > "$SYSTEMD_DIR/$TIMER_NAME" <<EOF
|
||||
[Unit]
|
||||
Description=Retrain TradeBot LSTM forecast parameters every 6 hours
|
||||
|
||||
[Timer]
|
||||
OnBootSec=5min
|
||||
OnUnitActiveSec=6h
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
EOF
|
||||
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable --now "$TIMER_NAME"
|
||||
|
||||
echo "Enabled user timer $TIMER_NAME. Check with: systemctl --user list-timers $TIMER_NAME"
|
||||
+13
-7
@@ -1,23 +1,29 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$TaskName = "TradeBot LSTM Retrainer",
|
||||
[string]$TaskName = "TradeBot PyTorch Forecaster Retrainer",
|
||||
[int]$EveryHours = 6,
|
||||
[string]$Symbols = "BTCUSDT,ETHUSDT,SOLUSDT,XRPUSDT,LTCUSDT",
|
||||
[string]$Symbols = "BTCUSDT,ETHUSDT,HYPEUSDT,SOLUSDT,LTCUSDT,XRPUSDT",
|
||||
[int]$Limit = 1000,
|
||||
[ValidateSet("torch", "reservoir")]
|
||||
[string]$Trainer = "torch",
|
||||
[int]$FirstRunMinutes = 0
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
||||
$Runner = Join-Path $RepoRoot "tools\run_lstm_retrain.ps1"
|
||||
$Runner = Join-Path $RepoRoot "tools\run_torch_retrain.ps1"
|
||||
if (-not (Test-Path $Runner)) {
|
||||
throw "Runner not found: $Runner"
|
||||
}
|
||||
|
||||
$actionArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$Runner`" -Trainer $Trainer"
|
||||
$LegacyTaskName = "TradeBot LSTM Retrainer"
|
||||
if ($TaskName -ne $LegacyTaskName) {
|
||||
$legacyTask = Get-ScheduledTask -TaskName $LegacyTaskName -ErrorAction SilentlyContinue
|
||||
if ($legacyTask) {
|
||||
Unregister-ScheduledTask -TaskName $LegacyTaskName -Confirm:$false
|
||||
}
|
||||
}
|
||||
|
||||
$actionArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$Runner`""
|
||||
if ($Symbols) {
|
||||
$actionArgs += " -Symbols `"$Symbols`""
|
||||
}
|
||||
@@ -46,7 +52,7 @@ Register-ScheduledTask `
|
||||
-Trigger $trigger `
|
||||
-Principal $principal `
|
||||
-Settings $settings `
|
||||
-Description "Retrains TradeBot LSTM forecast parameters every $EveryHours hours." `
|
||||
-Description "Retrains TradeBot PyTorch recurrent forecast parameters every $EveryHours hours." `
|
||||
-Force | Out-Null
|
||||
|
||||
Write-Host "Registered scheduled task '$TaskName' every $EveryHours hours."
|
||||
@@ -1,131 +0,0 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateSet("torch", "reservoir")]
|
||||
[string]$Trainer = "torch",
|
||||
[string]$Symbols = "",
|
||||
[int]$Limit = 0,
|
||||
[string]$Lookbacks = "",
|
||||
[string]$Units = "",
|
||||
[string]$Ridges = "",
|
||||
[string]$Architectures = "",
|
||||
[string]$HiddenSizes = "",
|
||||
[string]$Layers = "",
|
||||
[string]$Dropouts = "",
|
||||
[int]$Epochs = 0,
|
||||
[int]$Patience = 0,
|
||||
[string]$Interval = "",
|
||||
[string]$EnvFile = ""
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
||||
$RuntimeDir = Join-Path $RepoRoot "runtime"
|
||||
$LogFile = Join-Path $RuntimeDir "lstm_retrain.log"
|
||||
New-Item -ItemType Directory -Force -Path $RuntimeDir | Out-Null
|
||||
|
||||
function Write-RetrainLog {
|
||||
param([string]$Message)
|
||||
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ssK"
|
||||
"[$timestamp] $Message" | Tee-Object -FilePath $LogFile -Append
|
||||
}
|
||||
|
||||
function Resolve-Python {
|
||||
$venvPython = Join-Path $RepoRoot ".venv\Scripts\python.exe"
|
||||
if (Test-Path $venvPython) {
|
||||
return $venvPython
|
||||
}
|
||||
|
||||
$userPython = Join-Path $env:LOCALAPPDATA "Programs\TradeBotPython312\python.exe"
|
||||
if (Test-Path $userPython) {
|
||||
return $userPython
|
||||
}
|
||||
|
||||
foreach ($candidate in @("python.exe", "python")) {
|
||||
$command = Get-Command $candidate -ErrorAction SilentlyContinue
|
||||
if (-not $command) {
|
||||
continue
|
||||
}
|
||||
return $command.Source
|
||||
}
|
||||
throw "Python was not found. Create .venv or install Python 3.12."
|
||||
}
|
||||
|
||||
if (-not $Symbols -and $env:LSTM_RETRAIN_SYMBOLS) { $Symbols = $env:LSTM_RETRAIN_SYMBOLS }
|
||||
if ($Limit -le 0) {
|
||||
$Limit = if ($env:LSTM_RETRAIN_LIMIT) { [int]$env:LSTM_RETRAIN_LIMIT } else { 1000 }
|
||||
}
|
||||
if (-not $Lookbacks) { $Lookbacks = if ($env:LSTM_RETRAIN_LOOKBACKS) { $env:LSTM_RETRAIN_LOOKBACKS } else { "32,64" } }
|
||||
if (-not $Units) { $Units = if ($env:LSTM_RETRAIN_UNITS) { $env:LSTM_RETRAIN_UNITS } else { "4,6" } }
|
||||
if (-not $Ridges) { $Ridges = if ($env:LSTM_RETRAIN_RIDGES) { $env:LSTM_RETRAIN_RIDGES } else { "0.001" } }
|
||||
if (-not $Architectures) { $Architectures = if ($env:LSTM_RETRAIN_ARCHITECTURES) { $env:LSTM_RETRAIN_ARCHITECTURES } else { "lstm,gru" } }
|
||||
if (-not $HiddenSizes) { $HiddenSizes = if ($env:LSTM_RETRAIN_HIDDEN_SIZES) { $env:LSTM_RETRAIN_HIDDEN_SIZES } else { "16,32" } }
|
||||
if (-not $Layers) { $Layers = if ($env:LSTM_RETRAIN_LAYERS) { $env:LSTM_RETRAIN_LAYERS } else { "1" } }
|
||||
if (-not $Dropouts) { $Dropouts = if ($env:LSTM_RETRAIN_DROPOUTS) { $env:LSTM_RETRAIN_DROPOUTS } else { "0.0" } }
|
||||
if ($Epochs -le 0) { $Epochs = if ($env:LSTM_RETRAIN_EPOCHS) { [int]$env:LSTM_RETRAIN_EPOCHS } else { 60 } }
|
||||
if ($Patience -le 0) { $Patience = if ($env:LSTM_RETRAIN_PATIENCE) { [int]$env:LSTM_RETRAIN_PATIENCE } else { 10 } }
|
||||
if (-not $Interval -and $env:LSTM_RETRAIN_INTERVAL) { $Interval = $env:LSTM_RETRAIN_INTERVAL }
|
||||
if (-not $EnvFile -and $env:LSTM_RETRAIN_ENV) { $EnvFile = $env:LSTM_RETRAIN_ENV }
|
||||
if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" }
|
||||
|
||||
$mutex = New-Object System.Threading.Mutex($false, "TradeBotLstmRetrainer")
|
||||
$hasLock = $false
|
||||
$pushedLocation = $false
|
||||
|
||||
try {
|
||||
$hasLock = $mutex.WaitOne(0)
|
||||
if (-not $hasLock) {
|
||||
Write-RetrainLog "Another LSTM retrain is already running; skipping."
|
||||
exit 0
|
||||
}
|
||||
|
||||
$python = Resolve-Python
|
||||
if ($Trainer -eq "torch") {
|
||||
$trainerArgs = @(
|
||||
"-u",
|
||||
"tools\train_torch_recurrent_forecaster.py",
|
||||
"--limit", $Limit.ToString(),
|
||||
"--lookbacks", $Lookbacks,
|
||||
"--architectures", $Architectures,
|
||||
"--hidden-sizes", $HiddenSizes,
|
||||
"--layers", $Layers,
|
||||
"--dropouts", $Dropouts,
|
||||
"--epochs", $Epochs.ToString(),
|
||||
"--patience", $Patience.ToString()
|
||||
)
|
||||
} else {
|
||||
$trainerArgs = @(
|
||||
"-u",
|
||||
"tools\train_lstm_forecaster.py",
|
||||
"--limit", $Limit.ToString(),
|
||||
"--lookbacks", $Lookbacks,
|
||||
"--units", $Units,
|
||||
"--ridges", $Ridges
|
||||
)
|
||||
}
|
||||
if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) }
|
||||
if ($Interval) { $trainerArgs += @("--interval", $Interval) }
|
||||
if ($EnvFile) { $trainerArgs += @("--env", $EnvFile) }
|
||||
|
||||
Push-Location $RepoRoot
|
||||
$pushedLocation = $true
|
||||
Write-RetrainLog "Starting LSTM retrain: $python $($trainerArgs -join ' ')"
|
||||
& $python @trainerArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Trainer failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
Write-RetrainLog "Finished LSTM retrain."
|
||||
}
|
||||
catch {
|
||||
Write-RetrainLog "ERROR: $($_.Exception.Message)"
|
||||
exit 1
|
||||
}
|
||||
finally {
|
||||
if ($pushedLocation) {
|
||||
Pop-Location -ErrorAction SilentlyContinue
|
||||
}
|
||||
if ($hasLock) {
|
||||
$mutex.ReleaseMutex()
|
||||
}
|
||||
$mutex.Dispose()
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
RUNTIME_DIR="$REPO_ROOT/runtime"
|
||||
LOG_FILE="$RUNTIME_DIR/lstm_retrain.log"
|
||||
LOCK_FILE="$RUNTIME_DIR/lstm_retrain.lock"
|
||||
|
||||
mkdir -p "$RUNTIME_DIR"
|
||||
|
||||
log() {
|
||||
printf '[%s] %s\n' "$(date -Is)" "$*" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
if command -v flock >/dev/null 2>&1; then
|
||||
exec 9>"$LOCK_FILE"
|
||||
if ! flock -n 9; then
|
||||
log "Another LSTM retrain is already running; skipping."
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
LOCK_DIR="$LOCK_FILE.d"
|
||||
if ! mkdir "$LOCK_DIR" 2>/dev/null; then
|
||||
log "Another LSTM retrain is already running; skipping."
|
||||
exit 0
|
||||
fi
|
||||
trap 'rmdir "$LOCK_DIR"' EXIT
|
||||
fi
|
||||
|
||||
if [[ -x "$REPO_ROOT/.venv/bin/python" ]]; then
|
||||
PYTHON="$REPO_ROOT/.venv/bin/python"
|
||||
elif command -v python3 >/dev/null 2>&1; then
|
||||
PYTHON="$(command -v python3)"
|
||||
elif command -v python >/dev/null 2>&1; then
|
||||
PYTHON="$(command -v python)"
|
||||
else
|
||||
log "ERROR: Python was not found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SYMBOLS="${LSTM_RETRAIN_SYMBOLS:-}"
|
||||
LIMIT="${LSTM_RETRAIN_LIMIT:-1000}"
|
||||
LOOKBACKS="${LSTM_RETRAIN_LOOKBACKS:-16,32}"
|
||||
UNITS="${LSTM_RETRAIN_UNITS:-4,6}"
|
||||
RIDGES="${LSTM_RETRAIN_RIDGES:-0.001}"
|
||||
INTERVAL="${LSTM_RETRAIN_INTERVAL:-}"
|
||||
ENV_FILE="${LSTM_RETRAIN_ENV:-}"
|
||||
|
||||
if [[ -z "$ENV_FILE" && -f "$REPO_ROOT/.env" ]]; then
|
||||
ENV_FILE="$REPO_ROOT/.env"
|
||||
fi
|
||||
|
||||
args=(
|
||||
"tools/train_lstm_forecaster.py"
|
||||
"--limit" "$LIMIT"
|
||||
"--lookbacks" "$LOOKBACKS"
|
||||
"--units" "$UNITS"
|
||||
"--ridges" "$RIDGES"
|
||||
)
|
||||
|
||||
if [[ -n "$SYMBOLS" ]]; then args+=("--symbols" "$SYMBOLS"); fi
|
||||
if [[ -n "$INTERVAL" ]]; then args+=("--interval" "$INTERVAL"); fi
|
||||
if [[ -n "$ENV_FILE" ]]; then args+=("--env" "$ENV_FILE"); fi
|
||||
|
||||
cd "$REPO_ROOT"
|
||||
log "Starting LSTM retrain: $PYTHON -u ${args[*]}"
|
||||
"$PYTHON" -u "${args[@]}" 2>&1 | tee -a "$LOG_FILE"
|
||||
log "Finished LSTM retrain."
|
||||
@@ -0,0 +1,114 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$Symbols = "",
|
||||
[int]$Limit = 0,
|
||||
[string]$Lookbacks = "",
|
||||
[string]$Architectures = "",
|
||||
[string]$HiddenSizes = "",
|
||||
[string]$Layers = "",
|
||||
[string]$Dropouts = "",
|
||||
[int]$Epochs = 0,
|
||||
[int]$Patience = 0,
|
||||
[string]$Interval = "",
|
||||
[string]$EnvFile = ""
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
||||
$RuntimeDir = Join-Path $RepoRoot "runtime"
|
||||
$LogFile = Join-Path $RuntimeDir "torch_retrain.log"
|
||||
New-Item -ItemType Directory -Force -Path $RuntimeDir | Out-Null
|
||||
|
||||
function Write-RetrainLog {
|
||||
param([string]$Message)
|
||||
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ssK"
|
||||
"[$timestamp] $Message" | Tee-Object -FilePath $LogFile -Append
|
||||
}
|
||||
|
||||
function Resolve-Python {
|
||||
$venvPython = Join-Path $RepoRoot ".venv\Scripts\python.exe"
|
||||
if (Test-Path $venvPython) {
|
||||
return $venvPython
|
||||
}
|
||||
|
||||
$userPython = Join-Path $env:LOCALAPPDATA "Programs\TradeBotPython312\python.exe"
|
||||
if (Test-Path $userPython) {
|
||||
return $userPython
|
||||
}
|
||||
|
||||
foreach ($candidate in @("python.exe", "python")) {
|
||||
$command = Get-Command $candidate -ErrorAction SilentlyContinue
|
||||
if (-not $command) {
|
||||
continue
|
||||
}
|
||||
return $command.Source
|
||||
}
|
||||
throw "Python was not found. Create .venv or install Python 3.12."
|
||||
}
|
||||
|
||||
if (-not $Symbols -and $env:TORCH_RETRAIN_SYMBOLS) { $Symbols = $env:TORCH_RETRAIN_SYMBOLS }
|
||||
if ($Limit -le 0) {
|
||||
$Limit = if ($env:TORCH_RETRAIN_LIMIT) { [int]$env:TORCH_RETRAIN_LIMIT } else { 1000 }
|
||||
}
|
||||
if (-not $Lookbacks) { $Lookbacks = if ($env:TORCH_RETRAIN_LOOKBACKS) { $env:TORCH_RETRAIN_LOOKBACKS } else { "32,64" } }
|
||||
if (-not $Architectures) { $Architectures = if ($env:TORCH_RETRAIN_ARCHITECTURES) { $env:TORCH_RETRAIN_ARCHITECTURES } else { "lstm,gru" } }
|
||||
if (-not $HiddenSizes) { $HiddenSizes = if ($env:TORCH_RETRAIN_HIDDEN_SIZES) { $env:TORCH_RETRAIN_HIDDEN_SIZES } else { "16,32" } }
|
||||
if (-not $Layers) { $Layers = if ($env:TORCH_RETRAIN_LAYERS) { $env:TORCH_RETRAIN_LAYERS } else { "1" } }
|
||||
if (-not $Dropouts) { $Dropouts = if ($env:TORCH_RETRAIN_DROPOUTS) { $env:TORCH_RETRAIN_DROPOUTS } else { "0.0" } }
|
||||
if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 60 } }
|
||||
if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 10 } }
|
||||
if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RETRAIN_INTERVAL }
|
||||
if (-not $EnvFile -and $env:TORCH_RETRAIN_ENV) { $EnvFile = $env:TORCH_RETRAIN_ENV }
|
||||
if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" }
|
||||
|
||||
$mutex = New-Object System.Threading.Mutex($false, "TradeBotTorchRecurrentRetrainer")
|
||||
$hasLock = $false
|
||||
$pushedLocation = $false
|
||||
|
||||
try {
|
||||
$hasLock = $mutex.WaitOne(0)
|
||||
if (-not $hasLock) {
|
||||
Write-RetrainLog "Another PyTorch recurrent retrain is already running; skipping."
|
||||
exit 0
|
||||
}
|
||||
|
||||
$python = Resolve-Python
|
||||
$trainerArgs = @(
|
||||
"-u",
|
||||
"tools\train_torch_recurrent_forecaster.py",
|
||||
"--limit", $Limit.ToString(),
|
||||
"--lookbacks", $Lookbacks,
|
||||
"--architectures", $Architectures,
|
||||
"--hidden-sizes", $HiddenSizes,
|
||||
"--layers", $Layers,
|
||||
"--dropouts", $Dropouts,
|
||||
"--epochs", $Epochs.ToString(),
|
||||
"--patience", $Patience.ToString()
|
||||
)
|
||||
if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) }
|
||||
if ($Interval) { $trainerArgs += @("--interval", $Interval) }
|
||||
if ($EnvFile) { $trainerArgs += @("--env", $EnvFile) }
|
||||
|
||||
Push-Location $RepoRoot
|
||||
$pushedLocation = $true
|
||||
Write-RetrainLog "Starting PyTorch recurrent retrain: $python $($trainerArgs -join ' ')"
|
||||
& $python @trainerArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Trainer failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
Write-RetrainLog "Finished PyTorch recurrent retrain."
|
||||
}
|
||||
catch {
|
||||
Write-RetrainLog "ERROR: $($_.Exception.Message)"
|
||||
exit 1
|
||||
}
|
||||
finally {
|
||||
if ($pushedLocation) {
|
||||
Pop-Location -ErrorAction SilentlyContinue
|
||||
}
|
||||
if ($hasLock) {
|
||||
$mutex.ReleaseMutex()
|
||||
}
|
||||
$mutex.Dispose()
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import replace
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from crypto_spot_bot.bybit import BybitClient
|
||||
from crypto_spot_bot.config import Settings, load_settings
|
||||
from crypto_spot_bot.time_series import _log_returns, _validate_candidates
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = _parse_args()
|
||||
settings = load_settings(args.env)
|
||||
client = BybitClient(settings)
|
||||
symbols = _symbols(args.symbols, settings, client)
|
||||
interval = args.interval or settings.base_interval
|
||||
output = Path(args.output) if args.output else settings.time_series_lstm_model_path
|
||||
|
||||
artifact: dict[str, Any] = {
|
||||
"version": 1,
|
||||
"type": "lstm_reservoir_ridge_params",
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"interval": interval,
|
||||
"limit": args.limit,
|
||||
"symbols": {},
|
||||
}
|
||||
|
||||
for symbol in symbols:
|
||||
result = _train_symbol(
|
||||
client=client,
|
||||
settings=settings,
|
||||
symbol=symbol,
|
||||
interval=interval,
|
||||
limit=args.limit,
|
||||
lookbacks=_ints(args.lookbacks),
|
||||
units_values=_ints(args.units),
|
||||
ridges=_floats(args.ridges),
|
||||
)
|
||||
if result is None:
|
||||
print(f"{symbol}: skipped, not enough candles or returns")
|
||||
continue
|
||||
artifact["symbols"][symbol] = result
|
||||
print(
|
||||
f"{symbol}: lookback={result['lookback']} units={result['units']} "
|
||||
f"ridge={result['ridge']} mae={result['validation_mae_percent']:.5f}% "
|
||||
f"baseline={result['baseline_mae_percent']:.5f}% skill={result['skill']:.4f}"
|
||||
)
|
||||
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_output = output.with_name(f"{output.name}.tmp")
|
||||
tmp_output.write_text(json.dumps(artifact, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
tmp_output.replace(output)
|
||||
print(f"saved {output}")
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Train lightweight LSTM forecast params on Bybit spot candles.")
|
||||
parser.add_argument("--env", default=None, help="Path to .env file.")
|
||||
parser.add_argument("--symbols", default="", help="Comma-separated symbols. Defaults to configured or popular spot pairs.")
|
||||
parser.add_argument("--interval", default="", help="Bybit kline interval. Defaults to BASE_INTERVAL.")
|
||||
parser.add_argument("--limit", type=int, default=1000, help="Kline limit per symbol.")
|
||||
parser.add_argument("--lookbacks", default="16,32", help="Comma-separated LSTM lookback candidates.")
|
||||
parser.add_argument("--units", default="4,6", help="Comma-separated LSTM unit candidates.")
|
||||
parser.add_argument("--ridges", default="0.001", help="Comma-separated ridge candidates.")
|
||||
parser.add_argument("--output", default="", help="Output JSON path. Defaults to TIME_SERIES_LSTM_MODEL_PATH.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _symbols(raw: str, settings: Settings, client: BybitClient) -> list[str]:
|
||||
if raw.strip():
|
||||
return [item.strip().upper() for item in raw.split(",") if item.strip()]
|
||||
if settings.symbols:
|
||||
return list(settings.symbols)
|
||||
return client.popular_spot_symbols(settings.top_symbols_count)
|
||||
|
||||
|
||||
def _train_symbol(
|
||||
*,
|
||||
client: BybitClient,
|
||||
settings: Settings,
|
||||
symbol: str,
|
||||
interval: str,
|
||||
limit: int,
|
||||
lookbacks: list[int],
|
||||
units_values: list[int],
|
||||
ridges: list[float],
|
||||
) -> dict[str, Any] | None:
|
||||
candles = client.klines(symbol, interval, limit)
|
||||
closes = [float(candle.close) for candle in candles if candle.close > 0]
|
||||
returns = _log_returns(closes)
|
||||
if len(returns) < 80:
|
||||
return None
|
||||
validation_window = min(max(8, settings.time_series_validation_window), max(8, len(returns) // 3))
|
||||
best: dict[str, Any] | None = None
|
||||
for lookback in lookbacks:
|
||||
for units in units_values:
|
||||
for ridge in ridges:
|
||||
candidate_settings = replace(
|
||||
settings,
|
||||
time_series_lstm_enabled=True,
|
||||
time_series_lstm_lookback=lookback,
|
||||
time_series_lstm_units=units,
|
||||
time_series_lstm_ridge=ridge,
|
||||
)
|
||||
candidates = _validate_candidates(returns, validation_window, candidate_settings, symbol, {})
|
||||
baseline = next((item for item in candidates if item["model"] == "naive"), None)
|
||||
lstm = next((item for item in candidates if item["model"] == "lstm"), None)
|
||||
if baseline is None or lstm is None:
|
||||
continue
|
||||
baseline_mae = float(baseline["mae"])
|
||||
lstm_mae = float(lstm["mae"])
|
||||
skill = (baseline_mae - lstm_mae) / baseline_mae if baseline_mae > 0 else 0.0
|
||||
row = {
|
||||
"lookback": lookback,
|
||||
"units": units,
|
||||
"ridge": ridge,
|
||||
"validation_mae_percent": lstm_mae * 100,
|
||||
"baseline_mae_percent": baseline_mae * 100,
|
||||
"skill": skill,
|
||||
"candles": len(candles),
|
||||
"returns": len(returns),
|
||||
}
|
||||
if best is None or lstm_mae < best["validation_mae_percent"] / 100:
|
||||
best = row
|
||||
return best
|
||||
|
||||
|
||||
def _ints(raw: str) -> list[int]:
|
||||
return [int(item.strip()) for item in raw.split(",") if item.strip()]
|
||||
|
||||
|
||||
def _floats(raw: str) -> list[float]:
|
||||
return [float(item.strip()) for item in raw.split(",") if item.strip()]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user