Codex
812 lines
29 KiB
Python
812 lines
29 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
from dataclasses import asdict, dataclass, field
|
|
from typing import Any
|
|
|
|
from crypto_spot_bot.config import Settings
|
|
from crypto_spot_bot.models import Candle
|
|
|
|
|
|
DEFAULT_TORCH_FEATURES = (
|
|
"return_1",
|
|
"return_3",
|
|
"return_6",
|
|
"range_percent",
|
|
"body_percent",
|
|
"upper_wick_percent",
|
|
"lower_wick_percent",
|
|
"volume_change",
|
|
"volume_ratio",
|
|
"atr_percent",
|
|
"rsi_centered",
|
|
"macd_hist_percent",
|
|
"ema50_gap_percent",
|
|
"ema200_gap_percent",
|
|
"pattern_score",
|
|
"pattern_bullish",
|
|
"pattern_bearish",
|
|
"pattern_range",
|
|
"pattern_pullback",
|
|
"pattern_oversold_reversal",
|
|
"pattern_stabilized_drop",
|
|
"pattern_breakout",
|
|
"pattern_breakdown",
|
|
"pattern_fast_drop",
|
|
"pattern_volume_spike",
|
|
"pattern_range_position_20",
|
|
)
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class TimeSeriesForecast:
|
|
enabled: bool
|
|
usable: bool
|
|
model: str
|
|
volatility_model: str
|
|
expected_return_percent: float
|
|
expected_price: float
|
|
volatility_percent: float
|
|
probability_up: float
|
|
confidence_adjustment: float
|
|
block_entry: bool
|
|
validation_mae_percent: float
|
|
baseline_mae_percent: float
|
|
skill: float
|
|
horizon: int
|
|
reason: str
|
|
candidates: list[dict[str, Any]] = field(default_factory=list)
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return asdict(self)
|
|
|
|
|
|
class TimeSeriesForecaster:
|
|
def __init__(self, settings: Settings):
|
|
self.settings = settings
|
|
self._lstm_artifact_mtime: float | None = None
|
|
self._lstm_artifact: dict[str, Any] = {}
|
|
|
|
def forecast(self, candles: list[Candle], symbol: str | None = None) -> TimeSeriesForecast:
|
|
if not self.settings.time_series_forecast_enabled:
|
|
return _empty_forecast(False, "time-series forecast is disabled")
|
|
closes = [float(candle.close) for candle in candles if candle.close > 0]
|
|
min_candles = max(30, self.settings.time_series_min_candles)
|
|
if len(closes) < min_candles:
|
|
return _empty_forecast(True, "not enough candles for PyTorch forecast")
|
|
returns = _log_returns(closes)
|
|
if len(returns) < 20:
|
|
return _empty_forecast(True, "not enough returns for PyTorch forecast")
|
|
|
|
artifact = self._load_lstm_artifact()
|
|
entry = _torch_recurrent_entry(symbol, artifact)
|
|
model = _torch_recurrent_model_name(symbol, artifact)
|
|
feature_rows = _feature_matrix(candles, _feature_names(entry)) if entry else []
|
|
if not model or not _can_use_torch_recurrent(returns, symbol, artifact, feature_rows):
|
|
return _empty_forecast(True, "no valid PyTorch LSTM/GRU model for symbol")
|
|
|
|
prediction = _torch_recurrent_predict(
|
|
returns,
|
|
symbol,
|
|
artifact,
|
|
feature_rows=feature_rows,
|
|
closes=closes,
|
|
)
|
|
if entry is None or prediction is None:
|
|
return _empty_forecast(True, "PyTorch LSTM/GRU model could not build a forecast")
|
|
|
|
direct_horizon = _is_direct_horizon(entry)
|
|
horizon = _entry_horizon(entry, self.settings.time_series_forecast_horizon)
|
|
expected_return = prediction if direct_horizon else prediction * horizon
|
|
expected_price = closes[-1] * math.exp(expected_return)
|
|
model_mae = _torch_validation_mae(entry, returns)
|
|
baseline_mae = max(_float_entry(entry, "baseline_mae_percent", model_mae * 100) / 100, model_mae)
|
|
if direct_horizon:
|
|
uncertainty = max(model_mae, _horizon_return_scale(closes, horizon) * 0.25, 1e-9)
|
|
volatility_model = "direct horizon validation MAE"
|
|
else:
|
|
uncertainty_one_step = max(model_mae, _return_scale(returns) * 0.25, 1e-9)
|
|
uncertainty = uncertainty_one_step * math.sqrt(horizon)
|
|
volatility_model = "one-step validation MAE scaled by horizon"
|
|
volatility_percent = uncertainty * 100
|
|
expected_return_percent = (math.exp(expected_return) - 1) * 100
|
|
probability_up = _normal_cdf(expected_return / max(uncertainty, 1e-9))
|
|
skill = _clamp(_float_entry(entry, "skill", 0.0), -1.0, 1.0)
|
|
min_edge = max(0.0, self.settings.time_series_min_edge_percent)
|
|
confidence_adjustment = _confidence_adjustment(
|
|
expected_return_percent=expected_return_percent,
|
|
probability_up=probability_up,
|
|
skill=skill,
|
|
min_edge=min_edge,
|
|
max_adjustment=self.settings.time_series_max_adjustment,
|
|
)
|
|
block_entry = bool(expected_return_percent <= -min_edge and probability_up <= 0.45)
|
|
reason = _reason(
|
|
model=model,
|
|
expected_return_percent=expected_return_percent,
|
|
probability_up=probability_up,
|
|
skill=skill,
|
|
block_entry=block_entry,
|
|
)
|
|
return TimeSeriesForecast(
|
|
enabled=True,
|
|
usable=True,
|
|
model=model,
|
|
volatility_model=volatility_model,
|
|
expected_return_percent=round(expected_return_percent, 4),
|
|
expected_price=round(expected_price, 8),
|
|
volatility_percent=round(volatility_percent, 4),
|
|
probability_up=round(probability_up, 4),
|
|
confidence_adjustment=round(confidence_adjustment, 4),
|
|
block_entry=block_entry,
|
|
validation_mae_percent=round(model_mae * 100, 4),
|
|
baseline_mae_percent=round(baseline_mae * 100, 4),
|
|
skill=round(skill, 4),
|
|
horizon=horizon,
|
|
reason=reason,
|
|
candidates=[{"model": model, "mae_percent": round(model_mae * 100, 4)}],
|
|
)
|
|
|
|
def _load_lstm_artifact(self) -> dict[str, Any]:
|
|
if not self.settings.time_series_lstm_enabled:
|
|
return {}
|
|
path = self.settings.time_series_lstm_model_path
|
|
try:
|
|
stat = path.stat()
|
|
except OSError:
|
|
self._lstm_artifact_mtime = None
|
|
self._lstm_artifact = {}
|
|
return {}
|
|
if self._lstm_artifact_mtime == stat.st_mtime:
|
|
return self._lstm_artifact
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
data = {}
|
|
self._lstm_artifact = data if isinstance(data, dict) else {}
|
|
self._lstm_artifact_mtime = stat.st_mtime
|
|
return self._lstm_artifact
|
|
|
|
|
|
def _empty_forecast(enabled: bool, reason: str) -> TimeSeriesForecast:
|
|
return TimeSeriesForecast(
|
|
enabled=enabled,
|
|
usable=False,
|
|
model="none",
|
|
volatility_model="none",
|
|
expected_return_percent=0.0,
|
|
expected_price=0.0,
|
|
volatility_percent=0.0,
|
|
probability_up=0.5,
|
|
confidence_adjustment=0.0,
|
|
block_entry=False,
|
|
validation_mae_percent=0.0,
|
|
baseline_mae_percent=0.0,
|
|
skill=0.0,
|
|
horizon=0,
|
|
reason=reason,
|
|
)
|
|
|
|
|
|
def _log_returns(closes: list[float]) -> list[float]:
|
|
return [math.log(closes[index] / closes[index - 1]) for index in range(1, len(closes))]
|
|
|
|
|
|
def _feature_matrix(candles: list[Candle], feature_names: list[str] | tuple[str, ...] | None = None) -> list[list[float]]:
|
|
names = list(feature_names or DEFAULT_TORCH_FEATURES)
|
|
rows: list[list[float]] = []
|
|
for index, candle in enumerate(candles):
|
|
rows.append([_feature_value(name, candles, index, candle) for name in names])
|
|
return rows
|
|
|
|
|
|
def _feature_value(name: str, candles: list[Candle], index: int, candle: Candle) -> float:
|
|
close = max(float(candle.close), 1e-12)
|
|
previous = candles[index - 1] if index >= 1 else candle
|
|
if name == "return_1":
|
|
return _log_change(candle.close, previous.close)
|
|
if name == "return_3":
|
|
return _log_change(candle.close, candles[index - 3].close) if index >= 3 else 0.0
|
|
if name == "return_6":
|
|
return _log_change(candle.close, candles[index - 6].close) if index >= 6 else 0.0
|
|
if name == "range_percent":
|
|
return _safe_feature((candle.high - candle.low) / close)
|
|
if name == "body_percent":
|
|
return _safe_feature((candle.close - candle.open) / close)
|
|
if name == "upper_wick_percent":
|
|
return _safe_feature((candle.high - max(candle.open, candle.close)) / close)
|
|
if name == "lower_wick_percent":
|
|
return _safe_feature((min(candle.open, candle.close) - candle.low) / close)
|
|
if name == "volume_change":
|
|
return _log_change(max(candle.volume, 1e-12), max(previous.volume, 1e-12))
|
|
if name == "volume_ratio":
|
|
return _safe_feature((candle.volume / candle.volume_ma_20) - 1.0) if candle.volume_ma_20 else 0.0
|
|
if name == "atr_percent":
|
|
return _safe_feature(candle.atr_14 / close) if candle.atr_14 is not None else 0.0
|
|
if name == "rsi_centered":
|
|
return _safe_feature((candle.rsi_14 - 50.0) / 50.0) if candle.rsi_14 is not None else 0.0
|
|
if name == "macd_hist_percent":
|
|
return _safe_feature(candle.macd_hist / close) if candle.macd_hist is not None else 0.0
|
|
if name == "ema50_gap_percent":
|
|
return _safe_feature((candle.close - candle.ema_50) / close) if candle.ema_50 is not None else 0.0
|
|
if name == "ema200_gap_percent":
|
|
return _safe_feature((candle.close - candle.ema_200) / close) if candle.ema_200 is not None else 0.0
|
|
if name.startswith("pattern_"):
|
|
return _pattern_feature_value(name, candles, index)
|
|
return 0.0
|
|
|
|
|
|
def _pattern_feature_value(name: str, candles: list[Candle], index: int) -> float:
|
|
pattern = _pattern_snapshot(candles, index)
|
|
if name == "pattern_score":
|
|
return pattern["score"]
|
|
if name == "pattern_bullish":
|
|
return pattern["bullish"]
|
|
if name == "pattern_bearish":
|
|
return pattern["bearish"]
|
|
if name == "pattern_range":
|
|
return pattern["range"]
|
|
if name == "pattern_pullback":
|
|
return pattern["pullback"]
|
|
if name == "pattern_oversold_reversal":
|
|
return pattern["oversold_reversal"]
|
|
if name == "pattern_stabilized_drop":
|
|
return pattern["stabilized_drop"]
|
|
if name == "pattern_breakout":
|
|
return pattern["breakout"]
|
|
if name == "pattern_breakdown":
|
|
return pattern["breakdown"]
|
|
if name == "pattern_fast_drop":
|
|
return pattern["fast_drop"]
|
|
if name == "pattern_volume_spike":
|
|
return pattern["volume_spike"]
|
|
if name == "pattern_range_position_20":
|
|
return pattern["range_position_20"]
|
|
return 0.0
|
|
|
|
|
|
def _pattern_snapshot(candles: list[Candle], index: int) -> dict[str, float]:
|
|
if index < 29:
|
|
return {
|
|
"score": 0.0,
|
|
"bullish": 0.0,
|
|
"bearish": 0.0,
|
|
"range": 0.0,
|
|
"pullback": 0.0,
|
|
"oversold_reversal": 0.0,
|
|
"stabilized_drop": 0.0,
|
|
"breakout": 0.0,
|
|
"breakdown": 0.0,
|
|
"fast_drop": 0.0,
|
|
"volume_spike": 0.0,
|
|
"range_position_20": 0.5,
|
|
}
|
|
|
|
window = candles[: index + 1]
|
|
latest = window[-1]
|
|
previous = window[-2]
|
|
high20 = max(candle.high for candle in window[-20:])
|
|
low20 = min(candle.low for candle in window[-20:])
|
|
width20 = max(0.0, high20 - low20)
|
|
range_position_20 = _clamp((latest.close - low20) / width20, 0.0, 1.0) if width20 else 0.5
|
|
close_3 = window[-4].close if len(window) >= 4 else window[0].close
|
|
close_10 = window[-11].close if len(window) >= 11 else window[0].close
|
|
close_20 = window[-21].close if len(window) >= 21 else window[0].close
|
|
ret_3 = _percent_change(latest.close, close_3)
|
|
ret_10 = _percent_change(latest.close, close_10)
|
|
ret_20 = _percent_change(latest.close, close_20)
|
|
body = abs(latest.close - latest.open)
|
|
lower_wick = max(0.0, min(latest.open, latest.close) - latest.low)
|
|
atr_percent = (latest.atr_14 / latest.close * 100) if latest.atr_14 and latest.close else 0.0
|
|
volume_ratio = (
|
|
latest.volume / latest.volume_ma_20
|
|
if latest.volume_ma_20 and latest.volume_ma_20 > 0
|
|
else 0.0
|
|
)
|
|
ema_gap_percent = (
|
|
(latest.ema_50 - latest.ema_200) / latest.ema_200 * 100
|
|
if latest.ema_50 and latest.ema_200
|
|
else 0.0
|
|
)
|
|
uptrend = bool(
|
|
latest.ema_20
|
|
and latest.ema_50
|
|
and latest.ema_200
|
|
and latest.ema_20 >= latest.ema_50 >= latest.ema_200
|
|
and latest.close >= latest.ema_50
|
|
)
|
|
downtrend = bool(
|
|
latest.ema_20
|
|
and latest.ema_50
|
|
and latest.ema_200
|
|
and latest.ema_20 <= latest.ema_50 <= latest.ema_200
|
|
and latest.close <= latest.ema_50
|
|
)
|
|
pullback = bool(
|
|
latest.ema_20
|
|
and uptrend
|
|
and latest.close <= latest.ema_20 * 1.012
|
|
and latest.rsi_14 is not None
|
|
and 35 <= latest.rsi_14 <= 58
|
|
)
|
|
oversold_reversal = bool(
|
|
latest.rsi_14 is not None
|
|
and latest.rsi_14 <= 35
|
|
and latest.close > previous.close
|
|
and lower_wick >= body * 1.2
|
|
)
|
|
stabilized_drop = _pattern_stabilized_drop(
|
|
candles=window,
|
|
latest=latest,
|
|
previous=previous,
|
|
ret_3=ret_3,
|
|
ret_10=ret_10,
|
|
ret_20=ret_20,
|
|
atr_percent=atr_percent,
|
|
volume_ratio=volume_ratio,
|
|
lower_wick=lower_wick,
|
|
body=body,
|
|
)
|
|
breakout = bool(latest.close >= high20 * 0.995 and volume_ratio >= 1.15 and latest.close > latest.open)
|
|
breakdown = bool(latest.close <= low20 * 1.005 and volume_ratio >= 1.1 and latest.close < latest.open)
|
|
fast_drop = bool(ret_3 <= -max(1.2, atr_percent * 1.8) or (latest.rsi_14 or 100) <= 25)
|
|
range_market = bool(abs(ret_20) <= max(0.8, atr_percent * 1.2) and abs(ema_gap_percent) <= 0.35)
|
|
volume_spike = bool(volume_ratio >= 1.6)
|
|
|
|
score = 0.50
|
|
if fast_drop and breakdown:
|
|
score = 0.18
|
|
elif breakdown:
|
|
score = 0.24
|
|
elif pullback:
|
|
score = 0.76
|
|
elif oversold_reversal:
|
|
score = 0.68
|
|
elif stabilized_drop:
|
|
score = 0.58
|
|
elif breakout:
|
|
score = 0.72
|
|
elif uptrend:
|
|
score = 0.64
|
|
elif range_market:
|
|
score = 0.48
|
|
elif downtrend:
|
|
score = 0.28
|
|
bullish = float(pullback or oversold_reversal or stabilized_drop or breakout or uptrend)
|
|
bearish = float((fast_drop and breakdown) or breakdown or downtrend)
|
|
return {
|
|
"score": score,
|
|
"bullish": bullish,
|
|
"bearish": bearish,
|
|
"range": float(range_market),
|
|
"pullback": float(pullback),
|
|
"oversold_reversal": float(oversold_reversal),
|
|
"stabilized_drop": float(stabilized_drop),
|
|
"breakout": float(breakout),
|
|
"breakdown": float(breakdown),
|
|
"fast_drop": float(fast_drop),
|
|
"volume_spike": float(volume_spike),
|
|
"range_position_20": range_position_20,
|
|
}
|
|
|
|
|
|
def _percent_change(current: float, previous: float) -> float:
|
|
return ((current - previous) / previous * 100) if previous else 0.0
|
|
|
|
|
|
def _pattern_stabilized_drop(
|
|
*,
|
|
candles: list[Candle],
|
|
latest: Candle,
|
|
previous: Candle,
|
|
ret_3: float,
|
|
ret_10: float,
|
|
ret_20: float,
|
|
atr_percent: float,
|
|
volume_ratio: float,
|
|
lower_wick: float,
|
|
body: float,
|
|
) -> bool:
|
|
recent_drop = ret_10 <= -max(0.35, atr_percent * 1.1) or ret_20 <= -max(0.6, atr_percent * 1.6)
|
|
if not recent_drop or latest.rsi_14 is None or latest.rsi_14 > 52:
|
|
return False
|
|
recent_lows = [candle.low for candle in candles[-5:-1]]
|
|
no_new_low = bool(recent_lows) and latest.low >= min(recent_lows) * 0.999
|
|
bounce_from_low = ((latest.close - min(candle.low for candle in candles[-6:])) / latest.close * 100) if latest.close else 0.0
|
|
body_base = max(body, latest.close * 0.0001)
|
|
absorption = lower_wick >= body_base * 0.6 or bounce_from_low >= max(0.08, atr_percent * 0.3)
|
|
momentum_stabilized = latest.close >= previous.close or abs(ret_3) <= max(0.25, atr_percent * 0.8) or no_new_low
|
|
volume_present = volume_ratio >= 0.55
|
|
continuing_drop = latest.close < previous.close and not no_new_low and ret_3 <= -max(0.6, atr_percent * 1.2)
|
|
return bool(momentum_stabilized and absorption and volume_present and not continuing_drop)
|
|
|
|
|
|
def _log_change(current: float, previous: float) -> float:
|
|
if current <= 0 or previous <= 0:
|
|
return 0.0
|
|
return _safe_feature(math.log(current / previous))
|
|
|
|
|
|
def _safe_feature(value: float) -> float:
|
|
if not math.isfinite(value):
|
|
return 0.0
|
|
return _clamp(float(value), -50.0, 50.0)
|
|
|
|
|
|
def _torch_recurrent_model_name(symbol: str | None, artifact: dict[str, Any]) -> str | None:
|
|
entry = _torch_recurrent_entry(symbol, artifact)
|
|
if not entry:
|
|
return None
|
|
architecture = str(entry.get("architecture", "")).strip().lower()
|
|
if architecture in {"lstm", "gru"}:
|
|
return f"torch_{architecture}"
|
|
model = str(entry.get("model", "")).strip().lower()
|
|
return model if model in {"torch_lstm", "torch_gru"} else None
|
|
|
|
|
|
def _torch_recurrent_entry(symbol: str | None, artifact: dict[str, Any]) -> dict[str, Any] | None:
|
|
if artifact.get("type") != "pytorch_recurrent_forecaster":
|
|
return None
|
|
symbols = artifact.get("symbols")
|
|
entry = symbols.get(symbol.upper()) if symbol and isinstance(symbols, dict) else None
|
|
if not isinstance(entry, dict):
|
|
default = artifact.get("default")
|
|
entry = default if isinstance(default, dict) else None
|
|
if not isinstance(entry, dict):
|
|
return None
|
|
if not isinstance(entry.get("state_dict"), dict):
|
|
return None
|
|
return entry
|
|
|
|
|
|
def _can_use_torch_recurrent(
|
|
returns: list[float],
|
|
symbol: str | None,
|
|
artifact: dict[str, Any],
|
|
feature_rows: list[list[float]] | None = None,
|
|
) -> bool:
|
|
entry = _torch_recurrent_entry(symbol, artifact)
|
|
if not entry:
|
|
return False
|
|
lookback = int(_clamp(_float_entry(entry, "lookback", 0.0), 4.0, 512.0))
|
|
hidden_size = int(_clamp(_float_entry(entry, "hidden_size", 0.0), 1.0, 512.0))
|
|
num_layers = int(_clamp(_float_entry(entry, "num_layers", 1.0), 1.0, 8.0))
|
|
if hidden_size <= 0 or num_layers <= 0:
|
|
return False
|
|
if _is_direct_horizon(entry):
|
|
return bool(feature_rows and len(feature_rows) >= lookback)
|
|
return len(returns) >= lookback + 1
|
|
|
|
|
|
def _torch_recurrent_predict(
|
|
returns: list[float],
|
|
symbol: str | None,
|
|
artifact: dict[str, Any],
|
|
*,
|
|
feature_rows: list[list[float]] | None = None,
|
|
closes: list[float] | None = None,
|
|
) -> float | None:
|
|
entry = _torch_recurrent_entry(symbol, artifact)
|
|
model_name = _torch_recurrent_model_name(symbol, artifact)
|
|
if not entry or not model_name:
|
|
return None
|
|
lookback = int(_clamp(_float_entry(entry, "lookback", 0.0), 4.0, 512.0))
|
|
hidden_size = int(_clamp(_float_entry(entry, "hidden_size", 0.0), 1.0, 512.0))
|
|
num_layers = int(_clamp(_float_entry(entry, "num_layers", 1.0), 1.0, 8.0))
|
|
clip = _clamp(_float_entry(entry, "clip", 8.0), 1.0, 50.0)
|
|
direct_horizon = _is_direct_horizon(entry)
|
|
|
|
if direct_horizon:
|
|
rows = feature_rows or []
|
|
if len(rows) < lookback:
|
|
return None
|
|
sequence = _normalize_feature_rows(rows[-lookback:], entry, clip)
|
|
target_mean = _float_entry(entry, "target_mean", 0.0)
|
|
target_scale = max(_float_entry(entry, "target_scale", _return_scale(returns)), 1e-8)
|
|
else:
|
|
mean = _float_entry(entry, "mean", 0.0)
|
|
scale = max(_float_entry(entry, "scale", _return_scale(returns)), 1e-8)
|
|
if len(returns) < lookback:
|
|
return None
|
|
sequence = [[_clamp((value - mean) / scale, -clip, clip)] for value in returns[-lookback:]]
|
|
target_mean = mean
|
|
target_scale = scale
|
|
|
|
try:
|
|
hidden = _torch_recurrent_hidden(
|
|
sequence,
|
|
entry=entry,
|
|
model_name=model_name,
|
|
hidden_size=hidden_size,
|
|
num_layers=num_layers,
|
|
)
|
|
if hidden is None:
|
|
return None
|
|
head_weight = _float_vector(entry.get("head_weight"))
|
|
head_bias = _float_entry(entry, "head_bias", 0.0)
|
|
if len(head_weight) != hidden_size:
|
|
return None
|
|
normalized_prediction = sum(weight * value for weight, value in zip(head_weight, hidden)) + head_bias
|
|
if not math.isfinite(normalized_prediction):
|
|
return None
|
|
prediction = _clamp(normalized_prediction, -clip, clip) * target_scale + target_mean
|
|
except (IndexError, KeyError, TypeError, ValueError, OverflowError):
|
|
return None
|
|
|
|
if direct_horizon and closes:
|
|
horizon = _entry_horizon(entry, 1)
|
|
recent_abs = sorted(abs(value) for value in _horizon_log_returns(closes, horizon)[-48:])
|
|
else:
|
|
recent_abs = sorted(abs(value) for value in returns[-48:]) if len(returns) >= 8 else [0.01]
|
|
cap = max(recent_abs[int(len(recent_abs) * 0.9)] if recent_abs else 0.0, 0.0002)
|
|
return _clamp(prediction, -cap, cap)
|
|
|
|
|
|
def _normalize_feature_rows(rows: list[list[float]], entry: dict[str, Any], clip: float) -> list[list[float]]:
|
|
means = _float_vector(entry.get("feature_means"))
|
|
scales = _float_vector(entry.get("feature_scales"))
|
|
input_size = int(_clamp(_float_entry(entry, "input_size", len(rows[-1]) if rows else 1), 1.0, 256.0))
|
|
if len(means) != input_size:
|
|
means = [0.0 for _ in range(input_size)]
|
|
if len(scales) != input_size:
|
|
scales = [1.0 for _ in range(input_size)]
|
|
normalized = []
|
|
for row in rows:
|
|
normalized.append(
|
|
[
|
|
_clamp(((row[index] if index < len(row) else 0.0) - means[index]) / max(scales[index], 1e-8), -clip, clip)
|
|
for index in range(input_size)
|
|
]
|
|
)
|
|
return normalized
|
|
|
|
|
|
def _torch_recurrent_hidden(
|
|
sequence: list[list[float]],
|
|
*,
|
|
entry: dict[str, Any],
|
|
model_name: str,
|
|
hidden_size: int,
|
|
num_layers: int,
|
|
) -> list[float] | None:
|
|
state = entry.get("state_dict")
|
|
if not isinstance(state, dict):
|
|
return None
|
|
h_layers = [[0.0 for _ in range(hidden_size)] for _ in range(num_layers)]
|
|
c_layers = [[0.0 for _ in range(hidden_size)] for _ in range(num_layers)]
|
|
for row in sequence:
|
|
layer_input = list(row)
|
|
for layer in range(num_layers):
|
|
if model_name == "torch_lstm":
|
|
next_hidden, next_cell = _torch_lstm_step(layer_input, h_layers[layer], c_layers[layer], state, layer)
|
|
h_layers[layer] = next_hidden
|
|
c_layers[layer] = next_cell
|
|
elif model_name == "torch_gru":
|
|
h_layers[layer] = _torch_gru_step(layer_input, h_layers[layer], state, layer)
|
|
else:
|
|
return None
|
|
layer_input = h_layers[layer]
|
|
return h_layers[-1]
|
|
|
|
|
|
def _torch_lstm_step(
|
|
inputs: list[float],
|
|
hidden: list[float],
|
|
cell: list[float],
|
|
state: dict[str, Any],
|
|
layer: int,
|
|
) -> tuple[list[float], list[float]]:
|
|
hidden_size = len(hidden)
|
|
gates = _torch_gate_values(inputs, hidden, state, layer, gate_count=4)
|
|
input_gate = [_sigmoid(value) for value in gates[0]]
|
|
forget_gate = [_sigmoid(value) for value in gates[1]]
|
|
cell_gate = [math.tanh(value) for value in gates[2]]
|
|
output_gate = [_sigmoid(value) for value in gates[3]]
|
|
next_cell = [
|
|
forget_gate[index] * cell[index] + input_gate[index] * cell_gate[index]
|
|
for index in range(hidden_size)
|
|
]
|
|
next_hidden = [
|
|
output_gate[index] * math.tanh(next_cell[index])
|
|
for index in range(hidden_size)
|
|
]
|
|
return next_hidden, next_cell
|
|
|
|
|
|
def _torch_gru_step(
|
|
inputs: list[float],
|
|
hidden: list[float],
|
|
state: dict[str, Any],
|
|
layer: int,
|
|
) -> list[float]:
|
|
hidden_size = len(hidden)
|
|
weight_ih = _float_matrix(state[f"weight_ih_l{layer}"])
|
|
weight_hh = _float_matrix(state[f"weight_hh_l{layer}"])
|
|
bias_ih = _float_vector(state[f"bias_ih_l{layer}"])
|
|
bias_hh = _float_vector(state[f"bias_hh_l{layer}"])
|
|
|
|
def gate_input(gate: int) -> list[float]:
|
|
start = gate * hidden_size
|
|
output = []
|
|
for index in range(hidden_size):
|
|
row = start + index
|
|
output.append(_dot(weight_ih[row], inputs) + bias_ih[row])
|
|
return output
|
|
|
|
def gate_hidden(gate: int) -> list[float]:
|
|
start = gate * hidden_size
|
|
output = []
|
|
for index in range(hidden_size):
|
|
row = start + index
|
|
output.append(_dot(weight_hh[row], hidden) + bias_hh[row])
|
|
return output
|
|
|
|
reset_input = gate_input(0)
|
|
update_input = gate_input(1)
|
|
new_input = gate_input(2)
|
|
reset_hidden = gate_hidden(0)
|
|
update_hidden = gate_hidden(1)
|
|
new_hidden = gate_hidden(2)
|
|
|
|
reset_gate = [_sigmoid(reset_input[index] + reset_hidden[index]) for index in range(hidden_size)]
|
|
update_gate = [_sigmoid(update_input[index] + update_hidden[index]) for index in range(hidden_size)]
|
|
candidate = [
|
|
math.tanh(new_input[index] + reset_gate[index] * new_hidden[index])
|
|
for index in range(hidden_size)
|
|
]
|
|
return [
|
|
(1 - update_gate[index]) * candidate[index] + update_gate[index] * hidden[index]
|
|
for index in range(hidden_size)
|
|
]
|
|
|
|
|
|
def _torch_gate_values(
|
|
inputs: list[float],
|
|
hidden: list[float],
|
|
state: dict[str, Any],
|
|
layer: int,
|
|
gate_count: int,
|
|
) -> list[list[float]]:
|
|
hidden_size = len(hidden)
|
|
weight_ih = _float_matrix(state[f"weight_ih_l{layer}"])
|
|
weight_hh = _float_matrix(state[f"weight_hh_l{layer}"])
|
|
bias_ih = _float_vector(state[f"bias_ih_l{layer}"])
|
|
bias_hh = _float_vector(state[f"bias_hh_l{layer}"])
|
|
gates: list[list[float]] = []
|
|
for gate in range(gate_count):
|
|
values = []
|
|
start = gate * hidden_size
|
|
for index in range(hidden_size):
|
|
row = start + index
|
|
values.append(_dot(weight_ih[row], inputs) + _dot(weight_hh[row], hidden) + bias_ih[row] + bias_hh[row])
|
|
gates.append(values)
|
|
return gates
|
|
|
|
|
|
def _torch_validation_mae(entry: dict[str, Any], returns: list[float]) -> float:
|
|
mae_percent = _float_entry(entry, "validation_mae_percent", 0.0)
|
|
if mae_percent > 0:
|
|
return mae_percent / 100
|
|
return _return_scale(returns)
|
|
|
|
|
|
def _feature_names(entry: dict[str, Any] | None) -> list[str]:
|
|
if not entry:
|
|
return list(DEFAULT_TORCH_FEATURES)
|
|
names = entry.get("feature_names")
|
|
if isinstance(names, list) and names:
|
|
return [str(name) for name in names]
|
|
return list(DEFAULT_TORCH_FEATURES)
|
|
|
|
|
|
def _is_direct_horizon(entry: dict[str, Any]) -> bool:
|
|
return bool(entry.get("direct_horizon")) or "target_horizon" in entry
|
|
|
|
|
|
def _entry_horizon(entry: dict[str, Any], default: int) -> int:
|
|
return int(_clamp(_float_entry(entry, "target_horizon", float(max(1, default))), 1.0, 96.0))
|
|
|
|
|
|
def _float_entry(data: dict[str, Any], key: str, default: float) -> float:
|
|
value = data.get(key)
|
|
if isinstance(value, (int, float)):
|
|
return float(value)
|
|
if isinstance(value, str):
|
|
try:
|
|
return float(value)
|
|
except ValueError:
|
|
return default
|
|
return default
|
|
|
|
|
|
def _float_vector(data: Any) -> list[float]:
|
|
if not isinstance(data, list):
|
|
return []
|
|
return [float(value) for value in data]
|
|
|
|
|
|
def _float_matrix(data: Any) -> list[list[float]]:
|
|
if not isinstance(data, list):
|
|
return []
|
|
return [_float_vector(row) for row in data]
|
|
|
|
|
|
def _dot(left: list[float], right: list[float]) -> float:
|
|
return sum(left[index] * right[index] for index in range(min(len(left), len(right))))
|
|
|
|
|
|
def _return_scale(returns: list[float]) -> float:
|
|
recent = returns[-120:] if len(returns) > 120 else returns
|
|
values = sorted(abs(value) for value in recent if math.isfinite(value))
|
|
if not values:
|
|
return 0.0005
|
|
median = values[len(values) // 2]
|
|
mean = sum(values) / len(values)
|
|
return max(max(median, mean * 0.5), 1e-5)
|
|
|
|
|
|
def _horizon_log_returns(closes: list[float], horizon: int) -> list[float]:
|
|
horizon = max(1, horizon)
|
|
values = []
|
|
for index in range(0, len(closes) - horizon):
|
|
current = closes[index]
|
|
future = closes[index + horizon]
|
|
if current > 0 and future > 0:
|
|
values.append(math.log(future / current))
|
|
return values
|
|
|
|
|
|
def _horizon_return_scale(closes: list[float], horizon: int) -> float:
|
|
values = _horizon_log_returns(closes, horizon)
|
|
return _return_scale(values) if values else 0.0005
|
|
|
|
|
|
def _sigmoid(value: float) -> float:
|
|
if value >= 40:
|
|
return 1.0
|
|
if value <= -40:
|
|
return 0.0
|
|
return 1 / (1 + math.exp(-value))
|
|
|
|
|
|
def _confidence_adjustment(
|
|
*,
|
|
expected_return_percent: float,
|
|
probability_up: float,
|
|
skill: float,
|
|
min_edge: float,
|
|
max_adjustment: float,
|
|
) -> float:
|
|
edge = abs(expected_return_percent) - min_edge
|
|
if edge <= 0:
|
|
return 0.0
|
|
direction = 1.0 if expected_return_percent > 0 and probability_up >= 0.55 else -1.0
|
|
if direction < 0 and probability_up > 0.45:
|
|
return 0.0
|
|
strength = _clamp(edge / max(min_edge, 0.05), 0.0, 1.0)
|
|
probability_strength = _clamp(abs(probability_up - 0.5) / 0.25, 0.0, 1.0)
|
|
skill_strength = _clamp((skill + 0.03) / 0.18, 0.25, 1.0)
|
|
return direction * _clamp(max_adjustment, 0.0, 0.18) * strength * probability_strength * skill_strength
|
|
|
|
|
|
def _reason(
|
|
*,
|
|
model: str,
|
|
expected_return_percent: float,
|
|
probability_up: float,
|
|
skill: float,
|
|
block_entry: bool,
|
|
) -> str:
|
|
if block_entry:
|
|
return f"model {model}: expected move down {expected_return_percent:.3f}%, P(up)={probability_up:.2f}"
|
|
return f"model {model}: forecast {expected_return_percent:.3f}%, P(up)={probability_up:.2f}, skill={skill:.3f}"
|
|
|
|
|
|
def _normal_cdf(value: float) -> float:
|
|
return 0.5 * (1 + math.erf(value / math.sqrt(2)))
|
|
|
|
|
|
def _clamp(value: float, low: float, high: float) -> float:
|
|
return max(low, min(high, value))
|