from __future__ import annotations import json import math from dataclasses import asdict, dataclass, field from functools import lru_cache from typing import Any from crypto_spot_bot.config import Settings from crypto_spot_bot.models import Candle @dataclass(slots=True) class TimeSeriesForecast: enabled: bool usable: bool model: str volatility_model: str expected_return_percent: float expected_price: float volatility_percent: float probability_up: float confidence_adjustment: float block_entry: bool validation_mae_percent: float baseline_mae_percent: float skill: float horizon: int reason: str candidates: list[dict[str, Any]] = field(default_factory=list) def as_dict(self) -> dict[str, Any]: return asdict(self) class TimeSeriesForecaster: def __init__(self, settings: Settings): self.settings = settings self._lstm_artifact_mtime: float | None = None self._lstm_artifact: dict[str, Any] = {} def forecast(self, candles: list[Candle], symbol: str | None = None) -> TimeSeriesForecast: if not self.settings.time_series_forecast_enabled: return _empty_forecast(False, "прогноз временных рядов выключен") closes = [float(candle.close) for candle in candles if candle.close > 0] min_candles = max(30, self.settings.time_series_min_candles) if len(closes) < min_candles: return _empty_forecast(True, "недостаточно свечей для прогноза") returns = _log_returns(closes) if len(returns) < 20: return _empty_forecast(True, "недостаточно доходностей для прогноза") validation_window = min( max(8, self.settings.time_series_validation_window), max(8, len(returns) // 3), ) lstm_artifact = self._load_lstm_artifact() candidates = _validate_candidates(returns, validation_window, self.settings, symbol, lstm_artifact) best = min(candidates, key=lambda item: item["mae"]) baseline = next(item for item in candidates if item["model"] == "naive") latest_prediction = _predict_next_return(best["model"], returns, self.settings, symbol, lstm_artifact) horizon = max(1, self.settings.time_series_forecast_horizon) expected_return = latest_prediction * horizon expected_price = closes[-1] * math.exp(expected_return) ewma_vol = _ewma_volatility(returns, self.settings.time_series_ewma_lambda) garch_vol = _fixed_garch_volatility(returns) vol_one_step = max(ewma_vol, garch_vol) volatility_percent = vol_one_step * math.sqrt(horizon) * 100 expected_return_percent = (math.exp(expected_return) - 1) * 100 probability_up = _normal_cdf(expected_return / max(vol_one_step * math.sqrt(horizon), 1e-9)) baseline_mae = float(baseline["mae"]) model_mae = float(best["mae"]) skill = (baseline_mae - model_mae) / baseline_mae if baseline_mae > 0 else 0.0 skill = _clamp(skill, -1.0, 1.0) min_edge = max(0.0, self.settings.time_series_min_edge_percent) usable_skill = skill > 0.02 and best["model"] != "naive" confidence_adjustment = _confidence_adjustment( expected_return_percent=expected_return_percent, probability_up=probability_up, skill=skill, min_edge=min_edge, max_adjustment=self.settings.time_series_max_adjustment, usable_skill=usable_skill, ) block_entry = bool( usable_skill and expected_return_percent <= -min_edge and probability_up <= 0.45 ) reason = _reason( model=best["model"], expected_return_percent=expected_return_percent, probability_up=probability_up, skill=skill, block_entry=block_entry, usable_skill=usable_skill, ) return TimeSeriesForecast( enabled=True, usable=True, model=best["model"], volatility_model="max(EWMA,GARCH-like)", expected_return_percent=round(expected_return_percent, 4), expected_price=round(expected_price, 8), volatility_percent=round(volatility_percent, 4), probability_up=round(probability_up, 4), confidence_adjustment=round(confidence_adjustment, 4), block_entry=block_entry, validation_mae_percent=round(model_mae * 100, 4), baseline_mae_percent=round(baseline_mae * 100, 4), skill=round(skill, 4), horizon=horizon, reason=reason, candidates=[ {"model": item["model"], "mae_percent": round(float(item["mae"]) * 100, 4)} for item in sorted(candidates, key=lambda item: item["mae"]) ], ) def _load_lstm_artifact(self) -> dict[str, Any]: if not self.settings.time_series_lstm_enabled: return {} path = self.settings.time_series_lstm_model_path try: stat = path.stat() except OSError: self._lstm_artifact_mtime = None self._lstm_artifact = {} return {} if self._lstm_artifact_mtime == stat.st_mtime: return self._lstm_artifact try: data = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): data = {} self._lstm_artifact = data if isinstance(data, dict) else {} self._lstm_artifact_mtime = stat.st_mtime return self._lstm_artifact def _empty_forecast(enabled: bool, reason: str) -> TimeSeriesForecast: return TimeSeriesForecast( enabled=enabled, usable=False, model="none", volatility_model="none", expected_return_percent=0.0, expected_price=0.0, volatility_percent=0.0, probability_up=0.5, confidence_adjustment=0.0, block_entry=False, validation_mae_percent=0.0, baseline_mae_percent=0.0, skill=0.0, horizon=0, reason=reason, ) def _log_returns(closes: list[float]) -> list[float]: return [math.log(closes[index] / closes[index - 1]) for index in range(1, len(closes))] def _validate_candidates( returns: list[float], validation_window: int, settings: Settings, symbol: str | None = None, lstm_artifact: dict[str, Any] | None = None, ) -> list[dict[str, float | str]]: models = ["naive", "drift", "ewma", "ar1", "ar3"] torch_model = _torch_recurrent_model_name(symbol, lstm_artifact or {}) if torch_model and _can_use_torch_recurrent(returns, symbol, lstm_artifact or {}): models.append(torch_model) if _can_use_lstm(returns, settings, symbol, lstm_artifact or {}): models.append("lstm") rows: list[dict[str, float | str]] = [] start = max(8, len(returns) - validation_window) for model in models: errors: list[float] = [] for index in range(start, len(returns)): history = returns[:index] if len(history) < 8: continue predicted = _predict_next_return(model, history, settings, symbol, lstm_artifact) errors.append(abs(predicted - returns[index])) mae = sum(errors) / len(errors) if errors else 1e9 rows.append({"model": model, "mae": mae}) return rows def _predict_next_return( model: str, returns: list[float], settings: Settings | None = None, symbol: str | None = None, lstm_artifact: dict[str, Any] | None = None, ) -> float: if model == "naive": return 0.0 if model == "drift": window = returns[-24:] if len(returns) >= 24 else returns return sum(window) / len(window) if window else 0.0 if model == "ewma": return _ewma_mean(returns, 0.82) if model == "ar1": return _ar_predict(returns, 1) if model == "ar3": return _ar_predict(returns, 3) if model in {"torch_lstm", "torch_gru"}: return _torch_recurrent_predict(returns, symbol, lstm_artifact or {}) if model == "lstm" and settings is not None: return _lstm_predict(returns, settings, symbol, lstm_artifact or {}) return 0.0 def _ewma_mean(values: list[float], decay: float) -> float: if not values: return 0.0 estimate = values[0] alpha = 1 - _clamp(decay, 0.01, 0.99) for value in values[1:]: estimate = alpha * value + (1 - alpha) * estimate return estimate def _ar_predict(returns: list[float], lag_count: int) -> float: if len(returns) <= lag_count + 6: return _predict_next_return("drift", returns) rows: list[list[float]] = [] targets: list[float] = [] for index in range(lag_count, len(returns)): rows.append([1.0] + [returns[index - lag] for lag in range(1, lag_count + 1)]) targets.append(returns[index]) coeffs = _ols(rows, targets) if not coeffs: return _predict_next_return("drift", returns) features = [1.0] + [returns[-lag] for lag in range(1, lag_count + 1)] prediction = sum(coeff * feature for coeff, feature in zip(coeffs, features)) recent_abs = sorted(abs(value) for value in returns[-48:]) if len(returns) >= 8 else [0.01] cap = max(recent_abs[int(len(recent_abs) * 0.9)], 0.0002) return _clamp(prediction, -cap, cap) def _can_use_lstm( returns: list[float], settings: Settings, symbol: str | None, lstm_artifact: dict[str, Any], ) -> bool: if not settings.time_series_lstm_enabled: return False params = _lstm_params(settings, symbol, lstm_artifact) return len(returns) >= params["lookback"] + 16 def _lstm_params(settings: Settings, symbol: str | None, lstm_artifact: dict[str, Any]) -> dict[str, float | int]: params: dict[str, float | int] = { "lookback": settings.time_series_lstm_lookback, "units": settings.time_series_lstm_units, "ridge": settings.time_series_lstm_ridge, } default_params = lstm_artifact.get("default") if isinstance(default_params, dict): params.update(_clean_lstm_params(default_params)) symbols = lstm_artifact.get("symbols") symbol_params = symbols.get(symbol.upper()) if symbol and isinstance(symbols, dict) else None if isinstance(symbol_params, dict): params.update(_clean_lstm_params(symbol_params)) return { "lookback": int(_clamp(float(params["lookback"]), 6.0, 128.0)), "units": int(_clamp(float(params["units"]), 2.0, 16.0)), "ridge": _clamp(float(params["ridge"]), 1e-8, 0.5), } def _clean_lstm_params(data: dict[str, Any]) -> dict[str, float | int]: clean: dict[str, float | int] = {} for key in ("lookback", "units", "ridge"): value = data.get(key) if isinstance(value, (int, float)): clean[key] = value elif isinstance(value, str): try: clean[key] = float(value) except ValueError: continue return clean def _torch_recurrent_model_name(symbol: str | None, lstm_artifact: dict[str, Any]) -> str | None: entry = _torch_recurrent_entry(symbol, lstm_artifact) if not entry: return None architecture = str(entry.get("architecture", "")).strip().lower() if architecture in {"lstm", "gru"}: return f"torch_{architecture}" model = str(entry.get("model", "")).strip().lower() return model if model in {"torch_lstm", "torch_gru"} else None def _torch_recurrent_entry(symbol: str | None, lstm_artifact: dict[str, Any]) -> dict[str, Any] | None: symbols = lstm_artifact.get("symbols") entry = symbols.get(symbol.upper()) if symbol and isinstance(symbols, dict) else None if not isinstance(entry, dict): default = lstm_artifact.get("default") entry = default if isinstance(default, dict) else None if not isinstance(entry, dict): return None if not isinstance(entry.get("state_dict"), dict): return None return entry def _can_use_torch_recurrent(returns: list[float], symbol: str | None, lstm_artifact: dict[str, Any]) -> bool: entry = _torch_recurrent_entry(symbol, lstm_artifact) if not entry: return False lookback = int(_clamp(_float_entry(entry, "lookback", 0.0), 4.0, 512.0)) hidden_size = int(_clamp(_float_entry(entry, "hidden_size", 0.0), 1.0, 512.0)) num_layers = int(_clamp(_float_entry(entry, "num_layers", 1.0), 1.0, 8.0)) return len(returns) >= lookback + 1 and hidden_size > 0 and num_layers > 0 def _torch_recurrent_predict( returns: list[float], symbol: str | None, lstm_artifact: dict[str, Any], ) -> float: entry = _torch_recurrent_entry(symbol, lstm_artifact) model_name = _torch_recurrent_model_name(symbol, lstm_artifact) if not entry or not model_name: return _predict_next_return("drift", returns) lookback = int(_clamp(_float_entry(entry, "lookback", 0.0), 4.0, 512.0)) hidden_size = int(_clamp(_float_entry(entry, "hidden_size", 0.0), 1.0, 512.0)) num_layers = int(_clamp(_float_entry(entry, "num_layers", 1.0), 1.0, 8.0)) mean = _float_entry(entry, "mean", 0.0) scale = max(_float_entry(entry, "scale", _return_scale(returns)), 1e-8) clip = _clamp(_float_entry(entry, "clip", 8.0), 1.0, 50.0) if len(returns) < lookback: return _predict_next_return("drift", returns) normalized = [_clamp((value - mean) / scale, -clip, clip) for value in returns[-lookback:]] try: hidden = _torch_recurrent_hidden( normalized, entry=entry, model_name=model_name, hidden_size=hidden_size, num_layers=num_layers, ) if hidden is None: return _predict_next_return("drift", returns) head_weight = _float_vector(entry.get("head_weight")) head_bias = _float_entry(entry, "head_bias", 0.0) if len(head_weight) != hidden_size: return _predict_next_return("drift", returns) normalized_prediction = sum(weight * value for weight, value in zip(head_weight, hidden)) + head_bias if not math.isfinite(normalized_prediction): return _predict_next_return("drift", returns) prediction = _clamp(normalized_prediction, -clip, clip) * scale + mean except (IndexError, KeyError, TypeError, ValueError, OverflowError): return _predict_next_return("drift", returns) recent_abs = sorted(abs(value) for value in returns[-48:]) if len(returns) >= 8 else [0.01] cap = max(recent_abs[int(len(recent_abs) * 0.9)], 0.0002) return _clamp(prediction, -cap, cap) def _torch_recurrent_hidden( normalized: list[float], *, entry: dict[str, Any], model_name: str, hidden_size: int, num_layers: int, ) -> list[float] | None: state = entry.get("state_dict") if not isinstance(state, dict): return None h_layers = [[0.0 for _ in range(hidden_size)] for _ in range(num_layers)] c_layers = [[0.0 for _ in range(hidden_size)] for _ in range(num_layers)] for value in normalized: layer_input = [value] for layer in range(num_layers): if model_name == "torch_lstm": next_hidden, next_cell = _torch_lstm_step(layer_input, h_layers[layer], c_layers[layer], state, layer) h_layers[layer] = next_hidden c_layers[layer] = next_cell elif model_name == "torch_gru": h_layers[layer] = _torch_gru_step(layer_input, h_layers[layer], state, layer) else: return None layer_input = h_layers[layer] return h_layers[-1] def _torch_lstm_step( inputs: list[float], hidden: list[float], cell: list[float], state: dict[str, Any], layer: int, ) -> tuple[list[float], list[float]]: hidden_size = len(hidden) gates = _torch_gate_values(inputs, hidden, state, layer, gate_count=4) input_gate = [_sigmoid(value) for value in gates[0]] forget_gate = [_sigmoid(value) for value in gates[1]] cell_gate = [math.tanh(value) for value in gates[2]] output_gate = [_sigmoid(value) for value in gates[3]] next_cell = [ forget_gate[index] * cell[index] + input_gate[index] * cell_gate[index] for index in range(hidden_size) ] next_hidden = [ output_gate[index] * math.tanh(next_cell[index]) for index in range(hidden_size) ] return next_hidden, next_cell def _torch_gru_step( inputs: list[float], hidden: list[float], state: dict[str, Any], layer: int, ) -> list[float]: hidden_size = len(hidden) weight_ih = _float_matrix(state[f"weight_ih_l{layer}"]) weight_hh = _float_matrix(state[f"weight_hh_l{layer}"]) bias_ih = _float_vector(state[f"bias_ih_l{layer}"]) bias_hh = _float_vector(state[f"bias_hh_l{layer}"]) def gate_input(gate: int) -> list[float]: start = gate * hidden_size output = [] for index in range(hidden_size): row = start + index output.append(_dot(weight_ih[row], inputs) + bias_ih[row]) return output def gate_hidden(gate: int) -> list[float]: start = gate * hidden_size output = [] for index in range(hidden_size): row = start + index output.append(_dot(weight_hh[row], hidden) + bias_hh[row]) return output reset_input = gate_input(0) update_input = gate_input(1) new_input = gate_input(2) reset_hidden = gate_hidden(0) update_hidden = gate_hidden(1) new_hidden = gate_hidden(2) reset_gate = [_sigmoid(reset_input[index] + reset_hidden[index]) for index in range(hidden_size)] update_gate = [_sigmoid(update_input[index] + update_hidden[index]) for index in range(hidden_size)] candidate = [ math.tanh(new_input[index] + reset_gate[index] * new_hidden[index]) for index in range(hidden_size) ] return [ (1 - update_gate[index]) * candidate[index] + update_gate[index] * hidden[index] for index in range(hidden_size) ] def _torch_gate_values( inputs: list[float], hidden: list[float], state: dict[str, Any], layer: int, gate_count: int, ) -> list[list[float]]: hidden_size = len(hidden) weight_ih = _float_matrix(state[f"weight_ih_l{layer}"]) weight_hh = _float_matrix(state[f"weight_hh_l{layer}"]) bias_ih = _float_vector(state[f"bias_ih_l{layer}"]) bias_hh = _float_vector(state[f"bias_hh_l{layer}"]) gates: list[list[float]] = [] for gate in range(gate_count): values = [] start = gate * hidden_size for index in range(hidden_size): row = start + index values.append(_dot(weight_ih[row], inputs) + _dot(weight_hh[row], hidden) + bias_ih[row] + bias_hh[row]) gates.append(values) return gates def _float_entry(data: dict[str, Any], key: str, default: float) -> float: value = data.get(key) if isinstance(value, (int, float)): return float(value) if isinstance(value, str): try: return float(value) except ValueError: return default return default def _float_vector(data: Any) -> list[float]: if not isinstance(data, list): return [] return [float(value) for value in data] def _float_matrix(data: Any) -> list[list[float]]: if not isinstance(data, list): return [] return [_float_vector(row) for row in data] def _dot(left: list[float], right: list[float]) -> float: return sum(left[index] * right[index] for index in range(min(len(left), len(right)))) def _lstm_predict( returns: list[float], settings: Settings, symbol: str | None, lstm_artifact: dict[str, Any], ) -> float: params = _lstm_params(settings, symbol, lstm_artifact) lookback = int(params["lookback"]) units = int(params["units"]) ridge = float(params["ridge"]) if len(returns) <= lookback + 8: return _predict_next_return("drift", returns) scale = _return_scale(returns) normalized = [_clamp(value / scale, -6.0, 6.0) for value in returns] states = _lstm_states(normalized, units) rows: list[list[float]] = [] targets: list[float] = [] for index in range(lookback, len(returns)): rows.append([1.0] + states[index - 1]) targets.append(normalized[index]) coeffs = _ols(rows, targets, ridge) if not coeffs: return _predict_next_return("drift", returns) features = [1.0] + states[-1] prediction = sum(coeff * feature for coeff, feature in zip(coeffs, features)) prediction = _clamp(prediction, -4.0, 4.0) * scale recent_abs = sorted(abs(value) for value in returns[-48:]) if len(returns) >= 8 else [0.01] cap = max(recent_abs[int(len(recent_abs) * 0.9)], 0.0002) return _clamp(prediction, -cap, cap) def _return_scale(returns: list[float]) -> float: recent = returns[-120:] if len(returns) > 120 else returns values = sorted(abs(value) for value in recent if math.isfinite(value)) if not values: return 0.0005 median = values[len(values) // 2] mean = sum(values) / len(values) return max(max(median, mean * 0.5), 1e-5) def _lstm_states(normalized_returns: list[float], units: int) -> list[list[float]]: weights = _lstm_weights(units) hidden = [0.0 for _ in range(units)] cell = [0.0 for _ in range(units)] states: list[list[float]] = [] for value in normalized_returns: hidden, cell = _lstm_step(value, hidden, cell, weights) states.append(hidden[:]) return states @lru_cache(maxsize=16) def _lstm_weights(units: int) -> tuple[list[list[float]], list[list[list[float]]], list[list[float]]]: input_weights: list[list[float]] = [] recurrent_weights: list[list[list[float]]] = [] biases: list[list[float]] = [] base_biases = (-0.15, 0.70, 0.05, 0.0) for gate in range(4): gate_input: list[float] = [] gate_recurrent: list[list[float]] = [] gate_bias: list[float] = [] for unit in range(units): gate_input.append(0.55 * math.sin((gate + 1) * (unit + 1) * 1.61803398875)) gate_recurrent.append( [ 0.14 * math.sin((gate + 3) * (unit + 1) * (source + 1) * 0.731) for source in range(units) ] ) gate_bias.append(base_biases[gate] + 0.03 * math.sin((gate + 1) * (unit + 1))) input_weights.append(gate_input) recurrent_weights.append(gate_recurrent) biases.append(gate_bias) return input_weights, recurrent_weights, biases def _lstm_step( value: float, hidden: list[float], cell: list[float], weights: tuple[list[list[float]], list[list[list[float]]], list[list[float]]], ) -> tuple[list[float], list[float]]: input_weights, recurrent_weights, biases = weights units = len(hidden) next_hidden = [0.0 for _ in range(units)] next_cell = [0.0 for _ in range(units)] for unit in range(units): gate_values = [] for gate in range(4): raw = input_weights[gate][unit] * value + biases[gate][unit] raw += sum(recurrent_weights[gate][unit][source] * hidden[source] for source in range(units)) gate_values.append(raw) input_gate = _sigmoid(gate_values[0]) forget_gate = _sigmoid(gate_values[1]) output_gate = _sigmoid(gate_values[2]) candidate = math.tanh(gate_values[3]) next_cell[unit] = forget_gate * cell[unit] + input_gate * candidate next_hidden[unit] = output_gate * math.tanh(next_cell[unit]) return next_hidden, next_cell def _sigmoid(value: float) -> float: if value >= 40: return 1.0 if value <= -40: return 0.0 return 1 / (1 + math.exp(-value)) def _ols(rows: list[list[float]], targets: list[float], ridge: float = 1e-8) -> list[float] | None: if not rows: return None columns = len(rows[0]) xtx = [[0.0 for _ in range(columns)] for _ in range(columns)] xty = [0.0 for _ in range(columns)] for row, target in zip(rows, targets): for i in range(columns): xty[i] += row[i] * target for j in range(columns): xtx[i][j] += row[i] * row[j] for i in range(columns): xtx[i][i] += ridge return _solve_linear_system(xtx, xty) def _solve_linear_system(matrix: list[list[float]], vector: list[float]) -> list[float] | None: size = len(vector) augmented = [row[:] + [vector[index]] for index, row in enumerate(matrix)] for col in range(size): pivot = max(range(col, size), key=lambda row: abs(augmented[row][col])) if abs(augmented[pivot][col]) < 1e-12: return None augmented[col], augmented[pivot] = augmented[pivot], augmented[col] pivot_value = augmented[col][col] for item in range(col, size + 1): augmented[col][item] /= pivot_value for row in range(size): if row == col: continue factor = augmented[row][col] for item in range(col, size + 1): augmented[row][item] -= factor * augmented[col][item] return [augmented[row][size] for row in range(size)] def _ewma_volatility(returns: list[float], decay: float) -> float: if not returns: return 0.0 decay = _clamp(decay, 0.80, 0.995) variance = returns[0] * returns[0] for value in returns[1:]: variance = decay * variance + (1 - decay) * value * value return math.sqrt(max(variance, 0.0)) def _fixed_garch_volatility(returns: list[float]) -> float: if not returns: return 0.0 long_variance = sum(value * value for value in returns) / len(returns) alpha = 0.08 beta = 0.90 omega = max(1e-12, (1 - alpha - beta) * long_variance) variance = long_variance for value in returns: variance = omega + alpha * value * value + beta * variance return math.sqrt(max(variance, 0.0)) def _confidence_adjustment( *, expected_return_percent: float, probability_up: float, skill: float, min_edge: float, max_adjustment: float, usable_skill: bool, ) -> float: if not usable_skill: return 0.0 edge = abs(expected_return_percent) - min_edge if edge <= 0: return 0.0 direction = 1.0 if expected_return_percent > 0 and probability_up >= 0.55 else -1.0 if direction < 0 and probability_up > 0.45: return 0.0 strength = _clamp(edge / max(min_edge, 0.05), 0.0, 1.0) probability_strength = _clamp(abs(probability_up - 0.5) / 0.25, 0.0, 1.0) skill_strength = _clamp(skill / 0.18, 0.0, 1.0) return direction * _clamp(max_adjustment, 0.0, 0.18) * strength * probability_strength * skill_strength def _reason( *, model: str, expected_return_percent: float, probability_up: float, skill: float, block_entry: bool, usable_skill: bool, ) -> str: if not usable_skill: return f"модель {model} не лучше baseline на walk-forward проверке" if block_entry: return f"модель {model}: ожидаемое движение вниз {expected_return_percent:.3f}%, P(рост)={probability_up:.2f}" return f"модель {model}: прогноз {expected_return_percent:.3f}%, P(рост)={probability_up:.2f}, skill={skill:.3f}" def _normal_cdf(value: float) -> float: return 0.5 * (1 + math.erf(value / math.sqrt(2))) def _clamp(value: float, low: float, high: float) -> float: return max(low, min(high, value))