Fix remote training and model validation pipeline

2026-07-12 22:56:49 +03:00
parent 18936cf8b1
commit da53483164
22 changed files with 1691 additions and 246 deletions
@@ -160,6 +160,9 @@ class TimeSeriesForecast:
    model_created_at: str = ""
    model_age_hours: float | None = None
    model_fresh: bool = False
+    calibrated_min_edge_percent: float = 0.0
+    calibrated_min_probability_up: float = 0.0
+    calibrated_min_confidence: float = 0.0

    def as_dict(self) -> dict[str, Any]:
        return asdict(self)
@@ -196,8 +199,20 @@ class TimeSeriesForecaster:
            artifact,
            self.settings.time_series_model_max_age_hours,
        )
-        quality_gate = self._load_quality_gate()
+        calibration = self._load_quality_gate()
+        quality_gate = (
+            calibration.get("validation")
+            if isinstance(calibration.get("validation"), dict)
+            else calibration
+        )
        quality_gate_passed = _quality_gate_passed(quality_gate)
+        calibrated = _calibrated_thresholds(
+            calibration,
+            symbol,
+            edge=self.settings.time_series_min_edge_percent,
+            probability=self.settings.time_series_min_probability_up,
+            confidence=self.settings.time_series_min_confidence,
+        )
        entry = _torch_recurrent_entry(symbol, artifact)
        model = _torch_recurrent_model_name(symbol, artifact)
        clip = _clamp(_float_entry(entry or {}, "clip", 8.0), 1.0, 50.0)
@@ -249,7 +264,7 @@ class TimeSeriesForecaster:
            q90_percent = (math.exp(float(selected.get("q90", expected_return))) - 1) * 100
            skill = _clamp(_float_entry(entry, "skill", 0.0), -1.0, 1.0)
            horizon = int(selected.get("horizon", _entry_horizon(entry, self.settings.time_series_forecast_horizon)))
-            min_edge = max(0.0, self.settings.time_series_min_edge_percent)
+            min_edge = calibrated["edge"]
            confidence_adjustment = _confidence_adjustment(
                expected_return_percent=expected_return_percent,
                probability_up=probability_up,
@@ -299,6 +314,9 @@ class TimeSeriesForecaster:
                model_created_at=model_created_at,
                model_age_hours=model_age_hours,
                model_fresh=model_fresh,
+                calibrated_min_edge_percent=calibrated["edge"],
+                calibrated_min_probability_up=calibrated["probability"],
+                calibrated_min_confidence=calibrated["confidence"],
            )

        direct_horizon = _is_direct_horizon(entry)
@@ -318,7 +336,7 @@ class TimeSeriesForecaster:
        expected_return_percent = (math.exp(expected_return) - 1) * 100
        probability_up = _normal_cdf(expected_return / max(uncertainty, 1e-9))
        skill = _clamp(_float_entry(entry, "skill", 0.0), -1.0, 1.0)
-        min_edge = max(0.0, self.settings.time_series_min_edge_percent)
+        min_edge = calibrated["edge"]
        confidence_adjustment = _confidence_adjustment(
            expected_return_percent=expected_return_percent,
            probability_up=probability_up,
@@ -364,6 +382,9 @@ class TimeSeriesForecaster:
            model_created_at=model_created_at,
            model_age_hours=model_age_hours,
            model_fresh=model_fresh,
+            calibrated_min_edge_percent=calibrated["edge"],
+            calibrated_min_probability_up=calibrated["probability"],
+            calibrated_min_confidence=calibrated["confidence"],
        )

    def _load_lstm_artifact(self) -> dict[str, Any]:
@@ -400,8 +421,7 @@ class TimeSeriesForecaster:
            data = json.loads(path.read_text(encoding="utf-8"))
        except (OSError, json.JSONDecodeError):
            data = {}
-        validation = data.get("validation") if isinstance(data, dict) else {}
-        self._quality_gate = validation if isinstance(validation, dict) else {}
+        self._quality_gate = data if isinstance(data, dict) else {}
        self._calibration_mtime = stat.st_mtime
        return self._quality_gate

@@ -443,6 +463,9 @@ def _empty_forecast(enabled: bool, reason: str) -> TimeSeriesForecast:
 def _quality_gate_passed(quality_gate: dict[str, Any]) -> bool | None:
    if not quality_gate:
        return None
+    validation = quality_gate.get("validation")
+    if isinstance(validation, dict):
+        return _quality_gate_passed(validation)
    if "passed" in quality_gate:
        return bool(quality_gate.get("passed"))
    status = str(quality_gate.get("status", "")).strip().lower()
@@ -453,6 +476,26 @@ def _quality_gate_passed(quality_gate: dict[str, Any]) -> bool | None:
    return None


+def _calibrated_thresholds(
+    calibration: dict[str, Any],
+    symbol: str | None,
+    *,
+    edge: float,
+    probability: float,
+    confidence: float,
+) -> dict[str, float]:
+    recommended = calibration.get("recommended") if isinstance(calibration, dict) else None
+    per_symbol = calibration.get("symbol_recommendations") if isinstance(calibration, dict) else None
+    if symbol and isinstance(per_symbol, dict) and isinstance(per_symbol.get(symbol.upper()), dict):
+        recommended = per_symbol[symbol.upper()]
+    row = recommended if isinstance(recommended, dict) else {}
+    return {
+        "edge": max(0.0, float(row.get("edge", edge) or edge)),
+        "probability": _clamp(float(row.get("probability", probability) or probability), 0.5, 0.95),
+        "confidence": _clamp(float(row.get("confidence", confidence) or confidence), 0.0, 1.0),
+    }
+
+
 def _model_freshness(artifact: dict[str, Any], max_age_hours: float) -> tuple[str, float | None, bool]:
    raw = str(artifact.get("created_at", "")).strip() if isinstance(artifact, dict) else ""
    if not raw:
@@ -525,6 +568,8 @@ def _feature_context(
 def _feature_value(name: str, candles: list[Candle], index: int, candle: Candle, context: dict[str, Any]) -> float:
    close = max(float(candle.close), 1e-12)
    previous = candles[index - 1] if index >= 1 else candle
+    if name.startswith("symbol_is_"):
+        return 1.0 if context.get("symbol") == name.removeprefix("symbol_is_").upper() else 0.0
    if name == "return_1":
        return _log_change(candle.close, previous.close)
    if name == "return_3":
@@ -990,6 +1035,31 @@ def _torch_recurrent_predict(
    model_name = _torch_recurrent_model_name(symbol, artifact)
    if not entry or not model_name:
        return None
+    ensemble_members = entry.get("ensemble_members")
+    if isinstance(ensemble_members, list) and ensemble_members:
+        predictions: list[float | dict[str, Any]] = []
+        for member in ensemble_members:
+            if not isinstance(member, dict):
+                continue
+            member_entry = {**entry, **member}
+            member_entry.pop("ensemble_members", None)
+            member_entry.pop("ensemble_size", None)
+            member_artifact: dict[str, Any] = {"type": "pytorch_recurrent_forecaster"}
+            if symbol:
+                member_artifact["symbols"] = {symbol.upper(): member_entry}
+            else:
+                member_artifact["default"] = member_entry
+            prediction = _torch_recurrent_predict(
+                returns,
+                symbol,
+                member_artifact,
+                feature_rows=feature_rows,
+                closes=closes,
+                candles=candles,
+            )
+            if isinstance(prediction, (int, float, dict)):
+                predictions.append(prediction)
+        return _average_ensemble_predictions(predictions)
    lookback = int(_clamp(_float_entry(entry, "lookback", 0.0), 4.0, 512.0))
    hidden_size = int(_clamp(_float_entry(entry, "hidden_size", 0.0), 1.0, 512.0))
    num_layers = int(_clamp(_float_entry(entry, "num_layers", 1.0), 1.0, 8.0))
@@ -1050,6 +1120,33 @@ def _torch_recurrent_predict(
    return _clamp(prediction, -cap, cap)


+def _average_ensemble_predictions(predictions: list[float | dict[str, Any]]) -> float | dict[str, Any] | None:
+    if not predictions:
+        return None
+    numeric = [float(value) for value in predictions if isinstance(value, (int, float))]
+    if numeric:
+        return sum(numeric) / len(numeric)
+    mappings = [value for value in predictions if isinstance(value, dict)]
+    if not mappings:
+        return None
+    first = mappings[0]
+    output: dict[str, Any] = {}
+    for key, value in first.items():
+        if key == "horizons" and isinstance(value, dict):
+            horizons: dict[str, Any] = {}
+            for horizon, row in value.items():
+                rows = [item.get("horizons", {}).get(horizon) for item in mappings]
+                rows = [item for item in rows if isinstance(item, dict)]
+                if rows:
+                    horizons[horizon] = _average_ensemble_predictions(rows)
+            output[key] = horizons
+            continue
+        values = [item.get(key) for item in mappings]
+        finite = [float(item) for item in values if isinstance(item, (int, float)) and math.isfinite(float(item))]
+        output[key] = sum(finite) / len(finite) if finite else value
+    return output
+
+
 def _torch_head_outputs(context: list[float], entry: dict[str, Any], hidden_size: int) -> list[float]:
    context = _apply_context_norm(context, entry)
    raw_weight = entry.get("head_weight")