sevenhill/TradeBot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix remote training and model validation pipeline

Browse Source
This commit is contained in:
Курнат Андрей
2026-07-12 22:56:49 +03:00
parent 18936cf8b1
commit da53483164
22 changed files with 1691 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crypto_spot_bot/strategy.py
+14 -6
View File
@@ -644,8 +644,12 @@ def _torch_forecast_entry_signal(
expected_return = _safe_float(forecast.get("expected_return_percent"), 0.0)
probability_up = _safe_float(forecast.get("probability_up"), 0.5)
skill = _safe_float(forecast.get("skill"), 0.0)
min_edge = max(0.0, settings.time_series_min_edge_percent)
min_probability = _torch_min_probability(settings)
min_edge = max(0.0, _safe_float(forecast.get("calibrated_min_edge_percent"), settings.time_series_min_edge_percent))
min_probability = _clamp(
_safe_float(forecast.get("calibrated_min_probability_up"), _torch_min_probability(settings)),
0.5,
0.95,
)
probe_min_edge = max(0.0, min(settings.time_series_probe_min_edge_percent, min_edge))
probe_min_probability = round(
_clamp(settings.time_series_probe_min_probability_up, min_probability, 0.85),
@@ -716,7 +720,7 @@ def _torch_forecast_entry_signal(
and expected_return >= 0.0
and probability_up >= rebound_model_probability_min
and skill > 0.0
and confidence >= settings.time_series_min_confidence
and confidence >= _safe_float(forecast.get("calibrated_min_confidence"), settings.time_series_min_confidence)
)
fallback_rebound_entry_ok = bool(
settings.time_series_rebound_fallback_enabled
@@ -725,7 +729,7 @@ def _torch_forecast_entry_signal(
and quality_gate_ok
and model_fresh_ok
and not bool(forecast.get("block_entry", False))
and confidence >= settings.time_series_min_confidence
and confidence >= _safe_float(forecast.get("calibrated_min_confidence"), settings.time_series_min_confidence)
)
rebound_entry_ok = model_rebound_entry_ok or fallback_rebound_entry_ok
if rebound_entry_ok and position_notional > 0:
@@ -896,8 +900,12 @@ def _torch_forecast_exit_signal(
expected_return = _safe_float(forecast.get("expected_return_percent"), 0.0)
probability_up = _safe_float(forecast.get("probability_up"), 0.5)
skill = _safe_float(forecast.get("skill"), 0.0)
min_edge = max(0.0, settings.time_series_min_edge_percent)
min_probability = _torch_min_probability(settings)
min_edge = max(0.0, _safe_float(forecast.get("calibrated_min_edge_percent"), settings.time_series_min_edge_percent))
min_probability = _clamp(
_safe_float(forecast.get("calibrated_min_probability_up"), _torch_min_probability(settings)),
0.5,
0.95,
)
estimated_exit_net_percent = _estimated_exit_net_percent(position, price, settings)
min_exit_net_percent = _min_exit_net_percent(settings)
entry_path = str(position.entry_diagnostics.get("entry_path", ""))
crypto_spot_bot/time_series.py
+102 -5
View File
@@ -160,6 +160,9 @@ class TimeSeriesForecast:
model_created_at: str = ""
model_age_hours: float | None = None
model_fresh: bool = False
calibrated_min_edge_percent: float = 0.0
calibrated_min_probability_up: float = 0.0
calibrated_min_confidence: float = 0.0
def as_dict(self) -> dict[str, Any]:
return asdict(self)
@@ -196,8 +199,20 @@ class TimeSeriesForecaster:
artifact,
self.settings.time_series_model_max_age_hours,
)
quality_gate = self._load_quality_gate()
calibration = self._load_quality_gate()
quality_gate = (
calibration.get("validation")
if isinstance(calibration.get("validation"), dict)
else calibration
)
quality_gate_passed = _quality_gate_passed(quality_gate)
calibrated = _calibrated_thresholds(
calibration,
symbol,
edge=self.settings.time_series_min_edge_percent,
probability=self.settings.time_series_min_probability_up,
confidence=self.settings.time_series_min_confidence,
)
entry = _torch_recurrent_entry(symbol, artifact)
model = _torch_recurrent_model_name(symbol, artifact)
clip = _clamp(_float_entry(entry or {}, "clip", 8.0), 1.0, 50.0)
@@ -249,7 +264,7 @@ class TimeSeriesForecaster:
q90_percent = (math.exp(float(selected.get("q90", expected_return))) - 1) * 100
skill = _clamp(_float_entry(entry, "skill", 0.0), -1.0, 1.0)
horizon = int(selected.get("horizon", _entry_horizon(entry, self.settings.time_series_forecast_horizon)))
min_edge = max(0.0, self.settings.time_series_min_edge_percent)
min_edge = calibrated["edge"]
confidence_adjustment = _confidence_adjustment(
expected_return_percent=expected_return_percent,
probability_up=probability_up,
@@ -299,6 +314,9 @@ class TimeSeriesForecaster:
model_created_at=model_created_at,
model_age_hours=model_age_hours,
model_fresh=model_fresh,
calibrated_min_edge_percent=calibrated["edge"],
calibrated_min_probability_up=calibrated["probability"],
calibrated_min_confidence=calibrated["confidence"],
)
direct_horizon = _is_direct_horizon(entry)
@@ -318,7 +336,7 @@ class TimeSeriesForecaster:
expected_return_percent = (math.exp(expected_return) - 1) * 100
probability_up = _normal_cdf(expected_return / max(uncertainty, 1e-9))
skill = _clamp(_float_entry(entry, "skill", 0.0), -1.0, 1.0)
min_edge = max(0.0, self.settings.time_series_min_edge_percent)
min_edge = calibrated["edge"]
confidence_adjustment = _confidence_adjustment(
expected_return_percent=expected_return_percent,
probability_up=probability_up,
@@ -364,6 +382,9 @@ class TimeSeriesForecaster:
model_created_at=model_created_at,
model_age_hours=model_age_hours,
model_fresh=model_fresh,
calibrated_min_edge_percent=calibrated["edge"],
calibrated_min_probability_up=calibrated["probability"],
calibrated_min_confidence=calibrated["confidence"],
)
def _load_lstm_artifact(self) -> dict[str, Any]:
@@ -400,8 +421,7 @@ class TimeSeriesForecaster:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
data = {}
validation = data.get("validation") if isinstance(data, dict) else {}
self._quality_gate = validation if isinstance(validation, dict) else {}
self._quality_gate = data if isinstance(data, dict) else {}
self._calibration_mtime = stat.st_mtime
return self._quality_gate
@@ -443,6 +463,9 @@ def _empty_forecast(enabled: bool, reason: str) -> TimeSeriesForecast:
def _quality_gate_passed(quality_gate: dict[str, Any]) -> bool | None:
if not quality_gate:
return None
validation = quality_gate.get("validation")
if isinstance(validation, dict):
return _quality_gate_passed(validation)
if "passed" in quality_gate:
return bool(quality_gate.get("passed"))
status = str(quality_gate.get("status", "")).strip().lower()
@@ -453,6 +476,26 @@ def _quality_gate_passed(quality_gate: dict[str, Any]) -> bool | None:
return None
def _calibrated_thresholds(
calibration: dict[str, Any],
symbol: str | None,
*,
edge: float,
probability: float,
confidence: float,
) -> dict[str, float]:
recommended = calibration.get("recommended") if isinstance(calibration, dict) else None
per_symbol = calibration.get("symbol_recommendations") if isinstance(calibration, dict) else None
if symbol and isinstance(per_symbol, dict) and isinstance(per_symbol.get(symbol.upper()), dict):
recommended = per_symbol[symbol.upper()]
row = recommended if isinstance(recommended, dict) else {}
return {
"edge": max(0.0, float(row.get("edge", edge) or edge)),
"probability": _clamp(float(row.get("probability", probability) or probability), 0.5, 0.95),
"confidence": _clamp(float(row.get("confidence", confidence) or confidence), 0.0, 1.0),
}
def _model_freshness(artifact: dict[str, Any], max_age_hours: float) -> tuple[str, float | None, bool]:
raw = str(artifact.get("created_at", "")).strip() if isinstance(artifact, dict) else ""
if not raw:
@@ -525,6 +568,8 @@ def _feature_context(
def _feature_value(name: str, candles: list[Candle], index: int, candle: Candle, context: dict[str, Any]) -> float:
close = max(float(candle.close), 1e-12)
previous = candles[index - 1] if index >= 1 else candle
if name.startswith("symbol_is_"):
return 1.0 if context.get("symbol") == name.removeprefix("symbol_is_").upper() else 0.0
if name == "return_1":
return _log_change(candle.close, previous.close)
if name == "return_3":
@@ -990,6 +1035,31 @@ def _torch_recurrent_predict(
model_name = _torch_recurrent_model_name(symbol, artifact)
if not entry or not model_name:
return None
ensemble_members = entry.get("ensemble_members")
if isinstance(ensemble_members, list) and ensemble_members:
predictions: list[float | dict[str, Any]] = []
for member in ensemble_members:
if not isinstance(member, dict):
continue
member_entry = {**entry, **member}
member_entry.pop("ensemble_members", None)
member_entry.pop("ensemble_size", None)
member_artifact: dict[str, Any] = {"type": "pytorch_recurrent_forecaster"}
if symbol:
member_artifact["symbols"] = {symbol.upper(): member_entry}
else:
member_artifact["default"] = member_entry
prediction = _torch_recurrent_predict(
returns,
symbol,
member_artifact,
feature_rows=feature_rows,
closes=closes,
candles=candles,
)
if isinstance(prediction, (int, float, dict)):
predictions.append(prediction)
return _average_ensemble_predictions(predictions)
lookback = int(_clamp(_float_entry(entry, "lookback", 0.0), 4.0, 512.0))
hidden_size = int(_clamp(_float_entry(entry, "hidden_size", 0.0), 1.0, 512.0))
num_layers = int(_clamp(_float_entry(entry, "num_layers", 1.0), 1.0, 8.0))
@@ -1050,6 +1120,33 @@ def _torch_recurrent_predict(
return _clamp(prediction, -cap, cap)
def _average_ensemble_predictions(predictions: list[float | dict[str, Any]]) -> float | dict[str, Any] | None:
if not predictions:
return None
numeric = [float(value) for value in predictions if isinstance(value, (int, float))]
if numeric:
return sum(numeric) / len(numeric)
mappings = [value for value in predictions if isinstance(value, dict)]
if not mappings:
return None
first = mappings[0]
output: dict[str, Any] = {}
for key, value in first.items():
if key == "horizons" and isinstance(value, dict):
horizons: dict[str, Any] = {}
for horizon, row in value.items():
rows = [item.get("horizons", {}).get(horizon) for item in mappings]
rows = [item for item in rows if isinstance(item, dict)]
if rows:
horizons[horizon] = _average_ensemble_predictions(rows)
output[key] = horizons
continue
values = [item.get(key) for item in mappings]
finite = [float(item) for item in values if isinstance(item, (int, float)) and math.isfinite(float(item))]
output[key] = sum(finite) / len(finite) if finite else value
return output
def _torch_head_outputs(context: list[float], entry: dict[str, Any], hidden_size: int) -> list[float]:
context = _apply_context_norm(context, entry)
raw_weight = entry.get("head_weight")
crypto_spot_bot/training_coordination.py
+8 -3
View File
@@ -215,6 +215,10 @@ class TrainingCoordinator:
job["message"] = str(payload.get("message") or "")
if isinstance(payload.get("summary"), dict):
job["summary"] = payload["summary"]
if isinstance(payload["summary"].get("accepted"), bool):
job["model_decision"] = (
"accepted" if payload["summary"]["accepted"] else "rejected"
)
self._save_state(state)
return {"ok": True, "job": job, "status": self._public_status(state)}
@@ -294,10 +298,11 @@ class TrainingCoordinator:
os.replace(tmp, self.state_path)
def _worker_from_payload(self, payload: dict[str, Any]) -> dict[str, Any]:
worker_id = str(payload.get("worker_id") or payload.get("id") or "windows-training-host").strip()
return {
"id": str(payload.get("worker_id") or payload.get("id") or "windows-training-host"),
"name": str(payload.get("name") or "DESKTOP-TMFDL0H"),
"path": str(payload.get("path") or "C:\\Repos\\TradeBot"),
"id": worker_id,
"name": str(payload.get("name") or worker_id).strip(),
"path": str(payload.get("path") or "").strip(),
"version": str(payload.get("version") or "1"),
"last_seen_at": _now(),
}