sevenhill/TradeBot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix remote training and model validation pipeline

Browse Source
This commit is contained in:
Курнат Андрей
2026-07-12 22:56:49 +03:00
parent 18936cf8b1
commit da53483164
22 changed files with 1691 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/calibrate_torch_thresholds.py
+256 -16
View File
@@ -6,7 +6,7 @@ import json
import math
import sys
import time
from dataclasses import dataclass
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Any
@@ -130,13 +130,15 @@ def main() -> None:
if not records:
raise SystemExit("No forecast records could be built for calibration.")
results = _calibrate(
results = _calibrate_strategy(
records,
edges=_float_grid(args.edge_grid),
probabilities=_float_grid(args.probability_grid),
confidences=_float_grid(args.confidence_grid),
min_trades=args.min_trades,
horizon=horizon,
round_trip_cost=round_trip_cost,
settings=settings,
)
if not results:
raise SystemExit("No calibration result produced trades. Use wider grids or more history.")
@@ -156,6 +158,33 @@ def main() -> None:
round_trip_cost=round_trip_cost,
settings=settings,
)
symbol_recommendations: dict[str, dict[str, Any]] = {}
for symbol in symbols:
symbol_records = [record for record in records if record.symbol == symbol]
symbol_results = _calibrate_strategy(
symbol_records,
edges=_float_grid(args.edge_grid),
probabilities=_float_grid(args.probability_grid),
confidences=_float_grid(args.confidence_grid),
min_trades=max(3, min(args.min_trades, len(symbol_records) // 8)),
horizon=horizon,
round_trip_cost=round_trip_cost,
settings=settings,
)
symbol_selected = _choose_recommendation(
symbol_results,
min_trades=max(3, min(args.min_trades, len(symbol_records) // 8)),
) if symbol_results else None
if symbol_selected is not None:
symbol_recommendations[symbol] = _result_dict(symbol_selected)
calibration_insufficient = recommended is None
if recommended is None:
recommended = _empty_recommendation(
_float_grid(args.edge_grid),
_float_grid(args.probability_grid),
_float_grid(args.confidence_grid),
)
full_backtest = {**_stats([]), "trades_detail": [], "symbol_breakdown": []}
print("\nRECOMMENDED")
print(_result_line(recommended))
print("\nFULL_REPLAY")
@@ -188,6 +217,16 @@ def main() -> None:
min_profit_factor=args.min_oos_profit_factor,
min_benchmark_edge=args.min_benchmark_edge_percent,
)
deployment_recommended = recommended
deployment_symbol_recommendations = symbol_recommendations
if walk_forward.get("folds"):
last_fold = walk_forward["folds"][-1]
fold_thresholds = last_fold.get("thresholds")
if isinstance(fold_thresholds, dict):
deployment_recommended = _result_from_dict(fold_thresholds)
fold_symbols = last_fold.get("symbol_thresholds")
if isinstance(fold_symbols, dict):
deployment_symbol_recommendations = fold_symbols
print("\nWALK_FORWARD")
print(json.dumps(walk_forward["summary"], ensure_ascii=False, sort_keys=True))
print("\nBENCHMARK")
@@ -206,7 +245,9 @@ def main() -> None:
"artifact_sha256": artifact_sha256,
"artifact": _artifact_summary(artifact),
"records_by_symbol": per_symbol_counts,
"recommended": _result_dict(recommended),
"recommended": _result_dict(deployment_recommended),
"calibration_insufficient": calibration_insufficient,
"symbol_recommendations": deployment_symbol_recommendations,
"full_replay": full_backtest,
"walk_forward": walk_forward,
"benchmark": benchmark,
@@ -306,7 +347,9 @@ def _forecast_records(
return batched_records
records: list[ForecastRecord] = []
skill = float(entry.get("skill", 0.0) or 0.0)
# Entry eligibility may use validation-derived quality only. Holdout metrics
# belong exclusively to the final quality gate and cannot influence replay.
skill = _entry_validation_skill(entry)
for index in range(start, max(start, end)):
prediction = _torch_recurrent_predict(
_log_returns(closes[: index + 1]),
@@ -394,7 +437,7 @@ def _batch_forecast_records(
return []
records: list[ForecastRecord] = []
skill = float(entry.get("skill", 0.0) or 0.0)
skill = _entry_validation_skill(entry)
model.eval()
with torch.no_grad():
for offset in range(0, len(indices), max(1, batch_size)):
@@ -460,6 +503,10 @@ def _batch_forecast_records(
def _build_torch_model(entry: dict[str, Any], model_name: str) -> Any | None:
if isinstance(entry.get("ensemble_members"), list) and entry["ensemble_members"]:
# Ensemble inference is handled by the shared pure-Python runtime so
# calibration and production use the exact same averaging path.
return None
if torch is None or RecurrentReturnModel is None:
return None
architecture = "lstm" if model_name == "torch_lstm" else "gru" if model_name == "torch_gru" else ""
@@ -590,6 +637,7 @@ def _full_backtest(
round_trip_cost: float,
settings: Any,
detail_limit: int = 50,
symbol_thresholds: dict[str, CalibrationResult] | None = None,
) -> dict[str, Any]:
positions: dict[str, dict[str, Any]] = {}
trades: list[float] = []
@@ -600,6 +648,7 @@ def _full_backtest(
stop_loss_exit_enabled = bool(getattr(settings, "stop_loss_exit_enabled", True))
atr_multiplier = max(0.5, min(10.0, float(settings.atr_trailing_multiplier)))
for record in sorted(records, key=lambda item: (item.timestamp, item.symbol)):
active_thresholds = (symbol_thresholds or {}).get(record.symbol, thresholds)
position = positions.get(record.symbol)
if position is not None:
position["highest"] = max(position["highest"], record.high)
@@ -616,8 +665,8 @@ def _full_backtest(
and (stop_loss_exit_enabled or atr_stop_level > position["entry_price"])
)
weak_forecast = (
record.expected_percent < thresholds.edge
or record.probability_up < thresholds.probability
record.expected_percent < active_thresholds.edge
or record.probability_up < active_thresholds.probability
or record.skill <= 0.0
)
exit_reason = ""
@@ -633,7 +682,7 @@ def _full_backtest(
elif atr_stop:
exit_reason = "atr_trailing_stop"
exit_price = float(atr_stop_level)
elif (record.expected_percent <= 0.0 or record.probability_up <= 0.50 or _candidate_blocks(record, thresholds.edge)):
elif (record.expected_percent <= 0.0 or record.probability_up <= 0.50 or _candidate_blocks(record, active_thresholds.edge)):
exit_reason = "forecast_negative"
elif weak_forecast and net_percent >= 0:
exit_reason = "forecast_weak_profit_lock"
@@ -659,7 +708,7 @@ def _full_backtest(
if record.symbol in positions:
continue
if _candidate_allows(record, thresholds.edge, thresholds.probability, thresholds.confidence):
if _candidate_allows(record, active_thresholds.edge, active_thresholds.probability, active_thresholds.confidence):
positions[record.symbol] = {
"entry_price": record.next_open,
"entry_index": record.index + 1,
@@ -815,24 +864,49 @@ def _walk_forward(
test_end = timestamps[(fold + 1) * fold_size - 1] if fold < folds - 1 else timestamps[-1]
train = [record for record in ordered if record.timestamp < test_start]
test = [record for record in ordered if test_start <= record.timestamp <= test_end]
train_results = _calibrate(
train,
probability_calibration = _fit_platt_calibration(train)
calibrated_train = _apply_platt_calibration(train, probability_calibration)
calibrated_test = _apply_platt_calibration(test, probability_calibration)
train_results = _calibrate_strategy(
calibrated_train,
edges=edges,
probabilities=probabilities,
confidences=confidences,
min_trades=max(4, min_trades // 2),
horizon=horizon,
round_trip_cost=round_trip_cost,
settings=settings,
)
if not train_results:
continue
selected = _choose_recommendation(train_results, min_trades=max(4, min_trades // 2))
if selected is None:
continue
symbol_thresholds: dict[str, CalibrationResult] = {}
train_symbols = sorted({record.symbol for record in calibrated_train})
symbol_min_trades = max(3, min_trades // max(2, len(train_symbols) * 2))
for symbol in train_symbols:
symbol_results = _calibrate_strategy(
[record for record in calibrated_train if record.symbol == symbol],
edges=edges,
probabilities=probabilities,
confidences=confidences,
min_trades=symbol_min_trades,
horizon=horizon,
round_trip_cost=round_trip_cost,
settings=settings,
)
symbol_selected = _choose_recommendation(symbol_results, min_trades=symbol_min_trades) if symbol_results else None
if symbol_selected is not None:
symbol_thresholds[symbol] = symbol_selected
test_backtest = _full_backtest(
test,
calibrated_test,
selected,
horizon=horizon,
round_trip_cost=round_trip_cost,
settings=settings,
detail_limit=0,
symbol_thresholds=symbol_thresholds,
)
test_rows = test_backtest.get("trades_detail", [])
test_trades = [float(row.get("net_percent", 0.0) or 0.0) for row in test_rows if isinstance(row, dict)]
@@ -844,6 +918,10 @@ def _walk_forward(
"train_records": len(train),
"test_records": len(test),
"thresholds": _result_dict(selected),
"symbol_thresholds": {
symbol: _result_dict(value) for symbol, value in symbol_thresholds.items()
},
"probability_calibration": probability_calibration,
"test": {key: value for key, value in test_backtest.items() if key != "trades_detail"},
}
)
@@ -980,6 +1058,11 @@ def _candidate_blocks(record: ForecastRecord, edge: float) -> bool:
)
def _entry_validation_skill(entry: dict[str, Any]) -> float:
value = entry.get("validation_skill")
return float(value) if isinstance(value, (int, float)) and math.isfinite(float(value)) else 0.0
def _candidate_allows(record: ForecastRecord, edge: float, probability: float, confidence: float) -> bool:
dynamic_confidence = _forecast_confidence(record.expected_percent, record.probability_up, record.skill, edge)
return (
@@ -1146,6 +1229,101 @@ def _calibrate(
return results
def _calibrate_strategy(
records: list[ForecastRecord],
*,
edges: list[float],
probabilities: list[float],
confidences: list[float],
min_trades: int,
horizon: int,
round_trip_cost: float,
settings: Any,
) -> list[CalibrationResult]:
results: list[CalibrationResult] = []
for edge in edges:
for probability in probabilities:
for confidence in confidences:
thresholds = CalibrationResult(
edge=edge,
probability=probability,
confidence=confidence,
trades=0,
wins=0,
win_rate=0.0,
total_net_percent=0.0,
average_net_percent=0.0,
max_drawdown_percent=0.0,
profit_factor=0.0,
score=0.0,
)
replay = _full_backtest(
records,
thresholds,
horizon=horizon,
round_trip_cost=round_trip_cost,
settings=settings,
detail_limit=0,
)
trades = int(replay.get("trades", 0) or 0)
if trades <= 0:
continue
wins = int(replay.get("wins", 0) or 0)
total = float(replay.get("total_net_percent", 0.0) or 0.0)
average = float(replay.get("avg_net_percent", 0.0) or 0.0)
drawdown = float(replay.get("max_drawdown_percent", 0.0) or 0.0)
profit_factor = float(replay.get("profit_factor", 0.0) or 0.0)
trade_factor = min(1.0, trades / max(1, min_trades))
score = (
average * trade_factor
+ total * 0.015
- drawdown * 0.03
+ (wins / trades) * 0.04
)
results.append(
CalibrationResult(
edge=edge,
probability=probability,
confidence=confidence,
trades=trades,
wins=wins,
win_rate=wins / trades,
total_net_percent=total,
average_net_percent=average,
max_drawdown_percent=drawdown,
profit_factor=profit_factor,
score=score,
)
)
results.sort(
key=lambda item: (
item.score,
item.average_net_percent,
item.total_net_percent,
item.profit_factor,
item.trades,
),
reverse=True,
)
return results
def _result_from_dict(value: dict[str, Any]) -> CalibrationResult:
return CalibrationResult(
edge=float(value.get("edge", 0.1) or 0.1),
probability=float(value.get("probability", 0.7) or 0.7),
confidence=float(value.get("confidence", 0.4) or 0.4),
trades=int(value.get("trades", 0) or 0),
wins=int(value.get("wins", 0) or 0),
win_rate=float(value.get("win_rate", 0.0) or 0.0),
total_net_percent=float(value.get("total_net_percent", 0.0) or 0.0),
average_net_percent=float(value.get("average_net_percent", 0.0) or 0.0),
max_drawdown_percent=float(value.get("max_drawdown_percent", 0.0) or 0.0),
profit_factor=float(value.get("profit_factor", 0.0) or 0.0),
score=float(value.get("score", 0.0) or 0.0),
)
def _selected_trades(
records: list[ForecastRecord],
edge: float,
@@ -1164,7 +1342,7 @@ def _selected_trades(
return trades
def _choose_recommendation(results: list[CalibrationResult], *, min_trades: int) -> CalibrationResult:
def _choose_recommendation(results: list[CalibrationResult], *, min_trades: int) -> CalibrationResult | None:
viable = [
result
for result in results
@@ -1173,7 +1351,67 @@ def _choose_recommendation(results: list[CalibrationResult], *, min_trades: int)
and result.total_net_percent > 0
and result.profit_factor >= 1.05
]
return viable[0] if viable else results[0]
return viable[0] if viable else None
def _empty_recommendation(
edges: list[float], probabilities: list[float], confidences: list[float]
) -> CalibrationResult:
return CalibrationResult(
edge=max(edges or [1.0]),
probability=max(probabilities or [0.95]),
confidence=max(confidences or [1.0]),
trades=0,
wins=0,
win_rate=0.0,
total_net_percent=0.0,
average_net_percent=0.0,
max_drawdown_percent=0.0,
profit_factor=0.0,
score=-1.0,
)
def _fit_platt_calibration(records: list[ForecastRecord]) -> dict[str, float]:
samples = [
(
math.log(_clamp(record.probability_up, 1e-5, 1.0 - 1e-5) / (1.0 - _clamp(record.probability_up, 1e-5, 1.0 - 1e-5))),
1.0 if record.future_net_percent > 0 else 0.0,
)
for record in records
]
if len(samples) < 30:
return {"slope": 1.0, "intercept": 0.0, "samples": float(len(samples))}
slope = 1.0
intercept = 0.0
learning_rate = 0.05
for _ in range(300):
grad_slope = 0.0
grad_intercept = 0.0
for logit, target in samples:
probability = 1.0 / (1.0 + math.exp(-_clamp(slope * logit + intercept, -30.0, 30.0)))
error = probability - target
grad_slope += error * logit
grad_intercept += error
grad_slope = grad_slope / len(samples) + 0.001 * (slope - 1.0)
grad_intercept /= len(samples)
slope -= learning_rate * grad_slope
intercept -= learning_rate * grad_intercept
return {"slope": round(slope, 8), "intercept": round(intercept, 8), "samples": float(len(samples))}
def _apply_platt_calibration(
records: list[ForecastRecord], calibration: dict[str, float]
) -> list[ForecastRecord]:
slope = float(calibration.get("slope", 1.0))
intercept = float(calibration.get("intercept", 0.0))
output: list[ForecastRecord] = []
for record in records:
probability = _clamp(record.probability_up, 1e-5, 1.0 - 1e-5)
logit = math.log(probability / (1.0 - probability))
calibrated = 1.0 / (1.0 + math.exp(-_clamp(slope * logit + intercept, -30.0, 30.0)))
output.append(replace(record, probability_up=calibrated))
return output
def _choose_replay_recommendation(
@@ -1185,8 +1423,10 @@ def _choose_replay_recommendation(
horizon: int,
round_trip_cost: float,
settings: Any,
) -> tuple[CalibrationResult, dict[str, Any]]:
) -> tuple[CalibrationResult | None, dict[str, Any]]:
fallback = _choose_recommendation(results, min_trades=min_trades)
if fallback is None:
return None, {**_stats([]), "trades_detail": [], "symbol_breakdown": []}
fallback_replay = _full_backtest(records, fallback, horizon=horizon, round_trip_cost=round_trip_cost, settings=settings)
if min_full_replay_trades <= 0:
return fallback, fallback_replay
@@ -1207,7 +1447,7 @@ def _choose_replay_recommendation(
viable.append((result, replay))
if not viable:
return fallback, fallback_replay
return None, fallback_replay
viable.sort(
key=lambda item: (
item[0].score,