Add honest Torch validation gate
This commit is contained in:
@@ -284,6 +284,40 @@ def test_torch_forecast_blocks_without_valid_torch_model(make_settings, tmp_path
|
||||
assert signal.diagnostics["checks"]["torch_model_ok"] is False
|
||||
|
||||
|
||||
def test_torch_forecast_blocks_failed_quality_gate(make_settings, tmp_path) -> None:
|
||||
settings = make_settings(
|
||||
tmp_path,
|
||||
strategy_mode="torch_forecast",
|
||||
time_series_min_edge_percent=0.10,
|
||||
time_series_min_probability_up=0.57,
|
||||
max_position_usdt=25,
|
||||
stop_loss_percent=0.04,
|
||||
)
|
||||
strategy = SpotStrategy(settings)
|
||||
ticker = Ticker("BTCUSDT", 105, 104.99, 105.01, 10_000_000, 1000, 1.0)
|
||||
|
||||
signal = strategy.entry_signal(
|
||||
"BTCUSDT",
|
||||
[],
|
||||
ticker,
|
||||
open_positions_for_symbol=0,
|
||||
forecast={
|
||||
"usable": True,
|
||||
"model": "torch_gru",
|
||||
"expected_return_percent": 0.36,
|
||||
"probability_up": 0.66,
|
||||
"skill": 0.22,
|
||||
"block_entry": False,
|
||||
"quality_gate_passed": False,
|
||||
"quality_gate": {"status": "fail"},
|
||||
},
|
||||
account={"equity": 100.0},
|
||||
)
|
||||
|
||||
assert signal.action == "HOLD"
|
||||
assert signal.diagnostics["checks"]["quality_gate_ok"] is False
|
||||
|
||||
|
||||
def test_torch_forecast_probe_buys_on_positive_high_probability(make_settings, tmp_path) -> None:
|
||||
settings = make_settings(
|
||||
tmp_path,
|
||||
|
||||
@@ -282,6 +282,28 @@ def test_time_series_forecaster_reads_torch_gru_artifact(make_settings, tmp_path
|
||||
assert forecast.probability_up > 0.5
|
||||
|
||||
|
||||
def test_time_series_forecaster_attaches_quality_gate(make_settings, tmp_path) -> None:
|
||||
artifact_path = tmp_path / "lstm_forecaster.json"
|
||||
_write_torch_gru_artifact(artifact_path, head_bias=0.2)
|
||||
(tmp_path / "torch_threshold_calibration.json").write_text(
|
||||
json.dumps({"validation": {"status": "fail", "passed": False, "checks": []}}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
settings = make_settings(
|
||||
tmp_path,
|
||||
time_series_min_candles=80,
|
||||
time_series_lstm_enabled=True,
|
||||
time_series_lstm_model_path=artifact_path,
|
||||
)
|
||||
returns = [0.00015 if index % 4 else -0.00005 for index in range(140)]
|
||||
|
||||
forecast = TimeSeriesForecaster(settings).forecast(_candles_from_returns(returns), symbol="BTCUSDT")
|
||||
|
||||
assert forecast.usable is True
|
||||
assert forecast.quality_gate_passed is False
|
||||
assert forecast.quality_gate["status"] == "fail"
|
||||
|
||||
|
||||
def test_time_series_forecaster_reads_multifeature_direct_horizon_artifact(make_settings, tmp_path) -> None:
|
||||
artifact_path = tmp_path / "lstm_forecaster.json"
|
||||
_write_multifeature_torch_gru_artifact(artifact_path, head_bias=0.2)
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from tools.accept_torch_candidate import _decision
|
||||
|
||||
|
||||
def _report(*, validation_passed: bool = True, trades: int = 30, total: float = 10.0) -> dict:
|
||||
return {
|
||||
"recommended": {"score": 0.5},
|
||||
"full_replay": {
|
||||
"trades": trades,
|
||||
"avg_net_percent": 0.4,
|
||||
"total_net_percent": total,
|
||||
"profit_factor": 2.0,
|
||||
"max_drawdown_percent": 1.0,
|
||||
},
|
||||
"walk_forward": {"summary": {"trades": trades, "avg_net_percent": 0.3}},
|
||||
"validation": {"passed": validation_passed, "status": "pass" if validation_passed else "fail"},
|
||||
}
|
||||
|
||||
|
||||
def test_guard_rejects_candidate_without_honest_validation() -> None:
|
||||
decision = _decision(
|
||||
_report(),
|
||||
_report(validation_passed=False),
|
||||
min_trades=8,
|
||||
min_profit_factor=1.05,
|
||||
min_avg_net_percent=0.0,
|
||||
max_score_regression=0.05,
|
||||
)
|
||||
|
||||
assert decision == {"accepted": False, "reason": "candidate_failed_honest_validation"}
|
||||
|
||||
|
||||
def test_guard_accepts_candidate_that_passes_honest_validation() -> None:
|
||||
decision = _decision(
|
||||
_report(total=9.0),
|
||||
_report(total=12.0),
|
||||
min_trades=8,
|
||||
min_profit_factor=1.05,
|
||||
min_avg_net_percent=0.0,
|
||||
max_score_regression=0.05,
|
||||
)
|
||||
|
||||
assert decision == {"accepted": True, "reason": "candidate_passed_guard"}
|
||||
Reference in New Issue
Block a user