feat: train forecasts on trade outcomes
This commit is contained in:
Курнат Андрей
@@ -7,12 +7,20 @@ from tools.calibrate_torch_thresholds import (
|
||||
ForecastRecord,
|
||||
_average_selected_predictions,
|
||||
_apply_platt_calibration,
|
||||
_build_torch_model,
|
||||
_choose_recommendation,
|
||||
_full_backtest,
|
||||
_fit_platt_calibration,
|
||||
_record_event_target,
|
||||
_entry_validation_skill,
|
||||
)
|
||||
from tools.train_torch_recurrent_forecaster import _ensemble_candidate
|
||||
from tools.train_torch_recurrent_forecaster import (
|
||||
OUTPUT_LAYOUT,
|
||||
RecurrentReturnModel,
|
||||
_ensemble_candidate,
|
||||
_export_head_state,
|
||||
_export_recurrent_state,
|
||||
)
|
||||
|
||||
|
||||
def _result(*, trades: int, average: float, total: float, profit_factor: float) -> CalibrationResult:
|
||||
@@ -85,6 +93,13 @@ def test_platt_calibration_learns_probability_direction_from_train_records() ->
|
||||
assert calibrated[0].probability_up < calibrated[1].probability_up
|
||||
|
||||
|
||||
def test_barrier_event_target_takes_precedence_over_terminal_profit() -> None:
|
||||
record = _record(1, 0.8, 3.0)
|
||||
record.take_profit_first = False
|
||||
|
||||
assert _record_event_target(record) == 0.0
|
||||
|
||||
|
||||
def test_entry_quality_never_falls_back_to_holdout_skill() -> None:
|
||||
entry = {"validation_skill": 0.12, "skill": 0.99, "holdout_skill": 0.99}
|
||||
|
||||
@@ -107,6 +122,36 @@ def test_batched_ensemble_averages_decoded_predictions() -> None:
|
||||
}
|
||||
|
||||
|
||||
def test_calibrator_loads_multitask_head() -> None:
|
||||
model = RecurrentReturnModel(
|
||||
architecture="gru",
|
||||
input_size=2,
|
||||
hidden_size=4,
|
||||
num_layers=1,
|
||||
dropout=0.0,
|
||||
output_size=len(OUTPUT_LAYOUT),
|
||||
attention_pooling=False,
|
||||
context_norm=False,
|
||||
multitask_head=True,
|
||||
head_hidden_size=6,
|
||||
)
|
||||
entry = {
|
||||
"input_size": 2,
|
||||
"hidden_size": 4,
|
||||
"num_layers": 1,
|
||||
"output_size": len(OUTPUT_LAYOUT),
|
||||
"multitask_head": True,
|
||||
"head_hidden_size": 6,
|
||||
"state_dict": _export_recurrent_state(model),
|
||||
**_export_head_state(model),
|
||||
}
|
||||
|
||||
loaded = _build_torch_model(entry, "torch_gru")
|
||||
|
||||
assert loaded is not None
|
||||
assert loaded.multitask_head is True
|
||||
|
||||
|
||||
def test_multi_seed_export_does_not_duplicate_first_member_weights() -> None:
|
||||
members = [
|
||||
{
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_retrain_runner_passes_training_horizon_to_calibrator() -> None:
|
||||
runner = (
|
||||
Path(__file__).resolve().parents[1] / "tools" / "run_torch_retrain.ps1"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
calibration_start = runner.index("$calibrationBaseArgs = @(")
|
||||
calibration_end = runner.index("\n )", calibration_start)
|
||||
calibration_args = runner[calibration_start:calibration_end]
|
||||
|
||||
assert '"--horizon", $Horizon.ToString()' in calibration_args
|
||||
|
||||
|
||||
def test_retrain_runner_uses_a_regime_sized_validation_window() -> None:
|
||||
runner = (
|
||||
Path(__file__).resolve().parents[1] / "tools" / "run_torch_retrain.ps1"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "[int]$ValidationWindow = 0" in runner
|
||||
assert "else { 720 }" in runner
|
||||
assert '"--validation-window", $ValidationWindow.ToString()' in runner
|
||||
@@ -2,6 +2,8 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from crypto_spot_bot.models import Candle
|
||||
from crypto_spot_bot.time_series import TimeSeriesForecaster
|
||||
|
||||
@@ -191,6 +193,77 @@ def _write_probabilistic_torch_gru_artifact(path) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _write_barrier_multitask_gru_artifact(path) -> None:
|
||||
hidden_size = 2
|
||||
head_hidden_size = 2
|
||||
input_size = 2
|
||||
output_size = 5
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"version": 6,
|
||||
"type": "pytorch_recurrent_forecaster",
|
||||
"target_horizon": 3,
|
||||
"target_horizons": [3],
|
||||
"direct_horizon": True,
|
||||
"target_transform": "barrier_net_return",
|
||||
"event_target": "take_profit_before_stop_loss",
|
||||
"round_trip_cost": 0.0026,
|
||||
"output_layout": ["mean", "q10", "q50", "q90", "logit_tp_first"],
|
||||
"feature_names": ["return_1", "range_percent"],
|
||||
"symbols": {
|
||||
"BTCUSDT": {
|
||||
"model": "torch_gru",
|
||||
"architecture": "gru",
|
||||
"lookback": 8,
|
||||
"target_horizon": 3,
|
||||
"target_horizons": [3],
|
||||
"direct_horizon": True,
|
||||
"target_transform": "barrier_net_return",
|
||||
"event_target": "take_profit_before_stop_loss",
|
||||
"target_stop_loss_percent": 0.04,
|
||||
"target_take_profit_percent": 0.035,
|
||||
"round_trip_cost": 0.0026,
|
||||
"output_layout": ["mean", "q10", "q50", "q90", "logit_tp_first"],
|
||||
"input_size": input_size,
|
||||
"output_size": output_size,
|
||||
"feature_names": ["return_1", "range_percent"],
|
||||
"feature_means": [0.0, 0.0],
|
||||
"feature_scales": [0.001, 0.001],
|
||||
"target_means": [0.0],
|
||||
"target_scales": [1.0],
|
||||
"target_mean": 0.0,
|
||||
"target_scale": 1.0,
|
||||
"hidden_size": hidden_size,
|
||||
"num_layers": 1,
|
||||
"clip": 8.0,
|
||||
"validation_mae_by_horizon": {"3": 0.01},
|
||||
"baseline_mae_by_horizon": {"3": 0.02},
|
||||
"validation_mae_percent": 1.0,
|
||||
"baseline_mae_percent": 2.0,
|
||||
"skill": 0.2,
|
||||
"multitask_head": True,
|
||||
"head_hidden_size": head_hidden_size,
|
||||
"state_dict": {
|
||||
"weight_ih_l0": [[0.0, 0.0] for _ in range(3 * hidden_size)],
|
||||
"weight_hh_l0": [[0.0, 0.0] for _ in range(3 * hidden_size)],
|
||||
"bias_ih_l0": [0.0 for _ in range(3 * hidden_size)],
|
||||
"bias_hh_l0": [0.0 for _ in range(3 * hidden_size)],
|
||||
},
|
||||
"head_hidden_weight": [[0.0, 0.0], [0.0, 0.0]],
|
||||
"head_hidden_bias": [0.0, 0.0],
|
||||
"return_head_weight": [[0.0, 0.0] for _ in range(4)],
|
||||
"return_head_bias": [0.01, -0.01, 0.005, 0.02],
|
||||
"event_head_weight": [[0.0, 0.0]],
|
||||
"event_head_bias": [1.38629436112],
|
||||
}
|
||||
},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def test_time_series_forecaster_requires_torch_artifact(make_settings, tmp_path) -> None:
|
||||
settings = make_settings(
|
||||
tmp_path,
|
||||
@@ -428,3 +501,24 @@ def test_time_series_forecaster_reads_probabilistic_multi_horizon_artifact(make_
|
||||
assert forecast.feature_snapshot[0]["label"] == "Доходность 1ч"
|
||||
assert forecast.feature_snapshot[0]["raw_display"].endswith("%")
|
||||
assert "диапазон" in forecast.feature_snapshot[0]["interpretation"]
|
||||
|
||||
|
||||
def test_time_series_forecaster_reads_barrier_multitask_artifact(make_settings, tmp_path) -> None:
|
||||
artifact_path = tmp_path / "lstm_forecaster.json"
|
||||
_write_barrier_multitask_gru_artifact(artifact_path)
|
||||
settings = make_settings(
|
||||
tmp_path,
|
||||
time_series_lstm_model_path=artifact_path,
|
||||
time_series_min_candles=80,
|
||||
time_series_forecast_horizon=3,
|
||||
)
|
||||
|
||||
forecast = TimeSeriesForecaster(settings).forecast(
|
||||
_candles_from_returns([0.0002] * 140), symbol="BTCUSDT"
|
||||
)
|
||||
|
||||
assert forecast.usable is True
|
||||
assert forecast.target_transform == "barrier_net_return"
|
||||
assert forecast.expected_return_percent == pytest.approx(1.005, abs=0.01)
|
||||
assert forecast.probability_take_profit_first == pytest.approx(0.8, abs=0.001)
|
||||
assert "P(TP before SL)" in forecast.reason
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from crypto_spot_bot.models import Candle
|
||||
from crypto_spot_bot.time_series import _torch_head_outputs
|
||||
from tools.train_torch_recurrent_forecaster import (
|
||||
OUTPUT_LAYOUT,
|
||||
RecurrentReturnModel,
|
||||
_barrier_outcome,
|
||||
_export_head_state,
|
||||
)
|
||||
|
||||
|
||||
def _candle(index: int, *, open_: float, high: float, low: float, close: float) -> Candle:
|
||||
return Candle(index, open_, high, low, close, 100.0)
|
||||
|
||||
|
||||
def test_barrier_target_uses_next_open_and_marks_take_profit_first() -> None:
|
||||
candles = [
|
||||
_candle(0, open_=90.0, high=101.0, low=89.0, close=100.0),
|
||||
_candle(1, open_=100.0, high=102.0, low=99.0, close=101.0),
|
||||
_candle(2, open_=101.0, high=104.0, low=100.0, close=103.0),
|
||||
]
|
||||
|
||||
net_return, event = _barrier_outcome(
|
||||
candles,
|
||||
end_index=0,
|
||||
horizon=2,
|
||||
stop_loss_percent=0.02,
|
||||
take_profit_percent=0.03,
|
||||
round_trip_cost=0.002,
|
||||
) or (math.nan, math.nan)
|
||||
|
||||
assert event == 1.0
|
||||
assert net_return == pytest.approx(math.log(1.03) - 0.002)
|
||||
|
||||
|
||||
def test_barrier_target_resolves_same_candle_tie_as_stop_loss() -> None:
|
||||
candles = [
|
||||
_candle(0, open_=100.0, high=101.0, low=99.0, close=100.0),
|
||||
_candle(1, open_=100.0, high=104.0, low=97.0, close=101.0),
|
||||
]
|
||||
|
||||
net_return, event = _barrier_outcome(
|
||||
candles,
|
||||
end_index=0,
|
||||
horizon=1,
|
||||
stop_loss_percent=0.02,
|
||||
take_profit_percent=0.03,
|
||||
round_trip_cost=0.002,
|
||||
) or (math.nan, math.nan)
|
||||
|
||||
assert event == 0.0
|
||||
assert net_return == pytest.approx(math.log(0.98) - 0.002)
|
||||
|
||||
|
||||
def test_multitask_head_export_matches_runtime_inference() -> None:
|
||||
torch.manual_seed(7)
|
||||
model = RecurrentReturnModel(
|
||||
architecture="gru",
|
||||
input_size=2,
|
||||
hidden_size=4,
|
||||
num_layers=1,
|
||||
dropout=0.0,
|
||||
output_size=2 * len(OUTPUT_LAYOUT),
|
||||
attention_pooling=False,
|
||||
context_norm=False,
|
||||
multitask_head=True,
|
||||
head_hidden_size=6,
|
||||
)
|
||||
model.eval()
|
||||
context = torch.tensor([[0.2, -0.1, 0.4, 0.3]], dtype=torch.float32)
|
||||
with torch.no_grad():
|
||||
shared = model.head_activation(model.head_hidden(context))
|
||||
returns = model.return_head(shared).view(1, 2, 4)
|
||||
events = model.event_head(shared).view(1, 2, 1)
|
||||
expected = torch.cat((returns, events), dim=2).reshape(-1).tolist()
|
||||
|
||||
entry = {"multitask_head": True, **_export_head_state(model)}
|
||||
actual = _torch_head_outputs(context[0].tolist(), entry, hidden_size=4)
|
||||
|
||||
assert actual == pytest.approx(expected, abs=2e-6)
|
||||
@@ -6,7 +6,7 @@ import json
|
||||
|
||||
import pytest
|
||||
|
||||
from crypto_spot_bot.training_coordination import TrainingCoordinator
|
||||
from crypto_spot_bot.training_coordination import TrainingCoordinator, _validate_symbol_models
|
||||
|
||||
|
||||
def test_training_coordinator_claims_and_completes_job(tmp_path) -> None:
|
||||
@@ -57,10 +57,15 @@ def test_training_coordinator_sanitizes_independent_training_parameters(tmp_path
|
||||
"parameters": {
|
||||
"pooled": False,
|
||||
"limit": 6000,
|
||||
"validation_window": 720,
|
||||
"ensemble_seeds": "7,19",
|
||||
"selection_folds": 3,
|
||||
"learning_rate": 0.0007,
|
||||
"weight_decay": 0.0005,
|
||||
"horizon": 12,
|
||||
"horizons": "3,6,12,24",
|
||||
"patience": 8,
|
||||
"seed": 7,
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -68,10 +73,15 @@ def test_training_coordinator_sanitizes_independent_training_parameters(tmp_path
|
||||
assert requested["job"]["parameters"] == {
|
||||
"pooled": False,
|
||||
"limit": 6000,
|
||||
"validation_window": 720,
|
||||
"ensemble_seeds": "7,19",
|
||||
"selection_folds": 3,
|
||||
"learning_rate": 0.0007,
|
||||
"weight_decay": 0.0005,
|
||||
"horizon": 12,
|
||||
"horizons": "3,6,12,24",
|
||||
"patience": 8,
|
||||
"seed": 7,
|
||||
}
|
||||
|
||||
|
||||
@@ -148,6 +158,30 @@ def test_training_coordinator_accepts_chunked_artifact_upload(tmp_path) -> None:
|
||||
assert coordinator.status()["latest_job"]["artifacts"][0]["sha256"] == sha256
|
||||
|
||||
|
||||
def test_model_validation_accepts_multitask_ensemble_members() -> None:
|
||||
head = {
|
||||
"state_dict": {"weight_ih_l0": [[0.0]]},
|
||||
"head_hidden_weight": [[0.0]],
|
||||
"head_hidden_bias": [0.0],
|
||||
"return_head_weight": [[0.0]],
|
||||
"return_head_bias": [0.0],
|
||||
"event_head_weight": [[0.0]],
|
||||
"event_head_bias": [0.0],
|
||||
}
|
||||
symbols = {
|
||||
"BTCUSDT": {
|
||||
"model": "torch_gru",
|
||||
"lookback": 8,
|
||||
"input_size": 2,
|
||||
"hidden_size": 4,
|
||||
"multitask_head": True,
|
||||
"ensemble_members": [head, head],
|
||||
}
|
||||
}
|
||||
|
||||
_validate_symbol_models(symbols)
|
||||
|
||||
|
||||
def test_running_claimed_job_keeps_agent_online_when_heartbeat_is_stale(tmp_path) -> None:
|
||||
coordinator = TrainingCoordinator(tmp_path)
|
||||
coordinator.request_retrain({"source": "android"})
|
||||
|
||||
Reference in New Issue
Block a user