sevenhill/TradeBot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Keep bot operational when forecast model is unavailable

Browse Source
This commit is contained in:
Курнат Андрей
2026-07-13 11:57:25 +03:00
parent da53483164
commit 668e606ee2
22 changed files with 706 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/calibrate_torch_thresholds.py
+65 -19
View File
@@ -159,6 +159,7 @@ def main() -> None:
settings=settings,
)
symbol_recommendations: dict[str, dict[str, Any]] = {}
symbol_threshold_results: dict[str, CalibrationResult] = {}
for symbol in symbols:
symbol_records = [record for record in records if record.symbol == symbol]
symbol_results = _calibrate_strategy(
@@ -177,7 +178,8 @@ def main() -> None:
) if symbol_results else None
if symbol_selected is not None:
symbol_recommendations[symbol] = _result_dict(symbol_selected)
calibration_insufficient = recommended is None
symbol_threshold_results[symbol] = symbol_selected
calibration_insufficient = recommended is None or not symbol_threshold_results
if recommended is None:
recommended = _empty_recommendation(
_float_grid(args.edge_grid),
@@ -185,6 +187,16 @@ def main() -> None:
_float_grid(args.confidence_grid),
)
full_backtest = {**_stats([]), "trades_detail": [], "symbol_breakdown": []}
elif symbol_threshold_results:
full_backtest = _full_backtest(
records,
recommended,
horizon=horizon,
round_trip_cost=round_trip_cost,
settings=settings,
symbol_thresholds=symbol_threshold_results,
require_symbol_thresholds=True,
)
print("\nRECOMMENDED")
print(_result_line(recommended))
print("\nFULL_REPLAY")
@@ -248,6 +260,7 @@ def main() -> None:
"recommended": _result_dict(deployment_recommended),
"calibration_insufficient": calibration_insufficient,
"symbol_recommendations": deployment_symbol_recommendations,
"eligible_symbols": sorted(deployment_symbol_recommendations),
"full_replay": full_backtest,
"walk_forward": walk_forward,
"benchmark": benchmark,
@@ -419,8 +432,8 @@ def _batch_forecast_records(
horizons = _entry_target_horizons(entry)
if not horizons:
return None
model = _build_torch_model(entry, model_name)
if model is None:
models = _build_torch_models(entry, model_name)
if not models:
return None
lookback = int(_clamp(_float_entry(entry, "lookback", 64.0), 4.0, 512.0))
@@ -438,7 +451,8 @@ def _batch_forecast_records(
records: list[ForecastRecord] = []
skill = _entry_validation_skill(entry)
model.eval()
for model in models:
model.eval()
with torch.no_grad():
for offset in range(0, len(indices), max(1, batch_size)):
batch_indices = indices[offset : offset + max(1, batch_size)]
@@ -453,17 +467,25 @@ def _batch_forecast_records(
for index in batch_indices
]
batch = torch.tensor(windows, dtype=torch.float32)
outputs = model(batch).detach().cpu().tolist()
for index, output in zip(batch_indices, outputs):
selected = _decode_selected_output(
output,
entry=entry,
candles=candles,
closes=closes,
index=index,
horizon=decision_horizon,
clip=clip,
round_trip_cost=round_trip_cost,
outputs_by_model = [model(batch).detach().cpu().tolist() for model in models]
for batch_offset, index in enumerate(batch_indices):
selected = _average_selected_predictions(
[
decoded
for outputs in outputs_by_model
if (
decoded := _decode_selected_output(
outputs[batch_offset],
entry=entry,
candles=candles,
closes=closes,
index=index,
horizon=decision_horizon,
clip=clip,
round_trip_cost=round_trip_cost,
)
) is not None
]
)
if selected is None:
continue
@@ -502,11 +524,21 @@ def _batch_forecast_records(
return records
def _build_torch_models(entry: dict[str, Any], model_name: str) -> list[Any]:
members = entry.get("ensemble_members")
if isinstance(members, list) and members:
base = {key: value for key, value in entry.items() if key != "ensemble_members"}
models = [
_build_torch_model({**base, **member}, model_name)
for member in members
if isinstance(member, dict)
]
return [model for model in models if model is not None]
model = _build_torch_model(entry, model_name)
return [model] if model is not None else []
def _build_torch_model(entry: dict[str, Any], model_name: str) -> Any | None:
if isinstance(entry.get("ensemble_members"), list) and entry["ensemble_members"]:
# Ensemble inference is handled by the shared pure-Python runtime so
# calibration and production use the exact same averaging path.
return None
if torch is None or RecurrentReturnModel is None:
return None
architecture = "lstm" if model_name == "torch_lstm" else "gru" if model_name == "torch_gru" else ""
@@ -560,6 +592,15 @@ def _build_torch_model(entry: dict[str, Any], model_name: str) -> Any | None:
return model
def _average_selected_predictions(rows: list[dict[str, float]]) -> dict[str, float] | None:
if not rows:
return None
return {
name: sum(float(row[name]) for row in rows) / len(rows)
for name in ("expected_return", "q50", "probability_up")
}
def _decode_selected_output(
output: list[float],
*,
@@ -638,6 +679,7 @@ def _full_backtest(
settings: Any,
detail_limit: int = 50,
symbol_thresholds: dict[str, CalibrationResult] | None = None,
require_symbol_thresholds: bool = False,
) -> dict[str, Any]:
positions: dict[str, dict[str, Any]] = {}
trades: list[float] = []
@@ -708,6 +750,8 @@ def _full_backtest(
if record.symbol in positions:
continue
if require_symbol_thresholds and record.symbol not in (symbol_thresholds or {}):
continue
if _candidate_allows(record, active_thresholds.edge, active_thresholds.probability, active_thresholds.confidence):
positions[record.symbol] = {
"entry_price": record.next_open,
@@ -907,6 +951,7 @@ def _walk_forward(
settings=settings,
detail_limit=0,
symbol_thresholds=symbol_thresholds,
require_symbol_thresholds=True,
)
test_rows = test_backtest.get("trades_detail", [])
test_trades = [float(row.get("net_percent", 0.0) or 0.0) for row in test_rows if isinstance(row, dict)]
@@ -921,6 +966,7 @@ def _walk_forward(
"symbol_thresholds": {
symbol: _result_dict(value) for symbol, value in symbol_thresholds.items()
},
"eligible_symbols": sorted(symbol_thresholds),
"probability_calibration": probability_calibration,
"test": {key: value for key, value in test_backtest.items() if key != "trades_detail"},
}
tools/compact_runtime_db.py
+136
View File
@@ -0,0 +1,136 @@
from __future__ import annotations
import argparse
import json
import sqlite3
import sys
from pathlib import Path
from typing import Any
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from crypto_spot_bot.storage import Storage
PRESERVED_TABLES = ("positions", "trades", "runtime", "orders")
DEFAULT_RECENT_ROWS = {
"signals": 5_000,
"equity": 5_000,
"events": 2_000,
"llm_advice": 1_000,
}
def compact_database(
database: Path,
*,
recent_rows: dict[str, int] | None = None,
backup: Path | None = None,
) -> dict[str, Any]:
database = database.resolve()
if not database.is_file():
raise FileNotFoundError(database)
limits = dict(DEFAULT_RECENT_ROWS)
if recent_rows:
limits.update({key: max(0, int(value)) for key, value in recent_rows.items()})
temp = database.with_name(database.name + ".compact")
backup = (backup or database.with_name(database.name + ".precompact.bak")).resolve()
if temp.exists():
temp.unlink()
if backup.exists():
raise FileExistsError(f"backup already exists: {backup}")
source_bytes = database.stat().st_size
Storage(temp)
counts: dict[str, int] = {}
conn = sqlite3.connect(temp)
try:
conn.execute("PRAGMA foreign_keys=OFF")
conn.execute("ATTACH DATABASE ? AS source", (str(database),))
for table in PRESERVED_TABLES:
counts[table] = _copy_table(conn, table, limit=None)
for table, limit in limits.items():
counts[table] = _copy_table(conn, table, limit=limit)
conn.commit()
# Check only the newly built main database. The attached multi-gigabyte
# source is preserved as the rollback copy and must not be rescanned here.
integrity = str(conn.execute("PRAGMA main.integrity_check").fetchone()[0])
if integrity.lower() != "ok":
raise RuntimeError(f"compacted database integrity check failed: {integrity}")
conn.execute("DETACH DATABASE source")
conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
conn.execute("PRAGMA journal_mode=DELETE")
conn.commit()
finally:
conn.close()
database.replace(backup)
temp.replace(database)
compacted_bytes = database.stat().st_size
return {
"database": str(database),
"backup": str(backup),
"source_bytes": source_bytes,
"compacted_bytes": compacted_bytes,
"reclaimed_bytes": max(0, source_bytes - compacted_bytes),
"rows": counts,
}
def _copy_table(conn: sqlite3.Connection, table: str, *, limit: int | None) -> int:
destination_columns = _columns(conn, "main", table)
source_columns = set(_columns(conn, "source", table))
columns = [column for column in destination_columns if column in source_columns]
if not columns:
return 0
quoted = ", ".join(f'"{column}"' for column in columns)
if limit is None:
conn.execute(
f'INSERT INTO main."{table}" ({quoted}) SELECT {quoted} FROM source."{table}"'
)
elif limit > 0:
conn.execute(
f'INSERT INTO main."{table}" ({quoted}) '
f'SELECT {quoted} FROM source."{table}" ORDER BY id DESC LIMIT ?',
(limit,),
)
row = conn.execute(f'SELECT COUNT(*) FROM main."{table}"').fetchone()
return int(row[0] if row else 0)
def _columns(conn: sqlite3.Connection, schema: str, table: str) -> list[str]:
return [str(row[1]) for row in conn.execute(f'PRAGMA {schema}.table_info("{table}")')]
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Atomically compact the TradeBot runtime database while preserving durable trading state."
)
parser.add_argument("--database", required=True)
parser.add_argument("--backup", default="")
parser.add_argument("--signals", type=int, default=DEFAULT_RECENT_ROWS["signals"])
parser.add_argument("--equity", type=int, default=DEFAULT_RECENT_ROWS["equity"])
parser.add_argument("--events", type=int, default=DEFAULT_RECENT_ROWS["events"])
parser.add_argument("--llm-advice", type=int, default=DEFAULT_RECENT_ROWS["llm_advice"])
return parser.parse_args()
def main() -> None:
args = _parse_args()
result = compact_database(
Path(args.database),
backup=Path(args.backup) if args.backup else None,
recent_rows={
"signals": args.signals,
"equity": args.equity,
"events": args.events,
"llm_advice": args.llm_advice,
},
)
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
if __name__ == "__main__":
main()
tools/run_torch_retrain.ps1
+10 -3
View File
@@ -27,6 +27,7 @@ param(
[string]$PiRoot = "",
[string]$PiSshKeyPath = "",
[switch]$NoPiRestart,
[switch]$Pooled,
[switch]$SkipGuard,
[switch]$ResumeCandidate
)
@@ -148,13 +149,13 @@ if (-not $Horizons) { $Horizons = if ($env:TORCH_RETRAIN_HORIZONS) { $env:TORCH_
if (-not $Features -and $env:TORCH_RETRAIN_FEATURES) { $Features = $env:TORCH_RETRAIN_FEATURES }
if (-not $ContextSymbols -and $env:TORCH_RETRAIN_CONTEXT_SYMBOLS) { $ContextSymbols = $env:TORCH_RETRAIN_CONTEXT_SYMBOLS }
if ($Seed -le 0 -and $env:TORCH_RETRAIN_SEED) { $Seed = [int]$env:TORCH_RETRAIN_SEED }
if (-not $EnsembleSeeds) { $EnsembleSeeds = if ($env:TORCH_RETRAIN_ENSEMBLE_SEEDS) { $env:TORCH_RETRAIN_ENSEMBLE_SEEDS } else { "7,19,43" } }
if (-not $EnsembleSeeds) { $EnsembleSeeds = if ($env:TORCH_RETRAIN_ENSEMBLE_SEEDS) { $env:TORCH_RETRAIN_ENSEMBLE_SEEDS } else { "7,19" } }
if ($SelectionFolds -le 0) { $SelectionFolds = if ($env:TORCH_RETRAIN_SELECTION_FOLDS) { [int]$env:TORCH_RETRAIN_SELECTION_FOLDS } else { 3 } }
if ($LearningRate -le 0) { $LearningRate = if ($env:TORCH_RETRAIN_LEARNING_RATE) { [double]$env:TORCH_RETRAIN_LEARNING_RATE } else { 0.0007 } }
if ($WeightDecay -le 0) { $WeightDecay = if ($env:TORCH_RETRAIN_WEIGHT_DECAY) { [double]$env:TORCH_RETRAIN_WEIGHT_DECAY } else { 0.0005 } }
if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 70 } }
if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 8 } }
if ($HoldoutWindow -le 0) { $HoldoutWindow = if ($env:TORCH_RETRAIN_HOLDOUT_WINDOW) { [int]$env:TORCH_RETRAIN_HOLDOUT_WINDOW } else { 240 } }
if ($HoldoutWindow -le 0) { $HoldoutWindow = if ($env:TORCH_RETRAIN_HOLDOUT_WINDOW) { [int]$env:TORCH_RETRAIN_HOLDOUT_WINDOW } else { 1000 } }
if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RETRAIN_INTERVAL }
if (-not $EnvFile -and $env:TORCH_RETRAIN_ENV) { $EnvFile = $env:TORCH_RETRAIN_ENV }
if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" }
@@ -196,6 +197,12 @@ try {
"--weight-decay", $WeightDecay.ToString([Globalization.CultureInfo]::InvariantCulture),
"--output", $CandidateFile
)
if ($Pooled) {
$trainerArgs += "--pooled"
}
else {
$trainerArgs += "--no-pooled"
}
if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) }
if ($Interval) { $trainerArgs += @("--interval", $Interval) }
if ($EnvFile) { $trainerArgs += @("--env", $EnvFile) }
@@ -236,7 +243,7 @@ try {
"tools\calibrate_torch_thresholds.py",
"--limit", $Limit.ToString(),
"--calibration-window", ([Math]::Min(2400, [Math]::Max(1200, [int]($Limit / 2)))).ToString(),
"--min-trades", "60",
"--min-trades", "24",
"--walk-forward-folds", "8",
"--confidence-grid", "0.40"
)
tools/train_torch_recurrent_forecaster.py
+15 -5
View File
@@ -1020,12 +1020,22 @@ def _ensemble_candidate(members: list[dict[str, Any]], seeds: list[int]) -> dict
"context_norm_weight",
"context_norm_bias",
)
result["ensemble_members"] = [
{name: member[name] for name in export_names if name in member}
| {"seed": seeds[index] if index < len(seeds) else index}
for index, member in enumerate(members)
]
result["ensemble_size"] = len(members)
result["ensemble_seeds"] = [seeds[index] if index < len(seeds) else index for index in range(len(members))]
if len(members) > 1:
result["ensemble_members"] = [
{name: member[name] for name in export_names if name in member}
| {"seed": seeds[index] if index < len(seeds) else index}
for index, member in enumerate(members)
]
# Ensemble inference uses the member payloads. Keeping the first
# member at the top level duplicated a complete network in every
# exported symbol and could push an otherwise valid artifact over
# the server upload limit.
for name in export_names:
result.pop(name, None)
else:
result.pop("ensemble_members", None)
symbol_names = sorted(
{
symbol
tools/windows_training_agent.py
+2
View File
@@ -118,6 +118,8 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
value = parameters.get(key)
if value not in (None, ""):
cmd.extend([ps_arg, str(value)])
if parameters.get("pooled") is True:
cmd.append("-Pooled")
if parameters.get("resume_candidate") is True:
cmd.append("-ResumeCandidate")
log(log_path, "Running retrain: " + " ".join(quote_for_log(part) for part in cmd))