Add analytics risk guard and redesigned dashboard
This commit is contained in:
Codex
@@ -0,0 +1,114 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = _parse_args()
|
||||
current = _read_json(args.current_report)
|
||||
candidate = _read_json(args.candidate_report)
|
||||
decision = _decision(
|
||||
current,
|
||||
candidate,
|
||||
min_trades=args.min_trades,
|
||||
min_profit_factor=args.min_profit_factor,
|
||||
min_avg_net_percent=args.min_avg_net_percent,
|
||||
max_score_regression=args.max_score_regression,
|
||||
)
|
||||
payload = {
|
||||
"accepted": decision["accepted"],
|
||||
"reason": decision["reason"],
|
||||
"current": _summary(current),
|
||||
"candidate": _summary(candidate),
|
||||
}
|
||||
if args.report:
|
||||
Path(args.report).write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
||||
if not decision["accepted"]:
|
||||
raise SystemExit(2)
|
||||
target = Path(args.target_artifact)
|
||||
candidate_artifact = Path(args.candidate_artifact)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(candidate_artifact, target)
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Accept or reject a retrained Torch candidate artifact.")
|
||||
parser.add_argument("--current-report", required=True)
|
||||
parser.add_argument("--candidate-report", required=True)
|
||||
parser.add_argument("--candidate-artifact", required=True)
|
||||
parser.add_argument("--target-artifact", required=True)
|
||||
parser.add_argument("--report", default="")
|
||||
parser.add_argument("--min-trades", type=int, default=8)
|
||||
parser.add_argument("--min-profit-factor", type=float, default=1.05)
|
||||
parser.add_argument("--min-avg-net-percent", type=float, default=0.0)
|
||||
parser.add_argument("--max-score-regression", type=float, default=0.05)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _decision(
|
||||
current: dict[str, Any],
|
||||
candidate: dict[str, Any],
|
||||
*,
|
||||
min_trades: int,
|
||||
min_profit_factor: float,
|
||||
min_avg_net_percent: float,
|
||||
max_score_regression: float,
|
||||
) -> dict[str, Any]:
|
||||
candidate_score = _score(candidate)
|
||||
current_score = _score(current)
|
||||
candidate_replay = candidate.get("full_replay") if isinstance(candidate.get("full_replay"), dict) else {}
|
||||
candidate_walk = candidate.get("walk_forward") if isinstance(candidate.get("walk_forward"), dict) else {}
|
||||
walk_summary = candidate_walk.get("summary") if isinstance(candidate_walk.get("summary"), dict) else {}
|
||||
if int(candidate_replay.get("trades", 0) or 0) < min_trades:
|
||||
return {"accepted": False, "reason": "candidate_has_too_few_full_replay_trades"}
|
||||
if float(candidate_replay.get("profit_factor", 0.0) or 0.0) < min_profit_factor:
|
||||
return {"accepted": False, "reason": "candidate_profit_factor_below_min"}
|
||||
if float(candidate_replay.get("avg_net_percent", 0.0) or 0.0) <= min_avg_net_percent:
|
||||
return {"accepted": False, "reason": "candidate_expectancy_non_positive"}
|
||||
if int(walk_summary.get("trades", 0) or 0) >= min_trades and float(walk_summary.get("avg_net_percent", 0.0) or 0.0) <= min_avg_net_percent:
|
||||
return {"accepted": False, "reason": "candidate_walk_forward_expectancy_non_positive"}
|
||||
if current_score > 0 and candidate_score < current_score * (1.0 - max_score_regression):
|
||||
return {"accepted": False, "reason": "candidate_score_regressed_vs_current"}
|
||||
return {"accepted": True, "reason": "candidate_passed_guard"}
|
||||
|
||||
|
||||
def _score(report: dict[str, Any]) -> float:
|
||||
replay = report.get("full_replay") if isinstance(report.get("full_replay"), dict) else {}
|
||||
recommended = report.get("recommended") if isinstance(report.get("recommended"), dict) else {}
|
||||
replay_score = (
|
||||
float(replay.get("avg_net_percent", 0.0) or 0.0)
|
||||
+ float(replay.get("total_net_percent", 0.0) or 0.0) * 0.02
|
||||
- float(replay.get("max_drawdown_percent", 0.0) or 0.0) * 0.05
|
||||
+ min(float(replay.get("profit_factor", 0.0) or 0.0), 10.0) * 0.03
|
||||
)
|
||||
return replay_score + float(recommended.get("score", 0.0) or 0.0) * 0.25
|
||||
|
||||
|
||||
def _summary(report: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"score": round(_score(report), 6),
|
||||
"recommended": report.get("recommended", {}),
|
||||
"full_replay": report.get("full_replay", {}),
|
||||
"walk_forward_summary": (report.get("walk_forward") or {}).get("summary", {})
|
||||
if isinstance(report.get("walk_forward"), dict)
|
||||
else {},
|
||||
}
|
||||
|
||||
|
||||
def _read_json(path: str) -> dict[str, Any]:
|
||||
if not path:
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -48,6 +48,8 @@ class ForecastRecord:
|
||||
symbol: str
|
||||
index: int
|
||||
timestamp: int
|
||||
close: float
|
||||
atr: float
|
||||
expected_percent: float
|
||||
probability_up: float
|
||||
confidence: float
|
||||
@@ -139,6 +141,20 @@ def main() -> None:
|
||||
recommended = _choose_recommendation(results, min_trades=args.min_trades)
|
||||
print("\nRECOMMENDED")
|
||||
print(_result_line(recommended))
|
||||
full_backtest = _full_backtest(records, recommended, horizon=horizon, round_trip_cost=round_trip_cost, settings=settings)
|
||||
print("\nFULL_REPLAY")
|
||||
print(_stats_line(full_backtest))
|
||||
walk_forward = _walk_forward(
|
||||
records,
|
||||
edges=_float_grid(args.edge_grid),
|
||||
probabilities=_float_grid(args.probability_grid),
|
||||
confidences=_float_grid(args.confidence_grid),
|
||||
min_trades=args.min_trades,
|
||||
horizon=horizon,
|
||||
folds=args.walk_forward_folds,
|
||||
)
|
||||
print("\nWALK_FORWARD")
|
||||
print(json.dumps(walk_forward["summary"], ensure_ascii=False, sort_keys=True))
|
||||
print(
|
||||
"env "
|
||||
f"TIME_SERIES_MIN_EDGE_PERCENT={recommended.edge:.4f} "
|
||||
@@ -151,6 +167,9 @@ def main() -> None:
|
||||
"artifact": _artifact_summary(artifact),
|
||||
"records_by_symbol": per_symbol_counts,
|
||||
"recommended": _result_dict(recommended),
|
||||
"full_replay": full_backtest,
|
||||
"walk_forward": walk_forward,
|
||||
"probability_calibration": _probability_calibration(records),
|
||||
"top_results": [_result_dict(result) for result in results[: args.top]],
|
||||
}
|
||||
Path(args.output).write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
@@ -174,6 +193,7 @@ def _parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--output", default="", help="Optional JSON output path.")
|
||||
parser.add_argument("--batch-size", type=int, default=256, help="Torch inference batch size.")
|
||||
parser.add_argument("--threads", type=int, default=0, help="Torch CPU threads; 0 keeps torch default.")
|
||||
parser.add_argument("--walk-forward-folds", type=int, default=4, help="Threshold walk-forward folds.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -258,6 +278,8 @@ def _forecast_records(
|
||||
symbol=symbol,
|
||||
index=index,
|
||||
timestamp=candles[index].timestamp,
|
||||
close=closes[index],
|
||||
atr=float(candles[index].atr_14 or 0.0),
|
||||
expected_percent=expected_percent,
|
||||
probability_up=probability_up,
|
||||
confidence=_forecast_confidence(expected_percent, probability_up, skill, 0.04),
|
||||
@@ -349,6 +371,8 @@ def _batch_forecast_records(
|
||||
symbol=symbol,
|
||||
index=index,
|
||||
timestamp=candles[index].timestamp,
|
||||
close=closes[index],
|
||||
atr=float(candles[index].atr_14 or 0.0),
|
||||
expected_percent=expected_percent,
|
||||
probability_up=probability_up,
|
||||
confidence=_forecast_confidence(expected_percent, probability_up, skill, 0.04),
|
||||
@@ -484,6 +508,212 @@ def _feature_vector(entry: dict[str, Any], key: str, size: int, default: float)
|
||||
return [default for _ in range(size)]
|
||||
|
||||
|
||||
def _full_backtest(
|
||||
records: list[ForecastRecord],
|
||||
thresholds: CalibrationResult,
|
||||
*,
|
||||
horizon: int,
|
||||
round_trip_cost: float,
|
||||
settings: Any,
|
||||
) -> dict[str, Any]:
|
||||
positions: dict[str, dict[str, Any]] = {}
|
||||
trades: list[float] = []
|
||||
rows: list[dict[str, Any]] = []
|
||||
max_hold = max(12, horizon * 8)
|
||||
stop_loss_percent = max(0.003, min(0.08, float(settings.stop_loss_percent))) * 100.0
|
||||
atr_multiplier = max(0.5, min(10.0, float(settings.atr_trailing_multiplier)))
|
||||
for record in sorted(records, key=lambda item: (item.timestamp, item.symbol)):
|
||||
position = positions.get(record.symbol)
|
||||
if position is not None:
|
||||
position["highest"] = max(position["highest"], record.close)
|
||||
net_percent = _net_percent(position["entry_price"], record.close, round_trip_cost)
|
||||
held = record.index - int(position["entry_index"])
|
||||
atr_stop = (
|
||||
record.close <= position["highest"] - record.atr * atr_multiplier
|
||||
if record.atr > 0 and position["highest"] > position["entry_price"]
|
||||
else False
|
||||
)
|
||||
weak_forecast = (
|
||||
record.expected_percent < thresholds.edge
|
||||
or record.probability_up < thresholds.probability
|
||||
or record.skill <= 0.0
|
||||
)
|
||||
exit_reason = ""
|
||||
if net_percent <= -stop_loss_percent:
|
||||
exit_reason = "stop_loss"
|
||||
elif atr_stop:
|
||||
exit_reason = "atr_trailing_stop"
|
||||
elif (record.expected_percent <= 0.0 or record.probability_up <= 0.50 or _candidate_blocks(record, thresholds.edge)):
|
||||
exit_reason = "forecast_negative"
|
||||
elif weak_forecast and net_percent >= 0:
|
||||
exit_reason = "forecast_weak_profit_lock"
|
||||
elif held >= max_hold:
|
||||
exit_reason = "max_hold"
|
||||
if exit_reason:
|
||||
trades.append(net_percent)
|
||||
rows.append(
|
||||
{
|
||||
"symbol": record.symbol,
|
||||
"entry_timestamp": position["timestamp"],
|
||||
"exit_timestamp": record.timestamp,
|
||||
"net_percent": round(net_percent, 4),
|
||||
"reason": exit_reason,
|
||||
"held_bars": held,
|
||||
"entry_probability": round(float(position["probability_up"]), 4),
|
||||
"entry_expected_percent": round(float(position["expected_percent"]), 4),
|
||||
}
|
||||
)
|
||||
positions.pop(record.symbol, None)
|
||||
continue
|
||||
|
||||
if record.symbol in positions:
|
||||
continue
|
||||
if _candidate_allows(record, thresholds.edge, thresholds.probability, thresholds.confidence):
|
||||
positions[record.symbol] = {
|
||||
"entry_price": record.close,
|
||||
"entry_index": record.index,
|
||||
"timestamp": record.timestamp,
|
||||
"highest": record.close,
|
||||
"probability_up": record.probability_up,
|
||||
"expected_percent": record.expected_percent,
|
||||
}
|
||||
for symbol, position in list(positions.items()):
|
||||
tail = next((record for record in reversed(records) if record.symbol == symbol), None)
|
||||
if tail is None:
|
||||
continue
|
||||
net_percent = _net_percent(position["entry_price"], tail.close, round_trip_cost)
|
||||
trades.append(net_percent)
|
||||
rows.append(
|
||||
{
|
||||
"symbol": symbol,
|
||||
"entry_timestamp": position["timestamp"],
|
||||
"exit_timestamp": tail.timestamp,
|
||||
"net_percent": round(net_percent, 4),
|
||||
"reason": "end_of_replay",
|
||||
"held_bars": tail.index - int(position["entry_index"]),
|
||||
"entry_probability": round(float(position["probability_up"]), 4),
|
||||
"entry_expected_percent": round(float(position["expected_percent"]), 4),
|
||||
}
|
||||
)
|
||||
return {**_stats(trades), "trades_detail": rows[-50:]}
|
||||
|
||||
|
||||
def _walk_forward(
|
||||
records: list[ForecastRecord],
|
||||
*,
|
||||
edges: list[float],
|
||||
probabilities: list[float],
|
||||
confidences: list[float],
|
||||
min_trades: int,
|
||||
horizon: int,
|
||||
folds: int,
|
||||
) -> dict[str, Any]:
|
||||
ordered = sorted(records, key=lambda item: item.timestamp)
|
||||
if folds < 2 or len(ordered) < folds * 20:
|
||||
return {"summary": {"status": "insufficient"}, "folds": []}
|
||||
timestamps = sorted({record.timestamp for record in ordered})
|
||||
fold_size = max(1, len(timestamps) // folds)
|
||||
rows = []
|
||||
all_test_trades: list[float] = []
|
||||
for fold in range(1, folds):
|
||||
test_start = timestamps[fold * fold_size]
|
||||
test_end = timestamps[(fold + 1) * fold_size - 1] if fold < folds - 1 else timestamps[-1]
|
||||
train = [record for record in ordered if record.timestamp < test_start]
|
||||
test = [record for record in ordered if test_start <= record.timestamp <= test_end]
|
||||
train_results = _calibrate(
|
||||
train,
|
||||
edges=edges,
|
||||
probabilities=probabilities,
|
||||
confidences=confidences,
|
||||
min_trades=max(4, min_trades // 2),
|
||||
horizon=horizon,
|
||||
)
|
||||
if not train_results:
|
||||
continue
|
||||
selected = _choose_recommendation(train_results, min_trades=max(4, min_trades // 2))
|
||||
test_trades = _selected_trades(test, selected.edge, selected.probability, selected.confidence, horizon)
|
||||
all_test_trades.extend(test_trades)
|
||||
rows.append(
|
||||
{
|
||||
"fold": fold,
|
||||
"train_records": len(train),
|
||||
"test_records": len(test),
|
||||
"thresholds": _result_dict(selected),
|
||||
"test": _stats(test_trades),
|
||||
}
|
||||
)
|
||||
summary = _stats(all_test_trades)
|
||||
summary["status"] = "ok" if summary["trades"] >= min_trades and summary["avg_net_percent"] > 0 else "warn"
|
||||
return {"summary": summary, "folds": rows}
|
||||
|
||||
|
||||
def _probability_calibration(records: list[ForecastRecord]) -> dict[str, Any]:
|
||||
buckets: dict[str, list[ForecastRecord]] = {}
|
||||
for record in records:
|
||||
low = max(0.0, min(0.95, int(record.probability_up * 20) / 20))
|
||||
key = f"{low:.2f}-{low + 0.05:.2f}"
|
||||
buckets.setdefault(key, []).append(record)
|
||||
rows = []
|
||||
for key in sorted(buckets):
|
||||
items = buckets[key]
|
||||
wins = sum(1 for item in items if item.future_net_percent > 0)
|
||||
avg_probability = sum(item.probability_up for item in items) / len(items)
|
||||
rows.append(
|
||||
{
|
||||
"bucket": key,
|
||||
"samples": len(items),
|
||||
"avg_probability": round(avg_probability, 4),
|
||||
"actual_win_rate": round(wins / len(items), 4),
|
||||
"avg_future_net_percent": round(sum(item.future_net_percent for item in items) / len(items), 4),
|
||||
}
|
||||
)
|
||||
return {"samples": len(records), "buckets": rows}
|
||||
|
||||
|
||||
def _candidate_blocks(record: ForecastRecord, edge: float) -> bool:
|
||||
return (
|
||||
record.expected_percent <= -edge
|
||||
and record.probability_up <= 0.45
|
||||
) or (
|
||||
record.q50_percent <= -edge
|
||||
and record.probability_up <= 0.48
|
||||
)
|
||||
|
||||
|
||||
def _candidate_allows(record: ForecastRecord, edge: float, probability: float, confidence: float) -> bool:
|
||||
dynamic_confidence = _forecast_confidence(record.expected_percent, record.probability_up, record.skill, edge)
|
||||
return (
|
||||
not _candidate_blocks(record, edge)
|
||||
and record.expected_percent >= edge
|
||||
and record.probability_up >= probability
|
||||
and dynamic_confidence >= confidence
|
||||
and record.skill > 0.0
|
||||
)
|
||||
|
||||
|
||||
def _net_percent(entry_price: float, exit_price: float, round_trip_cost: float) -> float:
|
||||
if entry_price <= 0 or exit_price <= 0:
|
||||
return 0.0
|
||||
return (math.exp(math.log(exit_price / entry_price) - round_trip_cost) - 1.0) * 100.0
|
||||
|
||||
|
||||
def _stats(values: list[float]) -> dict[str, Any]:
|
||||
wins = sum(1 for value in values if value > 0)
|
||||
total = sum(values)
|
||||
gross_profit = sum(value for value in values if value > 0)
|
||||
gross_loss = abs(sum(value for value in values if value < 0))
|
||||
profit_factor = gross_profit / gross_loss if gross_loss > 0 else (999.0 if gross_profit > 0 else 0.0)
|
||||
return {
|
||||
"trades": len(values),
|
||||
"wins": wins,
|
||||
"win_rate": round(wins / len(values), 4) if values else 0.0,
|
||||
"total_net_percent": round(total, 4),
|
||||
"avg_net_percent": round(total / len(values), 4) if values else 0.0,
|
||||
"max_drawdown_percent": round(_max_drawdown(values), 4),
|
||||
"profit_factor": round(profit_factor, 4),
|
||||
}
|
||||
|
||||
|
||||
def _calibrate(
|
||||
records: list[ForecastRecord],
|
||||
*,
|
||||
@@ -551,26 +781,7 @@ def _selected_trades(
|
||||
for record in sorted(records, key=lambda item: (item.timestamp, item.symbol)):
|
||||
if record.index < next_allowed_by_symbol.get(record.symbol, -1):
|
||||
continue
|
||||
dynamic_confidence = _forecast_confidence(
|
||||
record.expected_percent,
|
||||
record.probability_up,
|
||||
record.skill,
|
||||
edge,
|
||||
)
|
||||
block_entry = (
|
||||
record.expected_percent <= -edge
|
||||
and record.probability_up <= 0.45
|
||||
) or (
|
||||
record.q50_percent <= -edge
|
||||
and record.probability_up <= 0.48
|
||||
)
|
||||
if (
|
||||
not block_entry
|
||||
and record.expected_percent >= edge
|
||||
and record.probability_up >= probability
|
||||
and dynamic_confidence >= confidence
|
||||
and record.skill > 0.0
|
||||
):
|
||||
if _candidate_allows(record, edge, probability, confidence):
|
||||
trades.append(record.future_net_percent)
|
||||
next_allowed_by_symbol[record.symbol] = record.index + max(1, horizon)
|
||||
return trades
|
||||
@@ -648,6 +859,14 @@ def _result_line(result: CalibrationResult) -> str:
|
||||
)
|
||||
|
||||
|
||||
def _stats_line(stats: dict[str, Any]) -> str:
|
||||
return (
|
||||
f"trades={stats.get('trades', 0)} win={stats.get('win_rate', 0):.3f} "
|
||||
f"avg={stats.get('avg_net_percent', 0):.4f}% total={stats.get('total_net_percent', 0):.4f}% "
|
||||
f"dd={stats.get('max_drawdown_percent', 0):.4f}% pf={stats.get('profit_factor', 0):.3f}"
|
||||
)
|
||||
|
||||
|
||||
def _result_dict(result: CalibrationResult) -> dict[str, Any]:
|
||||
return {
|
||||
"edge": result.edge,
|
||||
|
||||
@@ -14,7 +14,8 @@ param(
|
||||
[int]$Epochs = 0,
|
||||
[int]$Patience = 0,
|
||||
[string]$Interval = "",
|
||||
[string]$EnvFile = ""
|
||||
[string]$EnvFile = "",
|
||||
[switch]$SkipGuard
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
@@ -70,6 +71,13 @@ if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RE
|
||||
if (-not $EnvFile -and $env:TORCH_RETRAIN_ENV) { $EnvFile = $env:TORCH_RETRAIN_ENV }
|
||||
if (-not $EnvFile -and (Test-Path (Join-Path $RepoRoot ".env"))) { $EnvFile = Join-Path $RepoRoot ".env" }
|
||||
|
||||
$ModelFile = if ($env:TIME_SERIES_LSTM_MODEL_PATH) { $env:TIME_SERIES_LSTM_MODEL_PATH } else { Join-Path $RuntimeDir "lstm_forecaster.json" }
|
||||
if (-not [System.IO.Path]::IsPathRooted($ModelFile)) { $ModelFile = Join-Path $RepoRoot $ModelFile }
|
||||
$CandidateFile = Join-Path $RuntimeDir "lstm_forecaster.candidate.json"
|
||||
$CurrentCalibration = Join-Path $RuntimeDir "torch_guard_current.json"
|
||||
$CandidateCalibration = Join-Path $RuntimeDir "torch_guard_candidate.json"
|
||||
$GuardReport = Join-Path $RuntimeDir "torch_retrain_guard.json"
|
||||
|
||||
$mutex = New-Object System.Threading.Mutex($false, "TradeBotTorchRecurrentRetrainer")
|
||||
$hasLock = $false
|
||||
$pushedLocation = $false
|
||||
@@ -92,7 +100,8 @@ try {
|
||||
"--layers", $Layers,
|
||||
"--dropouts", $Dropouts,
|
||||
"--epochs", $Epochs.ToString(),
|
||||
"--patience", $Patience.ToString()
|
||||
"--patience", $Patience.ToString(),
|
||||
"--output", $CandidateFile
|
||||
)
|
||||
if ($Symbols) { $trainerArgs += @("--symbols", $Symbols) }
|
||||
if ($Interval) { $trainerArgs += @("--interval", $Interval) }
|
||||
@@ -109,7 +118,51 @@ try {
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Trainer failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
Write-RetrainLog "Finished PyTorch recurrent retrain."
|
||||
Write-RetrainLog "Finished PyTorch recurrent retrain candidate: $CandidateFile"
|
||||
|
||||
if ($SkipGuard -or -not (Test-Path $ModelFile)) {
|
||||
Move-Item -Force -LiteralPath $CandidateFile -Destination $ModelFile
|
||||
Write-RetrainLog "Accepted candidate without guard. Active artifact: $ModelFile"
|
||||
exit 0
|
||||
}
|
||||
|
||||
$calibrationBaseArgs = @(
|
||||
"-u",
|
||||
"tools\calibrate_torch_thresholds.py",
|
||||
"--limit", "2000",
|
||||
"--calibration-window", "720",
|
||||
"--min-trades", "12"
|
||||
)
|
||||
if ($Symbols) { $calibrationBaseArgs += @("--symbols", $Symbols) }
|
||||
if ($EnvFile) { $calibrationBaseArgs += @("--env", $EnvFile) }
|
||||
|
||||
Write-RetrainLog "Calibrating current artifact for guard."
|
||||
& $python @($calibrationBaseArgs + @("--artifact", $ModelFile, "--output", $CurrentCalibration)) 2>&1 | Tee-Object -FilePath $LogFile -Append
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Current artifact calibration failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
|
||||
Write-RetrainLog "Calibrating candidate artifact for guard."
|
||||
& $python @($calibrationBaseArgs + @("--artifact", $CandidateFile, "--output", $CandidateCalibration)) 2>&1 | Tee-Object -FilePath $LogFile -Append
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Candidate artifact calibration failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
|
||||
Write-RetrainLog "Running retrain guard."
|
||||
& $python -u "tools\accept_torch_candidate.py" `
|
||||
--current-report $CurrentCalibration `
|
||||
--candidate-report $CandidateCalibration `
|
||||
--candidate-artifact $CandidateFile `
|
||||
--target-artifact $ModelFile `
|
||||
--report $GuardReport 2>&1 | Tee-Object -FilePath $LogFile -Append
|
||||
if ($LASTEXITCODE -eq 2) {
|
||||
Write-RetrainLog "Candidate rejected by guard; keeping active artifact: $ModelFile"
|
||||
exit 0
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Retrain guard failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
Write-RetrainLog "Candidate accepted by guard. Active artifact: $ModelFile"
|
||||
}
|
||||
catch {
|
||||
Write-RetrainLog "ERROR: $($_.Exception.Message)"
|
||||
|
||||
Reference in New Issue
Block a user