sevenhill/TradeBot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: production paper trading platform

Browse Source
This commit is contained in:
Курнат Андрей
2026-07-14 22:52:47 +03:00
parent 7186acb9a1
commit 5c4aecfe5f
29 changed files with 396 additions and 564 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/install_windows_torch_retrainer.ps1
-94
View File
@@ -1,94 +0,0 @@
[CmdletBinding()]
param(
[string]$TaskName = "TradeBot PyTorch Forecaster Retrainer",
[int]$EveryHours = 6,
[string]$Symbols = "",
[int]$Limit = 3000,
[int]$Horizon = 0,
[string]$Horizons = "",
[string]$Features = "",
[string]$ContextSymbols = "",
[int]$FirstRunMinutes = 0,
[switch]$DeployToPi,
[string]$PiHost = "192.168.0.185",
[string]$PiUser = "sevenhill",
[string]$PiRoot = "/mnt/data/tradebot",
[string]$PiSshKeyPath = ""
)
$ErrorActionPreference = "Stop"
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
$Runner = Join-Path $RepoRoot "tools\run_torch_retrain.ps1"
if (-not (Test-Path $Runner)) {
throw "Runner not found: $Runner"
}
$LegacyTaskName = "TradeBot LSTM Retrainer"
if ($TaskName -ne $LegacyTaskName) {
$legacyTask = Get-ScheduledTask -TaskName $LegacyTaskName -ErrorAction SilentlyContinue
if ($legacyTask) {
Unregister-ScheduledTask -TaskName $LegacyTaskName -Confirm:$false
}
}
$actionArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$Runner`""
if ($Symbols) {
$actionArgs += " -Symbols `"$Symbols`""
}
if ($Limit -gt 0) {
$actionArgs += " -Limit $Limit"
}
if ($Horizon -gt 0) {
$actionArgs += " -Horizon $Horizon"
}
if ($Horizons) {
$actionArgs += " -Horizons `"$Horizons`""
}
if ($Features) {
$actionArgs += " -Features `"$Features`""
}
if ($ContextSymbols) {
$actionArgs += " -ContextSymbols `"$ContextSymbols`""
}
if ($DeployToPi) {
$actionArgs += " -DeployToPi"
}
if ($PiHost) {
$actionArgs += " -PiHost `"$PiHost`""
}
if ($PiUser) {
$actionArgs += " -PiUser `"$PiUser`""
}
if ($PiRoot) {
$actionArgs += " -PiRoot `"$PiRoot`""
}
if ($PiSshKeyPath) {
$actionArgs += " -PiSshKeyPath `"$PiSshKeyPath`""
}
$action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $actionArgs -WorkingDirectory $RepoRoot
$trigger = New-ScheduledTaskTrigger `
-Once `
-At (Get-Date).AddMinutes($(if ($FirstRunMinutes -gt 0) { $FirstRunMinutes } else { $EveryHours * 60 })) `
-RepetitionInterval (New-TimeSpan -Hours $EveryHours) `
-RepetitionDuration (New-TimeSpan -Days 3650)
$principal = New-ScheduledTaskPrincipal `
-UserId ([System.Security.Principal.WindowsIdentity]::GetCurrent().Name) `
-LogonType Interactive `
-RunLevel Limited
$settings = New-ScheduledTaskSettingsSet `
-StartWhenAvailable `
-MultipleInstances IgnoreNew `
-AllowStartIfOnBatteries `
-DontStopIfGoingOnBatteries
Register-ScheduledTask `
-TaskName $TaskName `
-Action $action `
-Trigger $trigger `
-Principal $principal `
-Settings $settings `
-Description "Retrains TradeBot PyTorch recurrent forecast parameters every $EveryHours hours." `
-Force | Out-Null
Write-Host "Registered scheduled task '$TaskName' every $EveryHours hours."
tools/run_retrain_until_replay8.ps1
-152
View File
@@ -1,152 +0,0 @@
[CmdletBinding()]
param(
[int]$MinReplayTrades = 8,
[int]$MaxAttempts = 0,
[string]$Symbols = "",
[int]$Limit = 3000,
[switch]$DeployToPi,
[string]$PiHost = "192.168.0.185",
[string]$PiUser = "sevenhill",
[string]$PiRoot = "/mnt/data/tradebot",
[string]$PiSshKeyPath = "",
[int]$SeedStart = 0
)
$ErrorActionPreference = "Stop"
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
$RuntimeDir = Join-Path $RepoRoot "runtime"
$LoopLog = Join-Path $RuntimeDir "torch_retrain_until_replay8.log"
$GuardReport = Join-Path $RuntimeDir "torch_retrain_guard.json"
$ActiveCalibration = Join-Path $RuntimeDir "torch_threshold_calibration.json"
$Runner = Join-Path $RepoRoot "tools\run_torch_retrain.ps1"
New-Item -ItemType Directory -Force -Path $RuntimeDir | Out-Null
function Write-LoopLog {
param([string]$Message)
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ssK"
"[$timestamp] $Message" | Tee-Object -FilePath $LoopLog -Append
}
function ConvertTo-IntOrZero {
param($Value)
try {
if ($null -eq $Value) {
return 0
}
return [int]$Value
}
catch {
return 0
}
}
function Read-GuardSummary {
if (-not (Test-Path $GuardReport)) {
return [pscustomobject]@{
Accepted = $false
Reason = "guard_report_missing"
CandidateReplayTrades = 0
CurrentReplayTrades = 0
WalkForwardTrades = 0
}
}
try {
$payload = Get-Content -Raw -LiteralPath $GuardReport | ConvertFrom-Json
return [pscustomobject]@{
Accepted = [bool]$payload.accepted
Reason = [string]$payload.reason
CandidateReplayTrades = ConvertTo-IntOrZero $payload.candidate.full_replay.trades
CurrentReplayTrades = ConvertTo-IntOrZero $payload.current.full_replay.trades
WalkForwardTrades = ConvertTo-IntOrZero $payload.candidate.walk_forward_summary.trades
}
}
catch {
return [pscustomobject]@{
Accepted = $false
Reason = "guard_report_unreadable"
CandidateReplayTrades = 0
CurrentReplayTrades = 0
WalkForwardTrades = 0
}
}
}
function Read-ActiveReplayTrades {
if (-not (Test-Path $ActiveCalibration)) {
return 0
}
try {
$payload = Get-Content -Raw -LiteralPath $ActiveCalibration | ConvertFrom-Json
return ConvertTo-IntOrZero $payload.full_replay.trades
}
catch {
return 0
}
}
function Read-ActiveValidationPassed {
if (-not (Test-Path $ActiveCalibration)) {
return $false
}
try {
$payload = Get-Content -Raw -LiteralPath $ActiveCalibration | ConvertFrom-Json
return [bool]$payload.validation.passed
}
catch {
return $false
}
}
$attempt = 0
while ($true) {
$activeReplayTrades = Read-ActiveReplayTrades
if (Read-ActiveValidationPassed) {
Write-LoopLog "Stop condition reached: active calibration passed honest validation with full_replay.trades=$activeReplayTrades."
exit 0
}
$attempt += 1
if ($SeedStart -gt 0) {
$attemptSeed = $SeedStart + $attempt - 1
}
else {
$attemptSeed = Get-Random -Minimum 1 -Maximum 2147483647
}
Write-LoopLog "Attempt $attempt started; seed=$attemptSeed; target full_replay.trades >= $MinReplayTrades."
$runnerArgs = @(
"-NoProfile",
"-ExecutionPolicy", "Bypass",
"-File", $Runner,
"-Limit", $Limit.ToString(),
"-Seed", $attemptSeed.ToString()
)
if ($Symbols) {
$runnerArgs += @("-Symbols", $Symbols)
}
if ($DeployToPi) {
$runnerArgs += "-DeployToPi"
if ($PiHost) { $runnerArgs += @("-PiHost", $PiHost) }
if ($PiUser) { $runnerArgs += @("-PiUser", $PiUser) }
if ($PiRoot) { $runnerArgs += @("-PiRoot", $PiRoot) }
if ($PiSshKeyPath) { $runnerArgs += @("-PiSshKeyPath", $PiSshKeyPath) }
}
& powershell.exe @runnerArgs 2>&1 | Tee-Object -FilePath $LoopLog -Append
$runnerExit = $LASTEXITCODE
$summary = Read-GuardSummary
Write-LoopLog "Attempt $attempt finished; runner_exit=$runnerExit accepted=$($summary.Accepted) reason=$($summary.Reason) candidate_full_replay.trades=$($summary.CandidateReplayTrades) current_full_replay.trades=$($summary.CurrentReplayTrades) walk_forward.trades=$($summary.WalkForwardTrades)."
if ($summary.Accepted -and (Read-ActiveValidationPassed)) {
Write-LoopLog "Stop condition reached: accepted candidate passed honest validation with full_replay.trades=$($summary.CandidateReplayTrades)."
exit 0
}
if ($MaxAttempts -gt 0 -and $attempt -ge $MaxAttempts) {
Write-LoopLog "MaxAttempts=$MaxAttempts reached before replay target."
exit 2
}
Start-Sleep -Seconds 10
}
tools/run_torch_retrain.ps1
+5 -43
View File
@@ -22,12 +22,6 @@ param(
[int]$HoldoutWindow = 0,
[string]$Interval = "",
[string]$EnvFile = "",
[switch]$DeployToPi,
[string]$PiHost = "",
[string]$PiUser = "",
[string]$PiRoot = "",
[string]$PiSshKeyPath = "",
[switch]$NoPiRestart,
[switch]$Pooled,
[switch]$SkipGuard,
[switch]$ResumeCandidate
@@ -105,44 +99,13 @@ function Test-TorchArtifactFile {
}
}
function Sync-AcceptedArtifactsToPi {
if (-not ($DeployToPi -or $env:TORCH_RETRAIN_DEPLOY_TO_PI)) {
Write-RetrainLog "Pi artifact sync disabled."
return
}
$syncScript = Join-Path $RepoRoot "tools\sync_torch_artifacts_to_pi.ps1"
if (-not (Test-Path $syncScript)) {
throw "Pi sync script not found: $syncScript"
}
$syncArgs = @(
"-NoProfile",
"-ExecutionPolicy", "Bypass",
"-File", $syncScript,
"-RepoRoot", $RepoRoot
)
if ($PiHost) { $syncArgs += @("-RemoteHost", $PiHost) }
if ($PiUser) { $syncArgs += @("-RemoteUser", $PiUser) }
if ($PiRoot) { $syncArgs += @("-RemoteRoot", $PiRoot) }
if ($PiSshKeyPath) { $syncArgs += @("-SshKeyPath", $PiSshKeyPath) }
if ($NoPiRestart) { $syncArgs += "-NoRestart" }
Write-RetrainLog "Syncing accepted Torch artifacts to Raspberry Pi."
& powershell.exe @syncArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
if ($LASTEXITCODE -ne 0) {
throw "Pi artifact sync failed with exit code $LASTEXITCODE."
}
Write-RetrainLog "Pi artifact sync completed."
}
if (-not $Symbols -and $env:TORCH_RETRAIN_SYMBOLS) { $Symbols = $env:TORCH_RETRAIN_SYMBOLS }
if ($Limit -le 0) {
$Limit = if ($env:TORCH_RETRAIN_LIMIT) { [int]$env:TORCH_RETRAIN_LIMIT } else { 6000 }
$Limit = if ($env:TORCH_RETRAIN_LIMIT) { [int]$env:TORCH_RETRAIN_LIMIT } else { 4000 }
}
if (-not $Lookbacks) { $Lookbacks = if ($env:TORCH_RETRAIN_LOOKBACKS) { $env:TORCH_RETRAIN_LOOKBACKS } else { "32,64,128" } }
if (-not $Architectures) { $Architectures = if ($env:TORCH_RETRAIN_ARCHITECTURES) { $env:TORCH_RETRAIN_ARCHITECTURES } else { "lstm,gru" } }
if (-not $HiddenSizes) { $HiddenSizes = if ($env:TORCH_RETRAIN_HIDDEN_SIZES) { $env:TORCH_RETRAIN_HIDDEN_SIZES } else { "64,96" } }
if (-not $Lookbacks) { $Lookbacks = if ($env:TORCH_RETRAIN_LOOKBACKS) { $env:TORCH_RETRAIN_LOOKBACKS } else { "64" } }
if (-not $Architectures) { $Architectures = if ($env:TORCH_RETRAIN_ARCHITECTURES) { $env:TORCH_RETRAIN_ARCHITECTURES } else { "lstm" } }
if (-not $HiddenSizes) { $HiddenSizes = if ($env:TORCH_RETRAIN_HIDDEN_SIZES) { $env:TORCH_RETRAIN_HIDDEN_SIZES } else { "64" } }
if (-not $Layers) { $Layers = if ($env:TORCH_RETRAIN_LAYERS) { $env:TORCH_RETRAIN_LAYERS } else { "2" } }
if (-not $Dropouts) { $Dropouts = if ($env:TORCH_RETRAIN_DROPOUTS) { $env:TORCH_RETRAIN_DROPOUTS } else { "0.20" } }
if ($Horizon -le 0) { $Horizon = if ($env:TORCH_RETRAIN_HORIZON) { [int]$env:TORCH_RETRAIN_HORIZON } else { 12 } }
@@ -154,7 +117,7 @@ if (-not $EnsembleSeeds) { $EnsembleSeeds = if ($env:TORCH_RETRAIN_ENSEMBLE_SEED
if ($SelectionFolds -le 0) { $SelectionFolds = if ($env:TORCH_RETRAIN_SELECTION_FOLDS) { [int]$env:TORCH_RETRAIN_SELECTION_FOLDS } else { 3 } }
if ($LearningRate -le 0) { $LearningRate = if ($env:TORCH_RETRAIN_LEARNING_RATE) { [double]$env:TORCH_RETRAIN_LEARNING_RATE } else { 0.0007 } }
if ($WeightDecay -le 0) { $WeightDecay = if ($env:TORCH_RETRAIN_WEIGHT_DECAY) { [double]$env:TORCH_RETRAIN_WEIGHT_DECAY } else { 0.0005 } }
if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 70 } }
if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 50 } }
if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 8 } }
if ($ValidationWindow -le 0) { $ValidationWindow = if ($env:TORCH_RETRAIN_VALIDATION_WINDOW) { [int]$env:TORCH_RETRAIN_VALIDATION_WINDOW } else { 720 } }
if ($HoldoutWindow -le 0) { $HoldoutWindow = if ($env:TORCH_RETRAIN_HOLDOUT_WINDOW) { [int]$env:TORCH_RETRAIN_HOLDOUT_WINDOW } else { 1000 } }
@@ -302,7 +265,6 @@ try {
Write-RetrainLog "Updated active threshold calibration: $(Join-Path $RuntimeDir "torch_threshold_calibration.json")"
}
Write-RetrainLog "Candidate accepted by guard. Active artifact: $ModelFile"
Sync-AcceptedArtifactsToPi
}
catch {
Write-RetrainLog "ERROR: $($_.Exception.Message)"
tools/sync_torch_artifacts_to_pi.ps1
-95
View File
@@ -1,95 +0,0 @@
[CmdletBinding()]
param(
[string]$RepoRoot = "",
[string]$RemoteHost = "",
[string]$RemoteUser = "",
[string]$RemoteRoot = "",
[string]$SshKeyPath = "",
[string]$ServiceName = "tradebot",
[switch]$NoRestart,
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
if (-not $RepoRoot) { $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path }
if (-not $RemoteHost -and $env:TORCH_DEPLOY_PI_HOST) { $RemoteHost = $env:TORCH_DEPLOY_PI_HOST }
if (-not $RemoteUser -and $env:TORCH_DEPLOY_PI_USER) { $RemoteUser = $env:TORCH_DEPLOY_PI_USER }
if (-not $RemoteRoot -and $env:TORCH_DEPLOY_PI_ROOT) { $RemoteRoot = $env:TORCH_DEPLOY_PI_ROOT }
if (-not $SshKeyPath -and $env:TORCH_DEPLOY_PI_SSH_KEY) { $SshKeyPath = $env:TORCH_DEPLOY_PI_SSH_KEY }
if (-not $RemoteHost) { $RemoteHost = "192.168.0.185" }
if (-not $RemoteUser) { $RemoteUser = "sevenhill" }
if (-not $RemoteRoot) { $RemoteRoot = "/mnt/data/tradebot" }
$RuntimeDir = Join-Path $RepoRoot "runtime"
$artifactNames = @(
"lstm_forecaster.json",
"torch_retrain_guard.json",
"torch_threshold_calibration.json"
)
$localFiles = @()
foreach ($name in $artifactNames) {
$path = Join-Path $RuntimeDir $name
if (Test-Path $path) {
$localFiles += (Resolve-Path $path).Path
}
}
if ($localFiles.Count -eq 0) {
throw "No Torch artifacts found in $RuntimeDir."
}
function ConvertTo-RemoteSingleQuoted {
param([string]$Value)
return "'" + ($Value -replace "'", "'\''") + "'"
}
function Invoke-LoggedCommand {
param(
[string]$Exe,
[string[]]$Arguments
)
$rendered = @($Exe) + $Arguments
Write-Host ($rendered -join " ")
if ($DryRun) {
return
}
& $Exe @Arguments
if ($LASTEXITCODE -ne 0) {
throw "$Exe failed with exit code $LASTEXITCODE."
}
}
$ssh = (Get-Command "ssh.exe" -ErrorAction SilentlyContinue)
if (-not $ssh) { $ssh = Get-Command "ssh" -ErrorAction Stop }
$scp = (Get-Command "scp.exe" -ErrorAction SilentlyContinue)
if (-not $scp) { $scp = Get-Command "scp" -ErrorAction Stop }
$commonSshArgs = @("-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=15")
if ($SshKeyPath) {
$expandedKey = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($SshKeyPath)
$commonSshArgs += @("-i", $expandedKey)
}
$remote = "${RemoteUser}@${RemoteHost}"
$remoteRuntime = "$RemoteRoot/runtime"
$remoteIncoming = "$remoteRuntime/.incoming-torch"
$mkdirCommand = "mkdir -p $(ConvertTo-RemoteSingleQuoted $remoteIncoming) $(ConvertTo-RemoteSingleQuoted $remoteRuntime)"
Invoke-LoggedCommand $ssh.Source (@($commonSshArgs + @($remote, $mkdirCommand)))
$destination = "${remote}:$remoteIncoming/"
Invoke-LoggedCommand $scp.Source (@($commonSshArgs + $localFiles + @($destination)))
$moveParts = @()
foreach ($path in $localFiles) {
$name = Split-Path $path -Leaf
$moveParts += "mv -f $(ConvertTo-RemoteSingleQuoted "$remoteIncoming/$name") $(ConvertTo-RemoteSingleQuoted "$remoteRuntime/$name")"
}
$moveCommand = $moveParts -join " && "
Invoke-LoggedCommand $ssh.Source (@($commonSshArgs + @($remote, $moveCommand)))
if (-not $NoRestart) {
$restartCommand = "cd $(ConvertTo-RemoteSingleQuoted $RemoteRoot) && docker compose restart $(ConvertTo-RemoteSingleQuoted $ServiceName)"
Invoke-LoggedCommand $ssh.Source (@($commonSshArgs + @($remote, $restartCommand)))
}
Write-Host "Synced Torch artifacts to ${remote}:$remoteRuntime"
tools/windows_training_agent.py
+88 -14
View File
@@ -54,23 +54,34 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
return
job = claim.get("job") if isinstance(claim.get("job"), dict) else {}
job_id = str(job.get("id") or "")
lease_token = str(claim.get("lease_token") or "")
if not job_id:
return
if not lease_token:
raise RuntimeError("training server did not issue a job lease")
log(log_path, f"Claimed retrain job {job_id}")
report_progress(args, job_id, "running", "claimed", 2, "Задание получено Windows-agent")
report_progress(args, job_id, lease_token, "running", "claimed", 2, "Задание получено Windows-agent")
success = False
message = ""
summary: dict[str, Any] = {}
try:
run_retrain(args, job_id, job, repo_root, log_path)
run_retrain(args, job_id, lease_token, job, repo_root, log_path)
summary = read_json(runtime_dir / "torch_retrain_guard.json")
accepted = summary.get("accepted") is True
if accepted:
report_progress(args, job_id, "running", "uploading", 72, "Обучение завершено, загружаю артефакты")
report_progress(
args,
job_id,
lease_token,
"running",
"uploading",
72,
"Обучение завершено, загружаю артефакты",
)
for name in ARTIFACT_NAMES:
path = runtime_dir / name
if path.is_file():
upload_artifact(args, job_id, path, log_path)
upload_artifact(args, job_id, lease_token, path, log_path)
message = "training completed; candidate accepted"
log(log_path, f"Completed retrain job {job_id}; candidate accepted")
else:
@@ -82,11 +93,23 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
message = str(exc)
log(log_path, f"Job {job_id} failed: {message}")
finally:
payload = {"success": success, "message": message, "summary": summary}
payload = {
"success": success,
"message": message,
"summary": summary,
"lease_token": lease_token,
}
api_json(args, f"/api/training/jobs/{job_id}/complete", payload)
def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo_root: Path, log_path: Path) -> None:
def run_retrain(
args: argparse.Namespace,
job_id: str,
lease_token: str,
job: dict[str, Any],
repo_root: Path,
log_path: Path,
) -> None:
script = repo_root / "tools" / "run_torch_retrain.ps1"
if not script.is_file():
raise RuntimeError(f"retrain script not found: {script}")
@@ -126,12 +149,20 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
value = parameters.get(key)
if value not in (None, ""):
cmd.extend([ps_arg, str(value)])
if parameters.get("pooled") is True:
if parameters.get("pooled", True) is True:
cmd.append("-Pooled")
if parameters.get("resume_candidate") is True:
cmd.append("-ResumeCandidate")
log(log_path, "Running retrain: " + " ".join(quote_for_log(part) for part in cmd))
report_progress(args, job_id, "running", "training", 8, "PyTorch retrain запущен")
report_progress(
args,
job_id,
lease_token,
"running",
"training",
8,
"PyTorch retrain запущен",
)
line_count = 0
output_queue: queue.Queue[str] = queue.Queue()
@@ -175,7 +206,16 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
report_message = last_message
if not got_line:
report_message = training_heartbeat_message(now, started_at, last_output_at, last_message)
safe_report_progress(args, job_id, "running", "training", progress, report_message, log_path)
safe_report_progress(
args,
job_id,
lease_token,
"running",
"training",
progress,
report_message,
log_path,
)
last_report_at = now
if process.poll() is not None and output_queue.empty():
@@ -185,7 +225,15 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
code = process.wait()
if code != 0:
raise RuntimeError(f"retrain failed with exit code {code}")
report_progress(args, job_id, "running", "guard", 70, "Guard завершён, подготавливаю артефакты")
report_progress(
args,
job_id,
lease_token,
"running",
"guard",
70,
"Guard завершён, подготавливаю артефакты",
)
def friendly_training_message(message: str) -> str:
@@ -282,7 +330,13 @@ def format_duration(seconds: float) -> str:
return f"{seconds_part}с"
def upload_artifact(args: argparse.Namespace, job_id: str, path: Path, log_path: Path) -> None:
def upload_artifact(
args: argparse.Namespace,
job_id: str,
lease_token: str,
path: Path,
log_path: Path,
) -> None:
digest = hashlib.sha256(path.read_bytes()).hexdigest()
size = path.stat().st_size
chunk_size = max(64 * 1024, args.chunk_size)
@@ -297,16 +351,26 @@ def upload_artifact(args: argparse.Namespace, job_id: str, path: Path, log_path:
"total": total,
"sha256": digest,
"data_base64": base64.b64encode(data).decode("ascii"),
"lease_token": lease_token,
}
api_json(args, f"/api/training/jobs/{job_id}/artifacts/chunk", payload, timeout=120)
if index == 0 or index == total - 1 or index % 10 == 0:
progress = 72 + int(((index + 1) / total) * 23)
report_progress(args, job_id, "running", "uploading", progress, f"Загружаю {path.name}: {index + 1}/{total}")
report_progress(
args,
job_id,
lease_token,
"running",
"uploading",
progress,
f"Загружаю {path.name}: {index + 1}/{total}",
)
def report_progress(
args: argparse.Namespace,
job_id: str,
lease_token: str,
status: str,
phase: str,
progress_percent: int,
@@ -321,6 +385,7 @@ def report_progress(
"progress_percent": progress_percent,
"message": message,
"worker": worker_payload(args, Path(args.repo_root).resolve()),
"lease_token": lease_token,
},
)
@@ -328,6 +393,7 @@ def report_progress(
def safe_report_progress(
args: argparse.Namespace,
job_id: str,
lease_token: str,
status: str,
phase: str,
progress_percent: int,
@@ -337,7 +403,15 @@ def safe_report_progress(
last_error: Exception | None = None
for attempt in range(1, 4):
try:
report_progress(args, job_id, status, phase, progress_percent, message)
report_progress(
args,
job_id,
lease_token,
status,
phase,
progress_percent,
message,
)
return
except Exception as exc: # noqa: BLE001 - keep the local training process alive.
last_error = exc
@@ -385,7 +459,7 @@ def worker_payload(args: argparse.Namespace, repo_root: Path) -> dict[str, Any]:
"worker_id": args.worker_id or f"{name}:{repo_root}",
"name": name,
"path": str(repo_root),
"version": "1",
"version": "2",
}