sevenhill/TradeBot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Torch probe entries and Pi artifact sync

Browse Source
This commit is contained in:
Курнат Андрей
2026-06-24 21:31:05 +03:00
parent 15a50fb575
commit cb8efb7cd7
16 changed files with 307266 additions and 363669 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/install_windows_torch_retrainer.ps1
+21 -1
View File
@@ -8,7 +8,12 @@ param(
[string]$Horizons = "",
[string]$Features = "",
[string]$ContextSymbols = "",
[int]$FirstRunMinutes = 0
[int]$FirstRunMinutes = 0,
[switch]$DeployToPi,
[string]$PiHost = "192.168.0.185",
[string]$PiUser = "sevenhill",
[string]$PiRoot = "/mnt/data/tradebot",
[string]$PiSshKeyPath = ""
)
$ErrorActionPreference = "Stop"
@@ -46,6 +51,21 @@ if ($Features) {
if ($ContextSymbols) {
$actionArgs += " -ContextSymbols `"$ContextSymbols`""
}
if ($DeployToPi) {
$actionArgs += " -DeployToPi"
}
if ($PiHost) {
$actionArgs += " -PiHost `"$PiHost`""
}
if ($PiUser) {
$actionArgs += " -PiUser `"$PiUser`""
}
if ($PiRoot) {
$actionArgs += " -PiRoot `"$PiRoot`""
}
if ($PiSshKeyPath) {
$actionArgs += " -PiSshKeyPath `"$PiSshKeyPath`""
}
$action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $actionArgs -WorkingDirectory $RepoRoot
$trigger = New-ScheduledTaskTrigger `
-Once `
tools/run_retrain_until_replay8.ps1
+114
View File
@@ -0,0 +1,114 @@
[CmdletBinding()]
param(
[int]$MinReplayTrades = 8,
[int]$MaxAttempts = 0,
[string]$Symbols = "BTCUSDT,ETHUSDT,SOLUSDT,LTCUSDT",
[int]$Limit = 3000,
[switch]$DeployToPi,
[string]$PiHost = "192.168.0.185",
[string]$PiUser = "sevenhill",
[string]$PiRoot = "/mnt/data/tradebot",
[string]$PiSshKeyPath = "",
[int]$SeedStart = 0
)
$ErrorActionPreference = "Stop"
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
$RuntimeDir = Join-Path $RepoRoot "runtime"
$LoopLog = Join-Path $RuntimeDir "torch_retrain_until_replay8.log"
$GuardReport = Join-Path $RuntimeDir "torch_retrain_guard.json"
$Runner = Join-Path $RepoRoot "tools\run_torch_retrain.ps1"
New-Item -ItemType Directory -Force -Path $RuntimeDir | Out-Null
function Write-LoopLog {
param([string]$Message)
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ssK"
"[$timestamp] $Message" | Tee-Object -FilePath $LoopLog -Append
}
function ConvertTo-IntOrZero {
param($Value)
try {
if ($null -eq $Value) {
return 0
}
return [int]$Value
}
catch {
return 0
}
}
function Read-GuardSummary {
if (-not (Test-Path $GuardReport)) {
return [pscustomobject]@{
Accepted = $false
Reason = "guard_report_missing"
ReplayTrades = 0
WalkForwardTrades = 0
}
}
try {
$payload = Get-Content -Raw -LiteralPath $GuardReport | ConvertFrom-Json
return [pscustomobject]@{
Accepted = [bool]$payload.accepted
Reason = [string]$payload.reason
ReplayTrades = ConvertTo-IntOrZero $payload.candidate.full_replay.trades
WalkForwardTrades = ConvertTo-IntOrZero $payload.candidate.walk_forward_summary.trades
}
}
catch {
return [pscustomobject]@{
Accepted = $false
Reason = "guard_report_unreadable"
ReplayTrades = 0
WalkForwardTrades = 0
}
}
}
$attempt = 0
while ($true) {
$attempt += 1
if ($SeedStart -gt 0) {
$attemptSeed = $SeedStart + $attempt - 1
}
else {
$attemptSeed = Get-Random -Minimum 1 -Maximum 2147483647
}
Write-LoopLog "Attempt $attempt started; seed=$attemptSeed; target full_replay.trades >= $MinReplayTrades."
$runnerArgs = @(
"-NoProfile",
"-ExecutionPolicy", "Bypass",
"-File", $Runner,
"-Symbols", $Symbols,
"-Limit", $Limit.ToString(),
"-Seed", $attemptSeed.ToString()
)
if ($DeployToPi) {
$runnerArgs += "-DeployToPi"
if ($PiHost) { $runnerArgs += @("-PiHost", $PiHost) }
if ($PiUser) { $runnerArgs += @("-PiUser", $PiUser) }
if ($PiRoot) { $runnerArgs += @("-PiRoot", $PiRoot) }
if ($PiSshKeyPath) { $runnerArgs += @("-PiSshKeyPath", $PiSshKeyPath) }
}
& powershell.exe @runnerArgs 2>&1 | Tee-Object -FilePath $LoopLog -Append
$runnerExit = $LASTEXITCODE
$summary = Read-GuardSummary
Write-LoopLog "Attempt $attempt finished; runner_exit=$runnerExit accepted=$($summary.Accepted) reason=$($summary.Reason) full_replay.trades=$($summary.ReplayTrades) walk_forward.trades=$($summary.WalkForwardTrades)."
if ($summary.ReplayTrades -ge $MinReplayTrades) {
Write-LoopLog "Stop condition reached: full_replay.trades=$($summary.ReplayTrades) >= $MinReplayTrades."
exit 0
}
if ($MaxAttempts -gt 0 -and $attempt -ge $MaxAttempts) {
Write-LoopLog "MaxAttempts=$MaxAttempts reached before replay target."
exit 2
}
Start-Sleep -Seconds 10
}
tools/run_torch_retrain.ps1
+68 -2
View File
@@ -11,10 +11,17 @@ param(
[string]$Horizons = "",
[string]$Features = "",
[string]$ContextSymbols = "",
[int]$Seed = 0,
[int]$Epochs = 0,
[int]$Patience = 0,
[string]$Interval = "",
[string]$EnvFile = "",
[switch]$DeployToPi,
[string]$PiHost = "",
[string]$PiUser = "",
[string]$PiRoot = "",
[string]$PiSshKeyPath = "",
[switch]$NoPiRestart,
[switch]$SkipGuard
)
@@ -52,6 +59,51 @@ function Resolve-Python {
throw "Python was not found. Create .venv or install Python 3.12."
}
function Test-TorchArtifactFile {
param([string]$Path)
if (-not (Test-Path $Path)) {
return $false
}
try {
$payload = Get-Content -Raw -LiteralPath $Path | ConvertFrom-Json
return $payload.type -eq "pytorch_recurrent_forecaster" -and $null -ne $payload.symbols
}
catch {
return $false
}
}
function Sync-AcceptedArtifactsToPi {
if (-not ($DeployToPi -or $env:TORCH_RETRAIN_DEPLOY_TO_PI)) {
Write-RetrainLog "Pi artifact sync disabled."
return
}
$syncScript = Join-Path $RepoRoot "tools\sync_torch_artifacts_to_pi.ps1"
if (-not (Test-Path $syncScript)) {
throw "Pi sync script not found: $syncScript"
}
$syncArgs = @(
"-NoProfile",
"-ExecutionPolicy", "Bypass",
"-File", $syncScript,
"-RepoRoot", $RepoRoot
)
if ($PiHost) { $syncArgs += @("-RemoteHost", $PiHost) }
if ($PiUser) { $syncArgs += @("-RemoteUser", $PiUser) }
if ($PiRoot) { $syncArgs += @("-RemoteRoot", $PiRoot) }
if ($PiSshKeyPath) { $syncArgs += @("-SshKeyPath", $PiSshKeyPath) }
if ($NoPiRestart) { $syncArgs += "-NoRestart" }
Write-RetrainLog "Syncing accepted Torch artifacts to Raspberry Pi."
& powershell.exe @syncArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
if ($LASTEXITCODE -ne 0) {
throw "Pi artifact sync failed with exit code $LASTEXITCODE."
}
Write-RetrainLog "Pi artifact sync completed."
}
if (-not $Symbols -and $env:TORCH_RETRAIN_SYMBOLS) { $Symbols = $env:TORCH_RETRAIN_SYMBOLS }
if ($Limit -le 0) {
$Limit = if ($env:TORCH_RETRAIN_LIMIT) { [int]$env:TORCH_RETRAIN_LIMIT } else { 3000 }
@@ -65,6 +117,7 @@ if ($Horizon -le 0 -and $env:TORCH_RETRAIN_HORIZON) { $Horizon = [int]$env:TORCH
if (-not $Horizons -and $env:TORCH_RETRAIN_HORIZONS) { $Horizons = $env:TORCH_RETRAIN_HORIZONS }
if (-not $Features -and $env:TORCH_RETRAIN_FEATURES) { $Features = $env:TORCH_RETRAIN_FEATURES }
if (-not $ContextSymbols -and $env:TORCH_RETRAIN_CONTEXT_SYMBOLS) { $ContextSymbols = $env:TORCH_RETRAIN_CONTEXT_SYMBOLS }
if ($Seed -le 0 -and $env:TORCH_RETRAIN_SEED) { $Seed = [int]$env:TORCH_RETRAIN_SEED }
if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 70 } }
if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 8 } }
if (-not $Interval -and $env:TORCH_RETRAIN_INTERVAL) { $Interval = $env:TORCH_RETRAIN_INTERVAL }
@@ -110,19 +163,27 @@ try {
if ($Horizons) { $trainerArgs += @("--horizons", $Horizons) }
if ($Features) { $trainerArgs += @("--features", $Features) }
if ($ContextSymbols) { $trainerArgs += @("--context-symbols", $ContextSymbols) }
if ($Seed -gt 0) { $trainerArgs += @("--seed", $Seed.ToString()) }
Push-Location $RepoRoot
$pushedLocation = $true
Write-RetrainLog "Starting PyTorch recurrent retrain: $python $($trainerArgs -join ' ')"
& $python @trainerArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
if ($LASTEXITCODE -ne 0) {
throw "Trainer failed with exit code $LASTEXITCODE."
$trainerExitCode = $LASTEXITCODE
if ($trainerExitCode -ne 0) {
if (Test-TorchArtifactFile $CandidateFile) {
Write-RetrainLog "WARNING: Trainer exited with code $trainerExitCode after writing a valid candidate artifact; continuing to guard."
}
else {
throw "Trainer failed with exit code $trainerExitCode."
}
}
Write-RetrainLog "Finished PyTorch recurrent retrain candidate: $CandidateFile"
if ($SkipGuard -or -not (Test-Path $ModelFile)) {
Move-Item -Force -LiteralPath $CandidateFile -Destination $ModelFile
Write-RetrainLog "Accepted candidate without guard. Active artifact: $ModelFile"
Sync-AcceptedArtifactsToPi
exit 0
}
@@ -162,7 +223,12 @@ try {
if ($LASTEXITCODE -ne 0) {
throw "Retrain guard failed with exit code $LASTEXITCODE."
}
if (Test-Path $CandidateCalibration) {
Copy-Item -Force -LiteralPath $CandidateCalibration -Destination (Join-Path $RuntimeDir "torch_threshold_calibration.json")
Write-RetrainLog "Updated active threshold calibration: $(Join-Path $RuntimeDir "torch_threshold_calibration.json")"
}
Write-RetrainLog "Candidate accepted by guard. Active artifact: $ModelFile"
Sync-AcceptedArtifactsToPi
}
catch {
Write-RetrainLog "ERROR: $($_.Exception.Message)"
tools/sync_torch_artifacts_to_pi.ps1
+95
View File
@@ -0,0 +1,95 @@
[CmdletBinding()]
param(
[string]$RepoRoot = "",
[string]$RemoteHost = "",
[string]$RemoteUser = "",
[string]$RemoteRoot = "",
[string]$SshKeyPath = "",
[string]$ServiceName = "tradebot",
[switch]$NoRestart,
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
if (-not $RepoRoot) { $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path }
if (-not $RemoteHost -and $env:TORCH_DEPLOY_PI_HOST) { $RemoteHost = $env:TORCH_DEPLOY_PI_HOST }
if (-not $RemoteUser -and $env:TORCH_DEPLOY_PI_USER) { $RemoteUser = $env:TORCH_DEPLOY_PI_USER }
if (-not $RemoteRoot -and $env:TORCH_DEPLOY_PI_ROOT) { $RemoteRoot = $env:TORCH_DEPLOY_PI_ROOT }
if (-not $SshKeyPath -and $env:TORCH_DEPLOY_PI_SSH_KEY) { $SshKeyPath = $env:TORCH_DEPLOY_PI_SSH_KEY }
if (-not $RemoteHost) { $RemoteHost = "192.168.0.185" }
if (-not $RemoteUser) { $RemoteUser = "sevenhill" }
if (-not $RemoteRoot) { $RemoteRoot = "/mnt/data/tradebot" }
$RuntimeDir = Join-Path $RepoRoot "runtime"
$artifactNames = @(
"lstm_forecaster.json",
"torch_retrain_guard.json",
"torch_threshold_calibration.json"
)
$localFiles = @()
foreach ($name in $artifactNames) {
$path = Join-Path $RuntimeDir $name
if (Test-Path $path) {
$localFiles += (Resolve-Path $path).Path
}
}
if ($localFiles.Count -eq 0) {
throw "No Torch artifacts found in $RuntimeDir."
}
function ConvertTo-RemoteSingleQuoted {
param([string]$Value)
return "'" + ($Value -replace "'", "'\''") + "'"
}
function Invoke-LoggedCommand {
param(
[string]$Exe,
[string[]]$Arguments
)
$rendered = @($Exe) + $Arguments
Write-Host ($rendered -join " ")
if ($DryRun) {
return
}
& $Exe @Arguments
if ($LASTEXITCODE -ne 0) {
throw "$Exe failed with exit code $LASTEXITCODE."
}
}
$ssh = (Get-Command "ssh.exe" -ErrorAction SilentlyContinue)
if (-not $ssh) { $ssh = Get-Command "ssh" -ErrorAction Stop }
$scp = (Get-Command "scp.exe" -ErrorAction SilentlyContinue)
if (-not $scp) { $scp = Get-Command "scp" -ErrorAction Stop }
$commonSshArgs = @("-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=15")
if ($SshKeyPath) {
$expandedKey = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($SshKeyPath)
$commonSshArgs += @("-i", $expandedKey)
}
$remote = "${RemoteUser}@${RemoteHost}"
$remoteRuntime = "$RemoteRoot/runtime"
$remoteIncoming = "$remoteRuntime/.incoming-torch"
$mkdirCommand = "mkdir -p $(ConvertTo-RemoteSingleQuoted $remoteIncoming) $(ConvertTo-RemoteSingleQuoted $remoteRuntime)"
Invoke-LoggedCommand $ssh.Source (@($commonSshArgs + @($remote, $mkdirCommand)))
$destination = "${remote}:$remoteIncoming/"
Invoke-LoggedCommand $scp.Source (@($commonSshArgs + $localFiles + @($destination)))
$moveParts = @()
foreach ($path in $localFiles) {
$name = Split-Path $path -Leaf
$moveParts += "mv -f $(ConvertTo-RemoteSingleQuoted "$remoteIncoming/$name") $(ConvertTo-RemoteSingleQuoted "$remoteRuntime/$name")"
}
$moveCommand = $moveParts -join " && "
Invoke-LoggedCommand $ssh.Source (@($commonSshArgs + @($remote, $moveCommand)))
if (-not $NoRestart) {
$restartCommand = "cd $(ConvertTo-RemoteSingleQuoted $RemoteRoot) && docker compose restart $(ConvertTo-RemoteSingleQuoted $ServiceName)"
Invoke-LoggedCommand $ssh.Source (@($commonSshArgs + @($remote, $restartCommand)))
}
Write-Host "Synced Torch artifacts to ${remote}:$remoteRuntime"