[CmdletBinding()] param( [int[]] $Milestones = @(100, 300, 500), [int] $PollSeconds = 20 ) $ErrorActionPreference = 'Stop' $root = $PSScriptRoot $trainingDir = Join-Path $root 'training' $checkpointRoot = Join-Path $trainingDir 'checkpoints' $configSource = Join-Path $trainingDir 'aletheia_ru.onnx.json' $python = Join-Path $root '.venv\Scripts\python.exe' $outputRoot = Join-Path $trainingDir 'milestones' $monitorLog = Join-Path $outputRoot 'monitor.log' New-Item -ItemType Directory -Force -Path $outputRoot | Out-Null function Write-MonitorLog([string] $Message) { $line = '{0} {1}' -f (Get-Date).ToUniversalTime().ToString('o'), $Message Add-Content -LiteralPath $monitorLog -Value $line -Encoding utf8 } function Invoke-LoggedProcess( [string] $FilePath, [string[]] $Arguments, [string] $StdoutPath, [string] $StderrPath ) { $process = Start-Process -FilePath $FilePath ` -ArgumentList $Arguments ` -RedirectStandardOutput $StdoutPath ` -RedirectStandardError $StderrPath ` -WindowStyle Hidden ` -Wait ` -PassThru if ($process.ExitCode -ne 0) { throw "Process failed with exit code $($process.ExitCode): $FilePath" } } function Get-LatestCheckpoint { Get-ChildItem -LiteralPath $checkpointRoot -Filter '*.ckpt' -File -Recurse -ErrorAction SilentlyContinue | Where-Object { $_.Name -match '^epoch=(\d+)-step=(\d+)\.ckpt$' } | Sort-Object LastWriteTimeUtc -Descending | Select-Object -First 1 } function Show-TrainingStatus { $trainingLog = Join-Path $trainingDir 'scheduled-training.stdout.log' $trainingTask = Get-ScheduledTask -TaskName 'AletheiaTTS-Training' -ErrorAction SilentlyContinue $progressLine = '' if (Test-Path -LiteralPath $trainingLog -PathType Leaf) { $progressLine = Get-Content -LiteralPath $trainingLog -Tail 30 | Where-Object { $_ -match '^Epoch\s+\d+:' } | Select-Object -Last 1 } $checkpoint = Get-LatestCheckpoint $gpu = (& nvidia-smi --query-gpu=name,utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw --format=csv,noheader,nounits 2>$null) -join [Environment]::NewLine try { $Host.UI.RawUI.WindowTitle = 'Aletheia TTS - training monitor' Clear-Host } catch { # The monitor also works when no interactive console is attached. } Write-Host 'Aletheia Russian TTS training' Write-Host ('Updated: {0}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss')) Write-Host ('Training task: {0}' -f $(if ($trainingTask) { $trainingTask.State } else { 'Not found' })) Write-Host ('Progress: {0}' -f $(if ($progressLine) { $progressLine.Trim() } else { 'Waiting for log data' })) Write-Host ('GPU: {0}' -f $(if ($gpu) { $gpu.Trim() } else { 'nvidia-smi unavailable' })) if ($checkpoint) { Write-Host ('Latest checkpoint: {0} ({1:N0} bytes)' -f $checkpoint.Name, $checkpoint.Length) } else { Write-Host 'Latest checkpoint: waiting' } Write-Host '' Write-Host 'Evaluation milestones:' foreach ($milestone in $Milestones) { $completePath = Join-Path $outputRoot ('epoch-{0:d4}\complete.json' -f $milestone) $state = if (Test-Path -LiteralPath $completePath -PathType Leaf) { 'READY' } else { 'waiting' } Write-Host (' {0,4} epochs: {1}' -f $milestone, $state) } Write-Host '' Write-Host 'This window may be minimized. Closing it stops milestone exports only.' } function Save-Milestone([int] $Milestone, [IO.FileInfo] $SourceCheckpoint) { $milestoneName = 'epoch-{0:d4}' -f $Milestone $milestoneDir = Join-Path $outputRoot $milestoneName $completePath = Join-Path $milestoneDir 'complete.json' if (Test-Path -LiteralPath $completePath -PathType Leaf) { return } New-Item -ItemType Directory -Force -Path $milestoneDir | Out-Null $checkpointPath = Join-Path $milestoneDir "aletheia_ru_$milestoneName.ckpt" if (-not (Test-Path -LiteralPath $checkpointPath -PathType Leaf)) { $firstLength = $SourceCheckpoint.Length Start-Sleep -Seconds 10 $refreshed = Get-Item -LiteralPath $SourceCheckpoint.FullName if ($firstLength -ne $refreshed.Length -or $refreshed.Length -le 0) { throw "Checkpoint is not stable yet: $($SourceCheckpoint.FullName)" } $partialPath = "$checkpointPath.partial" Copy-Item -LiteralPath $refreshed.FullName -Destination $partialPath -Force Move-Item -LiteralPath $partialPath -Destination $checkpointPath -Force } $modelPath = Join-Path $milestoneDir "aletheia_ru_$milestoneName.onnx" $configPath = "$modelPath.json" if (-not (Test-Path -LiteralPath $modelPath -PathType Leaf)) { Invoke-LoggedProcess $python @( '-m', 'piper.train.export_onnx', '--checkpoint', $checkpointPath, '--output-file', $modelPath ) (Join-Path $milestoneDir 'export.stdout.log') (Join-Path $milestoneDir 'export.stderr.log') } Copy-Item -LiteralPath $configSource -Destination $configPath -Force $inputPath = Join-Path $milestoneDir 'test_sentences.txt' $testText = @( '0JIg0YLQuNGI0LjQvdC1INCy0LXRh9C10YDQvdC10Lkg0LHQuNCx0LvQuNC+0YLQtdC60Lgg0YjQtdC70LXRgdGC0LXQu9C4INGB0YLRgNCw0L3QuNGG0YssINC4INC60LDQttC00LDRjyDQvdC+0LLQsNGPINCz0LvQsNCy0LAg0L7RgtC60YDRi9Cy0LDQu9CwINGD0LTQuNCy0LjRgtC10LvRjNC90YvQuSDQvNC40YAu', '0JrQvtCz0LTQsCDRh9C10LvQvtCy0LXQuiDQtNC10LvQsNC10YIg0LLRi9Cx0L7RgCwg0L7QvSDQvdC1INCy0YHQtdCz0LTQsCDQt9Cw0YDQsNC90LXQtSDQt9C90LDQtdGCLCDQuiDQutCw0LrQuNC8INC/0L7RgdC70LXQtNGB0YLQstC40Y/QvCDQv9GA0LjQstC10LTRkdGCINC10LPQviDRgNC10YjQtdC90LjQtS4=', '0JfQsCDQvtC60L3QvtC8INC80LXQtNC70LXQvdC90L4g0L3QsNGH0LjQvdCw0LvRgdGPINC00L7QttC00YwsINC90L4g0L/Rg9GC0LXRiNC10YHRgtCy0LjQtSDQs9C10YDQvtC10LIg0YLQvtC70YzQutC+INC90LDQsdC40YDQsNC70L4g0YHQuNC70YMu' ) | ForEach-Object { [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($_)) } $testText = $testText -join [Environment]::NewLine [IO.File]::WriteAllText($inputPath, $testText, [Text.UTF8Encoding]::new($false)) $samples = @( @{ Name = 'neutral'; Length = '1.0'; Noise = '0.667'; Width = '0.8'; Silence = '0.0' }, @{ Name = 'book'; Length = '1.5'; Noise = '0.75'; Width = '0.95'; Silence = '0.25' }, @{ Name = 'fast'; Length = '0.82'; Noise = '0.70'; Width = '0.85'; Silence = '0.0' } ) foreach ($sample in $samples) { $wavPath = Join-Path $milestoneDir ("sample_{0}.wav" -f $sample.Name) if (Test-Path -LiteralPath $wavPath -PathType Leaf) { continue } Invoke-LoggedProcess $python @( '-m', 'piper', '--model', $modelPath, '--config', $configPath, '--input-file', $inputPath, '--output-file', $wavPath, '--length-scale', $sample.Length, '--noise-scale', $sample.Noise, '--noise-w-scale', $sample.Width, '--sentence-silence', $sample.Silence ) (Join-Path $milestoneDir "$($sample.Name).stdout.log") (Join-Path $milestoneDir "$($sample.Name).stderr.log") } $sourceMatch = [regex]::Match($SourceCheckpoint.Name, '^epoch=(\d+)-step=(\d+)\.ckpt$') $artifacts = Get-ChildItem -LiteralPath $milestoneDir -File | Where-Object { $_.Extension -in @('.ckpt', '.onnx', '.json', '.wav') } | ForEach-Object { [ordered]@{ name = $_.Name bytes = $_.Length sha256 = (Get-FileHash -LiteralPath $_.FullName -Algorithm SHA256).Hash.ToLowerInvariant() } } [ordered]@{ requested_completed_epochs = $Milestone source_epoch_zero_based = [int]$sourceMatch.Groups[1].Value source_step = [int]$sourceMatch.Groups[2].Value created_utc = (Get-Date).ToUniversalTime().ToString('o') artifacts = @($artifacts) } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $completePath -Encoding utf8 Write-MonitorLog "Completed milestone $Milestone from $($SourceCheckpoint.Name)" } Write-MonitorLog "Monitor started for milestones: $($Milestones -join ', ')" while ($true) { Show-TrainingStatus $pending = @($Milestones | Where-Object { -not (Test-Path -LiteralPath (Join-Path $outputRoot ('epoch-{0:d4}\complete.json' -f $_)) -PathType Leaf) }) if ($pending.Count -eq 0) { Write-MonitorLog 'All milestones completed' exit 0 } try { $checkpoint = Get-LatestCheckpoint if ($checkpoint -and $checkpoint.Name -match '^epoch=(\d+)-step=(\d+)\.ckpt$') { $completedEpochs = [int]$Matches[1] + 1 foreach ($milestone in $pending) { if ($completedEpochs -ge $milestone) { Save-Milestone $milestone $checkpoint } } } } catch { Write-MonitorLog ("Retryable error: " + ($_ | Out-String).Trim()) } Start-Sleep -Seconds $PollSeconds }