56 lines
1.8 KiB
PowerShell
56 lines
1.8 KiB
PowerShell
[CmdletBinding()]
|
|
param()
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
$root = $PSScriptRoot
|
|
$runDir = Join-Path $root 'training'
|
|
$stdoutPath = Join-Path $runDir 'scheduled-training.stdout.log'
|
|
$stderrPath = Join-Path $runDir 'scheduled-training.stderr.log'
|
|
$exitPath = Join-Path $runDir 'scheduled-training-exit.json'
|
|
New-Item -ItemType Directory -Force -Path $runDir | Out-Null
|
|
if (Test-Path -LiteralPath $exitPath) {
|
|
Remove-Item -LiteralPath $exitPath -Force
|
|
}
|
|
|
|
$exitCode = 1
|
|
try {
|
|
$latestCheckpoint = Get-ChildItem -LiteralPath (Join-Path $runDir 'checkpoints') `
|
|
-Filter '*.ckpt' `
|
|
-File `
|
|
-Recurse `
|
|
-ErrorAction SilentlyContinue |
|
|
Sort-Object LastWriteTimeUtc -Descending |
|
|
Select-Object -First 1
|
|
$arguments = @(
|
|
'-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass',
|
|
'-File', (Join-Path $root 'train_piper_windows.ps1'),
|
|
'-DatasetDir', (Join-Path $root 'dataset'),
|
|
'-OutputDir', $runDir,
|
|
'-CacheDir', (Join-Path $root 'smoke\cache'),
|
|
'-PythonExe', (Join-Path $root '.venv\Scripts\python.exe'),
|
|
'-BatchSize', '16', '-NumWorkers', '0', '-MaxEpochs', '2000'
|
|
)
|
|
if ($latestCheckpoint) {
|
|
$arguments += @('-CheckpointPath', $latestCheckpoint.FullName)
|
|
}
|
|
$process = Start-Process -FilePath 'powershell.exe' `
|
|
-ArgumentList $arguments `
|
|
-RedirectStandardOutput $stdoutPath `
|
|
-RedirectStandardError $stderrPath `
|
|
-WindowStyle Hidden `
|
|
-Wait `
|
|
-PassThru
|
|
$exitCode = $process.ExitCode
|
|
} catch {
|
|
$_ | Out-String | Add-Content -LiteralPath $stderrPath -Encoding utf8
|
|
$exitCode = 1
|
|
} finally {
|
|
[ordered]@{
|
|
exit_code = $exitCode
|
|
finished_utc = (Get-Date).ToUniversalTime().ToString('o')
|
|
} | ConvertTo-Json | Set-Content -LiteralPath $exitPath -Encoding utf8
|
|
}
|
|
|
|
exit $exitCode
|