feat: production paper trading platform

This commit is contained in:
Курнат Андрей
2026-07-14 22:52:47 +03:00
parent 7186acb9a1
commit 5c4aecfe5f
29 changed files with 396 additions and 564 deletions
+7 -2
View File
@@ -3,13 +3,18 @@ HOST=127.0.0.1
PORT=8787 PORT=8787
BYBIT_TESTNET=false BYBIT_TESTNET=false
# Official regional public endpoints for this deployment. Before future live
# trading they must match the Bybit site where the API key was created.
BYBIT_REST_BASE_URL=https://api.bybit.kz
BYBIT_WEBSOCKET_URL=wss://stream.bybit.kz/v5/public/spot
BYBIT_API_KEY= BYBIT_API_KEY=
BYBIT_API_SECRET= BYBIT_API_SECRET=
STARTING_BALANCE_USDT=100 STARTING_BALANCE_USDT=100
AUTO_SELECT_SYMBOLS=false AUTO_SELECT_SYMBOLS=true
TOP_SYMBOLS_COUNT=12 TOP_SYMBOLS_COUNT=12
SYMBOLS=BTCUSDT,ETHUSDT,HYPEUSDT,SOLUSDT,XRPUSDT,XPLUSDT,WLDUSDT,MNTUSDT,HUSDT,XAUTUSDT,IPUSDT,AAVEUSDT # Leave empty to discover the most liquid eligible USDT Spot pairs from Bybit.
SYMBOLS=
STRATEGY_MODE=torch_forecast STRATEGY_MODE=torch_forecast
BASE_INTERVAL=60 BASE_INTERVAL=60
+9 -5
View File
@@ -5,12 +5,16 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
WORKDIR /app WORKDIR /app
COPY requirements.txt /app/requirements.txt COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir --upgrade pip \ RUN pip install --no-cache-dir --disable-pip-version-check --upgrade pip \
&& pip install --no-cache-dir -r /app/requirements.txt && pip install --no-cache-dir --disable-pip-version-check -r /app/requirements.txt \
&& groupadd --gid 1000 tradebot \
&& useradd --uid 1000 --gid tradebot --home-dir /app --shell /usr/sbin/nologin tradebot
COPY crypto_spot_bot /app/crypto_spot_bot COPY --chown=1000:1000 crypto_spot_bot /app/crypto_spot_bot
COPY README.md /app/README.md COPY --chown=1000:1000 README.md /app/README.md
RUN mkdir -p /app/runtime RUN mkdir -p /app/runtime && chown -R 1000:1000 /app/runtime
EXPOSE 8787 EXPOSE 8787
USER 1000:1000
STOPSIGNAL SIGTERM
CMD ["python", "-m", "crypto_spot_bot.main"] CMD ["python", "-m", "crypto_spot_bot.main"]
+22 -24
View File
@@ -5,7 +5,7 @@ Spot-бот для демо-торговли криптовалютой на р
## Что реализовано ## Что реализовано
- Реальные market data Bybit Spot: REST bootstrap и WebSocket-обновления. - Реальные market data Bybit Spot: REST bootstrap и WebSocket-обновления.
- Фиксированный набор 12 USDT spot-пар для основной стратегии: `BTCUSDT`, `ETHUSDT`, `HYPEUSDT`, `SOLUSDT`, `XRPUSDT`, `XPLUSDT`, `WLDUSDT`, `MNTUSDT`, `HUSDT`, `XAUTUSDT`, `IPUSDT`, `AAVEUSDT`. - Торговый universe автоматически строится из актуальных Bybit Spot-инструментов: выбираются до 12 ликвидных USDT-пар по `turnover24h`, исключаются stablecoin-to-stablecoin и leveraged-token пары; фиксированный список можно задать только явным `SYMBOLS`.
- Paper trading с учетом cash, комиссий, проскальзывания, stop-loss, take-profit и trailing stop. - Paper trading с учетом cash, комиссий, проскальзывания, stop-loss, take-profit и trailing stop.
- Spot-only логика: покупка базовой монеты за USDT и продажа обратно, без short и без плеча. - Spot-only логика: покупка базовой монеты за USDT и продажа обратно, без short и без плеча.
- Live spot-ордеры явно отправляются без плеча: `category=spot`, `isLeverage=0`. - Live spot-ордеры явно отправляются без плеча: `category=spot`, `isLeverage=0`.
@@ -18,13 +18,14 @@ Spot-бот для демо-торговли криптовалютой на р
- DCA/мартингейл отключены: в режиме `trend_macd` брокер не разрешает вторую позицию по той же паре. - DCA/мартингейл отключены: в режиме `trend_macd` брокер не разрешает вторую позицию по той же паре.
- Grid, rebound, adaptive learning, Kelly sizing и time-series forecast выключены по умолчанию и не участвуют в принятии решений `trend_macd`. - Grid, rebound, adaptive learning, Kelly sizing и time-series forecast выключены по умолчанию и не участвуют в принятии решений `trend_macd`.
- Быстрый режим торговли: отдельный короткий интервал цикла, короткий cooldown после выхода и лимит новых входов в минуту; выходы по риску этим лимитом не блокируются. - Быстрый режим торговли: отдельный короткий интервал цикла, короткий cooldown после выхода и лимит новых входов в минуту; выходы по риску этим лимитом не блокируются.
- Веб-dashboard на русском: equity, cash, PnL, позиции, сделки, сигналы, события, свечные графики, переключатель быстрой торговли и индикаторы работы обучения. - Защищённый JSON API: equity, cash, PnL, позиции, сделки, сигналы, события, свечи, управление paper-циклом и состоянием обучения.
- Android-монитор в `android/TradeBotMonitor`: русский мобильный интерфейс для просмотра 12 пар, свечей, Torch/Kelly параметров, расписания удалённого retrain и live-чеклиста. - Android-монитор в `android/TradeBotMonitor`: русский мобильный интерфейс для динамического списка Bybit-пар, свечей, Torch/Kelly параметров, WorkManager-расписания удалённого retrain и live-чеклиста.
- SQLite runtime-хранилище в `runtime/tradebot.sqlite3`. - SQLite runtime-хранилище в `runtime/tradebot.sqlite3`.
- Liveness `/api/health`, readiness `/api/ready`, объединенный mobile snapshot `/api/mobile/snapshot` и Prometheus-compatible `/metrics`. - Liveness `/api/health`, readiness `/api/ready`, объединенный mobile snapshot `/api/mobile/snapshot` и Prometheus-compatible `/metrics`.
- Все приватные API endpoints требуют токен или подтвержденный reverse-proxy user header; health и metrics остаются доступными для локального мониторинга. - Все приватные API endpoints требуют токен или подтвержденный reverse-proxy user header; health и metrics остаются доступными для локального мониторинга.
- Docker Compose для установки на Raspberry Pi 5 или другой Linux-хост. - Hardened Docker Compose для установки на Dell/Linux: non-root user, read-only root filesystem, dropped capabilities, healthcheck и ротация container logs.
- Live trading guard: live не стартует без `ENABLE_LIVE_TRADING=true`, `LIVE_TRADING_CONFIRM=I_ACCEPT_REAL_RISK` и Bybit API-ключей. - Live trading guard: live не стартует без `ENABLE_LIVE_TRADING=true`, `LIVE_TRADING_CONFIRM=I_ACCEPT_REAL_RISK` и Bybit API-ключей.
- Внешний production endpoint сохраняется на `https://tb.kusoft.xyz`; Caddy завершает TLS и проксирует API к контейнеру на loopback.
## Источники и принятые параметры ## Источники и принятые параметры
@@ -34,6 +35,8 @@ Spot-бот для демо-торговли криптовалютой на р
Популярность пар определяется через `/v5/market/tickers`, потому что Bybit Spot ticker возвращает `turnover24h`, `volume24h`, `bid1Price`, `ask1Price` и `lastPrice`: <https://bybit-exchange.github.io/docs/v5/market/tickers>. Популярность пар определяется через `/v5/market/tickers`, потому что Bybit Spot ticker возвращает `turnover24h`, `volume24h`, `bid1Price`, `ask1Price` и `lastPrice`: <https://bybit-exchange.github.io/docs/v5/market/tickers>.
Для текущего paper/training-развёртывания используется официальный региональный endpoint `api.bybit.kz`; Bybit перечисляет его в Integration Guidance. `BYBIT_REST_BASE_URL` и `BYBIT_WEBSOCKET_URL` остаются явными настройками, потому что перед будущим live-режимом домен обязан соответствовать площадке выпуска API-ключа: <https://bybit-exchange.github.io/docs/v5/guide>.
Лучшие bid/ask берутся из `/v5/market/orderbook`; документация Bybit описывает `GET /v5/market/orderbook` с `category=spot`: <https://bybit-exchange.github.io/docs/v5/market/orderbook>. Лучшие bid/ask берутся из `/v5/market/orderbook`; документация Bybit описывает `GET /v5/market/orderbook` с `category=spot`: <https://bybit-exchange.github.io/docs/v5/market/orderbook>.
WebSocket-стакан использует topic `orderbook.{depth}.{symbol}`; Bybit документирует snapshot/delta-поведение и частоты push для Spot depth 1/50/200/1000: <https://bybit-exchange.github.io/docs/v5/websocket/public/orderbook>. WebSocket-стакан использует topic `orderbook.{depth}.{symbol}`; Bybit документирует snapshot/delta-поведение и частоты push для Spot depth 1/50/200/1000: <https://bybit-exchange.github.io/docs/v5/websocket/public/orderbook>.
@@ -45,7 +48,7 @@ Live market orders используют `/v5/order/create`; Bybit докумен
- Investopedia перечисляет важные свойства algo trading software: real-time market data, low latency, configurability, backtesting, broker/exchange integration, fees/costs и APIs: <https://www.investopedia.com/articles/active-trading/090815/picking-right-algorithmic-trading-software.asp>. - Investopedia перечисляет важные свойства algo trading software: real-time market data, low latency, configurability, backtesting, broker/exchange integration, fees/costs и APIs: <https://www.investopedia.com/articles/active-trading/090815/picking-right-algorithmic-trading-software.asp>.
- Investopedia отдельно указывает, что automated trading systems задают правила entry/exit/money management, но требуют мониторинга и несут риск mechanical failures и over-optimization: <https://www.investopedia.com/articles/trading/11/automated-trading-systems.asp>. - Investopedia отдельно указывает, что automated trading systems задают правила entry/exit/money management, но требуют мониторинга и несут риск mechanical failures и over-optimization: <https://www.investopedia.com/articles/trading/11/automated-trading-systems.asp>.
- QuantInsti описывает типовой путь разработки: стратегия, backtesting, paper trading, затем live trading, плюс GUI, order management и risk management: <https://www.quantinsti.com/articles/automated-trading-system/>. - QuantInsti описывает типовой путь разработки: стратегия, backtesting, paper trading, затем live trading, плюс GUI, order management и risk management: <https://www.quantinsti.com/articles/automated-trading-system/>.
- Hochreiter и Schmidhuber описали LSTM как recurrent neural network architecture для последовательностей; обучение LSTM/GRU в проекте выполняется локально через PyTorch, а Raspberry Pi исполняет только экспортированные JSON-веса без PyTorch runtime: <https://direct.mit.edu/neco/article/9/8/1735/6109/Long-Short-Term-Memory>. - Hochreiter и Schmidhuber описали LSTM как recurrent neural network architecture для последовательностей; обучение LSTM/GRU в проекте выполняется локально через PyTorch, а Dell исполняет только прошедшие quality gate экспортированные JSON-веса без PyTorch runtime: <https://direct.mit.edu/neco/article/9/8/1735/6109/Long-Short-Term-Memory>.
Я не могу подтвердить, что эта стратегия будет прибыльной. Источники выше описывают технические свойства и риски автоматической торговли, но не гарантируют прибыль. Я не могу подтвердить, что эта стратегия будет прибыльной. Источники выше описывают технические свойства и риски автоматической торговли, но не гарантируют прибыль.
@@ -59,17 +62,17 @@ Copy-Item .env.example .env
python -m crypto_spot_bot.main python -m crypto_spot_bot.main
``` ```
Dashboard: <http://127.0.0.1:8787/> Liveness: <http://127.0.0.1:8787/api/health>
## Локальное обучение PyTorch LSTM/GRU ## Локальное обучение PyTorch LSTM
Обучение запускается на основной Windows-машине, а Raspberry Pi остается только для исполнения торгового цикла. PyTorch нужен только на машине обучения; в JSON экспортируются веса, а runtime на Raspberry Pi считает inference обычным Python-кодом: Обучение запускается на основной Windows-машине, а Dell остается для исполнения торгового цикла. PyTorch нужен только на машине обучения; в JSON экспортируются веса, а runtime на Dell считает inference обычным Python-кодом:
```powershell ```powershell
.\.venv\Scripts\python.exe -m pip install torch --index-url https://download.pytorch.org/whl/cpu .\.venv\Scripts\python.exe -m pip install torch --index-url https://download.pytorch.org/whl/cpu
.\.venv\Scripts\python.exe tools\train_torch_recurrent_forecaster.py ` .\.venv\Scripts\python.exe tools\train_torch_recurrent_forecaster.py `
--limit 3000 ` --limit 3000 `
--architectures lstm,gru ` --architectures lstm `
--lookbacks 64 ` --lookbacks 64 `
--hidden-sizes 64,96 ` --hidden-sizes 64,96 `
--layers 2 ` --layers 2 `
@@ -86,11 +89,10 @@ Dashboard: <http://127.0.0.1:8787/>
Файл из `TIME_SERIES_LSTM_MODEL_PATH` читается ботом автоматически, если `TIME_SERIES_FORECAST_ENABLED=true`. В стратегии `torch_forecast` экспортированная PyTorch LSTM/GRU модель является единственным направляющим сигналом для входа и forecast-выхода. Экспортированные модели появляются в dashboard как `PyTorch LSTM` или `PyTorch GRU`; старый легкий reservoir LSTM-кандидат и все встроенные не-torch прогнозы удалены. Файл из `TIME_SERIES_LSTM_MODEL_PATH` читается ботом автоматически, если `TIME_SERIES_FORECAST_ENABLED=true`. В стратегии `torch_forecast` экспортированная PyTorch LSTM/GRU модель является единственным направляющим сигналом для входа и forecast-выхода. Экспортированные модели появляются в dashboard как `PyTorch LSTM` или `PyTorch GRU`; старый легкий reservoir LSTM-кандидат и все встроенные не-torch прогнозы удалены.
Автопереобучение на Windows запускает PyTorch trainer, пишет лог в `runtime/torch_retrain.log` и защищается от параллельных запусков: Локальный retrain на Windows запускает PyTorch trainer, пишет лог в `runtime/torch_retrain.log` и защищается от параллельных запусков:
```powershell ```powershell
powershell -ExecutionPolicy Bypass -File tools\run_torch_retrain.ps1 powershell -ExecutionPolicy Bypass -File tools\run_torch_retrain.ps1
powershell -ExecutionPolicy Bypass -File tools\install_windows_torch_retrainer.ps1
``` ```
Для удалённого запуска с телефона или с бота используется Windows training agent. Бот на `tb.kusoft.xyz` хранит очередь заданий, а Windows-машина сама подключается к интернету, забирает задания, обучает модель и загружает артефакты обратно: Для удалённого запуска с телефона или с бота используется Windows training agent. Бот на `tb.kusoft.xyz` хранит очередь заданий, а Windows-машина сама подключается к интернету, забирает задания, обучает модель и загружает артефакты обратно:
@@ -99,21 +101,15 @@ powershell -ExecutionPolicy Bypass -File tools\install_windows_torch_retrainer.p
powershell -ExecutionPolicy Bypass -File tools\install_windows_training_agent.ps1 -ApiAuth "<TRADEBOT_TRAINING_TOKEN>" -StartNow powershell -ExecutionPolicy Bypass -File tools\install_windows_training_agent.ps1 -ApiAuth "<TRADEBOT_TRAINING_TOKEN>" -StartNow
``` ```
Установщик сохраняет worker-токен через Windows DPAPI, удаляет его старую plaintext-копию из пользовательского окружения и включает постоянный запуск агента. С правами администратора используется Scheduled Task с watchdog; без повышения прав — штатный ярлык в пользовательской папке Startup. Старые локальные retrain-задачи удаляются, чтобы обучение запускалось через очередь, а не двумя независимыми механизмами. Установщик сохраняет worker-токен через Windows DPAPI, удаляет его старую plaintext-копию из пользовательского окружения и включает постоянный запуск агента. С правами администратора используется Scheduled Task с watchdog; без повышения прав — штатный ярлык в пользовательской папке Startup. Сервер выдаёт каждой попытке 10-минутную возобновляемую lease; зависшая попытка автоматически возвращается в очередь, а устаревший процесс не может загрузить артефакты по старой lease.
По умолчанию Windows-agent обучает отдельную PyTorch `LSTM/GRU` для каждой пары на `6000` часовых свечах. Это не заставляет разнородные активы делить одну архитектуру и один набор recurrent-весов. Прогноз усредняется по seed `7/19`, модели сравниваются на validation-folds, а пороги калибруются отдельно для каждой пары. Ensemble guard выполняется пакетно на GPU, а экспорт не дублирует первый набор весов. Search space использует lookback `32/64/128`, hidden `64/96`, dropout `0.20`, AdamW learning rate `0.0007` и weight decay `0.0005`; untouched holdout и quality gate не ослабляются. Для диагностического pooled-запуска используется ключ `-Pooled`. Параметры можно переопределить через env: `TORCH_RETRAIN_SYMBOLS`, `TORCH_RETRAIN_LIMIT`, `TORCH_RETRAIN_LOOKBACKS`, `TORCH_RETRAIN_ARCHITECTURES`, `TORCH_RETRAIN_HIDDEN_SIZES`, `TORCH_RETRAIN_LAYERS`, `TORCH_RETRAIN_DROPOUTS`, `TORCH_RETRAIN_HORIZON`, `TORCH_RETRAIN_HORIZONS`, `TORCH_RETRAIN_CONTEXT_SYMBOLS`, `TORCH_RETRAIN_FEATURES`, `TORCH_RETRAIN_SEED`, `TORCH_RETRAIN_ENSEMBLE_SEEDS`, `TORCH_RETRAIN_SELECTION_FOLDS`, `TORCH_RETRAIN_LEARNING_RATE`, `TORCH_RETRAIN_WEIGHT_DECAY`, `TORCH_RETRAIN_EPOCHS`, `TORCH_RETRAIN_PATIENCE`, `TORCH_RETRAIN_INTERVAL`, `TORCH_RETRAIN_ENV`. По умолчанию Windows-agent обучает одну pooled PyTorch LSTM на динамическом наборе пар и `4000` часовых свечах на пару. Базовый профиль использует lookback `64`, hidden size `64`, два recurrent-слоя, dropout `0.20`, до `50` эпох, seed-ensemble `7/19`, три validation-fold и AdamW с learning rate `0.0007`/weight decay `0.0005`. Untouched holdout и quality gate не ослабляются. Параметр задания `pooled=false` включает независимые модели по парам; `architectures=gru` оставлен только как явная экспериментальная опция. Параметры можно переопределить через env: `TORCH_RETRAIN_SYMBOLS`, `TORCH_RETRAIN_LIMIT`, `TORCH_RETRAIN_LOOKBACKS`, `TORCH_RETRAIN_ARCHITECTURES`, `TORCH_RETRAIN_HIDDEN_SIZES`, `TORCH_RETRAIN_LAYERS`, `TORCH_RETRAIN_DROPOUTS`, `TORCH_RETRAIN_HORIZON`, `TORCH_RETRAIN_HORIZONS`, `TORCH_RETRAIN_CONTEXT_SYMBOLS`, `TORCH_RETRAIN_FEATURES`, `TORCH_RETRAIN_SEED`, `TORCH_RETRAIN_ENSEMBLE_SEEDS`, `TORCH_RETRAIN_SELECTION_FOLDS`, `TORCH_RETRAIN_LEARNING_RATE`, `TORCH_RETRAIN_WEIGHT_DECAY`, `TORCH_RETRAIN_EPOCHS`, `TORCH_RETRAIN_PATIENCE`, `TORCH_RETRAIN_INTERVAL`, `TORCH_RETRAIN_ENV`.
Loss и выбор гиперпараметров учитывают after-cost trading utility, ошибку ожидаемого чистого PnL, quantile-loss и focal BCE для события `TP before SL`, а не только MAE направления цены. В каждом walk-forward fold вероятность успеха калибруется Platt-моделью исключительно на train-части; затем на этой же train-части выбираются глобальные и per-symbol пороги, которые применяются к test-части. Для выбора порога требуется минимум 24 непересекающиеся сделки, а финальный quality gate по-прежнему требует не менее 30 OOS-сделок. Калибратор не имеет fallback на единичные сделки: если минимальная статистика не набрана, кандидат получает `calibration_insufficient` и не может пройти gate. Loss и выбор гиперпараметров учитывают after-cost trading utility, ошибку ожидаемого чистого PnL, quantile-loss и focal BCE для события `TP before SL`, а не только MAE направления цены. В каждом walk-forward fold вероятность успеха калибруется Platt-моделью исключительно на train-части; затем на этой же train-части выбираются глобальные и per-symbol пороги, которые применяются к test-части. Для выбора порога требуется минимум 24 непересекающиеся сделки, а финальный quality gate по-прежнему требует не менее 30 OOS-сделок. Калибратор не имеет fallback на единичные сделки: если минимальная статистика не набрана, кандидат получает `calibration_insufficient` и не может пройти gate.
Основной decision horizon — `12h`, дополнительные горизонты — `3/6/12/24`. Размеры обучающих барьеров берутся из `STOP_LOSS_PERCENT` и `TAKE_PROFIT_PERCENT`, а round-trip cost — из fee/slippage настроек. Threshold search оценивается тем же execution replay со stop-loss, take-profit, ATR trailing и forecast-exit, который используется в walk-forward. `holdout_skill` остаётся только в финальном отчёте и никогда не участвует в фильтрации входов или подборе порогов. Основной decision horizon — `12h`, дополнительные горизонты — `3/6/12/24`. Размеры обучающих барьеров берутся из `STOP_LOSS_PERCENT` и `TAKE_PROFIT_PERCENT`, а round-trip cost — из fee/slippage настроек. Threshold search оценивается тем же execution replay со stop-loss, take-profit, ATR trailing и forecast-exit, который используется в walk-forward. `holdout_skill` остаётся только в финальном отчёте и никогда не участвует в фильтрации входов или подборе порогов.
Если retrain запускается с `-DeployToPi`, после успешного guard он синхронизирует `runtime/lstm_forecaster.json`, `runtime/torch_retrain_guard.json` и `runtime/torch_threshold_calibration.json` на Raspberry Pi через SSH-ключ и перезапускает сервис `tradebot`. Отдельный запуск sync: Внутри recurrent модели используются exportable attention pooling и LayerNorm. После recurrent-контекста добавлена нелинейная GELU-проекция и две отдельные экспортируемые головы: одна для ожидаемого PnL/quantiles, вторая для `P(TP before SL)`. Принятый bundle загружается агентом через защищённый API `tb.kusoft.xyz`, проходит серверную проверку SHA-256/guard/calibration и атомарно становится активным на Dell.
```powershell
powershell -ExecutionPolicy Bypass -File tools\sync_torch_artifacts_to_pi.ps1 -RemoteHost 192.168.0.185 -RemoteUser sevenhill -RemoteRoot /mnt/data/tradebot
```
Внутри recurrent модели используются exportable attention pooling и LayerNorm. После recurrent-контекста добавлена нелинейная GELU-проекция и две отдельные экспортируемые головы: одна для ожидаемого PnL/quantiles, вторая для `P(TP before SL)`. Raspberry Pi по-прежнему исполняет модель из JSON без PyTorch runtime.
## Docker ## Docker
@@ -123,18 +119,20 @@ docker compose up -d --build
docker compose logs -f tradebot docker compose logs -f tradebot
``` ```
Dashboard: `http://<host>:8787/` Локальная проверка: `http://127.0.0.1:8787/api/health`; внешний адрес: `https://tb.kusoft.xyz`.
Для Raspberry Pi 5 проект использует `python:3.12-slim`, без Node.js build step. Runtime-данные лежат в volume `./runtime:/app/runtime`; на внешнем диске можно разместить папку проекта или заменить volume на абсолютный путь внешнего диска. На Dell проект использует `python:3.12-slim`, без Node.js build step. Runtime-данные лежат в bind mount `./runtime:/app/runtime`; корневая файловая система контейнера read-only, процесс работает как UID/GID 1000, а container logs ротируются по `10 MiB × 3`.
## Основные env-параметры ## Основные env-параметры
```env ```env
TRADING_MODE=paper TRADING_MODE=paper
STARTING_BALANCE_USDT=100 STARTING_BALANCE_USDT=100
AUTO_SELECT_SYMBOLS=false BYBIT_REST_BASE_URL=https://api.bybit.kz
BYBIT_WEBSOCKET_URL=wss://stream.bybit.kz/v5/public/spot
AUTO_SELECT_SYMBOLS=true
TOP_SYMBOLS_COUNT=12 TOP_SYMBOLS_COUNT=12
SYMBOLS=BTCUSDT,ETHUSDT,HYPEUSDT,SOLUSDT,XRPUSDT,XPLUSDT,WLDUSDT,MNTUSDT,HUSDT,XAUTUSDT,IPUSDT,AAVEUSDT SYMBOLS=
STRATEGY_MODE=torch_forecast STRATEGY_MODE=torch_forecast
BASE_INTERVAL=60 BASE_INTERVAL=60
TREND_INTERVAL=D TREND_INTERVAL=D
+2 -2
View File
@@ -11,7 +11,7 @@
- Kelly/размер позиции: текущий размер, Kelly-цель, занятая экспозиция, остаток, множители edge/P(up)/skill. - Kelly/размер позиции: текущий размер, Kelly-цель, занятая экспозиция, остаток, множители edge/P(up)/skill.
- Обзор equity/cash/exposure/PnL и последних решений. - Обзор equity/cash/exposure/PnL и последних решений.
- Удалённый запуск retrain через очередь заданий на боте и закреплённый Windows-компьютер обучения. - Удалённый запуск retrain через очередь заданий на боте и закреплённый Windows-компьютер обучения.
- Расписание retrain на телефоне: Android отправляет команду по расписанию, но обучение идёт на Windows-машине. - Расписание retrain через Android WorkManager: команда отправляется только при наличии сети, а обучение идёт на Windows-машине.
- Настройки API, токена команд, тёмной/светлой темы. - Настройки API, токена команд, тёмной/светлой темы.
- Live-чеклист: приложение показывает, готов ли сервер к реальной торговле, и не включает live одной опасной кнопкой. - Live-чеклист: приложение показывает, готов ли сервер к реальной торговле, и не включает live одной опасной кнопкой.
@@ -42,7 +42,7 @@ https://tb.kusoft.xyz
## Переобучение ## Переобучение
Телефон не обучает модель локально. Вкладка `Обучение` ставит задание в очередь на `tb.kusoft.xyz`, а Windows-agent на закреплённой машине `SEVENHILL` (`G:\Repos\TradeBot`) сам выходит в интернет, забирает задание, обучает модель и отправляет артефакты обратно боту. Так телефон становится пультом запуска/расписания, а тяжёлый PyTorch retrain остаётся на нормальном компьютере даже если он находится в другой сети. Телефон не обучает модель локально. Вкладка `Обучение` ставит задание в очередь на `tb.kusoft.xyz`, а Windows-agent на этой машине сам выходит в интернет, забирает задание, обучает модель и отправляет проверенный bundle обратно боту. Имя и путь активного worker приложение получает от сервера, без прошитого имени компьютера.
## Live-торговля ## Live-торговля
+8 -4
View File
@@ -4,13 +4,17 @@ plugins {
android { android {
namespace = "xyz.kusoft.tradebotmonitor" namespace = "xyz.kusoft.tradebotmonitor"
compileSdk = 36 compileSdk = 37
defaultConfig { defaultConfig {
applicationId = "xyz.kusoft.tradebotmonitor" applicationId = "xyz.kusoft.tradebotmonitor"
minSdk = 26 minSdk = 26
targetSdk = 36 targetSdk = 37
versionCode = 21 versionCode = 22
versionName = "0.4.2" versionName = "0.5.0"
} }
} }
dependencies {
implementation("androidx.work:work-runtime:2.11.2")
}
@@ -2,12 +2,13 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android"> <manifest xmlns:android="http://schemas.android.com/apk/res/android">
<uses-permission android:name="android.permission.INTERNET" /> <uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" /> <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
<uses-permission android:name="android.permission.RECEIVE_BOOT_COMPLETED" />
<application <application
android:allowBackup="false" android:allowBackup="false"
android:dataExtractionRules="@xml/data_extraction_rules"
android:fullBackupContent="@xml/backup_rules"
android:icon="@drawable/ic_launcher" android:icon="@drawable/ic_launcher"
android:label="TradeBot AI" android:label="TradeBot AI"
android:networkSecurityConfig="@xml/network_security_config"
android:roundIcon="@drawable/ic_launcher" android:roundIcon="@drawable/ic_launcher"
android:supportsRtl="true" android:supportsRtl="true"
android:theme="@style/AppTheme" android:theme="@style/AppTheme"
@@ -22,16 +23,5 @@
</intent-filter> </intent-filter>
</activity> </activity>
<receiver
android:name=".RetrainAlarmReceiver"
android:exported="false" />
<receiver
android:name=".BootReceiver"
android:exported="false">
<intent-filter>
<action android:name="android.intent.action.BOOT_COMPLETED" />
</intent-filter>
</receiver>
</application> </application>
</manifest> </manifest>
@@ -18,12 +18,9 @@ class AppPrefs(context: Context) {
} }
val trainingComputerName = prefs.getString("training_computer_name", null)?.trim() val trainingComputerName = prefs.getString("training_computer_name", null)?.trim()
val trainingComputerPath = prefs.getString("training_computer_path", null)?.trim() val trainingComputerPath = prefs.getString("training_computer_path", null)?.trim()
if ( val staleFallback = trainingComputerName in setOf("SEVENHILL", "DESKTOP-TMFDL0H") ||
trainingComputerName.isNullOrBlank() || trainingComputerPath in setOf("G:\\Repos\\TradeBot", "C:\\Repos\\TradeBot")
trainingComputerName == LEGACY_TRAINING_COMPUTER_NAME || if (trainingComputerName.isNullOrBlank() || trainingComputerPath.isNullOrBlank() || staleFallback) {
trainingComputerPath.isNullOrBlank() ||
trainingComputerPath == LEGACY_TRAINING_COMPUTER_PATH
) {
pinDefaultTrainingComputer() pinDefaultTrainingComputer()
} }
} }
@@ -141,10 +138,8 @@ class AppPrefs(context: Context) {
private companion object { private companion object {
const val DEFAULT_API_BASE_URL = "https://tb.kusoft.xyz" const val DEFAULT_API_BASE_URL = "https://tb.kusoft.xyz"
const val LEGACY_PI_API_BASE_URL = "http://192.168.0.185:8787" const val LEGACY_PI_API_BASE_URL = "http://192.168.0.185:8787"
const val DEFAULT_TRAINING_COMPUTER_NAME = "SEVENHILL" const val DEFAULT_TRAINING_COMPUTER_NAME = "Ожидание Windows-agent"
const val DEFAULT_TRAINING_COMPUTER_PATH = "G:\\Repos\\TradeBot" const val DEFAULT_TRAINING_COMPUTER_PATH = "Имя и путь поступят от сервера"
const val LEGACY_TRAINING_COMPUTER_NAME = "DESKTOP-TMFDL0H"
const val LEGACY_TRAINING_COMPUTER_PATH = "C:\\Repos\\TradeBot"
const val TOKEN_KEY_ALIAS = "tradebot_api_auth_v1" const val TOKEN_KEY_ALIAS = "tradebot_api_auth_v1"
} }
} }
@@ -1512,12 +1512,15 @@ class MainActivity : Activity() {
private fun rankedMarkets(data: BotSnapshot): List<MarketItem> = private fun rankedMarkets(data: BotSnapshot): List<MarketItem> =
orderedMarkets(data.markets).sortedWith( orderedMarkets(data.markets).sortedWith(
compareByDescending<MarketItem> { marketRankScore(it, data.signalsBySymbol[it.symbol]) } compareByDescending<MarketItem> { marketRankScore(it, data.signalsBySymbol[it.symbol]) }
.thenBy { fixedSymbolIndex(it.symbol) } .thenByDescending { it.ticker?.turnover24h ?: 0.0 }
.thenBy { it.symbol }, .thenBy { it.symbol },
) )
private fun orderedMarkets(markets: List<MarketItem>): List<MarketItem> = private fun orderedMarkets(markets: List<MarketItem>): List<MarketItem> =
markets.sortedWith(compareBy({ fixedSymbolIndex(it.symbol) }, { it.symbol })) markets.sortedWith(
compareByDescending<MarketItem> { it.ticker?.turnover24h ?: 0.0 }
.thenBy { it.symbol },
)
private fun marketRankScore(market: MarketItem, signal: SignalData?): Double { private fun marketRankScore(market: MarketItem, signal: SignalData?): Double {
val actionScore = when (normalizedAction(signal?.action)) { val actionScore = when (normalizedAction(signal?.action)) {
@@ -1615,11 +1618,6 @@ class MainActivity : Activity() {
} }
} }
private fun fixedSymbolIndex(symbol: String): Int {
val index = FIXED_SYMBOLS.indexOf(symbol.uppercase(Locale.US))
return if (index >= 0) index else FIXED_SYMBOLS.size + 1
}
private fun trainingStatusSignature(retrain: JSONObject): String = private fun trainingStatusSignature(retrain: JSONObject): String =
(retrain.optJSONObject("coordination") ?: JSONObject()).toString() (retrain.optJSONObject("coordination") ?: JSONObject()).toString()
@@ -1895,20 +1893,4 @@ class MainActivity : Activity() {
Toast.makeText(this, message, Toast.LENGTH_LONG).show() Toast.makeText(this, message, Toast.LENGTH_LONG).show()
} }
private companion object {
val FIXED_SYMBOLS = listOf(
"BTCUSDT",
"ETHUSDT",
"HYPEUSDT",
"SOLUSDT",
"XRPUSDT",
"XPLUSDT",
"WLDUSDT",
"MNTUSDT",
"HUSDT",
"XAUTUSDT",
"IPUSDT",
"AAVEUSDT",
)
}
} }
@@ -1,65 +1,55 @@
package xyz.kusoft.tradebotmonitor package xyz.kusoft.tradebotmonitor
import android.app.AlarmManager
import android.app.PendingIntent
import android.content.BroadcastReceiver
import android.content.Context import android.content.Context
import android.content.Intent import androidx.work.Constraints
import java.util.concurrent.Executors import androidx.work.ExistingPeriodicWorkPolicy
import androidx.work.NetworkType
import androidx.work.PeriodicWorkRequest
import androidx.work.WorkManager
import androidx.work.Worker
import androidx.work.WorkerParameters
import java.util.concurrent.TimeUnit
object RetrainScheduler { object RetrainScheduler {
private const val ACTION_RETRAIN = "xyz.kusoft.tradebotmonitor.RETRAIN" private const val UNIQUE_WORK_NAME = "tradebot-periodic-retrain"
private const val REQUEST_CODE = 6406
fun schedule(context: Context, hours: Int) { fun schedule(context: Context, hours: Int) {
val interval = hours.coerceAtLeast(1) * 60L * 60L * 1000L val constraints = Constraints.Builder()
val manager = context.getSystemService(Context.ALARM_SERVICE) as AlarmManager .setRequiredNetworkType(NetworkType.CONNECTED)
manager.setInexactRepeating( .build()
AlarmManager.RTC_WAKEUP, val request = PeriodicWorkRequest.Builder(
System.currentTimeMillis() + interval, RetrainWorker::class.java,
interval, hours.coerceAtLeast(1).toLong(),
pendingIntent(context), TimeUnit.HOURS,
)
.setConstraints(constraints)
.build()
WorkManager.getInstance(context).enqueueUniquePeriodicWork(
UNIQUE_WORK_NAME,
ExistingPeriodicWorkPolicy.UPDATE,
request,
) )
} }
fun cancel(context: Context) { fun cancel(context: Context) {
val manager = context.getSystemService(Context.ALARM_SERVICE) as AlarmManager WorkManager.getInstance(context).cancelUniqueWork(UNIQUE_WORK_NAME)
manager.cancel(pendingIntent(context)) }
} }
private fun pendingIntent(context: Context): PendingIntent = class RetrainWorker(
PendingIntent.getBroadcast( context: Context,
context, parameters: WorkerParameters,
REQUEST_CODE, ) : Worker(context, parameters) {
Intent(context, RetrainAlarmReceiver::class.java).setAction(ACTION_RETRAIN), override fun doWork(): Result {
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE, val prefs = AppPrefs(applicationContext)
) if (!prefs.retrainScheduleEnabled || prefs.commandToken.isBlank()) {
return Result.success()
} }
return try {
class RetrainAlarmReceiver : BroadcastReceiver() {
override fun onReceive(context: Context, intent: Intent) {
val pending = goAsync()
val executor = Executors.newSingleThreadExecutor()
executor.execute {
try {
val prefs = AppPrefs(context)
if (prefs.retrainScheduleEnabled) {
TradeBotApi(prefs.apiBaseUrl, prefs.commandToken).requestRetrain() TradeBotApi(prefs.apiBaseUrl, prefs.commandToken).requestRetrain()
} Result.success()
} finally { } catch (_: Exception) {
pending.finish() Result.retry()
executor.shutdown()
}
}
}
}
class BootReceiver : BroadcastReceiver() {
override fun onReceive(context: Context, intent: Intent) {
if (intent.action != Intent.ACTION_BOOT_COMPLETED) return
val prefs = AppPrefs(context)
if (prefs.retrainScheduleEnabled) {
RetrainScheduler.schedule(context, prefs.retrainIntervalHours)
} }
} }
} }
@@ -183,7 +183,7 @@ class TradeBotApi(
qualityScore = quality.optDouble("score", 0.0), qualityScore = quality.optDouble("score", 0.0),
) )
} }
return output.sortedBy { it.symbol } return output
} }
private fun parseTicker(row: JSONObject): TickerData = private fun parseTicker(row: JSONObject): TickerData =
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<full-backup-content>
<exclude domain="root" path="." />
<exclude domain="file" path="." />
<exclude domain="database" path="." />
<exclude domain="sharedpref" path="." />
<exclude domain="external" path="." />
</full-backup-content>
@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<data-extraction-rules>
<cloud-backup disableIfNoEncryptionCapabilities="true">
<exclude domain="root" path="." />
<exclude domain="file" path="." />
<exclude domain="database" path="." />
<exclude domain="sharedpref" path="." />
<exclude domain="external" path="." />
</cloud-backup>
<device-transfer>
<exclude domain="root" path="." />
<exclude domain="file" path="." />
<exclude domain="database" path="." />
<exclude domain="sharedpref" path="." />
<exclude domain="external" path="." />
</device-transfer>
</data-extraction-rules>
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<network-security-config>
<base-config cleartextTrafficPermitted="false">
<trust-anchors>
<certificates src="system" />
</trust-anchors>
</base-config>
</network-security-config>
@@ -142,7 +142,7 @@
<rect x="168" y="29" width="84" height="8" rx="4" fill="#151922"/> <rect x="168" y="29" width="84" height="8" rx="4" fill="#151922"/>
<text x="338" y="48" class="text small">91%</text> <text x="338" y="48" class="text small">91%</text>
<text x="34" y="88" class="text h2">Рынки</text> <text x="34" y="88" class="text h2">Рынки</text>
<text x="34" y="114" class="muted small">12 фиксированных spot-пар</text> <text x="34" y="114" class="muted small">Динамические Bybit spot-пары</text>
<rect x="34" y="136" width="340" height="42" rx="7" fill="#11141a" stroke="#242a36"/> <rect x="34" y="136" width="340" height="42" rx="7" fill="#11141a" stroke="#242a36"/>
<text x="52" y="162" class="dim small">Поиск пары или сигнала</text> <text x="52" y="162" class="dim small">Поиск пары или сигнала</text>

Before

Width:  |  Height:  |  Size: 17 KiB

After

Width:  |  Height:  |  Size: 17 KiB

@@ -1,7 +1,7 @@
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-9.4.1-bin.zip distributionUrl=https\://services.gradle.org/distributions/gradle-9.4.1-bin.zip
networkTimeout=10000 networkTimeout=60000
validateDistributionUrl=true validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists zipStorePath=wrapper/dists
+1 -1
View File
@@ -1,3 +1,3 @@
"""Crypto spot trading bot package.""" """Crypto spot trading bot package."""
__version__ = "0.1.0" __version__ = "1.0.0"
+14 -3
View File
@@ -169,17 +169,23 @@ class Settings:
hold_signal_sample_seconds: int = 60 hold_signal_sample_seconds: int = 60
storage_retention_days: int = 30 storage_retention_days: int = 30
storage_prune_interval_seconds: int = 3600 storage_prune_interval_seconds: int = 3600
bybit_rest_base_url_override: str = ""
bybit_websocket_url_override: str = ""
@property @property
def rest_base_url(self) -> str: def rest_base_url(self) -> str:
return "https://api-testnet.bybit.com" if self.bybit_testnet else "https://api.bybit.com" if self.bybit_rest_base_url_override:
return self.bybit_rest_base_url_override.rstrip("/")
return "https://api-testnet.bybit.com" if self.bybit_testnet else "https://api.bybit.kz"
@property @property
def websocket_url(self) -> str: def websocket_url(self) -> str:
if self.bybit_websocket_url_override:
return self.bybit_websocket_url_override
return ( return (
"wss://stream-testnet.bybit.com/v5/public/spot" "wss://stream-testnet.bybit.com/v5/public/spot"
if self.bybit_testnet if self.bybit_testnet
else "wss://stream.bybit.com/v5/public/spot" else "wss://stream.bybit.kz/v5/public/spot"
) )
@property @property
@@ -220,7 +226,7 @@ def load_settings(env_file: str | Path | None = None) -> Settings:
strategy_mode = os.getenv("STRATEGY_MODE", "torch_forecast").strip().lower() strategy_mode = os.getenv("STRATEGY_MODE", "torch_forecast").strip().lower()
if strategy_mode not in STRATEGY_MODES: if strategy_mode not in STRATEGY_MODES:
raise ValueError("STRATEGY_MODE must be legacy, trend_macd or torch_forecast") raise ValueError("STRATEGY_MODE must be legacy, trend_macd or torch_forecast")
auto_select_symbols = _bool_env("AUTO_SELECT_SYMBOLS", False) auto_select_symbols = _bool_env("AUTO_SELECT_SYMBOLS", True)
top_symbols_count = _int_env("TOP_SYMBOLS_COUNT", len(FIXED_SPOT_SYMBOLS)) top_symbols_count = _int_env("TOP_SYMBOLS_COUNT", len(FIXED_SPOT_SYMBOLS))
requested_symbols = _symbols_env("SYMBOLS") requested_symbols = _symbols_env("SYMBOLS")
symbols = requested_symbols if requested_symbols else (() if auto_select_symbols else FIXED_SPOT_SYMBOLS) symbols = requested_symbols if requested_symbols else (() if auto_select_symbols else FIXED_SPOT_SYMBOLS)
@@ -343,6 +349,11 @@ def load_settings(env_file: str | Path | None = None) -> Settings:
hold_signal_sample_seconds=_int_env("HOLD_SIGNAL_SAMPLE_SECONDS", 60), hold_signal_sample_seconds=_int_env("HOLD_SIGNAL_SAMPLE_SECONDS", 60),
storage_retention_days=_int_env("STORAGE_RETENTION_DAYS", 30), storage_retention_days=_int_env("STORAGE_RETENTION_DAYS", 30),
storage_prune_interval_seconds=_int_env("STORAGE_PRUNE_INTERVAL_SECONDS", 3600), storage_prune_interval_seconds=_int_env("STORAGE_PRUNE_INTERVAL_SECONDS", 3600),
bybit_rest_base_url_override=os.getenv(
"BYBIT_REST_BASE_URL",
"" if _bool_env("BYBIT_TESTNET", False) else "https://api.bybit.kz",
).strip(),
bybit_websocket_url_override=os.getenv("BYBIT_WEBSOCKET_URL", "").strip(),
) )
_validate_settings(settings) _validate_settings(settings)
if settings.trading_mode == "live" and not settings.live_ready: if settings.trading_mode == "live" and not settings.live_ready:
+3 -1
View File
@@ -12,6 +12,7 @@ from fastapi.responses import JSONResponse, PlainTextResponse
from crypto_spot_bot.analytics import analytics_snapshot from crypto_spot_bot.analytics import analytics_snapshot
from crypto_spot_bot.auth import ApiAuthorizer from crypto_spot_bot.auth import ApiAuthorizer
from crypto_spot_bot.bot import CryptoSpotBot from crypto_spot_bot.bot import CryptoSpotBot
from crypto_spot_bot import __version__
from crypto_spot_bot.bybit import BybitClient from crypto_spot_bot.bybit import BybitClient
from crypto_spot_bot.config import Settings, load_settings, update_env_value from crypto_spot_bot.config import Settings, load_settings, update_env_value
from crypto_spot_bot.execution import LiveBroker, PaperBroker from crypto_spot_bot.execution import LiveBroker, PaperBroker
@@ -58,7 +59,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
finally: finally:
await bot.stop() await bot.stop()
app = FastAPI(title="Крипто спот-бот", lifespan=lifespan) app = FastAPI(title="Крипто спот-бот", version=__version__, lifespan=lifespan)
app.state.settings = settings app.state.settings = settings
app.state.storage = storage app.state.storage = storage
app.state.bot = bot app.state.bot = bot
@@ -76,6 +77,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
"running": bot.running, "running": bot.running,
"mode": settings.trading_mode, "mode": settings.trading_mode,
"auth_configured": authorizer.configured(), "auth_configured": authorizer.configured(),
"version": __version__,
} }
@app.get("/api/ready") @app.get("/api/ready")
+83 -13
View File
@@ -2,9 +2,11 @@ from __future__ import annotations
import base64 import base64
import hashlib import hashlib
import hmac
import json import json
import os import os
import re import re
import secrets
import shutil import shutil
import uuid import uuid
from datetime import UTC from datetime import UTC
@@ -20,11 +22,11 @@ ALLOWED_TRAINING_ARTIFACTS = {
"torch_retrain_guard.json", "torch_retrain_guard.json",
"torch_threshold_calibration.json", "torch_threshold_calibration.json",
} }
RUNNING_TIMEOUT = timedelta(hours=12) RUNNING_LEASE_TIMEOUT = timedelta(minutes=10)
ONLINE_WINDOW = timedelta(minutes=3) ONLINE_WINDOW = timedelta(minutes=3)
MAX_JOB_ATTEMPTS = 3
MAX_ARTIFACT_CHUNK_BYTES = 1024 * 1024 MAX_ARTIFACT_CHUNK_BYTES = 1024 * 1024
# Independent per-symbol ensembles are intentionally larger than pooled models. # Keep uploads bounded while leaving room for explicitly requested per-symbol bundles.
# Keep a bounded limit, but leave enough room for the supported 12-symbol bundle.
MAX_ARTIFACT_BYTES = 256 * 1024 * 1024 MAX_ARTIFACT_BYTES = 256 * 1024 * 1024
MAX_ARTIFACT_CHUNKS = 1024 MAX_ARTIFACT_CHUNKS = 1024
REQUIRED_MODEL_BUNDLE = set(ALLOWED_TRAINING_ARTIFACTS) REQUIRED_MODEL_BUNDLE = set(ALLOWED_TRAINING_ARTIFACTS)
@@ -52,7 +54,12 @@ class TrainingCoordinator:
existing = self._active_job(state) existing = self._active_job(state)
if existing is not None: if existing is not None:
self._save_state(state) self._save_state(state)
return {"queued": False, "reason": "active_job_exists", "job": existing, "status": self._public_status(state)} return {
"queued": False,
"reason": "active_job_exists",
"job": self._public_job(existing),
"status": self._public_status(state),
}
now = _now() now = _now()
job = { job = {
@@ -63,11 +70,16 @@ class TrainingCoordinator:
"parameters": _safe_parameters(payload.get("parameters")), "parameters": _safe_parameters(payload.get("parameters")),
"message": "", "message": "",
"artifacts": [], "artifacts": [],
"attempts": 0,
} }
state.setdefault("jobs", []).append(job) state.setdefault("jobs", []).append(job)
self._trim_jobs(state) self._trim_jobs(state)
self._save_state(state) self._save_state(state)
return {"queued": True, "job": job, "status": self._public_status(state)} return {
"queued": True,
"job": self._public_job(job),
"status": self._public_status(state),
}
def heartbeat(self, payload: dict[str, Any] | None = None) -> dict[str, Any]: def heartbeat(self, payload: dict[str, Any] | None = None) -> dict[str, Any]:
payload = payload or {} payload = payload or {}
@@ -91,12 +103,21 @@ class TrainingCoordinator:
return {"claimed": False, "job": None, "status": self._public_status(state)} return {"claimed": False, "job": None, "status": self._public_status(state)}
now = _now() now = _now()
lease_token = secrets.token_urlsafe(32)
job["status"] = "running" job["status"] = "running"
job["claimed_at"] = now job["claimed_at"] = now
job["updated_at"] = now
job["claimed_by"] = worker["id"] job["claimed_by"] = worker["id"]
job["worker"] = worker job["worker"] = worker
job["lease_token"] = lease_token
job["attempts"] = int(job.get("attempts", 0)) + 1
self._save_state(state) self._save_state(state)
return {"claimed": True, "job": job, "status": self._public_status(state)} return {
"claimed": True,
"job": self._public_job(job),
"lease_token": lease_token,
"status": self._public_status(state),
}
def save_artifact_chunk(self, job_id: str, payload: dict[str, Any]) -> dict[str, Any]: def save_artifact_chunk(self, job_id: str, payload: dict[str, Any]) -> dict[str, Any]:
job_id = _valid_job_id(job_id) job_id = _valid_job_id(job_id)
@@ -126,6 +147,7 @@ class TrainingCoordinator:
raise ValueError(f"training job not found: {job_id}") raise ValueError(f"training job not found: {job_id}")
if job.get("status") != "running" or not job.get("claimed_by"): if job.get("status") != "running" or not job.get("claimed_by"):
raise ValueError("training job is not claimed and running") raise ValueError("training job is not claimed and running")
self._require_lease(job, payload)
uploads = job.setdefault("uploads", {}) uploads = job.setdefault("uploads", {})
upload = uploads.setdefault(name, {"sha256": sha256, "total": total}) upload = uploads.setdefault(name, {"sha256": sha256, "total": total})
if upload.get("sha256") != sha256 or int(upload.get("total", 0)) != total: if upload.get("sha256") != sha256 or int(upload.get("total", 0)) != total:
@@ -138,6 +160,7 @@ class TrainingCoordinator:
received = sum(1 for part in range(total) if (chunk_dir / f"{part:06d}.part").is_file()) received = sum(1 for part in range(total) if (chunk_dir / f"{part:06d}.part").is_file())
if received < total: if received < total:
upload["received"] = received upload["received"] = received
job["updated_at"] = _now()
self._save_state(state) self._save_state(state)
return {"complete": False, "received": received, "total": total} return {"complete": False, "received": received, "total": total}
@@ -169,6 +192,7 @@ class TrainingCoordinator:
{"name": name, "sha256": sha256, "size": size, "staged_at": _now()} {"name": name, "sha256": sha256, "size": size, "staged_at": _now()}
) )
job["artifacts"] = artifacts job["artifacts"] = artifacts
job["updated_at"] = _now()
upload["received"] = total upload["received"] = total
upload["complete"] = True upload["complete"] = True
self._save_state(state) self._save_state(state)
@@ -184,6 +208,7 @@ class TrainingCoordinator:
raise ValueError(f"training job not found: {job_id}") raise ValueError(f"training job not found: {job_id}")
if job.get("status") != "running" or not job.get("claimed_by"): if job.get("status") != "running" or not job.get("claimed_by"):
raise ValueError("training job is not claimed and running") raise ValueError("training job is not claimed and running")
self._require_lease(job, payload)
if isinstance(payload.get("worker"), dict): if isinstance(payload.get("worker"), dict):
state["worker"] = self._worker_from_payload(payload["worker"]) state["worker"] = self._worker_from_payload(payload["worker"])
job["status"] = "running" job["status"] = "running"
@@ -194,7 +219,11 @@ class TrainingCoordinator:
if isinstance(payload.get("details"), dict): if isinstance(payload.get("details"), dict):
job["details"] = payload["details"] job["details"] = payload["details"]
self._save_state(state) self._save_state(state)
return {"ok": True, "job": job, "status": self._public_status(state)} return {
"ok": True,
"job": self._public_job(job),
"status": self._public_status(state),
}
def complete(self, job_id: str, payload: dict[str, Any] | None = None) -> dict[str, Any]: def complete(self, job_id: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
payload = payload or {} payload = payload or {}
@@ -206,6 +235,7 @@ class TrainingCoordinator:
raise ValueError(f"training job not found: {job_id}") raise ValueError(f"training job not found: {job_id}")
if job.get("status") != "running" or not job.get("claimed_by"): if job.get("status") != "running" or not job.get("claimed_by"):
raise ValueError("training job is not claimed and running") raise ValueError("training job is not claimed and running")
self._require_lease(job, payload)
success = bool(payload.get("success", payload.get("status") == "completed")) success = bool(payload.get("success", payload.get("status") == "completed"))
if success and job.get("artifacts"): if success and job.get("artifacts"):
promoted = self._validate_and_promote(job_id, job) promoted = self._validate_and_promote(job_id, job)
@@ -221,8 +251,13 @@ class TrainingCoordinator:
job["model_decision"] = ( job["model_decision"] = (
"accepted" if payload["summary"]["accepted"] else "rejected" "accepted" if payload["summary"]["accepted"] else "rejected"
) )
job.pop("lease_token", None)
self._save_state(state) self._save_state(state)
return {"ok": True, "job": job, "status": self._public_status(state)} return {
"ok": True,
"job": self._public_job(job),
"status": self._public_status(state),
}
def _validate_and_promote(self, job_id: str, job: dict[str, Any]) -> list[dict[str, Any]]: def _validate_and_promote(self, job_id: str, job: dict[str, Any]) -> list[dict[str, Any]]:
ready_dir = self.upload_root / job_id / "ready" ready_dir = self.upload_root / job_id / "ready"
@@ -327,11 +362,27 @@ class TrainingCoordinator:
"agent_recently_seen": recently_seen, "agent_recently_seen": recently_seen,
"agent_busy": agent_busy, "agent_busy": agent_busy,
"worker": worker, "worker": worker,
"active_job": active, "active_job": self._public_job(active),
"latest_job": latest, "latest_job": self._public_job(latest),
"pending_jobs": sum(1 for job in state.get("jobs", []) if job.get("status") == "pending"), "pending_jobs": sum(1 for job in state.get("jobs", []) if job.get("status") == "pending"),
} }
@staticmethod
def _public_job(job: dict[str, Any] | None) -> dict[str, Any] | None:
if job is None:
return None
public = dict(job)
public.pop("lease_token", None)
public.pop("uploads", None)
return public
@staticmethod
def _require_lease(job: dict[str, Any], payload: dict[str, Any]) -> None:
expected = str(job.get("lease_token") or "")
supplied = str(payload.get("lease_token") or "")
if not expected or not supplied or not hmac.compare_digest(expected, supplied):
raise ValueError("training job lease is invalid or expired")
def _active_job(self, state: dict[str, Any]) -> dict[str, Any] | None: def _active_job(self, state: dict[str, Any]) -> dict[str, Any] | None:
for job in reversed(state.get("jobs", [])): for job in reversed(state.get("jobs", [])):
if job.get("status") in {"pending", "running"}: if job.get("status") in {"pending", "running"}:
@@ -355,11 +406,30 @@ class TrainingCoordinator:
for job in state.get("jobs", []): for job in state.get("jobs", []):
if job.get("status") != "running": if job.get("status") != "running":
continue continue
claimed_at = _parse_time(str(job.get("claimed_at") or "")) lease_updated_at = _parse_time(
if claimed_at and now - claimed_at > RUNNING_TIMEOUT: str(job.get("updated_at") or job.get("claimed_at") or "")
)
if not lease_updated_at or now - lease_updated_at <= RUNNING_LEASE_TIMEOUT:
continue
job_id = str(job.get("id") or "")
if job_id:
_remove_tree(self.upload_root / job_id)
job.pop("lease_token", None)
job.pop("uploads", None)
attempts = int(job.get("attempts", 0))
if attempts < MAX_JOB_ATTEMPTS:
job["status"] = "pending"
job["phase"] = "queued"
job["progress_percent"] = 0
job["message"] = "training worker lease expired; queued for retry"
job["retry_queued_at"] = _now()
for key in ("claimed_at", "claimed_by", "worker", "updated_at"):
job.pop(key, None)
else:
job["status"] = "failed" job["status"] = "failed"
job["phase"] = "failed"
job["completed_at"] = _now() job["completed_at"] = _now()
job["message"] = "training worker timeout" job["message"] = "training worker lease expired after maximum retries"
def _trim_jobs(self, state: dict[str, Any]) -> None: def _trim_jobs(self, state: dict[str, Any]) -> None:
jobs = state.get("jobs", []) jobs = state.get("jobs", [])
+7 -1
View File
@@ -7,9 +7,9 @@ services:
environment: environment:
HOST: 0.0.0.0 HOST: 0.0.0.0
PYTHONDONTWRITEBYTECODE: "1" PYTHONDONTWRITEBYTECODE: "1"
user: "1000:1000"
init: true init: true
read_only: true read_only: true
pids_limit: 128
cap_drop: cap_drop:
- ALL - ALL
security_opt: security_opt:
@@ -27,4 +27,10 @@ services:
timeout: 10s timeout: 10s
retries: 3 retries: 3
start_period: 30s start_period: 30s
stop_grace_period: 30s
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
restart: unless-stopped restart: unless-stopped
+1 -1
View File
@@ -1,2 +1,2 @@
-r requirements.txt -r requirements.txt
pytest==8.4.2 pytest==9.1.1
+4 -4
View File
@@ -1,4 +1,4 @@
fastapi==0.115.6 fastapi==0.139.0
uvicorn[standard]==0.34.0 uvicorn[standard]==0.51.0
requests==2.32.3 requests==2.34.2
websockets==14.1 websockets==16.1
+3 -3
View File
@@ -78,7 +78,7 @@ def test_llm_advisor_is_disabled_by_default(tmp_path, monkeypatch) -> None:
assert settings.llm_advisor_enabled is False assert settings.llm_advisor_enabled is False
def test_default_symbols_are_fixed_trend_pairs(tmp_path, monkeypatch) -> None: def test_default_symbols_are_discovered_from_bybit(tmp_path, monkeypatch) -> None:
monkeypatch.delenv("AUTO_SELECT_SYMBOLS", raising=False) monkeypatch.delenv("AUTO_SELECT_SYMBOLS", raising=False)
monkeypatch.delenv("TOP_SYMBOLS_COUNT", raising=False) monkeypatch.delenv("TOP_SYMBOLS_COUNT", raising=False)
monkeypatch.delenv("SYMBOLS", raising=False) monkeypatch.delenv("SYMBOLS", raising=False)
@@ -89,9 +89,9 @@ def test_default_symbols_are_fixed_trend_pairs(tmp_path, monkeypatch) -> None:
settings = load_settings(env_file) settings = load_settings(env_file)
assert settings.auto_select_symbols is False assert settings.auto_select_symbols is True
assert settings.top_symbols_count == len(FIXED_SPOT_SYMBOLS) assert settings.top_symbols_count == len(FIXED_SPOT_SYMBOLS)
assert settings.symbols == FIXED_SPOT_SYMBOLS assert settings.symbols == ()
assert settings.strategy_mode == "torch_forecast" assert settings.strategy_mode == "torch_forecast"
assert settings.base_interval == "60" assert settings.base_interval == "60"
assert settings.trend_interval == "D" assert settings.trend_interval == "D"
+53 -6
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import base64 import base64
import hashlib import hashlib
import json import json
from datetime import UTC, datetime, timedelta
import pytest import pytest
@@ -16,6 +17,7 @@ def test_training_coordinator_claims_and_completes_job(tmp_path) -> None:
job_id = requested["job"]["id"] job_id = requested["job"]["id"]
heartbeat = coordinator.heartbeat({"worker_id": "win-1", "name": "DESKTOP-TMFDL0H"}) heartbeat = coordinator.heartbeat({"worker_id": "win-1", "name": "DESKTOP-TMFDL0H"})
claimed = coordinator.claim({"worker_id": "win-1", "name": "DESKTOP-TMFDL0H"}) claimed = coordinator.claim({"worker_id": "win-1", "name": "DESKTOP-TMFDL0H"})
lease_token = claimed["lease_token"]
assert requested["queued"] is True assert requested["queued"] is True
assert heartbeat["status"]["agent_online"] is True assert heartbeat["status"]["agent_online"] is True
@@ -25,14 +27,23 @@ def test_training_coordinator_claims_and_completes_job(tmp_path) -> None:
progress = coordinator.progress( progress = coordinator.progress(
job_id, job_id,
{"status": "running", "phase": "training", "progress_percent": 42, "message": "epoch 1"}, {
"status": "running",
"phase": "training",
"progress_percent": 42,
"message": "epoch 1",
"lease_token": lease_token,
},
) )
assert progress["job"]["phase"] == "training" assert progress["job"]["phase"] == "training"
assert progress["job"]["progress_percent"] == 42 assert progress["job"]["progress_percent"] == 42
assert coordinator.status()["active_job"]["message"] == "epoch 1" assert coordinator.status()["active_job"]["message"] == "epoch 1"
completed = coordinator.complete(job_id, {"success": True, "message": "ok"}) completed = coordinator.complete(
job_id,
{"success": True, "message": "ok", "lease_token": lease_token},
)
assert completed["job"]["status"] == "completed" assert completed["job"]["status"] == "completed"
assert coordinator.status()["active_job"] is None assert coordinator.status()["active_job"] is None
@@ -104,7 +115,7 @@ def test_training_coordinator_reports_worker_identity_from_heartbeat(tmp_path) -
def test_training_coordinator_records_rejected_candidate_as_completed_training(tmp_path) -> None: def test_training_coordinator_records_rejected_candidate_as_completed_training(tmp_path) -> None:
coordinator = TrainingCoordinator(tmp_path) coordinator = TrainingCoordinator(tmp_path)
job = coordinator.request_retrain({"source": "android"})["job"] job = coordinator.request_retrain({"source": "android"})["job"]
coordinator.claim({"worker_id": "worker-1"}) lease_token = coordinator.claim({"worker_id": "worker-1"})["lease_token"]
completed = coordinator.complete( completed = coordinator.complete(
job["id"], job["id"],
@@ -112,6 +123,7 @@ def test_training_coordinator_records_rejected_candidate_as_completed_training(t
"success": True, "success": True,
"message": "training completed; candidate rejected by quality gate", "message": "training completed; candidate rejected by quality gate",
"summary": {"accepted": False, "reason": "candidate_failed_honest_validation"}, "summary": {"accepted": False, "reason": "candidate_failed_honest_validation"},
"lease_token": lease_token,
}, },
) )
@@ -124,7 +136,7 @@ def test_training_coordinator_records_rejected_candidate_as_completed_training(t
def test_training_coordinator_accepts_chunked_artifact_upload(tmp_path) -> None: def test_training_coordinator_accepts_chunked_artifact_upload(tmp_path) -> None:
coordinator = TrainingCoordinator(tmp_path) coordinator = TrainingCoordinator(tmp_path)
job = coordinator.request_retrain({"source": "test"})["job"] job = coordinator.request_retrain({"source": "test"})["job"]
coordinator.claim({"worker_id": "test-worker"}) lease_token = coordinator.claim({"worker_id": "test-worker"})["lease_token"]
payload = b'{"type":"pytorch_recurrent_forecaster","symbols":{}}\n' payload = b'{"type":"pytorch_recurrent_forecaster","symbols":{}}\n'
sha256 = hashlib.sha256(payload).hexdigest() sha256 = hashlib.sha256(payload).hexdigest()
first = payload[:20] first = payload[:20]
@@ -138,6 +150,7 @@ def test_training_coordinator_accepts_chunked_artifact_upload(tmp_path) -> None:
"total": 2, "total": 2,
"sha256": sha256, "sha256": sha256,
"data_base64": base64.b64encode(first).decode("ascii"), "data_base64": base64.b64encode(first).decode("ascii"),
"lease_token": lease_token,
}, },
) )
part_2 = coordinator.save_artifact_chunk( part_2 = coordinator.save_artifact_chunk(
@@ -148,6 +161,7 @@ def test_training_coordinator_accepts_chunked_artifact_upload(tmp_path) -> None:
"total": 2, "total": 2,
"sha256": sha256, "sha256": sha256,
"data_base64": base64.b64encode(second).decode("ascii"), "data_base64": base64.b64encode(second).decode("ascii"),
"lease_token": lease_token,
}, },
) )
@@ -199,6 +213,35 @@ def test_running_claimed_job_keeps_agent_online_when_heartbeat_is_stale(tmp_path
assert status["agent_online"] is True assert status["agent_online"] is True
def test_stale_training_lease_is_requeued_and_old_lease_is_rejected(tmp_path) -> None:
coordinator = TrainingCoordinator(tmp_path)
job = coordinator.request_retrain({"source": "android"})["job"]
first_claim = coordinator.claim({"worker_id": "worker-1"})
state_path = tmp_path / "training_coordination.json"
state = json.loads(state_path.read_text(encoding="utf-8"))
state["jobs"][0]["updated_at"] = (
datetime.now(UTC) - timedelta(minutes=11)
).isoformat()
state_path.write_text(json.dumps(state), encoding="utf-8")
second_claim = coordinator.claim({"worker_id": "worker-2"})
assert second_claim["claimed"] is True
assert second_claim["job"]["id"] == job["id"]
assert second_claim["job"]["attempts"] == 2
assert second_claim["lease_token"] != first_claim["lease_token"]
with pytest.raises(ValueError, match="lease"):
coordinator.progress(
job["id"],
{
"phase": "training",
"progress_percent": 10,
"lease_token": first_claim["lease_token"],
},
)
def test_training_upload_rejects_unknown_job(tmp_path) -> None: def test_training_upload_rejects_unknown_job(tmp_path) -> None:
coordinator = TrainingCoordinator(tmp_path) coordinator = TrainingCoordinator(tmp_path)
payload = b"{}" payload = b"{}"
@@ -219,7 +262,7 @@ def test_training_upload_rejects_unknown_job(tmp_path) -> None:
def test_training_bundle_promotes_only_after_successful_guard(tmp_path) -> None: def test_training_bundle_promotes_only_after_successful_guard(tmp_path) -> None:
coordinator = TrainingCoordinator(tmp_path) coordinator = TrainingCoordinator(tmp_path)
job = coordinator.request_retrain({"source": "test"})["job"] job = coordinator.request_retrain({"source": "test"})["job"]
coordinator.claim({"worker_id": "worker-1"}) lease_token = coordinator.claim({"worker_id": "worker-1"})["lease_token"]
model = { model = {
"type": "pytorch_recurrent_forecaster", "type": "pytorch_recurrent_forecaster",
"symbols": { "symbols": {
@@ -260,10 +303,14 @@ def test_training_bundle_promotes_only_after_successful_guard(tmp_path) -> None:
"total": 1, "total": 1,
"sha256": hashlib.sha256(payload).hexdigest(), "sha256": hashlib.sha256(payload).hexdigest(),
"data_base64": base64.b64encode(payload).decode("ascii"), "data_base64": base64.b64encode(payload).decode("ascii"),
"lease_token": lease_token,
}, },
) )
completed = coordinator.complete(job["id"], {"success": True}) completed = coordinator.complete(
job["id"],
{"success": True, "lease_token": lease_token},
)
assert completed["job"]["status"] == "completed" assert completed["job"]["status"] == "completed"
assert json.loads((tmp_path / "lstm_forecaster.json").read_text())["symbols"]["BTCUSDT"] assert json.loads((tmp_path / "lstm_forecaster.json").read_text())["symbols"]["BTCUSDT"]
-94
View File
@@ -1,94 +0,0 @@
[CmdletBinding()]
param(
[string]$TaskName = "TradeBot PyTorch Forecaster Retrainer",
[int]$EveryHours = 6,
[string]$Symbols = "",
[int]$Limit = 3000,
[int]$Horizon = 0,
[string]$Horizons = "",
[string]$Features = "",
[string]$ContextSymbols = "",
[int]$FirstRunMinutes = 0,
[switch]$DeployToPi,
[string]$PiHost = "192.168.0.185",
[string]$PiUser = "sevenhill",
[string]$PiRoot = "/mnt/data/tradebot",
[string]$PiSshKeyPath = ""
)
$ErrorActionPreference = "Stop"
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
$Runner = Join-Path $RepoRoot "tools\run_torch_retrain.ps1"
if (-not (Test-Path $Runner)) {
throw "Runner not found: $Runner"
}
$LegacyTaskName = "TradeBot LSTM Retrainer"
if ($TaskName -ne $LegacyTaskName) {
$legacyTask = Get-ScheduledTask -TaskName $LegacyTaskName -ErrorAction SilentlyContinue
if ($legacyTask) {
Unregister-ScheduledTask -TaskName $LegacyTaskName -Confirm:$false
}
}
$actionArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$Runner`""
if ($Symbols) {
$actionArgs += " -Symbols `"$Symbols`""
}
if ($Limit -gt 0) {
$actionArgs += " -Limit $Limit"
}
if ($Horizon -gt 0) {
$actionArgs += " -Horizon $Horizon"
}
if ($Horizons) {
$actionArgs += " -Horizons `"$Horizons`""
}
if ($Features) {
$actionArgs += " -Features `"$Features`""
}
if ($ContextSymbols) {
$actionArgs += " -ContextSymbols `"$ContextSymbols`""
}
if ($DeployToPi) {
$actionArgs += " -DeployToPi"
}
if ($PiHost) {
$actionArgs += " -PiHost `"$PiHost`""
}
if ($PiUser) {
$actionArgs += " -PiUser `"$PiUser`""
}
if ($PiRoot) {
$actionArgs += " -PiRoot `"$PiRoot`""
}
if ($PiSshKeyPath) {
$actionArgs += " -PiSshKeyPath `"$PiSshKeyPath`""
}
$action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $actionArgs -WorkingDirectory $RepoRoot
$trigger = New-ScheduledTaskTrigger `
-Once `
-At (Get-Date).AddMinutes($(if ($FirstRunMinutes -gt 0) { $FirstRunMinutes } else { $EveryHours * 60 })) `
-RepetitionInterval (New-TimeSpan -Hours $EveryHours) `
-RepetitionDuration (New-TimeSpan -Days 3650)
$principal = New-ScheduledTaskPrincipal `
-UserId ([System.Security.Principal.WindowsIdentity]::GetCurrent().Name) `
-LogonType Interactive `
-RunLevel Limited
$settings = New-ScheduledTaskSettingsSet `
-StartWhenAvailable `
-MultipleInstances IgnoreNew `
-AllowStartIfOnBatteries `
-DontStopIfGoingOnBatteries
Register-ScheduledTask `
-TaskName $TaskName `
-Action $action `
-Trigger $trigger `
-Principal $principal `
-Settings $settings `
-Description "Retrains TradeBot PyTorch recurrent forecast parameters every $EveryHours hours." `
-Force | Out-Null
Write-Host "Registered scheduled task '$TaskName' every $EveryHours hours."
-152
View File
@@ -1,152 +0,0 @@
[CmdletBinding()]
param(
[int]$MinReplayTrades = 8,
[int]$MaxAttempts = 0,
[string]$Symbols = "",
[int]$Limit = 3000,
[switch]$DeployToPi,
[string]$PiHost = "192.168.0.185",
[string]$PiUser = "sevenhill",
[string]$PiRoot = "/mnt/data/tradebot",
[string]$PiSshKeyPath = "",
[int]$SeedStart = 0
)
$ErrorActionPreference = "Stop"
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
$RuntimeDir = Join-Path $RepoRoot "runtime"
$LoopLog = Join-Path $RuntimeDir "torch_retrain_until_replay8.log"
$GuardReport = Join-Path $RuntimeDir "torch_retrain_guard.json"
$ActiveCalibration = Join-Path $RuntimeDir "torch_threshold_calibration.json"
$Runner = Join-Path $RepoRoot "tools\run_torch_retrain.ps1"
New-Item -ItemType Directory -Force -Path $RuntimeDir | Out-Null
function Write-LoopLog {
param([string]$Message)
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ssK"
"[$timestamp] $Message" | Tee-Object -FilePath $LoopLog -Append
}
function ConvertTo-IntOrZero {
param($Value)
try {
if ($null -eq $Value) {
return 0
}
return [int]$Value
}
catch {
return 0
}
}
function Read-GuardSummary {
if (-not (Test-Path $GuardReport)) {
return [pscustomobject]@{
Accepted = $false
Reason = "guard_report_missing"
CandidateReplayTrades = 0
CurrentReplayTrades = 0
WalkForwardTrades = 0
}
}
try {
$payload = Get-Content -Raw -LiteralPath $GuardReport | ConvertFrom-Json
return [pscustomobject]@{
Accepted = [bool]$payload.accepted
Reason = [string]$payload.reason
CandidateReplayTrades = ConvertTo-IntOrZero $payload.candidate.full_replay.trades
CurrentReplayTrades = ConvertTo-IntOrZero $payload.current.full_replay.trades
WalkForwardTrades = ConvertTo-IntOrZero $payload.candidate.walk_forward_summary.trades
}
}
catch {
return [pscustomobject]@{
Accepted = $false
Reason = "guard_report_unreadable"
CandidateReplayTrades = 0
CurrentReplayTrades = 0
WalkForwardTrades = 0
}
}
}
function Read-ActiveReplayTrades {
if (-not (Test-Path $ActiveCalibration)) {
return 0
}
try {
$payload = Get-Content -Raw -LiteralPath $ActiveCalibration | ConvertFrom-Json
return ConvertTo-IntOrZero $payload.full_replay.trades
}
catch {
return 0
}
}
function Read-ActiveValidationPassed {
if (-not (Test-Path $ActiveCalibration)) {
return $false
}
try {
$payload = Get-Content -Raw -LiteralPath $ActiveCalibration | ConvertFrom-Json
return [bool]$payload.validation.passed
}
catch {
return $false
}
}
$attempt = 0
while ($true) {
$activeReplayTrades = Read-ActiveReplayTrades
if (Read-ActiveValidationPassed) {
Write-LoopLog "Stop condition reached: active calibration passed honest validation with full_replay.trades=$activeReplayTrades."
exit 0
}
$attempt += 1
if ($SeedStart -gt 0) {
$attemptSeed = $SeedStart + $attempt - 1
}
else {
$attemptSeed = Get-Random -Minimum 1 -Maximum 2147483647
}
Write-LoopLog "Attempt $attempt started; seed=$attemptSeed; target full_replay.trades >= $MinReplayTrades."
$runnerArgs = @(
"-NoProfile",
"-ExecutionPolicy", "Bypass",
"-File", $Runner,
"-Limit", $Limit.ToString(),
"-Seed", $attemptSeed.ToString()
)
if ($Symbols) {
$runnerArgs += @("-Symbols", $Symbols)
}
if ($DeployToPi) {
$runnerArgs += "-DeployToPi"
if ($PiHost) { $runnerArgs += @("-PiHost", $PiHost) }
if ($PiUser) { $runnerArgs += @("-PiUser", $PiUser) }
if ($PiRoot) { $runnerArgs += @("-PiRoot", $PiRoot) }
if ($PiSshKeyPath) { $runnerArgs += @("-PiSshKeyPath", $PiSshKeyPath) }
}
& powershell.exe @runnerArgs 2>&1 | Tee-Object -FilePath $LoopLog -Append
$runnerExit = $LASTEXITCODE
$summary = Read-GuardSummary
Write-LoopLog "Attempt $attempt finished; runner_exit=$runnerExit accepted=$($summary.Accepted) reason=$($summary.Reason) candidate_full_replay.trades=$($summary.CandidateReplayTrades) current_full_replay.trades=$($summary.CurrentReplayTrades) walk_forward.trades=$($summary.WalkForwardTrades)."
if ($summary.Accepted -and (Read-ActiveValidationPassed)) {
Write-LoopLog "Stop condition reached: accepted candidate passed honest validation with full_replay.trades=$($summary.CandidateReplayTrades)."
exit 0
}
if ($MaxAttempts -gt 0 -and $attempt -ge $MaxAttempts) {
Write-LoopLog "MaxAttempts=$MaxAttempts reached before replay target."
exit 2
}
Start-Sleep -Seconds 10
}
+5 -43
View File
@@ -22,12 +22,6 @@ param(
[int]$HoldoutWindow = 0, [int]$HoldoutWindow = 0,
[string]$Interval = "", [string]$Interval = "",
[string]$EnvFile = "", [string]$EnvFile = "",
[switch]$DeployToPi,
[string]$PiHost = "",
[string]$PiUser = "",
[string]$PiRoot = "",
[string]$PiSshKeyPath = "",
[switch]$NoPiRestart,
[switch]$Pooled, [switch]$Pooled,
[switch]$SkipGuard, [switch]$SkipGuard,
[switch]$ResumeCandidate [switch]$ResumeCandidate
@@ -105,44 +99,13 @@ function Test-TorchArtifactFile {
} }
} }
function Sync-AcceptedArtifactsToPi {
if (-not ($DeployToPi -or $env:TORCH_RETRAIN_DEPLOY_TO_PI)) {
Write-RetrainLog "Pi artifact sync disabled."
return
}
$syncScript = Join-Path $RepoRoot "tools\sync_torch_artifacts_to_pi.ps1"
if (-not (Test-Path $syncScript)) {
throw "Pi sync script not found: $syncScript"
}
$syncArgs = @(
"-NoProfile",
"-ExecutionPolicy", "Bypass",
"-File", $syncScript,
"-RepoRoot", $RepoRoot
)
if ($PiHost) { $syncArgs += @("-RemoteHost", $PiHost) }
if ($PiUser) { $syncArgs += @("-RemoteUser", $PiUser) }
if ($PiRoot) { $syncArgs += @("-RemoteRoot", $PiRoot) }
if ($PiSshKeyPath) { $syncArgs += @("-SshKeyPath", $PiSshKeyPath) }
if ($NoPiRestart) { $syncArgs += "-NoRestart" }
Write-RetrainLog "Syncing accepted Torch artifacts to Raspberry Pi."
& powershell.exe @syncArgs 2>&1 | Tee-Object -FilePath $LogFile -Append
if ($LASTEXITCODE -ne 0) {
throw "Pi artifact sync failed with exit code $LASTEXITCODE."
}
Write-RetrainLog "Pi artifact sync completed."
}
if (-not $Symbols -and $env:TORCH_RETRAIN_SYMBOLS) { $Symbols = $env:TORCH_RETRAIN_SYMBOLS } if (-not $Symbols -and $env:TORCH_RETRAIN_SYMBOLS) { $Symbols = $env:TORCH_RETRAIN_SYMBOLS }
if ($Limit -le 0) { if ($Limit -le 0) {
$Limit = if ($env:TORCH_RETRAIN_LIMIT) { [int]$env:TORCH_RETRAIN_LIMIT } else { 6000 } $Limit = if ($env:TORCH_RETRAIN_LIMIT) { [int]$env:TORCH_RETRAIN_LIMIT } else { 4000 }
} }
if (-not $Lookbacks) { $Lookbacks = if ($env:TORCH_RETRAIN_LOOKBACKS) { $env:TORCH_RETRAIN_LOOKBACKS } else { "32,64,128" } } if (-not $Lookbacks) { $Lookbacks = if ($env:TORCH_RETRAIN_LOOKBACKS) { $env:TORCH_RETRAIN_LOOKBACKS } else { "64" } }
if (-not $Architectures) { $Architectures = if ($env:TORCH_RETRAIN_ARCHITECTURES) { $env:TORCH_RETRAIN_ARCHITECTURES } else { "lstm,gru" } } if (-not $Architectures) { $Architectures = if ($env:TORCH_RETRAIN_ARCHITECTURES) { $env:TORCH_RETRAIN_ARCHITECTURES } else { "lstm" } }
if (-not $HiddenSizes) { $HiddenSizes = if ($env:TORCH_RETRAIN_HIDDEN_SIZES) { $env:TORCH_RETRAIN_HIDDEN_SIZES } else { "64,96" } } if (-not $HiddenSizes) { $HiddenSizes = if ($env:TORCH_RETRAIN_HIDDEN_SIZES) { $env:TORCH_RETRAIN_HIDDEN_SIZES } else { "64" } }
if (-not $Layers) { $Layers = if ($env:TORCH_RETRAIN_LAYERS) { $env:TORCH_RETRAIN_LAYERS } else { "2" } } if (-not $Layers) { $Layers = if ($env:TORCH_RETRAIN_LAYERS) { $env:TORCH_RETRAIN_LAYERS } else { "2" } }
if (-not $Dropouts) { $Dropouts = if ($env:TORCH_RETRAIN_DROPOUTS) { $env:TORCH_RETRAIN_DROPOUTS } else { "0.20" } } if (-not $Dropouts) { $Dropouts = if ($env:TORCH_RETRAIN_DROPOUTS) { $env:TORCH_RETRAIN_DROPOUTS } else { "0.20" } }
if ($Horizon -le 0) { $Horizon = if ($env:TORCH_RETRAIN_HORIZON) { [int]$env:TORCH_RETRAIN_HORIZON } else { 12 } } if ($Horizon -le 0) { $Horizon = if ($env:TORCH_RETRAIN_HORIZON) { [int]$env:TORCH_RETRAIN_HORIZON } else { 12 } }
@@ -154,7 +117,7 @@ if (-not $EnsembleSeeds) { $EnsembleSeeds = if ($env:TORCH_RETRAIN_ENSEMBLE_SEED
if ($SelectionFolds -le 0) { $SelectionFolds = if ($env:TORCH_RETRAIN_SELECTION_FOLDS) { [int]$env:TORCH_RETRAIN_SELECTION_FOLDS } else { 3 } } if ($SelectionFolds -le 0) { $SelectionFolds = if ($env:TORCH_RETRAIN_SELECTION_FOLDS) { [int]$env:TORCH_RETRAIN_SELECTION_FOLDS } else { 3 } }
if ($LearningRate -le 0) { $LearningRate = if ($env:TORCH_RETRAIN_LEARNING_RATE) { [double]$env:TORCH_RETRAIN_LEARNING_RATE } else { 0.0007 } } if ($LearningRate -le 0) { $LearningRate = if ($env:TORCH_RETRAIN_LEARNING_RATE) { [double]$env:TORCH_RETRAIN_LEARNING_RATE } else { 0.0007 } }
if ($WeightDecay -le 0) { $WeightDecay = if ($env:TORCH_RETRAIN_WEIGHT_DECAY) { [double]$env:TORCH_RETRAIN_WEIGHT_DECAY } else { 0.0005 } } if ($WeightDecay -le 0) { $WeightDecay = if ($env:TORCH_RETRAIN_WEIGHT_DECAY) { [double]$env:TORCH_RETRAIN_WEIGHT_DECAY } else { 0.0005 } }
if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 70 } } if ($Epochs -le 0) { $Epochs = if ($env:TORCH_RETRAIN_EPOCHS) { [int]$env:TORCH_RETRAIN_EPOCHS } else { 50 } }
if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 8 } } if ($Patience -le 0) { $Patience = if ($env:TORCH_RETRAIN_PATIENCE) { [int]$env:TORCH_RETRAIN_PATIENCE } else { 8 } }
if ($ValidationWindow -le 0) { $ValidationWindow = if ($env:TORCH_RETRAIN_VALIDATION_WINDOW) { [int]$env:TORCH_RETRAIN_VALIDATION_WINDOW } else { 720 } } if ($ValidationWindow -le 0) { $ValidationWindow = if ($env:TORCH_RETRAIN_VALIDATION_WINDOW) { [int]$env:TORCH_RETRAIN_VALIDATION_WINDOW } else { 720 } }
if ($HoldoutWindow -le 0) { $HoldoutWindow = if ($env:TORCH_RETRAIN_HOLDOUT_WINDOW) { [int]$env:TORCH_RETRAIN_HOLDOUT_WINDOW } else { 1000 } } if ($HoldoutWindow -le 0) { $HoldoutWindow = if ($env:TORCH_RETRAIN_HOLDOUT_WINDOW) { [int]$env:TORCH_RETRAIN_HOLDOUT_WINDOW } else { 1000 } }
@@ -302,7 +265,6 @@ try {
Write-RetrainLog "Updated active threshold calibration: $(Join-Path $RuntimeDir "torch_threshold_calibration.json")" Write-RetrainLog "Updated active threshold calibration: $(Join-Path $RuntimeDir "torch_threshold_calibration.json")"
} }
Write-RetrainLog "Candidate accepted by guard. Active artifact: $ModelFile" Write-RetrainLog "Candidate accepted by guard. Active artifact: $ModelFile"
Sync-AcceptedArtifactsToPi
} }
catch { catch {
Write-RetrainLog "ERROR: $($_.Exception.Message)" Write-RetrainLog "ERROR: $($_.Exception.Message)"
-95
View File
@@ -1,95 +0,0 @@
[CmdletBinding()]
param(
[string]$RepoRoot = "",
[string]$RemoteHost = "",
[string]$RemoteUser = "",
[string]$RemoteRoot = "",
[string]$SshKeyPath = "",
[string]$ServiceName = "tradebot",
[switch]$NoRestart,
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
if (-not $RepoRoot) { $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path }
if (-not $RemoteHost -and $env:TORCH_DEPLOY_PI_HOST) { $RemoteHost = $env:TORCH_DEPLOY_PI_HOST }
if (-not $RemoteUser -and $env:TORCH_DEPLOY_PI_USER) { $RemoteUser = $env:TORCH_DEPLOY_PI_USER }
if (-not $RemoteRoot -and $env:TORCH_DEPLOY_PI_ROOT) { $RemoteRoot = $env:TORCH_DEPLOY_PI_ROOT }
if (-not $SshKeyPath -and $env:TORCH_DEPLOY_PI_SSH_KEY) { $SshKeyPath = $env:TORCH_DEPLOY_PI_SSH_KEY }
if (-not $RemoteHost) { $RemoteHost = "192.168.0.185" }
if (-not $RemoteUser) { $RemoteUser = "sevenhill" }
if (-not $RemoteRoot) { $RemoteRoot = "/mnt/data/tradebot" }
$RuntimeDir = Join-Path $RepoRoot "runtime"
$artifactNames = @(
"lstm_forecaster.json",
"torch_retrain_guard.json",
"torch_threshold_calibration.json"
)
$localFiles = @()
foreach ($name in $artifactNames) {
$path = Join-Path $RuntimeDir $name
if (Test-Path $path) {
$localFiles += (Resolve-Path $path).Path
}
}
if ($localFiles.Count -eq 0) {
throw "No Torch artifacts found in $RuntimeDir."
}
function ConvertTo-RemoteSingleQuoted {
param([string]$Value)
return "'" + ($Value -replace "'", "'\''") + "'"
}
function Invoke-LoggedCommand {
param(
[string]$Exe,
[string[]]$Arguments
)
$rendered = @($Exe) + $Arguments
Write-Host ($rendered -join " ")
if ($DryRun) {
return
}
& $Exe @Arguments
if ($LASTEXITCODE -ne 0) {
throw "$Exe failed with exit code $LASTEXITCODE."
}
}
$ssh = (Get-Command "ssh.exe" -ErrorAction SilentlyContinue)
if (-not $ssh) { $ssh = Get-Command "ssh" -ErrorAction Stop }
$scp = (Get-Command "scp.exe" -ErrorAction SilentlyContinue)
if (-not $scp) { $scp = Get-Command "scp" -ErrorAction Stop }
$commonSshArgs = @("-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=15")
if ($SshKeyPath) {
$expandedKey = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($SshKeyPath)
$commonSshArgs += @("-i", $expandedKey)
}
$remote = "${RemoteUser}@${RemoteHost}"
$remoteRuntime = "$RemoteRoot/runtime"
$remoteIncoming = "$remoteRuntime/.incoming-torch"
$mkdirCommand = "mkdir -p $(ConvertTo-RemoteSingleQuoted $remoteIncoming) $(ConvertTo-RemoteSingleQuoted $remoteRuntime)"
Invoke-LoggedCommand $ssh.Source (@($commonSshArgs + @($remote, $mkdirCommand)))
$destination = "${remote}:$remoteIncoming/"
Invoke-LoggedCommand $scp.Source (@($commonSshArgs + $localFiles + @($destination)))
$moveParts = @()
foreach ($path in $localFiles) {
$name = Split-Path $path -Leaf
$moveParts += "mv -f $(ConvertTo-RemoteSingleQuoted "$remoteIncoming/$name") $(ConvertTo-RemoteSingleQuoted "$remoteRuntime/$name")"
}
$moveCommand = $moveParts -join " && "
Invoke-LoggedCommand $ssh.Source (@($commonSshArgs + @($remote, $moveCommand)))
if (-not $NoRestart) {
$restartCommand = "cd $(ConvertTo-RemoteSingleQuoted $RemoteRoot) && docker compose restart $(ConvertTo-RemoteSingleQuoted $ServiceName)"
Invoke-LoggedCommand $ssh.Source (@($commonSshArgs + @($remote, $restartCommand)))
}
Write-Host "Synced Torch artifacts to ${remote}:$remoteRuntime"
+88 -14
View File
@@ -54,23 +54,34 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
return return
job = claim.get("job") if isinstance(claim.get("job"), dict) else {} job = claim.get("job") if isinstance(claim.get("job"), dict) else {}
job_id = str(job.get("id") or "") job_id = str(job.get("id") or "")
lease_token = str(claim.get("lease_token") or "")
if not job_id: if not job_id:
return return
if not lease_token:
raise RuntimeError("training server did not issue a job lease")
log(log_path, f"Claimed retrain job {job_id}") log(log_path, f"Claimed retrain job {job_id}")
report_progress(args, job_id, "running", "claimed", 2, "Задание получено Windows-agent") report_progress(args, job_id, lease_token, "running", "claimed", 2, "Задание получено Windows-agent")
success = False success = False
message = "" message = ""
summary: dict[str, Any] = {} summary: dict[str, Any] = {}
try: try:
run_retrain(args, job_id, job, repo_root, log_path) run_retrain(args, job_id, lease_token, job, repo_root, log_path)
summary = read_json(runtime_dir / "torch_retrain_guard.json") summary = read_json(runtime_dir / "torch_retrain_guard.json")
accepted = summary.get("accepted") is True accepted = summary.get("accepted") is True
if accepted: if accepted:
report_progress(args, job_id, "running", "uploading", 72, "Обучение завершено, загружаю артефакты") report_progress(
args,
job_id,
lease_token,
"running",
"uploading",
72,
"Обучение завершено, загружаю артефакты",
)
for name in ARTIFACT_NAMES: for name in ARTIFACT_NAMES:
path = runtime_dir / name path = runtime_dir / name
if path.is_file(): if path.is_file():
upload_artifact(args, job_id, path, log_path) upload_artifact(args, job_id, lease_token, path, log_path)
message = "training completed; candidate accepted" message = "training completed; candidate accepted"
log(log_path, f"Completed retrain job {job_id}; candidate accepted") log(log_path, f"Completed retrain job {job_id}; candidate accepted")
else: else:
@@ -82,11 +93,23 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
message = str(exc) message = str(exc)
log(log_path, f"Job {job_id} failed: {message}") log(log_path, f"Job {job_id} failed: {message}")
finally: finally:
payload = {"success": success, "message": message, "summary": summary} payload = {
"success": success,
"message": message,
"summary": summary,
"lease_token": lease_token,
}
api_json(args, f"/api/training/jobs/{job_id}/complete", payload) api_json(args, f"/api/training/jobs/{job_id}/complete", payload)
def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo_root: Path, log_path: Path) -> None: def run_retrain(
args: argparse.Namespace,
job_id: str,
lease_token: str,
job: dict[str, Any],
repo_root: Path,
log_path: Path,
) -> None:
script = repo_root / "tools" / "run_torch_retrain.ps1" script = repo_root / "tools" / "run_torch_retrain.ps1"
if not script.is_file(): if not script.is_file():
raise RuntimeError(f"retrain script not found: {script}") raise RuntimeError(f"retrain script not found: {script}")
@@ -126,12 +149,20 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
value = parameters.get(key) value = parameters.get(key)
if value not in (None, ""): if value not in (None, ""):
cmd.extend([ps_arg, str(value)]) cmd.extend([ps_arg, str(value)])
if parameters.get("pooled") is True: if parameters.get("pooled", True) is True:
cmd.append("-Pooled") cmd.append("-Pooled")
if parameters.get("resume_candidate") is True: if parameters.get("resume_candidate") is True:
cmd.append("-ResumeCandidate") cmd.append("-ResumeCandidate")
log(log_path, "Running retrain: " + " ".join(quote_for_log(part) for part in cmd)) log(log_path, "Running retrain: " + " ".join(quote_for_log(part) for part in cmd))
report_progress(args, job_id, "running", "training", 8, "PyTorch retrain запущен") report_progress(
args,
job_id,
lease_token,
"running",
"training",
8,
"PyTorch retrain запущен",
)
line_count = 0 line_count = 0
output_queue: queue.Queue[str] = queue.Queue() output_queue: queue.Queue[str] = queue.Queue()
@@ -175,7 +206,16 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
report_message = last_message report_message = last_message
if not got_line: if not got_line:
report_message = training_heartbeat_message(now, started_at, last_output_at, last_message) report_message = training_heartbeat_message(now, started_at, last_output_at, last_message)
safe_report_progress(args, job_id, "running", "training", progress, report_message, log_path) safe_report_progress(
args,
job_id,
lease_token,
"running",
"training",
progress,
report_message,
log_path,
)
last_report_at = now last_report_at = now
if process.poll() is not None and output_queue.empty(): if process.poll() is not None and output_queue.empty():
@@ -185,7 +225,15 @@ def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo
code = process.wait() code = process.wait()
if code != 0: if code != 0:
raise RuntimeError(f"retrain failed with exit code {code}") raise RuntimeError(f"retrain failed with exit code {code}")
report_progress(args, job_id, "running", "guard", 70, "Guard завершён, подготавливаю артефакты") report_progress(
args,
job_id,
lease_token,
"running",
"guard",
70,
"Guard завершён, подготавливаю артефакты",
)
def friendly_training_message(message: str) -> str: def friendly_training_message(message: str) -> str:
@@ -282,7 +330,13 @@ def format_duration(seconds: float) -> str:
return f"{seconds_part}с" return f"{seconds_part}с"
def upload_artifact(args: argparse.Namespace, job_id: str, path: Path, log_path: Path) -> None: def upload_artifact(
args: argparse.Namespace,
job_id: str,
lease_token: str,
path: Path,
log_path: Path,
) -> None:
digest = hashlib.sha256(path.read_bytes()).hexdigest() digest = hashlib.sha256(path.read_bytes()).hexdigest()
size = path.stat().st_size size = path.stat().st_size
chunk_size = max(64 * 1024, args.chunk_size) chunk_size = max(64 * 1024, args.chunk_size)
@@ -297,16 +351,26 @@ def upload_artifact(args: argparse.Namespace, job_id: str, path: Path, log_path:
"total": total, "total": total,
"sha256": digest, "sha256": digest,
"data_base64": base64.b64encode(data).decode("ascii"), "data_base64": base64.b64encode(data).decode("ascii"),
"lease_token": lease_token,
} }
api_json(args, f"/api/training/jobs/{job_id}/artifacts/chunk", payload, timeout=120) api_json(args, f"/api/training/jobs/{job_id}/artifacts/chunk", payload, timeout=120)
if index == 0 or index == total - 1 or index % 10 == 0: if index == 0 or index == total - 1 or index % 10 == 0:
progress = 72 + int(((index + 1) / total) * 23) progress = 72 + int(((index + 1) / total) * 23)
report_progress(args, job_id, "running", "uploading", progress, f"Загружаю {path.name}: {index + 1}/{total}") report_progress(
args,
job_id,
lease_token,
"running",
"uploading",
progress,
f"Загружаю {path.name}: {index + 1}/{total}",
)
def report_progress( def report_progress(
args: argparse.Namespace, args: argparse.Namespace,
job_id: str, job_id: str,
lease_token: str,
status: str, status: str,
phase: str, phase: str,
progress_percent: int, progress_percent: int,
@@ -321,6 +385,7 @@ def report_progress(
"progress_percent": progress_percent, "progress_percent": progress_percent,
"message": message, "message": message,
"worker": worker_payload(args, Path(args.repo_root).resolve()), "worker": worker_payload(args, Path(args.repo_root).resolve()),
"lease_token": lease_token,
}, },
) )
@@ -328,6 +393,7 @@ def report_progress(
def safe_report_progress( def safe_report_progress(
args: argparse.Namespace, args: argparse.Namespace,
job_id: str, job_id: str,
lease_token: str,
status: str, status: str,
phase: str, phase: str,
progress_percent: int, progress_percent: int,
@@ -337,7 +403,15 @@ def safe_report_progress(
last_error: Exception | None = None last_error: Exception | None = None
for attempt in range(1, 4): for attempt in range(1, 4):
try: try:
report_progress(args, job_id, status, phase, progress_percent, message) report_progress(
args,
job_id,
lease_token,
status,
phase,
progress_percent,
message,
)
return return
except Exception as exc: # noqa: BLE001 - keep the local training process alive. except Exception as exc: # noqa: BLE001 - keep the local training process alive.
last_error = exc last_error = exc
@@ -385,7 +459,7 @@ def worker_payload(args: argparse.Namespace, repo_root: Path) -> dict[str, Any]:
"worker_id": args.worker_id or f"{name}:{repo_root}", "worker_id": args.worker_id or f"{name}:{repo_root}",
"name": name, "name": name,
"path": str(repo_root), "path": str(repo_root),
"version": "1", "version": "2",
} }