Show retrain progress in Android

android/TradeBotMonitor/app/build.gradle.kts

@@ -10,7 +10,7 @@ android {
         applicationId = "xyz.kusoft.tradebotmonitor"
         minSdk = 26
         targetSdk = 36
-        versionCode = 6
+        versionCode = 7
-        versionName = "0.2.3"
+        versionName = "0.2.4"
     }
 }

android/TradeBotMonitor/app/src/main/java/xyz/kusoft/tradebotmonitor/MainActivity.kt

@@ -38,6 +38,7 @@ class MainActivity : Activity() {
     private var lastError: String = ""
     private var activeTab: String = "ai"
     private var palette: AppPalette = AppPalette.dark()
+    private var trainingStatusSignature: String = ""
 
     private val refreshRunnable = object : Runnable {
         override fun run() {
@@ -103,10 +104,13 @@ class MainActivity : Activity() {
                 mainHandler.post {
                     val hadSnapshot = snapshot != null
                     val hadError = lastError.isNotBlank()
+                    val newTrainingStatusSignature = trainingStatusSignature(data.retrain)
+                    val trainingChanged = newTrainingStatusSignature != trainingStatusSignature
                     snapshot = data
+                    trainingStatusSignature = newTrainingStatusSignature
                     lastError = ""
                     ensureSelectedSymbol()
-                    if (shouldRenderAfterRefresh(silent, hadSnapshot, hadError)) {
+                    if (shouldRenderAfterRefresh(silent, hadSnapshot, hadError, trainingChanged)) {
                         render()
                     }
                 }
@@ -116,7 +120,7 @@ class MainActivity : Activity() {
                     val hadError = lastError.isNotBlank()
                     lastError = error.message ?: "ошибка подключения"
                     if (!silent) toast(lastError)
-                    if (shouldRenderAfterRefresh(silent, hadSnapshot, hadError)) {
+                    if (shouldRenderAfterRefresh(silent, hadSnapshot, hadError, trainingChanged = false)) {
                         render()
                     }
                 }
@@ -124,7 +128,7 @@ class MainActivity : Activity() {
         }
     }
 
-    private fun shouldRenderAfterRefresh(silent: Boolean, hadSnapshot: Boolean, hadError: Boolean): Boolean {
+    private fun shouldRenderAfterRefresh(silent: Boolean, hadSnapshot: Boolean, hadError: Boolean, trainingChanged: Boolean): Boolean {
         val focused = contentHost.findFocus()
         if (activeTab == "settings" && focused is EditText) {
             return false
@@ -132,6 +136,9 @@ class MainActivity : Activity() {
         if (!silent) {
             return true
         }
+        if (activeTab == "settings" && trainingChanged) {
+            return true
+        }
         return !hadSnapshot || hadError
     }
 
@@ -568,42 +575,71 @@ class MainActivity : Activity() {
     private fun trainingComputerPanel(retrain: JSONObject): View =
         LinearLayout(this).apply {
             val coordination = retrain.optJSONObject("coordination") ?: JSONObject()
-            val activeJob = coordination.optJSONObject("active_job")
             val agentOnline = coordination.optBoolean("agent_online", false)
             orientation = LinearLayout.VERTICAL
             addView(text("Компьютер обучения", 12f, Typeface.NORMAL, palette.muted))
             addView(text(prefs.trainingComputerName, 18f, Typeface.BOLD, palette.green).top(dp(5)))
             addView(text(prefs.trainingComputerPath, 12f, Typeface.NORMAL, palette.muted).top(dp(4)))
-            addView(keyValueLine("Статус", if (prefs.trainingComputerPinned) "закреплен" else "не закреплен", if (prefs.trainingComputerPinned) palette.green else palette.amber).top(dp(8)))
             addView(keyValueLine("Связь агента", if (agentOnline) "онлайн" else "нет связи", if (agentOnline) palette.green else palette.amber).top(dp(4)))
-            if (activeJob != null) {
+            addView(text("Бот ставит задания через tb.kusoft.xyz, а этот Windows-agent сам забирает их через интернет и возвращает результат.", 12f, Typeface.NORMAL, palette.muted).top(dp(8)))
-                addView(keyValueLine("Задание", activeJob.optStringClean("status").ifBlank { "активно" }).top(dp(4)))
+        }
+
+    private fun trainingProcessPanel(coordination: JSONObject): View =
+        LinearLayout(this).apply {
+            val activeJob = coordination.optJSONObject("active_job")
+            val latestJob = coordination.optJSONObject("latest_job")
+            val job = activeJob ?: latestJob
+            orientation = LinearLayout.VERTICAL
+            addView(text("Процесс переобучения", 12f, Typeface.NORMAL, palette.muted))
+            if (job == null) {
+                val agentOnline = coordination.optBoolean("agent_online", false)
+                addView(keyValueLine("Состояние", if (agentOnline) "готов к запуску" else "ждет Windows-agent", if (agentOnline) palette.green else palette.amber).top(dp(8)))
+                return@apply
+            }
+            val status = job.optStringClean("status")
+            val phase = job.optStringClean("phase")
+            val progress = job.optInt("progress_percent", if (status == "completed") 100 else 0).coerceIn(0, 100)
+            val message = job.optStringClean("message")
+            addView(keyValueLine("Состояние", trainingJobLabel(status, phase), trainingJobColor(status, phase)).top(dp(8)))
+            addView(allocationBar(progress / 100.0).top(dp(10)))
+            addView(keyValueLine("Прогресс", "$progress%", trainingJobColor(status, phase)).top(dp(8)))
+            if (message.isNotBlank()) {
+                addView(text(message.take(160), 12f, Typeface.NORMAL, palette.text).top(dp(8)))
+            }
+            val completedAt = job.optStringClean("completed_at")
+            if (completedAt.isNotBlank()) {
+                addView(keyValueLine("Завершено", completedAt.take(19).replace("T", " ")).top(dp(8)))
+            }
+            val artifacts = job.optJSONArray("artifacts")
+            if (artifacts != null && artifacts.length() > 0) {
+                addView(keyValueLine("Артефакты", artifacts.length().toString(), palette.green).top(dp(4)))
             }
-            addView(text("Переобучение Torch закреплено за этой Windows-машиной. Телефон только управляет запуском и расписанием, Pi только исполняет бота.", 12f, Typeface.NORMAL, palette.muted).top(dp(8)))
-            addView(actionRow(
-                "Закрепить эту машину" to {
-                    prefs.pinDefaultTrainingComputer()
-                    toast("Компьютер обучения закреплен")
-                    render()
-                },
-            ).top(dp(10)))
         }
 
     private fun retrainSettingsBlock(retrain: JSONObject, backtest: JSONObject): View {
+        val coordination = retrain.optJSONObject("coordination") ?: JSONObject()
+        val activeJob = coordination.optJSONObject("active_job")
+        val isTrainingActive = activeJob?.optStringClean("status") in setOf("pending", "running")
         return LinearLayout(this).apply {
             orientation = LinearLayout.VERTICAL
-            addView(keyValueLine("Доступно", yesNo(retrain.optBoolean("available", false))))
-            addView(keyValueLine("Принята новая модель", yesNo(retrain.optBoolean("accepted", false))).top(dp(4)))
             addView(trainingComputerPanel(retrain).top(dp(12)))
             addView(thinDivider().top(dp(12)))
+            addView(trainingProcessPanel(coordination))
+            if (retrain.optBoolean("available", false)) {
+                addView(keyValueLine("Последний guard", if (retrain.optBoolean("accepted", false)) "модель принята" else "модель отклонена", if (retrain.optBoolean("accepted", false)) palette.green else palette.amber).top(dp(10)))
+            }
             val replay = backtest.optJSONObject("full_replay") ?: JSONObject()
             addView(keyValueLine("Replay сделок", replay.optInt("trades", 0).toString()).top(dp(4)))
             addView(keyValueLine("Replay PnL", signedMoney(replay.optDouble("net_pnl", 0.0)), colorForSigned(replay.optDouble("net_pnl", 0.0))).top(dp(4)))
+            if (isTrainingActive) {
+                addView(disabledActionButton("Обучение уже идет").top(dp(10)))
+            } else {
                 addView(actionRow(
                     "Запустить сейчас" to {
                         triggerRetrainNow()
                     },
                 ).top(dp(10)))
+            }
             addView(actionRow(
                 "1ч" to { setRetrainInterval(1) },
                 "3ч" to { setRetrainInterval(3) },
@@ -842,6 +878,21 @@ class MainActivity : Activity() {
             isFocusableInTouchMode = false
         }
 
+    private fun disabledActionButton(title: String): TextView =
+        TextView(this).apply {
+            text = title
+            textSize = 13f
+            typeface = Typeface.DEFAULT_BOLD
+            gravity = Gravity.CENTER
+            setTextColor(palette.muted)
+            background = boxDrawable(withAlpha(palette.muted, if (palette.isDark) 14 else 8), palette.line, 7)
+            isEnabled = false
+            isClickable = false
+            isFocusable = false
+            isFocusableInTouchMode = false
+            layoutParams = LinearLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, dp(48))
+        }
+
     private fun navItem(title: String, selected: Boolean, action: () -> Unit): TextView =
         TextView(this).apply {
             text = title
@@ -957,9 +1008,15 @@ class MainActivity : Activity() {
         executor.execute {
             try {
                 val message = TradeBotApi(prefs.apiBaseUrl, prefs.commandToken).requestRetrain()
-                mainHandler.post { toast(message.take(180)) }
+                mainHandler.post {
+                    toast(message.take(180))
+                    refreshData(silent = false)
+                }
             } catch (error: Exception) {
-                mainHandler.post { toast(error.message ?: "retrain не запущен") }
+                mainHandler.post {
+                    toast(error.message ?: "retrain не запущен")
+                    refreshData(silent = false)
+                }
             }
         }
     }
@@ -984,6 +1041,30 @@ class MainActivity : Activity() {
         return if (index >= 0) index else FIXED_SYMBOLS.size + 1
     }
 
+    private fun trainingStatusSignature(retrain: JSONObject): String =
+        (retrain.optJSONObject("coordination") ?: JSONObject()).toString()
+
+    private fun trainingJobLabel(status: String, phase: String): String =
+        when {
+            status == "pending" -> "ждет Windows-agent"
+            phase == "claimed" -> "задание получено"
+            phase == "training" -> "идет обучение"
+            phase == "guard" -> "проверка качества"
+            phase == "uploading" -> "загрузка результата"
+            status == "completed" -> "завершено успешно"
+            status == "failed" -> "ошибка обучения"
+            status == "running" -> "идет работа"
+            else -> "готов к запуску"
+        }
+
+    private fun trainingJobColor(status: String, phase: String): Int =
+        when {
+            status == "completed" -> palette.green
+            status == "failed" -> palette.red
+            status == "pending" || status == "running" || phase.isNotBlank() -> palette.amber
+            else -> palette.text
+        }
+
     private fun normalizedAction(raw: String?): String {
         val value = raw.orEmpty().uppercase(Locale.US)
         return when {

crypto_spot_bot/dashboard.py

@@ -144,6 +144,13 @@ def create_app(settings: Settings | None = None) -> FastAPI:
         except ValueError as exc:
             raise HTTPException(status_code=400, detail=str(exc)) from exc
 
+    @app.post("/api/training/jobs/{job_id}/progress")
+    async def training_progress(job_id: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
+        try:
+            return training.progress(job_id, payload)
+        except ValueError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+
     @app.post("/api/training/jobs/{job_id}/complete")
     async def training_complete(job_id: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
         try:

crypto_spot_bot/training_coordination.py

@@ -139,6 +139,25 @@ class TrainingCoordinator:
                 self._save_state(state)
         return {"complete": True, "name": name, "sha256": sha256}
 
+    def progress(self, job_id: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
+        payload = payload or {}
+        with self._lock:
+            state = self._load_state()
+            job = self._job_by_id(state, job_id)
+            if job is None:
+                raise ValueError(f"training job not found: {job_id}")
+            if isinstance(payload.get("worker"), dict):
+                state["worker"] = self._worker_from_payload(payload["worker"])
+            job["status"] = str(payload.get("status") or job.get("status") or "running")
+            job["phase"] = str(payload.get("phase") or job.get("phase") or "running")
+            job["message"] = str(payload.get("message") or job.get("message") or "")
+            job["progress_percent"] = _coerce_percent(payload.get("progress_percent"), job.get("progress_percent", 0))
+            job["updated_at"] = _now()
+            if isinstance(payload.get("details"), dict):
+                job["details"] = payload["details"]
+            self._save_state(state)
+            return {"ok": True, "job": job, "status": self._public_status(state)}
+
     def complete(self, job_id: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
         payload = payload or {}
         with self._lock:
@@ -148,6 +167,8 @@ class TrainingCoordinator:
                 raise ValueError(f"training job not found: {job_id}")
             success = bool(payload.get("success", payload.get("status") == "completed"))
             job["status"] = "completed" if success else "failed"
+            job["phase"] = "completed" if success else "failed"
+            job["progress_percent"] = 100 if success else _coerce_percent(payload.get("progress_percent"), job.get("progress_percent", 0))
             job["completed_at"] = _now()
             job["message"] = str(payload.get("message") or "")
             if isinstance(payload.get("summary"), dict):
@@ -257,6 +278,17 @@ def _parse_time(value: str) -> datetime | None:
         return None
 
 
+def _coerce_percent(value: Any, default: Any = 0) -> int:
+    try:
+        number = int(float(value))
+    except (TypeError, ValueError):
+        try:
+            number = int(float(default))
+        except (TypeError, ValueError):
+            number = 0
+    return max(0, min(number, 100))
+
+
 def _remove_tree(path: Path) -> None:
     if not path.exists():
         return

tests/test_training_coordination.py

@@ -20,6 +20,15 @@ def test_training_coordinator_claims_and_completes_job(tmp_path) -> None:
     assert claimed["job"]["id"] == job_id
     assert coordinator.status()["active_job"]["status"] == "running"
 
+    progress = coordinator.progress(
+        job_id,
+        {"status": "running", "phase": "training", "progress_percent": 42, "message": "epoch 1"},
+    )
+
+    assert progress["job"]["phase"] == "training"
+    assert progress["job"]["progress_percent"] == 42
+    assert coordinator.status()["active_job"]["message"] == "epoch 1"
+
     completed = coordinator.complete(job_id, {"success": True, "message": "ok"})
 
     assert completed["job"]["status"] == "completed"

tools/windows_training_agent.py

@@ -54,12 +54,14 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
     if not job_id:
         return
     log(log_path, f"Claimed retrain job {job_id}")
+    report_progress(args, job_id, "running", "claimed", 2, "Задание получено Windows-agent")
     success = False
     message = ""
     summary: dict[str, Any] = {}
     try:
-        run_retrain(job, repo_root, log_path)
+        run_retrain(args, job_id, job, repo_root, log_path)
         summary = read_json(runtime_dir / "torch_retrain_guard.json")
+        report_progress(args, job_id, "running", "uploading", 72, "Обучение завершено, загружаю артефакты")
         for name in ARTIFACT_NAMES:
             path = runtime_dir / name
             if path.is_file():
@@ -75,7 +77,7 @@ def poll_once(args: argparse.Namespace, repo_root: Path, runtime_dir: Path, log_
         api_json(args, f"/api/training/jobs/{job_id}/complete", payload)
 
 
-def run_retrain(job: dict[str, Any], repo_root: Path, log_path: Path) -> None:
+def run_retrain(args: argparse.Namespace, job_id: str, job: dict[str, Any], repo_root: Path, log_path: Path) -> None:
     script = repo_root / "tools" / "run_torch_retrain.ps1"
     if not script.is_file():
         raise RuntimeError(f"retrain script not found: {script}")
@@ -103,6 +105,8 @@ def run_retrain(job: dict[str, Any], repo_root: Path, log_path: Path) -> None:
         if value not in (None, ""):
             cmd.extend([ps_arg, str(value)])
     log(log_path, "Running retrain: " + " ".join(quote_for_log(part) for part in cmd))
+    report_progress(args, job_id, "running", "training", 8, "PyTorch retrain запущен")
+    line_count = 0
     with subprocess.Popen(
         cmd,
         cwd=str(repo_root),
@@ -114,10 +118,16 @@ def run_retrain(job: dict[str, Any], repo_root: Path, log_path: Path) -> None:
     ) as process:
         assert process.stdout is not None
         for line in process.stdout:
-            log(log_path, line.rstrip())
+            message = line.rstrip()
+            log(log_path, message)
+            line_count += 1
+            if line_count == 1 or line_count % 12 == 0:
+                progress = min(70, 8 + line_count // 3)
+                report_progress(args, job_id, "running", "training", progress, message[-220:])
         code = process.wait()
     if code != 0:
         raise RuntimeError(f"retrain failed with exit code {code}")
+    report_progress(args, job_id, "running", "guard", 70, "Guard завершён, подготавливаю артефакты")
 
 
 def upload_artifact(args: argparse.Namespace, job_id: str, path: Path, log_path: Path) -> None:
@@ -137,6 +147,30 @@ def upload_artifact(args: argparse.Namespace, job_id: str, path: Path, log_path:
                 "data_base64": base64.b64encode(data).decode("ascii"),
             }
             api_json(args, f"/api/training/jobs/{job_id}/artifacts/chunk", payload, timeout=120)
+            if index == 0 or index == total - 1 or index % 10 == 0:
+                progress = 72 + int(((index + 1) / total) * 23)
+                report_progress(args, job_id, "running", "uploading", progress, f"Загружаю {path.name}: {index + 1}/{total}")
+
+
+def report_progress(
+    args: argparse.Namespace,
+    job_id: str,
+    status: str,
+    phase: str,
+    progress_percent: int,
+    message: str,
+) -> None:
+    api_json(
+        args,
+        f"/api/training/jobs/{job_id}/progress",
+        {
+            "status": status,
+            "phase": phase,
+            "progress_percent": progress_percent,
+            "message": message,
+            "worker": worker_payload(args, Path(args.repo_root).resolve()),
+        },
+    )
 
 
 def api_json(args: argparse.Namespace, path: str, payload: dict[str, Any], timeout: int = 30) -> dict[str, Any]: