from __future__ import annotations import argparse import json import sqlite3 import sys from pathlib import Path from typing import Any PROJECT_ROOT = Path(__file__).resolve().parents[1] if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) from crypto_spot_bot.storage import Storage PRESERVED_TABLES = ("positions", "trades", "runtime", "orders") DEFAULT_RECENT_ROWS = { "signals": 5_000, "equity": 5_000, "events": 2_000, "llm_advice": 1_000, "market_observations": 100_000, } def compact_database( database: Path, *, recent_rows: dict[str, int] | None = None, backup: Path | None = None, ) -> dict[str, Any]: database = database.resolve() if not database.is_file(): raise FileNotFoundError(database) limits = dict(DEFAULT_RECENT_ROWS) if recent_rows: limits.update({key: max(0, int(value)) for key, value in recent_rows.items()}) temp = database.with_name(database.name + ".compact") backup = (backup or database.with_name(database.name + ".precompact.bak")).resolve() if temp.exists(): temp.unlink() if backup.exists(): raise FileExistsError(f"backup already exists: {backup}") source_bytes = database.stat().st_size Storage(temp) counts: dict[str, int] = {} conn = sqlite3.connect(temp) try: conn.execute("PRAGMA foreign_keys=OFF") conn.execute("ATTACH DATABASE ? AS source", (str(database),)) for table in PRESERVED_TABLES: counts[table] = _copy_table(conn, table, limit=None) for table, limit in limits.items(): counts[table] = _copy_table(conn, table, limit=limit) conn.commit() # Check only the newly built main database. The attached multi-gigabyte # source is preserved as the rollback copy and must not be rescanned here. integrity = str(conn.execute("PRAGMA main.integrity_check").fetchone()[0]) if integrity.lower() != "ok": raise RuntimeError(f"compacted database integrity check failed: {integrity}") conn.execute("DETACH DATABASE source") conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") conn.execute("PRAGMA journal_mode=DELETE") conn.commit() finally: conn.close() database.replace(backup) temp.replace(database) compacted_bytes = database.stat().st_size return { "database": str(database), "backup": str(backup), "source_bytes": source_bytes, "compacted_bytes": compacted_bytes, "reclaimed_bytes": max(0, source_bytes - compacted_bytes), "rows": counts, } def _copy_table(conn: sqlite3.Connection, table: str, *, limit: int | None) -> int: destination_columns = _columns(conn, "main", table) source_columns = set(_columns(conn, "source", table)) columns = [column for column in destination_columns if column in source_columns] if not columns: return 0 quoted = ", ".join(f'"{column}"' for column in columns) if limit is None: conn.execute( f'INSERT INTO main."{table}" ({quoted}) SELECT {quoted} FROM source."{table}"' ) elif limit > 0: conn.execute( f'INSERT INTO main."{table}" ({quoted}) ' f'SELECT {quoted} FROM source."{table}" ORDER BY id DESC LIMIT ?', (limit,), ) row = conn.execute(f'SELECT COUNT(*) FROM main."{table}"').fetchone() return int(row[0] if row else 0) def _columns(conn: sqlite3.Connection, schema: str, table: str) -> list[str]: return [str(row[1]) for row in conn.execute(f'PRAGMA {schema}.table_info("{table}")')] def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Atomically compact the TradeBot runtime database while preserving durable trading state." ) parser.add_argument("--database", required=True) parser.add_argument("--backup", default="") parser.add_argument("--signals", type=int, default=DEFAULT_RECENT_ROWS["signals"]) parser.add_argument("--equity", type=int, default=DEFAULT_RECENT_ROWS["equity"]) parser.add_argument("--events", type=int, default=DEFAULT_RECENT_ROWS["events"]) parser.add_argument("--llm-advice", type=int, default=DEFAULT_RECENT_ROWS["llm_advice"]) parser.add_argument( "--market-observations", type=int, default=DEFAULT_RECENT_ROWS["market_observations"], ) return parser.parse_args() def main() -> None: args = _parse_args() result = compact_database( Path(args.database), backup=Path(args.backup) if args.backup else None, recent_rows={ "signals": args.signals, "equity": args.equity, "events": args.events, "llm_advice": args.llm_advice, "market_observations": args.market_observations, }, ) print(json.dumps(result, ensure_ascii=False, sort_keys=True)) if __name__ == "__main__": main()