137 lines
4.7 KiB
Python
137 lines
4.7 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sqlite3
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
if str(PROJECT_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
from crypto_spot_bot.storage import Storage
|
|
|
|
|
|
PRESERVED_TABLES = ("positions", "trades", "runtime", "orders")
|
|
DEFAULT_RECENT_ROWS = {
|
|
"signals": 5_000,
|
|
"equity": 5_000,
|
|
"events": 2_000,
|
|
"llm_advice": 1_000,
|
|
}
|
|
|
|
|
|
def compact_database(
|
|
database: Path,
|
|
*,
|
|
recent_rows: dict[str, int] | None = None,
|
|
backup: Path | None = None,
|
|
) -> dict[str, Any]:
|
|
database = database.resolve()
|
|
if not database.is_file():
|
|
raise FileNotFoundError(database)
|
|
limits = dict(DEFAULT_RECENT_ROWS)
|
|
if recent_rows:
|
|
limits.update({key: max(0, int(value)) for key, value in recent_rows.items()})
|
|
temp = database.with_name(database.name + ".compact")
|
|
backup = (backup or database.with_name(database.name + ".precompact.bak")).resolve()
|
|
if temp.exists():
|
|
temp.unlink()
|
|
if backup.exists():
|
|
raise FileExistsError(f"backup already exists: {backup}")
|
|
|
|
source_bytes = database.stat().st_size
|
|
Storage(temp)
|
|
counts: dict[str, int] = {}
|
|
conn = sqlite3.connect(temp)
|
|
try:
|
|
conn.execute("PRAGMA foreign_keys=OFF")
|
|
conn.execute("ATTACH DATABASE ? AS source", (str(database),))
|
|
for table in PRESERVED_TABLES:
|
|
counts[table] = _copy_table(conn, table, limit=None)
|
|
for table, limit in limits.items():
|
|
counts[table] = _copy_table(conn, table, limit=limit)
|
|
conn.commit()
|
|
# Check only the newly built main database. The attached multi-gigabyte
|
|
# source is preserved as the rollback copy and must not be rescanned here.
|
|
integrity = str(conn.execute("PRAGMA main.integrity_check").fetchone()[0])
|
|
if integrity.lower() != "ok":
|
|
raise RuntimeError(f"compacted database integrity check failed: {integrity}")
|
|
conn.execute("DETACH DATABASE source")
|
|
conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
conn.execute("PRAGMA journal_mode=DELETE")
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
database.replace(backup)
|
|
temp.replace(database)
|
|
compacted_bytes = database.stat().st_size
|
|
return {
|
|
"database": str(database),
|
|
"backup": str(backup),
|
|
"source_bytes": source_bytes,
|
|
"compacted_bytes": compacted_bytes,
|
|
"reclaimed_bytes": max(0, source_bytes - compacted_bytes),
|
|
"rows": counts,
|
|
}
|
|
|
|
|
|
def _copy_table(conn: sqlite3.Connection, table: str, *, limit: int | None) -> int:
|
|
destination_columns = _columns(conn, "main", table)
|
|
source_columns = set(_columns(conn, "source", table))
|
|
columns = [column for column in destination_columns if column in source_columns]
|
|
if not columns:
|
|
return 0
|
|
quoted = ", ".join(f'"{column}"' for column in columns)
|
|
if limit is None:
|
|
conn.execute(
|
|
f'INSERT INTO main."{table}" ({quoted}) SELECT {quoted} FROM source."{table}"'
|
|
)
|
|
elif limit > 0:
|
|
conn.execute(
|
|
f'INSERT INTO main."{table}" ({quoted}) '
|
|
f'SELECT {quoted} FROM source."{table}" ORDER BY id DESC LIMIT ?',
|
|
(limit,),
|
|
)
|
|
row = conn.execute(f'SELECT COUNT(*) FROM main."{table}"').fetchone()
|
|
return int(row[0] if row else 0)
|
|
|
|
|
|
def _columns(conn: sqlite3.Connection, schema: str, table: str) -> list[str]:
|
|
return [str(row[1]) for row in conn.execute(f'PRAGMA {schema}.table_info("{table}")')]
|
|
|
|
|
|
def _parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Atomically compact the TradeBot runtime database while preserving durable trading state."
|
|
)
|
|
parser.add_argument("--database", required=True)
|
|
parser.add_argument("--backup", default="")
|
|
parser.add_argument("--signals", type=int, default=DEFAULT_RECENT_ROWS["signals"])
|
|
parser.add_argument("--equity", type=int, default=DEFAULT_RECENT_ROWS["equity"])
|
|
parser.add_argument("--events", type=int, default=DEFAULT_RECENT_ROWS["events"])
|
|
parser.add_argument("--llm-advice", type=int, default=DEFAULT_RECENT_ROWS["llm_advice"])
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> None:
|
|
args = _parse_args()
|
|
result = compact_database(
|
|
Path(args.database),
|
|
backup=Path(args.backup) if args.backup else None,
|
|
recent_rows={
|
|
"signals": args.signals,
|
|
"equity": args.equity,
|
|
"events": args.events,
|
|
"llm_advice": args.llm_advice,
|
|
},
|
|
)
|
|
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|