feat: add expressive offline Russian book TTS

tools/tts/audit_booktts_text_corpus.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Audit a BookTTS text corpus and emit reproducible statistics and samples."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import random
+import re
+import sys
+from pathlib import Path
+
+
+MARKUP_PATTERNS = ("{{", "}}", "[[", "]]", "<ref", "</", "{|", "|}", "http://", "https://")
+RUSSIAN = re.compile(r"[А-Яа-яЁё]")
+LETTERS = re.compile(r"[^\W\d_]", re.UNICODE)
+
+
+def percentile(sorted_values: list[int], fraction: float) -> int:
+    if not sorted_values:
+        return 0
+    return sorted_values[round((len(sorted_values) - 1) * fraction)]
+
+
+def audit(args: argparse.Namespace) -> None:
+    rng = random.Random(args.seed)
+    lengths: list[int] = []
+    words: list[int] = []
+    samples: list[str] = []
+    suspicious = {pattern: 0 for pattern in MARKUP_PATTERNS}
+    low_russian_ratio = 0
+    with_digits = 0
+    with_dialogue_dash = 0
+    digest = hashlib.sha256()
+    with args.corpus.open("rb") as binary:
+        for raw in binary:
+            digest.update(raw)
+            line = raw.decode("utf-8").rstrip("\r\n")
+            index = len(lengths)
+            lengths.append(len(line)); words.append(len(line.split()))
+            if any(char.isdigit() for char in line):
+                with_digits += 1
+            if line.startswith(("—", "–", "-")):
+                with_dialogue_dash += 1
+            letters = LETTERS.findall(line)
+            russian = RUSSIAN.findall(line)
+            if letters and len(russian) / len(letters) < 0.75:
+                low_russian_ratio += 1
+            for pattern in MARKUP_PATTERNS:
+                if pattern in line:
+                    suspicious[pattern] += 1
+            if len(samples) < args.samples:
+                samples.append(line)
+            else:
+                replacement = rng.randint(0, index)
+                if replacement < args.samples:
+                    samples[replacement] = line
+    ordered_lengths = sorted(lengths); ordered_words = sorted(words)
+    report = {
+        "corpus": str(args.corpus), "sha256": digest.hexdigest().upper(), "lines": len(lengths),
+        "bytes": args.corpus.stat().st_size,
+        "characters": {"min": min(lengths, default=0), "mean": sum(lengths) / len(lengths) if lengths else 0, "p50": percentile(ordered_lengths, 0.50), "p95": percentile(ordered_lengths, 0.95), "max": max(lengths, default=0)},
+        "words": {"min": min(words, default=0), "mean": sum(words) / len(words) if words else 0, "p50": percentile(ordered_words, 0.50), "p95": percentile(ordered_words, 0.95), "max": max(words, default=0)},
+        "with_digits": with_digits, "with_dialogue_dash": with_dialogue_dash,
+        "below_75_percent_russian_letters": low_russian_ratio, "suspicious_markup": suspicious,
+        "sample_seed": args.seed, "samples": samples,
+    }
+    rendered = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
+    if args.output:
+        if args.output.exists():
+            raise FileExistsError(f"Refusing to overwrite {args.output}")
+        args.output.write_text(rendered, encoding="utf-8")
+    print(rendered, end="")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--corpus", type=Path, required=True)
+    parser.add_argument("--output", type=Path)
+    parser.add_argument("--samples", type=int, default=20)
+    parser.add_argument("--seed", type=int, default=20260721)
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    sys.exit(audit(parse_args()) or 0)