88 lines
3.4 KiB
Python
88 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
||
"""Audit a BookTTS text corpus and emit reproducible statistics and samples."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import random
|
||
import re
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
|
||
MARKUP_PATTERNS = ("{{", "}}", "[[", "]]", "<ref", "</", "{|", "|}", "http://", "https://")
|
||
RUSSIAN = re.compile(r"[А-Яа-яЁё]")
|
||
LETTERS = re.compile(r"[^\W\d_]", re.UNICODE)
|
||
|
||
|
||
def percentile(sorted_values: list[int], fraction: float) -> int:
|
||
if not sorted_values:
|
||
return 0
|
||
return sorted_values[round((len(sorted_values) - 1) * fraction)]
|
||
|
||
|
||
def audit(args: argparse.Namespace) -> None:
|
||
rng = random.Random(args.seed)
|
||
lengths: list[int] = []
|
||
words: list[int] = []
|
||
samples: list[str] = []
|
||
suspicious = {pattern: 0 for pattern in MARKUP_PATTERNS}
|
||
low_russian_ratio = 0
|
||
with_digits = 0
|
||
with_dialogue_dash = 0
|
||
digest = hashlib.sha256()
|
||
with args.corpus.open("rb") as binary:
|
||
for raw in binary:
|
||
digest.update(raw)
|
||
line = raw.decode("utf-8").rstrip("\r\n")
|
||
index = len(lengths)
|
||
lengths.append(len(line)); words.append(len(line.split()))
|
||
if any(char.isdigit() for char in line):
|
||
with_digits += 1
|
||
if line.startswith(("—", "–", "-")):
|
||
with_dialogue_dash += 1
|
||
letters = LETTERS.findall(line)
|
||
russian = RUSSIAN.findall(line)
|
||
if letters and len(russian) / len(letters) < 0.75:
|
||
low_russian_ratio += 1
|
||
for pattern in MARKUP_PATTERNS:
|
||
if pattern in line:
|
||
suspicious[pattern] += 1
|
||
if len(samples) < args.samples:
|
||
samples.append(line)
|
||
else:
|
||
replacement = rng.randint(0, index)
|
||
if replacement < args.samples:
|
||
samples[replacement] = line
|
||
ordered_lengths = sorted(lengths); ordered_words = sorted(words)
|
||
report = {
|
||
"corpus": str(args.corpus), "sha256": digest.hexdigest().upper(), "lines": len(lengths),
|
||
"bytes": args.corpus.stat().st_size,
|
||
"characters": {"min": min(lengths, default=0), "mean": sum(lengths) / len(lengths) if lengths else 0, "p50": percentile(ordered_lengths, 0.50), "p95": percentile(ordered_lengths, 0.95), "max": max(lengths, default=0)},
|
||
"words": {"min": min(words, default=0), "mean": sum(words) / len(words) if words else 0, "p50": percentile(ordered_words, 0.50), "p95": percentile(ordered_words, 0.95), "max": max(words, default=0)},
|
||
"with_digits": with_digits, "with_dialogue_dash": with_dialogue_dash,
|
||
"below_75_percent_russian_letters": low_russian_ratio, "suspicious_markup": suspicious,
|
||
"sample_seed": args.seed, "samples": samples,
|
||
}
|
||
rendered = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||
if args.output:
|
||
if args.output.exists():
|
||
raise FileExistsError(f"Refusing to overwrite {args.output}")
|
||
args.output.write_text(rendered, encoding="utf-8")
|
||
print(rendered, end="")
|
||
|
||
|
||
def parse_args() -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("--corpus", type=Path, required=True)
|
||
parser.add_argument("--output", type=Path)
|
||
parser.add_argument("--samples", type=int, default=20)
|
||
parser.add_argument("--seed", type=int, default=20260721)
|
||
return parser.parse_args()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(audit(parse_args()) or 0)
|