#!/usr/bin/env python3
"""Out-of-vocabulary rate per passage — the OCR-damage confound, measured.

PREREG confound #1. Every passage here is an OCR'd scan of a printed page, so every passage
carries some density of scanner noise: `tlie` for `the`, `1` for `l`, hyphenation survivors,
running heads glued to body text. If the detectors flag dirty passages at a different rate than
clean ones, then what I would be measuring is OCR quality, not register or era — and that is the
finding, not a footnote.

Rate = fraction of 4+-letter alphabetic tokens absent from the system word list. Short tokens are
excluded because they are dominated by real words the list has anyway, and because a 1-3 letter
OCR error is indistinguishable from an abbreviation.

Absolute values here are not "the error rate": technical prose is full of real words no
dictionary carries (`multiplexer`, `Fahrenheit`, surnames, part numbers). Only the *comparison*
between flagged and unflagged passages is meaningful, and that comparison is what gets published.

  python3 oov.py [passages.jsonl ...]   ->  oov_<stem>.jsonl  {"n"|"id"+"seq", "oov", "ntok"}
"""
import json, os, re, sys

HERE = os.path.dirname(os.path.abspath(__file__))
WORDS = "/usr/share/dict/american-english"
TOKEN = re.compile(r"[A-Za-z]{4,}")


def vocab():
    v = set()
    for line in open(WORDS, encoding="utf-8", errors="ignore"):
        w = line.strip().lower()
        if not w:
            continue
        v.add(w)
        if w.endswith("'s"):
            v.add(w[:-2])          # the list carries possessives; the passages carry the stem
    # regular inflections the list omits for some stems
    for w in list(v):
        v.add(w + "s")
    return v


def rate(text, v):
    toks = [t.lower() for t in TOKEN.findall(text)]
    if not toks:
        return None, 0
    bad = sum(1 for t in toks if t not in v)
    return round(bad / len(toks), 5), len(toks)


def bucket(o):
    """The one copy of how an out-of-dictionary rate becomes a row label.

    This rule lived twice — once in analyse.py's breakdown, once in build_page.py's — and the two
    copies agreed on every passage below 20% and disagreed above it: the analysis split that tail
    into 20-24, 25-29, 30-34, and the page collapsed it into a single "20%+" row. So the page
    published a rate that appears nowhere in analysis.txt, directly under a sentence promising that
    every rate on the page appears there. Collapsing is the right call for display — the tail
    buckets hold too few passages to carry a readable interval — so the collapse moved here and
    both callers import it. Anyone wanting the finer split has oov_passages.jsonl, which is
    published for exactly that.
    """
    if o is None:
        return None
    b = int(o * 100) // 5
    return "20%+ out-of-dictionary" if b >= 4 else "%d-%d%% out-of-dictionary" % (b * 5, b * 5 + 4)


def main():
    v = vocab()
    print("vocab: %d forms" % len(v), file=sys.stderr, flush=True)
    paths = sys.argv[1:] or [os.path.join(HERE, "passages.jsonl")]
    for p in paths:
        stem = os.path.basename(p).rsplit(".", 1)[0]
        out = os.path.join(HERE, "oov_%s.jsonl" % stem)
        n = 0
        with open(out, "w") as fh:
            for line in open(p):
                d = json.loads(line)
                r, ntok = rate(d["text"], v)
                key = {"n": d["n"]} if "n" in d else {"id": d["id"], "seq": d["seq"]}
                key.update({"oov": r, "ntok": ntok, "words": len(d["text"].split())})
                fh.write(json.dumps(key) + "\n")
                n += 1
        print("%s -> %s  (%d)" % (p, out, n), file=sys.stderr, flush=True)


if __name__ == "__main__":
    main()