#!/usr/bin/env python3
"""Corpus #7 -- the register I never tested: warm, second-person, advisory prose.

The first six corpora were all flat technical instruction (military, legal, corporate,
RFC, programmed-instruction electronics). Nine of those passages went to Pangram and every
one came back Human. They share a property I did not notice until now: a chat model does
not write like that. RLHF'd assistant prose is warm, second-person, structured, and
benefit-listed -- the register of 1990s self-help, management, communication-skills and
study-skills books. That is what this corpus collects.

Also folded in at the book level: the archive.org mis-dating filter. An "Encyclopedia of
Essential Oils" filed under 1992 turned out to contain "with this new 2014 edition", and a
passage from it would have been a false claim. Any book whose own OCR text mentions
post-1999 years more than twice is dropped outright, not just its passages.
"""
import json, os, re, sys, urllib.parse

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from fetch_corpus import get, search, djvu_name, normalise, passages

OUT = "corpus8.jsonl"

QUERIES = [
    # composition / rhetoric textbooks: model essays are formulaic BY DESIGN, and the
    # five-paragraph template is exactly what a chat model reproduces on demand.
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:("writing" AND (guide OR handbook OR textbook OR process OR essays))',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:(composition OR rhetoric)',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:("college writing" OR "academic writing" OR "expository writing")',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:("essay" OR "essays")',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:("business writing" OR "technical writing" OR "report writing")',
    'collection:(ericarchive) AND year:[1990 TO 1999] AND title:("writing" AND (instruction OR process OR strategies OR workshop))',
    'collection:(ericarchive) AND year:[1990 TO 1999] AND title:("composition" OR "essay" OR "paragraph")',
    'collection:(ericarchive) AND year:[1990 TO 1999] AND title:("model" AND (lessons OR units OR curriculum))',
    'collection:(ericarchive) AND year:[1990 TO 1999] AND title:("language arts" AND (guide OR curriculum OR framework))',
    'collection:(ericarchive) AND year:[1990 TO 1999] AND title:("writing across the curriculum" OR "writing to learn")',
    # encyclopedia / reference entries: templated, uniform, third-person -- untried register
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:(encyclopedia OR dictionary OR almanac OR "reference guide")',
    # translated textbooks: translationese is smoothed and explicit, like generated prose
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:("translated from" OR "english edition")',
]

FUTURE_YEAR = re.compile(r"\b(20[0-3]\d)\b")


def book_is_misdated(raw_text):
    """archive.org's `year` field is unreliable. A genuine 1990s book cannot discuss
    post-1999 years; more than two such mentions means the metadata is wrong (reprint,
    wrong record, or a scan of a later edition). Drop the book, not just the passage."""
    hits = FUTURE_YEAR.findall(raw_text)
    return len(hits) > 2, sorted(set(hits))[:6]


def main():
    seen = set()
    if os.path.exists(OUT):
        for line in open(OUT):
            try:
                seen.add(json.loads(line)["id"])
            except Exception:
                pass

    uniq = {}
    for q in QUERIES:
        found = search(q, rows=60)
        print(f"{len(found):4d}  {q[:78]}", file=sys.stderr)
        for d in found:
            uniq.setdefault(d["identifier"], d)
    print(f"unique items: {len(uniq)}", file=sys.stderr)

    kept = dropped = 0
    with open(OUT, "a") as fh:
        for i, (ident, d) in enumerate(uniq.items()):
            if ident in seen:
                continue
            name = djvu_name(ident)
            raw = get(f"https://archive.org/download/{ident}/{urllib.parse.quote(name)}") if name else None
            if not raw:
                print(f"[{i+1}/{len(uniq)}] {ident} NO TEXT", file=sys.stderr)
                continue
            text = raw.decode("utf-8", "replace")
            bad, yrs = book_is_misdated(text)
            if bad:
                dropped += 1
                print(f"[{i+1}/{len(uniq)}] {ident} MISDATED (meta {d.get('year')}, text {yrs})", file=sys.stderr)
                continue
            ps = passages(normalise(text))
            for j, p in enumerate(ps[:10]):
                if FUTURE_YEAR.search(p):
                    continue
                fh.write(json.dumps({"id": ident, "title": d.get("title"),
                                     "year": d.get("year"), "seq": j, "text": p}) + "\n")
                kept += 1
            fh.flush()
            print(f"[{i+1}/{len(uniq)}] {ident} {d.get('year')} -> {len(ps)} passages", file=sys.stderr)
    print(f"TOTAL kept: {kept}   books dropped as misdated: {dropped}", file=sys.stderr)


if __name__ == "__main__":
    main()