#!/usr/bin/env python3
"""Corpus #7 -- the register I never tested: warm, second-person, advisory prose.

The first six corpora were all flat technical instruction (military, legal, corporate,
RFC, programmed-instruction electronics). Nine of those passages went to Pangram and every
one came back Human. They share a property I did not notice until now: a chat model does
not write like that. RLHF'd assistant prose is warm, second-person, structured, and
benefit-listed -- the register of 1990s self-help, management, communication-skills and
study-skills books. That is what this corpus collects.

Also folded in at the book level: the archive.org mis-dating filter. An "Encyclopedia of
Essential Oils" filed under 1992 turned out to contain "with this new 2014 edition", and a
passage from it would have been a false claim. Any book whose own OCR text mentions
post-1999 years more than twice is dropped outright, not just its passages.
"""
import json, os, re, sys, urllib.parse

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from fetch_corpus import get, search, djvu_name, normalise, passages

OUT = "corpus9.jsonl"

QUERIES = [
    # mid-1990s popular introductions to the Internet and computers. This is the genre the
    # whole dispute is about: the Reddit complainant wrote a textbook ~1996 and worked at
    # one of the first commercial internet companies. It is also, independently, the exact
    # register a chat model uses when explaining technology to a beginner -- second person,
    # benefit-framed, evenly paced, heavily signposted.
    'collection:(folkscanomy_computer OR folkscanomy OR opensource) AND year:[1993 TO 1999] AND title:(internet AND (guide OR introduction OR beginners OR complete OR using))',
    'collection:(folkscanomy_computer OR folkscanomy OR opensource) AND year:[1993 TO 1999] AND title:("world wide web" OR "information superhighway" OR cyberspace)',
    'collection:(folkscanomy_computer OR folkscanomy OR opensource) AND year:[1993 TO 1999] AND title:("electronic mail" OR email OR "online services" OR usenet)',
    'collection:(folkscanomy_computer OR folkscanomy OR opensource) AND year:[1993 TO 1999] AND title:(html OR "web page" OR "web site" OR webmaster)',
    'collection:(folkscanomy_computer OR folkscanomy OR opensource) AND year:[1992 TO 1999] AND title:(computers AND (introduction OR beginners OR basics OR understanding))',
    'collection:(folkscanomy_computer OR folkscanomy OR opensource) AND year:[1992 TO 1999] AND title:("for dummies" OR "idiots guide" OR "made easy" OR "in plain english")',
    'collection:(folkscanomy_computer OR folkscanomy OR opensource) AND year:[1992 TO 1999] AND title:(multimedia OR networking OR "computer literacy")',
    'collection:(folkscanomy OR opensource) AND year:[1992 TO 1999] AND title:("information technology" AND (introduction OR guide OR textbook))',
    'collection:(folkscanomy OR opensource) AND year:[1992 TO 1999] AND title:("business" AND (internet OR online OR "electronic commerce"))',
]

FUTURE_YEAR = re.compile(r"\b(20[0-3]\d)\b")


def book_is_misdated(raw_text):
    """archive.org's `year` field is unreliable. A genuine 1990s book cannot discuss
    post-1999 years; more than two such mentions means the metadata is wrong (reprint,
    wrong record, or a scan of a later edition). Drop the book, not just the passage."""
    hits = FUTURE_YEAR.findall(raw_text)
    return len(hits) > 2, sorted(set(hits))[:6]


def main():
    seen = set()
    if os.path.exists(OUT):
        for line in open(OUT):
            try:
                seen.add(json.loads(line)["id"])
            except Exception:
                pass

    uniq = {}
    for q in QUERIES:
        found = search(q, rows=60)
        print(f"{len(found):4d}  {q[:78]}", file=sys.stderr)
        for d in found:
            uniq.setdefault(d["identifier"], d)
    print(f"unique items: {len(uniq)}", file=sys.stderr)

    kept = dropped = 0
    with open(OUT, "a") as fh:
        for i, (ident, d) in enumerate(uniq.items()):
            if ident in seen:
                continue
            name = djvu_name(ident)
            raw = get(f"https://archive.org/download/{ident}/{urllib.parse.quote(name)}") if name else None
            if not raw:
                print(f"[{i+1}/{len(uniq)}] {ident} NO TEXT", file=sys.stderr)
                continue
            text = raw.decode("utf-8", "replace")
            bad, yrs = book_is_misdated(text)
            if bad:
                dropped += 1
                print(f"[{i+1}/{len(uniq)}] {ident} MISDATED (meta {d.get('year')}, text {yrs})", file=sys.stderr)
                continue
            ps = passages(normalise(text))
            for j, p in enumerate(ps[:10]):
                if FUTURE_YEAR.search(p):
                    continue
                fh.write(json.dumps({"id": ident, "title": d.get("title"),
                                     "year": d.get("year"), "seq": j, "text": p}) + "\n")
                kept += 1
            fh.flush()
            print(f"[{i+1}/{len(uniq)}] {ident} {d.get('year')} -> {len(ps)} passages", file=sys.stderr)
    print(f"TOTAL kept: {kept}   books dropped as misdated: {dropped}", file=sys.stderr)


if __name__ == "__main__":
    main()