#!/usr/bin/env python3
"""Corpus #7 -- the register I never tested: warm, second-person, advisory prose.

The first six corpora were all flat technical instruction (military, legal, corporate,
RFC, programmed-instruction electronics). Nine of those passages went to Pangram and every
one came back Human. They share a property I did not notice until now: a chat model does
not write like that. RLHF'd assistant prose is warm, second-person, structured, and
benefit-listed -- the register of 1990s self-help, management, communication-skills and
study-skills books. That is what this corpus collects.

Also folded in at the book level: the archive.org mis-dating filter. An "Encyclopedia of
Essential Oils" filed under 1992 turned out to contain "with this new 2014 edition", and a
passage from it would have been a false claim. Any book whose own OCR text mentions
post-1999 years more than twice is dropped outright, not just its passages.
"""
import json, os, re, sys, urllib.parse

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from fetch_corpus import get, search, djvu_name, normalise, passages

OUT = "corpus10.jsonl"

QUERIES = [
    # Translated and non-native-English textbooks. AI detectors are documented to
    # false-positive heavily on non-native English writing (Liang et al. found detectors
    # flagged most TOEFL essays as AI): simplified syntax, limited vocabulary, explicit
    # connectives, low idiom. Translationese has the same shape. Mir Publishers (Moscow)
    # put out English translations of Soviet textbooks in quantity, and Indian academic
    # publishers printed English-language textbooks throughout the 1990s.
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND publisher:("Mir Publishers")',
    'collection:(folkscanomy OR opensource) AND publisher:("Mir Publishers")',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:("translated from the russian")',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND publisher:("Tata McGraw" OR "Prentice-Hall of India" OR "New Age International" OR "Wiley Eastern")',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND publisher:("Foreign Languages Press" OR "Peace Publishers" OR "Progress Publishers")',
    'collection:(folkscanomy OR opensource) AND year:[1990 TO 2000] AND title:(textbook) AND publisher:(India OR Delhi OR Moscow OR Beijing)',
    'collection:(opensource) AND year:[1990 TO 2000] AND title:("english edition" OR "translated by") AND title:(textbook OR introduction OR course)',
]

FUTURE_YEAR = re.compile(r"\b(20[0-3]\d)\b")


def book_is_misdated(raw_text):
    """archive.org's `year` field is unreliable. A genuine 1990s book cannot discuss
    post-1999 years; more than two such mentions means the metadata is wrong (reprint,
    wrong record, or a scan of a later edition). Drop the book, not just the passage."""
    hits = FUTURE_YEAR.findall(raw_text)
    return len(hits) > 2, sorted(set(hits))[:6]


def main():
    seen = set()
    if os.path.exists(OUT):
        for line in open(OUT):
            try:
                seen.add(json.loads(line)["id"])
            except Exception:
                pass

    uniq = {}
    for q in QUERIES:
        found = search(q, rows=60)
        print(f"{len(found):4d}  {q[:78]}", file=sys.stderr)
        for d in found:
            uniq.setdefault(d["identifier"], d)
    print(f"unique items: {len(uniq)}", file=sys.stderr)

    kept = dropped = 0
    with open(OUT, "a") as fh:
        for i, (ident, d) in enumerate(uniq.items()):
            if ident in seen:
                continue
            name = djvu_name(ident)
            raw = get(f"https://archive.org/download/{ident}/{urllib.parse.quote(name)}") if name else None
            if not raw:
                print(f"[{i+1}/{len(uniq)}] {ident} NO TEXT", file=sys.stderr)
                continue
            text = raw.decode("utf-8", "replace")
            bad, yrs = book_is_misdated(text)
            if bad:
                dropped += 1
                print(f"[{i+1}/{len(uniq)}] {ident} MISDATED (meta {d.get('year')}, text {yrs})", file=sys.stderr)
                continue
            ps = passages(normalise(text))
            for j, p in enumerate(ps[:10]):
                if FUTURE_YEAR.search(p):
                    continue
                fh.write(json.dumps({"id": ident, "title": d.get("title"),
                                     "year": d.get("year"), "seq": j, "text": p}) + "\n")
                kept += 1
            fh.flush()
            print(f"[{i+1}/{len(uniq)}] {ident} {d.get('year')} -> {len(ps)} passages", file=sys.stderr)
    print(f"TOTAL kept: {kept}   books dropped as misdated: {dropped}", file=sys.stderr)


if __name__ == "__main__":
    main()