#!/usr/bin/env python3
"""Corpus #7 -- the register I never tested: warm, second-person, advisory prose.

The first six corpora were all flat technical instruction (military, legal, corporate,
RFC, programmed-instruction electronics). Nine of those passages went to Pangram and every
one came back Human. They share a property I did not notice until now: a chat model does
not write like that. RLHF'd assistant prose is warm, second-person, structured, and
benefit-listed -- the register of 1990s self-help, management, communication-skills and
study-skills books. That is what this corpus collects.

Also folded in at the book level: the archive.org mis-dating filter. An "Encyclopedia of
Essential Oils" filed under 1992 turned out to contain "with this new 2014 edition", and a
passage from it would have been a false claim. Any book whose own OCR text mentions
post-1999 years more than twice is dropped outright, not just its passages.
"""
import json, os, re, sys, urllib.parse

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from fetch_corpus import get, search, djvu_name, normalise, passages

OUT = "corpus7.jsonl"

QUERIES = [
    # self-help / popular psychology: the second-person advisory register
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:("how to" AND (win OR improve OR succeed OR "get" OR build))',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:(self-esteem OR assertiveness OR motivation OR confidence)',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:("time management" OR "stress management" OR "goal setting")',
    # management / business communication: bullet-listed benefits, "effective" everything
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:(leadership OR management) AND title:(effective OR successful OR skills)',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:("customer service" OR teamwork OR negotiation)',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:("business communication" OR "public speaking" OR presentation)',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:("interpersonal" OR "communication skills")',
    # study skills / career: textbook-shaped, advisory in tone
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:("study skills" OR "college success" OR "learning strategies")',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:(resume OR "job search" OR "career planning" OR interview)',
    # ERIC's advisory literature: "tips for", parent/teacher guides -- warm and listy
    'collection:(ericarchive) AND year:[1990 TO 1999] AND title:("tips for" OR "helping your" OR "a guide for parents")',
    'collection:(ericarchive) AND year:[1990 TO 1999] AND title:("study skills" OR "learning strategies" OR "critical thinking")',
    'collection:(ericarchive) AND year:[1990 TO 1999] AND title:("effective communication" OR "conflict resolution" OR "self-esteem")',
    'collection:(ericarchive) AND year:[1990 TO 1999] AND title:("career development" OR "job readiness" OR "workplace skills")',
    # health / wellness advice, same voice
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:(wellness OR nutrition OR fitness) AND title:(guide OR handbook OR "how to")',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2000] AND title:(parenting OR "your child") AND title:(guide OR handbook)',
]

FUTURE_YEAR = re.compile(r"\b(20[0-3]\d)\b")


def book_is_misdated(raw_text):
    """archive.org's `year` field is unreliable. A genuine 1990s book cannot discuss
    post-1999 years; more than two such mentions means the metadata is wrong (reprint,
    wrong record, or a scan of a later edition). Drop the book, not just the passage."""
    hits = FUTURE_YEAR.findall(raw_text)
    return len(hits) > 2, sorted(set(hits))[:6]


def main():
    seen = set()
    if os.path.exists(OUT):
        for line in open(OUT):
            try:
                seen.add(json.loads(line)["id"])
            except Exception:
                pass

    uniq = {}
    for q in QUERIES:
        found = search(q, rows=60)
        print(f"{len(found):4d}  {q[:78]}", file=sys.stderr)
        for d in found:
            uniq.setdefault(d["identifier"], d)
    print(f"unique items: {len(uniq)}", file=sys.stderr)

    kept = dropped = 0
    with open(OUT, "a") as fh:
        for i, (ident, d) in enumerate(uniq.items()):
            if ident in seen:
                continue
            name = djvu_name(ident)
            raw = get(f"https://archive.org/download/{ident}/{urllib.parse.quote(name)}") if name else None
            if not raw:
                print(f"[{i+1}/{len(uniq)}] {ident} NO TEXT", file=sys.stderr)
                continue
            text = raw.decode("utf-8", "replace")
            bad, yrs = book_is_misdated(text)
            if bad:
                dropped += 1
                print(f"[{i+1}/{len(uniq)}] {ident} MISDATED (meta {d.get('year')}, text {yrs})", file=sys.stderr)
                continue
            ps = passages(normalise(text))
            for j, p in enumerate(ps[:10]):
                if FUTURE_YEAR.search(p):
                    continue
                fh.write(json.dumps({"id": ident, "title": d.get("title"),
                                     "year": d.get("year"), "seq": j, "text": p}) + "\n")
                kept += 1
            fh.flush()
            print(f"[{i+1}/{len(uniq)}] {ident} {d.get('year')} -> {len(ps)} passages", file=sys.stderr)
    print(f"TOTAL kept: {kept}   books dropped as misdated: {dropped}", file=sys.stderr)


if __name__ == "__main__":
    main()