#!/usr/bin/env python3
"""Sixth pass. The ranking is dominated by ERIC education reports and WHO health
charters -- genuine 1990s smooth-register prose, but NOT "textbooks", which is the
exact word poidh #144 requires. The one true textbook that ranked high (Eysenck
Cognitive Psychology) was a mis-dated 2005+ edition.

So target genuine ~30-year-old TEXTBOOKS in the smoothest, most hedged and
encouraging register a pre-LLM human ever wrote: the helping/soft professions --
counseling, nursing, social work, education methods, communication, human resources.
Tight year window 1993-1999 (2026 - ~30 = ~1996). Downloadable full text only.
"""
import json, sys, urllib.parse
import fetch_corpus as F

QUERIES = [
    'collection:(folkscanomy OR opensource) AND year:[1993 TO 1999] AND title:(counseling OR counselling OR therapy OR "helping")',
    'collection:(folkscanomy OR opensource) AND year:[1993 TO 1999] AND title:(nursing OR "patient care" OR caregiving)',
    'collection:(folkscanomy OR opensource) AND year:[1993 TO 1999] AND title:("social work" OR "human services" OR "case management")',
    'collection:(folkscanomy OR opensource) AND year:[1993 TO 1999] AND title:("human resource" OR "organizational behavior" OR "organizational behaviour")',
    'collection:(folkscanomy OR opensource) AND year:[1993 TO 1999] AND title:(teaching OR "classroom" OR "educational psychology")',
    'collection:(folkscanomy OR opensource) AND year:[1993 TO 1999] AND title:("interpersonal communication" OR "group dynamics" OR "conflict resolution")',
    'collection:(folkscanomy OR opensource) AND year:[1993 TO 1999] AND title:(textbook) AND subject:(psychology OR education OR nursing OR management)',
    'collection:(folkscanomy OR opensource) AND year:[1993 TO 1999] AND subject:(textbook) AND title:(introduction OR principles OR fundamentals)',
    'collection:(folkscanomy OR opensource) AND year:[1993 TO 1999] AND title:("child development" OR "adolescent" OR "lifespan" OR "human development")',
    'collection:(folkscanomy OR opensource) AND year:[1993 TO 1999] AND title:(wellness OR "health education" OR "stress management")',
]

F.OUT = "corpus6.jsonl"


def main():
    uniq = {}
    for q in QUERIES:
        found = F.search(q, rows=60)
        print(f"{len(found):4d}  {q[:78]}", file=sys.stderr)
        for d in found:
            uniq.setdefault(d["identifier"], d)
    print(f"unique items: {len(uniq)}", file=sys.stderr)
    kept = 0
    with open(F.OUT, "w") as fh:
        for i, (ident, d) in enumerate(uniq.items()):
            name = F.djvu_name(ident)
            raw = F.get(f"https://archive.org/download/{ident}/{urllib.parse.quote(name)}") if name else None
            if not raw:
                print(f"[{i+1}/{len(uniq)}] {ident} -- no djvu", file=sys.stderr)
                continue
            ps = F.passages(F.normalise(raw.decode("utf-8", "replace")))
            for j, p in enumerate(ps[:12]):
                fh.write(json.dumps({"id": ident, "title": d.get("title"),
                                     "year": d.get("year"), "seq": j, "text": p}) + "\n")
                kept += 1
            fh.flush()
            print(f"[{i+1}/{len(uniq)}] {ident} {d.get('year')} -> {len(ps)} passages", file=sys.stderr)
    print(f"TOTAL kept: {kept}", file=sys.stderr)


if __name__ == "__main__":
    main()