#!/usr/bin/env python3
"""Fifth pass, after Pangram falsified the first hypothesis.

Uniformity was the wrong target: Pangram called a 1992 Navy training manual human and
called my own prose AI, on the same subject at the same reading level. So the signal is
not flatness, it is *register* -- the discourse habits a modern chat model was tuned
toward. Across every pre-2001 passage I had, marker density topped out near 16 per
thousand words while my control ran about 30, and the technical genres sat near zero.

So: stop searching manuals and start searching the 1990s writing that already sounds like
that. Business and management, self-help and motivation, popular science for a general
audience, corporate training -- prose written to be smooth, hedged, balanced and
encouraging, which is exactly what RLHF optimised toward.
"""
import json, sys, urllib.parse
import fetch_corpus as F

QUERIES = [
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:(management OR managing OR leadership)',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:("business communication" OR "business writing" OR "customer service")',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:(success OR motivation OR "positive thinking" OR habits)',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:("how to win" OR "self help" OR "personal growth" OR "self-improvement")',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:("time management" OR "stress" OR "communication skills")',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:("popular science" OR "science for" OR "explained")',
    'collection:(ericarchive) AND year:[1988 TO 2000] AND title:("staff development" OR "professional development")',
    'collection:(ericarchive) AND year:[1988 TO 2000] AND title:("total quality" OR "strategic planning" OR "school improvement")',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:(marketing OR entrepreneur OR "small business")',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:(psychology OR wellness OR "health promotion")',
]

F.OUT = "corpus5.jsonl"


def main():
    uniq = {}
    for q in QUERIES:
        found = F.search(q, rows=60)
        print(f"{len(found):4d}  {q[:74]}", file=sys.stderr)
        for d in found:
            uniq.setdefault(d["identifier"], d)
    print(f"unique items: {len(uniq)}", file=sys.stderr)
    kept = 0
    with open(F.OUT, "a") as fh:
        for i, (ident, d) in enumerate(uniq.items()):
            name = F.djvu_name(ident)
            raw = F.get(f"https://archive.org/download/{ident}/{urllib.parse.quote(name)}") if name else None
            if not raw:
                continue
            ps = F.passages(F.normalise(raw.decode("utf-8", "replace")))
            for j, p in enumerate(ps[:12]):
                fh.write(json.dumps({"id": ident, "title": d.get("title"),
                                     "year": d.get("year"), "seq": j, "text": p}) + "\n")
                kept += 1
            fh.flush()
            print(f"[{i+1}/{len(uniq)}] {ident} {d.get('year')} -> {len(ps)}", file=sys.stderr)
    print(f"TOTAL kept: {kept}", file=sys.stderr)


if __name__ == "__main__":
    main()