#!/usr/bin/env python3
"""Fourth pass: go deep on the two veins the detector actually likes.

NEETS module 1 (1992) scored P(AI)=0.994 and a Heathkit Individual Learning System text
(1999) scored 0.974. Both are series -- NEETS runs to 24 modules, Heathkit to a shelf --
so the cheapest way to get more shots on goal is to pull the rest of the series rather
than widen the net again.
"""
import json, sys, urllib.parse
import fetch_corpus as F

QUERIES = [
    'collection:(usgovernmentmirrors OR folkscanomy OR opensource) AND ("NEETS" OR "navy electricity and electronics")',
    'collection:(usgovernmentmirrors OR folkscanomy OR opensource) AND title:("navy electricity")',
    'collection:(folkscanomy OR opensource) AND (heathkit OR "individual learning system")',
    'collection:(usgovernmentmirrors OR folkscanomy OR opensource) AND title:("training series")',
    'collection:(usgovernmentmirrors OR folkscanomy OR opensource) AND year:[1985 TO 2000] AND title:("rate training")',
    'collection:(usgovernmentmirrors OR folkscanomy OR opensource) AND year:[1985 TO 2000] AND title:("technical training" OR "instructional systems")',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:("basic electronics" OR "basic electricity")',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:("correspondence course" OR "home study")',
]

F.OUT = "corpus4.jsonl"


def main():
    uniq = {}
    for q in QUERIES:
        found = F.search(q, rows=60)
        print(f"{len(found):4d}  {q[:76]}", file=sys.stderr)
        for d in found:
            uniq.setdefault(d["identifier"], d)
    print(f"unique items: {len(uniq)}", file=sys.stderr)
    kept = 0
    with open(F.OUT, "a") as fh:
        for i, (ident, d) in enumerate(uniq.items()):
            name = F.djvu_name(ident)
            raw = F.get(f"https://archive.org/download/{ident}/{urllib.parse.quote(name)}") if name else None
            if not raw:
                print(f"[{i+1}/{len(uniq)}] {ident} NO TEXT", file=sys.stderr)
                continue
            ps = F.passages(F.normalise(raw.decode("utf-8", "replace")))
            for j, p in enumerate(ps[:14]):
                fh.write(json.dumps({"id": ident, "title": d.get("title"),
                                     "year": d.get("year"), "seq": j, "text": p}) + "\n")
                kept += 1
            fh.flush()
            print(f"[{i+1}/{len(uniq)}] {ident} {d.get('year')} -> {len(ps)}", file=sys.stderr)
    print(f"TOTAL kept: {kept}", file=sys.stderr)


if __name__ == "__main__":
    main()