#!/usr/bin/env python3
"""Second, targeted pass: the genres with the strongest prior for reading as machine-written.

The broad pass pulls mostly scholarly monographs, which are bursty and idiosyncratic --
the opposite of what a detector flags. These queries go after text that is uniform *by
construction*:

  * non-native-authored English textbooks (India, China, Russia, Japan). Detectors are
    documented to over-flag ESL writing; a 1990s Indian or Chinese exam textbook is
    formulaic English written to a template.
  * programmed instruction / self-study courses -- short declarative sentences, heavy
    repetition, deliberately zero variance.
  * controlled/simplified English technical writing, where sentence length and word
    sense are mandated by a standard.
  * translated textbooks: translationese is flat by nature.
"""
import json, sys
import fetch_corpus as F

QUERIES = [
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:(textbook) AND (india OR indian OR "jee" OR ncert OR arihant)',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND (mir OR "moscow" OR "foreign languages publishing") AND title:(textbook OR course OR problems)',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:(textbook OR coursebook) AND (chinese OR china OR japan OR japanese OR korea)',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:("programmed instruction" OR "programmed learning" OR "self-study" OR "self study")',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:("individual learning" OR heathkit)',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:("simplified english" OR "basic english" OR "controlled language")',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:("translated from" OR "english translation") AND title:(textbook OR introduction)',
    'collection:(ericarchive) AND year:[1988 TO 2000] AND title:("competency based" OR "learning activity packet" OR "instructional module")',
    'collection:(ericarchive) AND year:[1988 TO 2000] AND title:("module" ) AND title:(student OR learner)',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:("maintenance manual" OR "operators manual" OR "technical manual")',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:("english grammar" OR "english usage" OR "spoken english")',
    'collection:(folkscanomy OR opensource) AND year:[1985 TO 2001] AND title:("nursing" OR "first aid" OR "safety") AND title:(textbook OR manual OR handbook)',
]

F.OUT = "corpus2.jsonl"


def main():
    uniq = {}
    for q in QUERIES:
        found = F.search(q, rows=60)
        print(f"{len(found):4d}  {q[:78]}", file=sys.stderr)
        for d in found:
            uniq.setdefault(d["identifier"], d)
    print(f"unique items: {len(uniq)}", file=sys.stderr)
    kept = 0
    with open(F.OUT, "a") as fh:
        for i, (ident, d) in enumerate(uniq.items()):
            name = F.djvu_name(ident)
            raw = F.get(f"https://archive.org/download/{ident}/{__import__('urllib.parse', fromlist=['quote']).quote(name)}") if name else None
            if not raw:
                print(f"[{i+1}/{len(uniq)}] {ident} NO TEXT", file=sys.stderr)
                continue
            ps = F.passages(F.normalise(raw.decode("utf-8", "replace")))
            for j, p in enumerate(ps[:10]):
                fh.write(json.dumps({"id": ident, "title": d.get("title"),
                                     "year": d.get("year"), "seq": j, "text": p}) + "\n")
                kept += 1
            fh.flush()
            print(f"[{i+1}/{len(uniq)}] {ident} {d.get('year')} -> {len(ps)}", file=sys.stderr)
    print(f"TOTAL kept: {kept}", file=sys.stderr)


if __name__ == "__main__":
    main()