#!/usr/bin/env python3
"""Third pass: 1990s beginner-tutorial prose.

The register a modern chat model produces when asked to explain something -- friendly,
numbered, "in this chapter you will learn", every term defined on first use, hedged
reassurance -- was not invented by the model. It is the house style of 1990s computing
tutorials and study-skills books. If any human text from that decade reads as machine
written, this is where it lives.
"""
import json, sys, urllib.parse
import fetch_corpus as F

QUERIES = [
    'collection:(folkscanomy_computer OR folkscanomy OR opensource) AND year:[1988 TO 2001] AND title:("teach yourself")',
    'collection:(folkscanomy_computer OR folkscanomy OR opensource) AND year:[1988 TO 2001] AND title:("for dummies")',
    'collection:(folkscanomy_computer OR folkscanomy OR opensource) AND year:[1988 TO 2001] AND title:("idiot\'s guide" OR "idiots guide")',
    'collection:(folkscanomy_computer OR folkscanomy OR opensource) AND year:[1988 TO 2001] AND title:("step by step")',
    'collection:(folkscanomy_computer OR folkscanomy OR opensource) AND year:[1988 TO 2001] AND title:("beginner" OR "beginners" OR "for the absolute beginner")',
    'collection:(folkscanomy_computer OR folkscanomy OR opensource) AND year:[1988 TO 2001] AND title:("getting started" OR "made easy" OR "made simple")',
    'collection:(folkscanomy_computer OR folkscanomy OR opensource) AND year:[1988 TO 2001] AND title:("user guide" OR "user\'s guide" OR tutorial)',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2001] AND title:("how to study" OR "study skills" OR "study guide")',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2001] AND title:("how to write" OR "writing skills" OR "business writing")',
    'collection:(folkscanomy OR opensource) AND year:[1988 TO 2001] AND title:("complete guide" OR "practical guide" OR "essential guide")',
]

F.OUT = "corpus3.jsonl"


def main():
    uniq = {}
    for q in QUERIES:
        found = F.search(q, rows=60)
        print(f"{len(found):4d}  {q[:78]}", file=sys.stderr)
        for d in found:
            uniq.setdefault(d["identifier"], d)
    print(f"unique items: {len(uniq)}", file=sys.stderr)
    kept = 0
    with open(F.OUT, "a") as fh:
        for i, (ident, d) in enumerate(uniq.items()):
            name = F.djvu_name(ident)
            raw = F.get(f"https://archive.org/download/{ident}/{urllib.parse.quote(name)}") if name else None
            if not raw:
                print(f"[{i+1}/{len(uniq)}] {ident} NO TEXT", file=sys.stderr)
                continue
            ps = F.passages(F.normalise(raw.decode("utf-8", "replace")))
            for j, p in enumerate(ps[:10]):
                fh.write(json.dumps({"id": ident, "title": d.get("title"),
                                     "year": d.get("year"), "seq": j, "text": p}) + "\n")
                kept += 1
            fh.flush()
            print(f"[{i+1}/{len(uniq)}] {ident} {d.get('year')} -> {len(ps)}", file=sys.stderr)
    print(f"TOTAL kept: {kept}", file=sys.stderr)


if __name__ == "__main__":
    main()