#!/usr/bin/env python3
"""Genre-neutral control corpus: pre-2001 books drawn WITHOUT register targeting.

The 12,247-passage corpus this study's headline comes from was assembled adversarially — every
query behind it was written to find prose that looks machine-written. Reporting its flag rate as
"how often detectors are wrong on old books" would be a lie by sampling. This builds the other
half: archive.org texts from the same era and the same collections, selected only on
"is a book, has full text, catalogued 1970-2000", with no title keywords at all.

It is not a random sample of twentieth-century publishing — archive.org's holdings have their own
shape, and that limitation goes in the writeup. It is dramatically less biased than the
adversarial set, which is the comparison that matters.

Same extraction as the adversarial corpus (fetch_corpus.passages, same target/lo/hi), so the two
rates differ by corpus and not by method.
"""
import json, os, random, re, sys, urllib.parse

sys.path.insert(0, "/home/agent/work/pangram")
from fetch_corpus import get, search, djvu_name, normalise, passages

OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "control.jsonl")
SEED = 20260828
TARGET_ITEMS = 260
FUTURE_YEAR = re.compile(r"\b(20[0-3]\d)\b")

# No title terms anywhere. One query per collection per era-slice, so the draw is not dominated
# by whichever collection or year archive.org happens to hold most of.
#
# `folkscanomy` is the same holding the adversarial corpus was drawn from, which makes this a
# *paired* control: same archive, same era, same extraction — the only thing removed is the
# register targeting. `americana` is added because folkscanomy alone skews technical, and a
# control that shares the treatment's genre skew cannot measure genre.
#
# Excluded: `sim_` identifiers. Those are microfilmed journal front matter (indexes, tables of
# contents) and yield no prose — the first draw spent eleven fetches to collect twenty-nine
# passages because of them.
QUERIES = [
    '%s AND mediatype:texts AND year:[%d TO %d]' % (c, a, b)
    for c in ('collection:folkscanomy', 'collection:americana')
    for a, b in ((1970, 1979), (1980, 1986), (1987, 1993), (1994, 2000))
]

# The adversarial corpus is English by construction (its queries were English title terms).
# folkscanomy is not: the unfiltered draw returned Hungarian and French computer books, and an
# English-trained detector scoring Hungarian measures nothing. `language:` is absent on many
# items, so this is checked against the text: in English prose "the" is the most common token
# by a wide margin, and in nothing else is it even close.
def is_english(text):
    from collections import Counter
    toks = re.findall(r"[a-z']+", text.lower()[:400000])
    if len(toks) < 2000:
        return False
    top = [w for w, _ in Counter(toks).most_common(3)]
    return "the" in top


def main():
    seen = set()
    if os.path.exists(OUT):
        for line in open(OUT):
            try:
                seen.add(json.loads(line)["id"])
            except Exception:
                pass

    pool = {}
    for q in QUERIES:
        found = search(q, rows=200)
        print("%4d  %s" % (len(found), q[:90]), file=sys.stderr, flush=True)
        for d in found:
            if d["identifier"].startswith("sim_"):
                continue
            pool.setdefault(d["identifier"], d)
    ids = sorted(pool)
    random.Random(SEED).shuffle(ids)          # fixed draw, never re-drawn (B5)
    print("pool: %d items; drawing %d" % (len(ids), TARGET_ITEMS), file=sys.stderr, flush=True)

    kept = books = dropped = 0
    with open(OUT, "a") as fh:
        for ident in ids:
            if books >= TARGET_ITEMS:
                break
            if ident in seen:
                books += 1
                continue
            d = pool[ident]
            name = djvu_name(ident)
            raw = get("https://archive.org/download/%s/%s" % (ident, urllib.parse.quote(name))) if name else None
            if not raw:
                print("  %-52s NO TEXT" % ident[:52], file=sys.stderr, flush=True)
                continue
            text = raw.decode("utf-8", "replace") if isinstance(raw, bytes) else raw
            if not is_english(text):
                print("  %-52s NOT ENGLISH" % ident[:52], file=sys.stderr, flush=True)
                continue
            # same book-level date filter as fetch9.py: archive.org's year field is unreliable
            if len(FUTURE_YEAR.findall(text)) > 2:
                dropped += 1
                print("  %-52s MISDATED" % ident[:52], file=sys.stderr, flush=True)
                continue
            ps = [p for p in passages(normalise(text)) if not FUTURE_YEAR.search(p)]
            for j, p in enumerate(ps[:10]):
                fh.write(json.dumps({"id": ident, "title": d.get("title"),
                                     "year": d.get("year"), "seq": j, "text": p}) + "\n")
                kept += 1
            fh.flush()
            books += 1
            print("  %-52s %s -> %d passages (%d books, %d passages)"
                  % (ident[:52], d.get("year"), len(ps[:10]), books, kept),
                  file=sys.stderr, flush=True)
    print("books kept: %d   misdated-dropped: %d   passages: %d" % (books, dropped, kept),
          file=sys.stderr, flush=True)


if __name__ == "__main__":
    main()