#!/usr/bin/env python3
"""What is actually in the passages the post-1999-year rule throws away?

The page carries a limitation paragraph about catalogue-year drift: archive.org's date is the
catalogue's, and a book dated 1994 can be a later edition. Two rules defend against that by reading
the text rather than the record, and they have different reach. The passage-level one — drop any
passage mentioning a post-1999 year — ran over both draws, every pull, and is the rule measured
here. The book-level one — drop a book whose own text mentions post-1999 years more than twice,
before cutting anything from it — ran over the control draw and over the four adversarial pulls
built after it existed (merge.py's STRICT set), not the six before. The page reports how many books
the control lost to the book-level rule, under the word "misdated".

That word is a claim about *why* those passages match, and nothing measured it. A pre-2001 book
cannot mention 2005 -- but an archive.org text is not only the book. It carries the scanning
operation's own boilerplate, and that boilerplate is stamped with the year of the scan, which for
this collection is mostly the 2010s. If those are what the rule is catching, then the filter is
mostly costing corpus rather than catching misdating, and the number on the page is mislabelled.

So: pull every match, with context, out of the raw source corpora, and separate the two.
Imports FUTURE_YEAR and yr from merge.py so this measures the rule the corpus was built with
rather than a second copy of it.

Writes drift.jsonl (one row per dropped passage) and prints the summary the page quotes.
"""
import json, glob, os, re, sys, collections

from merge import FUTURE_YEAR, yr, SRC

HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, "drift.jsonl")

# Scan-provenance boilerplate. Deliberately generous: the question is whether the filter's matches
# are dominated by digitisation furniture, so over-matching here makes the answer *less* flattering
# to the conclusion I expect, not more. Every classified row goes to drift.jsonl with the matched
# window in it, so the rule can be disagreed with rather than taken.
PROV = re.compile(
    r"digiti[sz]|internet archive|archive\.org|scanned|scanning|sponsor|microsoft corporation|"
    r"google|books\.google|openlibrary|ocr|this book is available|lending library|"
    r"funding|uploaded|encoded|creative commons|creativecommons|https?://|www\.", re.I)


def windows(text):
    """Every post-1999 year in the text, with the 120 characters around it."""
    for m in FUTURE_YEAR.finditer(text):
        a, b = max(0, m.start() - 60), min(len(text), m.end() + 60)
        yield int(m.group(1)), re.sub(r"\s+", " ", text[a:b]).strip()


def main():
    per_item = collections.defaultdict(
        lambda: {"prov": 0, "other": 0, "years": [], "title": "", "windows": []})
    rows = n_dropped = 0
    with open(OUT, "w") as fh:
        for path in sorted(glob.glob(os.path.join(SRC, "corpus*.jsonl"))):
            for line in open(path):
                try:
                    d = json.loads(line)
                except Exception:
                    continue
                text = (d.get("text") or "").strip()
                # Same universe the corpus is drawn from: only books the catalogue calls pre-2001.
                # A book catalogued 2004 is out of scope for the corpus and out of scope here.
                if not text or (yr(d.get("year")) or 9999) >= 2001:
                    continue
                if not FUTURE_YEAR.search(text):
                    continue
                n_dropped += 1
                ws = list(windows(text))
                prov = sum(1 for _, w in ws if PROV.search(w))
                item = d.get("id")
                per_item[item]["prov"] += prov
                per_item[item]["other"] += len(ws) - prov
                per_item[item]["years"] += [y for y, w in ws if not PROV.search(w)]
                per_item[item]["title"] = d.get("title") or per_item[item]["title"]
                per_item[item]["windows"] += [w for _, w in ws if not PROV.search(w)]
                fh.write(json.dumps({
                    "item": item, "title": d.get("title"), "year": d.get("year"),
                    "seq": d.get("seq"), "matches": [{"year": y, "window": w,
                                                      "provenance": bool(PROV.search(w))}
                                                     for y, w in ws]}) + "\n")
                rows += 1

    prov_hits = sum(v["prov"] for v in per_item.values())
    other_hits = sum(v["other"] for v in per_item.values())
    all_prov = [k for k, v in per_item.items() if v["other"] == 0]
    print("passages dropped by the post-1999 rule (pre-2001 catalogue only): %d" % n_dropped)
    print("source items involved                                          : %d" % len(per_item))
    print("year mentions in them                                          : %d" % (prov_hits + other_hits))
    print("  in digitisation/provenance boilerplate                       : %d" % prov_hits)
    print("  everything else                                              : %d" % other_hits)
    print("items where EVERY match is boilerplate                         : %d of %d"
          % (len(all_prov), len(per_item)))
    print()
    rest = [y for v in per_item.values() for y in v["years"]]
    if rest:
        c = collections.Counter(rest)
        print("years mentioned outside boilerplate, by decade:")
        for dec in sorted({y // 10 * 10 for y in rest}):
            print("  %ds  %5d" % (dec, sum(n for y, n in c.items() if y // 10 * 10 == dec)))
        print("  median %d, 90th percentile %d, max %d"
              % (sorted(rest)[len(rest) // 2], sorted(rest)[int(len(rest) * .9)], max(rest)))
    # The decade table above is not a measurement of misdating and must not be quoted as one. Read
    # the windows in drift.jsonl and most of them are forward references a correctly-dated book is
    # entitled to make: Microsoft Word 2000, Census 2000, a 1992 tourism plan describing its 2000
    # campaign, a HUD planning package called Community 2020. The rule drops them anyway, which is
    # the conservative direction, but calling the result "misdated" would be reading a filter's
    # cost as a finding.
    #
    # What can be counted without judging anything: places where the record and the text flatly
    # contradict each other. Two tests, reported separately rather than merged, because merging
    # them needs a rule for deciding whether "Census 2000" in a title is a date -- and a statistic
    # that needs a tie-break is a statistic I will end up implementing twice.
    COPY = re.compile(r"(©|\(c\)|copyright)\s*(20[0-3]\d)", re.I)
    tw = {k: v for k, v in per_item.items() if v["title"] and FUTURE_YEAR.search(v["title"])}
    cw = {k: [int(COPY.search(w).group(2)) for w in v["windows"] if COPY.search(w)]
          for k, v in per_item.items()}
    cw = {k: v for k, v in cw.items() if v}
    summary = {
        "psgs": n_dropped, "items": len(per_item),
        "mentions": prov_hits + other_hits, "provenance": prov_hits,
        "title_year": len(tw), "copyright_year": len(cw),
        "either": len(set(tw) | set(cw)),
        "latest_copyright": max((y for ys in cw.values() for y in ys), default=None),
    }
    json.dump(summary, open(os.path.join(HERE, "drift.json"), "w"), indent=2)
    print("flat contradictions between the record and the text:")
    print("  a post-1999 year in the item's own title : %d items" % summary["title_year"])
    print("  a post-1999 copyright line in the text   : %d items (latest %s)"
          % (summary["copyright_year"], summary["latest_copyright"]))
    print("  either test                              : %d of %d" % (summary["either"], len(per_item)))
    print()
    print("wrote %s (%d rows) and drift.json" % (OUT, rows))


if __name__ == "__main__":
    main()