#!/usr/bin/env python3 """Analysis for the detector false-positive study. Every number the writeup quotes comes from here. Runs on whatever has been scored so far. That is safe *only* because score.py shuffles with a fixed seed before scoring, so any prefix of scores_*.jsonl is a uniform random sample of the corpus rather than a genre-ordered slice of it. Partial output is labelled as partial. The headline is one number under one rule, fixed in PREREG.md before any scoring: flagged = P(AI) > 0.5 Everything else here — the threshold sweep, the OOV split, the length bins — is context for that number, not an alternative source for it. (I have published a count from one decision rule beside a worst case from another. It flattered me, and I had to correct it in public on two networks.) """ import json, math, os, sys from collections import defaultdict sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import clusterci import registers from oov import bucket as oov_bucket # the page bins OOV with this too; see oov.bucket from padding_check import THRESHOLDS as SWEEP_T # the sweep points, one copy, shared with the page HERE = os.path.dirname(os.path.abspath(__file__)) THRESH = 0.5 # The era-matched window. The two corpora's medians sit a decade apart, so the headline difference # is also recomputed on the years where they overlap. build_page.py imports this rather than # keeping its own 1990, and the page's prose interpolates it: three copies of one cut point, and # the two that are computed feed a check that says the page and this file hold the same numbers. ERA = 1990 def load(path, key="n"): if not os.path.exists(path): return {} out = {} for line in open(path): try: d = json.loads(line) except Exception: continue # a killed writer can leave one torn final line out[d[key]] = d return out # Wilson lives in clusterci, which is where the other interval on this page is computed. It used # to live here too, character for character — two copies that agreed, which is the state a rule is # in immediately before it stops agreeing, and the page's own claim is that every number on it # appears in this script's output. One implementation, imported twice. wilson = clusterci.wilson # The pre-registration's one live decision rule splits the corpus on this predicate, and three # programs had written it out for themselves: here, twice in build_page.py, once in selftest.py. # They agreed, which is where a rule sits right before it stops agreeing — and this is the split # that decides which number is the headline, so a drift in it is not a cosmetic drift. The display # labels stay local to each caller, because they are prose and they differ on purpose; what is # shared is what counts as strict. See TIER_LIMIT in build_page.py for the threshold, which was # the same duplication one level up. def tier(row): return "strict" if row.get("strict") else "loose" def check_keys(scores, index, what): """Every scored id must exist in the corpus it claims to come from. The corpus is digest-pinned, so this should never fire — but the failure it guards against is a corpus rebuilt underneath a running scan, and the two sides of this study fail differently under it. On the adversarial side `by_n[m]` raises KeyError, eleven hours in, at the end of the chain. On the control side nothing raises at all: control rows are looked up from the corpus, so an orphaned score is simply never read and the rate is computed over whatever survived. Silence is the worse of the two. Both are refusals here. """ orphans = sorted(m for m in scores if m not in index) if orphans: print("refusing to analyse: %d of %d %s scores name passages absent from the corpus " "(e.g. %s) — the score file and the corpus are not about the same text." % (len(orphans), len(scores), what, ", ".join(orphans[:3])), file=sys.stderr) sys.exit(1) def pct(k, n): if not n: return " n/a" lo, hi = wilson(k, n) return "%5.2f%% [%.2f-%.2f]" % (100 * k / n, 100 * lo, 100 * hi) def bucketed(rows, scores, keyfn, label, idkey="item"): """Every breakdown table, with both intervals. The by-book column was added when the page's breakdown tables started publishing it: the page says every rate and interval on it appears in this file, and a bootstrap interval printed in one place and not the other would make that sentence false the same way it was false three ways before. Same estimator, same seed, same shared clusterci — so the two files cannot disagree unless the corpus does. idkey is which field names the book: the adversarial rows call it "item", the control's call it "id". Defaulting it would have quietly clustered the control by a field it does not have. """ print("\n %-28s %7s %6s %8s %-24s %s" % (label, "n", "books", "flagged", "rate [95% CI passages]", "[95% CI by book]")) groups = defaultdict(lambda: [0, 0]) grows = defaultdict(list) for r in rows: s = scores.get(r["n"]) if not s: continue g = keyfn(r) if g is None: continue groups[g][1] += 1 groups[g][0] += s["p_ai"] > THRESH grows[g].append(r) for g in sorted(groups): k, n = groups[g] cl = clusterci.by_cluster(grows[g], scores, idkey, THRESH) blo, bhi = clusterci.cluster_bootstrap(cl) print(" %-28s %7d %6d %8d %-24s [%.2f-%.2f]" % (g, n, len(cl), k, pct(k, n).strip(), 100 * blo, 100 * bhi)) return groups # the tier gap is computed from this; see the call site def main(): rows = [json.loads(l) for l in open(os.path.join(HERE, "passages.jsonl"))] by_n = {r["n"]: r for r in rows} oov = load(os.path.join(HERE, "oov_passages.jsonl")) toks = load(os.path.join(HERE, "tokens.jsonl")) for key in ("hello", "openai"): scores = load(os.path.join(HERE, "scores_%s.jsonl" % key)) if not scores: continue check_keys(scores, by_n, key) n = len(scores) k = sum(1 for s in scores.values() if s["p_ai"] > THRESH) state = "COMPLETE" if n >= len(rows) else "PARTIAL (%.1f%% of corpus, seeded shuffle)" % (100 * n / len(rows)) print("\n" + "=" * 78) print("%s %s" % (key, state)) print("=" * 78) print(" adversarial corpus, P(AI)>0.5: %d / %d %s" % (k, n, pct(k, n))) print(" ** ceiling on these detectors' error, NOT a rate for 1990s books **") cl = clusterci.both([by_n[m] for m in scores], scores, "item", THRESH) print(" clustered by book (%d books, %.1f passages each): [%.2f-%.2f] %.2fx the " "passage-level width" % (cl[2], cl[1] / cl[2], 100 * cl[5], 100 * cl[6], cl[7])) # Are false positives a property of the passage or of the book? If whole books read as # machine-written, flags cluster and the variance across books exceeds binomial. Only # books with 2+ scored passages can show this at all, so the line is meaningless until # coverage is high — at 1.6 passages/book most books CANNOT show two flags, and the # apparent 1-flag-per-book pattern would be measuring the scan's progress, not the text. cl2 = [c for c in clusterci.by_cluster([by_n[m] for m in scores], scores, "item", THRESH) if c[1] >= 2] if cl2: nn = sum(c[1] for c in cl2) kk = sum(c[0] for c in cl2) p = kk / nn expected = sum(c[1] * p * (1 - p) for c in cl2) observed = sum((c[0] - c[1] * p) ** 2 for c in cl2) print(" dispersion across books with 2+ scored (%d books, %.1f passages each): " "%.2fx binomial%s" % (len(cl2), nn / len(cl2), observed / expected if expected else float("nan"), " [too little coverage to read]" if nn / len(cl2) < 3 else "")) ps = sorted(s["p_ai"] for s in scores.values()) qs = [0.5, 0.75, 0.9, 0.95, 0.99] print(" quantiles: " + " ".join("p%02d=%.4f" % (100 * q, ps[int(q * (len(ps) - 1))]) for q in qs)) print(" sweep: " + " ".join( ">%.2f: %.2f%%" % (t, 100 * sum(1 for p in ps if p > t) / n) for t in SWEEP_T)) # How close does anything get to the decision boundary? This is what decides whether the # headline count could be moved by float-level noise — batch padding width, thread count, # a torch version. Those perturb a logit by ~1e-6. If nothing sits within orders of # magnitude of that, the count is not a knife-edge and the question is closed. near = ["%.2f:%d" % (t, sum(1 for p in ps if abs(p - t) < 1e-2)) for t in SWEEP_T] print(" within 0.01 of each threshold: " + " ".join(near)) # And the answer to that question, from the check that measures it directly rather than # bounding it. The page publishes this table; this file promised to publish every number # the page does, and did not, because padding_check.py writes its own output and nothing # here read it. Same two fields the page reads, out of the same four files. for tag, what in ((key, "adversarial"), ("control_%s" % key, "control")): pd = list(load(os.path.join(HERE, "padding_%s.jsonl" % tag)).values()) if not pd: continue print(" batching check, %-11s %d re-scored singly, largest |delta| %.3g, " "verdict flips %d" % (what + ":", len(pd), max(abs(d["delta"]) for d in pd), sum(any((d["batched"] > t) != (d["single"] > t) for t in SWEEP_T) for d in pd))) scored = [by_n[m] for m in scores] TIER_LABEL = {"strict": "strict (book-date-verified)", "loose": "loose (catalogue year only)"} tiers = bucketed(scored, scores, lambda r: TIER_LABEL[tier(r)], "date verification") # The pre-registration's one live decision rule lives on the difference between those two # rows, and the page quotes it as a number. It was derivable from the table and printed # nowhere, which is not the same as published — the page says every rate, interval and # difference on it appears in this file, and a reader should not have to do the subtraction # to check the one number the pre-registration turns on. # strict minus loose, in that order, because build_page.py computes the sign that way and # two files disagreeing on the sign of one number is worse than not printing it. st = next((v for kk, v in tiers.items() if kk.startswith("strict")), None) lo_ = next((v for kk, v in tiers.items() if kk.startswith("loose")), None) if st and lo_ and st[1] and lo_[1]: print(" TIER GAP (strict minus loose): %+.2f pp [PREREG: strict becomes the " "headline above 2.00 pp]" % (100 * (st[0] / st[1] - lo_[0] / lo_[1]))) bucketed(scored, scores, registers.family, "source query family") bucketed(scored, scores, lambda r: (lambda w: "%3d-%3d words" % (w // 60 * 60, w // 60 * 60 + 59))(len(r["text"].split())), "passage length") # score.py truncates at 512 BPE tokens, so a long passage is silently scored as a shorter # one. Measured rather than waved at: it reaches 0.2% of the corpus, which is too few to # distort anything, but "too few" is a number I had to go and get. if toks: tr = [r for r in scored if toks.get(r["n"], {}).get("trunc")] trk = sum(1 for r in tr if scores[r["n"]]["p_ai"] > THRESH) print(" truncated at 512 tokens: %d of %d scored (%.2f%%)%s" % (len(tr), len(scored), 100 * len(tr) / len(scored), " flagged %d, %s" % (trk, pct(trk, len(tr))) if tr else "")) if oov: def oovbin(r): return oov_bucket(oov.get(r["n"], {}).get("oov")) bucketed(scored, scores, oovbin, "OCR damage") cpath = os.path.join(HERE, "scores_control_%s.jsonl" % key) cscores = load(cpath) if cscores: crows = [] for line in open(os.path.join(HERE, "control.jsonl")): d = json.loads(line) d["n"] = "%s#%d" % (d["id"], d["seq"]) crows.append(d) check_keys(cscores, {r["n"] for r in crows}, "control/" + key) cn = len(cscores) ck = sum(1 for s in cscores.values() if s["p_ai"] > THRESH) print("\n CONTROL (genre-neutral draw, same archive/era/extraction):") print(" control corpus, P(AI)>0.5: %d / %d %s" % (ck, cn, pct(ck, cn))) ccl = clusterci.both(crows, cscores, "id", THRESH) print(" clustered by book (%d books, %.1f passages each): [%.2f-%.2f] %.2fx the " "passage-level width" % (ccl[2], ccl[1] / ccl[2], 100 * ccl[5], 100 * ccl[6], ccl[7])) print(" this is the number that describes 1990s books; the one above describes the detectors") # The page's sweep table has four columns — both detectors over both corpora — and # this file had one, the adversarial. Ten of its twenty cells appeared nowhere in the # output that claims to hold every rate on the page. cps = sorted(s["p_ai"] for s in cscores.values()) print(" sweep: " + " ".join( ">%.2f: %.2f%%" % (t, 100 * sum(1 for p in cps if p > t) / cn) for t in SWEEP_T)) dlo, dhi = clusterci.diff_bootstrap( clusterci.by_cluster([by_n[m] for m in scores], scores, "item", THRESH), clusterci.by_cluster(crows, cscores, "id", THRESH)) print(" DIFFERENCE (adversarial - control): %+.2f pp 95%% [%+.2f, %+.2f]" % (100 * (k / n - ck / cn), 100 * dlo, 100 * dhi)) # Four archive.org items were drawn into both corpora, contributing byte-identical # passages to each side. They stay in — see PREREG — because removing them would # strip from the control exactly the books that matched a register-targeted query, # which inflates the difference. This is the robustness line that shows it does not # matter either way. shared = {r["item"] for r in rows} & {r["id"] for r in crows} if shared: arows = [by_n[m] for m in scores if by_n[m]["item"] not in shared] brows = [r for r in crows if r["id"] not in shared] ak = sum(1 for r in arows if scores[r["n"]]["p_ai"] > THRESH) bk = sum(1 for r in brows if r["n"] in cscores and cscores[r["n"]]["p_ai"] > THRESH) bn = sum(1 for r in brows if r["n"] in cscores) xlo, xhi = clusterci.diff_bootstrap( clusterci.by_cluster(arows, scores, "item", THRESH), clusterci.by_cluster(brows, cscores, "id", THRESH)) # Both component rates, not just their difference: the page's table publishes all # three, and a number a reader has to reconstruct is not one this file published. print(" excluding the %d books in both corpora: adv %.2f%% ctl %s " "%+.2f pp 95%% [%+.2f, %+.2f]" % (len(shared), 100 * ak / len(arows), "%.2f%%" % (100 * bk / bn) if bn else "n/a", 100 * (ak / len(arows) - bk / bn) if bn else float("nan"), 100 * xlo, 100 * xhi)) # Free padding check by a different route: identical text, scored twice, in two # corpora under two shuffles — so each copy sat in a different batch beside # different passages at a different pad width. If batching perturbed a score, # this is where it would show, on real data and at no compute cost. import hashlib ah = {} for r in rows: if r["item"] in shared and r["n"] in scores: ah[hashlib.sha1(r["text"].encode()).hexdigest()] = scores[r["n"]]["p_ai"] pairs = [] for r in crows: if r["id"] in shared and r["n"] in cscores: h = hashlib.sha1(r["text"].encode()).hexdigest() if h in ah: pairs.append((ah[h], cscores[r["n"]]["p_ai"])) if pairs: worst = max(abs(a - b) for a, b in pairs) flips = sum((a > THRESH) != (b > THRESH) for a, b in pairs) print(" same text scored in both corpora: %d passages, largest delta %.3g, " "verdict flips %d" % (len(pairs), worst, flips)) # Era-matched difference. The control's median year is ~9 years earlier than the # adversarial corpus's, because a genre-neutral draw lands where the archive's mass # is. Era could move a detector on its own, so the same statistic is recomputed on # the window where the two corpora actually overlap. Pre-registered before any # control rate was computed; published whatever it shows. ea = [by_n[m] for m in scores if by_n[m]["year"] >= ERA] eb = [r for r in crows if r["year"] >= ERA and r["n"] in cscores] if ea and eb: eak = sum(1 for r in ea if scores[r["n"]]["p_ai"] > THRESH) ebk = sum(1 for r in eb if cscores[r["n"]]["p_ai"] > THRESH) elo, ehi = clusterci.diff_bootstrap( clusterci.by_cluster(ea, scores, "item", THRESH), clusterci.by_cluster(eb, cscores, "id", THRESH)) print(" ERA-MATCHED (%d+ only, %d adv / %d ctl): adv %s ctl %s" % (ERA, len(ea), len(eb), pct(eak, len(ea)), pct(ebk, len(eb)))) print(" DIFFERENCE, era-matched: %+.2f pp 95%% [%+.2f, %+.2f]" % (100 * (eak / len(ea) - ebk / len(eb)), 100 * elo, 100 * ehi)) # And the per-decade rates that let a reader judge whether era moves it at all. for nm, rr, ss in (("adversarial", [by_n[m] for m in scores], scores), ("control", [r for r in crows if r["n"] in cscores], cscores)): bucketed(rr, ss, lambda r: "%ds" % (r["year"] // 10 * 10), "decade, " + nm, "item" if nm == "adversarial" else "id") # "2000s" reads as 2000-2009, and the whole study rests on nothing being # catalogued after 2000. Say what that bucket actually holds rather than let a # decade label imply a decade the corpus does not contain. last = [r["year"] for r in rr if r["year"] >= 2000] if last: lo, hi = min(last), max(last) print(" (the 2000s bucket is %s, not a decade: nothing in this corpus is " "catalogued later than %d)" % ("%d only" % lo if lo == hi else "%d-%d" % (lo, hi), hi)) else: print("\n CONTROL: not yet scored") if __name__ == "__main__": main()