#!/usr/bin/env python3 """Render the writeup from the data. No number that this study measured is typed by hand: every rate, count, interval and median on the page is substituted from the score files. What is left hand-written is the citations of the earlier Pangram writeup — 0.994 for the Navy manual, and sixteen passages from fourteen documents plus one control. Those are measurements from another study, checkable at their source, and on 2026-08-28 I went and checked them there rather than trusting the sentence I had written: p_ai 0.99435 on seq 2 of the 1992 NEETS module in that study's `detected.jsonl`, which Pangram called Human; and its own results file holds seventeen non-control submissions, one of which died on "you're out of free scans" and is correctly not among the sixteen. (The Hugging Face download counts used to be in this list. They are now read once into a dated `hf_downloads.json`, published, and substituted from it — a citation with a fetch date beats a citation with a recollection, and mine was 5% out.) Numbers are the easy half. The page also *describes* things — "three of them are books about MS-DOS", "two items alone supplied ten passages each" — and a description sitting beside a correctly substituted number is the failure this build cannot see by substituting. Two of those were wrong. Where the page enumerates its own evidence, the enumeration is pinned by a refusal below, so the count moving forces the sentence to be rewritten rather than silently invalidating it. Two inputs: page.tmpl.html structure and the prose that is fixed regardless of how the numbers land copy.json the narrative strings that depend on the result (title, lede, the sentence that says what the headline means) — authored AFTER the numbers are in The split is deliberate. Prose written while the answer is still unknown cannot be written toward the answer; prose written afterwards has to be, and keeping it in a separate file makes it obvious which is which. The build fails if any `{{PLACEHOLDER}}` survives substitution. That check runs over the whole document at once, not line by line: a placeholder that straddles a newline is exactly the kind a per-line grep misses, and I have shipped that bug before — the tell was a count that went *down* after an edit. python3 build_page.py [outdir] python3 build_page.py --dateline='28 August 2026' # republish under the original date python3 build_page.py --anyway # override the refuse-beside-a-live-scan check """ import hashlib, html, json, math, os, re, statistics, sys, time from collections import Counter, defaultdict sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import clusterci import registers # One source for the published thresholds. Written out here once, they would drift # from padding_check's copy on the branch no test reaches. score.py imports torch # lazily, so this costs nothing at import time. from padding_check import THRESHOLDS as SWEEP_T, NEAR as SWEEP_NEAR, select_sample from oov import bucket as oov_bucket # one copy of the OOV row labels; see oov.bucket # And the decision rule itself, from the script whose docstring says every number the writeup # quotes comes out of it. It was 0.5 here and 0.5 there, which is two places to change a # pre-registered rule and one place to forget. from analyse import THRESH, tier, ERA HERE = os.path.dirname(os.path.abspath(__file__)) # The pre-registered tier rule's threshold. Module scope rather than buried in build(), because # selftest.py reports the current gap against it every run and had its own hard-coded 2.0 — a # second copy of the one number this study's headline can turn on, in the file whose job is to # tell me early that it is about to fire. MED_GAP_LIMIT was the same story one refusal over: # 15 words, 5% of the cutter's 300-word target, written down before either median existed. TIER_LIMIT = 2.0 MED_GAP_LIMIT = 15 def median_words(rows): """The length statistic the page publishes and the build refuses on. One expression: selftest reported this gap early from its own identical copy, which is a warning system that can go stale in exactly the direction that keeps it quiet.""" return int(statistics.median(len(r["text"].split()) for r in rows)) MODELS = [("hello", "Hello-SimpleAI/chatgpt-detector-roberta"), ("openai", "openai-community/roberta-base-openai-detector")] # Module scope so selftest.py can check every href on the page against what actually gets # written, without keeping a second hand-copied list of filenames to drift from this one. SCRIPT_FILES = ("merge.py", "fetch_control.py", "score.py", "oov.py", "analyse.py", "drift.py", "registers.py", "clusterci.py", "padding_check.py", "build_page.py", "selftest.py", "tokens.py", "chain.sh", "run_scan.sh", # The analysis stage and its waiter. finish.sh is not a tidy-up script: it is the # only thing that produces analysis.txt for this run, because the chain copy that # is executing predates that stage. Publishing chain.sh alone would describe a # pipeline with a step nobody ran. "finish.sh", "run_analysis.sh", # The frozen copies that actually executed. See the divergence check in main(). "chain.run.sh", "finish.run.sh") # The code that chose and cut every adversarial passage is not in this directory: the corpus was # built in the earlier project this study grew out of, and merge.py only merges its output. So the # guard that keeps "every script is on this page" true — it walks *this* directory — is blind to # exactly the scripts the selection claim rests on. index.jsonl tells a reader which corpus file # each passage came from, by name, and until now nothing published the script that wrote that file. # fetch_corpus.py is the shared library: the archive search, the OCR normaliser, and the passages() # cutter that both corpora were cut by, which is what "same extraction code" above means. The nine # numbered scripts are the query families themselves. UPSTREAM_DIR = "/home/agent/work/pangram" UPSTREAM_SCRIPTS = ("fetch_corpus.py",) + tuple("fetch%d.py" % i for i in range(2, 11)) def unbacked_rates(authored, computed_values): """Rates in the hand-written copy that no computed value backs. See the call site for why.""" computed = re.sub(r"\s+", " ", "\n".join( v for k, v in computed_values.items() if k not in authored and isinstance(v, str))) cpcts = [float(x) for x in re.findall(r"(\d+(?:\.\d+)?)\s*%", computed)] out = [] for k, v in sorted(authored.items()): if not isinstance(v, str) or k.startswith("_"): continue flat = re.sub(r"\s+", " ", v) for m in re.finditer(r"(\d+(?:\.\d+)?)\s*%", flat): dec = len(m.group(1).split(".")[1]) if "." in m.group(1) else 0 if not any(round(c, dec) == float(m.group(1)) for c in cpcts): out.append("%s: %r" % (k, m.group(0))) for m in re.finditer(r"\b\d[\d,]*\s+of\s+\d[\d,]*\b", flat): if m.group(0) not in computed: out.append("%s: %r" % (k, m.group(0))) return out DATA_FILES = ("scores_hello.jsonl", "scores_openai.jsonl", "scores_control_hello.jsonl", "scores_control_openai.jsonl", "oov_passages.jsonl", "control.jsonl", "control_v0_discarded.jsonl", "padding_hello.jsonl", "padding_openai.jsonl", "padding_control_hello.jsonl", "padding_control_openai.jsonl", "index.jsonl", # The draw log. Published because the page quotes a number out of it — how many # books the read-the-text date filter rejected — and a number a reader cannot check # is a number they have to take on trust. It also lists every item the draw saw and # why each was skipped, which is the only surviving record of that. "control.log", # The discarded-passage evidence behind the catalogue-drift paragraph. The # adversarial corpus itself stays unpublished; this is 196 windows of 120 # characters from passages that were thrown away, which is the difference # between citing a source and redistributing it, and without it the drift # numbers are three assertions in a limitations section. "drift.jsonl", # The download reading behind the first paragraph. Dated, fetched once, # and not refetched at build time: a page whose numbers move under the # reader cannot be checked against anything. "hf_downloads.json", # BPE token count and truncation flag per passage. analysis.txt prints a # truncation rate off this file, and analysis.txt is published — but the input # was not, and it is the one input a reader cannot rebuild: it needs the # adversarial passage text, which stays unpublished by design. index.jsonl # carries words, not tokens, so there was no second route to the number either. # Every other per-passage intermediate the analysis reads is already here # (oov_passages, padding_*); this one was missed because its number never made # the page, which is not the same as never being published. "tokens.jsonl", # The twelve passages that sit in the gap between B3 as written ("no post-1999 year") # and B3 as enforced (merge.py's FUTURE_YEAR, which stops at 2039). The limitations # section used to say the passage-level rule ran with no exceptions; these are the # exceptions, so they ship with the reason each one was kept. A reader who thinks # "Project 2061" should have cost a 1991 book its place can find every instance here # and re-run the numbers without them. "b3_wide_exceptions.json") ROOT_FILES = ("prereg.txt", "analysis.txt", "corpus.sha256") def refuse_if_scan_live(argv=None): """Refuse to load the corpus and every score file while a scan holds a gigabyte of this box. selftest.py has had this since I broke the rule six times in one sitting. build_page.py did not, and it is the larger reader of the two — which I discovered by running it twice against a live scan inside a *control test for an unrelated refusal*, i.e. through the same hole the heredoc slipped through: it did not feel like reading data, it felt like checking a guard. The kernel kills the largest RSS, which is always the scan and never this process, so nothing about the damage is visible from here. Override with --anyway when the scan is finished or when you mean it. """ argv = sys.argv if argv is None else argv if "--anyway" in argv: return me, busy = str(os.getpid()), [] for p in os.listdir("/proc"): # Own pid excluded: a checker whose own command line matches its pattern is the oldest # trap in this repo, and here the argv of a --anyway run would contain the words too. if not p.isdigit() or p == me: continue try: argv = open("/proc/%s/cmdline" % p, "rb").read().decode("utf8", "replace").split("\0") except OSError: continue argv = [a for a in argv if a] if not argv: continue # An interpreter running the script, not any process whose command line contains its name. # A substring test matched the harness shell that had launched the chain hours earlier -- # a stale bash holding the words "score.py" in its argv -- so the guard would have kept # refusing long after the scan finished, and a guard that never clears is one I pass # --anyway to by reflex until it means nothing. SCANS = ("score.py", "padding_check.py") head = os.path.basename(argv[0]) # `python3 score.py hello` (how run_scan.sh calls it) or a shebang exec of `./score.py`. hit = (head.startswith("python") and any(os.path.basename(a) in SCANS for a in argv[1:]) or head in SCANS) if hit: busy.append("%s %s" % (p, " ".join(argv))) if busy: print("refusing to run: a scan is live and this would push the box into the OOM killer, " "which takes the scan and not this process:\n " + "\n ".join(busy) + "\nRe-run with --anyway if you have a reason.", file=sys.stderr) sys.exit(2) def orphans(written, hrefs): """Shipped files that no link on the page reaches. Asked twice — by the build against the rendered page, and by selftest.py against the template plus copy.json, hours earlier — so it is one function. Both copies had independently grown a `scripts/|data/` prefix filter, which exempted exactly the three ROOT_FILES from the check, one of which is the pre-registration the page's strongest claim rests on. A guard with a filter in it is two rules, and the second one is the one nobody wrote down.""" return sorted(set(written) - set(hrefs)) # B1/B2, as revised in the pre-registration once the merge rule was written down. The original # ceilings (12,206 / 1,814) were recollections of a one-off shell computation and turned out to be # unreproducible by any of six rules; these are outputs of merge.py, fixed before any passage was # scored. merge.py prints its counts against the *old* numbers and asserts nothing, so until now # these bounds were narrative. B1_PASSAGES, B2_ITEMS = 12247, 1809 # "If both detectors flag under ~2% of the adversarial corpus, there is no story and I will say so # and publish the number anyway." That is a rule about what the page is allowed to claim, and like # the tier rule it was written down and then computed by nothing. NULL_RATE = 0.02 def jl(path, key="n"): if not os.path.exists(path): return {} out = {} for line in open(path): try: d = json.loads(line) except Exception: continue out[d[key]] = d return out def rate(k, n): # clusterci.wilson, not a second copy of it. There was one here, hand-written, arithmetically # identical to the other — which is the state a shared function is in right before the two # copies stop matching, on whichever branch the tests do not reach. The headline table compares # this interval's width against the bootstrap's to decide which one it is allowed to call the # headline, so "identical" needed to be a fact about the code rather than about the day I wrote # it. lo, hi = clusterci.wilson(k, n) return "%.2f%%" % (100 * k / n), "%.2f–%.2f%%" % (100 * lo, 100 * hi) def losing_bounds(authored, lost, won): """Authored strings that quote a *losing* interval bound without the winning one beside it. The rates check upstream asks whether a percentage in the copy is one the build computed. Both intervals are computed, and both are rendered into the table, so the narrower one satisfies it perfectly — which means the one guard standing between me and the flattering number does not cover the flattering number. The narrower interval is always the more impressive claim, it is always sitting right there in the same row, and taking the count from one decision rule and the worst case from another is a mistake I have shipped and had to correct publicly, twice. Quoting the narrow bound is allowed when the wide one appears in the same string, because "0.38-0.52% by passage, 0.30-0.60% clustered by book" is the honest version of the sentence. """ out = [] for k, v in sorted(authored.items()): if not isinstance(v, str) or k.startswith("_"): continue flat = re.sub(r"\s+", " ", v) here = {m.group(1) for m in re.finditer(r"(\d+(?:\.\d+)?)\s*%", flat)} for b in sorted(here & lost): if b in won or (won & here): continue out.append("%s: %s%% is a bound of the narrower interval" % (k, b)) return out def intervals(f, n, cl, wins=None, bounds=None): """Both intervals for one row, with the wider marked as the headline. The one copy of the pre-registered rule. It lived inline in headline(), which was fine while the headline table was the only place two intervals were published — and it stopped being fine the moment the breakdown tables started publishing them too. Two hand-written copies of a comparison rule drift toward whichever branch nothing checks, and this one decides which number a reader is told to believe. cl is the (flagged, total)-per-book list the bootstrap resamples; f/n are the passage counts. Returns (rate, passage-level cell, by-book cell), one of the two cells bolded. """ r, ci = rate(f, n) blo, bhi = clusterci.cluster_bootstrap(cl) wlo, whi = clusterci.wilson(f, n) boot = "%.2f–%.2f%%" % (100 * blo, 100 * bhi) won = "by book" if (bhi - blo) >= (whi - wlo) else "by passage" if won == "by book": boot = "%s" % boot else: ci = "%s" % ci if wins is not None: wins[won] += 1 if bounds is not None: wide, narrow = ((blo, bhi), (wlo, whi)) if won == "by book" else ((wlo, whi), (blo, bhi)) bounds["won"].update("%.2f" % (100 * x) for x in wide) bounds["lost"].update("%.2f" % (100 * x) for x in narrow) return r, ci, boot def flagged(scores): return sum(1 for s in scores.values() if s["p_ai"] > THRESH) def table(head, rows): h = "".join("%s" % html.escape(c) for c in head) body = "" for r in rows: body += "" + "".join( '%s' % c if i else "%s" % c for i, c in enumerate(r)) + "" return "%s%s
" % (h, body) def grouped(rows, scores, keyfn): g = defaultdict(lambda: [0, 0]) for r in rows: s = scores.get(r["n"]) if not s: continue k = keyfn(r) if k is None: continue g[k][1] += 1 g[k][0] += s["p_ai"] > THRESH return g def diff_row(label, arows, asc, brows, bsc): """One row of the adversarial-minus-control table. Module scope rather than a closure inside main() so it can be exercised on partial scores. Everything downstream of the completeness guard is otherwise untestable until the last scan lands, which is the worst possible moment to discover a typo in it. The interval is the cluster difference bootstrap — books resampled independently within each corpus — and not a comparison of the two per-corpus intervals. Overlapping intervals do not imply no difference, and that inference is ruled out in the pre-registration by name. """ ak = sum(1 for r in arows if asc[r["n"]]["p_ai"] > THRESH) bk = sum(1 for r in brows if bsc[r["n"]]["p_ai"] > THRESH) lo, hi = clusterci.diff_bootstrap( clusterci.by_cluster(arows, asc, "item", THRESH), clusterci.by_cluster(brows, bsc, "id", THRESH)) return [label, "%.2f%%" % (100 * ak / len(arows)), "%.2f%%" % (100 * bk / len(brows)), "%+.2f pp" % (100 * (ak / len(arows) - bk / len(brows))), "%+.2f to %+.2f pp" % (100 * lo, 100 * hi)] def headline(S, C, corpus, control): """The headline table, plus the two pre-registered rules that decide what it may claim. Returns (rows, refusals, adversarial rates by detector). Module scope, and returning refusals rather than calling sys.exit, so `selftest.py` can run the same code on a partial scan. Both rules would otherwise sit behind the completeness guard and be evaluated for the first time at publication — and one of them, if it fires, means the copy is wrong rather than the data. Rule one: "Both intervals are published side by side and the wider one is the headline." I had implemented that as "the by-book interval is the headline", on the assumption that clustering always widens. It does not always: with few flagged books the percentile bootstrap is discrete and piles mass at zero, and it came out *narrower* than Wilson on a partial control scan — 5.88 points against 6.67. So the rule is implemented as written instead. Both are published; the wider is marked as the headline, row by row, and which method won is stated on the page. This cannot flatter in either direction, because wider is always the weaker claim. Rule two: "If both detectors flag under ~2% of the adversarial corpus, there is no story and I will say so and publish the number anyway." A null result needs different copy; it should not be possible to ship the confident version by not noticing. """ rows, refusals, adv_rates, wins = [], [], {}, {"by book": 0, "by passage": 0} bounds = {"won": set(), "lost": set()} for k, name in MODELS: for label, sc, src, idkey in (("adversarial", S[k], corpus, "item"), ("control", C[k], control, "id")): n, f = len(sc), flagged(sc) if not n: continue # unreachable in a real build: the completeness guard runs first cl = clusterci.by_cluster(src, sc, idkey, THRESH) # A seeded cluster bootstrap draws cluster *positions*, so the order of the list handed # to it is part of its input: same books, same seed, different input order, different # published bound. `by_cluster` sorts before returning for exactly that reason, and the # bootstrap sees nothing but its output — so if this permutation agrees, the interval # below is order-invariant too. The fix was already in the code and nothing computed # it; a rule nothing computes never fires. Fixed permutation, not a random one: a check # that draws its own randomness fails on a different build than the one you are running. perm = list(src) perm.reverse() perm = perm[1::2] + perm[0::2] if clusterci.by_cluster(perm, sc, idkey, THRESH) != cl: print("refusing to build: %s/%s cluster aggregation depends on input order, so the " "published interval does too. clusterci.by_cluster must sort before it " "returns." % (k, label), file=sys.stderr) sys.exit(1) if label == "adversarial": adv_rates[k] = f / n r, ci, boot = intervals(f, n, cl, wins, bounds) rows.append(["%s · %s" % (name.split("/")[-1], label), "{:,}".format(n), "{:,}".format(len(cl)), "{:,}".format(f), r, ci, boot]) if adv_rates and all(v < NULL_RATE for v in adv_rates.values()): refusals.append("the pre-registered null condition HOLDS (%s, both under %.0f%%). There is " "no story, the copy has to say so, and the numbers get published anyway." % (", ".join("%s %.2f%%" % (k, 100 * v) for k, v in sorted(adv_rates.items())), 100 * NULL_RATE)) return rows, refusals, adv_rates, wins, bounds def main(): # --dateline="15 August 2026" republishes under the original date; see the DATE_LONG refusal. flags = [a for a in sys.argv[1:] if a.startswith("--")] rest = [a for a in sys.argv[1:] if not a.startswith("--")] dateline = next((f.split("=", 1)[1] for f in flags if f.startswith("--dateline=")), None) bad_flags = [f for f in flags if not f.startswith("--dateline=") and f != "--anyway"] if bad_flags: print("refusing to build: unknown flag %s" % ", ".join(bad_flags), file=sys.stderr) sys.exit(1) # Before the first big read, not after: the next two lines pull a 27 MB corpus and every # score file into a 1967 MB box that a live scan already holds a gigabyte of. refuse_if_scan_live() outdir = rest[0] if rest else os.path.join(HERE, "out") os.makedirs(outdir, exist_ok=True) corpus = [json.loads(l) for l in open(os.path.join(HERE, "passages.jsonl"))] by_n = {r["n"]: r for r in corpus} control = [] for line in open(os.path.join(HERE, "control.jsonl")): d = json.loads(line) d["n"] = "%s#%d" % (d["id"], d["seq"]) control.append(d) ctl_by_n = {r["n"]: r for r in control} oov = jl(os.path.join(HERE, "oov_passages.jsonl")) # B1 and B2. The pre-registration is explicit about which way this cuts: "If the count of # scored passages comes out above B1 or B2, the merge is wrong, not the corpus." Coming out # *under* is a different thing and not a breach — it would mean a file was truncated — so it # refuses on either side, with the direction named. n_items = len({r["item"] for r in corpus}) for label, got, want in (("B1 passages", len(corpus), B1_PASSAGES), ("B2 source items", n_items, B2_ITEMS)): if got != want: print("refusing to build: %s is %d, pre-registered at %d — %s" % ( label, got, want, "the merge is wrong, not the corpus" if got > want else "under the bound, so something has been dropped since it was fixed"), file=sys.stderr) sys.exit(1) # B5: the control is drawn once and not re-drawn after its rate is seen. Nothing was enforcing # that either. CORPUS.sha256 freezes both corpus files; see that file for what the digests can # and cannot testify to, given they were recorded partway through the control scan. frozen = {} for line in open(os.path.join(HERE, "CORPUS.sha256")): if not line.startswith("#") and line.strip(): digest, name = line.split()[0], line.split()[1] frozen[name] = digest for name, want in sorted(frozen.items()): h = hashlib.sha256() with open(os.path.join(HERE, name), "rb") as fh: for chunk in iter(lambda: fh.read(1 << 20), b""): h.update(chunk) if h.hexdigest() != want: print("refusing to build: %s has changed since it was frozen (%s, expected %s)" % (name, h.hexdigest()[:16], want[:16]), file=sys.stderr) sys.exit(1) S = {k: jl(os.path.join(HERE, "scores_%s.jsonl" % k)) for k, _ in MODELS} C = {k: jl(os.path.join(HERE, "scores_control_%s.jsonl" % k)) for k, _ in MODELS} for k, _ in MODELS: if len(S[k]) < len(corpus) or len(C[k]) < len(control): print("refusing to build: %s has %d/%d corpus and %d/%d control scored" % (k, len(S[k]), len(corpus), len(C[k]), len(control)), file=sys.stderr) sys.exit(1) # B3, pre-registered at 100%: no scored passage contains a post-1999 year in its own text. # merge.py enforces this for the adversarial corpus at draw time. fetch_control.py has no such # filter — the control satisfies B3, but it did so by luck until the day I measured it, and a # bound that holds by accident is not a bound. Check both corpora and refuse. # # The rule is imported, not re-typed. This block used to hand-write its own year regex, which # matched 2000-2099 where merge.py's FUTURE_YEAR stops at 2039, so the corpus and the check on # the corpus disagreed about twelve passages for as long as both copies existed. from merge import FUTURE_YEAR b3 = [(label, r["n"]) for label, rows in (("adversarial", corpus), ("control", control)) for r in rows if FUTURE_YEAR.search(r["text"])] if b3: print("refusing to build: B3 breached — %d scored passages carry a post-1999 year in " "their own text, e.g. %r" % (len(b3), b3[:3]), file=sys.stderr) sys.exit(1) # The gap between B3 as written ("no post-1999 year") and B3 as enforced (FUTURE_YEAR, which # stops at 2039) is the 2040-2099 range. Twelve adversarial passages fall in it. Every one was # read in context: seven say "Project 2061", the AAAS programme; the rest are a fax number, a # grant number, a solar eclipse in 2088 in a logic exercise, and two sentences about the # future. None is evidence that the text postdates 1999, and dropping a 1991 book for # containing a writing prompt set in 2050 would be a worse error than keeping it. So they stay, # enumerated in b3_wide_exceptions.json — and the build refuses if the set ever differs from # that file, because a thirteenth has not been read and this is not a rule I get to widen once # and forget. wide = re.compile(r"\b(20[4-9]\d)\b") wide_hits = sorted(r["n"] for label, rows in (("adversarial", corpus), ("control", control)) for r in rows if wide.search(r["text"])) known = json.load(open(os.path.join(HERE, "b3_wide_exceptions.json"))) known_ns = sorted(p["n"] for p in known["passages"]) if wide_hits != known_ns: print("refusing to build: %d passages carry a 2040-2099 token, b3_wide_exceptions.json " "records %d; unrecorded %r, stale %r — read them before rendering" % (len(wide_hits), len(known_ns), sorted(set(wide_hits) - set(known_ns)), sorted(set(known_ns) - set(wide_hits))), file=sys.stderr) sys.exit(1) # The premise of the whole study, PREREG line 23: every item is catalogued with a year before # 2001, which is what "guaranteed human" rests on. That is the one claim on this page with no # statistical hedge in front of it, and until now the only thing enforcing it was the draw — # a filter in a script that has already been revised twice. Same lesson as B3 one block up: a # bound the data happens to satisfy is not a bound. Both corpora currently top out at 2000. late = [(label, r["n"], r["year"]) for label, rows in (("adversarial", corpus), ("control", control)) for r in rows if r.get("year") is None or r["year"] > 2000] if late: print("refusing to build: %d scored passages are catalogued after 2000 or carry no year at " "all, e.g. %r — the page calls this corpus guaranteed human on the strength of that " "date" % (len(late), late[:3]), file=sys.stderr) sys.exit(1) # The discarded first control draw is published and the page describes it, so the description # has to be read out of the file rather than remembered. It wasn't: an earlier draft of the # page and of PREREG.md both said eleven books and twenty-nine passages, numbers taken from a # log of a different run entirely — an aborted first attempt at the *accepted* draw, which had # been misnamed control_v0_discarded.log. The published file says something else, and the file # is what a reader can check. v0 = [json.loads(l) for l in open(os.path.join(HERE, "control_v0_discarded.jsonl"))] sha = lambda t: hashlib.sha1(t.encode()).hexdigest() v0_reused = len({sha(r["text"]) for r in v0} & {sha(r["text"]) for r in control}) if v0_reused != 1: # The page says this in the singular — one book re-selected, one of its passages carried # into the scored control. If that stops being one, the sentence is wrong in a way no # substitution fixes. print("refusing to build: %d discarded passages also appear in the scored control, and " "the page describes that overlap in the singular" % v0_reused, file=sys.stderr) sys.exit(1) # "two items alone supplying ten passages each — an issue of a yoga magazine and an astrology # treatise". Another hand-written enumeration beside a computed count, which is the pairing # that has now gone wrong twice on this page. Both halves are pinned: exactly two items at ten, # nothing else at ten, and the two are the ones the sentence names. v0_sizes = Counter(r["id"] for r in v0) tens = [i for i, c in v0_sizes.items() if c == 10] if len(tens) != 2 or not (any("yoga" in i for i in tens) and any("jyothisha" in i for i in tens)): print("refusing to build: the page says exactly two items supplied ten passages each to the " "discarded draw, a yoga magazine and an astrology treatise; the file now says %s" % sorted((c, i) for i, c in v0_sizes.items())[-3:], file=sys.stderr) sys.exit(1) # How many books the read-the-text date filter threw out, taken from the draw's own log rather # than retyped from a terminal I once looked at — which is how the discarded-draw counts a few # lines up went wrong. The log is published alongside the data for the same reason. tail = [l for l in open(os.path.join(HERE, "control.log")) if l.startswith("books kept:")] m = re.search(r"misdated-dropped:\s*(\d+)", tail[-1]) if tail else None if not m: print("refusing to build: control.log has no 'books kept:' summary line, so the number of " "date-rejected books on the page would be invented", file=sys.stderr) sys.exit(1) misdated = int(m.group(1)) # The padding check ran four times — two detectors over two corpora — and until now nothing # read its output. A check whose result never reaches the page is not a check; the page even # referred to "the float-level noise measured below" when no such measurement was published # anywhere on it. Read padding_*.jsonl rather than the logs, and never retype a number. prows, pmiss, pworst = [], [], 0.0 for k, _ in MODELS: for corpname, tag, src in (("adversarial", k, S), ("control", "control_%s" % k, C)): path = os.path.join(HERE, "padding_%s.jsonl" % tag) ds = [json.loads(l) for l in open(path)] if os.path.exists(path) else [] if not ds: pmiss.append(os.path.basename(path)) continue # "Every passage within 0.01 of any published threshold is included by construction" # is a claim about a set, and the count beside it came from the file rather than from # the rule. padding_check.py opens its output with "w" and has no resume, so a kill # partway through leaves a well-formed jsonl holding a prefix of the sample — and the # sentence stays on the page with a smaller number beside it that reads like a design # choice. Rebuild the sample from the scores, through the same function the run used, # and demand the same set. want = set(select_sample({n: r["p_ai"] for n, r in src[k].items()})[0]) got = {d["n"] for d in ds} if got != want: print("refusing to build: %s holds %d of the %d passages its own selection rule " "picks — %d missing, %d it does not pick. The page claims every " "near-threshold passage was re-scored singly. Re-run " "`python3 padding_check.py %s%s`." % (os.path.basename(path), len(got & want), len(want), len(want - got), len(got - want), k, " --control" if corpname == "control" else ""), file=sys.stderr) sys.exit(1) worst = max(abs(d["delta"]) for d in ds) pworst = max(pworst, worst) fl = sum(any((d["batched"] > t) != (d["single"] > t) for t in SWEEP_T) for d in ds) prows.append(["%s, %s corpus" % (k, corpname), "{:,}".format(len(ds)), "%.3g" % worst, str(fl)]) if pmiss: print("refusing to build: padding check has not run (%s)" % ", ".join(pmiss), file=sys.stderr) sys.exit(1) # The page sends the reader to the analysis output for the per-decade rates. It said so in # prose, with no link, pointing at a file nothing produced — so the link checker below could # not see it either. Refuse rather than ship a promise to a file that is not there. apath = os.path.join(HERE, "analysis.txt") if not os.path.exists(apath): print("refusing to build: analysis.txt missing (python3 analyse.py > analysis.txt 2>&1)", file=sys.stderr) sys.exit(1) # And it has to be newer than everything it summarises. The chain script that is currently # executing was copied before the analyse.py stage was added to it, so this file will be # produced by hand — and a hand-run step is exactly the one that gets done early, forgotten, # and shipped stale next to numbers that have since moved. Staleness is invisible in a text # file; a timestamp comparison is not. stale = [os.path.basename(p) for p in [os.path.join(HERE, "scores_%s.jsonl" % t) for t in ["%s%s" % (c, k) for k, _ in MODELS for c in ("", "control_")]] + [os.path.join(HERE, "padding_%s.jsonl" % t) for t in ["%s%s" % (c, k) for k, _ in MODELS for c in ("", "control_")]] if os.path.exists(p) and os.path.getmtime(p) > os.path.getmtime(apath)] if stale: print("refusing to build: analysis.txt predates %s — re-run " "`python3 analyse.py > analysis.txt 2>&1`" % ", ".join(sorted(stale)), file=sys.stderr) sys.exit(1) # The page opens its reproduction section with "Every script is on this page." That is a claim # about a set, and nothing computed it: SCRIPT_FILES is a hand-maintained tuple, so the # sentence stays on the page and quietly stops being true the first time a script is added # without being listed. It is true right now — this is what keeps it true. unlisted = sorted(f for f in os.listdir(HERE) if f.endswith((".py", ".sh")) and f not in SCRIPT_FILES) if unlisted: print("refusing to build: the page claims every script is published and these are not: %s" " — add them to SCRIPT_FILES, or delete them if they are scratch." % ", ".join(unlisted), file=sys.stderr) sys.exit(1) # Publish what ran, not what I meant to run. A long chain is launched from a copy # (`cp chain.sh chain.run.sh && setsid ./chain.run.sh`) because bash reads a script by byte # offset and editing the original mid-run corrupts it hours later. The copy is then frozen # while the source keeps being edited — and it was: chain.sh grew its analysis stage nineteen # minutes after chain.run.sh was taken, so the published chain.sh describes a pipeline # slightly different from the one that produced the numbers. Nothing would have said so. If a # frozen copy exists and differs, it publishes alongside its source. frozen = [] for f in SCRIPT_FILES: run = f[:-3] + ".run.sh" if f.endswith(".sh") else None if run and os.path.exists(os.path.join(HERE, run)) and run not in SCRIPT_FILES: if open(os.path.join(HERE, run)).read() != open(os.path.join(HERE, f)).read(): frozen.append(run) if frozen: print("refusing to build: %s differ(s) from the source script(s) they were copied from, " "so they are what actually ran — add them to SCRIPT_FILES." % ", ".join(frozen), file=sys.stderr) sys.exit(1) dpath = os.path.join(HERE, "drift.json") if not os.path.exists(dpath) or os.path.getmtime(dpath) < os.path.getmtime( os.path.join(HERE, "merge.py")): print("refusing to build: drift.json is missing or older than the merge rule it " "measures — re-run `python3 drift.py`", file=sys.stderr) sys.exit(1) D = json.load(open(dpath)) HF = json.load(open(os.path.join(HERE, "hf_downloads.json"))) V = dict(json.load(open(os.path.join(HERE, "copy.json")))) # The dateline is a fact the build knows, and it was the one hand-written string that could go # stale without anyone touching it: a nine-hour scan chain that lands after midnight publishes # a page dated the day before, contradicting the mtimes of the very files it ships. Computed # here, with an explicit escape for a later correction that must keep its original date. if "DATE_LONG" in V and not dateline: print("refusing to build: copy.json sets DATE_LONG, but the build computes the dateline. " "Delete the key, or pass --dateline='28 August 2026' to republish under the " "original date.", file=sys.stderr) sys.exit(1) V["DATE_LONG"] = dateline or time.strftime("%-d %B %Y") V["PADDING_BLOCK"] = table( ["detector and corpus", "re-scored one at a time", "largest change", "verdict flips"], prows) # These three are spliced into blocks below rather than substituted into the template, so an # absent one would raise a KeyError halfway through instead of joining the same refusal as # every other missing string. Same failure, one message. missing = [k for k in ("HEADLINE_INTRO", "SWEEP_INTRO", "GENRE_INTRO") if k not in V] if missing: print("refusing to build: copy.json is missing %s" % ", ".join(missing), file=sys.stderr) sys.exit(1) V.update({ "N_CORPUS": "{:,}".format(len(corpus)), "N_ITEMS": "{:,}".format(len({r["item"] for r in corpus})), "N_CONTROL": "{:,}".format(len(control)), "N_CONTROL_ITEMS": "{:,}".format(len({r["id"] for r in control})), "MED_ADV": str(median_words(corpus)), "MED_CTL": str(median_words(control)), # The bound the paragraph above the table quotes. It said "fifteen" in prose, beside a # guard that said 15 in code: the pair a reader cannot see disagree. "MED_GAP": str(MED_GAP_LIMIT), # The batching check's inclusion radius, which the paragraph above that table # quotes as a number. Same rule as the two bounds above: padding_check owns it. "NEAR": ("%g" % SWEEP_NEAR), # The era-matched window, quoted in the prose above the difference table. analyse.py owns # it; this page had its own 1990 and the paragraph a third one. "ERA": str(ERA), # The era mismatch the draft page asserted away. Computed, not typed. "MAX_YEAR": str(max(r["year"] for r in corpus + control)), "MISDATED": "{:,}".format(misdated), # Catalogue-drift evidence, straight out of drift.py's own summary rather than # recomputed here, because a second implementation of "what counts as a copyright # line" is a second answer. "DRIFT_PSGS": "{:,}".format(D["psgs"]), "DRIFT_ITEMS": "{:,}".format(D["items"]), "DRIFT_MENTIONS": "{:,}".format(D["mentions"]), "DRIFT_PROV": str(D["provenance"]), "DRIFT_TITLE": str(D["title_year"]), "DRIFT_COPY": str(D["copyright_year"]), "DRIFT_LATEST": str(D["latest_copyright"]), "DL_OPENAI": "{:,}".format(round(HF["models"][MODELS[1][1]]["downloads_30d"], -3)), "DL_HELLO": "{:,}".format(round(HF["models"][MODELS[0][1]]["downloads_30d"], -3)), "DL_DATE": HF["read_at"][:10], "V0_BOOKS": str(len({r["id"] for r in v0})), "V0_PSGS": str(len(v0)), # The paragraph characterises the discarded draw and the file is published so a reader can # check the characterisation. "Microfilmed" is not in that file and its log is gone, so the # page now says only what the titles say: a single numbered issue, "Vol N Iss M". "V0_ISSUES": str(sum(1 for t in {r["id"]: r["title"] for r in v0}.values() if re.search(r"Vol\s+\d+\s+Iss\s+\d+", t))), # The date bullet used to say both draws ran the same book-level filter. The control did; # the adversarial corpus ran it only on the pulls made after that filter existed — merge.py's # STRICT set, four files out of ten. The split is computed here rather than typed into the # prose, because a hand-typed "four out of ten" stops being true the moment the corpus grows. # Same split as the tier table and the pre-registered rule, so the same function: these # two counts are the page's statement of how big each subset is, and a page whose counts # and whose table disagreed about what "strict" means would be wrong in the quietest way. "N_STRICT": "{:,}".format(sum(1 for r in corpus if tier(r) == "strict")), "N_LOOSE": "{:,}".format(sum(1 for r in corpus if tier(r) == "loose")), "MED_YEAR_ADV": str(int(statistics.median(r["year"] for r in corpus))), "MED_YEAR_CTL": str(int(statistics.median(r["year"] for r in control))), }) # "Same extraction code" is a claim about a mechanism, and the paragraph offers the two medians # in the table as its visible consequence — one `passages()`, one target, one pair of bounds, so # the two sides should land in the same place. An earlier draft asserted the stronger form ("the # same median length") outright, next to a table that publishes both numbers and could have # contradicted it in the same screenful. Bound written down before the adversarial median had # ever been computed: 15 words, 5% of the cutter's 300-word target. if abs(int(V["MED_ADV"]) - int(V["MED_CTL"])) > MED_GAP_LIMIT: print("refusing to build: the page says both corpora were cut by one function with one " "target, and the medians are %s and %s words — more than %d apart. Either the draws " "did not share the cutter or the sentence needs rewriting; do not ship both." % (V["MED_ADV"], V["MED_CTL"], MED_GAP_LIMIT), file=sys.stderr) sys.exit(1) # How compressed the top of the distribution actually is. The draft asserted that "several # passages tie" at six decimal places; on the data, nothing in the top two hundred ties with # anything. The true statement is a different one — the top scores are separated by amounts far # too small to mean anything, while being perfectly reproducible — and it needs three measured # numbers rather than a sentence. This is the same mistake as the "float-level noise measured # below" that was measured nowhere: a claim about the data written from an impression of it. top = sorted((s["p_ai"] for s in S["hello"].values()), reverse=True) V["TOP20_SPREAD"] = "%.4f" % (top[0] - top[19]) V["TOP13_GAP"] = "%.6f" % (top[0] - top[2]) V["P200"] = "%.3f" % top[199] # That paragraph calls the ordering of the top three real, and the reason it is allowed to is # that the batching wobble is smaller than the gap between them. Whether that holds is a fact # about the padding check's output, so check it rather than assert it. # # It used to be asserted twice over: the prose said "far lower", and this guard read "far" as # an order of magnitude. Both were true when the padding check had scored a couple of hundred # pairs and stopped being true as it scored more — the worst wobble grew, the gap did not, and # the margin closed to under 2x. The qualifier is now computed into PAD_RATIO and written into # the sentence, so the page states the margin instead of characterising it, and what remains # here is a refusal on the claim itself: if the wobble reaches the gap, the ordering is not # reproducible and that paragraph is wrong rather than overstated. if pworst >= (top[0] - top[2]): print("refusing to build: the page calls the ordering of the top three real, but the worst " "batched-vs-single difference is %.3g against a first-to-third gap of %.3g — the " "ordering is inside the run-to-run wobble" % (pworst, top[0] - top[2]), file=sys.stderr) sys.exit(1) V["PAD_RATIO"] = "%.1f" % ((top[0] - top[2]) / pworst) # The second procedural difference between the draws: fetch_control.py gates on language and # merge.py does not. Measured rather than argued — a passage counts as non-English if a # non-English function word sits among its three commonest tokens. Deliberately crude and # deliberately not silent about it: three of the four adversarial hits are MS-DOS manuals, where # the token is "dos", which is what a crude test looks like when you publish its output instead # of its conclusion. NON_EN = {"de", "het", "der", "die", "das", "el", "la", "les", "des", "und", "van", "il", "dos", "da", "na", "ir", "og", "att", "och", "ja", "ei", "ne", "se", "los", "le", "du"} def non_english(rows): n = 0 for r in rows: toks = re.findall(r"[a-z']+", r["text"].lower()) if len(toks) < 50: continue top = {w for w, _ in Counter(toks).most_common(3)} if "the" not in top and top & NON_EN: n += 1 return n V["NONENG_ADV"] = str(non_english(corpus)) V["NONENG_CTL"] = str(non_english(control)) # The paragraph does not just quote NONENG_ADV, it accounts for every one of them by hand, and # that enumeration is the whole argument — "the gate was removing something that was not # there". Prose cannot be kept true by substitution, so the count is pinned here. # # Writing this check is what caught the sentence being wrong. It said three MS-DOS books; there # are two (ms-dos-users-guide-and-reference, bp-341-ms-dos-explained). The third "dos" is # ERIC_ED402584, whose commonest tokens are and/of/dos/passos — a report about John Dos Passos. # I had written the plausible generalisation from a token I recognised, and the guard I added # to protect the sentence is what made me read the four passages. if V["NONENG_ADV"] != "4": print("refusing to build: the page enumerates the non-English hits in the adversarial " "corpus one by one — two MS-DOS manuals, one Dos Passos report, one Spanish passage " "— and there are now %s. Read them and rewrite the sentence." % V["NONENG_ADV"], file=sys.stderr) sys.exit(1) # Books drawn into both corpora. Computed here rather than written into the prose, because a # disclosure with a hand-typed count in it drifts away from the files the moment either corpus # changes — and this one was first written down against a control that was still being drawn. # Defined before the tables below, both of which need it. shared = {r["item"] for r in corpus} & {r["id"] for r in control} # headline: one rate per model per corpus, all at the one pre-registered threshold. # Two intervals per row. Passages are clustered inside books — passages from one book share an # author, a register, a translator and one scanner's OCR — so a Wilson interval over passages # claims a precision the design does not have. The pre-registration says the wider # cluster-bootstrap interval is the headline, and it is the one in the last column. rows, refusals, adv_rates, wins, bounds = headline(S, C, corpus, control) for r in refusals: print("refusing to build: " + r, file=sys.stderr) if refusals: sys.exit(1) null_margin = max(adv_rates.values()) V["HEADLINE_BLOCK"] = V["HEADLINE_INTRO"] + table( ["detector and corpus", "passages", "books", "called AI", "rate", "95% CI (passages)", "95% CI (by book)"], rows) + ( "\n

Two intervals, because passages are not independent draws: they come " "in books, and passages from one book share an author, a register, a translator and one " "scanner's OCR. The first interval treats every passage as its own observation; the second " "resamples whole books. The pre-registration says the wider of the two is the " "headline, and it is the one in bold on each row — the by-book interval on %d of the " "%d rows and the passage-level one on %d. Taking the wider is always the weaker claim, so " "the rule cannot be steered by which number I would prefer.

" % (wins["by book"], sum(wins.values()), wins["by passage"])) + ( "\n

The pre-registration fixed a condition under which this page has no finding in it: " "if both detectors flagged under %.0f%% of the adversarial corpus, there is no " "story, and I said in advance that I would publish the numbers and say so. The higher of " "the two adversarial rates is %.2f%%, so the condition does not hold. The build refuses " "to render if it does — a null result would need different copy, and it should not be " "possible to ship the confident version by not noticing.

" % (100 * NULL_RATE, 100 * null_margin)) # The comparison itself. Without this the page shows two rates and invites the reader to # eyeball the gap between them — which is exactly the argument from interval overlap the # pre-registration rules out, since overlapping intervals do not imply no difference. # Three rows per detector: as drawn, with the shared books removed, and era-matched. drows = [] for k, name in MODELS: a, b = [r for r in corpus if r["n"] in S[k]], [r for r in control if r["n"] in C[k]] pre = "%s · " % name.split("/")[-1] drows.append(diff_row(pre + "as drawn", a, S[k], b, C[k])) if shared: drows.append(diff_row(pre + "excluding books in both corpora", [r for r in a if r["item"] not in shared], S[k], [r for r in b if r["id"] not in shared], C[k])) ea, eb = [r for r in a if r["year"] >= ERA], [r for r in b if r["year"] >= ERA] if ea and eb: drows.append(diff_row(pre + "%d onward only" % ERA, ea, S[k], eb, C[k])) V["DIFF_BLOCK"] = table(["comparison", "adversarial", "control", "difference", "95% CI on the difference"], drows) if shared: adup = [r for r in corpus if r["item"] in shared] cdup = [r for r in control if r["id"] in shared] ah = {hashlib.sha1(r["text"].encode()).hexdigest(): r["n"] for r in adup} pairs = [] for k, _ in MODELS: # both detectors, because the sentence below says both for r in cdup: m = ah.get(hashlib.sha1(r["text"].encode()).hexdigest()) if m is not None and m in S[k] and r["n"] in C[k]: pairs.append((S[k][m]["p_ai"], C[k][r["n"]]["p_ai"])) worst = max((abs(a - b) for a, b in pairs), default=0.0) flips = sum((a > THRESH) != (b > THRESH) for a, b in pairs) V["CONTAM_BLOCK"] = ( "

%d of the source items were drawn into both corpora, " "contributing %d passages to the adversarial side and %d to the control — " "byte-identical text, since both corpora run the same extractor over the same scan. " "They are left in. Removing them would strip from the control precisely those books " "that happened to match a register-targeted query, which biases the control away from " "the thing under test and makes the difference larger; leaving them in pulls " "the two rates together, so the difference is if anything understated. The difference " "with them excluded is published beside the headline one.

" "

It also buys a check for nothing. Those passages are scored twice " "by each detector, in two corpora under two different shuffles — so each copy " "sat in a different batch, beside different passages, at a different padding width. " "Across the %d such comparisons — each passage, on each detector, scored once " "in each corpus — the largest difference " "between the two scores of the same text is %.3g, and the number of verdict " "flips is %d. Scores are recorded to six decimal places, so a difference of " "zero here means below that resolution and not a proof of bit-identity.

" ) % (len(shared), len(adup), len(cdup), len(pairs), worst, flips) else: V["CONTAM_BLOCK"] = ("

No source item was drawn into both corpora; the " "adversarial and control draws are disjoint at the book level.

") # threshold sweep — context for the headline, never a second source for it sweep = [] for t in SWEEP_T: row = ["P(AI) > %.2f" % t] for k, _ in MODELS: for sc in (S[k], C[k]): row.append("%.2f%%" % (100 * sum(1 for s in sc.values() if s["p_ai"] > t) / len(sc))) sweep.append(row) V["SWEEP_BLOCK"] = V["SWEEP_INTRO"] + table( ["threshold", "hello adv", "hello ctl", "openai adv", "openai ctl"], sweep) # Worked examples, chosen by a rule rather than by me: the three highest-scoring passages, # whatever they turn out to be. Discretion here is where a page like this quietly becomes an # argument — pick the three most striking of the flagged set and the reader is looking at my # taste, not the detector's behaviour. # Which detector's scores every single-detector section on this page is built from. The page # has one headline number per detector, but the worked examples and all four breakdown tables # split one detector only — and nothing said which. A table headed with an empty cell reading # "20%+ out-of-dictionary … 1.90%" is silently a claim about a detector the reader cannot name, # and could as easily be read as both averaged. Same shape as the date-tier sentence that was # true of four pulls out of ten: the scope of a table is a claim like any other. So the model # key travels with the data — gsplit() picks the scores and the label in one argument, block() # prints that label in the header cell — and the build refuses if the sections stop agreeing. BREAKDOWN_SRC = set() EX_MK = "hello" BREAKDOWN_SRC.add(EX_MK) top = sorted(S[EX_MK].values(), key=lambda s: -s["p_ai"])[:3] ex = "" for s in top: r = by_n[s["n"]] ex += ('
%.4f P(AI) — ' '%s (%s)
' '
%s
') % ( s["p_ai"], html.escape(r["item"]), html.escape((r["title"] or r["item"])[:110]), r["year"], html.escape(r["text"][:1200]) + ("…" if len(r["text"]) > 1200 else "")) V["EXAMPLES_BLOCK"] = ex def gsplit(keyfn, mk="hello"): BREAKDOWN_SRC.add(mk) g = defaultdict(list) for r in corpus: if r["n"] in S[mk]: kk = keyfn(r) if kk is not None: g[kk].append(r) return mk, g # Every breakdown row now carries both intervals, on the same rule as the headline: resample # whole books, publish both, bold the wider. Until now these tables published a Wilson interval # over passages and headed the column "95% CI" — the exact estimator the headline table says # "claims a precision the design does not have", on the same page, three tables further down. # Nothing here was wrong; the interval was simply the narrower kind and the column did not say # so. A subgroup is where that matters most, because a register or an OOV band can be a handful # of books with many passages each, which is precisely when a passage-level interval is most # over-confident. BD_WINS = {"by book": 0, "by passage": 0} def block(gg, order=None): mk, g = gg # unpacked, not defaulted: a table cannot be built without its label keys = order or sorted(g) out = [] for kk in keys: rws = g[kk] if not rws: continue f = sum(1 for r in rws if S[mk][r["n"]]["p_ai"] > THRESH) cl = clusterci.by_cluster(rws, S[mk], "item", THRESH) out.append([kk, "{:,}".format(len(rws)), "{:,}".format(len(cl)), # Deliberately not feeding `bounds` here: the pre-registered wider-interval # rule is about the headline claim, and pouring every genre row's narrow # bound into that set would eventually refuse a build over a coincidence. # A guard that fires on coincidences is a guard I will start disabling. "{:,}".format(f), *intervals(f, len(rws), cl, BD_WINS)]) return table([mk, "passages", "books", "called AI", "rate", "95% CI (passages)", "95% CI (by book)"], out) def oovbin(r): return oov_bucket(oov.get(r["n"], {}).get("oov")) V["OOV_BLOCK"] = block(gsplit(oovbin)) TIER_LABEL = {"strict": "book-date-verified", "loose": "catalogue year only"} strict = gsplit(lambda r: TIER_LABEL[tier(r)]) # The pre-registration's one live decision rule: if the book-date-verified subset and the # catalogue-year-only subset diverge by more than two percentage points, the strict subset # becomes the headline. Until now that rule was written in the pre-registration, restated in # this page's prose, and printed as a breakdown by analyse.py — and evaluated by nothing. A # decision rule nobody computes is a decision rule that never fires, which is the same as not # having pre-registered it. Refuse rather than switch automatically: promoting the strict # subset changes what the headline sentence means, and that is a rewrite, not a substitution. tiers = {} for k, _ in MODELS: g = grouped(corpus, S[k], tier) if g["strict"][1] and g["loose"][1]: tiers[k] = 100 * (g["strict"][0] / g["strict"][1] - g["loose"][0] / g["loose"][1]) if not tiers: print("refusing to build: no date-verification tiers to compare, so the pre-registered " "rule cannot be evaluated", file=sys.stderr) sys.exit(1) fired = {k: v for k, v in tiers.items() if abs(v) > TIER_LIMIT} if fired: print("refusing to build: pre-registered tier rule FIRED (%s). The strict subset becomes " "the headline and the copy has to say so." % ", ".join("%s %+.2f pp" % (k, v) for k, v in sorted(fired.items())), file=sys.stderr) sys.exit(1) worst_tier = max(abs(v) for v in tiers.values()) V["GENRE_BLOCK"] = V["GENRE_INTRO"] + block(gsplit(registers.family)) \ + ("

And the two date-verification tiers. The pre-registration set a rule here rather " "than an observation: if the book-date-verified subset and the catalogue-year-only " "subset diverged by more than %.0f percentage points, the strict subset would become " "the headline and this page would be about that number instead. The largest gap " "between the tiers, across both detectors, is %.2f points, so the rule did not " "fire and the headline stands as drawn. The build refuses to render if it does fire, " "which is the only version of a pre-registered rule that means anything.

" % (TIER_LIMIT, worst_tier)) + block(strict) # The page introduces all of the above with one detector's name. That sentence is only true # while the sections agree on the detector, so the agreement is checked rather than assumed — # and the name itself is substituted from the same set, so nobody can change the split without # changing the prose. Splitting by detector is a legitimate future edit; shipping the sentence # afterwards is not. if len(BREAKDOWN_SRC) != 1: print("refusing to build: the worked examples and the breakdown tables no longer come " "from one detector (%s), but the page names a single one. Say which is which in the " "template, or put them back on one." % ", ".join(sorted(BREAKDOWN_SRC)), file=sys.stderr) sys.exit(1) V["BD_KEY"] = sorted(BREAKDOWN_SRC)[0] V["BD_MODEL"] = dict(MODELS)[V["BD_KEY"]] V["BD_WINS_BOOK"] = str(BD_WINS["by book"]) V["BD_WINS_PSG"] = str(BD_WINS["by passage"]) V["BD_ROWS"] = str(sum(BD_WINS.values())) # Every rate in the hand-written copy has to be a rate this build computed. # # copy.json is the one part of the page that is written after the answer is known — that is the # whole reason it is a separate file — which makes it the one place a number can be shaped by # the answer instead of read off it. A headline is where that happens: a percentage rounded the # flattering way, or a "9 of 11" that takes its count from one decision rule and its worst case # from another. I have shipped exactly that second mistake before, on two networks. # # So: pull every percentage and every "k of n" out of the authored strings and require it to be # backed by the computed output. Percentages match numerically at the precision they are written # to, so "1.7%" is satisfied by a computed 1.70%. A "k of n" has to appear verbatim, which in # practice means the build must be computing that pair into a key of its own — and if it isn't, # the honest fix is to compute it, not to type it. Version numbers and dates carry neither a # per-cent sign nor an "of", so nothing innocent trips this. unbacked = unbacked_rates(json.load(open(os.path.join(HERE, "copy.json"))), V) if unbacked: print("refusing to build: copy.json states rates the build never computed, so they were " "typed from somewhere else: %s. Compute the number into a key and substitute it, or " "take it out." % "; ".join(unbacked), file=sys.stderr) sys.exit(1) # ...and the check above cannot see the one number I am most likely to reach for. Both # intervals are computed and both are rendered, so the narrower — the more impressive ceiling, # sitting in the same table row — is "backed" by construction. The pre-registered rule is that # the headline quotes the wider one. narrow = losing_bounds(json.load(open(os.path.join(HERE, "copy.json"))), bounds["lost"] - bounds["won"], bounds["won"]) if narrow: print("refusing to build: copy.json quotes the narrower interval where the pre-registered " "rule says the wider one: %s. Quote the wider bound, or put both in the same sentence." % "; ".join(narrow), file=sys.stderr) sys.exit(1) tmpl = open(os.path.join(HERE, "page.tmpl.html")).read() # Every bound this build refuses on is also *explained* in the prose, and the prose used to say # it in its own words: "fifteen words", "0.01 of any published threshold", "passages from 1990 # onward". Four constants, each with a hand-typed twin in the paragraph that describes it — the # copy a reader checks the guard against, and the copy nothing checks. They agreed when I found # them, which is not a defence; the era pair was one edit away from putting two different # windows under one sentence, in a page whose own claim is that its numbers and analysis.txt's # are the same numbers. # # Run against the template, not the rendered page: after substitution {{MED_GAP}} *is* "15", # and a check that cannot tell the placeholder from the literal is not a check. for what, pat in (("ERA (%d)" % ERA, r"\b%d\b" % ERA), ("MED_GAP_LIMIT (%d)" % MED_GAP_LIMIT, r"\b%d\b|\bfifteen\b" % MED_GAP_LIMIT), ("NEAR (%g)" % SWEEP_NEAR, r"\b%g\b" % SWEEP_NEAR), ("TIER_LIMIT (%g)" % TIER_LIMIT, r"\b(?:%g|two)\s+percentage" % TIER_LIMIT)): hit = re.search(pat, tmpl) if hit: ln = tmpl[:hit.start()].count("\n") + 1 print("refusing to build: page.tmpl.html line %d writes out %s by hand (%r). It is a " "constant this build owns and interpolates — use the placeholder, or the prose " "and the guard drift apart with nothing to notice." % (ln, what, hit.group(0)), file=sys.stderr) sys.exit(1) page = re.sub(r"\{\{(\w+)\}\}", lambda m: V.get(m.group(1), m.group(0)), tmpl) left = sorted(set(re.findall(r"\{\{(\w+)\}\}", page))) # whole document, not per line if left: print("refusing to write: unsubstituted placeholders %s" % left, file=sys.stderr) sys.exit(1) # "analysis.txt is the unedited output of analyse.py: every rate, interval and difference on # this page appears in it, next to the ones that did not make the page." A claim about a set, # stated in the reproduction section, computed by nothing — and false three ways at once when I # finally checked it by hand. The page's OOV table collapsed into a "20%+" row a tail the # analysis split across four; the sweep table's two control columns had no counterpart in the # analysis at all, which is ten of its twenty cells; and the batching table's numbers came out # of files analyse.py never opened. Each is fixed at the source — one shared binning function, # a control sweep, a batching summary, the tier gap — and this is what keeps them fixed. # # Compares two-decimal numbers, because that is how every rate, interval bound and difference # on this page is formatted. Counts carry no decimal point and the worked examples' scores # carry four, so both fall outside the claim and outside this check. When it fires the fix is a # print in analyse.py and a re-run of it, which costs seconds — not an exemption here, which # would cost the sentence. NUM = r"(?]+>", " ", tables))) absent = sorted(onpage - set(re.findall(NUM, open(apath).read())), key=float) if absent: print("refusing to write: %d number(s) in the page's tables appear nowhere in " "analysis.txt — %s — and the reproduction section tells the reader every rate, " "interval and difference here is in that file. Print them there or stop claiming it." % (len(absent), ", ".join(absent)), file=sys.stderr) sys.exit(1) # B4: "Adversarial-corpus flag rate is reported as a ceiling, never as a population rate." Be # exact about what this can and cannot do. Whether a rate is *used* as a population rate is a # question about meaning, and no regex settles it — a guard that pretended otherwise would be # worse than none, because it would read as having checked something it did not. What it does # check is that the disclosure is still on the page at all. The bias section is the single most # deletable paragraph here — it is the one that weakens the headline — and it runs through both # the template and the outcome-dependent copy I have yet to write. Checking the rendered page # rather than either source covers both, and covers whichever one the wording migrates into. for phrase in ("ceiling", "adversarially"): if phrase not in page.lower(): print("refusing to write: the page never says %r. B4 requires the adversarial rate to " "be presented as a ceiling on how bad these detectors get, not as a rate for " "1990s books — this only checks the disclosure is present, not that it is used " "correctly, and the absent word means it is not even present." % phrase, file=sys.stderr) sys.exit(1) out = os.path.join(outdir, "index.html") open(out, "w").write(page) open(os.path.join(outdir, "prereg.txt"), "w").write(open(os.path.join(HERE, "PREREG.md")).read()) open(os.path.join(outdir, "analysis.txt"), "w").write(open(apath).read()) open(os.path.join(outdir, "corpus.sha256"), "w").write( open(os.path.join(HERE, "CORPUS.sha256")).read()) # Ship what the page promises, from the page builder, so the promise cannot outlive the # mechanism. Scripts and raw scores go up; the adversarial corpus text does not, because it is # twelve thousand verbatim runs of OCR from other people's scans. Instead every scored passage # gets a metadata row — the score files are keyed by an integer index, which is unusable on its # own, and a reproducibility offer that needs a file I did not publish is not an offer. sdir, ddir = os.path.join(outdir, "scripts"), os.path.join(outdir, "data") os.makedirs(sdir, exist_ok=True) os.makedirs(ddir, exist_ok=True) # The three files written at the output root count as shipped too. They are the ones the page's # claims actually rest on — a pre-registration a reader cannot open is a pre-registration on my # word — and they were outside the orphan check below purely because it was written while # thinking about the two subdirectories. written = list(ROOT_FILES) for f in SCRIPT_FILES: p = os.path.join(HERE, f) if os.path.exists(p): open(os.path.join(sdir, f), "w").write(open(p).read()) written.append("scripts/" + f) udir = os.path.join(sdir, "upstream") os.makedirs(udir, exist_ok=True) for f in UPSTREAM_SCRIPTS: p = os.path.join(UPSTREAM_DIR, f) if os.path.exists(p): open(os.path.join(udir, f), "w").write(open(p).read()) written.append("scripts/upstream/" + f) # The padding files go out too. The table above summarises them to a worst case and a flip # count; publishing the per-passage batched-vs-single pairs is what lets someone disagree with # that summary instead of taking it. for f in DATA_FILES: p = os.path.join(HERE, f) if os.path.exists(p): open(os.path.join(ddir, f), "w").write(open(p).read()) written.append("data/" + f) written.append("data/index.jsonl") with open(os.path.join(ddir, "index.jsonl"), "w") as fh: for r in corpus: # sha1 of the passage text. The text itself stays unpublished, so without this the # adversarial half of the study is unverifiable by construction: a reader can re-fetch # the same scans and re-run merge.py, but has nothing to compare the result against. # control.jsonl ships whole, so its counterpart is the file digest in corpus.sha256. fh.write(json.dumps({"n": r["n"], "item": r["item"], "seq": r["seq"], "year": r["year"], "strict": r["strict"], "files": r["files"], "words": len(r["text"].split()), "sha1": hashlib.sha1(r["text"].encode()).hexdigest()}) + "\n") # Every relative link on the page must resolve to a file that was actually written. The copy # loops above skip anything missing without comment, so a promise like "the discarded draw is # published here" could ship as a 404 and nothing would say so. On a page whose whole claim is # that you can check it yourself, a dead evidence link is worse than no link. # Two roots, because the page carries two kinds of link. Relative ones ("data/...") resolve # inside this build's output; site-absolute ones ("/pangram/") are cross-links to other pages # already published and resolve against the web root. Checking both against outdir would # condemn every working cross-link; skipping absolutes would stop checking them at all. WEBROOT = "/var/www/aaw" dead, unchecked, rel = [], [], set() for href in re.findall(r'href="([^"#:]+)"', page): if not href.startswith("/"): rel.add(href) if href.startswith("/"): if not os.path.isdir(WEBROOT): unchecked.append(href) elif not os.path.exists(os.path.join(WEBROOT, href.lstrip("/"))): dead.append(href) elif not os.path.exists(os.path.join(outdir, href)): dead.append(href) if unchecked: print("note: %s not checked (no web root on this box)" % ", ".join(sorted(set(unchecked))), file=sys.stderr) if dead: print("refusing to publish: page links to files that were not written: %s" % ", ".join(sorted(set(dead))), file=sys.stderr) sys.exit(1) # And the same check in the other direction. "Every script is on this page" is a claim about # reachability; the guard near the top of this file made it true of a *directory*, which is a # different sentence. Twelve of the eighteen scripts were copied to a URL a reader could only # reach by guessing it, and on the data side the prose did the gesturing out loud — "the two # control files", "its three companions" — naming files it did not link. A file that ships # unlinked is not published to the reader, it is published to whoever already knows the path. # Checked against every relative href the page carries, not against a `scripts/|data/` pattern: # the pattern silently exempted whatever it did not match, which is the same directory-shaped # blindness one paragraph up, one level higher. orphan = orphans(written, rel) if orphan: print("refusing to publish: %d file(s) shipped that the page never links, so no reader can " "reach them: %s — link them or stop shipping them." % (len(orphan), ", ".join(orphan)), file=sys.stderr) sys.exit(1) # os.path.getsize, not len(page): the page holds a few dozen non-ASCII characters and len() # counts characters, so this line said 51,407 bytes about a 51,425-byte file. Reporting a # number I did not measure, in a build whose whole point is not doing that. print("wrote %s (%d bytes) + scripts/ + data/" % (out, os.path.getsize(out))) if __name__ == "__main__": main()