#!/bin/bash
# Run every remaining scan back-to-back. Sequential on purpose: this box is 1 core / ~2 GB, and
# two roberta-base checkpoints resident at once is most of the RAM — starvation here looks
# exactly like a hang.
#
# Each stage waits on a FILE, never on a pgrep. A waiter whose pattern matches its own command
# line waits forever, and I have lost a night to exactly that.
#
# ORDERING, revised mid-run and deliberately not by convenience. The first version ran the whole
# second detector over the adversarial corpus before touching the control. That is the wrong
# order by risk: the control is the number that actually describes 1990s books, it is a twelfth
# the size, and the second adversarial scan is both the longest stage and the least load-bearing
# one. If this box dies overnight I would rather hold a complete paired control plus one complete
# adversarial scan than two adversarial scans and nothing to compare them against. Nothing about
# the corpora, the seed or the threshold changed — only which stage runs first, and that was
# decided before any control passage had been scored.
#
# SUPERVISION, added after the kernel killed the hello scan at 2,384/12,247 with no traceback.
# Every stage now goes through run_scan.sh, which retries until the output file is as long as its
# input. The version of this script that stage replaced waited on a "done" marker that a dead
# process was never going to write — and it would have sat there all night looking exactly like
# patience.
#
# LAUNCH IT AS `./chain.run.sh`, A COPY. Bash reads a script incrementally, by file offset, and
# keeps reading it while it runs — so editing this file in place while it is executing can make
# a live shell resume mid-token, hours later, in a script that looked fine when I saved it. I did
# exactly that today and had to restart the scan to be sure. Running from a copy taken at launch
# means edits here are edits to the *next* run, which is what I actually mean every time.
#
#   cp chain.sh chain.run.sh && setsid nohup ./chain.run.sh > chain.log 2>&1 < /dev/null &
# REORDERED AGAIN, once the control corpus finished being drawn. The rule has not changed — the
# control is the number that actually describes 1990s books, and it is a tenth the size — but when
# that rule was written the control was still being fetched, so the adversarial scan had to go
# first by default. It no longer does. Running both control scans first means a complete paired
# difference estimate exists in about an hour instead of in five.
#
# This cannot steer a result and it is worth saying why once, rather than hedging about it: every
# score is deterministic given the corpus, the model and the fixed seed. Stage order has no effect
# on any published number. It decides only which numbers exist if the run is cut short, which is
# an operational question and not a statistical one.
cd /home/agent/work/aidetect-fpr

until grep -q '^books kept:' control.log 2>/dev/null; do sleep 30; done
echo "[chain] control corpus complete, scoring it with both detectors" >&2
./run_scan.sh hello  control control_hello.log
echo "[chain] control/hello done" >&2
./run_scan.sh openai control control_openai.log
echo "[chain] control/openai done" >&2

./run_scan.sh hello corpus hello.log
echo "[chain] hello finished on the adversarial corpus" >&2

# Longest stage last. Its partial output is still publishable: score.py shuffles under a fixed
# seed before scoring and both detectors walk that same order, so the first N of the openai scan
# are the same N passages hello already scored — a paired comparison on a prefix, not a different
# sample.
./run_scan.sh openai corpus openai.log
echo "[chain] openai finished on the adversarial corpus" >&2

# Owed from the pre-registration: batches are padded to their longest member, so a passage's
# score depends slightly on which passages it was batched with. Re-score everything near a
# threshold one passage at a time, where there is nothing to pad to. Folded in here from a
# separate follow-on script — one chain is one thing to check on.
python3 padding_check.py hello            > padding_hello.log          2>&1
python3 padding_check.py openai           > padding_openai.log         2>&1
python3 padding_check.py hello  --control > padding_control_hello.log  2>&1
python3 padding_check.py openai --control > padding_control_openai.log 2>&1

# The page links this file for the per-decade rates, and build_page.py refuses to render without
# it. It used to be a prose reference to an "analysis output" that nothing produced.
#
# NOT in chain.run.sh: this stage was added at 08:24 and the copy that is executing was taken at
# 08:05. Tonight's analysis.txt therefore comes from finish.sh, not from here. Both now call the
# same script, so the two paths cannot disagree about what a valid analysis is.
./run_analysis.sh
echo "[chain] all done" >&2