#!/usr/bin/env python3
"""Pull per-claim detail for every poidh bounty from the site's own /data endpoint.

Why: the bounty-level pull already on disk carries claimants[] *deduped* per
bounty, so it cannot say how many claims a wallet actually filed. The
per-bounty endpoint returns the full claims array, and each claim carries the
claimer's `farcasterHandle` -- which is where the address <-> Farcaster
username map comes from. No API key, no Neynar, poidh's own data.

Resumable by design (see the 200k-scan lesson): every response is appended to
claims_raw.jsonl as one line, and a restart skips whatever is already there.
Never analyse this file while the pull is still running.

  python3 pull_claims.py            # resume / run
"""
import json, os, sys, threading, time, urllib.request, urllib.error
from queue import Queue

HERE = os.path.dirname(os.path.abspath(__file__))
OUT = f"{HERE}/claims_raw.jsonl"

# site ids are contiguous 1..max per chain; probe a little past the known max
MAXID = {"base": 1339, "degen": 1394, "mainnet": 25, "arbitrum": 329}
PROBE = 20
WORKERS = 4

done = set()
if os.path.exists(OUT):
    for line in open(OUT):
        try:
            d = json.loads(line)
            done.add((d["chain"], d["site_id"]))
        except Exception:
            pass
print(f"resuming with {len(done)} already fetched", flush=True)

work = Queue()
for chain, mx in MAXID.items():
    for i in range(1, mx + PROBE + 1):
        if (chain, i) not in done:
            work.put((chain, i))
total = work.qsize()
print(f"{total} to fetch", flush=True)

lock = threading.Lock()
fh = open(OUT, "a")
n = [0]


def get(url):
    req = urllib.request.Request(url, headers={"User-Agent": "poidh-type/1.0"})
    with urllib.request.urlopen(req, timeout=30) as r:
        return json.load(r)


def worker():
    while True:
        try:
            chain, i = work.get_nowait()
        except Exception:
            return
        rec = None
        for attempt in range(4):
            try:
                d = get(f"https://poidh.xyz/{chain}/bounty/{i}/data")
                if isinstance(d, dict) and d.get("error"):
                    rec = {"chain": chain, "site_id": i, "missing": True}
                else:
                    rec = {
                        "chain": chain, "site_id": i,
                        "on_chain_id": d.get("onChainId"),
                        "issuer": (d.get("issuer") or "").lower() or None,
                        "created_at": int(d["createdAt"]) if d.get("createdAt") else None,
                        "claims": [
                            {"id": c.get("claimId"),
                             "a": (c.get("issuerAddress") or "").lower() or None,
                             "fc": c.get("farcasterHandle"),
                             "tw": c.get("twitterHandle")}
                            for c in (d.get("claims") or [])
                        ],
                    }
                break
            except urllib.error.HTTPError as e:
                if e.code == 404:
                    rec = {"chain": chain, "site_id": i, "missing": True}
                    break
                time.sleep(2 * (attempt + 1))
            except Exception:
                time.sleep(2 * (attempt + 1))
        if rec is None:
            rec = {"chain": chain, "site_id": i, "failed": True}
        with lock:
            fh.write(json.dumps(rec, separators=(",", ":")) + "\n")
            fh.flush()
            n[0] += 1
            if n[0] % 100 == 0:
                print(f"  {n[0]}/{total}", flush=True)
        time.sleep(0.15)


ts = [threading.Thread(target=worker, daemon=True) for _ in range(WORKERS)]
[t.start() for t in ts]
[t.join() for t in ts]
fh.close()
print(f"done: {n[0]} fetched, file now {sum(1 for _ in open(OUT))} lines", flush=True)