Worker: manifest-based deployed-list (2 subrequests, not N)

With ~80 entries in users.txt, the per-user probe approach blew the
Cloudflare Workers 50-subrequests-per-request limit (CF error 1101).

Switch to: CI writes /deployed.json (a JSON array of user logins
whose dashboard HTML is in /public) before every deploy. Worker
reads users.txt + deployed.json (2 subrequests total) and computes
the diff. Scales to any users.txt size.

regen_manifest() is now called inside the mining-loop's deploy_now()
and from the final-deploy step, so the manifest tracks reality even
as users complete mid-run.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: pirate

.github/workflows/mine-and-deploy.yml

@@ -140,6 +140,14 @@ jobs:
           # (before the mining loop) means every interim deploy also has
           # a fresh users.txt available to the homepage.
           cp cloudflare/users.txt cloudflare/public/users.txt
+          # Generate deployed.json — a JSON array of user logins whose
+          # dashboard HTML is currently in /public. Updated on every
+          # deploy_now() call below so the homepage stays accurate.
+          ls cloudflare/public/*.html 2>/dev/null \
+            | sed -E 's|cloudflare/public/||;s|\.html$||' \
+            | grep -vE '^(index|404)$' \
+            | python3 -c "import sys,json; print(json.dumps(sorted([l.strip() for l in sys.stdin if l.strip()])))" \
+            > cloudflare/public/deployed.json
           echo "Targets:"
           cat /tmp/targets.txt
 
@@ -156,7 +164,16 @@ jobs:
         run: |
           set -e
 
+          regen_manifest() {
+            ls cloudflare/public/*.html 2>/dev/null \
+              | sed -E 's|cloudflare/public/||;s|\.html$||' \
+              | grep -vE '^(index|404)$' \
+              | python3 -c "import sys,json; print(json.dumps(sorted([l.strip() for l in sys.stdin if l.strip()])))" \
+              > cloudflare/public/deployed.json
+          }
+
           deploy_now() {
+            regen_manifest
             (cd cloudflare && npx --yes wrangler@latest deploy --minify 2>&1 |
               tail -2) || echo "::warning::interim deploy failed"
           }
@@ -221,8 +238,15 @@ jobs:
           git push || echo "::warning::push failed (no commit permission?)"
 
       - name: Final deploy
-        working-directory: cloudflare
-        run: npx --yes wrangler@latest deploy
+        working-directory: .
+        run: |
+          # Regenerate deployed.json one last time before the final push.
+          ls cloudflare/public/*.html 2>/dev/null \
+            | sed -E 's|cloudflare/public/||;s|\.html$||' \
+            | grep -vE '^(index|404)$' \
+            | python3 -c "import sys,json; print(json.dumps(sorted([l.strip() for l in sys.stdin if l.strip()])))" \
+            > cloudflare/public/deployed.json
+          cd cloudflare && npx --yes wrangler@latest deploy
         env:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}

cloudflare/worker/index.ts

@@ -429,55 +429,55 @@ async function handleStatus(
 }
 
 
-// Dynamic homepage. Reads /users.txt (also deployed as a static asset
-// by CI) for the canonical list of users we want dashboards for, then
-// probes /{user}.html via the ASSETS binding to see which are ready vs
-// still queued/mining. Output is cached in Workers Cache for 30s so
-// repeated visits don't fan out to N internal asset probes each time.
+// Dynamic homepage. Reads /deployed.json (a CI-generated manifest of
+// users whose dashboard HTML actually exists) and /users.txt (the
+// canonical list of users we want), then computes deployed vs queued.
+// Only 2 subrequests instead of probing each /{user}.html individually
+// (which blew the Worker's 50/req subrequest limit at 80+ users).
 async function handleIndex(env: Env, url: URL): Promise<Response> {
   const cache = caches.default;
-  const cacheKey = new Request("https://internal-index.invalid/v1");
+  const cacheKey = new Request("https://internal-index.invalid/v2");
   const cached = await cache.match(cacheKey);
   if (cached) return cached;
 
-  // Read users.txt from the deployed assets.
   const probeBase = new URL(url.toString());
   probeBase.search = "";
-  const usersTxtUrl = new URL(probeBase.toString());
-  usersTxtUrl.pathname = "/users.txt";
-  let users: string[] = [];
-  try {
-    const r = await env.ASSETS.fetch(new Request(usersTxtUrl.toString()));
-    if (r.ok) {
-      const txt = await r.text();
-      users = txt.split("\n")
-        .map((l) => l.split("#", 1)[0].trim())
-        .filter((l) => l.length > 0);
-    }
-  } catch {}
 
-  // Add pirate (intentionally not in users.txt — built locally).
-  if (!users.includes("pirate")) users.unshift("pirate");
-
-  // Probe each user's /<u>.html for deploy status in parallel.
-  const states = await Promise.all(users.map(async (u) => {
-    const probeUrl = new URL(probeBase.toString());
-    probeUrl.pathname = `/${u}.html`;
+  async function fetchAsset(path: string): Promise<string | null> {
     try {
-      const r = await env.ASSETS.fetch(new Request(probeUrl.toString()));
-      return { user: u, deployed: r.status === 200 };
+      const u = new URL(probeBase.toString());
+      u.pathname = path;
+      const r = await env.ASSETS.fetch(new Request(u.toString()));
+      if (!r.ok) return null;
+      return await r.text();
     } catch {
-      return { user: u, deployed: false };
+      return null;
     }
-  }));
+  }
+
+  // Parse users.txt — comment-aware, one login per line.
+  const usersTxt = await fetchAsset("/users.txt");
+  const wanted: string[] = (usersTxt ?? "")
+    .split("\n")
+    .map((l) => l.split("#", 1)[0].trim())
+    .filter((l) => l.length > 0);
+
+  // Parse deployed.json — JSON array of logins.
+  let deployedSet = new Set<string>();
+  const depJson = await fetchAsset("/deployed.json");
+  if (depJson) {
+    try {
+      const arr = JSON.parse(depJson) as string[];
+      if (Array.isArray(arr)) deployedSet = new Set(arr);
+    } catch {}
+  }
+  // Pirate is always deployed (committed in /public).
+  deployedSet.add("pirate");
 
-  const deployed = states
-    .filter((s) => s.deployed)
-    .map((s) => s.user)
+  const allUsers = new Set<string>([...wanted, ...deployedSet]);
+  const deployed = [...allUsers].filter((u) => deployedSet.has(u))
     .sort((a, b) => a.toLowerCase().localeCompare(b.toLowerCase()));
-  const queued = states
-    .filter((s) => !s.deployed)
-    .map((s) => s.user)
+  const queued = [...allUsers].filter((u) => !deployedSet.has(u))
     .sort((a, b) => a.toLowerCase().localeCompare(b.toLowerCase()));
 
   const html = indexPage(deployed, queued);