fix(sync): batch push queries + implement force flag

W2: Replace N+1 push handler with batch UNNEST upsert — single
conflict check query + single upsert query instead of 2N round trips.
Added 500-entity batch size limit.

W4: Wire force flag through stack (MCP handler → manager → engine).
Force push resets cloud_sync_state and cursors, re-pushes all data.

Author: StackMemory Bot (CLI)

packages/provenant-api/src/index.js

@@ -79,34 +79,70 @@ async function handlePush(request, sql, projectId, clientId) {
     });
   }
 
-  const rejected = [];
-  let accepted = 0;
+  if (entities.length > 500) {
+    return json({ error: 'Batch too large (max 500)' }, 400);
+  }
+
+  // Batch conflict check — single query instead of N+1
+  const incomingIds = entities.map((e) => e.id);
+  const incomingTables = entities.map((e) => e.table);
+  const existingRows = await sql`
+    SELECT id, table_name, version FROM sync_entities
+    WHERE project_id = ${projectId}
+      AND (id, table_name) IN (
+        SELECT UNNEST(${incomingIds}::text[]), UNNEST(${incomingTables}::text[])
+      )
+  `;
+
+  const existingMap = new Map();
+  for (const row of existingRows) {
+    existingMap.set(`${row.table_name}:${row.id}`, Number(row.version));
+  }
+
+  // Separate conflicts from upsertable entities
   const conflicts = [];
+  const toUpsert = [];
 
   for (const entity of entities) {
-    try {
-      // Check for conflict (newest_wins)
-      const existing = await sql`
-        SELECT version FROM sync_entities
-        WHERE project_id = ${projectId}
-          AND table_name = ${entity.table}
-          AND id = ${entity.id}
-      `;
+    const key = `${entity.table}:${entity.id}`;
+    const serverVersion = existingMap.get(key);
+    if (serverVersion !== undefined && serverVersion > entity.version) {
+      conflicts.push({
+        id: entity.id,
+        table: entity.table,
+        serverVersion,
+        clientVersion: entity.version,
+      });
+    } else {
+      toUpsert.push(entity);
+    }
+  }
 
-      if (existing.length > 0 && existing[0].version > entity.version) {
-        conflicts.push({
-          id: entity.id,
-          table: entity.table,
-          serverVersion: Number(existing[0].version),
-          clientVersion: entity.version,
-        });
-        continue;
-      }
+  // Batch upsert — single query
+  const rejected = [];
+  let accepted = 0;
+
+  if (toUpsert.length > 0) {
+    try {
+      const ids = toUpsert.map((e) => e.id);
+      const tableNames = toUpsert.map((e) => e.table);
+      const versions = toUpsert.map((e) => e.version);
+      const tiers = toUpsert.map((e) => e.tier);
+      const dataArr = toUpsert.map((e) => JSON.stringify(e.data));
+      const clientIds = toUpsert.map(() => clientId);
+      const projectIds = toUpsert.map(() => projectId);
 
-      // Upsert
       await sql`
         INSERT INTO sync_entities (id, project_id, table_name, version, tier, data, client_id)
-        VALUES (${entity.id}, ${projectId}, ${entity.table}, ${entity.version}, ${entity.tier}, ${JSON.stringify(entity.data)}, ${clientId})
+        SELECT * FROM UNNEST(
+          ${ids}::text[],
+          ${projectIds}::text[],
+          ${tableNames}::text[],
+          ${versions}::bigint[],
+          ${tiers}::text[],
+          ${dataArr}::jsonb[],
+          ${clientIds}::text[]
+        )
         ON CONFLICT (project_id, table_name, id)
         DO UPDATE SET
           version = EXCLUDED.version,
@@ -115,9 +151,12 @@ async function handlePush(request, sql, projectId, clientId) {
           client_id = EXCLUDED.client_id,
           pushed_at = NOW()
       `;
-      accepted++;
+      accepted = toUpsert.length;
     } catch (err) {
-      rejected.push({ id: entity.id, reason: String(err) });
+      // If batch fails, report all as rejected
+      for (const entity of toUpsert) {
+        rejected.push({ id: entity.id, reason: String(err) });
+      }
     }
   }
 

scripts/gepa/.before-optimize.md

@@ -111,14 +111,13 @@ When adding or renaming GitHub Actions workflows that should be triggerable via
 
 | Workflow | Script path | Category |
 |---|---|---|
-| `weekly-start.yml` | `voyager/scripts/content-brief.mjs` + `voyager/scripts/content-audit.mjs` + `ops/fathom-social-content.mjs` + `ops/fathom-testimonial-scan.mjs` + `ops/perplexity-citation-audit.mjs` + `commit/profound-aeo-pulse.mjs` + `commit/citation-rank-tracker.mjs` + `diag/ranking-snapshot.mjs` + `voyager/scripts/generate-blog-scaffold.mjs` + `ops/ahrefs-firehose-digest.mjs` + `ops/export-dripify.mjs` + `commit/prospect-discovery.mjs` + `commit/growth-signal-leads.mjs` + `ops/repush-clay-leads.mjs` + `ops/snitcher-outreach.mjs` | GHA cron (Mon) |
-| `weekly-end.yml` | `diag/fathom-demo-scorecard.mjs` + `commit/feedback/collect-*.mjs` + `commit/feedback/collect-ops-feedback.mjs` + `diag/weekly-retro.mjs` | GHA cron (Fri) |
+| `weekly-start.yml` | `voyager/scripts/content-brief.mjs` + `voyager/scripts/content-audit.mjs` + `ops/fathom-social-content.mjs` + `ops/fathom-testimonial-scan.mjs` + `ops/perplexity-citation-audit.mjs` + `commit/profound-aeo-pulse.mjs` + `commit/citation-rank-tracker.mjs` + `diag/ranking-snapshot.mjs` + `voyager/scripts/generate-blog-scaffold.mjs` + `ops/ahrefs-firehose-digest.mjs` + `ops/export-dripify.mjs` + `commit/prospect-discovery.mjs` + `commit/growth-signal-leads.mjs` + `ops/repush-clay-leads.mjs` + `ops/snitcher-outreach.mjs` | GHA cron (1st Mon) |
+| `weekly-end.yml` | `diag/fathom-demo-scorecard.mjs` + `commit/feedback/collect-*.mjs` + `commit/feedback/collect-ops-feedback.mjs` + `diag/weekly-retro.mjs` + `commit/sync-llms-txt.mjs` | GHA cron (Fri) |
 | `anneal-keywords.yml` | `commit/anneal-keywords.mjs` | GHA cron (Sun) |
-| `g2-review-monitor.yml` | `ops/g2-to-senja.mjs` | GHA cron (Daily) |
-| `testimonial-pipeline.yml` | `commit/testimonial-pipeline.mjs` | Manual |
+| `daily-ops.yml` | `ops/slack-digest.mjs` + `ops/fathom-meeting-digest.mjs` + `ops/ops-daily-briefing.mjs` + `ops/g2-to-senja.mjs` + `ops/review-intercept.mjs` | GHA cron (weekdays) |
+| `midweek-ops.yml` | `ops/sequence-orchestrator.mjs` + `ops/push-drafts-to-instantly.mjs` | GHA cron (Tue/Thu) |
+| `monthly-ops.yml` | `diag/pagespeed-audit.mjs` + `commit/pagespeed-improvements.mjs` + `commit/icp-tune.mjs` | GHA cron (1st of month) |
 | `video-pipeline.yml` | `ops/video-clips.mjs` | Manual |
-| `pagespeed-audit.yml` | `diag/pagespeed-audit.mjs` + `commit/pagespeed-improvements.mjs` | GHA cron (1st of month) |
-| `daily-ops.yml` | `ops/slack-digest.mjs` + `ops/fathom-meeting-digest.mjs` + `ops/ops-daily-briefing.mjs` | GHA cron (weekdays) |
 | `indexnow-submit.yml` | (inline curl) | Push to master (voyager) / Manual |
 
 ## GitHub Actions (`.github/workflows/`)
@@ -133,12 +132,12 @@ When adding or renaming GitHub Actions workflows that should be triggerable via
 In `actions/github-script@v7`, `github.rest.issues.createComment` posts plain issue comments on PRs (PRs are issues in GitHub's API). For inline code suggestions on specific files/lines, use `github.rest.pulls.createReview` or `github.rest.pulls.createReviewComment` instead.
 
 ### Scheduled (cron)
-- `weekly-start.yml` — Mon 9am ET (content review, social content, testimonial scan, Perplexity audit, AEO pulse → blog scaffold, Ahrefs digest, Dripify export, prospect discovery, growth-signal leads → snitcher outreach)
-- `weekly-end.yml` — Fri 9am ET (demo scorecard + pipeline health)
+- `weekly-start.yml` — 1st Mon 9am EDT (content review, social content, testimonial scan, Perplexity audit, AEO pulse → blog scaffold, Ahrefs digest, Dripify export, prospect discovery, growth-signal leads → snitcher outreach)
+- `weekly-end.yml` — Fri 10am EDT / 9am EST (demo scorecard, pipeline health, llms.txt sync, freshness audit, SEO/AEO snapshot)
 - `anneal-keywords.yml` — Sun 11am ET (keyword annealing + kill pattern updates)
-- `g2-review-monitor.yml` — Daily 10am ET
-- `pagespeed-audit.yml` — 1st of month 9am ET (PSI audit → Claude recommendations → PR)
-- `daily-ops.yml` — Weekdays 10am ET (signal monitor, G2 reviews, review intercept, Slack digest → meeting digest → daily briefing)
+- `daily-ops.yml` — Weekdays 10am EDT (signal monitor, G2 reviews, review intercept, Slack digest → meeting digest → daily briefing)
+- `midweek-ops.yml` — Tue/Thu (sequence orchestrator + push drafts to Instantly)
+- `monthly-ops.yml` — 1st of month 10am EDT (PSI audit → Claude recommendations → PR, ICP tuning)
 - `indexnow-submit.yml` — On push to master (voyager pages) + manual (`/run indexnow urls=...`)
 
 ## Deployments

scripts/gepa/generations/current

@@ -1 +1 @@
-/Users/jwu/Dev/stackmemory/scripts/gepa/generations/gen-002/baseline.md
+/Users/jwu/Dev/stackmemory/scripts/gepa/generations/gen-001/baseline.md

src/core/storage/cloud-sync-manager.ts

@@ -99,9 +99,10 @@ export class CloudSyncManager extends EventEmitter {
    * Perform a push
    */
   async performPush(
-    trigger: 'manual' | 'frame-close' | 'session-end' | 'periodic'
+    trigger: 'manual' | 'frame-close' | 'session-end' | 'periodic',
+    force = false
   ): Promise<CloudSyncPushResult> {
-    const result = await this.engine.push();
+    const result = await this.engine.push(force);
     this.emit('push', { trigger, result });
     return result;
   }

src/core/storage/cloud-sync.ts

@@ -177,9 +177,16 @@ export class CloudSyncEngine {
   /**
    * Push local changes to cloud
    */
-  async push(): Promise<CloudSyncPushResult> {
+  async push(force = false): Promise<CloudSyncPushResult> {
     return this.mutex.withLock(async () => {
       try {
+        if (force) {
+          // Reset all sync state — re-push everything
+          this.db.prepare(`DELETE FROM cloud_sync_state`).run();
+          this.db
+            .prepare(`DELETE FROM cloud_sync_cursors WHERE direction = 'push'`)
+            .run();
+        }
         const pending = this.collectPendingEntities();
         if (pending.length === 0) {
           return { success: true, pushed: 0, rejected: 0, conflicts: 0 };

src/integrations/mcp/handlers/cloud-sync-handlers.ts

@@ -38,7 +38,7 @@ export class CloudSyncHandlers {
     force?: boolean;
   }): Promise<{ content: Array<{ type: string; text: string }> }> {
     const manager = this.ensureManager();
-    const result = await manager.performPush('manual');
+    const result = await manager.performPush('manual', args.force);
 
     return {
       content: [