fix: add release smoke tests to prevent broken binary publishes

.github/meta/commit.txt

@@ -1,16 +1,26 @@
-fix: Codespaces — skip machine-scoped `GITHUB_TOKEN`, cap retries, fix phantom command
+fix: add release smoke tests to prevent broken binary publishes
 
-Closes #413
+Closes #462
 
-- Skip auto-enabling `github-models` and `github-copilot` providers in
-  machine environments (Codespaces: `CODESPACES=true`, GitHub Actions:
-  `GITHUB_ACTIONS=true`) when only machine-scoped tokens (`GITHUB_TOKEN`,
-  `GH_TOKEN`) are available. The Codespace/Actions token lacks
-  `models:read` scope needed for GitHub Models API.
-- Cap retry attempts at 5 (`RETRY_MAX_ATTEMPTS`) to prevent infinite
-  retry loops. Log actionable warning when retries exhaust.
-- Replace phantom `/discover-and-add-mcps` toast with actionable message.
-- Add `.devcontainer/` config (Node 22, Bun 1.3.10) for Codespaces.
-- Add 32 adversarial e2e tests covering full Codespace/Actions env
-  simulation, `GH_TOKEN`, token variations, config overrides, retry bounds.
-- Update docs to reference `mcp_discover` tool.
+v0.5.10 shipped with `@altimateai/altimate-core` missing from standalone
+distributions, crashing on startup. Add three-layer defense:
+
+- **CI smoke test**: run the compiled `linux-x64` binary after build and
+  before npm publish — catches runtime crashes that compile fine
+- **Build-time verification**: validate all `requiredExternals` are in
+  `package.json` `dependencies` (not just `devDependencies`) so they
+  ship in the npm wrapper package
+- **Local pre-release script**: `bun run pre-release` builds + smoke-tests
+  the binary before tagging — mandatory step in RELEASING.md
+
+Also adds `smoke-test-binary.test.ts` with 3 tests: version check,
+standalone graceful-failure, and `--help` output.
+
+Reviewed by 5 AI models (Claude, GPT 5.2 Codex, Gemini 3.1 Pro,
+Kimi K2.5, MiniMax M2.5). Key fixes from review:
+- Include workspace `node_modules` in `NODE_PATH` (Gemini)
+- Restrict dep check to `dependencies` only (GPT, Gemini, Kimi)
+- Hard-fail pre-publish gate when binary not found (Claude, GPT)
+- Tighten exit code assertion for signal safety (MiniMax)
+
+Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

.github/workflows/release.yml

@@ -99,6 +99,25 @@ jobs:
           GH_REPO: ${{ env.GH_REPO }}
           MODELS_DEV_API_JSON: test/tool/fixtures/models-api.json
 
+      # Smoke-test: verify the compiled binary actually starts.
+      # Only possible for native linux-x64 builds on the ubuntu runner.
+      # This catches missing externals (e.g. @altimateai/altimate-core)
+      # that compile fine but crash at runtime.
+      - name: Smoke test binary
+        if: matrix.name == 'linux-x64'
+        run: |
+          BINARY=$(find packages/opencode/dist -name altimate -type f | head -1)
+          if [ -z "$BINARY" ]; then
+            echo "::error::No binary found in dist/"
+            exit 1
+          fi
+          chmod +x "$BINARY"
+
+          # Set NODE_PATH so the binary can resolve external NAPI modules
+          # (mirrors what the npm bin wrapper does at runtime)
+          NODE_PATH="$(pwd)/packages/opencode/node_modules:$(pwd)/node_modules" "$BINARY" --version
+          echo "Smoke test passed: binary starts and prints version"
+
       - name: Upload build artifact
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
@@ -168,6 +187,21 @@ jobs:
       #   env:
       #     AUR_SSH_PRIVATE_KEY: ${{ secrets.AUR_SSH_PRIVATE_KEY }}
 
+      # Smoke-test a linux-x64 binary from the downloaded artifacts before publishing.
+      # This is the last gate before npm publish — catches runtime crashes that
+      # compile-time checks miss (e.g. missing NAPI externals like v0.5.10).
+      - name: Pre-publish smoke test
+        run: |
+          BINARY=$(find packages/opencode/dist -path '*altimate-code-linux-x64/bin/altimate' -type f | head -1)
+          if [ -z "$BINARY" ]; then
+            echo "::error::No linux-x64 binary found in artifacts — cannot verify release"
+            exit 1
+          else
+            chmod +x "$BINARY"
+            NODE_PATH="$(pwd)/packages/opencode/node_modules:$(pwd)/node_modules" "$BINARY" --version
+            echo "Pre-publish smoke test passed"
+          fi
+
       - name: Publish to npm
         run: bun run packages/opencode/script/publish.ts
         env:

docs/RELEASING.md

@@ -48,7 +48,23 @@ Add a new section at the top of `CHANGELOG.md`:
 - ...
 ```
 
-### 2. Commit and tag
+### 2. Run pre-release sanity check
+
+**MANDATORY** — this catches broken binaries before they reach users:
+
+```bash
+cd packages/opencode
+bun run pre-release
+```
+
+This verifies:
+- All required NAPI externals are in `package.json` dependencies
+- They're installed in `node_modules`
+- A local build produces a binary that actually starts
+
+Do NOT proceed if any check fails.
+
+### 3. Commit and tag
 
 ```bash
 git add -A
@@ -57,7 +73,7 @@ git tag v0.5.0
 git push origin main v0.5.0
 ```
 
-### 3. What happens automatically
+### 4. What happens automatically
 
 The `v*` tag triggers `.github/workflows/release.yml` which:
 
@@ -67,7 +83,7 @@ The `v*` tag triggers `.github/workflows/release.yml` which:
 4. **Updates AUR** — pushes PKGBUILD update to `altimate-code-bin`
 5. **Publishes Docker image** — to `ghcr.io/altimateai/altimate-code`
 
-### 4. Verify
+### 5. Verify
 
 After the workflow completes:
 

packages/opencode/package.json

@@ -12,7 +12,8 @@
     "build:local": "bun run script/build.ts --single --skip-install",
     "local": "./script/local.sh",
     "dev": "bun run --conditions=browser ./src/index.ts",
-    "db": "bun drizzle-kit"
+    "db": "bun drizzle-kit",
+    "pre-release": "bun run script/pre-release-check.ts"
   },
   "bin": {
     "altimate": "./bin/altimate",

packages/opencode/script/build.ts

@@ -193,6 +193,21 @@ const targets = targetIndexFlag !== undefined
 
 await $`rm -rf dist`
 
+// Packages excluded from the compiled binary — must be resolvable from
+// node_modules at runtime. Split into required (must ship with the wrapper
+// package) and optional (user installs on demand).
+const requiredExternals = [
+  // NAPI native module — cannot be embedded in Bun single-file executable.
+  "@altimateai/altimate-core",
+]
+const optionalExternals = [
+  // Database drivers — native addons, users install on demand per warehouse
+  "pg", "snowflake-sdk", "@google-cloud/bigquery", "@databricks/sql",
+  "mysql2", "mssql", "oracledb", "duckdb",
+  // Optional infra packages — native addons or heavy optional deps
+  "keytar", "ssh2", "dockerode",
+]
+
 const binaries: Record<string, string> = {}
 if (!skipInstall) {
   await $`bun install --os="*" --cpu="*" @opentui/core@${pkg.dependencies["@opentui/core"]}`
@@ -225,27 +240,11 @@ for (const item of targets) {
     tsconfig: "./tsconfig.json",
     plugins: [solidPlugin],
     sourcemap: "external",
-    // Packages excluded from the compiled binary — resolved from node_modules
-    // at runtime. Bun compiled binaries resolve externals via standard Node
-    // resolution from the binary's location, walking up to the wrapper
-    // package's node_modules.
-    //
     // IMPORTANT: Without code splitting, Bun inlines dynamic import() targets
     // into the main chunk. Any external require() in those targets will fail
     // at startup — not when the import() is called. Only mark packages as
     // external when they truly cannot be bundled (e.g. NAPI native addons).
-    external: [
-      // NAPI native module — cannot be embedded in Bun single-file executable.
-      // The JS loader dynamically require()s platform-specific .node binaries
-      // (e.g. @altimateai/altimate-core-darwin-arm64).
-      // Must be installed as a dependency of the published wrapper package.
-      "@altimateai/altimate-core",
-      // Database drivers — native addons, users install on demand per warehouse
-      "pg", "snowflake-sdk", "@google-cloud/bigquery", "@databricks/sql",
-      "mysql2", "mssql", "oracledb", "duckdb",
-      // Optional infra packages — native addons or heavy optional deps
-      "keytar", "ssh2", "dockerode",
-    ],
+    external: [...requiredExternals, ...optionalExternals],
     compile: {
       autoloadBunfig: false,
       autoloadDotenv: false,
@@ -293,6 +292,37 @@ for (const item of targets) {
   binaries[name] = Script.version
 }
 
+// ---------------------------------------------------------------------------
+// Build-time verification: ensure required externals are in package.json
+// dependencies so they ship with the npm wrapper package. This catches the
+// scenario where a new NAPI module is added to `external` but not to
+// package.json dependencies — which would compile fine but crash at runtime.
+// ---------------------------------------------------------------------------
+{
+  // Only check dependencies (not devDependencies) — publish.ts only ships
+  // dependencies to end users. A required external in devDependencies would
+  // pass this check but be missing for npm users.
+  const pkgDeps: Record<string, string> = {
+    ...pkg.dependencies,
+  }
+  const missing = requiredExternals.filter((ext) => !pkgDeps[ext])
+  if (missing.length > 0) {
+    const msg =
+      `Required external(s) not in package.json: ${missing.join(", ")}\n` +
+      `These packages are marked as external in the binary build but are not\n` +
+      `listed as dependencies. The binary will crash at runtime.\n` +
+      `Add them to "dependencies" in packages/opencode/package.json.`
+    if (Script.release) {
+      console.error(`FATAL: ${msg}`)
+      process.exit(1)
+    } else {
+      console.warn(`WARNING: ${msg}`)
+    }
+  } else {
+    console.log(`Verified ${requiredExternals.length} required external(s) are in package.json`)
+  }
+}
+
 if (Script.release) {
   for (const key of Object.keys(binaries)) {
     const archiveName = key.replace(/^@altimateai\//, "")

packages/opencode/script/pre-release-check.ts

@@ -0,0 +1,151 @@
+#!/usr/bin/env bun
+
+/**
+ * Pre-release sanity check — run BEFORE tagging a release.
+ *
+ * Verifies:
+ * 1. All required external NAPI modules are in package.json dependencies
+ * 2. The publish script will include them in the wrapper package
+ * 3. A local build produces a binary that actually starts
+ *
+ * Usage: bun run packages/opencode/script/pre-release-check.ts
+ */
+
+import fs from "fs"
+import path from "path"
+import { spawnSync } from "child_process"
+import { fileURLToPath } from "url"
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const pkgDir = path.resolve(__dirname, "..")
+const repoRoot = path.resolve(pkgDir, "../..")
+
+const pkg = JSON.parse(fs.readFileSync(path.join(pkgDir, "package.json"), "utf-8"))
+
+let failures = 0
+
+function pass(msg: string) {
+  console.log(`  ✓ ${msg}`)
+}
+
+function fail(msg: string) {
+  console.error(`  ✗ ${msg}`)
+  failures++
+}
+
+// ---------------------------------------------------------------------------
+// Check 1: Required externals are in package.json dependencies
+// ---------------------------------------------------------------------------
+console.log("\n[1/4] Checking required externals in package.json...")
+
+const requiredExternals = ["@altimateai/altimate-core"]
+
+for (const ext of requiredExternals) {
+  if (pkg.dependencies?.[ext]) {
+    pass(`${ext} is in dependencies (${pkg.dependencies[ext]})`)
+  } else {
+    fail(`${ext} is NOT in dependencies — binary will crash at runtime`)
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Check 2: Required externals are resolvable in node_modules
+// ---------------------------------------------------------------------------
+console.log("\n[2/4] Checking required externals are installed...")
+
+for (const ext of requiredExternals) {
+  try {
+    require.resolve(ext)
+    pass(`${ext} resolves from node_modules`)
+  } catch {
+    fail(`${ext} is NOT installed — run \`bun install\``)
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Check 3: Build and smoke-test the binary
+// ---------------------------------------------------------------------------
+console.log("\n[3/4] Building local binary...")
+
+const buildResult = spawnSync("bun", ["run", "build:local"], {
+  cwd: pkgDir,
+  encoding: "utf-8",
+  timeout: 120_000,
+  env: {
+    ...process.env,
+    MODELS_DEV_API_JSON: path.join(pkgDir, "test/tool/fixtures/models-api.json"),
+  },
+})
+
+if (buildResult.status !== 0) {
+  fail(`Build failed:\n${buildResult.stderr}`)
+} else {
+  pass("Local build succeeded")
+
+  // Find the binary — walk recursively for scoped packages (@altimateai/...)
+  const distDir = path.join(pkgDir, "dist")
+  let binaryPath: string | undefined
+  const binaryNames = process.platform === "win32" ? ["altimate.exe", "altimate"] : ["altimate"]
+  function searchDist(dir: string): string | undefined {
+    if (!fs.existsSync(dir)) return undefined
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      if (!entry.isDirectory()) continue
+      const sub = path.join(dir, entry.name)
+      for (const name of binaryNames) {
+        const candidate = path.join(sub, "bin", name)
+        if (fs.existsSync(candidate)) return candidate
+      }
+      const nested = searchDist(sub)
+      if (nested) return nested
+    }
+    return undefined
+  }
+  binaryPath = searchDist(distDir)
+
+  if (!binaryPath) {
+    fail("No binary found in dist/ after build")
+  } else {
+    console.log("\n[4/4] Smoke-testing compiled binary...")
+
+    // Resolve NODE_PATH like the bin wrapper does — start from pkgDir
+    // to include workspace-level node_modules where NAPI modules live
+    const nodePaths: string[] = []
+    let current = pkgDir
+    for (;;) {
+      const nm = path.join(current, "node_modules")
+      if (fs.existsSync(nm)) nodePaths.push(nm)
+      const parent = path.dirname(current)
+      if (parent === current) break
+      current = parent
+    }
+
+    const smokeResult = spawnSync(binaryPath, ["--version"], {
+      encoding: "utf-8",
+      timeout: 15_000,
+      env: {
+        ...process.env,
+        NODE_PATH: nodePaths.join(path.delimiter),
+        OPENCODE_DISABLE_TELEMETRY: "1",
+      },
+    })
+
+    if (smokeResult.status === 0) {
+      const version = (smokeResult.stdout ?? "").trim()
+      pass(`Binary starts successfully (${version})`)
+    } else {
+      const output = (smokeResult.stdout ?? "") + (smokeResult.stderr ?? "")
+      fail(`Binary crashed on startup:\n${output}`)
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Summary
+// ---------------------------------------------------------------------------
+console.log("")
+if (failures > 0) {
+  console.error(`FAILED: ${failures} check(s) failed. Do NOT tag a release.`)
+  process.exit(1)
+} else {
+  console.log("ALL CHECKS PASSED. Safe to tag a release.")
+}