feat: PostHog tracking for Capgo Builder onboarding + build lifecycle

cli/src/ai/telemetry.ts

@@ -0,0 +1,86 @@
+import { sendEvent } from '../utils.js'
+
+export type AiAnalysisChoice = 'capgo_ai' | 'local_ai' | 'skip' | 'auto_upload'
+export type AiAnalysisTriggeredBy = 'menu' | 'ci_flag'
+export type AiAnalysisResult = 'success' | 'already_analyzed' | 'too_big' | 'error'
+
+export interface TrackAiAnalysisChoiceInput {
+  apikey: string
+  orgId: string
+  appId: string
+  platform: 'ios' | 'android'
+  jobId: string
+  choice: AiAnalysisChoice
+  triggeredBy: AiAnalysisTriggeredBy
+}
+
+export interface TrackAiAnalysisResultInput {
+  apikey: string
+  orgId: string
+  appId: string
+  platform: 'ios' | 'android'
+  jobId: string
+  result: AiAnalysisResult
+  errorStatus?: number
+}
+
+/**
+ * Emit `CLI AI Build Analysis Choice` for every branch the user (or CI flag) selected.
+ *
+ * Privacy boundary: only closed-enum choice + triggered_by metadata is sent. The
+ * AI diagnosis text is never observed at this stage.
+ */
+export async function trackAiAnalysisChoice(input: TrackAiAnalysisChoiceInput): Promise<void> {
+  try {
+    await sendEvent(input.apikey, {
+      event: 'CLI AI Build Analysis Choice',
+      channel: 'build-lifecycle',
+      icon: '🤖',
+      notify: false,
+      user_id: input.orgId,
+      tags: {
+        app_id: input.appId,
+        platform: input.platform,
+        job_id: input.jobId,
+        choice: input.choice,
+        triggered_by: input.triggeredBy,
+      },
+    })
+  }
+  catch {
+    // Telemetry must never break the build flow.
+  }
+}
+
+/**
+ * Emit `CLI AI Build Analysis Result` only for paths that actually hit the server
+ * (capgo_ai or auto_upload).
+ *
+ * Privacy boundary: the AI analysis text (`result.analysis` in PostAnalyzeResult)
+ * MUST NEVER appear in any tag here. Only the closed-enum `result` and optional
+ * `error_status` cross the boundary.
+ */
+export async function trackAiAnalysisResult(input: TrackAiAnalysisResultInput): Promise<void> {
+  const tags: Record<string, string> = {
+    app_id: input.appId,
+    platform: input.platform,
+    job_id: input.jobId,
+    result: input.result,
+  }
+  if (input.result === 'error' && typeof input.errorStatus === 'number' && Number.isFinite(input.errorStatus))
+    tags.error_status = String(input.errorStatus)
+
+  try {
+    await sendEvent(input.apikey, {
+      event: 'CLI AI Build Analysis Result',
+      channel: 'build-lifecycle',
+      icon: '🤖',
+      notify: false,
+      user_id: input.orgId,
+      tags,
+    })
+  }
+  catch {
+    // Telemetry must never break the build flow.
+  }
+}

cli/src/build/onboarding/android/types.ts

@@ -51,6 +51,12 @@ export type AndroidOnboardingStep
     | 'build-complete'
     | 'error'
 
+export type AndroidOnboardingErrorCategory
+  = | 'keystore_invalid'
+    | 'google_oauth_failed'
+    | 'play_account_id_invalid'
+    | 'unknown'
+
 export type KeystoreMethod = 'existing' | 'generate'
 
 export interface KeystoreReady {

cli/src/build/onboarding/android/ui/app.tsx

@@ -2,6 +2,7 @@ import type { FC } from 'react'
 import type { BuildLogger } from '../../../request.js'
 import type { GcpProject } from '../gcp-api.js'
 import type {
+  AndroidOnboardingErrorCategory,
   AndroidOnboardingProgress,
   AndroidOnboardingStep,
   AndroidPackageChoice,
@@ -22,14 +23,16 @@ import { Alert, ProgressBar, Select } from '@inkjs/ui'
 import { Box, Newline, Text, useApp, useInput, useStdout } from 'ink'
 // src/build/onboarding/android/ui/app.tsx
 import React, { useCallback, useEffect, useRef, useState } from 'react'
-import { findSavedKey } from '../../../../utils.js'
+import { createSupabaseClient, findSavedKey, findSavedKeySilent, getOrganizationId } from '../../../../utils.js'
 import { loadSavedCredentials, updateSavedCredentials } from '../../../credentials.js'
 import { requestBuildInternal } from '../../../request.js'
-import { createCiSecretEntries, detectCiSecretTargets, getCiSecretTargetLabel, listExistingCiSecretKeys, uploadCiSecrets } from '../../ci-secrets.js'
 import type { CiSecretEntry, CiSecretSetupAdvice, CiSecretTarget } from '../../ci-secrets.js'
+import { createCiSecretEntries, detectCiSecretTargets, getCiSecretTargetLabel, listExistingCiSecretKeys, uploadCiSecrets } from '../../ci-secrets.js'
+import { mapAndroidOnboardingError } from '../../error-categories.js'
 import { canUseFilePicker, openKeystorePicker } from '../../file-picker.js'
-import { findAndroidApplicationIds } from '../gradle-parser.js'
+import { trackBuilderOnboardingStep } from '../../telemetry.js'
 import { Divider, ErrorLine, FilteredTextInput, Header, SpinnerLine, SuccessLine } from '../../ui/components.js'
+import { findAndroidApplicationIds } from '../gradle-parser.js'
 import {
   ANDROIDPUBLISHER_API,
   createServiceAccountKey,
@@ -119,8 +122,126 @@ const AndroidOnboardingApp: FC<AppProps> = ({ appId, initialProgress, androidDir
   const [step, setStep] = useState<AndroidOnboardingStep>(
     startStep === 'welcome' ? 'welcome' : startStep,
   )
+
+  // Telemetry: resolve org id once + emit per-step events
+  const stepTimingRef = useRef<{ step: AndroidOnboardingStep | null, startedAt: number }>({
+    step: null,
+    startedAt: Date.now(),
+  })
+  // Buffer of telemetry events that occurred before `resolvedOrgId` landed.
+  // Drained in order when the org id becomes available. Without this buffer,
+  // any step transitions during the async org-id resolution (which involves
+  // two HTTP round-trips: createSupabaseClient + getOrganizationId) would be
+  // dropped from the funnel.
+  const pendingTelemetryRef = useRef<Array<{
+    step: AndroidOnboardingStep
+    durationMs?: number
+    errorCategory?: AndroidOnboardingErrorCategory
+  }>>([])
+  const [resolvedOrgId, setResolvedOrgId] = useState<string | null>(null)
+  const resolvedApiKeyRef = useRef<string | null>(apikey ?? null)
+  const orgIdResolvedRef = useRef(false)
+  // Captures the mapped error category at handleError time so the telemetry
+  // useEffect can pass it through without re-mapping a reconstructed Error
+  // (which would have lost the .phase / instanceof discriminators).
+  const errorCategoryRef = useRef<AndroidOnboardingErrorCategory | undefined>(undefined)
+
+  useEffect(() => {
+    if (resolvedApiKeyRef.current)
+      return
+    const saved = findSavedKeySilent()
+    if (saved)
+      resolvedApiKeyRef.current = saved
+  }, [])
+
+  useEffect(() => {
+    if (orgIdResolvedRef.current || !resolvedApiKeyRef.current)
+      return
+    orgIdResolvedRef.current = true
+
+    let cancelled = false
+    void (async () => {
+      const supabase = await createSupabaseClient(resolvedApiKeyRef.current!, undefined, undefined, true)
+        .catch(() => null)
+      if (!supabase || cancelled)
+        return
+      const orgId = await getOrganizationId(supabase, appId).catch(() => null)
+      if (orgId && !cancelled)
+        setResolvedOrgId(orgId)
+    })()
+
+    return () => {
+      cancelled = true
+    }
+  }, [appId])
+
   const [logLines, setLogLines] = useState<LogEntry[]>([])
   const [error, setError] = useState<string | null>(null)
+
+  // Emit telemetry on every step transition (including initial mount).
+  // Sequencing:
+  //   1. If `resolvedOrgId` just became available, drain the backlog first.
+  //   2. Skip same-step re-renders (orgId-lands triggers a re-fire — we don't
+  //      want to re-emit the current step, only drain the backlog).
+  //   3. Otherwise compute the new event, then either emit immediately (orgId
+  //      available) or queue it (orgId still loading).
+  useEffect(() => {
+    if (!resolvedApiKeyRef.current)
+      return
+
+    const previous = stepTimingRef.current
+    const isDuplicateStep = previous.step !== null && previous.step === step && step !== 'error'
+
+    // (1) Drain the backlog if org id is now available, even when the current
+    // step is a duplicate (e.g., this effect fired because resolvedOrgId moved
+    // from null to a real value, not because step changed).
+    if (resolvedOrgId && pendingTelemetryRef.current.length > 0) {
+      for (const queued of pendingTelemetryRef.current) {
+        void trackBuilderOnboardingStep({
+          apikey: resolvedApiKeyRef.current,
+          appId,
+          orgId: resolvedOrgId,
+          platform: 'android',
+          ...queued,
+        })
+      }
+      pendingTelemetryRef.current = []
+    }
+
+    // (2) Now safely skip the duplicate-step path.
+    if (isDuplicateStep)
+      return
+
+    const now = Date.now()
+    // Initial step (previous.step === null) and same-step error re-entries have
+    // no meaningful previous-step duration.
+    const durationMs = previous.step === null || previous.step === step
+      ? undefined
+      : now - previous.startedAt
+
+    const eventPayload = {
+      step,
+      durationMs,
+      errorCategory: step === 'error' ? errorCategoryRef.current : undefined,
+    }
+
+    stepTimingRef.current = { step, startedAt: now }
+
+    // (3) Either fire immediately or buffer.
+    if (resolvedOrgId) {
+      void trackBuilderOnboardingStep({
+        apikey: resolvedApiKeyRef.current,
+        appId,
+        orgId: resolvedOrgId,
+        platform: 'android',
+        ...eventPayload,
+      })
+    }
+    else {
+      pendingTelemetryRef.current.push(eventPayload)
+    }
+  }, [step, appId, resolvedOrgId, error])
+
   const [retryCount, setRetryCount] = useState(0)
   const [retryStep, setRetryStep] = useState<AndroidOnboardingStep | null>(null)
   const exitRequestedRef = useRef(false)
@@ -351,6 +472,10 @@ const AndroidOnboardingApp: FC<AppProps> = ({ appId, initialProgress, androidDir
 
   const handleError = useCallback(
     (err: unknown, failedStep: AndroidOnboardingStep) => {
+      // Capture the mapped category BEFORE we collapse err to a string.
+      // The telemetry useEffect will read this ref instead of re-mapping a
+      // reconstructed `new Error(message)` (which has no discriminators).
+      errorCategoryRef.current = mapAndroidOnboardingError(err)
       const message = err instanceof Error ? err.message : String(err)
       if (retryCount === 0) {
         setError(message)
@@ -2067,6 +2192,7 @@ const AndroidOnboardingApp: FC<AppProps> = ({ appId, initialProgress, androidDir
             onChange={(value) => {
               if (value === 'retry') {
                 setError(null)
+                errorCategoryRef.current = undefined
                 const target = retryStep
                 setRetryStep(null)
                 setStep(target)

cli/src/build/onboarding/error-categories.ts

@@ -0,0 +1,84 @@
+import type { AndroidOnboardingErrorCategory } from './android/types.js'
+import type { OnboardingErrorCategory, OnboardingStep } from './types.js'
+import { MissingScopesError } from './android/oauth-google.js'
+import { CertificateLimitError } from './apple-api.js'
+
+interface MaybeStatus {
+  status?: unknown
+}
+
+interface MaybePhase {
+  phase?: string
+}
+
+function getStatus(error: unknown): number | undefined {
+  if (!error || typeof error !== 'object')
+    return undefined
+  const candidate = (error as MaybeStatus).status
+  return typeof candidate === 'number' ? candidate : undefined
+}
+
+function getPhase(error: unknown): string | undefined {
+  if (!error || typeof error !== 'object')
+    return undefined
+  const candidate = (error as MaybePhase).phase
+  return typeof candidate === 'string' ? candidate : undefined
+}
+
+export function mapIosOnboardingError(
+  error: unknown,
+  failedStep?: OnboardingStep,
+): OnboardingErrorCategory {
+  // Structural discriminators take precedence so an ASC API error thrown
+  // during an import step (e.g. fetching a profile via the API) still maps
+  // to the correct category instead of being shadowed by the step-derived
+  // fallback below.
+  if (error instanceof CertificateLimitError)
+    return 'cert_limit_reached'
+
+  const status = getStatus(error)
+  if (status === 401)
+    return 'apple_api_unauthorized'
+  if (status === 429)
+    return 'apple_api_rate_limited'
+
+  const phase = getPhase(error)
+  if (phase === 'profile')
+    return 'profile_creation_failed'
+  if (phase === 'p8')
+    return 'p8_invalid'
+
+  // Import-flow step-derived categories. The import path throws
+  // MacOSSigningError / generic Error without a `phase` or `status`
+  // discriminator, so we derive the category from the step at which the
+  // failure occurred.
+  if (failedStep === 'import-scanning')
+    return 'keychain_no_identities'
+  if (failedStep === 'import-compiling-helper')
+    return 'keychain_helper_compile_failed'
+  if (failedStep === 'import-exporting')
+    return 'keychain_export_failed'
+  if (failedStep === 'import-fetching-profile')
+    return 'profile_read_failed'
+  if (failedStep === 'import-pick-profile' || failedStep === 'import-no-match-recovery')
+    return 'profile_no_match'
+
+  return 'unknown'
+}
+
+export function mapAndroidOnboardingError(error: unknown): AndroidOnboardingErrorCategory {
+  // MissingScopesError has no `phase` property, so the instanceof check must
+  // precede the phase-based dispatching below.
+  if (error instanceof MissingScopesError)
+    return 'google_oauth_failed'
+
+  const phase = getPhase(error)
+  if (phase === 'keystore')
+    return 'keystore_invalid'
+  if (phase === 'oauth')
+    return 'google_oauth_failed'
+  if (phase === 'play_account_id')
+    return 'play_account_id_invalid'
+
+  return 'unknown'
+}