Skip to content

Commit 08ceb4a

Browse files
brandonkachenbrandonkachen
committed
feat: implement comprehensive analytics event tracking
- Add tracking for backend events: credit_consumed, user_input, flush_failed - Add tracking for CLI events: slash_command_used, invalid_command, terminal_command_completed, change_directory, login, slash_menu_activated, user_input_complete, knowledge_file_updated, app_launched, update_codebuff_failed - Remove 12 unused analytics events that referred to non-existent features - All events now have production tracking implementations
1 parent b380812 commit 08ceb4a

File tree

13 files changed

+341
-55
lines changed

13 files changed

+341
-55
lines changed

‎cli/release-staging/index.js‎

Lines changed: 77 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
const { spawn } = require('child_process')
44
const fs = require('fs')
5+
const http = require('http')
56
const https = require('https')
67
const os = require('os')
78
const path = require('path')
@@ -31,6 +32,70 @@ function createConfig(packageName) {
3132

3233
const CONFIG = createConfig(packageName)
3334

35+
function getPostHogConfig() {
36+
const apiKey =
37+
process.env.CODEBUFF_POSTHOG_API_KEY ||
38+
process.env.NEXT_PUBLIC_POSTHOG_API_KEY
39+
const host =
40+
process.env.CODEBUFF_POSTHOG_HOST ||
41+
process.env.NEXT_PUBLIC_POSTHOG_HOST_URL
42+
43+
if (!apiKey || !host) {
44+
return null
45+
}
46+
47+
return { apiKey, host }
48+
}
49+
50+
/**
51+
* Track update failure event to PostHog.
52+
* Fire-and-forget - errors are silently ignored.
53+
*/
54+
function trackUpdateFailed(errorMessage, version, context = {}) {
55+
try {
56+
const posthogConfig = getPostHogConfig()
57+
if (!posthogConfig) {
58+
return
59+
}
60+
61+
const payload = JSON.stringify({
62+
api_key: posthogConfig.apiKey,
63+
event: 'cli.update_codebuff_failed',
64+
properties: {
65+
distinct_id: `anonymous-${CONFIG.homeDir}`,
66+
error: errorMessage,
67+
version: version || 'unknown',
68+
platform: process.platform,
69+
arch: process.arch,
70+
isStaging: true,
71+
...context,
72+
},
73+
timestamp: new Date().toISOString(),
74+
})
75+
76+
const parsedUrl = new URL(`${posthogConfig.host}/capture/`)
77+
const isHttps = parsedUrl.protocol === 'https:'
78+
const options = {
79+
hostname: parsedUrl.hostname,
80+
port: parsedUrl.port || (isHttps ? 443 : 80),
81+
path: parsedUrl.pathname + parsedUrl.search,
82+
method: 'POST',
83+
headers: {
84+
'Content-Type': 'application/json',
85+
'Content-Length': Buffer.byteLength(payload),
86+
},
87+
}
88+
89+
const transport = isHttps ? https : http
90+
const req = transport.request(options)
91+
req.on('error', () => {}) // Silently ignore errors
92+
req.write(payload)
93+
req.end()
94+
} catch (e) {
95+
// Silently ignore any tracking errors
96+
}
97+
}
98+
3499
const PLATFORM_TARGETS = {
35100
'linux-x64': `${packageName}-linux-x64.tar.gz`,
36101
'linux-arm64': `${packageName}-linux-arm64.tar.gz`,
@@ -256,7 +321,9 @@ async function downloadBinary(version) {
256321
const fileName = PLATFORM_TARGETS[platformKey]
257322

258323
if (!fileName) {
259-
throw new Error(`Unsupported platform: ${process.platform} ${process.arch}`)
324+
const error = new Error(`Unsupported platform: ${process.platform} ${process.arch}`)
325+
trackUpdateFailed(error.message, version, { stage: 'platform_check' })
326+
throw error
260327
}
261328

262329
const downloadUrl = `${
@@ -278,7 +345,9 @@ async function downloadBinary(version) {
278345

279346
if (res.statusCode !== 200) {
280347
fs.rmSync(CONFIG.tempDownloadDir, { recursive: true })
281-
throw new Error(`Download failed: HTTP ${res.statusCode}`)
348+
const error = new Error(`Download failed: HTTP ${res.statusCode}`)
349+
trackUpdateFailed(error.message, version, { stage: 'http_download', statusCode: res.statusCode })
350+
throw error
282351
}
283352

284353
const totalSize = parseInt(res.headers['content-length'] || '0', 10)
@@ -318,9 +387,11 @@ async function downloadBinary(version) {
318387
if (!fs.existsSync(tempBinaryPath)) {
319388
const files = fs.readdirSync(CONFIG.tempDownloadDir)
320389
fs.rmSync(CONFIG.tempDownloadDir, { recursive: true })
321-
throw new Error(
390+
const error = new Error(
322391
`Binary not found after extraction. Expected: ${CONFIG.binaryName}, Available files: ${files.join(', ')}`,
323392
)
393+
trackUpdateFailed(error.message, version, { stage: 'extraction' })
394+
throw error
324395
}
325396

326397
// Set executable permissions
@@ -334,7 +405,9 @@ async function downloadBinary(version) {
334405

335406
if (!smokeTestPassed) {
336407
fs.rmSync(CONFIG.tempDownloadDir, { recursive: true })
337-
throw new Error('Downloaded binary failed smoke test (--version check)')
408+
const error = new Error('Downloaded binary failed smoke test (--version check)')
409+
trackUpdateFailed(error.message, version, { stage: 'smoke_test' })
410+
throw error
338411
}
339412

340413
// Smoke test passed - move binary to final location

‎cli/release/index.js‎

Lines changed: 76 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
const { spawn } = require('child_process')
44
const fs = require('fs')
5+
const http = require('http')
56
const https = require('https')
67
const os = require('os')
78
const path = require('path')
@@ -31,6 +32,69 @@ function createConfig(packageName) {
3132

3233
const CONFIG = createConfig(packageName)
3334

35+
function getPostHogConfig() {
36+
const apiKey =
37+
process.env.CODEBUFF_POSTHOG_API_KEY ||
38+
process.env.NEXT_PUBLIC_POSTHOG_API_KEY
39+
const host =
40+
process.env.CODEBUFF_POSTHOG_HOST ||
41+
process.env.NEXT_PUBLIC_POSTHOG_HOST_URL
42+
43+
if (!apiKey || !host) {
44+
return null
45+
}
46+
47+
return { apiKey, host }
48+
}
49+
50+
/**
51+
* Track update failure event to PostHog.
52+
* Fire-and-forget - errors are silently ignored.
53+
*/
54+
function trackUpdateFailed(errorMessage, version, context = {}) {
55+
try {
56+
const posthogConfig = getPostHogConfig()
57+
if (!posthogConfig) {
58+
return
59+
}
60+
61+
const payload = JSON.stringify({
62+
api_key: posthogConfig.apiKey,
63+
event: 'cli.update_codebuff_failed',
64+
properties: {
65+
distinct_id: `anonymous-${CONFIG.homeDir}`,
66+
error: errorMessage,
67+
version: version || 'unknown',
68+
platform: process.platform,
69+
arch: process.arch,
70+
...context,
71+
},
72+
timestamp: new Date().toISOString(),
73+
})
74+
75+
const parsedUrl = new URL(`${posthogConfig.host}/capture/`)
76+
const isHttps = parsedUrl.protocol === 'https:'
77+
const options = {
78+
hostname: parsedUrl.hostname,
79+
port: parsedUrl.port || (isHttps ? 443 : 80),
80+
path: parsedUrl.pathname + parsedUrl.search,
81+
method: 'POST',
82+
headers: {
83+
'Content-Type': 'application/json',
84+
'Content-Length': Buffer.byteLength(payload),
85+
},
86+
}
87+
88+
const transport = isHttps ? https : http
89+
const req = transport.request(options)
90+
req.on('error', () => {}) // Silently ignore errors
91+
req.write(payload)
92+
req.end()
93+
} catch (e) {
94+
// Silently ignore any tracking errors
95+
}
96+
}
97+
3498
const PLATFORM_TARGETS = {
3599
'linux-x64': `${packageName}-linux-x64.tar.gz`,
36100
'linux-arm64': `${packageName}-linux-arm64.tar.gz`,
@@ -256,7 +320,9 @@ async function downloadBinary(version) {
256320
const fileName = PLATFORM_TARGETS[platformKey]
257321

258322
if (!fileName) {
259-
throw new Error(`Unsupported platform: ${process.platform} ${process.arch}`)
323+
const error = new Error(`Unsupported platform: ${process.platform} ${process.arch}`)
324+
trackUpdateFailed(error.message, version, { stage: 'platform_check' })
325+
throw error
260326
}
261327

262328
const downloadUrl = `${
@@ -278,7 +344,9 @@ async function downloadBinary(version) {
278344

279345
if (res.statusCode !== 200) {
280346
fs.rmSync(CONFIG.tempDownloadDir, { recursive: true })
281-
throw new Error(`Download failed: HTTP ${res.statusCode}`)
347+
const error = new Error(`Download failed: HTTP ${res.statusCode}`)
348+
trackUpdateFailed(error.message, version, { stage: 'http_download', statusCode: res.statusCode })
349+
throw error
282350
}
283351

284352
const totalSize = parseInt(res.headers['content-length'] || '0', 10)
@@ -318,9 +386,11 @@ async function downloadBinary(version) {
318386
if (!fs.existsSync(tempBinaryPath)) {
319387
const files = fs.readdirSync(CONFIG.tempDownloadDir)
320388
fs.rmSync(CONFIG.tempDownloadDir, { recursive: true })
321-
throw new Error(
389+
const error = new Error(
322390
`Binary not found after extraction. Expected: ${CONFIG.binaryName}, Available files: ${files.join(', ')}`,
323391
)
392+
trackUpdateFailed(error.message, version, { stage: 'extraction' })
393+
throw error
324394
}
325395

326396
// Set executable permissions
@@ -334,7 +404,9 @@ async function downloadBinary(version) {
334404

335405
if (!smokeTestPassed) {
336406
fs.rmSync(CONFIG.tempDownloadDir, { recursive: true })
337-
throw new Error('Downloaded binary failed smoke test (--version check)')
407+
const error = new Error('Downloaded binary failed smoke test (--version check)')
408+
trackUpdateFailed(error.message, version, { stage: 'smoke_test' })
409+
throw error
338410
}
339411

340412
// Smoke test passed - move binary to final location

‎cli/src/chat.tsx‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { AnalyticsEvent } from '@codebuff/common/constants/analytics-events'
12
import { RECONNECTION_MESSAGE_DURATION_MS } from '@codebuff/sdk'
23
import open from 'open'
34
import { useQueryClient } from '@tanstack/react-query'
@@ -81,6 +82,7 @@ import { createPasteHandler } from './utils/strings'
8182
import { computeInputLayoutMetrics } from './utils/text-layout'
8283
import { createMarkdownPalette } from './utils/theme-system'
8384
import { reportActivity } from './utils/activity-tracker'
85+
import { trackEvent } from './utils/analytics'
8486

8587
import type { CommandResult } from './commands/command-registry'
8688
import type { MultilineInputHandle } from './components/multiline-input'
@@ -482,6 +484,17 @@ export const Chat = ({
482484
}
483485
}, [mentionContext.active])
484486

487+
// Track when slash menu is activated
488+
const prevSlashActiveRef = useRef(false)
489+
useEffect(() => {
490+
if (slashContext.active && !prevSlashActiveRef.current) {
491+
trackEvent(AnalyticsEvent.SLASH_MENU_ACTIVATED, {
492+
query: slashContext.query,
493+
})
494+
}
495+
prevSlashActiveRef.current = slashContext.active
496+
}, [slashContext.active, slashContext.query])
497+
485498
// Reset suggestion menu indexes when context changes
486499
useEffect(() => {
487500
if (!slashContext.active) {

‎cli/src/commands/init.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import { existsSync, mkdirSync, writeFileSync } from 'fs'
22
import path from 'path'
33

4+
import { AnalyticsEvent } from '@codebuff/common/constants/analytics-events'
5+
46
// @ts-expect-error - Bun text import attribute not supported by TypeScript
57
import agentDefinitionSource from '../../../common/src/templates/initial-agents-dir/types/agent-definition' with { type: 'text' }
68
// @ts-expect-error - Bun text import attribute not supported by TypeScript
@@ -9,6 +11,7 @@ import toolsSource from '../../../common/src/templates/initial-agents-dir/types/
911
import utilTypesSource from '../../../common/src/templates/initial-agents-dir/types/util-types' with { type: 'text' }
1012

1113
import { getProjectRoot } from '../project-files'
14+
import { trackEvent } from '../utils/analytics'
1215
import { getSystemMessage } from '../utils/message-history'
1316

1417
import type { PostUserMessageFn } from '../types/contracts/send-message'
@@ -61,6 +64,12 @@ export function handleInitializationFlowLocally(): {
6164
} else {
6265
writeFileSync(knowledgePath, INITIAL_KNOWLEDGE_FILE)
6366
messages.push(`✅ Created \`${KNOWLEDGE_FILE_NAME}\``)
67+
68+
// Track knowledge file creation
69+
trackEvent(AnalyticsEvent.KNOWLEDGE_FILE_UPDATED, {
70+
action: 'created',
71+
fileName: KNOWLEDGE_FILE_NAME,
72+
})
6473
}
6574

6675
const agentsDir = path.join(projectRoot, '.agents')

‎cli/src/commands/router.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { runTerminalCommand } from '@codebuff/sdk'
22

3+
import { AnalyticsEvent } from '@codebuff/common/constants/analytics-events'
34

45
import {
56
findCommand,
@@ -29,6 +30,7 @@ import {
2930
import { showClipboardMessage } from '../utils/clipboard'
3031
import { getSystemProcessEnv } from '../utils/env'
3132
import { getSystemMessage, getUserMessage } from '../utils/message-history'
33+
import { trackEvent } from '../utils/analytics'
3234

3335
/**
3436
* Run a bash command with automatic ghost/direct mode selection.
@@ -82,6 +84,14 @@ export function runBashCommand(command: string) {
8284
const stderr = 'stderr' in value ? value.stderr || '' : ''
8385
const exitCode = 'exitCode' in value ? value.exitCode ?? 0 : 0
8486

87+
// Track terminal command completion
88+
trackEvent(AnalyticsEvent.TERMINAL_COMMAND_COMPLETED, {
89+
command: command.split(' ')[0], // Just the command name, not args
90+
exitCode,
91+
success: exitCode === 0,
92+
ghost,
93+
})
94+
8595
if (ghost) {
8696
updatePendingBashMessage(id, {
8797
stdout,
@@ -132,6 +142,15 @@ export function runBashCommand(command: string) {
132142
const errorMessage =
133143
error instanceof Error ? error.message : String(error)
134144

145+
// Track terminal command completion with error
146+
trackEvent(AnalyticsEvent.TERMINAL_COMMAND_COMPLETED, {
147+
command: command.split(' ')[0], // Just the command name, not args
148+
exitCode: 1,
149+
success: false,
150+
ghost,
151+
error: true,
152+
})
153+
135154
if (ghost) {
136155
updatePendingBashMessage(id, {
137156
stdout: '',
@@ -241,6 +260,16 @@ export async function routeUserPrompt(
241260
// Allow empty messages if there are pending images attached
242261
if (!trimmed && pendingImages.length === 0) return
243262

263+
// Track user input complete
264+
trackEvent(AnalyticsEvent.USER_INPUT_COMPLETE, {
265+
inputLength: trimmed.length,
266+
mode: agentMode,
267+
inputMode,
268+
hasImages: pendingImages.length > 0,
269+
imageCount: pendingImages.length,
270+
isSlashCommand: isSlashCommand(trimmed),
271+
})
272+
244273
// Handle bash mode commands
245274
if (inputMode === 'bash') {
246275
const commandWithBang = '!' + trimmed
@@ -374,6 +403,12 @@ export async function routeUserPrompt(
374403
// Look up command in registry
375404
const commandDef = findCommand(cmd)
376405
if (commandDef) {
406+
// Track slash command usage
407+
trackEvent(AnalyticsEvent.SLASH_COMMAND_USED, {
408+
command: commandDef.name,
409+
hasArgs: args.trim().length > 0,
410+
})
411+
377412
// The command handler (via defineCommand/defineCommandWithArgs factories)
378413
// is responsible for validating and handling args
379414
return await commandDef.handler(params, args)
@@ -409,6 +444,11 @@ export async function routeUserPrompt(
409444

410445
// Unknown slash command - show error
411446
if (isSlashCommand(trimmed)) {
447+
// Track invalid/unknown command
448+
trackEvent(AnalyticsEvent.INVALID_COMMAND, {
449+
command: trimmed,
450+
})
451+
412452
setMessages((prev) => [
413453
...prev,
414454
getUserMessage(trimmed),

0 commit comments

Comments
 (0)