chore: add MCP token measurement script and docs (#507)

* feat: update project tool descriptions

* feat: update MCP tool descriptions

* chore: add MCP token measurement script and docs

Author: jonathannorris

mcp-worker/README.md

@@ -133,3 +133,27 @@ When a user completes OAuth on the hosted MCP Worker, the worker emits a single
 
 - **Channel**: `${orgId}-mcp-install`
 - **Event name**: `mcp-install`
+
+## MCP Tool Token Counts
+
+Measure how many AI tokens our MCP tool descriptions and schemas consume. Run the measurement script (from repo root):
+
+```bash
+yarn install &&
+yarn measure:mcp-tokens
+```
+
+What it does:
+
+- Uses `gpt-tokenizer` (OpenAI-style) and `@anthropic-ai/tokenizer` (Anthropic) to count tokens in each tool's `description` and `inputSchema`.
+
+Current token totals:
+
+```json
+{
+  "totals": {
+    "anthropic": 10428,
+    "openai": 10746
+  }
+}
+```

package.json

@@ -24,6 +24,7 @@
         "deploy:worker:prod": "cd mcp-worker && yarn deploy:prod",
         "deploy:worker:dev": "cd mcp-worker && yarn deploy:dev",
         "dev:worker": "cd mcp-worker && yarn dev",
+        "measure:mcp-tokens": "ts-node scripts/measureMcpTokens.ts",
         "format": "prettier --write \"src/**/*.{ts,js,json}\" \"test/**/*.{ts,js,json}\" \"test-utils/**/*.{ts,js,json}\" \"*.{ts,js,json,md}\"",
         "format:check": "prettier --check \"src/**/*.{ts,js,json}\" \"test/**/*.{ts,js,json}\" \"test-utils/**/*.{ts,js,json}\" \"*.{ts,js,json,md}\"",
         "lint": "eslint . --config eslint.config.mjs",
@@ -67,6 +68,7 @@
         "zod": "~3.25.76"
     },
     "devDependencies": {
+        "@anthropic-ai/tokenizer": "^0.0.4",
         "@babel/code-frame": "^7.27.1",
         "@babel/core": "^7.28.0",
         "@babel/generator": "^7.28.0",
@@ -85,6 +87,7 @@
         "chai": "^5.1.2",
         "eslint": "^9.18.0",
         "eslint-config-prettier": "^9.1.0",
+        "gpt-tokenizer": "^3.0.1",
         "mocha": "^10.8.2",
         "mocha-chai-jest-snapshot": "^1.1.6",
         "nock": "^13.5.6",

scripts/measureMcpTokens.ts

@@ -0,0 +1,135 @@
+#!/usr/bin/env ts-node
+import { registerAllToolsWithServer } from '../src/mcp/tools'
+import type { DevCycleMCPServerInstance } from '../src/mcp/server'
+import type { IDevCycleApiClient } from '../src/mcp/api/interface'
+
+type Collected = {
+    name: string
+    description: string
+    inputSchema?: unknown
+    outputSchema?: unknown
+}
+
+const collected: Collected[] = []
+
+const mockServer: DevCycleMCPServerInstance = {
+    registerToolWithErrorHandling(name, config) {
+        collected.push({
+            name,
+            description: config.description,
+            inputSchema: config.inputSchema,
+            outputSchema: config.outputSchema,
+        })
+    },
+}
+
+// We do not need a real client to collect tool metadata
+const fakeClient = {} as unknown as IDevCycleApiClient
+
+registerAllToolsWithServer(mockServer, fakeClient)
+
+let openAiEncoderPromise: Promise<(input: string) => number[]> | undefined
+async function countOpenAI(text: string): Promise<number> {
+    try {
+        if (!openAiEncoderPromise) {
+            openAiEncoderPromise = import('gpt-tokenizer').then((m) => m.encode)
+        }
+        const encode = await openAiEncoderPromise
+        return encode(text).length
+    } catch {
+        return 0
+    }
+}
+let anthropicCounterPromise: Promise<(input: string) => number> | undefined
+async function countAnthropic(text: string): Promise<number> {
+    try {
+        if (!anthropicCounterPromise) {
+            anthropicCounterPromise = import('@anthropic-ai/tokenizer').then(
+                (m) => m.countTokens,
+            )
+        }
+        const countTokens = await anthropicCounterPromise
+        return countTokens(text)
+    } catch {
+        return 0
+    }
+}
+
+type ResultRow = {
+    name: string
+    anthropic: {
+        description: number
+        inputSchema: number
+        outputSchema: number
+        total: number
+    }
+    openai: {
+        description: number
+        inputSchema: number
+        outputSchema: number
+        total: number
+    }
+}
+
+const rows: ResultRow[] = []
+let grandAnthropic = 0
+let grandOpenAI = 0
+
+async function main() {
+    for (const t of collected) {
+        const d = t.description ?? ''
+        const i = t.inputSchema ? JSON.stringify(t.inputSchema) : ''
+        const o = t.outputSchema ? JSON.stringify(t.outputSchema) : ''
+
+        const [aDesc, aIn, aOut] = await Promise.all([
+            countAnthropic(d),
+            i ? countAnthropic(i) : Promise.resolve(0),
+            o ? countAnthropic(o) : Promise.resolve(0),
+        ])
+        const aTotal = aDesc + aIn + aOut
+
+        const [oDesc, oIn, oOut] = await Promise.all([
+            countOpenAI(d),
+            i ? countOpenAI(i) : Promise.resolve(0),
+            o ? countOpenAI(o) : Promise.resolve(0),
+        ])
+        const oTotal = oDesc + oIn + oOut
+
+        grandAnthropic += aTotal
+        grandOpenAI += oTotal
+
+        rows.push({
+            name: t.name,
+            anthropic: {
+                description: aDesc,
+                inputSchema: aIn,
+                outputSchema: aOut,
+                total: aTotal,
+            },
+            openai: {
+                description: oDesc,
+                inputSchema: oIn,
+                outputSchema: oOut,
+                total: oTotal,
+            },
+        })
+    }
+
+    rows.sort((a, b) => a.name.localeCompare(b.name))
+
+    console.log(
+        JSON.stringify(
+            {
+                tools: rows,
+                totals: { anthropic: grandAnthropic, openai: grandOpenAI },
+            },
+            null,
+            2,
+        ),
+    )
+}
+
+main().catch((err) => {
+    console.error(err)
+    process.exit(1)
+})

yarn.lock

@@ -77,6 +77,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@anthropic-ai/tokenizer@npm:^0.0.4":
+  version: 0.0.4
+  resolution: "@anthropic-ai/tokenizer@npm:0.0.4"
+  dependencies:
+    "@types/node": "npm:^18.11.18"
+    tiktoken: "npm:^1.0.10"
+  checksum: 10c0/fddaa82c26228b6385a0a064c145450564d0288c51d0346a70ce62a716627b26c227077aa909405313668fb5cbef91f3e396783b83decfa01d1c8777d5141220
+  languageName: node
+  linkType: hard
+
 "@apidevtools/json-schema-ref-parser@npm:9.0.6":
   version: 9.0.6
   resolution: "@apidevtools/json-schema-ref-parser@npm:9.0.6"
@@ -734,6 +744,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@devcycle/cli@workspace:."
   dependencies:
+    "@anthropic-ai/tokenizer": "npm:^0.0.4"
     "@babel/code-frame": "npm:^7.27.1"
     "@babel/core": "npm:^7.28.0"
     "@babel/generator": "npm:^7.28.0"
@@ -769,6 +780,7 @@ __metadata:
     eslint-config-prettier: "npm:^9.1.0"
     estraverse: "npm:^5.3.0"
     fuzzy: "npm:^0.1.3"
+    gpt-tokenizer: "npm:^3.0.1"
     inquirer: "npm:^8.2.6"
     inquirer-autocomplete-prompt: "npm:^2.0.1"
     js-sha256: "npm:^0.11.0"
@@ -2824,6 +2836,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/node@npm:^18.11.18":
+  version: 18.19.123
+  resolution: "@types/node@npm:18.19.123"
+  dependencies:
+    undici-types: "npm:~5.26.4"
+  checksum: 10c0/8077177ee2019c4c8875784b367732813b07e533f24197d1fb3bb09e81335267de9da3e70326daaba7a6499df2410257f6099d82d15c9a903d1587a752563178
+  languageName: node
+  linkType: hard
+
 "@types/node@npm:^18.19.68":
   version: 18.19.118
   resolution: "@types/node@npm:18.19.118"
@@ -6115,6 +6136,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"gpt-tokenizer@npm:^3.0.1":
+  version: 3.0.1
+  resolution: "gpt-tokenizer@npm:3.0.1"
+  checksum: 10c0/e95c0825ccc13d27ff873b507c95eca1224f6c7aafdc825a49271959c1f8480368bad75728e32190d106489e25bae788c5932c7c8266ec58dc446ad0d5a26883
+  languageName: node
+  linkType: hard
+
 "graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.5, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9":
   version: 4.2.11
   resolution: "graceful-fs@npm:4.2.11"
@@ -10524,6 +10552,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tiktoken@npm:^1.0.10":
+  version: 1.0.22
+  resolution: "tiktoken@npm:1.0.22"
+  checksum: 10c0/4805cf957d32ee53707ea8416256b50f6b4e865fce1d36ba507bfcbf4dd31bfa69b31dbd1303ba216ae44eb680f54c46c11defa5305445ecea946f2c048fa87d
+  languageName: node
+  linkType: hard
+
 "tinybench@npm:^2.9.0":
   version: 2.9.0
   resolution: "tinybench@npm:2.9.0"