feat: Add handoff versioning, decision history, and accurate tokenization

- Handoff versioning: Keep last 10 handoffs in .stackmemory/handoffs/
- Decision history: Archive decisions on clear, view with --history flag
- Accurate tokens: Use @anthropic-ai/tokenizer instead of char estimation
- Decision clear now archives to ~/.stackmemory/decision-history/

Author: jonathanpeterwu

package-lock.json

@@ -82,6 +82,7 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.71.2",
+    "@anthropic-ai/tokenizer": "^0.0.4",
     "@aws-sdk/client-s3": "^3.958.0",
     "@browsermcp/mcp": "^0.1.3",
     "@google-cloud/storage": "^7.18.0",

package.json

@@ -8,8 +8,16 @@
  */
 
 import { Command } from 'commander';
-import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
-import { join } from 'path';
+import {
+  existsSync,
+  readFileSync,
+  writeFileSync,
+  mkdirSync,
+  readdirSync,
+} from 'fs';
+import { join, basename } from 'path';
+import { homedir } from 'os';
+import { createHash } from 'crypto';
 
 interface Decision {
   id: string;
@@ -29,6 +37,94 @@ function getDecisionStorePath(projectRoot: string): string {
   return join(projectRoot, '.stackmemory', 'session-decisions.json');
 }
 
+function getProjectId(projectRoot: string): string {
+  const hash = createHash('sha256').update(projectRoot).digest('hex');
+  return hash.slice(0, 12);
+}
+
+function getHistoryDir(): string {
+  return join(homedir(), '.stackmemory', 'decision-history');
+}
+
+function archiveDecisions(projectRoot: string, decisions: Decision[]): void {
+  if (decisions.length === 0) return;
+
+  const historyDir = getHistoryDir();
+  const projectId = getProjectId(projectRoot);
+  const projectDir = join(historyDir, projectId);
+
+  if (!existsSync(projectDir)) {
+    mkdirSync(projectDir, { recursive: true });
+  }
+
+  // Save with timestamp
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const archivePath = join(projectDir, `${timestamp}.json`);
+
+  const archive = {
+    projectRoot,
+    projectName: basename(projectRoot),
+    archivedAt: new Date().toISOString(),
+    decisions,
+  };
+
+  writeFileSync(archivePath, JSON.stringify(archive, null, 2));
+}
+
+interface HistoricalDecision extends Decision {
+  projectName: string;
+  archivedAt: string;
+}
+
+function loadDecisionHistory(projectRoot?: string): HistoricalDecision[] {
+  const historyDir = getHistoryDir();
+  if (!existsSync(historyDir)) return [];
+
+  const allDecisions: HistoricalDecision[] = [];
+
+  try {
+    const projectDirs = projectRoot
+      ? [getProjectId(projectRoot)]
+      : readdirSync(historyDir);
+
+    for (const projectId of projectDirs) {
+      const projectDir = join(historyDir, projectId);
+      if (!existsSync(projectDir)) continue;
+
+      try {
+        const files = readdirSync(projectDir).filter((f) =>
+          f.endsWith('.json')
+        );
+        for (const file of files) {
+          try {
+            const content = JSON.parse(
+              readFileSync(join(projectDir, file), 'utf-8')
+            );
+            for (const d of content.decisions || []) {
+              allDecisions.push({
+                ...d,
+                projectName: content.projectName || 'unknown',
+                archivedAt: content.archivedAt,
+              });
+            }
+          } catch {
+            // Skip invalid files
+          }
+        }
+      } catch {
+        // Skip unreadable directories
+      }
+    }
+  } catch {
+    // History dir unreadable
+  }
+
+  // Sort by timestamp descending
+  return allDecisions.sort(
+    (a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime()
+  );
+}
+
 function loadDecisions(projectRoot: string): DecisionStore {
   const storePath = getDecisionStorePath(projectRoot);
   if (existsSync(storePath)) {
@@ -104,10 +200,42 @@ export function createDecisionCommand(): Command {
     .command('list')
     .description('List all decisions from this session')
     .option('--json', 'Output as JSON')
+    .option('--history', 'Include historical decisions')
+    .option('--all', 'Show all projects (with --history)')
     .action((options) => {
       const projectRoot = process.cwd();
       const store = loadDecisions(projectRoot);
 
+      if (options.history) {
+        const history = loadDecisionHistory(
+          options.all ? undefined : projectRoot
+        );
+
+        if (options.json) {
+          console.log(JSON.stringify(history, null, 2));
+          return;
+        }
+
+        if (history.length === 0) {
+          console.log('No decision history found.');
+          return;
+        }
+
+        console.log(`Decision History (${history.length}):\n`);
+        for (const d of history.slice(0, 50)) {
+          const category = d.category ? `[${d.category}] ` : '';
+          const project = options.all ? `(${d.projectName}) ` : '';
+          console.log(`${project}${category}${d.what}`);
+          if (d.why) {
+            console.log(`  Rationale: ${d.why}`);
+          }
+          const date = new Date(d.timestamp).toLocaleDateString();
+          console.log(`  Date: ${date}`);
+          console.log('');
+        }
+        return;
+      }
+
       if (options.json) {
         console.log(JSON.stringify(store.decisions, null, 2));
         return;
@@ -137,8 +265,9 @@ export function createDecisionCommand(): Command {
   // Clear decisions (for new session)
   cmd
     .command('clear')
-    .description('Clear all decisions (start fresh session)')
+    .description('Clear all decisions (archives to history first)')
     .option('--force', 'Skip confirmation')
+    .option('--no-archive', 'Do not archive decisions')
     .action((options) => {
       const projectRoot = process.cwd();
       const store = loadDecisions(projectRoot);
@@ -150,10 +279,17 @@ export function createDecisionCommand(): Command {
 
       if (!options.force) {
         console.log(`This will clear ${store.decisions.length} decisions.`);
+        console.log('Decisions will be archived to history.');
         console.log('Use --force to confirm.');
         return;
       }
 
+      // Archive before clearing (unless --no-archive)
+      if (options.archive !== false) {
+        archiveDecisions(projectRoot, store.decisions);
+        console.log(`Archived ${store.decisions.length} decisions to history.`);
+      }
+
       const newStore: DecisionStore = {
         decisions: [],
         sessionStart: new Date().toISOString(),

src/cli/commands/decision.ts

@@ -4,7 +4,14 @@
 
 import { Command } from 'commander';
 import { execSync, execFileSync } from 'child_process';
-import { existsSync, readFileSync, writeFileSync } from 'fs';
+import {
+  existsSync,
+  readFileSync,
+  writeFileSync,
+  mkdirSync,
+  readdirSync,
+  unlinkSync,
+} from 'fs';
 import { join } from 'path';
 import Database from 'better-sqlite3';
 import { z } from 'zod';
@@ -13,6 +20,46 @@ import { LinearTaskManager } from '../../features/tasks/linear-task-manager.js';
 import { logger } from '../../core/monitoring/logger.js';
 import { EnhancedHandoffGenerator } from '../../core/session/enhanced-handoff.js';
 
+// Handoff versioning - keep last N handoffs
+const MAX_HANDOFF_VERSIONS = 10;
+
+function saveVersionedHandoff(
+  projectRoot: string,
+  branch: string,
+  content: string
+): string {
+  const handoffsDir = join(projectRoot, '.stackmemory', 'handoffs');
+  if (!existsSync(handoffsDir)) {
+    mkdirSync(handoffsDir, { recursive: true });
+  }
+
+  // Generate versioned filename: YYYY-MM-DD-HH-mm-branch.md
+  const now = new Date();
+  const timestamp = now.toISOString().slice(0, 16).replace(/[T:]/g, '-');
+  const safeBranch = branch.replace(/[^a-zA-Z0-9-]/g, '-').slice(0, 30);
+  const filename = `${timestamp}-${safeBranch}.md`;
+  const versionedPath = join(handoffsDir, filename);
+
+  // Save versioned handoff
+  writeFileSync(versionedPath, content);
+
+  // Clean up old handoffs (keep last N)
+  try {
+    const files = readdirSync(handoffsDir)
+      .filter((f) => f.endsWith('.md'))
+      .sort()
+      .reverse();
+
+    for (const oldFile of files.slice(MAX_HANDOFF_VERSIONS)) {
+      unlinkSync(join(handoffsDir, oldFile));
+    }
+  } catch {
+    // Cleanup failed, not critical
+  }
+
+  return versionedPath;
+}
+
 // Input validation schemas
 const CommitMessageSchema = z
   .string()
@@ -268,14 +315,33 @@ Generated by stackmemory handoff at ${timestamp}
           console.log(`Estimated tokens: ~${enhancedHandoff.estimatedTokens}`);
         }
 
-        // 7. Save handoff prompt
+        // 7. Save handoff prompt (both latest and versioned)
         const handoffPath = join(
           projectRoot,
           '.stackmemory',
           'last-handoff.md'
         );
         writeFileSync(handoffPath, handoffPrompt);
 
+        // Save versioned copy
+        let branch = 'unknown';
+        try {
+          branch = execSync('git rev-parse --abbrev-ref HEAD', {
+            encoding: 'utf-8',
+            cwd: projectRoot,
+          }).trim();
+        } catch {
+          // Not a git repo
+        }
+        const versionedPath = saveVersionedHandoff(
+          projectRoot,
+          branch,
+          handoffPrompt
+        );
+        console.log(
+          `Versioned: ${versionedPath.split('/').slice(-2).join('/')}`
+        );
+
         // 8. Display the prompt
         console.log('\n' + '='.repeat(60));
         console.log(handoffPrompt);

src/cli/commands/handoff.ts

@@ -17,6 +17,17 @@ import { join, basename } from 'path';
 import { homedir, tmpdir } from 'os';
 import { globSync } from 'glob';
 
+// Token counting - use Anthropic's tokenizer for accurate counts
+let countTokens: (text: string) => number;
+try {
+  // Dynamic import for CommonJS compatibility
+  const tokenizer = await import('@anthropic-ai/tokenizer');
+  countTokens = tokenizer.countTokens;
+} catch {
+  // Fallback to estimation if tokenizer not available
+  countTokens = (text: string) => Math.ceil(text.length / 3.5);
+}
+
 // Load session decisions if available
 interface SessionDecision {
   id: string;
@@ -256,7 +267,7 @@ export class EnhancedHandoffGenerator {
 
     // Calculate estimated tokens
     const markdown = this.toMarkdown(handoff);
-    handoff.estimatedTokens = Math.ceil(markdown.length / 3.5);
+    handoff.estimatedTokens = countTokens(markdown);
 
     return handoff;
   }