|
| 1 | +/** |
| 2 | + * Skill Integration Test — Golden test: seed rules, match prompts, verify parity with CJS |
| 3 | + * |
| 4 | + * Tests that the TypeScript matcher produces the same results as skill-eval.cjs |
| 5 | + * when given the same rules and prompts. |
| 6 | + */ |
| 7 | + |
| 8 | +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; |
| 9 | +import * as fs from 'fs'; |
| 10 | +import * as os from 'os'; |
| 11 | +import * as path from 'path'; |
| 12 | +import { SkillRegistry } from '../skill-registry.js'; |
| 13 | +import { matchPrompt } from '../skill-matcher.js'; |
| 14 | +import type { SkillRulesFile } from '../types.js'; |
| 15 | + |
| 16 | +let tmpDir: string; |
| 17 | +let registry: SkillRegistry; |
| 18 | +let rulesFile: SkillRulesFile; |
| 19 | + |
| 20 | +// Load the actual skill-rules.json used by the CJS hook |
| 21 | +const RULES_PATH = path.resolve( |
| 22 | + __dirname, |
| 23 | + '../../../../.claude/hooks/skill-rules.json' |
| 24 | +); |
| 25 | + |
| 26 | +beforeEach(() => { |
| 27 | + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-integration-')); |
| 28 | + registry = new SkillRegistry(path.join(tmpDir, 'skills.db')); |
| 29 | + |
| 30 | + if (fs.existsSync(RULES_PATH)) { |
| 31 | + rulesFile = JSON.parse(fs.readFileSync(RULES_PATH, 'utf-8')); |
| 32 | + registry.seedFromRulesJson(rulesFile); |
| 33 | + } else { |
| 34 | + // Fallback minimal rules for CI where .claude/ may not exist |
| 35 | + rulesFile = { |
| 36 | + version: '2.0', |
| 37 | + config: { |
| 38 | + minConfidenceScore: 3, |
| 39 | + showMatchReasons: true, |
| 40 | + maxSkillsToShow: 5, |
| 41 | + }, |
| 42 | + scoring: { |
| 43 | + keyword: 2, |
| 44 | + keywordPattern: 3, |
| 45 | + pathPattern: 4, |
| 46 | + directoryMatch: 5, |
| 47 | + intentPattern: 4, |
| 48 | + contentPattern: 3, |
| 49 | + contextPattern: 2, |
| 50 | + }, |
| 51 | + directoryMappings: { 'src/core': 'frame-management' }, |
| 52 | + skills: { |
| 53 | + 'frame-management': { |
| 54 | + description: 'Frame management', |
| 55 | + priority: 9, |
| 56 | + triggers: { |
| 57 | + keywords: ['frame', 'stack', 'context'], |
| 58 | + keywordPatterns: ['\\bframe\\b'], |
| 59 | + pathPatterns: ['**/frame-*.ts'], |
| 60 | + contentPatterns: ['FrameManager'], |
| 61 | + }, |
| 62 | + relatedSkills: ['context-bridge'], |
| 63 | + }, |
| 64 | + 'linear-integration': { |
| 65 | + description: 'Linear integration', |
| 66 | + priority: 8, |
| 67 | + triggers: { |
| 68 | + keywords: ['linear', 'issue', 'sync'], |
| 69 | + keywordPatterns: ['\\blinear\\b'], |
| 70 | + }, |
| 71 | + }, |
| 72 | + 'testing-patterns': { |
| 73 | + description: 'Testing patterns', |
| 74 | + priority: 7, |
| 75 | + triggers: { |
| 76 | + keywords: ['test', 'jest', 'spec'], |
| 77 | + keywordPatterns: ['\\btest\\b'], |
| 78 | + pathPatterns: ['**/*.test.ts'], |
| 79 | + }, |
| 80 | + }, |
| 81 | + }, |
| 82 | + }; |
| 83 | + registry.seedFromRulesJson(rulesFile); |
| 84 | + } |
| 85 | +}); |
| 86 | + |
| 87 | +afterEach(() => { |
| 88 | + registry.close(); |
| 89 | + fs.rmSync(tmpDir, { recursive: true, force: true }); |
| 90 | +}); |
| 91 | + |
| 92 | +describe('Golden tests: TS matcher parity with CJS', () => { |
| 93 | + function matchWithRegistry(prompt: string) { |
| 94 | + const rules = registry.getAllRules(); |
| 95 | + const { config, scoring } = registry.getMatcherConfig(); |
| 96 | + const mappings = registry.getDirectoryMappings(); |
| 97 | + return matchPrompt(prompt, rules, config, scoring, mappings); |
| 98 | + } |
| 99 | + |
| 100 | + it('frame management prompt matches frame-management skill', () => { |
| 101 | + const result = matchWithRegistry( |
| 102 | + 'Fix the frame manager to handle nested contexts' |
| 103 | + ); |
| 104 | + const names = result.matches.map((m) => m.name); |
| 105 | + expect(names).toContain('frame-management'); |
| 106 | + }); |
| 107 | + |
| 108 | + it('linear sync prompt matches linear-integration skill', () => { |
| 109 | + const result = matchWithRegistry( |
| 110 | + 'Sync the linear issues and update tracking' |
| 111 | + ); |
| 112 | + const names = result.matches.map((m) => m.name); |
| 113 | + expect(names).toContain('linear-integration'); |
| 114 | + }); |
| 115 | + |
| 116 | + it('test file path triggers testing-patterns', () => { |
| 117 | + const result = matchWithRegistry( |
| 118 | + 'Fix src/core/skills/__tests__/skill-matcher.test.ts' |
| 119 | + ); |
| 120 | + const names = result.matches.map((m) => m.name); |
| 121 | + expect(names).toContain('testing-patterns'); |
| 122 | + }); |
| 123 | + |
| 124 | + it('directory mapping triggers frame-management from src/core path', () => { |
| 125 | + const result = matchWithRegistry('Edit src/core/context/frame-manager.ts'); |
| 126 | + const frameMatch = result.matches.find( |
| 127 | + (m) => m.name === 'frame-management' |
| 128 | + ); |
| 129 | + expect(frameMatch).toBeDefined(); |
| 130 | + }); |
| 131 | + |
| 132 | + it('unrelated prompt returns no matches', () => { |
| 133 | + const result = matchWithRegistry('How is the weather today?'); |
| 134 | + expect(result.matches).toEqual([]); |
| 135 | + }); |
| 136 | + |
| 137 | + it('results are sorted by score descending', () => { |
| 138 | + const result = matchWithRegistry('frame linear test sync context issue'); |
| 139 | + for (let i = 1; i < result.matches.length; i++) { |
| 140 | + const prev = result.matches[i - 1]; |
| 141 | + const curr = result.matches[i]; |
| 142 | + expect(prev.score).toBeGreaterThanOrEqual(curr.score); |
| 143 | + } |
| 144 | + }); |
| 145 | + |
| 146 | + it('related skills are resolved', () => { |
| 147 | + const result = matchWithRegistry('Push frame onto the stack'); |
| 148 | + if (result.matches.some((m) => m.name === 'frame-management')) { |
| 149 | + expect(result.relatedSkills).toContain('context-bridge'); |
| 150 | + } |
| 151 | + }); |
| 152 | + |
| 153 | + it('file paths are extracted and reported', () => { |
| 154 | + const result = matchWithRegistry( |
| 155 | + 'Check src/core/frame-manager.ts for bugs' |
| 156 | + ); |
| 157 | + expect(result.filePaths).toContain('src/core/frame-manager.ts'); |
| 158 | + }); |
| 159 | +}); |
| 160 | + |
| 161 | +describe('End-to-end: seed → store skill → match → query', () => { |
| 162 | + it('full workflow', () => { |
| 163 | + // 1. Registry is already seeded from beforeEach |
| 164 | + |
| 165 | + // 2. Create a skill |
| 166 | + const skill = registry.createSkill({ |
| 167 | + content: 'Always validate frame depth before push', |
| 168 | + category: 'pitfall', |
| 169 | + priority: 'high', |
| 170 | + tags: ['frames'], |
| 171 | + tool: 'frame-manager', |
| 172 | + source: 'correction', |
| 173 | + }); |
| 174 | + expect(skill.id).toBeDefined(); |
| 175 | + |
| 176 | + // 3. Match a prompt |
| 177 | + const rules = registry.getAllRules(); |
| 178 | + const { config, scoring } = registry.getMatcherConfig(); |
| 179 | + const mappings = registry.getDirectoryMappings(); |
| 180 | + const result = matchPrompt( |
| 181 | + 'Fix the frame manager depth validation', |
| 182 | + rules, |
| 183 | + config, |
| 184 | + scoring, |
| 185 | + mappings |
| 186 | + ); |
| 187 | + expect(result.matches.length).toBeGreaterThan(0); |
| 188 | + |
| 189 | + // 4. Query skills related to the match |
| 190 | + const relevant = registry.getRelevantSkills({ tool: 'frame-manager' }); |
| 191 | + expect( |
| 192 | + relevant.some((s) => s.content.includes('validate frame depth')) |
| 193 | + ).toBe(true); |
| 194 | + |
| 195 | + // 5. Validate the skill |
| 196 | + const validated = registry.validateSkill(skill.id); |
| 197 | + expect(validated!.validatedCount).toBe(1); |
| 198 | + }); |
| 199 | +}); |
0 commit comments