Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 75 additions & 8 deletions src/service/api.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import express from 'express';
import { dirname, join } from 'path';
import { fileURLToPath } from 'url';
import { rankResults, groupResultsByFile } from './search-ranking.js';

const __dirname = dirname(fileURLToPath(import.meta.url));
const SLOW_QUERY_MS = 100;
Expand Down Expand Up @@ -138,22 +139,44 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n

app.get('/find-type', async (req, res) => {
try {
const { name, fuzzy, project, language, maxResults } = req.query;
const { name, fuzzy, project, language, maxResults, useZoekt } = req.query;

if (!name) {
return res.status(400).json({ error: 'name parameter required' });
}

const mr = parseInt(maxResults, 10) || 10;

// Try Zoekt symbol search if requested and available
if (useZoekt === 'true' && zoektClient) {
try {
const symbolResult = await zoektClient.searchSymbols(name, {
project: project || null,
language: language || null,
caseSensitive: true,
maxResults: mr
});
if (symbolResult.results.length > 0) {
return res.json({
results: symbolResult.results.map(r => ({ ...r, file: cleanPath(r.file) })),
source: 'zoekt-symbol'
});
}
} catch (err) {
// Fall through to database search
}
}

const opts = {
fuzzy: fuzzy === 'true',
project: project || null,
language: language || null,
kind: req.query.kind || null,
maxResults: parseInt(maxResults, 10) || 10
maxResults: mr
};

const results = await poolQuery('findTypeByName', [name, opts]);
res.json({ results });
res.json({ results, source: 'database' });
} catch (err) {
res.status(500).json({ error: err.message });
}
Expand Down Expand Up @@ -260,23 +283,45 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n

app.get('/find-member', async (req, res) => {
try {
const { name, fuzzy, containingType, memberKind, project, language, maxResults } = req.query;
const { name, fuzzy, containingType, memberKind, project, language, maxResults, useZoekt } = req.query;

if (!name) {
return res.status(400).json({ error: 'name parameter required' });
}

const mr = parseInt(maxResults, 10) || 20;

// Try Zoekt symbol search if requested and available
if (useZoekt === 'true' && zoektClient) {
try {
const symbolResult = await zoektClient.searchSymbols(name, {
project: project || null,
language: language || null,
caseSensitive: true,
maxResults: mr
});
if (symbolResult.results.length > 0) {
return res.json({
results: symbolResult.results.map(r => ({ ...r, file: cleanPath(r.file) })),
source: 'zoekt-symbol'
});
}
} catch (err) {
// Fall through to database search
}
}

const opts = {
fuzzy: fuzzy === 'true',
containingType: containingType || null,
memberKind: memberKind || null,
project: project || null,
language: language || null,
maxResults: parseInt(maxResults, 10) || 20
maxResults: mr
};

const results = await poolQuery('findMember', [name, opts]);
res.json({ results });
res.json({ results, source: 'database' });
} catch (err) {
res.status(500).json({ error: err.message });
}
Expand Down Expand Up @@ -441,7 +486,7 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n
// --- Content search (grep) ---

app.get('/grep', async (req, res) => {
const { pattern, project, language, caseSensitive: cs, maxResults: mr, contextLines: cl } = req.query;
const { pattern, project, language, caseSensitive: cs, maxResults: mr, contextLines: cl, grouped } = req.query;

if (!pattern) {
return res.status(400).json({ error: 'pattern parameter required' });
Expand Down Expand Up @@ -482,8 +527,30 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n
})
]);

// Clean paths and rank results
let results = sourceResult.results.map(r => ({ ...r, file: cleanPath(r.file) }));

// Fetch mtime metadata for ranking
const uniquePaths = [...new Set(results.map(r => r.file))];
const mtimeMap = database.getFilesMtime(uniquePaths);
results = rankResults(results, mtimeMap);

if (grouped === 'true') {
return res.json({
results: groupResultsByFile(results),
totalMatches: sourceResult.totalMatches,
truncated: sourceResult.results.length < sourceResult.totalMatches,
timedOut: false,
filesSearched: sourceResult.filesSearched,
searchEngine: 'zoekt',
zoektDurationMs: sourceResult.zoektDurationMs,
grouped: true,
assets: assetResult.results.length > 0 ? assetResult.results : undefined
});
}

const response = {
results: sourceResult.results.map(r => ({ ...r, file: cleanPath(r.file) })),
results,
totalMatches: sourceResult.totalMatches,
truncated: sourceResult.results.length < sourceResult.totalMatches,
timedOut: false,
Expand Down
18 changes: 18 additions & 0 deletions src/service/database.js
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,24 @@ export class IndexDatabase {
return this.db.prepare(sql).all(...params);
}

getFilesMtime(filePaths) {
if (!filePaths || filePaths.length === 0) return new Map();
// Batch query — SQLite max variable limit is 999, chunk if needed
const result = new Map();
const chunkSize = 900;
for (let i = 0; i < filePaths.length; i += chunkSize) {
const chunk = filePaths.slice(i, i + chunkSize);
const placeholders = chunk.map(() => '?').join(',');
const rows = this.db.prepare(
`SELECT path, mtime FROM files WHERE path IN (${placeholders})`
).all(...chunk);
for (const row of rows) {
result.set(row.path, row.mtime);
}
}
return result;
}

upsertFile(path, project, module, mtime, language = 'angelscript') {
const stmt = this.db.prepare(`
INSERT INTO files (path, project, module, mtime, language)
Expand Down
118 changes: 118 additions & 0 deletions src/service/search-ranking.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
// Search result ranking and grouping helpers for grep results.
// Extracted for testability — no Express or database dependencies in pure functions.

const DEFINITION_PATTERNS = [
/^\s*(class|struct|enum)\s+\w+/, // class/struct/enum definition
/^\s*UCLASS\s*\(/, // UE macros
/^\s*USTRUCT\s*\(/,
/^\s*UENUM\s*\(/,
/^\s*UFUNCTION\s*\(/,
/^\s*UPROPERTY\s*\(/,
/^\s*UINTERFACE\s*\(/,
/^\s*(virtual\s+)?(void|int|float|bool|double|auto|const\s+\w+&?|[\w:]+\*?)\s+\w+::\w+\s*\(/, // Method implementation
/^\s*(void|int|float|bool|double|auto|FString|FName|FVector|TArray|TMap)\s+\w+\s*\(/, // Function definition
/^\s*#define\s+\w+/, // Macro definition
/^\s*(mixin\s+)?class\s+\w+/, // AngelScript class
/^\s*(event\s+|delegate\s+)?\w+\s+\w+\s*\(/, // AngelScript event/delegate
];

/**
* Check if a match line looks like a symbol definition (class, function, macro, etc.)
*/
export function isDefinitionLine(line) {
if (!line) return false;
// Skip comments
const trimmed = line.trimStart();
if (trimmed.startsWith('//') || trimmed.startsWith('/*') || trimmed.startsWith('*')) return false;
return DEFINITION_PATTERNS.some(p => p.test(line));
}

const DAY_MS = 24 * 60 * 60 * 1000;

/**
* Score file recency based on mtime. Returns 0-10.
* @param {number} mtime - File modification time in milliseconds
* @param {number} [now] - Current time (for testing)
*/
export function recencyScore(mtime, now) {
if (!mtime) return 0;
now = now || Date.now();
const ageDays = (now - mtime) / DAY_MS;
if (ageDays < 1) return 10;
if (ageDays < 7) return 8;
if (ageDays < 30) return 5;
if (ageDays < 90) return 3;
return 1;
}

/**
* Rank grep results using multiple signals.
* Mutates and returns the results array (sorted in-place).
*
* @param {Array} results - Flat grep results [{file, project, language, line, match, context?}]
* @param {Map<string, number>} mtimeMap - Map of file path -> mtime (from database)
* @returns {Array} Same array, sorted by relevance
*/
export function rankResults(results, mtimeMap) {
if (!results || results.length === 0) return results;

const now = Date.now();

// Compute per-file scores
const fileMatchCount = new Map();
for (const r of results) {
fileMatchCount.set(r.file, (fileMatchCount.get(r.file) || 0) + 1);
}

const fileScores = new Map();
for (const [file, count] of fileMatchCount) {
let score = count; // Match density

// Header / Public boost
if (file.endsWith('.h') || file.endsWith('.hpp')) score += 5;
if (file.includes('/Public/')) score += 3;

// Recency boost
const mtime = mtimeMap ? mtimeMap.get(file) : undefined;
score += recencyScore(mtime, now);

fileScores.set(file, score);
}

// Per-result scoring: file score + definition boost
results.sort((a, b) => {
const sa = (fileScores.get(a.file) || 0) + (isDefinitionLine(a.match) ? 8 : 0);
const sb = (fileScores.get(b.file) || 0) + (isDefinitionLine(b.match) ? 8 : 0);
return sb - sa || a.line - b.line;
});

return results;
}

/**
* Group flat grep results by file.
*
* @param {Array} results - Flat grep results [{file, project, language, line, match, context?}]
* @returns {Array} Grouped results [{file, project, language, matches: [{line, match, context?}]}]
*/
export function groupResultsByFile(results) {
if (!results || results.length === 0) return [];

const fileMap = new Map();
for (const r of results) {
if (!fileMap.has(r.file)) {
fileMap.set(r.file, {
file: r.file,
project: r.project,
language: r.language,
matches: []
});
}
const entry = { line: r.line, match: r.match };
if (r.context) entry.context = r.context;
fileMap.get(r.file).matches.push(entry);
}

return Array.from(fileMap.values())
.sort((a, b) => b.matches.length - a.matches.length);
}
35 changes: 19 additions & 16 deletions src/service/zoekt-client.js
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,23 @@ export class ZoektClient {
return result;
}

async searchSymbols(symbolName, options = {}) {
const { project, language, caseSensitive = true, maxResults = 20 } = options;

const parts = [caseSensitive ? 'case:yes' : 'case:no'];
parts.push(`sym:${symbolName}`);

if (language && language !== 'all' && LANGUAGE_EXTENSIONS[language]) {
parts.push(`file:${LANGUAGE_EXTENSIONS[language]}`);
}
parts.push('-file:^_assets/');
if (project) {
parts.push(`file:${project}/`);
}

return this._executeQuery(parts.join(' '), maxResults, 0);
}

async _executeQuery(query, maxResults, contextLines) {
const body = {
Q: query,
Expand Down Expand Up @@ -246,22 +263,8 @@ export class ZoektClient {
}
}

// Rank results: header files and high match-density files first
const fileScores = new Map();
for (const r of results) {
const prev = fileScores.get(r.file) || 0;
let score = prev + 1; // +1 per match in same file
if (!prev) {
if (r.file.endsWith('.h') || r.file.endsWith('.hpp')) score += 5;
if (r.file.includes('/Public/')) score += 3;
}
fileScores.set(r.file, score);
}
results.sort((a, b) => {
const sa = fileScores.get(a.file) || 0;
const sb = fileScores.get(b.file) || 0;
return sb - sa || a.line - b.line;
});
// Results returned unranked — ranking is done in the API layer (search-ranking.js)
// with access to database metadata (mtime, symbol detection, etc.)

return {
results,
Expand Down
Loading