Optimize Zoekt mirror: direct-to-WSL streaming and scoped reindexing

package.json

@@ -27,6 +27,7 @@
     "better-sqlite3": "^11.0.0",
     "chokidar": "^4.0.0",
     "express": "^4.21.0",
+    "tar-stream": "^3.1.7",
     "undici": "^7.21.0"
   }
 }

src/service/index.js

@@ -177,46 +177,73 @@ class UnrealIndexService {
 
         this.zoektMirror = new ZoektMirror(mirrorDir);
 
+        this.zoektManager = new ZoektManager({
+          indexDir,
+          webPort: zoektConfig.webPort || 6070,
+          parallelism: zoektConfig.parallelism || 4,
+          fileLimitBytes: zoektConfig.fileLimitBytes || 524288,
+          reindexDebounceMs: zoektConfig.reindexDebounceMs || 5000,
+          zoektBin: zoektConfig.zoektBin || null
+        });
+
+        if (!this.zoektManager.init()) {
+          throw new Error('Zoekt binaries not found');
+        }
+
         const mirrorProgress = (p) => {
           console.log(`[Startup] Mirror: ${p.written}/${p.total} (${Math.round(p.written / p.total * 100)}%) — ETA ${p.etaSeconds}s`);
         };
 
-        if (!this.zoektMirror.isReady()) {
-          this.zoektMirror.bootstrapFromDatabase(this.database, mirrorProgress);
-        } else {
+        const needsBootstrap = !this.zoektMirror.isReady();
+        let mirrorIntegrityFailed = false;
+
+        if (!needsBootstrap) {
           this.zoektMirror.loadPrefix(this.database);
-          // Verify mirror integrity against database
           const check = this.zoektMirror.verifyIntegrity(this.database);
           if (!check.valid) {
             console.warn(`[Startup] Mirror integrity check failed: ${check.reason}, rebuilding...`);
-            this.zoektMirror.bootstrapFromDatabase(this.database, mirrorProgress);
+            mirrorIntegrityFailed = true;
           } else {
             console.log(`[Startup] Mirror OK (${check.mirrorCount} files)`);
           }
         }
 
-        this.zoektManager = new ZoektManager({
-          indexDir,
-          webPort: zoektConfig.webPort || 6070,
-          parallelism: zoektConfig.parallelism || 4,
-          fileLimitBytes: zoektConfig.fileLimitBytes || 524288,
-          reindexDebounceMs: zoektConfig.reindexDebounceMs || 5000,
-          zoektBin: zoektConfig.zoektBin || null
-        });
-
-        // Init → sync → start: sync must complete BEFORE webserver starts,
-        // as concurrent WSL processes cause zoekt-webserver to exit
-        if (!this.zoektManager.init()) {
-          throw new Error('Zoekt binaries not found');
+        if (needsBootstrap || mirrorIntegrityFailed) {
+          // First-time or rebuild: try direct-to-WSL bootstrap (skips Windows mirror)
+          if (this.zoektManager.useWsl && this.zoektManager.wslMirrorDir) {
+            console.log('[Startup] Direct-to-WSL bootstrap (skipping Windows mirror)...');
+            await this.zoektManager.bootstrapDirect(this.database, this.zoektMirror, mirrorProgress);
+            this.zoektManager.mirrorRoot = this.zoektMirror.getMirrorRoot();
+
+            // Populate Windows mirror in background (needed for watcher incremental updates)
+            setTimeout(() => {
+              console.log('[Startup] Background: populating Windows mirror for watcher...');
+              try {
+                this.zoektMirror.bootstrapFromDatabase(this.database, (p) => {
+                  if (p.written % 50000 === 0) {
+                    console.log(`[Startup] Background mirror: ${p.written}/${p.total}`);
+                  }
+                });
+                console.log('[Startup] Background mirror population complete');
+              } catch (err) {
+                console.warn(`[Startup] Background mirror failed: ${err.message}`);
+              }
+            }, 5000);
+          } else {
+            // Non-WSL or WSL not available: use existing bootstrap flow
+            this.zoektMirror.bootstrapFromDatabase(this.database, mirrorProgress);
+          }
         }
-        if (process.env.SKIP_SYNC !== '1') {
+
+        // Sync mirror to WSL if needed (skipped if bootstrapDirect already set effective path)
+        if (process.env.SKIP_SYNC !== '1' && !this.zoektManager._effectiveMirrorPath) {
           await this.zoektManager.syncMirror(this.zoektMirror.getMirrorRoot());
-        } else {
-          // Set effective mirror path to WSL-native dir so runIndex skips sync too
+        } else if (!this.zoektManager._effectiveMirrorPath) {
           this.zoektManager._effectiveMirrorPath = this.zoektManager.wslMirrorDir || this.zoektMirror.getMirrorRoot();
           this.zoektManager.mirrorRoot = this.zoektMirror.getMirrorRoot();
           console.log('[Startup] WSL mirror sync skipped (SKIP_SYNC=1)');
         }
+
         const started = await this.zoektManager.start();
         if (started) {
           this.zoektClient = new ZoektClient(this.zoektManager.getPort(), {

src/service/watcher.js

@@ -103,6 +103,7 @@ export class FileWatcher {
     let added = 0;
     let changed = 0;
     let deleted = 0;
+    const affectedProjects = new Set();
 
     // Phase 1: Handle deletes immediately (no I/O needed)
     const ioTasks = [];
@@ -114,9 +115,15 @@ export class FileWatcher {
 
       if (eventType === 'unlink') {
         if (language === 'content') {
-          if (this.database.deleteAsset(filePath)) deleted++;
+          if (this.database.deleteAsset(filePath)) {
+            deleted++;
+            affectedProjects.add('_assets');
+          }
         } else {
-          if (this.database.deleteFile(filePath)) deleted++;
+          if (this.database.deleteFile(filePath)) {
+            deleted++;
+            if (project) affectedProjects.add(project.name);
+          }
           // Remove from Zoekt mirror (Windows + WSL)
           if (this.zoektMirror) {
             const relativePath = this.zoektMirror._toRelativePath(filePath);
@@ -148,6 +155,12 @@ export class FileWatcher {
         this._writeToDatabase(result);
         if (result.eventType === 'add') added++;
         else changed++;
+        // Track affected project for scoped Zoekt reindexing
+        if (result.type === 'asset') {
+          affectedProjects.add('_assets');
+        } else if (result.project) {
+          affectedProjects.add(result.project.name);
+        }
       } catch (err) {
         console.error(`Error writing ${result.filePath}:`, err.message);
       }
@@ -158,9 +171,9 @@ export class FileWatcher {
       console.log(`[Watcher] +${added} ~${changed} -${deleted} (${updates.size} files) — ${ms}ms`);
       this.onUpdate({ added, changed, deleted });
 
-      // Trigger Zoekt re-indexing (adaptive debounce based on change volume)
+      // Trigger scoped Zoekt re-indexing (only affected projects)
       if (this.zoektManager) {
-        this.zoektManager.triggerReindex(added + changed + deleted);
+        this.zoektManager.triggerReindex(added + changed + deleted, affectedProjects);
       }
     }
   }

src/service/zoekt-client.js

@@ -35,10 +35,10 @@ export class ZoektClient {
   async searchAssets(pattern, options = {}) {
     const { project, caseSensitive = true, maxResults = 20 } = options;
 
-    // Asset query: only _assets/ paths, no language filter, no context lines
+    // Asset query: only _assets/ paths or repo, no language filter, no context lines
     const parts = [caseSensitive ? 'case:yes' : 'case:no'];
     if (project) parts.push(`file:${project}/`);
-    parts.push('file:^_assets/');
+    parts.push('(file:^_assets/ or repo:_assets)');
     if (this._hasRegexMeta(pattern)) {
       parts.push(`regex:${pattern}`);
     } else {
@@ -49,6 +49,8 @@ export class ZoektClient {
 
     // Map asset paths back to original content paths
     // Mirror adds .uasset extension to avoid file/directory collisions — strip it
+    // Per-project shard: file is "Game/Discovery/..." (no _assets/ prefix)
+    // Monolithic shard: file is "_assets/Game/Discovery/..." (with _assets/ prefix)
     result.results = result.results.map(r => {
       let file = '/' + r.file.replace(/^_assets\//, '');
       file = file.replace(/\.uasset$/, '');
@@ -91,8 +93,9 @@ export class ZoektClient {
       parts.push(`file:${LANGUAGE_EXTENSIONS[language]}`);
     }
     parts.push('-file:^_assets/');
+    parts.push('-repo:_assets');
     if (project) {
-      parts.push(`file:${project}/`);
+      parts.push(`(file:${project}/ or repo:${project})`);
     }
 
     return this._executeQuery(parts.join(' '), maxResults, 0);
@@ -159,15 +162,17 @@ export class ZoektClient {
       parts.push(`file:${LANGUAGE_EXTENSIONS[language]}`);
     }
 
-    // Exclude asset files from source searches
+    // Exclude asset files from source searches.
+    // Supports both monolithic shards (file path prefix) and per-project shards (repo name).
     if (excludeAssets) {
       parts.push('-file:^_assets/');
+      parts.push('-repo:_assets');
     }
 
-    // Project filter via path prefix
+    // Project filter — supports both monolithic (file:prefix) and per-project shards (repo:name).
+    // Use OR so it matches either shard type.
     if (project) {
-      // Project name appears as a path component in the mirror directory
-      parts.push(`file:${project}/`);
+      parts.push(`(file:${project}/ or repo:${project})`);
     }
 
     // The search pattern itself — use regex if it contains regex metacharacters
@@ -207,7 +212,13 @@ export class ZoektClient {
     const stats = data.Result?.Stats || data.Stats || {};
 
     for (const file of files) {
-      const filePath = file.FileName || '';
+      // Per-project shards: FileName is relative to project dir (e.g., "Script/Camera/Aim.as").
+      // Use Repository field (e.g., "Discovery") to reconstruct the full path with project prefix.
+      let filePath = file.FileName || '';
+      const repo = file.Repository || '';
+      if (repo && !filePath.startsWith(repo + '/')) {
+        filePath = repo + '/' + filePath;
+      }
       const fileProject = this._inferProject(filePath);
       const fileLanguage = this._inferLanguage(filePath);
       matchedFiles++;