feat(util): introduce uniq() utility and standardize array dedup across codebase

Centralizes array dedup using a new uniq() function to replace scattered Set-based patterns and prepare groundwork for removing lodash replacements.

🤖 Generated with Codebuff
Co-Authored-By: Codebuff <noreply@codebuff.com>

Author: brandonkachen

common/src/util/lodash-replacements.ts

@@ -54,11 +54,11 @@ export function union<T>(arr1: T[], arr2: T[]): T[] {
 export function partition<T, S extends T>(
   array: T[],
   predicate: (value: T) => value is S,
-): [S[], Exclude<T, S>[]];
+): [S[], Exclude<T, S>[]]
 export function partition<T>(
   array: T[],
   predicate: (value: T) => boolean,
-): [T[], T[]];
+): [T[], T[]]
 export function partition<T>(
   array: T[],
   predicate: (value: T) => boolean,
@@ -74,3 +74,8 @@ export function partition<T>(
   }
   return [truthy, falsy]
 }
+
+// Remove duplicates from an array
+export function uniq<T>(arr: T[]): T[] {
+  return Array.from(new Set(arr))
+}

packages/agent-runtime/src/find-files/request-files-prompt.ts

@@ -6,7 +6,7 @@ import {
   type FinetunedVertexModel,
 } from '@codebuff/common/old-constants'
 import { getAllFilePaths } from '@codebuff/common/project-file-tree'
-import { range, shuffle } from '@codebuff/common/util/lodash-replacements'
+import { range, shuffle, uniq } from '@codebuff/common/util/lodash-replacements'
 
 import { promptFlashWithFallbacks } from '../llm-api/gemini-with-fallbacks'
 import {
@@ -82,7 +82,7 @@ export async function requestRelevantFiles(
 
   const candidateFiles = (await keyPromise).files
 
-  validateFilePaths(Array.from(new Set(candidateFiles)))
+  validateFilePaths(uniq(candidateFiles))
 
   // logger.info(
   //   {
@@ -149,7 +149,7 @@ export async function requestRelevantFilesForTraining(
   })
 
   const candidateFiles = [...keyFiles.files, ...nonObviousFiles.files]
-  const validatedFiles = validateFilePaths(Array.from(new Set(candidateFiles)))
+  const validatedFiles = validateFilePaths(uniq(candidateFiles))
   logger.debug(
     { keyFiles, nonObviousFiles, validatedFiles },
     'requestRelevantFilesForTraining: results',
@@ -322,10 +322,7 @@ function getExampleFileList(params: {
     selectedDirectories.add(dirname(filePath))
   }
 
-  return Array.from(new Set([...selectedFiles, ...randomFilePaths])).slice(
-    0,
-    count,
-  )
+  return uniq([...selectedFiles, ...randomFilePaths]).slice(0, count)
 }
 
 function generateNonObviousRequestFilesPrompt(

packages/agent-runtime/src/get-file-reading-updates.ts

@@ -1,4 +1,5 @@
 import type { RequestFilesFn } from '@codebuff/common/types/contracts/client'
+import { uniq } from '@codebuff/common/util/lodash-replacements'
 
 export async function getFileReadingUpdates(params: {
   requestFiles: RequestFilesFn
@@ -11,7 +12,7 @@ export async function getFileReadingUpdates(params: {
 > {
   const { requestFiles, requestedFiles } = params
 
-  const allFilePaths = Array.from(new Set(requestedFiles))
+  const allFilePaths = uniq(requestedFiles)
   const loadedFiles = await requestFiles({ filePaths: allFilePaths })
 
   const addedFiles = allFilePaths

packages/code-map/src/parse.ts

@@ -1,6 +1,8 @@
 import * as fs from 'fs'
 import * as path from 'path'
 
+import { uniq } from '@codebuff/common/util/lodash-replacements'
+
 import { getLanguageConfig, LanguageConfig } from './languages'
 import type { Parser, Query } from 'web-tree-sitter'
 
@@ -172,8 +174,8 @@ export function parseTokens(
       throw new Error('Parser or query not found')
     }
     const parseResults = parseFile(parser, query, sourceCode)
-    const identifiers = Array.from(new Set(parseResults.identifier))
-    const calls = Array.from(new Set(parseResults['call.identifier']))
+    const identifiers = uniq(parseResults.identifier)
+    const calls = uniq(parseResults['call.identifier'])
 
     if (DEBUG_PARSING) {
       console.log(`\nParsing ${filePath}:`)

scripts/generate-ci-env.js

@@ -7,6 +7,7 @@
 import fs from 'fs'
 import path from 'path'
 import { fileURLToPath } from 'url'
+import { uniq } from '../common/src/util/lodash-replacements.ts'
 
 const __filename = fileURLToPath(import.meta.url)
 const __dirname = path.dirname(__filename)
@@ -88,7 +89,7 @@ function generateGitHubEnv() {
   } else if (scope === 'client') {
     selected = varsByScope.client
   } else {
-    selected = Array.from(new Set([...varsByScope.server, ...varsByScope.client]))
+    selected = uniq([...varsByScope.server, ...varsByScope.client])
   }
 
   if (prefix) {

web/src/lib/docs.ts

@@ -34,5 +34,5 @@ export async function getNewsArticles(): Promise<NewsArticle[]> {
 
 // export function getAllCategories() {
 //   if (!allDocs) return []
-//   return Array.from(new Set((allDocs as Doc[]).map((doc: Doc) => doc.category)))
+//   return uniq((allDocs as Doc[]).map((doc: Doc) => doc.category))
 // }