feat: new build script

Author: ghostdevv

package.json

@@ -5,6 +5,7 @@
         "start:discord": "pnpm --filter discord-qa -r start",
         "start:api": "pnpm --filter api -r start",
         "dev:api": "pnpm --filter api -r dev",
-        "generate:discord-index": "python3 scripts/generate_discord_index.py > search-index/discord_theads_store.json"
+        "generate:discord-index": "python3 scripts/generate_discord_index.py > search-index/discord_theads_store.json",
+        "build:index": "pnpm --filter docs-qa -r build"
     }
 }

packages/qa/package.json

@@ -2,6 +2,9 @@
     "name": "@gitpod/docs-qa",
     "private": true,
     "main": "./src/index.ts",
+    "scripts": {
+        "build": "tsm src/build.ts"
+    },
     "devDependencies": {
         "@types/marked": "^5.0.0",
         "@types/node": "^20.2.1",

packages/qa/src/build.ts

@@ -0,0 +1,73 @@
+// import { UnstructuredLoader } from 'langchain/document_loaders/fs/unstructured';
+import { OpenAIEmbeddings } from 'langchain/embeddings/openai';
+import { MarkdownTextSplitter } from 'langchain/text_splitter';
+import { HNSWLib } from 'langchain/vectorstores/hnswlib';
+import { TEMP_PATH, SEARCH_PATH } from './paths';
+import { Document } from 'langchain/document';
+import { simpleGit } from 'simple-git';
+import { readFile } from 'fs/promises';
+import { totalist } from 'totalist';
+import { existsSync } from 'fs';
+
+const repoPath = `${TEMP_PATH}/gitpod`;
+
+if (!existsSync(repoPath)) {
+    console.log('Cloning Gitpod Repo');
+
+    await simpleGit().clone('https://github.com/gitpod-io/website', repoPath, {
+        '--depth': 1,
+    });
+}
+
+console.log('Finding MD files');
+
+const paths: { path: string; url: string }[] = [];
+
+await totalist(repoPath, (relative, path) => {
+    if (!relative.endsWith('.md') || !relative.startsWith('src/routes')) return;
+
+    const pathname = relative
+        .replace('src/routes', '')
+        .slice(0, -3)
+        .replace(/\/index$/gm, '/')
+        .replace(/\/$/gm, '');
+
+    paths.push({
+        url: `https://www.gitpod.io${pathname}`,
+        path,
+    });
+});
+
+console.log('Getting documents');
+
+const documents: Document[] = [];
+
+for (const { path, url } of paths) {
+    // const loader = new UnstructuredLoader(path, {
+    //     // Switch to this if you are getting ratelimits, needs unstructured running locally
+    //     // https://js.langchain.com/docs/modules/indexes/document_loaders/examples/file_loaders/unstructured#setup
+    //     apiUrl: 'http://127.0.0.1:8000/general/v0/general',
+    // });
+
+    // const docs = await loader.loadAndSplit(new MarkdownTextSplitter());
+
+    // for (const doc of docs) {
+    //     doc.metadata.source = url;
+    // }
+
+    const content = await readFile(path, 'utf-8');
+    const splitter = new MarkdownTextSplitter();
+
+    const docs = await splitter.createDocuments([content], [{ source: url }]);
+
+    documents.push(...docs);
+}
+
+console.log(`Found ${documents.length} documents`);
+
+console.log('Creating Store');
+
+const store = await HNSWLib.fromDocuments(documents, new OpenAIEmbeddings());
+store.save(SEARCH_PATH);
+
+console.log('Done');