Skip to content

Commit 33be413

Browse files
amandeepmittalamandeepmittal
authored
[docs] Refactor llms-*.txt scripts (expo#43007)
1 parent 205ab6d commit 33be413

9 files changed

Lines changed: 353 additions & 813 deletions

File tree

‎docs/scripts/generate-llms/compileTalks.js‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎docs/scripts/generate-llms/index.js‎

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,13 @@ import fs from 'node:fs';
22
import path from 'node:path';
33
import process from 'node:process';
44

5-
import { compileTalksFile } from './compileTalks.js';
65
import { generateLlmsEasTxt } from './llms-eas-txt.js';
76
import { generateLlmsFullTxt } from './llms-full-txt.js';
87
import { generateLlmsSdkTxt } from './llms-sdk.js';
98
import { generateLlmsTxt } from './llms-txt.js';
10-
import {
11-
OUTPUT_DIRECTORY_NAME,
12-
OUTPUT_FILENAME_EAS_DOCS,
13-
OUTPUT_FILENAME_EXPO_DOCS,
14-
OUTPUT_FILENAME_LLMS_TXT,
15-
} from './utils.js';
16-
17-
const GENERATED_LLMS_FILES = [
18-
OUTPUT_FILENAME_LLMS_TXT,
19-
OUTPUT_FILENAME_EXPO_DOCS,
20-
OUTPUT_FILENAME_EAS_DOCS,
21-
'llms-sdk.txt',
22-
];
9+
import { OUTPUT_DIRECTORY_NAME } from './shared.js';
10+
11+
const GENERATED_LLMS_FILES = ['llms.txt', 'llms-full.txt', 'llms-eas.txt', 'llms-sdk.txt'];
2312

2413
async function syncGeneratedLlmsToOut() {
2514
const outDir = path.join(process.cwd(), 'out');
@@ -42,8 +31,6 @@ async function syncGeneratedLlmsToOut() {
4231
);
4332
}
4433

45-
await compileTalksFile();
46-
4734
const results = await Promise.allSettled([
4835
generateLlmsSdkTxt(),
4936
generateLlmsEasTxt(),

‎docs/scripts/generate-llms/llms-eas-txt.js‎

Lines changed: 21 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1,90 +1,39 @@
11
import fs from 'node:fs';
2-
import { createRequire } from 'node:module';
32
import path from 'node:path';
4-
import { fileURLToPath } from 'node:url';
53

64
import {
75
OUTPUT_DIRECTORY_NAME,
8-
OUTPUT_FILENAME_EAS_DOCS,
9-
TITLE_EAS,
10-
DESCRIPTION_EAS,
11-
generateSectionMarkdown,
12-
processSection,
13-
} from './utils.js';
6+
collectPageHrefs,
7+
composeMarkdownDocument,
8+
ensureBuildOutputDir,
9+
getMarkdownPathFromHref,
10+
readUniqueMarkdownContent,
11+
uniqueInternalHrefs,
12+
} from './shared.js';
1413
import { eas } from '../../constants/navigation.js';
1514

16-
const __filename = fileURLToPath(import.meta.url);
17-
const require = createRequire(__filename);
18-
const easCliData = require('../../ui/components/EASCLIReference/data/eas-cli-commands.json');
15+
const OUTPUT_FILENAME_EAS_DOCS = 'llms-eas.txt';
16+
const TITLE_EAS = 'Expo Application Services (EAS) Documentation';
17+
const DESCRIPTION_EAS =
18+
'Expo Application Services (EAS) are deeply integrated cloud services for Expo and React Native apps, from the team behind Expo.';
1919

20-
function generateFullMarkdown({ title, description, sections }) {
21-
return `# ${title}\n\n${description}\n\n` + sections.map(generateSectionMarkdown).join('');
22-
}
23-
24-
function formatEasCliDescription(description) {
25-
if (!description) {
26-
return '';
27-
}
28-
const normalized = description.trim();
29-
if (!normalized) {
30-
return '';
31-
}
32-
const prefixed = /^[A-Z]/.test(normalized)
33-
? normalized
34-
: normalized[0].toUpperCase() + normalized.slice(1);
35-
return prefixed.endsWith('.') ? prefixed : `${prefixed}.`;
36-
}
37-
38-
function generateEasCliCommandsMarkdown(data) {
39-
if (!data?.commands?.length) {
40-
return '';
41-
}
20+
function generateFullMarkdown({ title, description }) {
21+
const buildDir = ensureBuildOutputDir();
22+
const allHrefs = uniqueInternalHrefs(collectPageHrefs(eas));
23+
const markdownPaths = allHrefs.map(href => getMarkdownPathFromHref(buildDir, href));
24+
const contentChunks = readUniqueMarkdownContent(markdownPaths, { warnOnMissing: true });
4225

43-
const cliVersion = data?.source?.cliVersion;
44-
const header = cliVersion ? `# EAS CLI commands (v${cliVersion})` : '# EAS CLI commands';
45-
46-
const commands = data.commands
47-
.map(command => {
48-
const description = formatEasCliDescription(command.description);
49-
const usage = command.usage?.trim();
50-
51-
let content = `## ${command.command}\n\n`;
52-
if (description) {
53-
content += `${description}\n\n`;
54-
}
55-
if (usage) {
56-
content += '```\n' + usage + '\n```\n\n';
57-
}
58-
return content;
59-
})
60-
.join('');
61-
62-
return `${header}\n\n${commands}`.trimEnd();
26+
return composeMarkdownDocument({ title, description, contentChunks });
6327
}
6428

6529
export async function generateLlmsEasTxt() {
6630
try {
67-
const sections = eas.map(section => ({ ...processSection(section), category: 'Eas' }));
68-
69-
const easCliMarkdown = generateEasCliCommandsMarkdown(easCliData);
70-
const baseContent = generateFullMarkdown({
71-
title: TITLE_EAS,
72-
description: DESCRIPTION_EAS,
73-
sections,
74-
});
75-
76-
const placeholderRegex = /<EASCLIReference\s*\/>/g;
77-
let finalContent = easCliMarkdown
78-
? baseContent.replace(placeholderRegex, `\n${easCliMarkdown}\n`)
79-
: baseContent.replace(placeholderRegex, '');
80-
81-
if (easCliMarkdown && finalContent === baseContent) {
82-
finalContent = `${baseContent.replace(placeholderRegex, '')}\n\n${easCliMarkdown}`;
83-
}
84-
8531
await fs.promises.writeFile(
8632
path.join(process.cwd(), OUTPUT_DIRECTORY_NAME, OUTPUT_FILENAME_EAS_DOCS),
87-
finalContent
33+
generateFullMarkdown({
34+
title: TITLE_EAS,
35+
description: DESCRIPTION_EAS,
36+
})
8837
);
8938

9039
console.log(` \x1b[1m\x1b[32m✓\x1b[0m Successfully generated ${OUTPUT_FILENAME_EAS_DOCS}`);

‎docs/scripts/generate-llms/llms-full-txt.js‎

Lines changed: 17 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -3,95 +3,36 @@ import path from 'node:path';
33

44
import {
55
OUTPUT_DIRECTORY_NAME,
6-
OUTPUT_FILENAME_EXPO_DOCS,
7-
TITLE,
8-
DESCRIPTION,
9-
generateSectionMarkdown,
10-
processSection,
11-
readFrontmatter,
12-
} from './utils.js';
6+
collectPageHrefs,
7+
composeMarkdownDocument,
8+
ensureBuildOutputDir,
9+
getMarkdownPathFromHref,
10+
readUniqueMarkdownContent,
11+
uniqueInternalHrefs,
12+
} from './shared.js';
1313
import { home, learn, general } from '../../constants/navigation.js';
1414

15-
function readInstructions() {
16-
const instructionsDir = path.join(
17-
process.cwd(),
18-
'scenes/get-started/set-up-your-environment/instructions'
19-
);
15+
const OUTPUT_FILENAME_EXPO_DOCS = 'llms-full.txt';
16+
const TITLE = 'Expo Documentation';
17+
const DESCRIPTION =
18+
'Expo is an open-source React Native framework for apps that run natively on Android, iOS, and the web. Expo brings together the best of mobile and the web and enables many important features for building and scaling an app such as live updates, instantly sharing your app, and web support. The company behind Expo also offers Expo Application Services (EAS), which are deeply integrated cloud services for Expo and React Native apps.';
2019

21-
const headerMapping = {
22-
_androidEmulatorInstructions: 'Android Emulator Setup',
23-
_androidStudioEnvironmentInstructions: 'Android Studio Environment Setup',
24-
_androidStudioInstructions: 'Android Studio Setup',
25-
_xcodeInstructions: 'Xcode Setup',
26-
androidPhysicalDevelopmentBuild:
27-
'Create a development build for a physical Android device with EAS',
28-
androidPhysicalDevelopmentBuildLocal:
29-
'Create a development build for a physical Android device locally',
30-
androidPhysicalExpoGo: 'Run on a physical Android device with Expo Go',
31-
androidSimulatedDevelopmentBuild: 'Create a development build for Android Emulator with EAS',
32-
androidSimulatedDevelopmentBuildLocal:
33-
'Create a development build for Android Emulator locally',
34-
androidSimulatedExpoGo: 'Run on Android Emulator with Expo Go',
35-
iosPhysicalDevelopmentBuild: 'Create a development build for a physical iOS device with EAS',
36-
iosPhysicalDevelopmentBuildLocal:
37-
'Create a development build for a physical iOS device locally',
38-
iosPhysicalExpoGo: 'Run on a physical iOS device with Expo Go',
39-
iosSimulatedDevelopmentBuild: 'Create a development build for iOS Simulator with EAS',
40-
iosSimulatedDevelopmentBuildLocal: 'Create a development build for iOS Simulator locally',
41-
iosSimulatedExpoGo: 'Run on iOS Simulator with Expo Go',
42-
};
20+
function generateFullMarkdown({ title, description }) {
21+
const buildDir = ensureBuildOutputDir();
22+
const allHrefs = uniqueInternalHrefs(collectPageHrefs([...home, ...learn, ...general]));
23+
const markdownPaths = allHrefs.map(href => getMarkdownPathFromHref(buildDir, href));
24+
const contentChunks = readUniqueMarkdownContent(markdownPaths, { warnOnMissing: true });
4325

44-
const instructionFiles = Object.keys(headerMapping);
45-
46-
return instructionFiles
47-
.map(filename => {
48-
const filePath = path.join(instructionsDir, `${filename}.mdx`);
49-
if (fs.existsSync(filePath)) {
50-
const { title, content } = readFrontmatter(filePath);
51-
const header = headerMapping[filename] || title || filename;
52-
return `# ${header}\n\n${content}\n\n`;
53-
}
54-
return '';
55-
})
56-
.join('');
57-
}
58-
59-
function processEnvironmentPage(section) {
60-
if (section.items) {
61-
section.items = section.items.map(item => {
62-
if (item.url && item.url.includes('set-up-your-environment')) {
63-
const instructions = readInstructions();
64-
item.content = item.content.replace('<DevelopmentEnvironmentInstructions />', instructions);
65-
}
66-
return item;
67-
});
68-
}
69-
70-
if (section.sections) {
71-
section.sections = section.sections.map(processEnvironmentPage);
72-
}
73-
74-
return section;
75-
}
76-
77-
function generateFullMarkdown({ title, description, sections }) {
78-
return `# ${title}\n\n${description}\n\n` + sections.map(generateSectionMarkdown).join('');
26+
return composeMarkdownDocument({ title, description, contentChunks });
7927
}
8028

8129
export async function generateLlmsFullTxt() {
8230
try {
83-
const sections = [
84-
...home.map(section => ({ ...processSection(section), category: 'Home' })),
85-
...learn.map(section => ({ ...processSection(section), category: 'Learn' })),
86-
...general.map(section => ({ ...processSection(section), category: 'General' })),
87-
].map(processEnvironmentPage);
88-
8931
await fs.promises.writeFile(
9032
path.join(process.cwd(), OUTPUT_DIRECTORY_NAME, OUTPUT_FILENAME_EXPO_DOCS),
9133
generateFullMarkdown({
9234
title: TITLE,
9335
description: DESCRIPTION,
94-
sections,
9536
})
9637
);
9738

‎docs/scripts/generate-llms/llms-sdk.js‎

Lines changed: 11 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,22 @@
11
import fs from 'node:fs';
22
import path from 'node:path';
33

4-
const OUTPUT_DIRECTORY_NAME = 'public';
4+
import {
5+
OUTPUT_DIRECTORY_NAME,
6+
composeMarkdownDocument,
7+
ensureBuildOutputDir,
8+
findMarkdownFiles,
9+
readUniqueMarkdownContent,
10+
} from './shared.js';
11+
512
const OUTPUT_FILENAME = 'llms-sdk.txt';
613
const TITLE = 'Expo SDK Documentation';
714
const DESCRIPTION =
815
'Documentation for Expo SDK libraries, app configuration files, Expo CLI, create-expo-app, and more.';
9-
const BUILD_OUTPUT_DIR = 'out';
1016
const PATHS_TO_INCLUDE = ['versions/latest/', 'more/', 'technical-specs/'];
1117

12-
function findMarkdownFiles(dir) {
13-
const results = [];
14-
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
15-
const fullPath = path.join(dir, entry.name);
16-
if (entry.isDirectory()) {
17-
results.push(...findMarkdownFiles(fullPath));
18-
} else if (entry.name === 'index.md') {
19-
results.push(fullPath);
20-
}
21-
}
22-
return results;
23-
}
24-
2518
function generateFullMarkdown() {
26-
let fullContent = `# ${TITLE}\n\n${DESCRIPTION}\n\n`;
27-
28-
const buildDir = path.join(process.cwd(), BUILD_OUTPUT_DIR);
29-
if (!fs.existsSync(buildDir)) {
30-
throw new Error(`Build output directory not found: ${buildDir}. Run "next build" first.`);
31-
}
32-
19+
const buildDir = ensureBuildOutputDir();
3320
const allFiles = findMarkdownFiles(buildDir);
3421

3522
const matchingFiles = allFiles
@@ -42,25 +29,9 @@ function generateFullMarkdown() {
4229
const relB = path.relative(buildDir, b);
4330
return relA.localeCompare(relB);
4431
});
32+
const contentChunks = readUniqueMarkdownContent(matchingFiles);
4533

46-
const processedContent = new Set();
47-
48-
for (const filePath of matchingFiles) {
49-
const content = fs.readFileSync(filePath, 'utf8').trim();
50-
if (!content) {
51-
continue;
52-
}
53-
54-
const normalizedContent = content.toLowerCase();
55-
if (processedContent.has(normalizedContent)) {
56-
continue;
57-
}
58-
59-
processedContent.add(normalizedContent);
60-
fullContent += content + '\n\n---\n\n';
61-
}
62-
63-
return fullContent;
34+
return composeMarkdownDocument({ title: TITLE, description: DESCRIPTION, contentChunks });
6435
}
6536

6637
export async function generateLlmsSdkTxt() {

0 commit comments

Comments
 (0)