Add llms.txt and llms-full.txt

src/app/layout.tsx

@@ -23,6 +23,14 @@ export const metadata: Metadata = {
     icon: '/favicon.svg',
     shortcut: '/favicon.svg',
   },
+  alternates: {
+    types: {
+      'text/plain': [
+        { url: '/llms.txt', title: 'Cloudsmith Documentation – LLM index' },
+        { url: '/llms-full.txt', title: 'Cloudsmith Documentation – LLM full content' },
+      ],
+    },
+  },
 };
 
 export default function RootLayout({ children }: { children: React.ReactNode }) {

src/app/llms-full.txt/route.ts

@@ -0,0 +1,11 @@
+import { NextResponse } from 'next/server';
+import { buildFullDocs } from '@/lib/llms';
+
+export const dynamic = 'force-static';
+
+export async function GET() {
+  const content = await buildFullDocs();
+  return new NextResponse(content, {
+    headers: { 'Content-Type': 'text/plain; charset=utf-8' },
+  });
+}

src/app/llms.txt/route.ts

@@ -0,0 +1,11 @@
+import { NextResponse } from 'next/server';
+import { buildIndex } from '@/lib/llms';
+
+export const dynamic = 'force-static';
+
+export async function GET() {
+  const content = await buildIndex();
+  return new NextResponse(content, {
+    headers: { 'Content-Type': 'text/plain; charset=utf-8' },
+  });
+}

src/lib/llms.ts

@@ -0,0 +1,170 @@
+import fs from 'fs';
+import path from 'path';
+
+import menuData from '@/content/menu.json';
+
+import { loadMdxInfo } from './markdown/util';
+
+const BASE_URL = 'https://docs.cloudsmith.com';
+
+interface MenuItem {
+  title: string;
+  path?: string;
+  children?: MenuItem[];
+}
+
+interface MenuSection {
+  title: string;
+  path?: string;
+  icon?: string;
+  children?: MenuItem[];
+}
+
+// Strip trailing slash so URLs are consistent
+function cleanPath(p: string): string {
+  return p.endsWith('/') ? p.slice(0, -1) : p;
+}
+
+// Recursively collects all internal page paths from a menu subtree
+function collectMenuPaths(items: MenuItem[], paths: Set<string>): void {
+  for (const item of items) {
+    if (item.path && !item.path.startsWith('http')) {
+      const slug = cleanPath(item.path).slice(1);
+      if (slug) paths.add(slug);
+    }
+    if (item.children) collectMenuPaths(item.children, paths);
+  }
+}
+
+// Recursively renders menu items as indented markdown links
+function renderItems(items: MenuItem[], depth: number): string[] {
+  const indent = '  '.repeat(depth);
+  const lines: string[] = [];
+  for (const item of items) {
+    if (!item.path || item.path.startsWith('http')) {
+      // Group label with no path — render children at same depth
+      if (item.children) lines.push(...renderItems(item.children, depth));
+      continue;
+    }
+    const url = `${BASE_URL}${cleanPath(item.path)}`;
+    lines.push(`${indent}- [${item.title}](${url})`);
+    if (item.children) lines.push(...renderItems(item.children, depth + 1));
+  }
+  return lines;
+}
+
+// Builds the llms.txt index: a compact nav tree of all doc sections and pages
+export async function buildIndex(): Promise<string> {
+  const menu = menuData as Record<string, MenuSection>;
+
+  const lines: string[] = [
+    '# Cloudsmith Documentation',
+    '',
+    `> The universal artifact registry for secure software distribution and dependency management. Full content dump: ${BASE_URL}/llms-full.txt`,
+    '',
+    `[Cloudsmith Documentation](${BASE_URL}): The universal artifact registry for secure software distribution and dependency management.`,
+    '',
+  ];
+
+  for (const [, section] of Object.entries(menu)) {
+    // Skip non-content sections (e.g. mobileNavbar) — they have no internal path
+    if (!section.path?.startsWith('/') || !section.children) continue;
+
+    const sectionUrl = `${BASE_URL}${cleanPath(section.path)}`;
+    lines.push(`- [${section.title}](${sectionUrl})`);
+    lines.push(...renderItems(section.children, 1));
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+// Strips MDX-specific syntax (imports, JSX components) to leave plain markdown
+function stripMdx(content: string): string {
+  let result = content;
+
+  // Remove import statements
+  result = result.replace(/^import\s+[^\n]+\n?/gm, '');
+
+  // Extract prose from props before tags are stripped
+
+  // Note: headline/heading prop becomes a bold label before the note content
+  result = result.replace(/<Note[^>]*\bheadline="([^"]+)"[^>]*>/g, '\n**$1**\n');
+  result = result.replace(/<Note[^>]*\bheading="([^"]+)"[^>]*>/g, '\n**$1**\n');
+
+  // Card: self-closing, so props are the only content — render as "**[title](href)**: description"
+  result = result.replace(/<Card([^>]*\/>)/g, (_, props) => {
+    const title = props.match(/\btitle="([^"]+)"/)?.[1];
+    const description = props.match(/\bdescription="([^"]+)"/)?.[1];
+    const href = props.match(/\bhref="([^"]+)"/)?.[1];
+    if (!title && !description) return '';
+    const titlePart = title && href ? `**[${title}](${href})**` : title ? `**${title}**` : '';
+    return '\n' + [titlePart, description].filter(Boolean).join(': ') + '\n';
+  });
+
+  // BlockImage: render alt text as a plain image description
+  result = result.replace(/<BlockImage[^>]*\balt="([^"]+)"[^>]*\/?>/g, '\n[Image: $1]\n');
+
+  // Remove remaining self-closing JSX components (no extractable prose)
+  result = result.replace(/<[A-Z][a-zA-Z]*[^>]*\/>/g, '');
+
+  // Strip JSX block tags but keep their inner content
+  result = result.replace(/<[A-Z][a-zA-Z]*[^>]*>/g, '');
+  result = result.replace(/<\/[A-Z][a-zA-Z]*>/g, '');
+
+  // Remove common HTML-in-MDX elements
+  result = result.replace(/<br\s*\/?>/gi, '');
+
+  return result.replace(/\n{3,}/g, '\n\n').trim();
+}
+
+// Builds the llms-full.txt dump: full MDX content for every page present in the menu
+export async function buildFullDocs(): Promise<string> {
+  const KNOWN_SECTIONS = ['documentation', 'guides', 'api'] as const;
+  type SectionKey = (typeof KNOWN_SECTIONS)[number];
+
+  const menu = menuData as Record<string, MenuSection>;
+  const sections = Object.entries(menu)
+    .filter(([key, s]) => s.path?.startsWith('/') && (KNOWN_SECTIONS as readonly string[]).includes(key))
+    .map(([key, s]) => ({ key: key as SectionKey, label: s.title }));
+
+  const menuPaths = new Set<string>();
+  for (const [, section] of Object.entries(menu)) {
+    if (section.path?.startsWith('/')) menuPaths.add(cleanPath(section.path).slice(1));
+    if (section.children) collectMenuPaths(section.children, menuPaths);
+  }
+
+  const parts: string[] = [
+    '# Cloudsmith Documentation',
+    '',
+    '> The universal artifact registry for secure software distribution and dependency management.',
+    '',
+    'Cloudsmith docs cover repositories, package formats, integrations, authentication, CI/CD workflows, and more.',
+    '',
+  ];
+
+  for (const section of sections) {
+    const prefix = section.key !== 'documentation' ? `${section.key}/` : '';
+    const files = (await loadMdxInfo(section.key)).filter((info) => menuPaths.has(cleanPath(`${prefix}${info.slug}`)));
+
+    parts.push('---');
+    parts.push('');
+    parts.push(`# ${section.label}`);
+    parts.push('');
+
+    for (const info of files) {
+      const filePath = path.join(process.cwd(), 'src/content', info.file);
+      const raw = fs.readFileSync(filePath, 'utf-8');
+      const cleaned = stripMdx(raw);
+
+      if (!cleaned) continue;
+
+      parts.push(`<!-- ${BASE_URL}/${cleanPath(`${prefix}${info.slug}`)} -->`);
+      parts.push('');
+      parts.push(cleaned);
+      parts.push('');
+    }
+  }
+
+  return parts.join('\n');
+}