fix(seo): block non-indexable routes from crawlers

Author: fullstackjam

src/hooks.server.ts

@@ -160,5 +160,24 @@ export const handle: Handle = async ({ event, resolve }) => {
 	}
 
 	const response = await resolve(event);
-	return withSecurityHeaders(response);
+	const securedResponse = withSecurityHeaders(response);
+
+	// Prevent indexing of non-content routes
+	const noindexPrefixes = ['/api/', '/dashboard/', '/cli-auth/', '/install'];
+	const noindexSuffixes = ['/install', '/config', '/og'];
+	const shouldNoindex =
+		noindexPrefixes.some((p) => path.startsWith(p)) ||
+		noindexSuffixes.some((s) => path.endsWith(s));
+
+	if (shouldNoindex) {
+		const headers = new Headers(securedResponse.headers);
+		headers.set('X-Robots-Tag', 'noindex');
+		return new Response(securedResponse.body, {
+			status: securedResponse.status,
+			statusText: securedResponse.statusText,
+			headers
+		});
+	}
+
+	return securedResponse;
 };

static/robots.txt

@@ -1,6 +1,11 @@
-# allow crawling everything by default
+# allow crawling important pages
 User-agent: *
-Disallow:
+Disallow: /api/
+Disallow: /dashboard/
+Disallow: /cli-auth/
+Disallow: */install$
+Disallow: */config$
+Disallow: */og$
 
 # explicitly allow AI search crawlers
 User-agent: GPTBot