Add robots meta tags and implement robots.txt handling in worker

Leon-JavaScript · Leon-JavaScript · commit c6d206b833a9 · 2026-03-29T09:36:31.000+02:00
diff --git a/pages/defaultPage.html b/pages/defaultPage.html
@@ -3,6 +3,8 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta name="robots" content="noindex, nofollow, noarchive, nosnippet, noimageindex, notranslate" />
+    <meta name="googlebot" content="noindex, nofollow, noarchive, nosnippet, noimageindex, notranslate" />
     <title>MineTools API</title>
     <style>/*__INLINE_CSS__*/</style>
   </head>
diff --git a/worker.js b/worker.js
@@ -4,11 +4,33 @@ import { handleProfileRoute } from "./routes/profileRoute";
 import { handleUuidRoute } from "./routes/uuidRoute";
 import { internalError, notFound } from "./utils/responses";
 
+const ROBOTS_POLICY = "noindex, nofollow, noarchive, nosnippet, noimageindex, notranslate";
+
 const ROUTE_HANDLERS = {
   uuid: handleUuidRoute,
   profile: handleProfileRoute
 };
 
+function withRobotsBlocked(response) {
+  const headers = new Headers(response.headers);
+  headers.set("X-Robots-Tag", ROBOTS_POLICY);
+
+  return new Response(response.body, {
+    status: response.status,
+    statusText: response.statusText,
+    headers
+  });
+}
+
+function robotsTxtResponse() {
+  return new Response("User-agent: *\nDisallow: /\n", {
+    headers: {
+      "content-type": "text/plain; charset=utf-8",
+      "cache-control": "public, max-age=3600"
+    }
+  });
+}
+
 function resolveRoute(pathname) {
   if (!pathname || pathname === "/") {
     return null;
@@ -39,24 +61,28 @@ export default {
   async fetch(request, env) {
     try {
       const pathname = new URL(request.url).pathname;
+      if (pathname === "/robots.txt") {
+        return withRobotsBlocked(robotsTxtResponse());
+      }
+
       if (pathname === "/") {
-        return serveDefaultPage(request);
+        return withRobotsBlocked(serveDefaultPage(request));
       }
 
       const route = resolveRoute(pathname);
       if (!route) {
-        return notFound("Route not found");
+        return withRobotsBlocked(notFound("Route not found"));
       }
 
       const rateLimitedResponse = await enforceRateLimit(request, env);
       if (rateLimitedResponse) {
-        return rateLimitedResponse;
+        return withRobotsBlocked(rateLimitedResponse);
       }
 
-      return route.handler(route.identifier, env);
+      return withRobotsBlocked(await route.handler(route.identifier, env));
     } catch (error) {
       console.error("Unhandled worker error", error);
-      return internalError();
+      return withRobotsBlocked(internalError());
     }
   }
 };
