Add googlebot IP passthru [minor] (#71)

Author: joecorall

CLAUDE.md

@@ -62,9 +62,10 @@ services:
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.captchaProvider: turnstile
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.siteKey: ${TURNSTILE_SITE_KEY}
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.secretKey: ${TURNSTILE_SECRET_KEY}
-            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.goodBots: apple.com,archive.org,commoncrawl.org,duckduckgo.com,facebook.com,google.com,googlebot.com,googleusercontent.com,instagram.com,kagibot.org,linkedin.com,msn.com,openalex.org,twitter.com,x.com
+            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.goodBots: apple.com,archive.org,commoncrawl.org,duckduckgo.com,facebook.com,google.com,instagram.com,kagibot.org,linkedin.com,msn.com,openalex.org,twitter.com,x.com
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.persistentStateFile: /tmp/state.json
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableStateReconciliation: "false"
+            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableGooglebotIPCheck: "true"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.periodSeconds: 30
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.failureThreshold: 3
         networks:
@@ -82,7 +83,7 @@ services:
             --providers.docker=true
             --providers.docker.network=default
             --experimental.plugins.captcha-protect.modulename=github.com/libops/captcha-protect
-            --experimental.plugins.captcha-protect.version=v1.11.1
+            --experimental.plugins.captcha-protect.version=v1.12.0
         volumes:
             - /var/run/docker.sock:/var/run/docker.sock:z
             - /CHANGEME/TO/A/HOST/PATH/FOR/STATE/FILE:/tmp/state.json:rw
@@ -117,6 +118,7 @@ services:
 | `ipForwardedHeader`     | `string`                | `""`                     | Header to check for the original client IP if Traefik is behind a load balancer.                                                                                                                 |
 | `ipDepth`               | `int`                   | `0`                      | How deep past the last non-exempt IP to fetch the real IP from `ipForwardedHeader`. Default 0 returns the last IP in the forward header                                                          |
 | `goodBots`              | `[]string` (encouraged) | *see below*              | List of second-level domains for bots that are never challenged or rate-limited.                                                                                                                 |
+| `enableGooglebotIPCheck`| `string`.               | `"false"`                | Treat IPs coming from googlebot's known IP ranges as good bots                                                                                                                                   |
 | `protectParameters`     | `string`                | `"false"`                | Forces rate limiting even for good bots if URL parameters are present. Useful for protecting faceted search pages.                                                                               |
 | `protectFileExtensions` | `[]string`              | `""`                     | Comma-separated file extensions to protect. By default, your protected routes only protect html files. This is to prevent files like CSS/JS/img from tripping the rate limit.                    |
 | `protectHttpMethods`    | `[]string`              | `"GET,HEAD"`             | Comma-separated list of HTTP methods to protect against                                                                                                                                          |
@@ -152,14 +154,17 @@ The circuit breaker provides automatic failover when the primary captcha provide
 
 ### Good Bots
 
-To avoid having this middleware impact your SEO score, it's recommended to provide a value for `goodBots`. By default, no bots will be allowed to crawl your protected routes beyond the rate limit unless their second level domain (e.g. `google.com`) is configured as a good bot.
+To avoid having this middleware impact your SEO score, it's recommended to provide a value for `goodBots`. By default, no bots will be allowed to crawl your protected routes beyond the rate limit unless their second level domain (e.g. `bing.com`) is configured as a good bot.
 
 A good default value for `goodBots` would be:
 
 ```
-goodBots: apple.com,archive.org,duckduckgo.com,facebook.com,google.com,googlebot.com,googleusercontent.com,instagram.com,kagibot.org,linkedin.com,msn.com,openalex.org,twitter.com,x.com
+enableGooglebotIPCheck: "true"
+goodBots: apple.com,archive.org,duckduckgo.com,facebook.com,google.com,instagram.com,kagibot.org,linkedin.com,msn.com,openalex.org,twitter.com,x.com
 ```
 
+Since google publishes their bot IPs, we can also leverage their API to let google crawl the site unchallenged based on client IP. This can be enabled with `enableGooglebotIPCheck: "true"`
+
 **However** if you set the config parameter `protectParameters="true"`, even good bots won't be allowed to crawl protected routes if a URL parameter is on the request (e.g. `/foo?bar=baz`). This `protectParameters` feature is meant to help protect faceted search pages.
 
 

README.md

@@ -18,7 +18,9 @@ services:
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableStatsPage: "true"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.ipForwardedHeader: "X-Forwarded-For"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.logLevel: "DEBUG"
+            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.protectParameters: "${PROTECT_PARAMETERS:-false}"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.goodBots: ""
+            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableGooglebotIPCheck: "true"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.protectRoutes: "/"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.persistentStateFile: "/tmp/state.json"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableStateReconciliation: "true"
@@ -47,7 +49,9 @@ services:
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableStatsPage: "true"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.ipForwardedHeader: "X-Forwarded-For"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.logLevel: "DEBUG"
+            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.protectParameters: "${PROTECT_PARAMETERS:-false}"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.goodBots: ""
+            traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableGooglebotIPCheck: "true"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.protectRoutes: "/"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.persistentStateFile: "/tmp/state.json"
             traefik.http.middlewares.captcha-protect.plugin.captcha-protect.enableStateReconciliation: "true"