New Citations Logic

Author: moneuron

.github/workflows/update_citations.yml

@@ -25,8 +25,7 @@ jobs:
 
       - name: Fetch citation counts
         env:
-          SCHOLARLY_USE_PROXY: "scraperapi"
-          SCRAPERAPI_KEY: ${{ secrets.SCRAPERAPI_KEY }}
+          SCRAPINGBEE_API_KEY: ${{ secrets.SCRAPINGBEE_API_KEY }}
         run: python fetch_citations.py
 
       - name: Commit and push citations.json

fetch_citations.py

@@ -34,18 +34,20 @@ def get_title_from_doi(doi: str) -> str | None:
 
 pg = ProxyGenerator()
 
-proxy_key = os.getenv("SCRAPERAPI_KEY")
+SCRAPINGBEE_API_KEY = os.getenv("SCRAPINGBEE_API_KEY")
+
 proxy_enabled = False
 
-if proxy_key:
+if SCRAPINGBEE_API_KEY:
     try:
-        proxy_enabled = pg.ScraperAPI(proxy_key)
+        proxy_enabled = pg.SingleProxy(
+            http=f"http://{SCRAPINGBEE_API_KEY}:@proxy.scrapingbee.com:8886",
+            https=f"http://{SCRAPINGBEE_API_KEY}:@proxy.scrapingbee.com:8886",
+        )
 
         if proxy_enabled:
             scholarly.use_proxy(pg)
-            print("ScraperAPI proxy enabled")
-        else:
-            print("Failed to enable ScraperAPI proxy")
+            print("ScrapingBee proxy enabled")
 
     except Exception as exc:
         print(f"Proxy setup failed: {exc}")