New Citations Logic

Author: moneuron

.github/workflows/update_citations.yml

@@ -25,7 +25,8 @@ jobs:
 
       - name: Fetch citation counts
         env:
-          SCRAPINGBEE_API_KEY: ${{ secrets.SCRAPINGBEE_API_KEY }}
+          SCHOLARLY_USE_PROXY: "scraperapi"
+          SCRAPERAPI_KEY: ${{ secrets.SCRAPERAPI_KEY }}
         run: python fetch_citations.py
 
       - name: Commit and push citations.json

fetch_citations.py

@@ -34,20 +34,18 @@ def get_title_from_doi(doi: str) -> str | None:
 
 pg = ProxyGenerator()
 
-SCRAPINGBEE_API_KEY = os.getenv("SCRAPINGBEE_API_KEY")
-
+proxy_key = os.getenv("SCRAPERAPI_KEY")
 proxy_enabled = False
 
-if SCRAPINGBEE_API_KEY:
+if proxy_key:
     try:
-        proxy_enabled = pg.SingleProxy(
-            http=f"http://{SCRAPINGBEE_API_KEY}:@proxy.scrapingbee.com:8886",
-            https=f"http://{SCRAPINGBEE_API_KEY}:@proxy.scrapingbee.com:8886",
-        )
+        proxy_enabled = pg.ScraperAPI(proxy_key)
 
         if proxy_enabled:
             scholarly.use_proxy(pg)
-            print("ScrapingBee proxy enabled")
+            print("ScraperAPI proxy enabled")
+        else:
+            print("Failed to enable ScraperAPI proxy")
 
     except Exception as exc:
         print(f"Proxy setup failed: {exc}")