[pull] main from firecrawl:main

.github/workflows/test-server.yml

@@ -203,15 +203,6 @@ jobs:
         env:
           npm_config_ignore_scripts: 'true' # required currently to prevent re-building cached native lib
 
-      - name: Publish test report
-        if: always()
-        uses: dorny/test-reporter@v1
-        with:
-          name: Test Report (${{ matrix.engine }}, ${{ matrix.proxy }}, ${{ matrix.search }}, ${{ matrix.ai }})
-          path: apps/api/test-results/junit.xml
-          reporter: jest-junit
-          fail-on-error: true
-
       - name: Copy log files
         if: always()
         run: |

apps/api/jest.config.ts

@@ -8,18 +8,6 @@ const config: JestConfigWithTsJest = {
   detectOpenHandles: true,
   openHandlesTimeout: 120000,
   watchAll: false,
-  reporters: [
-    "default",
-    [
-      "jest-junit",
-      {
-        outputDirectory: "<rootDir>/test-results",
-        outputName: "junit.xml",
-        addFileAttribute: true,
-        suiteNameTemplate: "{filepath}",
-      },
-    ],
-  ],
 };
 
 export default config;

apps/api/package.json

@@ -58,7 +58,6 @@
     "@types/tough-cookie": "^4.0.5",
     "husky": "^9.1.7",
     "jest": "^30.2.0",
-    "jest-junit": "^16.0.0",
     "knip": "^5.70.1",
     "lint-staged": "^16.1.6",
     "supertest": "^6.3.3",

apps/api/src/controllers/v2/scrape.ts

@@ -23,6 +23,8 @@ import { logRequest } from "../../services/logging/log_job";
 import { getErrorContactMessage } from "../../lib/deployment";
 import { captureExceptionWithZdrCheck } from "../../services/sentry";
 
+const AGENT_INTEROP_CONCURRENCY_BOOST = 3;
+
 export async function scrapeController(
   req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
   res: Response<ScrapeResponse>,
@@ -100,6 +102,8 @@ export async function scrapeController(
 
       const shouldBill = req.body.__agentInterop?.shouldBill ?? true;
       const agentRequestId = req.body.__agentInterop?.requestId ?? null;
+      const boostConcurrency =
+        req.body.__agentInterop?.boostConcurrency ?? false;
 
       const logger = _logger.child({
         method: "scrapeController",
@@ -172,10 +176,15 @@ export async function scrapeController(
         }
         req.on("close", () => aborter.abort());
 
+        const baseConcurrency = req.acuc?.concurrency || 1;
+        const concurrency = boostConcurrency
+          ? baseConcurrency * AGENT_INTEROP_CONCURRENCY_BOOST
+          : baseConcurrency;
+
         doc = await teamConcurrencySemaphore.withSemaphore(
           req.auth.team_id,
           jobId,
-          req.acuc?.concurrency || 1,
+          concurrency,
           aborter.signal,
           timeout ?? 60_000,
           async limited => {

apps/api/src/controllers/v2/types.ts

@@ -771,6 +771,7 @@ const scrapeRequestSchemaBase = baseScrapeOptions.extend({
       auth: z.string(),
       requestId: z.string(),
       shouldBill: z.boolean(),
+      boostConcurrency: z.boolean().optional()
     })
     .optional(),
 });

apps/api/src/services/rate-limiter.ts

@@ -33,8 +33,12 @@ export function getRateLimiter(
   mode: RateLimiterMode,
   rate_limits: AuthCreditUsageChunk["rate_limits"] | null,
 ): RateLimiterRedis {
-  return createRateLimiter(
-    `${mode}`,
-    rate_limits?.[mode] ?? fallbackRateLimits?.[mode] ?? 500,
-  );
+  let rateLimit = rate_limits?.[mode] ?? fallbackRateLimits?.[mode] ?? 500;
+
+  if (mode === RateLimiterMode.Search || mode === RateLimiterMode.Scrape) {
+    // TEMP: Mogery
+    rateLimit = Math.max(rateLimit, 100);
+  }
+
+  return createRateLimiter(`${mode}`, rateLimit);
 }