feat: Add Firecrawl web crawling built-in tool with configurable API …

config.py

@@ -55,4 +55,5 @@ class Config:
     DOCSTAR_ORG_ID = os.getenv('DOCSTAR_ORG_ID')
     DOCSTAR_COLLECTION_ID = os.getenv('DOCSTAR_COLLECTION_ID')
     AI_ML_APIKEY = os.getenv('AI_ML_APIKEY')
-    AI_MIDDLEWARE_PAUTH_KEY = os.getenv('AI_MIDDLEWARE_PAUTH_KEY')
+    AI_MIDDLEWARE_PAUTH_KEY = os.getenv('AI_MIDDLEWARE_PAUTH_KEY')
+    FIRECRAWL_API_KEY = os.getenv('FIRECRAWL_API_KEY')

src/services/commonServices/baseService/utils.py

@@ -12,6 +12,7 @@
 from globals import *
 from src.db_services.ConfigurationServices import get_bridges_without_tools, update_bridge
 from src.services.utils.ai_call_util import call_gtwy_agent
+from src.services.utils.built_in_tools.firecrawl import call_firecrawl_scrape
 from globals import *
 from src.services.cache_service import store_in_cache, find_in_cache, client, REDIS_PREFIX
 from src.configs.constant import redis_keys
@@ -220,7 +221,7 @@ async def process_data_and_run_tools(codes_mapping, self):
         for tool_call_key, tool in codes_mapping.items():
             name = tool['name']
 
-            # Get corresponding function code mapping
+            # Get corresponding function code mapping``
             tool_mapping = {} if self.tool_id_and_name_mapping[name] else {"error": True, "response": "Wrong Function name"}
             tool_data = {**tool, **tool_mapping}
 
@@ -250,8 +251,10 @@ async def process_data_and_run_tools(codes_mapping, self):
                     # Pass bridge_configurations if available
                     if hasattr(self, 'bridge_configurations') and self.bridge_configurations:
                         agent_args["bridge_configurations"] = self.bridge_configurations
-                    
+
                     task = call_gtwy_agent(agent_args)
+                elif self.tool_id_and_name_mapping[name].get('type') == 'WEB_CRAWL':
+                    task = call_firecrawl_scrape(tool_data.get("args"))
                 else: 
                     task = axios_work(tool_data.get("args"), self.tool_id_and_name_mapping[name])
                 tasks.append((tool_call_key, tool_data, task))
@@ -496,5 +499,3 @@ async def save_files_to_redis(thread_id, sub_thread_id, bridge_id, files):
             await store_in_cache(cache_key, files, 604800)
     else:
         await store_in_cache(cache_key, files, 604800)
-
-

src/services/utils/built_in_tools/firecrawl.py

@@ -0,0 +1,63 @@
+from config import Config
+from src.services.utils.apiservice import fetch
+from src.services.utils.logger import logger
+
+FIRECRAWL_SCRAPE_URL = "https://api.firecrawl.dev/v2/scrape"
+
+async def call_firecrawl_scrape(args):
+    url = (args or {}).get('url') if isinstance(args, dict) else None
+    if not url:
+        return {
+            'response': {'error': 'url is required for web_crawling tool'},
+            'metadata': {'type': 'function'},
+            'status': 0
+        }
+
+    api_key = Config.FIRECRAWL_API_KEY
+    if not api_key:
+        return {
+            'response': {'error': 'web_crawling tool is not configured'},
+            'metadata': {'type': 'function'},
+            'status': 0
+        }
+
+    payload = {'url': url}
+    formats = args.get('formats') if isinstance(args, dict) else None
+    if formats:
+        if isinstance(formats, list):
+            payload['formats'] = formats
+        elif isinstance(formats, str):
+            payload['formats'] = [formats]
+        else:
+            payload['formats'] = [str(formats)]
+
+    request_headers = {
+        'Content-Type': 'application/json',
+        'Authorization': f'Bearer {api_key}'
+    }
+    endpoint = FIRECRAWL_SCRAPE_URL
+
+    try:
+        response, headers = await fetch(
+            endpoint,
+            'POST',
+            request_headers,
+            None,
+            payload
+        )
+        data = response.get('data') if isinstance(response, dict) and 'data' in response else response
+        return {
+            'response': data,
+            'metadata': {
+                'type': 'function',
+                'flowHitId': headers.get('flowHitId') if isinstance(headers, dict) else None
+            },
+            'status': 1
+        }
+    except Exception as exc:
+        logger.error(f"Firecrawl scrape failed: {exc}")
+        return {
+            'response': {'error': str(exc)},
+            'metadata': {'type': 'function'},
+            'status': 0
+        }

src/services/utils/getConfiguration.py

@@ -6,7 +6,7 @@
 from .getConfiguration_utils import (
     validate_bridge, get_bridge_data, setup_configuration, setup_tool_choice,
     setup_tools, setup_api_key, setup_pre_tools, add_rag_tool,
-    add_anthropic_json_schema, add_connected_agents
+    add_anthropic_json_schema, add_connected_agents, add_web_crawling_tool
 )
 from .update_and_check_cost import check_bridge_api_folder_limits
 
@@ -99,6 +99,9 @@ async def _prepare_configuration_response(configuration, service, bridge_id, api
     configuration.pop('tools', None)
     configuration['tools'] = tools
 
+    db_built_in_tools = result.get('bridges', {}).get('built_in_tools') or []
+    active_built_in_tools = built_in_tools or db_built_in_tools or []
+
     RTLayer = True if configuration and 'RTLayer' in configuration else False
 
     template_content = await ConfigurationService.get_template_by_id(template_id) if template_id else None
@@ -127,6 +130,7 @@ async def _prepare_configuration_response(configuration, service, bridge_id, api
     )
 
     add_rag_tool(tools, tool_id_and_name_mapping, rag_data)
+    add_web_crawling_tool(tools, tool_id_and_name_mapping, active_built_in_tools)
     add_anthropic_json_schema(service, configuration, tools)
 
     if rag_data:
@@ -162,7 +166,7 @@ async def _prepare_configuration_response(configuration, service, bridge_id, api
         'org_name': org_name,
         'bridge_id': result['bridges'].get('parent_id', result['bridges'].get('_id')),
         'variables_state': result.get('bridges', {}).get('variables_state', {}),
-        'built_in_tools': built_in_tools or result.get('bridges', {}).get('built_in_tools'),
+        'built_in_tools': active_built_in_tools,
         'fall_back': result.get('bridges', {}).get('fall_back') or {},
         'guardrails': guardrails_value,
         "is_embed": result.get('bridges', {}).get("folder_type") == 'embed',

src/services/utils/getConfiguration_utils.py

@@ -281,6 +281,51 @@ def add_rag_tool(tools, tool_id_and_name_mapping, rag_data):
         "type": "RAG"
     }
 
+def _should_enable_web_crawling_tool(built_in_tools):
+    if not built_in_tools:
+        return False
+    try:
+        normalized = {tool.lower() for tool in built_in_tools}
+    except AttributeError:
+        return False
+    return 'web_crawling' in normalized
+
+def add_web_crawling_tool(tools, tool_id_and_name_mapping, built_in_tools):
+    """Add Firecrawl-based web crawling tool when requested via built-in tools."""
+    if not _should_enable_web_crawling_tool(built_in_tools):
+        return
+
+    tools.append({
+        'type': 'function',
+        'name': 'web_crawling',
+        'description': 'Use this to crawl or scrape website content via Firecrawl. Always provide the `url` argument (must include http/https) and optionally pass a `formats` list if you need specific output formats.',
+        'properties': {
+            'url': {
+                'description': 'Fully qualified URL to scrape (must start with http or https).',
+                'type': 'string',
+                'enum': [],
+                'required_params': [],
+                'parameter': {}
+            },
+            'formats': {
+                'description': 'Optional list of formats to return (e.g., markdown, screenshot).',
+                'type': 'array',
+                'items': {
+                    'type': 'string'
+                },
+                'enum': [],
+                'required_params': [],
+                'parameter': {}
+            }
+        },
+        'required': ['url']
+    })
+
+    tool_id_and_name_mapping['web_crawling'] = {
+        'type': 'WEB_CRAWL',
+        'name': 'web_crawling'
+    }
+
 def add_anthropic_json_schema(service, configuration, tools):
     """Add JSON schema response format for Anthropic service"""
     if (service != 'anthropic' or