diff --git a/requirements.txt b/requirements.txt index 88ac716c..7982e816 100644 --- a/requirements.txt +++ b/requirements.txt @@ -82,7 +82,8 @@ faker==22.0.0 playwright>=1.49.0 pytest-playwright==0.4.4 -# --- Web Scraper (optional, feature-gated) --- +# --- Web Scraper + Search (optional, feature-gated) --- +ddgs>=0.1.0 crawl4ai==0.4.247 firecrawl-py==1.4.0 scrapegraphai==1.13.3 diff --git a/src/api/routers/chat_tools/web_tools.py b/src/api/routers/chat_tools/web_tools.py index cac4952f..cc1ca50f 100644 --- a/src/api/routers/chat_tools/web_tools.py +++ b/src/api/routers/chat_tools/web_tools.py @@ -17,7 +17,7 @@ log = get_logger("api.chat.tools.web") def _check_duckduckgo() -> bool: try: - import duckduckgo_search # noqa: F401 + import ddgs # noqa: F401 return True except ImportError: return False @@ -28,25 +28,29 @@ def _check_serper() -> bool: async def _search_duckduckgo(query: str, max_results: int) -> List[Dict[str, Any]]: - """Search using duckduckgo_search library (no API key required).""" + """Search using ddgs library (no API key required).""" try: - from duckduckgo_search import AsyncDDGS + from ddgs import DDGS except ImportError as exc: raise RuntimeError( - "duckduckgo_search is not installed. Run: pip install duckduckgo-search" + "ddgs is not installed. Run: pip install ddgs" ) from exc - results: List[Dict[str, Any]] = [] - async with AsyncDDGS() as ddgs: - async for r in ddgs.text(query, max_results=max_results): - results.append({ - "title": r.get("title", ""), - "url": r.get("href", ""), - "snippet": r.get("body", ""), - }) - if len(results) >= max_results: - break - return results + loop = asyncio.get_event_loop() + + def _do_search(): + with DDGS() as client: + return list(client.text(query, max_results=max_results)) + + raw = await loop.run_in_executor(None, _do_search) + return [ + { + "title": r.get("title", ""), + "url": r.get("href", ""), + "snippet": r.get("body", ""), + } + for r in raw + ] async def _search_serper(query: str, max_results: int) -> List[Dict[str, Any]]: @@ -135,7 +139,7 @@ class InternetSearchTool(ChatTool): "success": False, "error": ( "No search backend available. " - "Install duckduckgo-search (`pip install duckduckgo-search`) " + "Install ddgs (`pip install ddgs`) " "or set SERPER_API_KEY." ), "results": [],