diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 1756b925f..22e061096 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -92,6 +92,15 @@ def _attach_declarative_hooks(crawler, hooks_config: dict) -> dict: logger = logging.getLogger(__name__) + +def _raise_for_crawl_failure(result): + if not result.success: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=result.error_message, + ) + + # --- Helper to get memory --- def _get_memory_mb(): try: @@ -147,11 +156,7 @@ async def handle_llm_qa( enforce_egress(browser_cfg) crawler = await get_crawler(browser_cfg) result = await crawler.arun(url) - if not result.success: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=result.error_message - ) + _raise_for_crawl_failure(result) content = result.markdown.fit_markdown or result.markdown.raw_markdown # Create prompt and get LLM response @@ -179,6 +184,8 @@ async def handle_llm_qa( ) return response.choices[0].message.content + except HTTPException: + raise except LLMProviderNotAllowed as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: @@ -389,11 +396,7 @@ async def handle_markdown_request( ) ) - if not result.success: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=result.error_message - ) + _raise_for_crawl_failure(result) return (result.markdown.raw_markdown if filter_type == FilterType.RAW @@ -1023,4 +1026,4 @@ async def _runner(): except HTTPException: await redis.delete(f"task:{task_id}") raise - return {"task_id": task_id} \ No newline at end of file + return {"task_id": task_id} diff --git a/deploy/docker/tests/test_api_crawl_failures.py b/deploy/docker/tests/test_api_crawl_failures.py new file mode 100644 index 000000000..04b7037c1 --- /dev/null +++ b/deploy/docker/tests/test_api_crawl_failures.py @@ -0,0 +1,58 @@ +from types import SimpleNamespace + +import pytest +from fastapi import HTTPException + +from api import _raise_for_crawl_failure + + +def test_crawl_failure_is_reported_as_bad_gateway(): + result = SimpleNamespace(success=False, error_message="Blocked by anti-bot protection: challenge") + + with pytest.raises(HTTPException) as raised: + _raise_for_crawl_failure(result) + + assert raised.value.status_code == 502 + assert raised.value.detail == result.error_message + + +@pytest.mark.parametrize( + ("method", "path", "payload"), + [ + ("post", "/md", {"url": "https://example.com", "f": "raw"}), + ("get", "/llm/example.com?q=summarize", None), + ], +) +def test_single_url_crawl_failure_reaches_client( + stock_client, server_module, monkeypatch, method, path, payload +): + error_message = "Blocked by anti-bot protection: challenge" + failed_result = SimpleNamespace(success=False, error_message=error_message) + + class FailedCrawler: + async def arun(self, *args, **kwargs): + return failed_result + + async def get_failed_crawler(*args, **kwargs): + return FailedCrawler() + + async def release_crawler(*args, **kwargs): + return None + + import api + import crawler_pool + from auth import create_access_token + + monkeypatch.setattr(api, "validate_url_destination", lambda url: None) + monkeypatch.setattr(crawler_pool, "get_crawler", get_failed_crawler) + monkeypatch.setattr(crawler_pool, "release_crawler", release_crawler) + + token = create_access_token({"sub": "test@example.com"}) + request = getattr(stock_client, method) + kwargs = {"json": payload} if payload is not None else {} + response = request( + path, headers={"Authorization": f"Bearer {token}"}, **kwargs + ) + + assert response.status_code == 502 + assert response.json() == {"detail": error_message}