fix(docker): preserve failed crawl results

deploy/docker/server.py

@@ -906,9 +906,6 @@ async def crawl(
         hooks_config=hooks_config,
         crawler_configs=crawl_request.crawler_configs,
     )
-    # check if all of the results are not successful
-    if all(not result["success"] for result in results["results"]):
-        raise HTTPException(500, f"Crawl request failed: {results['results'][0]['error_message']}")
     return JSONResponse(results)
 
 

deploy/docker/tests/test_crawl_failure_response.py

@@ -0,0 +1,28 @@
+def test_all_failed_crawl_returns_results(stock_client, server_module, monkeypatch):
+    async def failed_crawl(**kwargs):
+        return {
+            "success": True,
+            "results": [
+                {
+                    "url": "https://example.com",
+                    "success": False,
+                    "error_message": "Wait condition failed: selector not found",
+                }
+            ],
+        }
+
+    monkeypatch.setattr(server_module, "handle_crawl_request", failed_crawl)
+
+    from auth import create_access_token
+
+    token = create_access_token({"sub": "test@example.com"})
+    response = stock_client.post(
+        "/crawl",
+        json={"urls": ["https://example.com"]},
+        headers={"Authorization": f"Bearer {token}"},
+    )
+
+    assert response.status_code == 200
+    result = response.json()["results"][0]
+    assert result["success"] is False
+    assert "Wait condition failed" in result["error_message"]

tests/docker/test_server_requests.py

@@ -683,9 +683,9 @@ async def test_invalid_url_handling(self, async_client: httpx.AsyncClient):
         # Should return 200 with failed results, not 500
         print(f"Status code: {response.status_code}")
         print(f"Response: {response.text}")
-        assert response.status_code == 500
+        assert response.status_code == 200
         data = response.json()
-        assert data["detail"].startswith("Crawl request failed:")
+        assert all(not result["success"] for result in data["results"])
 
     async def test_mixed_success_failure_urls(self, async_client: httpx.AsyncClient):
         """Test handling of mixed success/failure URLs."""
@@ -887,4 +887,4 @@ async def test_malformed_request_handling(self, async_client: httpx.AsyncClient)
     # Execute pytest
     exit_code = pytest.main(pytest_args)
 
-    print(f"Pytest finished with exit code: {exit_code}")
+    print(f"Pytest finished with exit code: {exit_code}")