From 60032d8b6969badc61830a63f90e7b10277fcd96 Mon Sep 17 00:00:00 2001 From: gonzalo-zenrows Date: Mon, 12 Oct 2026 17:46:59 +0200 Subject: [PATCH] feat(crawl): make depth optional; defaults are 100 pages and 100 items The Crawl API now accepts a create without depth: the crawl has no depth limit and follows links until max_pages or max_items stops it, or no link is left. max_pages and max_items now default to 100. - create(): depth is an optional keyword, sent only when given. - Crawl.depth is optional: a crawl created without depth omits it. - README and docstrings state the new defaults. - Acceptance: check 21 expects the documented default of 100; check 1's offline half creates without depth and expects none sent or read back. --- README.md | 14 ++++++------ acceptance/crawl/README.md | 12 ++++++----- acceptance/crawl/surface/test.py | 37 ++++++++++++++++---------------- src/zenrows/crawl/client.py | 16 ++++++++------ src/zenrows/crawl/models.py | 4 +++- tests/test_crawl_client.py | 34 +++++++++++++++++++++++------ 6 files changed, 74 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 97a67ac..9550c8e 100644 --- a/README.md +++ b/README.md @@ -496,9 +496,11 @@ See `src/zenrows/batch/client.py` or `help(ZenRowsBatchClient)`. Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change. -A crawl starts from one URL, follows the links on each page up to -`depth` hops, and keeps the URLs that match your patterns, up to -`max_items`. It stays on the start URL's registrable domain; +A crawl starts from one URL, follows the links on each page, and keeps +the URLs that match your patterns, up to `max_items`. It stops at +`max_items` or `max_pages`, or when no link is left. `depth` limits the +link hops from the start URL; without it the crawl has no depth limit. +It stays on the start URL's registrable domain; subdomains count. It runs in the background: `create` returns at once with `status="running"`, and `wait` blocks until it ends. Without `output_format` a crawl returns URLs only; `output_format="html"` also @@ -512,9 +514,9 @@ client = ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"]) crawl = client.create( "https://www.scrapingcourse.com/ecommerce/", - depth=1, # link hops from the start URL - max_items=50, # stop after keeping 50 URLs (default 10) - max_pages=100, # stop after fetching 100 pages (default 10) + depth=1, # link hops from the start URL (default: no limit) + max_items=50, # stop after keeping 50 URLs (default 100) + max_pages=100, # stop after fetching 100 pages (default 100) include_patterns=["/product/"], # keep only URLs containing this ) crawl = client.wait(crawl.crawl_id) # polls up to timeout=600 s diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index 9a45870..86d782f 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -60,7 +60,7 @@ and signal checks. | # | Feature | Expected | | --- | --- | --- | -| 1 | Create | returns at once: `running`, a `crawl_id`, the echoed parameters; only the fields set are sent | +| 1 | Create | returns at once: `running`, a `crawl_id`, the echoed parameters; only the fields set are sent, and a create without `depth` sends none and reads back none | | 2 | Create with html | results get `content_status` and, once fetched, `content_url` | | 3 | Idempotency | same key and body → same `crawl_id`; different body → 422 `idempotency_key_reused` (MCP, CLI: `CRAWL_INVALID_REQUEST` with the advice to send a new key) | | 4 | Wait | returns the ended crawl; `failed` is returned, not raised (CLI: exit 1) | @@ -80,17 +80,19 @@ and signal checks. | 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | | 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | | 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish | -| 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (10) | +| 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) | Extra checks for this surface: `mypy --strict` sees the package as typed (`py.typed`) and reads the real signatures. ## What it costs -Every crawl uses `depth` 1 and `max_pages` 10 or less. A run starts five crawls: the README run, +Every crawl uses `depth` 1. Every crawl but check 19's sets `max_pages` 10 or less; check 19's +leaves it out so check 21 can read the default (100). A run starts five crawls: the README run, one URLs-only crawl, one idempotency crawl that is stopped at once, one html crawl that is stopped with its pages in flight (check 20), and one that keeps nothing (check 19). On the default `CRAWL_URL`, a URLs-only crawl fetches 1 page and the -html crawl 6, so a run costs about **10 pages**, and at most 50 if every crawl reaches its -`max_pages`. Check 9 starts a sixth crawl, one html item, when the stop in check 20 lands +html crawl 6, so a run costs about **10 pages**. Check 19's crawl keeps nothing and fetches only +the start page at `depth` 1, so its default of 100 does not raise the cost. At most 140 if every +crawl reaches its `max_pages`. Check 9 starts a sixth crawl, one html item, when the stop in check 20 lands before any page is fetched. It adds at most 10 pages. The report lists each crawl id and its `pages_fetched`. An account can run 3 active jobs at once, shared with Batch jobs. A run has at most 2 crawls diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index 410f5cb..1f1ba40 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -27,7 +27,7 @@ NO_MATCH = ( "/crawl-acceptance-matches-nothing/" # no URL contains it: the crawl keeps nothing (check 19) ) -DOCUMENTED_DEFAULT = 10 # the default the README documents for max_pages and max_items (check 21) +DOCUMENTED_DEFAULT = 100 # the default the README documents for max_pages and max_items (check 21) RESULTS = "/results" PASS, FAIL, SKIP = "PASS", "FAIL", "SKIP" TERMINAL = {"completed", "stopped", "failed"} @@ -219,20 +219,18 @@ def do_POST(self): self.send_response(201) self.send_header("Content-Type", "application/json") self.end_headers() - self.wfile.write( - json.dumps( - { - "crawl_id": "c_capture", - "status": "running", - "url": body.get("url"), - "depth": body.get("depth"), - "max_items": 10, - "max_pages": 10, - "created_at": datetime.now(UTC).isoformat(), - "coverage": {"pages_fetched": 0, "pages_failed": 0, "items_found": 0}, - } - ).encode() - ) + crawl = { + "crawl_id": "c_capture", + "status": "running", + "url": body.get("url"), + "max_items": DOCUMENTED_DEFAULT, + "max_pages": DOCUMENTED_DEFAULT, + "created_at": datetime.now(UTC).isoformat(), + "coverage": {"pages_fetched": 0, "pages_failed": 0, "items_found": 0}, + } + if "depth" in body: # as the API: a crawl created without depth omits it + crawl["depth"] = body["depth"] + self.wfile.write(json.dumps(crawl).encode()) def do_GET(self): if "c_nobody" in self.path: @@ -255,7 +253,7 @@ def do_GET(self): cap = ZenRowsCrawlClient( "capture-key", base_url=f"http://127.0.0.1:{srv.server_address[1]}/v1", retries=0 ) -cap.create("https://example.com/", depth=1) +minimal = cap.create("https://example.com/") # depth is optional: no depth limit cap.create( "https://example.com/", depth=1, @@ -264,8 +262,11 @@ def do_GET(self): include_patterns=["/p/"], idempotency_key="idem-1", ) -sent_ok = sorted(captured[0]) == ["depth", "url"] and sorted(captured[1]) == sorted( - ["url", "depth", "max_items", "max_pages", "include_patterns"] +sent_ok = ( + sorted(captured[0]) == ["url"] + and minimal.depth is None + and sorted(captured[1]) + == sorted(["url", "depth", "max_items", "max_pages", "include_patterns"]) ) sent_line = ( f"sent (stand-in server): minimal={json.dumps(captured[0])} full={json.dumps(captured[1])}" diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 59bff3d..75c1bfa 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -1,7 +1,8 @@ """ZenRowsCrawlClient — the typed facade over the Crawl API (New). -A crawl starts from one URL, follows links up to `depth` hops, and -keeps the URLs that match its patterns, up to `max_items`. It runs as +A crawl starts from one URL, follows links (up to `depth` hops when +set), and keeps the URLs that match its patterns, up to `max_items`. It +stops at `max_items` or `max_pages`, or when no link is left. It runs as a job: `create` returns at once with `status=running`, and the URLs grow as pages finish. @@ -163,7 +164,7 @@ def create( self, url: str, *, - depth: int, + depth: int | None = None, max_items: int | None = None, max_pages: int | None = None, include_patterns: list[str] | None = None, @@ -174,10 +175,12 @@ def create( """`POST /crawls` — start a crawl and return it, `running`. - `depth`: link hops to follow from `url` (1 to 100,000). + Omit it for no depth limit: the crawl follows links until + `max_items` or `max_pages` stops it, or no link is left. - `max_items`: stop after keeping this many URLs (server - default 10). + default 100). - `max_pages`: stop after fetching this many pages, which - bounds the cost (server default 10). + bounds the cost (server default 100). - `include_patterns` / `exclude_patterns`: substrings matched against each normalized URL. A URL is kept when it matches an include pattern (if any) and no exclude pattern. A crawl @@ -192,8 +195,9 @@ def create( Batch jobs, `create` raises 429 `too_many_crawls` without retrying; retry after the error's `retry_after` seconds. """ - body: dict[str, Any] = {"url": url, "depth": depth} + body: dict[str, Any] = {"url": url} optional: dict[str, Any] = { + "depth": depth, "max_items": max_items, "max_pages": max_pages, "include_patterns": include_patterns, diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index eaea5d9..e1ab707 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -98,7 +98,9 @@ class Crawl(BaseModel): status: CrawlStatus url: str """The page the crawl started from.""" - depth: int + depth: int | None = None + """Link hops the crawl follows from `url`. Absent when the crawl was + created without one: it has no depth limit.""" max_items: int max_pages: int coverage: Coverage diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index c93fd20..6d9f933 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -29,18 +29,20 @@ API_KEY = "test-key" -def crawl_body(**overrides): +def crawl_body(*, omit: tuple[str, ...] = (), **overrides): body = { "crawl_id": "c_1", "status": "running", "url": "https://example.com/", "depth": 1, - "max_items": 10, - "max_pages": 10, + "max_items": 100, + "max_pages": 100, "coverage": {"pages_fetched": 0, "pages_failed": 0, "items_found": 0}, "created_at": "2026-10-08T09:00:00Z", } body.update(overrides) + for key in omit: + del body[key] return body @@ -93,18 +95,34 @@ def test_base_url_is_a_constructor_option_only(monkeypatch): @respx.mock def test_create_sends_only_required_fields(client: ZenRowsCrawlClient): route = respx.post(f"{BASE_URL}/crawls").mock( - return_value=Response(202, headers={"Location": "/v1/crawls/c_1"}, json=crawl_body()) + return_value=Response( + 202, headers={"Location": "/v1/crawls/c_1"}, json=crawl_body(omit=("depth",)) + ) ) - crawl = client.create("https://example.com/", depth=1) + crawl = client.create("https://example.com/") assert crawl.crawl_id == "c_1" assert crawl.status is CrawlStatus.RUNNING + assert crawl.depth is None # created without depth: no depth limit, none echoed sent = route.calls.last.request assert sent.headers["X-API-Key"] == API_KEY assert sent.headers["Content-Type"] == "application/json" assert "Idempotency-Key" not in sent.headers - assert json.loads(sent.content) == {"url": "https://example.com/", "depth": 1} + assert json.loads(sent.content) == {"url": "https://example.com/"} + + +@respx.mock +def test_create_sends_depth_when_given(client: ZenRowsCrawlClient): + route = respx.post(f"{BASE_URL}/crawls").mock(return_value=Response(202, json=crawl_body())) + + crawl = client.create("https://example.com/", depth=1) + + assert crawl.depth == 1 + assert json.loads(route.calls.last.request.content) == { + "url": "https://example.com/", + "depth": 1, + } @respx.mock @@ -213,7 +231,7 @@ def test_list_returns_one_page(client: ZenRowsCrawlClient): route = respx.get(f"{BASE_URL}/crawls").mock( side_effect=[ Response(200, json={"crawls": [crawl_body(crawl_id="c_2")], "next_cursor": "n1"}), - Response(200, json={"crawls": [crawl_body(crawl_id="c_1")]}), + Response(200, json={"crawls": [crawl_body(crawl_id="c_1", omit=("depth",))]}), ] ) @@ -222,6 +240,8 @@ def test_list_returns_one_page(client: ZenRowsCrawlClient): assert [c.crawl_id for c in first.crawls] == ["c_2"] assert [c.crawl_id for c in last.crawls] == ["c_1"] + assert first.crawls[0].depth == 1 + assert last.crawls[0].depth is None # created without depth assert last.next_cursor is None assert dict(route.calls[0].request.url.params) == {"limit": "1"} assert route.calls[1].request.url.params["cursor"] == "n1"