feat(crawl): make depth optional; defaults are 100 pages and 100 items

README.md

@@ -496,9 +496,11 @@ See `src/zenrows/batch/client.py` or `help(ZenRowsBatchClient)`.
 
 Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change.
 
-A crawl starts from one URL, follows the links on each page up to
-`depth` hops, and keeps the URLs that match your patterns, up to
-`max_items`. It stays on the start URL's registrable domain;
+A crawl starts from one URL, follows the links on each page, and keeps
+the URLs that match your patterns, up to `max_items`. It stops at
+`max_items` or `max_pages`, or when no link is left. `depth` limits the
+link hops from the start URL; without it the crawl has no depth limit.
+It stays on the start URL's registrable domain;
 subdomains count. It runs in the background: `create` returns at once with
 `status="running"`, and `wait` blocks until it ends. Without
 `output_format` a crawl returns URLs only; `output_format="html"` also
@@ -512,9 +514,9 @@ client = ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"])
 
 crawl = client.create(
     "https://www.scrapingcourse.com/ecommerce/",
-    depth=1,                         # link hops from the start URL
-    max_items=50,                    # stop after keeping 50 URLs (default 10)
-    max_pages=100,                   # stop after fetching 100 pages (default 10)
+    depth=1,                         # link hops from the start URL (default: no limit)
+    max_items=50,                    # stop after keeping 50 URLs (default 100)
+    max_pages=100,                   # stop after fetching 100 pages (default 100)
     include_patterns=["/product/"],  # keep only URLs containing this
 )
 crawl = client.wait(crawl.crawl_id)            # polls up to timeout=600 s

acceptance/crawl/README.md

@@ -60,7 +60,7 @@ and signal checks.
 
 | # | Feature | Expected |
 | --- | --- | --- |
-| 1 | Create | returns at once: `running`, a `crawl_id`, the echoed parameters; only the fields set are sent |
+| 1 | Create | returns at once: `running`, a `crawl_id`, the echoed parameters; only the fields set are sent, and a create without `depth` sends none and reads back none |
 | 2 | Create with html | results get `content_status` and, once fetched, `content_url` |
 | 3 | Idempotency | same key and body → same `crawl_id`; different body → 422 `idempotency_key_reused` (MCP, CLI: `CRAWL_INVALID_REQUEST` with the advice to send a new key) |
 | 4 | Wait | returns the ended crawl; `failed` is returned, not raised (CLI: exit 1) |
@@ -80,17 +80,19 @@ and signal checks.
 | 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch |
 | 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) |
 | 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish |
-| 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (10) |
+| 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) |
 
 Extra checks for this surface: `mypy --strict` sees the package as typed (`py.typed`) and reads the real signatures.
 
 ## What it costs
 
-Every crawl uses `depth` 1 and `max_pages` 10 or less. A run starts five crawls: the README run,
+Every crawl uses `depth` 1. Every crawl but check 19's sets `max_pages` 10 or less; check 19's
+leaves it out so check 21 can read the default (100). A run starts five crawls: the README run,
 one URLs-only crawl, one idempotency crawl that is stopped at once, one html crawl that is
 stopped with its pages in flight (check 20), and one that keeps nothing (check 19). On the default `CRAWL_URL`, a URLs-only crawl fetches 1 page and the
-html crawl 6, so a run costs about **10 pages**, and at most 50 if every crawl reaches its
-`max_pages`. Check 9 starts a sixth crawl, one html item, when the stop in check 20 lands
+html crawl 6, so a run costs about **10 pages**. Check 19's crawl keeps nothing and fetches only
+the start page at `depth` 1, so its default of 100 does not raise the cost. At most 140 if every
+crawl reaches its `max_pages`. Check 9 starts a sixth crawl, one html item, when the stop in check 20 lands
 before any page is fetched. It adds at most 10 pages. The report lists each crawl id and its `pages_fetched`.
 
 An account can run 3 active jobs at once, shared with Batch jobs. A run has at most 2 crawls

acceptance/crawl/surface/test.py

@@ -27,7 +27,7 @@
 NO_MATCH = (
     "/crawl-acceptance-matches-nothing/"  # no URL contains it: the crawl keeps nothing (check 19)
 )
-DOCUMENTED_DEFAULT = 10  # the default the README documents for max_pages and max_items (check 21)
+DOCUMENTED_DEFAULT = 100  # the default the README documents for max_pages and max_items (check 21)
 RESULTS = "/results"
 PASS, FAIL, SKIP = "PASS", "FAIL", "SKIP"
 TERMINAL = {"completed", "stopped", "failed"}
@@ -219,20 +219,18 @@ def do_POST(self):
         self.send_response(201)
         self.send_header("Content-Type", "application/json")
         self.end_headers()
-        self.wfile.write(
-            json.dumps(
-                {
-                    "crawl_id": "c_capture",
-                    "status": "running",
-                    "url": body.get("url"),
-                    "depth": body.get("depth"),
-                    "max_items": 10,
-                    "max_pages": 10,
-                    "created_at": datetime.now(UTC).isoformat(),
-                    "coverage": {"pages_fetched": 0, "pages_failed": 0, "items_found": 0},
-                }
-            ).encode()
-        )
+        crawl = {
+            "crawl_id": "c_capture",
+            "status": "running",
+            "url": body.get("url"),
+            "max_items": DOCUMENTED_DEFAULT,
+            "max_pages": DOCUMENTED_DEFAULT,
+            "created_at": datetime.now(UTC).isoformat(),
+            "coverage": {"pages_fetched": 0, "pages_failed": 0, "items_found": 0},
+        }
+        if "depth" in body:  # as the API: a crawl created without depth omits it
+            crawl["depth"] = body["depth"]
+        self.wfile.write(json.dumps(crawl).encode())
 
     def do_GET(self):
         if "c_nobody" in self.path:
@@ -255,7 +253,7 @@ def do_GET(self):
 cap = ZenRowsCrawlClient(
     "capture-key", base_url=f"http://127.0.0.1:{srv.server_address[1]}/v1", retries=0
 )
-cap.create("https://example.com/", depth=1)
+minimal = cap.create("https://example.com/")  # depth is optional: no depth limit
 cap.create(
     "https://example.com/",
     depth=1,
@@ -264,8 +262,11 @@ def do_GET(self):
     include_patterns=["/p/"],
     idempotency_key="idem-1",
 )
-sent_ok = sorted(captured[0]) == ["depth", "url"] and sorted(captured[1]) == sorted(
-    ["url", "depth", "max_items", "max_pages", "include_patterns"]
+sent_ok = (
+    sorted(captured[0]) == ["url"]
+    and minimal.depth is None
+    and sorted(captured[1])
+    == sorted(["url", "depth", "max_items", "max_pages", "include_patterns"])
 )
 sent_line = (
     f"sent (stand-in server): minimal={json.dumps(captured[0])} full={json.dumps(captured[1])}"

src/zenrows/crawl/client.py

@@ -1,7 +1,8 @@
 """ZenRowsCrawlClient — the typed facade over the Crawl API (New).
 
-A crawl starts from one URL, follows links up to `depth` hops, and
-keeps the URLs that match its patterns, up to `max_items`. It runs as
+A crawl starts from one URL, follows links (up to `depth` hops when
+set), and keeps the URLs that match its patterns, up to `max_items`. It
+stops at `max_items` or `max_pages`, or when no link is left. It runs as
 a job: `create` returns at once with `status=running`, and the URLs
 grow as pages finish.
 
@@ -163,7 +164,7 @@ def create(
         self,
         url: str,
         *,
-        depth: int,
+        depth: int | None = None,
         max_items: int | None = None,
         max_pages: int | None = None,
         include_patterns: list[str] | None = None,
@@ -174,10 +175,12 @@ def create(
         """`POST /crawls` — start a crawl and return it, `running`.
 
         - `depth`: link hops to follow from `url` (1 to 100,000).
+          Omit it for no depth limit: the crawl follows links until
+          `max_items` or `max_pages` stops it, or no link is left.
         - `max_items`: stop after keeping this many URLs (server
-          default 10).
+          default 100).
         - `max_pages`: stop after fetching this many pages, which
-          bounds the cost (server default 10).
+          bounds the cost (server default 100).
         - `include_patterns` / `exclude_patterns`: substrings matched
           against each normalized URL. A URL is kept when it matches
           an include pattern (if any) and no exclude pattern. A crawl
@@ -192,8 +195,9 @@ def create(
         Batch jobs, `create` raises 429 `too_many_crawls` without
         retrying; retry after the error's `retry_after` seconds.
         """
-        body: dict[str, Any] = {"url": url, "depth": depth}
+        body: dict[str, Any] = {"url": url}
         optional: dict[str, Any] = {
+            "depth": depth,
             "max_items": max_items,
             "max_pages": max_pages,
             "include_patterns": include_patterns,

src/zenrows/crawl/models.py

@@ -98,7 +98,9 @@ class Crawl(BaseModel):
     status: CrawlStatus
     url: str
     """The page the crawl started from."""
-    depth: int
+    depth: int | None = None
+    """Link hops the crawl follows from `url`. Absent when the crawl was
+    created without one: it has no depth limit."""
     max_items: int
     max_pages: int
     coverage: Coverage

tests/test_crawl_client.py

@@ -29,18 +29,20 @@
 API_KEY = "test-key"
 
 
-def crawl_body(**overrides):
+def crawl_body(*, omit: tuple[str, ...] = (), **overrides):
     body = {
         "crawl_id": "c_1",
         "status": "running",
         "url": "https://example.com/",
         "depth": 1,
-        "max_items": 10,
-        "max_pages": 10,
+        "max_items": 100,
+        "max_pages": 100,
         "coverage": {"pages_fetched": 0, "pages_failed": 0, "items_found": 0},
         "created_at": "2026-10-08T09:00:00Z",
     }
     body.update(overrides)
+    for key in omit:
+        del body[key]
     return body
 
 
@@ -93,18 +95,34 @@ def test_base_url_is_a_constructor_option_only(monkeypatch):
 @respx.mock
 def test_create_sends_only_required_fields(client: ZenRowsCrawlClient):
     route = respx.post(f"{BASE_URL}/crawls").mock(
-        return_value=Response(202, headers={"Location": "/v1/crawls/c_1"}, json=crawl_body())
+        return_value=Response(
+            202, headers={"Location": "/v1/crawls/c_1"}, json=crawl_body(omit=("depth",))
+        )
     )
 
-    crawl = client.create("https://example.com/", depth=1)
+    crawl = client.create("https://example.com/")
 
     assert crawl.crawl_id == "c_1"
     assert crawl.status is CrawlStatus.RUNNING
+    assert crawl.depth is None  # created without depth: no depth limit, none echoed
     sent = route.calls.last.request
     assert sent.headers["X-API-Key"] == API_KEY
     assert sent.headers["Content-Type"] == "application/json"
     assert "Idempotency-Key" not in sent.headers
-    assert json.loads(sent.content) == {"url": "https://example.com/", "depth": 1}
+    assert json.loads(sent.content) == {"url": "https://example.com/"}
+
+
+@respx.mock
+def test_create_sends_depth_when_given(client: ZenRowsCrawlClient):
+    route = respx.post(f"{BASE_URL}/crawls").mock(return_value=Response(202, json=crawl_body()))
+
+    crawl = client.create("https://example.com/", depth=1)
+
+    assert crawl.depth == 1
+    assert json.loads(route.calls.last.request.content) == {
+        "url": "https://example.com/",
+        "depth": 1,
+    }
 
 
 @respx.mock
@@ -213,7 +231,7 @@ def test_list_returns_one_page(client: ZenRowsCrawlClient):
     route = respx.get(f"{BASE_URL}/crawls").mock(
         side_effect=[
             Response(200, json={"crawls": [crawl_body(crawl_id="c_2")], "next_cursor": "n1"}),
-            Response(200, json={"crawls": [crawl_body(crawl_id="c_1")]}),
+            Response(200, json={"crawls": [crawl_body(crawl_id="c_1", omit=("depth",))]}),
         ]
     )
 
@@ -222,6 +240,8 @@ def test_list_returns_one_page(client: ZenRowsCrawlClient):
 
     assert [c.crawl_id for c in first.crawls] == ["c_2"]
     assert [c.crawl_id for c in last.crawls] == ["c_1"]
+    assert first.crawls[0].depth == 1
+    assert last.crawls[0].depth is None  # created without depth
     assert last.next_cursor is None
     assert dict(route.calls[0].request.url.params) == {"limit": "1"}
     assert route.calls[1].request.url.params["cursor"] == "n1"