feat(extract): remove the autoparse fallback and add domain preparation

README.md

@@ -30,6 +30,7 @@ This package ships three clients:
   - [PUT Requests](#put-requests)
   - [Concurrency](#concurrency)
   - [Extract](#extract)
+    - [Preparing a domain](#preparing-a-domain)
 - [Quickstart — Batch API (`ZenRowsBatchClient`)](#quickstart--batch-api-zenrowsbatchclient)
   - [Extract in a batch](#extract-in-a-batch)
   - [Upload URLs from a CSV](#upload-urls-from-a-csv)
@@ -68,7 +69,7 @@ client = ZenRowsClient("YOUR-API-KEY", retries=1)
 url = "https://www.zenrows.com/"
 
 response = client.fetch(url, params={
-    # Our algorithm allows to automatically extract content from any website
+    # Deprecated: use client.extract() for structured data instead
     "autoparse": False,
 
     # CSS Selectors for data extraction (i.e. {"links":"a @href"} to get href attributes from links)
@@ -204,6 +205,62 @@ print(response.json())
 
 `extract()` also sends [Adaptive Stealth Mode](https://docs.zenrows.com/fetch/features/adaptive-stealth-mode) (`mode: "auto"`) by default, except when `params` sets `js_render` or `premium_proxy`: the API rejects either combined with Adaptive Stealth Mode, so your fixed configuration is sent as is. Pass `adaptive_stealth=False` to turn it off entirely.
 
+Extract works on any prepared domain (you can prepare one yourself with `POST /v1/extract/prepared-domains`). A `403` with `code: "REQS007"` means Extract can't extract this page. Check `reason` in the problem body when present; older responses may not carry it:
+
+- `domain_not_prepared`: prepare the domain with `client.prepare_domain()` (see below), poll `client.prepared_domain_status()` until `status` is `"ready"`, then retry.
+- `page_not_recognised`: the domain is served, but this page isn't one Extract extracts (a home, search, login or category page). Try a product, article or listing page, or retry once in a few minutes. Preparing the domain again changes nothing.
+- `domain_not_enabled`: preparing doesn't help. Use `css_extractor`, or contact support.
+
+```python
+response = client.extract(url)
+if response.status_code == 403:
+    problem = response.json()
+    if problem.get("code") == "REQS007":
+        reason = problem.get("reason")  # None on older responses
+        if reason == "domain_not_prepared":
+            ...  # client.prepare_domain(...), then poll client.prepared_domain_status(...)
+        elif reason == "page_not_recognised":
+            ...  # try a product, article or listing page
+        elif reason == "domain_not_enabled":
+            ...  # use css_extractor, or contact support
+```
+
+The legacy `402` with `code: "AUTH010"` means the domain isn't enabled. `extract()` returns both error responses to you as is, in one request, and never retries them with autoparse. `fallback_to_autoparse` is deprecated and has no effect; passing `True` emits a `DeprecationWarning`.
+
+#### Preparing a domain
+
+`client.prepare_domain(domain, example_urls)` submits a domain for Extract (`POST /v1/extract/prepared-domains`), and `client.prepared_domain_status(domain)` reads where it is (`GET /v1/extract/prepared-domains/{domain}`). Both return the parsed status object. Pass the domain without a scheme and 1 to 5 example URLs, one per kind of page you want to extract. These calls cost no credits.
+
+Preparation takes minutes, and the client does not wait for it. Poll the status until it is no longer `"preparing"`:
+
+```python
+import time
+
+from zenrows import PreparedDomainError, ZenRowsClient
+
+client = ZenRowsClient("YOUR-API-KEY")
+
+try:
+    status = client.prepare_domain(
+        "www.scrapingcourse.com",
+        ["https://www.scrapingcourse.com/ecommerce/"],
+    )
+except PreparedDomainError as err:
+    # e.g. 409 in_progress, 429 out_of_allowance, 503 overloaded
+    print(err.status_code, err.code, err.detail)
+    raise
+
+while status["status"] == "preparing":
+    time.sleep(30)
+    status = client.prepared_domain_status("www.scrapingcourse.com")
+
+print(status["status"])  # "ready" or "failed" (see status["failure"])
+```
+
+`prepare_domain_async()` and `prepared_domain_status_async()` are the async counterparts. The client's `retries` setting doesn't apply to these calls, and an empty `domain` raises `ValueError` before any request.
+
+Both methods raise `PreparedDomainError` on an error answer, with `status_code`, `code` (`invalid_argument`, `not_found`, `in_progress`, `out_of_allowance`, `overloaded`, `disabled`, ...), `detail`, `reason` and the whole `problem` body. See the [Extract endpoints docs](https://docs.zenrows.com/extract/endpoints) for every status and failure code.
+
 A `422` with `code: "no_data_extracted"` means Extract knows this kind of page but found no data on it. It isn't charged, and the client never retries it, even with `retries` set. `is_no_data_extracted(response)` tells you when you got one. See the [Extract docs](https://docs.zenrows.com/extract/setup).
 
 ## Quickstart — Batch API (`ZenRowsBatchClient`)

src/zenrows/__init__.py

@@ -15,11 +15,12 @@
 
 from zenrows.__version__ import __version__
 from zenrows.batch import ZenRowsBatchClient
-from zenrows.client import ZenRowsClient, is_no_data_extracted
+from zenrows.client import PreparedDomainError, ZenRowsClient, is_no_data_extracted
 from zenrows.crawl import CrawlAPIError, ZenRowsCrawlClient
 
 __all__ = [
     "CrawlAPIError",
+    "PreparedDomainError",
     "ZenRowsBatchClient",
     "ZenRowsClient",
     "ZenRowsCrawlClient",

src/zenrows/client.py

@@ -12,11 +12,14 @@
 """
 
 import asyncio
+import math
 import os
 import time
+import warnings
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 from typing import Any
+from urllib.parse import quote
 
 import requests
 import urllib3
@@ -56,19 +59,93 @@ def is_no_data_extracted(response: requests.Response) -> bool:
     return response.status_code == 422 and code is not None and code.lower() == _NO_DATA_EXTRACTED
 
 
-# Extract answers these when it can't serve the target domain: 403 REQS007
-# (domain not prepared yet) or the legacy 402 AUTH010 (domain not enabled).
-# extract(mode="auto") retries both once with Autoparse.
-_EXTRACT_DOMAIN_UNAVAILABLE = {(403, "REQS007"), (402, "AUTH010")}
+_FALLBACK_DEPRECATED = (
+    "fallback_to_autoparse is deprecated and has no effect: extract() no longer "
+    "retries with autoparse. On a 403 REQS007, check `reason` in the problem "
+    "body; for domain_not_prepared, prepare the domain with prepare_domain()."
+)
 
 
-def _is_extract_domain_unavailable(response: requests.Response) -> bool:
-    """True when the response is Extract saying it can't serve the target
-    domain yet (403 REQS007, or the legacy 402 AUTH010)."""
-    code = _error_code(response)
-    if code is None:
-        return False
-    return (response.status_code, code.upper()) in _EXTRACT_DOMAIN_UNAVAILABLE
+def _check_domain(domain: str) -> None:
+    # An empty domain would request the list endpoint and read its answer as
+    # a status, so it never leaves the client.
+    if not isinstance(domain, str) or not domain.strip():
+        raise ValueError("domain must be a non-empty string")
+
+
+def _prepared_domains_request(method: str, url: str, **kwargs: Any) -> requests.Response:
+    # Not through `requests_session`: its retry adapter would retry a 429 or
+    # 503, which these endpoints use for answers to read (out_of_allowance,
+    # overloaded). The Go and Node SDKs skip client retries here too.
+    with requests.Session() as session:
+        return session.request(method, url, **kwargs)
+
+
+def _prepared_domain_body(response: requests.Response) -> dict:
+    if not response.ok:
+        raise PreparedDomainError.from_response(response)
+    try:
+        body = response.json()
+    except ValueError:
+        body = None
+    if not isinstance(body, dict):
+        raise PreparedDomainError(response.status_code, None)
+    return body
+
+
+class PreparedDomainError(Exception):
+    """A non-2xx answer from the prepared-domains endpoints
+    (`prepare_domain()` / `prepared_domain_status()`).
+
+    Branch on `status_code` + `code`; display `detail`, don't parse it.
+    `code` is the problem's `code` member, for example `invalid_argument`
+    (400), `not_found` (404), `in_progress` (409: a preparation is already
+    running, read its status instead of resubmitting), `out_of_allowance`
+    (429: no preparations left this month), or `overloaded` / `disabled`
+    (503); None when the body has no code. `problem` holds the whole body,
+    or None when it was not JSON.
+    """
+
+    def __init__(
+        self,
+        status_code: int,
+        problem: dict | None,
+        *,
+        retry_after: float | None = None,
+    ):
+        self.status_code = status_code
+        self.problem = problem
+        # Seconds the server asked to wait (`Retry-After`), when it sent one.
+        self.retry_after = retry_after
+
+        def member(name: str) -> str | None:
+            value = problem.get(name) if problem else None
+            return value if isinstance(value, str) else None
+
+        self.code = member("code")
+        self.detail = member("detail")
+        self.reason = member("reason")
+        title = member("title") or "Prepared-domains request failed"
+        explanation = self.detail or self.code
+        super().__init__(f"{status_code} {title}" + (f": {explanation}" if explanation else ""))
+
+    @classmethod
+    def from_response(cls, response: requests.Response) -> "PreparedDomainError":
+        try:
+            body = response.json()
+        except ValueError:
+            body = None
+        try:
+            retry_after: float | None = float(response.headers.get("Retry-After", ""))
+        except ValueError:
+            retry_after = None
+        if retry_after is not None and (not math.isfinite(retry_after) or retry_after < 0):
+            retry_after = None
+        return cls(
+            response.status_code,
+            body if isinstance(body, dict) else None,
+            retry_after=retry_after,
+        )
 
 
 # Adaptive Stealth Mode (wire `mode=auto`) picks js_render / premium_proxy
@@ -168,7 +245,7 @@ def extract(
         params: dict | None = None,
         headers: dict | None = None,
         mode: str = "auto",
-        fallback_to_autoparse: bool = True,
+        fallback_to_autoparse: bool = False,
         adaptive_stealth: bool = True,
         **kwargs: Any,
     ) -> requests.Response:
@@ -177,39 +254,47 @@ def extract(
         or "standard". Thin wrapper over `fetch()` with the `extract` param set:
         no separate endpoint or auth.
 
-        Extract works on any domain once Zenrows has prepared it, with any
-        active API key. A domain that isn't prepared yet answers 403 with
-        `code: "REQS007"` (prepare it with `POST /v1/extract/prepared-domains`);
-        the legacy domain-not-enabled code is a 402 with `code: "AUTH010"`.
-        On either, `mode="auto"` retries once with `autoparse=True` instead of
-        returning the error response. Pass `fallback_to_autoparse=False` to
-        disable that and get the raw error response back.
+        Extract works on any prepared domain (you can prepare one yourself
+        with `POST /v1/extract/prepared-domains`), with any active API key.
+        A 403 with `code: "REQS007"` means Extract can't extract this page.
+        Check `reason` in the problem body (`response.json().get("reason")`)
+        when present; older responses may not carry it:
+
+          - `domain_not_prepared`: prepare the domain with `prepare_domain()`,
+            poll `prepared_domain_status()` until `status` is `"ready"`, then
+            retry.
+          - `page_not_recognised`: the domain is served, but this page isn't
+            one Extract extracts (a home, search, login or category page).
+            Try a product, article or listing page, or retry once in a few
+            minutes. Preparing the domain again changes nothing.
+          - `domain_not_enabled`: preparing doesn't help. Use
+            `css_extractor`, or contact support.
+
+        The legacy domain-not-enabled code is a 402 with `code: "AUTH010"`.
+        Both error responses are returned to you as is, in one request.
+
+        `fallback_to_autoparse` is deprecated and has no effect: extract()
+        never retries with autoparse. Passing True emits a
+        `DeprecationWarning`.
 
         A 422 with `code: "no_data_extracted"` means Extract knows this kind
         of page but found no data on it. It is not charged and is never
         retried; check it with `is_no_data_extracted(response)`.
 
         `adaptive_stealth=True` (default) also sends Adaptive Stealth Mode
         (`mode: "auto"` at the wire-param level - unrelated to this method's own
-        `mode` argument) on both the extract attempt and the Autoparse fallback,
-        so a target needing `js_render`/`premium_proxy` gets escalated
+        `mode` argument), so a target needing `js_render`/`premium_proxy` gets escalated
         automatically instead of failing with REQS002. It is skipped when
         `params` sets `js_render` or `premium_proxy` (the API rejects either
         combined with Adaptive Stealth Mode) or its own wire `mode`. Pass
         `adaptive_stealth=False` to turn it off entirely.
         """
+        if fallback_to_autoparse:
+            warnings.warn(_FALLBACK_DEPRECATED, DeprecationWarning, stacklevel=2)
         final_params = dict(params) if params else {}
         final_params["extract"] = mode
         _with_adaptive_stealth(final_params, adaptive_stealth)
-        response = self.fetch(url, final_params, headers, **kwargs)
-
-        if mode == "auto" and fallback_to_autoparse and _is_extract_domain_unavailable(response):
-            autoparse_params = dict(params) if params else {}
-            autoparse_params["autoparse"] = True
-            _with_adaptive_stealth(autoparse_params, adaptive_stealth)
-            return self.fetch(url, autoparse_params, headers, **kwargs)
-
-        return response
+        return self.fetch(url, final_params, headers, **kwargs)
 
     def post(
         self,
@@ -231,6 +316,64 @@ def put(
     ) -> requests.Response:
         return self._worker("PUT", url, params, headers, data=data, **kwargs)
 
+    # ---- Extract prepared domains ----
+
+    def prepare_domain(self, domain: str, example_urls: list[str]) -> dict:
+        """Submit `domain` for Extract preparation
+        (`POST /v1/extract/prepared-domains`) and return the parsed status
+        object the API answers with `202`: `domain`, `status` (`preparing`,
+        `ready` or `failed`), `state`, `submitted_at`, `updated_at`.
+
+        `domain` is the domain without a scheme. `example_urls` are 1 to 5
+        URLs on that domain, one per kind of page you want to extract.
+        Submitting a domain that is already prepared returns its current
+        status instead of starting a new run.
+
+        Preparation takes minutes. This method does not wait for it: call
+        `prepared_domain_status(domain)` until `status` is no longer
+        `"preparing"`. These endpoints don't fetch pages and cost no
+        credits.
+
+        The client's `retries` setting doesn't apply to these endpoints.
+        Raises `ValueError` for an empty `domain`, before any request, and
+        `PreparedDomainError` on a non-2xx answer, for example 400
+        `invalid_argument`, 409 `in_progress`, 429 `out_of_allowance` or
+        503 `overloaded` / `disabled`.
+        """
+        _check_domain(domain)
+        response = _prepared_domains_request(
+            "POST",
+            self._prepared_domains_url(),
+            json={"domain": domain, "example_urls": list(example_urls)},
+            headers=self._prepared_domains_headers(),
+        )
+        return _prepared_domain_body(response)
+
+    def prepared_domain_status(self, domain: str) -> dict:
+        """Read the preparation status of `domain`
+        (`GET /v1/extract/prepared-domains/{domain}`) and return the parsed
+        status object. `status` is `preparing`, `ready` or `failed`; a
+        failed preparation carries a `failure` object with `code`, `detail`
+        and `slot_consumed`. `state` and `roles` report progress.
+
+        Raises `ValueError` for an empty `domain`, before any request, and
+        `PreparedDomainError` on a non-2xx answer, for example 404
+        `not_found` when your account never submitted `domain`.
+        """
+        _check_domain(domain)
+        response = _prepared_domains_request(
+            "GET",
+            f"{self._prepared_domains_url()}/{quote(domain, safe='')}",
+            headers=self._prepared_domains_headers(),
+        )
+        return _prepared_domain_body(response)
+
+    def _prepared_domains_url(self) -> str:
+        return self.api_url.rstrip("/") + "/extract/prepared-domains"
+
+    def _prepared_domains_headers(self) -> dict:
+        return {"X-API-Key": self.apikey, "User-Agent": DEFAULT_USER_AGENT}
+
     # ---- async-flavoured wrappers (thread-pool offload) ----
 
     async def fetch_async(
@@ -258,25 +401,34 @@ async def extract_async(
         params: dict | None = None,
         headers: dict | None = None,
         mode: str = "auto",
-        fallback_to_autoparse: bool = True,
+        fallback_to_autoparse: bool = False,
         adaptive_stealth: bool = True,
         **kwargs: Any,
     ) -> requests.Response:
         """Async counterpart of `extract()` - see its docstring for the
-        REQS007 / AUTH010 -> Autoparse fallback behavior,
-        `fallback_to_autoparse`, and `adaptive_stealth`."""
+        REQS007 `reason` cases, the deprecated `fallback_to_autoparse`, and
+        `adaptive_stealth`."""
+        if fallback_to_autoparse:
+            warnings.warn(_FALLBACK_DEPRECATED, DeprecationWarning, stacklevel=2)
         final_params = dict(params) if params else {}
         final_params["extract"] = mode
         _with_adaptive_stealth(final_params, adaptive_stealth)
-        response = await self.fetch_async(url, final_params, headers, **kwargs)
+        return await self.fetch_async(url, final_params, headers, **kwargs)
 
-        if mode == "auto" and fallback_to_autoparse and _is_extract_domain_unavailable(response):
-            autoparse_params = dict(params) if params else {}
-            autoparse_params["autoparse"] = True
-            _with_adaptive_stealth(autoparse_params, adaptive_stealth)
-            return await self.fetch_async(url, autoparse_params, headers, **kwargs)
+    async def prepare_domain_async(self, domain: str, example_urls: list[str]) -> dict:
+        """Async counterpart of `prepare_domain()`, run on the client's thread pool."""
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(
+            self.executor, partial(self.prepare_domain, domain, example_urls)
+        )
 
-        return response
+    async def prepared_domain_status_async(self, domain: str) -> dict:
+        """Async counterpart of `prepared_domain_status()`, run on the client's
+        thread pool."""
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(
+            self.executor, partial(self.prepared_domain_status, domain)
+        )
 
     async def post_async(
         self,