From 5a049f7aa1721fb0d5db265c3817b3d2f34b2605 Mon Sep 17 00:00:00 2001 From: Siddharth Mishra <33321224+Siddharth-DWT@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:28:02 +0530 Subject: [PATCH] Crawls: a crawl's webhook takes include, and the MCP tools stop saying spreadsheets are read - crawl()'s docstring and the README describe `"include": ["markdown"]` on a crawl's webhook, which the API has accepted since 23 October 2026: each page's markdown in the crawl.page messages, up to about 1 MB a message, a page that does not fit (or was not stored) as `markdown: None` with `markdownOmitted` and a `contentUrl`, an API path read with the key (or `job.page(url)`); it needs crawl.page among the events, as it is when they are left out. The client already sent it. - The MCP settings guide names `webhook_include`, the project setting that puts added and modified pages' markdown in page.changed (not sent to subscriptions). - The MCP tools' parse_documents parameter, its helper's docstring and the settings guide said spreadsheets are read; PDFs (a scanned one by OCR) and Word (.docx) files are, other Office files are recorded as seen. - A test that the webhook is sent as given; CHANGELOG, Unreleased. --- CHANGELOG.md | 2 ++ README.md | 2 +- mesharc/__init__.py | 7 +++++++ mesharc/mcp.py | 11 +++++++---- tests/test_client.py | 8 ++++++++ 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06aeea6..cc10868 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,9 +43,11 @@ All notable changes to this package are recorded here. The format follows Each returns the API's JSON. `request` is a POST /search or POST /agent body in the API's own (camelCase) names, so a news monitor says `"sources": ["news"]` (`web_search(news=True)` maps to that; `monitors.create` does not); a monitored search reads its results page only, so `scrape` is refused; a search baseline's `limit` must be at least the monitor's. The webhook hears `search.changed` / `agent.changed`, and its secret comes back once as `webhookSecret`. Creating and running a monitor spend credits, and every call but the three reads needs a key that can write. `arc.monitor()`, the workspace's job queue, is unchanged. **This needs an API that has the `/monitors` route**, which is on the API's feature branch and not yet on mesharc.dev. - **`run_agent`, a sixteenth MCP tool, so an assistant can hand a research question to the agent.** It needs write access; a read-only connection is answered `{"code": "read_only"}`. It returns at once with a job, hosted or local, and a retry that lands on a finished run gets its result; `get_job` and `cancel_job` take `kind="agent"` to follow and stop it, and a run past its 7 days answers `status: "expired"`. A finished answer carries `fieldSources`, where each value of `data` came from, and keeps within 60,000 characters: `fieldSources` is left out first, then its sources are cut to the first 50, then `data` to its JSON text, with a note saying where the full answer is. The model's tokens are charged at the model provider's price plus 20%. It sends its own idempotency key, made from its inputs and the ten-minute window, so a retry in the same ten minutes lands on the same run; a 409 saying that key is still being accepted (code `in_flight`, or an older API's message naming the Idempotency-Key) is answered as running, and any other 409 as an error. An output schema describes its answer. **This needs an API that has the `/agent` route**, which is not live on mesharc.dev yet. - **`continue_agent`, a seventeenth MCP tool, carries on an agent run that stopped at its credit limit.** `continue_agent(id, max_credits=None, max_steps=None)` starts a new run on the same thread, which resumes where the stopped one was and keeps the pages it read (not paid for again); left out, the budget and steps are the stopped run's. It needs write access; a read-only connection is answered `{"code": "read_only"}`. It returns at once with the new run's job, as `run_agent` does, for `get_job` and `cancel_job` with `kind="agent"`, and sends its own idempotency key, made from its inputs and the ten-minute window. A run already carried on (a retry past that window included) is answered with the run that carried it on, and nothing new is started. A run that did not stop at its limit is answered with code `conflict`, an unknown one `not_found`, one past its 7 days `expired`. A `credit_limit` answer's note now points at `continue_agent` with the run's id, or at the run that already carried it on. **This needs an API that has the `/agent/{id}/continue` route**, which is not live on mesharc.dev yet. +- **A crawl's `webhook` takes `"include": ["markdown"]`**, which puts each page's markdown in the `crawl.page` messages (about 1 MB a message, the rest by a `contentUrl`); `crawl()`'s docstring and the README say so. The MCP settings guide names `webhook_include`, the project setting that does the same for `page.changed`. Live on the API since 23 October 2026. ### Fixed +- The MCP tools' `parse_documents` parameter and settings guide said spreadsheets are read; it reads PDFs (a scanned one by OCR) and Word (`.docx`) files, and records other Office files as seen. - `agent_batches.wait()` waits while a table is `cancelling` -- its rows still stopping -- and returns once it has ended. It returned at `cancelling`, before the rows had stopped. - `AgentRun.follows_run_id` and `followed_by` name the message a follow-up follows and the one asked after it, as `continues_run_id` and `continued_by` do for a continuation. - `monitors.create` and `update` type `sentiment` as `False` or `{"connectionId", "only"}`; `True` was allowed by the type but always refused by the API. diff --git a/README.md b/README.md index b2a538e..e1b6389 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ A one-shot crawl expires after 30 days. If the site is worth watching: project = job.keep(name="Docs", schedule="weekly") ``` -A `webhook=` in the options (`{"url", "events", "metadata"}`) is told about `crawl.started`, `crawl.page` (fifty pages a message) and `crawl.completed`; its signing secret comes back once as `job.webhook_secret`. +A `webhook=` in the options (`{"url", "events", "metadata", "include"}`) is told about `crawl.started`, `crawl.page` (fifty pages a message) and `crawl.completed`; its signing secret comes back once as `job.webhook_secret`. With `"include": ["markdown"]` the `crawl.page` messages carry each page's markdown too, up to about 1 MB a message: a page that does not fit, or was not stored, comes as `"markdown": null` with `markdownOmitted` (`too_large` or `not_stored`) and a `contentUrl`, an API path read with your key (`job.page(url)` reads the same). It needs `crawl.page` among the events, as it is when `events` is left out. ## Mapping a site diff --git a/mesharc/__init__.py b/mesharc/__init__.py index 9e92f15..61c56a7 100644 --- a/mesharc/__init__.py +++ b/mesharc/__init__.py @@ -1230,6 +1230,13 @@ def crawl(self, url: str, wait: bool = False, poll: float = 3.0, timeout: float ``limit``, ``maxDepth``, ``includePaths``, ``excludePaths``, ``crawlMode``, ``maxAge``, ``maxTier``, ``scrapeOptions``, ``webhook`` -- and ``config`` takes any project setting directly. + + ``webhook`` is a URL, or ``{"url", "events", "metadata", "include"}``: + ``"include": ["markdown"]`` (the one value) puts each page's markdown + in the ``crawl.page`` messages, up to about 1 MB a message; a page that + does not fit, or was not stored, comes as ``markdown: None`` with + ``markdownOmitted`` and a ``contentUrl``, an API path read with your + key (``job.page(url)`` reads the same). """ job = Crawl(self._h, self._h("POST", "/crawl", json={"url": url, **opts}, idempotency_key=idempotency_key)) if wait: diff --git a/mesharc/mcp.py b/mesharc/mcp.py index f37b187..7847df6 100644 --- a/mesharc/mcp.py +++ b/mesharc/mcp.py @@ -124,8 +124,8 @@ def _acts(*, destructive=False, idempotent=False, open_world=True): FetchConfig = Annotated[dict | None, Field( description="Fetch settings to override, as {key: value}; omit for the defaults.")] ParseDocuments = Annotated[bool, Field( - description="Read PDF, Word and spreadsheet URLs as text (default true). A parse_documents key in config " - "wins.")] + description="Read PDF (a scanned one by OCR) and Word (.docx) URLs as text (default true). A " + "parse_documents key in config wins.")] # What each tool answers, as output schemas. Every field is optional and any @@ -906,7 +906,7 @@ def _shape_batch(base, rows, reserve): def _with_documents(config, parse_documents): - """A PDF, a Word file or a spreadsheet handed to an assistant should be + """A PDF or a Word file handed to an assistant should be read, not reported as skipped: documents are parsed unless the caller says otherwise or the config already decides.""" cfg = dict(config or {}) @@ -1789,7 +1789,8 @@ def go(): "exclude_tags": "CSS selectors to drop, e.g. ['.comments', '#newsletter']", "min_words": "pages shorter than this are recorded but not compared (0 = keep all)", "languages": "list of language codes to keep, e.g. ['en']; empty = all", - "parse_documents": "true reads PDFs, Word and spreadsheet files the crawl meets", + "parse_documents": "true reads the PDFs (a scanned one by OCR, its first 30 pages) and Word (.docx) files " + "the crawl meets; other Office files are recorded as seen, not read", "respect_robots": "obey robots.txt (default true)", "allow_subdomains": "follow links to subdomains of the seed's domain", "use_proxy": "route through the residential exits (costs more; for walled sites)", @@ -1807,6 +1808,8 @@ def go(): "max_output_tokens 0 = sized to the schema (default) else 200-16000", "webhook_url": "where to POST run and change events", "webhook_events": "which events to send, e.g. ['run.finished', 'page.changed']", + "webhook_include": "['markdown'] puts each added or modified page's markdown in page.changed (about 1 MB " + "a message, the rest by a contentUrl); not sent to subscriptions; [] = off (default)", "notify_min_words": "a change smaller than this many words is not notified", "chat_url": "a Slack, Discord, Teams, Mattermost or Google Chat incoming-webhook URL: the run record is posted " "there when a run finds something (never when nothing changed). Read back masked; sending the " diff --git a/tests/test_client.py b/tests/test_client.py index 3ce59f8..622aa69 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -111,6 +111,14 @@ def test_a_crawl_handle_pages_through_its_rows_and_follows_the_cursor(): assert "cursor=abc" in str(arc.calls[2].url) +def test_a_crawl_sends_its_webhook_as_given_include_and_all(): + arc = scripted({"id": "c3", "status": "running", "url": "https://d.test/", "webhookSecret": "whsec_y"}) + webhook = {"url": "https://hooks.test/h", "events": ["crawl.page"], "include": ["markdown"]} + job = arc.crawl("https://d.test/", limit=5, webhook=webhook) + assert json.loads(arc.calls[0].content) == {"url": "https://d.test/", "limit": 5, "webhook": webhook} + assert job.webhook_secret == "whsec_y" + + def test_a_web_search_posts_its_body_with_the_key_and_polls_by_id(): arc = scripted({"id": "s1", "status": "running", "data": []}, {"id": "s1", "status": "done", "data": [{"url": "https://a.test/"}]})