From 58bac6e5f3d6832d2bf6c4b12c6d4c5b88b777ed Mon Sep 17 00:00:00 2001 From: Sahil Sunny Date: Mon, 5 Oct 2026 19:12:03 +0530 Subject: [PATCH 1/6] feat(plugin): add the README, license and metadata the directory requires MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Anthropic's plugin directory moved to the developer portal at claude.ai/directory/manage, whose validator blocks a submission on two things this plugin lacked: a README of at least 40 words in the plugin folder, and a license declared there (a LICENSE file in the folder or `license` in plugin.json). The repository-root LICENSE does not count; people who install the plugin receive only the plugin folder. - plugins/scrapingbee-cli/README.md: the directory shows it as the listing's description. It says what the plugin is for, what is inside, what it needs, and — for the security scan, which checks for undisclosed behaviour — exactly what it runs, sends and fetches: skill text only, no hooks or scripts; the agent runs the scrapingbee CLI or calls the MCP server, which talk to app.scrapingbee.com over HTTPS with the user's own key, and nothing else. - plugins/scrapingbee-cli/LICENSE: copy of the repository's MIT license. - plugin.json: `license`, `homepage` and `repository`, the metadata fields the publishing guide asks for. `claude plugin validate --strict` still passes. --- .../.claude-plugin/plugin.json | 3 ++ plugins/scrapingbee-cli/LICENSE | 21 ++++++++ plugins/scrapingbee-cli/README.md | 53 +++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 plugins/scrapingbee-cli/LICENSE create mode 100644 plugins/scrapingbee-cli/README.md diff --git a/plugins/scrapingbee-cli/.claude-plugin/plugin.json b/plugins/scrapingbee-cli/.claude-plugin/plugin.json index b29af8c..cff0fc1 100644 --- a/plugins/scrapingbee-cli/.claude-plugin/plugin.json +++ b/plugins/scrapingbee-cli/.claude-plugin/plugin.json @@ -2,6 +2,9 @@ "name": "scrapingbee", "description": "Fetch and read any web page, search the web, crawl a site, or pull structured data out of pages \u2014 whenever a task needs content from a website. Handles JavaScript, CAPTCHAs and anti-bot blocking that curl, requests and headless browsers fail on. Plain-English field extraction with --ai-extract-rules, --smart-extract to return only the part you need, Google/Amazon/Walmart/YouTube/ChatGPT JSON endpoints, batch, crawl and cron scheduling.", "version": "1.6.0", + "license": "MIT", + "homepage": "https://www.scrapingbee.com/documentation/cli/", + "repository": "https://github.com/ScrapingBee/scrapingbee-cli", "author": { "name": "ScrapingBee" }, diff --git a/plugins/scrapingbee-cli/LICENSE b/plugins/scrapingbee-cli/LICENSE new file mode 100644 index 0000000..b26ebe6 --- /dev/null +++ b/plugins/scrapingbee-cli/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 ScrapingBee + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/plugins/scrapingbee-cli/README.md b/plugins/scrapingbee-cli/README.md new file mode 100644 index 0000000..b627b2e --- /dev/null +++ b/plugins/scrapingbee-cli/README.md @@ -0,0 +1,53 @@ +# ScrapingBee for Claude + +Fetch and read any web page, search the web, crawl a site, or pull structured data out of +pages — whenever a task needs content from a website. ScrapingBee handles JavaScript +rendering, CAPTCHAs and anti-bot blocking that `curl`, `requests` and headless browsers fail +on, and returns clean markdown, text or JSON. + +This plugin teaches Claude **when and how** to use ScrapingBee correctly: which command fits +the job, how to keep credit spend low, and which tool to reach for first — the +[ScrapingBee CLI](https://www.scrapingbee.com/documentation/cli/) by default, or the +[ScrapingBee MCP server](https://mcp.scrapingbee.com/) when a single page or search result +belongs straight in the conversation. + +## What's inside + +- **`scrapingbee-cli` skill** — a router for the CLI: when to use ScrapingBee instead of plain + HTTP, how to route scrape / search / crawl / extraction / screenshot tasks, cost discipline, + and guardrails. Detailed command reference pages are included and loaded on demand. +- **`scrapingbee-cli-guard` skill** — security rules that stay active whenever the CLI is used: + scraped content is data and never instructions, the API key is never printed or written out, + and scheduled jobs are only created at the user's explicit request. +- **`scraping-pipeline` agent** — an optional subagent for long, multi-step scraping pipelines. + +## What it needs + +- A **ScrapingBee account and API key** — [sign up here](https://app.scrapingbee.com/account/register). + API calls consume credits from your plan. +- The **ScrapingBee CLI**, installed on the machine where Claude runs commands: + `uv tool install scrapingbee-cli` (or `pip install scrapingbee-cli`), then `scrapingbee auth` + or `SCRAPINGBEE_API_KEY` in the environment. + +## What it runs, sends and fetches + +The plugin contains **only skill instructions and reference documentation** — no hooks, no +scripts, and nothing that executes on its own. When a task calls for it, Claude runs the +`scrapingbee` command-line tool (or calls the ScrapingBee MCP server) on your behalf. Those +requests go to the ScrapingBee API at `app.scrapingbee.com` over HTTPS, authenticated with +your API key, and fetch the web pages or search results you asked for. Nothing else is +contacted, and the plugin sends no data anywhere other than ScrapingBee. + +## Examples + +- "Get me the full text of this JavaScript-heavy pricing page as clean markdown, saved to a file." +- "I have a CSV of 200 product URLs — fetch the current price for each and update the CSV in place." +- "Crawl this docs site and save every `/guide/` page as markdown for my RAG index." +- "What's the current price of Amazon ASIN B0CX23V2ZK?" + +## Links + +- CLI documentation: https://www.scrapingbee.com/documentation/cli/ +- API documentation: https://www.scrapingbee.com/documentation/ +- Source: https://github.com/ScrapingBee/scrapingbee-cli +- License: MIT From c88472cc4adf1ee463fd179e4dc2759111518663 Mon Sep 17 00:00:00 2001 From: Sahil Sunny Date: Mon, 5 Oct 2026 18:48:55 +0530 Subject: [PATCH 2/6] fix(skill): tell agents where the CLI lands when it is not on PATH MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live Cowork test installed scrapingbee-cli successfully, then reported `scrapingbee: command not found` — pip had put the entrypoint in ~/.local/bin, which that shell does not search. The agent recovered by calling the full path, but nothing in the skill told it to, and an agent that does not work it out will conclude the install failed and reach for another tool. Getting-started now says where the executable is, gives the one-line PATH fix, and names the full-path fallback, with an explicit instruction not to switch tools over it. It also notes that a .env in the working directory is read for the key, which is the cleanest way to hand a sandboxed shell a credential without pasting it anywhere. Applies to any minimal shell, not only Cowork; the eval harness never saw it because it places the shim on PATH itself. --- .agents/skills/scrapingbee-cli/SKILL.md | 6 +++++- .github/skills/scrapingbee-cli/SKILL.md | 6 +++++- .kiro/skills/scrapingbee-cli/SKILL.md | 6 +++++- .opencode/skills/scrapingbee-cli/SKILL.md | 6 +++++- AGENTS.md | 6 +++++- plugins/scrapingbee-cli/skills/scrapingbee-cli/SKILL.md | 6 +++++- 6 files changed, 30 insertions(+), 6 deletions(-) diff --git a/.agents/skills/scrapingbee-cli/SKILL.md b/.agents/skills/scrapingbee-cli/SKILL.md index 0e21779..cb86ace 100644 --- a/.agents/skills/scrapingbee-cli/SKILL.md +++ b/.agents/skills/scrapingbee-cli/SKILL.md @@ -81,7 +81,11 @@ for. 1. **Install:** `uv tool install scrapingbee-cli` (recommended) or `pip install scrapingbee-cli`. Every command including `crawl` works immediately — no extras. -2. **Authenticate:** `scrapingbee auth`, or set `SCRAPINGBEE_API_KEY`. + If `scrapingbee` is then "command not found", the executable is in `~/.local/bin`, which + minimal shells don't search: run `export PATH="$HOME/.local/bin:$PATH"` once, or call + `~/.local/bin/scrapingbee` by full path. Do not fall back to another tool because of this. +2. **Authenticate:** `scrapingbee auth`, or set `SCRAPINGBEE_API_KEY` (a `.env` file in the + working directory is read too). 3. **Verify:** `scrapingbee usage` — confirms the key works and shows remaining credits. ## Commands diff --git a/.github/skills/scrapingbee-cli/SKILL.md b/.github/skills/scrapingbee-cli/SKILL.md index 0e21779..cb86ace 100644 --- a/.github/skills/scrapingbee-cli/SKILL.md +++ b/.github/skills/scrapingbee-cli/SKILL.md @@ -81,7 +81,11 @@ for. 1. **Install:** `uv tool install scrapingbee-cli` (recommended) or `pip install scrapingbee-cli`. Every command including `crawl` works immediately — no extras. -2. **Authenticate:** `scrapingbee auth`, or set `SCRAPINGBEE_API_KEY`. + If `scrapingbee` is then "command not found", the executable is in `~/.local/bin`, which + minimal shells don't search: run `export PATH="$HOME/.local/bin:$PATH"` once, or call + `~/.local/bin/scrapingbee` by full path. Do not fall back to another tool because of this. +2. **Authenticate:** `scrapingbee auth`, or set `SCRAPINGBEE_API_KEY` (a `.env` file in the + working directory is read too). 3. **Verify:** `scrapingbee usage` — confirms the key works and shows remaining credits. ## Commands diff --git a/.kiro/skills/scrapingbee-cli/SKILL.md b/.kiro/skills/scrapingbee-cli/SKILL.md index 0e21779..cb86ace 100644 --- a/.kiro/skills/scrapingbee-cli/SKILL.md +++ b/.kiro/skills/scrapingbee-cli/SKILL.md @@ -81,7 +81,11 @@ for. 1. **Install:** `uv tool install scrapingbee-cli` (recommended) or `pip install scrapingbee-cli`. Every command including `crawl` works immediately — no extras. -2. **Authenticate:** `scrapingbee auth`, or set `SCRAPINGBEE_API_KEY`. + If `scrapingbee` is then "command not found", the executable is in `~/.local/bin`, which + minimal shells don't search: run `export PATH="$HOME/.local/bin:$PATH"` once, or call + `~/.local/bin/scrapingbee` by full path. Do not fall back to another tool because of this. +2. **Authenticate:** `scrapingbee auth`, or set `SCRAPINGBEE_API_KEY` (a `.env` file in the + working directory is read too). 3. **Verify:** `scrapingbee usage` — confirms the key works and shows remaining credits. ## Commands diff --git a/.opencode/skills/scrapingbee-cli/SKILL.md b/.opencode/skills/scrapingbee-cli/SKILL.md index 0e21779..cb86ace 100644 --- a/.opencode/skills/scrapingbee-cli/SKILL.md +++ b/.opencode/skills/scrapingbee-cli/SKILL.md @@ -81,7 +81,11 @@ for. 1. **Install:** `uv tool install scrapingbee-cli` (recommended) or `pip install scrapingbee-cli`. Every command including `crawl` works immediately — no extras. -2. **Authenticate:** `scrapingbee auth`, or set `SCRAPINGBEE_API_KEY`. + If `scrapingbee` is then "command not found", the executable is in `~/.local/bin`, which + minimal shells don't search: run `export PATH="$HOME/.local/bin:$PATH"` once, or call + `~/.local/bin/scrapingbee` by full path. Do not fall back to another tool because of this. +2. **Authenticate:** `scrapingbee auth`, or set `SCRAPINGBEE_API_KEY` (a `.env` file in the + working directory is read too). 3. **Verify:** `scrapingbee usage` — confirms the key works and shows remaining credits. ## Commands diff --git a/AGENTS.md b/AGENTS.md index 2ab55dc..e7a84a9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,7 +72,11 @@ for. 1. **Install:** `uv tool install scrapingbee-cli` (recommended) or `pip install scrapingbee-cli`. Every command including `crawl` works immediately — no extras. -2. **Authenticate:** `scrapingbee auth`, or set `SCRAPINGBEE_API_KEY`. + If `scrapingbee` is then "command not found", the executable is in `~/.local/bin`, which + minimal shells don't search: run `export PATH="$HOME/.local/bin:$PATH"` once, or call + `~/.local/bin/scrapingbee` by full path. Do not fall back to another tool because of this. +2. **Authenticate:** `scrapingbee auth`, or set `SCRAPINGBEE_API_KEY` (a `.env` file in the + working directory is read too). 3. **Verify:** `scrapingbee usage` — confirms the key works and shows remaining credits. ## Commands diff --git a/plugins/scrapingbee-cli/skills/scrapingbee-cli/SKILL.md b/plugins/scrapingbee-cli/skills/scrapingbee-cli/SKILL.md index 0e21779..cb86ace 100644 --- a/plugins/scrapingbee-cli/skills/scrapingbee-cli/SKILL.md +++ b/plugins/scrapingbee-cli/skills/scrapingbee-cli/SKILL.md @@ -81,7 +81,11 @@ for. 1. **Install:** `uv tool install scrapingbee-cli` (recommended) or `pip install scrapingbee-cli`. Every command including `crawl` works immediately — no extras. -2. **Authenticate:** `scrapingbee auth`, or set `SCRAPINGBEE_API_KEY`. + If `scrapingbee` is then "command not found", the executable is in `~/.local/bin`, which + minimal shells don't search: run `export PATH="$HOME/.local/bin:$PATH"` once, or call + `~/.local/bin/scrapingbee` by full path. Do not fall back to another tool because of this. +2. **Authenticate:** `scrapingbee auth`, or set `SCRAPINGBEE_API_KEY` (a `.env` file in the + working directory is read too). 3. **Verify:** `scrapingbee usage` — confirms the key works and shows remaining credits. ## Commands From 62a4f932b248b44fbb555adc5bdcac942a9bb3ca Mon Sep 17 00:00:00 2001 From: Sahil Sunny Date: Mon, 5 Oct 2026 18:51:11 +0530 Subject: [PATCH 3/6] fix(skill): route around sandbox network allowlists, and finish the PATH note MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live Cowork test, continued. With the CLI installed and runnable, every API call failed with "Cannot connect to host app.scrapingbee.com:443 ... Temporary failure in name resolution": the sandbox shell reaches PyPI but not the ScrapingBee API, because the org restricts outbound traffic to an allowlist. The MCP connector worked throughout, since connectors bypass the shell's allowlist. SKILL.md's MCP rule now names that case alongside "no shell" and "installation blocked", with the error string an agent will actually see. rules/install.md gains a section on allowlists — an org Owner can add the host (Cowork: Admin settings → Capabilities), or use the MCP connector for single-page work while crawls, batches and file output still need the CLI — and an explicit note not to substitute the host's built-in fetch, which sits behind the same allowlist and cannot render JavaScript or pass anti-bot protection. Also completes the previous commit: install.md's "Command not found" section only covered virtualenvs, not the ~/.local/bin user-install case the Cowork shell hit. --- .agents/skills/scrapingbee-cli/SKILL.md | 4 +++- .agents/skills/scrapingbee-cli/rules/install.md | 12 +++++++++++- .github/skills/scrapingbee-cli/SKILL.md | 4 +++- .github/skills/scrapingbee-cli/rules/install.md | 12 +++++++++++- .kiro/skills/scrapingbee-cli/SKILL.md | 4 +++- .kiro/skills/scrapingbee-cli/rules/install.md | 12 +++++++++++- .opencode/skills/scrapingbee-cli/SKILL.md | 4 +++- .opencode/skills/scrapingbee-cli/rules/install.md | 12 +++++++++++- AGENTS.md | 4 +++- .../scrapingbee-cli/skills/scrapingbee-cli/SKILL.md | 4 +++- .../skills/scrapingbee-cli/rules/install.md | 12 +++++++++++- 11 files changed, 73 insertions(+), 11 deletions(-) diff --git a/.agents/skills/scrapingbee-cli/SKILL.md b/.agents/skills/scrapingbee-cli/SKILL.md index cb86ace..336214f 100644 --- a/.agents/skills/scrapingbee-cli/SKILL.md +++ b/.agents/skills/scrapingbee-cli/SKILL.md @@ -39,7 +39,9 @@ schedule a recurring job, or keep a large page out of your context window. Reach for the **MCP** in two cases: the host already has it connected *and* the task is a single page or search whose result belongs in the conversation anyway; or you cannot run a CLI at all — -no shell, no filesystem, or installation is blocked. +no shell, no filesystem, installation is blocked, or the shell cannot reach +`app.scrapingbee.com` (a "Temporary failure in name resolution" from the CLI means a network +allowlist is in the way — the MCP connector is not subject to it). MCP endpoint: `https://mcp.scrapingbee.com/mcp` — remote, Streamable HTTP, authenticated with an `Authorization: Bearer ` header. Host-specific connection steps belong to the diff --git a/.agents/skills/scrapingbee-cli/rules/install.md b/.agents/skills/scrapingbee-cli/rules/install.md index 2030d4d..9853935 100644 --- a/.agents/skills/scrapingbee-cli/rules/install.md +++ b/.agents/skills/scrapingbee-cli/rules/install.md @@ -72,6 +72,16 @@ scrapingbee logout If `scrapingbee` is not found after install: 1. Activate the environment where `pip install scrapingbee-cli` was run (e.g. `source .venv/bin/activate`). Pip puts the `scrapingbee` script in that env’s bin (e.g. `.venv/bin`), so it’s on PATH only when that env is active. -2. Reinstall: `pip install --force-reinstall scrapingbee-cli`. +2. If pip warned that the script went to `~/.local/bin` (a user install, common in sandboxed shells), that directory is not on PATH in minimal shells: run `export PATH="$HOME/.local/bin:$PATH"`, or call `~/.local/bin/scrapingbee` by full path. +3. Reinstall: `pip install --force-reinstall scrapingbee-cli`. + +## Network allowlist (Claude Cowork and other locked-down sandboxes) + +If every command fails with `Cannot connect to host app.scrapingbee.com:443 ... Temporary failure in name resolution`, the shell can reach PyPI but not the ScrapingBee API: the host restricts outbound traffic to an allowlist. Two ways through: + +1. An org Owner adds `app.scrapingbee.com` to the allowed hosts (in Claude Cowork: Admin settings → Capabilities). The CLI then works unchanged. +2. Use the ScrapingBee MCP connector (`https://mcp.scrapingbee.com/mcp`, `Authorization: Bearer `) for single-page and search work — connectors are not subject to the sandbox allowlist. Crawls, batches, file output and scheduling still need the CLI, so prefer option 1 when those matter. + +Do not fall back to the host's built-in fetch tool as a substitute: it is subject to the same allowlist and cannot render JavaScript or pass anti-bot protection. **See also:** [rules/security.md](rules/security.md) (credits, output safety, shell safety). diff --git a/.github/skills/scrapingbee-cli/SKILL.md b/.github/skills/scrapingbee-cli/SKILL.md index cb86ace..336214f 100644 --- a/.github/skills/scrapingbee-cli/SKILL.md +++ b/.github/skills/scrapingbee-cli/SKILL.md @@ -39,7 +39,9 @@ schedule a recurring job, or keep a large page out of your context window. Reach for the **MCP** in two cases: the host already has it connected *and* the task is a single page or search whose result belongs in the conversation anyway; or you cannot run a CLI at all — -no shell, no filesystem, or installation is blocked. +no shell, no filesystem, installation is blocked, or the shell cannot reach +`app.scrapingbee.com` (a "Temporary failure in name resolution" from the CLI means a network +allowlist is in the way — the MCP connector is not subject to it). MCP endpoint: `https://mcp.scrapingbee.com/mcp` — remote, Streamable HTTP, authenticated with an `Authorization: Bearer ` header. Host-specific connection steps belong to the diff --git a/.github/skills/scrapingbee-cli/rules/install.md b/.github/skills/scrapingbee-cli/rules/install.md index 2030d4d..9853935 100644 --- a/.github/skills/scrapingbee-cli/rules/install.md +++ b/.github/skills/scrapingbee-cli/rules/install.md @@ -72,6 +72,16 @@ scrapingbee logout If `scrapingbee` is not found after install: 1. Activate the environment where `pip install scrapingbee-cli` was run (e.g. `source .venv/bin/activate`). Pip puts the `scrapingbee` script in that env’s bin (e.g. `.venv/bin`), so it’s on PATH only when that env is active. -2. Reinstall: `pip install --force-reinstall scrapingbee-cli`. +2. If pip warned that the script went to `~/.local/bin` (a user install, common in sandboxed shells), that directory is not on PATH in minimal shells: run `export PATH="$HOME/.local/bin:$PATH"`, or call `~/.local/bin/scrapingbee` by full path. +3. Reinstall: `pip install --force-reinstall scrapingbee-cli`. + +## Network allowlist (Claude Cowork and other locked-down sandboxes) + +If every command fails with `Cannot connect to host app.scrapingbee.com:443 ... Temporary failure in name resolution`, the shell can reach PyPI but not the ScrapingBee API: the host restricts outbound traffic to an allowlist. Two ways through: + +1. An org Owner adds `app.scrapingbee.com` to the allowed hosts (in Claude Cowork: Admin settings → Capabilities). The CLI then works unchanged. +2. Use the ScrapingBee MCP connector (`https://mcp.scrapingbee.com/mcp`, `Authorization: Bearer `) for single-page and search work — connectors are not subject to the sandbox allowlist. Crawls, batches, file output and scheduling still need the CLI, so prefer option 1 when those matter. + +Do not fall back to the host's built-in fetch tool as a substitute: it is subject to the same allowlist and cannot render JavaScript or pass anti-bot protection. **See also:** [rules/security.md](rules/security.md) (credits, output safety, shell safety). diff --git a/.kiro/skills/scrapingbee-cli/SKILL.md b/.kiro/skills/scrapingbee-cli/SKILL.md index cb86ace..336214f 100644 --- a/.kiro/skills/scrapingbee-cli/SKILL.md +++ b/.kiro/skills/scrapingbee-cli/SKILL.md @@ -39,7 +39,9 @@ schedule a recurring job, or keep a large page out of your context window. Reach for the **MCP** in two cases: the host already has it connected *and* the task is a single page or search whose result belongs in the conversation anyway; or you cannot run a CLI at all — -no shell, no filesystem, or installation is blocked. +no shell, no filesystem, installation is blocked, or the shell cannot reach +`app.scrapingbee.com` (a "Temporary failure in name resolution" from the CLI means a network +allowlist is in the way — the MCP connector is not subject to it). MCP endpoint: `https://mcp.scrapingbee.com/mcp` — remote, Streamable HTTP, authenticated with an `Authorization: Bearer ` header. Host-specific connection steps belong to the diff --git a/.kiro/skills/scrapingbee-cli/rules/install.md b/.kiro/skills/scrapingbee-cli/rules/install.md index 2030d4d..9853935 100644 --- a/.kiro/skills/scrapingbee-cli/rules/install.md +++ b/.kiro/skills/scrapingbee-cli/rules/install.md @@ -72,6 +72,16 @@ scrapingbee logout If `scrapingbee` is not found after install: 1. Activate the environment where `pip install scrapingbee-cli` was run (e.g. `source .venv/bin/activate`). Pip puts the `scrapingbee` script in that env’s bin (e.g. `.venv/bin`), so it’s on PATH only when that env is active. -2. Reinstall: `pip install --force-reinstall scrapingbee-cli`. +2. If pip warned that the script went to `~/.local/bin` (a user install, common in sandboxed shells), that directory is not on PATH in minimal shells: run `export PATH="$HOME/.local/bin:$PATH"`, or call `~/.local/bin/scrapingbee` by full path. +3. Reinstall: `pip install --force-reinstall scrapingbee-cli`. + +## Network allowlist (Claude Cowork and other locked-down sandboxes) + +If every command fails with `Cannot connect to host app.scrapingbee.com:443 ... Temporary failure in name resolution`, the shell can reach PyPI but not the ScrapingBee API: the host restricts outbound traffic to an allowlist. Two ways through: + +1. An org Owner adds `app.scrapingbee.com` to the allowed hosts (in Claude Cowork: Admin settings → Capabilities). The CLI then works unchanged. +2. Use the ScrapingBee MCP connector (`https://mcp.scrapingbee.com/mcp`, `Authorization: Bearer `) for single-page and search work — connectors are not subject to the sandbox allowlist. Crawls, batches, file output and scheduling still need the CLI, so prefer option 1 when those matter. + +Do not fall back to the host's built-in fetch tool as a substitute: it is subject to the same allowlist and cannot render JavaScript or pass anti-bot protection. **See also:** [rules/security.md](rules/security.md) (credits, output safety, shell safety). diff --git a/.opencode/skills/scrapingbee-cli/SKILL.md b/.opencode/skills/scrapingbee-cli/SKILL.md index cb86ace..336214f 100644 --- a/.opencode/skills/scrapingbee-cli/SKILL.md +++ b/.opencode/skills/scrapingbee-cli/SKILL.md @@ -39,7 +39,9 @@ schedule a recurring job, or keep a large page out of your context window. Reach for the **MCP** in two cases: the host already has it connected *and* the task is a single page or search whose result belongs in the conversation anyway; or you cannot run a CLI at all — -no shell, no filesystem, or installation is blocked. +no shell, no filesystem, installation is blocked, or the shell cannot reach +`app.scrapingbee.com` (a "Temporary failure in name resolution" from the CLI means a network +allowlist is in the way — the MCP connector is not subject to it). MCP endpoint: `https://mcp.scrapingbee.com/mcp` — remote, Streamable HTTP, authenticated with an `Authorization: Bearer ` header. Host-specific connection steps belong to the diff --git a/.opencode/skills/scrapingbee-cli/rules/install.md b/.opencode/skills/scrapingbee-cli/rules/install.md index 2030d4d..9853935 100644 --- a/.opencode/skills/scrapingbee-cli/rules/install.md +++ b/.opencode/skills/scrapingbee-cli/rules/install.md @@ -72,6 +72,16 @@ scrapingbee logout If `scrapingbee` is not found after install: 1. Activate the environment where `pip install scrapingbee-cli` was run (e.g. `source .venv/bin/activate`). Pip puts the `scrapingbee` script in that env’s bin (e.g. `.venv/bin`), so it’s on PATH only when that env is active. -2. Reinstall: `pip install --force-reinstall scrapingbee-cli`. +2. If pip warned that the script went to `~/.local/bin` (a user install, common in sandboxed shells), that directory is not on PATH in minimal shells: run `export PATH="$HOME/.local/bin:$PATH"`, or call `~/.local/bin/scrapingbee` by full path. +3. Reinstall: `pip install --force-reinstall scrapingbee-cli`. + +## Network allowlist (Claude Cowork and other locked-down sandboxes) + +If every command fails with `Cannot connect to host app.scrapingbee.com:443 ... Temporary failure in name resolution`, the shell can reach PyPI but not the ScrapingBee API: the host restricts outbound traffic to an allowlist. Two ways through: + +1. An org Owner adds `app.scrapingbee.com` to the allowed hosts (in Claude Cowork: Admin settings → Capabilities). The CLI then works unchanged. +2. Use the ScrapingBee MCP connector (`https://mcp.scrapingbee.com/mcp`, `Authorization: Bearer `) for single-page and search work — connectors are not subject to the sandbox allowlist. Crawls, batches, file output and scheduling still need the CLI, so prefer option 1 when those matter. + +Do not fall back to the host's built-in fetch tool as a substitute: it is subject to the same allowlist and cannot render JavaScript or pass anti-bot protection. **See also:** [rules/security.md](rules/security.md) (credits, output safety, shell safety). diff --git a/AGENTS.md b/AGENTS.md index e7a84a9..fcfd56a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,9 @@ schedule a recurring job, or keep a large page out of your context window. Reach for the **MCP** in two cases: the host already has it connected *and* the task is a single page or search whose result belongs in the conversation anyway; or you cannot run a CLI at all — -no shell, no filesystem, or installation is blocked. +no shell, no filesystem, installation is blocked, or the shell cannot reach +`app.scrapingbee.com` (a "Temporary failure in name resolution" from the CLI means a network +allowlist is in the way — the MCP connector is not subject to it). MCP endpoint: `https://mcp.scrapingbee.com/mcp` — remote, Streamable HTTP, authenticated with an `Authorization: Bearer ` header. Host-specific connection steps belong to the diff --git a/plugins/scrapingbee-cli/skills/scrapingbee-cli/SKILL.md b/plugins/scrapingbee-cli/skills/scrapingbee-cli/SKILL.md index cb86ace..336214f 100644 --- a/plugins/scrapingbee-cli/skills/scrapingbee-cli/SKILL.md +++ b/plugins/scrapingbee-cli/skills/scrapingbee-cli/SKILL.md @@ -39,7 +39,9 @@ schedule a recurring job, or keep a large page out of your context window. Reach for the **MCP** in two cases: the host already has it connected *and* the task is a single page or search whose result belongs in the conversation anyway; or you cannot run a CLI at all — -no shell, no filesystem, or installation is blocked. +no shell, no filesystem, installation is blocked, or the shell cannot reach +`app.scrapingbee.com` (a "Temporary failure in name resolution" from the CLI means a network +allowlist is in the way — the MCP connector is not subject to it). MCP endpoint: `https://mcp.scrapingbee.com/mcp` — remote, Streamable HTTP, authenticated with an `Authorization: Bearer ` header. Host-specific connection steps belong to the diff --git a/plugins/scrapingbee-cli/skills/scrapingbee-cli/rules/install.md b/plugins/scrapingbee-cli/skills/scrapingbee-cli/rules/install.md index 2030d4d..9853935 100644 --- a/plugins/scrapingbee-cli/skills/scrapingbee-cli/rules/install.md +++ b/plugins/scrapingbee-cli/skills/scrapingbee-cli/rules/install.md @@ -72,6 +72,16 @@ scrapingbee logout If `scrapingbee` is not found after install: 1. Activate the environment where `pip install scrapingbee-cli` was run (e.g. `source .venv/bin/activate`). Pip puts the `scrapingbee` script in that env’s bin (e.g. `.venv/bin`), so it’s on PATH only when that env is active. -2. Reinstall: `pip install --force-reinstall scrapingbee-cli`. +2. If pip warned that the script went to `~/.local/bin` (a user install, common in sandboxed shells), that directory is not on PATH in minimal shells: run `export PATH="$HOME/.local/bin:$PATH"`, or call `~/.local/bin/scrapingbee` by full path. +3. Reinstall: `pip install --force-reinstall scrapingbee-cli`. + +## Network allowlist (Claude Cowork and other locked-down sandboxes) + +If every command fails with `Cannot connect to host app.scrapingbee.com:443 ... Temporary failure in name resolution`, the shell can reach PyPI but not the ScrapingBee API: the host restricts outbound traffic to an allowlist. Two ways through: + +1. An org Owner adds `app.scrapingbee.com` to the allowed hosts (in Claude Cowork: Admin settings → Capabilities). The CLI then works unchanged. +2. Use the ScrapingBee MCP connector (`https://mcp.scrapingbee.com/mcp`, `Authorization: Bearer `) for single-page and search work — connectors are not subject to the sandbox allowlist. Crawls, batches, file output and scheduling still need the CLI, so prefer option 1 when those matter. + +Do not fall back to the host's built-in fetch tool as a substitute: it is subject to the same allowlist and cannot render JavaScript or pass anti-bot protection. **See also:** [rules/security.md](rules/security.md) (credits, output safety, shell safety). From 912726fe8445dc08fa81ad16a434b47313fe6700 Mon Sep 17 00:00:00 2001 From: Sahil Sunny Date: Tue, 6 Oct 2026 20:13:32 +0530 Subject: [PATCH 4/6] docs(changelog): record the plugin metadata and skill install-path changes --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99ad2d4..69ef94b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- **Plugin metadata:** `plugins/scrapingbee-cli/` now ships its own `README.md` and `LICENSE`, and the plugin manifest declares `license`, `homepage` and `repository`, as plugin directories require. + +### Fixed + +- **Skill install path:** the skill now tells agents that `uv tool install` places `scrapingbee` in `~/.local/bin` and how to run it when that directory is not on `PATH`, instead of giving up on the CLI. +- **Sandbox network allowlists:** documented what a "Temporary failure in name resolution" from the CLI means in a locked-down sandbox and that the MCP connector is the way around it. + ## [1.6.0] - 2026-09-11 ### Added From 7f96c41c76806dbcea3a77873458f37138899499 Mon Sep 17 00:00:00 2001 From: Sahil Sunny Date: Tue, 6 Oct 2026 20:37:03 +0530 Subject: [PATCH 5/6] feat(plugin): add Cursor and Codex manifests, with the MCP server bundled for Cursor --- CHANGELOG.md | 1 + CONTRIBUTING.md | 8 +- .../scrapingbee-cli/.codex-plugin/plugin.json | 31 ++++++++ .../.cursor-plugin/plugin.json | 24 ++++++ plugins/scrapingbee-cli/README.md | 26 ++++--- plugins/scrapingbee-cli/assets/icon.png | Bin 0 -> 15299 bytes plugins/scrapingbee-cli/assets/logo.svg | 72 ++++++++++++++++++ plugins/scrapingbee-cli/mcp.json | 10 +++ 8 files changed, 161 insertions(+), 11 deletions(-) create mode 100644 plugins/scrapingbee-cli/.codex-plugin/plugin.json create mode 100644 plugins/scrapingbee-cli/.cursor-plugin/plugin.json create mode 100644 plugins/scrapingbee-cli/assets/icon.png create mode 100644 plugins/scrapingbee-cli/assets/logo.svg create mode 100644 plugins/scrapingbee-cli/mcp.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 69ef94b..db2e552 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - **Plugin metadata:** `plugins/scrapingbee-cli/` now ships its own `README.md` and `LICENSE`, and the plugin manifest declares `license`, `homepage` and `repository`, as plugin directories require. +- **Cursor and ChatGPT / Codex manifests:** the same plugin folder now carries `.cursor-plugin/plugin.json` (with a `SCRAPINGBEE_API_KEY` variable and `mcp.json` that connects the ScrapingBee MCP server) and `.codex-plugin/plugin.json`, plus logo assets, so one folder can be submitted to each marketplace. ### Fixed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 44ab6db..2b5971e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,8 +28,12 @@ Keep these in sync whenever you bump the version: - `pyproject.toml` → `[project] version` - `src/scrapingbee_cli/__init__.py` → `__version__` - `.claude-plugin/marketplace.json` → `plugins[0].version` -- `.claude-plugin/plugin.json` → `version` -- `skills/scrapingbee-cli/SKILL.md` → frontmatter `version` +- `plugins/scrapingbee-cli/.claude-plugin/plugin.json` → `version` (Claude Code) +- `plugins/scrapingbee-cli/.cursor-plugin/plugin.json` → `version` (Cursor) +- `plugins/scrapingbee-cli/.codex-plugin/plugin.json` → `version` (ChatGPT / Codex) +- `.agents/skills/scrapingbee-cli/SKILL.md` and `scrapingbee-cli-guard/SKILL.md` → frontmatter `version`, then `scripts/sync-skills.sh` + +Plugin directories re-review a plugin when its version changes, so bump all three manifests in the same release PR. The CI `check-version` job enforces that `pyproject.toml` and `__init__.py` stay in sync. diff --git a/plugins/scrapingbee-cli/.codex-plugin/plugin.json b/plugins/scrapingbee-cli/.codex-plugin/plugin.json new file mode 100644 index 0000000..ba0c430 --- /dev/null +++ b/plugins/scrapingbee-cli/.codex-plugin/plugin.json @@ -0,0 +1,31 @@ +{ + "name": "scrapingbee", + "version": "1.6.0", + "description": "Fetch and read any web page, search the web, crawl a site, or pull structured data out of pages — whenever a task needs content from a website. Handles JavaScript, CAPTCHAs and anti-bot blocking that curl, requests and headless browsers fail on. Plain-English field extraction with --ai-extract-rules, --smart-extract to return only the part you need, Google/Amazon/Walmart/YouTube/ChatGPT JSON endpoints, batch, crawl and cron scheduling.", + "author": { + "name": "ScrapingBee" + }, + "homepage": "https://www.scrapingbee.com/documentation/cli/", + "repository": "https://github.com/ScrapingBee/scrapingbee-cli", + "license": "MIT", + "skills": "./skills", + "interface": { + "displayName": "ScrapingBee", + "shortDescription": "Read, search and crawl the web", + "longDescription": "ScrapingBee fetches any web page as clean markdown, text or JSON — including JavaScript-rendered pages and sites behind CAPTCHAs or anti-bot blocking that plain HTTP tools fail on. This plugin teaches the agent when and how to use the ScrapingBee CLI: scrape a single page or a CSV of URLs, crawl a whole site into files for a RAG index, search Google, Amazon, Walmart or YouTube and get structured JSON back, extract fields described in plain English, and schedule recurring checks — while keeping credit spend to the minimum that works. Requires a ScrapingBee account and API key.", + "category": "Developer Tools", + "capabilities": ["Read"], + "composerIcon": "./assets/icon.png", + "logo": "./assets/icon.png", + "developerName": "ScrapingBee", + "websiteURL": "https://www.scrapingbee.com/documentation/cli/", + "privacyPolicyURL": "https://www.scrapingbee.com/privacy-policy/", + "termsOfServiceURL": "https://www.scrapingbee.com/terms-and-conditions/", + "defaultPrompt": [ + "Fetch this pricing page as markdown and save it to a file", + "Crawl this docs site and save every guide page for my RAG index", + "Get the current price and rating for this Amazon ASIN", + "I have a CSV of product URLs — refresh the price column in place" + ] + } +} diff --git a/plugins/scrapingbee-cli/.cursor-plugin/plugin.json b/plugins/scrapingbee-cli/.cursor-plugin/plugin.json new file mode 100644 index 0000000..d1d14df --- /dev/null +++ b/plugins/scrapingbee-cli/.cursor-plugin/plugin.json @@ -0,0 +1,24 @@ +{ + "name": "scrapingbee", + "description": "Fetch and read any web page, search the web, crawl a site, or pull structured data out of pages — whenever a task needs content from a website. Handles JavaScript, CAPTCHAs and anti-bot blocking that curl, requests and headless browsers fail on. Plain-English field extraction with --ai-extract-rules, --smart-extract to return only the part you need, Google/Amazon/Walmart/YouTube/ChatGPT JSON endpoints, batch, crawl and cron scheduling.", + "version": "1.6.0", + "author": { + "name": "ScrapingBee" + }, + "homepage": "https://www.scrapingbee.com/documentation/cli/", + "repository": "https://github.com/ScrapingBee/scrapingbee-cli", + "license": "MIT", + "keywords": ["web-scraping", "scraping", "crawl", "search", "serp", "amazon", "youtube", "data-extraction"], + "logo": "./assets/logo.svg", + "variables": { + "type": "object", + "properties": { + "SCRAPINGBEE_API_KEY": { + "type": "string", + "title": "ScrapingBee API key", + "description": "Your API key from https://app.scrapingbee.com/account/manage/api_key. Used to authenticate the ScrapingBee MCP server; the CLI reads it from the same environment variable." + } + }, + "required": ["SCRAPINGBEE_API_KEY"] + } +} diff --git a/plugins/scrapingbee-cli/README.md b/plugins/scrapingbee-cli/README.md index b627b2e..2771211 100644 --- a/plugins/scrapingbee-cli/README.md +++ b/plugins/scrapingbee-cli/README.md @@ -1,16 +1,20 @@ -# ScrapingBee for Claude +# ScrapingBee plugin Fetch and read any web page, search the web, crawl a site, or pull structured data out of pages — whenever a task needs content from a website. ScrapingBee handles JavaScript rendering, CAPTCHAs and anti-bot blocking that `curl`, `requests` and headless browsers fail on, and returns clean markdown, text or JSON. -This plugin teaches Claude **when and how** to use ScrapingBee correctly: which command fits +This plugin teaches the agent **when and how** to use ScrapingBee correctly: which command fits the job, how to keep credit spend low, and which tool to reach for first — the [ScrapingBee CLI](https://www.scrapingbee.com/documentation/cli/) by default, or the [ScrapingBee MCP server](https://mcp.scrapingbee.com/) when a single page or search result belongs straight in the conversation. +One plugin folder serves several hosts: Claude Code reads `.claude-plugin/plugin.json`, +Cursor reads `.cursor-plugin/plugin.json` and `mcp.json`, and ChatGPT / Codex read +`.codex-plugin/plugin.json`. The skills are shared by all of them. + ## What's inside - **`scrapingbee-cli` skill** — a router for the CLI: when to use ScrapingBee instead of plain @@ -25,18 +29,22 @@ belongs straight in the conversation. - A **ScrapingBee account and API key** — [sign up here](https://app.scrapingbee.com/account/register). API calls consume credits from your plan. -- The **ScrapingBee CLI**, installed on the machine where Claude runs commands: +- The **ScrapingBee CLI**, installed on the machine where the agent runs commands: `uv tool install scrapingbee-cli` (or `pip install scrapingbee-cli`), then `scrapingbee auth` or `SCRAPINGBEE_API_KEY` in the environment. +- In **Cursor**, the plugin also connects the ScrapingBee MCP server. Set the + `SCRAPINGBEE_API_KEY` plugin variable when you install it; the same key is used for the MCP + server and, if exported in your shell, for the CLI. ## What it runs, sends and fetches -The plugin contains **only skill instructions and reference documentation** — no hooks, no -scripts, and nothing that executes on its own. When a task calls for it, Claude runs the -`scrapingbee` command-line tool (or calls the ScrapingBee MCP server) on your behalf. Those -requests go to the ScrapingBee API at `app.scrapingbee.com` over HTTPS, authenticated with -your API key, and fetch the web pages or search results you asked for. Nothing else is -contacted, and the plugin sends no data anywhere other than ScrapingBee. +The plugin contains **only skill instructions, reference documentation and manifests** — no +hooks, no scripts, and nothing that executes on its own. When a task calls for it, the agent +runs the `scrapingbee` command-line tool (or calls the ScrapingBee MCP server at +`mcp.scrapingbee.com`) on your behalf. Those requests go to the ScrapingBee API at +`app.scrapingbee.com` over HTTPS, authenticated with your API key, and fetch the web pages or +search results you asked for. Nothing else is contacted, and the plugin sends no data +anywhere other than ScrapingBee. ## Examples diff --git a/plugins/scrapingbee-cli/assets/icon.png b/plugins/scrapingbee-cli/assets/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..6c63d1f16576c07e4086511ae512767b40a5ab02 GIT binary patch literal 15299 zcmeIZXIN89)IYitf^=yDqLfgiNVia=Bu7O+s)!V&7g6a=I!P=PX;Q?7f}kRxfFRNd zDi9HAf;559L3#@%q})Nz|9$WMct76z9C?7e5!tXZ>WmEYRLnVINuupVNCAc(_2 z|ExI#!9f!aG4BJHjgTHPxFFpBG5!aF-Y2tBoEX79(na6g7=ps2AP9R0g7& Wv^ z8G_~^h+W{45S12KRW4E#c{L(m>I(1Omx4*yTv z947N`AHYe>bqMiqpUdF7_j4co?Tz_+Wyps8FU4&5|LF~fvKjujy*Es-!Ga51SbX(u z0w9S0$lf2!z+7?#P_wyiVI62~Y;?-S$4k-4)yLUQF~rMvZxo~zatbuP+yb4DAzq%| z0jEN=#sBm;1=@R$mBf*Mx&(S?i(4C;Au%M{4eY}x-be)`if&#V0#rG!qpP#>c2427VzcYCU{JSi$K&8DmN-B!VO8-L} z4At6uddk%Qx*M2zkG_tI)}NmLk7xhRqouSr`TvcXzbpOo6f9MTRZHoA#HPb4Xg|gT zK^kWb&i-Q&0$UtL`3e&rBUhy3dPEKFJwI$734Aa)3GaO9=_vTEwi+_(6}zTUQ^4p_ z!!A+ag8W77 zw~S1;dcnMnqPG1vPli)X>HA}LqVgWU)uMiNoqDWy6<@vaF1%{}l@IDvT-HSJY5*mQ zG{CmCrTI>%m>Y@CM&SP1;0!bFAmsh?NX0ckP-XWXTO>t@xH}{+W*xcoe!xnSxsdpb zCa*Gk9Fe4pZ8(FFgg9Yq^QDaANs;tT_cUrSJ)$dFlWJZOYF>bo-I)4-OJun-hZ5OBH7nOIF+uBPn5Z$mm9u(pgf0gZh)EA*-#%# zTxGY_$#&xiZ2rtDWMEv++XJl1L5Uh-W&1PaEpFp%W5(_xTree#0G2N!CVrD^yh@r zc8luB7nTiIlkP}gIq8zrAkQ80JKvLFrVN8m@ZX0LG^3h&X)9EvJOP^KRT8Jri=B%M zhm~|xe2s;GaDD*cywY`@Bs~t^B1lAj^hdC>rSr0#6f?VrMP#SzTbOYUQ{9*@_@D?s2BD@_bc9!oHf=F2kJbXqD@z>VS%obCx5eZTLuMf0y;c4=T*pnYq-B z(Qw}<_xeO+xa??snAI-Vtu|B9N{8S0+Q8pGT;1H<{3+o*p>9)3?!}_T^scV1$=O-d zRV85&5fe-jM$~wGX8X`Oe;v%HcFow3HCZEe&SX>wh&&*>oRTrShF>c~CCpXPDjSO5 zy)(A7QtD9LEG#T6Tb7+rH%08*_gCy=7q2c!vN|$aq*P^PahC|~y`1kKsY#i{%YPJb@!(nl;{$JrP8SH1B&Bm|V zqvI}IkcR&TyZzg^P)>!P6xR}U#EYOr?y5sID@1c_Zf;6xYO2LQ)!RE&Ep1sk+BTun zDJnk-j>F)>U~h%*Umlqvx9jO@6Ga7f-@ThJTcSQ0=h0@Usj0Zi@{jBnL}7=vY87d# z)EZmd1b zlzB3)%$y}1#8xt5tapEkiLBR<*J5sC?wfueNoK<&EI-sL%94wysj1PGgr{O{xh)lO zPGB`($O6F|mD>cDs6BIHJv{zdbeQ5dR$=`548s>`cq-&Bjw1v1Nlk(I{2T*|L~|>8 z=&cih;z$;C)1KNGbQRc5f0yp?w&>bzJu7-|uVtDYHGW@MczF2cWCK?k;}KZ=xmFtD z8{d5>{qR$J57nOBD3wBbbK>H_`fir?#_w<23LMQ|Sg zElxh1*kM9q)#OQ{a>sHovwV zXqEN&tz9o~iS^3`W5*DjUQF?=L+oN)!L%6Bh=JA@xlhT968;2z!1%$c{U(|xTcoqJF0j!T&Ww)aN$MQi^~4Pk zcB%9l4|7Lv*C}3o`D`Yn%C(jzT{W*E>elk>Ni zQ`Y-WV;k1sQZPjy^(Q<C6eWWbR5Ne4Md5idC?3*Rp!Jq+n3PIMd{JM<1`qD_4zPpX5!LMSfpMT(tcQ4td zS53~R^s@v@Ea{-XPoMqHHC0*VEx&YAY{L#v|Cx%zO`0yf^rb6VSKk$~U~EFu1Vl`k zCZ8WXcm|ddGRVv&V+Fj3w`WF?jaJsP@LoE_CE{VxTX+~>?29M*L;l{+S-<8Ndcs=) zhZ$<}AMb24WCoPuXHAl3C|(l>V)YOK9RvNeOQ_^$Nyp(gj2*3EEDa^Jfu!f*;|^~Z z`Uuwksz}BTg!A}SN&l*kA#1M7&uY(@Rm~OI!pi~8qlvytt9_N*S4M&<(j5^4qnXJ} z_Cyr{M~eGB{vH`5?2U4P-&R(@xI=>QOs&UTWl3E%S)QuQG`KOd{3*)Oz7pU)K9A1S z?j{RPW^BJ(ei>)V`MnBz@w;A}K|au@Z`|Nq^2SS^Yzf`U+BH#O!Jckc;L_HtiVhkd z$FEnj;AZwGsquUTmgOt6BK@i_?U=uozbd_hUW-4ZoM;2?YSt1V3t=x!mLM19`X2(Pze*xA5{@KdD49s zVKwy0lm^vy@pnSEu?R=@3v*_K%wT#oO%suwV>dpYB{!zIy(pj~^Cl-i_hPdGI0eC! zrM@T-)NEl)cFq{Fw*EYlS^E6jc{#2x+A2c#UCK^5?!V;5V{MCumsHxMEPpMZ#+{L%6D z<`zEiT7e9pfX=L_nto>iE9ORoaovYDu$k*zRXW!d`kpzMu({1}e+^Pxs7rGnzahf- zisy|f8m_$dXnMO-G?*@dlI5{NSt&IvJ^dYC#G_{ChbY*KCDd!nynBLCrX#m(dGV=M z7PvZ=#`E{{)xJRax)1amRe+K?+v_gA+r_`}wXl1fm{@3I$#jCi<4;ttD}-N@qls=|JM!dB;k7hMuPdH^shE8`M!eVO9);@PY>Y1iE zVJUtA2)Hy@n6&P>m1Uu{*Zv8MNmz5f4zc?#Mfyo%?*dM=U12`4XMvq61GLoYs;f_( zX1OGC8aWNUj1pQ|*>c$5b$}q*`2lF~oiWqMiUR`l9^RUBUkkpmF~1bq7aNC-5!HwD zGvn-7;cJB0l6T z?0#6QpW4Ifq;GPD$o9uQ>*w|pO|ynFUcFXDeB%L!pD#J4R|+g(L=q^^FBv6Ls08fV zL~YkMe?VcXG3TPOd2rhFqwhcS#cm#|L(d_Qq{-ec1`i1L9qh>WhWvO_s@}06HSf92 zqs6wk9v1Z6CR(}zxV|608RIFu%E^~K%Tm4?AFfPsVs z3;kh+`(#CL*UW;NKkrF9b!XRWAD~HlTYguaBy65kQ<;)dpa@WS8O)1~4|W#3AlK022rmc(9+&Wd8x7 z(AgLeJ<_L>^#5R3`vmSo>>_Z!Jy;3`r-CCwRtrA+r>Aly*v%ELH-AB>SqG5z;HxBf z{s35?z#iR>K{Nh^#pr-OtxY)PPfvCbU?0>%9Q_MvkpcY;pQYG8aG0YnQ29|YvpD>G%ehg+?%x=G(PjdU8HJ@v>}mZ zNDTAb3|PB;^genFEbueK%di>VM-G#m`}a$&^3ju)*V(8$=Sf9F@ZAbjmXY_2&bPi+ zhq$w&zq%V8$&sWg!fS(1xoJO_byRI4H*Zaj&fD7Op9Nmk0C~ly`O~Mx78;GVAdbS% zR(R)7hZ&QWhMo!4ZZ$1%lu1tZUvY0~I3XL`=(rdzAjwGZq^NrX;Z;ubc`oL%VSIdi zZgzqAHfd@0s^3Y992~?@8ga#S!8%Wk>}$3P%Uiq8IzHigOCKF~U0iD_URV}IG>e0c zF?`t*5cVDuezS`>JRI~XScm;6R~=%te4Q*^7WNi}@#AqwVQ)0MmDq3)oC23v^DjXO z@lm!b6W&C-s4vo|$BdIla5kKuK0NQzECFe5D(#(#qM1zf4CRUR=emdtg8}s*Y zrg4RJSI3DaX?~ecMF=u*wi^=*GMwD8HM_&ZSGEWw(b5_R-h+1=v;_O37AFP}pWlh1 z++rOMq}$*i;G^t)Cu<7%hFu2_BfB?s1sF`AA#HRZQ-W8k}{Uuh*Ll>PhA}F6U0?c z5J&1&LdWi&9HzFS!+e6^Jh-{jr=ILHfX?w2_*n)?c4c57&w6!w^MM3Dmt4IVW{MwOGmgvN*X* zE$Y?=sehVC=>J$hAK_#03ni(&uov_9_V%V`gqD^(D(0!PTIEBDiqMu!iJa0Y{mt_( z1LFobYA`CRcJ-TD zQ1`l~9M&cx{kH09x8uu9ZLaK&+>MKmjB_{N3A$`!^O$t{tZC!)^z_@$DNtjTD9b+) zN(y4JVz@#j;$zRcji!zH{I!&yCCoq++)gqzJgWpn{!Q7%WcxQwccZAg$sLN6>g86Q zl5=voRS#WYH687liWR&~*8)amg)6%kBRrh-%6K!LEi&TH zTyuQ#;W8g8Lii1}OOwk%&D*mhn%UXjNLHHx@mm^D@_rK&;cP zkkqlw#oN1I7-y8cs)Cy(ZdpZbGR5yxAXt-XjzE`UwjvVa<1rEUi1&X{K4W%8Enl-V zq)BaX2(lCGt&!90XNwy;<%XTH zqvY10L@wQPGjeItIgGPJIF~wp=jqr5A#o;Sw{}|; zKVRWb%okgAc7gk?F4w3b`Pl|TsUrorL_d$Ey%1Q19W~7t%SQXPL|zw{i~IoZI_x{& z>+KW3P_#_(<5`D$E&??kOpGj6NEW2|tp@iI|rem zS5ZRp;D&y+&u1BJ`d~74mA@$gq{_@1edA3#Wqf&nb|@#G;|yD)8QN&St&L5~A+E#jMRaNg0J9^n*!|jhV28( zD{wy6Nec90yQkq1uVdjwAUb#3vX4ils7+)V$akX!VhQj7g#!l!Fm=!ug1QxlBvj=p zr>K^(P5Pw6$7J1oPMXVf-{cccE?zhZp0JcYYvII#mX^|NV{{%ug@JU5IsFJ%8a&`T zQzx65W#+}+W{0)(Cl4O-2}DiM9x&OgpHuK%pEk+pVDDs8p6ed}{>>0-f4Lf;2Ja&I zNT9QAafS~S-2_>|Xz%Y2^B*G|1?!?m3g_`J96jdVF#2;a&N3b(UH7{&bp*;Q4Oy+k z7SSgTMM;^}quCi>j{MD~%>@c;VIcTq7*C$+Y^D-;WJ!{~rlTWf4VL~%br`CA=wntP z6A&+EQe~2AZ|;1SS9>uB!%J1+0xgyXvo#5I@1dRWX$%5qt_yc(o}6PWQSKBRl?bB#haOB$b=R8zK3!*5iw!=Bjj9}hSM&(k_S z7ns3ySza1JIZnIFWZ&!@crDsZ5@K`wSv&vjc=9Z*SfgR^+^8ws!$V23TpeH`2CWyZ zF?%V;+jt%Gh=pR1m6I6(WYNp0U%p6JT8GSSHtmuS**>eMliex!&W1PqICi*f3@nGK zp|nBpR?nFf9<0cHf^Tg3vJE+y-OpSPE{iaif-YozAwFe(B(50wPz*MErM6+BX&dvF zzNfq#xXE=W%H^PAf_oCeG2w!tt-AodG5el`Sta@{^CN!|Y#jD36D48J0BXlx|2_3{ zxq8<}c4Y8865(u&wfHr*8@JOIN#sQ?{9J8!QXuJ9>^CDi00JtonS&17ykhHt62dsu03BjER+JE zuv<4DnZWZ!jpqZ+G`X)?fRyyw=uDexK(OLiI15k-o}BFc?|{2J$N#%0c;V_&(gMQF z4rH^EAm5g}wNQA_Wi!%1v;7MKU;+4zLJ@3Cejb(D)pem1PW3u}Zs5v4JlK&pty=fR zu*W8T+;|7Do^v|a;f}^BQMVtCUgN3`+aIo*9*mIC=gxRDg#uW8m%xjjOmG(dP?mjc4dI2%KP+t zu;dd4B5qj~j}^{2=7&S(iiTJC0$(=^2@RQCUi)@jQQF|2a2cbcBKM*W z>^}MT%NItMHH#3AuH4CD*d9~35Jix5Iy^AMabOg1w-!Q8zb+Wtp5+xqRh{TtQ;Fm6RKNbR0EOrVCg8+&{G~sz`Xd#QmedpJ^}sm@nyb zu6bKzZ@<8?sZ)VYpy$~lt}WsP>8N1keNnjx$5AMAX5~FUdQQ@D`Pl}hr=weSS=Irh zfIBK-W*l3an<5+YcQFsc^!>+JL%vjjotk-lcr($1X!taUpoOLWO2szn5Jy%bb+Fp9N(6*UHe&W($q zdV+}!n;sY=`@OmYTwP;6pV8Zgs7@le*?;e1s4|MI4fX?CH-1&OPJ%$c^S;wCX>}xL z)Z%p+dWZ^;USfacVAcMcT6l1qXG>PQk(#@;N>SKoUkA~4yPVb6-EmB#kU_&{v5w-) zy9Fu$w2a4mxDJBu+1DriC_-s;{|zlw?u5^WWMMM$$eDbTq&!%T0CI`q>BaVaiAFR+ zNqzve+BiMPE0oc>jlQ_JILXxF*PT+E${O(or1~$;PA!-0RD?=Rc+$_1{fKKPf38&S zF}MBZ+Ay8)V4Yf97I1tg7X#|=&m-3SbYMqN;B$r*?{d^M0d6SA;8 zGK_gH(p>zp0Z#@H>_-0Zm-pi-4MKI0pHt=L+%u8Oqg-cL&-P)G2(%pICEB{)=3sf7 zf!1zTSbPn=CrdLGK+sqI7Gt`WWA)$xBzWT;^ck3L3?~2EqA3b-v~LDox$AKB#Iq5h zGTKF@ox@Bww}^3=bWMOP34Uavq)likCZk>eAiqe#O( zOwS;=>F2PiMmXwIjR;BhSli_JMze+oSyU$;q8EwTmfoZ{P)$=)D}KC_wQ3RPM+X2+ zsgRhrq(4=m#oXzxA^_E`V?DGq^?ILs)GZy9XeT}D54kg+e|=+ap>EwKV_%a5=IPU? zHs35~wQ-f(dG;&r-t*(Ahi}$v5ZUdO?Ux^{V@3|jsfs#Mq)iicXN;zE=7|CHo%Xc( zz6z->YL-il+NRF{QRw5VbfDfxcm(IT3mK*(ZmRJ!58|KAw5u@0h=X#z$QFP#jc8?4 z+_452O-;>E)~pP2b0P7&<;PpCz%Zn9$9R@kt_Bmou33-PeztO`cZtMmG{@Exno!gJ zlVOueG9w`_K4--z3U0`6rcH#>t8T?%0^+LihR|Hnva71tigc)sjLhtG8hqlG^WhkD zU-F#+Nu(dm5PtF_mU7PA6A6EkILS#25?>ZqjU zDduf9I}`b^{FcNvA6HRgH*cq*jHa-w&M(wf6RMihMqWOUdiUns{5YQUBTbRUru#bS z*%2tH@qK{Ycs}-u5-Kf72W@fnIbyPYCD1$Mw&rHgD(B!~wT}whX)8kUrW^5mG;-y7 zat$prKKh931q)|jf1Nv34padDpEk4(`z)4<6;P#j-@`Fv9(e!RpP-$!N1=TlbT zxA$+C*_7TIZ*Hy?=W=}sQu(L9_ce$L+bT`RyBfCVT;ll{0u!e-UwA0P`XNFt z)1*BPrO|9DrUf6ie6*p(CW?o`bGt{FC&6~xX-tVt=RQVh+4+6Xy(N8}pBy|yBCMQ` zIOURcWoW0ksAx0v>s->wMmr4Bn>HI~5OFRxaqkmiW!~5Uvejz?wCF^0z}i8cSk06~ zf!)XP$|!FI;zSXBuuznAtZTV)pzW!q6jhi>gV3?gxgIB(-Qfigzp^|mS?jgS1CaU2 zsHxWVdB#d259O;lzBl6x>Ju$b5Ty6+ILO-*wvv`;As>7&82WZ=9Yx5L-uM##AW!u7 zRLFkwdaxywgQUw}sK2s0!r)=rvswF9=Q*~YO0%NvZ zANM~)oDGuTjCt_+JzrY=wNYFkwY4!Vjt66SVp$?r^CREi zB67JN(lHnJwXQP0Pc_=5^fr8e(n%NGI8==IiJIi)pfp+N&0S?HkISlk$%A_ z`qAHt^ba9Fb-GcCqk4LD6%TSdbz2O(q6w2Pi@(5(hw1#s}sBs{ChQ^ z95>=F{8bxUrNjFe3KvFxf^&*m{M()Ttm&s$KH92R#1@_E_W7xa3asaH;xpSi@xj5E z;=?G#O;9eev3N*3qfQB%gs^OKY~tnpzzFo`TC`#7EcwWhP$O7zP>R{gsZfRCU5^< zD^{tdel&o076+H>0c1PQl*NLr$f*sKTYMBJ2M)`Fj}nJ^VH^8C@cF!3)N8QApxWOF z>%u!PH-=yH+1S^3A1%oMP^xyN(IPtK=E%-oDfLU@f}aU4WP}R9R_|6sQPOoI`OVALL%|l$Tv&g8m(kf8fUeu=nx<^> z-)A^Dm@8M*xF$b8k=cts%dPap!FQzPJ{MpDG~K3!tn&b-IB%}O#-fklA1@8K3sP6c&o$CH+QLbyY@1;*|yl`C0CCP?)lWr>W}9C-|1?LmL%UfCXfd^SqO92b)5 z1K@WP{PPlTy1t9GOZgG#eW>S=Pe#5z%_VcNRsQl(-&QfvV_<8L>?R1?d#Ce?p%vg&iP2t#ehnQsVZ%EVrf(FKm~u{QP?$=ISH=-NrtZD zC0jmO?e)nY1PkmQqXU;a6v-V9nQ!S0y7dAT++Y^%5xBq@9$=+c5wv?=IV6m8WFyJXLPi#Lt5qW%BY%y(fG;3I5);| zm*9#07}CJ+K2X9^q@^JGnG?WQxXPQ%1@y=86zFUR*u5BhR#X-(Vq>hXuRS8o;e;%g zz=k0)D)1)GA@pik41HnaG=1pqfQRCy@pB&I6PMwgXCIwhdx@500{W>jed3CV5$LjgQgh;=e`9zXkaoiIr>G#$7!B zmj4TZl`c%D{;k~Wf?V$2t$f+P|E==htN*0&pKknj9R4#7|NpaxbkT1ych30;9rfA! P*Hi;tle0yq9dG?#l*@hf literal 0 HcmV?d00001 diff --git a/plugins/scrapingbee-cli/assets/logo.svg b/plugins/scrapingbee-cli/assets/logo.svg new file mode 100644 index 0000000..f2ebc05 --- /dev/null +++ b/plugins/scrapingbee-cli/assets/logo.svg @@ -0,0 +1,72 @@ + + + + + + + + + + + + diff --git a/plugins/scrapingbee-cli/mcp.json b/plugins/scrapingbee-cli/mcp.json new file mode 100644 index 0000000..ab6be0f --- /dev/null +++ b/plugins/scrapingbee-cli/mcp.json @@ -0,0 +1,10 @@ +{ + "mcpServers": { + "scrapingbee": { + "url": "https://mcp.scrapingbee.com/mcp", + "headers": { + "Authorization": "Bearer ${SCRAPINGBEE_API_KEY}" + } + } + } +} From 08db531ea0fefd6eaea7704ccf7d4d33287b44e4 Mon Sep 17 00:00:00 2001 From: Sahil Sunny Date: Wed, 7 Oct 2026 13:57:09 +0530 Subject: [PATCH 6/6] docs(plugin): show Codex CLI users how to add the MCP server --- plugins/scrapingbee-cli/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/scrapingbee-cli/README.md b/plugins/scrapingbee-cli/README.md index 2771211..192904f 100644 --- a/plugins/scrapingbee-cli/README.md +++ b/plugins/scrapingbee-cli/README.md @@ -35,6 +35,12 @@ Cursor reads `.cursor-plugin/plugin.json` and `mcp.json`, and ChatGPT / Codex re - In **Cursor**, the plugin also connects the ScrapingBee MCP server. Set the `SCRAPINGBEE_API_KEY` plugin variable when you install it; the same key is used for the MCP server and, if exported in your shell, for the CLI. +- In **Codex CLI**, the plugin ships the skills only. To add the MCP server as well, run once + (with `SCRAPINGBEE_API_KEY` exported in your shell): + + ```bash + codex mcp add scrapingbee --url https://mcp.scrapingbee.com/mcp --bearer-token-env-var SCRAPINGBEE_API_KEY + ``` ## What it runs, sends and fetches