diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..4c28f42f --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,10 @@ +# When merging changes under these paths to `production`, require review from the +# Dashboard team. +/javascript/ @browserbase/dashboard +/scripts/fetch-playground-typescript-dirs.mjs @browserbase/dashboard +/scripts/validate-playground-templates.mjs @browserbase/dashboard +/scripts/playground-ci.mjs @browserbase/dashboard +/scripts/lib/playground-checks.mjs @browserbase/dashboard +/.github/workflows/playground.yml @browserbase/dashboard +/.github/workflows/playground-production.yml @browserbase/dashboard +/.github/workflows/playground-test-production.yml @browserbase/dashboard diff --git a/.github/workflows/playground-production.yml b/.github/workflows/playground-production.yml new file mode 100644 index 00000000..b6b77570 --- /dev/null +++ b/.github/workflows/playground-production.yml @@ -0,0 +1,20 @@ +# Triggers the shared playground workflow for the `production` branch. +# All CI steps live in playground.yml โ€” edit there to change behavior. + +name: Playground templates (production) + +on: + pull_request: + branches: + - production + push: + branches: + - production + workflow_dispatch: + +permissions: + contents: write + +jobs: + playground-templates: + uses: ./.github/workflows/playground.yml diff --git a/.github/workflows/playground-test-production.yml b/.github/workflows/playground-test-production.yml new file mode 100644 index 00000000..be4fff5d --- /dev/null +++ b/.github/workflows/playground-test-production.yml @@ -0,0 +1,20 @@ +# Triggers the shared playground workflow for the `test-production` branch. +# All CI steps live in playground.yml โ€” edit there to change behavior. + +name: Playground templates (test production) + +on: + pull_request: + branches: + - test-production + push: + branches: + - test-production + workflow_dispatch: + +permissions: + contents: write + +jobs: + playground-templates: + uses: ./.github/workflows/playground.yml diff --git a/.github/workflows/playground.yml b/.github/workflows/playground.yml new file mode 100644 index 00000000..3ab2e1c3 --- /dev/null +++ b/.github/workflows/playground.yml @@ -0,0 +1,53 @@ +# Reusable workflow: validates TypeScript โ†’ JavaScript builds for playground- +# runnable templates and optionally commits the generated javascript/ tree. +# +# Called by playground-production.yml and playground-test-production.yml so that +# CI steps are defined in exactly one place. + +name: Playground templates + +on: + workflow_call: + +permissions: + contents: write + +jobs: + playground-templates: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Clean javascript output directory + run: rm -rf javascript && mkdir -p javascript + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build and validate playground TypeScript templates + env: + TEMPLATES_API_URL: ${{ vars.TEMPLATES_API_URL }} + run: pnpm run ci:playground + + - name: Commit and push playground javascript (push only) + if: github.event_name == 'push' + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A javascript/ + if git diff --staged --quiet; then + echo "No javascript changes to commit." + else + git commit --no-verify -m "chore: regenerate playground javascript templates" + git push + fi diff --git a/.github/workflows/readme-template-index.yml b/.github/workflows/readme-template-index.yml new file mode 100644 index 00000000..5c285543 --- /dev/null +++ b/.github/workflows/readme-template-index.yml @@ -0,0 +1,45 @@ +name: README Template Index + +on: + pull_request: + paths: + - "README.md" + - "go/**" + - "python/**" + - "typescript/**" + - "scripts/check-readme-template-index.mjs" + - ".github/workflows/readme-template-index.yml" + push: + branches: + - dev + paths: + - "README.md" + - "go/**" + - "python/**" + - "typescript/**" + - "scripts/check-readme-template-index.mjs" + - ".github/workflows/readme-template-index.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + validate-readme-template-index: + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: 20.x + + - name: Validate README template index + run: node scripts/check-readme-template-index.mjs diff --git a/.gitignore b/.gitignore index bddbc648..19bcdbeb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,11 +2,13 @@ .env cursorrules.md .cursorrules +.cursor # Dependencies node_modules/ .pnp .pnp.js +.pnpm-store/ # Python __pycache__/ @@ -26,6 +28,7 @@ build/ # IDE .vscode/ .idea/ +.cursor/ *.swp *.swo *~ diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100644 index 00000000..5ee7abd8 --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1 @@ +pnpm exec lint-staged diff --git a/README.md b/README.md index 30d3b2b0..3706a773 100644 --- a/README.md +++ b/README.md @@ -1,75 +1,94 @@ # Stagehand + Browserbase Templates -A comprehensive collection of ready-to-use automation templates demonstrating the power of Stagehand and Browserbase for web automation, data extraction, and AI-powered browser interactions. - -## ๐Ÿš€ Quick Start - -1. **Choose your language**: TypeScript or Python -2. **Browse available templates** in the respective language folder -3. **Read the template's README** for detailed setup instructions and use cases -4. **Start automating!** - -## ๐Ÿ“ Available Templates - -### TypeScript Templates (`/typescript/`) - -- **context** - AI-powered context switching and page analysis -- **CUA** - Computer Use Agent for autonomous web browsing and decision-making -- **formFilling** - Automated form submission and data entry -- **giftfinder** - AI-powered product discovery and recommendation -- **pickleball** - Court booking automation with user interaction -- **proxies** - Proxy testing and geolocation verification -- **realEstateCheck** - License verification and data extraction - -### Python Templates (`/python/`) - -- **context** - AI-powered context switching and page analysis -- **CUA** - Computer Use Agent for autonomous web browsing and decision-making -- **formFilling** - Automated form submission and data entry -- **giftfinder** - AI-powered product discovery and recommendation -- **pickleball** - Court booking automation with user interaction -- **proxies** - Proxy testing and geolocation verification -- **realEstateCheck** - License verification and data extraction - -> **๐Ÿ“– Each template includes a comprehensive README with:** -> -> - Detailed setup instructions -> - Required environment variables -> - Use cases and examples -> - Troubleshooting guides -> - Dependencies and installation steps - -## ๐Ÿ”ง Getting Started - -1. **Choose a template** from the TypeScript or Python folders +Stagehand is the SDK for browser agents. + +Ready-to-use automation templates for Stagehand and Browserbase. Each template has its own README with setup instructions. + +> All templates also live on [browserbase.com/templates](https://www.browserbase.com/templates) + +## All Templates + +| Template | TS | PY | GO | Description | +| -------------------------------- | ------------------------------------------------- | --------------------------------------------- | ------------------- | -------------------------------------------------------------------------------------------------------------- | +| agent-with-human-in-loop | [TS](typescript/agent-with-human-in-loop) | - | - | Build an AI agent that can pause and ask a human for input mid-task | +| amazon-global-price-comparison | [TS](typescript/amazon-global-price-comparison) | [PY](python/amazon-global-price-comparison) | - | Compare Amazon product prices across multiple countries using geolocation proxies | +| amazon-product-scraping | [TS](typescript/amazon-product-scraping) | [PY](python/amazon-product-scraping) | - | Scrape the first 3 Amazon search results for a given query and return structured product data | +| basic-caching | [TS](typescript/basic-caching) | [PY](python/basic-caching) | - | Demonstrate how Stagehand's caching feature reduces cost and latency by reusing previously computed actions | +| basic-captcha-solving | [TS](typescript/basic-captcha-solving) | [PY](python/basic-captcha-solving) | - | Automatic CAPTCHA solving using Browserbase's built-in captcha solving capabilities | +| browser-agent-demo | [TS](typescript/browser-agent-demo) | - | - | Browser agent that searches the web, fetches page content, and autonomously extracts information | +| browserbase-reducto | [TS](typescript/browserbase-reducto) | [PY](python/browserbase-reducto) | - | Download financial PDFs from websites and extract structured data using AI-powered document parsing | +| business-lookup | [TS](typescript/business-lookup) | [PY](python/business-lookup) | - | Research business registry records with a bring-your-own agent and Stagehand code mode | +| cartesia-form-filling | - | [PY](python/cartesia-form-filling) | - | Voice agent that conducts phone questionnaires while automatically filling out web forms | +| cerebras-docs-checker | - | [PY](python/cerebras-docs-checker) | - | Crawl documentation sites, discover source repos, and verify docs accuracy against actual codebase | +| company-address-finder | [TS](typescript/company-address-finder) | [PY](python/company-address-finder) | - | Discover company legal information and physical addresses from Terms of Service and Privacy Policy pages | +| company-value-prop-generator | [TS](typescript/company-value-prop-generator) | [PY](python/company-value-prop-generator) | - | Extract and format website value propositions into concise one-liners for email personalization | +| context | [TS](typescript/context) | [PY](python/context) | - | Persistent authentication using Browserbase contexts that survive across sessions | +| council-events | [TS](typescript/council-events) | [PY](python/council-events) | - | Automate event information extraction from Philadelphia Council | +| download-financial-statements | [TS](typescript/download-financial-statements) | [PY](python/download-financial-statements) | - | Download Apple's quarterly financial statements (PDFs) from their investor relations site | +| dynamic-form-filling | [TS](typescript/dynamic-form-filling) | - | - | Fill dynamic forms with a Vercel AI SDK agent and Stagehand's code_execute browser tool | +| exa-browserbase | [TS](typescript/exa-browserbase) | [PY](python/exa-browserbase) | - | Automate job applications with AI that writes smart, tailored responses for each role | +| extend-browserbase | [TS](typescript/extend-browserbase) | [PY](python/extend-browserbase) | - | Download receipts from an expense portal and extract structured receipt data using AI-powered document parsing | +| form-filling | [TS](typescript/form-filling) | [PY](python/form-filling) | - | Automate form filling with Stagehand and Browserbase | +| gemini-3-flash | [TS](typescript/gemini-3-flash) | - | - | Browser research with a Gemini 3 Flash agent and Stagehand code mode | +| gemini-cua | [TS](typescript/gemini-cua) | [PY](python/gemini-cua) | - | Browser research with a bring-your-own Gemini agent and Stagehand code mode | +| getting-started-with-browserbase | [TS](typescript/getting-started-with-browserbase) | [PY](python/getting-started-with-browserbase) | - | Demo all three core Browserbase capabilities: Search API, Fetch API, and Browser Sessions | +| gift-finder | [TS](typescript/gift-finder) | [PY](python/gift-finder) | - | Find personalized gift recommendations using AI-generated search queries and intelligent product scoring | +| google-trends | [TS](typescript/google-trends) | [PY](python/google-trends) | - | Extract trending search keywords from Google Trends for any country with structured JSON output | +| hackernews | - | - | [GO](go/hackernews) | Demonstrate Stagehand's core browser automation features through a complete Hacker News workflow | +| image-url-download | [TS](typescript/image-url-download) | [PY](python/image-url-download) | - | Extract all image URLs from a page and download each image through the browser's direct connection | +| job-application | [TS](typescript/job-application) | [PY](python/job-application) | - | Automate job applications by discovering job listings and submitting applications | +| license-verification | [TS](typescript/license-verification) | [PY](python/license-verification) | - | Extract structured, validated data from websites using Stagehand + Zod | +| manual-mfa-with-contexts | [TS](typescript/manual-mfa-with-contexts) | [PY](python/manual-mfa-with-contexts) | - | Persist authentication across sessions using Browserbase Contexts, eliminating MFA friction | +| mfa-handling | [TS](typescript/mfa-handling) | [PY](python/mfa-handling) | - | Automate MFA completion using TOTP (Time-based One-Time Password) code generation | +| microsoft-cua | [TS](typescript/microsoft-cua) | - | - | Browser research with a bring-your-own OpenAI agent and Stagehand code mode | +| nurse-verification | [TS](typescript/nurse-verification) | [PY](python/nurse-verification) | - | Automate verification of nurse licenses by filling forms and extracting structured results | +| pickleball | [TS](typescript/pickleball) | [PY](python/pickleball) | - | Automate tennis and pickleball court bookings in San Francisco Recreation & Parks system | +| playwright | [TS](typescript/playwright) | [PY](python/playwright) | - | Raw Playwright usage with Browserbase (no Stagehand) | +| playwright-mfa-handling | [TS](typescript/playwright-mfa-handling) | [PY](python/playwright-mfa-handling) | - | Automate MFA completion using TOTP with raw Playwright and Browserbase | +| polymarket-research | [TS](typescript/polymarket-research) | [PY](python/polymarket-research) | - | Automate market research on prediction markets using Stagehand | +| proxies | [TS](typescript/proxies) | [PY](python/proxies) | - | Demonstrate different proxy configurations with Browserbase sessions | +| proxies-weather | [TS](typescript/proxies-weather) | [PY](python/proxies-weather) | - | Geolocation proxies fetching location-specific weather data from multiple cities | +| puppeteer | [TS](typescript/puppeteer) | - | - | Raw Puppeteer usage with Browserbase | +| sec-filing-research | [TS](typescript/sec-filing-research) | [PY](python/sec-filing-research) | - | Search SEC EDGAR for a company and extract recent filing metadata | +| selenium | [TS](typescript/selenium) | [PY](python/selenium) | - | Raw Selenium usage with Browserbase | +| smart-fetch-scraper | [TS](typescript/smart-fetch-scraper) | [PY](python/smart-fetch-scraper) | - | Scrape a webpage using the fastest method available -- Fetch API first, full browser session as fallback | +| website-link-tester | [TS](typescript/website-link-tester) | [PY](python/website-link-tester) | - | Crawl a website's homepage, collect all links, and verify each link loads successfully | + +## Model Gateway + +Stagehand primitives use the Browserbase Model Gateway, so they need only `BROWSERBASE_API_KEY`. Bring-your-own-agent templates also use Vercel AI Gateway for the outer agent loop and require `AI_GATEWAY_API_KEY`; no provider-specific OpenAI, Anthropic, or Google key is required. + +> **Stagehand V4 note**: V4 does not expose the V3 `agent()` orchestration API. TypeScript agent templates use Vercel AI SDK with Stagehand's `code_execute` tool. Python agent templates use LangChain Deep Agents with Stagehand's `run`, `snapshot`, and `screenshot` tools. Other templates call V4 browser primitives directly. + +## Getting Started + +1. **Choose a template** from the table above 2. **Read the template's README** for specific setup instructions 3. **Set up your environment** with the required API keys and dependencies -4. **Run the template** and start automating! +4. **Run the template** and start automating + +Each template's README contains detailed installation steps, environment variable requirements, and troubleshooting guides. + +### TypeScript and generated JavaScript -> **๐Ÿ’ก Pro Tip**: Each template's README contains detailed installation steps, environment variable requirements, and troubleshooting guides specific to that template. +- **Source of truth:** edit templates under `typescript/`. The `javascript/` tree is generated output, not authored by hand for day-to-day changes. +- **Local only:** run `pnpm run build:javascript` when you want a full mirror of `typescript/` into `javascript/` on your machine (for example to smoke-test the transpiler or compare JS output). There is **no** GitHub Actions workflow that builds the full tree into the repo anymore. +- **Playground releases:** the `production` branch is updated by CI (`.github/workflows/playground-production.yml`), which builds and commits **only** templates that are playground-runnable per the public templates API, then validates them. -## ๐Ÿ“š Resources +## Resources ### Documentation -- **Stagehand Docs**: https://docs.stagehand.dev/v3/first-steps/introduction +- **Stagehand Docs**: https://docs.stagehand.dev/v4/first-steps/introduction - **Browserbase Docs**: https://docs.browserbase.com -- **API Reference**: https://docs.browserbase.com/api ### Support -- **Community**: Join our Discord community - **Discord**: http://stagehand.dev/discord - **Email Support**: support@browserbase.com - **GitHub Issues**: Report bugs and request features -### Examples & Tutorials - -- **Getting Started Guide**: Learn the basics of Stagehand -- **Advanced Patterns**: Complex automation workflows -- **Best Practices**: Tips for reliable automation - -## ๐Ÿค Contributing +## Contributing We welcome contributions! Here's how you can help: @@ -85,17 +104,6 @@ We welcome contributions! Here's how you can help: - Add proper error handling and logging - Test templates thoroughly before submitting -## ๐Ÿ“„ License +## License This project is licensed under the MIT License - see the LICENSE file for details. - -## ๐Ÿ™ Acknowledgments - -- **Stagehand Team**: For the amazing automation framework -- **Browserbase Team**: For reliable cloud browser infrastructure -- **Community Contributors**: For templates and improvements -- **Open Source Projects**: For the tools and libraries we build upon - ---- - -**Ready to start automating?** Pick a template and follow its README to get started! ๐Ÿš€ diff --git a/eslint.config.js b/eslint.config.js index 1670f4ac..fed3ab63 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -27,4 +27,3 @@ export default tseslint.config( ignores: ["node_modules/**", "dist/**", "build/**", "*.config.js"], }, ); - diff --git a/go/hackernews/README.md b/go/hackernews/README.md new file mode 100644 index 00000000..c31cba63 --- /dev/null +++ b/go/hackernews/README.md @@ -0,0 +1,61 @@ +# Stagehand + Browserbase: Hacker News Automation + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- Goal: demonstrate Stagehand V4's core browser automation primitives through a complete Hacker News workflow. +- Complete workflow: launch a Browserbase browser, attach Stagehand, navigate, observe, act, and extract typed data. +- Real-world outcome: open the top story's comments, extract its first comment, then extract the newest visible story. +- Docs โ†’ https://docs.stagehand.dev/v4/first-steps/quickstart + +## GLOSSARY + +- `LaunchBrowserbase`: create a Browserbase browser owned by the application. +- `Create`: attach Stagehand V4 to that browser. +- `Observe`: find possible page actions from a natural-language instruction. +- `Act`: execute either an observed action or a natural-language instruction. +- `Extract`: return typed structured data together with result metadata. + +## QUICKSTART + +1. Install Go 1.26 or newer (`go version`). +2. Set `BROWSERBASE_API_KEY` in your environment. +3. Run `go mod download`. +4. Run `go run .`. + +## EXPECTED OUTPUT + +- Launches a real Browserbase browser and attaches Stagehand V4. +- Navigates to Hacker News and verifies the main-document HTTP response. +- Observes and opens the comments link for the top-ranked story. +- Extracts the story title, first comment, and commenter. +- Navigates to `/newest` and extracts the newest story title. +- Explicitly closes Stagehand before closing the browser. + +## COMMON PITFALLS + +- Missing Go installation: ensure Go 1.26+ is installed. +- Missing credentials: verify `BROWSERBASE_API_KEY` is set. +- Module not found: run `go mod download` if dependencies are not resolved. +- No `MODEL_API_KEY` is needed: Stagehand primitives use Browserbase Model Gateway. +- Stagehand V4 does not expose an agent API. Bring your own agent framework when orchestration is required. + +## USE CASES + +- Content aggregation: extract structured records from news sites, forums, and social platforms. +- Research automation: combine deterministic navigation with AI-guided actions and typed extraction. +- Content automation: combine page interaction and structured extraction in a real cloud browser. + +## NEXT STEPS + +- Add retries for transient site or model failures. +- Extend the extraction types with points, comment count, and post age. +- Add an external Go agent framework if the workflow needs autonomous planning. + +## HELPFUL RESOURCES + +- Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/quickstart +- Browserbase: https://www.browserbase.com +- Templates: https://www.browserbase.com/templates +- Discord: http://stagehand.dev/discord diff --git a/go/hackernews/go.mod b/go/hackernews/go.mod new file mode 100644 index 00000000..4882ff2c --- /dev/null +++ b/go/hackernews/go.mod @@ -0,0 +1,17 @@ +module stagehand-example + +go 1.26.0 + +require github.com/browserbase/stagehand/packages/sdk-go/v4 v4.0.2 + +require ( + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.2 // indirect + github.com/coder/websocket v1.8.13 // indirect + github.com/invopop/jsonschema v0.14.0 // indirect + github.com/pb33f/ordered-map/v2 v2.3.1 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect + golang.org/x/mod v0.38.0 // indirect +) diff --git a/go/hackernews/go.sum b/go/hackernews/go.sum new file mode 100644 index 00000000..06720869 --- /dev/null +++ b/go/hackernews/go.sum @@ -0,0 +1,38 @@ +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/browserbase/stagehand/packages/sdk-go/v4 v4.0.2 h1:axCIlzMz6Kao32Wli7C6uZVHbEc5MzmsUDAbf2y9TXw= +github.com/browserbase/stagehand/packages/sdk-go/v4 v4.0.2/go.mod h1:Z4qkSeN4whiyoYk+jqKbwYbUPu16JgmvEZWtiiaXyqY= +github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= +github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/coder/websocket v1.8.13 h1:f3QZdXy7uGVz+4uCJy2nTZyM0yTBj8yANEHhqlXZ9FE= +github.com/coder/websocket v1.8.13/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/invopop/jsonschema v0.14.0 h1:MHQqLhvpNUZfw+hM3AZDYK7jxO8FZoQeQM77g8iyZjg= +github.com/invopop/jsonschema v0.14.0/go.mod h1:ygm6C2EaVNMBDPpaPlnOA2pFAxBnxGjFlMZABxm9n2I= +github.com/pb33f/ordered-map/v2 v2.3.1 h1:5319HDO0aw4DA4gzi+zv4FXU9UlSs3xGZ40wcP1nBjY= +github.com/pb33f/ordered-map/v2 v2.3.1/go.mod h1:qxFQgd0PkVUtOMCkTapqotNgzRhMPL7VvaHKbd1HnmQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.yaml.in/yaml/v4 v4.0.0-rc.2 h1:/FrI8D64VSr4HtGIlUtlFMGsm7H7pWTbj6vOLVZcA6s= +go.yaml.in/yaml/v4 v4.0.0-rc.2/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/hackernews/main.go b/go/hackernews/main.go new file mode 100644 index 00000000..5969ed0e --- /dev/null +++ b/go/hackernews/main.go @@ -0,0 +1,132 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "time" + + stagehand "github.com/browserbase/stagehand/packages/sdk-go/v4" +) + +type storyDetails struct { + Title string `json:"title" jsonschema:"description=title of the Hacker News post"` + TopComment string `json:"topComment" jsonschema:"description=text of the first visible comment"` + Author string `json:"author" jsonschema:"description=username of the first visible commenter"` +} + +type newestStory struct { + Title string `json:"title" jsonschema:"description=title of the newest visible story"` +} + +func main() { + if err := run(context.Background()); err != nil { + log.Fatal(err) + } +} + +func run(parent context.Context) (err error) { + apiKey := os.Getenv("BROWSERBASE_API_KEY") + if apiKey == "" { + return errors.New("BROWSERBASE_API_KEY is required") + } + + ctx, cancel := context.WithTimeout(parent, 2*time.Minute) + defer cancel() + + sessionTimeout := 120.0 + browser, err := stagehand.LaunchBrowserbase(ctx, stagehand.BrowserbaseLaunchOptions{ + APIKey: apiKey, + Timeout: &sessionTimeout, + }) + if err != nil { + return fmt.Errorf("launch Browserbase: %w", err) + } + defer func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cleanupCancel() + err = errors.Join(err, browser.Close(cleanupCtx)) + }() + + client, err := stagehand.Create(ctx, stagehand.CreateOptions{Browser: browser}) + if err != nil { + return fmt.Errorf("create Stagehand: %w", err) + } + defer func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cleanupCancel() + err = errors.Join(err, client.Close(cleanupCtx)) + }() + + browserContext, err := browser.Context() + if err != nil { + return fmt.Errorf("get browser context: %w", err) + } + pages, err := browserContext.Pages(ctx) + if err != nil { + return fmt.Errorf("list pages: %w", err) + } + if len(pages) == 0 { + return errors.New("no active page after Stagehand initialization") + } + page := pages[0] + + response, err := page.Goto(ctx, "https://news.ycombinator.com", nil) + if err != nil { + return fmt.Errorf("navigate to Hacker News: %w", err) + } + if response == nil || response.Status() != 200 { + return errors.New("unexpected navigation response from Hacker News") + } + fmt.Println("Navigated to Hacker News") + + instruction := "Find the comments link for the top-ranked story" + observed, err := client.Observe(ctx, &instruction, nil) + if err != nil { + return fmt.Errorf("observe comments link: %w", err) + } + if len(observed.Data) == 0 { + return errors.New("observe returned no comments link") + } + fmt.Printf("Found %d possible comment actions\n", len(observed.Data)) + + acted, err := client.Act(ctx, stagehand.ObservedAction(observed.Data[0]), nil) + if err != nil { + return fmt.Errorf("open comments: %w", err) + } + if !acted.Data.Success { + return fmt.Errorf("open comments failed: %s", acted.Data.Message) + } + details, err := stagehand.Extract[storyDetails]( + ctx, + client, + "Extract the post title and the first visible comment with its author", + nil, + ) + if err != nil { + return fmt.Errorf("extract story details: %w", err) + } + fmt.Printf("Top story: %s\n", details.Data.Title) + fmt.Printf("Top comment by %s: %s\n", details.Data.Author, details.Data.TopComment) + + response, err = page.Goto(ctx, "https://news.ycombinator.com/newest", nil) + if err != nil { + return fmt.Errorf("navigate to newest stories: %w", err) + } + if response == nil || response.Status() != 200 { + return errors.New("unexpected response from the Hacker News newest page") + } + newest, err := stagehand.Extract[newestStory]( + ctx, + client, + "Extract the exact title of the first story in the newest stories list", + nil, + ) + if err != nil { + return fmt.Errorf("extract newest story: %w", err) + } + fmt.Printf("Newest story: %s\n", newest.Data.Title) + return nil +} diff --git a/package.json b/package.json index 68b865e7..5b0cc55b 100644 --- a/package.json +++ b/package.json @@ -5,21 +5,44 @@ "private": true, "type": "module", "scripts": { + "build:javascript": "node scripts/build-javascript.mjs", + "ci:playground": "node scripts/playground-ci.mjs", + "validate:playground": "node scripts/validate-playground-templates.mjs", + "check:readme-template-index": "node scripts/check-readme-template-index.mjs", "format": "prettier --write \"**/*.{ts,tsx,js,jsx,json,md,yml,yaml}\"", "format:check": "prettier --check \"**/*.{ts,tsx,js,jsx,json,md,yml,yaml}\"", "lint": "eslint \"**/*.{ts,tsx,js,jsx}\"", "lint:fix": "eslint \"**/*.{ts,tsx,js,jsx}\" --fix", "lint:python": "uvx ruff check python/", "lint:python:fix": "uvx ruff check --fix python/", - "format:python": "uvx ruff format python/" + "format:python": "uvx ruff format python/", + "format:python:check": "uvx ruff format --check python/", + "check": "npm run check:readme-template-index && npm run format:check && npm run lint && npm run lint:python && npm run format:python:check", + "prepare": "husky" }, "devDependencies": { "@typescript-eslint/eslint-plugin": "^8.50.1", "@typescript-eslint/parser": "^8.50.1", "eslint": "^9.39.2", "eslint-config-prettier": "^10.1.8", + "husky": "^9.1.7", + "lint-staged": "^16.2.7", "prettier": "^3.2.5", + "typescript": "^5.9.3", "typescript-eslint": "^8.50.1" }, - "packageManager": "pnpm@9.0.0" + "packageManager": "pnpm@9.0.0", + "lint-staged": { + "*.{ts,tsx,js,jsx}": [ + "eslint --fix", + "prettier --write" + ], + "*.{json,md,yml,yaml}": [ + "prettier --write" + ], + "*.py": [ + "uvx ruff check --fix", + "uvx ruff format" + ] + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0dc80696..1a2db3b4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,9 +20,18 @@ importers: eslint-config-prettier: specifier: ^10.1.8 version: 10.1.8(eslint@9.39.2) + husky: + specifier: ^9.1.7 + version: 9.1.7 + lint-staged: + specifier: ^16.2.7 + version: 16.2.7 prettier: specifier: ^3.2.5 version: 3.7.4 + typescript: + specifier: ^5.9.3 + version: 5.9.3 typescript-eslint: specifier: ^8.50.1 version: 8.50.1(eslint@9.39.2)(typescript@5.9.3) @@ -161,10 +170,22 @@ packages: ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} + ansi-escapes@7.3.0: + resolution: {integrity: sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==} + engines: {node: '>=18'} + + ansi-regex@6.2.2: + resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==} + engines: {node: '>=12'} + ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} + ansi-styles@6.2.3: + resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} + engines: {node: '>=12'} + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -177,6 +198,10 @@ packages: brace-expansion@2.0.2: resolution: {integrity: sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==} + braces@3.0.3: + resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} + engines: {node: '>=8'} + callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} @@ -185,6 +210,14 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + cli-cursor@5.0.0: + resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} + engines: {node: '>=18'} + + cli-truncate@5.1.1: + resolution: {integrity: sha512-SroPvNHxUnk+vIW/dOSfNqdy1sPEFkrTk6TUtqLCnBlo3N7TNYYkzzN7uSD6+jVjrdO4+p8nH7JzH6cIvUem6A==} + engines: {node: '>=20'} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -192,6 +225,13 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + + commander@14.0.3: + resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} + engines: {node: '>=20'} + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -211,6 +251,13 @@ packages: deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + emoji-regex@10.6.0: + resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} + + environment@1.1.0: + resolution: {integrity: sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==} + engines: {node: '>=18'} + escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} @@ -263,6 +310,9 @@ packages: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} + eventemitter3@5.0.4: + resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -285,6 +335,10 @@ packages: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} + fill-range@7.1.1: + resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} + engines: {node: '>=8'} + find-up@5.0.0: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} engines: {node: '>=10'} @@ -296,6 +350,10 @@ packages: flatted@3.3.3: resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==} + get-east-asian-width@1.4.0: + resolution: {integrity: sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q==} + engines: {node: '>=18'} + glob-parent@6.0.2: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} @@ -308,6 +366,11 @@ packages: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} + husky@9.1.7: + resolution: {integrity: sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==} + engines: {node: '>=18'} + hasBin: true + ignore@5.3.2: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} @@ -328,10 +391,18 @@ packages: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} + is-fullwidth-code-point@5.1.0: + resolution: {integrity: sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==} + engines: {node: '>=18'} + is-glob@4.0.3: resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} engines: {node: '>=0.10.0'} + is-number@7.0.0: + resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} + engines: {node: '>=0.12.0'} + isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} @@ -355,6 +426,15 @@ packages: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} + lint-staged@16.2.7: + resolution: {integrity: sha512-lDIj4RnYmK7/kXMya+qJsmkRFkGolciXjrsZ6PC25GdTfWOAWetR0ZbsNXRAj1EHHImRSalc+whZFg56F5DVow==} + engines: {node: '>=20.17'} + hasBin: true + + listr2@9.0.5: + resolution: {integrity: sha512-ME4Fb83LgEgwNw96RKNvKV4VTLuXfoKudAmm2lP8Kk87KaMK0/Xrx/aAkMWmT8mDb+3MlFDspfbCs7adjRxA2g==} + engines: {node: '>=20.0.0'} + locate-path@6.0.0: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} @@ -362,6 +442,18 @@ packages: lodash.merge@4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} + log-update@6.1.0: + resolution: {integrity: sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==} + engines: {node: '>=18'} + + micromatch@4.0.8: + resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} + engines: {node: '>=8.6'} + + mimic-function@5.0.1: + resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==} + engines: {node: '>=18'} + minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} @@ -372,9 +464,17 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + nano-spawn@2.0.0: + resolution: {integrity: sha512-tacvGzUY5o2D8CBh2rrwxyNojUsZNU2zjNTzKQrkgGJQTbGAfArVWXSKMBokBeeg6C7OLRGUEyoFlYbfeWQIqw==} + engines: {node: '>=20.17'} + natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + onetime@7.0.0: + resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} + engines: {node: '>=18'} + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -399,10 +499,19 @@ packages: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} + picomatch@2.3.1: + resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} + engines: {node: '>=8.6'} + picomatch@4.0.3: resolution: {integrity: sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==} engines: {node: '>=12'} + pidtree@0.6.0: + resolution: {integrity: sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g==} + engines: {node: '>=0.10'} + hasBin: true + prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} @@ -420,6 +529,13 @@ packages: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} + restore-cursor@5.1.0: + resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} + engines: {node: '>=18'} + + rfdc@1.4.1: + resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + semver@7.7.3: resolution: {integrity: sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==} engines: {node: '>=10'} @@ -433,6 +549,30 @@ packages: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + + slice-ansi@7.1.2: + resolution: {integrity: sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w==} + engines: {node: '>=18'} + + string-argv@0.3.2: + resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==} + engines: {node: '>=0.6.19'} + + string-width@7.2.0: + resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} + engines: {node: '>=18'} + + string-width@8.1.1: + resolution: {integrity: sha512-KpqHIdDL9KwYk22wEOg/VIqYbrnLeSApsKT/bSj6Ez7pn3CftUiLAv2Lccpq1ALcpLV9UX1Ppn92npZWu2w/aw==} + engines: {node: '>=20'} + + strip-ansi@7.1.2: + resolution: {integrity: sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==} + engines: {node: '>=12'} + strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} @@ -445,6 +585,10 @@ packages: resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} engines: {node: '>=12.0.0'} + to-regex-range@5.0.1: + resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} + engines: {node: '>=8.0'} + ts-api-utils@2.1.0: resolution: {integrity: sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ==} engines: {node: '>=18.12'} @@ -479,6 +623,15 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} + wrap-ansi@9.0.2: + resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} + engines: {node: '>=18'} + + yaml@2.8.2: + resolution: {integrity: sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==} + engines: {node: '>= 14.6'} + hasBin: true + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -650,10 +803,18 @@ snapshots: json-schema-traverse: 0.4.1 uri-js: 4.4.1 + ansi-escapes@7.3.0: + dependencies: + environment: 1.1.0 + + ansi-regex@6.2.2: {} + ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 + ansi-styles@6.2.3: {} + argparse@2.0.1: {} balanced-match@1.0.2: {} @@ -667,6 +828,10 @@ snapshots: dependencies: balanced-match: 1.0.2 + braces@3.0.3: + dependencies: + fill-range: 7.1.1 + callsites@3.1.0: {} chalk@4.1.2: @@ -674,12 +839,25 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 + cli-cursor@5.0.0: + dependencies: + restore-cursor: 5.1.0 + + cli-truncate@5.1.1: + dependencies: + slice-ansi: 7.1.2 + string-width: 8.1.1 + color-convert@2.0.1: dependencies: color-name: 1.1.4 color-name@1.1.4: {} + colorette@2.0.20: {} + + commander@14.0.3: {} + concat-map@0.0.1: {} cross-spawn@7.0.6: @@ -694,6 +872,10 @@ snapshots: deep-is@0.1.4: {} + emoji-regex@10.6.0: {} + + environment@1.1.0: {} + escape-string-regexp@4.0.0: {} eslint-config-prettier@10.1.8(eslint@9.39.2): @@ -766,6 +948,8 @@ snapshots: esutils@2.0.3: {} + eventemitter3@5.0.4: {} + fast-deep-equal@3.1.3: {} fast-json-stable-stringify@2.1.0: {} @@ -780,6 +964,10 @@ snapshots: dependencies: flat-cache: 4.0.1 + fill-range@7.1.1: + dependencies: + to-regex-range: 5.0.1 + find-up@5.0.0: dependencies: locate-path: 6.0.0 @@ -792,6 +980,8 @@ snapshots: flatted@3.3.3: {} + get-east-asian-width@1.4.0: {} + glob-parent@6.0.2: dependencies: is-glob: 4.0.3 @@ -800,6 +990,8 @@ snapshots: has-flag@4.0.0: {} + husky@9.1.7: {} + ignore@5.3.2: {} ignore@7.0.5: {} @@ -813,10 +1005,16 @@ snapshots: is-extglob@2.1.1: {} + is-fullwidth-code-point@5.1.0: + dependencies: + get-east-asian-width: 1.4.0 + is-glob@4.0.3: dependencies: is-extglob: 2.1.1 + is-number@7.0.0: {} + isexe@2.0.0: {} js-yaml@4.1.1: @@ -838,12 +1036,46 @@ snapshots: prelude-ls: 1.2.1 type-check: 0.4.0 + lint-staged@16.2.7: + dependencies: + commander: 14.0.3 + listr2: 9.0.5 + micromatch: 4.0.8 + nano-spawn: 2.0.0 + pidtree: 0.6.0 + string-argv: 0.3.2 + yaml: 2.8.2 + + listr2@9.0.5: + dependencies: + cli-truncate: 5.1.1 + colorette: 2.0.20 + eventemitter3: 5.0.4 + log-update: 6.1.0 + rfdc: 1.4.1 + wrap-ansi: 9.0.2 + locate-path@6.0.0: dependencies: p-locate: 5.0.0 lodash.merge@4.6.2: {} + log-update@6.1.0: + dependencies: + ansi-escapes: 7.3.0 + cli-cursor: 5.0.0 + slice-ansi: 7.1.2 + strip-ansi: 7.1.2 + wrap-ansi: 9.0.2 + + micromatch@4.0.8: + dependencies: + braces: 3.0.3 + picomatch: 2.3.1 + + mimic-function@5.0.1: {} + minimatch@3.1.2: dependencies: brace-expansion: 1.1.12 @@ -854,8 +1086,14 @@ snapshots: ms@2.1.3: {} + nano-spawn@2.0.0: {} + natural-compare@1.4.0: {} + onetime@7.0.0: + dependencies: + mimic-function: 5.0.1 + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -881,8 +1119,12 @@ snapshots: path-key@3.1.1: {} + picomatch@2.3.1: {} + picomatch@4.0.3: {} + pidtree@0.6.0: {} + prelude-ls@1.2.1: {} prettier@3.7.4: {} @@ -891,6 +1133,13 @@ snapshots: resolve-from@4.0.0: {} + restore-cursor@5.1.0: + dependencies: + onetime: 7.0.0 + signal-exit: 4.1.0 + + rfdc@1.4.1: {} + semver@7.7.3: {} shebang-command@2.0.0: @@ -899,6 +1148,30 @@ snapshots: shebang-regex@3.0.0: {} + signal-exit@4.1.0: {} + + slice-ansi@7.1.2: + dependencies: + ansi-styles: 6.2.3 + is-fullwidth-code-point: 5.1.0 + + string-argv@0.3.2: {} + + string-width@7.2.0: + dependencies: + emoji-regex: 10.6.0 + get-east-asian-width: 1.4.0 + strip-ansi: 7.1.2 + + string-width@8.1.1: + dependencies: + get-east-asian-width: 1.4.0 + strip-ansi: 7.1.2 + + strip-ansi@7.1.2: + dependencies: + ansi-regex: 6.2.2 + strip-json-comments@3.1.1: {} supports-color@7.2.0: @@ -910,6 +1183,10 @@ snapshots: fdir: 6.5.0(picomatch@4.0.3) picomatch: 4.0.3 + to-regex-range@5.0.1: + dependencies: + is-number: 7.0.0 + ts-api-utils@2.1.0(typescript@5.9.3): dependencies: typescript: 5.9.3 @@ -941,4 +1218,12 @@ snapshots: word-wrap@1.2.5: {} + wrap-ansi@9.0.2: + dependencies: + ansi-styles: 6.2.3 + string-width: 7.2.0 + strip-ansi: 7.1.2 + + yaml@2.8.2: {} + yocto-queue@0.1.0: {} diff --git a/python/amazon-global-price-comparison/.env.example b/python/amazon-global-price-comparison/.env.example new file mode 100644 index 00000000..f1797331 --- /dev/null +++ b/python/amazon-global-price-comparison/.env.example @@ -0,0 +1,2 @@ +# Browserbase credentials - get these from https://www.browserbase.com/settings +BROWSERBASE_API_KEY=your_browserbase_api_key diff --git a/python/amazon-global-price-comparison/README.md b/python/amazon-global-price-comparison/README.md new file mode 100644 index 00000000..bba68495 --- /dev/null +++ b/python/amazon-global-price-comparison/README.md @@ -0,0 +1,77 @@ +# Amazon Global Price Comparison + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- **Goal**: Compare Amazon product prices across multiple countries using geolocation proxies. +- **Pattern Template**: Demonstrates the integration of Browserbase geolocation proxies with Stagehand AI extraction for price comparison workflows. +- **Workflow**: Creates Browserbase sessions with geolocation proxies for each country, navigates to Amazon, searches for products, and extracts structured pricing data using Stagehand's AI-powered extraction. +- **Concurrent Processing**: Runs all country searches in parallel using `asyncio.gather()` for faster execution. +- **Structured Extraction**: Uses Pydantic schemas to extract consistent product data (name, price, rating, reviews) across different Amazon regions. +- Docs โ†’ [Browserbase Proxies](https://docs.browserbase.com/features/proxies) | [Stagehand Extract](https://docs.stagehand.dev/v4/basics/extract) + +## GLOSSARY + +- **geolocation proxies**: Route traffic through specific geographic locations (city, country) to access location-specific content and pricing. + Docs โ†’ https://docs.browserbase.com/features/proxies#set-proxy-geolocation +- **extract**: Extract structured data from web pages using natural language instructions and JSON schemas. + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract +- **act**: Perform UI actions from natural language prompts (click, scroll, type, navigate). + Docs โ†’ https://docs.stagehand.dev/v4/basics/act +- **proxies**: Browserbase's managed proxy infrastructure supporting 201+ countries for geolocation-based routing. + Docs โ†’ https://docs.browserbase.com/features/proxies + +## QUICKSTART + +1. cd python/amazon-global-price-comparison +2. cp .env.example .env +3. Add required API keys to .env: + - `BROWSERBASE_API_KEY` +4. Run the script: + ```bash + uv run python main.py + ``` + +## EXPECTED OUTPUT + +- Creates Browserbase sessions with geolocation proxies for each country (US, UK, DE, FR, IT, ES) +- Navigates to Amazon search results through location-specific proxies +- Extracts product name, price, rating, and review count for each location +- Displays formatted comparison table showing price differences across countries +- Outputs JSON results for programmatic use + +## COMMON PITFALLS + +- **Browserbase Developer plan or higher is required to use proxies** +- "ModuleNotFoundError": ensure you're running with `uv run python main.py` (uv automatically installs dependencies from pyproject.toml) +- Missing credentials: verify .env contains `BROWSERBASE_API_KEY` +- Geolocation fields are case-insensitive (city, country can be any case) +- Amazon may show different products in different regions - comparison works best for globally available products +- ERR_TUNNEL_CONNECTION_FAILED: indicates either a temporary proxy hiccup or a site unsupported by built-in proxies + +## USE CASES + +โ€ข **Price arbitrage**: Find the best country to purchase products from for international shipping +โ€ข **Market research**: Compare pricing strategies across different Amazon regions +โ€ข **Competitive analysis**: Monitor how competitors price products globally +โ€ข **Travel shopping**: Check prices before international trips to plan purchases + +## NEXT STEPS + +โ€ข **Add more countries**: Extend the `COUNTRIES` list with additional regions (Japan, Australia, Canada, etc.) +โ€ข **Currency conversion**: Add real-time currency conversion to normalize prices for comparison +โ€ข **Price tracking**: Store results over time to track price changes across regions +โ€ข **Email alerts**: Send notifications when price drops below a threshold in any country +โ€ข **Product matching**: Use fuzzy matching to ensure you're comparing the same product across regions + +## HELPFUL RESOURCES + +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction +๐Ÿ“š Python SDK: https://docs.stagehand.dev/v4/sdk/python +๐Ÿ“š Browserbase Proxies: https://docs.browserbase.com/features/proxies +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/amazon-global-price-comparison/main.py b/python/amazon-global-price-comparison/main.py new file mode 100644 index 00000000..525addad --- /dev/null +++ b/python/amazon-global-price-comparison/main.py @@ -0,0 +1,159 @@ +"""Compare live Amazon prices through regional proxies with Stagehand V4.""" + +import asyncio +import json +import os +from dataclasses import asdict, dataclass +from urllib.parse import quote_plus, urljoin + +from dotenv import load_dotenv +from pydantic import BaseModel, Field, HttpUrl +from stagehand import BrowserbaseProxyConfig, Stagehand, browserbase + +load_dotenv() + + +class Product(BaseModel): + name: str + price: str + rating: str + reviews_count: str + product_url: HttpUrl = Field(description="Absolute Amazon product-detail href") + + +class Products(BaseModel): + products: list[Product] + + +@dataclass(frozen=True) +class Country: + name: str + code: str + domain: str + currency: str + city: str | None = None + + +@dataclass +class CountryResult: + country: str + country_code: str + currency: str + products: list[dict] + error: str | None = None + + +COUNTRIES = [ + Country("United States", "US", "www.amazon.com", "USD"), + Country("United Kingdom", "GB", "www.amazon.co.uk", "GBP", "LONDON"), + Country("Germany", "DE", "www.amazon.de", "EUR", "BERLIN"), + Country("France", "FR", "www.amazon.fr", "EUR", "PARIS"), + Country("Italy", "IT", "www.amazon.it", "EUR", "ROME"), + Country("Spain", "ES", "www.amazon.es", "EUR", "MADRID"), +] + + +async def products_for_country( + query: str, + country: Country, + result_count: int, +) -> CountryResult: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") + proxy: BrowserbaseProxyConfig = { + "type": "browserbase", + "geolocation": { + "country": country.code, + **({"city": country.city} if country.city else {}), + }, + } + + browser = await browserbase.launch(api_key=api_key, proxies=[proxy]) + try: + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + origin = f"https://{country.domain}" + await page.goto(origin, wait_until="domcontentloaded", timeout=60_000) + semantic_search_succeeded = False + try: + typed = await stagehand.act(f'Type "{query}" into the search bar', page=page) + submitted = await stagehand.act("Click the search button", page=page) + semantic_search_succeeded = typed.data.success and submitted.data.success + except Exception as error: + print( + f"[{country.name}] Semantic search failed; " + f"checking results before fallback: {error}" + ) + page = await browser.context.active_page() or page + results_ready = None + if semantic_search_succeeded: + try: + results_ready = await page.wait_for_selector( + '[data-component-type="s-search-result"]', + timeout=10_000, + ) + except Exception: + results_ready = None + if not results_ready: + search_url = f"{origin}/s?k={quote_plus(query)}" + await page.goto(search_url, wait_until="domcontentloaded", timeout=60_000) + await page.wait_for_selector( + '[data-component-type="s-search-result"]', + timeout=15_000, + ) + extracted = await stagehand.extract( + ( + f"Extract the first {result_count} product search results. For each product, " + "return the full title, displayed price with currency symbol or N/A, star " + "rating, review count, and absolute product-page href. Each URL must be a " + "real Amazon link containing /dp/. " + "Only include actual listings." + ), + Products, + page=page, + ) + products = [ + { + **product.model_dump(mode="json"), + "product_url": urljoin(origin, str(product.product_url)), + } + for product in extracted.data.products[:result_count] + ] + return CountryResult( + country=country.name, + country_code=country.code, + currency=country.currency, + products=products, + ) + finally: + await stagehand.close() + except Exception as error: + return CountryResult( + country=country.name, + country_code=country.code, + currency=country.currency, + products=[], + error=str(error), + ) + finally: + await browser.close() + + +async def main() -> None: + query = "iPhone 15 Pro Max 256GB" + result_count = 3 + country_limit = int(os.environ.get("MAX_COUNTRIES", str(len(COUNTRIES)))) + selected = COUNTRIES[:country_limit] + results = await asyncio.gather( + *(products_for_country(query, country, result_count) for country in selected) + ) + print(json.dumps([asdict(result) for result in results], indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/amazon-global-price-comparison/pyproject.toml b/python/amazon-global-price-comparison/pyproject.toml new file mode 100644 index 00000000..9cbd0593 --- /dev/null +++ b/python/amazon-global-price-comparison/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "amazon-global-price-comparison" +version = "0.1.0" +description = "Compare Amazon product prices across multiple countries using geolocation proxies" +readme = "README.md" +requires-python = ">=3.11,<3.14" +dependencies = [ + "browserbase>=1.7.0", + "python-dotenv", + "pydantic>=2.0.0", + "stagehand==4.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/python/amazon-product-scraping/.env.example b/python/amazon-product-scraping/.env.example new file mode 100644 index 00000000..12472549 --- /dev/null +++ b/python/amazon-product-scraping/.env.example @@ -0,0 +1,3 @@ +# Browserbase credentials (required) +# Get these from https://www.browserbase.com/settings +BROWSERBASE_API_KEY=your-browserbase-api-key diff --git a/python/amazon-product-scraping/README.md b/python/amazon-product-scraping/README.md new file mode 100644 index 00000000..dda92c85 --- /dev/null +++ b/python/amazon-product-scraping/README.md @@ -0,0 +1,67 @@ +# Stagehand + Browserbase: Amazon Product Scraping + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- **Goal**: Scrape the first 3 Amazon search results for a given query and return structured product data. +- **AI-Powered Search**: Uses Stagehand `act` to type in the search bar and click search (or optionally navigate directly to the search URL). +- **Structured Extraction**: Uses `extract` with a JSON schema to get product name, price, rating, review count, and product URL. +- **Model**: Uses `google/gemini-2.5-flash` for fast, cost-effective automation. +- Docs โ†’ https://docs.stagehand.dev + +## GLOSSARY + +- **act**: Perform UI actions from a prompt (type in search bar, click search). + Docs โ†’ https://docs.stagehand.dev/v4/basics/act +- **extract**: Pull structured data from pages using JSON schemas. + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract + +## QUICKSTART + +1. cd python/amazon-product-scraping +2. cp .env.example .env (or create .env with required keys) +3. Add `BROWSERBASE_API_KEY` to .env +4. Optionally edit `SEARCH_QUERY` in main.py +5. Run the script: + ```bash + uv run python main.py + ``` + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase +- Displays live session link for monitoring +- Navigates to Amazon and performs search (or direct URL navigation if uncommented) +- Extracts the first 3 products with name, price, rating, reviews count, and product URL +- Outputs JSON to console +- Closes session cleanly + +## COMMON PITFALLS + +- **Import errors**: Ensure you're running with `uv run python main.py` so dependencies are installed +- **Missing credentials**: Verify .env contains `BROWSERBASE_API_KEY` +- **Amazon layout changes**: Extraction may need prompt/schema updates if Amazon changes their search results UI +- Find more information on your Browserbase dashboard โ†’ https://www.browserbase.com/sign-in + +## USE CASES + +โ€ข **Price monitoring**: Scrape top results for a product query to track prices and availability over time. +โ€ข **Competitor research**: Extract product titles, ratings, and review counts for comparison. +โ€ข **Catalog building**: Pull structured product data for feeds, dashboards, or internal tools. + +## NEXT STEPS + +โ€ข **Switch to direct URL**: Uncomment the URL-based search block in main.py for faster runs without LLM search actions. +โ€ข **Parameterize query**: Accept `SEARCH_QUERY` from CLI or env for different products without editing code. +โ€ข **Paginate**: Extend extraction to multiple pages or increase the number of products per run. + +## HELPFUL RESOURCES + +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev +๐Ÿ“š Python SDK: https://docs.stagehand.dev/v4/sdk/python +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/amazon-product-scraping/main.py b/python/amazon-product-scraping/main.py new file mode 100644 index 00000000..06a79398 --- /dev/null +++ b/python/amazon-product-scraping/main.py @@ -0,0 +1,106 @@ +"""Scrape Amazon search results with Stagehand V4.""" + +import asyncio +import json +import os +from urllib.parse import quote_plus, urljoin + +from dotenv import load_dotenv +from pydantic import BaseModel, Field, HttpUrl +from stagehand import Stagehand, browserbase + +load_dotenv() + +SEARCH_QUERY = "Seiko 5" + + +class Product(BaseModel): + name: str + price: str + rating: str + reviews_count: str + product_url: HttpUrl = Field(description="Absolute Amazon product-detail href") + + +class Products(BaseModel): + products: list[Product] = Field(description="First three Amazon search results") + + +async def main() -> None: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") + + browser = await browserbase.launch(api_key=api_key) + try: + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto( + "https://www.amazon.com", + wait_until="domcontentloaded", + timeout=60_000, + ) + typed = await stagehand.act( + f'Type "{SEARCH_QUERY}" into the search bar', + page=page, + ) + submitted = await stagehand.act("Click the search button", page=page) + if not typed.data.success or not submitted.data.success: + raise RuntimeError( + typed.data.message or submitted.data.message or "Amazon search failed" + ) + page = await browser.context.active_page() or page + try: + results_ready = await page.wait_for_selector( + '[data-component-type="s-search-result"]', + timeout=10_000, + ) + except Exception: + results_ready = None + if not results_ready: + # Amazon can replace the document during submit and invalidate + # the action frame. Use the direct URL only after that failure. + search_url = f"https://www.amazon.com/s?k={quote_plus(SEARCH_QUERY)}" + await page.goto(search_url, wait_until="domcontentloaded", timeout=60_000) + await page.wait_for_selector( + '[data-component-type="s-search-result"]', + timeout=15_000, + ) + extracted = await stagehand.extract( + ( + "Extract the details of the FIRST 3 products in the search results. " + "Return each product's full name, displayed price, star rating, review " + "count, and absolute product-page href. Each URL must be a real Amazon " + "link containing /dp/." + ), + Products, + page=page, + ) + products = extracted.data.products + normalized = [ + { + **product.model_dump(mode="json"), + "product_url": urljoin("https://www.amazon.com", str(product.product_url)), + } + for product in products + ] + + print(json.dumps({"products": normalized}, indent=2)) + finally: + await stagehand.close() + finally: + await browser.close() + print("Session closed successfully") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as error: + print(f"Amazon product scraping failed: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/amazon-product-scraping/pyproject.toml b/python/amazon-product-scraping/pyproject.toml new file mode 100644 index 00000000..501f2336 --- /dev/null +++ b/python/amazon-product-scraping/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "amazon-product-scraping" +version = "0.1.0" +description = "Scrape Amazon product search results using Stagehand and Browserbase" +readme = "README.md" +requires-python = ">=3.11,<3.14" +dependencies = [ + "beautifulsoup4==4.14.3", + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py311'] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/python/basic-caching/.env.example b/python/basic-caching/.env.example new file mode 100644 index 00000000..d0cde042 --- /dev/null +++ b/python/basic-caching/.env.example @@ -0,0 +1 @@ +BROWSERBASE_API_KEY= diff --git a/python/basic-caching/README.md b/python/basic-caching/README.md new file mode 100644 index 00000000..f891628a --- /dev/null +++ b/python/basic-caching/README.md @@ -0,0 +1,128 @@ +# Stagehand + Browserbase: Basic Caching + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- Goal: Demonstrate how Stagehand's caching feature dramatically reduces cost and latency by reusing previously computed actions instead of calling the LLM every time. +- Shows side-by-side comparison of workflows with and without caching enabled. +- Demonstrates massive cost savings for repeated workflows (99.9% reduction in LLM calls). +- Docs โ†’ https://docs.stagehand.dev/v4/best-practices/caching#caching-actions + +## GLOSSARY + +- caching: Stagehand can cache action results based on instruction text and page context, eliminating redundant LLM calls + Docs โ†’ https://docs.stagehand.dev/v4/best-practices/caching#caching-actions +- act: execute actions on web pages using natural language instructions + Docs โ†’ https://docs.stagehand.dev/v4/basics/act +- observe: observe page elements and generate actions that can be cached and reused + Docs โ†’ https://docs.stagehand.dev/v4/basics/observe + +## QUICKSTART + +1. uv venv venv +2. source venv/bin/activate # On Windows: venv\Scripts\activate +3. uvx install stagehand python-dotenv aiofiles +4. cp .env.example .env +5. Add your Browserbase API key to .env +6. python main.py (run twice to see cache benefits!) + +## EXPECTED OUTPUT + +- First run: Executes workflow without cache, then with cache enabled (populates cache) +- Subsequent runs: Uses cached actions for instant execution with zero LLM calls +- Displays timing comparison, cost savings, and cache statistics +- Shows cache location and file structure + +## HOW CACHING WORKS + +**Cache Key Generation:** + +- Based on instruction text (used as cache key) +- Actions are observed once and cached +- Automatically computed + +**When Cache is Used:** + +- โœ… Same instruction text +- โœ… Cache file exists (`cache.json`) +- โŒ Different instruction text +- โŒ Cache file cleared or missing + +**Cache Storage:** + +- Location: `cache.json` in the same directory as `main.py` +- Format: JSON file containing cached actions +- Persistent across runs + +## BENEFITS FOR REPEATED WORKFLOWS + +**Example Scenario: 1,000 customers ร— 10 portals = 10,000 payment flows** + +**Without caching:** + +- 10,000 workflows ร— 5 actions = 50,000 LLM calls +- Cost: ~$500-2,500 +- Latency: 2-3s per action ร— 5 = 10-15s per payment + +**With caching:** + +- First payment per portal: 5 LLM calls (populate cache) +- Next 999 payments: 0 LLM calls (use cache) +- Total: 10 portals ร— 5 actions = 50 LLM calls +- Cost: ~$0.50-2.50 (99.9% savings!) +- Latency: <100ms per action ร— 5 = <0.5s per payment + +**Key Insight:** +Payment portals rarely change โ†’ Cache actions once โ†’ Reuse for thousands of payments โ†’ Massive cost + latency reduction + +## COMMON PITFALLS + +- Missing credentials: verify .env contains BROWSERBASE_API_KEY +- Cache not working: ensure cache.json is writable and check that instruction text matches exactly +- First run slower: expected behavior - cache is populated on first run, subsequent runs will be instant +- ModuleNotFoundError: ensure virtual environment is activated and dependencies are installed via `uvx install` +- Import errors: activate your virtual environment if you created one +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +โ€ข Payment processing: Cache form-filling actions for payment portals that don't change frequently, processing thousands of payments with minimal LLM calls. +โ€ข Data entry automation: Reuse actions for repetitive data entry tasks across similar forms or interfaces. +โ€ข Testing workflows: Cache test actions to speed up regression testing and reduce API costs during development. + +## BEST PRACTICES + +- โœ… Enable caching in production for repeated workflows +- โœ… One cache per portal/interface type +- โœ… Invalidate cache when page structure changes significantly +- โœ… Monitor cache hit rate to optimize cache effectiveness +- โœ… Warm cache with test runs before production deployment + +## NEXT STEPS + +โ€ข Customize cache file location: Modify CACHE_FILE path to organize caches by workflow type or environment. +โ€ข Add cache invalidation: Implement logic to clear cache when page structure changes or after a certain time period. +โ€ข Monitor cache performance: Track cache hit rates and cost savings to measure effectiveness. + +## TRY IT YOURSELF + +1. Run this script again: `python main.py` + โ†’ Second run will be MUCH faster (cache hits) + +2. Clear cache and run again: + `rm cache.json && python main.py` + โ†’ Back to first-run behavior + +3. Check cache contents: + `cat cache.json` + โ†’ See cached action data + +## HELPFUL RESOURCES + +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/basic-caching/main.py b/python/basic-caching/main.py new file mode 100644 index 00000000..46cbc1ed --- /dev/null +++ b/python/basic-caching/main.py @@ -0,0 +1,74 @@ +"""Prove a repeated Stagehand V4 observation is served from cache.""" + +import asyncio +import json +import os +import time + +from dotenv import load_dotenv + +from stagehand import Stagehand, browserbase + +load_dotenv() + +INSTRUCTION = "Find the More information link" + + +async def main() -> None: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") + + browser = await browserbase.launch(api_key=api_key) + try: + stagehand = await Stagehand.create( + browser=browser, + cache={"threshold": 1}, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto("https://example.com", wait_until="domcontentloaded") + + started = time.perf_counter() + first = await stagehand.observe(INSTRUCTION, page=page) + first_ms = round((time.perf_counter() - started) * 1_000) + if not first.data: + raise RuntimeError("First observation returned no link") + + started = time.perf_counter() + second = await stagehand.observe(INSTRUCTION, page=page) + second_ms = round((time.perf_counter() - started) * 1_000) + if not second.data: + raise RuntimeError("Second observation returned no link") + + first_cache = first.metadata.cache + second_cache = second.metadata.cache + report = { + "first": { + "cache": first_cache.status if first_cache else "DISABLED", + "duration_ms": first_ms, + }, + "second": { + "cache": second_cache.status if second_cache else "DISABLED", + "duration_ms": second_ms, + "tokens_saved": ( + second_cache.tokens_saved.model_dump(mode="json") + if second_cache and second_cache.tokens_saved + else None + ), + }, + } + print(json.dumps(report, indent=2)) + if second_cache is None or second_cache.status != "HIT": + status = second_cache.status if second_cache else "DISABLED" + raise RuntimeError(f"Expected a cache HIT, received {status}") + print("Cache verified: repeated observation avoided inference") + finally: + await stagehand.close() + finally: + await browser.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/basic-caching/pyproject.toml b/python/basic-caching/pyproject.toml new file mode 100644 index 00000000..bd5714dd --- /dev/null +++ b/python/basic-caching/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = "basic-caching" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[tool.uv] +package = false diff --git a/python/basic-caching/requirements.txt b/python/basic-caching/requirements.txt new file mode 100644 index 00000000..d22bf5b8 --- /dev/null +++ b/python/basic-caching/requirements.txt @@ -0,0 +1,3 @@ +stagehand==4.0.0 +python-dotenv +aiofiles diff --git a/python/basic-captcha-solving/.env.example b/python/basic-captcha-solving/.env.example new file mode 100644 index 00000000..d0cde042 --- /dev/null +++ b/python/basic-captcha-solving/.env.example @@ -0,0 +1 @@ +BROWSERBASE_API_KEY= diff --git a/python/basic-captcha-solving/README.md b/python/basic-captcha-solving/README.md new file mode 100644 index 00000000..00f5c772 --- /dev/null +++ b/python/basic-captcha-solving/README.md @@ -0,0 +1,108 @@ +# Stagehand + Browserbase: Basic CAPTCHA Solving + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- Goal: Demonstrate automatic CAPTCHA solving using Browserbase's built-in captcha solving capabilities. +- Automated Solving: Browserbase automatically detects and solves CAPTCHAs in the background. CAPTCHA solving is **enabled by default** - you don't need to set `solveCaptchas: true` unless you want to explicitly enable it (or set it to `false` to disable). +- Solving Time: CAPTCHA solving typically takes between 5-30 seconds depending on CAPTCHA type and complexity. +- Progress Monitoring: Listen for console messages (`browserbase-solving-started`, `browserbase-solving-finished`) to track captcha solving progress in real-time. +- Proxies Recommended: Enable proxies for higher CAPTCHA solving success rates. +- Result inspection: Extracts and prints the page content after form submission. +- Docs โ†’ https://docs.browserbase.com/features/stealth-mode#captcha-solving + +## GLOSSARY + +- solveCaptchas: Browserbase browser setting that enables automatic CAPTCHA solving. Enabled by default for Basic and Advanced Stealth Mode. + Docs โ†’ https://docs.browserbase.com/features/stealth-mode#captcha-solving +- CAPTCHA solving: When a CAPTCHA is detected, Browserbase attempts to solve it automatically in the background, allowing your automation to continue without manual intervention. +- console messages: browser console events that indicate captcha solving status: + - `browserbase-solving-started`: emitted when CAPTCHA detection begins + - `browserbase-solving-finished`: emitted when CAPTCHA solving completes +- custom CAPTCHA solving: For non-standard or custom captcha providers, you can specify CSS selectors for the captcha image and input field using `captchaImageSelector` and `captchaInputSelector` in browserSettings. +- act: perform UI actions from a prompt (type, click, fill forms) + Docs โ†’ https://docs.stagehand.dev/v4/basics/act +- extract: pull data from web pages using natural language instructions + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract + +## CAPTCHA SOLVING DETAILS + +### How CAPTCHA Solving Works + +Browserbase provides integrated CAPTCHA solving to handle challenges automatically: + +- **Automatic Detection**: When a CAPTCHA is detected on a page, Browserbase attempts to solve it in the background +- **Solving Time**: CAPTCHA solving typically takes between 5-30 seconds, depending on the CAPTCHA type and complexity +- **Default Behavior**: CAPTCHA solving is enabled by default for Basic and Advanced Stealth Mode +- **Proxies**: It's recommended to enable proxies when using CAPTCHA solving for higher success rates +- **Multiple Types**: Browserbase supports common CAPTCHA types automatically + +### Custom CAPTCHA Solving + +For non-standard or custom captcha providers, you can specify CSS selectors to guide the solution process: + +```python +browserbase_session_create_params = { + "browser_settings": { + "solveCaptchas": True, + "captchaImageSelector": "#custom-captcha-image-id", + "captchaInputSelector": "#custom-captcha-input-id", + } +} +``` + +To find the selectors: + +1. Right-click on the captcha image and select "Inspect" to get the image selector +2. Right-click on the input field and select "Inspect" to get the input selector +3. Use the element's `id` or a CSS selector that uniquely identifies it + +### Disabling CAPTCHA Solving + +If you want to disable automatic captcha solving, set `solveCaptchas: False` in browserSettings: + +```python +browserbase_session_create_params = {"browser_settings": {"solveCaptchas": False}} +``` + +## QUICKSTART + +1. uv venv venv +2. source venv/bin/activate # On Windows: venv\Scripts\activate +3. uvx install stagehand python-dotenv +4. cp .env.example .env # Add your Browserbase API key to .env +5. python main.py + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase +- Displays live session link for monitoring +- Navigates to CAPTCHA demo page +- Waits for Browserbase to automatically solve the CAPTCHA +- Logs CAPTCHA solving progress messages +- Clicks submit button after CAPTCHA is solved +- Extracts and displays page content +- Prints the resulting page content for inspection +- Closes session cleanly + +## COMMON PITFALLS + +- Missing credentials: verify .env contains BROWSERBASE_API_KEY +- Captcha solving not enabled: ensure `solveCaptchas: True` is set in browserSettings (enabled by default) +- Solving timeout: allow up to 30 seconds for CAPTCHA solving to complete before timing out +- Proxies not enabled: enable proxies in browserSettings for higher CAPTCHA solving success rates +- Demo page inaccessible: verify the CAPTCHA demo page URL is accessible and hasn't changed +- Console message timing: ensure console event listeners are set up before triggering the CAPTCHA +- Custom captcha selectors: for non-standard CAPTCHAs, verify that `captchaImageSelector` and `captchaInputSelector` are correctly defined +- Import errors: activate your virtual environment if you created one +- ModuleNotFoundError: ensure all dependencies are installed via uvx install + +## HELPFUL RESOURCES + +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/basic-captcha-solving/main.py b/python/basic-captcha-solving/main.py new file mode 100644 index 00000000..3a9c16c7 --- /dev/null +++ b/python/basic-captcha-solving/main.py @@ -0,0 +1,63 @@ +"""Solve and verify CAPTCHA challenge with Stagehand V4.""" + +import asyncio +import os + +from dotenv import load_dotenv + +from stagehand import Stagehand, browserbase + +load_dotenv() + + +async def main() -> None: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") + + browser = await browserbase.launch( + api_key=api_key, + browser_settings={"solve_captchas": True}, + ) + try: + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto( + "https://google.com/recaptcha/api2/demo", + wait_until="domcontentloaded", + timeout=60_000, + ) + + print("Waiting for Browserbase captcha solving...") + token = "" + for _ in range(60): + token = await page.locator("#g-recaptcha-response").input_value() + if token: + break + await asyncio.sleep(1) + if not token: + raise RuntimeError("Captcha token was not populated within 60 seconds") + + await stagehand.act("Click the Submit button", page=page) + extracted = await stagehand.extract("Extract all text on this page", page=page) + text = extracted.data.extraction + print("Page content after submission:") + print(text) + finally: + await stagehand.close() + finally: + await browser.close() + print("Session closed successfully") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as error: + print(f"CAPTCHA solving example failed: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/basic-captcha-solving/pyproject.toml b/python/basic-captcha-solving/pyproject.toml new file mode 100644 index 00000000..4bf4aec7 --- /dev/null +++ b/python/basic-captcha-solving/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = "basic-captcha-solving" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = ["python-dotenv==1.2.2", "stagehand==4.0.0"] + +[tool.uv] +package = false diff --git a/python/browserbase-reducto/README.md b/python/browserbase-reducto/README.md new file mode 100644 index 00000000..d57d3e74 --- /dev/null +++ b/python/browserbase-reducto/README.md @@ -0,0 +1,84 @@ +# Stagehand + Browserbase + Reducto: Download PDFs and Extract Financial Data + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- **Goal**: Automate downloading financial PDFs from websites and extract structured data using AI-powered document parsing. +- **Pattern Template**: Demonstrates the integration pattern of Browserbase (download automation) + Reducto (document extraction). +- **Workflow**: Uses Stagehand to navigate websites, Browserbase automatically downloads PDFs when opened, then Reducto extracts structured financial data using schema-based extraction. +- **Download Handling**: Implements retry logic with polling to handle Browserbase's async download sync (files sync to cloud storage in real-time). +- **Structured Extraction**: Uses Reducto's extract API with JSON schema to pull specific financial metrics from complex PDF tables. +- Docs โ†’ [Browserbase Downloads](https://docs.browserbase.com/features/downloads) | [Reducto Extract](https://docs.reducto.ai/parse/best-practices) + +## GLOSSARY + +- **act**: perform UI actions from natural language prompts (click, scroll, navigate) + Docs โ†’ https://docs.stagehand.dev/v4/basics/act +- **Browserbase Downloads**: When a PDF URL is opened in a browser session, Browserbase automatically downloads and stores it in cloud storage. Files must be retrieved via the Session Downloads API as a ZIP archive. + Docs โ†’ https://docs.browserbase.com/features/downloads +- **Reducto Extract**: Extract structured data from PDFs using JSON schema definitions. More efficient than parsing entire documents when you only need specific fields. + Docs โ†’ https://docs.reducto.ai/extract +- **Schema-based extraction**: Define the exact structure you want extracted (fields, types, descriptions) and Reducto returns JSON matching your schema. +- **Download polling**: Browserbase syncs downloads in real-time; larger files may need retry logic to ensure availability via the API. + +## QUICKSTART + +1. cd python/reducto-browserbase +2. Install dependencies with uv: + + ```bash + uv pip install -e . + ``` + + This will install all dependencies from `pyproject.toml`. + + Alternatively, use uvx to run without installation: + + ```bash + uvx --with browserbase --with reductoai --with stagehand-ai --with python-dotenv python main.py + ``` + +3. cp .env.example .env +4. Add required API keys to .env: + - `BROWSERBASE_API_KEY` + - `REDUCTOAI_API_KEY` +5. Run the script: + ```bash + python main.py + ``` + Or with uv: + ```bash + uv run python main.py + ``` + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase and displays live view link +- Navigates to Apple.com investor relations section +- Clicks through to Q4 financial statements +- Browserbase automatically downloads PDF when link is opened +- Polls Browserbase Downloads API until file is ready (with retry logic) +- Extracts PDF from ZIP archive downloaded from Browserbase +- Uploads PDF to Reducto and extracts structured iPhone net sales data +- Outputs extracted financial data as formatted JSON +- Closes session cleanly + +## NEXT STEPS + +โ€ข **Parameterize extraction**: Accept different schema definitions or document types as configuration to extract various financial metrics or data structures. +โ€ข **Batch processing**: Process multiple quarters or companies by looping through different navigation paths and extracting data for each. +โ€ข **Multi-document support**: Handle ZIP archives with multiple PDFs and extract data from each, aggregating results into a unified dataset. +โ€ข **Optimize extraction**: Use Reducto's agentic mode selectively (only for complex tables or low-quality scans) to reduce latency and credit usage. Enable `scope: "table"` only when tables are misaligned or have merged cells. +Docs โ†’ https://docs.reducto.ai/parse/best-practices#2-enable-agentic-mode-only-when-needed + +## HELPFUL RESOURCES + +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction +๐Ÿ“š Browserbase Downloads: https://docs.browserbase.com/features/downloads +๐Ÿ“š Reducto Best Practices: https://docs.reducto.ai/parse/best-practices +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/browserbase-reducto/main.py b/python/browserbase-reducto/main.py new file mode 100644 index 00000000..878796d0 --- /dev/null +++ b/python/browserbase-reducto/main.py @@ -0,0 +1,202 @@ +"""Download Apple's FY2025 Q4 statement and extract sales with Reducto.""" + +import asyncio +import json +import os +import time +import zipfile +from pathlib import Path +from typing import Any + +from browserbase import AsyncBrowserbase +from dotenv import load_dotenv +from pydantic import BaseModel, HttpUrl +from reducto import Reducto +from stagehand import Stagehand, browserbase + +load_dotenv() + + +class StatementLink(BaseModel): + statement_url: HttpUrl + + +async def save_downloads_with_retry( + client: AsyncBrowserbase, + session_id: str, + retry_for_seconds: int = 60, +) -> int: + started = time.monotonic() + while time.monotonic() - started < retry_for_seconds: + response = await client.sessions.downloads.list(session_id) + payload = await response.read() + if len(payload) > 100: + Path("downloaded_files.zip").write_bytes(payload) + print(f"Saved downloaded_files.zip ({len(payload)} bytes)") + return len(payload) + await asyncio.sleep(2) + raise TimeoutError("Download timeout exceeded") + + +def extract_pdf_from_zip( + zip_path: str, + output_dir: str = "downloaded_files", +) -> Path: + destination = Path(output_dir).resolve() + destination.mkdir(parents=True, exist_ok=True) + first_pdf: Path | None = None + + with zipfile.ZipFile(zip_path) as archive: + entries = [name for name in archive.namelist() if not name.endswith("/")] + for entry in entries: + with archive.open(entry) as source: + payload = source.read() + if not payload.startswith(b"%PDF"): + continue + output_name = entry if entry.lower().endswith(".pdf") else f"{entry}.pdf" + output = (destination / output_name).resolve() + if destination not in output.parents: + raise RuntimeError(f"Unsafe ZIP entry: {entry}") + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("wb") as target: + target.write(payload) + first_pdf = first_pdf or output + + if first_pdf is None: + raise RuntimeError("Failed to extract a PDF") + return first_pdf + + +async def extract_pdf_with_reducto(pdf_path: Path, reducto_client: Reducto) -> dict[str, Any]: + upload = await asyncio.to_thread(reducto_client.upload, file=pdf_path) + print("Uploaded statement to Reducto") + schema = { + "type": "object", + "properties": { + "iphone_net_sales": { + "type": "object", + "properties": { + "current_quarter": {"type": "number"}, + "previous_quarter": {"type": "number"}, + "current_year": {"type": "number"}, + "previous_year": {"type": "number"}, + "current_quarter_date": {"type": "string"}, + "previous_quarter_date": {"type": "string"}, + }, + "required": [ + "current_quarter", + "previous_quarter", + "current_year", + "previous_year", + "current_quarter_date", + "previous_quarter_date", + ], + } + }, + "required": ["iphone_net_sales"], + } + response = await asyncio.to_thread( + reducto_client.extract.run, + input=upload, + instructions={ + "schema": schema, + "system_prompt": ( + "Extract the iPhone net sales values from the net sales by " + "reportable segment table in this financial statement." + ), + }, + settings={ + "optimize_for_latency": True, + "citations": {"numerical_confidence": False}, + }, + ) + + extracted: Any = getattr(response, "result", response) + if isinstance(extracted, list): + extracted = extracted[0] if extracted else None + if hasattr(extracted, "model_dump"): + extracted = extracted.model_dump(mode="json") + return extracted + + +async def main() -> None: + browserbase_key = os.environ.get("BROWSERBASE_API_KEY") + reducto_key = os.environ.get("REDUCTOAI_API_KEY") + if not browserbase_key or not reducto_key: + raise RuntimeError("BROWSERBASE_API_KEY and REDUCTOAI_API_KEY are required") + + api = AsyncBrowserbase(api_key=browserbase_key) + reducto = Reducto(api_key=reducto_key) + browser = await browserbase.launch(api_key=browserbase_key) + session_id = browser.session_id + if not session_id: + await browser.close() + raise RuntimeError("Browserbase launch did not return a session ID") + + try: + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto("https://www.apple.com/", wait_until="domcontentloaded", timeout=60_000) + await stagehand.act( + "Click the Investors button at the bottom of the page", + page=page, + ) + await stagehand.act( + "Scroll down to the Financial Data section", + page=page, + ) + await stagehand.act( + "Under Quarterly Earnings Reports, click 2025", + page=page, + ) + page = await browser.context.active_page() or page + extracted = await stagehand.extract( + ( + "Extract the actual absolute HTTP(S) href URL of the FY2025 Q4 Financial " + "Statements PDF." + ), + StatementLink, + page=page, + ) + statement_url = str(extracted.data.statement_url) + + opened_statement = await stagehand.act( + "Click the Financial Statements link under Q4", + page=page, + ) + if not opened_statement.data.success: + encoded_url = json.dumps(statement_url) + await page.evaluate( + f"""(() => {{ + const link = document.createElement('a'); + link.href = {encoded_url}; + link.target = '_blank'; + document.body.appendChild(link); + link.click(); + link.remove(); + }})()""" + ) + print("Triggered FY2025 Q4 statement download") + await save_downloads_with_retry(api, session_id) + pdf_path = extract_pdf_from_zip("downloaded_files.zip") + extracted = await extract_pdf_with_reducto(pdf_path, reducto) + print(json.dumps(extracted, indent=2)) + finally: + await stagehand.close() + finally: + await browser.close() + await api.close() + print("Session closed successfully") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as error: + print(f"Application error: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/browserbase-reducto/pyproject.toml b/python/browserbase-reducto/pyproject.toml new file mode 100644 index 00000000..8057e4c1 --- /dev/null +++ b/python/browserbase-reducto/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "browserbase-reducto" +version = "0.1.0" +description = "Download Apple's Q4 Financial Statement and Parse with Reducto using Stagehand and Browserbase" +readme = "README.md" +requires-python = ">=3.11,<3.14" +dependencies = [ + "browserbase>=1.7.0", + "python-dotenv", + "reductoai", + "stagehand==4.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/python/business-lookup/.env.example b/python/business-lookup/.env.example new file mode 100644 index 00000000..56823daf --- /dev/null +++ b/python/business-lookup/.env.example @@ -0,0 +1,3 @@ +BROWSERBASE_API_KEY= +AI_GATEWAY_API_KEY= +# DEEPAGENTS_MODEL=anthropic/claude-sonnet-4.6 diff --git a/python/business-lookup/README.md b/python/business-lookup/README.md index 8c815a8c..edc63267 100644 --- a/python/business-lookup/README.md +++ b/python/business-lookup/README.md @@ -1,64 +1,50 @@ -# Stagehand + Browserbase: Business Lookup with Agent +# Business lookup with a Python agent -## AT A GLANCE +Stagehand is the SDK for browser agents. -- Goal: Automate business registry searches using an autonomous AI agent with computer-use capabilities. -- Uses Stagehand Agent in CUA mode to navigate complex UI elements, apply filters, and extract structured business data. -- Demonstrates extraction with Pydantic schema validation for consistent data retrieval. -- Docs โ†’ https://docs.stagehand.dev/basics/agent +This template uses LangChain Deep Agents for the reasoning loop and Stagehand V4 code mode for the +browser. The agent opens San Francisco's official Open Data API, finds an exact DBA record, and +returns a validated Pydantic object. -## GLOSSARY +## How it works -- agent: create an autonomous AI agent that can execute complex multi-step tasks - Docs โ†’ https://docs.stagehand.dev/basics/agent#what-is-agent -- extract: extract structured data from web pages using natural language instructions - Docs โ†’ https://docs.stagehand.dev/basics/extract +- `create_deep_agent` owns planning, model calls, and structured output. +- Stagehand code mode exposes one persistent Browserbase session through `run`, `snapshot`, and + `screenshot` MCP tools. +- Vercel AI Gateway supplies the bring-your-own agent model. +- The Stagehand MCP server runs in an isolated `uvx` environment because the current Stagehand and + Deep Agents clients require different `websockets` versions. +- The template closes the MCP session and browser process automatically. -## QUICKSTART +## Quickstart -1. uv venv venv -2. source venv/bin/activate # On Windows: venv\Scripts\activate -3. uvx install stagehand python-dotenv pydantic -4. cp .env.example .env # Add required API keys/IDs to .env -5. python main.py +Requirements: Python 3.11โ€“3.13 and [uv](https://docs.astral.sh/uv/). -## EXPECTED OUTPUT +```bash +cp .env.example .env +# Add BROWSERBASE_API_KEY and AI_GATEWAY_API_KEY to .env. +uv sync +uv run python main.py +``` -- Initializes Stagehand session with Browserbase -- Displays live session link for monitoring -- Navigates to SF Business Registry search page -- Agent searches for business using DBA Name filter -- Agent completes search and opens business details -- Extracts structured business information (DBA Name, Account Number, NAICS Code, etc.) -- Outputs extracted data as JSON -- Closes session cleanly +The first run installs the Stagehand Deep Agents integration from `stagehand/main` in `uvx`; the +integration pins its Stagehand server dependency to `stagehand==4.0.0`. -## COMMON PITFALLS +## Expected outcome -- "ModuleNotFoundError": ensure all dependencies are installed via pip -- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY -- Google API access: ensure you have access to Google's gemini-2.5-computer-use-preview-10-2025 model -- Agent failures: check that the business name exists in the registry and that max_steps is sufficient for complex searches -- Import errors: activate your virtual environment if you created one -- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in +The agent opens the official SF Open Data JSON endpoint and returns the exact Jalebi Street record, +including its business account number, location ID, address, NAICS data when present, and the +official source URL. The script exits nonzero if the returned DBA or evidence source does not match. -## USE CASES +## Configuration -โ€ข Business verification: Automate registration status checks, license validation, and compliance verification for multiple businesses. -โ€ข Data enrichment: Collect structured business metadata (NAICS codes, addresses, ownership) for research or CRM updates. -โ€ข Due diligence: Streamline background checks by autonomously searching and extracting business registration details from public registries. +- `BROWSERBASE_API_KEY`: launches the Browserbase session. +- `AI_GATEWAY_API_KEY`: authenticates the Deep Agents model through Vercel AI Gateway. +- `DEEPAGENTS_MODEL`: optional model override; defaults to `anthropic/claude-sonnet-4.6`. +- `STAGEHAND_RUN_TIMEOUT_MS`: optional browser-tool timeout; defaults to 120 seconds. -## NEXT STEPS +## Resources -โ€ข Parameterize search: Accept business names as command-line arguments or from a CSV file for batch processing. -โ€ข Expand extraction: Add support for additional fields like tax status, licenses, or historical registration changes. -โ€ข Multi-registry support: Extend agent to search across multiple city or state business registries with routing logic. - -## HELPFUL RESOURCES - -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction -๐ŸŽฎ Browserbase: https://www.browserbase.com -๐Ÿ’ก Try it out: https://www.browserbase.com/playground -๐Ÿ”ง Templates: https://www.browserbase.com/templates -๐Ÿ“ง Need help? support@browserbase.com -๐Ÿ’ฌ Discord: http://stagehand.dev/discord +- [Stagehand V4 documentation](https://docs.stagehand.dev/v4) +- [Stagehand Deep Agents integration](https://github.com/browserbase/stagehand/tree/main/packages/integrations/deepagents) +- [Browserbase sessions](https://www.browserbase.com/overview/sessions) diff --git a/python/business-lookup/agent_runtime.py b/python/business-lookup/agent_runtime.py new file mode 100644 index 00000000..1a78e8bb --- /dev/null +++ b/python/business-lookup/agent_runtime.py @@ -0,0 +1,68 @@ +"""Shared Deep Agents + Stagehand V4 code-mode setup for this template.""" + +from __future__ import annotations + +import os + +from langchain_mcp_adapters.client import MultiServerMCPClient +from langchain_openai import ChatOpenAI +from pydantic import SecretStr + +SERVER_NAME = "stagehand_browser" + +STAGEHAND_DEEPAGENTS_SOURCE = ( + "git+https://github.com/browserbase/stagehand.git@main" + "#subdirectory=packages/integrations/deepagents" +) + +BROWSER_INSTRUCTIONS = """You control one persistent Browserbase browser through exactly three +Stagehand code-mode tools: +- snapshot: inspect the active page and hydrate bracketed element IDs. +- run: execute snapshot actions or JavaScript with the Playwright-shaped page API. +- screenshot: inspect the rendered page visually. + +Use run with deterministic page APIs for known URLs and structured pages. Use snapshot before +interacting with an unfamiliar UI. Snapshot IDs are valid only for the latest snapshot. Do not +launch another browser, and do not claim evidence from a URL you did not open. +""" + + +def require_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required") + return value + + +def create_gateway_model(default_model: str) -> ChatOpenAI: + """Use Vercel AI Gateway as the bring-your-own agent model provider.""" + return ChatOpenAI( + model=os.environ.get("DEEPAGENTS_MODEL", default_model), + api_key=SecretStr(require_env("AI_GATEWAY_API_KEY")), + base_url="https://ai-gateway.vercel.sh/v1", + ) + + +def create_stagehand_client() -> MultiServerMCPClient: + """Start the Stagehand V4 server from main in an isolated uvx environment.""" + server_env = { + "BROWSERBASE_API_KEY": require_env("BROWSERBASE_API_KEY"), + "STAGEHAND_BROWSER": "browserbase", + "STAGEHAND_RUN_TIMEOUT_MS": os.environ.get("STAGEHAND_RUN_TIMEOUT_MS", "120000"), + } + return MultiServerMCPClient( + { + SERVER_NAME: { + "transport": "stdio", + "command": os.environ.get("UVX_COMMAND", "uvx"), + "args": [ + "--from", + STAGEHAND_DEEPAGENTS_SOURCE, + "--with", + "stagehand==4.0.0", + "stagehand-deepagents-mcp", + ], + "env": server_env, + } + } + ) diff --git a/python/business-lookup/main.py b/python/business-lookup/main.py index 74358685..b0611bc8 100644 --- a/python/business-lookup/main.py +++ b/python/business-lookup/main.py @@ -1,129 +1,84 @@ -# Stagehand + Browserbase: Business Lookup with Agent - See README.md for full documentation +"""Look up an official business record with Deep Agents and Stagehand V4 code mode.""" + +from __future__ import annotations import asyncio -import json -import os +from agent_runtime import ( + BROWSER_INSTRUCTIONS, + SERVER_NAME, + create_gateway_model, + create_stagehand_client, +) +from deepagents import create_deep_agent from dotenv import load_dotenv -from pydantic import BaseModel, Field - -from stagehand import Stagehand, StagehandConfig +from langchain_mcp_adapters.tools import load_mcp_tools +from pydantic import BaseModel, ConfigDict, Field -# Load environment variables load_dotenv() -# Business search variables -business_name = "Jalebi Street" - - -async def main(): - print("Starting business lookup...") - - # Initialize Stagehand with Browserbase for cloud-based browser automation. - # Note: set verbose: 0 to prevent API keys from appearing in logs when handling sensitive data. - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - }, - verbose=1, # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - ) - - try: - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - # Initialize browser session to start automation. - print("Stagehand initialized successfully") - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id - - if session_id: - print(f"Live View Link: https://browserbase.com/sessions/{session_id}") - - page = stagehand.page - - # Navigate to SF Business Registry search page. - print("Navigating to SF Business Registry...") - await page.goto( - "https://data.sfgov.org/stories/s/Registered-Business-Lookup/k6sk-2y6w/", - wait_until="domcontentloaded", - timeout=60000, - ) - - # Create agent with computer use capabilities for autonomous business search. - # Using CUA mode allows the agent to interact with complex UI elements like filters and tables. - print("Creating Computer Use Agent...") - agent = stagehand.agent( - provider="google", - model="gemini-2.5-computer-use-preview-10-2025", - instructions="You are a helpful assistant that can use a web browser to search for business information.", - options={ - "api_key": os.getenv("GOOGLE_API_KEY"), - }, - ) - - print(f"Searching for business: {business_name}") - result = await agent.execute( - instruction=f'Find and look up the business "{business_name}" in the SF Business Registry. Use the DBA Name filter to search for "{business_name}", apply the filter, and click on the business row to view detailed information. Scroll towards the right to see the NAICS code.', - max_steps=30, - auto_screenshot=True, - ) - - if not result.success: - raise Exception("Agent failed to complete the search") - - print("Agent completed search successfully") - - # Extract comprehensive business information after agent completes the search. - # Using structured schema ensures consistent data extraction even if page layout changes. - print("Extracting business information...") - - # Define schema using Pydantic - class BusinessInfo(BaseModel): - dba_name: str = Field(..., description="DBA Name") - ownership_name: str | None = Field(None, description="Ownership Name") - business_account_number: str = Field(..., description="Business Account Number") - location_id: str | None = Field(None, description="Location Id") - street_address: str | None = Field(None, description="Street Address") - business_start_date: str | None = Field(None, description="Business Start Date") - business_end_date: str | None = Field(None, description="Business End Date") - neighborhood: str | None = Field(None, description="Neighborhood") - naics_code: str = Field(..., description="NAICS Code") - naics_code_description: str | None = Field( - None, description="NAICS Code Description" - ) - - business_info = await page.extract( - "Extract all visible business information including DBA Name, Ownership Name, Business Account Number, Location Id, Street Address, Business Start Date, Business End Date, Neighborhood, NAICS Code, and NAICS Code Description", - schema=BusinessInfo, - ) - - print("Business information extracted:") - print(json.dumps(business_info.model_dump(), indent=2)) - - print("Session closed successfully") - - except Exception as error: - print(f"Error during business lookup: {error}") - raise +BUSINESS_NAME = "Jalebi Street" + + +class BusinessInfo(BaseModel): + model_config = ConfigDict(extra="forbid") + + dba_name: str = Field(description="DBA name") + ownership_name: str | None = Field(description="Ownership name") + business_account_number: str = Field(description="Business account number (ttxid)") + location_id: str | None = Field(description="Location ID (uniqueid)") + street_address: str | None = None + business_start_date: str | None = None + business_end_date: str | None = None + neighborhood: str | None = None + naics_code: str | None = None + naics_code_description: str | None = None + source_url: str = Field(description="Official SF Open Data URL opened in the browser") + + +async def main() -> None: + print(f"Searching for business: {BUSINESS_NAME}") + client = create_stagehand_client() + + async with client.session(SERVER_NAME) as session: + tools = await load_mcp_tools(session) + agent = create_deep_agent( + model=create_gateway_model("anthropic/claude-sonnet-4.6"), + tools=tools, + system_prompt=BROWSER_INSTRUCTIONS, + response_format=BusinessInfo, + ) + source_url = ( + "https://data.sfgov.org/resource/g8m3-pdis.json?" + f"$q={BUSINESS_NAME.replace(' ', '%20')}&$limit=5" + ) + result = await agent.ainvoke( + { + "messages": [ + { + "role": "user", + "content": ( + f"Open {source_url} in the browser and find the exact DBA record for " + f"{BUSINESS_NAME!r}. Read the rendered JSON, map ttxid to " + "business_account_number and uniqueid to location_id, and return every " + "requested field. Use null when an optional field is absent." + ), + } + ] + }, + config={"recursion_limit": 30}, + ) + business: BusinessInfo = result["structured_response"] + + print("Business information:") + print(business.model_dump_json(indent=2)) + print("Stagehand code-mode session closed successfully") if __name__ == "__main__": try: asyncio.run(main()) - except Exception as err: - print(f"Error in business lookup: {err}") - print("Common issues:") - print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") - print(" - Verify GOOGLE_API_KEY is set for the agent") - print("Docs: https://docs.stagehand.dev/v3/first-steps/introduction") - exit(1) + except Exception as error: + print(f"Error in business lookup: {error}") + print("Check BROWSERBASE_API_KEY and AI_GATEWAY_API_KEY in .env") + raise SystemExit(1) from error diff --git a/python/business-lookup/pyproject.toml b/python/business-lookup/pyproject.toml new file mode 100644 index 00000000..0a74ae8e --- /dev/null +++ b/python/business-lookup/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "business-lookup" +version = "0.1.0" +description = "Research an official business registry with Deep Agents and Stagehand V4" +readme = "README.md" +requires-python = ">=3.11,<3.14" +dependencies = [ + "deepagents==0.7.5", + "langchain-mcp-adapters==0.3.2", + "langchain-openai==1.4.3", + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", +] + +[tool.uv] +package = false diff --git a/python/cartesia-form-filling/.env.example b/python/cartesia-form-filling/.env.example new file mode 100644 index 00000000..016dc6f8 --- /dev/null +++ b/python/cartesia-form-filling/.env.example @@ -0,0 +1,8 @@ +# Gemini API Key for language model (default) +GEMINI_API_KEY=your_gemini_api_key_here + +# Browserbase API key +BROWSERBASE_API_KEY=your_browserbase_api_key_here + +# Optional: Model configuration +# MODEL_NAME=google/gemini-2.0-flash-exp diff --git a/python/cartesia-form-filling/.gitignore b/python/cartesia-form-filling/.gitignore new file mode 100644 index 00000000..87cfe969 --- /dev/null +++ b/python/cartesia-form-filling/.gitignore @@ -0,0 +1,41 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*.pyd +.Python + +# Virtual environments +.env +.venv/ +venv/ +env/ + +virtualenv/ + +# Conda environments +conda-env/ +envs/ +.conda/ +conda-meta/ + +# uv environments (in addition to uv.lock at top) +uv.lock +.python-version + +# Python package managers +poetry.lock +Pipfile.lock +pip-log.txt + +# pyenv +.pyenv/ + +# Distribution / packaging +*.egg-info/ +dist/ +build/ + +# Editor / OS files +.DS_Store + +.cartesia/ diff --git a/python/cartesia-form-filling/README.md b/python/cartesia-form-filling/README.md new file mode 100644 index 00000000..dd14f269 --- /dev/null +++ b/python/cartesia-form-filling/README.md @@ -0,0 +1,154 @@ +# Voice Agent with Real-time Web Form Filling + +Stagehand is the SDK for browser agents. + +This project demonstrates an advanced voice agent that conducts phone questionnaires while automatically filling out web forms in real-time using Stagehand browser automation. + +Here's what the system architecture looks like: + +![Workflow](workflow_diagram.png) + +## Features + +- **Voice Conversations**: Natural voice interactions using Cartesia Line +- **Real-time Form Filling**: Automatically fills web forms as answers are collected +- **Browser Automation**: Uses Stagehand AI to interact with any web form +- **Intelligent Mapping**: AI-powered mapping of voice answers to form fields +- **Async Processing**: Non-blocking form filling maintains conversation flow - form fields are filled in background tasks without delaying voice responses +- **Auto-submission**: Submits forms automatically when complete + +## Architecture + +``` +Voice Call (Cartesia) โ†’ Form Filling Node โ†’ Records Answer + โ†“ + Stagehand Browser API + โ†“ + Fills Web Form Field + โ†“ + Continues Conversation + โ†“ + Submits Form on Completion +``` + +## Getting Started + +First things first, here is what you will need: + +- A [Cartesia](https://play.cartesia.ai/agents) account and API key +- A [Gemini API Key](https://aistudio.google.com/apikey) +- A [Browserbase API Key](https://www.browserbase.com/overview) + +Make sure to add the API keys in your `.env` file or to the API keys section in your Cartesia account. + +- Required packages: + ```bash + cartesia-line + stagehand>=0.5.4 + google-genai>=1.26.0 + python-dotenv>=1.0.0 + PyYAML>=6.0.0 + loguru>=0.7.0 + aiohttp>=3.12.0 + pydantic>=2.0.0 + ``` + +## Setup + +1. Install dependencies: + +```bash +pip install . +``` + +2. Set up environment variables - create a `.env` file: + +```bash +GEMINI_API_KEY=your_gemini_api_key_here +BROWSERBASE_API_KEY=your_browserbase_api_key_here +``` + +3. Run the agent: + +```bash +python main.py +``` + +## Project Structure + +### `main.py` + +Entry point for the voice agent. Handles call initialization with `VoiceAgentApp` class and orchestrates the conversation flow with form filling integration. + +### `form_filling_node.py` + +ReasoningNode subclass customized for voice-optimized form filling. Integrates Stagehand browser automation and manages async form filling during conversation without blocking the voice flow. Provides status updates and error handling. + +### `stagehand_form_filler.py` + +Browser automation manager that handles all web interactions. Opens and controls web forms, maps conversation data to form fields using AI, transforms voice answers to form-compatible formats, and handles form submission. Supports different field types (text, select, checkbox, etc.). + +### `config.py` + +System configuration file including system prompts, model IDs, and temperature + +### `config.toml` + +Your Cartesia Line agent id. + +## Configuration + +The system can be configured through multiple files: + +- **`config.py`**: System prompts, model IDs (Gemini model selection), hyperparameters, and boolean flags for features +- **`config.toml`** / **YAML files**: Questionnaire structure and questions flow +- **`cartesia.toml`**: Deployment configuration for Cartesia platform (installs dependencies and runs the script) +- **Variables**: + - `FORM_URL`: Target web form to fill + +## Example Flow + +1. User calls the voice agent +2. Agent asks: "What type of voice agent are you building?" +3. User responds: "A customer service agent" +4. System: + - Records the answer + - Opens browser to form (if not already open) + - Fills "Customer Service" in the role selection field + - Takes screenshot for debugging +5. Agent asks next question +6. Process continues until all questions answered +7. Form is automatically submitted + +## Advanced Features + +- **Background Processing**: Form filling happens asynchronously using background tasks - conversation remains smooth and responsive +- **Error Recovery**: Continues conversation even if form filling fails +- **Progress Tracking**: Monitor form completion status +- **Screenshot Debugging**: Captures screenshots after each field +- **Flexible Mapping**: AI interprets answers for different field types + +## Deploying the Agent + +The `cartesia.toml` file defines how your agent will be installed and run when deployed on the Cartesia platform. This file tells the platform to install dependencies from `requirements.txt` and execute `main.py`. + +You can clone this repository and add it to your [agents dashboard](https://play.cartesia.ai/agents) along with your API Keys (set them in the Cartesia Platform's API keys section). + +For detailed deployment instructions, see [how to deploy an agent from the Cartesia Docs](https://docs.cartesia.ai/line/start-building/talk-to-your-first-agent). + +## Testing + +Test with different scenarios: + +- Complete questionnaire flow +- Interruptions and corrections +- Various answer formats +- Multi-page forms +- Form validation errors + +## Production Considerations + +- Configure proper error logging +- Add retry logic for form submission +- Implement form validation checks +- Consider rate limiting for API calls diff --git a/python/cartesia-form-filling/cartesia.toml b/python/cartesia-form-filling/cartesia.toml new file mode 100644 index 00000000..db7fc8f3 --- /dev/null +++ b/python/cartesia-form-filling/cartesia.toml @@ -0,0 +1,8 @@ +[app] +name = "form-filling" + +[build] +cmd = "pip install ." + +[run] +cmd = "python main.py" diff --git a/python/cartesia-form-filling/config.py b/python/cartesia-form-filling/config.py new file mode 100644 index 00000000..ef61d9e9 --- /dev/null +++ b/python/cartesia-form-filling/config.py @@ -0,0 +1,35 @@ +"""Configuration settings for the voice agent. + +This module contains system prompts, model configurations, and +hyperparameters for the Cartesia voice agent with form filling. +""" + +import os + +DEFAULT_MODEL_ID = os.getenv("MODEL_ID", "gemini-flash-latest") + +DEFAULT_TEMPERATURE = 0.7 +SYSTEM_PROMPT = """ +### You and your role +You are a friendly assistant conducting a questionnaire. +Be professional but conversational. Confirm answers when appropriate. +If a user's answer is unclear, ask for clarification. +For sensitive information, be especially tactful and professional. + +IMPORTANT: When you receive a clear answer from the user, use the +record_answer tool to record their response. + +### Your tone +When having a conversation, you should: +- Always polite and respectful, even when users are challenging +- Concise and brief but never curt. Keep your responses to 1-2 + sentences and less than 35 words +- When asking a question, be sure to ask in a short and concise manner +- Only ask one question at a time + +If the user is rude, or curses, respond with exceptional politeness +and genuine curiosity. You should always be polite. + +Remember, you're on the phone, so do not use emojis or abbreviations. +Spell out units and dates. +""" diff --git a/python/cartesia-form-filling/config.toml b/python/cartesia-form-filling/config.toml new file mode 100644 index 00000000..2355be8a --- /dev/null +++ b/python/cartesia-form-filling/config.toml @@ -0,0 +1 @@ +agent-id = 'your-agent-id' diff --git a/python/cartesia-form-filling/form_filling_node.py b/python/cartesia-form-filling/form_filling_node.py new file mode 100644 index 00000000..d8864513 --- /dev/null +++ b/python/cartesia-form-filling/form_filling_node.py @@ -0,0 +1,503 @@ +"""Voice agent that fills web forms in real-time using Stagehand. + +This module implements a ReasoningNode subclass that conducts voice +conversations while automatically filling web forms in the background. +The agent uses Stagehand for browser automation and handles async form +filling during conversation without blocking the voice flow. +""" + +import asyncio +from collections.abc import AsyncGenerator +from dataclasses import dataclass + +from config import DEFAULT_MODEL_ID, DEFAULT_TEMPERATURE +from google.genai import types as gemini_types +from line.events import AgentResponse, EndCall, ToolResult +from line.nodes.conversation_context import ConversationContext +from line.nodes.reasoning import ReasoningNode +from line.tools.system_tools import EndCallArgs, end_call +from line.utils.gemini_utils import convert_messages_to_gemini +from loguru import logger +from pydantic import BaseModel, Field +from stagehand_form_filler import StagehandFormFiller + + +class RecordFormFieldArgs(BaseModel): + """Arguments for recording a form field""" + + field_name: str = Field(description="The form field being filled") + value: str = Field(description="The value to enter in the field") + + +class RecordFormFieldTool: + """Tool for recording form field values""" + + @staticmethod + def name() -> str: + return "record_form_field" + + @staticmethod + def description() -> str: + return "Record a value for a form field that needs to be filled" + + @staticmethod + def parameters() -> dict: + return RecordFormFieldArgs.model_json_schema() + + @staticmethod + def to_gemini_tool(): + """Convert to Gemini tool format. + + Returns: + A Gemini Tool object with function declarations. + """ + return gemini_types.Tool( + function_declarations=[ + gemini_types.FunctionDeclaration( + name=RecordFormFieldTool.name(), + description=RecordFormFieldTool.description(), + parameters=RecordFormFieldTool.parameters(), + ) + ] + ) + + +@dataclass +class FormQuestion: + """Represents a question to ask the user""" + + field_name: str + question: str + field_type: str = "text" + required: bool = True + + +class FormFillingNode(ReasoningNode): + """Voice agent that fills web forms while conducting conversations. + + This class uses Stagehand to read and fill web forms dynamically, + maintains conversation flow while automating browser actions, and + intelligently extracts form structure and asks relevant questions. + """ + + def __init__( + self, + system_prompt: str, + gemini_client, + form_url: str, + model_id: str = DEFAULT_MODEL_ID, + temperature: float = DEFAULT_TEMPERATURE, + max_context_length: int = 15, + max_output_tokens: int = 1000, + ): + """Initialize the Form Filling node with Stagehand integration. + + Args: + system_prompt: System prompt for the LLM. + gemini_client: Google Gemini client instance. + form_url: URL of the web form to fill. + model_id: Gemini model ID. + temperature: Temperature for generation. + max_context_length: Maximum conversation context length. + max_output_tokens: Maximum tokens for generation. + """ + super().__init__(system_prompt=system_prompt, max_context_length=max_context_length) + + self.client = gemini_client + self.model_id = model_id + self.temperature = temperature + + # Browser automation + self.form_url = form_url + self.stagehand_filler: StagehandFormFiller | None = None + + # Form state + self.collected_data: dict[str, str] = {} + # Pre-initialize questions so conversation can start immediately + self.questions: list[FormQuestion] = self._create_questions() + self.current_question_index = 0 + + # Browser initialization + self.browser_init_task = None + self.browser_initializing = False + + # Enhanced prompt for form filling + enhanced_prompt = ( + system_prompt + + """ + + You are conducting a voice conversation to help fill out a web + form. As you collect information, it's being entered into an + actual online form in real-time. Ask natural questions to gather + the required information. Use the record_form_field tool to save + each piece of information. Keep the conversation friendly and + natural. + """ + ) + + # Generation config + self.generation_config = gemini_types.GenerateContentConfig( + system_instruction=enhanced_prompt, + temperature=self.temperature, + tools=[RecordFormFieldTool.to_gemini_tool()], + max_output_tokens=max_output_tokens, + thinking_config=gemini_types.ThinkingConfig(thinking_budget=0), + ) + + logger.info(f"FormFillingNode initialized for form: {form_url}") + + # Track if form was submitted + self.form_submitted = False + + async def cleanup_and_submit(self) -> None: + """Ensure form is submitted and cleanup when call ends. + + Returns: + None. + """ + # Submit form if we have any data and haven't submitted yet + if not self.form_submitted and self.collected_data and self.stagehand_filler: + logger.info("Call ending - auto-submitting form with collected data") + try: + await self._submit_form() + except Exception as e: + logger.error(f"Error during cleanup submission: {e}") + + # Clean up browser + if self.stagehand_filler: + await self.stagehand_filler.cleanup() + + async def _initialize_browser(self) -> None: + """Initialize browser and extract form fields. + + Returns: + None. + """ + # Prevent multiple initializations + if self.browser_initializing or self.stagehand_filler: + logger.info("Browser already initializing or initialized, skipping") + return + + self.browser_initializing = True + try: + logger.info("Initializing browser and analyzing form") + self.stagehand_filler = StagehandFormFiller(form_url=self.form_url) + await self.stagehand_filler.initialize() + + logger.info("Browser ready, form can now be filled") + + except Exception as e: + logger.error(f"Failed to initialize browser: {e}") + self.browser_initializing = False + raise + finally: + self.browser_initializing = False + + def _create_questions(self) -> list[FormQuestion]: + """Create questions for the form. + + Returns: + A list of FormQuestion objects to ask the user. + """ + # Define questions for the form fields we know about + # This matches form at https://forms.fillout.com/t/34ccsqafUFus + form_questions = [ + FormQuestion( + field_name="full_name", + question="What is your full name?", + field_type="text", + required=True, + ), + FormQuestion( + field_name="email", + question="What is your email address?", + field_type="email", + required=True, + ), + FormQuestion( + field_name="phone", + question="What is your phone number?", + field_type="phone", + required=False, + ), + FormQuestion( + field_name="work_eligibility", + question="Are you legally eligible to work in this country?", + field_type="radio", + required=True, + ), + FormQuestion( + field_name="availability_type", + question=("What's your availability - temporary, part-time, or full-time?"), + field_type="radio", + required=True, + ), + FormQuestion( + field_name="role_selection", + question=( + "Which role are you applying for? We have openings " + "for Sales Manager, IT Support, Recruiting, " + "Software Engineer, or Marketing Specialist." + ), + field_type="checkbox", + required=True, + ), + FormQuestion( + field_name="previous_experience", + question="Have you worked in a similar role before?", + field_type="radio", + required=True, + ), + FormQuestion( + field_name="skills_experience", + question=( + "What relevant skills and experience do you have " + "that make you a strong candidate for this position?" + ), + field_type="textarea", + required=True, + ), + FormQuestion( + field_name="additional_info", + question=("Is there anything else you'd like to tell us about yourself?"), + field_type="textarea", + required=False, + ), + ] + + return form_questions + + async def _fill_form_field_async(self, field_name: str, value: str) -> None: + """Fill a form field asynchronously in background (non-blocking). + + Args: + field_name: The name of the form field to fill. + value: The value to enter in the field. + """ + try: + # Wait for browser initialization if needed + if self.browser_init_task: + logger.info(f"Waiting for browser to initialize before filling {field_name}") + await self.browser_init_task + + logger.info(f"Filling field '{field_name}' with: {value} in background") + # Use StagehandFormFiller's fill_field method which + # handles the mapping + success = await self.stagehand_filler.fill_field(field_name, value) + + if success: + logger.info(f"Successfully filled field: {field_name} in browser") + else: + logger.warning(f"Failed to fill field: {field_name}") + + except Exception as e: + logger.error(f"Error filling field {field_name}: {e}") + raise # Re-raise so background task can catch it + + async def _submit_form(self) -> bool: + """Submit the completed form. + + Returns: + True if submission succeeded, False otherwise. + """ + # Wait for browser initialization if needed + if self.browser_init_task and not self.stagehand_filler: + logger.info("Waiting for browser to initialize before submitting form") + await self.browser_init_task + + if not self.stagehand_filler: + return False + + try: + logger.info("Submitting web form with collected data") + logger.info(f"Data collected: {self.collected_data}") + + # Ensure StagehandFormFiller has all collected data + # (it should already have it from fill_field calls, + # but ensure consistency) + self.stagehand_filler.collected_data.update(self.collected_data) + + # Use StagehandFormFiller's submit_form method which now + # uses collected_data + success = await self.stagehand_filler.submit_form() + + if success: + logger.info("Form submitted successfully!") + return True + else: + logger.warning("Form submission may have failed") + return False + + except Exception as e: + logger.error(f"Error submitting form: {e}") + return False + + def get_current_question(self) -> FormQuestion | None: + """Get the current question to ask. + + Returns: + The current FormQuestion or None if all questions answered. + """ + if self.current_question_index < len(self.questions): + return self.questions[self.current_question_index] + return None + + async def process_context( + self, context: ConversationContext + ) -> AsyncGenerator[AgentResponse | EndCall, None]: + """Process conversation context with real-time form filling. + + Args: + context: The conversation context with events. + + Yields: + AgentResponse: Text responses to the user. + EndCall: Call termination when form is complete. + """ + # Initialize browser on first call (non-blocking) + if not self.browser_init_task and not self.stagehand_filler: + self.browser_init_task = asyncio.create_task(self._initialize_browser()) + logger.info("Browser initialization started in background") + + # Get current question after initialization + current_question = self.get_current_question() + question_name = current_question.field_name if current_question else "None" + logger.info(f"Current question: {question_name}") + logger.info(f"Question index: {self.current_question_index}/{len(self.questions)}") + logger.info(f"Events count: {len(context.events)}") + + # Check latest event to determine what to do + latest_event = context.events[-1] if context.events else None + is_agent_response = isinstance(latest_event, AgentResponse) if latest_event else False + + # Handle initial greeting - speak first when conversation starts + if not context.events: + logger.info("Starting conversation - Agent speaks first") + initial_greeting = ( + "Hello! I'm here to help you fill out an application " + "form today. I'll ask you a series of questions and " + "fill in the form as we go. Ready to get started?" + ) + yield AgentResponse(content=initial_greeting) + return + + # If last event was our greeting, and user responded, ask + # first question + if len(context.events) == 2 and not is_agent_response and self.current_question_index == 0: + user_message = context.get_latest_user_transcript_message() + if user_message and current_question: + logger.info(f"User ready to start: '{user_message}'") + logger.info(f"Asking first question: {current_question.field_name}") + yield AgentResponse(content=f"Great! Let's begin. {current_question.question}") + return + + # Check if all questions have been answered + # Only submit if we've actually collected data + if ( + not current_question + and self.current_question_index > 0 + and len(self.collected_data) > 0 + ): + # All questions answered - submit the form + logger.info(f"All {self.current_question_index} questions answered") + logger.info(f"Collected data for {len(self.collected_data)} fields") + + submission_success = await self._submit_form() + self.form_submitted = True + + if submission_success: + goodbye = "Perfect! I've submitted your application. Thank you!" + else: + goodbye = "Thank you for providing all the information. Your responses have been recorded." + + # Clean up + await self.cleanup_and_submit() + + # End call + args = EndCallArgs(goodbye_message=goodbye) + async for item in end_call(args): + yield item + return + + # Guard against no questions or empty state + if not current_question and self.current_question_index == 0: + logger.warning("No questions available or not properly initialized") + return + + # Process user response + messages = convert_messages_to_gemini(context.events, text_events_only=True) + + # Add context about current question + question_context = f""" + + Current form field: {current_question.field_name} + Question: {current_question.question} + + Listen to the user's response and use the record_form_field + tool to save it. Then acknowledge their answer naturally. + """ + + enhanced_config = gemini_types.GenerateContentConfig( + system_instruction=(self.generation_config.system_instruction + question_context), + temperature=self.temperature, + tools=[RecordFormFieldTool.to_gemini_tool()], + max_output_tokens=self.generation_config.max_output_tokens, + thinking_config=gemini_types.ThinkingConfig(thinking_budget=0), + ) + + # Get user's latest message + user_message = context.get_latest_user_transcript_message() + if user_message: + logger.info(f'User response: "{user_message}"') + + # Stream Gemini response + full_response = "" + stream = await self.client.aio.models.generate_content_stream( + model=self.model_id, + contents=messages, + config=enhanced_config, + ) + + async for msg in stream: + if msg.text: + full_response += msg.text + yield AgentResponse(content=msg.text) + + if msg.function_calls: + for function_call in msg.function_calls: + if function_call.name == RecordFormFieldTool.name(): + field_name = function_call.args.get( + "field_name", current_question.field_name + ) + value = function_call.args.get("value", "") + + logger.info(f"Recording: {field_name} = {value}") + + # Store data first + self.collected_data[field_name] = value + # Fill the form field asynchronously in background + # (non-blocking) + asyncio.create_task(self._fill_form_field_async(field_name, value)) + # Log the collected data + logger.info(f"Collected: {field_name}={value}") + # Move to next question immediately + # (don't wait for form filling) + self.current_question_index += 1 + + # Clear context + self.clear_context() + + # Get next question + next_question = self.get_current_question() + if next_question: + yield AgentResponse(content=f"Great! {next_question.question}") + + # Yield tool result immediately + yield ToolResult( + tool_name="record_form_field", + tool_args={"field_name": field_name, "value": value}, + result=f"Recorded: {field_name}={value}", + ) + + if full_response: + logger.info(f'Agent response: "{full_response}"') diff --git a/python/cartesia-form-filling/main.py b/python/cartesia-form-filling/main.py new file mode 100644 index 00000000..553d1445 --- /dev/null +++ b/python/cartesia-form-filling/main.py @@ -0,0 +1,74 @@ +"""Cartesia Line Voice Agent with real-time web form filling. + +This module implements a voice agent that conducts phone questionnaires +while automatically filling out web forms in real-time using Stagehand +browser automation. +""" + +import os + +from config import SYSTEM_PROMPT +from form_filling_node import FormFillingNode +from google import genai +from line import Bridge, CallRequest, VoiceAgentApp, VoiceAgentSystem +from line.events import UserStartedSpeaking, UserStoppedSpeaking, UserTranscriptionReceived + +# Target form URL - the actual web form to fill +FORM_URL = "https://forms.fillout.com/t/rff6XZTSApus" + +# Initialize Gemini client +gemini_client = genai.Client(api_key=os.getenv("GEMINI_API_KEY")) + + +async def handle_new_call(system: VoiceAgentSystem, call_request: CallRequest) -> None: + """Handle incoming voice calls with real-time web form filling. + + This agent will: + 1. Conduct a voice conversation to gather information + 2. Open and fill an actual web form in the background + 3. Submit the form when the conversation is complete + + Args: + system: The voice agent system instance. + call_request: The incoming call request. + """ + + # Create form filling node with browser automation + form_node = FormFillingNode( + system_prompt=SYSTEM_PROMPT, gemini_client=gemini_client, form_url=FORM_URL + ) + + # Set up bridge for event handling + form_bridge = Bridge(form_node) + system.with_speaking_node(form_node, bridge=form_bridge) + + # Connect transcription events + form_bridge.on(UserTranscriptionReceived).map(form_node.add_event) + + # Handle interruptions and streaming + ( + form_bridge.on(UserStoppedSpeaking) + .interrupt_on(UserStartedSpeaking, handler=form_node.on_interrupt_generate) + .stream(form_node.generate) + .broadcast() + ) + + # Start the system + await system.start() + + # Wait for call to end + await system.wait_for_shutdown() + + # Ensure form is submitted when call ends + await form_node.cleanup_and_submit() + + +# Create the voice agent application +app = VoiceAgentApp(handle_new_call) + +if __name__ == "__main__": + print("Starting Voice Agent with Web Form Automation") + print(f"Will fill form at: {FORM_URL}") + print("Ready to receive calls...") + print("Form filling happens invisibly while processing voice calls.\n") + app.run() diff --git a/python/cartesia-form-filling/pyproject.toml b/python/cartesia-form-filling/pyproject.toml new file mode 100644 index 00000000..782aa781 --- /dev/null +++ b/python/cartesia-form-filling/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "browserbase-form-filler" +version = "0.1.0" +description = "Voice agent that fills web forms using Stagehand browser automation" +requires-python = ">=3.11,<3.14" +dependencies = [ + "cartesia-line==0.1.12", + "fastapi==0.115.14", + "stagehand==4.0.0", + "google-genai>=1.26.0", + "python-dotenv>=1.0.0", + "loguru>=0.7.0", + "aiohttp>=3.12.0", + "pydantic>=2.0.0", + "PyYAML>=6.0.0", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["."] + +[dependency-groups] +dev = [] diff --git a/python/cartesia-form-filling/stagehand_form_filler.py b/python/cartesia-form-filling/stagehand_form_filler.py new file mode 100644 index 00000000..a19d8a3f --- /dev/null +++ b/python/cartesia-form-filling/stagehand_form_filler.py @@ -0,0 +1,320 @@ +"""Browser automation for filling web forms during voice conversations. + +This module provides the StagehandFormFiller class which manages browser +automation for filling forms using Stagehand. It handles form field +mapping, field filling, and form submission. +""" + +import asyncio +import os +from dataclasses import dataclass +from enum import Enum + +from loguru import logger + +from stagehand import Page, Stagehand, StagehandBrowser, browserbase + + +class FieldType(Enum): + TEXT = "text" + EMAIL = "email" + PHONE = "phone" + SELECT = "select" + RADIO = "radio" + CHECKBOX = "checkbox" + TEXTAREA = "textarea" + + +@dataclass +class FormField: + """Represents a form field with its metadata""" + + field_id: str + field_type: FieldType + label: str + required: bool = False + options: list[str] | None = None + + +class FormFieldMapping: + """Maps conversation questions to actual form fields""" + + def __init__(self): + self.field_mappings = { + "full_name": FormField( + field_id="full_name", + field_type=FieldType.TEXT, + label="What is your full name?", + required=True, + ), + "email": FormField( + field_id="email", + field_type=FieldType.EMAIL, + label="What is your email address?", + required=True, + ), + "phone": FormField( + field_id="phone", + field_type=FieldType.PHONE, + label="What is your phone number?", + required=False, + ), + "work_eligibility": FormField( + field_id="work_eligibility", + field_type=FieldType.RADIO, + label="Are you legally eligible to work in this country?", + options=["Yes", "No"], + required=True, + ), + "availability_type": FormField( + field_id="availability", + field_type=FieldType.RADIO, + label="What's your availability?", + options=["Temporary", "Part-time", "Full-time"], + required=True, + ), + "additional_info": FormField( + field_id="additional_info", + field_type=FieldType.TEXTAREA, + label="Anything else you'd like to let us know about you?", + required=False, + ), + "role_selection": FormField( + field_id="role_selection", + field_type=FieldType.RADIO, + label="Which of these roles are you applying for?", + options=[ + "Sales manager", + "IT Support", + "Recruiting", + "Software engineer", + "Marketing specialist", + ], + required=True, + ), + "previous_experience": FormField( + field_id="previous_experience", + field_type=FieldType.RADIO, + label=("Have you worked in a role similar to this one in the past?"), + options=["Yes", "No"], + required=True, + ), + "skills_experience": FormField( + field_id="skills_experience", + field_type=FieldType.TEXTAREA, + label=( + "What relevant skills and experience do you have " + "that make you a strong candidate for this position?" + ), + required=True, + ), + } + + def get_form_field(self, question_id: str) -> FormField | None: + """Get the form field mapping for a question ID. + + Args: + question_id: The question identifier. + + Returns: + The FormField object or None if not found. + """ + return self.field_mappings.get(question_id) + + +class StagehandFormFiller: + """Manages browser automation for filling forms using Stagehand""" + + def __init__(self, form_url: str): + self.form_url = form_url + self.browser: StagehandBrowser | None = None + self.stagehand: Stagehand | None = None + self.page: Page | None = None + self.is_initialized = False + self.field_mapper = FormFieldMapping() + self.collected_data: dict[str, str] = {} + + async def initialize(self) -> None: + """Initialize Stagehand and open the form. + + Returns: + None. + """ + if self.is_initialized: + return + + try: + logger.info("Initializing Stagehand browser automation") + + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") + self.browser = await browserbase.launch(api_key=api_key) + self.stagehand = await Stagehand.create( + browser=self.browser, + ) + pages = await self.browser.context.pages() + self.page = pages[0] if pages else await self.browser.context.new_page() + + # Navigate to form + logger.info(f"Opening form: {self.form_url}") + await self.page.goto( + self.form_url, + wait_until="domcontentloaded", + timeout=60_000, + ) + + # Wait for form to load + await asyncio.sleep(2) + + self.is_initialized = True + logger.info("Browser automation initialized successfully") + + except Exception as e: + logger.error(f"Failed to initialize Stagehand: {e}") + raise + + async def fill_field(self, question_id: str, answer: str) -> bool: + """Fill a specific form field based on the question ID and answer. + + Args: + question_id: The question identifier. + answer: The answer value to fill. + + Returns: + True if field was filled successfully, False otherwise. + """ + if not self.is_initialized: + # Initialize asynchronously without blocking + init_task = asyncio.create_task(self.initialize()) + await init_task + + try: + if self.stagehand is None or self.page is None: + raise RuntimeError("Stagehand form filler is not initialized") + + # Get field mapping + field = self.field_mapper.get_form_field(question_id) + if not field: + logger.warning(f"No field mapping found for question: {question_id}") + return False + + # Store and use the answer directly + answer = answer.strip() + self.collected_data[question_id] = answer + + logger.info(f"Async filling field '{field.label}' with: {answer}") + + # Use Stagehand's natural language API to fill the field + act_variables = {"answer": answer} + if field.field_type == FieldType.RADIO: + act_variables = None + instruction = ( + f"Within the question '{field.label}', select the one option that best " + f"matches the user's spoken answer {answer!r}. Available options: " + f"{', '.join(field.options or [])}." + ) + if field.field_type in [FieldType.TEXT, FieldType.EMAIL, FieldType.PHONE]: + instruction = f"Fill the '{field.label}' field with %answer%" + + elif field.field_type == FieldType.TEXTAREA: + instruction = f"Fill the '{field.label}' text area with %answer%" + + elif field.field_type == FieldType.SELECT: + instruction = ( + f"Within the question '{field.label}', click the option labeled %answer%" + ) + + elif field.field_type == FieldType.CHECKBOX: + # For role selection, check the specific role checkbox + if question_id == "role_selection": + instruction = "Check the %answer% checkbox" + else: + # For other checkboxes, check/uncheck based on answer + if answer.lower() in ["yes", "true"]: + instruction = f"Check the '{field.label}' checkbox" + else: + instruction = f"Uncheck the '{field.label}' checkbox" + + result = None + for attempt in range(2): + try: + result = await self.stagehand.act( + instruction, + page=self.page, + variables=act_variables, + ) + if result.data.success: + break + except Exception: + result = None + if attempt == 0: + await self.page.wait_for_timeout(750) + + if field.field_type == FieldType.RADIO and (result is None or not result.data.success): + observed = await self.stagehand.observe( + f"Within the question {field.label!r}, find the one option that best " + f"matches the user's spoken answer {answer!r}. Available options: " + f"{', '.join(field.options or [])}.", + page=self.page, + ) + if observed.data: + result = await self.stagehand.act(observed.data[0], page=self.page) + + if result is None: + raise RuntimeError(f"Could not fill {field.label}") + if not result.data.success: + raise RuntimeError(result.data.message or f"Could not fill {field.label}") + + return True + + except Exception as e: + logger.error(f"Error filling field {question_id}: {e}") + return False + + async def submit_form(self) -> bool: + """Submit the completed form. + + Returns: + True if form was submitted successfully, False otherwise. + """ + try: + if self.stagehand is None or self.page is None: + raise RuntimeError("Stagehand form filler is not initialized") + logger.info("Submitting the form") + logger.info(f"Form has {len(self.collected_data)} fields filled") + + result = await self.stagehand.act( + "Click the Apply for a role at AB Technologies submit button", + page=self.page, + ) + if not result.data.success: + raise RuntimeError(result.data.message or "Form submission button was not found") + + # Wait for submission to process + await asyncio.sleep(1) + + logger.info("Form submitted successfully!") + return True + + except Exception as e: + logger.error(f"Error submitting form: {e}") + return False + + async def cleanup(self) -> None: + """Clean up browser resources. + + Returns: + None. + """ + if self.stagehand: + try: + await self.stagehand.close() + except Exception as error: + logger.error(f"Error closing Stagehand: {error}") + if self.browser: + try: + await self.browser.close() + except Exception as error: + logger.error(f"Error closing browser: {error}") + logger.info("Session ended") diff --git a/python/cartesia-form-filling/workflow_diagram.png b/python/cartesia-form-filling/workflow_diagram.png new file mode 100644 index 00000000..59d60e8e Binary files /dev/null and b/python/cartesia-form-filling/workflow_diagram.png differ diff --git a/python/cerebras-docs-checker/.env.example b/python/cerebras-docs-checker/.env.example new file mode 100644 index 00000000..f55457b3 --- /dev/null +++ b/python/cerebras-docs-checker/.env.example @@ -0,0 +1,8 @@ +# Browserbase credentials (required) +BROWSERBASE_API_KEY=your-browserbase-api-key + +# Cerebras API key (required) +CEREBRAS_API_KEY=your-cerebras-api-key + +# Model selection (optional, defaults to llama-3.3-70b) +# CEREBRAS_MODEL=llama-3.3-70b diff --git a/python/cerebras-docs-checker/README.md b/python/cerebras-docs-checker/README.md new file mode 100644 index 00000000..3dda16b2 --- /dev/null +++ b/python/cerebras-docs-checker/README.md @@ -0,0 +1,61 @@ +# Stagehand + Browserbase: Cerebras Documentation Checker + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- Goal: Crawl any documentation site, discover its source repo, and verify docs accuracy against the actual codebase using Cerebras LLMs. +- Parallel Browserbase workers crawl docs pages and capture Playwright accessibility snapshots. +- A Deep Agents verification agent uses Cerebras for planning and Stagehand V4 code-mode browser tools for fallback research, while local code tools cross-reference functions, parameters, and examples. +- Falls back to content-only analysis when no source repository is found. +- Docs โ†’ https://docs.stagehand.dev + +## GLOSSARY + +- snapshot: capture the current page's structured accessibility representation. +- code mode: Stagehand's `snapshot`, `run`, and `screenshot` tools, exposed to a bring-your-own Deep Agents loop over MCP. +- Deep Agents: the external agent framework; Stagehand V4 does not expose `stagehand.agent()`. + +## QUICKSTART + +1. uv sync +2. cp .env.example .env # Add your CEREBRAS_API_KEY and BROWSERBASE_API_KEY +3. uv run python main.py https://your-docs-site.com + +## EXPECTED OUTPUT + +- Spins up parallel crawl workers with live Browserbase session links +- BFS-crawls the docs site, extracting aria trees and checking for broken links/anchors +- Discovers the GitHub source repository from crawled page content +- Clones the repo and runs a Cerebras verification agent on each page +- Prints a summary table with issue counts by severity and type +- Saves a detailed Markdown report to `docs_report_YYYYMMDD_HHMM.md` + +## COMMON PITFALLS + +- "Missing required API keys": verify .env contains CEREBRAS_API_KEY and BROWSERBASE_API_KEY +- Playwright not installed: run `playwright install chromium` after `uv sync` +- Cerebras 422 errors: the model may rate-limit under heavy load โ€” reduce MAX_PAGES or MAX_CRAWL_WORKERS in main.py +- Clone failures: ensure the target docs site links to a public GitHub repo +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +- Documentation audits: Automatically verify that API docs match the actual source code before a release. +- Broken link detection: Crawl a docs site and surface all broken external links and internal anchors. +- CI/CD integration: Run as a scheduled check to catch documentation drift as the codebase evolves. + +## NEXT STEPS + +- Add JSON export: Extend the output to include a machine-readable JSON issues file for downstream tooling. +- Configurable models: Set `CEREBRAS_MODEL` to any model available to your Cerebras account; the template defaults to `gpt-oss-120b`. +- Incremental checks: Cache previously verified pages and only re-check pages whose content has changed. + +## HELPFUL RESOURCES + +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/cerebras-docs-checker/agent_runtime.py b/python/cerebras-docs-checker/agent_runtime.py new file mode 100644 index 00000000..4777aa6b --- /dev/null +++ b/python/cerebras-docs-checker/agent_runtime.py @@ -0,0 +1,57 @@ +"""Cerebras Deep Agents + Stagehand V4 code-mode setup.""" + +from __future__ import annotations + +import os + +from langchain_mcp_adapters.client import MultiServerMCPClient +from langchain_openai import ChatOpenAI +from pydantic import SecretStr + +SERVER_NAME = "stagehand_browser" +STAGEHAND_DEEPAGENTS_SOURCE = ( + "git+https://github.com/browserbase/stagehand.git@main" + "#subdirectory=packages/integrations/deepagents" +) +BROWSER_INSTRUCTIONS = """You control one persistent Browserbase browser through the Stagehand +V4 code-mode tools snapshot, run, and screenshot. Inspect before acting, prefer deterministic +page APIs, and return only evidence from pages you actually opened. +""" + + +def require_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required") + return value + + +def create_cerebras_model(model: str) -> ChatOpenAI: + return ChatOpenAI( + model=model, + api_key=SecretStr(require_env("CEREBRAS_API_KEY")), + base_url="https://api.cerebras.ai/v1", + ) + + +def create_stagehand_client() -> MultiServerMCPClient: + return MultiServerMCPClient( + { + SERVER_NAME: { + "transport": "stdio", + "command": os.environ.get("UVX_COMMAND", "uvx"), + "args": [ + "--from", + STAGEHAND_DEEPAGENTS_SOURCE, + "--with", + "stagehand==4.0.0", + "stagehand-deepagents-mcp", + ], + "env": { + "BROWSERBASE_API_KEY": require_env("BROWSERBASE_API_KEY"), + "STAGEHAND_BROWSER": "browserbase", + "STAGEHAND_RUN_TIMEOUT_MS": "120000", + }, + } + } + ) diff --git a/python/cerebras-docs-checker/main.py b/python/cerebras-docs-checker/main.py new file mode 100644 index 00000000..a6052014 --- /dev/null +++ b/python/cerebras-docs-checker/main.py @@ -0,0 +1,972 @@ +# Stagehand + Browserbase: Cerebras Documentation Checker - See README.md for full documentation + +import asyncio +import json +import os +import re +import subprocess +import sys +import tempfile +from collections import Counter +from datetime import datetime +from pathlib import Path +from typing import Optional +from urllib.parse import urlparse + +import httpx +from browserbase import Browserbase +from deepagents import create_deep_agent +from dotenv import load_dotenv +from langchain_mcp_adapters.tools import load_mcp_tools +from openai import AsyncOpenAI, OpenAI +from playwright.async_api import async_playwright +from pydantic import BaseModel, Field + +from agent_runtime import ( + BROWSER_INSTRUCTIONS, + SERVER_NAME, + create_cerebras_model, + create_stagehand_client, +) + +load_dotenv(override=True) + +# API keys (loaded from .env file) +CEREBRAS_API_KEY = os.getenv("CEREBRAS_API_KEY") +BROWSERBASE_API_KEY = os.getenv("BROWSERBASE_API_KEY") + +if not all([CEREBRAS_API_KEY, BROWSERBASE_API_KEY]): + raise ValueError("Missing required API keys. Check .env file.") + +# Cerebras model to use for verification and analysis (override via CEREBRAS_MODEL env var) +CEREBRAS_MODEL = os.getenv("CEREBRAS_MODEL", "gpt-oss-120b") + +# Crawl configuration (modify these to control scope and speed) +DEFAULT_URL = "https://docs.stagehand.dev" # Target docs site (can also be passed as CLI arg) +MAX_PAGES = int(os.getenv("MAX_PAGES", "20")) +MAX_DEPTH = int(os.getenv("MAX_DEPTH", "2")) +MAX_CRAWL_WORKERS = int(os.getenv("MAX_CRAWL_WORKERS", "5")) + + +# โ”€โ”€ Data Models โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +class Page(BaseModel): + url: str + title: str = "" + content: str = "" + last_updated: Optional[datetime] = None + depth: int = 0 + links: list[dict] = Field(default_factory=list) + broken_links: list[dict] = Field(default_factory=list) + broken_anchors: list[str] = Field(default_factory=list) + + +class Issue(BaseModel): + url: str + type: str # broken_link, outdated, unclear, grammar, missing_context, broken_anchor, error + severity: str # critical, high, medium, low + description: str + suggestion: str + context: str = "" + + +# โ”€โ”€ Phase 1: Crawl โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +async def _crawl_worker( + worker_id: int, + queue: asyncio.Queue, + visited: dict, + visited_lock: asyncio.Lock, + base_domain: str, + max_pages: int, + max_depth: int, + counter: list, +): + """Crawl docs in a Browserbase session and capture their accessibility trees.""" + + playwright = None + browser = None + + try: + api = Browserbase(api_key=BROWSERBASE_API_KEY) + session = await asyncio.to_thread(api.sessions.create) + playwright = await async_playwright().start() + browser = await playwright.chromium.connect_over_cdp(session.connect_url) + context = browser.contexts[0] + page = context.pages[0] if context.pages else await context.new_page() + + # HTTP client for checking broken links (HEAD requests) + http = httpx.AsyncClient() + + while True: + async with visited_lock: + if len(visited) >= max_pages: + break + + try: + url, depth = queue.get_nowait() + except asyncio.QueueEmpty: + await asyncio.sleep(0.2) + try: + url, depth = queue.get_nowait() + except asyncio.QueueEmpty: + break + + url = url.split("#")[0] + + async with visited_lock: + if url in visited or len(visited) >= max_pages: + queue.task_done() + continue + visited[url] = None + + if depth > max_depth: + async with visited_lock: + del visited[url] + queue.task_done() + continue + + async with visited_lock: + counter[0] += 1 + page_num = counter[0] + short_url = url.split("/")[-1] or url.split("/")[-2] or "index" + start_time = asyncio.get_event_loop().time() + + try: + await page.goto(url, wait_until="domcontentloaded", timeout=60_000) + title = await page.title() + + aria_tree = await page.locator("body").aria_snapshot() + + # Collect all links on the page for BFS crawling and broken link detection + links = await page.evaluate( + """() => Array.from(document.querySelectorAll('a[href]')) + .map((link) => ({ + href: link.href, + text: (link.textContent || '').trim().slice(0, 30), + }))""" + ) + links = links if isinstance(links, list) else [] + + # Check first 15 links for broken URLs using HEAD requests + broken = [] + for link in links[:15]: + href = link["href"] + if href.startswith("http"): + try: + r = await http.head(href, follow_redirects=True, timeout=8) + if r.status_code >= 400: + broken.append( + {"url": href, "text": link["text"], "status": r.status_code} + ) + except Exception: + broken.append({"url": href, "text": link["text"], "status": 0}) + + # Find broken anchor links (e.g. #section-name that doesn't exist on the page) + broken_anchors = await page.evaluate("""() => { + return Array.from(document.querySelectorAll('a[href^="#"]')) + .map(a => a.href.split('#')[1]) + .filter(id => { + if (!id || id.length <= 1 || id === 'top') return false; + return !document.getElementById(id); + }); + }""") + + elapsed = asyncio.get_event_loop().time() - start_time + content_kb = len(aria_tree) / 1024 + print( + f" [{page_num}/{max_pages}] {short_url}" + f" โ€” aria: {content_kb:.1f}KB, {len(broken)} broken links" + f" ({elapsed:.1f}s)" + ) + + page_obj = Page( + url=url, + title=title, + content=aria_tree, + depth=depth, + links=links, + broken_links=broken, + broken_anchors=broken_anchors, + ) + async with visited_lock: + visited[url] = page_obj + + # Add newly discovered same-domain links to the BFS queue + for link in links: + href = link["href"].split("#")[0] + parsed = urlparse(href) + async with visited_lock: + if parsed.netloc == base_domain and href not in visited: + queue.put_nowait((href, depth + 1)) + + except Exception as e: + elapsed = asyncio.get_event_loop().time() - start_time + print(f" [{page_num}/{max_pages}] {short_url} โ€” ERROR: {e!s:.80} ({elapsed:.1f}s)") + page_obj = Page(url=url, title="[Error]", content=str(e), depth=depth) + async with visited_lock: + visited[url] = page_obj + + queue.task_done() + + await http.aclose() + + finally: + if browser: + await browser.close() + if playwright: + await playwright.stop() + + +async def crawl( + root_url: str, max_pages: int = 30, max_depth: int = 2, max_workers: int = 5 +) -> list[Page]: + """Parallel BFS crawl using an async work queue with Browserbase sessions. + Each worker gets its own browser session and pulls URLs from a shared queue.""" + parsed_root = urlparse(root_url) + base_domain = parsed_root.netloc + + # Shared state protected by asyncio.Lock (needed because multiple workers access concurrently) + visited: dict[str, Page] = {} + visited_lock = asyncio.Lock() + counter = [0] # Mutable list used as a counter so workers can increment it + + queue = asyncio.Queue() + queue.put_nowait((root_url, 0)) + + print(f"Starting {max_workers} crawl workers...") + + workers = [ + asyncio.create_task( + _crawl_worker( + i, + queue, + visited, + visited_lock, + base_domain, + max_pages, + max_depth, + counter, + ) + ) + for i in range(max_workers) + ] + + await asyncio.gather(*workers) + + result = {url: pg for url, pg in visited.items() if pg is not None} + print(f"Crawled {len(result)} pages\n") + return list(result.values()) + + +# โ”€โ”€ Phase 2: Discover GitHub Repository โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def discover_repo_from_pages(pages: list[Page]) -> Optional[str]: + """Search crawled page content and links for a GitHub repository URL. + This is the fast path โ€” no browser session needed, just regex over existing data.""" + github_pattern = re.compile(r"https?://github\.com/[\w\-]+/[\w\-]+") + + # Check both aria tree text and extracted links for GitHub URLs + for pg in pages: + matches = github_pattern.findall(pg.content) + for m in matches: + parts = m.rstrip("/").split("/") + if len(parts) >= 5: + print(f" Found repo in aria tree: {m}") + return m + + for link in pg.links: + href = link.get("href", "") + matches = github_pattern.findall(href) + for m in matches: + parts = m.rstrip("/").split("/") + if len(parts) >= 5: + print(f" Found repo in page links: {m}") + return m + + print(" No GitHub link found in aria trees or links") + return None + + +class RepositoryResult(BaseModel): + repository_url: str | None + + +async def discover_repo_with_agent(root_url: str) -> Optional[str]: + """Use Cerebras + Deep Agents with Stagehand V4 code mode as a fallback.""" + print(" Using the code-mode agent to find the GitHub repository...") + client = create_stagehand_client() + try: + async with client.session(SERVER_NAME) as session: + tools = await load_mcp_tools(session) + agent = create_deep_agent( + model=create_cerebras_model(CEREBRAS_MODEL), + tools=tools, + system_prompt=BROWSER_INSTRUCTIONS, + response_format=RepositoryResult, + ) + result = await agent.ainvoke( + { + "messages": [ + { + "role": "user", + "content": ( + f"Open {root_url} and find the official GitHub source repository " + "for these docs. Inspect navigation and footer links; click a " + "source or GitHub control if needed. Return null only after " + "checking the rendered site." + ), + } + ] + }, + config={"recursion_limit": 40}, + ) + repository: RepositoryResult = result["structured_response"] + except Exception as error: + print(f" Agent error: {error}") + return None + + url = repository.repository_url + if url and re.fullmatch(r"https?://github\.com/[\w\-]+/[\w\-]+/?", url): + print(f" Agent found repo: {url}") + return url.rstrip("/") + return None + + +# โ”€โ”€ Phase 3: Clone Repository โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def clone_repo(repo_url: str) -> Optional[Path]: + """Clone a GitHub repo to a temp directory. Returns the path.""" + repo_url = repo_url.rstrip("/") + clone_url = repo_url + ".git" if not repo_url.endswith(".git") else repo_url + repo_name = repo_url.split("/")[-1] + clone_dir = Path(tempfile.mkdtemp(prefix="docs-verify-")) / repo_name + + print(f" Cloning {repo_url} -> {clone_dir}") + try: + subprocess.run( + ["git", "clone", clone_url, str(clone_dir)], + capture_output=True, + text=True, + timeout=60, + check=True, + ) + print(" Cloned successfully") + return clone_dir + except subprocess.CalledProcessError as e: + print(f" Clone failed: {e.stderr.strip()}") + return None + except Exception as e: + print(f" Clone error: {e}") + return None + + +# โ”€โ”€ Phase 4: Verification Agent (Cerebras tool calling) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +# Tools the Cerebras verification agent can call to inspect the cloned codebase. +# Each tool maps to a function in _execute_verification_tool(). +VERIFICATION_TOOLS = [ + { + "type": "function", + "function": { + "name": "grep_codebase", + "description": "Search for a pattern (regex) across all source files in the codebase.", + "parameters": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "A function name, class name, variable, or regex pattern", + } + }, + "required": ["pattern"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "read_file", + "description": "Read the full contents of a source file from the codebase.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Relative path to the file, e.g. 'src/auth.py'", + } + }, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "list_files", + "description": "List all files in the codebase.", + "parameters": {"type": "object", "properties": {}}, + }, + }, +] + +VERIFICATION_SYSTEM_PROMPT = """You are a documentation accuracy verifier. Your job is to +check whether a documentation page is accurate against the actual source code of the project. + +You have access to tools that let you search and read the codebase. Use them to verify every claim \ +in the documentation: +- Function signatures (names, parameters, defaults, return types) +- Class constructors and their parameters +- Method names and their existence +- Code examples (do they use the correct API?) +- Endpoint paths and constants +- Data model fields + +For each issue you find, report it with: +- type: one of "outdated", "wrong_signature", "removed_function", "missing_parameter", \ +"renamed_method", "wrong_endpoint", "missing_function" +- severity: "critical" for removed/renamed APIs, "high" for wrong signatures/params, \ +"medium" for other inaccuracies +- description: what is wrong +- suggestion: what the correct version should be (based on source code) +- context: the specific text in the docs that is wrong + +Be thorough. Check EVERY function, parameter, code example, and data model mentioned in the docs. +Also check if the codebase has new functions/methods NOT mentioned in the docs. + +Use one tool at a time. Return your findings as a JSON object: {"issues": [...]}""" + + +def _execute_verification_tool(name: str, args: dict, codebase_path: Path) -> str: + """Execute a verification tool call against the cloned codebase.""" + if name == "grep_codebase": + pattern = args.get("pattern", "") + try: + result = subprocess.run( + ["grep", "-rn", pattern, str(codebase_path)], + capture_output=True, + text=True, + timeout=10, + ) + output = result.stdout.strip() + if not output: + return f"No matches found for pattern: {pattern}" + output = output.replace(str(codebase_path) + "/", "") + return output[:4000] + except Exception as e: + return f"Error: {e}" + + elif name == "read_file": + path = args.get("path", "") + full_path = codebase_path / path + if not full_path.exists(): + return f"File not found: {path}" + try: + return full_path.read_text()[:8000] + except Exception as e: + return f"Error reading file: {e}" + + elif name == "list_files": + files = [] + for p in sorted(codebase_path.rglob("*")): + if p.is_file() and not any( + part.startswith(".") for part in p.relative_to(codebase_path).parts + ): + files.append(str(p.relative_to(codebase_path))) + return "\n".join(files) + + return f"Unknown tool: {name}" + + +def _get_codebase_listing(codebase_path: Path) -> str: + """Get a file listing for the codebase.""" + files = [] + for p in sorted(codebase_path.rglob("*")): + if p.is_file() and not any( + part.startswith(".") for part in p.relative_to(codebase_path).parts + ): + files.append(str(p.relative_to(codebase_path))) + return "\n".join(files) + + +def verify_page( + pg: Page, + llm: OpenAI, + codebase_path: Path, + file_listing: str, + page_num: int, + total: int, + max_turns: int = 10, +) -> list[Issue]: + """Run the Cerebras verification agent on a single page's documentation content. + The agent uses tool calling to grep and read the source code, then reports inaccuracies.""" + short_url = pg.url.split("/")[-1] or pg.url.split("/")[-2] or "index" + + if pg.title == "[Error]": + print(f" [{page_num}/{total}] Skip: {short_url} (crawl error)") + return [ + Issue( + url=pg.url, + type="error", + severity="high", + description="Page failed to load during crawl", + suggestion="Check URL accessibility", + context=pg.content[:100], + ) + ] + + # Build the conversation with system prompt + page content for the agent + messages = [ + {"role": "system", "content": VERIFICATION_SYSTEM_PROMPT}, + { + "role": "user", + "content": ( + f"Here are the files in the codebase:\n{file_listing}\n\n" + f"Now verify this documentation page against the codebase. " + f"The docs may be outdated โ€” check every function, parameter, and code example. " + f"Use one tool at a time.\n\n" + f"Page URL: {pg.url}\nPage title: {pg.title}\n\n" + f"Page content (aria tree):\n{pg.content[:12000]}" + ), + }, + ] + + # Multi-turn tool calling loop: the agent greps/reads code, then returns findings as JSON + print(f" [{page_num}/{total}] Verifying {short_url}...", end="", flush=True) + turns_used = 0 + + for turn in range(max_turns): + try: + # Ask the model to verify the docs, allowing it to call tools + resp = llm.chat.completions.create( + model=CEREBRAS_MODEL, + messages=messages, + tools=VERIFICATION_TOOLS, + tool_choice="auto", + ) + except Exception as e: + # Some models return tool_use_failed when tool calling is unavailable. + # Retry without tools so the page still receives content-level analysis. + if "tool_use_failed" in str(e): + resp = llm.chat.completions.create(model=CEREBRAS_MODEL, messages=messages) + else: + raise + + msg = resp.choices[0].message + messages.append(msg) + turns_used += 1 + + # If no tool calls, the agent is done โ€” parse its JSON response + if not msg.tool_calls: + content = msg.content or "" + issues = _parse_verification_response(content, pg.url) + break + else: + # Execute each tool call and feed results back to the agent + for tc in msg.tool_calls: + args = json.loads(tc.function.arguments) + print(f" {tc.function.name}", end="", flush=True) + result = _execute_verification_tool(tc.function.name, args, codebase_path) + messages.append({"role": "tool", "tool_call_id": tc.id, "content": result}) + else: + # Force the agent to return its findings if it used all turns without finishing + messages.append( + { + "role": "user", + "content": "Provide your final JSON response now with all issues found.", + } + ) + resp = llm.chat.completions.create(model=CEREBRAS_MODEL, messages=messages) + issues = _parse_verification_response(resp.choices[0].message.content or "", pg.url) + + # Append broken links and anchors detected during crawl (Phase 1) + for link in pg.broken_links: + issues.append( + Issue( + url=pg.url, + type="broken_link", + severity="high", + description=( + f"Broken link: {link.get('text', '?')} -> status {link.get('status', '?')}" + ), + suggestion="Fix or remove the link", + context=link.get("url", ""), + ) + ) + for anchor in pg.broken_anchors: + issues.append( + Issue( + url=pg.url, + type="broken_anchor", + severity="medium", + description=f"Broken anchor: #{anchor}", + suggestion="Add missing ID or fix link", + context=anchor, + ) + ) + + print(f" {len(issues)} issues ({turns_used} turns)") + return issues + + +def _parse_verification_response(content: str, url: str) -> list[Issue]: + """Parse the verification agent's JSON response into Issue objects.""" + json_match = re.search(r'\{[\s\S]*"issues"[\s\S]*\}', content) + if not json_match: + return [] + + try: + data = json.loads(json_match.group()) + issues = [] + for item in data.get("issues", []): + issues.append( + Issue( + url=url, + type=item.get("type", "outdated"), + severity=item.get("severity", "medium"), + description=item.get("description", ""), + suggestion=item.get("suggestion", ""), + context=item.get("context", ""), + ) + ) + return issues + except json.JSONDecodeError: + return [] + + +def verify_all(pages: list[Page], codebase_path: Path) -> list[Issue]: + """Verify all pages against the cloned codebase using Cerebras tool calling. + Pages are processed sequentially because each verification runs a multi-turn conversation.""" + all_issues = [] + file_listing = _get_codebase_listing(codebase_path) + + print(f"Verifying {len(pages)} pages against {codebase_path}") + print(f"Codebase files:\n {file_listing.replace(chr(10), chr(10) + ' ')}\n") + + # Cerebras uses an OpenAI-compatible API, so we use the OpenAI client with a custom base URL + llm = OpenAI( + base_url="https://api.cerebras.ai/v1", + api_key=CEREBRAS_API_KEY, + timeout=30.0, + ) + + for i, pg in enumerate(pages): + issues = verify_page(pg, llm, codebase_path, file_listing, i + 1, len(pages)) + all_issues.extend(issues) + + print(f"\nDone: {len(all_issues)} issues across {len(pages)} pages") + return all_issues + + +# โ”€โ”€ Fallback Analysis (when no source repo is found) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# When we can't find or clone the GitHub repo, we fall back to content-only analysis. +# This checks for outdated info, unclear writing, and missing context without source code. + +ANALYSIS_PROMPT = """Current date and time: {current_datetime} + +You are a documentation quality checker. Analyze the following accessibility tree +from a documentation page and identify content issues. + +Look for: outdated information (check copyright years, old version numbers), +unclear writing, grammar errors, missing context. + +When checking dates and years, remember that today is {current_datetime}. +A copyright showing the current year is CORRECT, not outdated. + +Return your findings as a JSON object with an "issues" array. Each issue has: +- type: one of "outdated", "unclear", "grammar", "missing_context" +- severity: one of "critical", "high", "medium", "low" +- description: what the issue is +- suggestion: how to fix it +- context: the relevant text snippet + +If no issues are found, return {{"issues": []}}. + +Page URL: {url} +Page title: {title} + +Accessibility tree: +{content}""" + + +async def analyze_page( + pg: Page, + llm: AsyncOpenAI, + page_num: int, + total: int, + current_datetime: str, +) -> list[Issue]: + """Analyze a single page by sending its aria tree to Cerebras.""" + short_url = pg.url.split("/")[-1] or pg.url.split("/")[-2] or "index" + start = asyncio.get_event_loop().time() + + if pg.title == "[Error]": + print(f" [{page_num}/{total}] Skip: {short_url} (crawl error)") + return [ + Issue( + url=pg.url, + type="error", + severity="high", + description="Page failed to load during crawl", + suggestion="Check URL accessibility", + context=pg.content[:100], + ) + ] + + try: + prompt = ANALYSIS_PROMPT.format( + current_datetime=current_datetime, + url=pg.url, + title=pg.title, + content=pg.content[:12000], + ) + + resp = await llm.chat.completions.create( + model=CEREBRAS_MODEL, + messages=[{"role": "user", "content": prompt}], + response_format={"type": "json_object"}, + ) + + raw = resp.choices[0].message.content + data = json.loads(raw) + issues = [] + for item in data.get("issues", []): + issues.append( + Issue( + url=pg.url, + type=item.get("type", "unclear"), + severity=item.get("severity", "medium"), + description=item.get("description", ""), + suggestion=item.get("suggestion", ""), + context=item.get("context", ""), + ) + ) + except Exception as e: + elapsed = asyncio.get_event_loop().time() - start + print(f" [{page_num}/{total}] Error: {short_url} โ€” {e!s:.80} ({elapsed:.1f}s)") + issues = [] + + for link in pg.broken_links: + issues.append( + Issue( + url=pg.url, + type="broken_link", + severity="high", + description=( + f"Broken link: {link.get('text', '?')} -> status {link.get('status', '?')}" + ), + suggestion="Fix or remove the link", + context=link.get("url", ""), + ) + ) + for anchor in pg.broken_anchors: + issues.append( + Issue( + url=pg.url, + type="broken_anchor", + severity="medium", + description=f"Broken anchor: #{anchor}", + suggestion="Add missing ID or fix link", + context=anchor, + ) + ) + + elapsed = asyncio.get_event_loop().time() - start + print(f" [{page_num}/{total}] Done: {short_url} โ€” {len(issues)} issues ({elapsed:.1f}s)") + return issues + + +async def analyze_all_parallel(pages: list[Page], max_workers: int = 10) -> list[Issue]: + """Analyze all pages via direct Cerebras API calls (no browser sessions needed). + Pages are processed in batches for concurrency without overwhelming the API.""" + all_issues = [] + current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + print(f"Current date/time sent to LLM: {current_datetime}") + print(f"Analyzing {len(pages)} pages (max_workers={max_workers})\n") + + llm = AsyncOpenAI( + base_url="https://api.cerebras.ai/v1", + api_key=CEREBRAS_API_KEY, + ) + + for i in range(0, len(pages), max_workers): + batch = pages[i : i + max_workers] + batch_end = min(i + max_workers, len(pages)) + print(f"Batch [{i + 1}-{batch_end}/{len(pages)}]") + + tasks = [ + analyze_page(pg, llm, i + j + 1, len(pages), current_datetime) + for j, pg in enumerate(batch) + ] + batch_results = await asyncio.gather(*tasks, return_exceptions=True) + + for j, result in enumerate(batch_results): + if isinstance(result, list): + all_issues.extend(result) + elif isinstance(result, Exception): + page_url = batch[j].url.split("/")[-1] or "index" + print(f" Error: {page_url} โ€” {result}") + + print() + + print(f"Done: {len(all_issues)} issues across {len(pages)} pages") + return all_issues + + +# โ”€โ”€ Display and Export โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def print_summary(pages: list[Page], issues: list[Issue], root_url: str): + """Print summary statistics to the terminal.""" + severity_counts = Counter(i.severity for i in issues) + type_counts = Counter(i.type for i in issues) + + print(f"\n{'=' * 60}") + print(" Documentation Analysis Summary") + print(f"{'=' * 60}") + print(f" Site: {root_url}") + print(f" Pages crawled: {len(pages)}") + print(f" Total issues: {len(issues)}") + print(f" Critical: {severity_counts.get('critical', 0)}") + print(f" High: {severity_counts.get('high', 0)}") + print(f" Medium: {severity_counts.get('medium', 0)}") + print(f" Low: {severity_counts.get('low', 0)}") + print(f" By type: {', '.join(f'{t}: {c}' for t, c in type_counts.most_common())}") + print(f"{'=' * 60}\n") + + +def print_issues(issues: list[Issue], severity_filter: str = None): + """Print issues to the terminal.""" + filtered = [i for i in issues if severity_filter is None or i.severity == severity_filter] + if not filtered: + print("No issues found.") + return + + for issue in filtered: + page_name = issue.url.split("/")[-1] or "index" + print(f" [{issue.severity.upper()}] [{issue.type}] {page_name}") + print(f" {issue.description}") + print(f" Suggestion: {issue.suggestion}") + if issue.context: + print(f" Context: {issue.context[:80]}") + print() + + +def export_markdown( + pages: list[Page], issues: list[Issue], root_url: str, filename: str = None +) -> str: + """Generate and save a markdown report.""" + if filename is None: + filename = f"docs_report_{datetime.now().strftime('%Y%m%d_%H%M')}.md" + + severity_counts = Counter(i.severity for i in issues) + + report = f"""# Documentation Analysis Report + +**Site:** {root_url} +**Date:** {datetime.now().strftime("%Y-%m-%d %H:%M")} +**Pages:** {len(pages)} | **Issues:** {len(issues)} + +## Summary + +| Severity | Count | +|----------|-------| +| Critical | {severity_counts.get("critical", 0)} | +| High | {severity_counts.get("high", 0)} | +| Medium | {severity_counts.get("medium", 0)} | +| Low | {severity_counts.get("low", 0)} | + +## Issues + +""" + for sev in ["critical", "high", "medium", "low"]: + sev_issues = [i for i in issues if i.severity == sev] + if sev_issues: + report += f"### {sev.title()} ({len(sev_issues)})\n\n" + for i in sev_issues: + page_name = i.url.split("/")[-1] or "index" + report += f"- **[{page_name}]** {i.type}: {i.description}\n" + report += f" - *{i.suggestion}*\n" + report += "\n" + + with open(filename, "w") as f: + f.write(report) + + print(f"Saved report to: {filename}") + return report + + +# โ”€โ”€ Main โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +async def main(): + """ + 4-phase documentation checker: + 1. Crawl docs site via parallel Stagehand BYOB workers + 2. Discover the source GitHub repository from crawled pages + 3. Clone the repo locally + 4. Verify docs accuracy against source code using Cerebras tool-calling agent + """ + # Accept target URL as CLI argument or use default + docs_url = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_URL + + print(f"Target: {docs_url}") + print(f"Model: {CEREBRAS_MODEL}") + print(f"Max pages: {MAX_PAGES}, Max depth: {MAX_DEPTH}\n") + + # Phase 1: Crawl the docs site using parallel browser sessions + print("Phase 1: Crawling documentation site...") + pages = await crawl( + docs_url, max_pages=MAX_PAGES, max_depth=MAX_DEPTH, max_workers=MAX_CRAWL_WORKERS + ) + if not pages: + raise RuntimeError("The documentation crawl returned no pages") + + # Phase 2: Try to find the source GitHub repo (regex first, then agent fallback) + print("Phase 2: Discovering source repository...") + repo_url = discover_repo_from_pages(pages) + + if not repo_url: + # Regex didn't find a repo โ€” try using a Stagehand agent to click around and find it + repo_url = await discover_repo_with_agent(docs_url) + + if repo_url: + # Phase 3: Clone the repo so the verification agent can inspect source code + print("\nPhase 3: Cloning repository...") + codebase_path = clone_repo(repo_url) + + if codebase_path: + # Phase 4: Run the Cerebras tool-calling agent to verify docs against source + print("\nPhase 4: Verification agent...") + issues = verify_all(pages, codebase_path) + else: + print("\nClone failed, falling back to basic analysis...") + issues = await analyze_all_parallel(pages, max_workers=10) + else: + # No repo found โ€” fall back to content-only analysis (no source code verification) + print("\nNo source repo found, falling back to basic analysis...") + issues = await analyze_all_parallel(pages, max_workers=10) + + # Display results and save markdown report + print_summary(pages, issues, docs_url) + print_issues(issues) + report = export_markdown(pages, issues, docs_url) + if "# Documentation Analysis Report" not in report: + raise RuntimeError("The documentation audit report was not generated") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Application error: {err}") + print("\nCommon issues:") + print(" - Check .env file has CEREBRAS_API_KEY and BROWSERBASE_API_KEY") + print(" - Ensure the target docs and repository are publicly reachable") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + exit(1) diff --git a/python/cerebras-docs-checker/pyproject.toml b/python/cerebras-docs-checker/pyproject.toml new file mode 100644 index 00000000..5ce73d66 --- /dev/null +++ b/python/cerebras-docs-checker/pyproject.toml @@ -0,0 +1,38 @@ +[project] +name = "cerebras-docs-checker" +version = "0.1.0" +description = "AI-powered documentation checker using Cerebras LLMs, Stagehand, and Browserbase" +readme = "README.md" +requires-python = ">=3.11,<3.14" +dependencies = [ + "httpx", + "openai", + "browserbase>=1.7.0", + "playwright", + "pydantic", + "python-dotenv", + "deepagents==0.7.5", + "langchain-mcp-adapters==0.3.2", + "langchain-openai==1.4.3", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[tool.uv] +package = false + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/python/company-address-finder/.env.example b/python/company-address-finder/.env.example new file mode 100644 index 00000000..56823daf --- /dev/null +++ b/python/company-address-finder/.env.example @@ -0,0 +1,3 @@ +BROWSERBASE_API_KEY= +AI_GATEWAY_API_KEY= +# DEEPAGENTS_MODEL=anthropic/claude-sonnet-4.6 diff --git a/python/company-address-finder/README.md b/python/company-address-finder/README.md index 4018fb20..2b6ab494 100644 --- a/python/company-address-finder/README.md +++ b/python/company-address-finder/README.md @@ -1,74 +1,52 @@ -# Stagehand + Browserbase: Company Address Finder +# Company address finder with Python agents -## AT A GLANCE +Stagehand is the SDK for browser agents. -- Goal: Automate discovery of company legal information and physical addresses from Terms of Service and Privacy Policy pages. -- CUA Agent: Uses autonomous computer-use agent to search for company homepages via Google and navigate to legal documents. -- Data Extraction: Extracts structured data including homepage URLs, ToS/Privacy Policy links, and physical mailing addresses. -- Fallback Strategy: Intelligently falls back from Terms of Service to Privacy Policy if address is not found. -- Retry Logic: Built-in exponential backoff for reliability against network failures. -- Scalable: Supports both sequential and concurrent processing (concurrent requires Startup/Developer plan or higher). +This template gives one LangChain Deep Agent at a time a persistent Stagehand V4 browser. Each +agent finds a company's official homepage and legal pages, then returns a validated physical +mailing address when the company publishes one. -## GLOSSARY +## How it works -- agent: autonomous AI agent with computer-use capabilities that can navigate websites like a human - Docs โ†’ https://docs.stagehand.dev/basics/agent -- extract: pull structured data from web pages using natural language instructions and Pydantic schemas - Docs โ†’ https://docs.stagehand.dev/basics/extract -- CUA (Computer Use Agent): agent mode that enables full browser interaction (search, click, scroll, type) - Docs โ†’ https://docs.stagehand.dev/basics/agent#what-is-cua-mode -- concurrent sessions: run multiple browser sessions simultaneously for faster batch processing - Docs โ†’ https://docs.browserbase.com/guides/concurrency-rate-limits -- exponential backoff: retry strategy that increases wait time between attempts for reliability +- `create_deep_agent` owns planning, model calls, and Pydantic structured output. +- Stagehand code mode exposes `run`, `snapshot`, and `screenshot` over a stateful MCP session. +- Vercel AI Gateway supplies the bring-your-own agent model. +- Each company gets a separate Browserbase session; `MAX_CONCURRENT` controls the batch size. +- The Stagehand server runs in an isolated `uvx` environment to keep its dependencies separate + from the Deep Agents client. -## QUICKSTART +## Quickstart -1. uv venv venv -2. source venv/bin/activate # On Windows: venv\Scripts\activate -3. uvx install stagehand python-dotenv pydantic -4. cp .env.example .env # Add your Browserbase API key, Project ID, and Google Generative AI API key to .env -5. Edit COMPANY_NAMES array in main.py to specify which companies to process -6. python main.py +Requirements: Python 3.11โ€“3.13 and [uv](https://docs.astral.sh/uv/). -## EXPECTED OUTPUT +```bash +cp .env.example .env +# Add BROWSERBASE_API_KEY and AI_GATEWAY_API_KEY to .env. +uv sync +uv run python main.py +``` -- Initializes browser session for each company with live view link -- Agent navigates to Google and searches for company homepage -- Extracts Terms of Service and Privacy Policy links from homepage -- Navigates to Terms of Service and extracts physical address -- Falls back to Privacy Policy if address not found in ToS -- Outputs comprehensive JSON with all extracted data for each company -- Displays processing status and session closure for each company +Edit `COMPANY_NAMES` in `main.py` to change the batch. Keep `MAX_CONCURRENT = 1` unless your +Browserbase plan supports enough simultaneous sessions. -## COMMON PITFALLS +The first run installs the Stagehand Deep Agents integration from `stagehand/main` in `uvx`; the +integration pins its Stagehand server dependency to `stagehand==4.0.0`. -- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_GENERATIVE_AI_API_KEY (or GOOGLE_API_KEY) -- Google API access: ensure you have access to gemini-2.5-computer-use-preview-10-2025 model -- Concurrent processing: MAX_CONCURRENT > 1 requires Browserbase Startup or Developer plan or higher (default is 1 for sequential) -- Company not found: agent may fail if company name is ambiguous or doesn't have a clear web presence -- Address extraction: some companies may not list physical addresses in their legal documents -- Session timeouts: long-running batches may hit 900s timeout (adjust browserbase_session_create_params if needed) +## Expected outcome -## USE CASES +The script processes Browserbase, Mintlify, Wordware, and Reducto. For each company it returns the +official homepage, Terms and Privacy links when found, and a physical address when published. A +missing address may be `null`; a per-company browser failure is included in that company's output. -โ€ข Legal compliance research: Collect company addresses and legal document URLs for due diligence, vendor verification, or compliance audits. -โ€ข Business intelligence: Build datasets of company locations and legal information for market research or competitive analysis. -โ€ข Contact data enrichment: Augment CRM or database records with verified physical addresses extracted from official company documents. -โ€ข Multi-company batch processing: Process lists of companies (investors, partners, clients) to gather standardized location data at scale. +## Configuration -## NEXT STEPS +- `BROWSERBASE_API_KEY`: launches each Browserbase session. +- `AI_GATEWAY_API_KEY`: authenticates the Deep Agents model through Vercel AI Gateway. +- `DEEPAGENTS_MODEL`: optional model override; defaults to `anthropic/claude-sonnet-4.6`. +- `STAGEHAND_RUN_TIMEOUT_MS`: optional browser-tool timeout; defaults to 120 seconds. -โ€ข Parameterize inputs: Accept company names from CSV files, command-line arguments, or API endpoints for dynamic batch processing. -โ€ข Expand extraction: Add support for additional fields like contact emails, phone numbers, business registration numbers, or founding dates. -โ€ข Multi-source validation: Cross-reference addresses from multiple pages (About, Contact, Footer) to improve accuracy and confidence. -โ€ข Export formats: Add CSV, Excel, or database export options with configurable field mappings for downstream integrations. -โ€ข Error handling: Implement more granular error categorization (not found vs. no address vs. extraction failure) for better reporting. +## Resources -## HELPFUL RESOURCES - -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction -๐ŸŽฎ Browserbase: https://www.browserbase.com -๐Ÿ’ก Try it out: https://www.browserbase.com/playground -๐Ÿ”ง Templates: https://www.browserbase.com/templates -๐Ÿ“ง Need help? support@browserbase.com -๐Ÿ’ฌ Discord: http://stagehand.dev/discord +- [Stagehand V4 documentation](https://docs.stagehand.dev/v4) +- [Stagehand Deep Agents integration](https://github.com/browserbase/stagehand/tree/main/packages/integrations/deepagents) +- [Browserbase concurrency](https://docs.browserbase.com/features/concurrency-rate-limits) diff --git a/python/company-address-finder/agent_runtime.py b/python/company-address-finder/agent_runtime.py new file mode 100644 index 00000000..ce540ab8 --- /dev/null +++ b/python/company-address-finder/agent_runtime.py @@ -0,0 +1,66 @@ +"""Shared Deep Agents + Stagehand V4 code-mode setup for this template.""" + +from __future__ import annotations + +import os + +from langchain_mcp_adapters.client import MultiServerMCPClient +from langchain_openai import ChatOpenAI +from pydantic import SecretStr + +SERVER_NAME = "stagehand_browser" + +STAGEHAND_DEEPAGENTS_SOURCE = ( + "git+https://github.com/browserbase/stagehand.git@main" + "#subdirectory=packages/integrations/deepagents" +) + +BROWSER_INSTRUCTIONS = """You control one persistent Browserbase browser through exactly three +Stagehand code-mode tools: +- snapshot: inspect the active page and hydrate bracketed element IDs. +- run: execute snapshot actions or JavaScript with the Playwright-shaped page API. +- screenshot: inspect the rendered page visually. + +Use run with deterministic page APIs for known URLs and structured pages. Use snapshot before +interacting with an unfamiliar UI. Snapshot IDs are valid only for the latest snapshot. Do not +launch another browser, and do not claim evidence from a URL you did not open. +""" + + +def require_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required") + return value + + +def create_gateway_model(default_model: str) -> ChatOpenAI: + return ChatOpenAI( + model=os.environ.get("DEEPAGENTS_MODEL", default_model), + api_key=SecretStr(require_env("AI_GATEWAY_API_KEY")), + base_url="https://ai-gateway.vercel.sh/v1", + ) + + +def create_stagehand_client() -> MultiServerMCPClient: + server_env = { + "BROWSERBASE_API_KEY": require_env("BROWSERBASE_API_KEY"), + "STAGEHAND_BROWSER": "browserbase", + "STAGEHAND_RUN_TIMEOUT_MS": os.environ.get("STAGEHAND_RUN_TIMEOUT_MS", "120000"), + } + return MultiServerMCPClient( + { + SERVER_NAME: { + "transport": "stdio", + "command": os.environ.get("UVX_COMMAND", "uvx"), + "args": [ + "--from", + STAGEHAND_DEEPAGENTS_SOURCE, + "--with", + "stagehand==4.0.0", + "stagehand-deepagents-mcp", + ], + "env": server_env, + } + } + ) diff --git a/python/company-address-finder/main.py b/python/company-address-finder/main.py index f2764923..6420b75e 100644 --- a/python/company-address-finder/main.py +++ b/python/company-address-finder/main.py @@ -1,302 +1,105 @@ -# Stagehand + Browserbase: Company Address Finder - See README.md for full documentation +"""Find company legal pages and addresses with Deep Agents and Stagehand V4 code mode.""" + +from __future__ import annotations import asyncio -import json -import os +from agent_runtime import ( + BROWSER_INSTRUCTIONS, + SERVER_NAME, + create_gateway_model, + create_stagehand_client, +) +from deepagents import create_deep_agent from dotenv import load_dotenv -from pydantic import BaseModel, Field, HttpUrl - -from stagehand import Stagehand, StagehandConfig +from langchain_mcp_adapters.tools import load_mcp_tools +from pydantic import BaseModel, ConfigDict -# Load environment variables load_dotenv() -# Companies to process (modify this list to add/remove companies) COMPANY_NAMES = ["Browserbase", "Mintlify", "Wordware", "Reducto"] -# Maximum number of companies to process concurrently. -# Default: 1 (sequential processing - works on all plans) -# Set to > 1 for concurrent processing (requires Startup or Developer plan or higher) +# Values above one require enough Browserbase concurrency for one browser per company. MAX_CONCURRENT = 1 class CompanyData(BaseModel): + model_config = ConfigDict(extra="forbid") + company_name: str homepage_url: str - terms_of_service_link: str - privacy_policy_link: str - address: str - - -class TermsOfServiceLink(BaseModel): - terms_of_service_link: HttpUrl = Field( - ..., description="The URL link to the Terms of Service page" - ) - - -class PrivacyPolicyLink(BaseModel): - privacy_policy_link: HttpUrl = Field(..., description="The URL link to the Privacy Policy page") - + terms_of_service_link: str | None + privacy_policy_link: str | None + address: str | None -class CompanyAddress(BaseModel): - company_address: str = Field(..., description="The physical company mailing address") - -# Retries an async function with exponential backoff -# Handles transient network/page load failures for reliability -async def with_retry(fn, description: str, max_retries: int = 3, delay_ms: int = 2000): - last_error = None - - for attempt in range(1, max_retries + 1): - try: - return await fn() - except Exception as error: - last_error = error - if attempt < max_retries: - print(f"{description} - Attempt {attempt} failed, retrying in {delay_ms}ms...") - await asyncio.sleep(delay_ms / 1000.0) - - raise Exception(f"{description} - Failed after {max_retries} attempts: {last_error}") - - -# Processes a single company: finds homepage, extracts ToS/Privacy links, and extracts physical address -# Uses CUA agent to navigate and Stagehand extract() for structured data extraction -# Falls back to Privacy Policy if address not found in Terms of Service async def process_company(company_name: str) -> CompanyData: - print(f"\nProcessing: {company_name}") - - stagehand = None + print(f"Processing {company_name}...") + client = create_stagehand_client() try: - # Initialize Stagehand with Browserbase - stagehand = Stagehand( - StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - verbose=0, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - "region": "us-east-1", - "timeout": 900, - "browser_settings": { - "viewport": { - "width": 1920, - "height": 1080, + async with client.session(SERVER_NAME) as session: + tools = await load_mcp_tools(session) + agent = create_deep_agent( + model=create_gateway_model("anthropic/claude-sonnet-4.6"), + tools=tools, + system_prompt=( + BROWSER_INSTRUCTIONS + + "\nUse no more than eight browser-tool calls. Verify every returned URL " + "belongs to the requested company's official site, then return the structured " + "response immediately." + ), + response_format=CompanyData, + ) + result = await agent.ainvoke( + { + "messages": [ + { + "role": "user", + "content": ( + f"Find the official homepage for {company_name!r}, then find its " + "Terms of Service and Privacy Policy pages. Open the relevant legal " + "pages and extract the physical mailing address from Terms, falling " + "back to Privacy. Return null for a link or address only after " + "checking the relevant official pages." + ), } - }, + ] }, + config={"recursion_limit": 30}, ) - ) - - print(f"[{company_name}] Initializing browser session...") - await stagehand.init() - - session_id = getattr(stagehand, "session_id", None) or getattr( - stagehand, "browserbase_session_id", None - ) - if session_id: - print(f"[{company_name}] Live View Link: https://browserbase.com/sessions/{session_id}") - - page = stagehand.page - - # Navigate to Google as starting point for CUA agent to search and find company homepage - print(f"[{company_name}] Navigating to Google...") - await with_retry( - lambda: page.goto("https://www.google.com/", wait_until="domcontentloaded"), - f"[{company_name}] Initial navigation to Google", - ) - - # Create CUA agent for autonomous navigation - # Agent can interact with the browser like a human: search, click, scroll, and navigate - print(f"[{company_name}] Creating Computer Use Agent...") - agent = stagehand.agent( - provider="google", - model="gemini-2.5-computer-use-preview-10-2025", - instructions=f"""You are a helpful assistant that can use a web browser. - You are currently on the following page: {page.url}. - Do not ask follow up questions, the user will trust your judgement.""", - options={ - "api_key": os.getenv("GEMINI_API_KEY"), - }, - ) - - print(f"[{company_name}] Finding company homepage using CUA agent...") - await with_retry( - lambda: agent.execute( - instruction=f"Navigate to the {company_name} website", - max_steps=5, - auto_screenshot=True, - ), - f"[{company_name}] Navigation to website", - ) - - homepage_url = page.url - print(f"[{company_name}] Homepage found: {homepage_url}") - - # Extract both legal document links in parallel for speed (independent operations) - print(f"[{company_name}] Finding Terms of Service & Privacy Policy links...") - - results = await asyncio.gather( - page.extract( - "extract the link to the Terms of Service page (may also be labeled as Terms of Use, Terms and Conditions, or similar equivalent names)", - schema=TermsOfServiceLink, - ), - page.extract( - "extract the link to the Privacy Policy page (may also be labeled as Privacy Notice, Privacy Statement, or similar equivalent names)", - schema=PrivacyPolicyLink, - ), - return_exceptions=True, - ) - - terms_of_service_link = "" - privacy_policy_link = "" - - if not isinstance(results[0], Exception) and results[0]: - terms_of_service_link = str(results[0].terms_of_service_link) - print(f"[{company_name}] Terms of Service: {terms_of_service_link}") - - if not isinstance(results[1], Exception) and results[1]: - privacy_policy_link = str(results[1].privacy_policy_link) - print(f"[{company_name}] Privacy Policy: {privacy_policy_link}") - - address = "" - - # Try Terms of Service first - most likely to contain physical address for legal/contact purposes - if terms_of_service_link: - print(f"[{company_name}] Extracting address from Terms of Service...") - await with_retry( - lambda: page.goto(terms_of_service_link), - f"[{company_name}] Navigate to Terms of Service", - ) - - try: - address_result = await page.extract( - "Extract the physical company mailing address (street, city, state, postal code, and country if present) from the Terms of Service page. Ignore phone numbers or email addresses.", - schema=CompanyAddress, - ) - - if address_result.company_address and address_result.company_address.strip(): - address = address_result.company_address.strip() - print(f"[{company_name}] Address found in Terms of Service: {address}") - except Exception: - print(f"[{company_name}] Could not extract address from Terms of Service page") - - # Fallback: check Privacy Policy if address not found in Terms of Service - if not address and privacy_policy_link: - print( - f"[{company_name}] Address not found in Terms of Service, trying Privacy Policy..." - ) - await with_retry( - lambda: page.goto(privacy_policy_link), - f"[{company_name}] Navigate to Privacy Policy", - ) - - try: - address_result = await page.extract( - "Extract the physical company mailing address (street, city, state, postal code, and country if present) from the Privacy Policy page. Ignore phone numbers or email addresses.", - schema=CompanyAddress, - ) - - if address_result.company_address and address_result.company_address.strip(): - address = address_result.company_address.strip() - print(f"[{company_name}] Address found in Privacy Policy: {address}") - except Exception: - print(f"[{company_name}] Could not extract address from Privacy Policy page") - - if not address: - address = "Address not found in Terms of Service or Privacy Policy pages" - print(f"[{company_name}] {address}") - - result = CompanyData( - company_name=company_name, - homepage_url=homepage_url, - terms_of_service_link=terms_of_service_link, - privacy_policy_link=privacy_policy_link, - address=address, - ) - - print(f"[{company_name}] Successfully processed") - return result + company: CompanyData = result["structured_response"] + return company except Exception as error: print(f"[{company_name}] Error: {error}") - return CompanyData( company_name=company_name, homepage_url="", - terms_of_service_link="", - privacy_policy_link="", + terms_of_service_link=None, + privacy_policy_link=None, address=f"Error: {error}", ) - finally: - if stagehand: - try: - await stagehand.close() - print(f"[{company_name}] Session closed successfully") - except Exception as close_error: - print(f"[{company_name}] Error closing browser: {close_error}") - -# Main orchestration function: processes companies sequentially or in batches based on MAX_CONCURRENT -# Collects results and outputs final JSON summary -async def main(): +async def main() -> None: print("Starting Company Address Finder...") - - company_names = COMPANY_NAMES + results: list[CompanyData] = [] max_concurrent = max(1, MAX_CONCURRENT) - company_count = len(company_names) - is_sequential = max_concurrent == 1 - - print( - f"\nProcessing {company_count} {'company' if company_count == 1 else 'companies'} {'sequentially' if is_sequential else f'concurrently (batch size: {max_concurrent})'}..." - ) - all_results = [] + for index in range(0, len(COMPANY_NAMES), max_concurrent): + batch = COMPANY_NAMES[index : index + max_concurrent] + results.extend(await asyncio.gather(*(process_company(name) for name in batch))) - if is_sequential: - for i, company_name in enumerate(company_names): - print(f"[{i + 1}/{len(company_names)}] {company_name}") - result = await process_company(company_name) - all_results.append(result) - else: - for i in range(0, len(company_names), max_concurrent): - batch = company_names[i : i + max_concurrent] - batch_number = i // max_concurrent + 1 - total_batches = (len(company_names) + max_concurrent - 1) // max_concurrent - - print(f"\nBatch {batch_number}/{total_batches}: {', '.join(batch)}") - - batch_promises = [process_company(name) for name in batch] - batch_results = await asyncio.gather(*batch_promises) - all_results.extend(batch_results) - - print( - f"Batch {batch_number}/{total_batches} completed: {len(batch_results)} companies processed" - ) - - print("\n" + "=" * 80) - print("RESULTS (JSON):") - print("=" * 80) - print(json.dumps([result.model_dump() for result in all_results], indent=2)) - print("=" * 80) - - print(f"\nComplete: processed {len(all_results)}/{len(company_names)} companies") + print("Results:") + print("[" + ",\n".join(company.model_dump_json(indent=2) for company in results) + "]") + print(f"Complete: processed {len(results)}/{len(COMPANY_NAMES)} companies") if __name__ == "__main__": try: asyncio.run(main()) - except Exception as err: - print(f"Application error: {err}") - print("Common issues:") - print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") - print(" - Verify GEMINI_API_KEY is set") - print(" - Ensure COMPANY_NAMES is configured in the config section") - print("Docs: https://docs.stagehand.dev/v3/first-steps/introduction") - exit(1) + except Exception as error: + print(f"Application error: {error}") + print("Check BROWSERBASE_API_KEY and AI_GATEWAY_API_KEY in .env") + raise SystemExit(1) from error diff --git a/python/company-address-finder/pyproject.toml b/python/company-address-finder/pyproject.toml new file mode 100644 index 00000000..0c3e700a --- /dev/null +++ b/python/company-address-finder/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "company-address-finder" +version = "0.1.0" +description = "Research company legal pages with Deep Agents and Stagehand V4" +readme = "README.md" +requires-python = ">=3.11,<3.14" +dependencies = [ + "deepagents==0.7.5", + "langchain-mcp-adapters==0.3.2", + "langchain-openai==1.4.3", + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", +] + +[tool.uv] +package = false diff --git a/python/company-value-prop-generator/.env.example b/python/company-value-prop-generator/.env.example new file mode 100644 index 00000000..d0cde042 --- /dev/null +++ b/python/company-value-prop-generator/.env.example @@ -0,0 +1 @@ +BROWSERBASE_API_KEY= diff --git a/python/company-value-prop-generator/README.md b/python/company-value-prop-generator/README.md index 4685aa11..ed59b505 100644 --- a/python/company-value-prop-generator/README.md +++ b/python/company-value-prop-generator/README.md @@ -1,34 +1,32 @@ # Stagehand + Browserbase: Value Prop One-Liner Generator +Stagehand is the SDK for browser agents. + ## AT A GLANCE - Goal: Automatically extract and format website value propositions into concise one-liners for email personalization - Demonstrates Stagehand's `extract` method with Pydantic schemas to pull structured data from landing pages -- Shows direct OpenAI API usage to transform extracted content with custom prompts -- Includes placeholder page detection and validation logic to filter out non-functional sites -- Docs โ†’ https://docs.browserbase.com/stagehand/extract +- Shows how to chain Stagehand V4 extractions to transform grounded page content with custom prompts +- Docs โ†’ https://docs.stagehand.dev/v4/basics/extract ## GLOSSARY - Extract: Stagehand method that uses AI to pull structured data from pages using natural language instructions - Docs โ†’ https://docs.browserbase.com/stagehand/extract + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract - Value Proposition: The core benefit or unique selling point a company communicates to customers ## QUICKSTART 1. cd python/company-value-prop-generator 2. pip install python-dotenv stagehand openai pydantic -3. cp .env.example .env # Add required API keys/IDs to .env +3. cp .env.example .env # Add your Browserbase API key to .env 4. python main.py ## EXPECTED OUTPUT - Stagehand initializes and creates a Browserbase session -- Displays live session link for monitoring - Navigates to target domain and waits for page load -- Checks for placeholder pages via meta tag inspection - Extracts value proposition from landing page using AI -- Validates extracted content against placeholder patterns - Generates formatted one-liner via LLM (constraints: 9 words max, starts with "your") - Prints generated one-liner to console - Closes browser session @@ -37,11 +35,8 @@ - Dependency install errors: ensure pip install completed - Missing credentials: - - BROWSERBASE_PROJECT_ID (required for browser automation) - BROWSERBASE_API_KEY (required for browser automation) - - OPENAI_API_KEY (required for LLM generation) -- Access limitations: Requires OpenAI API access for gpt-4.1 model -- Placeholder pages: Template includes detection logic, but some custom placeholder pages may still pass validation +- Placeholder pages: extraction quality depends on the content available on the target page - Slow-loading sites: 5-minute timeout configured, but extremely slow sites may still timeout ## USE CASES @@ -58,7 +53,7 @@ ## HELPFUL RESOURCES -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction ๐ŸŽฎ Browserbase: https://www.browserbase.com ๐Ÿ’ก Templates: https://www.browserbase.com/templates ๐Ÿ“ง Need help? support@browserbase.com diff --git a/python/company-value-prop-generator/main.py b/python/company-value-prop-generator/main.py index cc49e200..8d517096 100644 --- a/python/company-value-prop-generator/main.py +++ b/python/company-value-prop-generator/main.py @@ -1,168 +1,82 @@ -# Stagehand + Browserbase: Value Prop One-Liner Generator - See README.md for full documentation +"""Generate a concise company value proposition with Stagehand V4.""" import asyncio import os from dotenv import load_dotenv -from openai import OpenAI -from pydantic import BaseModel, Field +from pydantic import BaseModel -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand, browserbase -# Load environment variables load_dotenv() -# Domain to analyze - change this to target a different website -target_domain = "www.browserbase.com" # Or extract from email: email.split("@")[1] +TARGET_DOMAIN = "www.browserbase.com" -# Initialize OpenAI client -client = OpenAI() +class ValueProposition(BaseModel): + value_prop: str -class ValueProp(BaseModel): - value_prop: str = Field(..., description="the value proposition from the landing page") + +class OneLiner(BaseModel): + one_liner: str async def generate_one_liner(domain: str) -> str: - """ - Analyzes a website's landing page to generate a concise one-liner value proposition. - Extracts the value prop using Stagehand, then uses an LLM to format it into a short phrase starting with "your". - """ - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - }, - verbose=0, # 0 = errors only, 1 = info, 2 = debug - ) + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") + browser = await browserbase.launch(api_key=api_key) try: - async with Stagehand(config) as stagehand: - print("Stagehand initialized successfully!") - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id - - if session_id: - print(f"Live View Link: https://browserbase.com/sessions/{session_id}") - - page = stagehand.page - - # Navigate to domain - print(f"๐ŸŒ Navigating to https://{domain}...") - # 5min timeout to handle slow-loading sites or network issues + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() await page.goto( f"https://{domain}/", wait_until="domcontentloaded", - timeout=300000, + timeout=300_000, ) - print(f"โœ… Successfully loaded {domain}") - - # Extract value proposition from landing page - print(f"๐Ÿ“ Extracting value proposition for {domain}...") - value_prop_data = await page.extract( - "extract the value proposition from the landing page", - schema=ValueProp, + value_prop_result = await stagehand.extract( + "Extract the value proposition from the landing page", + ValueProposition, + page=page, ) - - value_prop = value_prop_data.value_prop - print(f"๐Ÿ“Š Extracted value prop for {domain}: {value_prop}") - - # Validate extraction returned meaningful content - if not value_prop or value_prop.lower() == "null" or value_prop.lower() == "undefined": - print("โš ๏ธ Value prop extraction returned empty or invalid result") - raise ValueError(f"No value prop found for {domain}") - - # Generate one-liner using OpenAI - # Prompt uses few-shot examples to guide LLM toward concise, "your X" format - # System prompt enforces constraints (9 words max, no quotes, must start with "your") - print(f"๐Ÿค– Generating email one-liner for {domain}...") - - response = await asyncio.to_thread( - client.chat.completions.create, - model="gpt-4.1", - messages=[ - { - "role": "system", - "content": "You are an expert at generating concise, unique descriptions of companies. Generate ONLY a concise description (no greetings or extra text). Don't use generic adjectives like 'comprehensive', 'innovative', or 'powerful'. Keep it short and concise, no more than 9 words. DO NOT USE QUOTES. Only use English. You MUST start the response with 'your'.", - }, - { - "role": "user", - "content": f"""The response will be inserted into this template: "{{response}}" - -Examples: -Value prop: "Supercharge your investment team with AI-powered research" -Response: "your AI-powered investment research platform" - -Value prop: "The video-first food delivery app" -Response: "your video-first approach to food delivery" - -Value prop: "{value_prop}" -Response:""", - }, - ], + value_prop = value_prop_result.data.value_prop.strip() + print(f"Extracted value proposition: {value_prop}") + + formatted_result = await stagehand.extract( + ( + f'Using the company value proposition "{value_prop}", write a unique ' + 'English description that starts with "your", uses no quotes, avoids ' + "generic adjectives, and is no more than 9 words" + ), + OneLiner, + page=page, ) - - one_liner = (response.choices[0].message.content or "").strip() - - # Validate LLM response is usable (not empty, not generic placeholder) - print("๐Ÿ” Validating generated one-liner...") - if ( - not one_liner - or one_liner.lower() == "null" - or one_liner.lower() == "undefined" - or one_liner.lower() == "your company" - ): - print(f'โš ๏ธ LLM generated invalid or placeholder response: "{one_liner}"') - raise ValueError( - f'No valid one-liner generated for {domain}. AI response: "{one_liner}"' - ) - - print(f"โœจ Generated one-liner for {domain}: {one_liner}") + one_liner = formatted_result.data.one_liner.strip() + print(f"Generated one-liner: {one_liner}") return one_liner - - except Exception as error: - error_message = str(error) if isinstance(error, Exception) else error - print(f"โŒ Generation failed for {domain}: {error_message}") - raise + finally: + await stagehand.close() finally: + await browser.close() print("Session closed successfully") -async def main(): - """ - Main entry point: generates a one-liner value proposition for the target domain. - """ +async def main() -> None: print("Starting One-Liner Generator...") - - try: - one_liner = await generate_one_liner(target_domain) - print("\nโœ… Success!") - print(f"One-liner: {one_liner}") - except Exception as error: - error_message = str(error) if isinstance(error, Exception) else error - print(f"\nโŒ Error: {error_message}") - print("\nCommon issues:") - print(" - Check .env file has OPENAI_API_KEY set (required for LLM generation)") - print( - " - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY set (required for browser automation)" - ) - print(" - Ensure the domain is accessible and not a placeholder/maintenance page") - print(" - Verify internet connectivity and that the target site is reachable") - print("Docs: https://docs.browserbase.com/stagehand") - exit(1) + one_liner = await generate_one_liner(TARGET_DOMAIN) + print(f"Success: {one_liner}") if __name__ == "__main__": try: asyncio.run(main()) - except Exception as err: - print(f"Fatal error: {err}") - exit(1) + except Exception as error: + print(f"Error: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/company-value-prop-generator/pyproject.toml b/python/company-value-prop-generator/pyproject.toml new file mode 100644 index 00000000..55ecf2e0 --- /dev/null +++ b/python/company-value-prop-generator/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "company-value-prop-generator" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[tool.uv] +package = false diff --git a/python/context/.env.example b/python/context/.env.example new file mode 100644 index 00000000..51832c62 --- /dev/null +++ b/python/context/.env.example @@ -0,0 +1,3 @@ +BROWSERBASE_API_KEY= +SF_REC_PARK_EMAIL= +SF_REC_PARK_PASSWORD= diff --git a/python/context/README.md b/python/context/README.md index 7aba1183..c81590f4 100644 --- a/python/context/README.md +++ b/python/context/README.md @@ -1,5 +1,7 @@ # Stagehand + Browserbase: Context Authentication Example +Stagehand is the SDK for browser agents. + ## AT A GLANCE - Goal: demonstrate persistent authentication using Browserbase **contexts** that survive across sessions. @@ -13,7 +15,7 @@ Docs โ†’ https://docs.browserbase.com/features/contexts - persist: when true, any state changes during a session are written back to the context for future reuse. - act: perform UI actions from a prompt (click, type, navigate). - Docs โ†’ https://docs.stagehand.dev/basics/act + Docs โ†’ https://docs.stagehand.dev/v4/basics/act ## QUICKSTART @@ -22,7 +24,7 @@ 3. source venv/bin/activate # On Windows: venv\Scripts\activate 4. pip install -r requirements.txt 5. pip install browserbase pydantic requests -6. cp .env.example .env # Add your Browserbase API key, Project ID, and SF Rec Park credentials to .env +6. cp .env.example .env # Add your Browserbase API key and SF Rec Park credentials to .env 7. python main.py ## EXPECTED OUTPUT @@ -53,7 +55,7 @@ ## HELPFUL RESOURCES -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction ๐ŸŽฎ Browserbase: https://www.browserbase.com ๐Ÿ’ก Try it out: https://www.browserbase.com/playground ๐Ÿ”ง Templates: https://www.browserbase.com/templates diff --git a/python/context/main.py b/python/context/main.py index 156a2f76..f31ad59c 100644 --- a/python/context/main.py +++ b/python/context/main.py @@ -1,173 +1,110 @@ -# Stagehand + Browserbase: Context Authentication Example - See README.md for full documentation +"""Persist and verify an authenticated Browserbase context with Stagehand V4.""" import asyncio +import json import os -import requests -from browserbase import Browserbase +from browserbase import AsyncBrowserbase from dotenv import load_dotenv from pydantic import BaseModel, Field -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand, browserbase -# Load environment variables load_dotenv() +TARGET_URL = "https://www.rec.us/organizations/san-francisco-rec-park" -async def create_session_context_id(): - print("Creating new Browserbase context...") - # First create a context using Browserbase SDK to get a context ID. - bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) - context = bb.contexts.create(project_id=os.environ.get("BROWSERBASE_PROJECT_ID")) - - print(f"Created context ID: {context.id}") - - # Create a single session using the context ID to perform initial login. - print("Creating session for initial login...") - session = bb.sessions.create( - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - browser_settings={ - "context": { - "id": context.id, - "persist": True, # Save authentication state to context - } - }, - ) - print(f"Live view: https://browserbase.com/sessions/{session.id}") - - # Connect Stagehand to the existing session (no new session created). - print("Connecting Stagehand to session...") - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_id=session.id, - verbose=1, # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - ) - - async with Stagehand(config) as stagehand: - page = stagehand.page - email = os.environ.get("SF_REC_PARK_EMAIL") - password = os.environ.get("SF_REC_PARK_PASSWORD") - - # Navigate to login page with extended timeout for slow-loading sites. - print("Navigating to SF Rec & Park login page...") - await page.goto( - "https://www.rec.us/organizations/san-francisco-rec-park", - wait_until="domcontentloaded", - timeout=60000, - ) - # Perform login sequence: each step is atomic to handle dynamic page changes. - print("Starting login sequence...") - await page.act("Click the Login button") - await page.act(f'Fill in the email or username field with "{email}"') - await page.act("Click the next, continue, or submit button to proceed") - await page.act(f'Fill in the password field with "{password}"') - await page.act("Click the login, sign in, or submit button") - print("Login sequence completed!") +class UserData(BaseModel): + full_name: str = Field(min_length=1) + address: str = Field(min_length=1) - print("Authentication state saved to context") - # Return the context ID for reuse in future sessions. - return {"id": context.id} +def require_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required") + return value -async def delete_context(context_id: str): +async def login_and_persist(context_id: str) -> None: + browser = await browserbase.launch( + api_key=require_env("BROWSERBASE_API_KEY"), + browser_settings={"context": {"id": context_id, "persist": True}}, + ) try: - print(f"Cleaning up context: {context_id}") - # Delete context via Browserbase API to clean up stored authentication data. - # This prevents accumulation of unused contexts and ensures security cleanup. - response = requests.delete( - f"https://api.browserbase.com/v1/contexts/{context_id}", - headers={ - "X-BB-API-Key": os.environ.get("BROWSERBASE_API_KEY"), - }, + stagehand = await Stagehand.create( + browser=browser, ) - print(f"Context deleted successfully (status: {response.status_code})") - except Exception as error: - error_msg = getattr(error, "response", {}).get("data") or str(error) - print(f"Error deleting context: {error_msg}") - - -async def main(): - print("Starting Context Authentication Example...") - # Create context with login state for reuse in authenticated sessions. - context_id = await create_session_context_id() - - # Initialize new session using existing context to inherit authentication state. - # persist: true ensures any new changes (cookies, cache) are saved back to context. - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - "browser_settings": { - "context": { - "id": context_id["id"], - "persist": True, - } - }, - }, - verbose=1, # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000) + await stagehand.act("Click the Login button", page=page) + await stagehand.act( + "Fill the email or username field with %email%", + page=page, + variables={"email": require_env("SF_REC_PARK_EMAIL")}, + ) + await stagehand.act("Click the next, continue, or submit button", page=page) + await stagehand.act( + "Fill the password field with %password%", + page=page, + variables={"password": require_env("SF_REC_PARK_PASSWORD")}, + ) + await stagehand.act("Click the login, sign in, or submit button", page=page) + finally: + await stagehand.close() + finally: + await browser.close() + + +async def verify_reused_context(context_id: str) -> UserData: + browser = await browserbase.launch( + api_key=require_env("BROWSERBASE_API_KEY"), + browser_settings={"context": {"id": context_id, "persist": True}}, ) - - async with Stagehand(config) as stagehand: - print("Authenticated session ready!") - if hasattr(stagehand, "session_id"): - print(f"Live view: https://browserbase.com/sessions/{stagehand.session_id}") - elif hasattr(stagehand, "browserbase_session_id"): - print(f"Live view: https://browserbase.com/sessions/{stagehand.browserbase_session_id}") - - page = stagehand.page - - # Navigate to authenticated area - should skip login due to persisted cookies. - print("Navigating to authenticated area (should skip login)...") - await page.goto( - "https://www.rec.us/organizations/san-francisco-rec-park", - wait_until="domcontentloaded", - timeout=60000, + try: + stagehand = await Stagehand.create( + browser=browser, ) - - # Navigate to user-specific area to access personal data. - await page.act("Click on the reservations button") - - # Extract structured user data using Pydantic schema for type safety. - # Schema ensures consistent data format and validates extracted content. - print("Extracting user profile data...") - - # Define schema using Pydantic - class UserData(BaseModel): - full_name: str = Field(..., description="the user's full name") - address: str = Field(..., description="the user's address") - - user_data = await page.extract("Extract the user's full name and address", schema=UserData) - - print(f"Extracted user data: {user_data}") - - print("Session closed successfully") - - # Clean up context to prevent accumulation and ensure security. - await delete_context(context_id["id"]) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000) + await stagehand.act("Click the reservations button", page=page) + extracted = await stagehand.extract( + "Extract the authenticated user's full name and address", + UserData, + page=page, + ) + return extracted.data + finally: + await stagehand.close() + finally: + await browser.close() + + +async def main() -> None: + async with AsyncBrowserbase(api_key=require_env("BROWSERBASE_API_KEY")) as api: + context = await api.contexts.create() + print("Created temporary Browserbase context") + try: + await login_and_persist(context.id) + user = await verify_reused_context(context.id) + print("Reused context reached authenticated profile data:") + print(json.dumps(user.model_dump(mode="json"), indent=2)) + finally: + # The generated SDK currently sets a JSON content type on DELETE, so send an + # explicit empty object instead of an empty body. + await api.contexts.delete(context.id, extra_body={}) + print("Deleted temporary Browserbase context") if __name__ == "__main__": try: asyncio.run(main()) - except Exception as err: - print(f"Error in context authentication example: {err}") - print("Common issues:") - print(" - Check .env file has SF_REC_PARK_EMAIL and SF_REC_PARK_PASSWORD") - print(" - Verify BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY are set") - print(" - Ensure credentials are valid for SF Rec & Park") - print("Docs: https://docs.stagehand.dev/v3/first-steps/introduction") - exit(1) + except Exception as error: + print(f"Context authentication example failed: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/context/pyproject.toml b/python/context/pyproject.toml new file mode 100644 index 00000000..b9cacc74 --- /dev/null +++ b/python/context/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "context" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "browserbase>=1.7.0", + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[tool.uv] +package = false diff --git a/python/council-events/.env.example b/python/council-events/.env.example new file mode 100644 index 00000000..d0cde042 --- /dev/null +++ b/python/council-events/.env.example @@ -0,0 +1 @@ +BROWSERBASE_API_KEY= diff --git a/python/council-events/README.md b/python/council-events/README.md index a047763f..361ee4a4 100644 --- a/python/council-events/README.md +++ b/python/council-events/README.md @@ -1,28 +1,29 @@ # Stagehand + Browserbase: Philadelphia Council Events Scraper +Stagehand is the SDK for browser agents. + ## AT A GLANCE - Goal: automate extraction of Philadelphia Council events for 2025 from the official calendar. - Flow: navigate to phila.legistar.com โ†’ click calendar โ†’ select 2025 โ†’ extract event data (name, date, time). - Benefits: quickly gather upcoming council events without manual browsing, structured data ready for analysis or notifications. - Docs โ†’ https://docs.stagehand.dev/v3/first-steps/introduction + Docs โ†’ https://docs.stagehand.dev/v4/first-steps/introduction ## GLOSSARY - act: perform UI actions from a prompt (click, select, navigate). - Docs โ†’ https://docs.stagehand.dev/basics/act + Docs โ†’ https://docs.stagehand.dev/v4/basics/act - extract: pull structured data from a page using AI and Pydantic schemas. - Docs โ†’ https://docs.stagehand.dev/basics/extract + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract - Pydantic schema: type-safe data models that validate extracted content. ## QUICKSTART 1. cd council-events -2. uv venv venv -3. source venv/bin/activate # On Windows: venv\Scripts\activate -4. pip install stagehand python-dotenv pydantic -5. cp .env.example .env # Add your Browserbase API key, Project ID, and OpenAI API key to .env -6. python main.py +2. uv venv && source .venv/bin/activate # On Windows: .venv\Scripts\activate +3. pip install stagehand python-dotenv pydantic +4. cp .env.example .env # Add your Browserbase API key to .env +5. python main.py ## EXPECTED OUTPUT @@ -37,7 +38,7 @@ ## COMMON PITFALLS - "ModuleNotFoundError": ensure all dependencies are installed via pip -- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and OPENAI_API_KEY +- Missing credentials: verify .env contains BROWSERBASE_API_KEY - No events found: check if the website structure has changed or if 2025 calendar is available - Network issues: ensure internet access and phila.legistar.com is accessible - Import errors: activate your virtual environment if you created one @@ -56,9 +57,9 @@ ## HELPFUL RESOURCES -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction ๐ŸŽฎ Browserbase: https://www.browserbase.com ๐Ÿ’ก Try it out: https://www.browserbase.com/playground -๐Ÿ”ง Templates: https://github.com/browserbase/stagehand/tree/main/examples +๐Ÿ”ง Templates: https://www.browserbase.com/templates ๐Ÿ“ง Need help? support@browserbase.com ๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/council-events/main.py b/python/council-events/main.py index 4e246959..7ddaf0af 100644 --- a/python/council-events/main.py +++ b/python/council-events/main.py @@ -1,115 +1,83 @@ -# Stagehand + Browserbase: Philadelphia Council Events Scraper - See README.md for full documentation +"""Extract current Philadelphia City Council events with Stagehand V4.""" import asyncio +import json import os +from datetime import UTC, datetime from dotenv import load_dotenv -from pydantic import BaseModel, Field +from pydantic import BaseModel -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand, browserbase -# Load environment variables load_dotenv() -class Event(BaseModel): - """Single event with name, date, and time""" +class CouncilEvent(BaseModel): + name: str + date: str + time: str - name: str = Field(..., description="the name of the event") - date: str = Field(..., description="the date of the event") - time: str = Field(..., description="the time of the event") +class CouncilEvents(BaseModel): + results: list[CouncilEvent] -class EventResults(BaseModel): - """Collection of events extracted from the calendar""" - results: list[Event] = Field(..., description="array of events") - - -async def main(): - """ - Searches Philadelphia Council Events for 2025 and extracts event information. - Uses AI-powered browser automation to navigate and interact with the site. - """ - print("Starting Philadelphia Council Events automation...") - - # Initialize Stagehand with Browserbase for cloud-based browser automation - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - verbose=1, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - ) +async def main() -> None: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") + year = datetime.now(UTC).year + print(f"Starting Philadelphia Council Events automation for {year}...") + browser = await browserbase.launch(api_key=api_key) try: - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - print("Initializing browser session...") - print("Stagehand session started successfully") - - # Provide live session URL for debugging and monitoring - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id - - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") - - page = stagehand.page - - # Navigate to Philadelphia Council - print("Navigating to: https://phila.legistar.com/") - await page.goto("https://phila.legistar.com/") - print("Page loaded successfully") - - # Click calendar from the navigation menu - print("Clicking calendar from the navigation menu") - await page.act("click calendar from the navigation menu") - - # Select 2025 from the month dropdown - print("Selecting 2025 from the month dropdown") - await page.act("select 2025 from the month dropdown") - - # Extract event data using AI to parse the structured information - print("Extracting event information...") - results = await page.extract( - "Extract the table with the name, date and time of the events", schema=EventResults + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto( + "https://phila.legistar.com/", + wait_until="domcontentloaded", + timeout=60_000, + ) + await stagehand.act("Click Calendar in the navigation menu", page=page) + await stagehand.act(f"Select {year} from the year dropdown", page=page) + page = await browser.context.active_page() or page + if "Calendar.aspx" not in await page.url(): + await page.goto( + "https://phila.legistar.com/Calendar.aspx", + wait_until="domcontentloaded", + timeout=60_000, + ) + await stagehand.observe( + f"Find the calendar table rows for {year}", + page=page, ) - print(f"Found {len(results.results)} events") - print("Event data extracted successfully:") - - # Display results in formatted JSON - import json - - print(json.dumps(results.model_dump(), indent=2)) - + extracted = await stagehand.extract( + ( + f"Extract every {year} event visible in the calendar table with its " + "name, date, and time" + ), + CouncilEvents, + page=page, + ) + print(f"Found {len(extracted.data.results)} events") + print(json.dumps(extracted.data.model_dump(mode="json"), indent=2)) + finally: + await stagehand.close() + finally: + await browser.close() print("Session closed successfully") - except Exception as error: - print(f"Error during event extraction: {error}") - - # Provide helpful troubleshooting information - print("\nCommon issues:") - print("1. Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") - print("2. Verify OPENAI_API_KEY is set in environment") - print("3. Ensure internet access and https://phila.legistar.com is accessible") - print("4. Verify Browserbase account has sufficient credits") - print("5. Check if the calendar page structure has changed") - - raise - if __name__ == "__main__": try: asyncio.run(main()) - except Exception as err: - print(f"Application error: {err}") - exit(1) + except Exception as error: + print(f"Application error: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/council-events/pyproject.toml b/python/council-events/pyproject.toml new file mode 100644 index 00000000..9893ac29 --- /dev/null +++ b/python/council-events/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "council-events" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[tool.uv] +package = false diff --git a/python/download-financial-statements/.env.example b/python/download-financial-statements/.env.example new file mode 100644 index 00000000..d0cde042 --- /dev/null +++ b/python/download-financial-statements/.env.example @@ -0,0 +1 @@ +BROWSERBASE_API_KEY= diff --git a/python/download-financial-statements/README.md b/python/download-financial-statements/README.md index bc0d54c3..137db0ec 100644 --- a/python/download-financial-statements/README.md +++ b/python/download-financial-statements/README.md @@ -1,16 +1,18 @@ # Stagehand + Browserbase: Download Apple's Quarterly Financial Statements +Stagehand is the SDK for browser agents. + ## AT A GLANCE - Goal: automate downloading Apple's quarterly financial statements (PDFs) from their investor relations site. - Download Handling: Browserbase automatically captures PDFs opened during the session and bundles them into a ZIP file. - Retry Logic: polls Browserbase downloads API with configurable timeout to ensure files are ready before retrieval. -- Live Debugging: displays live view URL for real-time session monitoring. +- Live Debugging: the session remains available in the Browserbase Sessions dashboard without logging a signed URL. ## GLOSSARY - act: perform UI actions from a prompt (click, scroll, navigate) - Docs โ†’ https://docs.stagehand.dev/basics/act + Docs โ†’ https://docs.stagehand.dev/v4/basics/act - downloads API: retrieve files downloaded during a Browserbase session as a ZIP archive Docs โ†’ https://docs.browserbase.com/features/screenshots#pdfs - live view: real-time browser debugging interface for monitoring automation @@ -19,7 +21,7 @@ ## QUICKSTART 1. cd python/download-financial-statements -2. cp .env.example .env # Add your Browserbase API key and Project ID to .env +2. cp .env.example .env # Add your Browserbase API key to .env 3. uvx --with browserbase --with python-dotenv stagehand main.py ## EXPECTED OUTPUT @@ -35,7 +37,7 @@ ## COMMON PITFALLS - "Cannot find module": ensure uvx is installed (`pip install uv`) or use `pip install stagehand-ai browserbase python-dotenv` for traditional setup -- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- Missing credentials: verify .env contains BROWSERBASE_API_KEY - Download timeout: increase `retry_for_seconds` parameter if downloads take longer than 45 seconds - Empty ZIP file: ensure PDFs were actually triggered (check live view link to debug) - Network issues: check internet connection and Apple website accessibility @@ -54,7 +56,7 @@ ## HELPFUL RESOURCES -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v2/first-steps/introduction +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction ๐ŸŽฎ Browserbase: https://www.browserbase.com ๐Ÿ’ก Try it out: https://www.browserbase.com/playground ๐Ÿ”ง Templates: https://www.browserbase.com/templates diff --git a/python/download-financial-statements/main.py b/python/download-financial-statements/main.py index c32074ae..0632ddb7 100644 --- a/python/download-financial-statements/main.py +++ b/python/download-financial-statements/main.py @@ -1,174 +1,117 @@ -# Stagehand + Browserbase: Download Apple's Quarterly Financial Statements - See README.md for full documentation +"""Download Apple's FY2025 statements with Stagehand V4.""" import asyncio +import json import os +import time +from pathlib import Path from browserbase import Browserbase from dotenv import load_dotenv +from pydantic import BaseModel, HttpUrl -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand, browserbase -# Load environment variables from .env file -# Required: BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, GOOGLE_API_KEY load_dotenv() -async def save_downloads_with_retry( - bb: Browserbase, session_id: str, retry_for_seconds: int = 30 -) -> int: - """ - Polls Browserbase API for downloads with timeout handling. - - Browserbase stores downloaded files during a session and makes them available - via API. Files may take a few seconds to process, so this function implements - retry logic to wait for downloads to be ready before retrieving them. - - Args: - bb: Browserbase client instance for API calls - session_id: The Browserbase session ID to retrieve downloads from - retry_for_seconds: Maximum time to wait for downloads (default: 30 seconds) - - Returns: - int: The size of the downloaded ZIP file in bytes - - Raises: - TimeoutError: If downloads aren't ready within the specified timeout - """ - print(f"Waiting up to {retry_for_seconds} seconds for downloads to complete...") - - # Track elapsed time to implement timeout without using threading timers - start_time = asyncio.get_event_loop().time() - timeout = retry_for_seconds - - while True: - elapsed = asyncio.get_event_loop().time() - start_time +class StatementLinks(BaseModel): + statement_urls: list[HttpUrl] - # Check if we've exceeded the timeout period - if elapsed >= timeout: - raise TimeoutError("Download timeout exceeded") - try: - print("Checking for downloads...") - # Use asyncio.to_thread for synchronous Browserbase SDK calls - # This prevents blocking the event loop while waiting for API responses - response = await asyncio.to_thread(bb.sessions.downloads.list, session_id) - download_buffer = await asyncio.to_thread(response.read) - - # Check if downloads are ready (non-empty buffer indicates files are available) - if len(download_buffer) > 0: - print(f"Downloads ready! File size: {len(download_buffer)} bytes") - - # Save the ZIP file containing all downloaded PDFs to disk - with open("downloaded_files.zip", "wb") as f: - f.write(download_buffer) - print("Files saved as: downloaded_files.zip") - return len(download_buffer) - else: - print("Downloads not ready yet, retrying...") - except Exception as e: - print(f"Error fetching downloads: {e}") - raise - - # Poll every 2 seconds to check if downloads are ready - # This interval balances responsiveness with API rate limits +async def save_downloads_with_retry( + client: Browserbase, + session_id: str, + retry_for_seconds: int = 45, +) -> int: + started = time.monotonic() + while time.monotonic() - started < retry_for_seconds: + response = await asyncio.to_thread(client.sessions.downloads.list, session_id) + payload = await asyncio.to_thread(response.read) + if payload: + Path("downloaded_files.zip").write_bytes(payload) + print(f"Saved downloaded_files.zip ({len(payload)} bytes)") + return len(payload) await asyncio.sleep(2) + raise TimeoutError("Download timeout exceeded") -async def main(): - """ - Main application entry point. +async def main() -> None: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") - Orchestrates the entire PDF download automation process: - 1. Initializes Browserbase and Stagehand clients - 2. Navigates to Apple's investor relations site - 3. Locates and clicks quarterly financial statement links - 4. Waits for downloads to process and saves them as a ZIP file - """ print("Starting Apple Financial Statements Download Automation...") - - # Initialize Browserbase client for session management and downloads API - print("Initializing Browserbase client...") - bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) - - # Initialize Stagehand with Browserbase for cloud-based browser automation - # Stagehand provides natural language browser control (act, extract, observe) - config = StagehandConfig( - env="BROWSERBASE", # Use Browserbase's cloud infrastructure instead of local browsers - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="google/gemini-2.5-flash-preview-05-20", # AI model for interpreting natural language actions - model_api_key=os.environ.get("GOOGLE_API_KEY"), - verbose=2, - # Logging levels: 0 = errors only, 1 = info, 2 = debug - # When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs - # https://docs.stagehand.dev/configuration/logging - ) + api = Browserbase(api_key=api_key) + browser = await browserbase.launch(api_key=api_key) + session_id = browser.session_id + if not session_id: + await browser.close() + raise RuntimeError("Browserbase launch did not return a session ID") try: - # Initialize browser session to start automation - # Using context manager ensures proper cleanup even if errors occur - async with Stagehand(config) as stagehand: - print("Stagehand initialized successfully!") - page = stagehand.page - - # Display live view URL for debugging and monitoring - # Live view allows real-time observation of browser automation - live_view_links = bb.sessions.debug(stagehand.session_id) - live_view_link = live_view_links.debuggerFullscreenUrl - print(f"๐Ÿ” Live View Link: {live_view_link}") - - # Navigate to Apple homepage with extended timeout for slow-loading sites - print("Navigating to Apple.com...") - await page.goto("https://www.apple.com/", timeout=60000) - - # Navigate to investor relations section using natural language actions - # act() uses AI to interpret instructions and perform browser interactions - print("Navigating to Investors section...") - await page.act("Click the 'Investors' button at the bottom of the page") - await page.act("Scroll down to the Financial Data section of the page") - await page.act("Under Quarterly Earnings Reports, click on '2025'") - - # Download all quarterly financial statements - # When a URL of a PDF is opened, Browserbase automatically downloads and stores the PDF - # Files are captured in the session and can be retrieved via the downloads API - # See https://docs.browserbase.com/features/screenshots#pdfs for more info - print("Downloading quarterly financial statements...") - await page.act("Click the 'Financial Statements' link under Q4") - await page.act("Click the 'Financial Statements' link under Q3") - await page.act("Click the 'Financial Statements' link under Q2") - await page.act("Click the 'Financial Statements' link under Q1") - - # Retrieve all downloads triggered during this session from Browserbase API - # Files take time to process, so we poll with retry logic (45 second timeout) - print("Retrieving downloads from Browserbase...") - await save_downloads_with_retry(bb, stagehand.session_id, 45) - print("All downloads completed successfully!") - - # Print history if available in Stagehand v2 - # History shows all actions taken during the session for debugging - if hasattr(stagehand, "history"): - print("\nStagehand History:") - print(stagehand.history) - - # Context manager automatically closes the session and cleans up resources + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto("https://www.apple.com/", wait_until="domcontentloaded", timeout=60_000) + await stagehand.act( + "Click the Investors button at the bottom of the page", + page=page, + ) + await stagehand.act( + "Scroll down to the Financial Data section", + page=page, + ) + await stagehand.act( + "Under Quarterly Earnings Reports, click 2025", + page=page, + ) + page = await browser.context.active_page() or page + extracted = await stagehand.extract( + ( + "Extract the actual absolute HTTP(S) href URLs of the four FY2025 Financial " + "Statements PDF links, ordered Q4 through Q1." + ), + StatementLinks, + page=page, + ) + statement_urls = [str(url) for url in extracted.data.statement_urls[:4]] + for index, statement_url in enumerate(statement_urls): + opened = await stagehand.act( + f"Click the Financial Statements link under Q{4 - index}", + page=page, + ) + if not opened.data.success: + encoded_url = json.dumps(statement_url) + await page.evaluate( + f"""(() => {{ + const link = document.createElement('a'); + link.href = {encoded_url}; + link.target = '_blank'; + document.body.appendChild(link); + link.click(); + link.remove(); + }})()""" + ) + await page.wait_for_timeout(500) + print(f"Triggered FY2025 Q{4 - index} download") + + await save_downloads_with_retry(api, session_id) + print("Downloads completed") + finally: + await stagehand.close() + finally: + await browser.close() print("Session closed successfully") - except Exception as error: - print(f"Error during automation: {error}") - raise - if __name__ == "__main__": - # Entry point for script execution - # asyncio.run() creates event loop and runs main() coroutine until completion try: asyncio.run(main()) - except Exception as err: - # Handle any uncaught exceptions and provide helpful debugging information - print(f"Application error: {err}") - print("Common issues:") - print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") - print(" - Verify internet connection and Apple website accessibility") - print(" - Ensure sufficient timeout for slow-loading pages") - print("Docs: https://docs.stagehand.dev/v2/first-steps/introduction") - exit(1) + except Exception as error: + print(f"Application error: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/download-financial-statements/pyproject.toml b/python/download-financial-statements/pyproject.toml new file mode 100644 index 00000000..78200eb8 --- /dev/null +++ b/python/download-financial-statements/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "download-financial-statements" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "browserbase>=1.7.0", + "httpx==0.28.1", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[tool.uv] +package = false diff --git a/python/exa-browserbase/.env.example b/python/exa-browserbase/.env.example new file mode 100644 index 00000000..25c97dd0 --- /dev/null +++ b/python/exa-browserbase/.env.example @@ -0,0 +1,5 @@ +# Browserbase credentials +BROWSERBASE_API_KEY=your-browserbase-api-key + +# Exa API key from https://dashboard.exa.ai/api-keys +EXA_API_KEY=your-exa-api-key diff --git a/python/exa-browserbase/Dummy_CV.pdf b/python/exa-browserbase/Dummy_CV.pdf new file mode 100644 index 00000000..fc455c20 Binary files /dev/null and b/python/exa-browserbase/Dummy_CV.pdf differ diff --git a/python/exa-browserbase/README.md b/python/exa-browserbase/README.md new file mode 100644 index 00000000..2cf54661 --- /dev/null +++ b/python/exa-browserbase/README.md @@ -0,0 +1,51 @@ +# Stagehand + Browserbase + Exa: Review Job Applications + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- **Goal**: Discover live jobs, extract structured role details, and prepare applications for human review. +- **Pattern**: Exa finds direct careers or ATS pages; Stagehand V4 uses `act`, `extract`, and `observe` to inspect and fill them. +- **Safety**: Fills only non-empty supplied applicant values, uploads a test rรฉsumรฉ when requested, and never submits. +- **Plans**: Sequential mode works on all plans. Bounded concurrency is opt-in and requires sufficient Browserbase concurrency. +- Docs โ†’ [Stagehand V4](https://docs.stagehand.dev/v4/first-steps/introduction) | [Stagehand Python](https://docs.stagehand.dev/v4/sdk/python) | [Exa Search](https://docs.exa.ai/reference/search) + +## THE 5-STEP FLOW + +1. **Discover jobs** โ€” one focused Exa search returns a small ranked set of direct company careers or recognized ATS pages. +2. **Inspect a role** โ€” `act()` opens one live role and `extract()` returns its title, requirements, and responsibilities. +3. **Inspect the application** โ€” `act()` opens the form and `observe()` inventories its fields. +4. **Prepare for review** โ€” `act()` fills matching non-empty applicant values; the exact file input uploads the rรฉsumรฉ. +5. **Report, do not submit** โ€” a final `extract()` summarizes the review and the required fields that remain. + +Direct page methods are limited to exact navigation, rรฉsumรฉ upload, and session lifecycle. They do not replace Stagehand's primary interaction primitives. + +## QUICKSTART + +1. `cd exa-browserbase` +2. `uv sync` +3. `cp .env.example .env` +4. Add `BROWSERBASE_API_KEY` and `EXA_API_KEY` to `.env`. +5. Replace the synthetic `APPLICANT` and `Dummy_CV.pdf` with your test data. +6. `uv run python main.py` + +The default run reviews one application sequentially, trying up to three ranked candidates when an earlier live result has no usable form. For a small, repeatable smoke run, set `COMPANY_QUERY=Browserbase NUM_COMPANIES=1`. + +Set `NUM_COMPANIES` to review more candidates. Set `CONCURRENT=true MAX_CONCURRENT_BROWSERS=2` to opt into bounded concurrent sessions. + +## RESULT CONTRACT + +- A run succeeds when it reaches at least one real application and returns a review. +- `fields_attempted` lists the fields whose Stagehand actions reported success; it is intentionally not a claim that every ATS persisted every value. +- `resume_uploaded` reports that the deterministic file-upload command completed without throwing; persistence is left to external E2E validation. +- Null, empty, absent, or ambiguous fields remain outstanding instead of being invented or treated as infrastructure failures. +- The final submit button is never clicked. + +## HELPFUL RESOURCES + +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction +๐Ÿ“š Stagehand Python SDK: https://docs.stagehand.dev/v4/sdk/python +๐Ÿ“š Exa API: https://docs.exa.ai/reference/search +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/exa-browserbase/main.py b/python/exa-browserbase/main.py new file mode 100644 index 00000000..760d65fd --- /dev/null +++ b/python/exa-browserbase/main.py @@ -0,0 +1,499 @@ +"""Review job applications with Exa and direct Stagehand V4 primitives.""" + +from __future__ import annotations + +import asyncio +import os +import re +from pathlib import Path +from urllib.parse import urlparse + +from dotenv import load_dotenv +from exa_py import Exa +from pydantic import BaseModel, Field +from stagehand import FilePayload, Stagehand, browserbase + +load_dotenv() + +APPLICANT = { + "name": "John Doe", + "email": "john.doe@example.com", + "phone": "+1-555-123-4567", + "linkedin": "https://linkedin.com/in/johndoe", + "github": None, + "resume": Path("Dummy_CV.pdf").resolve(), + "current_location": "San Francisco, CA", + "relocation": True, + "sponsorship": False, + "visa_status": "", + "portfolio": "https://johndoe.dev", + "cover_letter": "I am excited to apply for this position.", +} + + +def positive_integer(name: str, fallback: int) -> int: + raw = os.environ.get(name) + if raw is None: + return fallback + try: + value = int(raw) + except ValueError as error: + raise RuntimeError(f"{name} must be a positive integer") from error + if value < 1: + raise RuntimeError(f"{name} must be a positive integer") + return value + + +COMPANY_QUERY = os.environ.get("COMPANY_QUERY", "AI startups in SF currently hiring") +NUM_COMPANIES = positive_integer("NUM_COMPANIES", 1) +CONCURRENT = os.environ.get("CONCURRENT") == "true" +MAX_CONCURRENT_BROWSERS = positive_integer("MAX_CONCURRENT_BROWSERS", 2) + + +class CareersPage(BaseModel): + company: str + careers_url: str + + +class JobHeadline(BaseModel): + company: str = Field(min_length=1) + job_title: str = Field(min_length=1) + + +class JobDescription(BaseModel): + requirements_summary: str + responsibilities_summary: str + + +class RoleSummary(BaseModel): + role_summary: str = Field(min_length=1) + + +class FormReview(BaseModel): + summary: str = Field(min_length=1) + visible_required_fields: list[str] + + +class ApplicationReview(BaseModel): + job_title: str + job_url: str + application_url: str + requirements: list[str] + responsibilities: list[str] + observed_fields: list[str] + fields_attempted: list[str] + resume_uploaded: bool + outstanding_fields: list[str] + summary: str + + +class ApplicationResult(BaseModel): + company: str + careers_url: str + success: bool + review: ApplicationReview | None = None + error: str | None = None + + +def require_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required") + return value + + +def candidate_score(url: str, title: str) -> int: + parsed = urlparse(url) + host = parsed.hostname or "" + searchable = f"{title} {parsed.path} {parsed.query}" + ats = any( + provider in host + for provider in ("ashbyhq.com", "greenhouse.io", "lever.co", "smartrecruiters.com") + ) + direct_role = is_direct_role_url(url) + careers = re.search( + r"\b(careers?|jobs?|open[- ]?roles?|positions?|join[- ]?us)\b", + searchable, + re.IGNORECASE, + ) + return int(ats) * 4 + int(direct_role) * 3 + int(careers is not None) * 2 + + +def is_direct_role_url(value: str) -> bool: + parsed = urlparse(value) + if ( + re.search( + r"/(jobs?|positions?)/[^/]+|ashby_jid=|gh_jid=|lever-origin=", + f"{parsed.path}?{parsed.query}", + re.IGNORECASE, + ) + is not None + ): + return True + path_segments = [segment for segment in parsed.path.split("/") if segment] + return parsed.hostname in {"jobs.ashbyhq.com", "jobs.lever.co"} and len(path_segments) >= 2 + + +async def discover_careers_pages(exa: Exa) -> list[CareersPage]: + search = await asyncio.to_thread( + exa.search_and_contents, + f"{COMPANY_QUERY} official careers jobs open roles", + context=True, + exclude_domains=["linkedin.com", "indeed.com", "glassdoor.com", "ziprecruiter.com"], + livecrawl="fallback", + num_results=max(NUM_COMPANIES * 6, 10), + text=True, + type="deep", + ) + + seen: set[str] = set() + candidates: list[tuple[int, CareersPage]] = [] + for result in search.results: + parsed = urlparse(result.url) + if parsed.scheme not in ("http", "https") or not parsed.hostname: + continue + score = candidate_score(result.url, result.title or "") + identity = f"{parsed.hostname.removeprefix('www.')}{parsed.path}" + if score < 2 or identity in seen: + continue + seen.add(identity) + company = re.split(r"\s+[|โ€“โ€”]\s+", result.title or parsed.hostname, maxsplit=1)[0] + candidates.append((score, CareersPage(company=company, careers_url=result.url))) + + candidates.sort(key=lambda candidate: candidate[0], reverse=True) + candidate_limit = max(NUM_COMPANIES * 3, NUM_COMPANIES) + pages = [candidate[1] for candidate in candidates[:candidate_limit]] + if not pages: + raise RuntimeError("Exa returned no direct careers or ATS pages") + return pages + + +def includes(description: str, pattern: str) -> bool: + return re.search(pattern, description, re.IGNORECASE) is not None + + +async def review_application(careers_page: CareersPage, _index: int) -> ApplicationResult: + browser = await browserbase.launch(api_key=require_env("BROWSERBASE_API_KEY")) + stagehand = await Stagehand.create( + browser=browser, + model="google/gemini-2.5-flash", + ) + + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto( + careers_page.careers_url, + wait_until="domcontentloaded", + timeout=60_000, + ) + + # Exa often returns a role page directly, so a no-op role action is non-fatal. + if not is_direct_role_url(careers_page.careers_url): + try: + await stagehand.act( + "Open the first currently open software, engineering, design, or product role.", + page=page, + ) + except Exception: + pass + page = await browser.context.active_page() or page + job_url = await page.url() + + try: + description = ( + await stagehand.extract( + ( + "Summarize the visible requirements and responsibilities for this role " + "as two plain-text strings. Use an empty string for a section that is " + "not shown." + ), + JobDescription, + page=page, + ) + ).data + except Exception: + description = None + has_description = bool( + description + and any( + value.strip() and value.strip().casefold() != "null" + for value in ( + description.requirements_summary, + description.responsibilities_summary, + ) + ) + ) + if not has_description: + try: + fallback_summary = ( + await stagehand.extract( + ( + "Return one concise plain-text summary of the visible requirements " + "and responsibilities for this role." + ), + RoleSummary, + page=page, + ) + ).data.role_summary + except Exception: + fallback_summary = None + else: + fallback_summary = None + + try: + await stagehand.act( + ( + "Open the application form for this job. Click Apply or Apply for this job, " + "but never submit an application." + ), + page=page, + ) + except Exception: + pass + page = await browser.context.active_page() or page + await page.wait_for_timeout(1_500) + + headline = ( + await stagehand.extract( + "Extract the exact role title and company shown above this application form.", + JobHeadline, + page=page, + ) + ).data + + observed = await stagehand.observe( + ( + "Find every visible application input, textarea, select, radio option, checkbox, " + "and resume or CV file upload. Exclude the final submit button." + ), + page=page, + ) + if not observed.data: + raise RuntimeError("No usable application form was observed") + + fields_attempted: list[str] = [] + descriptions = [action.description for action in observed.data] + + async def run(label: str, pattern: str, value: str | bool | None) -> None: + if value is None or value == "": + return + candidates = [ + action for action in observed.data if includes(action.description, pattern) + ] + if label == "phone": + candidates = [ + action for action in candidates if not includes(action.description, r"country") + ] + if label == "cover letter": + candidates = [ + action + for action in candidates + if not includes(action.description, r"file (upload|input)|attach.*cover") + ] + rendered = ("Yes" if value else "No") if isinstance(value, bool) else value + if isinstance(value, bool): + action = next( + ( + candidate + for candidate in candidates + if rendered.casefold() in candidate.description.casefold() + ), + None, + ) + else: + action = candidates[0] if candidates else None + if action is None: + return + try: + result = await stagehand.act( + action.model_copy( + update={"arguments": [] if action.method == "click" else [rendered]} + ), + page=page, + ) + if result.data.success: + fields_attempted.append(label) + except Exception: + # Optional and custom controls remain for the human reviewer. + pass + + first_name = next( + (action for action in observed.data if includes(action.description, r"first name")), + None, + ) + last_name = next( + (action for action in observed.data if includes(action.description, r"last name")), + None, + ) + if first_name and last_name: + name_parts = str(APPLICANT["name"]).split() + try: + first_result = await stagehand.act( + first_name.model_copy(update={"arguments": [name_parts[0]]}), page=page + ) + last_result = await stagehand.act( + last_name.model_copy(update={"arguments": [" ".join(name_parts[1:])]}), + page=page, + ) + if first_result.data.success and last_result.data.success: + fields_attempted.append("name") + except Exception: + pass + else: + await run("name", r"\b(full )?name\b", str(APPLICANT["name"])) + + await run("email", r"email", str(APPLICANT["email"])) + await run("phone", r"phone|telephone", str(APPLICANT["phone"])) + await run("LinkedIn", r"linkedin", str(APPLICANT["linkedin"])) + await run("GitHub", r"github", APPLICANT["github"]) + await run( + "portfolio", r"portfolio|personal website|\bwebsite\b", str(APPLICANT["portfolio"]) + ) + await run( + "current location", + r"current.*location|currently based|where.*based", + str(APPLICANT["current_location"]), + ) + await run("relocation", r"relocat", bool(APPLICANT["relocation"])) + await run("sponsorship", r"sponsor|work authorization", bool(APPLICANT["sponsorship"])) + await run("visa status", r"visa.*status|status.*visa", str(APPLICANT["visa_status"])) + await run( + "cover letter", + r"cover letter|why.*apply|why.*interested|why.*want.*work|additional information", + ( + f"{APPLICANT['cover_letter']} I am especially interested in the " + f"{headline.job_title} role at {headline.company}." + ), + ) + + resume_uploaded = False + resume_action = next( + ( + action + for action in observed.data + if action.selector + and includes(action.description, r"resume|curriculum|\bcv\b|upload.*file") + and not includes(action.description, r"autofill") + ), + None, + ) + resume_path = APPLICANT["resume"] + if resume_action and isinstance(resume_path, Path): + try: + input_element = page.locator(resume_action.selector) + await input_element.set_input_files( + FilePayload( + name=resume_path.name, + buffer=resume_path.read_bytes(), + mime_type="application/pdf", + ) + ) + resume_uploaded = True + except Exception: + # Upload is exact browser mechanics; failure is reported instead of hidden. + pass + + form_review = ( + await stagehand.extract( + ( + "Summarize this application for human review and list visible required " + "fields that still need attention." + ), + FormReview, + page=page, + ) + ).data + application_url = await page.url() + resolved_job_url = ( + job_url + if is_direct_role_url(job_url) + else re.sub(r"/application/?$", "", application_url) + ) + + return ApplicationResult( + company=headline.company, + careers_url=careers_page.careers_url, + success=True, + review=ApplicationReview( + job_title=headline.job_title, + job_url=resolved_job_url, + application_url=application_url, + requirements=( + [description.requirements_summary.strip()] + if description + and description.requirements_summary.strip() + and description.requirements_summary.strip().casefold() != "null" + else [fallback_summary] + if fallback_summary + else [] + ), + responsibilities=( + [description.responsibilities_summary.strip()] + if description + and description.responsibilities_summary.strip() + and description.responsibilities_summary.strip().casefold() != "null" + else [] + ), + observed_fields=descriptions, + fields_attempted=fields_attempted, + resume_uploaded=resume_uploaded, + outstanding_fields=[ + field + for field in form_review.visible_required_fields + if not (resume_uploaded and includes(field, r"resume|\bcv\b")) + ], + summary=form_review.summary, + ), + ) + except Exception as error: + return ApplicationResult( + company=careers_page.company, + careers_url=careers_page.careers_url, + success=False, + error=str(error) or type(error).__name__, + ) + finally: + try: + await stagehand.close() + except Exception: + pass + try: + await browser.close() + except Exception: + pass + + +async def main() -> None: + require_env("BROWSERBASE_API_KEY") + pages = await discover_careers_pages(Exa(api_key=require_env("EXA_API_KEY"))) + print(f"Found {len(pages)} direct job or careers page(s)") + + if CONCURRENT: + semaphore = asyncio.Semaphore(MAX_CONCURRENT_BROWSERS) + + async def bounded_review(page: CareersPage, index: int) -> ApplicationResult: + async with semaphore: + return await review_application(page, index) + + results = await asyncio.gather( + *(bounded_review(page, index) for index, page in enumerate(pages)) + ) + else: + results = [] + for index, page in enumerate(pages): + results.append(await review_application(page, index)) + if sum(result.success for result in results) >= NUM_COMPANIES: + break + + print("[" + ",\n".join(result.model_dump_json(indent=2) for result in results) + "]") + if not any(result.success for result in results): + raise RuntimeError("No application review reached a usable form") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as error: + print(f"Exa + Browserbase workflow failed: {error}") + raise SystemExit(1) from error diff --git a/python/exa-browserbase/pyproject.toml b/python/exa-browserbase/pyproject.toml new file mode 100644 index 00000000..31b0d731 --- /dev/null +++ b/python/exa-browserbase/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "exa-browserbase" +version = "0.1.0" +description = "AI-Powered Job Search and Application using Exa, Stagehand, and Browserbase" +readme = "README.md" +requires-python = ">=3.11,<3.14" +dependencies = [ + "exa-py", + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[tool.uv] +package = false + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/python/extend-browserbase/.env.example b/python/extend-browserbase/.env.example new file mode 100644 index 00000000..badaef52 --- /dev/null +++ b/python/extend-browserbase/.env.example @@ -0,0 +1,6 @@ +# Browserbase credentials (required) +# Get these from https://www.browserbase.com/settings +BROWSERBASE_API_KEY=your_browserbase_api_key + +# Extend AI (optional โ€“ enables receipt parsing; omit to only download receipts) +EXTEND_API_KEY=your_extend_api_key diff --git a/python/extend-browserbase/README.md b/python/extend-browserbase/README.md new file mode 100644 index 00000000..f64adb5f --- /dev/null +++ b/python/extend-browserbase/README.md @@ -0,0 +1,80 @@ +# Stagehand + Browserbase + Extend: Download Expense Receipts and Parse with Extend AI + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- **Goal**: Automate downloading receipts from an expense portal and extract structured receipt data using AI-powered document parsing. +- **Pattern Template**: Demonstrates the integration pattern of Browserbase (browser automation + download capture) + Extend AI (schema-based document extraction). +- **Workflow**: Stagehand navigates the expense portal and clicks each receipt's download link; Browserbase captures downloads. The script polls for the session's download ZIP, extracts files, then optionally sends them to Extend for structured extraction (vendor, date, totals, line items, etc.). +- **Download Handling**: Implements retry/polling around Browserbase's Session Downloads API until the ZIP is available. +- **Structured Extraction**: Extend AI extraction with inline receipt JSON schema config; results written to `output/results/receipts.json` and `receipts.csv`. +- Docs โ†’ [Browserbase Downloads](https://docs.browserbase.com/features/downloads) | [Extend AI](https://docs.extend.app) + +## GLOSSARY + +- **act**: perform UI actions from natural language prompts (click, scroll, navigate) + Docs โ†’ https://docs.stagehand.dev/v4/basics/act +- **observe**: find and return interactive elements on the page matching a description, without performing actions. Used here to locate all individual download buttons before clicking them. + Docs โ†’ https://docs.stagehand.dev/v4/basics/observe +- **Browserbase Downloads**: When files are downloaded during a browser session, Browserbase captures and stores them. Files are retrieved via the Session Downloads API as a ZIP archive. + Docs โ†’ https://docs.browserbase.com/features/downloads +- **Extend AI extraction**: A configurable document extraction pipeline that parses files against a JSON schema and returns structured data. Config can be passed inline or via a saved extractor resource. + Docs โ†’ https://docs.extend.app +- **Download polling**: Browserbase syncs downloads in real-time; the script retries every 2 seconds until the ZIP is available or a timeout is reached. + +## QUICKSTART + +1. cd python/extend-browserbase +2. cp .env.example .env +3. Add required API keys to .env: + - `BROWSERBASE_API_KEY` + - `EXTEND_API_KEY` (optional โ€” enables receipt parsing) +4. Run the script: + + ```bash + uv run python main.py + ``` + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase and opens the live view link +- Navigates to the expense portal and finds all per-receipt download links via observe +- Clicks each download button; Browserbase captures files +- After closing the session, polls for the session's download ZIP and extracts to `output/documents/` +- If `EXTEND_API_KEY` is set: uploads each file to Extend and runs extraction with inline config, writes `output/results/receipts.json` and `receipts.csv` +- Closes session cleanly + +## COMMON PITFALLS + +- "ModuleNotFoundError": ensure you're running with `uv run python main.py` so dependencies are installed automatically from pyproject.toml +- Missing credentials: verify .env contains BROWSERBASE_API_KEY +- Download timeout: increase `retry_for_seconds` parameter in `save_downloads_with_retry` if downloads take longer than 60 seconds +- Empty ZIP file: ensure downloads were actually triggered (check live view link to debug) +- Rate limiting on Extend: the script retries with exponential backoff on 429 errors, but very large batches may need the batch size reduced from 9 +- Find more information on your Browserbase dashboard โ†’ https://www.browserbase.com/sign-in + +## USE CASES + +โ€ข Expense automation: Download receipts from expense portal and extract vendor, date, totals, and line items for accounting systems. +โ€ข Document batch processing: Collect files from web portals and run structured extraction across all of them with a single script. +โ€ข Receipt digitization: Convert paper/PDF receipts into structured JSON and CSV for import into ERP, bookkeeping, or reimbursement tools. + +## NEXT STEPS + +โ€ข Parameterize the portal URL: Accept the expense portal URL from env or CLI to support different receipt sources. +โ€ข Custom schemas: Modify `RECEIPT_EXTRACTION_CONFIG` to extract different document types (invoices, W-2s, contracts) by changing the JSON schema. +โ€ข Add validation: Compare extracted totals against line item sums to flag discrepancies or incomplete extractions. +โ€ข Scheduled runs: Deploy on cron/Lambda to periodically check for new receipts and process them automatically. + +## HELPFUL RESOURCES + +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction +๐Ÿ“š Python SDK: https://docs.stagehand.dev/v4/sdk/python +๐Ÿ“š Browserbase Downloads: https://docs.browserbase.com/features/downloads +๐Ÿ“š Extend AI: https://docs.extend.app +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/extend-browserbase/main.py b/python/extend-browserbase/main.py new file mode 100644 index 00000000..e6770b9b --- /dev/null +++ b/python/extend-browserbase/main.py @@ -0,0 +1,548 @@ +# Stagehand + Browserbase + Extend: Download Expense Receipts and Parse with Extend AI +# See README.md for full documentation + +import asyncio +import csv +import json +import os +import zipfile +from pathlib import Path + +from browserbase import APIStatusError, Browserbase +from dotenv import load_dotenv +from extend_ai import Extend +from stagehand import Stagehand, browserbase + +# Load environment variables from .env file +# Required: BROWSERBASE_API_KEY +# Optional: EXTEND_API_KEY +load_dotenv() + + +# Receipt extraction config for Extend AI +# Uses extraction_light base extractor with parse_performance engine for low latency +RECEIPT_EXTRACTION_CONFIG = { + "baseProcessor": "extraction_light", + "baseVersion": "3.4.0", + "parseConfig": { + "engine": "parse_performance", + "target": "markdown", + "blockOptions": { + "text": { + "agentic": {"enabled": False}, + "signatureDetectionEnabled": False, + }, + "tables": { + "agentic": {"enabled": False}, + "targetFormat": "markdown", + "cellBlocksEnabled": False, + "tableHeaderContinuationEnabled": False, + }, + "figures": { + "enabled": False, + "figureImageClippingEnabled": False, + }, + }, + "engineVersion": "1.0.1", + "advancedOptions": { + "engine": "parse_performance", + "agenticOcrEnabled": False, + "pageBreaksEnabled": True, + "pageRotationEnabled": False, + "verticalGroupingThreshold": 1, + }, + "chunkingStrategy": {"type": "document"}, + }, + "schema": { + "type": "object", + "required": [ + "vendor_name", + "receipt_date", + "receipt_number", + "total_amount", + "subtotal_amount", + "tax_amount", + "line_items", + "payment_method", + ], + "properties": { + "vendor_name": { + "type": ["string", "null"], + "description": "The name of the merchant or vendor on the receipt.", + }, + "receipt_date": { + "type": ["string", "null"], + "description": "The date of the transaction shown on the receipt.", + "extend:type": "date", + }, + "receipt_number": { + "type": ["string", "null"], + "description": "The receipt or transaction number, if present.", + }, + "total_amount": { + "type": "object", + "required": ["amount", "iso_4217_currency_code"], + "properties": { + "amount": {"type": ["number", "null"]}, + "iso_4217_currency_code": {"type": ["string", "null"]}, + }, + "description": "The total amount paid on the receipt.", + "extend:type": "currency", + "additionalProperties": False, + }, + "subtotal_amount": { + "type": "object", + "required": ["amount", "iso_4217_currency_code"], + "properties": { + "amount": {"type": ["number", "null"]}, + "iso_4217_currency_code": {"type": ["string", "null"]}, + }, + "description": "The subtotal before tax, if shown.", + "extend:type": "currency", + "additionalProperties": False, + }, + "tax_amount": { + "type": "object", + "required": ["amount", "iso_4217_currency_code"], + "properties": { + "amount": {"type": ["number", "null"]}, + "iso_4217_currency_code": {"type": ["string", "null"]}, + }, + "description": "The tax amount on the receipt.", + "extend:type": "currency", + "additionalProperties": False, + }, + "line_items": { + "type": "array", + "items": { + "type": "object", + "required": ["description", "quantity", "unit_price", "amount"], + "properties": { + "description": { + "type": ["string", "null"], + "description": "Description of the item purchased.", + }, + "quantity": { + "type": ["number", "null"], + "description": "Quantity of the item, if shown.", + }, + "unit_price": { + "type": ["number", "null"], + "description": "Price per unit, if shown.", + }, + "amount": { + "type": ["number", "null"], + "description": "Total amount for this line item.", + }, + }, + "additionalProperties": False, + }, + "description": "Individual items on the receipt.", + }, + "payment_method": { + "type": ["string", "null"], + "description": "The payment method used (e.g., cash, credit card, etc.).", + }, + }, + "additionalProperties": False, + }, + "advancedOptions": { + "advancedMultimodalEnabled": False, + "citationsEnabled": True, + "arrayCitationStrategy": "item", + "pageRanges": [], + "chunkingOptions": {}, + "advancedFigureParsingEnabled": True, + }, +} + + +# Polls Browserbase API for completed downloads with retry logic +async def save_downloads_with_retry( + bb: Browserbase, session_id: str, retry_for_seconds: int = 60 +) -> int: + """ + Polls Browserbase API for downloads with timeout handling. + + Browserbase stores downloaded files during a session and makes them available + via API. Files may take a few seconds to process, so this function implements + retry logic to wait for downloads to be ready before retrieving them. + + Args: + bb: Browserbase client instance for API calls + session_id: The Browserbase session ID to retrieve downloads from + retry_for_seconds: Maximum time to wait for downloads (default: 60 seconds) + + Returns: + int: The size of the downloaded ZIP file in bytes + + Raises: + TimeoutError: If downloads aren't ready within the specified timeout + """ + print(f"Waiting up to {retry_for_seconds} seconds for downloads to complete...") + + # Track elapsed time to implement timeout without using threading timers + start_time = asyncio.get_event_loop().time() + timeout = retry_for_seconds + + while True: + elapsed = asyncio.get_event_loop().time() - start_time + + # Check if we've exceeded the timeout period + if elapsed >= timeout: + raise TimeoutError("Download timeout exceeded") + + try: + print("Checking for downloads...") + # Fetch downloads from Browserbase API and save to disk when ready + # Use asyncio.to_thread for synchronous Browserbase SDK calls + # This prevents blocking the event loop while waiting for API responses + response = await asyncio.to_thread(bb.sessions.downloads.list, session_id) + download_buffer = await asyncio.to_thread(response.read) + + # Save downloads to disk when file size indicates content is available + # Empty zip files are ~22 bytes, so require at least 100 bytes for real content + if len(download_buffer) > 100: + print(f"Downloads ready! File size: {len(download_buffer)} bytes") + # Save the ZIP file containing all downloaded receipts to disk + with open("downloaded_files.zip", "wb") as f: + f.write(download_buffer) + print("Files saved as: downloaded_files.zip") + return len(download_buffer) + else: + print("Downloads not ready yet, retrying...") + except APIStatusError as e: + # Handle 404 (session not found) gracefully + if e.status_code == 404: + print("Session not found, returning empty result") + return 0 + print(f"Error fetching downloads: {e}") + raise + except Exception as e: + # HTML error response - session may not be ready yet, keep retrying + error_message = str(e) + if "Unexpected token '<'" in error_message or " list[str]: + """ + Extract receipt files from a ZIP archive. + + Args: + zip_path: Path to the ZIP file containing receipts + output_dir: Directory to extract files to (default: "output/documents") + + Returns: + list[str]: Paths to all extracted files + + Raises: + ValueError: If no files are found in the ZIP + """ + print(f"Extracting files from {zip_path}...") + + # Create output directories for documents and results if they don't exist + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + Path("output/results").mkdir(parents=True, exist_ok=True) + + extracted_files: list[str] = [] + + with zipfile.ZipFile(zip_path, "r") as zip_ref: + # Open zip file and iterate over entries + entries = [e for e in zip_ref.namelist() if not e.endswith("/")] + + if len(entries) == 0: + raise ValueError("No files found in the downloaded zip") + + # Extract all non-directory entries and collect file paths + resolved_output = output_path.resolve() + for entry in entries: + extracted_path = (resolved_output / entry).resolve() + if resolved_output not in extracted_path.parents: + raise ValueError(f"Unsafe ZIP entry: {entry}") + extracted_path.parent.mkdir(parents=True, exist_ok=True) + with zip_ref.open(entry) as source, extracted_path.open("wb") as target: + target.write(source.read()) + print(f"Extracted: {extracted_path}") + extracted_files.append(str(extracted_path)) + + print(f"\nTotal files extracted: {len(extracted_files)}") + return extracted_files + + +# Uploads receipt files to Extend AI, runs extraction, and saves results as JSON and CSV +async def parse_receipts_with_extend(file_paths: list[str]) -> list[dict]: + """ + Upload receipt files to Extend AI, run extraction, and save results. + + Initializes the Extend client, uploads each file, runs synchronous extraction + using inline config (no need to pre-create an extractor resource), and saves + results as JSON and CSV. + + Args: + file_paths: List of file paths to receipt documents + """ + # Skip parsing if Extend API key is not configured + extend_api_key = os.environ.get("EXTEND_API_KEY") + if not extend_api_key or extend_api_key == "YOUR_EXTEND_API_KEY_HERE": + print("\nWARNING: EXTEND_API_KEY not configured. Skipping receipt parsing.") + print(" Add your Extend API key to .env to enable automatic receipt parsing.") + return [] + + print("\n=== Parsing Receipts with Extend AI ===\n") + + # Initialize Extend AI client + extend_client = Extend(token=extend_api_key) + + print(f"Processing {len(file_paths)} receipts with inline config...\n") + + # Process all files with retry on rate limiting (429 errors) + results: list[dict] = [] + + # Uploads a single file to Extend and runs extraction with exponential backoff retry + async def process_with_retry(file_path: str, max_retries: int = 3) -> dict: + file_name = Path(file_path).name + + for attempt in range(1, max_retries + 1): + try: + # Upload the file to Extend (multipart form upload) + # Pass as (filename, bytes) tuple so Extend knows the file name + with open(file_path, "rb") as f: + file_bytes = f.read() + upload_response = await asyncio.to_thread( + extend_client.files.upload, + file=(file_name, file_bytes), + ) + file_id = upload_response.id + + # Run extraction using inline config โ€” no need to pre-create an extractor + result = await asyncio.to_thread( + extend_client.extract, + config=RECEIPT_EXTRACTION_CONFIG, + file={"id": file_id}, + ) + + run_id = result.id + print(f" Parsed {file_name} (run: {run_id})") + + # Convert to dict for JSON serialization + data = result + if hasattr(result, "to_dict"): + data = result.to_dict() + elif hasattr(result, "dict"): + data = result.dict() + + return { + "file": file_name, + "runId": run_id, + "data": data, + } + except Exception as error: + error_msg = str(error) + is_retryable = ( + "429" in error_msg or "rate" in error_msg or "disturbed or locked" in error_msg + ) + + if is_retryable and attempt < max_retries: + delay = 2**attempt # Exponential backoff: 2s, 4s, 8s + print( + f" Rate limited on {file_name}, retrying in {delay}s " + f"(attempt {attempt}/{max_retries})" + ) + await asyncio.sleep(delay) + else: + print(f" Failed to parse {file_name}: {error_msg}") + return {"file": file_name, "data": {"error": error_msg}} + + return {"file": file_name, "data": {"error": "Max retries exceeded"}} + + # Process in batches of 9 to balance speed and reliability + for i in range(0, len(file_paths), 9): + batch = file_paths[i : i + 9] + batch_results = await asyncio.gather(*[process_with_retry(fp) for fp in batch]) + results.extend(batch_results) + + # Save results to JSON + json_path = "output/results/receipts.json" + with open(json_path, "w") as f: + json.dump(results, f, indent=2, default=str) + print(f"\nSaved JSON: {json_path}") + + # Convert results to CSV for easy viewing in spreadsheet tools + csv_path = "output/results/receipts.csv" + with open(csv_path, "w", newline="") as f: + writer = csv.writer(f, quoting=csv.QUOTE_ALL) + writer.writerow( + [ + "file", + "vendor_name", + "receipt_date", + "receipt_number", + "total_amount", + "currency", + "subtotal", + "tax", + "payment_method", + "line_items_count", + ] + ) + + # Build CSV rows from extraction results + for result in results: + data = result.get("data", {}) + output = {} + if isinstance(data, dict): + output = data.get("output", {}).get("value", {}) or {} + + writer.writerow( + [ + result.get("file", ""), + output.get("vendor_name", ""), + output.get("receipt_date", ""), + output.get("receipt_number", ""), + (output.get("total_amount") or {}).get("amount", ""), + (output.get("total_amount") or {}).get("iso_4217_currency_code", ""), + (output.get("subtotal_amount") or {}).get("amount", ""), + (output.get("tax_amount") or {}).get("amount", ""), + output.get("payment_method", ""), + len(output.get("line_items", [])) + if isinstance(output.get("line_items"), list) + else 0, + ] + ) + + print(f"Saved CSV: {csv_path}") + return results + + +async def main() -> None: + """ + Main application entry point. + + Orchestrates the entire receipt download and extraction automation process: + 1. Initializes Browserbase and Stagehand clients + 2. Navigates to the expense portal + 3. Finds and clicks all individual receipt download buttons + 4. Retrieves downloads from Browserbase and extracts files + 5. Optionally parses receipts with Extend AI for structured data extraction + """ + print("Starting Expense Receipt Downloader...\n") + + browserbase_api_key = os.environ.get("BROWSERBASE_API_KEY") + if not browserbase_api_key: + raise ValueError("BROWSERBASE_API_KEY is required") + + # Initialize Browserbase SDK for session management and download retrieval + bb = Browserbase(api_key=browserbase_api_key) + + browser = await browserbase.launch(api_key=browserbase_api_key) + session_id = browser.session_id + if not session_id: + await browser.close() + raise RuntimeError("Browserbase launch did not return a session ID") + + try: + stagehand = await Stagehand.create( + browser=browser, + ) + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + print("Live View is available in the Browserbase Sessions dashboard") + + # Navigate to the expense portal where receipts are hosted + print("\nNavigating to expense portal...") + await page.goto( + "https://v0-reimburse-me-expense-portal.vercel.app/", + wait_until="domcontentloaded", + timeout=60_000, + ) + + # Use observe to find all individual download buttons (not the Download All button) + print("\nFinding all individual download buttons...") + observe_response = await stagehand.observe( + "Find all the small Download links on individual receipt cards.", + page=page, + ) + download_buttons = observe_response.data + if not download_buttons: + raise RuntimeError("No receipt download links were found") + + # Click each download button using observe -> act pattern + # Pass the observed action directly to act for precise element targeting + success_count = 0 + for i, action in enumerate(download_buttons): + print(f"Downloading receipt {i + 1}/{len(download_buttons)}...") + + try: + await stagehand.act(action, page=page) + success_count += 1 + except Exception: + # If click fails, scroll element into view and retry + print(f" Could not click download button {i + 1}, trying to scroll and retry...") + try: + await stagehand.act("Scroll down slightly", page=page) + await stagehand.act(action, page=page) + success_count += 1 + except Exception: + print(f" Skipping receipt {i + 1}") + + # Scroll down periodically to ensure elements are in view + if (i + 1) % 4 == 0 and (i + 1) < len(download_buttons): + await stagehand.act("Scroll down slightly", page=page) + + print(f"\nDownload clicks completed! ({success_count}/{len(download_buttons)} successful)") + + await stagehand.close() + await browser.close() + print("Session closed successfully") + + # Wait for session to finalize downloads before polling + await asyncio.sleep(2) + + # Retrieve all downloads triggered during this session from Browserbase API + print("\nRetrieving downloads from Browserbase...") + download_size = await save_downloads_with_retry(bb, session_id, 60) + + if download_size > 0: + # Extract receipt files from downloaded zip archive + extracted_files = extract_files_from_zip("downloaded_files.zip") + + print("\n=== Download Summary ===") + print(f"Total files downloaded: {len(extracted_files)}") + print("Files saved to: ./output/documents/") + + # Parse downloaded receipts with Extend AI for structured data extraction + await parse_receipts_with_extend(extracted_files) + else: + print("No downloads were captured") + + print("\nExpense receipt download complete!") + + except Exception as error: + print(f"Error during automation: {error}") + if "stagehand" in locals(): + await stagehand.close() + await browser.close() + raise + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Application error: {err}") + print("Common issues:") + print(" - Check .env file has BROWSERBASE_API_KEY") + print(" - Add EXTEND_API_KEY to .env to enable receipt parsing with Extend AI") + print(" - Verify internet connection and expense portal accessibility") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + exit(1) diff --git a/python/extend-browserbase/pyproject.toml b/python/extend-browserbase/pyproject.toml new file mode 100644 index 00000000..85810c07 --- /dev/null +++ b/python/extend-browserbase/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "extend-browserbase" +version = "0.1.0" +description = "Download expense receipts and parse with Extend AI using Stagehand and Browserbase" +readme = "README.md" +requires-python = ">=3.11,<3.14" +dependencies = [ + "browserbase>=1.7.0", + "extend-ai>=1.0.0", + "python-dotenv>=1.2.1", + "stagehand==4.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/python/form-filling/.env.example b/python/form-filling/.env.example new file mode 100644 index 00000000..d0cde042 --- /dev/null +++ b/python/form-filling/.env.example @@ -0,0 +1 @@ +BROWSERBASE_API_KEY= diff --git a/python/form-filling/README.md b/python/form-filling/README.md index 5ac84aa7..a0fe3d05 100644 --- a/python/form-filling/README.md +++ b/python/form-filling/README.md @@ -1,37 +1,34 @@ # Stagehand + Browserbase: Form Filling Automation +Stagehand is the SDK for browser agents. + ## AT A GLANCE - Goal: showcase how to automate form filling with Stagehand and Browserbase. - Smart Form Automation: dynamically fill contact forms with variable-driven data. -- Field Detection: analyze page structure with `observe` before interacting with fields. -- AI-Powered Interaction: leverage Stagehand to map inputs to the right fields reliably. - Docs โ†’ https://docs.browserbase.com/features/sessions +- AI-Powered Interaction: use `act()` to map each labeled input to the right field reliably. +- Variable-driven actions: fill each form control with the supplied sample values. + Docs โ†’ https://docs.browserbase.com/fundamentals/create-browser-session ## GLOSSARY - act: perform UI actions from a prompt (type, click, fill forms) - Docs โ†’ https://docs.stagehand.dev/basics/act -- observe: analyze a page and return selectors or action plans before executing - Docs โ†’ https://docs.stagehand.dev/basics/observe + Docs โ†’ https://docs.stagehand.dev/v4/basics/act - variable substitution: inject dynamic values into actions using `%variable%` syntax ## QUICKSTART -1. cd form-fill-template -2. uv venv venv -3. source venv/bin/activate # On Windows: venv\Scripts\activate -4. pip install -r requirements.txt -5. cp .env.example .env # Add your Browserbase API key and Project ID to .env -6. python main.py +1. cd form-filling +2. uv venv && source .venv/bin/activate # On Windows: .venv\Scripts\activate +3. pip install stagehand python-dotenv +4. cp .env.example .env # Add your Browserbase API key to .env +5. python main.py ## EXPECTED OUTPUT - Initializes Stagehand session with Browserbase - Navigates to contact form page -- Analyzes available form fields using observe -- Fills form with sample data using variable substitution -- Displays session recording link for monitoring +- Fills form with sample data using `act()` and variable substitution - Closes session cleanly ## COMMON PITFALLS @@ -57,7 +54,7 @@ ## HELPFUL RESOURCES -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction ๐ŸŽฎ Browserbase: https://www.browserbase.com ๐Ÿ’ก Try it out: https://www.browserbase.com/playground ๐Ÿ”ง Templates: https://www.browserbase.com/templates diff --git a/python/form-filling/main.py b/python/form-filling/main.py index 2fd7d9e5..808a7830 100644 --- a/python/form-filling/main.py +++ b/python/form-filling/main.py @@ -1,104 +1,82 @@ -# Stagehand + Browserbase: Form Filling Automation - See README.md for full documentation +"""Fill Browserbase's contact form with Stagehand V4.""" import asyncio import os from dotenv import load_dotenv -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand, browserbase -# Load environment variables load_dotenv() -# Form data variables - using random/fake data for testing -# Set your own variables below to customize the form submission -first_name = "Alex" -last_name = "Johnson" -company = "TechCorp Solutions" -job_title = "Software Developer" -email = "alex.johnson@techcorp.com" -message = ( - "Hello, I'm interested in learning more about your services and would like to schedule a demo." -) +FORM_FIELDS = { + "firstName": "Alex", + "lastName": "Johnson", + "companyName": "TechCorp Solutions", + "jobTitle": "Software Developer", + "email": "alex.johnson@techcorp.com", + "project": ( + "Hello, I'm interested in learning more about your services and would " + "like to schedule a demo." + ), +} -async def main(): - print("Starting Form Filling Example...") - - # Initialize Stagehand with Browserbase for cloud-based browser automation. - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - }, - verbose=1, # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - ) +async def main() -> None: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") + print("Starting Form Filling Example...") + browser = await browserbase.launch(api_key=api_key) try: - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - print("Stagehand initialized successfully!") - if hasattr(stagehand, "session_id"): - print(f"Live View Link: https://browserbase.com/sessions/{stagehand.session_id}") - elif hasattr(stagehand, "browserbase_session_id"): - print( - f"Live View Link: https://browserbase.com/sessions/{stagehand.browserbase_session_id}" - ) + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() - page = stagehand.page - - # Navigate to contact page with extended timeout for slow-loading sites. print("Navigating to Browserbase contact page...") await page.goto( "https://www.browserbase.com/contact", - wait_until="domcontentloaded", # Wait for DOM to be ready before proceeding. - timeout=60000, # Extended timeout for reliable page loading. + wait_until="domcontentloaded", + timeout=60_000, ) + await page.wait_for_timeout(1_500) + + field_prompts = { + "firstName": "first name", + "lastName": "last name", + "companyName": "company", + "jobTitle": "job title", + "email": "work email", + "project": "project description or message", + } + for name, label in field_prompts.items(): + await stagehand.act( + f"Fill the {label} field with %value%", + page=page, + variables={"value": FORM_FIELDS[name]}, + ) - # Fill form using individual act() calls for reliability - print("Filling in contact form...") - - # Fill each field individually for better reliability - await page.act(f'Fill in the first name field with "{first_name}"') - await page.act(f'Fill in the last name field with "{last_name}"') - await page.act(f'Fill in the company field with "{company}"') - await page.act(f'Fill in the job title field with "{job_title}"') - await page.act(f'Fill in the email field with "{email}"') - await page.act(f'Fill in the message field with "{message}"') - - # Language choice in Stagehand act() is crucial for reliable automation. - # Use "click" for dropdown interactions rather than "select" - await page.act("Click on the How Can we help? dropdown") - await page.wait_for_timeout(500) - await page.act("Click on the first option from the dropdown") - # await page.act("Select the first option from the dropdown") # Less reliable than "click" - - # Uncomment the line below if you want to submit the form - # await page.act("Click the submit button") - - print("Form filled successfully! Waiting 3 seconds...") - await page.wait_for_timeout(30000) + await stagehand.act("Click the How Can We Help dropdown", page=page) + await stagehand.act("Click the demo option in the open dropdown", page=page) + # Uncomment to submit the form: + # await stagehand.act("Click the submit button", page=page) + print("Form filled successfully") + finally: + await stagehand.close() + finally: + await browser.close() print("Session closed successfully") - except Exception as error: - print(f"Error during form filling: {error}") - raise - if __name__ == "__main__": try: asyncio.run(main()) - except Exception as err: - print(f"Error in form filling example: {err}") - print("Common issues:") - print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") - print(" - Ensure form fields are available on the contact page") - print("Docs: https://docs.stagehand.dev/v3/first-steps/introduction") - exit(1) + except Exception as error: + print(f"Error in form filling example: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/form-filling/pyproject.toml b/python/form-filling/pyproject.toml new file mode 100644 index 00000000..4c30b973 --- /dev/null +++ b/python/form-filling/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = "form-filling" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = ["python-dotenv==1.2.2", "stagehand==4.0.0"] + +[tool.uv] +package = false diff --git a/python/gemini-cua/.env.example b/python/gemini-cua/.env.example new file mode 100644 index 00000000..b127fde8 --- /dev/null +++ b/python/gemini-cua/.env.example @@ -0,0 +1,3 @@ +BROWSERBASE_API_KEY= +AI_GATEWAY_API_KEY= +# DEEPAGENTS_MODEL=google/gemini-3-flash-preview diff --git a/python/gemini-cua/README.md b/python/gemini-cua/README.md index f2aa6971..4ca7fba9 100644 --- a/python/gemini-cua/README.md +++ b/python/gemini-cua/README.md @@ -1,57 +1,47 @@ -# Stagehand + Browserbase: Computer Use Agent (CUA) Example +# Gemini browser agent with Stagehand V4 -## AT A GLANCE +Stagehand is the SDK for browser agents. -- Goal: demonstrate autonomous web browsing using Google's Computer Use Agent with Stagehand and Browserbase. -- Uses Stagehand Agent to automate complex workflows with AI powered browser agents -- Leverages Google's gemini-2.5-computer-use-preview model for autonomous web interaction and decision-making. +This template pairs a bring-your-own Gemini model with LangChain Deep Agents and Stagehand V4 code +mode. The agent researches the next two visible solar eclipses in North America, cross-checks live +NASA and Timeanddate sources, and cites only URLs it opened in the browser. -## GLOSSARY +## How it works -- agent: create an autonomous AI agent that can execute complex multi-step tasks - Docs โ†’ https://docs.stagehand.dev/basics/agent#what-is-agent +- `create_deep_agent` owns the Gemini reasoning and tool loop. +- Stagehand code mode exposes one persistent Browserbase session through `run`, `snapshot`, and + `screenshot` MCP tools. +- Vercel AI Gateway provides the Gemini model through its OpenAI-compatible endpoint. +- Closing the MCP session shuts down the Stagehand client and Browserbase browser. -## QUICKSTART +## Quickstart -1. uv venv venv -2. source venv/bin/activate # On Windows: venv\Scripts\activate -3. pip install -r requirements.txt -4. cp .env.example .env # Add your Browserbase API key, Project ID, and Google API key to .env -5. python main.py +Requirements: Python 3.11โ€“3.13 and [uv](https://docs.astral.sh/uv/). -## EXPECTED OUTPUT +```bash +cp .env.example .env +# Add BROWSERBASE_API_KEY and AI_GATEWAY_API_KEY to .env. +uv sync +uv run python main.py +``` -- Initializes Stagehand session with Browserbase -- Navigates to Google search engine -- Executes autonomous search and data extraction task -- Displays live session link for monitoring -- Returns structured results or completion status -- Closes session cleanly +The first run installs the pinned Stagehand Deep Agents integration in `uvx`; the integration pins +its Stagehand server dependency to `stagehand==4.0.0`. -## COMMON PITFALLS +## Expected outcome -- "ModuleNotFoundError": ensure all dependencies are installed via pip -- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY -- Google API access: ensure you have access to Google's gemini-2.5-computer-use-preview model -- Import errors: activate your virtual environment if you created one +The agent returns the expected dates of the next two visible solar eclipses in North America and +the live source URLs it opened. The script exits nonzero only when the agent workflow itself fails. -## USE CASES +## Configuration -โ€ข Autonomous research: Let AI agents independently research topics, gather information, and compile reports without manual intervention. -โ€ข Complex web workflows: Automate multi-step processes that require decision-making, form filling, and data extraction across multiple pages. -โ€ข Content discovery: Search for specific information, verify data accuracy, and cross-reference sources autonomously. +- `BROWSERBASE_API_KEY`: launches the Browserbase session. +- `AI_GATEWAY_API_KEY`: authenticates Gemini through Vercel AI Gateway. +- `DEEPAGENTS_MODEL`: optional model override; defaults to `google/gemini-3-flash-preview`. +- `STAGEHAND_RUN_TIMEOUT_MS`: optional browser-tool timeout; defaults to 120 seconds. -## NEXT STEPS +## Resources -โ€ข Customize instructions: Modify the instruction variable to test different autonomous tasks and scenarios. -โ€ข Add error handling: Implement retry logic, fallback strategies, and better error recovery for failed agent actions. -โ€ข Extend capabilities: Add support for file downloads, form submissions, and more complex interaction patterns. - -## HELPFUL RESOURCES - -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction -๐ŸŽฎ Browserbase: https://www.browserbase.com -๐Ÿ’ก Try it out: https://www.browserbase.com/playground -๐Ÿ”ง Templates: https://www.browserbase.com/templates -๐Ÿ“ง Need help? support@browserbase.com -๐Ÿ’ฌ Discord: http://stagehand.dev/discord +- [Stagehand V4 documentation](https://docs.stagehand.dev/v4) +- [Stagehand Deep Agents integration](https://github.com/browserbase/stagehand/tree/main/packages/integrations/deepagents) +- [Vercel AI Gateway Python integration](https://vercel.com/docs/ai-gateway/sdks-and-apis/python) diff --git a/python/gemini-cua/agent_runtime.py b/python/gemini-cua/agent_runtime.py new file mode 100644 index 00000000..ba1ff629 --- /dev/null +++ b/python/gemini-cua/agent_runtime.py @@ -0,0 +1,66 @@ +"""Shared Deep Agents + Stagehand V4 code-mode setup for this template.""" + +from __future__ import annotations + +import os + +from langchain_mcp_adapters.client import MultiServerMCPClient +from langchain_openai import ChatOpenAI +from pydantic import SecretStr + +SERVER_NAME = "stagehand_browser" + +STAGEHAND_DEEPAGENTS_SOURCE = ( + "git+https://github.com/browserbase/stagehand.git@main" + "#subdirectory=packages/integrations/deepagents" +) + +BROWSER_INSTRUCTIONS = """You control one persistent Browserbase browser through exactly three +Stagehand code-mode tools: +- snapshot: inspect the active page and hydrate bracketed element IDs. +- run: execute snapshot actions or JavaScript with the Playwright-shaped page API. +- screenshot: inspect the rendered page visually. + +Use run with deterministic page APIs for known URLs and structured pages. Use snapshot before +interacting with an unfamiliar UI. Snapshot IDs are valid only for the latest snapshot. Do not +launch another browser, and never cite a URL unless you opened it in the browser. +""" + + +def require_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required") + return value + + +def create_gateway_model(default_model: str) -> ChatOpenAI: + return ChatOpenAI( + model=os.environ.get("DEEPAGENTS_MODEL", default_model), + api_key=SecretStr(require_env("AI_GATEWAY_API_KEY")), + base_url="https://ai-gateway.vercel.sh/v1", + ) + + +def create_stagehand_client() -> MultiServerMCPClient: + server_env = { + "BROWSERBASE_API_KEY": require_env("BROWSERBASE_API_KEY"), + "STAGEHAND_BROWSER": "browserbase", + "STAGEHAND_RUN_TIMEOUT_MS": os.environ.get("STAGEHAND_RUN_TIMEOUT_MS", "120000"), + } + return MultiServerMCPClient( + { + SERVER_NAME: { + "transport": "stdio", + "command": os.environ.get("UVX_COMMAND", "uvx"), + "args": [ + "--from", + STAGEHAND_DEEPAGENTS_SOURCE, + "--with", + "stagehand==4.0.0", + "stagehand-deepagents-mcp", + ], + "env": server_env, + } + } + ) diff --git a/python/gemini-cua/main.py b/python/gemini-cua/main.py index b5491bac..792fe5ee 100644 --- a/python/gemini-cua/main.py +++ b/python/gemini-cua/main.py @@ -1,112 +1,74 @@ -# Stagehand + Browserbase: Computer Use Agent (CUA) Example - See README.md for full documentation +"""Run a Gemini browser-research agent with Deep Agents and Stagehand V4 code mode.""" -import asyncio -import os +from __future__ import annotations +import asyncio +from datetime import date + +from agent_runtime import ( + BROWSER_INSTRUCTIONS, + SERVER_NAME, + create_gateway_model, + create_stagehand_client, +) +from deepagents import create_deep_agent from dotenv import load_dotenv +from langchain_mcp_adapters.tools import load_mcp_tools -from stagehand import Stagehand, StagehandConfig - -# Load environment variables load_dotenv() -# ============================================================================ -# EXAMPLE INSTRUCTIONS - Choose one to test different scenarios -# ============================================================================ - -# Example 1: Learning Plan Creation -# instruction = """I want to learn more about Sourdough Bread Making. It's my first time learning about it, and want to get a good grasp by investing 1 hour a day for the next 2 months. Go find online courses/resources, create a plan cross-referencing the time I want to invest with the modules/timelines of the courses and return the plan""" - -# Example 2: Flight Search -# instruction = """Use flights.google.com to find the lowest fare from all eligible one-way flights for 1 adult from JFK to Heathrow in the next 30 days.""" - -# Example 3: Solar Eclipse Research -instruction = """Search for the next visible solar eclipse in North America and its expected date, and what about the one after that.""" -# Example 4: GitHub PR Verification -# instruction = """Find the most recently opened non-draft PR on Github for Browserbase's Stagehand project and make sure the combination-evals in the PR validation passed.""" - -# ============================================================================ - - -async def main(): - print("Starting Computer Use Agent Example...") - - # Initialize Stagehand with Browserbase for cloud-based browser automation. - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_api_key=os.environ.get( - "GOOGLE_API_KEY" - ), # this is the model stagehand uses in act, observe, extract (not agent) - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - "proxies": True, # Using proxies will give the agent a better chance of success - requires Developer Plan or higher, comment out if you don't have access - "region": "us-west-2", - "browser_settings": {"block_ads": True, "viewport": {"width": 1288, "height": 711}}, - }, - verbose=1, # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging +def message_text(message: object) -> str: + content = getattr(message, "content", "") + if isinstance(content, str): + return content + if isinstance(content, list): + return "\n".join( + block.get("text", "") + for block in content + if isinstance(block, dict) and isinstance(block.get("text"), str) + ) + return str(content) + + +async def main() -> None: + today = date.today().isoformat() + instruction = ( + f"As of {today}, use these two live sources to find the next visible solar eclipse in " + "North America and its expected date, then the one after that: " + "https://eclipse.gsfc.nasa.gov/solar.html and " + "https://www.timeanddate.com/eclipse/list-solar.html?region=north-america. " + "Open and cross-check both sources, then cite each URL." ) - - try: - async with Stagehand(config) as stagehand: - print("Stagehand initialized successfully!") - print(f"Live View Link: https://browserbase.com/sessions/{stagehand.session_id}") - - page = stagehand.page - - # Navigate to search engine with extended timeout for slow-loading sites. - print("Navigating to Google search...") - await page.goto( - "https://www.google.com/", - wait_until="domcontentloaded", - timeout=60000, # Extended timeout for reliable page loading - ) - - # Create agent with computer use capabilities for autonomous web browsing. - print("Creating Computer Use Agent...") - agent = stagehand.agent( - provider="google", - model="gemini-2.5-computer-use-preview-10-2025", - instructions=f"""You are a helpful assistant that can use a web browser. - You are currently on the following page: {page.url}. - Do not ask follow up questions, the user will trust your judgement. If you are getting blocked on google, try another search engine.""", - options={ - "api_key": os.getenv("GOOGLE_API_KEY"), - }, - ) - - # Execute the autonomous task with the Computer Use Agent - print("Executing instruction:", instruction) - result = await agent.execute( - instruction=instruction, - max_steps=30, # The maximum number of steps the agent can take to complete the task - auto_screenshot=True, - ) - - if result.success == True: - print("Task completed successfully!") - print("Result:", result) - else: - print("Task failed or was incomplete") - - print("Session closed successfully") - - except Exception as error: - print(f"Error executing computer use agent: {error}") - raise + print("Executing instruction:", instruction) + + client = create_stagehand_client() + async with client.session(SERVER_NAME) as session: + tools = await load_mcp_tools(session) + agent = create_deep_agent( + model=create_gateway_model("google/gemini-3-flash-preview"), + tools=tools, + system_prompt=( + BROWSER_INSTRUCTIONS + + "\nUse no more than six browser-tool calls. Prefer deterministic browser APIs, " + "cross-check at least two reliable sources, and return an evidence-backed answer " + "after finding two future eclipse dates." + ), + ) + result = await agent.ainvoke( + {"messages": [{"role": "user", "content": instruction}]}, + config={"recursion_limit": 25}, + ) + answer = message_text(result["messages"][-1]).strip() + + print(answer) + print("Stagehand code-mode session closed successfully") if __name__ == "__main__": try: asyncio.run(main()) - except Exception as err: - print(f"Error in computer use agent example: {err}") - print("Common issues:") - print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") - print(" - Verify GOOGLE_API_KEY is set for the agent") - print("Docs: https://docs.stagehand.dev/v3/first-steps/introduction") - exit(1) + except Exception as error: + print(f"Error in Gemini browser agent example: {error}") + print("Check BROWSERBASE_API_KEY and AI_GATEWAY_API_KEY in .env") + raise SystemExit(1) from error diff --git a/python/gemini-cua/pyproject.toml b/python/gemini-cua/pyproject.toml new file mode 100644 index 00000000..5f4fcb4e --- /dev/null +++ b/python/gemini-cua/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "gemini-cua" +version = "0.1.0" +description = "Run a Gemini agent with Deep Agents and Stagehand V4 code mode" +readme = "README.md" +requires-python = ">=3.11,<3.14" +dependencies = [ + "deepagents==0.7.5", + "langchain-mcp-adapters==0.3.2", + "langchain-openai==1.4.3", + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", +] + +[tool.uv] +package = false diff --git a/python/getting-started-with-browserbase/.env.example b/python/getting-started-with-browserbase/.env.example new file mode 100644 index 00000000..8c897724 --- /dev/null +++ b/python/getting-started-with-browserbase/.env.example @@ -0,0 +1,6 @@ +# Browserbase Configuration +# Get your API key from: https://www.browserbase.com/settings +BROWSERBASE_API_KEY=your_browserbase_api_key + +# Optional โ€” inferred from your API key if not provided +# BROWSERBASE_PROJECT_ID=your_browserbase_project_id diff --git a/python/getting-started-with-browserbase/README.md b/python/getting-started-with-browserbase/README.md new file mode 100644 index 00000000..6f89e2ee --- /dev/null +++ b/python/getting-started-with-browserbase/README.md @@ -0,0 +1,82 @@ +# Browserbase: Getting Started + +## AT A GLANCE + +- Goal: demo all three core Browserbase capabilities in one script โ€” Search API, Fetch API, and Browser Sessions. +- No AI required: uses Playwright and the Browserbase SDK directly โ€” no model API key needed. +- Three demos in one: web search, lightweight HTTP fetch, and full cloud browser automation. + Docs โ†’ https://docs.browserbase.com + +## GLOSSARY + +- Search API: run web searches through Browserbase โ€” no browser session needed. + Docs โ†’ https://docs.browserbase.com/features/search +- Fetch API: lightweight HTTP fetch through Browserbase infrastructure โ€” no browser, no JS execution. + Docs โ†’ https://docs.browserbase.com/features/fetch +- Session: a full cloud browser instance you connect to via CDP and control with Playwright. + Docs โ†’ https://docs.browserbase.com/introduction/getting-started +- CDP (Chrome DevTools Protocol): the protocol Playwright uses to control the remote browser. + Docs โ†’ https://docs.browserbase.com/introduction/playwright +- Playwright: browser automation library โ€” `page.goto()`, `page.fill()`, `page.click()`, etc. + Docs โ†’ https://playwright.dev/docs/api/class-page + +## QUICKSTART + +### Python + +1. cd python/getting-started-with-browserbase +2. cp .env.example .env +3. Add BROWSERBASE_API_KEY to .env (get it from https://browserbase.com/settings) +4. `uv run python main.py` + +### TypeScript + +1. cd typescript/getting-started-with-browserbase +2. npm install +3. cp .env.example .env +4. Add BROWSERBASE_API_KEY to .env (get it from https://browserbase.com/settings) +5. npm start + +## EXPECTED OUTPUT + +- Search API: searches the web for "Browser automation" and prints the top 5 results with titles and URLs +- Fetch API: fetches a Wikipedia page (no browser), prints status code, content length, and page title +- Browser Session: creates a cloud browser, connects via CDP, navigates to Wikipedia, searches, and extracts the article title, summary, and section headings +- Prints a session replay link for the browser demo +- Closes all resources cleanly + +## COMMON PITFALLS + +- Missing API key: verify .env contains BROWSERBASE_API_KEY โ€” this is the only required credential +- Project ID confusion: BROWSERBASE_PROJECT_ID is optional โ€” the API infers it from your API key. You only need it if you have multiple projects +- Search API not enabled: if you get a 403, the Search API may need to be enabled on your account โ€” check your dashboard +- Fetch API size limit: responses over 1 MB return a 502 โ€” use a browser session for large pages +- Fetch API no JS: the Fetch API returns raw HTML without executing JavaScript โ€” JS-rendered pages will be empty shells +- Connection timeout: if CDP connection fails, check that your API key is valid at https://browserbase.com/settings +- Wikipedia layout changes: if selectors like `input[name="search"]` stop working, Wikipedia may have updated their HTML โ€” consider using Stagehand for self-healing selectors via AI +- Playwright version mismatch: use `playwright-core` (not `playwright`) in TypeScript to avoid downloading unnecessary browser binaries โ€” Browserbase provides the browser +- Session not closing: always close the browser in a `finally` block to avoid leaked sessions +- Find more information on your Browserbase dashboard โ†’ https://www.browserbase.com/sign-in + +## USE CASES + +โ€ข Getting started: understand all three Browserbase capabilities before deciding which to use in your project. +โ€ข Quick prototyping: use Search to find targets, Fetch for static pages, Sessions for JS-heavy ones. +โ€ข Cost-optimized scraping: Search and Fetch are cheaper than full browser sessions โ€” use them when you can. +โ€ข Building blocks: fork this template and build on whichever API fits your use case. + +## NEXT STEPS + +โ€ข Add AI extraction: swap in Stagehand for AI-powered `act()`, `extract()`, and `observe()` โ€” see the other templates in this repo. +โ€ข Chain the APIs: use Search to find URLs, Fetch to triage them, and Sessions for the ones that need a real browser. +โ€ข Enable proxies: pass `proxies: true` in `bb.sessions.create()` or `bb.fetchAPI.create()` for residential proxy support. +โ€ข Enable stealth mode: add `browserSettings: { advancedStealth: true, solveCaptchas: true }` to bypass bot detection. + +## HELPFUL RESOURCES + +๐Ÿ“š Browserbase Docs: https://docs.browserbase.com +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/getting-started-with-browserbase/main.py b/python/getting-started-with-browserbase/main.py new file mode 100644 index 00000000..3365d666 --- /dev/null +++ b/python/getting-started-with-browserbase/main.py @@ -0,0 +1,160 @@ +# Browserbase: Getting Started +# See README.md for full documentation +# +# Demos the three core Browserbase capabilities: +# 1. Search API โ€” web search without a browser +# 2. Fetch API โ€” lightweight HTTP fetch through Browserbase infra +# 3. Sessions โ€” full cloud browser controlled via Playwright + CDP + +import os +import re + +from browserbase import Browserbase +from dotenv import load_dotenv +from playwright.sync_api import sync_playwright + +load_dotenv() + +# ============= CONFIGURATION ============= +# The topic to search, fetch, and browse +SEARCH_QUERY = "Browser automation" +# ========================================= + + +# --------------------------------------------------------------------------- +# 1. Search API โ€” run a web search, no browser needed +# --------------------------------------------------------------------------- + + +def demo_search_api(bb: Browserbase) -> None: + print("\n" + "=" * 50) + print("1. SEARCH API") + print("=" * 50) + print(f'Searching the web for "{SEARCH_QUERY}"...') + + response = bb.search.web(query=SEARCH_QUERY, num_results=5) + + print(f"\nTop {len(response.results)} results:") + for result in response.results: + print(f" - {result.title}") + print(f" {result.url}") + + +# --------------------------------------------------------------------------- +# 2. Fetch API โ€” lightweight HTTP fetch, no browser session needed +# --------------------------------------------------------------------------- + + +def demo_fetch_api(bb: Browserbase) -> None: + print("\n" + "=" * 50) + print("2. FETCH API") + print("=" * 50) + + target_url = "https://en.wikipedia.org/wiki/Browser_automation" + print(f"Fetching {target_url} (no browser)...") + + response = bb.fetch_api.create(url=target_url, allow_redirects=True) + + print(f" Status: {response.status_code}") + print(f" Content length: {len(response.content)} chars") + + # Pull the title out of the raw HTML + title_match = re.search(r"]*>([^<]+)", response.content, re.IGNORECASE) + title = title_match.group(1).strip() if title_match else "Unknown" + print(f" Page title: {title}") + + +# --------------------------------------------------------------------------- +# 3. Sessions + Playwright โ€” full cloud browser via CDP +# --------------------------------------------------------------------------- + + +def demo_browser_session(bb: Browserbase) -> None: + print("\n" + "=" * 50) + print("3. BROWSER SESSION (Playwright + CDP)") + print("=" * 50) + + # Create a session + print("Creating a Browserbase session...") + session = bb.sessions.create() + + print(f"Session ID: {session.id}") + print(f"Live view: https://browserbase.com/sessions/{session.id}") + + # Connect via CDP + print("Connecting to browser via CDP...") + with sync_playwright() as pw: + browser = pw.chromium.connect_over_cdp(session.connect_url) + context = browser.contexts[0] + page = context.pages[0] if context.pages else None + + if not page: + browser.close() + raise RuntimeError("No page found โ€” session may have failed to start.") + + print("Connected!") + + try: + # Navigate to Wikipedia + print(f'\nSearching Wikipedia for "{SEARCH_QUERY}"...') + page.goto("https://www.wikipedia.org/", wait_until="domcontentloaded", timeout=30000) + + # Search + page.fill('input[name="search"]', SEARCH_QUERY) + page.click('button[type="submit"]') + page.wait_for_load_state("domcontentloaded") + page.wait_for_selector("h1", state="visible", timeout=10000) + + # Extract content + title = page.text_content("h1") or "No title found" + summary_text = page.locator( + "#mw-content-text p:not(.mw-empty-elt)" + ).first.text_content() + summary = (summary_text or "").strip() + headings = page.locator("#mw-content-text .mw-heading2 h2").all_text_contents() + sections = [h.replace("[edit]", "").strip() for h in headings if h.strip()] + + print(f"\n Title: {title}") + print(f" Summary: {summary}") + print(f" Sections ({len(sections)}):") + for section in sections: + print(f" - {section}") + print(f"\n Session replay: https://browserbase.com/sessions/{session.id}") + finally: + browser.close() + print(" Browser closed.") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main() -> None: + print("=" * 50) + print("GETTING STARTED WITH BROWSERBASE") + print("=" * 50) + print("Demos: Search API, Fetch API, and Browser Sessions\n") + + bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) + + demo_search_api(bb) + demo_fetch_api(bb) + demo_browser_session(bb) + + print("\n" + "=" * 50) + print("ALL DEMOS COMPLETE!") + print("=" * 50) + + +if __name__ == "__main__": + try: + main() + except Exception as err: + print(f"Error: {err}") + print("\nTroubleshooting:") + print(" 1. Check your .env file has BROWSERBASE_API_KEY") + print(" 2. Verify your API key at https://browserbase.com/settings") + print(" 3. View session logs at https://browserbase.com/sessions") + print("Docs: https://docs.browserbase.com") + exit(1) diff --git a/python/getting-started-with-browserbase/pyproject.toml b/python/getting-started-with-browserbase/pyproject.toml new file mode 100644 index 00000000..dd1ac62c --- /dev/null +++ b/python/getting-started-with-browserbase/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "getting-started-with-browserbase" +version = "1.0.0" +description = "Browserbase + Playwright: Getting Started โ€” connect to a cloud browser, navigate, and extract data" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "browserbase>=1.7.0", + "playwright", + "python-dotenv", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/python/gift-finder/.env.example b/python/gift-finder/.env.example new file mode 100644 index 00000000..fb300c35 --- /dev/null +++ b/python/gift-finder/.env.example @@ -0,0 +1,5 @@ +BROWSERBASE_API_KEY= +# Recommended: a Vercel AI Gateway key that can route to openai/gpt-4.1. +AI_GATEWAY_API_KEY= +# Alternative when not using AI Gateway. +OPENAI_API_KEY= diff --git a/python/gift-finder/README.md b/python/gift-finder/README.md index 4623ca0f..7ac0d153 100644 --- a/python/gift-finder/README.md +++ b/python/gift-finder/README.md @@ -1,5 +1,7 @@ # Stagehand + Browserbase: AI-Powered Gift Finder +Stagehand is the SDK for browser agents. + ## AT A GLANCE - Goal: find personalized gift recommendations using AI-generated search queries and intelligent product scoring. @@ -10,9 +12,9 @@ ## GLOSSARY - act: perform UI actions from a prompt (search, click, type) - Docs โ†’ https://docs.stagehand.dev/basics/act + Docs โ†’ https://docs.stagehand.dev/v4/basics/act - extract: pull structured data from pages using schemas - Docs โ†’ https://docs.stagehand.dev/basics/extract + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract - concurrent sessions: run multiple browser sessions simultaneously for faster searching Docs โ†’ https://docs.browserbase.com/guides/concurrency-rate-limits - proxies: use geolocation-based routing for European website access (Firebox.eu) @@ -25,7 +27,7 @@ 3. source venv/bin/activate # On Windows: venv\Scripts\activate 4. pip install -r requirements.txt 5. pip install InquirerPy pydantic openai -6. cp .env.example .env # Add your Browserbase API key, Project ID, and OpenAI API key to .env +6. cp .env.example .env # Add your Browserbase API key to .env 7. python main.py ## EXPECTED OUTPUT @@ -59,7 +61,7 @@ ## HELPFUL RESOURCES -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction ๐ŸŽฎ Browserbase: https://www.browserbase.com ๐Ÿ’ก Try it out: https://www.browserbase.com/playground ๐Ÿ”ง Templates: https://www.browserbase.com/templates diff --git a/python/gift-finder/main.py b/python/gift-finder/main.py index 980638bf..4cef0b4b 100644 --- a/python/gift-finder/main.py +++ b/python/gift-finder/main.py @@ -1,29 +1,20 @@ -# Stagehand + Browserbase: AI-Powered Gift Finder - See README.md for full documentation +"""Find, score, and verify live gift recommendations with Stagehand V4.""" import asyncio +import json import os +from urllib.parse import urljoin, urlparse from dotenv import load_dotenv from openai import OpenAI -from pydantic import BaseModel, Field, HttpUrl +from pydantic import BaseModel, Field -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand, browserbase -# Load environment variables load_dotenv() -# ============= CONFIGURATION ============= -# Update these values to customize your gift search -CONFIG = { - "recipient": "Friend", # Options: "Mum", "Dad", "Sister", "Brother", "Friend", "Boss" - "description": "loves cooking and trying new recipes", # Describe their interests, hobbies, age, etc. -} -# ========================================= - - -class GiftFinderAnswers(BaseModel): - recipient: str - description: str +RECIPIENT = "Friend" +DESCRIPTION = "loves cooking and trying new recipes" class Product(BaseModel): @@ -31,410 +22,165 @@ class Product(BaseModel): url: str price: str rating: str - ai_score: int | None = None - ai_reason: str | None = None + ai_score: int | None + ai_reason: str | None -class SearchResult(BaseModel): - query: str - session_index: int - products: list[Product] +class Products(BaseModel): + products: list[Product] = Field(max_length=3) -client = OpenAI() +class ProductScore(BaseModel): + product_index: int = Field(alias="productIndex") + score: int = Field(ge=1, le=10) + reason: str = Field(min_length=1, max_length=100) -async def generate_search_queries(recipient: str, description: str) -> list[str]: - """ - Generate intelligent search queries based on recipient profile. +def openai_client() -> tuple[OpenAI, str]: + gateway_key = os.environ.get("AI_GATEWAY_API_KEY") + if gateway_key: + return ( + OpenAI(api_key=gateway_key, base_url="https://ai-gateway.vercel.sh/v1"), + "openai/gpt-4.1", + ) + key = os.environ.get("OPENAI_API_KEY") + if not key: + raise RuntimeError("AI_GATEWAY_API_KEY or OPENAI_API_KEY is required") + return OpenAI(api_key=key), "gpt-4.1" - Uses AI to create thoughtful, complementary gift search terms that go beyond - obvious basics to find unique and meaningful gifts. - """ - print(f"Generating search queries for {recipient}...") - # Use AI to generate search terms based on recipient profile - # This avoids generic searches and focuses on thoughtful, complementary gifts - response = await asyncio.to_thread( - client.chat.completions.create, - model="gpt-4.1", +def generate_search_queries() -> list[str]: + client, model = openai_client() + response = client.chat.completions.create( + model=model, messages=[ { "role": "user", - "content": f"""Generate exactly 3 short gift search queries (1-2 words each) for finding gifts for a {recipient.lower()} who is described as: "{description}". - -IMPORTANT: Assume they already have the basic necessities related to their interests. Focus on: -- Complementary items that enhance their hobbies -- Thoughtful accessories or upgrades -- Related but unexpected items -- Premium or unique versions of things they might not buy themselves - -AVOID obvious basics like "poker set" for poker players, "dumbbells" for fitness enthusiasts, etc. - -Examples for "loves cooking": -spice rack -chef knife -herb garden - -Return ONLY the search terms, one per line, no dashes, bullets, or numbers. Just the plain search terms:""", + "content": ( + "Generate exactly three short gift search queries of one or two words " + f"for a {RECIPIENT.lower()} who {DESCRIPTION}. Focus on thoughtful " + "accessories, upgrades, and related unexpected items rather than basic " + "necessities. Return one plain query per line with no bullets." + ), } ], - max_completion_tokens=1000, + max_completion_tokens=200, ) + content = response.choices[0].message.content or "" + queries = [line.strip(" -0123456789.\t") for line in content.splitlines() if line.strip()] + if len(queries) != 3: + raise RuntimeError(f"OpenAI returned {len(queries)} queries instead of three") + return queries - # Parse AI response and clean up formatting - content = response.choices[0].message.content - queries = content.strip().split("\n") if content else [] - queries = [q.strip() for q in queries if q.strip()] - return queries[:3] - - -async def score_products( - products: list[Product], - recipient: str, - description: str, -) -> list[Product]: - """ - Score and rank products based on recipient profile using AI. - Analyzes each product against the recipient's interests, relationship context, - value, uniqueness, and practical usefulness to find the best gift matches. - """ - print("AI is analyzing gift options based on recipient profile...") - - # Flatten all products from multiple search sessions into single array - all_products = products - - if len(all_products) == 0: - print("No products to score") - return [] - - # Format products for AI analysis with index numbers for reference +def score_products(products: list[Product]) -> list[Product]: product_list = "\n".join( - [ - f"{index + 1}. {product.title} - {product.price} - {product.rating}" - for index, product in enumerate(all_products) - ] + f"{index + 1}. {product.title} - {product.price} - {product.rating}" + for index, product in enumerate(products) ) - - print(f"Scoring {len(all_products)} products...") - - response = await asyncio.to_thread( - client.chat.completions.create, - model="gpt-4.1", + client, model = openai_client() + response = client.chat.completions.create( + model=model, messages=[ { "role": "user", - "content": f"""You are a gift recommendation expert. Score each product based on how well it matches the recipient profile. - -RECIPIENT: {recipient} -DESCRIPTION: {description} - -PRODUCTS TO SCORE: -{product_list} - -For each product, provide a score from 1-10 (10 being perfect match) and a brief reason. Consider: -- How well it matches their interests/hobbies -- Appropriateness for the relationship ({recipient.lower()}) -- Value for money -- Uniqueness/thoughtfulness -- Practical usefulness - -Return ONLY a valid JSON array (no markdown, no code blocks) with this exact format: -[ - {{ - "productIndex": 1, - "score": 8, - "reason": "Perfect for poker enthusiasts, high quality chips enhance the gaming experience" - }}, - {{ - "productIndex": 2, - "score": 6, - "reason": "Useful but basic, might already own similar item" - }} -] - -IMPORTANT: -- Return raw JSON only, no code blocks -- Include all {len(all_products)} products -- Keep reasons under 100 characters -- Use productIndex 1-{len(all_products)}""", + "content": ( + f"Score every gift from 1-10 for a {RECIPIENT.lower()} who {DESCRIPTION}. " + "Return only a JSON array where every object has productIndex, score, " + f"and a reason under 100 characters.\n\n{product_list}" + ), } ], - max_completion_tokens=1000, + max_completion_tokens=1_000, ) - - try: - # Clean up AI response by removing markdown code blocks - response_content = ( - response.choices[0].message.content.strip() - if response.choices[0].message.content - else "[]" - ) - - response_content = ( - response_content.replace("```json\n", "") - .replace("```json", "") - .replace("```\n", "") - .replace("```", "") - ) - - # Parse JSON response from AI scoring - import json - - scores_data = json.loads(response_content) - - # Map AI scores back to products using index matching - scored_products = [] - for index, product in enumerate(all_products): - score_info = next((s for s in scores_data if s.get("productIndex") == index + 1), None) - product.ai_score = score_info.get("score", 0) if score_info else 0 - product.ai_reason = ( - score_info.get("reason", "No scoring available") - if score_info - else "No scoring available" - ) - scored_products.append(product) - - # Sort by AI score descending to show best matches first - scored_products.sort(key=lambda x: x.ai_score or 0, reverse=True) - return scored_products - except Exception as error: - print(f"Error parsing AI scores: {error}") - print("Using fallback scoring (all products scored as 5)") - - # Fallback scoring ensures app continues working even if AI fails - # Neutral score of 5 allows products to still be ranked and displayed - for product in all_products: - product.ai_score = 5 - product.ai_reason = "Scoring failed - using neutral score" - return all_products - - -async def get_user_input() -> GiftFinderAnswers: - """ - Collect user input for gift recipient and description. - - Uses the CONFIG dictionary at the top of the file for configuration. - """ - print("Welcome to the Gift Finder App!") - print("Find the perfect gift with intelligent web browsing") - print(f"\nSearching for gifts for: {CONFIG['recipient']}") - print(f"Profile: {CONFIG['description']}\n") - - # Validate description length - if len(CONFIG["description"].strip()) < 5: - raise ValueError( - "Description must be at least 5 characters long. Please update the CONFIG at the top of the file." + content = (response.choices[0].message.content or "[]").strip() + content = content.removeprefix("```json").removeprefix("```").removesuffix("```").strip() + raw_scores = json.loads(content) + scores = [ProductScore.model_validate(item) for item in raw_scores] + expected_indexes = set(range(1, len(products) + 1)) + if ( + len(scores) != len(products) + or {score.product_index for score in scores} != expected_indexes + ): + raise RuntimeError("OpenAI did not return one unique score for every product") + + by_index = {score.product_index: score for score in scores} + scored = [] + for index, product in enumerate(products, start=1): + score = by_index[index] + scored.append( + product.model_copy(update={"ai_score": score.score, "ai_reason": score.reason}) ) + return sorted(scored, key=lambda product: product.ai_score or 0, reverse=True) - return GiftFinderAnswers(recipient=CONFIG["recipient"], description=CONFIG["description"]) - - -async def main() -> None: - """ - Main application entry point. - - Orchestrates the entire gift finding process: - 1. Collects user input - 2. Generates intelligent search queries - 3. Runs concurrent browser searches - 4. Scores and ranks products with AI - 5. Displays top recommendations - """ - print("Starting Gift Finder Application...") - # Step 1: Collect user input - user_input = await get_user_input() - recipient = user_input.recipient - description = user_input.description - print(f"User input received: {recipient} - {description}") +async def search_products(query: str, index: int) -> list[Product]: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") - # Step 2: Generate search queries using AI - print("\nGenerating intelligent search queries...") + print(f"Search {index + 1}: {query}") + browser = await browserbase.launch(api_key=api_key, region="us-east-1") try: - search_queries = await generate_search_queries(recipient, description) - - print("\nGenerated Search Queries:") - for index, query in enumerate(search_queries): - cleaned_query = query.replace('"', "").replace("'", "") - print(f" {index + 1}. {cleaned_query}") - except Exception as error: - print(f"Error generating search queries: {error}") - # Fallback queries ensure app continues working - search_queries = ["gifts", "accessories", "items"] - print("Using fallback search queries") - - # Step 3: Start concurrent browser searches - print("\nStarting concurrent browser searches...") - - async def run_single_search(query: str, session_index: int) -> SearchResult: - print(f'Starting search session {session_index + 1} for: "{query}"') - - # Create separate Stagehand instance for each search to run concurrently - # Each session searches independently to maximize speed and parallel processing - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - verbose=1, # Silent logging to avoid cluttering output - # Logging levels: 0 = errors only, 1 = info, 2 = debug - # When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs - # https://docs.stagehand.dev/configuration/logging - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - # Proxies require Developer Plan or higher - uncomment if you have a Developer Plan or higher - # "proxies": [ - # { - # "type": "browserbase", - # "geolocation": { - # "city": "LONDON", - # "country": "GB" - # } - # } - # ], - "region": "us-east-1", - "timeout": 900, - "browser_settings": { - "viewport": { - "width": 1920, - "height": 1080, - } - }, - }, + stagehand = await Stagehand.create( + browser=browser, ) - try: - # Initialize browser session with Stagehand - async with Stagehand(config) as session_stagehand: - session_page = session_stagehand.page - - # Display live view URL for debugging and monitoring - session_id = None - if hasattr(session_stagehand, "session_id"): - session_id = session_stagehand.session_id - elif hasattr(session_stagehand, "browserbase_session_id"): - session_id = session_stagehand.browserbase_session_id - - if session_id: - live_view_url = f"https://www.browserbase.com/sessions/{session_id}" - print(f"Session {session_index + 1} Live View: {live_view_url}") - - # Navigate to European gift site - print(f"Session {session_index + 1}: Navigating to Firebox.eu...") - await session_page.goto("https://firebox.eu/") - - # Perform search using natural language actions - print(f'Session {session_index + 1}: Searching for "{query}"...') - await session_page.act(f"Type {query} into the search bar") - await session_page.act("Click the search button") - await session_page.wait_for_timeout(1000) - - # Extract structured product data using Pydantic schema for type safety - print(f"Session {session_index + 1}: Extracting product data...") - - # Define Pydantic schemas for structured data extraction - class ProductItem(BaseModel): - title: str = Field(..., description="the title/name of the product") - url: HttpUrl = Field(..., description="the full URL link to the product page") - price: str = Field( - ..., description="the price of the product (include currency symbol)" - ) - rating: str = Field( - ..., - description="the star rating or number of reviews (e.g., '4.5 stars' or '123 reviews')", - ) - - class ProductsData(BaseModel): - products: list[ProductItem] = Field( - ..., - max_length=3, - description="array of the first 3 products from search results", - ) - - products_data = await session_page.extract( - "Extract the first 3 products from the search results", schema=ProductsData - ) - - print( - f'Session {session_index + 1}: Found {len(products_data.products)} products for "{query}"' - ) - - # Convert to Product objects - products = [ - Product(title=p.title, url=str(p.url), price=p.price, rating=p.rating) - for p in products_data.products - ] - - return SearchResult(query=query, session_index=session_index + 1, products=products) - except Exception as error: - print(f"Session {session_index + 1} failed: {error}") - - return SearchResult(query=query, session_index=session_index + 1, products=[]) - - # Create concurrent search tasks for all generated queries - search_promises = [ - run_single_search(query, index) for index, query in enumerate(search_queries) - ] - - print("\nBrowser Sessions Starting...") - print("Live view links will appear as each session initializes") - - # Execute all searches concurrently using asyncio.gather() - all_results = await asyncio.gather(*search_promises) + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto("https://firebox.eu/", wait_until="domcontentloaded", timeout=60_000) + await stagehand.act(f"Type {query} into the search bar", page=page) + await stagehand.act("Click the search button", page=page) + await page.wait_for_timeout(1_000) + extracted = await stagehand.extract( + "Extract the first three products from the search results", + Products, + page=page, + ) + base_url = await page.url() + products = [] + for product in extracted.data.products: + if not product.title.strip() or not product.url.strip(): + continue + absolute_url = urljoin(base_url, product.url) + parsed_url = urlparse(absolute_url) + if parsed_url.scheme in {"http", "https"} and parsed_url.hostname: + products.append(product.model_copy(update={"url": absolute_url})) + if not products: + raise RuntimeError(f"No products found for {query!r}") + return products + finally: + await stagehand.close() + finally: + await browser.close() - # Calculate total products found across all search sessions - total_products = sum(len(result.products) for result in all_results) - print(f"\nTotal products found: {total_products} across {len(search_queries)} searches") - # Flatten all products into single array for AI scoring - all_products_flat = [] - for result in all_results: - all_products_flat.extend(result.products) +async def main() -> None: + if len(DESCRIPTION.strip()) < 5: + raise RuntimeError("Recipient description is too short") + queries = await asyncio.to_thread(generate_search_queries) + print(f"Generated queries: {queries}") - # Step 4: Score and rank products with AI - if len(all_products_flat) > 0: + products: list[Product] = [] + for index, query in enumerate(queries): try: - # AI scores all products and ranks them by relevance to recipient - scored_products = await score_products(all_products_flat, recipient, description) - top3_products = scored_products[:3] - - print("\nTOP 3 RECOMMENDED GIFTS:") - print("=" * 50) - - # Display top 3 products with AI reasoning for transparency - for index, product in enumerate(top3_products): - rank = f"#{index + 1}" - print(f"\n{rank} - {product.title}") - print(f"Price: {product.price}") - print(f"Rating: {product.rating}") - print(f"AI Score: {product.ai_score}/10") - print(f"Why: {product.ai_reason}") - print(f"Link: {product.url}") - print("-" * 30) - - print( - f"\nGift finding complete! Found {total_products} products, analyzed {len(scored_products)} with AI." - ) + products.extend(await search_products(query, index)) except Exception as error: - # Handle AI scoring errors - print(f"Error scoring products: {error}") - print(f"Target: {recipient}") - print(f"Profile: {description}") - else: - # Handle case where no products were found - print("No products found to score") - print("Try adjusting your recipient description or check if the website is accessible") + print(f"Search {index + 1} produced no usable products: {error}") + if len(products) < 3: + raise RuntimeError(f"Expected at least three products, received {len(products)}") - print("\nThank you for using Gift Finder!") + scored = await asyncio.to_thread(score_products, products) + top_three = scored[:3] + if len(top_three) != 3 or any(product.ai_score is None for product in top_three): + raise RuntimeError("Gift ranking did not produce three scored recommendations") + print("Top three recommendations:") + print(json.dumps([product.model_dump(mode="json") for product in top_three], indent=2)) if __name__ == "__main__": try: asyncio.run(main()) - except Exception as err: - print(f"Application error: {err}") - print("Check your environment variables") - exit(1) + except Exception as error: + print(f"Gift finder failed: {error}") + raise SystemExit(1) from error diff --git a/python/gift-finder/pyproject.toml b/python/gift-finder/pyproject.toml new file mode 100644 index 00000000..4cbd154d --- /dev/null +++ b/python/gift-finder/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "gift-finder" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "openai>=2.26,<3", + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[tool.uv] +package = false diff --git a/python/google-trends/.env.example b/python/google-trends/.env.example new file mode 100644 index 00000000..2e9ab652 --- /dev/null +++ b/python/google-trends/.env.example @@ -0,0 +1,2 @@ +# Browserbase credentials (required) +BROWSERBASE_API_KEY=your-browserbase-api-key diff --git a/python/google-trends/README.md b/python/google-trends/README.md new file mode 100644 index 00000000..af53fb83 --- /dev/null +++ b/python/google-trends/README.md @@ -0,0 +1,62 @@ +# Stagehand + Browserbase: Google Trends Keywords Extractor + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- Goal: Extract trending search keywords from Google Trends for any country with structured JSON output. +- Configurable by country code (US, GB, IN, DE, etc.) and language preference. +- Uses JSON schema validation for consistent, typed data extraction. +- Docs โ†’ https://docs.stagehand.dev/v4/basics/extract + +## GLOSSARY + +- extract: extract structured data from web pages using natural language instructions and JSON schemas + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract +- act: perform UI actions from a prompt (click, type, dismiss dialogs) + Docs โ†’ https://docs.stagehand.dev/v4/basics/act + +## QUICKSTART + +1. uv sync +2. cp .env.example .env # Add your Browserbase API key to .env +3. uv run python main.py + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase +- Displays live session link for monitoring +- Navigates to Google Trends trending page with configured country/language +- Dismisses any consent dialogs if present +- Extracts trending keywords with rank positions +- Outputs structured JSON with country code, language, timestamp, and keyword list +- Closes session cleanly + +## COMMON PITFALLS + +- "ModuleNotFoundError": ensure all dependencies are installed via `uv sync` +- Missing credentials: verify .env contains BROWSERBASE_API_KEY +- Invalid country code: ensure country code is a valid 2-letter ISO code (US, GB, IN, DE, FR, BR, etc.) +- Empty results: Google Trends may not have trending data for all country/language combinations +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +โ€ข Market research: Track trending topics across different regions to identify emerging interests and market opportunities. +โ€ข Content strategy: Discover popular search terms to inform blog posts, social media content, or SEO keyword targeting. +โ€ข Competitive intelligence: Monitor trending keywords in your industry to stay ahead of market shifts and consumer interests. + +## NEXT STEPS + +โ€ข Parameterize inputs: Accept country code and limit as command-line arguments or environment variables for flexible deployment. +โ€ข Historical tracking: Store extracted keywords with timestamps to build a trends database and analyze keyword momentum over time. +โ€ข Multi-region comparison: Extend to fetch trends from multiple countries in parallel and compare regional differences. + +## HELPFUL RESOURCES + +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/google-trends/main.py b/python/google-trends/main.py new file mode 100644 index 00000000..79d7dc6c --- /dev/null +++ b/python/google-trends/main.py @@ -0,0 +1,84 @@ +"""Extract current Google Trends keywords with Stagehand V4.""" + +import asyncio +import json +import os +from datetime import UTC, datetime + +from dotenv import load_dotenv +from pydantic import BaseModel, Field, RootModel +from stagehand import Stagehand, browserbase + +load_dotenv() + +COUNTRY_CODE = "US" +LANGUAGE = "en-US" +LIMIT = 20 + + +class TrendingKeyword(BaseModel): + rank: int = Field(description="Position in the visible trending list") + keyword: str = Field(description="Main trending search term") + + +class TrendingKeywords(RootModel[list[TrendingKeyword]]): + pass + + +async def main() -> None: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") + + print(f"Extracting up to {LIMIT} Google Trends keywords for {COUNTRY_CODE}") + browser = await browserbase.launch(api_key=api_key) + try: + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + url = f"https://trends.google.com/trending?geo={COUNTRY_CODE.upper()}&hl={LANGUAGE}" + await page.goto(url, wait_until="networkidle", timeout=60_000) + + try: + await stagehand.act( + 'Click the "Got it" button if it is visible', + page=page, + timeout=5_000, + ) + except Exception: + print("No consent dialog found") + + extracted = await stagehand.extract( + ( + "Extract the visible trending search keywords from the table. " + "Assign rank 1 to the first row and continue in order. " + f"Return at most {LIMIT} items." + ), + TrendingKeywords, + page=page, + ) + keywords = extracted.data.root[:LIMIT] + output = { + "country_code": COUNTRY_CODE, + "language": LANGUAGE, + "extracted_at": datetime.now(UTC).isoformat(), + "trending_keywords": [item.model_dump() for item in keywords], + } + print(json.dumps(output, indent=2)) + finally: + await stagehand.close() + finally: + await browser.close() + print("Session closed successfully") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as error: + print(f"Google Trends extraction failed: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/google-trends/pyproject.toml b/python/google-trends/pyproject.toml new file mode 100644 index 00000000..3c5ccc1b --- /dev/null +++ b/python/google-trends/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "google-trends" +version = "0.1.0" +description = "Extract trending keywords from Google Trends using Stagehand and Browserbase" +readme = "README.md" +requires-python = ">=3.11,<3.14" +dependencies = [ + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py311'] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/python/image-url-download/.env.example b/python/image-url-download/.env.example new file mode 100644 index 00000000..bec3c8be --- /dev/null +++ b/python/image-url-download/.env.example @@ -0,0 +1,5 @@ +# Browserbase Configuration +BROWSERBASE_API_KEY=your_browserbase_api_key + +# Optional: maximum number of images to download per run (default: 10) +# MAX_IMAGES=10 diff --git a/python/image-url-download/.gitignore b/python/image-url-download/.gitignore new file mode 100644 index 00000000..23048827 --- /dev/null +++ b/python/image-url-download/.gitignore @@ -0,0 +1,2 @@ +.env +images/ diff --git a/python/image-url-download/README.md b/python/image-url-download/README.md new file mode 100644 index 00000000..9a48bf4e --- /dev/null +++ b/python/image-url-download/README.md @@ -0,0 +1,76 @@ +# Stagehand + Browserbase: Image URL Download + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- Goal: extract all image URLs from a page with Stagehand and download each image with the Browserbase Fetch API. +- SDK-backed downloads: `AsyncBrowserbase.fetch_api.create()` retrieves each discovered image through Browserbase without handwritten HTTP requests. +- AI-powered URL extraction: uses `extract()` with a JSON schema to reliably pull `` src attributes and background image URLs from any page. +- Format-agnostic: uses the `Content-Type` response header to detect the real MIME type โ€” files are saved with the correct extension (`.jpg`, `.png`, `.svg`, `.webp`, etc.). +- Organized output: images are saved to `./images//` so runs against different sites never mix. +- V4 page access: the Python SDK exposes the active Stagehand page directly for navigation; URL discovery remains an `extract()` operation. + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract + +## GLOSSARY + +- extract: pull structured data from a page using a natural language instruction and a JSON schema. + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract +- Browserbase Fetch API: retrieve a discovered asset through the first-party Browserbase SDK. + Docs โ†’ https://docs.browserbase.com/features/fetch-api +- page.evaluate: a narrow fallback that reads exact DOM asset URLs only when semantic extraction + returns no fetchable candidates. + Docs โ†’ https://docs.stagehand.dev/v4/reference/page +- ImageUrls: Pydantic schema passed to `extract()` for typed URL discovery. +- MAX_IMAGES: configurable cap (default: 10) on how many images to download per run. Set via the `MAX_IMAGES` env var or the constant at the top of `main.py`. + +## QUICKSTART + +1. cd image-url-download +2. cp .env.example .env +3. Add BROWSERBASE_API_KEY to .env +4. uv run main.py \ โ€” e.g. `uv run main.py https://www.browserbase.com` + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase +- Connects Playwright to the same session via CDP +- Navigates to the target URL and waits for the page to fully render +- Extracts all image URLs from the page using `extract()` +- Deduplicates URLs and caps at `MAX_IMAGES` (default: 10) +- Downloads each image with the Browserbase Fetch API through the first-party Python SDK +- Saves images to `./images//`, named `-.` with the extension derived from the `Content-Type` header +- Logs per-image status (saved / failed) and a final summary count +- Closes session cleanly + +## COMMON PITFALLS + +- `ModuleNotFoundError`: ensure all dependencies are installed โ€” `uv run` handles this automatically via `pyproject.toml` +- Missing credentials: verify .env contains BROWSERBASE_API_KEY +- Zero images found: the page may load images lazily or use CSS background images โ€” try scrolling before extraction with `stagehand.act()`, or refine the extract instruction +- Download failures (403): some auth-gated image URLs require session cookies that the Fetch API request does not inherit +- MAX_IMAGES cap: if you need more than 10 images, set `MAX_IMAGES=50` in your .env or edit the constant at the top of `main.py` +- Large pages: pages with hundreds of images may slow down `extract()` โ€” use MAX_IMAGES to limit the download set + +## USE CASES + +โ€ข Asset archiving: bulk-save product images, thumbnails, or media assets from websites you own or have permission to scrape. +โ€ข Visual regression testing: download reference images from a staging environment to diff against production. +โ€ข Dataset collection: gather labeled image sets from public pages for ML training pipelines. +โ€ข Public media: archive images discovered on pages without maintaining separate download HTTP code. + +## NEXT STEPS + +โ€ข Scroll before extracting: use `stagehand.act()` to scroll the page before `extract()` to trigger lazy-loaded images. +โ€ข Concurrent downloads: fan out the Fetch API calls with `asyncio.gather()` for faster bulk downloads. +โ€ข Metadata CSV: write a `manifest.csv` alongside the images recording original URL, filename, MIME type, byte size, and download timestamp. +โ€ข Extend MIME support: add entries to the `MIME_TO_EXT` dict at the top of `main.py` for any formats not already covered. + +## HELPFUL RESOURCES + +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/image-url-download/main.py b/python/image-url-download/main.py new file mode 100644 index 00000000..fa229891 --- /dev/null +++ b/python/image-url-download/main.py @@ -0,0 +1,185 @@ +"""Download images discovered in a live page with Stagehand V4.""" + +import asyncio +import base64 +import os +import re +import sys +import time +from pathlib import Path +from urllib.parse import urljoin, urlparse + +from browserbase import AsyncBrowserbase +from dotenv import load_dotenv +from pydantic import BaseModel, Field +from stagehand import Page, Stagehand, browserbase + +load_dotenv() + +MAX_IMAGES = int(os.environ.get("MAX_IMAGES", "10")) +OUTPUT_DIR = Path("images") +MIME_TO_EXT = { + "image/jpeg": "jpg", + "image/png": "png", + "image/webp": "webp", + "image/gif": "gif", + "image/svg+xml": "svg", + "image/avif": "avif", + "image/bmp": "bmp", + "image/tiff": "tiff", +} + + +class ImageUrls(BaseModel): + urls: list[str] = Field( + description="Absolute HTTP(S) image resource URLs from src or background-image values" + ) + + +def image_filename(url: str, mime_type: str, index: int) -> str: + extension = MIME_TO_EXT.get(mime_type, "bin") + segment = Path(urlparse(url).path).name + stem = Path(segment).stem or f"image-{index}" + safe_stem = re.sub(r"[^a-zA-Z0-9_-]", "_", stem)[:80] + return f"{safe_stem}-{int(time.time() * 1000)}.{extension}" + + +async def fetch_image(api: AsyncBrowserbase, url: str) -> tuple[bytes, str] | None: + result = await api.fetch_api.create(url=url, format="raw", allow_redirects=True) + mime_type = (result.content_type or "").split(";", 1)[0] + if not 200 <= result.status_code < 300 or not mime_type.startswith("image/"): + return None + if not isinstance(result.content, str): + return None + if result.encoding == "base64": + payload = base64.b64decode(result.content) + else: + payload = result.content.encode(result.encoding or "utf-8") + return payload, mime_type + + +async def exact_dom_image_urls(page: Page, target_url: str) -> list[str]: + values = await page.evaluate( + r"""(() => { + const urls = new Set(); + for (const image of Array.from(document.images)) { + if (image.currentSrc) urls.add(image.currentSrc); + if (image.src) urls.add(image.src); + } + for (const element of Array.from(document.querySelectorAll('[style]'))) { + const background = getComputedStyle(element).backgroundImage; + for (const match of background.matchAll(/url\(["']?(.*?)["']?\)/g)) { + if (match[1]) urls.add(new URL(match[1], document.baseURI).href); + } + } + return [...urls]; + })()""" + ) + if not isinstance(values, list): + return [] + urls = [] + for value in values: + absolute = urljoin(target_url, str(value)) + if urlparse(absolute).scheme in {"http", "https"} and absolute not in urls: + urls.append(absolute) + return urls + + +async def download_images( + api: AsyncBrowserbase, urls: list[str], output_dir: Path +) -> tuple[int, int]: + saved = 0 + failed = 0 + for index, url in enumerate(urls): + try: + fetched = await fetch_image(api, url) + except Exception: + fetched = None + if fetched is None: + failed += 1 + continue + + payload, mime_type = fetched + filename = image_filename(url, mime_type, index) + (output_dir / filename).write_bytes(payload) + print(f"Saved {filename} ({len(payload)} bytes)") + saved += 1 + return saved, failed + + +async def main() -> None: + if len(sys.argv) < 2: + raise RuntimeError("Usage: uv run python main.py ") + target_url = sys.argv[1] + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") + + browser = await browserbase.launch(api_key=api_key) + try: + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto(target_url, wait_until="domcontentloaded", timeout=60_000) + await page.wait_for_timeout(3_000) + + extracted = await stagehand.extract( + ( + "Extract all rendered image URLs on this page, including image src " + "attributes and background-image URLs. Return absolute HTTP(S) image " + "resource URLs exactly as rendered. Preserve every hostname character, " + "including any www prefix." + ), + ImageUrls, + page=page, + ) + normalized = [] + for value in extracted.data.urls: + candidate = str(value) + if not candidate.lower().startswith(("http://", "https://")): + continue + absolute = candidate + if urlparse(absolute).scheme in {"http", "https"} and absolute not in normalized: + normalized.append(absolute) + if not normalized: + # Accessibility snapshots can omit decorative images. Inspect the exact DOM + # shape only when semantic extraction returns no usable candidates at all. + normalized = await exact_dom_image_urls(page, target_url) + urls = normalized[:MAX_IMAGES] + hostname = urlparse(target_url).hostname or "unknown" + output_dir = OUTPUT_DIR / hostname + output_dir.mkdir(parents=True, exist_ok=True) + async with AsyncBrowserbase(api_key=api_key) as api: + saved, failed = await download_images(api, urls, output_dir) + if saved == 0 and normalized: + # If semantic extraction produced URLs that cannot be fetched, fall back to + # exact DOM mechanics without changing the semantic-first discovery path. + fallback_urls = [ + url + for url in await exact_dom_image_urls(page, target_url) + if url not in normalized + ][:MAX_IMAGES] + fallback_saved, fallback_failed = await download_images( + api, fallback_urls, output_dir + ) + saved += fallback_saved + failed += fallback_failed + + print(f"Downloaded {saved} image(s); {failed} failed") + finally: + await stagehand.close() + finally: + await browser.close() + print("Session closed successfully") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as error: + print(f"Image download failed: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/image-url-download/pyproject.toml b/python/image-url-download/pyproject.toml new file mode 100644 index 00000000..0188c9f4 --- /dev/null +++ b/python/image-url-download/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "image-url-download" +version = "1.0.0" +description = "Stagehand + Browserbase: extract image URLs from a page and download them with the Browserbase Fetch API" +readme = "README.md" +requires-python = ">=3.11,<3.14" +dependencies = [ + "browserbase>=1.17.0", + "playwright", + "python-dotenv", + "stagehand==4.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/python/job-application/.env.example b/python/job-application/.env.example new file mode 100644 index 00000000..ec2065ba --- /dev/null +++ b/python/job-application/.env.example @@ -0,0 +1,4 @@ +BROWSERBASE_API_KEY= +# Optional controls for test volume and Browserbase concurrency. +MAX_CONCURRENCY=2 +MAX_JOBS=0 diff --git a/python/job-application/README.md b/python/job-application/README.md index 0bde93cb..54f37c63 100644 --- a/python/job-application/README.md +++ b/python/job-application/README.md @@ -1,89 +1,19 @@ -# Stagehand + Browserbase: Automated Job Application Agent +# Stagehand + Browserbase: Job Application Automation -## AT A GLANCE +Stagehand is the SDK for browser agents. -- Goal: Automate job applications by discovering job listings and submitting applications with unique agent identifiers. -- Concurrent Processing: applies to multiple jobs in parallel with configurable concurrency limits based on Browserbase project settings. -- Dynamic Data Generation: generates unique agent IDs and email addresses for each application. -- File Upload Support: automatically uploads resume PDF from a remote URL during the application process. -- Docs โ†’ https://docs.stagehand.dev/basics/agent +This template uses Stagehand V4 to discover every role on a public test job board, fill each application with unique test data, upload a PDF resume, and submit it. -## GLOSSARY +## Run -- agent: create an autonomous AI agent that can execute complex multi-step tasks - Docs โ†’ https://docs.stagehand.dev/basics/agent#what-is-agent -- act: perform UI actions from a prompt (click, type, fill forms) - Docs โ†’ https://docs.stagehand.dev/basics/act -- extract: extract structured data from web pages using natural language instructions - Docs โ†’ https://docs.stagehand.dev/basics/extract -- observe: analyze a page and return selectors or action plans before executing - Docs โ†’ https://docs.stagehand.dev/basics/observe -- asyncio.Semaphore: concurrency control mechanism to limit parallel job applications based on project limits +```bash +cp .env.example .env +uv sync +uv run python main.py +``` -## QUICKSTART +Set `BROWSERBASE_API_KEY` in `.env`. `MAX_CONCURRENCY` defaults to `2`; set `MAX_JOBS` to a positive number when you want a bounded test run. -1. uv venv venv -2. source venv/bin/activate # On Windows: venv\Scripts\activate -3. uvx install stagehand browserbase pydantic python-dotenv httpx -4. cp .env.example .env # Add required API keys/IDs to .env (BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, GOOGLE_GENERATIVE_AI_API_KEY) -5. python main.py +Expected output includes the number of discovered jobs, a submission line for every application, and a final completed-submission count. Runtime errors still make the process exit nonzero. -## EXPECTED OUTPUT - -- Fetches project concurrency limit from Browserbase (maxed at 5) -- Initializes main Stagehand session with Browserbase -- Displays live session link for monitoring -- Navigates to agent job board -- Clicks "View Jobs" button -- Extracts all job listings with titles and URLs using Pydantic schema validation -- Closes main session -- Creates asyncio.Semaphore for concurrency control -- Applies to all jobs in parallel (respecting concurrency limit) -- For each job application: - - Generates unique agent ID and email - - Navigates to job page - - Clicks on specific job - - Fills agent identifier field - - Fills contact endpoint (email) field - - Fills deployment region field - - Uploads resume PDF from remote URL using httpx - - Selects multi-region deployment option - - Submits application -- Displays completion message when all applications are finished - -## COMMON PITFALLS - -- "ModuleNotFoundError": ensure all dependencies are installed via uvx install -- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_GENERATIVE_AI_API_KEY -- Google API access: ensure you have access to Google's gemini-2.5-flash model -- Concurrency limits: script automatically respects Browserbase project concurrency (capped at 5) -- Resume URL: ensure the resume URL (https://agent-job-board.vercel.app/Agent%20Resume.pdf) is accessible -- Job detection: verify that job listings are visible on the page and match expected structure -- Network issues: check internet connection and website accessibility -- Import errors: activate your virtual environment if you created one -- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in - -## USE CASES - -โ€ข Bulk job applications: Automate applying to multiple job postings simultaneously with unique credentials for each application. -โ€ข Agent deployment automation: Streamline the process of deploying multiple AI agents by automating the application and registration workflow. -โ€ข Testing & QA: Validate job application forms and workflows across multiple listings to ensure consistent functionality. -โ€ข Recruitment automation: Scale agent recruitment processes by programmatically submitting applications with generated identifiers. - -## NEXT STEPS - -โ€ข Add filtering: Implement job filtering by title keywords, location, or other criteria before applying. -โ€ข Error handling: Add retry logic for failed applications and better error reporting with job-specific logs. -โ€ข Resume customization: Support multiple resume versions or dynamic resume generation based on job requirements. -โ€ข Application tracking: Store application status, timestamps, and results in a database for tracking and follow-up. -โ€ข Rate limiting: Add delays between applications to avoid overwhelming the target system. -โ€ข Multi-site support: Extend to support multiple job boards with site-specific form field mappings. - -## HELPFUL RESOURCES - -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v2/first-steps/introduction -๐ŸŽฎ Browserbase: https://www.browserbase.com -๐Ÿ’ก Try it out: https://www.browserbase.com/playground -๐Ÿ”ง Templates: https://www.browserbase.com/templates -๐Ÿ“ง Need help? support@browserbase.com -๐Ÿ’ฌ Discord: http://stagehand.dev/discord +Docs: https://docs.stagehand.dev/v4/first-steps/introduction diff --git a/python/job-application/main.py b/python/job-application/main.py index ee66a8c5..9818b9a9 100644 --- a/python/job-application/main.py +++ b/python/job-application/main.py @@ -1,246 +1,155 @@ -# Stagehand + Browserbase: Job Application Automation - See README.md for full documentation +"""Discover and submit test applications with Stagehand V4.""" import asyncio import os import random +import string import time import httpx -from browserbase import Browserbase from dotenv import load_dotenv from pydantic import BaseModel, Field, HttpUrl -from stagehand import Stagehand, StagehandConfig +from stagehand import FilePayload, Stagehand, browserbase -# Load environment variables load_dotenv() +JOB_BOARD_URL = "https://agent-job-board.vercel.app/" +RESUME_URL = f"{JOB_BOARD_URL}Agent%20Resume.pdf" + -# Define Pydantic schemas for structured data extraction -# Using schemas ensures consistent data extraction even if page layout changes class JobInfo(BaseModel): - url: HttpUrl = Field(..., description="Job URL") - title: str = Field(..., description="Job title") + url: HttpUrl = Field(description="Job URL") + title: str = Field(min_length=1, description="Job title") class JobsData(BaseModel): jobs: list[JobInfo] -async def get_project_concurrency() -> int: - """ - Fetch project concurrency limit from Browserbase SDK. - - Retrieves the maximum concurrent sessions allowed for the project, - capped at 5. - """ - bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) - - # Use asyncio.to_thread to run synchronous SDK call in thread pool - project = await asyncio.to_thread( - bb.projects.retrieve, os.environ.get("BROWSERBASE_PROJECT_ID") - ) - return min(project.concurrency, 5) +def require_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required") + return value def generate_random_email() -> str: - """ - Generate a random email address for form submission. - - """ - random_string = "".join(random.choices("abcdefghijklmnopqrstuvwxyz0123456789", k=8)) - return f"agent-{random_string}@example.com" + suffix = "".join(random.choices(string.ascii_lowercase + string.digits, k=8)) + return f"agent-{suffix}@example.com" def generate_agent_id() -> str: - """ - Generate a unique agent identifier for job applications. - - Combines timestamp and random string to ensure uniqueness across - multiple job applications and sessions. - """ - timestamp = int(time.time() * 1000) - random_string = "".join(random.choices("abcdefghijklmnopqrstuvwxyz0123456789", k=7)) - return f"agent-{timestamp}-{random_string}" + suffix = "".join(random.choices(string.ascii_lowercase + string.digits, k=7)) + return f"agent-{int(time.time() * 1000)}-{suffix}" -async def apply_to_job(job_info: JobInfo, semaphore: asyncio.Semaphore): - """ - Apply to a single job posting with automated form filling. +async def close_session(stagehand: Stagehand, browser: object) -> None: + await stagehand.close() + await browser.close() # type: ignore[attr-defined] - Uses Stagehand to navigate to job page, fill out application form, - upload resume, and submit the application. - """ - # Semaphore ensures we don't exceed project concurrency limits - async with semaphore: - # Initialize Stagehand with Browserbase for cloud-based browser automation - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="google/gemini-2.5-flash", - model_api_key=os.environ.get("GOOGLE_GENERATIVE_AI_API_KEY"), - ) - try: - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - print(f"[{job_info.title}] Session Started") - - # Get session ID for live viewing/debugging - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id - - if session_id: - print( - f"[{job_info.title}] Watch live: https://browserbase.com/sessions/{session_id}" - ) - - page = stagehand.page - - # Navigate to job URL - await page.goto(str(job_info.url)) - print(f"[{job_info.title}] Navigated to job page") - - # Click on the specific job listing to open application form - await page.act(f"click on {job_info.title}") - print(f"[{job_info.title}] Clicked on job") - - # Generate unique identifiers for this application - agent_id = generate_agent_id() - email = generate_random_email() - - print(f"[{job_info.title}] Agent ID: {agent_id}") - print(f"[{job_info.title}] Email: {email}") - - # Fill out application form fields using natural language actions - # Stagehand's act() method understands natural language instructions - await page.act(f"type '{agent_id}' into the agent identifier field") - - await page.act(f"type '{email}' into the contact endpoint field") - - await page.act("type 'us-west-2' into the deployment region field") - - # Upload agent profile/resume file - # Using observe() to find the upload button, then setting files programmatically - upload_actions = await page.observe("find the file upload button for agent profile") - if upload_actions and len(upload_actions) > 0: - upload_action = upload_actions[0] - upload_selector = str(upload_action.selector) - if upload_selector: - file_input = page.locator(upload_selector) - - # Fetch resume PDF from remote URL - # Using httpx to download the file before uploading - resume_url = "https://agent-job-board.vercel.app/Agent%20Resume.pdf" - async with httpx.AsyncClient() as client: - response = await client.get(resume_url) - if response.status_code != 200: - raise Exception(f"Failed to fetch resume: {response.status_code}") - resume_buffer = response.content - - # Upload file using Playwright's set_input_files with buffer - await file_input.set_input_files( - { - "name": "Agent Resume.pdf", - "mimeType": "application/pdf", - "buffer": resume_buffer, - } - ) - print(f"[{job_info.title}] Uploaded resume from {resume_url}") - - # Select multi-region deployment option - await page.act("select 'Yes' for multi region deployment") - - # Submit the application form - await page.act("click deploy agent button") - - print(f"[{job_info.title}] Application submitted successfully!") - - except Exception as error: - print(f"[{job_info.title}] Error: {error}") - raise error - - -async def main(): - """ - Main application entry point. - - Orchestrates the job application process: - 1. Fetches project concurrency limits - 2. Scrapes job listings from the job board - 3. Applies to all jobs in parallel with concurrency control - """ - print("Starting Job Application Automation...") - - # Get project concurrency limit to control parallel execution - max_concurrency = await get_project_concurrency() - print(f"Executing with concurrency limit: {max_concurrency}") - - # Initialize Stagehand with Browserbase for cloud-based browser automation - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="google/gemini-2.5-flash", - model_api_key=os.environ.get("GOOGLE_GENERATIVE_AI_API_KEY"), +async def discover_jobs() -> list[JobInfo]: + browser = await browserbase.launch(api_key=require_env("BROWSERBASE_API_KEY")) + stagehand = await Stagehand.create( + browser=browser, ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto(JOB_BOARD_URL, wait_until="domcontentloaded", timeout=60_000) + await stagehand.act("Click the View Jobs button", page=page) + extracted = await stagehand.extract( + "Extract every visible job listing with its title and absolute URL", + JobsData, + page=page, + ) + jobs = extracted.data.jobs + if not jobs: + raise RuntimeError("The job board returned no job listings") + return jobs + finally: + await close_session(stagehand, browser) - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - print("Main Stagehand Session Started") - - # Get session ID for live viewing/debugging - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id - - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") - - page = stagehand.page - - # Navigate to agent job board homepage - await page.goto("https://agent-job-board.vercel.app/") - print("Navigated to agent-job-board.vercel.app") - - # Click on "View Jobs" button to access job listings - await page.act("click on the view jobs button") - print("Clicked on view jobs button") - # Extract all job listings with titles and URLs using structured schema - # Using extract() with Pydantic schema ensures consistent data extraction - jobs_result = await page.extract( - "extract all job listings with their titles and URLs", schema=JobsData +async def apply_to_job(job: JobInfo, resume: bytes, semaphore: asyncio.Semaphore) -> str: + async with semaphore: + browser = await browserbase.launch(api_key=require_env("BROWSERBASE_API_KEY")) + stagehand = await Stagehand.create( + browser=browser, ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto(str(job.url), wait_until="domcontentloaded", timeout=60_000) + await stagehand.act(f"Click the job listing titled {job.title}", page=page) + + agent_id = generate_agent_id() + email = generate_random_email() + await stagehand.act( + "Fill the agent identifier field with %agent_id%", + page=page, + variables={"agent_id": agent_id}, + ) + await stagehand.act( + "Fill the contact endpoint field with %email%", + page=page, + variables={"email": email}, + ) + await stagehand.act("Fill the deployment region field with us-west-2", page=page) + + observed = await stagehand.observe( + "Find the file input for the agent profile or resume", + page=page, + ) + if not observed.data or not observed.data[0].selector: + raise RuntimeError(f"[{job.title}] Could not locate the resume upload input") + await page.locator(observed.data[0].selector).set_input_files( + FilePayload( + name="Agent Resume.pdf", + buffer=resume, + mime_type="application/pdf", + ) + ) + + await stagehand.act("Select Yes for multi-region deployment", page=page) + await stagehand.act("Click the Deploy Agent button", page=page) + print(f"[{job.title}] Application submitted ({agent_id}, {email})") + return job.title + finally: + await close_session(stagehand, browser) + + +async def main() -> None: + max_concurrency = max(1, int(os.environ.get("MAX_CONCURRENCY", "2"))) + max_jobs = int(os.environ.get("MAX_JOBS", "0")) + jobs = await discover_jobs() + if max_jobs > 0: + jobs = jobs[:max_jobs] + print(f"Discovered {len(jobs)} jobs; applying with concurrency {max_concurrency}") + + async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client: + response = await client.get(RESUME_URL) + response.raise_for_status() + resume = response.content + if not resume.startswith(b"%PDF"): + raise RuntimeError("The resume download was not a PDF") - jobs_data = jobs_result.jobs - print(f"Found {len(jobs_data)} jobs") - - # Create semaphore with concurrency limit to control parallel job applications - # Semaphore ensures we don't exceed Browserbase project limits semaphore = asyncio.Semaphore(max_concurrency) - - # Apply to all jobs in parallel with concurrency control - # Using asyncio.gather() to run all applications concurrently - print(f"Starting to apply to {len(jobs_data)} jobs with max concurrency of {max_concurrency}") - - application_tasks = [apply_to_job(job, semaphore) for job in jobs_data] - - # Wait for all applications to complete - await asyncio.gather(*application_tasks) - - print("All applications completed!") + results = await asyncio.gather( + *(apply_to_job(job, resume, semaphore) for job in jobs), + return_exceptions=True, + ) + failures = [result for result in results if isinstance(result, BaseException)] + if failures: + raise RuntimeError(f"{len(failures)} of {len(jobs)} applications failed: {failures}") + print(f"Completed {len(results)} job application submissions") if __name__ == "__main__": try: asyncio.run(main()) - except Exception as err: - print(f"Error: {err}") - exit(1) + except Exception as error: + print(f"Job application automation failed: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/job-application/pyproject.toml b/python/job-application/pyproject.toml new file mode 100644 index 00000000..6cf29901 --- /dev/null +++ b/python/job-application/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "job-application" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "httpx==0.28.1", + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[tool.uv] +package = false diff --git a/python/license-verification/.env.example b/python/license-verification/.env.example new file mode 100644 index 00000000..d0cde042 --- /dev/null +++ b/python/license-verification/.env.example @@ -0,0 +1 @@ +BROWSERBASE_API_KEY= diff --git a/python/license-verification/README.md b/python/license-verification/README.md index 98cf9c24..1d2b1b56 100644 --- a/python/license-verification/README.md +++ b/python/license-verification/README.md @@ -1,5 +1,7 @@ # Stagehand + Browserbase: Data Extraction with Structured Schemas +Stagehand is the SDK for browser agents. + ## AT A GLANCE - Goal: show how to extract structured, validated data from websites using Stagehand + Pydantic. @@ -10,9 +12,9 @@ ## GLOSSARY - act: perform UI actions from a prompt (type, click, navigate). - Docs โ†’ https://docs.stagehand.dev/basics/act + Docs โ†’ https://docs.stagehand.dev/v4/basics/act - extract: pull structured data from web pages into validated objects. - Docs โ†’ https://docs.stagehand.dev/basics/extract + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract - schema: a Pydantic definition that enforces data types, optional fields, and validation rules. Docs โ†’ https://docs.pydantic.dev/ - form automation: filling and submitting inputs to trigger results before extraction. @@ -20,13 +22,11 @@ ## QUICKSTART -1. cd license-verification-template -2. uv venv venv -3. source venv/bin/activate # On Windows: venv\Scripts\activate -4. pip install -r requirements.txt -5. pip install pydantic -6. cp .env.example .env # Add your Browserbase API key and Project ID to .env -7. python main.py +1. cd license-verification +2. uv venv && source .venv/bin/activate # On Windows: .venv\Scripts\activate +3. pip install stagehand python-dotenv pydantic +4. cp .env.example .env # Add your Browserbase API key to .env +5. python main.py ## EXPECTED OUTPUT @@ -57,7 +57,7 @@ ## HELPFUL RESOURCES -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction ๐ŸŽฎ Browserbase: https://www.browserbase.com ๐Ÿ’ก Try it out: https://www.browserbase.com/playground ๐Ÿ”ง Templates: https://www.browserbase.com/templates diff --git a/python/license-verification/main.py b/python/license-verification/main.py index 914f4cad..b6a43ddf 100644 --- a/python/license-verification/main.py +++ b/python/license-verification/main.py @@ -1,99 +1,76 @@ -# Stagehand + Browserbase: Data Extraction with Structured Schemas - See README.md for full documentation +"""Verify a California real-estate license with Stagehand V4.""" import asyncio +import json import os from dotenv import load_dotenv -from pydantic import BaseModel, Field +from pydantic import BaseModel -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand, browserbase -# Load environment variables load_dotenv() -# License verification variables -variables = { - "input1": "02237476" # DRE License ID to search for -} +LICENSE_ID = "02237476" -async def main(): - # Initialize Stagehand with Browserbase for cloud-based browser automation. - config = StagehandConfig( - env="BROWSERBASE", # Use Browserbase cloud browsers for reliable automation. - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - verbose=1, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - ) +class LicenseDetails(BaseModel): + license_type: str | None + name: str | None + mailing_address: str | None + license_id: str | None + expiration_date: str | None + license_status: str | None + salesperson_license_issued: str | None + former_names: str | None + responsible_broker: str | None + broker_license_id: str | None + broker_address: str | None + disciplinary_action: str | None + other_comments: str | None - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - print("Stagehand Session Started") - # Provide live session URL for debugging and monitoring extraction process. - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id +async def main() -> None: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") - - page = stagehand.page - - # Navigate to California DRE license verification website for data extraction. - print("Navigating to: https://www2.dre.ca.gov/publicasp/pplinfo.asp") - await page.goto("https://www2.dre.ca.gov/publicasp/pplinfo.asp") - - # Fill in license ID to search for specific real estate professional. - print(f"Performing action: type {variables['input1']} into the License ID input field") - await page.act(f"type {variables['input1']} into the License ID input field") - - # Submit search form to retrieve license verification data. - print("Performing action: click the Find button") - await page.act("click the Find button") - - # Define schema using Pydantic - class LicenseData(BaseModel): - license_type: str | None = Field(None, description="Type of real estate license") - name: str | None = Field(None, description="License holder's full name") - mailing_address: str | None = Field(None, description="Current mailing address") - license_id: str | None = Field(None, description="Unique license identifier") - expiration_date: str | None = Field(None, description="License expiration date") - license_status: str | None = Field( - None, description="Current status (active, expired, etc.)" - ) - salesperson_license_issued: str | None = Field( - None, description="Date salesperson license was issued" + browser = await browserbase.launch(api_key=api_key) + try: + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto( + "https://www2.dre.ca.gov/publicasp/pplinfo.asp", + wait_until="domcontentloaded", + timeout=60_000, ) - former_names: str | None = Field(None, description="Any previous names used") - responsible_broker: str | None = Field(None, description="Associated broker name") - broker_license_id: str | None = Field(None, description="Broker's license ID") - broker_address: str | None = Field(None, description="Broker's business address") - disciplinary_action: str | None = Field( - None, description="Any disciplinary actions taken" + await stagehand.act( + f"Type {LICENSE_ID} into the License ID input field", + page=page, ) - other_comments: str | None = Field(None, description="Additional relevant information") + await stagehand.act("Click the Find button", page=page) - # Extract structured license data using Pydantic schema for type safety and validation. - print("Extracting: extract all the license verification details for DRE#02237476") - extracted_data = await page.extract( - "extract all the license verification details for DRE#02237476", schema=LicenseData - ) - print(f"Extracted: {extracted_data}") - - print("Session closed successfully") + extracted = await stagehand.extract( + f"Extract all license verification details for DRE #{LICENSE_ID}", + LicenseDetails, + page=page, + ) + details = extracted.data + print(json.dumps(details.model_dump(mode="json"), indent=2)) + finally: + await stagehand.close() + finally: + await browser.close() if __name__ == "__main__": try: asyncio.run(main()) - except Exception as err: - print(f"Error: {err}") - exit(1) + except Exception as error: + print(f"License verification failed: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/license-verification/pyproject.toml b/python/license-verification/pyproject.toml new file mode 100644 index 00000000..ebb27e3c --- /dev/null +++ b/python/license-verification/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "license-verification" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[tool.uv] +package = false diff --git a/python/manual-mfa-with-contexts/.env.example b/python/manual-mfa-with-contexts/.env.example new file mode 100644 index 00000000..38bd3488 --- /dev/null +++ b/python/manual-mfa-with-contexts/.env.example @@ -0,0 +1,3 @@ +BROWSERBASE_API_KEY= +GITHUB_USERNAME= +GITHUB_PASSWORD= diff --git a/python/manual-mfa-with-contexts/README.md b/python/manual-mfa-with-contexts/README.md new file mode 100644 index 00000000..fc495ad9 --- /dev/null +++ b/python/manual-mfa-with-contexts/README.md @@ -0,0 +1,69 @@ +# Stagehand + Browserbase: Manual MFA with Contexts + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- Goal: demonstrate how to persist authentication across sessions using Browserbase Contexts, eliminating MFA friction after the first login. +- Flow: first session creates context and completes MFA manually โ†’ context saves auth state โ†’ second session reuses context with no MFA required. + +## GLOSSARY + +- context: a Browserbase feature that persists browser state (cookies, localStorage, sessionStorage) across sessions. + Docs โ†’ https://docs.browserbase.com/features/contexts +- persist: setting that saves authentication state including MFA trust/remember device state to the context. +- MFA (Multi-Factor Authentication): two-factor authentication requiring a code from an authenticator app. +- session persistence: maintaining logged-in state across multiple browser sessions without re-authentication. + +## QUICKSTART + +1. cd manual-mfa-with-contexts +2. uv venv venv +3. source venv/bin/activate # On Windows: venv\Scripts\activate +4. uvx install stagehand browserbase python-dotenv pydantic requests +5. cp .env.example .env +6. Add your Browserbase API key, GitHub username, and password to .env +7. Ensure 2FA is enabled on your GitHub test account (Settings โ†’ Password and authentication โ†’ Enable two-factor authentication) +8. python main.py + +## EXPECTED OUTPUT + +- Creates a new Browserbase context +- First session: navigates to GitHub login, fills credentials, detects MFA prompt +- Pauses and displays Browserbase session link for manual MFA completion +- Waits for MFA completion (2 minute timeout) +- Saves authentication state to context +- Second session: reuses context, navigates to GitHub (already logged in, no MFA) +- Extracts and prints the logged-in username from the reused context +- Cleans up context + +## COMMON PITFALLS + +- "ModuleNotFoundError": ensure all dependencies are installed via uvx install +- Missing credentials: verify .env contains BROWSERBASE_API_KEY, GITHUB_USERNAME, and GITHUB_PASSWORD +- MFA timeout: ensure you complete MFA within 2 minutes, or increase timeout value +- 2FA not enabled: GitHub account must have 2FA enabled for this demo to work +- Context not persisting: verify context.persist is set to true in browser_settings + +## USE CASES + +โ€ข Payment automation: Complete MFA once for utility portals, then automate future payments without MFA prompts. +โ€ข Account management: Persist authentication for services requiring MFA, enabling automated account management workflows. +โ€ข Compliance automation: Store trusted device state for regulatory portals, reducing friction for recurring compliance tasks. + +## NEXT STEPS + +โ€ข Store context IDs: Save context_id per customer in your database to reuse across sessions. +โ€ข Multi-portal support: Extend to multiple portals/services, each with their own context. +โ€ข Context management: Implement context cleanup, rotation, and expiration policies. +โ€ข Error handling: Add retry logic and better error messages for MFA timeouts. + +## HELPFUL RESOURCES + +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ“š Contexts Docs: https://docs.browserbase.com/features/contexts +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/manual-mfa-with-contexts/main.py b/python/manual-mfa-with-contexts/main.py new file mode 100644 index 00000000..6422c166 --- /dev/null +++ b/python/manual-mfa-with-contexts/main.py @@ -0,0 +1,132 @@ +"""Persist a manually completed GitHub MFA login with Stagehand V4.""" + +import asyncio +import os +import time + +from browserbase import AsyncBrowserbase +from dotenv import load_dotenv +from pydantic import BaseModel + +from stagehand import Stagehand, browserbase + +load_dotenv() + + +class MFAStatus(BaseModel): + mfa_required: bool + + +class AuthenticationState(BaseModel): + username: str + + +def require_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required") + return value + + +async def first_login(context_id: str) -> None: + browser = await browserbase.launch( + api_key=require_env("BROWSERBASE_API_KEY"), + browser_settings={"context": {"id": context_id, "persist": True}}, + ) + try: + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto("https://github.com/login", wait_until="domcontentloaded") + await stagehand.act( + "Fill the username field with %username%", + page=page, + variables={"username": require_env("GITHUB_USERNAME")}, + ) + await stagehand.act( + "Fill the password field with %password%", + page=page, + variables={"password": require_env("GITHUB_PASSWORD")}, + ) + await stagehand.act("Click the Sign in button", page=page) + + status = await stagehand.extract( + "Is a two-factor authentication or verification-code prompt visible?", + MFAStatus, + page=page, + ) + if status.data.mfa_required: + print("MFA is required. Open the newest Browserbase session and complete it.") + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + current_url = await page.url() + if "/login" not in current_url and "/sessions/two-factor" not in current_url: + break + await asyncio.sleep(3) + else: + raise TimeoutError("MFA was not completed within two minutes") + + print("First session authenticated and persisted") + finally: + await stagehand.close() + finally: + await browser.close() + + +async def verify_context(context_id: str) -> None: + browser = await browserbase.launch( + api_key=require_env("BROWSERBASE_API_KEY"), + browser_settings={"context": {"id": context_id, "persist": True}}, + ) + try: + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto("https://github.com", wait_until="domcontentloaded") + extracted = await stagehand.extract( + ( + "Extract the logged-in GitHub username. Return an empty string if the page " + "is not authenticated." + ), + AuthenticationState, + page=page, + ) + username = extracted.data.username + print("Second session reused GitHub authentication without another login") + print(f"Logged-in username: {username}") + finally: + await stagehand.close() + finally: + await browser.close() + + +async def main() -> None: + require_env("GITHUB_USERNAME") + require_env("GITHUB_PASSWORD") + async with AsyncBrowserbase(api_key=require_env("BROWSERBASE_API_KEY")) as api: + context = await api.contexts.create() + print("Created temporary Browserbase context") + try: + await first_login(context.id) + await asyncio.sleep(5) + await verify_context(context.id) + finally: + # The generated SDK currently sets a JSON content type on DELETE, so send an + # explicit empty object instead of an empty body. + await api.contexts.delete(context.id, extra_body={}) + print("Deleted temporary Browserbase context") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as error: + print(f"MFA context demo failed: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/manual-mfa-with-contexts/pyproject.toml b/python/manual-mfa-with-contexts/pyproject.toml new file mode 100644 index 00000000..715c30fc --- /dev/null +++ b/python/manual-mfa-with-contexts/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "manual-mfa-with-contexts" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "browserbase>=1.7.0", + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[tool.uv] +package = false diff --git a/python/manual-mfa-with-contexts/requirements.txt b/python/manual-mfa-with-contexts/requirements.txt new file mode 100644 index 00000000..f7af058e --- /dev/null +++ b/python/manual-mfa-with-contexts/requirements.txt @@ -0,0 +1,5 @@ +browserbase>=1.7.0 +python-dotenv +pydantic +stagehand==4.0.0 +requests diff --git a/python/mfa-handling/.env.example b/python/mfa-handling/.env.example new file mode 100644 index 00000000..d0cde042 --- /dev/null +++ b/python/mfa-handling/.env.example @@ -0,0 +1 @@ +BROWSERBASE_API_KEY= diff --git a/python/mfa-handling/README.md b/python/mfa-handling/README.md new file mode 100644 index 00000000..d64bfab2 --- /dev/null +++ b/python/mfa-handling/README.md @@ -0,0 +1,78 @@ +# Stagehand + Browserbase: MFA Handling - TOTP Automation + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- Goal: Automate MFA (Multi-Factor Authentication) completion using TOTP (Time-based One-Time Password) code generation. +- TOTP Generation: Implements RFC 6238 compliant algorithm to generate time-based authentication codes programmatically. +- Automatic Form Filling: Extracts TOTP secrets from pages and automatically fills MFA forms without user interaction. +- Retry Logic: Handles time window edge cases by regenerating codes and retrying authentication when needed. +- Docs โ†’ https://docs.stagehand.dev/v4/basics/act + +## GLOSSARY + +- act: perform UI actions from a prompt (type, click, fill forms) + Docs โ†’ https://docs.stagehand.dev/v4/basics/act +- extract: extract structured data from web pages using natural language instructions + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract +- TOTP: Time-based One-Time Password - a 6-digit code that changes every 30 seconds, generated using HMAC-SHA1 algorithm +- RFC 6238: Standard specification for TOTP authentication codes used by Google Authenticator, Authy, and other authenticator apps + +## QUICKSTART + +1. python -m venv venv +2. source venv/bin/activate # On Windows: venv\Scripts\activate +3. pip install stagehand python-dotenv pydantic +4. cp .env.example .env +5. Add your Browserbase API key to .env (BROWSERBASE_API_KEY) +6. python main.py + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase +- Displays live session link for monitoring +- Navigates to TOTP challenge demo page (authenticationtest.com/totpChallenge/) +- Extracts test credentials (email, password) and TOTP secret from the page +- Generates TOTP code using RFC 6238 algorithm +- Fills in email and password fields +- Fills in TOTP code field with generated code +- Submits authentication form +- Checks authentication result +- Retries with fresh code if initial attempt fails (handles time window edge cases) +- Closes session cleanly + +## COMMON PITFALLS + +- "ModuleNotFoundError": ensure all dependencies are installed via pip +- Missing credentials: verify .env contains BROWSERBASE_API_KEY +- TOTP code expiration: codes are valid for 30 seconds - if authentication fails, the script automatically retries with a fresh code +- Page structure changes: if the demo site structure changes, extraction may fail +- Network timeouts: ensure stable internet connection for reliable page loading +- Import errors: activate your virtual environment if you created one +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +โ€ข Automated authentication: Complete MFA challenges automatically when session persistence isn't enough (session expired, new device, etc.) +โ€ข TOTP integration: Store encrypted TOTP secrets during user onboarding and generate codes programmatically when needed +โ€ข Zero-touch MFA: Eliminate user interaction for MFA completion in automated workflows +โ€ข Session recovery: Automatically handle MFA prompts when re-authenticating expired sessions + +## NEXT STEPS + +โ€ข Secure storage: Implement encrypted TOTP secret storage (AES-256) in your database during user onboarding +โ€ข Multiple time windows: Add support for trying ยฑ1 time window (60s range) if current code fails +โ€ข SMS/Email MFA: Extend to support SMS codes (via Twilio/Bandwidth API) or email codes (via Gmail API/IMAP) +โ€ข Backup codes: Implement fallback to backup codes stored during initial MFA setup +โ€ข Context integration: Combine with Browserbase Contexts to minimize MFA prompts (95% context reuse, 4% auto TOTP, 1% user-mediated) +โ€ข Error handling: Add graceful fallback to user prompts when automation fails + +## HELPFUL RESOURCES + +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/mfa-handling/main.py b/python/mfa-handling/main.py new file mode 100644 index 00000000..0e9379b9 --- /dev/null +++ b/python/mfa-handling/main.py @@ -0,0 +1,118 @@ +"""Complete a live RFC 6238 TOTP challenge with Stagehand V4.""" + +import asyncio +import base64 +import hashlib +import hmac +import os +import struct +import time + +from dotenv import load_dotenv +from pydantic import BaseModel, Field + +from stagehand import Page, Stagehand, browserbase + +load_dotenv() + +DEMO_URL = "https://authenticationtest.com/totpChallenge/" + + +class Credentials(BaseModel): + email: str + password: str + totp_secret: str = Field(description="TOTP secret key shown by the demo") + + +class AuthResult(BaseModel): + success: bool + message: str + + +def generate_totp(secret: str, window: int = 0) -> str: + normalized = secret.upper().replace(" ", "").rstrip("=") + padding = "=" * ((8 - len(normalized) % 8) % 8) + key = base64.b32decode(normalized + padding) + counter = int(time.time() // 30) + window + digest = hmac.new(key, struct.pack(">Q", counter), hashlib.sha1).digest() + offset = digest[-1] & 0x0F + code = struct.unpack(">I", digest[offset : offset + 4])[0] & 0x7FFFFFFF + return str(code % 1_000_000).zfill(6) + + +async def submit(stagehand: Stagehand, page: Page, credentials: Credentials) -> None: + await stagehand.act( + "Fill the email field with %email%", + page=page, + variables={"email": credentials.email}, + ) + await stagehand.act( + "Fill the password field with %password%", + page=page, + variables={"password": credentials.password}, + ) + seconds_left = 30 - int(time.time()) % 30 + if seconds_left < 12: + await asyncio.sleep(seconds_left + 1) + code = generate_totp(credentials.totp_secret) + await stagehand.act( + "Fill the TOTP code field with %code%", + page=page, + variables={"code": code}, + ) + await stagehand.act("Click the submit or login button", page=page) + + +async def main() -> None: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") + + browser = await browserbase.launch(api_key=api_key) + try: + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto(DEMO_URL, wait_until="domcontentloaded", timeout=60_000) + extracted = await stagehand.extract( + "Extract the test email, password, and TOTP secret shown on the page", + Credentials, + page=page, + ) + credentials = extracted.data + await submit(stagehand, page, credentials) + await page.wait_for_timeout(1_000) + result = await stagehand.extract( + "Check whether the TOTP login succeeded and return its message", + AuthResult, + page=page, + ) + if not result.data.success: + await page.goto(DEMO_URL, wait_until="domcontentloaded") + await submit(stagehand, page, credentials) + await page.wait_for_timeout(1_000) + result = await stagehand.extract( + "Check whether the TOTP login succeeded and return its message", + AuthResult, + page=page, + ) + if not result.data.success: + raise RuntimeError(f"TOTP authentication failed: {result.data.message}") + print(f"TOTP authentication succeeded: {result.data.message}") + finally: + await stagehand.close() + finally: + await browser.close() + print("Session closed successfully") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as error: + print(f"TOTP example failed: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/mfa-handling/pyproject.toml b/python/mfa-handling/pyproject.toml new file mode 100644 index 00000000..4ddc019d --- /dev/null +++ b/python/mfa-handling/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "mfa-handling" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[tool.uv] +package = false diff --git a/python/mfa-handling/requirements.txt b/python/mfa-handling/requirements.txt new file mode 100644 index 00000000..b64f3057 --- /dev/null +++ b/python/mfa-handling/requirements.txt @@ -0,0 +1,3 @@ +stagehand==4.0.0 +python-dotenv +pydantic diff --git a/python/nurse-verification/.env.example b/python/nurse-verification/.env.example new file mode 100644 index 00000000..d0cde042 --- /dev/null +++ b/python/nurse-verification/.env.example @@ -0,0 +1 @@ +BROWSERBASE_API_KEY= diff --git a/python/nurse-verification/README.md b/python/nurse-verification/README.md index a827337f..c4fb8706 100644 --- a/python/nurse-verification/README.md +++ b/python/nurse-verification/README.md @@ -1,18 +1,20 @@ # Stagehand + Browserbase: Nurse License Verification +Stagehand is the SDK for browser agents. + ## AT A GLANCE - Goal: automate verification of nurse licenses by filling forms and extracting structured results from verification sites. - Flow: loop through license records โ†’ navigate to verification site โ†’ fill form โ†’ search โ†’ extract verification results. - Benefits: quickly verify multiple licenses without manual form filling, structured data ready for compliance tracking or HR systems. - Docs โ†’ https://docs.stagehand.dev/basics/act + Docs โ†’ https://docs.stagehand.dev/v4/basics/act ## GLOSSARY - act: perform UI actions from a prompt (type, click, fill forms). - Docs โ†’ https://docs.stagehand.dev/basics/act + Docs โ†’ https://docs.stagehand.dev/v4/basics/act - extract: pull structured data from a page using AI and Pydantic schemas. - Docs โ†’ https://docs.stagehand.dev/basics/extract + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract - schema: a Pydantic model that enforces data types, optional fields, and validation rules. Docs โ†’ https://docs.pydantic.dev/ - license verification: process of confirming the validity and status of professional licenses. @@ -23,7 +25,7 @@ 2. uv venv venv 3. source venv/bin/activate # On Windows: venv\Scripts\activate 4. pip install stagehand python-dotenv pydantic -5. cp .env.example .env # Add your Browserbase API key, Project ID, and OpenAI API key to .env +5. cp .env.example .env # Add your Browserbase API key to .env 6. python main.py ## EXPECTED OUTPUT @@ -39,7 +41,7 @@ ## COMMON PITFALLS - "ModuleNotFoundError": ensure all dependencies are installed via pip -- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and OPENAI_API_KEY +- Missing credentials: verify .env contains BROWSERBASE_API_KEY - No results found: check if license numbers are valid or if verification site structure has changed - Network issues: ensure internet access and verification sites are accessible - Schema validation errors: ensure extracted data matches Pydantic schema structure @@ -59,7 +61,7 @@ ## HELPFUL RESOURCES -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction ๐ŸŽฎ Browserbase: https://www.browserbase.com ๐Ÿ’ก Try it out: https://www.browserbase.com/playground ๐Ÿ”ง Templates: https://www.browserbase.com/templates diff --git a/python/nurse-verification/main.py b/python/nurse-verification/main.py index 2bce22f0..cd39cb6b 100644 --- a/python/nurse-verification/main.py +++ b/python/nurse-verification/main.py @@ -1,144 +1,98 @@ -# Stagehand + Browserbase: Automated Nurse License Verification - See README.md for full documentation +"""Verify nurse-license records with Stagehand V4.""" import asyncio +import json import os from dotenv import load_dotenv from pydantic import BaseModel, Field -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand, browserbase -# Load environment variables load_dotenv() class LicenseRecord(BaseModel): - """Single license verification record""" - - name: str = Field(..., description="the name of the license holder") - license_number: str = Field(..., description="the license number") - status: str = Field(..., description="the status of the license") - more_info_url: str = Field(..., description="URL for more information") + name: str = Field(min_length=1, description="License holder name") + license_number: str = Field(min_length=1, description="License number") + status: str = Field(min_length=1, description="License status") + more_info_url: str = Field(description="URL for more information") class LicenseResults(BaseModel): - """Collection of license verification results""" - - list_of_licenses: list[LicenseRecord] = Field( - ..., description="array of license verification results" - ) + list_of_licenses: list[LicenseRecord] -# License records to verify - add more records as needed LICENSE_RECORDS = [ { - "Site": "https://pod-search.kalmservices.net/", - "FirstName": "Ronald", - "LastName": "Agee", - "LicenseNumber": "346", - }, + "site": "https://pod-search.kalmservices.net/", + "first_name": "Ronald", + "last_name": "Agee", + "license_number": "346", + } ] -async def main(): - """ - Automated nurse license verification using AI-powered browser automation. - Processes multiple license records and extracts verification results. - """ - print("Starting Nurse License Verification Automation...") - - # Initialize Stagehand with Browserbase for cloud-based browser automation - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - verbose=1, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - ) +def require_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required") + return value + +async def main() -> None: + print("Starting nurse license verification...") + browser = await browserbase.launch(api_key=require_env("BROWSERBASE_API_KEY")) try: - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - print("Initializing browser session...") - print("Stagehand session started successfully") - - # Provide live session URL for debugging and monitoring - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id - - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") - - page = stagehand.page - - # Process each license record sequentially - for license_record in LICENSE_RECORDS: - print( - f"Verifying license for: {license_record['FirstName']} {license_record['LastName']}" + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + + for record in LICENSE_RECORDS: + expected_name = f"{record['first_name']} {record['last_name']}" + print(f"Verifying {expected_name}, license {record['license_number']}") + await page.goto( + record["site"], + wait_until="domcontentloaded", + timeout=60_000, ) - - # Navigate to license verification site - print(f"Navigating to: {license_record['Site']}") - await page.goto(license_record["Site"]) - await page.wait_for_load_state("domcontentloaded") - # Brief timeout to ensure form fields are interactive - await page.wait_for_timeout(1000) - - # Fill in form fields with license information - print("Filling in license information...") - await page.act(f'Type "{license_record["FirstName"]}" into the first name field') - await page.act(f'Type "{license_record["LastName"]}" into the last name field') - await page.act( - f'Type "{license_record["LicenseNumber"]}" into the license number field' + await stagehand.act( + f"Type {record['first_name']} into the first name field", + page=page, ) - - # Submit search - print("Clicking search button...") - await page.act("Click the search button") - - # Wait for search results to load - await page.wait_for_load_state("domcontentloaded") - await page.wait_for_timeout(1000) - - # Extract license verification results - print("Extracting license verification results...") - results = await page.extract( - "Extract ALL the license verification results from the page, including name, license number and status", - schema=LicenseResults, + await stagehand.act( + f"Type {record['last_name']} into the last name field", + page=page, + ) + await stagehand.act( + f"Type {record['license_number']} into the license number field", + page=page, + ) + await stagehand.act("Click the Search button", page=page) + await stagehand.observe( + "Find the first visible license result row", + page=page, ) - print("License verification results extracted:") - - # Display results in formatted JSON - import json - - print(json.dumps(results.model_dump(), indent=2)) - - print("Session closed successfully") - - except Exception as error: - print(f"Error during license verification: {error}") - - # Provide helpful troubleshooting information - print("\nCommon issues:") - print("1. Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") - print("2. Verify OPENAI_API_KEY is set in environment") - print("3. Ensure internet access and license verification site is accessible") - print("4. Verify Browserbase account has sufficient credits") - - raise + extracted = await stagehand.extract( + "Extract every license result with name, license number, status, and details URL", + LicenseResults, + page=page, + ) + print(json.dumps(extracted.data.model_dump(mode="json"), indent=2)) + finally: + await stagehand.close() + finally: + await browser.close() if __name__ == "__main__": try: asyncio.run(main()) - except Exception as err: - print(f"Application error: {err}") - exit(1) + except Exception as error: + print(f"Nurse license verification failed: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/nurse-verification/pyproject.toml b/python/nurse-verification/pyproject.toml new file mode 100644 index 00000000..16662c34 --- /dev/null +++ b/python/nurse-verification/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "nurse-verification" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[tool.uv] +package = false diff --git a/python/pickleball/.env.example b/python/pickleball/.env.example new file mode 100644 index 00000000..e7e9b99c --- /dev/null +++ b/python/pickleball/.env.example @@ -0,0 +1,9 @@ +BROWSERBASE_API_KEY= +SF_REC_PARK_EMAIL= +SF_REC_PARK_PASSWORD= +ACTIVITY=Pickleball +# ISO date, for example 2026-08-12. Defaults to tomorrow when omitted. +SELECTED_DATE= +TIME_OF_DAY=Evening +# Set true only when you intend to create a real reservation. +BOOK_COURT=false diff --git a/python/pickleball/README.md b/python/pickleball/README.md index 2dd2212a..e3e2dbb5 100644 --- a/python/pickleball/README.md +++ b/python/pickleball/README.md @@ -1,23 +1,25 @@ # Stagehand + Browserbase: AI-Powered Court Booking Automation +Stagehand is the SDK for browser agents. + ## AT A GLANCE - Goal: automate tennis and pickleball court bookings in San Francisco Recreation & Parks system. - AI Integration: Stagehand for UI interaction and data extraction. - Browser Automation: automates login, filtering, court selection, and booking confirmation. - User Interaction: prompts for activity type, date, and time preferences with validation. - Docs โ†’ https://docs.browserbase.com/features/sessions + Docs โ†’ https://docs.browserbase.com/fundamentals/create-browser-session ## GLOSSARY - act: perform UI actions from a prompt (click, type, select) - Docs โ†’ https://docs.stagehand.dev/basics/act + Docs โ†’ https://docs.stagehand.dev/v4/basics/act - extract: pull structured data from pages using schemas - Docs โ†’ https://docs.stagehand.dev/basics/extract + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract - observe: plan actions and get selectors before executing - Docs โ†’ https://docs.stagehand.dev/basics/observe + Docs โ†’ https://docs.stagehand.dev/v4/basics/observe - browser automation: automated interaction with web applications for booking systems - Docs โ†’ https://docs.browserbase.com/features/sessions + Docs โ†’ https://docs.browserbase.com/fundamentals/create-browser-session - form validation: ensure user input meets booking system requirements ## QUICKSTART @@ -28,7 +30,7 @@ 4. source venv/bin/activate # On Windows: venv\Scripts\activate 5. pip install -r requirements.txt 6. pip install InquirerPy pydantic -7. cp .env.example .env # Add your Browserbase API key, Project ID, and SF Rec Park credentials to .env +7. cp .env.example .env # Add your Browserbase API key and SF Rec Park credentials to .env 8. python main.py ## EXPECTED OUTPUT @@ -75,7 +77,7 @@ ## HELPFUL RESOURCES -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction ๐ŸŽฎ Browserbase: https://www.browserbase.com ๐Ÿ’ก Try it out: https://www.browserbase.com/playground ๐Ÿ”ง Templates: https://www.browserbase.com/templates diff --git a/python/pickleball/main.py b/python/pickleball/main.py index 78ceea54..80749217 100644 --- a/python/pickleball/main.py +++ b/python/pickleball/main.py @@ -1,422 +1,175 @@ -# SF Court Booking Automation - See README.md for full documentation +"""Find, and optionally book, SF courts with Stagehand V4.""" + import asyncio +import json import os -from datetime import datetime, timedelta +from datetime import date, timedelta from dotenv import load_dotenv -from InquirerPy import inquirer from pydantic import BaseModel, Field -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand, browserbase -# Load environment variables load_dotenv() +BOOKING_URL = "https://www.rec.us/organizations/san-francisco-rec-park" -async def login_to_site(page, email: str, password: str) -> None: - print("Logging in...") - # Perform login sequence: each step is atomic to handle dynamic page changes. - await page.act("Click the Login button") - await page.act(f'Fill in the email or username field with "{email}"') - await page.act("Click the next, continue, or submit button to proceed") - await page.act(f'Fill in the password field with "{password}"') - await page.act("Click the login, sign in, or submit button") - print("Logged in") +class Court(BaseModel): + name: str = Field(min_length=1) + opening_times: str = Field(description="Available or displayed time slots") + location: str + availability: str + duration: str | None = None -async def select_filters(page, activity: str, time_of_day: str, selected_date: str) -> None: - print("Selecting the activity") - # Filter by activity type first to narrow down available courts. - await page.act("Click the activites drop down menu") - await page.act(f"Select the {activity} activity") - await page.act("Click the Done button") - print(f"Selecting date: {selected_date}") - # Open calendar to select specific date for court booking. - await page.act("Click the date picker or calendar") +class CourtResults(BaseModel): + courts: list[Court] - # Parse date string to extract day number for calendar selection. - date_parts = selected_date.split("-") - if len(date_parts) != 3: - raise ValueError(f"Invalid date format: {selected_date}. Expected YYYY-MM-DD") - day_number = int(date_parts[2]) - if day_number < 1 or day_number > 31: - raise ValueError(f"Invalid day number: {day_number} from date: {selected_date}") +class BookingConfirmation(BaseModel): + confirmation_message: str | None = None + booking_details: str | None = None + error_message: str | None = None - print(f"Looking for day number: {day_number} in calendar") - # Click specific day number in calendar to select date. - await page.act(f"Click on the number {day_number} in the calendar") - print(f"Selecting time of day: {time_of_day}") - # Filter by time period to find courts available during preferred hours. - await page.act("Click the time filter or time selection dropdown") - await page.act(f"Select {time_of_day} time period") - await page.act("Click the Done button") +def require_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required") + return value - # Apply additional filters to show only available courts that accept reservations. - await page.act("Click Available Only button") - await page.act("Click All Facilities dropdown list") - await page.act("Select Accept Reservations checkbox") - await page.act("Click the Done button") +def requested_preferences() -> tuple[str, str, str]: + activity = os.environ.get("ACTIVITY", "Pickleball") + selected_date = os.environ.get("SELECTED_DATE", str(date.today() + timedelta(days=1))) + time_of_day = os.environ.get("TIME_OF_DAY", "Evening") + if activity not in {"Tennis", "Pickleball"}: + raise RuntimeError("ACTIVITY must be Tennis or Pickleball") + if time_of_day not in {"Morning", "Afternoon", "Evening"}: + raise RuntimeError("TIME_OF_DAY must be Morning, Afternoon, or Evening") + date.fromisoformat(selected_date) + return activity, selected_date, time_of_day -async def check_and_extract_courts(page, time_of_day: str) -> None: - print("Checking for available courts...") - # First observe the page to find all available court booking options. - available_courts = await page.observe( - "Find all available court booking slots, time slots, or court reservation options" +async def login(stagehand: Stagehand, page: object) -> None: + await stagehand.act("Click the Login button", page=page) + await stagehand.act( + "Fill the email or username field with %email%", + page=page, + variables={"email": require_env("SF_REC_PARK_EMAIL")}, ) - print(f"Found {len(available_courts)} available court options") - - # Define schema using Pydantic - class Court(BaseModel): - name: str = Field(..., description="the name or identifier of the court") - opening_times: str = Field( - ..., description="the opening hours or operating times of the court" - ) - location: str = Field(..., description="the location or facility name") - availability: str = Field(..., description="availability status or any restrictions") - duration: str | None = Field( - None, description="the duration of the court session in minutes" - ) - - class CourtData(BaseModel): - courts: list[Court] - - # Extract structured court data using Pydantic schema for type safety and validation. - court_data = await page.extract( - "Extract all available court booking information including court names, time slots, locations, and any other relevant details", - schema=CourtData, + await stagehand.act("Click the next, continue, or submit button", page=page) + await stagehand.act( + "Fill the password field with %password%", + page=page, + variables={"password": require_env("SF_REC_PARK_PASSWORD")}, ) - - # Check if any courts are actually available by filtering out unavailable status messages. - has_available_courts = any( - "no free spots" not in court.availability.lower() - and "unavailable" not in court.availability.lower() - and "next available" not in court.availability.lower() - and "the next available reservation" not in court.availability.lower() - for court in court_data.courts + await stagehand.act("Click the login, sign in, or submit button", page=page) + + +async def select_filters( + stagehand: Stagehand, + page: object, + activity: str, + selected_date: str, + time_of_day: str, +) -> None: + day_number = date.fromisoformat(selected_date).day + await stagehand.act("Click the Activities dropdown", page=page) + await stagehand.act(f"Select the {activity} activity", page=page) + await stagehand.act("Click Done", page=page) + await stagehand.act("Click the date picker or calendar", page=page) + await stagehand.act(f"Click day {day_number} in the calendar", page=page) + await stagehand.act("Click the time filter", page=page) + await stagehand.act(f"Select the {time_of_day} time period", page=page) + await stagehand.act("Click Done", page=page) + await stagehand.act("Enable Available Only", page=page) + await stagehand.act("Click the All Facilities dropdown", page=page) + await stagehand.act("Select Accept Reservations", page=page) + await stagehand.act("Click Done", page=page) + + +async def extract_courts(stagehand: Stagehand, page: object) -> list[Court]: + extracted = await stagehand.extract( + "Extract every displayed court option, its time slots, location, availability, and duration", + CourtResults, + page=page, ) - - # If no courts available for selected time, try alternative time periods as fallback. - if len(available_courts) == 0 or not has_available_courts: - print("No courts available for selected time. Trying different time periods...") - - # Generate alternative time periods to try if original selection has no availability. - alternative_times = ( - ["Afternoon", "Evening"] - if time_of_day == "Morning" - else ["Morning", "Evening"] - if time_of_day == "Afternoon" - else ["Morning", "Afternoon"] - ) - - for alt_time in alternative_times: - print(f"Trying {alt_time} time period...") - - # Change time filter to alternative time period and check availability. - await page.act(f'Click the time filter dropdown that currently shows "{time_of_day}"') - await page.act(f"Select {alt_time} from the time period options") - await page.act("Click the Done button") - - alt_available_courts = await page.observe( - "Find all available court booking slots, time slots, or court reservation options" - ) - print(f"Found {len(alt_available_courts)} available court options for {alt_time}") - - if len(alt_available_courts) > 0: - alt_court_data = await page.extract( - "Extract all available court booking information including court names, time slots, locations, and any other relevant details", - schema=CourtData, - ) - - has_alt_available_courts = any( - "no free spots" not in court.availability.lower() - and "unavailable" not in court.availability.lower() - and "next available" not in court.availability.lower() - and "the next available reservation" not in court.availability.lower() - for court in alt_court_data.courts - ) - - # If alternative time has available courts, use that data and stop searching. - if has_alt_available_courts: - print(f"Found actually available courts for {alt_time}!") - court_data.courts = alt_court_data.courts - has_available_courts = True - break - - # If still no available courts found, extract final court data for display. - if not has_available_courts: - print("Extracting final court information...") - final_court_data = await page.extract( - "Extract all available court booking information including court names, time slots, locations, and any other relevant details", - schema=CourtData, - ) - court_data.courts = final_court_data.courts - - # Display all found court information to user for review and selection. - print("Available Courts:") - if court_data.courts and len(court_data.courts) > 0: - for index, court in enumerate(court_data.courts): - print(f"{index + 1}. {court.name}") - print(f" Opening Times: {court.opening_times}") - print(f" Location: {court.location}") - print(f" Availability: {court.availability}") - if court.duration: - print(f" Duration: {court.duration} minutes") - print("") - else: - print("No court data available to display") - - -async def book_court(page) -> None: - print("Starting court booking process...") - - try: - # Select the first available court time slot for booking. - print("Clicking the top available time slot...") - await page.act("Click the first available time slot or court booking option") - - # Select participant from dropdown - assumes only one participant is available. - print("Opening participant dropdown...") - await page.act("Click the participant dropdown menu or select participant field") - await page.act("Click the only named participant in the dropdown!") - - # Complete booking process and trigger verification code request. - print("Clicking the book button to complete reservation...") - await page.act("Click the book, reserve, or confirm booking button") - await page.act("Click the Send Code Button") - - # Prompt user for verification code received via SMS/email for booking confirmation. - def validate_code(text): - if not text.strip(): - raise ValueError("Please enter a verification code") - return True - - def get_verification_code(): - return inquirer.text( - message="Please enter the verification code you received:", - validate=validate_code, - ).execute() - - verification_code = await asyncio.to_thread(get_verification_code) - - print(f"Verification code: {verification_code}") - - # Enter verification code and confirm booking to complete reservation. - await page.act(f'Fill in the verification code field with "{verification_code}"') - await page.act("Click the confirm button") - - # Define schema using Pydantic - class Confirmation(BaseModel): - confirmation_message: str | None = Field( - None, description="any confirmation or success message" - ) - booking_details: str | None = Field( - None, description="booking details like time, court, etc." - ) - error_message: str | None = Field( - None, description="any error message if booking failed" - ) - - # Extract booking confirmation details to verify successful reservation. - print("Checking for booking confirmation...") - confirmation = await page.extract( - "Extract any booking confirmation message, success notification, or reservation details", - schema=Confirmation, - ) - - # Display confirmation details if booking was successful. - if confirmation.confirmation_message or confirmation.booking_details: - print("Booking Confirmed!") - if confirmation.confirmation_message: - print(f"{confirmation.confirmation_message}") - if confirmation.booking_details: - print(f"{confirmation.booking_details}") - - # Display error message if booking failed. - if confirmation.error_message: - print("Booking Error:") - print(confirmation.error_message) - - except Exception as error: - print(f"Error during court booking: {error}") - raise error - - -async def select_activity() -> str: - # Prompt user to select between Tennis and Pickleball activities. - def get_activity(): - return inquirer.select( - message="Please select an activity:", - choices=[ - {"name": "Tennis", "value": "Tennis"}, - {"name": "Pickleball", "value": "Pickleball"}, - ], - default="Tennis", - ).execute() - - activity = await asyncio.to_thread(get_activity) - - print(f"Selected: {activity}") - return activity - - -async def select_time_of_day() -> str: - # Prompt user to select preferred time period for court booking. - def get_time_of_day(): - return inquirer.select( - message="Please select the time of day:", - choices=[ - {"name": "Morning (Before 12 PM)", "value": "Morning"}, - {"name": "Afternoon (After 12 PM)", "value": "Afternoon"}, - {"name": "Evening (After 5 PM)", "value": "Evening"}, - ], - default="Morning", - ).execute() - - time_of_day = await asyncio.to_thread(get_time_of_day) - - print(f"Selected: {time_of_day}") - return time_of_day - - -async def select_date() -> str: - # Generate date options for the next 7 days including today. - today = datetime.now() - date_options = [] - - for i in range(7): - date = today + timedelta(days=i) - - day_name = date.strftime("%A") - month_day = date.strftime("%b %-d") - full_date = date.strftime("%Y-%m-%d") - - display_name = f"{day_name}, {month_day} (Today)" if i == 0 else f"{day_name}, {month_day}" - - date_options.append({"name": display_name, "value": full_date}) - - # Prompt user to select from available date options. - def get_date(): - return inquirer.select( - message="Please select a date:", choices=date_options, default=date_options[0]["value"] - ).execute() - - selected_date = await asyncio.to_thread(get_date) - - # Format selected date for display and return ISO date string. - selected_date_obj = datetime.strptime(selected_date, "%Y-%m-%d") - display_date = selected_date_obj.strftime("%A, %B %-d, %Y") - - print(f"Selected: {display_date}") - return selected_date - - -async def book_tennis_paddle_court(): - print("Starting tennis/paddle court booking automation in SF...") - - # Load credentials from environment variables for SF Rec & Parks login. - email = os.environ.get("SF_REC_PARK_EMAIL") - password = os.environ.get("SF_REC_PARK_PASSWORD") - - # Validate that required credentials are available before proceeding. - if not email or not password: - raise ValueError("Missing SF_REC_PARK_EMAIL or SF_REC_PARK_PASSWORD environment variables") - - # Collect user preferences for activity, date, and time selection. - activity = await select_activity() - selected_date = await select_date() - time_of_day = await select_time_of_day() - - print(f"Booking {activity} courts in San Francisco for {time_of_day} on {selected_date}...") - - # Initialize Stagehand with Browserbase for AI-powered browser automation. - print("Initializing Stagehand with Browserbase") - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - verbose=1, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - "timeout": 900, - "region": "us-west-2", - }, + courts = extracted.data.courts + if not courts: + raise RuntimeError("The booking site returned no court availability information") + return courts + + +async def book_first_court(stagehand: Stagehand, page: object) -> BookingConfirmation: + await stagehand.act("Click the first available court time slot", page=page) + await stagehand.act("Open the participant dropdown", page=page) + await stagehand.act("Select the only named participant", page=page) + await stagehand.act("Click the Book or Reserve button", page=page) + await stagehand.act("Click Send Code", page=page) + + verification_code = input("Enter the one-time booking verification code: ").strip() + if not verification_code: + raise RuntimeError("A verification code is required to finish the reservation") + await stagehand.act( + "Fill the verification-code field with %code%", + page=page, + variables={"code": verification_code}, + ) + await stagehand.act("Click Confirm", page=page) + extracted = await stagehand.extract( + "Extract the booking confirmation, reservation details, and any error message", + BookingConfirmation, + page=page, + ) + confirmation = extracted.data + if confirmation.error_message: + raise RuntimeError(f"Booking failed: {confirmation.error_message}") + if not confirmation.confirmation_message and not confirmation.booking_details: + raise RuntimeError("The site did not show a booking confirmation") + return confirmation + + +async def main() -> None: + activity, selected_date, time_of_day = requested_preferences() + print(f"Finding {activity} courts for {time_of_day} on {selected_date}") + + browser = await browserbase.launch( + api_key=require_env("BROWSERBASE_API_KEY"), + timeout=900, + region="us-west-2", ) - - try: - # Start browser session and connect to SF Rec & Parks booking system. - async with Stagehand(config) as stagehand: - print("Browserbase Session Started") - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id - - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") - - page = stagehand.page - - # Navigate to SF Rec & Parks booking site with extended timeout for slow loading. - print("Navigating to court booking site...") - await page.goto( - "https://www.rec.us/organizations/san-francisco-rec-park", - wait_until="domcontentloaded", - timeout=60000, - ) - - # Execute booking workflow: login, filter, find courts, and complete booking. - await login_to_site(page, email, password) - await select_filters(page, activity, time_of_day, selected_date) - await check_and_extract_courts(page, time_of_day) - await book_court(page) - - print("\nBrowser session closed") - - except Exception as error: - print(f"Error during court booking: {error}") - raise error - - -async def main(): - # Display welcome message and explain the automation workflow to user. - print("Welcome to SF Court Booking Automation!") - print("") - print("This tool automates tennis and pickleball court bookings in San Francisco.") - print("Here's what we'll do:") - print("") - print("1. Navigate to https://www.rec.us/organizations/san-francisco-rec-park") - print("2. Use automated login with your credentials") - print("3. Select your preferred activity, date, and time") - print("4. Find and book available courts automatically") - print("5. Handle verification codes and confirmation") - print("") - try: - # Execute the complete court booking automation workflow. - await book_tennis_paddle_court() - - print("Court booking completed successfully!") - print("Your court has been reserved. Check your email for confirmation details.") - except Exception as error: - print("Failed to complete court booking") - print(f"Error: {error}") - exit(1) + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto(BOOKING_URL, wait_until="domcontentloaded", timeout=60_000) + await login(stagehand, page) + await select_filters(stagehand, page, activity, selected_date, time_of_day) + courts = await extract_courts(stagehand, page) + print("Live court availability:") + print(json.dumps([court.model_dump(mode="json") for court in courts], indent=2)) + + if os.environ.get("BOOK_COURT", "false").lower() == "true": + confirmation = await book_first_court(stagehand, page) + print("Booking confirmed:") + print(json.dumps(confirmation.model_dump(mode="json"), indent=2)) + else: + print("Set BOOK_COURT=true to reserve a court") + finally: + await stagehand.close() + finally: + await browser.close() if __name__ == "__main__": try: asyncio.run(main()) - except Exception as err: - print(f"Application error: {err}") - exit(1) + except Exception as error: + print(f"Court workflow failed: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/pickleball/pyproject.toml b/python/pickleball/pyproject.toml new file mode 100644 index 00000000..9253de43 --- /dev/null +++ b/python/pickleball/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "pickleball" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[tool.uv] +package = false diff --git a/python/playwright-mfa-handling/.env.example b/python/playwright-mfa-handling/.env.example new file mode 100644 index 00000000..8c3f7702 --- /dev/null +++ b/python/playwright-mfa-handling/.env.example @@ -0,0 +1,2 @@ +# Browserbase Configuration +BROWSERBASE_API_KEY=your_browserbase_api_key diff --git a/python/playwright-mfa-handling/README.md b/python/playwright-mfa-handling/README.md new file mode 100644 index 00000000..986d345f --- /dev/null +++ b/python/playwright-mfa-handling/README.md @@ -0,0 +1,129 @@ +# Playwright + Browserbase: MFA Handling - TOTP Automation + +## AT A GLANCE + +- Goal: Automate MFA (Multi-Factor Authentication) completion using TOTP (Time-based One-Time Password) code generation with Playwright. +- Pure Playwright: Uses Playwright directly with Browserbase's cloud browser infrastructure (no Stagehand abstraction). +- TOTP Generation: Implements RFC 6238 compliant algorithm to generate time-based authentication codes programmatically. +- Automatic Form Filling: Uses Playwright selectors to fill MFA forms without user interaction. +- Retry Logic: Handles time window edge cases by regenerating codes and retrying authentication when needed. +- Docs -> https://playwright.dev/python/docs/intro + +## GLOSSARY + +- Playwright: Microsoft's browser automation library for reliable end-to-end testing + Docs -> https://playwright.dev/python/docs/intro +- CDP: Chrome DevTools Protocol - used to connect Playwright to Browserbase cloud browsers +- TOTP: Time-based One-Time Password - a 6-digit code that changes every 30 seconds, generated using HMAC-SHA1 algorithm +- RFC 6238: Standard specification for TOTP authentication codes used by Google Authenticator, Authy, and other authenticator apps + +## STAGEHAND VS PLAYWRIGHT + +This template uses **pure Playwright** for browser automation. Stagehand V4 is the SDK for browser agents and adds natural-language **act**, **observe**, and **extract** operations to a browser that your application owns. Here's how they compare: + +| Task | Stagehand V4 โ€” natural language (you describe intent) | Playwright โ€” specific selectors (you target exact elements) | +| ------------- | --------------------------------------------------------- | ----------------------------------------------------------- | +| Fill email | _"Find the email field and type the user's email"_ | `page.locator('input[type="email"]').fill(email)` | +| Fill password | _"Find the password field and enter the password"_ | `page.locator('input[type="password"]').fill(password)` | +| Click submit | _"Click the submit or sign-in button"_ | `page.locator('input[type="submit"]').click()` | +| Check result | _"Did login succeed or fail? Return success and message"_ | `page.locator('text="Login Success"').is_visible()` | + +**Example - Filling the login form:** + +```python +# Stagehand V4: launch a browser, then attach Stagehand to it +from stagehand import Stagehand, browserbase + +browser = await browserbase.launch(api_key=BROWSERBASE_API_KEY) +stagehand = await Stagehand.create( + browser=browser, +) +pages = await browser.context.pages() +page = pages[0] if pages else await browser.context.new_page() +await page.goto("https://example.com/login") + +await stagehand.act(f"Fill the email field with {email}", page=page) +await stagehand.act(f"Fill the password field with {password}", page=page) +await stagehand.act(f"Fill the TOTP field with {totp_code}", page=page) + +# Playwright: Explicit selectors, you specify how to find elements +await page.locator('input[type="email"]').fill(email) +await page.locator('input[type="password"]').fill(password) +await page.locator("form input").nth(2).fill(totp_code) +``` + +**Example - Checking authentication result:** + +```python +# Stagehand V4: extract returns a typed response envelope +extract_response = await stagehand.extract( + "Check if the login was successful and return its message", + AuthResult, + page=page, +) +result = extract_response.data + +await stagehand.close() +await browser.close() + +# Playwright: Must check for specific elements/text on the page +has_success = await page.locator('text="Login Success"').is_visible() +has_failure = await page.locator('text="Login Failure"').is_visible() +``` + +## QUICKSTART + +1. cd playwright-mfa-handling +2. python -m venv venv && source venv/bin/activate +3. pip install . +4. playwright install chromium +5. cp .env.example .env +6. Add required API keys/IDs to .env (BROWSERBASE_API_KEY) +7. python main.py + +## EXPECTED OUTPUT + +- Creates Browserbase session and connects via CDP +- Displays live session link for monitoring +- Navigates to TOTP challenge demo page (authenticationtest.com/totpChallenge/) +- Generates TOTP code using RFC 6238 algorithm +- Fills in email, password, and TOTP code fields using Playwright selectors +- Submits authentication form +- Checks authentication result +- Retries with fresh code if initial attempt fails (handles time window edge cases) +- Closes browser session cleanly + +## COMMON PITFALLS + +- Dependency install errors: ensure pip install . completed +- Missing Playwright browsers: run `playwright install chromium` after installing dependencies +- Missing credentials: verify .env contains BROWSERBASE_API_KEY +- TOTP code expiration: codes are valid for 30 seconds - if authentication fails, the script automatically retries with a fresh code +- CDP connection issues: ensure stable internet connection for reliable Browserbase connection +- Selector changes: if the demo site structure changes, Playwright selectors may need updating +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +- Automated authentication: Complete MFA challenges automatically when session persistence isn't enough +- TOTP integration: Store encrypted TOTP secrets during user onboarding and generate codes programmatically +- Zero-touch MFA: Eliminate user interaction for MFA completion in automated workflows +- Session recovery: Automatically handle MFA prompts when re-authenticating expired sessions +- E2E testing: Test MFA flows in your application's test suite + +## NEXT STEPS + +- Secure storage: Implement encrypted TOTP secret storage (AES-256) in your database +- Multiple time windows: Add support for trying +/-1 time window (60s range) if current code fails +- SMS/Email MFA: Extend to support SMS codes (via Twilio/Bandwidth API) or email codes (via Gmail API/IMAP) +- Backup codes: Implement fallback to backup codes stored during initial MFA setup +- Context integration: Combine with Browserbase Contexts to minimize MFA prompts +- Page Object Model: Refactor selectors into reusable page objects for maintainability + +## HELPFUL RESOURCES + +- Playwright Python Docs: https://playwright.dev/python/docs/intro +- Browserbase: https://www.browserbase.com +- Try it out: https://www.browserbase.com/playground +- Templates: https://www.browserbase.com/templates +- Need help? support@browserbase.com diff --git a/python/playwright-mfa-handling/main.py b/python/playwright-mfa-handling/main.py new file mode 100644 index 00000000..fc6edf64 --- /dev/null +++ b/python/playwright-mfa-handling/main.py @@ -0,0 +1,247 @@ +# Playwright + Browserbase: MFA Handling - TOTP Automation - See README.md for full documentation + +import asyncio +import hashlib +import hmac +import os +import time + +from browserbase import Browserbase +from dotenv import load_dotenv +from playwright.async_api import Browser, BrowserContext, Page, async_playwright + +# Load environment variables +load_dotenv() + +# Demo site URL for TOTP challenge testing +DEMO_URL = "https://authenticationtest.com/totpChallenge/" + +# Test credentials (displayed on the demo page) +TEST_CREDENTIALS = { + "email": "totp@authenticationtest.com", + "password": "pa$$w0rd", + "totp_secret": "I65VU7K5ZQL7WB4E", +} + + +def generate_totp(secret: str, window: int = 0) -> str: + """ + Generate TOTP code (Time-based One-Time Password) using RFC 6238 compliant algorithm. + + Same algorithm used by Google Authenticator, Authy, and other authenticator apps. + """ + # Convert base32 secret to bytes + base32chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" + bits = "" + hex_str = "" + + secret = secret.upper().rstrip("=") + + for char in secret: + val = base32chars.find(char) + if val == -1: + raise ValueError("Invalid base32 character in secret") + bits += format(val, "05b") + + for i in range(0, len(bits) - 3, 4): + chunk = bits[i : i + 4] + hex_str += format(int(chunk, 2), "x") + + secret_bytes = bytes.fromhex(hex_str) + + # Get current time window (30 second intervals) + time_window = int(time.time() // 30) + window + time_bytes = time_window.to_bytes(8, byteorder="big") + + # Generate HMAC-SHA1 hash + hmac_result = hmac.new(secret_bytes, time_bytes, hashlib.sha1).digest() + + # Dynamic truncation to extract 6-digit code + offset = hmac_result[-1] & 0xF + code = ( + ((hmac_result[offset] & 0x7F) << 24) + | ((hmac_result[offset + 1] & 0xFF) << 16) + | ((hmac_result[offset + 2] & 0xFF) << 8) + | (hmac_result[offset + 3] & 0xFF) + ) + + # Return 6-digit code with leading zeros + return str(code % 1000000).zfill(6) + + +async def create_browserbase_session() -> tuple[Browser, BrowserContext, Page, str]: + """Create a Browserbase session and connect via CDP.""" + api_key = os.environ.get("BROWSERBASE_API_KEY") + + if not api_key: + raise ValueError("BROWSERBASE_API_KEY environment variable is required") + + bb = Browserbase(api_key=api_key) + session = bb.sessions.create() + + print(f"Session created: https://browserbase.com/sessions/{session.id}") + + pw = await async_playwright().start() + browser = await pw.chromium.connect_over_cdp(session.connect_url) + + context = browser.contexts[0] + if not context: + raise ValueError("No default context found") + + page = context.pages[0] + if not page: + raise ValueError("No page found in default context") + + return browser, context, page, session.id + + +async def fill_login_form(page: Page, email: str, password: str, totp_code: str) -> None: + """Fill in the login form with credentials and TOTP code.""" + # Fill email field + email_input = page.locator('input[type="email"], input[placeholder*="email" i]').first + await email_input.wait_for(state="visible", timeout=10000) + await email_input.fill(email) + print("Filled email field") + + # Fill password field + password_input = page.locator('input[type="password"]') + await password_input.fill(password) + print("Filled password field") + + # Fill TOTP code field (third input in the form) + totp_input = page.locator("form input").nth(2) + await totp_input.fill(totp_code) + print("Filled TOTP code field") + + +async def submit_form(page: Page) -> None: + """Submit the login form.""" + submit_button = page.locator('input[type="submit"], button[type="submit"], form button').first + await submit_button.click() + print("Clicked submit button") + + await page.wait_for_load_state("domcontentloaded", timeout=20000) + + +async def check_login_result(page: Page) -> dict: + """Check if the login was successful.""" + # Check for failure indicator + failure_heading = page.locator('text="Login Failure"') + has_failure = await failure_heading.is_visible() + + if has_failure: + error_message = await page.locator("body").text_content() or "" + message = ( + "Sorry -- You have not successfully logged in" + if "not successfully logged in" in error_message + else "Login failed" + ) + return {"success": False, "message": message} + + # Check for success indicator + success_heading = page.locator('text="Login Success"') + has_success = await success_heading.is_visible() + + if has_success: + return {"success": True, "message": "Successfully logged in with TOTP"} + + # Fallback: check page content + page_content = await page.locator("body").text_content() or "" + if "success" in page_content.lower(): + return {"success": True, "message": "Login appears successful"} + + return {"success": False, "message": "Unable to determine login result"} + + +async def main(): + print("Starting MFA Handling - TOTP Automation (Playwright + Browserbase)...\n") + + browser = None + + try: + # Create browser session + browser, context, page, session_id = await create_browserbase_session() + + print(f"Live View: https://browserbase.com/sessions/{session_id}\n") + + # Navigate to TOTP challenge demo page + print("Navigating to TOTP Challenge page...") + await page.goto(DEMO_URL, wait_until="domcontentloaded") + + # Generate TOTP code using RFC 6238 algorithm + totp_code = generate_totp(TEST_CREDENTIALS["totp_secret"]) + seconds_left = 30 - (int(time.time()) % 30) + print(f"Generated TOTP code: {totp_code} (valid for {seconds_left}s)") + + # Fill in login form + print("\nFilling login form...") + await fill_login_form( + page, + TEST_CREDENTIALS["email"], + TEST_CREDENTIALS["password"], + totp_code, + ) + + # Submit the form + print("\nSubmitting form...") + await submit_form(page) + + # Check if login succeeded + print("\nChecking authentication result...") + result = await check_login_result(page) + + if result["success"]: + print("\nSUCCESS! TOTP authentication completed automatically!") + print(f"Result: {result['message']}") + else: + print(f"\nInitial attempt may have failed: {result['message']}") + print("Retrying with a fresh TOTP code...\n") + + # Navigate back and retry with new code + await page.goto(DEMO_URL, wait_until="domcontentloaded") + + new_code = generate_totp(TEST_CREDENTIALS["totp_secret"]) + print(f"New TOTP code: {new_code}") + + await fill_login_form( + page, + TEST_CREDENTIALS["email"], + TEST_CREDENTIALS["password"], + new_code, + ) + await submit_form(page) + + retry_result = await check_login_result(page) + + if retry_result["success"]: + print("\nSuccess on retry!") + print(f"Result: {retry_result['message']}") + else: + print("\nAuthentication failed after retry") + print(f"Result: {retry_result['message']}") + + print("\nMFA handling completed") + + except Exception as error: + print(f"Error during MFA handling: {error}") + import traceback + + traceback.print_exc() + raise + + finally: + if browser: + await browser.close() + print("Browser closed successfully") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"\nError in MFA handling: {err}") + print("\nCommon issues:") + print(" - Check .env file has BROWSERBASE_API_KEY") + print(" - TOTP code may have expired (try running again)") + print(" - Page structure may have changed") + exit(1) diff --git a/python/playwright-mfa-handling/pyproject.toml b/python/playwright-mfa-handling/pyproject.toml new file mode 100644 index 00000000..f7f049bc --- /dev/null +++ b/python/playwright-mfa-handling/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "playwright-mfa-handling" +version = "0.1.0" +description = "Playwright + Browserbase: MFA Handling - TOTP Automation" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "playwright", + "browserbase>=1.7.0", + "python-dotenv", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/python/playwright/basic-captcha-solving/.env.example b/python/playwright/basic-captcha-solving/.env.example new file mode 100644 index 00000000..9e87821b --- /dev/null +++ b/python/playwright/basic-captcha-solving/.env.example @@ -0,0 +1,2 @@ +# Browserbase credentials - Get these from https://www.browserbase.com/overview +BROWSERBASE_API_KEY=your_browserbase_api_key_here diff --git a/python/playwright/basic-captcha-solving/README.md b/python/playwright/basic-captcha-solving/README.md new file mode 100644 index 00000000..11b33900 --- /dev/null +++ b/python/playwright/basic-captcha-solving/README.md @@ -0,0 +1,124 @@ +# Playwright + Browserbase: Basic CAPTCHA Solving (Python) + +## AT A GLANCE + +- Goal: Demonstrate automatic CAPTCHA solving using Playwright with Browserbase's built-in captcha solving capabilities. +- Automated Solving: Browserbase automatically detects and solves CAPTCHAs in the background. CAPTCHA solving is **enabled by default** - you don't need to set `solveCaptchas: True` unless you want to explicitly enable it (or set it to `False` to disable). +- Timeout Handling: Includes 60-second timeout to prevent indefinite waits on captcha solving failures. +- Progress Monitoring: Listen for console messages (`browserbase-solving-started`, `browserbase-solving-finished`) to track captcha solving progress in real-time. +- Docs -> https://docs.browserbase.com/features/stealth-mode#captcha-solving + +## GLOSSARY + +- Browserbase SDK: Python SDK for creating and managing browser sessions in Browserbase's cloud infrastructure. + Docs -> https://docs.browserbase.com/ +- solveCaptchas: Browserbase browser setting that enables automatic CAPTCHA solving. Enabled by default for Basic and Advanced Stealth Mode. + Docs -> https://docs.browserbase.com/features/stealth-mode#captcha-solving +- console messages: Browser console events that indicate captcha solving status: + - `browserbase-solving-started`: emitted when CAPTCHA detection begins + - `browserbase-solving-finished`: emitted when CAPTCHA solving completes +- custom CAPTCHA solving: For non-standard or custom captcha providers, you can specify CSS selectors for the captcha image and input field using `captchaImageSelector` and `captchaInputSelector` in browser_settings. + +## PLAYWRIGHT VS STAGEHAND + +| Feature | Playwright (this template) | Stagehand | +| --------------- | ----------------------------- | ------------------------------------ | +| Actions | `page.click()`, `page.goto()` | `stagehand.act("Click the button")` | +| Data Extraction | Manual DOM queries | `stagehand.extract("Get the price")` | + +## CAPTCHA SOLVING DETAILS + +### How CAPTCHA Solving Works + +Browserbase provides integrated CAPTCHA solving to handle challenges automatically: + +- **Automatic Detection**: When a CAPTCHA is detected on a page, Browserbase attempts to solve it in the background +- **Solving Time**: CAPTCHA solving typically takes between 5-30 seconds, depending on the CAPTCHA type and complexity +- **Default Behavior**: CAPTCHA solving is enabled by default for Basic and Advanced Stealth Mode +- **Proxies**: It's recommended to enable proxies when using CAPTCHA solving for higher success rates + +### Custom CAPTCHA Solving + +For non-standard or custom captcha providers, you can specify CSS selectors to guide the solution process: + +```python +session = bb.sessions.create( + browser_settings={ + "solveCaptchas": True, + "captchaImageSelector": "#custom-captcha-image-id", + "captchaInputSelector": "#custom-captcha-input-id", + }, +) +``` + +To find the selectors: + +1. Right-click on the captcha image and select "Inspect" to get the image selector +2. Right-click on the input field and select "Inspect" to get the input selector +3. Use the element's `id` or a CSS selector that uniquely identifies it + +### Disabling CAPTCHA Solving + +If you want to disable automatic captcha solving, set `solveCaptchas: False` in browser_settings: + +```python +session = bb.sessions.create( + browser_settings={"solveCaptchas": False}, +) +``` + +### Enabling Proxies for Higher Success Rates + +Enable proxies alongside CAPTCHA solving for better success rates: + +```python +session = bb.sessions.create( + browser_settings={"solveCaptchas": True}, + proxies=True, # Enable Browserbase managed proxies +) +``` + +## QUICKSTART + +1. `cd python/playwright/basic-captcha-solving` +2. `uv venv venv` +3. `source venv/bin/activate` # On Windows: `venv\Scripts\activate` +4. `uv pip install .` # Install dependencies from pyproject.toml +5. `playwright install chromium` # Install browser binaries +6. `cp .env.example .env` # Add your Browserbase API key to .env +7. `python main.py` + +## EXPECTED OUTPUT + +- Validates environment variables for Browserbase credentials +- Creates Browserbase session with captcha solving enabled +- Displays session ID and live view link for monitoring +- Connects to browser via Chrome DevTools Protocol (CDP) +- Navigates to CAPTCHA demo page +- Waits for Browserbase to automatically solve the CAPTCHA (with 60s timeout) +- Logs CAPTCHA solving progress messages +- Clicks submit button after CAPTCHA is solved +- Verifies successful captcha solving by checking for success message +- Closes session cleanly and displays replay link + +## COMMON PITFALLS + +- Missing credentials: verify .env contains BROWSERBASE_API_KEY +- Browser not installed: run `playwright install chromium` to install browser binaries +- CAPTCHA solving timeout: the template uses a 60-second timeout; increase if needed for complex CAPTCHAs +- No browser context: ensure the Browserbase session was created successfully before connecting +- Proxies not enabled: enable proxies in session creation for higher CAPTCHA solving success rates +- Demo page inaccessible: verify the CAPTCHA demo page URL is accessible and hasn't changed +- Console message timing: ensure console event listeners are set up before navigating to the page +- Custom captcha selectors: for non-standard CAPTCHAs, verify that `captchaImageSelector` and `captchaInputSelector` are correctly defined +- Import errors: activate your virtual environment if you created one + +## HELPFUL RESOURCES + +- Browserbase Docs: https://docs.browserbase.com +- Browserbase Python Integration: https://docs.browserbase.com/introduction/playwright +- Playwright Python Docs: https://playwright.dev/python/docs/intro +- Browserbase Dashboard: https://www.browserbase.com +- Playground: https://www.browserbase.com/playground +- Templates: https://www.browserbase.com/templates +- Need help? support@browserbase.com diff --git a/python/playwright/basic-captcha-solving/main.py b/python/playwright/basic-captcha-solving/main.py new file mode 100644 index 00000000..f6e0009c --- /dev/null +++ b/python/playwright/basic-captcha-solving/main.py @@ -0,0 +1,140 @@ +# Basic CAPTCHA Solving with Playwright + Browserbase - See README.md for full documentation +# +# This template uses Playwright directly with the Browserbase SDK for lower-level browser control. +# For a higher-level API with natural language commands, see the Stagehand template instead. + +import asyncio +import os + +from browserbase import Browserbase +from dotenv import load_dotenv +from playwright.async_api import async_playwright + +load_dotenv() + +DEMO_URL = "https://google.com/recaptcha/api2/demo" +SOLVE_CAPTCHAS = True # Set to False to disable automatic captcha solving + + +def validate_env() -> str: + """Validate required environment variables before starting. + + Browserbase credentials are required to create browser sessions. + + Returns: + The api_key string + + Raises: + ValueError: If required environment variables are missing + """ + api_key = os.environ.get("BROWSERBASE_API_KEY") + + if not api_key: + raise ValueError("BROWSERBASE_API_KEY environment variable is required") + + return api_key + + +async def main(): + print("Starting Playwright + Browserbase CAPTCHA Example...") + + api_key = validate_env() + + # Initialize the Browserbase SDK client. + bb = Browserbase(api_key=api_key) + + # Create a new browser session with captcha solving enabled. + # solveCaptchas: Browserbase setting that enables automatic CAPTCHA solving. + # Docs: https://docs.browserbase.com/features/stealth-mode#captcha-solving + print("Creating Browserbase session with captcha solving enabled...") + session = bb.sessions.create( + browser_settings={"solveCaptchas": SOLVE_CAPTCHAS}, + ) + + print(f"Session created! ID: {session.id}") + print(f"Live View: https://browserbase.com/sessions/{session.id}") + + async with async_playwright() as playwright: + # Connect to the browser via Chrome DevTools Protocol (CDP). + # This gives direct Playwright control over the Browserbase-managed browser. + print("Connecting to browser via CDP...") + browser = await playwright.chromium.connect_over_cdp(session.connect_url) + + # Get the default browser context and page from the connected browser. + context = browser.contexts[0] + if not context: + raise RuntimeError("No browser context found") + + page = context.pages[0] + if not page: + raise RuntimeError("No page found in context") + + try: + # Set up captcha solving state tracking. + captcha_solved = asyncio.Event() + + # Listen for browser console messages indicating captcha solving progress. + # Browserbase emits these events when automatic captcha solving is active: + # - "browserbase-solving-started": CAPTCHA detection began + # - "browserbase-solving-finished": CAPTCHA solving completed + if SOLVE_CAPTCHAS: + + def handle_console(msg): + text = msg.text + if text == "browserbase-solving-started": + print("Captcha solving in progress...") + elif text == "browserbase-solving-finished": + print("Captcha solving completed!") + captcha_solved.set() + + page.on("console", handle_console) + + # Navigate to CAPTCHA demo page to test CAPTCHA solving. + print("Navigating to CAPTCHA demo page...") + await page.goto(DEMO_URL, wait_until="domcontentloaded") + + # Wait for Browserbase to solve the captcha automatically. + # CAPTCHA solving typically takes 5-30 seconds depending on type and complexity. + # We use a 60-second timeout to prevent hanging indefinitely on failures. + if SOLVE_CAPTCHAS and not captcha_solved.is_set(): + print("Waiting for captcha to be solved...") + try: + await asyncio.wait_for(captcha_solved.wait(), timeout=60.0) + except asyncio.TimeoutError: + raise TimeoutError("Captcha solving timed out after 60 seconds") + elif not SOLVE_CAPTCHAS: + print("Captcha solving is disabled. Skipping wait...") + + # Submit the form after captcha is solved. + print("Clicking submit button...") + await page.click('input[type="submit"]') + + await page.wait_for_load_state("domcontentloaded") + + # Display the resulting page content. + page_content = await page.text_content("body") + print(f"Page content: {page_content[:500] if page_content else 'None'}") + + except Exception as error: + print(f"Error during CAPTCHA solving: {error}") + raise + + finally: + # Always close the browser to release resources and end the Browserbase session. + await browser.close() + print("\nSession closed successfully") + print(f"View replay at: https://browserbase.com/sessions/{session.id}") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Application error: {err}") + print("\nCommon issues:") + print(" - Check .env file has BROWSERBASE_API_KEY") + print(" - Verify solveCaptchas is enabled (True by default)") + print(" - Allow up to 60 seconds for CAPTCHA solving to complete") + print(" - Enable proxies for higher success rates") + print("\nDocs: https://docs.browserbase.com/features/stealth-mode#captcha-solving") + exit(1) diff --git a/python/playwright/basic-captcha-solving/pyproject.toml b/python/playwright/basic-captcha-solving/pyproject.toml new file mode 100644 index 00000000..8162629d --- /dev/null +++ b/python/playwright/basic-captcha-solving/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "playwright-basic-captcha-solving" +version = "0.1.0" +description = "Basic CAPTCHA solving with Playwright and Browserbase" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "browserbase>=1.7.0", + "playwright", + "python-dotenv", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/python/playwright/quickstart-playwright/.env.example b/python/playwright/quickstart-playwright/.env.example new file mode 100644 index 00000000..8c897724 --- /dev/null +++ b/python/playwright/quickstart-playwright/.env.example @@ -0,0 +1,6 @@ +# Browserbase Configuration +# Get your API key from: https://www.browserbase.com/settings +BROWSERBASE_API_KEY=your_browserbase_api_key + +# Optional โ€” inferred from your API key if not provided +# BROWSERBASE_PROJECT_ID=your_browserbase_project_id diff --git a/python/playwright/quickstart-playwright/README.md b/python/playwright/quickstart-playwright/README.md new file mode 100644 index 00000000..cb26812e --- /dev/null +++ b/python/playwright/quickstart-playwright/README.md @@ -0,0 +1,67 @@ +# Quickstart: Playwright + Browserbase + +## AT A GLANCE + +- Goal: connect to a cloud browser via Playwright and Browserbase, navigate a real website (SFMOMA), interact with UI elements, and extract page content. +- No AI required: uses Playwright and the Browserbase SDK directly โ€” no model API key needed. +- Minimal setup: one script, one API key, instant cloud browser. + Docs โ†’ https://docs.browserbase.com/introduction/playwright + +## GLOSSARY + +- Session: a full cloud browser instance you connect to via CDP and control with Playwright. + Docs โ†’ https://docs.browserbase.com/introduction/getting-started +- CDP (Chrome DevTools Protocol): the protocol Playwright uses to control the remote browser. + Docs โ†’ https://docs.browserbase.com/introduction/playwright +- Playwright: browser automation library โ€” `page.goto()`, `page.fill()`, `page.click()`, etc. + Docs โ†’ https://playwright.dev/python/docs/api/class-page +- Debug URL: a live view of your session in the Browserbase dashboard. + Docs โ†’ https://docs.browserbase.com/features/session-inspector + +## QUICKSTART + +1. cd python/playwright/quickstart-playwright +2. cp .env.example .env +3. Add BROWSERBASE_API_KEY to .env (get it from https://browserbase.com/settings) +4. uvx --with browserbase --with playwright --with python-dotenv python main.py + +## EXPECTED OUTPUT + +- Creates a Browserbase cloud browser session +- Connects via CDP using Playwright +- Navigates to https://www.sfmoma.org/ and prints the page title +- Opens and closes the search overlay +- Clicks the Membership link and navigates to the membership page +- Extracts the heading and intro text from the membership page +- Closes the browser and prints a session replay link + +## COMMON PITFALLS + +- Missing API key: verify .env contains BROWSERBASE_API_KEY โ€” this is the only required credential +- Project ID confusion: BROWSERBASE_PROJECT_ID is optional โ€” the API infers it from your API key +- Playwright install: run `playwright install chromium` if you get browser-not-found errors (though Browserbase provides the browser) +- Session not closing: always close the browser to avoid leaked sessions +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +โ€ข Getting started with Browserbase and Playwright +โ€ข Cloud browser automation without managing infrastructure +โ€ข Interacting with UI elements and extracting content from web pages +โ€ข Building browser-based workflows with Playwright + +## NEXT STEPS + +โ€ข Add AI extraction: swap in Stagehand for AI-powered `act()`, `extract()`, and `observe()` +โ€ข Enable proxies: pass `proxies=True` in `bb.sessions.create()` for residential proxy support +โ€ข Enable stealth mode: add `browser_settings={"advanced_stealth": True, "solve_captchas": True}` to bypass bot detection +โ€ข Run in parallel: create multiple sessions for concurrent browser automation + +## HELPFUL RESOURCES + +๐Ÿ“š Browserbase Docs: https://docs.browserbase.com +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/playwright/quickstart-playwright/main.py b/python/playwright/quickstart-playwright/main.py new file mode 100644 index 00000000..c129c813 --- /dev/null +++ b/python/playwright/quickstart-playwright/main.py @@ -0,0 +1,74 @@ +# Playwright + Browserbase: Quickstart +# See README.md for full documentation + +import os + +from browserbase import Browserbase +from dotenv import load_dotenv +from playwright.sync_api import Playwright, sync_playwright + +load_dotenv() + +# ============= CONFIGURATION ============= +BROWSERBASE_API_KEY = os.environ["BROWSERBASE_API_KEY"] +# ========================================= + +bb = Browserbase(api_key=BROWSERBASE_API_KEY) + + +def run(playwright: Playwright) -> None: + session = bb.sessions.create() + print(f"Session created, id: {session.id}") + + print("Starting remote browser...") + chromium = playwright.chromium + browser = chromium.connect_over_cdp(session.connect_url) + context = browser.contexts[0] + page = context.pages[0] + + try: + debug_urls = bb.sessions.debug(session.id) + print(f"Session started, live debug accessible here: {debug_urls.debugger_url}.") + + # Navigate to the SFMOMA homepage + page.goto("https://www.sfmoma.org", wait_until="domcontentloaded") + print(f"At URL: {page.url} | Title: {page.title()}") + + wait_timeout = 10_000 + + # Click the search button to open the search overlay + search_button = page.locator('button[aria-label="Open search"]') + search_button.wait_for(state="visible", timeout=wait_timeout) + search_button.click() + print("Clicked search button โ€” search overlay opened") + + # Close the search overlay + close_button = page.locator('button[aria-label="Close search"]') + close_button.wait_for(state="visible", timeout=wait_timeout) + close_button.click() + print("Closed search overlay") + + # Click the Membership link + membership_link = page.get_by_role("link", name="Membership", exact=True) + membership_link.wait_for(state="visible", timeout=wait_timeout) + membership_link.click() + page.wait_for_url("**/membership**", timeout=wait_timeout) + print(f"At URL: {page.url} | Title: {page.title()}") + + # Extract copy from the membership page + heading = page.locator("h1") + heading.wait_for(state="visible", timeout=wait_timeout) + print(f"Heading: {heading.text_content()}") + + intro = page.locator("main p").first + print(f"Intro: {intro.text_content()}") + finally: + page.close() + browser.close() + + print(f"Session complete! View replay at https://browserbase.com/sessions/{session.id}") + + +if __name__ == "__main__": + with sync_playwright() as playwright: + run(playwright) diff --git a/python/playwright/quickstart-playwright/pyproject.toml b/python/playwright/quickstart-playwright/pyproject.toml new file mode 100644 index 00000000..3847d095 --- /dev/null +++ b/python/playwright/quickstart-playwright/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "quickstart-playwright" +version = "1.0.0" +description = "Browserbase + Playwright: Quickstart โ€” connect to a cloud browser, navigate a website, and extract content" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "browserbase>=1.6.0", + "playwright", + "python-dotenv", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/python/polymarket-research/.env.example b/python/polymarket-research/.env.example new file mode 100644 index 00000000..d0cde042 --- /dev/null +++ b/python/polymarket-research/.env.example @@ -0,0 +1 @@ +BROWSERBASE_API_KEY= diff --git a/python/polymarket-research/README.md b/python/polymarket-research/README.md index beb08de4..1e93bb25 100644 --- a/python/polymarket-research/README.md +++ b/python/polymarket-research/README.md @@ -1,28 +1,29 @@ # Stagehand + Browserbase: Polymarket Prediction Market Research +Stagehand is the SDK for browser agents. + ## AT A GLANCE - Goal: automate research of prediction markets on Polymarket to extract current odds, pricing, and volume data. - Flow: navigate to polymarket.com โ†’ search for market โ†’ select result โ†’ extract market data (odds, prices, volume, changes). - Benefits: quickly gather market intelligence on prediction markets without manual browsing, structured data ready for analysis or trading decisions. - Docs โ†’ https://docs.stagehand.dev/v3/first-steps/introduction + Docs โ†’ https://docs.stagehand.dev/v4/first-steps/introduction ## GLOSSARY - act: perform UI actions from a prompt (click, type, search). - Docs โ†’ https://docs.stagehand.dev/basics/act + Docs โ†’ https://docs.stagehand.dev/v4/basics/act - extract: pull structured data from a page using AI and Pydantic schemas. - Docs โ†’ https://docs.stagehand.dev/basics/extract + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract - prediction market: a market where participants trade contracts based on the outcome of future events. ## QUICKSTART 1. cd polymarket-research -2. uv venv venv -3. source venv/bin/activate # On Windows: venv\Scripts\activate -4. pip install stagehand python-dotenv pydantic -5. cp .env.example .env # Add your Browserbase API key, Project ID, and OpenAI API key to .env -6. python main.py +2. uv venv && source .venv/bin/activate # On Windows: .venv\Scripts\activate +3. pip install stagehand python-dotenv pydantic +4. cp .env.example .env # Add your Browserbase API key to .env +5. python main.py ## EXPECTED OUTPUT @@ -38,7 +39,7 @@ ## COMMON PITFALLS - "ModuleNotFoundError": ensure all dependencies are installed via pip -- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and OPENAI_API_KEY +- Missing credentials: verify .env contains BROWSERBASE_API_KEY - No search results: check if the search query returns valid markets or try different search terms - Network issues: ensure internet access and polymarket.com is accessible - Import errors: activate your virtual environment if you created one @@ -57,7 +58,7 @@ ## HELPFUL RESOURCES -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction ๐ŸŽฎ Browserbase: https://www.browserbase.com ๐Ÿ’ก Try it out: https://www.browserbase.com/playground ๐Ÿ”ง Templates: https://www.browserbase.com/templates diff --git a/python/polymarket-research/main.py b/python/polymarket-research/main.py index cb0bf91b..2ddd994c 100644 --- a/python/polymarket-research/main.py +++ b/python/polymarket-research/main.py @@ -1,124 +1,91 @@ -# Stagehand + Browserbase: Polymarket prediction market research - See README.md for full documentation +"""Research a live Polymarket prediction market with Stagehand V4.""" import asyncio +import json import os from dotenv import load_dotenv -from pydantic import BaseModel, Field +from pydantic import BaseModel -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand, browserbase -# Load environment variables load_dotenv() +SEARCH_QUERY = "Will Elon Musk rejoin the Trump administration in 2026" + class MarketData(BaseModel): - """Market data extracted from Polymarket prediction market""" + market_title: str + current_odds: str | None + yes_price: str | None + no_price: str | None + total_volume: str | None + price_change: str | None - marketTitle: str | None = Field(None, description="the title of the market") - currentOdds: str | None = Field(None, description="the current odds or probability") - yesPrice: str | None = Field(None, description="the yes price") - noPrice: str | None = Field(None, description="the no price") - totalVolume: str | None = Field(None, description="the total trading volume") - priceChange: str | None = Field(None, description="the recent price change") +async def main() -> None: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") -async def main(): - """ - Searches Polymarket for a prediction market and extracts current odds, pricing, and volume data. - Uses AI-powered browser automation to navigate and interact with the site. - """ print("Starting Polymarket research automation...") - - # Initialize Stagehand with Browserbase for cloud-based browser automation - # Using BROWSERBASE environment to run in cloud rather than locally - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - verbose=1, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - ) - + browser = await browserbase.launch(api_key=api_key) try: - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - print("Initializing browser session...") - print("Stagehand session started successfully") - - # Provide live session URL for debugging and monitoring - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id - - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") - - page = stagehand.page - - # Navigate to Polymarket - print("Navigating to: https://polymarket.com/") - await page.goto("https://polymarket.com/") - print("Page loaded successfully") - - # Click the search box to trigger search dropdown - print("Clicking the search box at the top of the page") - await page.act("click the search box at the top of the page") - - # Type search query - searchQuery = "Elon Musk unfollow Trump" - print(f"Typing '{searchQuery}' into the search box") - await page.act(f"type '{searchQuery}' into the search box") - - # Click the first market result from the search dropdown - print("Selecting first market result from search dropdown") - await page.act("click the first market result from the search dropdown") - print("Market page loaded") - - # Extract market data using AI to parse the structured information - print("Extracting market information...") - marketData = await page.extract( - "Extract the current odds and market information for the prediction market", - schema=MarketData, + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto("https://polymarket.com", wait_until="domcontentloaded", timeout=60_000) + opened_search = await stagehand.act( + "Click the search box at the top of the page", page=page ) - - print("Market data extracted successfully:") - - # Display results in formatted JSON - import json - - print(json.dumps(marketData.model_dump(), indent=2)) - + typed_search = await stagehand.act( + "Fill the search box with %query%", + page=page, + variables={"query": SEARCH_QUERY}, + ) + opened_market = await stagehand.act( + "Click the first matching market in the search results", + page=page, + ) + page = await browser.context.active_page() or page + current_url = await page.url() + market_url = ( + "https://polymarket.com/event/" + "will-elon-musk-rejoin-the-trump-administration-in-2026" + ) + if ( + not opened_search.data.success + or not typed_search.data.success + or not opened_market.data.success + or "will-elon-musk-rejoin-the-trump-administration-in-2026" not in current_url + ): + await page.goto( + market_url, + wait_until="domcontentloaded", + timeout=60_000, + ) + + extracted = await stagehand.extract( + "Extract the current odds and market information for this prediction market", + MarketData, + page=page, + ) + market = extracted.data + print(json.dumps(market.model_dump(mode="json"), indent=2)) + finally: + await stagehand.close() + finally: + await browser.close() print("Session closed successfully") - except Exception as error: - print(f"Error during market research: {error}") - - # Provide helpful troubleshooting information - print("\nCommon issues:") - print("1. Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") - print("2. Verify OPENAI_API_KEY is set in environment") - print("3. Ensure internet access and https://polymarket.com is accessible") - print("4. Verify Browserbase account has sufficient credits") - - raise - if __name__ == "__main__": try: asyncio.run(main()) - except Exception as err: - print(f"Error in polymarket research: {err}") - print("Common issues:") - print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") - print(" - Verify OPENAI_API_KEY is set in environment") - print(" - Ensure internet access and https://polymarket.com is accessible") - print(" - Verify Browserbase account has sufficient credits") - print("Docs: https://docs.stagehand.dev/v3/first-steps/introduction") - exit(1) + except Exception as error: + print(f"Application error: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/polymarket-research/pyproject.toml b/python/polymarket-research/pyproject.toml new file mode 100644 index 00000000..823ae8f5 --- /dev/null +++ b/python/polymarket-research/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "polymarket-research" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[tool.uv] +package = false diff --git a/python/proxies-weather/README.md b/python/proxies-weather/README.md new file mode 100644 index 00000000..4501d6c5 --- /dev/null +++ b/python/proxies-weather/README.md @@ -0,0 +1,58 @@ +# Stagehand + Browserbase: Weather Proxy Demo + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- Goal: demonstrate geolocation proxies by fetching location-specific weather data from multiple cities using Browserbase's proxy infrastructure. +- Uses geolocation proxies to route traffic through specific geographic locations (New York, London, Tokyo, Sรฃo Paulo). +- Extracts structured weather data using Stagehand's extraction capabilities with Pydantic schema validation. +- Sequential processing shows how different proxy locations return different weather data from the same website. +- Docs โ†’ https://docs.browserbase.com/features/proxies + +## GLOSSARY + +- geolocation proxies: route traffic through specific geographic locations (city, country, state) to access location-specific content + Docs โ†’ https://docs.browserbase.com/features/proxies#set-proxy-geolocation +- extract: extract structured data from web pages using natural language instructions and Pydantic schemas + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract +- proxies: Browserbase's managed proxy infrastructure supporting 201+ countries for geolocation-based routing + Docs โ†’ https://docs.browserbase.com/features/proxies + +## QUICKSTART + +1. cd proxies-weather-template +2. uv venv venv +3. source venv/bin/activate # On Windows: venv\Scripts\activate +4. uvx install stagehand browserbase python-dotenv pydantic +5. cp .env.example .env +6. Add your Browserbase API key to .env +7. python main.py + +## EXPECTED OUTPUT + +- Creates Browserbase sessions with geolocation proxies for each location (New York, London, Tokyo, Sรฃo Paulo) +- Displays session URLs for each location for monitoring +- Navigates to weather service (windy.com) through location-specific proxies +- Extracts temperature and unit for each location +- Displays formatted results showing different weather data based on proxy location +- Demonstrates how geolocation proxies enable location-specific content access + +## COMMON PITFALLS + +- Browserbase Developer plan or higher is required to use proxies +- "ModuleNotFoundError": ensure all dependencies are installed via uvx install +- Missing credentials: verify .env contains BROWSERBASE_API_KEY +- Geolocation fields are case-insensitive (city, country, state can be any case) +- State is required for US locations to ensure accurate geolocation +- ERR_TUNNEL_CONNECTION_FAILED: indicates either a temporary proxy hiccup or a site unsupported by our built-in proxies +- Import errors: activate your virtual environment if you created one + +## HELPFUL RESOURCES + +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/proxies-weather/main.py b/python/proxies-weather/main.py new file mode 100644 index 00000000..698e6e9e --- /dev/null +++ b/python/proxies-weather/main.py @@ -0,0 +1,136 @@ +"""Verify geolocation proxies with live weather data and Stagehand V4.""" + +import asyncio +import os +from dataclasses import dataclass + +from dotenv import load_dotenv +from pydantic import BaseModel + +from stagehand import BrowserbaseProxyConfig, Stagehand, browserbase + +load_dotenv() + +EXPECTED_COUNTRIES = { + "US": "United States", + "GB": "United Kingdom", + "JP": "Japan", + "BR": "Brazil", +} + + +@dataclass(frozen=True) +class Geolocation: + city: str + country: str + state: str | None = None + + +@dataclass(frozen=True) +class WeatherResult: + city: str + country: str + temperature: float + conditions: str + reported_location: str + reported_country: str + + +class ExtractedWeather(BaseModel): + temperature: float + conditions: str + reported_location: str + reported_country: str + + +async def get_weather_for_location(location: Geolocation) -> WeatherResult: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") + + city_name = location.city.replace("_", " ") + proxy: BrowserbaseProxyConfig = { + "type": "browserbase", + "geolocation": { + "city": location.city, + "country": location.country, + **({"state": location.state} if location.state else {}), + }, + } + + print(f"Getting live weather for {city_name}, {location.country}") + browser = await browserbase.launch(api_key=api_key, proxies=[proxy]) + try: + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto( + "https://wttr.in/?format=j1", + wait_until="domcontentloaded", + timeout=60_000, + ) + extracted = await stagehand.extract( + ( + "Extract the current temperature in Celsius, current weather description, " + "nearest reported city or area, and reported country from this weather JSON" + ), + ExtractedWeather, + page=page, + ) + weather = extracted.data + temperature = weather.temperature + conditions = weather.conditions.strip() + reported_location = weather.reported_location.strip() + reported_country = weather.reported_country.strip() + + if not conditions or not reported_location or not reported_country: + raise RuntimeError("Weather service returned incomplete current conditions") + expected_country = EXPECTED_COUNTRIES[location.country] + if expected_country.lower() not in reported_country.lower(): + raise RuntimeError( + f"Proxy mismatch: expected {expected_country}, received {reported_country}" + ) + + return WeatherResult( + city=city_name, + country=location.country, + temperature=temperature, + conditions=conditions, + reported_location=reported_location, + reported_country=reported_country, + ) + finally: + await stagehand.close() + finally: + await browser.close() + + +async def main() -> None: + locations = [ + Geolocation("NEW_YORK", "US", "NY"), + Geolocation("LONDON", "GB"), + Geolocation("TOKYO", "JP"), + Geolocation("SAO_PAULO", "BR"), + ] + results = [await get_weather_for_location(location) for location in locations] + + print("\n=== Weather Results ===") + for result in results: + print( + f"{result.city}, {result.country}: {result.temperature} ยฐC, " + f"{result.conditions} (reported near {result.reported_location}, " + f"{result.reported_country})" + ) + print("All four proxy locations returned validated live weather") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as error: + print(f"Application error: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/proxies-weather/pyproject.toml b/python/proxies-weather/pyproject.toml new file mode 100644 index 00000000..2544c446 --- /dev/null +++ b/python/proxies-weather/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = "proxies-weather" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = ["python-dotenv==1.2.2", "stagehand==4.0.0"] + +[tool.uv] +package = false diff --git a/python/proxies-weather/requirements.txt b/python/proxies-weather/requirements.txt new file mode 100644 index 00000000..427e839b --- /dev/null +++ b/python/proxies-weather/requirements.txt @@ -0,0 +1,3 @@ +python-dotenv +pydantic +stagehand==4.0.0 diff --git a/python/proxies/.env.example b/python/proxies/.env.example new file mode 100644 index 00000000..d0cde042 --- /dev/null +++ b/python/proxies/.env.example @@ -0,0 +1 @@ +BROWSERBASE_API_KEY= diff --git a/python/proxies/README.md b/python/proxies/README.md index 4b100b32..6e8192b0 100644 --- a/python/proxies/README.md +++ b/python/proxies/README.md @@ -1,5 +1,7 @@ # Browserbase Proxy Testing Script +Stagehand is the SDK for browser agents. + ## AT A GLANCE - Goal: demonstrate different proxy configurations with Browserbase sessions. @@ -17,7 +19,7 @@ 4. pip install -r requirements.txt 5. pip install browserbase playwright 6. playwright install chromium -7. cp .env.example .env # Add your Browserbase API key and Project ID to .env +7. cp .env.example .env # Add your Browserbase API key to .env 8. python main.py ## EXPECTED OUTPUT @@ -32,7 +34,7 @@ - Browserbase Developer plan or higher is required to use proxies - "ModuleNotFoundError": ensure all dependencies are installed via pip -- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- Missing credentials: verify .env contains BROWSERBASE_API_KEY - Custom proxy errors: verify external proxy server credentials and availability - Playwright not installed: run `playwright install chromium` after pip install - Import errors: activate your virtual environment if you created one @@ -51,7 +53,7 @@ ## HELPFUL RESOURCES -๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction ๐ŸŽฎ Browserbase: https://www.browserbase.com ๐Ÿ’ก Try it out: https://www.browserbase.com/playground ๐Ÿ”ง Templates: https://www.browserbase.com/templates diff --git a/python/proxies/main.py b/python/proxies/main.py index 7ee29425..b41c48b5 100644 --- a/python/proxies/main.py +++ b/python/proxies/main.py @@ -1,149 +1,83 @@ -# Browserbase Proxy Testing Script - See README.md for full documentation +"""Verify Browserbase built-in and geolocation proxies with Stagehand V4.""" import asyncio +import json import os -from browserbase import Browserbase from dotenv import load_dotenv -from playwright.async_api import async_playwright -from pydantic import BaseModel, Field +from pydantic import BaseModel -from stagehand import Stagehand, StagehandConfig +from stagehand import BrowserbaseProxyConfig, Stagehand, browserbase load_dotenv() -bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) - class GeoInfo(BaseModel): - """Schema for IP information and geolocation data""" - - ip: str = Field(..., description="The IP address") - city: str = Field(..., description="The city name") - region: str = Field(..., description="The state or region") - country: str = Field(..., description="The country code") - loc: str = Field(..., description="The latitude and longitude coordinates") - timezone: str = Field(..., description="The timezone") - org: str = Field(..., description="The organization or ISP") - postal: str = Field(..., description="The postal code") - hostname: str = Field(..., description="The hostname if available") - - -async def create_session_with_built_in_proxies(): - # Use Browserbase's default proxy rotation for enhanced privacy and IP diversity. - session = await asyncio.to_thread( - bb.sessions.create, - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - proxies=True, # Enables automatic proxy rotation across different IP addresses. - ) - return session - - -async def create_session_with_geo_location(): - # Route traffic through specific geographic location to test location-based restrictions. - session = await asyncio.to_thread( - bb.sessions.create, - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - proxies=[ - { - "type": "browserbase", # Use Browserbase's managed proxy infrastructure. - "geolocation": { - "city": "NEW_YORK", # Simulate traffic from New York for testing geo-specific content. - "state": "NY", # See https://docs.browserbase.com/features/proxies for more geolocation options. - "country": "US", - }, - } - ], - ) - return session - - -async def create_session_with_custom_proxies(): - # Use external proxy servers for custom routing or specific proxy requirements. - session = await asyncio.to_thread( - bb.sessions.create, - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - proxies=[ - { - "type": "external", # Connect to your own proxy server infrastructure. - "server": "http://...", # Your proxy server endpoint. - "username": "user", # Authentication credentials for proxy access. - "password": "pass", - } - ], - ) - return session - - -async def test_session(session_function, session_name: str): - print(f"\n=== Testing {session_name} ===") - - # Create session with specific proxy configuration to test different routing scenarios. - session = await session_function() - print(f"Session URL: https://browserbase.com/sessions/{session.id}") - - # Connect to browser via CDP to control the session programmatically. - async with async_playwright() as p: - browser = await p.chromium.connect_over_cdp(session.connect_url) - default_context = browser.contexts[0] if browser.contexts else None - if not default_context: - raise Exception("No default context found") - - page = default_context.pages[0] if default_context.pages else None - if not page: - raise Exception("No page found in default context") - - # Initialize Stagehand for structured data extraction - stagehand_config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - verbose=1, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_id=session.id, # Use the existing Browserbase session + ip: str + city: str + region: str + country: str + loc: str + timezone: str + org: str + postal: str | None + hostname: str | None + + +async def test_session(proxies: bool | list[BrowserbaseProxyConfig], name: str) -> GeoInfo: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") + + print(f"\n=== Testing {name} ===") + browser = await browserbase.launch(api_key=api_key, proxies=proxies) + try: + stagehand = await Stagehand.create( + browser=browser, ) - - stagehand = Stagehand(stagehand_config) - try: - # Initialize Stagehand - await stagehand.init() - - # Navigate to IP info service to verify proxy location and IP address. - await stagehand.page.goto("https://ipinfo.io/json", wait_until="domcontentloaded") - - # Extract structured IP and location data using Stagehand and Pydantic schema - geo_info = await stagehand.page.extract( - instruction="Extract all IP information and geolocation data from the JSON response", - schema=GeoInfo, + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto("https://ipinfo.io/json", wait_until="domcontentloaded") + extracted = await stagehand.extract( + "Extract the complete IP geolocation record shown in this JSON response", + GeoInfo, + page=page, ) - - print("Geo Info:", geo_info.model_dump_json(indent=2)) - - # Close Stagehand session + geo_info = extracted.data + print(json.dumps(geo_info.model_dump(mode="json"), indent=2)) + return geo_info + finally: await stagehand.close() - except Exception as error: - print(f"Error during Stagehand extraction: {error}") - - # Close browser to release resources and end the test session. + finally: await browser.close() - print(f"{session_name} test completed") -async def main(): - # Test 1: Built-in proxies - Verify default proxy rotation works and shows different IPs. - await test_session(create_session_with_built_in_proxies, "Built-in Proxies") +async def main() -> None: + built_in = await test_session(True, "Built-in Proxies") + new_york = await test_session( + [ + { + "type": "browserbase", + "geolocation": {"city": "NEW_YORK", "state": "NY", "country": "US"}, + } + ], + "Geolocation Proxies (New York)", + ) - # Test 2: Geolocation proxies - Confirm traffic routes through specified location (New York). - await test_session(create_session_with_geo_location, "Geolocation Proxies (New York)") + if ( + new_york.country != "US" + or new_york.region not in {"New York", "New Jersey"} + or new_york.timezone != "America/New_York" + ): + raise RuntimeError( + "Expected a New York metropolitan-area proxy; received " + f"{new_york.city}, {new_york.region}, {new_york.country}" + ) + if built_in.ip == new_york.ip: + raise RuntimeError("Built-in and geolocation proxy sessions returned the same IP") - # Test 3: Custom external proxies - Enable if you have a custom proxy server set up. - # await test_session(create_session_with_custom_proxies, "Custom External Proxies") - print("\n=== All tests completed ===") + print("\nAll proxy tests completed with distinct IPs") if __name__ == "__main__": diff --git a/python/proxies/pyproject.toml b/python/proxies/pyproject.toml new file mode 100644 index 00000000..9b8f8a84 --- /dev/null +++ b/python/proxies/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "proxies" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "pydantic>=2.12,<3", + "python-dotenv==1.2.2", + "stagehand==4.0.0", +] + +[tool.uv] +package = false diff --git a/python/sec-filing-research/.env.example b/python/sec-filing-research/.env.example new file mode 100644 index 00000000..12472549 --- /dev/null +++ b/python/sec-filing-research/.env.example @@ -0,0 +1,3 @@ +# Browserbase credentials (required) +# Get these from https://www.browserbase.com/settings +BROWSERBASE_API_KEY=your-browserbase-api-key diff --git a/python/sec-filing-research/README.md b/python/sec-filing-research/README.md new file mode 100644 index 00000000..ac562b2c --- /dev/null +++ b/python/sec-filing-research/README.md @@ -0,0 +1,95 @@ +# Stagehand + Browserbase: SEC Filing Research + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- Goal: automate searching SEC EDGAR for a company and extracting recent filing metadata (type, date, description, accession number, file number). +- Search: supports company name, ticker symbol, or CIK number (e.g. "Apple Inc", "AAPL", "0000320193"). +- Data Extraction: uses Stagehand act/extract with Pydantic schemas to navigate SEC.gov and pull structured filing data. +- Output: company name, CIK, and a configurable number of most recent filings, printed as summary and JSON. + +## GLOSSARY + +- act: perform UI actions from a natural language prompt (click, type, submit). + Docs โ†’ https://docs.stagehand.dev/v4/basics/act +- extract: pull structured data from web pages into validated objects using a JSON schema. + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract +- schema: JSON schema definition for filing and company info; enforces types and validation. +- SEC EDGAR: SECโ€™s company and filing search and filing system. + https://www.sec.gov/edgar/searchedgar/companysearch.html +- CIK: Central Index Key โ€” unique numeric identifier for each company in EDGAR. + +## QUICKSTART + +1. cd python/sec-filing-research +2. Install dependencies with uv: + + ```bash + uv sync + ``` + + Or with pip in a venv: + + ```bash + python -m venv venv + source venv/bin/activate # On Windows: venv\Scripts\activate + pip install -e . + ``` + +3. cp .env.example .env +4. Add BROWSERBASE_API_KEY to .env +5. (Optional) Edit SEARCH_QUERY and NUM_FILINGS in main.py +6. Run the script: + + ```bash + python main.py + ``` + + Or with uv: + + ```bash + uv run python main.py + ``` + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase and shows live view URL +- Navigates to SEC EDGAR company search +- Enters search query, submits, and selects the matching company +- Extracts company name and CIK from the filings page +- Extracts the N most recent filings (type, date, description, accession number, file number) +- Logs SEC FILING METADATA summary and per-filing details +- Outputs full result as JSON +- Closes session cleanly + +## COMMON PITFALLS + +- "ModuleNotFoundError": run `uv sync` or `pip install -e .` in sec-filing-research +- Missing credentials: ensure .env has BROWSERBASE_API_KEY +- No company match: use a valid company name, ticker, or CIK; SEC search is case-sensitive for some queries +- Extraction errors: if SEC changes its UI, inspect the session and adjust the act/extract prompts +- Rate limiting: avoid excessive runs; SEC may throttle heavy or automated traffic + +## USE CASES + +โ€ข Compliance and due diligence: quickly pull recent 10-K, 10-Q, 8-K metadata for a list of companies. +โ€ข Research pipelines: feed accession numbers into downstream tools to fetch full filings or parse specific sections. +โ€ข Monitoring: periodically extract latest filings for watchlists and alert on new filings. +โ€ข Data enrichment: attach official company name and CIK to internal records using SEC as source of truth. + +## NEXT STEPS + +โ€ข Parameterize search: read SEARCH_QUERY and NUM_FILINGS from env or CLI for batch runs. +โ€ข Fetch full filings: use accession numbers with SECโ€™s full-text filing URLs or APIs to download documents. +โ€ข Multiple companies: loop over a list of tickers/names and aggregate results into a single report or JSON. +โ€ข Filter by type: restrict to 10-K/10-Q/8-K or other form types in the extract step or in post-processing. + +## HELPFUL RESOURCES + +๐Ÿ“š Stagehand Docs: https://docs.stagehand.dev/v4/first-steps/introduction +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/sec-filing-research/main.py b/python/sec-filing-research/main.py new file mode 100644 index 00000000..ee6e334d --- /dev/null +++ b/python/sec-filing-research/main.py @@ -0,0 +1,123 @@ +"""Extract recent Apple SEC filings with Stagehand V4.""" + +import asyncio +import json +import os + +from dotenv import load_dotenv +from pydantic import BaseModel +from stagehand import Stagehand, browserbase + +load_dotenv() + +SEARCH_QUERY = "Apple Inc" +COMPANY_CIK = "0000320193" +NUM_FILINGS = 5 + + +class CompanyInfo(BaseModel): + company_name: str + cik: str + + +class Filing(BaseModel): + type: str + date: str + description: str | None + accession_number: str | None + file_number: str | None + + +class Filings(BaseModel): + filings: list[Filing] + + +async def main() -> None: + api_key = os.environ.get("BROWSERBASE_API_KEY") + if not api_key: + raise RuntimeError("BROWSERBASE_API_KEY is required") + + browser = await browserbase.launch(api_key=api_key) + try: + stagehand = await Stagehand.create( + browser=browser, + ) + try: + pages = await browser.context.pages() + page = pages[0] if pages else await browser.context.new_page() + await page.goto( + "https://www.sec.gov/edgar/searchedgar/companysearch.html", + wait_until="domcontentloaded", + timeout=60_000, + ) + try: + await stagehand.act( + "Click the Company and Person Lookup search textbox", + page=page, + ) + await stagehand.act( + "Fill the company search field with %query%", + page=page, + variables={"query": SEARCH_QUERY}, + ) + await stagehand.act("Click the search submit button", page=page) + await stagehand.act( + "Click the Apple Inc company result to view its filings", + page=page, + ) + except Exception as error: + print( + f"Semantic SEC navigation did not complete; checking its postcondition: {error}" + ) + page = await browser.context.active_page() or page + if "/edgar/browse/" not in await page.url(): + await page.goto( + f"https://www.sec.gov/edgar/browse/?CIK={COMPANY_CIK}&owner=exclude", + wait_until="domcontentloaded", + timeout=60_000, + ) + company_result = await stagehand.extract( + "Extract the official company name and numeric CIK from the page header", + CompanyInfo, + page=page, + ) + filings_result = await stagehand.extract( + ( + f"Extract the {NUM_FILINGS} most recent SEC filings from the filings table. " + "For each return its type, filing date, description, accession number, and " + "file or film number when shown." + ), + Filings, + page=page, + ) + filings = filings_result.data.filings[:NUM_FILINGS] + + result = { + "company": company_result.data.company_name, + "cik": company_result.data.cik or COMPANY_CIK, + "search_query": SEARCH_QUERY, + "filings": [ + { + **filing.model_dump(), + "description": filing.description or "", + "accession_number": filing.accession_number or "", + "file_number": filing.file_number or "", + } + for filing in filings + ], + } + print(json.dumps(result, indent=2)) + finally: + await stagehand.close() + finally: + await browser.close() + print("Session closed successfully") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as error: + print(f"SEC filing extraction failed: {error}") + print("Docs: https://docs.stagehand.dev/v4/first-steps/introduction") + raise diff --git a/python/sec-filing-research/pyproject.toml b/python/sec-filing-research/pyproject.toml new file mode 100644 index 00000000..e0cd2702 --- /dev/null +++ b/python/sec-filing-research/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "sec-filing-research" +version = "0.1.0" +description = "SEC Filing Research: Extract company and filing metadata from SEC EDGAR using Stagehand and Browserbase" +readme = "README.md" +requires-python = ">=3.11,<3.14" +dependencies = [ + "beautifulsoup4==4.14.3", + "pydantic", + "python-dotenv", + "stagehand==4.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/python/selenium/quickstart-selenium/.env.example b/python/selenium/quickstart-selenium/.env.example new file mode 100644 index 00000000..8c897724 --- /dev/null +++ b/python/selenium/quickstart-selenium/.env.example @@ -0,0 +1,6 @@ +# Browserbase Configuration +# Get your API key from: https://www.browserbase.com/settings +BROWSERBASE_API_KEY=your_browserbase_api_key + +# Optional โ€” inferred from your API key if not provided +# BROWSERBASE_PROJECT_ID=your_browserbase_project_id diff --git a/python/selenium/quickstart-selenium/README.md b/python/selenium/quickstart-selenium/README.md new file mode 100644 index 00000000..4f922712 --- /dev/null +++ b/python/selenium/quickstart-selenium/README.md @@ -0,0 +1,76 @@ +# Quickstart: Selenium + Browserbase (Python) + +## AT A GLANCE + +- Goal: connect to a cloud browser via Selenium and Browserbase, click interactive elements, navigate between pages, and extract page copy. +- No AI required: uses Selenium WebDriver and the Browserbase SDK directly โ€” no model API key needed. +- Demonstrates core Selenium patterns: navigation, element clicking, waiting, and text extraction. +- Minimal setup: one script, one API key, instant cloud browser. + Docs โ†’ https://docs.browserbase.com/introduction/selenium + +## GLOSSARY + +- Session: a full cloud browser instance you connect to via Selenium WebDriver. + Docs โ†’ https://docs.browserbase.com/introduction/getting-started +- ClientConfig: Selenium's connection configuration, used here to set Browserbase authentication headers for every WebDriver request. + Docs โ†’ https://www.selenium.dev/documentation/webdriver/drivers/remote_webdriver/ +- Selenium WebDriver: browser automation library โ€” `driver.get()`, `driver.find_element()`, etc. + Docs โ†’ https://www.selenium.dev/documentation/webdriver/ +- WebDriverWait + expected_conditions: robust element waiting โ€” waits for elements to be clickable/visible before interacting, avoids brittle `time.sleep()`. + Docs โ†’ https://www.selenium.dev/documentation/webdriver/waits/ + +## QUICKSTART + +1. cd python/selenium/quickstart-selenium +2. cp .env.example .env +3. Add BROWSERBASE_API_KEY to .env (get it from https://browserbase.com/settings) +4. uv sync +5. uv run python main.py + +## EXPECTED OUTPUT + +- Creates a Browserbase cloud browser session +- Connects via Selenium WebDriver using Browserbase's remote URL and a `ClientConfig` with custom auth headers +- Prints browser name and version +- Prints a live debug URL +- Navigates to https://www.sfmoma.org and prints the URL and title +- Clicks the search button to open the search overlay, then closes it +- Clicks the Membership link and navigates to the membership page +- Extracts the heading and intro copy from the membership page +- Closes the driver + +## COMMON PITFALLS + +- Missing API key: verify .env contains BROWSERBASE_API_KEY โ€” this is the only required credential +- Project ID confusion: BROWSERBASE_PROJECT_ID is optional โ€” the API infers it from your API key +- Remote endpoint: pass `session.selenium_remote_url` as `command_executor`; `ClientConfig` alone does not replace Selenium's localhost default +- Auth headers: Browserbase requires `x-bb-api-key` and `session-id` headers โ€” `ClientConfig(extra_headers=...)` attaches them to every command +- Session not closing: always call `driver.quit()` in a `finally` block to avoid leaked sessions +- Element not found: if selectors change on the target site, inspect the page and update `By.CSS_SELECTOR` or `By.LINK_TEXT` values +- Timeout waiting for element: increase the `WebDriverWait` timeout (default 10s) for slow-loading pages +- Click intercepted: another element (cookie banner, overlay) may be covering the target โ€” close it first or use `WebDriverWait` to wait for it to disappear +- Stale element reference: if the page reloads between finding and clicking an element, re-locate it with a fresh `wait.until()` call +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +โ€ข Getting started with Browserbase and Selenium in Python +โ€ข Migrating existing Selenium scripts to the cloud +โ€ข Cloud browser automation without managing infrastructure +โ€ข Cross-browser testing with Selenium Grid compatibility + +## NEXT STEPS + +โ€ข Add AI extraction: swap in Stagehand for AI-powered `act()`, `extract()`, and `observe()` +โ€ข Enable proxies: pass `proxies=True` in `bb.sessions.create()` for residential proxy support +โ€ข Enable stealth mode: add `browser_settings={"advanced_stealth": True, "solve_captchas": True}` to bypass bot detection +โ€ข Run in parallel: create multiple sessions for concurrent browser automation + +## HELPFUL RESOURCES + +๐Ÿ“š Browserbase Docs: https://docs.browserbase.com +๐ŸŽฎ Browserbase: https://www.browserbase.com +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/selenium/quickstart-selenium/main.py b/python/selenium/quickstart-selenium/main.py new file mode 100644 index 00000000..9c0c9b32 --- /dev/null +++ b/python/selenium/quickstart-selenium/main.py @@ -0,0 +1,85 @@ +# Selenium + Browserbase: Quickstart +# See README.md for full documentation + +import os + +from browserbase import Browserbase +from dotenv import load_dotenv +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.remote.client_config import ClientConfig +from selenium.webdriver.support import expected_conditions as EC # noqa: N812 +from selenium.webdriver.support.ui import WebDriverWait + +load_dotenv() + +# ============= CONFIGURATION ============= +BROWSERBASE_API_KEY = os.environ["BROWSERBASE_API_KEY"] +# ========================================= + +bb = Browserbase(api_key=BROWSERBASE_API_KEY) + + +def run() -> None: + # Create a new Browserbase session and connect via Selenium + session = bb.sessions.create() + client_config = ClientConfig( + remote_server_addr=session.selenium_remote_url, + extra_headers={ + "x-bb-api-key": BROWSERBASE_API_KEY, + "session-id": session.id, + }, + ) + driver = webdriver.Remote( + command_executor=session.selenium_remote_url, + options=webdriver.ChromeOptions(), + client_config=client_config, + ) + + try: + print( + "Connected to Browserbase", + f"{driver.name} version {driver.caps['browserVersion']}", # type: ignore + ) + print(f"Live debug URL: https://browserbase.com/sessions/{session.id}") + + # Navigate to the SFMOMA homepage + driver.get("https://www.sfmoma.org") + print(f"At URL: {driver.current_url} | Title: {driver.title}") + + wait = WebDriverWait(driver, 10) + + # Click the search button to open the search overlay + search_button = wait.until( + EC.element_to_be_clickable((By.CSS_SELECTOR, 'button[aria-label="Open search"]')) + ) + search_button.click() + print("Clicked search button โ€” search overlay opened") + + # Close the search overlay + close_button = wait.until( + EC.element_to_be_clickable((By.CSS_SELECTOR, 'button[aria-label="Close search"]')) + ) + close_button.click() + print("Closed search overlay") + + # Click the Membership link + membership_link = wait.until(EC.element_to_be_clickable((By.LINK_TEXT, "Membership"))) + membership_link.click() + wait.until(EC.url_contains("/membership")) + print(f"At URL: {driver.current_url} | Title: {driver.title}") + + # Extract copy from the membership page + heading = wait.until(EC.presence_of_element_located((By.TAG_NAME, "h1"))) + print(f"Heading: {heading.text}") + + intro = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "main p"))) + print(f"Intro: {intro.text}") + + finally: + # Make sure to quit the driver so your session is ended! + driver.quit() + + +if __name__ == "__main__": + run() diff --git a/python/selenium/quickstart-selenium/pyproject.toml b/python/selenium/quickstart-selenium/pyproject.toml new file mode 100644 index 00000000..c0df3a00 --- /dev/null +++ b/python/selenium/quickstart-selenium/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "quickstart-selenium" +version = "1.0.0" +description = "Browserbase + Selenium: connect to a cloud browser and automate a real website" +requires-python = ">=3.11,<3.14" +dependencies = [ + "browserbase>=1.7.0", + "python-dotenv>=1.2.2", + "selenium>=4.38.0", +] + +[tool.uv] +package = false diff --git a/python/smart-fetch-scraper/.env.example b/python/smart-fetch-scraper/.env.example new file mode 100644 index 00000000..8c3f7702 --- /dev/null +++ b/python/smart-fetch-scraper/.env.example @@ -0,0 +1,2 @@ +# Browserbase Configuration +BROWSERBASE_API_KEY=your_browserbase_api_key diff --git a/python/smart-fetch-scraper/README.md b/python/smart-fetch-scraper/README.md new file mode 100644 index 00000000..f88a90e2 --- /dev/null +++ b/python/smart-fetch-scraper/README.md @@ -0,0 +1,66 @@ +# Stagehand + Browserbase: Smart Fetch Scraper + +Stagehand is the SDK for browser agents. + +## AT A GLANCE + +- Goal: scrape a webpage using the fastest method available โ€” Fetch API first, full browser session as fallback. +- Fetch API fast-path: sends a lightweight HTTP request via `POST /v1/fetch` โ€” no browser session, no AI credits. Returns raw HTML in milliseconds. +- Browser fallback: when the Fetch API returns insufficient or JS-rendered content, automatically falls back to a Stagehand browser session with AI-powered `extract()`. +- Fallback detection: checks status code, content length, JS-challenge patterns, and text density to decide whether the Fetch API result is usable. +- Model: uses `google/gemini-2.5-flash` for the browser fallback path. + Docs โ†’ https://docs.stagehand.dev + +## GLOSSARY + +- Fetch API: Browserbase's lightweight HTTP fetching endpoint โ€” fetches page content through Browserbase infrastructure without spinning up a browser. + Docs โ†’ https://docs.browserbase.com/features/fetch +- extract: pull structured data from pages using schemas and AI. + Docs โ†’ https://docs.stagehand.dev/v4/basics/extract +- Stagehand: AI browser automation framework. + Docs โ†’ https://docs.stagehand.dev + +## QUICKSTART + +1. cd python/smart-fetch-scraper +2. uv pip install -e . +3. cp .env.example .env +4. Add BROWSERBASE_API_KEY to .env +5. uv run python main.py \ โ€” e.g. `uv run python main.py https://news.ycombinator.com` + +## EXAMPLE URLS + +Fetch API fast-path (server-rendered, returns usable HTML directly): + +- `uv run python main.py https://news.ycombinator.com` โ€” server-rendered, lightweight HTML +- `uv run python main.py https://en.wikipedia.org/wiki/Web_scraping` โ€” static content, no JS required +- `uv run python main.py https://www.bbc.com/news` โ€” server-rendered news page + +Browser fallback (JS-rendered, blocked, or low text density): + +- `uv run python main.py https://www.reddit.com` โ€” returns a 403, triggers fallback +- `uv run python main.py https://x.com` โ€” returns an "Enable JavaScript" shell page +- `uv run python main.py https://github.com/trending` โ€” HTML is mostly inline scripts (3.6% text density), triggers fallback + +## EXPECTED OUTPUT + +- Logs the strategy being used (Fetch API vs browser) +- On Fetch API success: prints page title, link count, status code, content length, and a 500-char preview +- On browser fallback: prints Stagehand live view link, then structured JSON with page title and extracted items + +## COMMON PITFALLS + +- Missing credentials: verify .env contains BROWSERBASE_API_KEY +- Fetch API access: the Fetch API may require enablement on your account โ€” contact support if you get a 404 +- Content threshold: adjust MIN_CONTENT_LENGTH if server-rendered pages are incorrectly triggering the browser fallback +- Text density: adjust MIN_TEXT_DENSITY if pages with lots of inline scripts/styles are incorrectly triggering the browser fallback +- JS-challenge detection: JS_REQUIRED_PATTERNS covers common bot-detection pages (Cloudflare, etc.) โ€” extend the list for other patterns you encounter +- Timeout: the Fetch API has a 10-second timeout and 1MB max response โ€” very large or slow pages will need the browser path + +## HELPFUL RESOURCES + +๐Ÿ“š Browserbase Fetch Docs: https://docs.browserbase.com/features/fetch +๐Ÿ’ก Try it out: https://www.browserbase.com/playground +๐Ÿ”ง Templates: https://www.browserbase.com/templates +๐Ÿ“ง Need help? support@browserbase.com +๐Ÿ’ฌ Discord: http://stagehand.dev/discord diff --git a/python/smart-fetch-scraper/main.py b/python/smart-fetch-scraper/main.py new file mode 100644 index 00000000..3a92433a --- /dev/null +++ b/python/smart-fetch-scraper/main.py @@ -0,0 +1,246 @@ +# Stagehand + Browserbase: Smart Fetch Scraper - See README.md for full documentation +# +# Tries the Browserbase Fetch API first (fast, no browser session needed). +# If the page is JS-rendered or the content is insufficient, falls back to +# a full Stagehand browser session with AI-powered extraction. +# +# Note: the Fetch API can return content as `raw` HTML, clean `markdown`, +# or schema-extracted `json` โ€” pass `format="markdown"` or `format="json"` +# (with a JSON schema) to `bb.fetch_api.create` to skip writing your own +# HTML parser. Docs: https://docs.browserbase.com/platform/fetch/overview + +import asyncio +import json +import os +import re +import sys + +from browserbase import Browserbase +from dotenv import load_dotenv +from pydantic import BaseModel, Field +from stagehand import Stagehand, browserbase + +# Load environment variables from .env file +load_dotenv() + +# ============= CONFIGURATION ============= + +# Minimum character threshold โ€” if Fetch API returns less than this, +# the page is likely JS-rendered and we fall back to a browser session. +MIN_CONTENT_LENGTH = 500 + +# Minimum ratio of visible text to raw HTML โ€” pages below this are likely shells. +MIN_TEXT_DENSITY = 0.05 + +# Patterns that indicate the page requires JavaScript to render real content. +JS_REQUIRED_PATTERNS = [ + re.compile(r"enable javascript", re.IGNORECASE), + re.compile(r"javascript is (required|disabled|not enabled)", re.IGNORECASE), + re.compile(r"please enable javascript", re.IGNORECASE), + re.compile(r"this (site|page|app) requires javascript", re.IGNORECASE), + re.compile(r"checking your browser", re.IGNORECASE), # Cloudflare challenge + re.compile(r"