From 82272c6673d882dcdaf1cfc66f90bddfa7036e6e Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Mon, 29 Dec 2025 23:05:37 +0000 Subject: [PATCH 01/45] jaysahnan/gro-811-recaptcha-template (#28) * init commit * updating model --- .gitignore | 1 + python/basic-recaptcha/README.md | 111 +++++++++++++++++++++++++ python/basic-recaptcha/main.py | 116 +++++++++++++++++++++++++++ typescript/basic-recaptcha/README.md | 106 ++++++++++++++++++++++++ typescript/basic-recaptcha/index.ts | 90 +++++++++++++++++++++ 5 files changed, 424 insertions(+) create mode 100644 python/basic-recaptcha/README.md create mode 100644 python/basic-recaptcha/main.py create mode 100644 typescript/basic-recaptcha/README.md create mode 100644 typescript/basic-recaptcha/index.ts diff --git a/.gitignore b/.gitignore index bddbc648..b0f8e2d8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .env cursorrules.md .cursorrules +.cursor # Dependencies node_modules/ diff --git a/python/basic-recaptcha/README.md b/python/basic-recaptcha/README.md new file mode 100644 index 00000000..0ab61d8f --- /dev/null +++ b/python/basic-recaptcha/README.md @@ -0,0 +1,111 @@ +# Stagehand + Browserbase: Basic reCAPTCHA Solving + +## AT A GLANCE + +- Goal: Demonstrate automatic reCAPTCHA solving using Browserbase's built-in captcha solving capabilities. +- Automated Solving: Browserbase automatically detects and solves CAPTCHAs in the background. CAPTCHA solving is **enabled by default** - you don't need to set `solveCaptchas: true` unless you want to explicitly enable it (or set it to `false` to disable). +- Solving Time: CAPTCHA solving typically takes between 5-30 seconds depending on CAPTCHA type and complexity. +- Progress Monitoring: Listen for console messages (`browserbase-solving-started`, `browserbase-solving-finished`) to track captcha solving progress in real-time. +- Proxies Recommended: Enable proxies for higher CAPTCHA solving success rates. +- Verification: Extracts page content to verify successful captcha solving and form submission. +- Docs → https://docs.browserbase.com/features/stealth-mode#captcha-solving + +## GLOSSARY + +- solveCaptchas: Browserbase browser setting that enables automatic captcha solving for reCAPTCHA, CAPTCHA, and other captcha types. Enabled by default for Basic and Advanced Stealth Mode. + Docs → https://docs.browserbase.com/features/stealth-mode#captcha-solving +- CAPTCHA solving: When a CAPTCHA is detected, Browserbase attempts to solve it automatically in the background, allowing your automation to continue without manual intervention. +- console messages: browser console events that indicate captcha solving status: + - `browserbase-solving-started`: emitted when CAPTCHA detection begins + - `browserbase-solving-finished`: emitted when CAPTCHA solving completes +- custom CAPTCHA solving: For non-standard or custom captcha providers, you can specify CSS selectors for the captcha image and input field using `captchaImageSelector` and `captchaInputSelector` in browserSettings. +- act: perform UI actions from a prompt (type, click, fill forms) + Docs → https://docs.stagehand.dev/v2/basics/act +- extract: pull data from web pages using natural language instructions + Docs → https://docs.stagehand.dev/v2/basics/extract + +## CAPTCHA SOLVING DETAILS + +### How CAPTCHA Solving Works + +Browserbase provides integrated CAPTCHA solving to handle challenges automatically: + +- **Automatic Detection**: When a CAPTCHA is detected on a page, Browserbase attempts to solve it in the background +- **Solving Time**: CAPTCHA solving typically takes between 5-30 seconds, depending on the CAPTCHA type and complexity +- **Default Behavior**: CAPTCHA solving is enabled by default for Basic and Advanced Stealth Mode +- **Proxies**: It's recommended to enable proxies when using CAPTCHA solving for higher success rates +- **Multiple Types**: Browserbase supports reCAPTCHA, CAPTCHA, and other common captcha providers automatically + +### Custom CAPTCHA Solving + +For non-standard or custom captcha providers, you can specify CSS selectors to guide the solution process: + +```python +browserbase_session_create_params={ + "browser_settings": { + "solveCaptchas": True, + "captchaImageSelector": "#custom-captcha-image-id", + "captchaInputSelector": "#custom-captcha-input-id" + } +} +``` + +To find the selectors: + +1. Right-click on the captcha image and select "Inspect" to get the image selector +2. Right-click on the input field and select "Inspect" to get the input selector +3. Use the element's `id` or a CSS selector that uniquely identifies it + +### Disabling CAPTCHA Solving + +If you want to disable automatic captcha solving, set `solveCaptchas: False` in browserSettings: + +```python +browserbase_session_create_params={ + "browser_settings": { + "solveCaptchas": False + } +} +``` + +## QUICKSTART + +1. uv venv venv +2. source venv/bin/activate # On Windows: venv\Scripts\activate +3. uvx install stagehand python-dotenv +4. cp .env.example .env # Add your Browserbase API key, Project ID, and Google API key to .env +5. python main.py + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase +- Displays live session link for monitoring +- Navigates to Google reCAPTCHA demo page +- Waits for Browserbase to automatically solve the captcha +- Logs captcha solving progress messages +- Clicks submit button after captcha is solved +- Extracts and displays page content +- Verifies successful captcha solving by checking for success message +- Closes session cleanly + +## COMMON PITFALLS + +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY +- Captcha solving not enabled: ensure `solveCaptchas: True` is set in browserSettings (enabled by default) +- Solving timeout: allow up to 30 seconds for CAPTCHA solving to complete before timing out +- Proxies not enabled: enable proxies in browserSettings for higher CAPTCHA solving success rates +- Demo page inaccessible: verify the reCAPTCHA demo page URL is accessible and hasn't changed +- Console message timing: ensure console event listeners are set up before triggering the captcha +- Verification failure: success message check may fail if page structure changes; check extracted text manually +- Custom captcha selectors: for non-standard CAPTCHAs, verify that `captchaImageSelector` and `captchaInputSelector` are correctly defined +- Import errors: activate your virtual environment if you created one +- ModuleNotFoundError: ensure all dependencies are installed via uvx install + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v2/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/python/basic-recaptcha/main.py b/python/basic-recaptcha/main.py new file mode 100644 index 00000000..e8138a50 --- /dev/null +++ b/python/basic-recaptcha/main.py @@ -0,0 +1,116 @@ +# Basic reCAPTCHA Solving with Browserbase - See README.md for full documentation + +import asyncio +import os + +from dotenv import load_dotenv + +from stagehand import Stagehand, StagehandConfig + +# Load environment variables +load_dotenv() + +# Set to False to disable automatic captcha solving (True by default) +solve_captchas = True + + +async def main(): + # Initialize Stagehand with Browserbase for cloud-based browser automation. + # Enable captcha solving in browser settings for automatic reCAPTCHA handling. + config = StagehandConfig( + env="BROWSERBASE", + api_key=os.environ.get("BROWSERBASE_API_KEY"), + project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_name="google/gemini-2.5-pro", + model_api_key=os.environ.get("GOOGLE_API_KEY"), + verbose=0, + # 0 = errors only, 1 = info, 2 = debug + # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + # https://docs.stagehand.dev/configuration/logging + browserbase_session_create_params={ + "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), + "browser_settings": { + "solveCaptchas": solve_captchas, + }, + }, + ) + + try: + # Use async context manager for automatic resource management + async with Stagehand(config) as stagehand: + # Initialize browser session to start automation. + print("Stagehand initialized successfully!") + session_id = None + if hasattr(stagehand, "session_id"): + session_id = stagehand.session_id + elif hasattr(stagehand, "browserbase_session_id"): + session_id = stagehand.browserbase_session_id + + if session_id: + print(f"Live View Link: https://browserbase.com/sessions/{session_id}") + + page = stagehand.page + + # Navigate to Google reCAPTCHA demo page to test captcha solving. + print("Navigating to reCAPTCHA demo page...") + await page.goto("https://google.com/recaptcha/api2/demo") + + # Wait for Browserbase to solve the captcha automatically. + # Listen for console messages indicating captcha solving progress. + if solve_captchas: + print("Waiting for captcha to be solved...") + captcha_solved = asyncio.Event() + + def handle_console(msg): + if msg.text == "browserbase-solving-started": + print("Captcha solving in progress...") + elif msg.text == "browserbase-solving-finished": + print("Captcha solving completed!") + captcha_solved.set() + + page.on("console", handle_console) + await captcha_solved.wait() + else: + print("Captcha solving is disabled. Skipping wait...") + + # Click submit again after captcha is solved to complete the form submission. + print("Clicking submit button after captcha is solved...") + await page.act("Click the Submit button") + + # Extract and display the page content to verify successful submission. + print("Extracting page content...") + text = await page.extract("Extract all the text on this page") + print("Page content:") + print(text) + + # Check if captcha was successfully solved by looking for success message. + if isinstance(text, dict) and "extraction" in text: + extraction_text = text["extraction"] + elif isinstance(text, str): + extraction_text = text + else: + extraction_text = str(text) + + if "Verification Success... Hooray!" in extraction_text: + print("reCAPTCHA successfully solved!") + else: + print("Could not verify captcha success from page content") + + print("Session closed successfully") + + except Exception as error: + print(f"Error during reCAPTCHA solving: {error}") + raise + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Error in reCAPTCHA solving example: {err}") + print("Common issues:") + print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") + print(" - Verify solveCaptchas is enabled in browserSettings") + print(" - Ensure the demo page is accessible") + print("Docs: https://docs.stagehand.dev/v3/first-steps/introduction") + exit(1) diff --git a/typescript/basic-recaptcha/README.md b/typescript/basic-recaptcha/README.md new file mode 100644 index 00000000..cd0871e1 --- /dev/null +++ b/typescript/basic-recaptcha/README.md @@ -0,0 +1,106 @@ +# Stagehand + Browserbase: Basic reCAPTCHA Solving + +## AT A GLANCE + +- Goal: Demonstrate automatic reCAPTCHA solving using Browserbase's built-in captcha solving capabilities. +- Automated Solving: Browserbase automatically detects and solves CAPTCHAs in the background. CAPTCHA solving is **enabled by default** - you don't need to set `solveCaptchas: true` unless you want to explicitly enable it (or set it to `false` to disable). +- Solving Time: CAPTCHA solving typically takes between 5-30 seconds depending on CAPTCHA type and complexity. +- Progress Monitoring: Listen for console messages (`browserbase-solving-started`, `browserbase-solving-finished`) to track captcha solving progress in real-time. +- Proxies Recommended: Enable proxies for higher CAPTCHA solving success rates. +- Verification: Extracts page content to verify successful captcha solving and form submission. +- Docs → https://docs.browserbase.com/features/stealth-mode#captcha-solving + +## GLOSSARY + +- solveCaptchas: Browserbase browser setting that enables automatic captcha solving for reCAPTCHA, CAPTCHA, and other captcha types. Enabled by default for Basic and Advanced Stealth Mode. + Docs → https://docs.browserbase.com/features/stealth-mode#captcha-solving +- CAPTCHA solving: When a CAPTCHA is detected, Browserbase attempts to solve it automatically in the background, allowing your automation to continue without manual intervention. +- console messages: browser console events that indicate captcha solving status: + - `browserbase-solving-started`: emitted when CAPTCHA detection begins + - `browserbase-solving-finished`: emitted when CAPTCHA solving completes +- custom CAPTCHA solving: For non-standard or custom captcha providers, you can specify CSS selectors for the captcha image and input field using `captchaImageSelector` and `captchaInputSelector` in browserSettings. +- act: perform UI actions from a prompt (type, click, fill forms) + Docs → https://docs.stagehand.dev/basics/act +- extract: pull data from web pages using natural language instructions + Docs → https://docs.stagehand.dev/basics/extract + +## CAPTCHA SOLVING DETAILS + +### How CAPTCHA Solving Works + +Browserbase provides integrated CAPTCHA solving to handle challenges automatically: + +- **Automatic Detection**: When a CAPTCHA is detected on a page, Browserbase attempts to solve it in the background +- **Solving Time**: CAPTCHA solving typically takes between 5-30 seconds, depending on the CAPTCHA type and complexity +- **Default Behavior**: CAPTCHA solving is enabled by default for Basic and Advanced Stealth Mode +- **Proxies**: It's recommended to enable proxies when using CAPTCHA solving for higher success rates +- **Multiple Types**: Browserbase supports reCAPTCHA, CAPTCHA, and other common captcha providers automatically + +### Custom CAPTCHA Solving + +For non-standard or custom captcha providers, you can specify CSS selectors to guide the solution process: + +```typescript +browserSettings: { + solveCaptchas: true, + captchaImageSelector: "#custom-captcha-image-id", + captchaInputSelector: "#custom-captcha-input-id" +} +``` + +To find the selectors: + +1. Right-click on the captcha image and select "Inspect" to get the image selector +2. Right-click on the input field and select "Inspect" to get the input selector +3. Use the element's `id` or a CSS selector that uniquely identifies it + +### Disabling CAPTCHA Solving + +If you want to disable automatic captcha solving, set `solveCaptchas: false` in browserSettings: + +```typescript +browserSettings: { + solveCaptchas: false; +} +``` + +## QUICKSTART + +1. cd basic-recaptcha +2. pnpm install +3. cp .env.example .env +4. Add your Browserbase API key and Project ID to .env +5. pnpm start + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase +- Displays live session link for monitoring +- Navigates to Google reCAPTCHA demo page +- Clicks submit button to trigger reCAPTCHA challenge +- Waits for Browserbase to automatically solve the captcha +- Logs captcha solving progress messages +- Clicks submit again after captcha is solved +- Extracts and displays page content +- Verifies successful captcha solving by checking for success message +- Closes session cleanly + +## COMMON PITFALLS + +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- Captcha solving not enabled: ensure `solveCaptchas: true` is set in browserSettings (enabled by default) +- Solving timeout: allow up to 30 seconds for CAPTCHA solving to complete before timing out +- Proxies not enabled: enable proxies in browserSettings for higher CAPTCHA solving success rates +- Demo page inaccessible: verify the reCAPTCHA demo page URL is accessible and hasn't changed +- Console message timing: ensure console event listeners are set up before triggering the captcha +- Verification failure: success message check may fail if page structure changes; check extracted text manually +- Custom captcha selectors: for non-standard CAPTCHAs, verify that `captchaImageSelector` and `captchaInputSelector` are correctly defined + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/typescript/basic-recaptcha/index.ts b/typescript/basic-recaptcha/index.ts new file mode 100644 index 00000000..ea55e66c --- /dev/null +++ b/typescript/basic-recaptcha/index.ts @@ -0,0 +1,90 @@ +// Basic reCAPTCHA Solving with Browserbase - See README.md for full documentation + +import "dotenv/config"; +import { Stagehand } from "@browserbasehq/stagehand"; + +async function main() { + // Initialize Stagehand with Browserbase for cloud-based browser automation. + // Enable captcha solving in browser settings for automatic reCAPTCHA handling. + + const solveCaptchas = true; // Set to false to disable automatic captcha solving (true by default) + + const stagehand = new Stagehand({ + env: "BROWSERBASE", + verbose: 1, + // 0 = errors only, 1 = info, 2 = debug + // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + // https://docs.stagehand.dev/configuration/logging + browserbaseSessionCreateParams: { + browserSettings: { + solveCaptchas: solveCaptchas, + }, + }, + }); + + try { + // Initialize browser session to start automation. + await stagehand.init(); + console.log("Stagehand initialized successfully!"); + console.log( + `Live View Link: https://browserbase.com/sessions/${stagehand.browserbaseSessionId}`, + ); + + const page = stagehand.context.pages()[0]; + + // Navigate to Google reCAPTCHA demo page to test captcha solving. + console.log("Navigating to reCAPTCHA demo page..."); + await page.goto("https://google.com/recaptcha/api2/demo"); + + // Wait for Browserbase to solve the captcha automatically. + // Listen for console messages indicating captcha solving progress. + if (solveCaptchas) { + console.log("Waiting for captcha to be solved..."); + await new Promise((resolve) => { + page.on("console", (msg) => { + if (msg.text() === "browserbase-solving-started") { + console.log("Captcha solving in progress..."); + } else if (msg.text() === "browserbase-solving-finished") { + console.log("Captcha solving completed!"); + resolve(); + } + }); + }); + } else { + console.log("Captcha solving is disabled. Skipping wait..."); + } + + // Click submit again after captcha is solved to complete the form submission. + console.log("Clicking submit button after captcha is solved..."); + await stagehand.act("Click the Submit button"); + + // Extract and display the page content to verify successful submission. + console.log("Extracting page content..."); + const text = await stagehand.extract("Extract all the text on this page"); + console.log("Page content:"); + console.log(text); + + // Check if captcha was successfully solved by looking for success message. + if (text.extraction.includes("Verification Success... Hooray!")) { + console.log("reCAPTCHA successfully solved!"); + } else { + console.log("Could not verify captcha success from page content"); + } + } catch (error) { + console.error("Error during reCAPTCHA solving:", error); + } finally { + // Always close session to release resources and clean up. + await stagehand.close(); + console.log("Session closed successfully"); + } +} + +main().catch((err) => { + console.error("Error in reCAPTCHA solving example:", err); + console.error("Common issues:"); + console.error(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY"); + console.error(" - Verify solveCaptchas is enabled in browserSettings"); + console.error(" - Ensure the demo page is accessible"); + console.error("Docs: https://docs.stagehand.dev/v3/first-steps/introduction"); + process.exit(1); +}); From 4654cd633ac2b71d224721f0bac2701863872674 Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Tue, 6 Jan 2026 09:53:08 +0000 Subject: [PATCH 02/45] Add MFA Handling - TOTP Automation Template (#25) * init commit * Add MFA-handling template updates and check script - Updated MFA-handling Python and TypeScript templates - Added 'check' script to package.json for running all formatting and linting checks * addressing PR comments --- package.json | 4 +- python/MFA-handling/README.md | 78 ++++++++++ python/MFA-handling/main.py | 217 +++++++++++++++++++++++++++ python/MFA-handling/requirements.txt | 4 + typescript/MFA-handling/README.md | 73 +++++++++ typescript/MFA-handling/index.ts | 192 ++++++++++++++++++++++++ typescript/MFA-handling/package.json | 30 ++++ 7 files changed, 597 insertions(+), 1 deletion(-) create mode 100644 python/MFA-handling/README.md create mode 100644 python/MFA-handling/main.py create mode 100644 python/MFA-handling/requirements.txt create mode 100644 typescript/MFA-handling/README.md create mode 100644 typescript/MFA-handling/index.ts create mode 100644 typescript/MFA-handling/package.json diff --git a/package.json b/package.json index 68b865e7..ee1334cb 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,9 @@ "lint:fix": "eslint \"**/*.{ts,tsx,js,jsx}\" --fix", "lint:python": "uvx ruff check python/", "lint:python:fix": "uvx ruff check --fix python/", - "format:python": "uvx ruff format python/" + "format:python": "uvx ruff format python/", + "format:python:check": "uvx ruff format --check python/", + "check": "npm run format:check && npm run lint && npm run lint:python && npm run format:python:check" }, "devDependencies": { "@typescript-eslint/eslint-plugin": "^8.50.1", diff --git a/python/MFA-handling/README.md b/python/MFA-handling/README.md new file mode 100644 index 00000000..a77699ff --- /dev/null +++ b/python/MFA-handling/README.md @@ -0,0 +1,78 @@ +# Stagehand + Browserbase: MFA Handling - TOTP Automation + +## AT A GLANCE + +- Goal: Automate MFA (Multi-Factor Authentication) completion using TOTP (Time-based One-Time Password) code generation. +- TOTP Generation: Implements RFC 6238 compliant algorithm to generate time-based authentication codes programmatically. +- Automatic Form Filling: Extracts TOTP secrets from pages and automatically fills MFA forms without user interaction. +- Retry Logic: Handles time window edge cases by regenerating codes and retrying authentication when needed. +- Docs → https://docs.stagehand.dev/basics/act + +## GLOSSARY + +- act: perform UI actions from a prompt (type, click, fill forms) + Docs → https://docs.stagehand.dev/basics/act +- extract: extract structured data from web pages using natural language instructions + Docs → https://docs.stagehand.dev/basics/extract +- TOTP: Time-based One-Time Password - a 6-digit code that changes every 30 seconds, generated using HMAC-SHA1 algorithm +- RFC 6238: Standard specification for TOTP authentication codes used by Google Authenticator, Authy, and other authenticator apps + +## QUICKSTART + +1. python -m venv venv +2. source venv/bin/activate # On Windows: venv\Scripts\activate +3. pip install stagehand python-dotenv pydantic +4. cp .env.example .env +5. Add required API keys/IDs to .env (BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, GOOGLE_GENERATIVE_AI_API_KEY) +6. python main.py + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase +- Displays live session link for monitoring +- Navigates to TOTP challenge demo page (authenticationtest.com/totpChallenge/) +- Extracts test credentials (email, password) and TOTP secret from the page +- Generates TOTP code using RFC 6238 algorithm +- Fills in email and password fields +- Fills in TOTP code field with generated code +- Submits authentication form +- Checks authentication result +- Retries with fresh code if initial attempt fails (handles time window edge cases) +- Closes session cleanly + +## COMMON PITFALLS + +- "ModuleNotFoundError": ensure all dependencies are installed via pip +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_GENERATIVE_AI_API_KEY +- Google API access: ensure you have access to Google's gemini-2.5-flash model +- TOTP code expiration: codes are valid for 30 seconds - if authentication fails, the script automatically retries with a fresh code +- Page structure changes: if the demo site structure changes, extraction may fail +- Network timeouts: ensure stable internet connection for reliable page loading +- Import errors: activate your virtual environment if you created one +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +• Automated authentication: Complete MFA challenges automatically when session persistence isn't enough (session expired, new device, etc.) +• TOTP integration: Store encrypted TOTP secrets during user onboarding and generate codes programmatically when needed +• Zero-touch MFA: Eliminate user interaction for MFA completion in automated workflows +• Session recovery: Automatically handle MFA prompts when re-authenticating expired sessions + +## NEXT STEPS + +• Secure storage: Implement encrypted TOTP secret storage (AES-256) in your database during user onboarding +• Multiple time windows: Add support for trying ±1 time window (60s range) if current code fails +• SMS/Email MFA: Extend to support SMS codes (via Twilio/Bandwidth API) or email codes (via Gmail API/IMAP) +• Backup codes: Implement fallback to backup codes stored during initial MFA setup +• Context integration: Combine with Browserbase Contexts to minimize MFA prompts (95% context reuse, 4% auto TOTP, 1% user-mediated) +• Error handling: Add graceful fallback to user prompts when automation fails + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v2/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord + diff --git a/python/MFA-handling/main.py b/python/MFA-handling/main.py new file mode 100644 index 00000000..419feb78 --- /dev/null +++ b/python/MFA-handling/main.py @@ -0,0 +1,217 @@ +# Stagehand + Browserbase: MFA Handling - TOTP Automation - See README.md for full documentation + +import asyncio +import hashlib +import hmac +import os +import time + +from dotenv import load_dotenv +from pydantic import BaseModel, Field + +from stagehand import Stagehand + +# Load environment variables +load_dotenv() + +# Demo site URL for TOTP challenge testing +DEMO_URL = "https://authenticationtest.com/totpChallenge/" + + +def generate_totp(secret: str, window: int = 0) -> str: + """ + Generate TOTP code (Time-based One-Time Password) using RFC 6238 compliant algorithm. + + Same algorithm used by Google Authenticator, Authy, and other authenticator apps. + """ + # Convert base32 secret to bytes + base32chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" + bits = "" + hex_str = "" + + secret = secret.upper().rstrip("=") + + for char in secret: + val = base32chars.find(char) + if val == -1: + raise ValueError("Invalid base32 character in secret") + bits += format(val, "05b") + + for i in range(0, len(bits) - 3, 4): + chunk = bits[i : i + 4] + hex_str += format(int(chunk, 2), "x") + + secret_bytes = bytes.fromhex(hex_str) + + # Get current time window (30 second intervals) + time_window = int(time.time() // 30) + window + time_bytes = time_window.to_bytes(8, byteorder="big") + + # Generate HMAC-SHA1 hash + hmac_result = hmac.new(secret_bytes, time_bytes, hashlib.sha1).digest() + + # Dynamic truncation to extract 6-digit code + offset = hmac_result[-1] & 0xF + code = ( + ((hmac_result[offset] & 0x7F) << 24) + | ((hmac_result[offset + 1] & 0xFF) << 16) + | ((hmac_result[offset + 2] & 0xFF) << 8) + | (hmac_result[offset + 3] & 0xFF) + ) + + # Return 6-digit code with leading zeros + return str(code % 1000000).zfill(6) + + +# Define Pydantic schema for credentials extraction +class Credentials(BaseModel): + email: str = Field(..., description="Email address") + password: str = Field(..., description="Password") + totp_secret: str = Field(..., description="The TOTP secret key for generating codes") + + +class AuthResult(BaseModel): + success: bool = Field(..., description="Whether authentication was successful") + message: str = Field(..., description="Success or error message") + + +class RetryResult(BaseModel): + success: bool = Field(..., description="Whether the retry login was successful") + + +async def main(): + print("Starting MFA Handling - TOTP Automation...") + + # Initialize Stagehand with Browserbase for cloud-based browser automation + # Note: set verbose: 0 to prevent API keys from appearing in logs when handling sensitive data. + stagehand = Stagehand( + env="BROWSERBASE", + api_key=os.environ.get("BROWSERBASE_API_KEY"), + project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model="openai/gpt-4.1", + model_api_key=os.environ.get("OPENAI_API_KEY"), + verbose=0, # 0 = errors only, 1 = info, 2 = debug + # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + # https://docs.stagehand.dev/configuration/logging + ) + + try: + # Initialize browser session + await stagehand.init() + print("Stagehand initialized successfully!") + + # Get session ID if available + session_id = getattr(stagehand, "session_id", None) or getattr( + stagehand, "browserbase_session_id", None + ) + if session_id: + print(f"Live View Link: https://browserbase.com/sessions/{session_id}") + + # Get the page object + page = stagehand.page + + # Navigate to TOTP challenge demo page + print("Navigating to TOTP Challenge page...") + await page.goto(DEMO_URL, wait_until="domcontentloaded") + + # Extract test credentials and TOTP secret from the page + print("Extracting test credentials and TOTP secret...") + credentials = await page.extract( + instruction="Extract the test email, password, and TOTP secret key shown on the page", + schema=Credentials, + ) + + print(f"Credentials extracted - Email: {credentials.email}") + + # Generate TOTP code using RFC 6238 algorithm + totp_code = generate_totp(credentials.totp_secret) + seconds_left = 30 - (int(time.time()) % 30) + print(f"Generated TOTP code: {totp_code} (valid for {seconds_left} seconds)") + + # Fill in login form with email and password + print("Filling in email...") + await page.act(f"Type '{credentials.email}' into the email field") + + print("Filling in password...") + await page.act(f"Type '{credentials.password}' into the password field") + + # Fill in TOTP code + print("Filling in TOTP code...") + await page.act(f"Type '{totp_code}' into the TOTP code field") + + # Submit the form + print("Submitting form...") + await page.act("Click the submit or login button") + + # Wait for response - be tolerant of sites that never reach full "networkidle" + try: + print("Waiting for page to finish loading after submit...") + await page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + print( + "Timed out waiting for 'networkidle' after submit; continuing because the login likely succeeded." + ) + + # Check if login succeeded + print("Checking authentication result...") + result = await page.extract( + instruction="Check if the login was successful or if there's an error message", + schema=AuthResult, + ) + + if result.success: + print("SUCCESS! TOTP authentication completed automatically!") + print(f"Authentication Result: {result.message}") + else: + print(f"Authentication may have failed. Message: {result.message}") + print("Retrying with a fresh TOTP code...") + + # Regenerate and retry with new code (handles time window edge cases) + new_code = generate_totp(credentials.totp_secret) + print(f"New TOTP code: {new_code}") + + await page.act("Clear the TOTP code field") + await page.act(f"Type '{new_code}' into the TOTP code field") + await page.act("Click the submit or login button") + + try: + print("Waiting for page to finish loading after retry submit...") + await page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + print( + "Timed out waiting for 'networkidle' after retry submit; continuing because the login likely succeeded." + ) + + retry_result = await page.extract( + instruction="Check if the login was successful", + schema=RetryResult, + ) + + if retry_result.success: + print("Success on retry!") + else: + print("Authentication failed after retry") + + await stagehand.close() + print("Session closed successfully") + + except Exception as error: + print(f"Error during MFA handling: {error}") + import traceback + + traceback.print_exc() + raise + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Error in MFA handling: {err}") + print("Common issues:") + print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") + print(" - Check .env file has OPENAI_API_KEY (or set model_api_key for your chosen model)") + print(" - TOTP code may have expired (try running again)") + print(" - Page structure may have changed") + print("Docs: https://docs.stagehand.dev/v2/first-steps/introduction") + exit(1) diff --git a/python/MFA-handling/requirements.txt b/python/MFA-handling/requirements.txt new file mode 100644 index 00000000..4e90a161 --- /dev/null +++ b/python/MFA-handling/requirements.txt @@ -0,0 +1,4 @@ +stagehand +python-dotenv +pydantic + diff --git a/typescript/MFA-handling/README.md b/typescript/MFA-handling/README.md new file mode 100644 index 00000000..258b82e4 --- /dev/null +++ b/typescript/MFA-handling/README.md @@ -0,0 +1,73 @@ +# Stagehand + Browserbase: MFA Handling - TOTP Automation + +## AT A GLANCE + +- Goal: Automate MFA (Multi-Factor Authentication) completion using TOTP (Time-based One-Time Password) code generation. +- TOTP Generation: Implements RFC 6238 compliant algorithm to generate time-based authentication codes programmatically. +- Automatic Form Filling: Extracts TOTP secrets from pages and automatically fills MFA forms without user interaction. +- Retry Logic: Handles time window edge cases by regenerating codes and retrying authentication when needed. +- Docs → https://docs.stagehand.dev/basics/act + +## GLOSSARY + +- act: perform UI actions from a prompt (type, click, fill forms) + Docs → https://docs.stagehand.dev/basics/act +- extract: extract structured data from web pages using natural language instructions + Docs → https://docs.stagehand.dev/basics/extract +- TOTP: Time-based One-Time Password - a 6-digit code that changes every 30 seconds, generated using HMAC-SHA1 algorithm +- RFC 6238: Standard specification for TOTP authentication codes used by Google Authenticator, Authy, and other authenticator apps + +## QUICKSTART + +1. pnpm install +2. cp .env.example .env +3. Add required API keys/IDs to .env (BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, GOOGLE_GENERATIVE_AI_API_KEY) +4. pnpm start + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase +- Displays live session link for monitoring +- Navigates to TOTP challenge demo page (authenticationtest.com/totpChallenge/) +- Extracts test credentials (email, password) and TOTP secret from the page +- Generates TOTP code using RFC 6238 algorithm +- Fills in email and password fields +- Fills in TOTP code field with generated code +- Submits authentication form +- Checks authentication result +- Retries with fresh code if initial attempt fails (handles time window edge cases) +- Closes session cleanly + +## COMMON PITFALLS + +- Dependency install errors: ensure pnpm install completed +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_GENERATIVE_AI_API_KEY +- Google API access: ensure you have access to Google's gemini-2.5-flash model +- TOTP code expiration: codes are valid for 30 seconds - if authentication fails, the script automatically retries with a fresh code +- Page structure changes: if the demo site structure changes, extraction may fail +- Network timeouts: ensure stable internet connection for reliable page loading +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +• Automated authentication: Complete MFA challenges automatically when session persistence isn't enough (session expired, new device, etc.) +• TOTP integration: Store encrypted TOTP secrets during user onboarding and generate codes programmatically when needed +• Zero-touch MFA: Eliminate user interaction for MFA completion in automated workflows +• Session recovery: Automatically handle MFA prompts when re-authenticating expired sessions + +## NEXT STEPS + +• Secure storage: Implement encrypted TOTP secret storage (AES-256) in your database during user onboarding +• Multiple time windows: Add support for trying ±1 time window (60s range) if current code fails +• SMS/Email MFA: Extend to support SMS codes (via Twilio/Bandwidth API) or email codes (via Gmail API/IMAP) +• Backup codes: Implement fallback to backup codes stored during initial MFA setup +• Context integration: Combine with Browserbase Contexts to minimize MFA prompts (95% context reuse, 4% auto TOTP, 1% user-mediated) +• Error handling: Add graceful fallback to user prompts when automation fails + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com diff --git a/typescript/MFA-handling/index.ts b/typescript/MFA-handling/index.ts new file mode 100644 index 00000000..cdc4fd10 --- /dev/null +++ b/typescript/MFA-handling/index.ts @@ -0,0 +1,192 @@ +// Stagehand + Browserbase: MFA Handling - TOTP Automation - See README.md for full documentation + +import "dotenv/config"; +import { Stagehand } from "@browserbasehq/stagehand"; +import { z } from "zod"; +import crypto from "crypto"; + +// Demo site URL for TOTP challenge testing +const DEMO_URL = "https://authenticationtest.com/totpChallenge/"; + +// Generate TOTP code (Time-based One-Time Password) using RFC 6238 compliant algorithm +// Same algorithm used by Google Authenticator, Authy, and other authenticator apps +function generateTOTP(secret: string, window = 0): string { + // Convert base32 secret to buffer + const base32chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + let bits = ""; + let hex = ""; + + secret = secret.toUpperCase().replace(/=+$/, ""); + + for (let i = 0; i < secret.length; i++) { + const val = base32chars.indexOf(secret.charAt(i)); + if (val === -1) throw new Error("Invalid base32 character in secret"); + bits += val.toString(2).padStart(5, "0"); + } + + for (let i = 0; i + 4 <= bits.length; i += 4) { + const chunk = bits.substr(i, 4); + hex += parseInt(chunk, 2).toString(16); + } + + const secretBuffer = Buffer.from(hex, "hex"); + + // Get current time window (30 second intervals) + const time = Math.floor(Date.now() / 1000 / 30) + window; + const timeBuffer = Buffer.alloc(8); + timeBuffer.writeBigInt64BE(BigInt(time)); + + // Generate HMAC-SHA1 hash + const hmac = crypto.createHmac("sha1", secretBuffer); + hmac.update(timeBuffer); + const hmacResult = hmac.digest(); + + // Dynamic truncation to extract 6-digit code + const offset = hmacResult[hmacResult.length - 1] & 0xf; + const code = + ((hmacResult[offset] & 0x7f) << 24) | + ((hmacResult[offset + 1] & 0xff) << 16) | + ((hmacResult[offset + 2] & 0xff) << 8) | + (hmacResult[offset + 3] & 0xff); + + // Return 6-digit code with leading zeros + return (code % 1000000).toString().padStart(6, "0"); +} + +async function main() { + console.log("Starting MFA Handling - TOTP Automation..."); + + // Initialize Stagehand with Browserbase for cloud-based browser automation + const stagehand = new Stagehand({ + env: "BROWSERBASE", + verbose: 0, + // 0 = errors only, 1 = info, 2 = debug + // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + // https://docs.stagehand.dev/configuration/logging + model: { + modelName: "google/gemini-2.5-flash", + apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY, + }, + }); + + try { + // Initialize browser session to start automation + await stagehand.init(); + console.log("Stagehand initialized successfully!"); + console.log( + `Live View Link: https://browserbase.com/sessions/${stagehand.browserbaseSessionId}`, + ); + + const page = stagehand.context.pages()[0]; + + // Navigate to TOTP challenge demo page + console.log("Navigating to TOTP Challenge page..."); + await page.goto(DEMO_URL, { + waitUntil: "domcontentloaded", + }); + + // Extract test credentials and TOTP secret from the page + console.log("Extracting test credentials and TOTP secret..."); + const credentials = await stagehand.extract( + "Extract the test email, password, and TOTP secret key shown on the page", + z.object({ + email: z.string(), + password: z.string(), + totpSecret: z.string().describe("The TOTP secret key for generating codes"), + }), + ); + + console.log(`Credentials extracted - Email: ${credentials.email}`); + + // Generate TOTP code using RFC 6238 algorithm + const totpCode = generateTOTP(credentials.totpSecret); + const secondsLeft = 30 - (Math.floor(Date.now() / 1000) % 30); + console.log(`Generated TOTP code: ${totpCode} (valid for ${secondsLeft} seconds)`); + + // Fill in login form with email and password + console.log("Filling in email..."); + await stagehand.act(`Type '${credentials.email}' into the email field`); + + console.log("Filling in password..."); + await stagehand.act(`Type '${credentials.password}' into the password field`); + + // Fill in TOTP code + console.log("Filling in TOTP code..."); + await stagehand.act(`Type '${totpCode}' into the TOTP code field`); + + // Submit the form + console.log("Submitting form..."); + await stagehand.act("Click the submit or login button"); + + // Wait for response - be tolerant of sites that never reach full "networkidle" + try { + console.log("Waiting for page to finish loading after submit..."); + await page.waitForLoadState("networkidle", { timeout: 15000 }); + } catch (err) { + console.warn( + "Timed out waiting for 'networkidle' after submit; continuing because the login likely succeeded.", + err, + ); + } + + // Check if login succeeded + console.log("Checking authentication result..."); + const result = await stagehand.extract( + "Check if the login was successful or if there's an error message", + z.object({ + success: z.boolean(), + message: z.string(), + }), + ); + + if (result.success) { + console.log("SUCCESS! TOTP authentication completed automatically!"); + console.log("Authentication Result:", JSON.stringify(result, null, 2)); + } else { + console.log("Authentication may have failed. Message:", result.message); + console.log("Retrying with a fresh TOTP code..."); + + // Regenerate and retry with new code (handles time window edge cases) + const newCode = generateTOTP(credentials.totpSecret); + console.log(`New TOTP code: ${newCode}`); + + await stagehand.act("Clear the TOTP code field"); + await stagehand.act(`Type '${newCode}' into the TOTP code field`); + await stagehand.act("Click the submit or login button"); + + try { + console.log("Waiting for page to finish loading after retry submit..."); + await page.waitForLoadState("networkidle", { timeout: 15000 }); + } catch (err) { + console.warn( + "Timed out waiting for 'networkidle' after retry submit; continuing because the login likely succeeded.", + err, + ); + } + + const retryResult = await stagehand.extract("Check if the login was successful", z.boolean()); + + if (retryResult) { + console.log("Success on retry!"); + } else { + console.log("Authentication failed after retry"); + } + } + } catch (error) { + console.error("Error during MFA handling:", error); + } finally { + // Always close session to release resources and clean up + await stagehand.close(); + console.log("Session closed successfully"); + } +} + +main().catch((err) => { + console.error("Error in MFA handling:", err); + console.error("Common issues:"); + console.error(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY"); + console.error(" - TOTP code may have expired (try running again)"); + console.error(" - Page structure may have changed"); + console.error("Docs: https://docs.stagehand.dev/v3/first-steps/introduction"); + process.exit(1); +}); diff --git a/typescript/MFA-handling/package.json b/typescript/MFA-handling/package.json new file mode 100644 index 00000000..f30332db --- /dev/null +++ b/typescript/MFA-handling/package.json @@ -0,0 +1,30 @@ +{ + "name": "mfa-handling-template", + "version": "1.0.0", + "description": "Stagehand + Browserbase: MFA Handling - TOTP Automation", + "main": "index.ts", + "scripts": { + "start": "tsx index.ts", + "dev": "tsx watch index.ts" + }, + "keywords": [ + "stagehand", + "browserbase", + "automation", + "mfa", + "totp", + "2fa" + ], + "author": "", + "license": "MIT", + "dependencies": { + "@browserbasehq/stagehand": "^3.0.0", + "dotenv": "^16.0.0", + "zod": "^3.22.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "tsx": "^4.7.0", + "typescript": "^5.3.0" + } +} From 486093a216f1cd09c870d86b3e120d48f13e5e5e Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Tue, 6 Jan 2026 09:57:55 +0000 Subject: [PATCH 03/45] jaysahnan/gro-809-manual-mfa-with-contexts (#26) * init commit * addressing PR comments --- python/manual-MFA-with-contexts/README.md | 67 +++++ python/manual-MFA-with-contexts/main.py | 271 ++++++++++++++++++ .../manual-MFA-with-contexts/requirements.txt | 6 + typescript/manual-MFA-with-contexts/README.md | 65 +++++ typescript/manual-MFA-with-contexts/index.ts | 254 ++++++++++++++++ .../manual-MFA-with-contexts/package.json | 22 ++ 6 files changed, 685 insertions(+) create mode 100644 python/manual-MFA-with-contexts/README.md create mode 100644 python/manual-MFA-with-contexts/main.py create mode 100644 python/manual-MFA-with-contexts/requirements.txt create mode 100644 typescript/manual-MFA-with-contexts/README.md create mode 100644 typescript/manual-MFA-with-contexts/index.ts create mode 100644 typescript/manual-MFA-with-contexts/package.json diff --git a/python/manual-MFA-with-contexts/README.md b/python/manual-MFA-with-contexts/README.md new file mode 100644 index 00000000..35cbafb2 --- /dev/null +++ b/python/manual-MFA-with-contexts/README.md @@ -0,0 +1,67 @@ +# Stagehand + Browserbase: Manual MFA with Contexts + +## AT A GLANCE + +- Goal: demonstrate how to persist authentication across sessions using Browserbase Contexts, eliminating MFA friction after the first login. +- Flow: first session creates context and completes MFA manually → context saves auth state → second session reuses context with no MFA required. + +## GLOSSARY + +- context: a Browserbase feature that persists browser state (cookies, localStorage, sessionStorage) across sessions. + Docs → https://docs.browserbase.com/features/contexts +- persist: setting that saves authentication state including MFA trust/remember device state to the context. +- MFA (Multi-Factor Authentication): two-factor authentication requiring a code from an authenticator app. +- session persistence: maintaining logged-in state across multiple browser sessions without re-authentication. + +## QUICKSTART + +1. cd manual-MFA-with-contexts +2. uv venv venv +3. source venv/bin/activate # On Windows: venv\Scripts\activate +4. uvx install stagehand browserbase python-dotenv pydantic requests +5. cp .env.example .env +6. Add your Browserbase API key, Project ID, GitHub username, and password to .env +7. Ensure 2FA is enabled on your GitHub test account (Settings → Password and authentication → Enable two-factor authentication) +8. python main.py + +## EXPECTED OUTPUT + +- Creates a new Browserbase context +- First session: navigates to GitHub login, fills credentials, detects MFA prompt +- Pauses and displays Browserbase session link for manual MFA completion +- Waits for MFA completion (2 minute timeout) +- Saves authentication state to context +- Second session: reuses context, navigates to GitHub (already logged in, no MFA) +- Extracts logged-in username to verify authentication +- Cleans up context + +## COMMON PITFALLS + +- "ModuleNotFoundError": ensure all dependencies are installed via uvx install +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, GITHUB_USERNAME, and GITHUB_PASSWORD +- MFA timeout: ensure you complete MFA within 2 minutes, or increase timeout value +- 2FA not enabled: GitHub account must have 2FA enabled for this demo to work +- Context not persisting: verify context.persist is set to true in browser_settings + +## USE CASES + +• Payment automation: Complete MFA once for utility portals, then automate future payments without MFA prompts. +• Account management: Persist authentication for services requiring MFA, enabling automated account management workflows. +• Compliance automation: Store trusted device state for regulatory portals, reducing friction for recurring compliance tasks. + +## NEXT STEPS + +• Store context IDs: Save context_id per customer in your database to reuse across sessions. +• Multi-portal support: Extend to multiple portals/services, each with their own context. +• Context management: Implement context cleanup, rotation, and expiration policies. +• Error handling: Add retry logic and better error messages for MFA timeouts. + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +📚 Contexts Docs: https://docs.browserbase.com/features/contexts +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/python/manual-MFA-with-contexts/main.py b/python/manual-MFA-with-contexts/main.py new file mode 100644 index 00000000..c019c182 --- /dev/null +++ b/python/manual-MFA-with-contexts/main.py @@ -0,0 +1,271 @@ +# Manual MFA with Browserbase Contexts - See README.md for full documentation + +import asyncio +import os +import time + +import requests +from browserbase import Browserbase +from dotenv import load_dotenv +from pydantic import BaseModel, Field + +from stagehand import Stagehand, StagehandConfig + +# Load environment variables +load_dotenv() + +bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) + + +async def create_session_with_context(): + """First session: Create context and login (with MFA)""" + print("Creating new Browserbase context...") + + context = bb.contexts.create(project_id=os.environ.get("BROWSERBASE_PROJECT_ID")) + + print(f"Context created: {context.id}") + print("First session: Performing login with MFA...") + + config = StagehandConfig( + env="BROWSERBASE", + api_key=os.environ.get("BROWSERBASE_API_KEY"), + project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_name="google/gemini-2.5-flash", + model_api_key=os.environ.get("GOOGLE_GENERATIVE_AI_API_KEY"), + browserbase_session_create_params={ + "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), + "browser_settings": { + "context": { + "id": context.id, + "persist": True, # Save authentication state including MFA + } + }, + }, + verbose=0, # 0 = errors only, 1 = info, 2 = debug + # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + # https://docs.stagehand.dev/configuration/logging + ) + + async with Stagehand(config) as stagehand: + session_id = None + if hasattr(stagehand, "session_id"): + session_id = stagehand.session_id + elif hasattr(stagehand, "browserbase_session_id"): + session_id = stagehand.browserbase_session_id + + if session_id: + print(f"Watch live: https://browserbase.com/sessions/{session_id}") + + page = stagehand.page + + # Navigate to GitHub login + print("Navigating to GitHub login...") + await page.goto("https://github.com/login", wait_until="domcontentloaded") + + # Fill in credentials + print("Entering username...") + await page.act(f"Type '{os.environ.get('GITHUB_USERNAME')}' into the username field") + + print("Entering password...") + await page.act(f"Type '{os.environ.get('GITHUB_PASSWORD')}' into the password field") + + print("Clicking Sign in...") + await page.act("Click the Sign in button") + + await page.wait_for_load_state("networkidle") + + # Check if MFA is required + class MFARequired(BaseModel): + mfa_required: bool = Field(..., description="Whether MFA is required") + + mfa_check = await page.extract( + instruction="Is there a two-factor authentication or verification code prompt on the page?", + schema=MFARequired, + ) + + if mfa_check.mfa_required: + print("MFA DETECTED!") + print("═══════════════════════════════════════════════════════════") + print("PAUSED: Please complete MFA in the browser") + print("═══════════════════════════════════════════════════════════") + if session_id: + print( + f"1. Open the Browserbase session in your browser: https://browserbase.com/sessions/{session_id}" + ) + else: + print("1. Open the Browserbase session in your browser") + print("2. Enter your 2FA code from authenticator app") + print("3. Click 'Verify' or submit") + print("4. Wait for login to complete") + print("\nThe script will wait for you to complete MFA...\n") + + # Wait for MFA completion (poll until we're no longer on login page) + login_complete = False + start_time = time.time() + timeout = 120 # 2 minutes + + while not login_complete and (time.time() - start_time) < timeout: + await asyncio.sleep(3) # Check every 3 seconds + + current_url = page.url + if "/login" not in current_url and "/sessions/two-factor" not in current_url: + login_complete = True + + if not login_complete: + raise Exception("MFA timeout - login was not completed within 2 minutes") + + print("MFA completed! Login successful.\n") + else: + print("Login successful (no MFA required)\n") + + print(f"Context {context.id} now contains:") + print(" - Session cookies") + print(" - MFA trust/remember device state") + print(" - All authentication data\n") + + return context.id + + +async def reuse_context(context_id: str): + """Second session: Reuse context - NO MFA needed!""" + print(f"Second session: Reusing context {context_id}") + print(" (No login, no MFA required - auth state persisted)\n") + + config = StagehandConfig( + env="BROWSERBASE", + api_key=os.environ.get("BROWSERBASE_API_KEY"), + project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_name="google/gemini-2.5-flash", + model_api_key=os.environ.get("GOOGLE_GENERATIVE_AI_API_KEY"), + browserbase_session_create_params={ + "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), + "browser_settings": { + "context": { + "id": context_id, + "persist": True, + } + }, + }, + verbose=0, # 0 = errors only, 1 = info, 2 = debug + # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + # https://docs.stagehand.dev/configuration/logging + ) + + async with Stagehand(config) as stagehand: + session_id = None + if hasattr(stagehand, "session_id"): + session_id = stagehand.session_id + elif hasattr(stagehand, "browserbase_session_id"): + session_id = stagehand.browserbase_session_id + + if session_id: + print(f"Watch live: https://browserbase.com/sessions/{session_id}") + + page = stagehand.page + + # Navigate directly to GitHub (should already be logged in) + print("Navigating to GitHub...") + await page.goto("https://github.com", wait_until="domcontentloaded") + await page.wait_for_load_state("networkidle") + + # Check if we're logged in + class Username(BaseModel): + username: str = Field(..., description="The logged-in username") + + username_result = await page.extract( + instruction="Extract the logged-in username or check if we're authenticated", + schema=Username, + ) + + print("\nSUCCESS! Already logged in without MFA!") + print(f" Username: {username_result.username}") + print("\nThis is the power of Browserbase Contexts:") + print(" - First session: User completes MFA once") + print(" - Context saves trusted device state") + print(" - All future sessions: No MFA required\n") + + +async def delete_context(context_id: str): + """Clean up context""" + print(f"Deleting context: {context_id}") + try: + # Delete via API (SDK doesn't have delete method) + response = requests.delete( + f"https://api.browserbase.com/v1/contexts/{context_id}", + headers={ + "X-BB-API-Key": os.environ.get("BROWSERBASE_API_KEY"), + }, + ) + + if response.ok: + print("Context deleted\n") + else: + print(f"Could not delete context: {response.status_code} {response.reason}") + print(" Context will auto-expire after 30 days\n") + except Exception as error: + print(f"Could not delete context: {str(error)}") + print(" Context will auto-expire after 30 days\n") + + +async def main(): + print("Starting Browserbase Context MFA Persistence Demo...") + + # Check environment variables + if not os.environ.get("BROWSERBASE_API_KEY") or not os.environ.get("BROWSERBASE_PROJECT_ID"): + print("\nError: Missing Browserbase credentials") + print(" Set BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID in .env") + exit(1) + + if not os.environ.get("GITHUB_USERNAME") or not os.environ.get("GITHUB_PASSWORD"): + print("\nError: Missing GitHub credentials") + print(" Set GITHUB_USERNAME and GITHUB_PASSWORD in .env") + print("Setup Instructions:") + print(" 1. Create a test GitHub account") + print(" 2. Enable 2FA: Settings → Password and authentication") + print(" 3. Set credentials in .env file") + exit(1) + + try: + print("\nDemo Flow:") + print(" 1. First session: Login + complete MFA manually") + print(" 2. Second session: No login, no MFA needed") + print(" 3. Clean up context\n") + + # First session: Create context and login with MFA + context_id = await create_session_with_context() + + print("Waiting 5 seconds before reusing context...\n") + await asyncio.sleep(5) + + # Second session: Reuse context (NO MFA!) + await reuse_context(context_id) + + # Clean up + await delete_context(context_id) + + print("═══════════════════════════════════════════════════════════") + print("Key Takeaway:") + print("═══════════════════════════════════════════════════════════") + print("First session: User completes MFA once") + print("Context saves trusted device state") + print("All future sessions: No MFA prompt") + print("Store context_id per customer in database\n") + except Exception as error: + print(f"\nError: {str(error)}") + print("\nTroubleshooting:") + print(" - Ensure GitHub credentials are correct") + print(" - Ensure 2FA is enabled on the test account") + print(" - Check Browserbase dashboard for session details") + raise + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Application error: {err}") + print("Common issues:") + print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") + print(" - Verify GOOGLE_GENERATIVE_AI_API_KEY is set in environment") + print("Docs: https://docs.stagehand.dev/v2/first-steps/introduction") + exit(1) diff --git a/python/manual-MFA-with-contexts/requirements.txt b/python/manual-MFA-with-contexts/requirements.txt new file mode 100644 index 00000000..e180bbee --- /dev/null +++ b/python/manual-MFA-with-contexts/requirements.txt @@ -0,0 +1,6 @@ +browserbase +python-dotenv +pydantic +stagehand +requests + diff --git a/typescript/manual-MFA-with-contexts/README.md b/typescript/manual-MFA-with-contexts/README.md new file mode 100644 index 00000000..763a4ed2 --- /dev/null +++ b/typescript/manual-MFA-with-contexts/README.md @@ -0,0 +1,65 @@ +# Stagehand + Browserbase: Manual MFA with Contexts + +## AT A GLANCE + +- Goal: demonstrate how to persist authentication across sessions using Browserbase Contexts, eliminating MFA friction after the first login. +- Flow: first session creates context and completes MFA manually → context saves auth state → second session reuses context with no MFA required. + +## GLOSSARY + +- context: a Browserbase feature that persists browser state (cookies, localStorage, sessionStorage) across sessions. + Docs → https://docs.browserbase.com/features/contexts +- persist: setting that saves authentication state including MFA trust/remember device state to the context. +- MFA (Multi-Factor Authentication): two-factor authentication requiring a code from an authenticator app. +- session persistence: maintaining logged-in state across multiple browser sessions without re-authentication. + +## QUICKSTART + +1. cd manual-MFA-with-contexts +2. pnpm install +3. cp .env.example .env +4. Add your Browserbase API key, Project ID, GitHub username, and password to .env +5. Ensure 2FA is enabled on your GitHub test account (Settings → Password and authentication → Enable two-factor authentication) +6. pnpm start + +## EXPECTED OUTPUT + +- Creates a new Browserbase context +- First session: navigates to GitHub login, fills credentials, detects MFA prompt +- Pauses and displays Browserbase session link for manual MFA completion +- Waits for MFA completion (2 minute timeout) +- Saves authentication state to context +- Second session: reuses context, navigates to GitHub (already logged in, no MFA) +- Extracts logged-in username to verify authentication +- Cleans up context + +## COMMON PITFALLS + +- "Cannot find module 'dotenv'": ensure pnpm install ran successfully +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, GITHUB_USERNAME, and GITHUB_PASSWORD +- MFA timeout: ensure you complete MFA within 2 minutes, or increase timeout value +- 2FA not enabled: GitHub account must have 2FA enabled for this demo to work +- Context not persisting: verify context.persist is set to true in browserSettings + +## USE CASES + +• Payment automation: Complete MFA once for utility portals, then automate future payments without MFA prompts. +• Account management: Persist authentication for services requiring MFA, enabling automated account management workflows. +• Compliance automation: Store trusted device state for regulatory portals, reducing friction for recurring compliance tasks. + +## NEXT STEPS + +• Store context IDs: Save context_id per customer in your database to reuse across sessions. +• Multi-portal support: Extend to multiple portals/services, each with their own context. +• Context management: Implement context cleanup, rotation, and expiration policies. +• Error handling: Add retry logic and better error messages for MFA timeouts. + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +📚 Contexts Docs: https://docs.browserbase.com/features/contexts +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/typescript/manual-MFA-with-contexts/index.ts b/typescript/manual-MFA-with-contexts/index.ts new file mode 100644 index 00000000..785f8dac --- /dev/null +++ b/typescript/manual-MFA-with-contexts/index.ts @@ -0,0 +1,254 @@ +// Manual MFA with Browserbase Contexts - See README.md for full documentation + +import "dotenv/config"; +import { Stagehand } from "@browserbasehq/stagehand"; +import Browserbase from "@browserbasehq/sdk"; +import { z } from "zod"; + +const bb = new Browserbase({ + apiKey: process.env.BROWSERBASE_API_KEY, +}); + +/** + * First session: Create context and login (with MFA) + */ +async function createSessionWithContext() { + console.log("Creating new Browserbase context..."); + + const context = await bb.contexts.create({ + projectId: process.env.BROWSERBASE_PROJECT_ID!, + }); + + console.log(`Context created: ${context.id}`); + console.log("First session: Performing login with MFA..."); + + const stagehand = new Stagehand({ + env: "BROWSERBASE", + verbose: 0, + // 0 = errors only, 1 = info, 2 = debug + // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + // https://docs.stagehand.dev/configuration/logging + model: "openai/gpt-4.1-mini", + browserbaseSessionCreateParams: { + projectId: process.env.BROWSERBASE_PROJECT_ID!, + browserSettings: { + context: { + id: context.id, + persist: true, // Save authentication state including MFA + }, + }, + }, + }); + + await stagehand.init(); + console.log(`Watch live: https://browserbase.com/sessions/${stagehand.browserbaseSessionID}`); + + const page = stagehand.context.pages()[0]; + + // Navigate to GitHub login + console.log("Navigating to GitHub login..."); + await page.goto("https://github.com/login"); + await page.waitForLoadState("domcontentloaded"); + + // Fill in credentials + console.log("Entering username..."); + await stagehand.act(`Type '${process.env.GITHUB_USERNAME}' into the username field`); + + console.log("Entering password..."); + await stagehand.act(`Type '${process.env.GITHUB_PASSWORD}' into the password field`); + + console.log("Clicking Sign in..."); + await stagehand.act("Click the Sign in button"); + + await page.waitForLoadState("networkidle"); + + // Check if MFA is required + const mfaRequired = await stagehand.extract( + "Is there a two-factor authentication or verification code prompt on the page?", + z.boolean(), + ); + + if (mfaRequired) { + console.log("MFA DETECTED!"); + console.log("═══════════════════════════════════════════════════════════"); + console.log("PAUSED: Please complete MFA in the browser"); + console.log("═══════════════════════════════════════════════════════════"); + console.log( + `1. Open the Browserbase session in your browser: https://browserbase.com/sessions/${stagehand.browserbaseSessionID}`, + ); + console.log("2. Enter your 2FA code from authenticator app"); + console.log("3. Click 'Verify' or submit"); + console.log("4. Wait for login to complete"); + console.log("\nThe script will wait for you to complete MFA...\n"); + + // Wait for MFA completion (poll until we're no longer on login page) + let loginComplete = false; + const startTime = Date.now(); + const timeout = 120000; // 2 minutes + + while (!loginComplete && Date.now() - startTime < timeout) { + await new Promise((resolve) => setTimeout(resolve, 3000)); // Check every 3 seconds + + const currentUrl = page.url(); + if (!currentUrl.includes("/login") && !currentUrl.includes("/sessions/two-factor")) { + loginComplete = true; + } + } + + if (!loginComplete) { + throw new Error("MFA timeout - login was not completed within 2 minutes"); + } + + console.log("MFA completed! Login successful.\n"); + } else { + console.log("Login successful (no MFA required)\n"); + } + + console.log(`Context ${context.id} now contains:`); + console.log(" - Session cookies"); + console.log(" - MFA trust/remember device state"); + console.log(" - All authentication data\n"); + + await stagehand.close(); + + return context.id; +} + +/** + * Second session: Reuse context - NO MFA needed! + */ +async function reuseContext(contextId: string) { + console.log(`Second session: Reusing context ${contextId}`); + console.log(" (No login, no MFA required - auth state persisted)\n"); + + const stagehand = new Stagehand({ + env: "BROWSERBASE", + verbose: 0, + // 0 = errors only, 1 = info, 2 = debug + // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + // https://docs.stagehand.dev/configuration/logging + model: "openai/gpt-4.1-mini", + browserbaseSessionCreateParams: { + projectId: process.env.BROWSERBASE_PROJECT_ID!, + browserSettings: { + context: { + id: contextId, + persist: true, + }, + }, + }, + }); + + await stagehand.init(); + console.log(`Watch live: https://browserbase.com/sessions/${stagehand.browserbaseSessionID}`); + + const page = stagehand.context.pages()[0]; + + // Navigate directly to GitHub (should already be logged in) + console.log("Navigating to GitHub..."); + await page.goto("https://github.com"); + await page.waitForLoadState("networkidle"); + + // Check if we're logged in + const username = await stagehand.extract( + "Extract the logged-in username or check if we're authenticated", + z.string(), + ); + + console.log("\nSUCCESS! Already logged in without MFA!"); + console.log(` Username: ${username}`); + console.log("\nThis is the power of Browserbase Contexts:"); + console.log(" - First session: User completes MFA once"); + console.log(" - Context saves trusted device state"); + console.log(" - All future sessions: No MFA required\n"); + + await stagehand.close(); +} + +/** + * Clean up context + */ +async function deleteContext(contextId: string) { + console.log(`Deleting context: ${contextId}`); + try { + // Delete via API (SDK doesn't have delete method) + const response = await fetch(`https://api.browserbase.com/v1/contexts/${contextId}`, { + method: "DELETE", + headers: { + "X-BB-API-Key": process.env.BROWSERBASE_API_KEY!, + }, + }); + + if (response.ok) { + console.log("Context deleted\n"); + } else { + console.log(`Could not delete context: ${response.status} ${response.statusText}`); + console.log(" Context will auto-expire after 30 days\n"); + } + } catch (error) { + console.log( + `Could not delete context: ${error instanceof Error ? error.message : String(error)}`, + ); + console.log(" Context will auto-expire after 30 days\n"); + } +} + +async function main() { + console.log("Starting Browserbase Context MFA Persistence Demo..."); + + // Check environment variables + if (!process.env.BROWSERBASE_API_KEY || !process.env.BROWSERBASE_PROJECT_ID) { + console.error("\n❌ Missing Browserbase credentials"); + console.error(" Set BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID in .env"); + process.exit(1); + } + + if (!process.env.GITHUB_USERNAME || !process.env.GITHUB_PASSWORD) { + console.error("\nError: Missing GitHub credentials"); + console.error(" Set GITHUB_USERNAME and GITHUB_PASSWORD in .env"); + console.error("Setup Instructions:"); + console.error(" 1. Create a test GitHub account"); + console.error(" 2. Enable 2FA: Settings → Password and authentication"); + console.error(" 3. Set credentials in .env file"); + process.exit(1); + } + + try { + console.log("\n📋 Demo Flow:"); + console.log(" 1. First session: Login + complete MFA manually"); + console.log(" 2. Second session: No login, no MFA needed"); + console.log(" 3. Clean up context\n"); + + // First session: Create context and login with MFA + const contextId = await createSessionWithContext(); + + console.log("⏳ Waiting 5 seconds before reusing context...\n"); + await new Promise((resolve) => setTimeout(resolve, 5000)); + + // Second session: Reuse context (NO MFA!) + await reuseContext(contextId); + + // Clean up + await deleteContext(contextId); + + console.log("═══════════════════════════════════════════════════════════"); + console.log("Key Takeaway:"); + console.log("═══════════════════════════════════════════════════════════"); + console.log("✅ First session: User completes MFA once"); + console.log("✅ Context saves trusted device state"); + console.log("✅ All future sessions: No MFA prompt"); + console.log("✅ Store context_id per customer in database\n"); + } catch (error) { + console.error("\n❌ Error:", error instanceof Error ? error.message : String(error)); + console.error("\nTroubleshooting:"); + console.error(" - Ensure GitHub credentials are correct"); + console.error(" - Ensure 2FA is enabled on the test account"); + console.error(" - Check Browserbase dashboard for session details"); + throw error; + } +} + +main().catch((err) => { + console.error("Application error:", err); + process.exit(1); +}); diff --git a/typescript/manual-MFA-with-contexts/package.json b/typescript/manual-MFA-with-contexts/package.json new file mode 100644 index 00000000..4d63fd4e --- /dev/null +++ b/typescript/manual-MFA-with-contexts/package.json @@ -0,0 +1,22 @@ +{ + "name": "manual-mfa-with-contexts", + "version": "1.0.0", + "description": "Stagehand + Browserbase: Manual MFA with Contexts", + "type": "module", + "main": "index.ts", + "scripts": { + "start": "tsx index.ts", + "dev": "tsx watch index.ts" + }, + "dependencies": { + "@browserbasehq/sdk": "^1.0.0", + "@browserbasehq/stagehand": "^3.0.0", + "dotenv": "^16.4.5", + "zod": "^3.23.8" + }, + "devDependencies": { + "@types/node": "^20.14.0", + "tsx": "^4.16.0", + "typescript": "^5.5.0" + } +} From 78b6631673a7c37249ee5b87052c2c279a41077b Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Tue, 6 Jan 2026 17:06:55 +0000 Subject: [PATCH 04/45] Add basic-caching template (#27) --- python/basic-caching/README.md | 126 +++++++++++ python/basic-caching/main.py | 296 ++++++++++++++++++++++++++ python/basic-caching/requirements.txt | 4 + typescript/basic-caching/README.md | 121 +++++++++++ typescript/basic-caching/index.ts | 169 +++++++++++++++ typescript/basic-caching/package.json | 20 ++ 6 files changed, 736 insertions(+) create mode 100644 python/basic-caching/README.md create mode 100644 python/basic-caching/main.py create mode 100644 python/basic-caching/requirements.txt create mode 100644 typescript/basic-caching/README.md create mode 100644 typescript/basic-caching/index.ts create mode 100644 typescript/basic-caching/package.json diff --git a/python/basic-caching/README.md b/python/basic-caching/README.md new file mode 100644 index 00000000..4fb456f5 --- /dev/null +++ b/python/basic-caching/README.md @@ -0,0 +1,126 @@ +# Stagehand + Browserbase: Basic Caching + +## AT A GLANCE + +- Goal: Demonstrate how Stagehand's caching feature dramatically reduces cost and latency by reusing previously computed actions instead of calling the LLM every time. +- Shows side-by-side comparison of workflows with and without caching enabled. +- Demonstrates massive cost savings for repeated workflows (99.9% reduction in LLM calls). +- Docs → https://docs.stagehand.dev/v2/best-practices/caching#caching-actions + +## GLOSSARY + +- caching: Stagehand can cache action results based on instruction text and page context, eliminating redundant LLM calls + Docs → https://docs.stagehand.dev/v2/best-practices/caching#caching-actions +- act: execute actions on web pages using natural language instructions + Docs → https://docs.stagehand.dev/v2/basics/act +- observe: observe page elements and generate actions that can be cached and reused + Docs → https://docs.stagehand.dev/v2/basics/observe + +## QUICKSTART + +1. uv venv venv +2. source venv/bin/activate # On Windows: venv\Scripts\activate +3. uvx install stagehand python-dotenv aiofiles +4. cp .env.example .env +5. Add required API keys/IDs to .env +6. python main.py (run twice to see cache benefits!) + +## EXPECTED OUTPUT + +- First run: Executes workflow without cache, then with cache enabled (populates cache) +- Subsequent runs: Uses cached actions for instant execution with zero LLM calls +- Displays timing comparison, cost savings, and cache statistics +- Shows cache location and file structure + +## HOW CACHING WORKS + +**Cache Key Generation:** + +- Based on instruction text (used as cache key) +- Actions are observed once and cached +- Automatically computed + +**When Cache is Used:** + +- ✅ Same instruction text +- ✅ Cache file exists (`cache.json`) +- ❌ Different instruction text +- ❌ Cache file cleared or missing + +**Cache Storage:** + +- Location: `cache.json` in the same directory as `main.py` +- Format: JSON file containing cached actions +- Persistent across runs + +## BENEFITS FOR REPEATED WORKFLOWS + +**Example Scenario: 1,000 customers × 10 portals = 10,000 payment flows** + +**Without caching:** + +- 10,000 workflows × 5 actions = 50,000 LLM calls +- Cost: ~$500-2,500 +- Latency: 2-3s per action × 5 = 10-15s per payment + +**With caching:** + +- First payment per portal: 5 LLM calls (populate cache) +- Next 999 payments: 0 LLM calls (use cache) +- Total: 10 portals × 5 actions = 50 LLM calls +- Cost: ~$0.50-2.50 (99.9% savings!) +- Latency: <100ms per action × 5 = <0.5s per payment + +**Key Insight:** +Payment portals rarely change → Cache actions once → Reuse for thousands of payments → Massive cost + latency reduction + +## COMMON PITFALLS + +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY +- Cache not working: ensure cache.json is writable and check that instruction text matches exactly +- First run slower: expected behavior - cache is populated on first run, subsequent runs will be instant +- ModuleNotFoundError: ensure virtual environment is activated and dependencies are installed via `uvx install` +- Import errors: activate your virtual environment if you created one +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +• Payment processing: Cache form-filling actions for payment portals that don't change frequently, processing thousands of payments with minimal LLM calls. +• Data entry automation: Reuse actions for repetitive data entry tasks across similar forms or interfaces. +• Testing workflows: Cache test actions to speed up regression testing and reduce API costs during development. + +## BEST PRACTICES + +- ✅ Enable caching in production for repeated workflows +- ✅ One cache per portal/interface type +- ✅ Invalidate cache when page structure changes significantly +- ✅ Monitor cache hit rate to optimize cache effectiveness +- ✅ Warm cache with test runs before production deployment + +## NEXT STEPS + +• Customize cache file location: Modify CACHE_FILE path to organize caches by workflow type or environment. +• Add cache invalidation: Implement logic to clear cache when page structure changes or after a certain time period. +• Monitor cache performance: Track cache hit rates and cost savings to measure effectiveness. + +## TRY IT YOURSELF + +1. Run this script again: `python main.py` + → Second run will be MUCH faster (cache hits) + +2. Clear cache and run again: + `rm cache.json && python main.py` + → Back to first-run behavior + +3. Check cache contents: + `cat cache.json` + → See cached action data + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/python/basic-caching/main.py b/python/basic-caching/main.py new file mode 100644 index 00000000..e7fa99f0 --- /dev/null +++ b/python/basic-caching/main.py @@ -0,0 +1,296 @@ +# Stagehand + Browserbase: Basic Caching - See README.md for full documentation + +import asyncio +import json +import os +from pathlib import Path +from typing import Any + +import aiofiles +from dotenv import load_dotenv + +from stagehand import Stagehand, StagehandConfig + +# Load environment variables +load_dotenv() + +# Cache file location - stores observed actions for reuse +CACHE_FILE = Path(__file__).parent / "cache.json" + + +async def get_cache(key: str) -> dict[str, Any] | None: + """Get the cached value (None if it doesn't exist)""" + try: + # Read cache file asynchronously for better performance + async with aiofiles.open(CACHE_FILE) as f: + cache_content = await f.read() + parsed = json.loads(cache_content) + return parsed.get(key) + except (FileNotFoundError, json.JSONDecodeError): + # Cache file doesn't exist or is invalid - return None + return None + + +async def set_cache(key: str, value: Any) -> None: + """Set the cache value - converts ObserveResult (Pydantic model) to dict if needed""" + try: + # Read existing cache file to preserve other cached entries + async with aiofiles.open(CACHE_FILE) as f: + cache_content = await f.read() + parsed = json.loads(cache_content) + except (FileNotFoundError, json.JSONDecodeError): + # Cache file doesn't exist - start with empty dict + parsed = {} + + # Convert ObserveResult (Pydantic BaseModel) to dict for JSON serialization + # Supports both Pydantic v1 and v2 for compatibility + if hasattr(value, "model_dump"): + # Pydantic v2 + parsed[key] = value.model_dump() + elif hasattr(value, "dict"): + # Pydantic v1 + parsed[key] = value.dict() + elif isinstance(value, dict): + parsed[key] = value + else: + # Fallback: try to convert to dict + parsed[key] = dict(value) if hasattr(value, "__dict__") else value + + # Write updated cache back to file + async with aiofiles.open(CACHE_FILE, "w") as f: + await f.write(json.dumps(parsed, indent=2, default=str)) + + +async def act_with_cache(page, key: str, prompt: str, self_heal: bool = False): + """ + Check the cache, get the action, and run it. + If self_heal is true, we'll attempt to self-heal if the action fails. + + This function demonstrates manual caching by: + 1. Checking if action is cached (no LLM call) + 2. If not cached, observing the page to generate action (LLM call) + 3. Caching the observed action for future use + 4. Executing the action + """ + try: + # Check if action is already cached + cache_exists = await get_cache(key) + + if cache_exists: + # Use the already-retrieved cached action - no LLM inference needed + action = cache_exists + print(f" ✓ Cache hit for: {prompt}") + else: + # Get the observe result (the action) - this requires LLM inference + print(f" → Observing: {prompt}") + actions = await page.observe(prompt) + action = actions[0] + + # Cache the action for future use + await set_cache(key, action) + print(f" ✓ Cached action for: {prompt}") + + # Run the action (no LLM inference when using cached action) + await page.act(action) + except Exception as e: + print(f" ✗ Error: {e}") + # In self_heal mode, retry the action with a fresh LLM call + if self_heal: + print(" → Attempting to self-heal...") + await page.act(prompt) + else: + raise e + + +async def run_without_cache(): + """Run workflow without caching (baseline) - demonstrates normal LLM usage""" + print("RUN 1: WITHOUT CACHING") + + start_time = asyncio.get_event_loop().time() + + # Initialize Stagehand with Browserbase for cloud-based browser automation. + # Note: set verbose: 0 to prevent API keys from appearing in logs when handling sensitive data. + config = StagehandConfig( + env="BROWSERBASE", + api_key=os.getenv("BROWSERBASE_API_KEY"), + project_id=os.getenv("BROWSERBASE_PROJECT_ID"), + model_name="google/gemini-2.5-flash", + model_api_key=os.getenv("GOOGLE_API_KEY"), + verbose=0, # 0 = errors only, 1 = info, 2 = debug + # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + # https://docs.stagehand.dev/configuration/logging + browserbase_session_create_params={ + "project_id": os.getenv("BROWSERBASE_PROJECT_ID"), + }, + ) + + # Use async context manager for automatic resource management + async with Stagehand(config) as stagehand: + page = stagehand.page + + try: + # Navigate to Stripe checkout demo page + print("Navigating to Stripe checkout...") + await page.goto("https://checkout.stripe.dev/preview", wait_until="domcontentloaded") + + # Each act() call requires LLM inference - no caching enabled + await page.act("Click on the View Demo button") + await page.act("Type 'test@example.com' into the email field") + await page.act("Type '4242424242424242' into the card number field") + await page.act("Type '12/34' into the expiration date field") + + elapsed = f"{(asyncio.get_event_loop().time() - start_time):.2f}" + + print(f"Total time: {elapsed}s") + print("Cost: ~$0.01-0.05 (4 LLM calls)") + print("API calls: 4 (one per action)\n") + + return {"elapsed": elapsed, "llm_calls": 4} + except Exception as error: + print(f"Error: {error}") + raise + + +async def run_with_cache(): + """Run workflow with caching enabled - demonstrates cost and latency savings""" + print("RUN 2: WITH CACHING\n") + + start_time = asyncio.get_event_loop().time() + + # Initialize Stagehand with Browserbase for cloud-based browser automation. + config = StagehandConfig( + env="BROWSERBASE", + api_key=os.getenv("BROWSERBASE_API_KEY"), + project_id=os.getenv("BROWSERBASE_PROJECT_ID"), + model_name="google/gemini-2.5-flash", + model_api_key=os.getenv("GOOGLE_API_KEY"), + verbose=0, # 0 = errors only, 1 = info, 2 = debug + # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + # https://docs.stagehand.dev/configuration/logging + browserbase_session_create_params={ + "project_id": os.getenv("BROWSERBASE_PROJECT_ID"), + }, + ) + + # Use async context manager for automatic resource management + async with Stagehand(config) as stagehand: + page = stagehand.page + + try: + # Navigate to Stripe checkout demo page + print("Navigating to Stripe checkout...") + await page.goto("https://checkout.stripe.dev/preview", wait_until="domcontentloaded") + + # Use cached actions - first run will observe and cache, subsequent runs use cache + await act_with_cache( + page, "Click on the View Demo button", "Click on the View Demo button" + ) + await act_with_cache( + page, + "Type 'test@example.com' into the email field", + "Type 'test@example.com' into the email field", + ) + await act_with_cache( + page, + "Type '4242424242424242' into the card number field", + "Type '4242424242424242' into the card number field", + ) + await act_with_cache( + page, + "Type '12/34' into the expiration date field", + "Type '12/34' into the expiration date field", + ) + + elapsed = f"{(asyncio.get_event_loop().time() - start_time):.2f}" + cache_exists = CACHE_FILE.exists() + + # Count cache entries to determine if this was a cache hit or miss + if cache_exists: + async with aiofiles.open(CACHE_FILE) as f: + cache_content = await f.read() + cache_data = json.loads(cache_content) + cache_count = len(cache_data) + else: + cache_count = 0 + + print(f"\nTotal time: {elapsed}s") + + # Display results based on cache status + if cache_count >= 4: + # Cache was used - no LLM calls made + print("Cost: $0.00 (cache hits, no LLM calls)") + print("API calls: 0 (all from cache)") + print(f"Cache entries: {cache_count}") + else: + # First run - cache was populated + print("💰Cost: ~$0.01-0.05 (first run, populated cache)") + print("📡API calls: 4 (saved to cache for next run)") + print("📂Cache created") + print() + + return {"elapsed": elapsed, "llm_calls": 0 if cache_count >= 4 else 4} + except Exception as error: + print(f"Error: {error}") + raise + + +async def main(): + """Main function demonstrating caching benefits with side-by-side comparison""" + print("\n╔═══════════════════════════════════════════════════════════╗") + print("║ Caching Demo - Run This Script TWICE! ║") + print("╚═══════════════════════════════════════════════════════════╝\n") + + print("This demo shows caching impact by running the same workflow twice:\n") + print("First run:") + print(" 1. WITHOUT cache (baseline)") + print(" 2. WITH cache enabled (populates cache)\n") + + print("Second run:") + print(" - WITH cache (instant, $0 cost)\n") + + print("Run 'python main.py' twice to see the difference!\n") + + # Check if cache exists to determine if this is first or subsequent run + cache_exists = CACHE_FILE.exists() + + if cache_exists: + # Read cache file to count entries + async with aiofiles.open(CACHE_FILE) as f: + cache_content = await f.read() + cache_data = json.loads(cache_content) + cache_count = len(cache_data) + print(f"📂 Cache found: {cache_count} entries") + print(" This is a SUBSEQUENT run - cache will be used!\n") + else: + print("No cache found - first run will populate cache") + + print("\nRunning comparison: without cache vs with cache...\n") + + # Run both workflows for comparison + without_cache = await run_without_cache() + with_cache = await run_with_cache() + + # Display comparison results + print("\n=== Comparison ===") + print(f"Without caching: {without_cache['elapsed']}s, {without_cache['llm_calls']} LLM calls") + print(f"With caching: {with_cache['elapsed']}s, {with_cache['llm_calls']} LLM calls") + + # Calculate and display speedup if cache was used + if with_cache["llm_calls"] == 0: + speedup = float(without_cache["elapsed"]) / float(with_cache["elapsed"]) + print(f"\nSpeedup: {speedup:.1f}x faster with cache") + print("Cost savings: 100% (no LLM calls)") + + print("\nRun again to see cache benefits on subsequent runs!") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Error in caching demo: {err}") + print("Common issues:") + print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") + print(" - Verify GOOGLE_API_KEY is set for the model") + print("Docs: https://docs.stagehand.dev/v2/first-steps/introduction") + exit(1) diff --git a/python/basic-caching/requirements.txt b/python/basic-caching/requirements.txt new file mode 100644 index 00000000..2a48386c --- /dev/null +++ b/python/basic-caching/requirements.txt @@ -0,0 +1,4 @@ +stagehand +python-dotenv +aiofiles + diff --git a/typescript/basic-caching/README.md b/typescript/basic-caching/README.md new file mode 100644 index 00000000..1e6327ae --- /dev/null +++ b/typescript/basic-caching/README.md @@ -0,0 +1,121 @@ +# Stagehand + Browserbase: Basic Caching + +## AT A GLANCE + +- Goal: Demonstrate how Stagehand's caching feature dramatically reduces cost and latency by reusing previously computed actions instead of calling the LLM every time. +- Shows side-by-side comparison of workflows with and without caching enabled. +- Demonstrates massive cost savings for repeated workflows (99.9% reduction in LLM calls). +- Docs → https://docs.stagehand.dev/v3/best-practices/caching#caching-actions + +## GLOSSARY + +- caching: Stagehand can cache action results based on instruction text and page context, eliminating redundant LLM calls + Docs → https://docs.stagehand.dev/v3/best-practices/caching#caching-actions +- act: execute actions on web pages using natural language instructions + Docs → https://docs.stagehand.dev/basics/act + +## QUICKSTART + +1. pnpm install +2. cp .env.example .env +3. Add required API keys/IDs to .env +4. pnpm start (run twice to see cache benefits!) + +## EXPECTED OUTPUT + +- First run: Executes workflow without cache, then with cache enabled (populates cache) +- Subsequent runs: Uses cached actions for instant execution with zero LLM calls +- Displays timing comparison, cost savings, and cache statistics +- Shows cache location and file structure + +## HOW CACHING WORKS + +**Cache Key Generation:** + +- Based on instruction text +- Based on page context +- Automatically computed + +**When Cache is Used:** + +- ✅ Same instruction +- ✅ Same page structure +- ✅ Cache file exists +- ❌ Different instruction +- ❌ Page structure changed significantly + +**Cache Storage:** + +- Location: `.cache/stagehand-demo` +- Format: JSON files (one per cached action) +- Persistent across runs + +## BENEFITS FOR REPEATED WORKFLOWS + +**Example Scenario: 1,000 customers × 10 portals = 10,000 payment flows** + +**Without caching:** + +- 10,000 workflows × 5 actions = 50,000 LLM calls +- Cost: ~$500-2,500 +- Latency: 2-3s per action × 5 = 10-15s per payment + +**With caching:** + +- First payment per portal: 5 LLM calls (populate cache) +- Next 999 payments: 0 LLM calls (use cache) +- Total: 10 portals × 5 actions = 50 LLM calls +- Cost: ~$0.50-2.50 (99.9% savings!) +- Latency: <100ms per action × 5 = <0.5s per payment + +**Key Insight:** +Payment portals rarely change → Cache actions once → Reuse for thousands of payments → Massive cost + latency reduction + +## COMMON PITFALLS + +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY +- Cache not working: ensure cacheDir path is writable and check that instruction text matches exactly +- First run slower: expected behavior - cache is populated on first run, subsequent runs will be instant +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +• Payment processing: Cache form-filling actions for payment portals that don't change frequently, processing thousands of payments with minimal LLM calls. +• Data entry automation: Reuse actions for repetitive data entry tasks across similar forms or interfaces. +• Testing workflows: Cache test actions to speed up regression testing and reduce API costs during development. + +## BEST PRACTICES + +- ✅ Enable caching in production for repeated workflows +- ✅ One cache per portal/interface type +- ✅ Invalidate cache when page structure changes significantly +- ✅ Monitor cache hit rate to optimize cache effectiveness +- ✅ Warm cache with test runs before production deployment + +## NEXT STEPS + +• Customize cache directory: Modify cacheDir to organize caches by workflow type or environment. +• Add cache invalidation: Implement logic to clear cache when page structure changes or after a certain time period. +• Monitor cache performance: Track cache hit rates and cost savings to measure effectiveness. + +## TRY IT YOURSELF + +1. Run this script again: `pnpm start` + → Second run will be MUCH faster (cache hits) + +2. Clear cache and run again: + `rm -rf .cache/stagehand-demo && pnpm start` + → Back to first-run behavior + +3. Check cache contents: + `ls -la .cache/stagehand-demo` + → See cached action files + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/typescript/basic-caching/index.ts b/typescript/basic-caching/index.ts new file mode 100644 index 00000000..8fb9ccbc --- /dev/null +++ b/typescript/basic-caching/index.ts @@ -0,0 +1,169 @@ +// Stagehand + Browserbase: Basic Caching - See README.md for full documentation + +import "dotenv/config"; +import { Stagehand } from "@browserbasehq/stagehand"; +import fs from "fs"; +import path from "path"; + +const CACHE_DIR = path.join(process.cwd(), ".cache", "stagehand-demo"); + +async function runWithoutCache() { + console.log("RUN 1: WITHOUT CACHING"); + + const startTime = Date.now(); + + const stagehand = new Stagehand({ + env: "BROWSERBASE", + verbose: 0, + // 0 = errors only, 1 = info, 2 = debug + // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + // https://docs.stagehand.dev/configuration/logging + model: "google/gemini-2.5-flash", + enableCaching: false, + browserbaseSessionCreateParams: { + projectId: process.env.BROWSERBASE_PROJECT_ID!, + }, + }); + + await stagehand.init(); + const page = stagehand.context.pages()[0]; + + try { + console.log("Navigating to Stripe checkout..."); + await page.goto("https://checkout.stripe.dev/preview", { + waitUntil: "domcontentloaded", + }); + + await stagehand.act("Click on the View Demo button"); + await stagehand.act("Type 'test@example.com' into the email field"); + await stagehand.act("Type '4242424242424242' into the card number field"); + await stagehand.act("Type '12/34' into the expiration date field"); + + const elapsed = ((Date.now() - startTime) / 1000).toFixed(2); + + console.log(`Total time: ${elapsed}s`); + console.log("Cost: ~$0.01-0.05 (4 LLM calls)"); + console.log("API calls: 4 (one per action)\n"); + + await stagehand.close(); + + return { elapsed, llmCalls: 4 }; + } catch (error) { + console.error("Error:", error.message); + await stagehand.close(); + throw error; + } +} + +async function runWithCache() { + console.log("RUN 2: WITH CACHING"); + + const startTime = Date.now(); + + const stagehand = new Stagehand({ + env: "BROWSERBASE", + verbose: 0, + // 0 = errors only, 1 = info, 2 = debug + // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + // https://docs.stagehand.dev/configuration/logging + model: "google/gemini-2.5-flash", + enableCaching: true, + cacheDir: CACHE_DIR, + browserbaseSessionCreateParams: { + projectId: process.env.BROWSERBASE_PROJECT_ID!, + }, + }); + + await stagehand.init(); + const page = stagehand.context.pages()[0]; + + try { + console.log("Navigating to Stripe checkout..."); + await page.goto("https://checkout.stripe.dev/preview", { + waitUntil: "domcontentloaded", + }); + + await stagehand.act("Click on the View Demo button"); + await stagehand.act("Type 'test@example.com' into the email field"); + await stagehand.act("Type '4242424242424242' into the card number field"); + await stagehand.act("Type '12/34' into the expiration date field"); + + const elapsed = ((Date.now() - startTime) / 1000).toFixed(2); + const cacheExists = fs.existsSync(CACHE_DIR); + const cacheFiles = cacheExists ? fs.readdirSync(CACHE_DIR).length : 0; + + console.log(`Total time: ${elapsed}s`); + + if (cacheFiles > 0) { + console.log("Cost: $0.00 (cache hits, no LLM calls)"); + console.log("API calls: 0 (all from cache)"); + console.log(`Cache entries: ${cacheFiles}`); + } else { + console.log("💰Cost: ~$0.01-0.05 (first run, populated cache)"); + console.log("📡API calls: 4 (saved to cache for next run)"); + console.log("📂Cache created"); + } + console.log(); + + await stagehand.close(); + + return { elapsed, llmCalls: cacheFiles > 0 ? 0 : 4 }; + } catch (error) { + console.error("Error:", error.message); + await stagehand.close(); + throw error; + } +} + +async function main() { + console.log("\n╔═══════════════════════════════════════════════════════════╗"); + console.log("║ Caching Demo - Run This Script TWICE! ║"); + console.log("╚═══════════════════════════════════════════════════════════╝\n"); + + console.log("This demo shows caching impact by running the same workflow twice:\n"); + console.log("First run:"); + console.log(" 1. WITHOUT cache (baseline)"); + console.log(" 2. WITH cache enabled (populates cache)\n"); + + console.log("Second run:"); + console.log(" - WITH cache (instant, $0 cost)\n"); + + console.log("Run 'pnpm start' twice to see the difference!\n"); + + // Check if cache exists + const cacheExists = fs.existsSync(CACHE_DIR); + + if (cacheExists) { + const cacheFiles = fs.readdirSync(CACHE_DIR); + console.log(`📂 Cache found: ${cacheFiles.length} entries`); + console.log(" This is a SUBSEQUENT run - cache will be used!\n"); + } else { + console.log("No cache found - first run will populate cache"); + } + + console.log("\nRunning comparison: without cache vs with cache...\n"); + + const withoutCache = await runWithoutCache(); + const withCache = await runWithCache(); + + console.log("\n=== Comparison ==="); + console.log(`Without caching: ${withoutCache.elapsed}s, ${withoutCache.llmCalls} LLM calls`); + console.log(`With caching: ${withCache.elapsed}s, ${withCache.llmCalls} LLM calls`); + + if (withCache.llmCalls === 0) { + const speedup = (parseFloat(withoutCache.elapsed) / parseFloat(withCache.elapsed)).toFixed(1); + console.log(`\nSpeedup: ${speedup}x faster with cache`); + console.log("Cost savings: 100% (no LLM calls)"); + } + + console.log("\nRun again to see cache benefits on subsequent runs!"); +} + +main().catch((err) => { + console.error("Error in caching demo:", err); + console.error("Common issues:"); + console.error(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY"); + console.error(" - Verify GOOGLE_API_KEY is set for the model"); + console.error("Docs: https://docs.stagehand.dev/v3/first-steps/introduction"); + process.exit(1); +}); diff --git a/typescript/basic-caching/package.json b/typescript/basic-caching/package.json new file mode 100644 index 00000000..6e46064b --- /dev/null +++ b/typescript/basic-caching/package.json @@ -0,0 +1,20 @@ +{ + "name": "basic-caching", + "version": "1.0.0", + "description": "Stagehand + Browserbase: Basic Caching Demo", + "type": "module", + "main": "index.ts", + "scripts": { + "start": "tsx index.ts" + }, + "dependencies": { + "@browserbasehq/stagehand": "latest", + "dotenv": "latest" + }, + "devDependencies": { + "@types/node": "latest", + "tsx": "latest", + "typescript": "latest" + }, + "packageManager": "pnpm@9.0.0" +} From 80ce3016b5988740319e16db497111e01b0564f1 Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Tue, 6 Jan 2026 17:47:18 +0000 Subject: [PATCH 05/45] jaysahnan/gro-812-proxies-weather-template (#29) * init commit * addressing PR comments --- .gitignore | 1 + python/proxies-weather/README.md | 57 ++++++++ python/proxies-weather/main.py | 182 ++++++++++++++++++++++++ python/proxies-weather/requirements.txt | 4 + typescript/proxies-weather/README.md | 54 +++++++ typescript/proxies-weather/index.ts | 179 +++++++++++++++++++++++ typescript/proxies-weather/package.json | 17 +++ 7 files changed, 494 insertions(+) create mode 100644 python/proxies-weather/README.md create mode 100644 python/proxies-weather/main.py create mode 100644 python/proxies-weather/requirements.txt create mode 100644 typescript/proxies-weather/README.md create mode 100644 typescript/proxies-weather/index.ts create mode 100644 typescript/proxies-weather/package.json diff --git a/.gitignore b/.gitignore index b0f8e2d8..f3bab778 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ build/ # IDE .vscode/ .idea/ +.cursor/ *.swp *.swo *~ diff --git a/python/proxies-weather/README.md b/python/proxies-weather/README.md new file mode 100644 index 00000000..0ded77de --- /dev/null +++ b/python/proxies-weather/README.md @@ -0,0 +1,57 @@ +# Stagehand + Browserbase: Weather Proxy Demo + +## AT A GLANCE + +- Goal: demonstrate geolocation proxies by fetching location-specific weather data from multiple cities using Browserbase's proxy infrastructure. +- Uses geolocation proxies to route traffic through specific geographic locations (New York, London, Tokyo, São Paulo). +- Extracts structured weather data using Stagehand's extraction capabilities with Pydantic schema validation. +- Sequential processing shows how different proxy locations return different weather data from the same website. +- Docs → https://docs.browserbase.com/features/proxies + +## GLOSSARY + +- geolocation proxies: route traffic through specific geographic locations (city, country, state) to access location-specific content + Docs → https://docs.browserbase.com/features/proxies#set-proxy-geolocation +- extract: extract structured data from web pages using natural language instructions and Pydantic schemas + Docs → https://docs.stagehand.dev/v2/basics/extract +- proxies: Browserbase's managed proxy infrastructure supporting 201+ countries for geolocation-based routing + Docs → https://docs.browserbase.com/features/proxies + +## QUICKSTART + +1. cd proxies-weather-template +2. uv venv venv +3. source venv/bin/activate # On Windows: venv\Scripts\activate +4. uvx install stagehand browserbase python-dotenv pydantic +5. cp .env.example .env +6. Add your Browserbase API key and Project ID to .env +7. python main.py + +## EXPECTED OUTPUT + +- Creates Browserbase sessions with geolocation proxies for each location (New York, London, Tokyo, São Paulo) +- Displays session URLs for each location for monitoring +- Navigates to weather service (windy.com) through location-specific proxies +- Extracts temperature and unit for each location +- Displays formatted results showing different weather data based on proxy location +- Demonstrates how geolocation proxies enable location-specific content access + +## COMMON PITFALLS + +- Browserbase Developer plan or higher is required to use proxies +- "ModuleNotFoundError": ensure all dependencies are installed via uvx install +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- Geolocation fields are case-insensitive (city, country, state can be any case) +- State is required for US locations to ensure accurate geolocation +- ERR_TUNNEL_CONNECTION_FAILED: indicates either a temporary proxy hiccup or a site unsupported by our built-in proxies +- Import errors: activate your virtual environment if you created one + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v2/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord + diff --git a/python/proxies-weather/main.py b/python/proxies-weather/main.py new file mode 100644 index 00000000..18b3669d --- /dev/null +++ b/python/proxies-weather/main.py @@ -0,0 +1,182 @@ +# Stagehand + Browserbase: Weather Proxy Demo - See README.md for full documentation + +import asyncio +import os + +from dotenv import load_dotenv +from pydantic import BaseModel, Field + +from stagehand import Stagehand, StagehandConfig + +load_dotenv() + + +class GeolocationConfig(BaseModel): + """Configuration for geolocation proxy settings""" + + city: str + country: str + state: str | None = None + + +class WeatherResult(BaseModel): + """Result structure for weather data extraction""" + + city: str + country: str + temperature: float + unit: str + error: str | None = None + + +class TemperatureData(BaseModel): + """Schema for temperature extraction""" + + temperature: float = Field(..., description="The current temperature value") + unit: str = Field(..., description="The temperature unit") + + +async def get_weather_for_location(geolocation: GeolocationConfig) -> WeatherResult: + """Fetch weather data for a specific location using geolocation proxies.""" + city_name = geolocation.city.replace("_", " ") + print(f"\n=== Getting weather for {city_name}, {geolocation.country} ===") + + model_api_key = os.environ.get("MODEL_API_KEY") or os.environ.get("GOOGLE_API_KEY") + if not model_api_key: + raise ValueError( + "MODEL_API_KEY or GOOGLE_API_KEY environment variable is required. " + "Please set one in your .env file." + ) + + # Build proxy configuration for geolocation routing + proxy_config = { + "type": "browserbase", # Use Browserbase's managed proxy infrastructure for reliable geolocation routing + "geolocation": { + "city": geolocation.city, # City name (case-insensitive, e.g., "NEW_YORK", "new_york", "New York" all work) + "country": geolocation.country, # ISO country code (case-insensitive, e.g., "US", "us", "gb", "GB" all work) + }, + } + # Add state if provided (required for US locations to ensure accurate geolocation, case-insensitive) + if geolocation.state: + proxy_config["geolocation"]["state"] = geolocation.state + + # Initialize Stagehand with geolocation proxy configuration + # This ensures all browser traffic routes through the specified geographic location + config = StagehandConfig( + env="BROWSERBASE", + api_key=os.environ.get("BROWSERBASE_API_KEY"), + project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_name=os.environ.get("MODEL_NAME", "google/gemini-2.5-flash"), + model_api_key=model_api_key, + verbose=0, + # 0 = errors only, 1 = info, 2 = debug + # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + # https://docs.stagehand.dev/configuration/logging + browserbase_session_create_params={ + "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), + "proxies": [proxy_config], + }, + ) + + stagehand = Stagehand(config) + + try: + # Initialize browser session to start automation + print(f"Initializing Stagehand for {city_name}...") + await stagehand.init() + print(f"Stagehand initialized successfully for {city_name}") + + # Navigate to weather service - geolocation proxy ensures location-specific weather data + print(f"Navigating to weather service for {city_name}...") + await stagehand.page.goto("https://www.windy.com/", wait_until="networkidle") # Wait for network to be idle to ensure weather data is loaded + print(f"Page loaded for {city_name}") + + # Wait a bit for weather data to render + await asyncio.sleep(2) + + # Extract structured temperature data using Stagehand and Pydantic schema for type safety + print(f"Extracting temperature data for {city_name}...") + extract_result = await stagehand.page.extract( + instruction="Extract the current temperature and its unit", + schema=TemperatureData, + ) + + print( + f"Successfully extracted weather data for {city_name}: {extract_result.temperature} {extract_result.unit}" + ) + + # Close Stagehand session to release resources + await stagehand.close() + + return WeatherResult( + city=city_name, + country=geolocation.country, + temperature=extract_result.temperature, + unit=extract_result.unit, + ) + except Exception as error: + await stagehand.close() + print(f"Error getting weather for {city_name}: {error}") + return WeatherResult( + city=city_name, + country=geolocation.country, + temperature=0.0, + unit="", + error=str(error), + ) + + +def display_results(results: list[WeatherResult]): + """Display formatted weather results for all processed locations.""" + print("\n=== Weather Results ===") + for result in results: + if result.error: + print(f"{result.city}, {result.country}: Error - {result.error}") + else: + print(f"{result.city}, {result.country}: {result.temperature} {result.unit}") + + +async def main(): + """Main orchestration function: processes multiple locations sequentially using geolocation proxies.""" + # Define locations to test - demonstrating the power of geolocation proxies + # Each location will route traffic through its respective geographic proxy to get location-specific weather + # Note: All geolocation fields (city, country, state) are case-insensitive + locations = [ + GeolocationConfig(city="NEW_YORK", state="NY", country="US"), # State required for US locations (case-insensitive) + GeolocationConfig(city="LONDON", country="GB"), + GeolocationConfig(city="TOKYO", country="JP"), + GeolocationConfig(city="SAO_PAULO", country="BR"), + ] + + print("=== Weather Proxy Demo - Running Sequentially ===\n") + print(f"Processing {len(locations)} locations with geolocation proxies...") + print("Each location will use a different proxy to fetch location-specific weather data\n") + + # Collect all results for final summary + results: list[WeatherResult] = [] + + # Run each location sequentially to show different weather based on proxy location + # Sequential processing ensures clear demonstration of proxy-based location differences + for i, location in enumerate(locations, 1): + print(f"\n[{i}/{len(locations)}] Processing {location.city}, {location.country}...") + result = await get_weather_for_location(location) + results.append(result) + + # Display all results in formatted summary + display_results(results) + print("\n=== All locations completed ===") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Application error: {err}") + print("Common issues:") + print(" - Check .env file has BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY") + print(" - Set MODEL_API_KEY or GOOGLE_API_KEY (required for AI model)") + print( + " - Verify geolocation proxy locations are valid (see https://docs.browserbase.com/features/proxies)" + ) + print("Docs: https://docs.stagehand.dev/v3/first-steps/introduction") + exit(1) \ No newline at end of file diff --git a/python/proxies-weather/requirements.txt b/python/proxies-weather/requirements.txt new file mode 100644 index 00000000..1c92107f --- /dev/null +++ b/python/proxies-weather/requirements.txt @@ -0,0 +1,4 @@ +python-dotenv +pydantic +stagehand + diff --git a/typescript/proxies-weather/README.md b/typescript/proxies-weather/README.md new file mode 100644 index 00000000..665209f1 --- /dev/null +++ b/typescript/proxies-weather/README.md @@ -0,0 +1,54 @@ +# Stagehand + Browserbase: Weather Proxy Demo + +## AT A GLANCE + +- Goal: demonstrate geolocation proxies by fetching location-specific weather data from multiple cities using Browserbase's proxy infrastructure. +- Uses geolocation proxies to route traffic through specific geographic locations (New York, London, Tokyo, São Paulo). +- Extracts structured weather data using Stagehand's extraction capabilities with Zod schema validation. +- Sequential processing shows how different proxy locations return different weather data from the same website. +- Docs → https://docs.browserbase.com/features/proxies + +## GLOSSARY + +- geolocation proxies: route traffic through specific geographic locations (city, country, state) to access location-specific content + Docs → https://docs.browserbase.com/features/proxies#set-proxy-geolocation +- extract: extract structured data from web pages using natural language instructions and Zod schemas + Docs → https://docs.stagehand.dev/basics/extract +- proxies: Browserbase's managed proxy infrastructure supporting 201+ countries for geolocation-based routing + Docs → https://docs.browserbase.com/features/proxies + +## QUICKSTART + +1. cd proxies-weather-template +2. pnpm install +3. pnpm install @browserbasehq/sdk @browserbasehq/stagehand zod +4. cp .env.example .env +5. Add your Browserbase API key and Project ID to .env +6. pnpm start + +## EXPECTED OUTPUT + +- Creates Browserbase sessions with geolocation proxies for each location (New York, London, Tokyo, São Paulo) +- Displays session URLs for each location for monitoring +- Navigates to weather service (windy.com) through location-specific proxies +- Extracts temperature and unit for each location +- Displays formatted results showing different weather data based on proxy location +- Demonstrates how geolocation proxies enable location-specific content access + +## COMMON PITFALLS + +- Browserbase Developer plan or higher is required to use proxies +- "Cannot find module": ensure all dependencies are installed (@browserbasehq/sdk, @browserbasehq/stagehand, zod) +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- Geolocation fields are case-insensitive (city, country, state can be any case) +- State is required for US locations to ensure accurate geolocation +- ERR_TUNNEL_CONNECTION_FAILED: indicates either a temporary proxy hiccup or a site unsupported by our built-in proxies + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/typescript/proxies-weather/index.ts b/typescript/proxies-weather/index.ts new file mode 100644 index 00000000..d2c03eed --- /dev/null +++ b/typescript/proxies-weather/index.ts @@ -0,0 +1,179 @@ +// Stagehand + Browserbase: Weather Proxy Demo - See README.md for full documentation + +import "dotenv/config"; +import { Stagehand } from "@browserbasehq/stagehand"; +import { z } from "zod"; + +interface GeolocationConfig { + city: string; + country: string; + state?: string; +} + +interface WeatherResult { + city: string; + country: string; + temperature: number; + unit: string; + error?: string; +} + +// Fetches weather data for a specific location using geolocation proxies +// Configures Stagehand with location-specific proxy, navigates to weather site, +// and extracts temperature data using Stagehand's structured extraction capabilities +async function getWeatherForLocation(geolocation: GeolocationConfig): Promise { + const cityName = geolocation.city.replace(/_/g, " "); + console.log(`\n=== Getting weather for ${cityName}, ${geolocation.country} ===`); + + // Initialize Stagehand with geolocation proxy configuration + // This ensures all browser traffic routes through the specified geographic location + const stagehand = new Stagehand({ + env: "BROWSERBASE", + verbose: 0, + // 0 = errors only, 1 = info, 2 = debug + // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + // https://docs.stagehand.dev/configuration/logging + browserbaseSessionCreateParams: { + projectId: process.env.BROWSERBASE_PROJECT_ID!, + proxies: [ + { + type: "browserbase", // Use Browserbase's managed proxy infrastructure for reliable geolocation routing + geolocation: { + city: geolocation.city, // City name (case-insensitive, e.g., "NEW_YORK", "new_york", "New York" all work) + country: geolocation.country, // ISO country code (case-insensitive, e.g., "US", "us", "gb", "GB" all work) + ...(geolocation.state && { state: geolocation.state }), // State required for US locations (case-insensitive) + }, + }, + ], + }, + }); + + try { + // Initialize browser session to start automation + console.log(`Initializing Stagehand for ${cityName}...`); + await stagehand.init(); + console.log(`Stagehand initialized successfully for ${cityName}`); + + const page = stagehand.context.pages()[0]; + + // Navigate to weather service - geolocation proxy ensures location-specific weather data + console.log(`Navigating to weather service for ${cityName}...`); + await page.goto("https://www.windy.com/", { + waitUntil: "networkidle", // Wait for network to be idle to ensure weather data is loaded + }); + console.log(`Page loaded for ${cityName}`); + + // Wait a bit for weather data to render + await new Promise(resolve => setTimeout(resolve, 2000)); + + // Extract structured temperature data using Stagehand and Zod schema for type safety + console.log(`Extracting temperature data for ${cityName}...`); + const extractResult = await stagehand.extract( + "Extract the current temperature and its unit", + z.object({ + temperature: z.number().describe("The current temperature value"), + unit: z.string().describe("The temperature unit)"), + }), + ); + + console.log( + `Successfully extracted weather data for ${cityName}: ${extractResult.temperature} ${extractResult.unit}`, + ); + + // Close Stagehand session to release resources + await stagehand.close(); + + return { + city: cityName, + country: geolocation.country, + temperature: extractResult.temperature, + unit: extractResult.unit, + }; + } catch (error) { + await stagehand.close(); + console.error(`Error getting weather for ${cityName}:`, error); + return { + city: cityName, + country: geolocation.country, + temperature: 0, + unit: "", + error: error instanceof Error ? error.message : String(error), + }; + } +} + +// Displays formatted weather results for all processed locations +// Shows successful results with temperature data or error messages for failed locations +function displayResults(results: WeatherResult[]) { + console.log("\n=== Weather Results ==="); + + for (const result of results) { + if (result.error) { + console.log(`${result.city}, ${result.country}: Error - ${result.error}`); + } else { + console.log(`${result.city}, ${result.country}: ${result.temperature} ${result.unit}`); + } + } +} + +// Main orchestration function: processes multiple locations sequentially using geolocation proxies +// Demonstrates how different proxy locations return different weather data from the same website +async function main() { + // Define locations to test - demonstrating the power of geolocation proxies + // Each location will route traffic through its respective geographic proxy to get location-specific weather + // Note: All geolocation fields (city, country, state) are case-insensitive + const locations: GeolocationConfig[] = [ + { + city: "NEW_YORK", + state: "NY", // State required for US locations (case-insensitive) + country: "US", + }, + { + city: "LONDON", + country: "GB", + }, + { + city: "TOKYO", + country: "JP", + }, + { + city: "SAO_PAULO", + country: "BR", + }, + ]; + + console.log("=== Weather Proxy Demo - Running Sequentially ===\n"); + console.log(`Processing ${locations.length} locations with geolocation proxies...`); + console.log("Each location will use a different proxy to fetch location-specific weather data\n"); + + // Collect all results for final summary + const results: WeatherResult[] = []; + + // Run each location sequentially to show different weather based on proxy location + // Sequential processing ensures clear demonstration of proxy-based location differences + for (let i = 0; i < locations.length; i++) { + const location = locations[i]; + console.log( + `\n[${i + 1}/${locations.length}] Processing ${location.city}, ${location.country}...`, + ); + const result = await getWeatherForLocation(location); + results.push(result); + } + + // Display all results in formatted summary + displayResults(results); + + console.log("\n=== All locations completed ==="); +} + +main().catch((err) => { + console.error("Application error:", err); + console.error("Common issues:"); + console.error(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY"); + console.error( + " - Verify geolocation proxy locations are valid (see https://docs.browserbase.com/features/proxies)", + ); + console.error(" - Ensure locations array is properly configured"); + console.error("Docs: https://docs.stagehand.dev/v3/first-steps/introduction"); + process.exit(1); +}); \ No newline at end of file diff --git a/typescript/proxies-weather/package.json b/typescript/proxies-weather/package.json new file mode 100644 index 00000000..bff070c9 --- /dev/null +++ b/typescript/proxies-weather/package.json @@ -0,0 +1,17 @@ +{ + "name": "proxies-weather-template", + "type": "module", + "scripts": { + "build": "tsc", + "start": "tsx index.ts" + }, + "dependencies": { + "@browserbasehq/stagehand": "latest", + "dotenv": "^16.4.7", + "zod": "latest" + }, + "devDependencies": { + "tsx": "^4.19.2", + "typescript": "^5.0.0" + } +} From d75cd60b8fc3e314320c87543aba476fae9f9d90 Mon Sep 17 00:00:00 2001 From: Kyle Jeong <77771518+Kylejeong2@users.noreply.github.com> Date: Tue, 20 Jan 2026 13:31:24 -0800 Subject: [PATCH 06/45] add go hackernews extraction template (#37) --- go/hackernews/README.md | 78 ++++++++++++++++++++ go/hackernews/go.mod | 12 ++++ go/hackernews/go.sum | 12 ++++ go/hackernews/main.go | 156 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 258 insertions(+) create mode 100644 go/hackernews/README.md create mode 100644 go/hackernews/go.mod create mode 100644 go/hackernews/go.sum create mode 100644 go/hackernews/main.go diff --git a/go/hackernews/README.md b/go/hackernews/README.md new file mode 100644 index 00000000..16e7bcb2 --- /dev/null +++ b/go/hackernews/README.md @@ -0,0 +1,78 @@ +# Stagehand + Browserbase: Hacker News Automation + +## AT A GLANCE + +- Goal: Demonstrate Stagehand's core browser automation features through a complete Hacker News workflow. +- Comprehensive Example: Shows navigate, observe, act, extract, and execute (autonomous agent) in a single workflow. +- Real-world Scenario: Navigates to Hacker News, clicks on top post comments, extracts structured data, and uses an autonomous agent to find the newest post. +- Docs → https://docs.stagehand.dev/v3/sdk/go + +## GLOSSARY + +- navigate: Load a web page in the browser session + Docs → https://docs.stagehand.dev/basics/navigate +- observe: Analyze page elements and generate actionable steps based on natural language instructions + Docs → https://docs.stagehand.dev/basics/observe +- act: Execute actions on web pages using natural language instructions + Docs → https://docs.stagehand.dev/basics/act +- extract: Extract structured data from pages using JSON schema definitions + Docs → https://docs.stagehand.dev/basics/extract +- execute: Run an autonomous agent to complete multi-step tasks automatically + Docs → https://docs.stagehand.dev/basics/execute + +## QUICKSTART + +1. Ensure Go 1.22+ is installed (`go version`) +2. Set required environment variables: + ```bash + export BROWSERBASE_API_KEY="your-api-key" + export BROWSERBASE_PROJECT_ID="your-project-id" + export MODEL_API_KEY="your-model-api-key" + ``` +3. Run the example: + ```bash + go run main.go + ``` + +## EXPECTED OUTPUT + +- Session initialization with Browserbase +- Navigation to Hacker News homepage +- Observation of page to find comment links for the top post +- Action execution to click on the comment link +- Structured data extraction (title, top comment, author) +- Autonomous agent execution to navigate back and find the newest post +- Session cleanup and termination +- Live session recording link displayed in console + +## COMMON PITFALLS + +- Missing Go installation: Ensure Go 1.22+ is installed (`go version`) +- Missing environment variables: Verify BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, and MODEL_API_KEY are set +- Module not found: Run `go mod download` if dependencies aren't resolved +- Network issues: Check internet connection and website accessibility +- Session errors: Verify API keys are valid and project ID exists in Browserbase dashboard +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +• Content aggregation: Automate data extraction from news sites, forums, and social platforms for monitoring and analysis. +• Research automation: Collect structured information from multiple pages using autonomous agents for competitive intelligence. +• Testing workflows: Validate web interactions, form submissions, and navigation flows across different pages. +• Data scraping: Extract structured data from dynamic websites that require JavaScript execution and user interactions. + +## NEXT STEPS + +• Add error handling: Implement retry logic for failed actions and better error messages for debugging. +• Extend extraction schema: Add more fields to extract (upvotes, comment count, post date, etc.). +• Multi-page workflows: Chain multiple navigations and extractions to build comprehensive data collection pipelines. +• Customize agent instructions: Modify the execute instruction to perform different tasks (e.g., find posts by keyword, filter by score). + +## HELPFUL RESOURCES + +📚 Stagehand Go Docs: https://docs.stagehand.dev/v3/sdk/go +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/go/hackernews/go.mod b/go/hackernews/go.mod new file mode 100644 index 00000000..7bb96052 --- /dev/null +++ b/go/hackernews/go.mod @@ -0,0 +1,12 @@ +module stagehand-example + +go 1.22 + +require github.com/browserbase/stagehand-go v0.16.1 + +require ( + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect +) diff --git a/go/hackernews/go.sum b/go/hackernews/go.sum new file mode 100644 index 00000000..d9da97bf --- /dev/null +++ b/go/hackernews/go.sum @@ -0,0 +1,12 @@ +github.com/browserbase/stagehand-go v0.16.1 h1:Sl2h1kkzOpQtApZBKgPr2JTL7PlKAkDIQ6XlWJW0rfo= +github.com/browserbase/stagehand-go v0.16.1/go.mod h1:pzE7WVsYGBXQiH75NwFMn9m4U1CkzmmlW4HHn5yhd+o= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= diff --git a/go/hackernews/main.go b/go/hackernews/main.go new file mode 100644 index 00000000..84094c93 --- /dev/null +++ b/go/hackernews/main.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/browserbase/stagehand-go" + "github.com/browserbase/stagehand-go/option" +) + +const sdkVersion = "3.0.7" + +func main() { + // Create client using environment variables + client := stagehand.NewClient( + option.WithBrowserbaseAPIKey(os.Getenv("BROWSERBASE_API_KEY")), + option.WithBrowserbaseProjectID(os.Getenv("BROWSERBASE_PROJECT_ID")), + option.WithModelAPIKey(os.Getenv("MODEL_API_KEY")), + ) + + ctx := context.Background() + + // Start a new browser session + startResponse, err := client.Sessions.Start(ctx, stagehand.SessionStartParams{ + ModelName: "openai/gpt-4o-mini", + XLanguage: stagehand.SessionStartParamsXLanguageTypescript, + XSDKVersion: stagehand.String(sdkVersion), + }) + if err != nil { + fmt.Printf("Failed to start session: %v\n", err) + os.Exit(1) + } + + sessionID := startResponse.Data.SessionID + fmt.Printf("Session started: %s\n", sessionID) + fmt.Printf("Watch live: https://www.browserbase.com/sessions/%s\n", sessionID) + + // Ensure we clean up the session + defer func() { + _, _ = client.Sessions.End(ctx, sessionID, stagehand.SessionEndParams{ + XLanguage: stagehand.SessionEndParamsXLanguageTypescript, + XSDKVersion: stagehand.String(sdkVersion), + }) + fmt.Println("Session ended") + }() + + // Navigate to Hacker News + _, err = client.Sessions.Navigate(ctx, sessionID, stagehand.SessionNavigateParams{ + URL: "https://news.ycombinator.com", + FrameID: stagehand.String(""), + XLanguage: stagehand.SessionNavigateParamsXLanguageTypescript, + XSDKVersion: stagehand.String(sdkVersion), + }) + if err != nil { + fmt.Printf("Failed to navigate: %v\n", err) + return + } + fmt.Println("Navigated to Hacker News") + + // Observe available actions + observeResponse, err := client.Sessions.Observe(ctx, sessionID, stagehand.SessionObserveParams{ + Instruction: stagehand.String("find the link to view comments for the top post"), + XLanguage: stagehand.SessionObserveParamsXLanguageTypescript, + XSDKVersion: stagehand.String(sdkVersion), + }) + if err != nil { + fmt.Printf("Failed to observe: %v\n", err) + return + } + + actions := observeResponse.Data.Result + fmt.Printf("Found %d possible actions\n", len(actions)) + + if len(actions) == 0 { + fmt.Println("No actions found") + return + } + + // Act on the first action + action := actions[0] + fmt.Printf("Acting on: %s\n", action.Description) + + actResponse, err := client.Sessions.Act(ctx, sessionID, stagehand.SessionActParams{ + Input: stagehand.SessionActParamsInputUnion{ + OfAction: &stagehand.ActionParam{ + Description: action.Description, + Selector: action.Selector, + Method: stagehand.String(action.Method), + Arguments: action.Arguments, + }, + }, + XLanguage: stagehand.SessionActParamsXLanguageTypescript, + XSDKVersion: stagehand.String(sdkVersion), + }) + if err != nil { + fmt.Printf("Failed to act: %v\n", err) + return + } + fmt.Printf("Act completed: %s\n", actResponse.Data.Result.Message) + + // Extract structured data + extractResponse, err := client.Sessions.Extract(ctx, sessionID, stagehand.SessionExtractParams{ + Instruction: stagehand.String("extract the title and top comment from this page"), + Schema: map[string]any{ + "type": "object", + "properties": map[string]any{ + "title": map[string]any{ + "type": "string", + "description": "The title of the post", + }, + "topComment": map[string]any{ + "type": "string", + "description": "The text of the top comment", + }, + "author": map[string]any{ + "type": "string", + "description": "The username of the top commenter", + }, + }, + "required": []string{"title"}, + }, + XLanguage: stagehand.SessionExtractParamsXLanguageTypescript, + XSDKVersion: stagehand.String(sdkVersion), + }) + if err != nil { + fmt.Printf("Failed to extract: %v\n", err) + return + } + fmt.Printf("Extracted data: %+v\n", extractResponse.Data.Result) + + // Run autonomous agent + executeResponse, err := client.Sessions.Execute(ctx, sessionID, stagehand.SessionExecuteParams{ + ExecuteOptions: stagehand.SessionExecuteParamsExecuteOptions{ + Instruction: "Navigate back to the main Hacker News page and find the newest post", + MaxSteps: stagehand.Float(5), + }, + AgentConfig: stagehand.SessionExecuteParamsAgentConfig{ + Model: stagehand.ModelConfigUnionParam{ + OfModelConfigModelConfigObject: &stagehand.ModelConfigModelConfigObjectParam{ + ModelName: "openai/gpt-4o-mini", + APIKey: stagehand.String(os.Getenv("MODEL_API_KEY")), + }, + }, + Cua: stagehand.Bool(false), + }, + XLanguage: stagehand.SessionExecuteParamsXLanguageTypescript, + XSDKVersion: stagehand.String(sdkVersion), + }) + if err != nil { + fmt.Printf("Failed to execute agent: %v\n", err) + return + } + fmt.Printf("Agent result: %s\n", executeResponse.Data.Result.Message) + fmt.Printf("Agent success: %v\n", executeResponse.Data.Result.Success) +} From 1d7a23d9ccf8b2f75b69d47dda91a07a17c96879 Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Thu, 22 Jan 2026 14:53:56 -0800 Subject: [PATCH 07/45] init commit (#42) --- .gitignore | 1 + python/browserbase-reducto/README.md | 84 +++++ python/browserbase-reducto/main.py | 334 ++++++++++++++++++++ python/browserbase-reducto/pyproject.toml | 34 ++ typescript/browserbase-reducto/.env.example | 9 + typescript/browserbase-reducto/README.md | 63 ++++ typescript/browserbase-reducto/index.ts | 326 +++++++++++++++++++ typescript/browserbase-reducto/package.json | 23 ++ 8 files changed, 874 insertions(+) create mode 100644 python/browserbase-reducto/README.md create mode 100644 python/browserbase-reducto/main.py create mode 100644 python/browserbase-reducto/pyproject.toml create mode 100644 typescript/browserbase-reducto/.env.example create mode 100644 typescript/browserbase-reducto/README.md create mode 100644 typescript/browserbase-reducto/index.ts create mode 100644 typescript/browserbase-reducto/package.json diff --git a/.gitignore b/.gitignore index f3bab778..19bcdbeb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ cursorrules.md node_modules/ .pnp .pnp.js +.pnpm-store/ # Python __pycache__/ diff --git a/python/browserbase-reducto/README.md b/python/browserbase-reducto/README.md new file mode 100644 index 00000000..905494d0 --- /dev/null +++ b/python/browserbase-reducto/README.md @@ -0,0 +1,84 @@ +# Stagehand + Browserbase + Reducto: Download PDFs and Extract Financial Data + +## AT A GLANCE + +- **Goal**: Automate downloading financial PDFs from websites and extract structured data using AI-powered document parsing. +- **Pattern Template**: Demonstrates the integration pattern of Browserbase (download automation) + Reducto (document extraction). +- **Workflow**: Uses Stagehand to navigate websites, Browserbase automatically downloads PDFs when opened, then Reducto extracts structured financial data using schema-based extraction. +- **Download Handling**: Implements retry logic with polling to handle Browserbase's async download sync (files sync to cloud storage in real-time). +- **Structured Extraction**: Uses Reducto's extract API with JSON schema to pull specific financial metrics from complex PDF tables. +- Docs → [Browserbase Downloads](https://docs.browserbase.com/features/downloads) | [Reducto Extract](https://docs.reducto.ai/parse/best-practices) + +## GLOSSARY + +- **act**: perform UI actions from natural language prompts (click, scroll, navigate) + Docs → https://docs.stagehand.dev/basics/act +- **Browserbase Downloads**: When a PDF URL is opened in a browser session, Browserbase automatically downloads and stores it in cloud storage. Files must be retrieved via the Session Downloads API as a ZIP archive. + Docs → https://docs.browserbase.com/features/downloads +- **Reducto Extract**: Extract structured data from PDFs using JSON schema definitions. More efficient than parsing entire documents when you only need specific fields. + Docs → https://docs.reducto.ai/extract +- **Schema-based extraction**: Define the exact structure you want extracted (fields, types, descriptions) and Reducto returns JSON matching your schema. +- **Download polling**: Browserbase syncs downloads in real-time; larger files may need retry logic to ensure availability via the API. + +## QUICKSTART + +1. cd python/reducto-browserbase +2. Install dependencies with uv: + + ```bash + uv pip install -e . + ``` + + This will install all dependencies from `pyproject.toml`. + + Alternatively, use uvx to run without installation: + + ```bash + uvx --with browserbase --with reductoai --with stagehand-ai --with python-dotenv python main.py + ``` + +3. cp .env.example .env +4. Add required API keys to .env: + - `BROWSERBASE_PROJECT_ID` + - `BROWSERBASE_API_KEY` + - `REDUCTOAI_API_KEY` + - `GOOGLE_API_KEY` +5. Run the script: + ```bash + python main.py + ``` + Or with uv: + ```bash + uv run python main.py + ``` + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase and displays live view link +- Navigates to Apple.com investor relations section +- Clicks through to Q4 financial statements +- Browserbase automatically downloads PDF when link is opened +- Polls Browserbase Downloads API until file is ready (with retry logic) +- Extracts PDF from ZIP archive downloaded from Browserbase +- Uploads PDF to Reducto and extracts structured iPhone net sales data +- Outputs extracted financial data as formatted JSON +- Closes session cleanly + +## NEXT STEPS + +• **Parameterize extraction**: Accept different schema definitions or document types as configuration to extract various financial metrics or data structures. +• **Batch processing**: Process multiple quarters or companies by looping through different navigation paths and extracting data for each. +• **Multi-document support**: Handle ZIP archives with multiple PDFs and extract data from each, aggregating results into a unified dataset. +• **Optimize extraction**: Use Reducto's agentic mode selectively (only for complex tables or low-quality scans) to reduce latency and credit usage. Enable `scope: "table"` only when tables are misaligned or have merged cells. +Docs → https://docs.reducto.ai/parse/best-practices#2-enable-agentic-mode-only-when-needed + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +📚 Browserbase Downloads: https://docs.browserbase.com/features/downloads +📚 Reducto Best Practices: https://docs.reducto.ai/parse/best-practices +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/python/browserbase-reducto/main.py b/python/browserbase-reducto/main.py new file mode 100644 index 00000000..e2eb0896 --- /dev/null +++ b/python/browserbase-reducto/main.py @@ -0,0 +1,334 @@ +# Stagehand + Browserbase: Download Apple's Q4 Financial Statement and Parse with Reducto +# See README.md for full documentation + +import asyncio +import os +import zipfile +from pathlib import Path + +from browserbase import Browserbase +from dotenv import load_dotenv +from reducto import Reducto +from stagehand import AsyncStagehand + +# Load environment variables from .env file +# Required: BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, REDUCTOAI_API_KEY, GOOGLE_API_KEY +load_dotenv() + + +# Polls Browserbase API for completed downloads with retry logic +async def save_downloads_with_retry( + bb: Browserbase, session_id: str, retry_for_seconds: int = 30 +) -> int: + """ + Polls Browserbase API for downloads with timeout handling. + + Browserbase stores downloaded files during a session and makes them available + via API. Files may take a few seconds to process, so this function implements + retry logic to wait for downloads to be ready before retrieving them. + + Args: + bb: Browserbase client instance for API calls + session_id: The Browserbase session ID to retrieve downloads from + retry_for_seconds: Maximum time to wait for downloads (default: 30 seconds) + + Returns: + int: The size of the downloaded ZIP file in bytes + + Raises: + TimeoutError: If downloads aren't ready within the specified timeout + """ + print(f"Waiting up to {retry_for_seconds} seconds for downloads to complete...") + + # Track elapsed time to implement timeout without using threading timers + start_time = asyncio.get_event_loop().time() + timeout = retry_for_seconds + + while True: + elapsed = asyncio.get_event_loop().time() - start_time + + # Check if we've exceeded the timeout period + if elapsed >= timeout: + raise TimeoutError("Download timeout exceeded") + + try: + print("Checking for downloads...") + # Fetch downloads from Browserbase API and save to disk when ready + # Use asyncio.to_thread for synchronous Browserbase SDK calls + # This prevents blocking the event loop while waiting for API responses + response = await asyncio.to_thread(bb.sessions.downloads.list, session_id) + download_buffer = await asyncio.to_thread(response.read) + + # Save downloads to disk when file size indicates content is available + # Empty zip files are ~22 bytes, so require at least 100 bytes for real content + if len(download_buffer) > 100: + print(f"Downloads ready! File size: {len(download_buffer)} bytes") + # Save the ZIP file containing all downloaded PDFs to disk + with open("downloaded_files.zip", "wb") as f: + f.write(download_buffer) + print("Files saved as: downloaded_files.zip") + return len(download_buffer) + else: + print("Downloads not ready yet, retrying...") + except Exception as e: + error_message = str(e) + # Handle session not found errors gracefully (session may have expired) + if "Session with given id not found" in error_message or "-32001" in error_message: + print("Session not found, returning empty result") + return 0 + print(f"Error fetching downloads: {e}") + raise + + # Poll every 2 seconds to check if downloads are ready + await asyncio.sleep(2) + + +# Extracts PDF files from downloaded zip archive +def extract_pdf_from_zip(zip_path: str, output_dir: str = "downloaded_files") -> str: + """ + Extract PDF files from a ZIP archive. + + Args: + zip_path: Path to the ZIP file containing PDFs + output_dir: Directory to extract PDFs to (default: "downloaded_files") + + Returns: + str: Path to the first extracted PDF file + + Raises: + FileNotFoundError: If ZIP file doesn't exist + ValueError: If no PDF files are found in the ZIP + """ + print(f"Extracting PDF from {zip_path}...") + + # Create output directory if it doesn't exist + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + pdf_path = None + + with zipfile.ZipFile(zip_path, "r") as zip_ref: + # Open zip file and filter for PDF entries only + pdf_entries = [entry for entry in zip_ref.namelist() if entry.lower().endswith(".pdf")] + + if len(pdf_entries) == 0: + raise ValueError("No PDF files found in the downloaded zip") + + # Extract all PDF files and return path to first one + for entry in pdf_entries: + zip_ref.extract(entry, output_dir) + extracted_path = output_path / entry + print(f"Extracted: {extracted_path}") + + if pdf_path is None: + pdf_path = str(extracted_path) + + if pdf_path is None: + raise ValueError("Failed to extract PDF file") + + return pdf_path + + +# Uploads PDF to Reducto and extracts structured financial data +async def extract_pdf_with_reducto(pdf_path: str, reducto_client: Reducto) -> None: + """ + Extract structured financial data from PDF using Reducto. + + Uploads the PDF to Reducto and extracts iPhone net sales data + using a JSON schema definition. + + Args: + pdf_path: Path to the PDF file to process + reducto_client: Reducto client instance for API calls + """ + print(f"\nExtracting financial data with Reducto: {pdf_path}...") + + # Upload PDF to Reducto for processing + # Use asyncio.to_thread for synchronous SDK calls + upload_response = await asyncio.to_thread(reducto_client.upload, file=Path(pdf_path)) + print(f"Uploaded to Reducto: {upload_response}") + + # Define JSON schema to extract iPhone net sales from financial statements + schema = { + "type": "object", + "properties": { + "iphone_net_sales": { + "type": "object", + "properties": { + "current_quarter": { + "type": "number", + "description": "iPhone net sales for the current quarter (in millions)", + }, + "previous_quarter": { + "type": "number", + "description": "iPhone net sales for the previous quarter (in millions)", + }, + "current_year": { + "type": "number", + "description": "iPhone net sales for the current year (in millions)", + }, + "previous_year": { + "type": "number", + "description": "iPhone net sales for the previous year (in millions)", + }, + "current_quarter_date": { + "type": "string", + "description": "Date or period label for the current quarter", + }, + "previous_quarter_date": { + "type": "string", + "description": "Date or period label for the previous quarter", + }, + }, + "required": [ + "current_quarter", + "previous_quarter", + "current_year", + "previous_year", + "current_quarter_date", + "previous_quarter_date", + ], + "description": "iPhone net sales values from the financial statements", + } + }, + "required": ["iphone_net_sales"], + } + + # Configure extraction instructions + instructions = { + "schema": schema, + "system_prompt": ( + "Extract the iPhone net sales values from the financial statements. " + "Find the iPhone line item in the net sales by category table and extract " + "the values for current quarter, previous quarter, current year, and previous year " + "(typically shown in columns in the income statement or operations statement)." + ), + } + + # Configure extraction settings + settings = { + "optimize_for_latency": True, + "citations": {"numerical_confidence": False}, + } + + # Extract structured data using Reducto's AI extraction with schema + # Use asyncio.to_thread for synchronous SDK calls + result = await asyncio.to_thread( + reducto_client.extract.run, + input=upload_response, + instructions=instructions, + settings=settings, + ) + + # Display extracted financial data in formatted JSON + print("\n=== Extracted Financial Data ===\n") + # Handle different possible response structures + extracted_data = result + if hasattr(result, "result"): + extracted_data = result.result + elif hasattr(result, "data"): + extracted_data = result.data + + import json + + print(json.dumps(extracted_data, indent=2)) + + +async def main(): + """ + Main application entry point. + + Orchestrates the entire PDF download and extraction automation process: + 1. Initializes Browserbase, Reducto, and Stagehand clients + 2. Navigates to Apple's investor relations site + 3. Downloads Q4 financial statement PDF + 4. Extracts PDF from ZIP archive + 5. Uploads PDF to Reducto and extracts structured financial data + """ + print("Starting Apple Q4 Financial Statement Download and Parse Automation...") + + # Initialize Browserbase SDK for session management and download retrieval + bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) + + # Initialize Reducto AI client for PDF data extraction + reducto_client = Reducto(api_key=os.environ.get("REDUCTOAI_API_KEY")) + + # Initialize AsyncStagehand client (v3 BYOB architecture) + client = AsyncStagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("GOOGLE_API_KEY"), + ) + + # Start a Stagehand session (returns a response with session_id) + start_response = await client.sessions.start(model_name="google/gemini-2.5-pro") + session_id = start_response.data.session_id + print(f"Stagehand session started: {session_id}") + + try: + # Get live view URL for monitoring browser session in real-time + # Use asyncio.to_thread for synchronous SDK calls + live_view_links = await asyncio.to_thread(bb.sessions.debug, session_id) + live_view_link = live_view_links.debuggerFullscreenUrl + print(f"Live View Link: {live_view_link}") + + # Navigate to Apple homepage using Stagehand + print("Navigating to Apple.com...") + await client.sessions.navigate(id=session_id, url="https://www.apple.com/") + + # Navigate to investor relations section using Stagehand AI actions + print("Navigating to Investors section...") + await client.sessions.act( + id=session_id, input="Click the 'Investors' button at the bottom of the page" + ) + await client.sessions.act( + id=session_id, input="Scroll down to the Financial Data section of the page" + ) + await client.sessions.act( + id=session_id, input="Under Quarterly Earnings Reports, click on '2025'" + ) + + # Download Q4 quarterly financial statement + # When a URL of a PDF is opened, Browserbase automatically downloads and stores the PDF + # See https://docs.browserbase.com/features/downloads for more info + print("Downloading Q4 financial statement...") + await client.sessions.act( + id=session_id, input="Click the 'Financial Statements' link under Q4" + ) + + # Wait for the PDF download to be triggered and processed + print("Waiting for download to be triggered...") + await asyncio.sleep(10) + + # Retrieve all downloads triggered during this session from Browserbase API + print("Retrieving downloads from Browserbase...") + await save_downloads_with_retry(bb, session_id, 60) + print("Download completed successfully!") + + # Extract PDF from downloaded zip archive + pdf_path = extract_pdf_from_zip("downloaded_files.zip") + print(f"PDF extracted to: {pdf_path}") + + # Extract structured financial data using Reducto AI + await extract_pdf_with_reducto(pdf_path, reducto_client) + + finally: + # End the Stagehand session + await client.sessions.end(id=session_id) + print("Session closed successfully") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Application error: {err}") + print("Common issues:") + print( + " - Check .env file has BROWSERBASE_PROJECT_ID, " + "BROWSERBASE_API_KEY, and REDUCTOAI_API_KEY" + ) + print(" - Verify internet connection and Apple website accessibility") + print(" - Ensure sufficient timeout for slow-loading pages") + print("Docs: https://docs.stagehand.dev/reference/python/overview") + exit(1) diff --git a/python/browserbase-reducto/pyproject.toml b/python/browserbase-reducto/pyproject.toml new file mode 100644 index 00000000..c6ccd25a --- /dev/null +++ b/python/browserbase-reducto/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "browserbase-reducto" +version = "0.1.0" +description = "Download Apple's Q4 Financial Statement and Parse with Reducto using Stagehand and Browserbase" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "browserbase", + "python-dotenv", + "reductoai", + "stagehand", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/typescript/browserbase-reducto/.env.example b/typescript/browserbase-reducto/.env.example new file mode 100644 index 00000000..3c6562ac --- /dev/null +++ b/typescript/browserbase-reducto/.env.example @@ -0,0 +1,9 @@ +# Browserbase Configuration +BROWSERBASE_PROJECT_ID=your_browserbase_project_id +BROWSERBASE_API_KEY=your_browserbase_api_key + +# Pass your preferred LLM API key (choose a model for your provider in the Stagehand constructor) +GOOGLE_API_KEY="YOUR_GOOGLE_API_KEY" + +# Reducto Configuration +REDUCTOAI_API_KEY=your_reducto_api_key \ No newline at end of file diff --git a/typescript/browserbase-reducto/README.md b/typescript/browserbase-reducto/README.md new file mode 100644 index 00000000..387d354e --- /dev/null +++ b/typescript/browserbase-reducto/README.md @@ -0,0 +1,63 @@ +# Stagehand + Browserbase + Reducto: Download PDFs and Extract Financial Data + +## AT A GLANCE + +- **Goal**: Automate downloading financial PDFs from websites and extract structured data using AI-powered document parsing. +- **Pattern Template**: Demonstrates the integration pattern of Browserbase (download automation) + Reducto (document extraction). +- **Workflow**: Uses Stagehand to navigate websites, Browserbase automatically downloads PDFs when opened, then Reducto extracts structured financial data using schema-based extraction. +- **Download Handling**: Implements retry logic with polling to handle Browserbase's async download sync (files sync to cloud storage in real-time). +- **Structured Extraction**: Uses Reducto's extract API with JSON schema to pull specific financial metrics from complex PDF tables. +- Docs → [Browserbase Downloads](https://docs.browserbase.com/features/downloads) | [Reducto Extract](https://docs.reducto.ai/parse/best-practices) + +## GLOSSARY + +- **act**: perform UI actions from natural language prompts (click, scroll, navigate) + Docs → https://docs.stagehand.dev/basics/act +- **Browserbase Downloads**: When a PDF URL is opened in a browser session, Browserbase automatically downloads and stores it in cloud storage. Files must be retrieved via the Session Downloads API as a ZIP archive. + Docs → https://docs.browserbase.com/features/downloads +- **Reducto Extract**: Extract structured data from PDFs using JSON schema definitions. More efficient than parsing entire documents when you only need specific fields. + Docs → https://docs.reducto.ai/extract +- **Schema-based extraction**: Define the exact structure you want extracted (fields, types, descriptions) and Reducto returns JSON matching your schema. +- **Download polling**: Browserbase syncs downloads in real-time; larger files may need retry logic to ensure availability via the API. + +## QUICKSTART + +1. cd reducto-browserbase +2. pnpm install +3. cp .env.example .env +4. Add required API keys to .env: + - `BROWSERBASE_PROJECT_ID` + - `BROWSERBASE_API_KEY` + - `REDUCTOAI_API_KEY` +5. pnpm start + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase and displays live view link +- Navigates to Apple.com investor relations section +- Clicks through to Q4 financial statements +- Browserbase automatically downloads PDF when link is opened +- Polls Browserbase Downloads API until file is ready (with retry logic) +- Extracts PDF from ZIP archive downloaded from Browserbase +- Uploads PDF to Reducto and extracts structured iPhone net sales data +- Outputs extracted financial data as formatted JSON +- Closes session cleanly + +## NEXT STEPS + +• **Parameterize extraction**: Accept different schema definitions or document types as configuration to extract various financial metrics or data structures. +• **Batch processing**: Process multiple quarters or companies by looping through different navigation paths and extracting data for each. +• **Multi-document support**: Handle ZIP archives with multiple PDFs and extract data from each, aggregating results into a unified dataset. +• **Optimize extraction**: Use Reducto's agentic mode selectively (only for complex tables or low-quality scans) to reduce latency and credit usage. Enable `scope: "table"` only when tables are misaligned or have merged cells. +Docs → https://docs.reducto.ai/parse/best-practices#2-enable-agentic-mode-only-when-needed + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +📚 Browserbase Downloads: https://docs.browserbase.com/features/downloads +📚 Reducto Best Practices: https://docs.reducto.ai/parse/best-practices +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/typescript/browserbase-reducto/index.ts b/typescript/browserbase-reducto/index.ts new file mode 100644 index 00000000..cf37c016 --- /dev/null +++ b/typescript/browserbase-reducto/index.ts @@ -0,0 +1,326 @@ +// Stagehand + Browserbase: Download Apple's Q4 Financial Statement and Parse with Reducto - See README.md for full documentation + +import "dotenv/config"; +import { Browserbase } from "@browserbasehq/sdk"; +import { Stagehand } from "@browserbasehq/stagehand"; +import fs from "fs"; +import path from "path"; +import reductoai from "reductoai"; +import AdmZip from "adm-zip"; + +// Net sales data structure extracted from financial statements +interface IPhoneNetSales { + current_quarter: number; + previous_quarter: number; + current_year: number; + previous_year: number; + current_quarter_date?: string; + previous_quarter_date?: string; +} + +// Complete financial data structure returned from Reducto extraction +interface ExtractedFinancialData { + iphone_net_sales: IPhoneNetSales; +} + +// Reducto API response structure +interface ReductoExtractResult { + result?: ExtractedFinancialData; + data?: ExtractedFinancialData; +} + +// Polls Browserbase API for completed downloads with retry logic +function saveDownloadsWithRetry( + bb: Browserbase, + sessionId: string, + retryForSeconds: number = 30, +): { promise: Promise; stopPolling: () => void } { + // Track polling intervals and timeout for cleanup + const intervals = { + poller: undefined as NodeJS.Timeout | undefined, + timeout: undefined as NodeJS.Timeout | undefined, + isStopped: false, + }; + + // Cleanup function to stop all polling and timeouts + const stopPolling = (): void => { + if (intervals.isStopped) return; + intervals.isStopped = true; + if (intervals.poller) { + clearInterval(intervals.poller); + intervals.poller = undefined; + } + if (intervals.timeout) { + clearTimeout(intervals.timeout); + intervals.timeout = undefined; + } + }; + + const promise = new Promise((resolve, reject) => { + console.log(`Waiting up to ${retryForSeconds} seconds for downloads to complete...`); + + // Fetch downloads from Browserbase API and save to disk when ready + async function fetchDownloads(): Promise { + if (intervals.isStopped) { + return; + } + + try { + console.log("Checking for downloads..."); + const response = await bb.sessions.downloads.list(sessionId); + const downloadBuffer: ArrayBuffer = await response.arrayBuffer(); + + // Save downloads to disk when file size indicates content is available + if (downloadBuffer.byteLength > 0) { + console.log(`Downloads ready! File size: ${downloadBuffer.byteLength} bytes`); + fs.writeFileSync("downloaded_files.zip", Buffer.from(downloadBuffer)); + console.log("Files saved as: downloaded_files.zip"); + + stopPolling(); + resolve(downloadBuffer.byteLength); + } else { + console.log("Downloads not ready yet, retrying..."); + } + } catch (e: unknown) { + if (intervals.isStopped) { + return; + } + // Handle session not found errors gracefully (session may have expired) + const errorMessage = e instanceof Error ? e.message : String(e); + if ( + errorMessage.includes("Session with given id not found") || + errorMessage.includes("-32001") + ) { + stopPolling(); + resolve(0); + return; + } + console.error("Error fetching downloads:", e); + stopPolling(); + reject(e); + } + } + + // Set timeout to fail if downloads don't complete within retry window + intervals.timeout = setTimeout(() => { + if (!intervals.isStopped) { + stopPolling(); + reject(new Error("Download timeout exceeded")); + } + }, retryForSeconds * 1000); + + // Poll every 2 seconds to check if downloads are ready + intervals.poller = setInterval(fetchDownloads, 2000); + }); + + return { promise, stopPolling }; +} + +// Extracts PDF files from downloaded zip archive +function extractPdfFromZip(zipPath: string, outputDir: string = "downloaded_files"): string { + console.log(`Extracting PDF from ${zipPath}...`); + + // Create output directory if it doesn't exist + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + // Open zip file and filter for PDF entries only + const zip = new AdmZip(zipPath); + const pdfEntries = zip + .getEntries() + .filter((entry: AdmZip.IZipEntry) => entry.entryName.toLowerCase().endsWith(".pdf")); + + if (pdfEntries.length === 0) { + throw new Error("No PDF files found in the downloaded zip"); + } + + // Extract all PDF files and return path to first one + let pdfPath: string | null = null; + for (const entry of pdfEntries) { + const outputPath = path.join(outputDir, entry.entryName); + zip.extractEntryTo(entry, outputDir, false, true); + console.log(`Extracted: ${outputPath}`); + + if (!pdfPath) { + pdfPath = outputPath; + } + } + + if (!pdfPath) { + throw new Error("Failed to extract PDF file"); + } + + return pdfPath; +} + +// Uploads PDF to Reducto and extracts structured financial data +async function extractPDFWithReducto(pdfPath: string, reductoaiClient: reductoai): Promise { + console.log(`\nExtracting financial data with Reducto: ${pdfPath}...`); + + // Upload PDF to Reducto for processing + const upload = await reductoaiClient.upload({ + file: fs.createReadStream(pdfPath), + }); + console.log(`Uploaded to Reducto: ${upload.file_id}`); + + // Define JSON schema to extract iPhone net sales from financial statements + const schema = { + type: "object", + properties: { + iphone_net_sales: { + type: "object", + properties: { + current_quarter: { + type: "number", + description: "iPhone net sales for the current quarter (in millions)", + }, + previous_quarter: { + type: "number", + description: "iPhone net sales for the previous quarter (in millions)", + }, + current_year: { + type: "number", + description: "iPhone net sales for the current year (in millions)", + }, + previous_year: { + type: "number", + description: "iPhone net sales for the previous year (in millions)", + }, + current_quarter_date: { + type: "string", + description: "Date or period label for the current quarter", + }, + previous_quarter_date: { + type: "string", + description: "Date or period label for the previous quarter", + }, + }, + required: [ + "current_quarter", + "previous_quarter", + "current_year", + "previous_year", + "current_quarter_date", + "previous_quarter_date", + ], + description: "iPhone net sales values from the financial statements", + }, + }, + required: ["iphone_net_sales"], + }; + + // Extract structured data using Reducto's AI extraction with schema + const result = (await reductoaiClient.extract.run({ + input: upload.file_id, + instructions: { + schema: schema, + system_prompt: + "Extract the iPhone net sales values from the financial statements. Find the iPhone line item in the net sales by category table and extract the values for current quarter, previous quarter, current year, and previous year (typically shown in columns in the income statement or operations statement).", + }, + settings: { + optimize_for_latency: true, + citations: { + numerical_confidence: false, + }, + }, + })) as ReductoExtractResult; + + // Display extracted financial data in formatted JSON + console.log("\n=== Extracted Financial Data ===\n"); + const extractedData = result?.result || result?.data; + console.log(JSON.stringify(extractedData, null, 2)); +} + +async function main(): Promise { + console.log("Starting Apple Q4 Financial Statement Download and Parse Automation..."); + + // Initialize Browserbase SDK for session management and download retrieval + const bb = new Browserbase({ + apiKey: process.env.BROWSERBASE_API_KEY as string, + }); + + // Initialize Reducto AI client for PDF data extraction + const reductoaiClient = new reductoai({ + apiKey: process.env.REDUCTOAI_API_KEY, + }); + + // Initialize Stagehand with Browserbase for cloud-based browser automation + const stagehand = new Stagehand({ + env: "BROWSERBASE", + verbose: 0, + // 0 = errors only, 1 = info, 2 = debug + // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + model: "google/gemini-2.5-pro", + }); + + try { + // Initialize browser session to start automation + await stagehand.init(); + console.log("Stagehand initialized successfully!"); + const page = stagehand.context.pages()[0]; + + // Get live view URL for monitoring browser session in real-time + const liveViewLinks = await bb.sessions.debug(stagehand.browserbaseSessionId!); + console.log(`Live View Link: ${liveViewLinks.debuggerFullscreenUrl}`); + + // Navigate to Apple homepage. + console.log("Navigating to Apple.com..."); + await page.goto("https://www.apple.com/"); + + // Navigate to investor relations section using Stagehand + console.log("Navigating to Investors section..."); + await stagehand.act("Click the 'Investors' button at the bottom of the page'"); + await stagehand.act("Scroll down to the Financial Data section of the page"); + await stagehand.act("Under Quarterly Earnings Reports, click on '2025'"); + + // Download Q4 quarterly financial statement + // When a URL of a PDF is opened, Browserbase automatically downloads and stores the PDF + // See https://docs.browserbase.com/features/downloads for more info + console.log("Downloading Q4 financial statement..."); + await stagehand.act("Click the 'Financial Statements' link under Q4"); + + // Retrieve all downloads triggered during this session from Browserbase API + console.log("Retrieving downloads from Browserbase..."); + const { promise: downloadPromise, stopPolling } = saveDownloadsWithRetry( + bb, + stagehand.browserbaseSessionId!, + 45, + ); + + try { + await downloadPromise; + console.log("Download completed successfully!"); + stopPolling(); + + // Extract PDF from downloaded zip archive + const pdfPath = extractPdfFromZip("downloaded_files.zip"); + console.log(`PDF extracted to: ${pdfPath}`); + + // Extract structured financial data using Reducto AI + await extractPDFWithReducto(pdfPath, reductoaiClient); + } catch (error) { + stopPolling(); + throw error; + } finally { + // Always close session to release resources and clean up + await stagehand.close(); + console.log("Session closed successfully"); + } + } catch (error) { + console.error("Error during automation:", error); + throw error; + } +} + +main().catch((err) => { + console.error("Application error:", err); + console.error("Common issues:"); + console.error( + " - Check .env file has BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and REDUCTOAI_API_KEY", + ); + console.error(" - Verify internet connection and Apple website accessibility"); + console.error(" - Ensure sufficient timeout for slow-loading pages"); + console.error("Docs: https://docs.stagehand.dev/v3/first-steps/introduction"); + process.exit(1); +}); diff --git a/typescript/browserbase-reducto/package.json b/typescript/browserbase-reducto/package.json new file mode 100644 index 00000000..78f0b4aa --- /dev/null +++ b/typescript/browserbase-reducto/package.json @@ -0,0 +1,23 @@ +{ + "name": "browserbase-reducto", + "version": "1.0.0", + "description": "Stagehand + Browserbase: Download Apple's Q4 Financial Statement and Parse with Reducto", + "type": "module", + "main": "index.ts", + "scripts": { + "start": "tsx index.ts" + }, + "dependencies": { + "@browserbasehq/sdk": "latest", + "@browserbasehq/stagehand": "latest", + "adm-zip": "latest", + "dotenv": "latest", + "reductoai": "latest" + }, + "devDependencies": { + "@types/node": "latest", + "tsx": "latest", + "typescript": "latest" + }, + "packageManager": "pnpm@9.0.0" +} From a44a77d8d23423806e75696ad9c7b6c9e31a4eee Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Fri, 23 Jan 2026 09:28:10 -0800 Subject: [PATCH 08/45] init commit (#43) --- README.md | 40 +------------------ .../README.md | 2 +- .../main.py | 0 .../requirements.txt | 0 .../{MFA-handling => mfa-handling}/README.md | 0 python/{MFA-handling => mfa-handling}/main.py | 0 .../requirements.txt | 0 .../README.md | 2 +- .../index.ts | 0 .../package.json | 0 .../{MFA-handling => mfa-handling}/README.md | 0 .../{MFA-handling => mfa-handling}/index.ts | 0 .../package.json | 0 13 files changed, 4 insertions(+), 40 deletions(-) rename python/{manual-MFA-with-contexts => manual-mfa-with-contexts}/README.md (99%) rename python/{manual-MFA-with-contexts => manual-mfa-with-contexts}/main.py (100%) rename python/{manual-MFA-with-contexts => manual-mfa-with-contexts}/requirements.txt (100%) rename python/{MFA-handling => mfa-handling}/README.md (100%) rename python/{MFA-handling => mfa-handling}/main.py (100%) rename python/{MFA-handling => mfa-handling}/requirements.txt (100%) rename typescript/{manual-MFA-with-contexts => manual-mfa-with-contexts}/README.md (99%) rename typescript/{manual-MFA-with-contexts => manual-mfa-with-contexts}/index.ts (100%) rename typescript/{manual-MFA-with-contexts => manual-mfa-with-contexts}/package.json (100%) rename typescript/{MFA-handling => mfa-handling}/README.md (100%) rename typescript/{MFA-handling => mfa-handling}/index.ts (100%) rename typescript/{MFA-handling => mfa-handling}/package.json (100%) diff --git a/README.md b/README.md index 30d3b2b0..3c909353 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ A comprehensive collection of ready-to-use automation templates demonstrating the power of Stagehand and Browserbase for web automation, data extraction, and AI-powered browser interactions. +> All templates also live on [browserbase.com/templates](https://www.browserbase.com/templates) + ## 🚀 Quick Start 1. **Choose your language**: TypeScript or Python @@ -9,36 +11,6 @@ A comprehensive collection of ready-to-use automation templates demonstrating th 3. **Read the template's README** for detailed setup instructions and use cases 4. **Start automating!** -## 📁 Available Templates - -### TypeScript Templates (`/typescript/`) - -- **context** - AI-powered context switching and page analysis -- **CUA** - Computer Use Agent for autonomous web browsing and decision-making -- **formFilling** - Automated form submission and data entry -- **giftfinder** - AI-powered product discovery and recommendation -- **pickleball** - Court booking automation with user interaction -- **proxies** - Proxy testing and geolocation verification -- **realEstateCheck** - License verification and data extraction - -### Python Templates (`/python/`) - -- **context** - AI-powered context switching and page analysis -- **CUA** - Computer Use Agent for autonomous web browsing and decision-making -- **formFilling** - Automated form submission and data entry -- **giftfinder** - AI-powered product discovery and recommendation -- **pickleball** - Court booking automation with user interaction -- **proxies** - Proxy testing and geolocation verification -- **realEstateCheck** - License verification and data extraction - -> **📖 Each template includes a comprehensive README with:** -> -> - Detailed setup instructions -> - Required environment variables -> - Use cases and examples -> - Troubleshooting guides -> - Dependencies and installation steps - ## 🔧 Getting Started 1. **Choose a template** from the TypeScript or Python folders @@ -54,7 +26,6 @@ A comprehensive collection of ready-to-use automation templates demonstrating th - **Stagehand Docs**: https://docs.stagehand.dev/v3/first-steps/introduction - **Browserbase Docs**: https://docs.browserbase.com -- **API Reference**: https://docs.browserbase.com/api ### Support @@ -89,13 +60,6 @@ We welcome contributions! Here's how you can help: This project is licensed under the MIT License - see the LICENSE file for details. -## 🙏 Acknowledgments - -- **Stagehand Team**: For the amazing automation framework -- **Browserbase Team**: For reliable cloud browser infrastructure -- **Community Contributors**: For templates and improvements -- **Open Source Projects**: For the tools and libraries we build upon - --- **Ready to start automating?** Pick a template and follow its README to get started! 🚀 diff --git a/python/manual-MFA-with-contexts/README.md b/python/manual-mfa-with-contexts/README.md similarity index 99% rename from python/manual-MFA-with-contexts/README.md rename to python/manual-mfa-with-contexts/README.md index 35cbafb2..dbdd5a6b 100644 --- a/python/manual-MFA-with-contexts/README.md +++ b/python/manual-mfa-with-contexts/README.md @@ -15,7 +15,7 @@ ## QUICKSTART -1. cd manual-MFA-with-contexts +1. cd manual-mfa-with-contexts 2. uv venv venv 3. source venv/bin/activate # On Windows: venv\Scripts\activate 4. uvx install stagehand browserbase python-dotenv pydantic requests diff --git a/python/manual-MFA-with-contexts/main.py b/python/manual-mfa-with-contexts/main.py similarity index 100% rename from python/manual-MFA-with-contexts/main.py rename to python/manual-mfa-with-contexts/main.py diff --git a/python/manual-MFA-with-contexts/requirements.txt b/python/manual-mfa-with-contexts/requirements.txt similarity index 100% rename from python/manual-MFA-with-contexts/requirements.txt rename to python/manual-mfa-with-contexts/requirements.txt diff --git a/python/MFA-handling/README.md b/python/mfa-handling/README.md similarity index 100% rename from python/MFA-handling/README.md rename to python/mfa-handling/README.md diff --git a/python/MFA-handling/main.py b/python/mfa-handling/main.py similarity index 100% rename from python/MFA-handling/main.py rename to python/mfa-handling/main.py diff --git a/python/MFA-handling/requirements.txt b/python/mfa-handling/requirements.txt similarity index 100% rename from python/MFA-handling/requirements.txt rename to python/mfa-handling/requirements.txt diff --git a/typescript/manual-MFA-with-contexts/README.md b/typescript/manual-mfa-with-contexts/README.md similarity index 99% rename from typescript/manual-MFA-with-contexts/README.md rename to typescript/manual-mfa-with-contexts/README.md index 763a4ed2..788d7b6c 100644 --- a/typescript/manual-MFA-with-contexts/README.md +++ b/typescript/manual-mfa-with-contexts/README.md @@ -15,7 +15,7 @@ ## QUICKSTART -1. cd manual-MFA-with-contexts +1. cd manual-mfa-with-contexts 2. pnpm install 3. cp .env.example .env 4. Add your Browserbase API key, Project ID, GitHub username, and password to .env diff --git a/typescript/manual-MFA-with-contexts/index.ts b/typescript/manual-mfa-with-contexts/index.ts similarity index 100% rename from typescript/manual-MFA-with-contexts/index.ts rename to typescript/manual-mfa-with-contexts/index.ts diff --git a/typescript/manual-MFA-with-contexts/package.json b/typescript/manual-mfa-with-contexts/package.json similarity index 100% rename from typescript/manual-MFA-with-contexts/package.json rename to typescript/manual-mfa-with-contexts/package.json diff --git a/typescript/MFA-handling/README.md b/typescript/mfa-handling/README.md similarity index 100% rename from typescript/MFA-handling/README.md rename to typescript/mfa-handling/README.md diff --git a/typescript/MFA-handling/index.ts b/typescript/mfa-handling/index.ts similarity index 100% rename from typescript/MFA-handling/index.ts rename to typescript/mfa-handling/index.ts diff --git a/typescript/MFA-handling/package.json b/typescript/mfa-handling/package.json similarity index 100% rename from typescript/MFA-handling/package.json rename to typescript/mfa-handling/package.json From f96ef241cf2e48741eeb85909f0bfbdfc9e729b4 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Fri, 23 Jan 2026 21:59:48 -0500 Subject: [PATCH 09/45] Add cartesia-form-filling template (#47) * Add cartesia-form-filling template Adds a voice-based form-filling agent template using Cartesia's LINE SDK and Google Gemini for conducting structured questionnaires. Co-Authored-By: Claude Haiku 4.5 * Replace with browserbase integration example Uses the correct cartesia-ai/line example_integrations/browserbase folder which integrates Cartesia LINE with Stagehand for voice-driven form filling. Co-Authored-By: Claude Opus 4.5 * Fix README install instructions Use pip install . instead of pip install -r requirements.txt since pyproject.toml is used for dependencies. Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: Claude Haiku 4.5 --- python/cartesia-form-filling/.env.example | 10 + python/cartesia-form-filling/.gitignore | 41 ++ python/cartesia-form-filling/README.md | 143 +++++ python/cartesia-form-filling/cartesia.toml | 8 + python/cartesia-form-filling/config.py | 35 ++ python/cartesia-form-filling/config.toml | 1 + .../form_filling_node.py | 489 ++++++++++++++++++ python/cartesia-form-filling/main.py | 73 +++ python/cartesia-form-filling/pyproject.toml | 28 + .../stagehand_form_filler.py | 260 ++++++++++ .../workflow_diagram.png | Bin 0 -> 60691 bytes 11 files changed, 1088 insertions(+) create mode 100644 python/cartesia-form-filling/.env.example create mode 100644 python/cartesia-form-filling/.gitignore create mode 100644 python/cartesia-form-filling/README.md create mode 100644 python/cartesia-form-filling/cartesia.toml create mode 100644 python/cartesia-form-filling/config.py create mode 100644 python/cartesia-form-filling/config.toml create mode 100644 python/cartesia-form-filling/form_filling_node.py create mode 100644 python/cartesia-form-filling/main.py create mode 100644 python/cartesia-form-filling/pyproject.toml create mode 100644 python/cartesia-form-filling/stagehand_form_filler.py create mode 100644 python/cartesia-form-filling/workflow_diagram.png diff --git a/python/cartesia-form-filling/.env.example b/python/cartesia-form-filling/.env.example new file mode 100644 index 00000000..09723f01 --- /dev/null +++ b/python/cartesia-form-filling/.env.example @@ -0,0 +1,10 @@ +# Gemini API Key for language model (default) +GEMINI_API_KEY=your_gemini_api_key_here + +# Browserbase API key and Project ID +BROWSERBASE_API_KEY=your_browserbase_api_key_here +BROWSERBASE_PROJECT_ID=your_browserbase_project_id_here + +# Optional: Model configuration +# MODEL_NAME=google/gemini-2.0-flash-exp +# MODEL_API_KEY=your_model_api_key_here diff --git a/python/cartesia-form-filling/.gitignore b/python/cartesia-form-filling/.gitignore new file mode 100644 index 00000000..87cfe969 --- /dev/null +++ b/python/cartesia-form-filling/.gitignore @@ -0,0 +1,41 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*.pyd +.Python + +# Virtual environments +.env +.venv/ +venv/ +env/ + +virtualenv/ + +# Conda environments +conda-env/ +envs/ +.conda/ +conda-meta/ + +# uv environments (in addition to uv.lock at top) +uv.lock +.python-version + +# Python package managers +poetry.lock +Pipfile.lock +pip-log.txt + +# pyenv +.pyenv/ + +# Distribution / packaging +*.egg-info/ +dist/ +build/ + +# Editor / OS files +.DS_Store + +.cartesia/ diff --git a/python/cartesia-form-filling/README.md b/python/cartesia-form-filling/README.md new file mode 100644 index 00000000..1e188b0b --- /dev/null +++ b/python/cartesia-form-filling/README.md @@ -0,0 +1,143 @@ +# Voice Agent with Real-time Web Form Filling + +This project demonstrates an advanced voice agent that conducts phone questionnaires while automatically filling out web forms in real-time using Stagehand browser automation. + +Here's what the system architecture looks like: + +![Workflow](workflow_diagram.png) + +## Features + +- **Voice Conversations**: Natural voice interactions using Cartesia Line +- **Real-time Form Filling**: Automatically fills web forms as answers are collected +- **Browser Automation**: Uses Stagehand AI to interact with any web form +- **Intelligent Mapping**: AI-powered mapping of voice answers to form fields +- **Async Processing**: Non-blocking form filling maintains conversation flow - form fields are filled in background tasks without delaying voice responses +- **Auto-submission**: Submits forms automatically when complete + +## Architecture + +``` +Voice Call (Cartesia) → Form Filling Node → Records Answer + ↓ + Stagehand Browser API + ↓ + Fills Web Form Field + ↓ + Continues Conversation + ↓ + Submits Form on Completion +``` + +## Getting Started + +First things first, here is what you will need: +- A [Cartesia](https://play.cartesia.ai/agents) account and API key +- A [Gemini API Key](https://aistudio.google.com/apikey) +- A [Browserbase API Key and Project ID](https://www.browserbase.com/overview) + +Make sure to add the API keys in your `.env` file or to the API keys section in your Cartesia account. + +- Required packages: + ```bash + cartesia-line + stagehand>=0.5.4 + google-genai>=1.26.0 + python-dotenv>=1.0.0 + PyYAML>=6.0.0 + loguru>=0.7.0 + aiohttp>=3.12.0 + pydantic>=2.0.0 + ``` + +## Setup + +1. Install dependencies: +```bash +pip install . +``` + +2. Set up environment variables - create a `.env` file: +```bash +GEMINI_API_KEY=your_gemini_api_key_here +BROWSERBASE_API_KEY=your_browserbase_api_key_here +BROWSERBASE_PROJECT_ID=your_browserbase_project_id_here +``` + +3. Run the agent: +```bash +python main.py +``` + +## Project Structure + +### `main.py` +Entry point for the voice agent. Handles call initialization with `VoiceAgentApp` class and orchestrates the conversation flow with form filling integration. + +### `form_filling_node.py` +ReasoningNode subclass customized for voice-optimized form filling. Integrates Stagehand browser automation and manages async form filling during conversation without blocking the voice flow. Provides status updates and error handling. + +### `stagehand_form_filler.py` +Browser automation manager that handles all web interactions. Opens and controls web forms, maps conversation data to form fields using AI, transforms voice answers to form-compatible formats, and handles form submission. Supports different field types (text, select, checkbox, etc.). + +### `config.py` +System configuration file including system prompts, model IDs, and temperature + +### `config.toml` +Your Cartesia Line agent id. + +## Configuration + +The system can be configured through multiple files: + +- **`config.py`**: System prompts, model IDs (Gemini model selection), hyperparameters, and boolean flags for features +- **`config.toml`** / **YAML files**: Questionnaire structure and questions flow +- **`cartesia.toml`**: Deployment configuration for Cartesia platform (installs dependencies and runs the script) +- **Variables**: + - `FORM_URL`: Target web form to fill + +## Example Flow + +1. User calls the voice agent +2. Agent asks: "What type of voice agent are you building?" +3. User responds: "A customer service agent" +4. System: + - Records the answer + - Opens browser to form (if not already open) + - Fills "Customer Service" in the role selection field + - Takes screenshot for debugging +5. Agent asks next question +6. Process continues until all questions answered +7. Form is automatically submitted + +## Advanced Features + +- **Background Processing**: Form filling happens asynchronously using background tasks - conversation remains smooth and responsive +- **Error Recovery**: Continues conversation even if form filling fails +- **Progress Tracking**: Monitor form completion status +- **Screenshot Debugging**: Captures screenshots after each field +- **Flexible Mapping**: AI interprets answers for different field types + +## Deploying the Agent + +The `cartesia.toml` file defines how your agent will be installed and run when deployed on the Cartesia platform. This file tells the platform to install dependencies from `requirements.txt` and execute `main.py`. + +You can clone this repository and add it to your [agents dashboard](https://play.cartesia.ai/agents) along with your API Keys (set them in the Cartesia Platform's API keys section). + +For detailed deployment instructions, see [how to deploy an agent from the Cartesia Docs](https://docs.cartesia.ai/line/start-building/talk-to-your-first-agent). + +## Testing + +Test with different scenarios: +- Complete questionnaire flow +- Interruptions and corrections +- Various answer formats +- Multi-page forms +- Form validation errors + +## Production Considerations + +- Configure proper error logging +- Add retry logic for form submission +- Implement form validation checks +- Consider rate limiting for API calls diff --git a/python/cartesia-form-filling/cartesia.toml b/python/cartesia-form-filling/cartesia.toml new file mode 100644 index 00000000..979916c4 --- /dev/null +++ b/python/cartesia-form-filling/cartesia.toml @@ -0,0 +1,8 @@ +[app] +name = "form-filling" + +[build] +cmd = "pip install -r requirements.txt" + +[run] +cmd = "python main.py" diff --git a/python/cartesia-form-filling/config.py b/python/cartesia-form-filling/config.py new file mode 100644 index 00000000..e8b9bc2d --- /dev/null +++ b/python/cartesia-form-filling/config.py @@ -0,0 +1,35 @@ +"""Configuration settings for the voice agent. + +This module contains system prompts, model configurations, and +hyperparameters for the Cartesia voice agent with form filling. +""" + +import os + +DEFAULT_MODEL_ID = os.getenv("MODEL_ID", "gemini-2.5-flash") + +DEFAULT_TEMPERATURE = 0.7 +SYSTEM_PROMPT = """ +### You and your role +You are a friendly assistant conducting a questionnaire. +Be professional but conversational. Confirm answers when appropriate. +If a user's answer is unclear, ask for clarification. +For sensitive information, be especially tactful and professional. + +IMPORTANT: When you receive a clear answer from the user, use the +record_answer tool to record their response. + +### Your tone +When having a conversation, you should: +- Always polite and respectful, even when users are challenging +- Concise and brief but never curt. Keep your responses to 1-2 + sentences and less than 35 words +- When asking a question, be sure to ask in a short and concise manner +- Only ask one question at a time + +If the user is rude, or curses, respond with exceptional politeness +and genuine curiosity. You should always be polite. + +Remember, you're on the phone, so do not use emojis or abbreviations. +Spell out units and dates. +""" diff --git a/python/cartesia-form-filling/config.toml b/python/cartesia-form-filling/config.toml new file mode 100644 index 00000000..2355be8a --- /dev/null +++ b/python/cartesia-form-filling/config.toml @@ -0,0 +1 @@ +agent-id = 'your-agent-id' diff --git a/python/cartesia-form-filling/form_filling_node.py b/python/cartesia-form-filling/form_filling_node.py new file mode 100644 index 00000000..73b9f818 --- /dev/null +++ b/python/cartesia-form-filling/form_filling_node.py @@ -0,0 +1,489 @@ +"""Voice agent that fills web forms in real-time using Stagehand. + +This module implements a ReasoningNode subclass that conducts voice +conversations while automatically filling web forms in the background. +The agent uses Stagehand for browser automation and handles async form +filling during conversation without blocking the voice flow. +""" + +import asyncio +from dataclasses import dataclass +from typing import AsyncGenerator, Dict, List, Optional, Union + +from config import DEFAULT_MODEL_ID, DEFAULT_TEMPERATURE +from google.genai import types as gemini_types +from loguru import logger +from pydantic import BaseModel, Field +from stagehand_form_filler import StagehandFormFiller + +from line.events import AgentResponse, EndCall, ToolResult +from line.nodes.conversation_context import ConversationContext +from line.nodes.reasoning import ReasoningNode +from line.tools.system_tools import EndCallArgs, end_call +from line.utils.gemini_utils import convert_messages_to_gemini + + +class RecordFormFieldArgs(BaseModel): + """Arguments for recording a form field""" + + field_name: str = Field(description="The form field being filled") + value: str = Field(description="The value to enter in the field") + + +class RecordFormFieldTool: + """Tool for recording form field values""" + + @staticmethod + def name() -> str: + return "record_form_field" + + @staticmethod + def description() -> str: + return "Record a value for a form field that needs to be filled" + + @staticmethod + def parameters() -> Dict: + return RecordFormFieldArgs.model_json_schema() + + @staticmethod + def to_gemini_tool(): + """Convert to Gemini tool format. + + Returns: + A Gemini Tool object with function declarations. + """ + return gemini_types.Tool( + function_declarations=[ + gemini_types.FunctionDeclaration( + name=RecordFormFieldTool.name(), + description=RecordFormFieldTool.description(), + parameters=RecordFormFieldTool.parameters(), + ) + ] + ) + + +@dataclass +class FormQuestion: + """Represents a question to ask the user""" + + field_name: str + question: str + field_type: str = "text" + required: bool = True + + +class FormFillingNode(ReasoningNode): + """Voice agent that fills web forms while conducting conversations. + + This class uses Stagehand to read and fill web forms dynamically, + maintains conversation flow while automating browser actions, and + intelligently extracts form structure and asks relevant questions. + """ + + def __init__( + self, + system_prompt: str, + gemini_client, + form_url: str, + model_id: str = DEFAULT_MODEL_ID, + temperature: float = DEFAULT_TEMPERATURE, + max_context_length: int = 15, + max_output_tokens: int = 1000, + ): + """Initialize the Form Filling node with Stagehand integration. + + Args: + system_prompt: System prompt for the LLM. + gemini_client: Google Gemini client instance. + form_url: URL of the web form to fill. + model_id: Gemini model ID. + temperature: Temperature for generation. + max_context_length: Maximum conversation context length. + max_output_tokens: Maximum tokens for generation. + """ + super().__init__(system_prompt=system_prompt, max_context_length=max_context_length) + + self.client = gemini_client + self.model_id = model_id + self.temperature = temperature + + # Browser automation + self.form_url = form_url + self.stagehand_filler: Optional[StagehandFormFiller] = None + + # Form state + self.collected_data: Dict[str, str] = {} + # Pre-initialize questions so conversation can start immediately + self.questions: List[FormQuestion] = self._create_questions() + self.current_question_index = 0 + + # Browser initialization + self.browser_init_task = None + self.browser_initializing = False + + # Enhanced prompt for form filling + enhanced_prompt = ( + system_prompt + + """ + + You are conducting a voice conversation to help fill out a web + form. As you collect information, it's being entered into an + actual online form in real-time. Ask natural questions to gather + the required information. Use the record_form_field tool to save + each piece of information. Keep the conversation friendly and + natural. + """ + ) + + # Generation config + self.generation_config = gemini_types.GenerateContentConfig( + system_instruction=enhanced_prompt, + temperature=self.temperature, + tools=[RecordFormFieldTool.to_gemini_tool()], + max_output_tokens=max_output_tokens, + thinking_config=gemini_types.ThinkingConfig(thinking_budget=0), + ) + + logger.info(f"FormFillingNode initialized for form: {form_url}") + + # Track if form was submitted + self.form_submitted = False + + async def cleanup_and_submit(self) -> None: + """Ensure form is submitted and cleanup when call ends. + + Returns: + None. + """ + # Submit form if we have any data and haven't submitted yet + if not self.form_submitted and self.collected_data and self.stagehand_filler: + logger.info("Call ending - auto-submitting form with collected data") + try: + await self._submit_form() + except Exception as e: + logger.error(f"Error during cleanup submission: {e}") + + # Clean up browser + if self.stagehand_filler: + await self.stagehand_filler.cleanup() + + async def _initialize_browser(self) -> None: + """Initialize browser and extract form fields. + + Returns: + None. + """ + # Prevent multiple initializations + if self.browser_initializing or self.stagehand_filler: + logger.info("Browser already initializing or initialized, skipping") + return + + self.browser_initializing = True + try: + logger.info("Initializing browser and analyzing form") + self.stagehand_filler = StagehandFormFiller(form_url=self.form_url) + await self.stagehand_filler.initialize() + + logger.info("Browser ready, form can now be filled") + + except Exception as e: + logger.error(f"Failed to initialize browser: {e}") + self.browser_initializing = False + raise + finally: + self.browser_initializing = False + + def _create_questions(self) -> List[FormQuestion]: + """Create questions for the form. + + Returns: + A list of FormQuestion objects to ask the user. + """ + # Define questions for the form fields we know about + # This matches form at https://forms.fillout.com/t/34ccsqafUFus + form_questions = [ + FormQuestion( + field_name="full_name", question="What is your full name?", field_type="text", required=True + ), + FormQuestion( + field_name="email", question="What is your email address?", field_type="email", required=True + ), + FormQuestion( + field_name="phone", question="What is your phone number?", field_type="phone", required=False + ), + FormQuestion( + field_name="work_eligibility", + question="Are you legally eligible to work in this country?", + field_type="radio", + required=True, + ), + FormQuestion( + field_name="availability_type", + question=("What's your availability - temporary, part-time, or full-time?"), + field_type="radio", + required=True, + ), + FormQuestion( + field_name="role_selection", + question=( + "Which role are you applying for? We have openings " + "for Sales Manager, IT Support, Recruiting, " + "Software Engineer, or Marketing Specialist." + ), + field_type="checkbox", + required=True, + ), + FormQuestion( + field_name="previous_experience", + question="Have you worked in a similar role before?", + field_type="radio", + required=True, + ), + FormQuestion( + field_name="skills_experience", + question=( + "What relevant skills and experience do you have " + "that make you a strong candidate for this position?" + ), + field_type="textarea", + required=True, + ), + FormQuestion( + field_name="additional_info", + question=("Is there anything else you'd like to tell us about yourself?"), + field_type="textarea", + required=False, + ), + ] + + return form_questions + + async def _fill_form_field_async(self, field_name: str, value: str) -> None: + """Fill a form field asynchronously in background (non-blocking). + + Args: + field_name: The name of the form field to fill. + value: The value to enter in the field. + """ + try: + # Wait for browser initialization if needed + if self.browser_init_task: + logger.info(f"Waiting for browser to initialize before filling {field_name}") + await self.browser_init_task + + logger.info(f"Filling field '{field_name}' with: {value} in background") + # Use StagehandFormFiller's fill_field method which + # handles the mapping + success = await self.stagehand_filler.fill_field(field_name, value) + + if success: + logger.info(f"Successfully filled field: {field_name} in browser") + else: + logger.warning(f"Failed to fill field: {field_name}") + + except Exception as e: + logger.error(f"Error filling field {field_name}: {e}") + raise # Re-raise so background task can catch it + + async def _submit_form(self) -> bool: + """Submit the completed form. + + Returns: + True if submission succeeded, False otherwise. + """ + # Wait for browser initialization if needed + if self.browser_init_task and not self.stagehand_filler: + logger.info("Waiting for browser to initialize before submitting form") + await self.browser_init_task + + if not self.stagehand_filler: + return False + + try: + logger.info("Submitting web form with collected data") + logger.info(f"Data collected: {self.collected_data}") + + # Ensure StagehandFormFiller has all collected data + # (it should already have it from fill_field calls, + # but ensure consistency) + self.stagehand_filler.collected_data.update(self.collected_data) + + # Use StagehandFormFiller's submit_form method which now + # uses collected_data + success = await self.stagehand_filler.submit_form() + + if success: + logger.info("Form submitted successfully!") + return True + else: + logger.warning("Form submission may have failed") + return False + + except Exception as e: + logger.error(f"Error submitting form: {e}") + return False + + def get_current_question(self) -> Optional[FormQuestion]: + """Get the current question to ask. + + Returns: + The current FormQuestion or None if all questions answered. + """ + if self.current_question_index < len(self.questions): + return self.questions[self.current_question_index] + return None + + async def process_context( + self, context: ConversationContext + ) -> AsyncGenerator[Union[AgentResponse, EndCall], None]: + """Process conversation context with real-time form filling. + + Args: + context: The conversation context with events. + + Yields: + AgentResponse: Text responses to the user. + EndCall: Call termination when form is complete. + """ + # Initialize browser on first call (non-blocking) + if not self.browser_init_task and not self.stagehand_filler: + self.browser_init_task = asyncio.create_task(self._initialize_browser()) + logger.info("Browser initialization started in background") + + # Get current question after initialization + current_question = self.get_current_question() + question_name = current_question.field_name if current_question else "None" + logger.info(f"Current question: {question_name}") + logger.info(f"Question index: {self.current_question_index}/{len(self.questions)}") + logger.info(f"Events count: {len(context.events)}") + + # Check latest event to determine what to do + latest_event = context.events[-1] if context.events else None + is_agent_response = isinstance(latest_event, AgentResponse) if latest_event else False + + # Handle initial greeting - speak first when conversation starts + if not context.events: + logger.info("Starting conversation - Agent speaks first") + initial_greeting = ( + "Hello! I'm here to help you fill out an application " + "form today. I'll ask you a series of questions and " + "fill in the form as we go. Ready to get started?" + ) + yield AgentResponse(content=initial_greeting) + return + + # If last event was our greeting, and user responded, ask + # first question + if len(context.events) == 2 and not is_agent_response and self.current_question_index == 0: + user_message = context.get_latest_user_transcript_message() + if user_message and current_question: + logger.info(f"User ready to start: '{user_message}'") + logger.info(f"Asking first question: {current_question.field_name}") + yield AgentResponse(content=f"Great! Let's begin. {current_question.question}") + return + + # Check if all questions have been answered + # Only submit if we've actually collected data + if not current_question and self.current_question_index > 0 and len(self.collected_data) > 0: + # All questions answered - submit the form + logger.info(f"All {self.current_question_index} questions answered") + logger.info(f"Collected data for {len(self.collected_data)} fields") + + submission_success = await self._submit_form() + self.form_submitted = True + + if submission_success: + goodbye = "Perfect! I've submitted your application. Thank you!" + else: + goodbye = "Thank you for providing all the information. Your responses have been recorded." + + # Clean up + await self.cleanup_and_submit() + + # End call + args = EndCallArgs(goodbye_message=goodbye) + async for item in end_call(args): + yield item + return + + # Guard against no questions or empty state + if not current_question and self.current_question_index == 0: + logger.warning("No questions available or not properly initialized") + return + + # Process user response + messages = convert_messages_to_gemini(context.events, text_events_only=True) + + # Add context about current question + question_context = f""" + + Current form field: {current_question.field_name} + Question: {current_question.question} + + Listen to the user's response and use the record_form_field + tool to save it. Then acknowledge their answer naturally. + """ + + enhanced_config = gemini_types.GenerateContentConfig( + system_instruction=(self.generation_config.system_instruction + question_context), + temperature=self.temperature, + tools=[RecordFormFieldTool.to_gemini_tool()], + max_output_tokens=self.generation_config.max_output_tokens, + thinking_config=gemini_types.ThinkingConfig(thinking_budget=0), + ) + + # Get user's latest message + user_message = context.get_latest_user_transcript_message() + if user_message: + logger.info(f'User response: "{user_message}"') + + # Stream Gemini response + full_response = "" + stream = await self.client.aio.models.generate_content_stream( + model=self.model_id, + contents=messages, + config=enhanced_config, + ) + + async for msg in stream: + if msg.text: + full_response += msg.text + yield AgentResponse(content=msg.text) + + if msg.function_calls: + for function_call in msg.function_calls: + if function_call.name == RecordFormFieldTool.name(): + field_name = function_call.args.get("field_name", current_question.field_name) + value = function_call.args.get("value", "") + + logger.info(f"Recording: {field_name} = {value}") + + # Store data first + self.collected_data[field_name] = value + # Fill the form field asynchronously in background + # (non-blocking) + asyncio.create_task(self._fill_form_field_async(field_name, value)) + # Log the collected data + logger.info(f"Collected: {field_name}={value}") + # Move to next question immediately + # (don't wait for form filling) + self.current_question_index += 1 + + # Clear context + self.clear_context() + + # Get next question + next_question = self.get_current_question() + if next_question: + yield AgentResponse(content=f"Great! {next_question.question}") + + # Yield tool result immediately + yield ToolResult( + tool_name="record_form_field", + tool_args={"field_name": field_name, "value": value}, + result=f"Recorded: {field_name}={value}", + ) + + if full_response: + logger.info(f'Agent response: "{full_response}"') diff --git a/python/cartesia-form-filling/main.py b/python/cartesia-form-filling/main.py new file mode 100644 index 00000000..4d93c92c --- /dev/null +++ b/python/cartesia-form-filling/main.py @@ -0,0 +1,73 @@ +"""Cartesia Line Voice Agent with real-time web form filling. + +This module implements a voice agent that conducts phone questionnaires +while automatically filling out web forms in real-time using Stagehand +browser automation. +""" + +import os + +from config import SYSTEM_PROMPT +from form_filling_node import FormFillingNode +from google import genai + +from line import Bridge, CallRequest, VoiceAgentApp, VoiceAgentSystem +from line.events import UserStartedSpeaking, UserStoppedSpeaking, UserTranscriptionReceived + +# Target form URL - the actual web form to fill +FORM_URL = "https://forms.fillout.com/t/rff6XZTSApus" + +# Initialize Gemini client +gemini_client = genai.Client(api_key=os.getenv("GEMINI_API_KEY")) + + +async def handle_new_call(system: VoiceAgentSystem, call_request: CallRequest) -> None: + """Handle incoming voice calls with real-time web form filling. + + This agent will: + 1. Conduct a voice conversation to gather information + 2. Open and fill an actual web form in the background + 3. Submit the form when the conversation is complete + + Args: + system: The voice agent system instance. + call_request: The incoming call request. + """ + + # Create form filling node with browser automation + form_node = FormFillingNode(system_prompt=SYSTEM_PROMPT, gemini_client=gemini_client, form_url=FORM_URL) + + # Set up bridge for event handling + form_bridge = Bridge(form_node) + system.with_speaking_node(form_node, bridge=form_bridge) + + # Connect transcription events + form_bridge.on(UserTranscriptionReceived).map(form_node.add_event) + + # Handle interruptions and streaming + ( + form_bridge.on(UserStoppedSpeaking) + .interrupt_on(UserStartedSpeaking, handler=form_node.on_interrupt_generate) + .stream(form_node.generate) + .broadcast() + ) + + # Start the system + await system.start() + + # Wait for call to end + await system.wait_for_shutdown() + + # Ensure form is submitted when call ends + await form_node.cleanup_and_submit() + + +# Create the voice agent application +app = VoiceAgentApp(handle_new_call) + +if __name__ == "__main__": + print("Starting Voice Agent with Web Form Automation") + print(f"Will fill form at: {FORM_URL}") + print("Ready to receive calls...") + print("Form filling happens invisibly while processing voice calls.\n") + app.run() diff --git a/python/cartesia-form-filling/pyproject.toml b/python/cartesia-form-filling/pyproject.toml new file mode 100644 index 00000000..f570f870 --- /dev/null +++ b/python/cartesia-form-filling/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "browserbase-form-filler" +version = "0.1.0" +description = "Voice agent that fills web forms using Stagehand browser automation" +requires-python = ">=3.10" +dependencies = [ + "cartesia-line", + "stagehand>=3.0.0", + "google-genai>=1.26.0", + "python-dotenv>=1.0.0", + "loguru>=0.7.0", + "aiohttp>=3.12.0", + "pydantic>=2.0.0", + "PyYAML>=6.0.0", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["."] + +[dependency-groups] +dev = [] diff --git a/python/cartesia-form-filling/stagehand_form_filler.py b/python/cartesia-form-filling/stagehand_form_filler.py new file mode 100644 index 00000000..9a775af8 --- /dev/null +++ b/python/cartesia-form-filling/stagehand_form_filler.py @@ -0,0 +1,260 @@ +"""Browser automation for filling web forms during voice conversations. + +This module provides the StagehandFormFiller class which manages browser +automation for filling forms using Stagehand. It handles form field +mapping, field filling, and form submission. +""" + +import asyncio +from dataclasses import dataclass +from enum import Enum +import os +from typing import Dict, List, Optional + +from loguru import logger +from stagehand import AsyncStagehand + + +class FieldType(Enum): + TEXT = "text" + EMAIL = "email" + PHONE = "phone" + SELECT = "select" + RADIO = "radio" + CHECKBOX = "checkbox" + TEXTAREA = "textarea" + + +@dataclass +class FormField: + """Represents a form field with its metadata""" + + field_id: str + field_type: FieldType + label: str + required: bool = False + options: Optional[List[str]] = None + + +class FormFieldMapping: + """Maps conversation questions to actual form fields""" + + def __init__(self): + self.field_mappings = { + "full_name": FormField( + field_id="full_name", + field_type=FieldType.TEXT, + label="What is your full name?", + required=True, + ), + "email": FormField( + field_id="email", + field_type=FieldType.EMAIL, + label="What is your email address?", + required=True, + ), + "phone": FormField( + field_id="phone", + field_type=FieldType.PHONE, + label="What is your phone number?", + required=False, + ), + "work_eligibility": FormField( + field_id="work_eligibility", + field_type=FieldType.RADIO, + label="Are you legally eligible to work in this country?", + options=["Yes", "No"], + required=True, + ), + "availability_type": FormField( + field_id="availability", + field_type=FieldType.RADIO, + label="What's your availability?", + options=["Temporary", "Part-time", "Full-time"], + required=True, + ), + "additional_info": FormField( + field_id="additional_info", + field_type=FieldType.TEXTAREA, + label="Anything else you'd like to let us know about you?", + required=False, + ), + "role_selection": FormField( + field_id="role_selection", + field_type=FieldType.CHECKBOX, + label="Which of these roles are you applying for?", + options=[ + "Sales manager", + "IT Support", + "Recruiting", + "Software engineer", + "Marketing specialist", + ], + required=True, + ), + "previous_experience": FormField( + field_id="previous_experience", + field_type=FieldType.RADIO, + label=("Have you worked in a role similar to this one in the past?"), + options=["Yes", "No"], + required=True, + ), + "skills_experience": FormField( + field_id="skills_experience", + field_type=FieldType.TEXTAREA, + label=( + "What relevant skills and experience do you have " + "that make you a strong candidate for this position?" + ), + required=True, + ), + } + + def get_form_field(self, question_id: str) -> Optional[FormField]: + """Get the form field mapping for a question ID. + + Args: + question_id: The question identifier. + + Returns: + The FormField object or None if not found. + """ + return self.field_mappings.get(question_id) + + +class StagehandFormFiller: + """Manages browser automation for filling forms using Stagehand""" + + def __init__(self, form_url: str): + self.form_url = form_url + self.client: Optional[AsyncStagehand] = None + self.session = None + self.is_initialized = False + self.field_mapper = FormFieldMapping() + self.collected_data: Dict[str, str] = {} + + async def initialize(self) -> None: + """Initialize Stagehand and open the form. + + Returns: + None. + """ + if self.is_initialized: + return + + try: + logger.info("Initializing Stagehand browser automation") + + self.client = AsyncStagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("GEMINI_API_KEY"), + ) + + self.session = await self.client.sessions.create(model_name="google/gemini-3-flash-preview") + + logger.info(f"Session started: {self.session.id}") + + # Navigate to form + logger.info(f"Opening form: {self.form_url}") + await self.session.navigate(url=self.form_url) + + # Wait for form to load + await asyncio.sleep(2) + + self.is_initialized = True + logger.info("Browser automation initialized successfully") + + except Exception as e: + logger.error(f"Failed to initialize Stagehand: {e}") + raise + + async def fill_field(self, question_id: str, answer: str) -> bool: + """Fill a specific form field based on the question ID and answer. + + Args: + question_id: The question identifier. + answer: The answer value to fill. + + Returns: + True if field was filled successfully, False otherwise. + """ + if not self.is_initialized: + # Initialize asynchronously without blocking + init_task = asyncio.create_task(self.initialize()) + await init_task + + try: + # Get field mapping + field = self.field_mapper.get_form_field(question_id) + if not field: + logger.warning(f"No field mapping found for question: {question_id}") + return False + + # Store and use the answer directly + answer = answer.strip() + self.collected_data[question_id] = answer + + logger.info(f"Async filling field '{field.label}' with: {answer}") + + # Use Stagehand's natural language API to fill the field + if field.field_type in [FieldType.TEXT, FieldType.EMAIL, FieldType.PHONE]: + await self.session.act(input=f"Fill in the '{field.label}' field with: {answer}") + + elif field.field_type == FieldType.TEXTAREA: + await self.session.act(input=f"Type in the '{field.label}' text area: {answer}") + + elif field.field_type in [FieldType.SELECT, FieldType.RADIO]: + await self.session.act(input=f"Select '{answer}' for the '{field.label}' field") + + elif field.field_type == FieldType.CHECKBOX: + # For role selection, check the specific role checkbox + if question_id == "role_selection": + await self.session.act(input=f"Check the '{answer}' checkbox") + else: + # For other checkboxes, check/uncheck based on answer + if answer.lower() in ["yes", "true"]: + await self.session.act(input=f"Check the '{field.label}' checkbox") + else: + await self.session.act(input=f"Uncheck the '{field.label}' checkbox") + + return True + + except Exception as e: + logger.error(f"Error filling field {question_id}: {e}") + return False + + async def submit_form(self) -> bool: + """Submit the completed form. + + Returns: + True if form was submitted successfully, False otherwise. + """ + try: + logger.info("Submitting the form") + logger.info(f"Form has {len(self.collected_data)} fields filled") + + await self.session.act(input="Find and click the Submit button to submit the form") + + # Wait for submission to process + await asyncio.sleep(1) + + logger.info("Form submitted successfully!") + return True + + except Exception as e: + logger.error(f"Error submitting form: {e}") + return False + + async def cleanup(self) -> None: + """Clean up browser resources. + + Returns: + None. + """ + if self.session: + try: + await self.session.end() + logger.info("Session ended") + except Exception as e: + logger.error(f"Error ending session: {e}") diff --git a/python/cartesia-form-filling/workflow_diagram.png b/python/cartesia-form-filling/workflow_diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..59d60e8e6e7d0695b8021e3355bd276f622235e3 GIT binary patch literal 60691 zcmeFZbySpH8!rq9q9CCNC?S|Ms7SYjN-NT((lJ90Aq@g55(Xk&5>i9W3_T*!odXOo zl+-YUzz{>6n|R{&Ti;setaHvkXT7e)aJ%Q;ab0^~{oB0JRDVE4!9+nsL`3!Ip~4d) zqEp>OL?k_DP6I8&y-#e2h$x^o^75LG@A3h9=?f=A=msk#L&3BnGyQx z!>flJeVU~6q)``M63ve)JtQH$e~#<*>%@9`qsIygg?V#nuLJFf%^#nCY#)~P0!4P7 z9%9TpSXf1KDX`LE%xkTqt$Dk>dAmbuv%_8RfJk;=O8Le$Jyw7FA|oc&>DGs;%9i6H zM8w&|OzgzGF&rNYuV1etdhu~}8NYPezhP1+zR2WYf6q^dsz$cq6wzA+(P)7wi|gK` ztL~C-o>CLp%6t&f=Ztx`O8?5{iMi}6ws5B%Qol=6eNIXr8|+@B@6lQk`Exjx_qY;0 z?0qkJJtih%{c1R_R=th!?RkFLK^_S&Vzvq4SA0=SXMdRO>3_p)JUTG!d}^V=FJjzG z=XF`63aNSPyc6SIa^otJ&+zLPqo6mUT=GmT;R+PamXXB_uekLSn6BJuzQ6CWCu+qR zICU$QV@)N=JeD&=u-Uu7FS|IBQ_}uJ@2h|orql#FiDnXg*|r-hj0!v`wU0YR9I-r! zadPkJv)ZiAh_>u+UoJZPF@fhDjc&(hdOC6KsjM~?`tFwMGe6~Ow6LeSJFgnzN5_I$ zewF-wH~8$Am}Axqz6Evglka*b53y?r`BsUX)<#Vew;oyX!S zmX#F#H0k**ksHn*7$l6ch;JD#~0BUZ~PUj;SM()Wa%yDHek z1A_ME@Yp}33R&;>4J9MnY-wEWg!48zt=^(&a^|C z{_KKk(8hBYkJD{|Lynv*l%2u76D*=^{I5{8(T-GAY~VnRTE+>JG8$j%$P^-FGQ|g% zQ|&c*Ke2I>M88v(fAm=8aeX$$Q>q-|{p*I zt1nhF7zdAqf$zJgK~o9cb#u^5ca><&!kro&Z@>BC+IC;{d?txjnAlsfp!xc9FUFo1 z>4|@S1F1i4_OUQ_uznCW2pUwfmw=N?__NkdI1Y>2C2;VQG}bpeDVFh-?Uj8j%Po@| zA+Mqsqh0o=`N=U=zZJbju`9iMc2|3sZkJOo@tKk%hXU=^E5%n&et_@uOx%4Qbz8ZQ z{qCEjpL#EDIgu587?hQiuX^B}o|g6^Z!+^j#f8|EoncZ%(xor1NwU0(lSmXpa87ukb5 zS)`ge1->WSZ>Sf9_PQ;t2=(avkNzu;h&+gCNcod(? znv$4NEmx^esG&A!ROV2&O6_n<6ciV5wW+vLAvyYDRKCJ}{h1U`wRLs<%gS~B`S3aB zIr4dxb;MlqfZuajk(E15_c%pFpWilrD71d}BGYH)qt#Ho)3wbNF+Vvk)ax)DWj`ofF1#VU2X{r^Mn}Wj;R04)Ivyq!Jy$}#gSvIN zU8@{*bH{Pvui|-dhB#9RJ+T#*9M%Gf53HwHW5nR1HO|Fu7eqns6~9y=Nx!Ow%j>qo z!g9mc!%oS>L+t>C%|pRc;N@h*Y%;8pRfnxB{7 zs9X@ZHGkvmM}rSe&8y9@=IbS=EFI#cL>oQTyAzmQVqMl|l9ICoHSR5MdBe6xwnMdU zbDiZPJdt*}UM4j1{;B%HK;QEuHV46FF-)jw2ipY?iWX9h9?9O8o}^5#Ob1oAmX(g| zCXcJ@bUJ5EElyi4HF+n~zlU*P<2mK}zJzzGH%s5jd|mQ0k@6hFG~e8NJ(agNe8iT- z;+;b>QhN+8--ze2d*4g@-u2P*)GfL0WNfryBH{V--5+(HOrDcfDq(pH`TFZOWNshC z)X&vd)w7Z*k?~OM#1zIX8d31684nk=nd?s!*Zz`g-7H`)XfjwOC|5>X@L&p*_4*C6 zP0Okr$3s)t7Q&u}eN`0E^~xJty}t8cB68G@1!b?l2K!>_o%*xjYufah!PqdR%+SHp z?Op`L23w(Gtc-v*7BE7QMYUdaV|T-gcZ&nUON51^1zW;eG2l4lymzoxeMI}u_P6am z(a)p#3{??%nBV2x={bmDM1@oC*np5B#$n2#vSQTAtB!h)^`?aM9`Ek+I*j^EGK)=2DN;1r*EYq>w{f)hO%~K$3F4M~ zR?20`etf9cc(LH)tYjxF(Ot%Gkpz2xO)(}+Eha&Lb#HWgGkuhQR4v9>0Ab`**)uFy zGW%0|IXA0d+f>3U(Pi(2gNH+Bco{PS+)+N{9_ELg2X*6g-JO@;jT!sWxInjRcJ|(q zzofCdaX7K>8hIgx;Z9rd~OnB=bRPM^bO=%Wm1^n~9GVS=1Wn5R%XW$@o3{ zmGbHBxv(@Dq5YAap2=VDk>2|i_Yy=Ux0H7l&{&v5DQ3|VyAC(I;XGYix#*t0942G! zH%NHDQH0g{k;RctDKjOl>sz$jvm>;Qt#%l~)I2JCvQ&0a_9}H&2Jy;E8lveNMA@(T zIXaTM2BuD{?Xq3sSV{J{?-vU)TZ3_NlWn=~tx%0S{KkLhq5Kni>%| z@O*}dIMjxS6nG*AewctCz}fu}LPQSyz5x6vq>%je?WyjRQ-3{^^c=n@rzQXB5%62f z%-O;M>|*WUs%b0|0}M4_^Hk4OPfbYBBytE0GpfV;apzq=s6gR_;u9WgO6f!lWl?%w4C-r#fb1iL=>-~+o{KOW?- zaTF|E%$#i;U2Pn|?1$q%f8hXemAZEAaH7Be9k0{E!{(ovz%D0b0R##hej{*)|F*#2 zV*_0!4_n1GZ9FXO^%ZPD0DFKrq($!Dl{|X?|MSg1Gybcm-akFXL`44G^IzZm*;Cua z!dcz{1Wf5F{m;&vbpH33CmkgP4k`Z^Cyv>C)C%xfnnF_G?>&>Ia8+r)OhhD0^hiPO zsR!}mD7p7j?cv5%s}Hi-sq(V23W|%YUW$v=d5N&iu?=l^5YK=y1ji3I_NpE8S~h-M z8L0!ovF01Gk8U-qX6|(~np-Z7crA%xBuF}aI>hL>1pvE9Moda8OZ4v# zCVEa=R~onfY&?8NK$f$S^}-5PyIQ-4K7*Q#2XhYul_*`z+|gG7WQ8z|BKWAW%57F`0q*nN4Ed})CFh2 zu`~tTr{(0~DZ%5=jQ4ftuaoF=6!xE|F?LXpM=$giyQncE2xYz5bR{@H?~Cu@2@@O7Tg-1tMtx7h(1r_p^ceXIkc zZUyve$e_R#^D-MVAK4&%t8T4)>G^i3g0r0?h4^nKN$*1GS#s6j8)vV5C=zp)8PoELVy3vt8ZIk)mZ?O!YhgzfmCAn zVL3TDgYA62V~8q7fY0}bXoFTBvMy?l9QjS;?voEM9ubqwm4Y6#SXttpzcMq7eyZvJx#Txq=D zyPAn4*QJpm_^>TAv$VINR?0}i6C)#gEEJgvEYoBFYIeEAM?;FrU1rPj)+1^nIFh|b$m_h8tD#(?I*(_z zE3gG{Q{U|&b;;GINZe{8TM{>&qDpHglDJ^Z# zDM!xwLPJKX;fl|C>Re5;WcmJ%VcN!WQR5gM?J$K4r0lpuwoh30D;#yKlfiw|kY3kj zh4eWIyUuaD^*#23$y1B{fR8wPtY2JcMD9Z}d^Y-H+~!gr41Eu1HGn{1Tn`VnXOkEA zHZkE#t#l_OveOysq2kAm4kLS)Ik82$w%d?}`+=x=lO|y*@-n}?F^D2%wmsfzHqkxP z*rolKw4Kj-H%A%B1-DpoU9ME7X-|A_-DRX}B|_M;E!}Nvf_y0ZF(X=P^BsN&dp%Ne z0JVldCStL1LUx(Bs%m{OW(4Tj%4r^tpJISqImrl`+%6$h5(dVhFd?pu6*|%9J zwK>wrAYihH)$vDvN@IVy(1*b_U5|9BD<^ErlT$G*qJQHMn6h^}>=@w<#XXPxZL8fC zC!C&sd6QCvWScO?n@1+nunrNFb`^&hbxWVGjDYm|2Ojzbnq`381bgA;#f~NN5vBLu zhOV#ZW|G%_y_F5Uw=I8ZOQg&`ZT{>ghJ!E*nY~9&j4N}JQQD+RT3?p=E(7Y#%->k( zM;|=Z(b?g3(ljk}FpG>a^$lnB+vTbOA@_u9mraaVVyp3kW;8qBjV^=RK_2EHWgx_In&eWCluYha?+#Czn!2JVnm(Q)k0oN8hJ zt_|Occc}Rd$t(QvcK`7*WO?km!Sc^HbbD*jx+XU(_&n}D|KV})lo;PGV#;O@{+(Xj zY9PDVuWS0d?k9bdsSiZWr{$%cI^u47%V%;>k}xRF^wr`Ujw@juG2q|LVn=Ev2w9iG7aq8^ zXjzE2UHzh9hbU^yV@e3w0=rvd;9)=AB)_LTM1EyUH-`AiY=~Z@qkMj{+NC}Qdio&~ zuf3c2i@vo2AWi?gXQpAn-$(*i*uv+`@8oK*ZrFQnW%f->CW@VI-Nb$KhTn_(V=EL4)dTc-l1yMG}B+F zw)Aclpjn+M--W+5Y|MTfGyI7QNq~ll?{9tF3p8a?&Zqwlt=VxF;VAF%(;m3fnD8u! z96HWG#(W8JrCnp-zG<8cE1|c>hn%of=`b>a3&-`SK-Xg(Es|g)?wS{G=kn?meh*Cj zC9}U(J?f5;0vEV?NsRlyhcDNV$AM3uV-amy&Ru(aZ0^pG-v9OM*KRwl^~&qRgBgV& zv9PchfZ3s{aHe*yd1~>(z6C`j6BUYU>5O~^)@!q!P=Zc1%D`L|b2gv82K#xOsXa-m za>?)!a%rTzt<&A4%rQVlNxmvb7`vVQ7`4Mpoc;i#168kvG84D?6+Gyjq;i>QyR)Zx zsD6zY*4}~*&6&{dw9+KN-^c2^+#;oY z2qkh!>&2?zPE2IfO%lL(tzb5GSL!zDWUZQoQZFB>2!4vFHy!o4O7Ta7$MMSEs%SQ6 z@t0L5Yf{Mi^s$9jmOY4sm-n-rw|e*??f-bsQHFqcmP4wo`TnEXU-mTF7rMm1-j70( zs0?owzTJG=l35kf<5}@1^-xh2_R7M^$l|WrV^kx zm{XS4__c1L^!0y;KYY5U;?G{8Z_o1TKl!5h`cQ5CpB`S;mQ*~3VTjLf-VVxw%&Ye7 zVfUK&65uvzIT~;5`M60H0xwOFs>%u%^Y;H%lljRc@hz_2jC@!Q3wh%;SWV7nhFTop zSn_Hg&32hjs0($?)2^rHasQ2WH_ms-$3aZLrmNM86`)H-@CGHk9c*NH&b%DsH4^ewO<=X1HuV42Mf*&{MR42-=0Q;D0-{4zKc^ z`4RL{m;W0)XQ8nCAnxgK^u^en{t~m@XW-Qjh1wO#Pd}p?`Br8}+;PLT3(|Jc+c<}J z@tE=Xp6*+Tg=Xk$hLwI2Xs`B=psmx)s8=EyLur*#P96uhSZWK7HmLnPr%i_(OVA1T z`4~mLJZS7tx6IxB?xB}F9Nb1IP>ZOJ0i(>mnq>H(^d;o&icaNfv_XtpJ=iM)RPux) zzQplc`xj*2@Ws$PQ~fufqWk2`5|ARbM8r?F^mD&-!0~rmm-j2WGTXxKQSrGOe6VQ3 zr(qtMHbgAt=i!tk&|=A>u*FeR+YW?1LKt3~f`=D!+jn{kLYr%09VG*Iaf?ZjR_JN5 z_b$+-4vA4%c^^z4g2YdG$D0&AtyBu@s9besU8F%3xcizsTVq5an{PY0MK2SSt5x!U zLLA?O?fxqC?u6sxy&O8yY}vZN;$=b6FrH#}UwvAT{HXB!&$|W=vR=%S@BAE>*wnL~ ztefr(&&ee3Li8B5t|9$U!UJ`qPq*qegy8m*I~iCgp2yI)Ki6*y(I49WdHLC|M3u#o zH(~a9l?~iD<@-w|T^-Wfp!dj{9D;>=`_qaFY-I=`?_#+b1P4i!ez#qvHYOjnik)JU%pE_&*0&1GS_t$tY;n2JmU8V7ZIpxw)P@4Cf;H*(x`#lQ+(xX zdnI~aLlq_;0udrB3o~l)_^CE!d<)j5?xx?7k^r}GuiOg0EImgBzq#h|tDxG)atqBB z?Y1tDL}L)k7KtNWYt;N^xoOWAHN2%d3-$%SNV;IJNSB>k&~fy}YxPc^4(;@o=-jXH z)kT|yci>lMUG|(PLK?k=I$f_%CVQ=~2EZ1c$Q%$>St>V1^>q+x6vGa~Sx_2y&{_*V zo7~T#>&~DE-YH949DUK9PQJEK73#@%;Rg0rJ&u51BX%uBf&+~<8&t7V_-Y?yjr1JN z7{U4XHHrQAT<`+Q#Rs?^H0I*mB=~f8LbaL(dQAp7gy(TDrT2nup!tW25PUf~aF9dN zxak_@p##niBRMy#p3eb0PGv-W@+XnBV6ZA{!%k+bhKk5+%^R_iF1er-2%UrypJ$M^ zTcc?kxe=SjI{LLeu^Q?niQq6&wN7ut=kcZoxil$4GCNi0rHsd6ebkkH#jLxSN5Z3I zZsbr*y@i2_57AyavvSQIADy?-u2~F3YH0%Fb6Qxm+XbUNhy_0^CBbO1B`*Z7MzJ#X zMm$3{TRo#3A2nnJ|t%vI;iG>t%eD8kgo- zt~gg2g24{c$u{nNDLfA_|KS_J!YW_-MtU3xWuCQeDrl){8J71pv+KS+X=Nq+K1$*2=O{uk=IGKPs;BYxV)Suia338xmYS-;o=Cxn(JL zZYH=ObeMo`tWOko^ukoh*{8^#du5e;~$$g9IhJcV@ts}q5 zKm)Lk8+l@M$7c31F_|1i&pmI^$6`LYSKQDwJ|HQPuLOj?wlpVC#CCjT51AoF3Nv?* zmqN%E976m8$$xX->b->e(CjL|zm}VU+|)kSwTL6)5-sa6tKn>+1-g=~YX^NLH)g2s z?M>fo$H%G?FE&BVYx4zQdW$R}QiBYy&`Ik`p_m!&7D#4n_@|Ic$S1vyj?wbM za|>*Kj&kM2eW4X7XWc01yESL2YqRce_N|lq{fv3qg9;2kK};s4A5hdBW-y3I@;SaI z%ng*xs2)2NeHXsm^LoSOi2lKMErZua^AoI(VxTgAY)If;hG!SIKzTIcBbUX}RthJP z)^-C~h@YSD%fj-3^bk?IvK4(W_3t~(?j>uKU`F64fNF zUyP!agt{S^J(GA=NTS^p*l^~#N`_Gi5>>vR3>W!sXZ}`EeU^`sOUW1e1yQ*hQG2I0 z@I?zhmZu;v?dlklzhiAl?;a|Y6>p`PAAd=tTB>)ZVzIO(u>QH9%XLwH57k>!c{@S% z{ee>+N1oQhd_Wv*rM?|!U<1CADOKOC1G|Ok8JEfnc&&$d3J>LfsUGc$;J-dJ;8yJu z*m;+`F`IXeKNs?Q%`~XK?n0t@0P{dq0kcV7f1*&R9*;@#%-vz8h`jpR(rnZfyH7(w z$sMb;^L}{jtPa+M)|kH12fd@O)}Ey^^E6l6PIQ}AWY;iPw$ac@+&X4v)%Kx_X6g3< zEpT_Cc^JcfPtJ%{`PFtG@lN;)cWJ&Tx14yHMC-YVh5euTmer}P33l3c^kjP!Pb{*_ zn|h7lUK}E!dUA047h!N44;avYWI7A^tq?TAf_ucD!iJmqDl&b^u(KoA4Zq=lo8co@ zbQz*CLWJR_#|-Q9UloXTJ5=^JKwKty+=Ju;N;+gZs8&sCM@h@8Me>(-7R_;HRexE@zYvuH)pwhmFDu%xH8<#ecWjxn4!tV=zZ z2rQ(8=I^Jl0xora#|v*e#Es2Q{EYkg1u6}I5633gmd z?N=h(1u{CNHT%k`ZllW<)Ob|-jDDnRSPYrW1MtPmEi7no;?u)nH zZ@g)4n&~1KY|U92^4Sif5K-&7o(*m5c^Ig1aBeba^IoZuBIa_w=YY!SBbaz)pb3(D z`~2P>A*(DH-B39(5nmb49#dggR$jUN-hSZVj{9{bQ=80qQgLCoJA$TS$Rj=Uwe1+H%n` z83qOy1#bUfx|X-htTS-;arYeqPlL;SgG7&G&7vzXx_K!Y7A6*TYq=XfjV%i~b6#I0 z%ItF(POG>-#v5V+MhAD7Q&=g+gC3(}}-W^uVJ4gk>*?89{f z)B_h-kX`MjJqoF2BVyenS@CI-K9H-g+bsl7u zUB!gcv#z0&Yr%^(kf7P|>W0ACXUIVyVY!=dnq+(Y%Tvh)tcv_b17!*fxf5$|8trK^ zU0kuMe67)g(16aIAjHK;wdS|vpW6nU;Jt?dgKpT6+Mh%1m2#_I7n0w5;9eY6hGd5S(7Bl}fF*;DRAAa4YY6uyeNfUoPFa-z<*j(N#(-mW2@umu zc28kfQiLP^C?^yZe;AC$qr!Fm2*%8Ta+3I|xr7TxhWYOgb`fAam$iy++v6n3aX=3Q z)#uLK=J@;lloe9$*ozP*RGg6tvHJ9qKrvO8mps?snUc_meplX9O5Dp zA93m=i*Xcn0v{A={;IC2#!fq7hfZ>rElDCr)sXFM%O+i-*H~~(Sj@rHr1vuS-chjI z6LtR;tXf|303ovquLio)Y7r-~8jU{_y>{BY#_y6Y%-?Kk=x^{4t{e zuPp88i~HPvWXL%$0%PAV?^&n&n-#~AaXqjje7C-m{K1a!=y+2xmVxB4HWz5`-=tKAD-M1Qm! z0kd^P^C(LELCjr#Ab-EAB75quZ8}-Jvm&s$x&;?;$Km~7+@@_R1GcF_fSBqJ>YEPh z14iXfO8=mqeF~udGLrP(-y@tPq5ez238(+pcR==0gW*3L7NA7>SwGY7y4Q;EvCQXR z2hOK>N5Wrq?++I_`S@UJ!b6fvV}=e=X#;o~X%Ey)XR@gsv%vATKgz1cWPS zBu=U~w!nFK8uo%|i(J?Whz24M0M{AS?hkYqS#~c#`;^QYbY1vE%kUoNO@C7=texz|UtNkH0NUh#V}fGk6i|`=LGd-a zIwV8!LPgi5qodeHu_DzSyD__X%!;oH&t{{ChfPuZu*oQWSgzBL#SHu);40!)RhaS) zFcR*imnm~_%TWG21SaIv@ZfxWX^D!7sWqF1ZCTPwa*;wgx$RYEcnQ@`jGoP-Cf|51 z5UQr6tD#e;bt)#&=s>BUA3heJtZHJbTceG{ZMfObgcSEgwdbGXkgO`Ue` zj*mGHm|kN#%3b@*PLUX+gH>avA~^(#XgK&+FHiRh+hD92JXd=PwKMsp!Bwf?$Wth8 zSpH`_v*Ze*tqCZj;WxX@Pj7z)MLOH?{7@kcz+=aao9o-05Tek?30HY60ve!FJhxa4!#9e+}Q7Khk!?6;TdGxHWi z8C}8B+2QRjL;bP*r}mme!ayxmapxT0x0*wds&60;Y{fZo?cGYeXjqdD&e(o=5B@Vk zADL)OSh~je>H*1k{|G+%B|YJtvwSR;Z*-ecCm5H5xPEz>v5^n1C2VVj@GWh!6tETj zvcE|P^FBzUa@OceZZ4piGC~FwuYSJ}zd*9Y-V}inwy5oo5y<2%3P1fx?=h#j!@j|< z+_}j-o%wu_WsY4cdUt5~&9d+fXOt&nxg{A0ySQLntx?1k#E%$b4YQYb2g6cRRlU0e zt{mC6y8wtMaobLo^7srs{3e^Mt_W}c4N{%3zMViI74d#4#z>@F8D0-&i6g-g8^85- z+YG&|f@@=B*kUK_B#Ik|Fv>RaAaDcSPZ2HM*!c1ra4J^_=WQgoF(Q*lf|H^r(#N)j zW*`VuJLMtL=bc}3Q@GTsC>%1YLz^eN5dgg(e!5$ZwJ7od)YSIoQXfHDZ$bn*NwQrc zV&@Q9Ik70ysXQW_c(BWdCJ1wvX^TA%dLps7x()9i7Zxrv{3>HrUU6~d^ag=-YWMA| z?-B!U&vC^pIbk_+u*YINQV-Fm?&lkTD|^Aryi}m|_0zD5jCL&e!eQltn6%gs5VcAu zP5#LVctoP>fVo<6;D^l46G}$iG}s%NvBT(3hx*zTEY}er5lc{i4dV4R8J!`Yr9rJx z?>U%pVQz5{<<`X;V{7^u)wc^;z+w{ZbnSI}brPq<8NYWL(Yndm-?%gYulSYoWq%YmoF=<17Gs04g2CbH z$O|G0-fvh7Ex!dD`}<*HqWL-1M}#frwC`~c9H9J;`UJS!r`9nC9Ld}r3?IB7$U#c! z&Ms}X=c3=egfcrr5o^iJ!Y(2-pKzGXT2J90WW8otDDxgV);hOim{3o#;BjI|-_3Nz9X8hfG7Q+ci+L<_S!aV>s|WKq|z z;Bt**8y`vL?{}1vJPe^~Lw4=81Ckbew3#W8%ZLw%5DOZo%O=$(1@Y*s>W>}se}C5v z!?`hP`mN8VSs0wbMQAW}{Io5D(F_=WFYwp@l?vtA+uolfv9TBudbOh^vi`Ct?9oRX zw4e+xo*<3AV)1oiQH_tPoFm6>a>Q1~YA-EcVvqp#*o9K@=6%klh95cLr8R)9#>TrS z9qZnnTPqu-8TjeJdN+I+rFUV~c1oPTru+wb;0PPAtDK4pDzAiWcE7OTo&}F}2phwH z4sZ%M<-UBdfbF_9txGDfY-C>BDZtO(cg}urM1y5W1=)F#h<_)^Gt2%{w+bv8<)hKR|+T~4&k#N%J%u=<0gC{YU2igNNKZlIa z^+O^ml}uIj7X-)BHy5Xk>2R&b8?$&mdplmpYgj?$FN;k^auckbXO5iGkF|=XfB7?# zJ=sOR3SnH7pWK?a-DYrUacA?IJZ!ddHVz12Dc5|I5Jj2T!5+~AlI|Z??R@FRr}r@u zF@wZ*pN4P5Oe8M)K4r2synES-Y`O)^Q>G68Ss~!m$Q4DSZ0}zX&%{_${M}+PY%vZe zQBmH58)3o!lphKbKhCc-<;f->f80g0cSM|&IocV!lshS?VVW@v=$F4{o2oDkwoe&! zYO_MjO(m;>47O zQ4}3KP&LCv->7)sJ_Bhfy(@5XNJ?}a^cKh~1P3Mn%Vsk}C{?SFYjnUvQ3py+B{ z5SGn~?;M(v-^-iO_+G5xxuTu#cetO#4o|Vv?0D#%@O95D6{*RmnIc4t`t&=6BgyozC#Jz&UJ#r`{RUslPj_ zLtO?u`?fL{on!sutSoDKJ!9Eb*lY}}ZY;`l6c$7|6N}J=9;FWsiuFU|blq3D8 z^gk`fPol&Ce)RO8tv9%UMDE{U(*XPPTV^|@_BIr_<0G3g4tSxIOCNflAL)A|hA%S6 zAIE$G6iYzcp;Ifklt6zJw*39UP7ajWXU7%On~$ao=IEO5ji-S2-jS-=7?9Y@1@KV! zEO|7J!(?k8fYt$#*>}fFyQj}yT3U!qE;oME6i;6v|4~Mi<%m9j-Vm3pgijQIMAyHy z0S22wdH0TR7$=ejEahQ2pu<}peWU$*S;re-3ZzcfHd=ZFj(K~cABh4!9fnE&uME%L zPxA4Wnkpo^FZAea$o|iP0HT6O08CVKtxm7`(SCc=0kH;xN7?T$E}ihQ=N91q2^(-L z9kKN9k0z0T58hbXy%T`o`|lras$8h9$MZ6t39ZXyD|r6dlr!#dK^Fn6wkxrbdBu_Q z{!G|?$><?tf1XplAQDixuai&9m8ThkAtl zfb`|&5(qa8YehzMr{~dI_hyFkm8kkLi#k3VfWJS=9i6$S@ob^*n~uKzP*sqi6`4J#91je` z;nB-Q63bPc;Bt#n9P*&nv%9||Q3AEl7{Ux*B#2mdIm6(H^?Cr%JiAccyR*sG)srK7 z<Iyg0|qic2E_S1jyRZzfsM_#z4cyE01*z)5dW?U3i}8>0_p9EueE(l zC)d{IE6Pxb*-4&jxF|W_lM!Z`o(e+4xw6B#4-IMqV4RpmUeNqyo7jJmgkE|1iRNCi z_irb!#iG#30XB+Y`*5Lw-}(c~$b*iCa64YU82LroOM$5=2u6e1WDA%9+{0$@(w=P>n*c z`EHH*?yh&lnwbMp&&oeca&*2zA2r_u;|U882p_Ba~@P4&G4 z0JXP=oeNO&=s`Hhm^P*U;)@%=e0wJG~!|U_C zs(XeG)l_-e*-gyN28tjPaCt3$17$jFV3%)ZT;U1>u&`HuiIDF$AEJ8n4{yjs#{?gN z^AA;i4P{<6UYvnC6!$IbN+epFYNKYcIQI}g-@WO#GykA+`U52hF^VsJUMKv!G3JxP zzL<#9q=OrP@f`wA%=D>w0ASioUPwXGDNXQZtKwb(dhpm%`RLGkc8F!|Jo`QUv9s`| z%4a2A44dH0d5Ou_W_cRP&Okbh`jQQRi!a{;VBT*1+NkpHgNh9Wi`ARH_+dMNLr66| zh(`J+D`xC~1%UG6*E**b0jNLR2S0>a_S%`}hzndubQ^AGML3jFTYQ67F4+=(w|3$N z%`{4>S;e=+U86$EkoaL9x6M)aE%6U~LN9*e-Npm_!k46-r=Nr8Hj$%Lp=^t<=+t3A ze!3G}v;Fd5e`Nw4x8uIIfhuD@$1~jhfQ=1={ufsLD%!a<9~5JZUm->obWhet<9|?g zDs|QZ2=gXC;u(e61EI{&6XP6ZK|&#ohV#LOO04v?wY5hESUsnmr2Tz=H#Zt-dX^nR z)}_nPFR|x+{VddzU)sK7_1%(_=E)JOO_YckIz_XKLLv!EUW5njmj`NSq+F9(-3A{^ z`|FJR_B{?A45HcZwE7S!z4OYN`Ci-ZQssdefV(fP-W-zv-f=sGI}2L0BlmZB*b;bh zy;}K1%7}LFvH3f|`h{yEcnZq;fW+-=NjV@={xmzuo->d9>>V$RUeYAW-|LWAR%QEw z4Y_YGFul3;}j^Wc+gnp`F~zmCMA!E;Fj} zcYnDjlhJUaz1Q-0VVVsG08o4bL_lLYHgH-x-TE@l}XLU;dl+r*Lh!zI-{##7io$PthB~VduQH!QsnImxTB^@1lPQRJ_ zB}FDP4-k$<>bm&T=h=zES^~)^cUN#K0@b%CZvU=;@Bm4jp{!vBLwgW#`oAWl^TrK^ z)*q+yyM?*c(LGufJ_#^?fy|d zuzv9E$oYr+8pj`|X8Edn+PpDLc zM&MmM5hKoJi7yxQ!QkKM?&V0|{p~CtS+pJp^)QcE5^J)(b}qo?j+(iC|1Y}_j zDtZ*jfB2+6Rh5lpg2!)rlL+T)o@Neh0k*nXpkei5)z* z#7>8V!?tJB=9pI&r6U0yXIFspTTcQYDUM~^Glr-IQ&A&hEX>5ZW63feG(STMpi?x^ zfQvPCN}=6C=ke`t&&PJ+xZK|wD6WNit- z>oVyTWPUfDa%;FeyWIvGLm8P-U8i|l172ugKKRAJBhgIWJ6up-Sx|2I38Ux}4hl2j z;I8$G2UmFJ8v}0A905@HS%i4{VGWmDU1JF3<7N=e*z9a%u7GVNgPK;iE!v=K@sn1^ z@3rO!?{2dV@&x6xi;J4(e_SO2^rABRQxlVY+BH?4a1UNBvoXm+>{ke+!6bc)3P;1EIOb1;Aqb!!Cg`$yia4OLh zeSxg3C@jhbIY?vx{b=K!CcLoqG#Ws*`1}Suo!^uh^8~n%Bw(i;!o0WxXUll)6;j5@ z^b#w_Wk`rl3c`sbk&?TzKSK7F{oFacBSmN*Dy8J*wcS$P>dzPT4fk28^=fAMahMwN zXJn`Q=NfkL6ZXf#;+Wzt{WqpW;V)Ne59nnD=C6^Q`6>@2!^EF{`wMXZfQ|&1)fmKY zmOecSu)9G_e>G)qQX;pyM#=i{Bf!mkNKO&2I&tEcHy zfUxn~wG_tSUNW-oqbpMZBjGRj$0x<3Etbp2$H5XX1R>VO}g)4^@?2Z`JbM5=4LRN6J@^Ol-i9ew0 zG`XSm&)5xSA+I#;1Fyy%v{u>8(gQUH1K-}4LLo@sh973Y;^>kQD+w4r@6J>m*A?rM zE3Nt)8M%Z#Yz;hOG=FY7hHrNue?QA4;}KXh-hMKsAi^;1!K=^cuKVO;mi-4(KNe9m z>FAH17lv_c3{2ubat`1x?pBu29I`@|8pzAduGFW`9}{|muj^ZkKAPH6t1*r*fmw=I z((eAcXGPMhPx|2AG5g&IFpWDKp-tGE(U#g_w#b+{?Xuy#m)~L|Y~)r!&TM=drh(czap8iz>f3Ke*A4Ma&$3gVDpyU4KT6cNcBN|IiIhD0q=7$E>%t z)@W2H&0zo|!7Ox>_f=6RQ88Igh=Ac=1GhCFe_i4bC|vp3=|h#8iq_l8a&POyOY`V& zeGFY78%&LcHk}}SC&8D~1dr)oDa=B8dTuEBwS=C(vPTbI1JEq=L z6kUBzk>-uGlMf}8UL)B&=7D6o!^#n;n?2?bF}A(XeN-PJCVg^)mdP`Q^0QrtQK-=~ zJ)b(fCBFwXt7C3(PK?6cA#scy%FsH|r){{>^E=2mWQd;aen-6JK)L($27Xq5E7(tO zYgA$ntAKnzGFcHQvUIyCzGgrn^jZQGK|Oz|iE1t{kXi?3Bo;_ls!Z3>}}w8sB&} z6I{qU9<5{!8Y2kCjG7^}6zd zR$O8fcO=|x*SEUBG22AqyA5hOKFtdqE}tlWk5c(p7lbd>9pB*{p=YQ2m&)og4_MLF zh-;6#BnJM$@xBp;Uy1CGEDvrRpLMj-tELp=knRW_)IV|^iuNOwzRc^*77^na6d`+V&!Htn7TaF;&8N4EWqF8=%@`-`gl?WD3IOS z^IB+sLtXCKEovK^Y4@d^X99!7 z=LUwt#hcq_P3A(qLy_KTCUiN=l3j1r{m}Q(NpwC?xsT&B-#4L3o6wD~r5`0c2k%#W zqy5BXX6#jS)@1eg;$z)m#NoYaeS``mBKbyprZTnZYC8tv@*1>L_AuJw7k`*x$y;~s zj*m}STgDmaQHutfkamf1X-XN#z~Bz=PH)=$>d_xBCKwB{{Sx9gS+BT6v@)U2nL2dc zC>y%nN5yxrKC!c*-LV$ny@3Odc~I(}hnk8R9%P?Wb&X&F3Oa(N)C!vcPd=t1WDlUnqM|6E5@GsCD z*y-q>N4#BOG~<c^(0>$U&g3hr97X>Ie5gtyr%Xar&VM=H|DO1O)U zAKyv=^ft_xnfJ(b-_LWupZERG$DFp$-e>K#*ZQvCX0H84?p24E;S(Pi z0%=8wS$zzhcY|vP^k%VnH*wI33Q1nZGu~0q{To3gC53|hm z3^gn{{|x)SKK;b$U(9Nl{B!@nfH{7buHe)@L^A(6A$CZjhf$_$lOZwnOx!u&m_d`6$)-rFd!zxpuz zD}Qz0iWU}xy$`ox*?%=eWxTVYZb#|<_TICtIho2;&y4H^DwX|;j2{e!uGc+tpW#zp z>sMyawH7_D881M+u4bACMtg8GeX6*7v8I;RhARH`r~F?+|1OTM6OIW|Gu7wg?W16C z>sZ{4xul<=-)su*VM>LabW0+eykS$mU9W;JF?FN<+ft(;w(ctOh%Bj!y(M7?{R?8( zLAzQ`zuMLKq@A)7$Lb1msJjeY5f$C+m_0Nq<^8X}b`Kcp;G99R|2Rb1lSJ(>gS7~U z2z1%%hHgqDWU~?O@1Md{+v+7(QupWwoq$38Z{iX8h%z&D5s$ATh&Nk{G-M}jns8R* zVb;|Uv0(r_ptRtkD+AoCbmi^AHPvRF_jZtaOLg3%&U=k*CnGt0)1+m{@%XMXr*;P5 zxPYtU@WQ!a|4!<_G(SZyB=l9H_rtVg*$(Et=z++C=*&~bw43*XWQR$CEF(hTF2#S+ zO~Ef2t?lo4E}DpoM6(bsRa~!?>@#CT7kt4z2$HP?8rb)`;=fEJd;9|e3X6)#FaP;9u|*r{WVD;3_3zJ#n~8j1QVk*_ccm_t+bxC@Gr#C znOi#)lW;HO%#EuyUL{L*TBWQ1^)0VtBMfyM&^hcQ2G@yx17FIbk6v0A&2vQR%C@T#ZTgXu{i(+n>(6KjZAbg(NL{z{kVOJ{(JRm_y?}GJx(J>$ER()hf9o zM-KZ~ne^3LrQi;z7XK6vxPv;LJEg+BM?78W-PELWzit2U7hB?SvN9&zoB+YjPIGKE z8Gwjh`OGv51!y{1f1pcbuLEAE6--g@cS~-=RlWg0Vck^U?JHX3b$(q=1B3@V*&zeqWGQgD@4TkZ6CSRgEN2dLN>h9r!5@qAR zFJKiAL+Y`xsfj7Nl`&~e(7s_TzLVxO{ry2G%x}p2&t3aT^`OPu7H5j=SD~OERfxgA zqXhgAU{qg^JS`v%iI`EXSaMXZeIatqURmnt(BqkcisYB<&li2z!j|zdX+3G)GlzOK zA4^Ef|7?|00C{SlIo+96e|_TWTww8yG~weW3BN^rJ!E*{Taaks^FqiY_nO$3j+tR5 z;+V1&X=EEh$|I-Bi~rH@*n)Nu;M)YuVu1PZzq#lu@BYLvXqgRDH29*VrdYK0O$g$| zP<~SNNv09}!=#buZ%Q0KS{sT3kycLo$Q+4RS{pZGX93BoKER*la_Y+Z+a@ty;Hq=v zjF%l!BE6G$-_dN+N*0{y3D<}ajq|oe5`=q;YCd_6A;zmuNs34lOOTYG;=!*EQh%^I zYY~0|Dz(My${4|a(E{smpjppHR+-<4{+<*lBFWHHUu=A@3E9pUG7K_l+_IeY`HX{; zh1k_p}>7**t=_pmA>l zmGR(_?2}A5C=@vD$Hg?^^aa*C+w=d^n8j-H>3hLddI|>%)yQvsE#{D(iyel=P*{hy zjM2|bNioh2%1WYGN%LLz(|$E|5l(M0PF)R?o?HYDn!QVgXa!N%pLYiCNyNweN9Zh9eM9i{Qt%3 z1`7dY1xhAJ%aG8(>ilE2Qt<);Yx+#3;AB>wi>sGtJY6O1Bk#O;uRr5~Auz(XzWp-# zYk*2=xKG%{jSe#QOltSUlwuk`FN@jX;$ErCZ{Hl-Y8Fe(Vts2TLh_K#KuKC3ez9>R zHV|H-<>derIW*h7iMz5CA#~gm?2V4>C4u(&5?)ggC5}&oEe9VEp?>hKQMzKuFSZLH z`)C6qMRN9TKJ>9#*l+PH%K3L!Ig;0E+kX*?-q*n|$#ao6VRFl*fjaq3gsvccJ`;7% zz(S}uK?O+8A6K6+CaDx_tHb`IR5;kZBfNQD68IiA8eDGwyJ91~{hFGEsebdQ+VX(b zcW+A4iPHZTd1Xc5x?xM=XEQfmaXX z_vPH|uknRWQJR_G_%$W%nQJ$EwnMXgoFFxw`&OTpNvDU2tW2iv!=0N(g`oI+R608_ z!kL4w*4gpq1Q*N-JvX`0#MzbpU|lnqdVYoQx+~Iv%q&E6u3_|>{6Mk(43YcR%vpi5hwgM5l0S4a__G%LH=lL%m@H zc=nIpsL<@jy#uGp3EhAdy&`X5Iwl;l`Fqo6T=D91Zq%|?yqMCGJ4~{4Bo=$FnI%>m z$B8A{W##iMf`wC{4glWo0YYErqd>3!j((GJ)$1?mdQ;fNdzUSI&fP|0RV~`ccaPKK zuoMJas)5PJqsh|0jM0>qKc#W}Hwy_Xa^K7C~`L=`u z$Fr04bj1rmN4ZATO+z|+ocP~K{~j#~fi?DlsxjNI9#zK5ksqxD7rwYx*)+PRQh~n8 zFfvWqSsSca8LZ*ixaXof9(0^fuK78@>SfT%!v7K30&~WL`HYD>XhT=NfN!=(9T~Bw zzAuK|0YRFf3qIyR9dC$5 zlaI>n+(?7#3D6Qj`>*3vI=XKI{z(CKV)5UPQcsCpI}|ZBy7KtyMXr!JhgLd6{O!&% zfcH`(_O}F3HkyzmOTV+H?dk|Q67KjZI3t>)P#{ueX*=^_q@(+%H-FceBe#PagR5dE zWxjkeo$-*zGE1B_g)ePL5$Zk8hNa^0IqYw(3EJZKULr-Y4pM+c{XYfP%4CH2gE+&- zs=-wuA@)X39{245B9&lXS&=f$1~FX*(t}x8Zghf`8b{=C75VF{GM!)p`bfqXC4v|6 z?y)VLax&(1hkvyFszYc**@{~?(wItBQpy+r1_F0Uh5;yqoh4tT`Svh!*G69}kbkTH zQJ>P;(SV>(5nHB&XRY_~hVWMEhg}2I0+VBX1AVDfD4+DZpOKYa)T6-SgeNt;2#;3G& zEo$@#5=>`r!OFtg|EkwrJ&6SHB+X4fi}vFgI2sXIqphpyj_5cr6c;WXDZtD7R|_ow zZIfZV=5VndF+%_ZpG&R5E@}fEbnk33j=!tZ8rSInp2fU4OdF(BW)n76<+hJd(FN z{?DmX`{zS}?8m43z$>&sf<*^WPkV8^+{V7Ur=?fK$*dJ<24Fw@`~l%V{}Cx5=ss@} z8nbY=wTF%Kw+zIdiXP}<`x3d({--5Q@POLxdv>$`;Ym=IbF9^TXA6OsPdiTAc7PO0 z9dMwHsPSq8lHyc8kK`s5>&_tY9MFNb0JV@1ns)Bn8=q4<_L{G02L-!o)p?%#?iG&$ z^1j7TI(BO4yX%fUO^SN9g#_0HZ@K0n^I=i1<5;Dzh;?9WRo6`E^?!AfL39}7e?FAAR|>O4&I#{F6^W$DF2&t8mqX1aXPZ3P zB(Bu3+Cqo|nre9>CI5VNbpV*25fx*hTbB#2u<}gLr6DaNZ%ql@`2#?q@1hoaCV_G2 zVf9UgJ3(4-s}Zz(z}!8et%gJ92}md9;pVu*}wHbCX< zR*JCkc2%KI;9#oK>djFg_xy{8MDw6_-qCk<(G%aJU5CdJH+euDui%M5u>FoQSEi$2 z%Sp406(GB4xVUSGQLg|6vRb7We9Ga;mPD4~pMx17xwX>AgLG#H9aQy=sJ}$bJpJ3^ z-dy>JSn0@VyT!Nl0Ip)+8M(Xr`v$&Pt@SIjck1G&0qAaC+)~{Y3%WA{F6#f6Z~;ZdvPhs4CUN_5aA zAUAn-y((nVA0_phxFKrTP;YM7W6_-|FiTBz@qP|QQybf{pXZU~F?ZOCXEc`V$MC#z zdhHE>RDSu`A68k_K*8+7*^IdpX2AHY4!D0;zUsxI@j!O#kSM(0stJn6Ni+c}nOK8Q zi?>FfoB-Ta&!X(?qLJTWgNdgTCJ^r#e*E~0;~(X1BB8)5!lgE@$&uIn`QThWPz#jR z^x(nU+d}_x_;*g(z z9Xucr_Lz5b&K2gr;Jmo*Kcy)jd6gm${2Kl)qQex0Gro*ptE_zHyVp?Q_oU2wWd%2zh+68C@7f}X}1Npg{K*X?xk9GFvynW#{)}5ah#rM)6Al)j$;lL6Gm`V) z$(V~^YnAb#>(cIpFpzVb03~dwmP_-4Iq3PZwtcv6I!UUzyVrl~ct|FI>Qo@cmn=?j zuMEttAp=Y}&xvyTpV+GQffK`%!*L~{@-IK4d(f4XXb`9q{m=*%_oqq zrjtpPA>G@2I&W34XSJ^Ka^G78%hne2eq#ZoLt`Mu(NUB9dYKa3Z6QXXH~AUuzgT8- zsimzk@FN)(mu3<>sAQ%Df4aAmDxqDCy;gn+bV!#^nN}dmPQ!giQ+*e)>Bo8k`i^1? zRP)k^jB!7DK;cxons-rhl-X(V;i!8tz-tD7dh*U>p`I=8_*Wir16P`zy0Q^FNi0*p zfKtf1(_*{vp?3e2HK8HfYb1&F&)x*3fBu1WW@$`6sgG?tlc$WjRYhfV94Y&r5bz>! z^-Xrg8uksx!}FK$wu*VLSXlw`TTHb%ES?5fC-oKBtU+t>?4rltu&nN=IFSNbmJnR` z`Zq1LsgSp5-k*(%u0BbcQl}HocN3IY56Hu8jLNlkwMJhr^#|GP9m!K+VbiC5Tx<;& zBTRF^%(sUcg+vRWBiN3>fjcGEcu~0DkjI>jHo)`=97n$cm$y^14H|o|TLgf*19SF($&V{j$cqKfm}o7ukS8n-Aq zi@1W2nwA0;Jhk(6rj10Ize^>DItp0_H)Knm&B!m6E5&7vLZ}kEdJ`hhnnS{?vY_h} zd4Yh%&FfeW{o`FqLlUyi-o)HoHv8b{ajyVUa$$v{r3*ElX;`2jZtcHK(YRb6U8ILxuzly3VMGFKBC|cKybC-<QR@mW#?R!%Zj zdQ5Cy%SQO54|+uUit>s7C6kQpvW_=L>(C9&WHTr9?|YbCa|6}M6Kqmz30!N0HAbnm z!I#W_kKUJqw1Ty$!HUDfGWf?JvB?+$qVX%i+HechLB~K1)R8R1rP~`vWz(W%c<&|- zm@^U_nszDY*0hsb54(|<=DBCwfQ(+wmRtiK=Zif!J%&MNKpK1mlQrv!vd&Yg;B zj0SuXU;kd!d>l!$5$a+wC#=Z+0t9ck7D&|QLj5?JeTHFxhOkG@LwBEO-VF7}+kPV zk2nvjAF_qb&60=E{HoH5ilNbwDQSP(XC@WPuMkujvI_?@04|vo3T^#DsV$V#hqC?GCH53&Ciz;zoax zCG|6nl5c!X;CJuFx+it@>}DLAU8lhk34ldN!{~K1p2n3gvt8a}g{0p-dYg>iNLRv4 zf?$$WxN=^LDa{bgne_Q^14#jruT>m6Q#B=p*0auzt*ga0pqGZ8F(b;~W3(6mT6{iD zi^ul?wUR8HFoBy1{P=_q(!2>Jcs874|B}6c=mb$ZHEsi9zxoBBv{T3Z2NB)snc|ZW zitl{>59u=bw0Vi=ixl%#lT75mGpJP}5TFHQ5@=x(dNI{j`$3wMEGyKt6PYrXR@l-63du2<|3^pBY02HiV)9V>deY&Wf1Gzu$X^!NK+L-U+ z3!|1q3cAosiL~}=4>en`$Xn`?UFfWSK%^Pf}3f)0FG;2&Dc!hWjS+6+i^0>@E#AbY|a?|6RbMSMvx$O_*w7dz{)@jI&lrzjSANjk;7EE^4EzDjn`RO`NL(doB%V?X(sj zCdSEp0#)tc10TG%+!q{N!jm>Q4q-m{c_1`JX0T#F(8kH3KN|KC3#uAITKL4Fe`5m^ z5{*~B8zo!s0*Ko85H5Zi9V>LeRKB<`p4L_w{z9aNAUc|1#;S$&t5$K)m3*~3oMq`b z`c%dhF0SM39}hQbFzML8mRrYR^1H_OXJ%8-e-@~>WzY_*{5E0sNP2u~~cM~~-C zvuYleR+dmM(CiQI{Z{=}R7)Rb(RI%sb!zYt9PJ?)+Tx+`N|P>lpMU?`_Xqo5$_ zO#D`o@E%hzP-ZY)+$Xwlkf`_T0SzJh|9uUW%0Z@R`?37TB(lFnoR;i?6`J!m1<5QB zE9B#i^Z$Ds`$!=PdoGiv=eQ1ukYw=K*t;g6%Aunwd)*;mEg|At(p;^B-I1%iYi=Ux z|DJX&_$IsT3(p&}1+PeC+2z>v9_|Da?>*%g$)5%y_Fzw^EQw(;g8$Pi?D6S@)T0fv zf)iu}F|o1_3%OM5A)_^!H|78BwW}vzqq^SSa9jW>T>S|#3l+kn2 z?(zh44n^-SgTB@-DY?~g4yEhNqQPei(?%5*`=z&ZQ{1dA^|exVZJr9V1}9GdT!-`^%I; zJ@?nuA9v08pa>>Qa;|B+0kaNii;Bdus-wURi`q?W+Ogy0=obP1?o~X-)k{;2S>jI| z15&EJKhWEG(opLJjMww)vX6#uI24;z_)}KRRKP2a^xwSRi=q&lGK(-WSr5dje$tl< zJYee16}((5?V&!Vc~e!X#t?{Pjg~nazjX2#ptMZa;+oeea24?Wuq)H~1{2m5&WsS< z0uPEnYU+)jZw)uiA#@VnwL8pPR_)Gj`)$4~al)9)lXVZZq8-uTA$n@nIww#zvHdvQ9X0zn$m0$Mv;&()((CAfd` zK}}UVvLpm>(ZiSKX4JnSyGKmjdZVIQ<@G3p0Hh|?7lVJGu79cPgdq=nHNRILI3~{k z#UE8;W_2V8A~)w;?oYDP-Z=$h!TX0T4t-1m4jq>BG z;@8^JikcpSeqyqk{c%5kt;ZSqHT~d8Vy2#(0I3B8=L}8)&sS`p`XB`7YrUJ2ocQa! zy#16~-bshYXhQd2^>LA`@dwY$;;kxz*`Ci=wmFsI>?+>)4>jH@*MXVJQBT-))MA!R z8jJ&LXl|l4xZ!(@h~#R`$eMkJsY>%X|Eb23u3E>6>Xe=AJOfEeVY~53du(Or;h$j< zq`NG-narWY_v06Vz&=G|@+zXl$9uQTq)>v9S`7ZX70WKX_|pW5x%Vlqjy)PR=np5F zfoU(P*fI>OMpmC2RqLSS82D@wam^)~u~7<*tme|aG|*jxXqaCErBc4Sp4;i|X!SIc zLsGa3C_AXOn^E08!li514}QZxsBnUKhc-0$P?-Fz{FhCj#@Q3tf`nu({Ez`Dv$P7s zPp!TO+3H;1^)l-*u$-!ZbnQ1xtd1_1;EZRsLsgq9Kc2|`*=#ag>0Jv@aO2$i=J`z) zs(w}hW~!ZgZ;UqnlrAl_|JKY<$^8g_&8!wx(PEFxcc|T5|HYv;J(>wlhia)AI*=Q` zwpWp=N6|AlC-5x(H&C2Z`z zxeDCClr}ZNrm4C2(Qevo1ZueKp(KWRvV#FyIViphZxi2{AGTbO>8W)(OkX5>k6pKL zn*F7yw||tu#W2%}3L|7qN1#&jDqy0Xy|PD)MuX zk=T5HV4xp$-HMjHQOQseapR!VD|7#Ue(+}+^(E3U`F-t@`E>p1?-rrVv)c1&#GuxY z@VW!IN$1iq&LiPYa>R%kL*OQ{C7(3e5LM6XhbQ+eD;z^ZM1>+Py%07x)SPvicqNiA(zvui9Fb=s$4Uu#=OKtOKX3Bm!DYyAfsK~{OoAB*Q2q;O|TH% ztyhb}`AylMUMjr5eKxZo^|ORg#~!cF>1b^jIbeqI!ZQkk-5h6@%=tIW3<;j=xFh0~ zh&x3~NkqFUa(Ze7QvGeaep11PrwRvkHH~klUal`&mt|~t~XM0p{ z@a?t4P+&h3wXLnzZi_K)=NyBfN>3hxeiTB6)qhPiWbFCf^5w&8+?A6R%v#l;;EKqU zaGCIyJaDdZ#OV1AcaG%8m)np>pR(8;>O$%n=jS)KRukEjZ)d0Dt!#SUTv3(4-tf6_XXP=oKiCnY=(Bz%`y(w^au6b_#|&a@Nfd zbItPHVH!_f6Zw@SK^}8{x@Tm?2TwltKeJ=UwevF=O9_79yX0|Pora+7B9<6H*txR2 z!im6u#fOk47O_JcQBtAVp+7Hf5xC!HKb&@P_C-EzE#VKSX{TEiJCX5*uwP4>U4h=S zntxq43#Ia#iM<%$m^wDW)093TJs?*1;`MTSbWOb;Nn$r<*0WXe_CVCp3#NB!y@H9o zOxI_)`)@BgrX;2*j? z_gx$M6B?N`k{r`=bj|lrg3e(tkvx=JW~hM!4 zu+Oe`glPNRPDwXHZa1Vr=f(}%cK${o&jfh&yLpuqC(3#Gma-SWmoC?RfgOUO359Ll z#?QwZV|Q*hkK$R6-%{|L%MJ%Ky{g*E-vZ# zW}twXx>t^%BF>Q6Rucz_rWN(nV0lke4WD52-_@up!~EmRUuzSSgr8DOhV~5CcbTzL z+CO3)-e$m@YeuC%U^H35D6_U=iSe0t{oF|MrfSdC8hP}-zo>PKi54ccmpMxmNLzPE zJ#OrhHm31GEetW>_xIazJ{Nyp8Z*q}KYxx)W}H$~n-S?$oEQkC0+FApt7e^hEjYcz zr?y4~nCrUjIgqZb^O*JEe|K!*KOPo1RnH_CdpdHV57$it&x?%Ca;Sd$jqNNShs4L? zw@IMhM%0eW(XDPu)K3rNjEXOq#gAbO;}oBSoDP3M^;COaUlviaJ~Ip$GKu!?3>r@8 zHRCd}JH!Z(- zX=m*hGN%!9MLcTlv~tlm?++3edc#tPx=ln~iC<{*^h z%-KYQIPlk;^WjOxb*lv{-$n#_N7hgUbyB|QGb_Ng!!Vqto;&sXXeanSKW@S4vDUb^ z{wX?v*t_eo3?65GueUUs-hV?>`p%rF1`ZMEln?)dVk>dISn;!8LDb$iS|D&y#3@X7 zaFrdw;UFST4msgKGb_w~#m(f)IypR7CuFIoAor60a1nBN1ofLk&cnV4_>4gCQyeL8 zlgZDHp!>@iY5s$O@OJhzr?38({S$_MOE(CXRPGQyQ(qop;=%;!HLzf&<##)M9{pL|mt^?Mu>ZhmSt#t&>7m&ZBtZ8}9i zWrZbF)F5Y_4cH7C=h?+1cjq-bCE6ap_H-o_|Jnvu-k-!nXvi7Ne;tfoa-A`+-Au}C z*$9Q*GeM|I^EXQ$8$&NXC9)?6Ej9b7Sob_5r*OclOLc7kY6!SkB`(NiTGzj z4;$e}xbI%lLchwskiBh1v7FX}GaJg1#S(83=yERSowdnVm|0eu42345UM}6x-B|5Ge_4s zF`Ei@)_(l*s;WmeqQWI93E$l!dg{=4*KSX=T3;ls21(P|^!bN^1<6xL>J1%u*M@3< zE%fFsx93dWA|)RLA_deLp-8(ZX+TvF5pBHL?8Eu4u}3QG4BXn}$_~P5qw-mg!33Z97w12-CBh$s!=b$Nyv(r4RLF*F!c(lRK zJl7#AZQoBie@89AP2Kd)*VNUQk7BysU@WL_(aNpMFx^%7c6vAAcJB(cw{l=JsEe&h z?!L9Eew>pGxa-3 za~?MFUI!oVGZH@vV8iC~d!QkqBa{SlQoj#<1*<+WMw3C~plhN0HDXTh%RuMRfhTQ= z`XNEF(8K*sr;4&kz0_@+FD7qWG>J(!7rA9_PgZyK7|miS4kTsN}xo#@(4*;c!JS%816VR4Y`K|UXUubR~^2D)NEmD;RRKI=sA=jf*G~9>V zb^mQ!ZR=`{YLAnyHp9z`M^g{I^c%QzS1a8iS{x6X_qlwC)q%}DMtHqz;7#i{pm=^y z7irWe7U`i@hM2X{$qdAF zz~vk2&gWc%(Ui)qkgV3JCD-fsj09FK{AKobkDlejh4~`igjGP$CNVbjE<<^g8>j|+ z(-Fn|hUwmDyIs1r11Wi>vL7V9iNkB3KCRVs{=oM+Q{=OZ3L7}`u zzDIfzX7Q0j;Z0VL02UiQ9u`b~oit4c)z+7{CYBsTYU)OI!(er~hJ0mt#f3_>6xFMs z7d6St06Nw)t6&Y|e_d`8o%s7Z&|Ct)B~!ocL=LxiqAO+$FHQJjNfWN8P2c^ox5`YG1Ljph44ok8yqPcQqj%TMM88lTkVKvTnbu9My67vB2Y?cuL!(9b zgsWp`5r^i*4LIp2Er)R?i062vM2f9sLom6~qmv`1Ywa`%HQExHc8}(B*_h_RXwNy) zlkh1u=v)xLI^Er>WaFw=awzUryN;KfCZz%5cG)D8ZB3U)-|VSM&fV0 zgOWO;9gxtAK9>IRBXGp;wEWeqKtRFmHk!}1_}bZe!OxFZSsUmi4)0MNOGt6t6z1Ma zczVz8)(u7a9PGXGk=#J7=wp#nG%4bzv8btRH*vn3+N=bsKnv;!@+|O6DtGzyqrG3V zRq5TPPq`Rf$Np#ji{BlCP19i3t#@{(&J;IOE`D!5iSrr@GIXm#nr>+FD|nyRXqCx{ zf;?{DOfTkK#W>P%&w=w^kPtM<$r60X85a!j&83=RoPC9A*Rmoulp2%vMkJUn?Kg@s zg_uOsH?6uQj!ZPKuP{o# z@0a;Z|Nh;M=U3)57W|?*VDqxyKzK(I(6;O*ow<2xcSahk} z7b_dZ&FOUcjn6%#SRF=8TK1cz=_42d@rTYg=$Wv0QL%0wf{XqPsD>r9!3d{QyaH&0 zs-TL}zkRBn(B&h2^s`@0D>0C?lvkQGlx%^A$qK3KGtcQ<>5Q>TW`PdFIg{jhv~=WI z4We(BK6yt)x1!78j7`(CPG6#1vERD7v*@)pD5C#q5N)hw7{4@$%Ce5Fqnw}!P{exn zD}_=_X2vlYu)lg`C+#n8H@2YowIE9MINqJ5W}Dh!_OML6fj#b*L4<^dmfsw`JjF{* zFU9qi&SGloX}$`%+_Lsn4Z>K$0(}P<=1Yg&DFXZs)#~ZzO|e8817w&jHpf5+~g|vdGvN=uXtLr?-qeD?Z|z7Vq?*{xxHP{V*31Yiy@3=Zyf-QpkJEO z!qQy%B#ws?`QD*V@&}CSV3NRMdUUQZtZS*!rc!yPzxu^&9>`n3?+tdmOR9Tv)+=z~ zy~cPhH3NSeUvSxU*<;$(Oq7aiVtI7vv}WlPn5aG1d(|+UYrEd@EZmUU5ZaU%YysvJ zC_(!p(GDDbm_xzlWsQiWFdvbu7DFibQC+txJ8p%X;kaSA;(g#A$7i@|_Q6G|3)Q;H zt?`R|e&nM5_)>?tp|chpL5Aop_o`Czxt9AbsKL%yoEEjnQEhj1c`C#Bx8EOLnCGPV z8nFE4^7Ma?}IQ_mI+Yaz1-%gJf;G_ZT;|$RM$@{W(RABbA;NzbF<_ zy&Pv`x>pYSL%D?b17*8;z5Rjb!*aoT_~S?S?|&F#oGBh&{o*#VfSZs`7PUWa+@>t@ z@b$}&_6D2D;1)V_l{H?wOi0c=!v~f))LGk+2vW3L#zb$;>7De7{epM7-;o^Qg*}in zD_Dq+BuhX&e+CU_s!uPp?;P(vNBPeU);`|=waZj8a0c2 zMq%6M#$uCI9|3*vlkU|v(rGv1IkH6I$mToY+KLofJ%Z4Md4s-7fZcC&E>jS78p^uQNQ;T92FSvnUA1N1jtiA2WI!>V9nOwsIqD z7<9a2TCJD*h-)wm^4WQt@AFyFCooo-IO)uAgP^~KAx#v?5O_Mpo?YVuxk;Q0>}7LJ zk3;|BGoSpi83+sRl0L+i>u4uLYj}Hg1Pam4w!Uj*xE;;1x-#)VovQsY4i{(}_6TRx zZda<o8+bfsNknH?W2>ol@?y z$(fw-6<#cZ*1!s^n0{&@2&^iDj%;g^G)%7 z{CUwxel9udtCNCfW5Ro@PbI0O?%f_e`{(0I^GR^#Oug&nDG{6X!Nma!#%D8wirFUS z^SxnTaIOH~QCqh)M}6*ovzd>Z&Zno%@wpo^X?cS0dtJ`KQfrcYjpqr}uUUuXGS$&B zr;S@Jo0ph&N!t>ncZppmBg6ZTYB~yby%m;U8fOhl>@J%0hxeEZG<#C~sBf0GyKjbu zJ~gIRgBrI@4N^hxRy+lfLzxb#Ed8k*@^gjK}^zaG)9frs22NPC&GOjCp9+yx`svLw=w>yPv zb&~WW?_aQ}!AsUDsSQB`=~3yk=n9cN%!78zA+CV4+>{QhNjt9b?)sA1+bU`)dXV09 z=9U6xTIue(eQwz$&!@>;2FD;_HmOt0JTyS&mf7ahj}hMf7G&N|+olJ12#>x5Md+o# z{i^Bx*WVZqd47kGSqqR=NyRPP6V-*u1+B2wkQ|q!rcrS5z|d?w9SfC$nDfW za@r2lT10Rk#;U*|h}Xusx3}O)5csVwO3%Z1TVS(1$XDMbkGX=k8(~rKx#h5f>chG{ zHD>;}@cwCosW0h^6ZYgARWi1m0V$A5f3zBd#JNh8rFYYxL{_&^BzY&WyBNE`x+z7z zcZ;Xtw^Uf_oYb6d5tO#C@QW!sFRxB@YH{8Uwt?FlmQ5p3 znHOA)j7wHac)bN>@@3`tR};2xU%6h;9p9IVlKjepF!u6G(!zNh(qeXoastwNAl_5T zD&W^?XOG4&hMHK(3+%W=)?Pes`H1T+c`|)eVHYy}RMPBK+}(HnHTN4gJ~vwVbE)tT zNcH--ebah3?YiJ_hGf9^=uR3%m}=UXAGOIU+I8Z0`Kl%MM^e&c44)FUuO{V@c?!hyH{GX3F17eJJ~!3w z5kAKODPZIj|0DatHoJ*>z9+U92(XPeIdW~~0XD5pja;5%eivn29XY|+mBrgv=fQ_klkPjsXq;67dtlSRZB$_MO=t;KO?~Ri z(f35YubQJ}s-*m!f62hAORW=I#=BZLbV6x#YHX7nX5QA><=I*QRJ83-x;EQCdcyv- zt?+5rgcGDMJ@$No0KO=Nr*t z+G*jL+L-lPmp7|*`@REBGZOlKPt%#J1-j;UfgHX3kRFil9unVc7tKB}jtX{LK^|R- z*wM2pY9aD%AMLxr?w29Oe!fM<@OkWkCJ^2#o!mmtPcu#t2WgUTkKSooy{_`v8}`;t zY0clg_jY?{I()-at3~7k^vs`@6_$4Ns!q_S+tpUb!|?7u7q+c6XhqiP+t+D#mxLgL zwPkQe*E*3J@|2A|?Apcd)Lg-QDeAhA)9?s`#wqelfKo4#chU8Wo%sG_%Ck)E)uN&s zTSu>c7GFRDRHg;h1b?44{ZvH0gKcrwHD=X`N7OnlX~YV$-Z>`16jz;hO0aqWQJLLg1|;`*Ms8qXAZC{z<1l$r?{A;o)*4iI{(;iub0= zhUn3j>Z*w+(M$2`;Ah)DlwkdDWtK@!-yT^Pe_6N2*`f33^z9P!^Z&d@@!U6GS)1UO zkS=j-(%0$&loWql6f{2jN~xvOeDint$qh-Gl(b>Y#IdSmUeV3b3U_Zvk)SQwuo&6! z;gtE^=QoW-l=SXY<09KtmxfP9>1c|Yg$-@nxm0n( z{=vmYI<9faN{blB4 z8!8a}4>9uhCw6TBgK+)DRY?3_aNr6ix%z2M@=AQtk8xd_=(M?77svhJ766*a%F~X# zK1yde=WWx;<0yA{-yrxz86=uU>GYXmT@*~?A+9#83*?tGcGE6PXLtiCpWFmsi0m9v zdqZSgLHj-W>*!_;ch5g|rM^5bdYFvs##a{}1p(ECt$ihFUFPdrS_T(lFp9|_PPH_> zfn|iNn`*6km~U~kFwiM*80#q>#C;$Wxk<2Zl1BV5O49qB;DR&v z2VJHW*PY24>axsxwJ~007+nqF?-I1*p~O^&2Pi6%(=v&7vH$75D=f(RAwY9DK4dW@ z8R|UFm^FGhAyMV$%Ht_H+AzU1LG+&n1p6H;JjfNZa?QH7D%NE9IdxcIKb)I_W|4Y= zhwi_}Yy-ojGex(+jIgx4+qI(fc`k-FmO;*2iB%@4dTMuU@%VuNGXe&!A7Ysd8I>B_M9AWgJ5G8v@yzBH~6d z?<7uq7aw_ai9S(o-h!DZHQtTf4w|F6++@49^Fqta6BzkP?{7}}`~sjv*tuhDgZ$!xVuY0FTMkeDdugUA%t#E_y3m_2ltv zkYg9|>?6Z~D4^B((KQmS;Kx7ke**<#a`zoK@GR}PM4}}iUlEaQy0}ZobJT=8(p9RB ztU37VZ&KY`)%qALNuN?85%c~Zb|lUu1TuRp?hlAjh}(ihe!eHmVvuQ2$D&{n#L7DU z6xG#vA|bx;woZ}d@ANJa1IEu*43*yi@x3rcm8>F9Dy1Qg-+QtYsH3+m4&8^m&<{w^>y7%$(Bw?@ef& zeLx^}&*Pub>CCVr)MG!PV1BO zdi76~@6gASk3DBRm)9bx%-U_ME|^tZyP%TyycDlSY7b!p8IAkJ!euq zt}z`Ge=uoqooQjV;^W__KKsb@9v~7(hy=_SlGedq)&|6xL|8o^Z21cw{|5Az#(4-Z zMgFdA5JDz-^bYoNtot>?S-%!at%)~3i2lL-nKJoXXuP@G!MsVfsh0A5aN{paOIsm{ z1y%2WyDk@d?2o?0P}$YSOSdU{v%Rz8Gai;>D#x>M!E>x$!d?ylUa6JhPH_beDUxbX zIuTO7>@zVX!+p%dPw@Ex)}L{8%K^FVK&dqYQruF$o80I4pnFr6WOB6qmingGIWL_} z&3Ex2X$W^KiW~Lu`H*Q$94i)S?<2MdR%F}FtX@V24L!6xoiZAJ$(^bePM39 zo|hP$6&9@@{rT>1I5{5zV?Q4l<0Qk{m!R`^Z7a}F0{=mrhbX{ZY_sEO78l_8^;59` zNN3#f5B>RDkey5N&j36G6%Hnp|8uN<7GPCtrtfk5>Hlve+aIAZYnN#>YQf&(AKAWzYU5?{g?bYDh)I5iIgg{|L3eKmGrIDs+Q2E*Qs}`KYv? z($dno;Brg#hj|-hHY<4!NN>WG&`U8~EJZ-Uc6E~_Cg65+m!a`WJIX5Hf zlVf1qrG&-jP^{JLeVh58+<1|TjF(wMzx$Of31Dg5DjZ{62aXy>dD_^@FCBeZat$b3 zI<8nq7$0Ncy>SuwJEFjfu-_rz2?gp}HxA~3!QTkK8uE$h0;;4cw*yEjwvYD4zPMU# z_kd+*S|Sq}?@O2C@`4V&0Zc5NKFi%_Nh;Wub;pB`<&r9!HboVp-s#v?4d3{~7(h8D zVnEejwSkfsvsiI9Fx4IMsG26;Da;zD3?%1Qb@q4uN4n7g1^wkNk&JyitW(J=_yIhv zRKb1mS_W&tDRn~>K}z@IoIoyNKa~f>n{|`dOA)(qJI{%*}VNf747!{R?6_WVsq zD0W2NxN(QdTqMG>yB^f39~V~fPP8+&dQ=(_J}0W2dF7925CH*8jJP6h*aLf^_fsN| znd(jG73`RZck=ThJc?KUhzbv^yFj|Nhu!_o-v+~P&iD990#Hd1E8lNx53t04(S?Wy z@PJ&+rFzjhwwp`^;&Sxp=p`Q4W~ago%FbY z`rnRnjJzG?5x6Na3rWy=Zd(jZM0RlUlClSnv_Hr}9SbWEp!}PE`hz)(EMm(HVevCi z&h`ie4HKmkeo*S%r<1jJEE6i01;^`o@$Du9scD2qn!;v?x;*ulKg)8*^$T>i?C(6L zIBX0d>swFJp)g!2NBUz_h9xm;yoomsHjGDLx&7mZ$NitJT9Z*0bp_Xp)2#9fiO_Ks zO6umn^$M^?_a6az;`*-9RuJRVbJv0G(s1eF>i0Hj=$+%V7hC#fy(vRwxerQqws=&! z#wBd_BT&yYSEWT%g$nH_DudSEkZxo#R3cg{aYDYBkg&eIGeJ|10+%E5ivu4i=kmT4 ztE(;uw_-1sU}v|Z9}~n~!fn*6`^!pZLd~)dy>}n^x)6OBFWjbKYoXUMp+5D2?=9&7 zzk`3wEp_V>1{N^*ZEQq@hRnqHp)bv#lNyc-YPtMM@LN=j#{S^udPSy5$OHDdCo9Pn zS33f{FQ~0mK)sLGGu%z&J_y|FP^}H&w(=_7y)(h(ctTcJHdkVg zYb7^w;=7fW>RU4*4WnkZAjVHFwBwE3Y!6Bq3s_6O+~~|*NFLL?o?E=L#r^uB5s+^* zG%L(l{i!F2+oiPZcBtye7r%~Fxv3gEw94S|ceZD%Y*2_(;JB36^sZoTRd}9EKM&X4 z(k$R?0m#+D9F@CMKa;iH%*M^Y@?OEt0izA<|M2@(^gEj!dS9h%q;nT^m8oGf&5_-2 z|HuZ{jSPwZFy!~Gc{C7dq9z$TJ6=C<`y*Mq*}ecvUSq9Ap4Vo+wJs zi`7_|q88iRNr{YDv(>&Irqjyi<`9lLtT@@N&!a`|;+Io|Q87>1dfO+_@ zo05C3>XUC%T=L6}Pdu%>_S#2T2_YVvmlmL{btOCU!0jUuab9eRP=yXHr+c51BQtmf zA!ElBl3SU#|1M81kBMJ&q4N_yy5t%SKuPAu@}nJZh4~9!l{KK9JnLClA|Xq>g{UmZr) zRrTVli%T{Xk9^D5hS>$@2_?pwn9#kA^U`LE!USO=X4gh7c{z)zUTfgq=I!~s z%Z#1TgWq4v?bZt)k5Uc{>%>3wGsauF@o{8i*~H9z7|deY{sD49d0!RR^>Hb5rwEsk zGwI6+k3U+p{t?;qwV|3Zsob$`(MAW-tW9XcJ;Qx4()dL=(pPhL{=LiUwozz<+6Sxb ziEiQd?Hpy6YudB96{KQSi`f^0pPCAkfavgLK&YYV2xAXkwY4aQquEk7bER z28Q5g%CSv?8u6WKw2i4j{;;H$(4#`v`J>2`djjLstx7tsJqZQGOULr7H+Kp7jA%!e zZQN*2kG7Iok>8v(<_Zs~qi?k^dogz$X4`BQ#+Yk!*^plf49&atZUmI*eKj>P2TMqc}vUE-v`c3m5_fe0)*?%H~{8de{c-I}R2 z!edqP_5s1_ZLYv!U16U$o1z96RzA=Nhz2!6Sgq-kDQl*|Gc@>Ptd5wsRf;d$vIT23 zs%7*=#z%Fbr|b%AWxc5IhIHXdH~FaGik~%I#nEi~VQ)`t0zP2M#(?CTs*|=x%5Z_> zE>Dc&h!TP{Ks-(S0~yswNAoAqgu4nwy^@25Z-9%ILmykI%t?p+kfBj0Cprw5tAe~W z4rp+_{Xt-1;rZ@0+i_t5U$Z%-J}i@VgI;vHXFa)>ckt;_6ou;b$hM&QXF0NjbhmCZLxphS z!-@NlK^HXVEg|5%9LH#DtZL*iPPwXf=8J|`5&RQMTkc$voXCuKW_~lsnCiOX``0xR z8M$*J@=#fW#byjtrjkWv7}lk75U1t`V#Q*FLzF`$wYMt3 zw(_vl<&%E2p->Y2v&?5$gg`tZNi0(pE=EeTnZ2c@6wLY+P7&2DINLJ;qPtG@n!Gpn za{Nat-tL+@vTH8wgO;UaTK-}>ChtW1uy*ToKNHzMB3Q^*`8MbTPPw`>daKrIL;U{n z{9NqE&+@HBqs{hegd+}XD=7joRs}X<#|ac#AQf~Jj9 zXfgNl;LN0DeMf#px3b(7wr072JX@2LYU8?VnY;XDMo}F3Hj0Q}VfudzV#PhcwIS?o z)gXSs1kazJYUeC}Ri>wBZ`D+4969gZuEYvQ%g;Ax&^(V8gcsAyR1Q^h61;l=E9-Bp zDwPVWuBN_LPC#ED6Payq_b!B$-Ec*lRZiP6Z<3WgmFB8Syo9!#Q%0XUGx_^sIktB& zq^$*NhqlhZ=j2F)=<0+pRf=*}VPeJT*s{@$P&%RM=ZC_CpV(fN1g^b8EvPaBDclM(esoQ znWkJJsrj^DO}C;o{vPhCNLfCI{8C8T${KZEt;t^nuZ*S&Nn@DAB0$7V4JTlRTvGs&;>Ot z9jq7Wk~d7OGXYgUmlGQ4sr-HA;2T?KY6L=4+s+RP+I(S=X(p5`>a;02eo|CB6m#u5 zC)}YR7=w@QNl_zhSXRhu$!nfY*9xGJz9{BNv<%iC+TmvIQ&w%>*1hr?nK4DBLtEmgH$ z5gmHXD;$*MR`^rHODW{9%NmAm&ll`tAVF&+*{KgmLP{NOZ2CLg=^w8d?QHa)%wma= z%&9ih_&~PZM{%jr@>QR>)5eA9fT9|9Z)-Q13MFMe*X^kuWtODJC?2b*TMe&?-aV);QyTBqWRyW4J^ z*Bf5#gkfIgfLHc}Br&inZjXBz;D%8FsN|6lU+u{)WyV)rr|M)*_rQC1eN|*NZj0}S zxC!!l>Y3TS@lugY|Gc{){FenpGfe7H-Eu&D+6^{a2Zg zah-|PD@1S(|H|4_@~HFAp&>M3xfoQ4I~qnTe2;9(RDbh+$CVn3KKs}cbab-LfOe-A zdFU}j!q!QL%SsI{Gq?~UWSnxD*3ZfdYzD2k_Imq%Nzh>_kjdjI? z?vt_HrUluxprwj-(t#e16bI(u1ODfFgZN5${N!MpS73Vwbb`wmLuk1|LaXfuVd=Zi z7_D^pSvH!p?lr&;%47}AysYz>YTIhMLQe%*am)H!iLP$A?nt?;kfXg8-e%;ORTlSJ zC>?VJ3px+(gt+nYK6<~q{z3sA<~p@$S7sj0Za;sTu)4hvzq`=x?P1usn$+3qxqO}f zTLza!f>5`(O%m7rkhM4^DLWgCpjlCYv7#tkTSZt(q~n%E($!&3LOT2s)ziR@!na<$ zlEL3!X%PChQRZHbLdGo9O;PRCv?E;2p>iJ zefSE@tm^DkGN`lwx@FqJ9oIhJ~}e79weHcG1Rj3xrsr-h562ziMwB8;=72D4_wqME->uDx?@5zGJgz7e3HGtazBipNz* z!Mgp|eFPsx@`HqM3nBdqD%~i_<>~I(uNz16v3G z9pryt^8YNlA26C8gYa zSpO{k2+*hTM6H4GYm=q7;2LMUDX!0C7YyrJ*0VwsWha;K95=d7hV6Qu7Aml|F8Ij z^T$Yxk~J=`2VkQGH0u}Me~iV**eMzi!)c=58Oy_7IM_o)GF0uvL3hr{BHF-a>w;nV z%ca*?T2=OP$L$MPbU=ao)^2~5JPnFt@Fi-EG(12+Ns6oLHK_ic)}{}zm$cuf+JEcM zF52I0KoNBoF%@v1roZHY?E#d1)`;b{sK2OlJmQ#uZ6J9EqVSI*W4N}9-97nrC+YDw zG^*?aA(*O26s%_1o7(~zC2B$006cmgE8q}xJj<+_!3zBubp-|M6EXuiZ^Je>-vb+N z4ygH9E7@3k1#akm2l>!toKLV0wyL4&jlG07%|#cq<5elZx^a(}4y9nf%nD2Q`UdQL zngx)tN`FwRaW(|X1JMFSm&x^#%h0Y1y~|%Xhj!1d17B8UF9MR*h6&gOjmD}lg{YBo zqpw21j%@;l8{78z;+H>#e%jms1$%@R)41djk1HFu7c*wvnipP%Ssfm6-d!~T3Xui{ z^yKMH6zVlvtPDOSA6v--Dp~BsEv7YWM82aJnYDqoE?43L~eo&ba-TZIW*Hyd?uER(2K*t*SRF z368J+fN(*0fmEW#B8Q1%c9-R>);=a`K}TQ0JB0l)LfTBVk!xi>Sr*^?a``Z$e5v4b~%6`2=hrN7W1U(uLn) zO&{11E&^__W()?h`MjuI*Y(P5mq=ru8a_=CeBu4aiJ+@HQx#RGnILRaxg;Rdba$sf zhYWk#3c4i&RAg2dUJ!Cf2LWXn2kTCS*6J-uv~>=VJQjnw!J=wAXZ454wf9PA4Vr4# zy=+iDlFUQEhPr_ym30_oSt$eA5=fZ`7~5>G%D)k#jBsdpg>awa$q#?Hx`o0K{>I>X zg0ecDH39p;GZLIDI}?Qw(7i^MMoi3!>)sTs@bI(PjS`?BdiED`jbPrx9|}-QV4MHN z+akTjYgL==9kS?1?E?$)j*-yd_88733!p@D*cb#;hXRTS6if3 zSfv@Q3WN4ffEkEZ-JgAcUIlKPC2!g_jMUOQM7xQoJvKi+u-i>Mm4dab&PD1J2C!8& z&4MB_95-h%rsk{D%>lp{=|i9C4~G*^wuxN!rf|-Rg&bFpbtgVdU_PVe^8z z6Vtt}KRCGLam4J-45S->CKJ(R#_G#ZD8ZrmW|V=__#~5~A+z^lF+9Q&`9Kavj~{G_ zgsBsMLbCPTXU`RE?16?W$FqXV9<7=V=H9C|n_IUWtMpla>r{^c(o6_;kf+x{`?GEh z4f+Vpt-i_dK}|z1C9_gXKV%zF(~WZ8KyrdhQea@MVxepDrO-})Un3O?sIYv^BR=MN zVt5_)?x(-{SWwFYISzaIm%aMmH$SJ@IxFve560L~Pbv(4a4=s>>Lx<29mLIjY%j?NIo3@BD=+dUa zg*+<}OJce>0iFX74Ur^myz5lF_dNw!R@DoDOJ?|e`#acA6MQ(VN&u;uQ%g5=Eiv+F zhwiQNUW3a}vkAtzrWO(VTxa8p0W zJleBH2Cls?GLU%8aa0Y(%<`@SK<)5-I?vL!$cC2N z+0f?O6GW|uZArbw&3nKyv*Jc{(LlD73-?n3YDwBbYu{$Ha!(sp2JSubPv7sK*lQ`r zy2*+JZhv0z+>Qx`&V>uxTJ3k!#VO18r`ygI68^Cq=-hB8{kjrsQ__huxjX=W+00p@ zOs~y9nAjIU zmHDhM-G@|SPL6T*q^GYm+)ZDrJGIoowJ2f#WS?P&LbCco{IsGyi;&s-6Ur;M=5ku) z0hK&XF5tq8yfuECf+kw!Y9!A7cAqe<-~{P`*;_mAL%toh;DRBA>>hCG4!~A;4JK zt|s&1>s{K+tdz4!U>s+3cQB0U$g3D2ZO%#$uiN>w+SZp)sj!t+TFj#o?S75DiTV#% z`+&6+A6)Z1J`@PCyts9BW2$mxyLzFfw@kt&xu^k1cwqYk{d)Uv#`I{BcIWE}EL||+ z!G8bwJGH|RhhS@k=|GC4q4HQ4;MV@}kS=(inGx>ux6JZwm1i~-mHpPK|IkAcaLsTE zrN7{uC~##W-mgR~`2fGb3sQhk97q)tnS20bluIf6ozmqjZ+;a$HziY= zo4&!O;jN|c;Ox_RH@8hAFs&hnP>kMsmuO<%@Y^DFaIqq>#5ss(;?lS4ROstfbtXlq zt{I0u?^9_=mPK*Zg0QK|bTid#b)EX8uMpEH-0 z4!9$wN8MeP@yzUD@$iiFZeUtu`_||GHiWsac;29HtB&#C~{;+%4i;^9!ts1aoIXPU)Uz z(DbjYZBX|?B+qGM|;YXi|j?cC;L zcc}GU%9n00XLC!nJ!g6*kiPPGW$ELUo00OxUX+pIRSk#V?IC$?RW+zmMk^ot=K|r- zHw(w+a>JF6zbRH)*uZjn4^=UTADWLQxU7yI_UH6=nbt~823gyujqr?syDVo0<>=== z-#8qaM}fcWXXYWfP3CY^=8p_XN=(e&dZW3H9|T8^C!S~@QKG4e{Q zEe41$0{FHY^V5b~9_7JF*T0B$xy*HnHL6k<6leFdRfBYYuG-2((L`?!tw z>?IFVJmZtKDNw8W=xI&aJ(4cS@cNdyi)$4ZI~!gr=<$jl9Yx(jifn@X@bF=RY4A^{ zmV~&|6s=WlbH7@h+N}j3vn;jF64P}Q4B|sy4RFfbQckUVTcU~0ZePFigq@}Ml+C&G zgnNfgU`VXwU`!X^;H0q?I^6(dSz(q=T1NzljB1y6biC0#)JIoV28(U>a=DMD7{(`xg^@ndR=8Li zCcQf=NxN5i2lHa#eB_C5X^N7d%gYOX)#%6*H1e_9)9*O+Y@kUQ`(R8D9&3>7NCDRq zIYfc)qAlnL_c`w?I$(L)Amaa$6ai9o^Fiz2-4!vYu}g)dYKjBIKT;LzfVOX~G(hLb z12jvsVs+pO#+#0vI25ir3STkbi72(#pGf^!s8!??PH6pviE?554YkYkxTF`_$CmbH zuiz7OU47`Iki%vpb|NQXE_CzcK<&Fa4zXcj$K&URYdJha*_~#nuh3R(H0lRFR&;3Ri>oe7nRxx<|Ago3WpCTe@P{*Cfgx^AbOm-7m2^gigh7 z<^`6rn^1}`9~1{BTacpK!5^dV54NyC#W~ zU@0G#a?9qdAf#{46P)WScaeC&CtvyW=?Wd7W*oX4ht~GipT!p0mq7+=;umfRft)5a zh)ke^USWdKjLmW%lfw604TfFnS8^cuCMv$00Rc=J9OUNsBSL8ttsnQkl&jEJweIje z!mUF@)~@K)$uw54po1#g#d~KW$8CU zGU`nK>cQM}r4#)$AQ=iEnA>uMq>Cm^2BM04T%gG-y!sN$tSq<0OjA~-`*{Siz(DY| z&VEKt?7^GegME*1A(l@%PQj?Bg4a#2uW+`_AAYdK=TkrwlXX67BU`DGAh@FvxOqND87WXK5gXwh91npy z$_=I}Pe0M7aL>$puzHM|Q(C3LGzFB4Z!+)l3pzQP)~#h*;t#Oz-X5*vnyw~pIi(C1 zRpdS8@s6h?OLu8mfOqunn1wmW$vO-a{E*(de8K9I$Bw$G#jJVH`}-Qn=klT#p-rv} z*?et%_g5BZM%$jN`bq(L7+1fMS!t3rZP@F0F4Gf*TF%jov{CLa#u+v0ZE9$r?$S1X z&=uZQ+$_O~r)bD8RmypB3f&Y=U0pdEZvjFZmw$2T!%pWsA(Y&2FAU0|GfrnxVZ}6bed$+5;%RahB=1>&9 zw?s-l46WTm9tyeBQ4X<@kmf^TX!EX)F!WK)wy9JxtJ^DLLUuQ&c~Uz|p{lfP+vZEQ zWjU3816Rf1g;Vj~o*q8s5ehd5{HPZR{Ie3dR5c>G_-)}+g)A+00uiHhcY8k;OgH3O&Gcxx>4M2ze8P_h($o5>eKZVsZnQsXA9wWUIH z!#|v9N@--$ZfDNbA1`blHf(4-UfCpB@R77oPi$HqAWe>OWA%vX;w;~PlZeQDXXy2% zL+LOI?iI0Qt%@&)kYNC)*!vRHmvw6RbX#AVij5g$1`W5kTzbpm=gp0~TBw=hJm zb-e?2e}sKPF89reJG^5YvG66wywpHqX?YT)%10zW2;K&z}+5G zeLxs8am-yfB=X(zrmr1(yk;|g!qTYOgSbuTY?A^AX@OM;#-SmR3pseU=3`xzWh<`> z3?&Kl)4*yn!EVxH`3pB!rsW{x#nsgsMf_Y^ub){&4$PVioY;3Fu|vMqCzj?Gns&zL zKWx7jRbEjzuoYq`O%9@vvG^kF?Qy z_a|)Uo(FgOC;XtIPIbUD=zJm{509&?6B0taF3&#BUSD5~o_smXwPU6g{oKitZL4CV zgWS77LLs+kC|7*Z6($^t26jfufe6;|{UX^xk3Z-2zOMn>Drh#opbk7NFQyb5tcT&o zw*Yr4FeTRH-fs>37v1;e(ZAfKBmTqliaPVWh6`Z7&nBPm*grjU?=EM)!8>pH$4~Kk z0l0Tj$I>|${FYxo>8=2M{VxV_l(*H}Wyx#D@{j{_kHi`JNn#r&^ZNfg58%B!5QfI# z_jowV?9mA>!Dx;h-4oBdm~o-#cm^&a19ra=Tc4o)sV3GikJb3-wHs7#fP?f!X)=*m z*nh2M zAl9QtDvl88fd81F*x3ZJWXk>ni@(9by*T*UTTg-LK|Foo8-oA9q9lOD{|@p$F!_Hg zx=ts? zrNE7+%Gv4JVfRgXr5Qa$yzak;)h&ZU+^bib1iqltx$OH3!?e#v7Q54UH4QVGNW{B!xd^$WJ=;ul?I znG0=~u?B7ur{wUYJB$CTl@aKGu|7NE&(}aV+01m}s#dZ)zW-{KCS3$>bKKQ^VV4sj zvV=7@HkOrqp~@%gfR{W44eA z^3>hls|SaNN>)}@ODN;tU?_Tv82uHV9SV%)bGOiRpdAM6m~xu*&eGFTa+%utyS+3m z1ugjqS6j6mYX6$`uZi*^u_A0}Ev!)_bltCYx8^(Lf)PYPG-J_gz)tCb%mm)~t1xj9 zUrW5I=Xk_-*v__AGV;LX*M&8!xe3O?nxCB`jSLsJhgAe|?=lt+zToN6rfuP{aY;9h zGoR>hNV_R6(^qn>X)uZ;`H7UAJ(WwMI>)?mv^(+GQ)j}PbdNQYhRtE>i&N6tH_uk- z3|`nP0Eo2OGCX2l5#7^YJ+JegFIu-g&~x*ZBquCvZ~RqZN((aQ%s}A2=6~jpN4_4n zI2i1~LibqTBZzOzWSdbeF->rqsJzEdM{6GiTlci-MV6YqTF7`?+7n(^B||-0$vgmc zU6gdKsfvO4|6a#QDFWOcNsZSKY-zGPrSDvSu)=B&P+1WPi>J!cH(&A79Du>SM?Knk zw0NrLz&+L!@ugv#=JMbsxx-OeC70iCNF%G8RK`L$fBN7YM$}z8@p8CY#Vaf^Y%9s9PL zgr@z$)8Y^$H-cF;)|k7!#U>L8uek>?*mvlQL0OS4Jc_c_KWIB5N_W$OO3rZ9xpnQB z%nvOsp`rWifd-9kd($cKjF;4@L{42NiTj}hCy$#;3E45ZM~T}%(-*vHcVo`y`knYP zM~@h9HC^{Omf3_xqRN6I#i>1NEL8puL{y{BKxsy8;(J7DZ~X|_srJFN6Au+EFS5K= zLvw@?M9$f@|2;0C_rnY3w62$_0S2l3{F^H``+`6MlnYdTv`qGLZOJaL8^5lV+Nqv~ zWNK5NuiYVkYLCjGmj`%Pr#Q?O*$0;&5XlAU$3gR^xnF*C_*Qt+u9zngd6xb19qrdCevt1Hk(4$hLu!G+f^Ijt++$c|{ z?HE9k!STC3bZkj0&(m1SSr z?FV1hG8tJb@11t7sks7aHe#AhqJg-Xbg-ZuSv&LVpPb7I162`UA^X;ra1!1WEFShF zO@4#T0gieE*nO#QFCcv<=!f?X|8(%})03mIW|!J?=7H4#vhTzII6MFR z-uik>Yu|i4WXZ@Z;w!;hHfh!Q!(icvD>H7jM-zc?qS;}DoAUuU>n?lR$*Ku0JJy=8 z@D_@J7rJ!B{+0C#xGzDl@hDoUfDLO~;r-{p$1Wxb8$s(7&oJN)w5;1k?MQFFCPG+n zXXp7Cz*Eh!G6BTYdQK;v!+du#ww-SbZZZFSAP+K`u7=qm%3NI^>vV9L9IjFGeHsrR zDM*VY*NzklGwP>RKXkOV{Aqf69I7K@(~7x;piVwijZIMK*phu;Ldr4d=CYK?J$}%k z3QgsnL*;bgH)_&_=zsFF?X zOBNh+f-2YiY*$h!C+wZ>fx_ZX7pp1z{&Fe;(|K$JNY;hj1~azO4I;@nCS_bA(`+*0 z)qftfl#0I#B9&=2NXNhgL#Y`fv|D$YHPI`(m=6_I;P&hY(=>f@z+McAh#C$ir#G z?^hA*?trZ?D}e$$zNMm!j$_bqTGCPKSpI7OrJ#wbaQxYmlJgr|410e3+6Fb~mJ?ekEY0zychoG@j?nXM*nn47Q^rNL9(%^c3QVe&i z%fkFv{!HZzbU~OG;}Bpt-i)^|gMDDU&@T89B2!DmC%P|YMw3$qNaYC$)IWh6s!)n>AxVdZ1M%R}E5JfMB=LHQkb(m;yROjs-=DMrjjMYXj z;dYdM}a@8K1`G^doUbvTfkoE%v7tT zLt6b^w%l$B^jfifdsxFxxwBv-iEtRSd<#RfEnNX;ncZNTIuxOn{Jm%>*6t@RyQ>GM z6K3dyZwE84lB%i%cDA>vcp&JgzW7Q@WO_?Jc8u8E1kts}^?c+RR*w$lozqSOr?-dfoLi!noR+rJ)2kHVKdW?GOG0A@ zbgZoOq3N4J$RPTKCObm68@6$@3(vExpk%_W^Tv>>vf0j7{;f~jQR#^RTQ@SAN>U8? zoXka3-a0vs^-dGrDP;t|LbV=7ghmdJ7rkO>&W>Gpk#+8Boo=7#*j= zArAo>yW6pztUus(L6UsbU2FY!{&)hf-p4DrAG>X7)KG4;gCmq0cv*< zN_^ZN3tnTk=!+d-Yn*uw|F?o`Tb8b#N6P#D>|loA`|vEki&OUj=!(c=CeJ`eS66p( zs7^{%>JM&346z?AT9yw{z~=o>_qau%3(U z*VBT|O~PNT!$1ea4a>~uq<%(n?EuMz|0PX*fnPJvIcKSVMxG#EYoJeh5&J>f8Hvp8slX2keXvG8N%}q44!{Kn(mBvHwNv|9gr( aqW!Rg*VdkRkK_{Y_d-HmJYQ7z!~X+RoGsn} literal 0 HcmV?d00001 From dc79a4bf8c374756ab5babddaeced3aabf0f761c Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Wed, 28 Jan 2026 10:34:00 -0800 Subject: [PATCH 10/45] Jaysahnan/gro 926 mfa template with playwright (#48) * init commit * changed to toml * changes to readme * fixed NITS --- python/playwright-mfa-handling/.env.example | 3 + python/playwright-mfa-handling/README.md | 126 +++++++++ python/playwright-mfa-handling/main.py | 250 ++++++++++++++++++ python/playwright-mfa-handling/pyproject.toml | 33 +++ .../playwright-mfa-handling/.env.example | 3 + typescript/playwright-mfa-handling/README.md | 111 ++++++++ typescript/playwright-mfa-handling/index.ts | 243 +++++++++++++++++ .../playwright-mfa-handling/package.json | 31 +++ 8 files changed, 800 insertions(+) create mode 100644 python/playwright-mfa-handling/.env.example create mode 100644 python/playwright-mfa-handling/README.md create mode 100644 python/playwright-mfa-handling/main.py create mode 100644 python/playwright-mfa-handling/pyproject.toml create mode 100644 typescript/playwright-mfa-handling/.env.example create mode 100644 typescript/playwright-mfa-handling/README.md create mode 100644 typescript/playwright-mfa-handling/index.ts create mode 100644 typescript/playwright-mfa-handling/package.json diff --git a/python/playwright-mfa-handling/.env.example b/python/playwright-mfa-handling/.env.example new file mode 100644 index 00000000..36c24713 --- /dev/null +++ b/python/playwright-mfa-handling/.env.example @@ -0,0 +1,3 @@ +# Browserbase Configuration +BROWSERBASE_PROJECT_ID=your_browserbase_project_id +BROWSERBASE_API_KEY=your_browserbase_api_key diff --git a/python/playwright-mfa-handling/README.md b/python/playwright-mfa-handling/README.md new file mode 100644 index 00000000..7980e10e --- /dev/null +++ b/python/playwright-mfa-handling/README.md @@ -0,0 +1,126 @@ +# Playwright + Browserbase: MFA Handling - TOTP Automation + +## AT A GLANCE + +- Goal: Automate MFA (Multi-Factor Authentication) completion using TOTP (Time-based One-Time Password) code generation with Playwright. +- Pure Playwright: Uses Playwright directly with Browserbase's cloud browser infrastructure (no Stagehand abstraction). +- TOTP Generation: Implements RFC 6238 compliant algorithm to generate time-based authentication codes programmatically. +- Automatic Form Filling: Uses Playwright selectors to fill MFA forms without user interaction. +- Retry Logic: Handles time window edge cases by regenerating codes and retrying authentication when needed. +- Docs -> https://playwright.dev/python/docs/intro + +## GLOSSARY + +- Playwright: Microsoft's browser automation library for reliable end-to-end testing + Docs -> https://playwright.dev/python/docs/intro +- CDP: Chrome DevTools Protocol - used to connect Playwright to Browserbase cloud browsers +- TOTP: Time-based One-Time Password - a 6-digit code that changes every 30 seconds, generated using HMAC-SHA1 algorithm +- RFC 6238: Standard specification for TOTP authentication codes used by Google Authenticator, Authy, and other authenticator apps + +## STAGEHAND VS PLAYWRIGHT + +This template uses **pure Playwright** for browser automation. The Stagehand v3 Python SDK uses a session-based API with **observe** (find actions) and **act** (execute an action) instead. Here's how they compare: + +| Task | Stagehand v3 — natural language (you describe intent) | Playwright — specific selectors (you target exact elements) | +| ------------- | ------------------------------------------------------- | ------------------------------------------------------------- | +| Fill email | *"Find the email field and type the user's email"* | `page.locator('input[type="email"]').fill(email)` | +| Fill password | *"Find the password field and enter the password"* | `page.locator('input[type="password"]').fill(password)` | +| Click submit | *"Click the submit or sign-in button"* | `page.locator('input[type="submit"]').click()` | +| Check result | *"Did login succeed or fail? Return success and message"* | `page.locator('text="Login Success"').is_visible()` | + +**Example - Filling the login form:** + +```python +# Stagehand v3: session-based; observe finds actions, act executes one +from stagehand import AsyncStagehand + +client = AsyncStagehand() +session = await client.sessions.create(model_name="openai/gpt-5-nano") +await session.navigate(url="https://example.com/login") + +observe_resp = await session.observe(instruction="find the email input and fill it") +action = observe_resp.data.result[0].to_dict(exclude_none=True) +await session.act(input=action) +# repeat observe/act for password and TOTP field + +# Playwright: Explicit selectors, you specify how to find elements +await page.locator('input[type="email"]').fill(email) +await page.locator('input[type="password"]').fill(password) +await page.locator("form input").nth(2).fill(totp_code) +``` + +**Example - Checking authentication result:** + +```python +# Stagehand v3: extract returns structured data via response.data.result +extract_response = await session.extract( + instruction="Check if the login was successful or if there's an error message", + schema={ + "type": "object", + "properties": {"success": {"type": "boolean"}, "message": {"type": "string"}}, + "required": ["success"], + }, +) +result = extract_response.data.result + +# Playwright: Must check for specific elements/text on the page +has_success = await page.locator('text="Login Success"').is_visible() +has_failure = await page.locator('text="Login Failure"').is_visible() +``` + +## QUICKSTART + +1. cd playwright-mfa-handling +2. python -m venv venv && source venv/bin/activate +3. pip install . +4. playwright install chromium +5. cp .env.example .env +6. Add required API keys/IDs to .env (BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY) +7. python main.py + +## EXPECTED OUTPUT + +- Creates Browserbase session and connects via CDP +- Displays live session link for monitoring +- Navigates to TOTP challenge demo page (authenticationtest.com/totpChallenge/) +- Generates TOTP code using RFC 6238 algorithm +- Fills in email, password, and TOTP code fields using Playwright selectors +- Submits authentication form +- Checks authentication result +- Retries with fresh code if initial attempt fails (handles time window edge cases) +- Closes browser session cleanly + +## COMMON PITFALLS + +- Dependency install errors: ensure pip install . completed +- Missing Playwright browsers: run `playwright install chromium` after installing dependencies +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- TOTP code expiration: codes are valid for 30 seconds - if authentication fails, the script automatically retries with a fresh code +- CDP connection issues: ensure stable internet connection for reliable Browserbase connection +- Selector changes: if the demo site structure changes, Playwright selectors may need updating +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +- Automated authentication: Complete MFA challenges automatically when session persistence isn't enough +- TOTP integration: Store encrypted TOTP secrets during user onboarding and generate codes programmatically +- Zero-touch MFA: Eliminate user interaction for MFA completion in automated workflows +- Session recovery: Automatically handle MFA prompts when re-authenticating expired sessions +- E2E testing: Test MFA flows in your application's test suite + +## NEXT STEPS + +- Secure storage: Implement encrypted TOTP secret storage (AES-256) in your database +- Multiple time windows: Add support for trying +/-1 time window (60s range) if current code fails +- SMS/Email MFA: Extend to support SMS codes (via Twilio/Bandwidth API) or email codes (via Gmail API/IMAP) +- Backup codes: Implement fallback to backup codes stored during initial MFA setup +- Context integration: Combine with Browserbase Contexts to minimize MFA prompts +- Page Object Model: Refactor selectors into reusable page objects for maintainability + +## HELPFUL RESOURCES + +- Playwright Python Docs: https://playwright.dev/python/docs/intro +- Browserbase: https://www.browserbase.com +- Try it out: https://www.browserbase.com/playground +- Templates: https://www.browserbase.com/templates +- Need help? support@browserbase.com diff --git a/python/playwright-mfa-handling/main.py b/python/playwright-mfa-handling/main.py new file mode 100644 index 00000000..761802b0 --- /dev/null +++ b/python/playwright-mfa-handling/main.py @@ -0,0 +1,250 @@ +# Playwright + Browserbase: MFA Handling - TOTP Automation - See README.md for full documentation + +import asyncio +import hashlib +import hmac +import os +import time + +from browserbase import Browserbase +from dotenv import load_dotenv +from playwright.async_api import Browser, BrowserContext, Page, async_playwright + +# Load environment variables +load_dotenv() + +# Demo site URL for TOTP challenge testing +DEMO_URL = "https://authenticationtest.com/totpChallenge/" + +# Test credentials (displayed on the demo page) +TEST_CREDENTIALS = { + "email": "totp@authenticationtest.com", + "password": "pa$$w0rd", + "totp_secret": "I65VU7K5ZQL7WB4E", +} + + +def generate_totp(secret: str, window: int = 0) -> str: + """ + Generate TOTP code (Time-based One-Time Password) using RFC 6238 compliant algorithm. + + Same algorithm used by Google Authenticator, Authy, and other authenticator apps. + """ + # Convert base32 secret to bytes + base32chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" + bits = "" + hex_str = "" + + secret = secret.upper().rstrip("=") + + for char in secret: + val = base32chars.find(char) + if val == -1: + raise ValueError("Invalid base32 character in secret") + bits += format(val, "05b") + + for i in range(0, len(bits) - 3, 4): + chunk = bits[i : i + 4] + hex_str += format(int(chunk, 2), "x") + + secret_bytes = bytes.fromhex(hex_str) + + # Get current time window (30 second intervals) + time_window = int(time.time() // 30) + window + time_bytes = time_window.to_bytes(8, byteorder="big") + + # Generate HMAC-SHA1 hash + hmac_result = hmac.new(secret_bytes, time_bytes, hashlib.sha1).digest() + + # Dynamic truncation to extract 6-digit code + offset = hmac_result[-1] & 0xF + code = ( + ((hmac_result[offset] & 0x7F) << 24) + | ((hmac_result[offset + 1] & 0xFF) << 16) + | ((hmac_result[offset + 2] & 0xFF) << 8) + | (hmac_result[offset + 3] & 0xFF) + ) + + # Return 6-digit code with leading zeros + return str(code % 1000000).zfill(6) + + +async def create_browserbase_session() -> tuple[Browser, BrowserContext, Page, str]: + """Create a Browserbase session and connect via CDP.""" + api_key = os.environ.get("BROWSERBASE_API_KEY") + project_id = os.environ.get("BROWSERBASE_PROJECT_ID") + + if not api_key: + raise ValueError("BROWSERBASE_API_KEY environment variable is required") + if not project_id: + raise ValueError("BROWSERBASE_PROJECT_ID environment variable is required") + + bb = Browserbase(api_key=api_key) + session = bb.sessions.create(project_id=project_id) + + print(f"Session created: https://browserbase.com/sessions/{session.id}") + + pw = await async_playwright().start() + browser = await pw.chromium.connect_over_cdp(session.connect_url) + + context = browser.contexts[0] + if not context: + raise ValueError("No default context found") + + page = context.pages[0] + if not page: + raise ValueError("No page found in default context") + + return browser, context, page, session.id + + +async def fill_login_form(page: Page, email: str, password: str, totp_code: str) -> None: + """Fill in the login form with credentials and TOTP code.""" + # Fill email field + email_input = page.locator('input[type="email"], input[placeholder*="email" i]').first + await email_input.wait_for(state="visible", timeout=10000) + await email_input.fill(email) + print("Filled email field") + + # Fill password field + password_input = page.locator('input[type="password"]') + await password_input.fill(password) + print("Filled password field") + + # Fill TOTP code field (third input in the form) + totp_input = page.locator("form input").nth(2) + await totp_input.fill(totp_code) + print("Filled TOTP code field") + + +async def submit_form(page: Page) -> None: + """Submit the login form.""" + submit_button = page.locator('input[type="submit"], button[type="submit"], form button').first + await submit_button.click() + print("Clicked submit button") + + await page.wait_for_load_state("domcontentloaded", timeout=20000) + + +async def check_login_result(page: Page) -> dict: + """Check if the login was successful.""" + # Check for failure indicator + failure_heading = page.locator('text="Login Failure"') + has_failure = await failure_heading.is_visible() + + if has_failure: + error_message = await page.locator("body").text_content() or "" + message = ( + "Sorry -- You have not successfully logged in" + if "not successfully logged in" in error_message + else "Login failed" + ) + return {"success": False, "message": message} + + # Check for success indicator + success_heading = page.locator('text="Login Success"') + has_success = await success_heading.is_visible() + + if has_success: + return {"success": True, "message": "Successfully logged in with TOTP"} + + # Fallback: check page content + page_content = await page.locator("body").text_content() or "" + if "success" in page_content.lower(): + return {"success": True, "message": "Login appears successful"} + + return {"success": False, "message": "Unable to determine login result"} + + +async def main(): + print("Starting MFA Handling - TOTP Automation (Playwright + Browserbase)...\n") + + browser = None + + try: + # Create browser session + browser, context, page, session_id = await create_browserbase_session() + + print(f"Live View: https://browserbase.com/sessions/{session_id}\n") + + # Navigate to TOTP challenge demo page + print("Navigating to TOTP Challenge page...") + await page.goto(DEMO_URL, wait_until="domcontentloaded") + + # Generate TOTP code using RFC 6238 algorithm + totp_code = generate_totp(TEST_CREDENTIALS["totp_secret"]) + seconds_left = 30 - (int(time.time()) % 30) + print(f"Generated TOTP code: {totp_code} (valid for {seconds_left}s)") + + # Fill in login form + print("\nFilling login form...") + await fill_login_form( + page, + TEST_CREDENTIALS["email"], + TEST_CREDENTIALS["password"], + totp_code, + ) + + # Submit the form + print("\nSubmitting form...") + await submit_form(page) + + # Check if login succeeded + print("\nChecking authentication result...") + result = await check_login_result(page) + + if result["success"]: + print("\nSUCCESS! TOTP authentication completed automatically!") + print(f"Result: {result['message']}") + else: + print(f"\nInitial attempt may have failed: {result['message']}") + print("Retrying with a fresh TOTP code...\n") + + # Navigate back and retry with new code + await page.goto(DEMO_URL, wait_until="domcontentloaded") + + new_code = generate_totp(TEST_CREDENTIALS["totp_secret"]) + print(f"New TOTP code: {new_code}") + + await fill_login_form( + page, + TEST_CREDENTIALS["email"], + TEST_CREDENTIALS["password"], + new_code, + ) + await submit_form(page) + + retry_result = await check_login_result(page) + + if retry_result["success"]: + print("\nSuccess on retry!") + print(f"Result: {retry_result['message']}") + else: + print("\nAuthentication failed after retry") + print(f"Result: {retry_result['message']}") + + print("\nMFA handling completed") + + except Exception as error: + print(f"Error during MFA handling: {error}") + import traceback + + traceback.print_exc() + raise + + finally: + if browser: + await browser.close() + print("Browser closed successfully") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"\nError in MFA handling: {err}") + print("\nCommon issues:") + print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") + print(" - TOTP code may have expired (try running again)") + print(" - Page structure may have changed") + exit(1) diff --git a/python/playwright-mfa-handling/pyproject.toml b/python/playwright-mfa-handling/pyproject.toml new file mode 100644 index 00000000..ae919862 --- /dev/null +++ b/python/playwright-mfa-handling/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "playwright-mfa-handling" +version = "0.1.0" +description = "Playwright + Browserbase: MFA Handling - TOTP Automation" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "playwright", + "browserbase", + "python-dotenv", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/typescript/playwright-mfa-handling/.env.example b/typescript/playwright-mfa-handling/.env.example new file mode 100644 index 00000000..36c24713 --- /dev/null +++ b/typescript/playwright-mfa-handling/.env.example @@ -0,0 +1,3 @@ +# Browserbase Configuration +BROWSERBASE_PROJECT_ID=your_browserbase_project_id +BROWSERBASE_API_KEY=your_browserbase_api_key diff --git a/typescript/playwright-mfa-handling/README.md b/typescript/playwright-mfa-handling/README.md new file mode 100644 index 00000000..2b4cfd8e --- /dev/null +++ b/typescript/playwright-mfa-handling/README.md @@ -0,0 +1,111 @@ +# Playwright + Browserbase: MFA Handling - TOTP Automation + +## AT A GLANCE + +- Goal: Automate MFA (Multi-Factor Authentication) completion using TOTP (Time-based One-Time Password) code generation with Playwright. +- Pure Playwright: Uses Playwright directly with Browserbase's cloud browser infrastructure (no Stagehand abstraction). +- TOTP Generation: Implements RFC 6238 compliant algorithm to generate time-based authentication codes programmatically. +- Automatic Form Filling: Uses Playwright selectors to fill MFA forms without user interaction. +- Retry Logic: Handles time window edge cases by regenerating codes and retrying authentication when needed. +- Docs -> https://playwright.dev/docs/intro + +## GLOSSARY + +- Playwright: Microsoft's browser automation library for reliable end-to-end testing + Docs -> https://playwright.dev/docs/intro +- CDP: Chrome DevTools Protocol - used to connect Playwright to Browserbase cloud browsers +- TOTP: Time-based One-Time Password - a 6-digit code that changes every 30 seconds, generated using HMAC-SHA1 algorithm +- RFC 6238: Standard specification for TOTP authentication codes used by Google Authenticator, Authy, and other authenticator apps + +## STAGEHAND VS PLAYWRIGHT + +This template uses **pure Playwright** for browser automation. The Stagehand version of this template uses AI-powered natural language commands instead. Here's how they compare: + +| Task | Stagehand (AI) | Playwright (Selectors) | +| ------------- | --------------------------------------------------------- | ------------------------------------------------------------- | +| Fill email | `await page.act("Type email into the email field")` | `await page.locator('input[type="email"]').fill(email)` | +| Fill password | `await page.act("Type password into the password field")` | `await page.locator('input[type="password"]').fill(password)` | +| Click submit | `await page.act("Click the submit button")` | `await page.locator('input[type="submit"]').click()` | +| Check result | AI interprets page content | Explicit selectors required | + +**Example - Filling the login form:** + +```typescript +// Stagehand: Natural language, AI finds the elements +await page.act(`Type '${email}' into the email field`); +await page.act(`Type '${password}' into the password field`); +await page.act(`Type '${totpCode}' into the TOTP code field`); + +// Playwright: Explicit selectors, you specify how to find elements +await page.locator('input[type="email"]').fill(email); +await page.locator('input[type="password"]').fill(password); +await page.locator("form input").nth(2).fill(totpCode); +``` + +**Example - Checking authentication result:** + +```typescript +// Stagehand: AI understands context and extracts structured data +const result = await page.extract({ + instruction: "Check if the login was successful or if there's an error message", + schema: AuthResultSchema, +}); + +// Playwright: Must check for specific elements/text on the page +const hasSuccess = await page.locator('text="Login Success"').isVisible(); +const hasFailure = await page.locator('text="Login Failure"').isVisible(); +``` + +## QUICKSTART + +1. cd playwright-mfa-handling +2. pnpm install +3. cp .env.example .env +4. Add required API keys/IDs to .env (BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY) +5. pnpm start + +## EXPECTED OUTPUT + +- Creates Browserbase session and connects via CDP +- Displays live session link for monitoring +- Navigates to TOTP challenge demo page (authenticationtest.com/totpChallenge/) +- Generates TOTP code using RFC 6238 algorithm +- Fills in email, password, and TOTP code fields using Playwright selectors +- Submits authentication form +- Checks authentication result +- Retries with fresh code if initial attempt fails (handles time window edge cases) +- Closes browser session cleanly + +## COMMON PITFALLS + +- Dependency install errors: ensure pnpm install completed +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- TOTP code expiration: codes are valid for 30 seconds - if authentication fails, the script automatically retries with a fresh code +- CDP connection issues: ensure stable internet connection for reliable Browserbase connection +- Selector changes: if the demo site structure changes, Playwright selectors may need updating +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +- Automated authentication: Complete MFA challenges automatically when session persistence isn't enough +- TOTP integration: Store encrypted TOTP secrets during user onboarding and generate codes programmatically +- Zero-touch MFA: Eliminate user interaction for MFA completion in automated workflows +- Session recovery: Automatically handle MFA prompts when re-authenticating expired sessions +- E2E testing: Test MFA flows in your application's test suite + +## NEXT STEPS + +- Secure storage: Implement encrypted TOTP secret storage (AES-256) in your database +- Multiple time windows: Add support for trying +/-1 time window (60s range) if current code fails +- SMS/Email MFA: Extend to support SMS codes (via Twilio/Bandwidth API) or email codes (via Gmail API/IMAP) +- Backup codes: Implement fallback to backup codes stored during initial MFA setup +- Context integration: Combine with Browserbase Contexts to minimize MFA prompts +- Page Object Model: Refactor selectors into reusable page objects for maintainability + +## HELPFUL RESOURCES + +- Playwright Docs: https://playwright.dev/docs/intro +- Browserbase: https://www.browserbase.com +- Try it out: https://www.browserbase.com/playground +- Templates: https://www.browserbase.com/templates +- Need help? support@browserbase.com diff --git a/typescript/playwright-mfa-handling/index.ts b/typescript/playwright-mfa-handling/index.ts new file mode 100644 index 00000000..a730c53d --- /dev/null +++ b/typescript/playwright-mfa-handling/index.ts @@ -0,0 +1,243 @@ +// Playwright + Browserbase: MFA Handling - TOTP Automation +// See README.md for full documentation + +import "dotenv/config"; +import { chromium, Browser, BrowserContext, Page } from "playwright-core"; +import Browserbase from "@browserbasehq/sdk"; +import crypto from "crypto"; + +// Demo site URL for TOTP challenge testing +const DEMO_URL = "https://authenticationtest.com/totpChallenge/"; + +// Test credentials (displayed on the demo page) +const TEST_CREDENTIALS = { + email: "totp@authenticationtest.com", + password: "pa$$w0rd", + totpSecret: "I65VU7K5ZQL7WB4E", +}; + +// Type for browser session result +interface BrowserSession { + browser: Browser; + context: BrowserContext; + page: Page; + sessionId: string; +} + +// Generate TOTP code (Time-based One-Time Password) using RFC 6238 compliant algorithm +// Same algorithm used by Google Authenticator, Authy, and other authenticator apps +function generateTOTP(secret: string, window = 0): string { + // Convert base32 secret to buffer + const base32chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + let bits = ""; + let hex = ""; + + secret = secret.toUpperCase().replace(/=+$/, ""); + + for (let i = 0; i < secret.length; i++) { + const val = base32chars.indexOf(secret.charAt(i)); + if (val === -1) throw new Error("Invalid base32 character in secret"); + bits += val.toString(2).padStart(5, "0"); + } + + for (let i = 0; i + 4 <= bits.length; i += 4) { + const chunk = bits.slice(i, i + 4); + hex += parseInt(chunk, 2).toString(16); + } + + const secretBuffer = Buffer.from(hex, "hex"); + + // Get current time window (30 second intervals) + const time = Math.floor(Date.now() / 1000 / 30) + window; + const timeBuffer = Buffer.alloc(8); + timeBuffer.writeBigInt64BE(BigInt(time)); + + // Generate HMAC-SHA1 hash + const hmac = crypto.createHmac("sha1", secretBuffer); + hmac.update(timeBuffer); + const hmacResult = hmac.digest(); + + // Dynamic truncation to extract 6-digit code + const offset = hmacResult[hmacResult.length - 1] & 0xf; + const code = + ((hmacResult[offset] & 0x7f) << 24) | + ((hmacResult[offset + 1] & 0xff) << 16) | + ((hmacResult[offset + 2] & 0xff) << 8) | + (hmacResult[offset + 3] & 0xff); + + // Return 6-digit code with leading zeros + return (code % 1000000).toString().padStart(6, "0"); +} + +async function createBrowserbaseSession(): Promise { + if (!process.env.BROWSERBASE_API_KEY) { + throw new Error("BROWSERBASE_API_KEY environment variable is required"); + } + if (!process.env.BROWSERBASE_PROJECT_ID) { + throw new Error("BROWSERBASE_PROJECT_ID environment variable is required"); + } + + const bb = new Browserbase({ apiKey: process.env.BROWSERBASE_API_KEY }); + + const session = await bb.sessions.create({ + projectId: process.env.BROWSERBASE_PROJECT_ID, + }); + + console.log(`Session created: https://browserbase.com/sessions/${session.id}`); + + const browser = await chromium.connectOverCDP(session.connectUrl); + const context = browser.contexts()[0]; + if (!context) { + throw new Error("No default context found"); + } + const page = context.pages()[0]; + if (!page) { + throw new Error("No page found in default context"); + } + + return { browser, context, page, sessionId: session.id }; +} + +async function fillLoginForm( + page: Page, + email: string, + password: string, + totpCode: string, +): Promise { + // Fill email field + const emailInput = page.locator('input[type="email"], input[placeholder*="email" i]').first(); + await emailInput.waitFor({ state: "visible", timeout: 10000 }); + await emailInput.fill(email); + console.log("Filled email field"); + + // Fill password field + const passwordInput = page.locator('input[type="password"]'); + await passwordInput.fill(password); + console.log("Filled password field"); + + // Fill TOTP code field (third input in the form) + const totpInput = page.locator("form input").nth(2); + await totpInput.fill(totpCode); + console.log("Filled TOTP code field"); +} + +async function submitForm(page: Page): Promise { + const submitButton = page + .locator('input[type="submit"], button[type="submit"], form button') + .first(); + await submitButton.click(); + console.log("Clicked submit button"); + + await page.waitForLoadState("domcontentloaded", { timeout: 15000 }); +} + +async function checkLoginResult(page: Page): Promise<{ success: boolean; message: string }> { + // Check for failure indicator + const failureHeading = page.locator('text="Login Failure"'); + const hasFailure = await failureHeading.isVisible().catch(() => false); + + if (hasFailure) { + const errorMessage = (await page.locator("body").textContent()) ?? ""; + const message = errorMessage.includes("not successfully logged in") + ? "Sorry -- You have not successfully logged in" + : "Login failed"; + return { success: false, message }; + } + + // Check for success indicator + const successHeading = page.locator('text="Login Success"'); + const hasSuccess = await successHeading.isVisible().catch(() => false); + + if (hasSuccess) { + return { success: true, message: "Successfully logged in with TOTP" }; + } + + // Fallback: check page content + const pageContent = (await page.locator("body").textContent()) ?? ""; + if (pageContent.toLowerCase().includes("success")) { + return { success: true, message: "Login appears successful" }; + } + + return { success: false, message: "Unable to determine login result" }; +} + +async function main() { + console.log("Starting MFA Handling - TOTP Automation (Playwright + Browserbase)...\n"); + + let browser: Browser | null = null; + + try { + // Create browser session + const { browser: b, page, sessionId } = await createBrowserbaseSession(); + browser = b; + + console.log(`Live View: https://browserbase.com/sessions/${sessionId}\n`); + + // Navigate to TOTP challenge demo page + console.log("Navigating to TOTP Challenge page..."); + await page.goto(DEMO_URL, { waitUntil: "domcontentloaded" }); + + // Generate TOTP code using RFC 6238 algorithm + const totpCode = generateTOTP(TEST_CREDENTIALS.totpSecret); + const secondsLeft = 30 - (Math.floor(Date.now() / 1000) % 30); + console.log(`Generated TOTP code: ${totpCode} (valid for ${secondsLeft}s)`); + + // Fill in login form + console.log("\nFilling login form..."); + await fillLoginForm(page, TEST_CREDENTIALS.email, TEST_CREDENTIALS.password, totpCode); + + // Submit the form + console.log("\nSubmitting form..."); + await submitForm(page); + + // Check if login succeeded + console.log("\nChecking authentication result..."); + const result = await checkLoginResult(page); + + if (result.success) { + console.log("\nSUCCESS! TOTP authentication completed automatically!"); + console.log(`Result: ${result.message}`); + } else { + console.log(`\nInitial attempt may have failed: ${result.message}`); + console.log("Retrying with a fresh TOTP code...\n"); + + // Navigate back and retry with new code + await page.goto(DEMO_URL, { waitUntil: "domcontentloaded" }); + + const newCode = generateTOTP(TEST_CREDENTIALS.totpSecret); + console.log(`New TOTP code: ${newCode}`); + + await fillLoginForm(page, TEST_CREDENTIALS.email, TEST_CREDENTIALS.password, newCode); + await submitForm(page); + + const retryResult = await checkLoginResult(page); + + if (retryResult.success) { + console.log("\nSuccess on retry!"); + console.log(`Result: ${retryResult.message}`); + } else { + console.log("\nAuthentication failed after retry"); + console.log(`Result: ${retryResult.message}`); + } + } + + console.log("\nMFA handling completed"); + } catch (error) { + console.error("Error during MFA handling:", error); + throw error; + } finally { + if (browser) { + await browser.close(); + console.log("Browser closed successfully"); + } + } +} + +main().catch((err) => { + console.error("\nError in MFA handling:", err); + console.error("\nCommon issues:"); + console.error(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY"); + console.error(" - TOTP code may have expired (try running again)"); + console.error(" - Page structure may have changed"); + process.exit(1); +}); diff --git a/typescript/playwright-mfa-handling/package.json b/typescript/playwright-mfa-handling/package.json new file mode 100644 index 00000000..17d5a8c4 --- /dev/null +++ b/typescript/playwright-mfa-handling/package.json @@ -0,0 +1,31 @@ +{ + "name": "playwright-mfa-handling-template", + "version": "1.0.0", + "description": "Playwright + Browserbase: MFA Handling - TOTP Automation", + "type": "module", + "main": "index.ts", + "scripts": { + "start": "tsx index.ts", + "dev": "tsx watch index.ts" + }, + "keywords": [ + "playwright", + "browserbase", + "automation", + "mfa", + "totp", + "2fa" + ], + "author": "", + "license": "MIT", + "dependencies": { + "@browserbasehq/sdk": "^2.0.0", + "dotenv": "^16.0.0", + "playwright-core": "^1.40.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "tsx": "^4.7.0", + "typescript": "^5.3.0" + } +} From 3ddbece89b5a21a813b08166bf687328b1343dea Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Wed, 28 Jan 2026 10:34:51 -0800 Subject: [PATCH 11/45] init commit (#45) --- eslint.config.js | 1 - python/mfa-handling/README.md | 1 - .../playwright/basic-recaptcha/.env.example | 3 + python/playwright/basic-recaptcha/README.md | 128 +++++++++++++++ python/playwright/basic-recaptcha/main.py | 153 ++++++++++++++++++ .../playwright/basic-recaptcha/pyproject.toml | 33 ++++ python/proxies-weather/README.md | 1 - typescript/basic-recaptcha/README.md | 7 + typescript/company-address-finder/index.ts | 8 +- typescript/dynamic-form-filling/README.md | 1 - typescript/gemini-3-flash/README.md | 26 +-- typescript/gemini-3-flash/index.ts | 24 +-- typescript/pickleball/index.ts | 19 +-- .../playwright/basic-recaptcha/.env.example | 3 + .../playwright/basic-recaptcha/README.md | 107 ++++++++++++ .../playwright/basic-recaptcha/index.ts | 152 +++++++++++++++++ .../playwright/basic-recaptcha/package.json | 17 ++ typescript/proxies-weather/index.ts | 4 +- 18 files changed, 650 insertions(+), 38 deletions(-) create mode 100644 python/playwright/basic-recaptcha/.env.example create mode 100644 python/playwright/basic-recaptcha/README.md create mode 100644 python/playwright/basic-recaptcha/main.py create mode 100644 python/playwright/basic-recaptcha/pyproject.toml create mode 100644 typescript/playwright/basic-recaptcha/.env.example create mode 100644 typescript/playwright/basic-recaptcha/README.md create mode 100644 typescript/playwright/basic-recaptcha/index.ts create mode 100644 typescript/playwright/basic-recaptcha/package.json diff --git a/eslint.config.js b/eslint.config.js index 1670f4ac..fed3ab63 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -27,4 +27,3 @@ export default tseslint.config( ignores: ["node_modules/**", "dist/**", "build/**", "*.config.js"], }, ); - diff --git a/python/mfa-handling/README.md b/python/mfa-handling/README.md index a77699ff..91ceac11 100644 --- a/python/mfa-handling/README.md +++ b/python/mfa-handling/README.md @@ -75,4 +75,3 @@ 🔧 Templates: https://www.browserbase.com/templates 📧 Need help? support@browserbase.com 💬 Discord: http://stagehand.dev/discord - diff --git a/python/playwright/basic-recaptcha/.env.example b/python/playwright/basic-recaptcha/.env.example new file mode 100644 index 00000000..dfaaf699 --- /dev/null +++ b/python/playwright/basic-recaptcha/.env.example @@ -0,0 +1,3 @@ +# Browserbase credentials - Get these from https://www.browserbase.com/overview +BROWSERBASE_API_KEY=your_browserbase_api_key_here +BROWSERBASE_PROJECT_ID=your_browserbase_project_id_here diff --git a/python/playwright/basic-recaptcha/README.md b/python/playwright/basic-recaptcha/README.md new file mode 100644 index 00000000..51f708be --- /dev/null +++ b/python/playwright/basic-recaptcha/README.md @@ -0,0 +1,128 @@ +# Playwright + Browserbase: Basic reCAPTCHA Solving (Python) + +## AT A GLANCE + +- Goal: Demonstrate automatic reCAPTCHA solving using Playwright with Browserbase's built-in captcha solving capabilities. +- Automated Solving: Browserbase automatically detects and solves CAPTCHAs in the background. CAPTCHA solving is **enabled by default** - you don't need to set `solveCaptchas: True` unless you want to explicitly enable it (or set it to `False` to disable). +- Timeout Handling: Includes 60-second timeout to prevent indefinite waits on captcha solving failures. +- Progress Monitoring: Listen for console messages (`browserbase-solving-started`, `browserbase-solving-finished`) to track captcha solving progress in real-time. +- Docs -> https://docs.browserbase.com/features/stealth-mode#captcha-solving + +## GLOSSARY + +- Browserbase SDK: Python SDK for creating and managing browser sessions in Browserbase's cloud infrastructure. + Docs -> https://docs.browserbase.com/ +- solveCaptchas: Browserbase browser setting that enables automatic captcha solving for reCAPTCHA, CAPTCHA, and other captcha types. Enabled by default for Basic and Advanced Stealth Mode. + Docs -> https://docs.browserbase.com/features/stealth-mode#captcha-solving +- console messages: Browser console events that indicate captcha solving status: + - `browserbase-solving-started`: emitted when CAPTCHA detection begins + - `browserbase-solving-finished`: emitted when CAPTCHA solving completes +- custom CAPTCHA solving: For non-standard or custom captcha providers, you can specify CSS selectors for the captcha image and input field using `captchaImageSelector` and `captchaInputSelector` in browser_settings. + +## PLAYWRIGHT VS STAGEHAND + +| Feature | Playwright (this template) | Stagehand | +| --------------- | ----------------------------- | ------------------------------------ | +| Actions | `page.click()`, `page.goto()` | `stagehand.act("Click the button")` | +| Data Extraction | Manual DOM queries | `stagehand.extract("Get the price")` | + +## CAPTCHA SOLVING DETAILS + +### How CAPTCHA Solving Works + +Browserbase provides integrated CAPTCHA solving to handle challenges automatically: + +- **Automatic Detection**: When a CAPTCHA is detected on a page, Browserbase attempts to solve it in the background +- **Solving Time**: CAPTCHA solving typically takes between 5-30 seconds, depending on the CAPTCHA type and complexity +- **Default Behavior**: CAPTCHA solving is enabled by default for Basic and Advanced Stealth Mode +- **Proxies**: It's recommended to enable proxies when using CAPTCHA solving for higher success rates + +### Custom CAPTCHA Solving + +For non-standard or custom captcha providers, you can specify CSS selectors to guide the solution process: + +```python +session = bb.sessions.create( + project_id=project_id, + browser_settings={ + "solveCaptchas": True, + "captchaImageSelector": "#custom-captcha-image-id", + "captchaInputSelector": "#custom-captcha-input-id" + }, +) +``` + +To find the selectors: + +1. Right-click on the captcha image and select "Inspect" to get the image selector +2. Right-click on the input field and select "Inspect" to get the input selector +3. Use the element's `id` or a CSS selector that uniquely identifies it + +### Disabling CAPTCHA Solving + +If you want to disable automatic captcha solving, set `solveCaptchas: False` in browser_settings: + +```python +session = bb.sessions.create( + project_id=project_id, + browser_settings={"solveCaptchas": False}, +) +``` + +### Enabling Proxies for Higher Success Rates + +Enable proxies alongside CAPTCHA solving for better success rates: + +```python +session = bb.sessions.create( + project_id=project_id, + browser_settings={"solveCaptchas": True}, + proxies=True, # Enable Browserbase managed proxies +) +``` + +## QUICKSTART + +1. `cd python/playwright/basic-recaptcha` +2. `uv venv venv` +3. `source venv/bin/activate` # On Windows: `venv\Scripts\activate` +4. `uv pip install .` # Install dependencies from pyproject.toml +5. `playwright install chromium` # Install browser binaries +6. `cp .env.example .env` # Add your Browserbase API key and Project ID to .env +7. `python main.py` + +## EXPECTED OUTPUT + +- Validates environment variables for Browserbase credentials +- Creates Browserbase session with captcha solving enabled +- Displays session ID and live view link for monitoring +- Connects to browser via Chrome DevTools Protocol (CDP) +- Navigates to Google reCAPTCHA demo page +- Waits for Browserbase to automatically solve the captcha (with 60s timeout) +- Logs captcha solving progress messages +- Clicks submit button after captcha is solved +- Verifies successful captcha solving by checking for success message +- Closes session cleanly and displays replay link + +## COMMON PITFALLS + +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- Browser not installed: run `playwright install chromium` to install browser binaries +- Captcha solving timeout: the template uses a 60-second timeout; increase if needed for complex CAPTCHAs +- No browser context: ensure the Browserbase session was created successfully before connecting +- Proxies not enabled: enable proxies in session creation for higher CAPTCHA solving success rates +- Demo page inaccessible: verify the reCAPTCHA demo page URL is accessible and hasn't changed +- Console message timing: ensure console event listeners are set up before navigating to the page +- Verification failure: success message check may fail if page structure changes; check page content manually +- Custom captcha selectors: for non-standard CAPTCHAs, verify that `captchaImageSelector` and `captchaInputSelector` are correctly defined +- Import errors: activate your virtual environment if you created one + +## HELPFUL RESOURCES + +- Browserbase Docs: https://docs.browserbase.com +- Browserbase Python Integration: https://docs.browserbase.com/integrations/playwright/python +- Playwright Python Docs: https://playwright.dev/python/docs/intro +- Browserbase Dashboard: https://www.browserbase.com +- Playground: https://www.browserbase.com/playground +- Templates: https://www.browserbase.com/templates +- Need help? support@browserbase.com diff --git a/python/playwright/basic-recaptcha/main.py b/python/playwright/basic-recaptcha/main.py new file mode 100644 index 00000000..2a03264a --- /dev/null +++ b/python/playwright/basic-recaptcha/main.py @@ -0,0 +1,153 @@ +# Basic reCAPTCHA Solving with Playwright + Browserbase - See README.md for full documentation +# +# This template uses Playwright directly with the Browserbase SDK for lower-level browser control. +# For a higher-level API with natural language commands, see the Stagehand template instead. + +import asyncio +import os + +from browserbase import Browserbase +from dotenv import load_dotenv +from playwright.async_api import async_playwright + +load_dotenv() + +DEMO_URL = "https://google.com/recaptcha/api2/demo" +SOLVE_CAPTCHAS = True # Set to False to disable automatic captcha solving + + +def validate_env() -> tuple[str, str]: + """Validate required environment variables before starting. + + Browserbase credentials are required to create browser sessions. + + Returns: + Tuple of (api_key, project_id) + + Raises: + ValueError: If required environment variables are missing + """ + api_key = os.environ.get("BROWSERBASE_API_KEY") + project_id = os.environ.get("BROWSERBASE_PROJECT_ID") + + if not api_key: + raise ValueError("BROWSERBASE_API_KEY environment variable is required") + if not project_id: + raise ValueError("BROWSERBASE_PROJECT_ID environment variable is required") + + return api_key, project_id + + +async def main(): + print("Starting Playwright + Browserbase reCAPTCHA Example...") + + api_key, project_id = validate_env() + + # Initialize the Browserbase SDK client. + bb = Browserbase(api_key=api_key) + + # Create a new browser session with captcha solving enabled. + # solveCaptchas: Browserbase setting that enables automatic solving for + # reCAPTCHA, CAPTCHA, and other captcha types. + # Docs: https://docs.browserbase.com/features/stealth-mode#captcha-solving + print("Creating Browserbase session with captcha solving enabled...") + session = bb.sessions.create( + project_id=project_id, + browser_settings={"solveCaptchas": SOLVE_CAPTCHAS}, + ) + + print(f"Session created! ID: {session.id}") + print(f"Live View: https://browserbase.com/sessions/{session.id}") + + async with async_playwright() as playwright: + # Connect to the browser via Chrome DevTools Protocol (CDP). + # This gives direct Playwright control over the Browserbase-managed browser. + print("Connecting to browser via CDP...") + browser = await playwright.chromium.connect_over_cdp(session.connect_url) + + # Get the default browser context and page from the connected browser. + context = browser.contexts[0] + if not context: + raise RuntimeError("No browser context found") + + page = context.pages[0] + if not page: + raise RuntimeError("No page found in context") + + try: + # Set up captcha solving state tracking. + captcha_solved = asyncio.Event() + + # Listen for browser console messages indicating captcha solving progress. + # Browserbase emits these events when automatic captcha solving is active: + # - "browserbase-solving-started": CAPTCHA detection began + # - "browserbase-solving-finished": CAPTCHA solving completed + if SOLVE_CAPTCHAS: + + def handle_console(msg): + text = msg.text + if text == "browserbase-solving-started": + print("Captcha solving in progress...") + elif text == "browserbase-solving-finished": + print("Captcha solving completed!") + captcha_solved.set() + + page.on("console", handle_console) + + # Navigate to Google reCAPTCHA demo page to test captcha solving. + print("Navigating to reCAPTCHA demo page...") + await page.goto(DEMO_URL, wait_until="domcontentloaded") + + # Wait for Browserbase to solve the captcha automatically. + # CAPTCHA solving typically takes 5-30 seconds depending on type and complexity. + # We use a 60-second timeout to prevent hanging indefinitely on failures. + if SOLVE_CAPTCHAS and not captcha_solved.is_set(): + print("Waiting for captcha to be solved...") + try: + await asyncio.wait_for(captcha_solved.wait(), timeout=60.0) + except asyncio.TimeoutError: + raise TimeoutError("Captcha solving timed out after 60 seconds") + elif not SOLVE_CAPTCHAS: + print("Captcha solving is disabled. Skipping wait...") + + # Submit the form after captcha is solved. + print("Clicking submit button...") + await page.click('input[type="submit"]') + + await page.wait_for_load_state("domcontentloaded") + + # Verify captcha was successfully solved by checking for success + # message in page content. + print("Checking for success message...") + page_content = await page.text_content("body") + + if page_content and "Verification Success... Hooray!" in page_content: + print("\nSUCCESS! reCAPTCHA was solved and form was submitted!") + print("Page content confirms: Verification Success... Hooray!") + else: + print("\nCould not verify captcha success from page content") + print(f"Page content: {page_content[:500] if page_content else 'None'}") + + except Exception as error: + print(f"Error during reCAPTCHA solving: {error}") + raise + + finally: + # Always close the browser to release resources and end the Browserbase session. + await browser.close() + print("\nSession closed successfully") + print(f"View replay at: https://browserbase.com/sessions/{session.id}") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Application error: {err}") + print("\nCommon issues:") + print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") + print(" - Verify solveCaptchas is enabled (True by default)") + print(" - Allow up to 60 seconds for CAPTCHA solving to complete") + print(" - Enable proxies for higher success rates") + print("\nDocs: https://docs.browserbase.com/features/stealth-mode#captcha-solving") + exit(1) diff --git a/python/playwright/basic-recaptcha/pyproject.toml b/python/playwright/basic-recaptcha/pyproject.toml new file mode 100644 index 00000000..9ea927b8 --- /dev/null +++ b/python/playwright/basic-recaptcha/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "playwright-basic-recaptcha" +version = "0.1.0" +description = "Basic reCAPTCHA solving with Playwright and Browserbase" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "browserbase", + "playwright", + "python-dotenv", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/python/proxies-weather/README.md b/python/proxies-weather/README.md index 0ded77de..f8164373 100644 --- a/python/proxies-weather/README.md +++ b/python/proxies-weather/README.md @@ -54,4 +54,3 @@ 🔧 Templates: https://www.browserbase.com/templates 📧 Need help? support@browserbase.com 💬 Discord: http://stagehand.dev/discord - diff --git a/typescript/basic-recaptcha/README.md b/typescript/basic-recaptcha/README.md index cd0871e1..b41a055f 100644 --- a/typescript/basic-recaptcha/README.md +++ b/typescript/basic-recaptcha/README.md @@ -24,6 +24,13 @@ - extract: pull data from web pages using natural language instructions Docs → https://docs.stagehand.dev/basics/extract +## STAGEHAND VS PLAYWRIGHT + +| Feature | Stagehand (this template) | Playwright | +| --------------- | ------------------------------------ | ----------------------------- | +| Actions | `stagehand.act("Click the button")` | `page.click()`, `page.goto()` | +| Data Extraction | `stagehand.extract("Get the price")` | Manual DOM queries | + ## CAPTCHA SOLVING DETAILS ### How CAPTCHA Solving Works diff --git a/typescript/company-address-finder/index.ts b/typescript/company-address-finder/index.ts index 8bf2f206..4e4ee9ee 100644 --- a/typescript/company-address-finder/index.ts +++ b/typescript/company-address-finder/index.ts @@ -170,7 +170,9 @@ async function processCompany(companyName: string): Promise { console.log(`[${companyName}] Address found in Terms of Service: ${address}`); } } catch (error) { - console.log(`[${companyName}] Could not extract address from Terms of Service page: ${error}`); + console.log( + `[${companyName}] Could not extract address from Terms of Service page: ${error}`, + ); } } @@ -197,7 +199,9 @@ async function processCompany(companyName: string): Promise { console.log(`[${companyName}] Address found in Privacy Policy: ${address}`); } } catch (error) { - console.log(`[${companyName}] Could not extract address from Privacy Policy page: ${error}`); + console.log( + `[${companyName}] Could not extract address from Privacy Policy page: ${error}`, + ); } } diff --git a/typescript/dynamic-form-filling/README.md b/typescript/dynamic-form-filling/README.md index 8302454a..b9c875c0 100644 --- a/typescript/dynamic-form-filling/README.md +++ b/typescript/dynamic-form-filling/README.md @@ -62,4 +62,3 @@ 🔧 Templates: https://www.browserbase.com/templates 📧 Need help? support@browserbase.com 💬 Discord: http://stagehand.dev/discord - diff --git a/typescript/gemini-3-flash/README.md b/typescript/gemini-3-flash/README.md index 5cf92857..03c7bbaf 100644 --- a/typescript/gemini-3-flash/README.md +++ b/typescript/gemini-3-flash/README.md @@ -1,21 +1,25 @@ # Stagehand + Browserbase: Gemini 3 Flash Agent Example ## AT A GLANCE + - Goal: demonstrate autonomous web browsing using Google's Gemini 3 Flash with Stagehand and Browserbase. - Uses Stagehand Agent to automate complex workflows with AI powered browser agents. - Leverages Gemini 3 Flash model for autonomous web interaction and decision-making. ## GLOSSARY + - agent: create an autonomous AI agent that can execute complex multi-step tasks Docs → https://docs.stagehand.dev/basics/agent#what-is-agent ## QUICKSTART - 1) npm install - 2) cp .env.example .env - 3) Add your Browserbase API key, Project ID, and Google API key to .env - 4) npm start + +1. npm install +2. cp .env.example .env +3. Add your Browserbase API key, Project ID, and Google API key to .env +4. npm start ## EXPECTED OUTPUT + - Initializes Stagehand session with Browserbase - Navigates to Google search engine - Executes autonomous search and data extraction task @@ -24,23 +28,27 @@ - Closes session cleanly ## COMMON PITFALLS + - "Cannot find module": ensure all dependencies are installed - Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY - Google API access: ensure you have access to Gemini 3 Flash via the Google AI API ## USE CASES + • Autonomous research: Let AI agents independently research topics, gather information, and compile reports without manual intervention. • Complex web workflows: Automate multi-step processes that require decision-making, form filling, and data extraction across multiple pages. • Content discovery: Search for specific information, verify data accuracy, and cross-reference sources autonomously. ## NEXT STEPS + • Customize instructions: Modify the instruction variable to test different autonomous tasks and scenarios. • Add error handling: Implement retry logic, fallback strategies, and better error recovery for failed agent actions. • Extend capabilities: Add support for file downloads, form submissions, and more complex interaction patterns. ## HELPFUL RESOURCES -📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction -🎮 Browserbase: https://www.browserbase.com -💡 Try it out: https://www.browserbase.com/playground -🔧 Templates: https://www.browserbase.com/templates -📧 Need help? support@browserbase.com + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com diff --git a/typescript/gemini-3-flash/index.ts b/typescript/gemini-3-flash/index.ts index ed808b28..802cee77 100644 --- a/typescript/gemini-3-flash/index.ts +++ b/typescript/gemini-3-flash/index.ts @@ -21,13 +21,12 @@ const instruction = `Search for the next visible solar eclipse in North America // ============================================================================ async function main() { - const stagehand = new Stagehand({ env: "BROWSERBASE", // model: "google/gemini-2.5-pro", // this is the model Stagehand uses for act, observe, extract (not agent) verbose: 1, - // 0 = errors only, 1 = info, 2 = debug - // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + // 0 = errors only, 1 = info, 2 = debug + // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) // https://docs.stagehand.dev/configuration/logging browserbaseSessionCreateParams: { projectId: process.env.BROWSERBASE_PROJECT_ID!, @@ -37,8 +36,8 @@ async function main() { blockAds: true, viewport: { width: 1288, - height: 711 - } + height: 711, + }, }, }, }); @@ -47,31 +46,33 @@ async function main() { // Initialize browser session to start automation. await stagehand.init(); console.log("Stagehand initialized successfully!"); - console.log(`Live View Link: https://browserbase.com/sessions/${stagehand.browserbaseSessionId}`); + console.log( + `Live View Link: https://browserbase.com/sessions/${stagehand.browserbaseSessionId}`, + ); const page = stagehand.context.pages()[0]; // Navigate to search engine with extended timeout for slow-loading sites. await page.goto("https://www.google.com/", { - waitUntil: 'domcontentloaded', + waitUntil: "domcontentloaded", }); // Create agent with Gemini 3 Flash for autonomous web browsing. const agent = stagehand.agent({ model: { modelName: "google/gemini-3-flash-preview", - apiKey: process.env.GOOGLE_API_KEY + apiKey: process.env.GOOGLE_API_KEY, }, systemPrompt: `You are a helpful assistant that can use a web browser. You are currently on the following page: ${page.url()}. - Do not ask follow up questions, the user will trust your judgement. If you are getting blocked on google, try another search engine.` + Do not ask follow up questions, the user will trust your judgement. If you are getting blocked on google, try another search engine.`, }); console.log("Executing instruction:", instruction); const result = await agent.execute({ instruction: instruction, maxSteps: 30, - highlightCursor: true + highlightCursor: true, }); if (result.success === true) { @@ -80,7 +81,6 @@ async function main() { } else { console.log("Task failed or was incomplete"); } - } catch (error) { console.error("Error executing Gemini 3 Flash agent:", error); } finally { @@ -96,4 +96,4 @@ main().catch((err) => { console.error(" - Verify GOOGLE_API_KEY is set for the agent"); console.error("Docs: https://docs.stagehand.dev/v3/first-steps/introduction"); process.exit(1); -}); \ No newline at end of file +}); diff --git a/typescript/pickleball/index.ts b/typescript/pickleball/index.ts index e513eca4..2c5a54e2 100644 --- a/typescript/pickleball/index.ts +++ b/typescript/pickleball/index.ts @@ -195,15 +195,16 @@ async function checkAndExtractCourts(stagehand: Stagehand, timeOfDay: string): P }, index: number, ) => { - console.log(`${index + 1}. ${court.name}`); - console.log(` Opening Times: ${court.openingTimes}`); - console.log(` Location: ${court.location}`); - console.log(` Availability: ${court.availability}`); - if (court.duration) { - console.log(` Duration: ${court.duration} minutes`); - } - console.log(""); - }); + console.log(`${index + 1}. ${court.name}`); + console.log(` Opening Times: ${court.openingTimes}`); + console.log(` Location: ${court.location}`); + console.log(` Availability: ${court.availability}`); + if (court.duration) { + console.log(` Duration: ${court.duration} minutes`); + } + console.log(""); + }, + ); } else { console.log("No court data available to display"); } diff --git a/typescript/playwright/basic-recaptcha/.env.example b/typescript/playwright/basic-recaptcha/.env.example new file mode 100644 index 00000000..36c24713 --- /dev/null +++ b/typescript/playwright/basic-recaptcha/.env.example @@ -0,0 +1,3 @@ +# Browserbase Configuration +BROWSERBASE_PROJECT_ID=your_browserbase_project_id +BROWSERBASE_API_KEY=your_browserbase_api_key diff --git a/typescript/playwright/basic-recaptcha/README.md b/typescript/playwright/basic-recaptcha/README.md new file mode 100644 index 00000000..8e7a01d9 --- /dev/null +++ b/typescript/playwright/basic-recaptcha/README.md @@ -0,0 +1,107 @@ +# Playwright + Browserbase: Basic reCAPTCHA Solving + +## AT A GLANCE + +- Goal: Demonstrate automatic reCAPTCHA solving using Playwright with Browserbase's built-in captcha solving capabilities. +- Automated Solving: Browserbase automatically detects and solves CAPTCHAs in the background. CAPTCHA solving is **enabled by default** - you don't need to set `solveCaptchas: true` unless you want to explicitly enable it (or set it to `false` to disable). +- Timeout Handling: Includes 60-second timeout to prevent indefinite waits on captcha solving failures. +- Progress Monitoring: Listen for console messages (`browserbase-solving-started`, `browserbase-solving-finished`) to track captcha solving progress in real-time. +- Docs -> https://docs.browserbase.com/features/stealth-mode#captcha-solving + +## GLOSSARY + +- Browserbase SDK: JavaScript/TypeScript SDK for creating and managing browser sessions in Browserbase's cloud infrastructure. + Docs -> https://docs.browserbase.com/ +- solveCaptchas: Browserbase browser setting that enables automatic captcha solving for reCAPTCHA, CAPTCHA, and other captcha types. Enabled by default for Basic and Advanced Stealth Mode. + Docs -> https://docs.browserbase.com/features/stealth-mode#captcha-solving +- console messages: Browser console events that indicate captcha solving status: + - `browserbase-solving-started`: emitted when CAPTCHA detection begins + - `browserbase-solving-finished`: emitted when CAPTCHA solving completes +- custom CAPTCHA solving: For non-standard or custom captcha providers, you can specify CSS selectors for the captcha image and input field using `captchaImageSelector` and `captchaInputSelector` in browserSettings. + +## PLAYWRIGHT VS STAGEHAND + +| Feature | Playwright (this template) | Stagehand | +| --------------- | ----------------------------- | ------------------------------------ | +| Actions | `page.click()`, `page.goto()` | `stagehand.act("Click the button")` | +| Data Extraction | Manual DOM queries | `stagehand.extract("Get the price")` | + +## CAPTCHA SOLVING DETAILS + +### How CAPTCHA Solving Works + +Browserbase provides integrated CAPTCHA solving to handle challenges automatically: + +- **Automatic Detection**: When a CAPTCHA is detected on a page, Browserbase attempts to solve it in the background +- **Solving Time**: CAPTCHA solving typically takes between 5-30 seconds, depending on the CAPTCHA type and complexity +- **Default Behavior**: CAPTCHA solving is enabled by default for Basic and Advanced Stealth Mode +- **Proxies**: It's recommended to enable proxies when using CAPTCHA solving for higher success rates + +### Custom CAPTCHA Solving + +For non-standard or custom captcha providers, you can specify CSS selectors to guide the solution process: + +```typescript +browserSettings: { + solveCaptchas: true, + captchaImageSelector: "#custom-captcha-image-id",xc + captchaInputSelector: "#custom-captcha-input-id" +} +``` + +To find the selectors: + +1. Right-click on the captcha image and select "Inspect" to get the image selector +2. Right-click on the input field and select "Inspect" to get the input selector +3. Use the element's `id` or a CSS selector that uniquely identifies it + +### Disabling CAPTCHA Solving + +If you want to disable automatic captcha solving, set `solveCaptchas: false` in browserSettings: + +```typescript +browserSettings: { + solveCaptchas: false; +} +``` + +## QUICKSTART + +1. cd typescript/playwright/basic-recaptcha +2. pnpm install +3. cp .env.example .env +4. Add your Browserbase API key and Project ID to .env +5. pnpm start + +## EXPECTED OUTPUT + +- Validates environment variables for Browserbase credentials +- Creates Browserbase session with captcha solving enabled +- Displays session ID and live view link for monitoring +- Connects to browser via Chrome DevTools Protocol (CDP) +- Navigates to Google reCAPTCHA demo page +- Waits for Browserbase to automatically solve the captcha (with 60s timeout) +- Logs captcha solving progress messages +- Clicks submit button after captcha is solved +- Verifies successful captcha solving by checking for success message +- Closes session cleanly and displays replay link + +## COMMON PITFALLS + +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- Captcha solving timeout: the template uses a 60-second timeout; increase if needed for complex CAPTCHAs +- No browser context: ensure the Browserbase session was created successfully before connecting +- Proxies not enabled: enable proxies in browserSettings for higher CAPTCHA solving success rates +- Demo page inaccessible: verify the reCAPTCHA demo page URL is accessible and hasn't changed +- Console message timing: ensure console event listeners are set up before navigating to the page +- Verification failure: success message check may fail if page structure changes; check page content manually +- Custom captcha selectors: for non-standard CAPTCHAs, verify that `captchaImageSelector` and `captchaInputSelector` are correctly defined + +## HELPFUL RESOURCES + +- Browserbase Docs: https://docs.browserbase.com +- Playwright Docs: https://playwright.dev/docs/intro +- Browserbase Dashboard: https://www.browserbase.com +- Playground: https://www.browserbase.com/playground +- Templates: https://www.browserbase.com/templates +- Need help? support@browserbase.com diff --git a/typescript/playwright/basic-recaptcha/index.ts b/typescript/playwright/basic-recaptcha/index.ts new file mode 100644 index 00000000..93fe05ac --- /dev/null +++ b/typescript/playwright/basic-recaptcha/index.ts @@ -0,0 +1,152 @@ +// Basic reCAPTCHA Solving with Playwright + Browserbase - See README.md for full documentation +// +// This template uses Playwright directly with the Browserbase SDK for lower-level browser control. +// For a higher-level API with natural language commands, see the Stagehand template instead. + +import { chromium, Browser, Page } from "playwright-core"; +import Browserbase from "@browserbasehq/sdk"; +import "dotenv/config"; + +const DEMO_URL = "https://google.com/recaptcha/api2/demo"; +const SOLVE_CAPTCHAS = true; // Set to false to disable automatic captcha solving + +// Validate required environment variables before starting. +// Browserbase credentials are required to create browser sessions. +function validateEnv(): { apiKey: string; projectId: string } { + const apiKey = process.env.BROWSERBASE_API_KEY; + const projectId = process.env.BROWSERBASE_PROJECT_ID; + + if (!apiKey) { + throw new Error("BROWSERBASE_API_KEY environment variable is required"); + } + if (!projectId) { + throw new Error("BROWSERBASE_PROJECT_ID environment variable is required"); + } + + return { apiKey, projectId }; +} + +async function main() { + console.log("Starting Playwright + Browserbase reCAPTCHA Example..."); + + const { apiKey, projectId } = validateEnv(); + + // Initialize the Browserbase SDK client. + const bb = new Browserbase({ apiKey }); + + // Create a new browser session with captcha solving enabled. + // solveCaptchas: Browserbase setting that enables automatic solving for reCAPTCHA, CAPTCHA, and other captcha types. + // Docs: https://docs.browserbase.com/features/stealth-mode#captcha-solving + console.log("Creating Browserbase session with captcha solving enabled..."); + const session = await bb.sessions.create({ + projectId, + browserSettings: { + solveCaptchas: SOLVE_CAPTCHAS, + }, + }); + + console.log(`Session created! ID: ${session.id}`); + console.log(`Live View: https://browserbase.com/sessions/${session.id}`); + + // Connect to the browser via Chrome DevTools Protocol (CDP). + // This gives direct Playwright control over the Browserbase-managed browser. + console.log("Connecting to browser via CDP..."); + const browser: Browser = await chromium.connectOverCDP(session.connectUrl); + + // Get the default browser context and page from the connected browser. + const context = browser.contexts()[0]; + if (!context) { + throw new Error("No browser context found"); + } + + const page: Page = context.pages()[0]; + if (!page) { + throw new Error("No page found in context"); + } + + try { + // Set up captcha solving state tracking. + let captchaSolved = false; + let captchaResolve: () => void; + + const captchaPromise = new Promise((resolve) => { + captchaResolve = resolve; + }); + + // Listen for browser console messages indicating captcha solving progress. + // Browserbase emits these events when automatic captcha solving is active: + // - "browserbase-solving-started": CAPTCHA detection began + // - "browserbase-solving-finished": CAPTCHA solving completed + if (SOLVE_CAPTCHAS) { + page.on("console", (msg) => { + const text = msg.text(); + if (text === "browserbase-solving-started") { + console.log("Captcha solving in progress..."); + } else if (text === "browserbase-solving-finished") { + console.log("Captcha solving completed!"); + captchaSolved = true; + captchaResolve(); + } + }); + } + + // Navigate to Google reCAPTCHA demo page to test captcha solving. + console.log("Navigating to reCAPTCHA demo page..."); + await page.goto(DEMO_URL, { waitUntil: "domcontentloaded" }); + + // Wait for Browserbase to solve the captcha automatically. + // CAPTCHA solving typically takes 5-30 seconds depending on type and complexity. + // We use a 60-second timeout to prevent hanging indefinitely on failures. + if (SOLVE_CAPTCHAS && !captchaSolved) { + console.log("Waiting for captcha to be solved..."); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + if (!captchaSolved) { + reject(new Error("Captcha solving timed out after 60 seconds")); + } + }, 60000); + }); + + await Promise.race([captchaPromise, timeoutPromise]); + } else if (!SOLVE_CAPTCHAS) { + console.log("Captcha solving is disabled. Skipping wait..."); + } + + // Submit the form after captcha is solved. + console.log("Clicking submit button..."); + await page.click('input[type="submit"]'); + + await page.waitForLoadState("domcontentloaded"); + + // Verify captcha was successfully solved by checking for success message in page content. + console.log("Checking for success message..."); + const pageContent = await page.textContent("body"); + + if (pageContent?.includes("Verification Success... Hooray!")) { + console.log("\nSUCCESS! reCAPTCHA was solved and form was submitted!"); + console.log("Page content confirms: Verification Success... Hooray!"); + } else { + console.log("\nCould not verify captcha success from page content"); + console.log("Page content:", pageContent?.substring(0, 500)); + } + } catch (error) { + console.error("Error during reCAPTCHA solving:", error); + throw error; + } finally { + // Always close the browser to release resources and end the Browserbase session. + await browser.close(); + console.log("\nSession closed successfully"); + console.log(`View replay at: https://browserbase.com/sessions/${session.id}`); + } +} + +main().catch((err) => { + console.error("Application error:", err); + console.error("\nCommon issues:"); + console.error(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY"); + console.error(" - Verify solveCaptchas is enabled (true by default)"); + console.error(" - Allow up to 60 seconds for CAPTCHA solving to complete"); + console.error(" - Enable proxies for higher success rates"); + console.error("\nDocs: https://docs.browserbase.com/features/stealth-mode#captcha-solving"); + process.exit(1); +}); diff --git a/typescript/playwright/basic-recaptcha/package.json b/typescript/playwright/basic-recaptcha/package.json new file mode 100644 index 00000000..547782eb --- /dev/null +++ b/typescript/playwright/basic-recaptcha/package.json @@ -0,0 +1,17 @@ +{ + "name": "playwright-basic-recaptcha-template", + "type": "module", + "scripts": { + "build": "tsc", + "start": "tsx index.ts" + }, + "dependencies": { + "@browserbasehq/sdk": "latest", + "playwright-core": "latest", + "dotenv": "^16.4.7" + }, + "devDependencies": { + "tsx": "^4.19.2", + "typescript": "^5.0.0" + } +} diff --git a/typescript/proxies-weather/index.ts b/typescript/proxies-weather/index.ts index d2c03eed..b8ef16aa 100644 --- a/typescript/proxies-weather/index.ts +++ b/typescript/proxies-weather/index.ts @@ -64,7 +64,7 @@ async function getWeatherForLocation(geolocation: GeolocationConfig): Promise setTimeout(resolve, 2000)); + await new Promise((resolve) => setTimeout(resolve, 2000)); // Extract structured temperature data using Stagehand and Zod schema for type safety console.log(`Extracting temperature data for ${cityName}...`); @@ -176,4 +176,4 @@ main().catch((err) => { console.error(" - Ensure locations array is properly configured"); console.error("Docs: https://docs.stagehand.dev/v3/first-steps/introduction"); process.exit(1); -}); \ No newline at end of file +}); From ac1b0ed9303b7f86db856276ce4998655a800f71 Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Mon, 2 Feb 2026 13:19:53 -0800 Subject: [PATCH 12/45] jaysahnan/gro-954-amazon-global-price-comparision (#52) * init commit * addressing PR comment --- .../.env.example | 7 + .../amazon-global-price-comparison/README.md | 77 ++++ python/amazon-global-price-comparison/main.py | 377 ++++++++++++++++++ .../pyproject.toml | 34 ++ .../.env.example | 3 + .../amazon-global-price-comparison/README.md | 66 +++ .../amazon-global-price-comparison/index.ts | 266 ++++++++++++ .../package.json | 17 + 8 files changed, 847 insertions(+) create mode 100644 python/amazon-global-price-comparison/.env.example create mode 100644 python/amazon-global-price-comparison/README.md create mode 100644 python/amazon-global-price-comparison/main.py create mode 100644 python/amazon-global-price-comparison/pyproject.toml create mode 100644 typescript/amazon-global-price-comparison/.env.example create mode 100644 typescript/amazon-global-price-comparison/README.md create mode 100644 typescript/amazon-global-price-comparison/index.ts create mode 100644 typescript/amazon-global-price-comparison/package.json diff --git a/python/amazon-global-price-comparison/.env.example b/python/amazon-global-price-comparison/.env.example new file mode 100644 index 00000000..8cf6d6e1 --- /dev/null +++ b/python/amazon-global-price-comparison/.env.example @@ -0,0 +1,7 @@ +# Browserbase credentials - get these from https://www.browserbase.com/settings +BROWSERBASE_API_KEY=your_browserbase_api_key +BROWSERBASE_PROJECT_ID=your_browserbase_project_id + +# Model API key for Stagehand AI capabilities +# Use Google API key for Gemini models, or OpenAI API key for GPT models +MODEL_API_KEY=your_model_api_key diff --git a/python/amazon-global-price-comparison/README.md b/python/amazon-global-price-comparison/README.md new file mode 100644 index 00000000..31be6bb6 --- /dev/null +++ b/python/amazon-global-price-comparison/README.md @@ -0,0 +1,77 @@ +# Amazon Global Price Comparison + +## AT A GLANCE + +- **Goal**: Compare Amazon product prices across multiple countries using geolocation proxies. +- **Pattern Template**: Demonstrates the integration of Browserbase geolocation proxies with Stagehand AI extraction for price comparison workflows. +- **Workflow**: Creates Browserbase sessions with geolocation proxies for each country, navigates to Amazon, searches for products, and extracts structured pricing data using Stagehand's AI-powered extraction. +- **Concurrent Processing**: Runs all country searches in parallel using `asyncio.gather()` for faster execution. +- **Structured Extraction**: Uses Pydantic schemas to extract consistent product data (name, price, rating, reviews) across different Amazon regions. +- Docs → [Browserbase Proxies](https://docs.browserbase.com/features/proxies) | [Stagehand Extract](https://docs.stagehand.dev/basics/extract) + +## GLOSSARY + +- **geolocation proxies**: Route traffic through specific geographic locations (city, country) to access location-specific content and pricing. + Docs → https://docs.browserbase.com/features/proxies#set-proxy-geolocation +- **extract**: Extract structured data from web pages using natural language instructions and JSON schemas. + Docs → https://docs.stagehand.dev/basics/extract +- **act**: Perform UI actions from natural language prompts (click, scroll, type, navigate). + Docs → https://docs.stagehand.dev/basics/act +- **proxies**: Browserbase's managed proxy infrastructure supporting 201+ countries for geolocation-based routing. + Docs → https://docs.browserbase.com/features/proxies + +## QUICKSTART + +1. cd python/amazon-global-price-comparison +2. cp .env.example .env +3. Add required API keys to .env: + - `BROWSERBASE_PROJECT_ID` + - `BROWSERBASE_API_KEY` + - `MODEL_API_KEY` (Google API key for Gemini, or OpenAI API key) +4. Run the script: + ```bash + uv run python main.py + ``` + +## EXPECTED OUTPUT + +- Creates Browserbase sessions with geolocation proxies for each country (US, UK, DE, FR, IT, ES) +- Navigates to Amazon search results through location-specific proxies +- Extracts product name, price, rating, and review count for each location +- Displays formatted comparison table showing price differences across countries +- Outputs JSON results for programmatic use + +## COMMON PITFALLS + +- **Browserbase Developer plan or higher is required to use proxies** +- "ModuleNotFoundError": ensure you're running with `uv run python main.py` (uv automatically installs dependencies from pyproject.toml) +- Missing credentials: verify .env contains `BROWSERBASE_PROJECT_ID`, `BROWSERBASE_API_KEY`, and `MODEL_API_KEY` +- Geolocation fields are case-insensitive (city, country can be any case) +- Amazon may show different products in different regions - comparison works best for globally available products +- ERR_TUNNEL_CONNECTION_FAILED: indicates either a temporary proxy hiccup or a site unsupported by built-in proxies + +## USE CASES + +• **Price arbitrage**: Find the best country to purchase products from for international shipping +• **Market research**: Compare pricing strategies across different Amazon regions +• **Competitive analysis**: Monitor how competitors price products globally +• **Travel shopping**: Check prices before international trips to plan purchases + +## NEXT STEPS + +• **Add more countries**: Extend the `COUNTRIES` list with additional regions (Japan, Australia, Canada, etc.) +• **Currency conversion**: Add real-time currency conversion to normalize prices for comparison +• **Price tracking**: Store results over time to track price changes across regions +• **Email alerts**: Send notifications when price drops below a threshold in any country +• **Product matching**: Use fuzzy matching to ensure you're comparing the same product across regions + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +📚 Python SDK: https://docs.stagehand.dev/v3/sdk/python +📚 Browserbase Proxies: https://docs.browserbase.com/features/proxies +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/python/amazon-global-price-comparison/main.py b/python/amazon-global-price-comparison/main.py new file mode 100644 index 00000000..908df56a --- /dev/null +++ b/python/amazon-global-price-comparison/main.py @@ -0,0 +1,377 @@ +# Amazon Global Price Comparison - See README.md for full documentation + +import asyncio +import json +import os +from dataclasses import dataclass +from typing import Optional + +from browserbase import Browserbase +from dotenv import load_dotenv +from pydantic import BaseModel, Field + +from stagehand import AsyncStagehand + +# Load environment variables from .env file +# Required: BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, MODEL_API_KEY +load_dotenv() + + +# Schema for a single product with structured extraction fields +class Product(BaseModel): + """Schema for extracted product data from Amazon search results""" + + name: str = Field(..., description="The full product title/name") + price: str = Field( + default="N/A", + description=( + "The product price including currency symbol (e.g., '$29.99', '29,99 EUR', " + "'29.99 GBP'). If no price is visible, return 'N/A'" + ), + ) + rating: str = Field( + default="N/A", description="The star rating (e.g., '4.5 out of 5 stars')" + ) + reviews_count: str = Field( + default="N/A", description="The number of customer reviews (e.g., '1,234')" + ) + product_url: str = Field( + default="N/A", + description="The full href URL link to the product detail page (starting with https:// or /dp/)", + ) + + +# Schema for extracting multiple products from search results +class ProductsResult(BaseModel): + """Schema for extracting multiple products from Amazon search results""" + + products: list[Product] = Field( + default_factory=list, description="Array of products from search results" + ) + + +# Country configuration with geolocation proxy settings +# Each country routes traffic through its geographic location to see local pricing +@dataclass +class CountryConfig: + name: str + code: str + city: Optional[str] + currency: str + + +# Supported countries for price comparison +# Add or remove countries as needed - see https://docs.browserbase.com/features/proxies for available geolocations +COUNTRIES: list[CountryConfig] = [ + CountryConfig(name="United States", code="US", city=None, currency="USD"), + CountryConfig(name="United Kingdom", code="GB", city="LONDON", currency="GBP"), + CountryConfig(name="Germany", code="DE", city="BERLIN", currency="EUR"), + CountryConfig(name="France", code="FR", city="PARIS", currency="EUR"), + CountryConfig(name="Italy", code="IT", city="ROME", currency="EUR"), + CountryConfig(name="Spain", code="ES", city="MADRID", currency="EUR"), +] + + +# Results structure for each country +@dataclass +class CountryResult: + country: str + country_code: str + currency: str + products: list[dict] + error: Optional[str] = None + + +# Initialize Browserbase SDK for session management with proxies +bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) + + +async def get_products_for_country( + search_query: str, + country: CountryConfig, + results_count: int = 3, +) -> CountryResult: + """ + Fetches products from Amazon for a specific country using geolocation proxy. + + Uses Browserbase's managed proxy infrastructure to route traffic through the target country. + This ensures Amazon shows location-specific pricing and availability. + + Args: + search_query: The product search term to look up on Amazon + country: Configuration for the target country including geolocation settings + results_count: Number of products to extract (default: 3) + + Returns: + CountryResult with extracted products or error information + """ + print(f'\n=== Searching Amazon for "{search_query}" in {country.name} ===') + + # Build geolocation config for proxy routing + geolocation: dict = {"country": country.code} + if country.city: + geolocation["city"] = country.city + + # Create Browserbase session with geolocation proxy configuration + # This ensures all browser traffic routes through the specified geographic location + print(f"Creating Browserbase session with {country.name} proxy...") + session = await asyncio.to_thread( + bb.sessions.create, + project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + proxies=[ + { + "type": "browserbase", # Use Browserbase's managed proxy infrastructure + "geolocation": geolocation, + } + ], + ) + session_id = session.id + print(f"Session created: https://browserbase.com/sessions/{session_id}") + + # Initialize AsyncStagehand client (v3 API) + client = AsyncStagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("MODEL_API_KEY"), + ) + + try: + # Start Stagehand session connected to our proxy-enabled Browserbase session + print(f"[{country.name}] Initializing Stagehand session...") + stagehand_session = await client.sessions.create( + model_name="google/gemini-2.5-flash", + browserbase_session_id=session_id, # Connect to existing proxy session + ) + + # Navigate to Amazon homepage to begin search + print(f"[{country.name}] Navigating to Amazon...") + await stagehand_session.navigate(url="https://www.amazon.com") + + # Perform search using natural language actions + print(f"[{country.name}] Searching for: {search_query}") + await stagehand_session.act(input=f'Type "{search_query}" into the search bar') + await stagehand_session.act(input="Click the search button") + + # Wait for search results to load + await asyncio.sleep(2) + + # Extract products from search results using Stagehand's structured extraction + print(f"[{country.name}] Extracting top {results_count} products...") + + # Use a flattened schema that Gemini can understand (avoids $ref issues) + products_schema = { + "type": "object", + "properties": { + "products": { + "type": "array", + "description": "Array of products from search results", + "items": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "The full product title/name"}, + "price": {"type": "string", "description": "The product price including currency symbol (e.g., '$29.99'). If no price is visible, return 'N/A'"}, + "rating": {"type": "string", "description": "The star rating (e.g., '4.5 out of 5 stars')"}, + "reviews_count": {"type": "string", "description": "The number of customer reviews (e.g., '1,234')"}, + "product_url": {"type": "string", "description": "The full href URL link to the product detail page"}, + }, + "required": ["name"], + }, + } + }, + "required": ["products"], + } + + extract_response = await stagehand_session.extract( + instruction=f"""Extract the first {results_count} product search results from this Amazon page. For each product, extract: + 1. name: the full product title + 2. price: the displayed price WITH currency symbol (like $599.99 or 599,99 EUR). If no price shown, use "N/A" + 3. rating: the star rating text (like "4.5 out of 5 stars") + 4. reviews_count: the number of reviews (like "2,508") + 5. product_url: the href link to the product page (starts with /dp/ or https://) + + Only extract actual product listings, skip sponsored ads or recommendations.""", + schema=products_schema, + ) + + # Parse the extracted data + extracted_data = extract_response.data.result + if isinstance(extracted_data, str): + extracted_data = json.loads(extracted_data) + + products = extracted_data.get("products", []) + + # Clean up products - ensure price is never null and URLs are absolute + cleaned_products = [] + for p in products: + product_url = p.get("product_url", "N/A") + if product_url and product_url.startswith("/"): + product_url = f"https://www.amazon.com{product_url}" + elif not product_url: + product_url = "N/A" + + cleaned_products.append( + { + "name": p.get("name", "Unknown"), + "price": p.get("price") or "N/A", + "rating": p.get("rating") or "N/A", + "reviews_count": p.get("reviews_count") or "N/A", + "product_url": product_url, + } + ) + + print(f"Found {len(cleaned_products)} products in {country.name}") + + # End the Stagehand session + await stagehand_session.end() + + return CountryResult( + country=country.name, + country_code=country.code, + currency=country.currency, + products=cleaned_products[:results_count], + ) + + except Exception as error: + print(f"Error fetching products from {country.name}: {error}") + + return CountryResult( + country=country.name, + country_code=country.code, + currency=country.currency, + products=[], + error=str(error), + ) + + +def display_comparison_table(results: list[CountryResult]) -> None: + """ + Displays results in a formatted comparison table. + + Shows product name, price, rating, and review count for each country. + + Args: + results: List of CountryResult objects containing extracted product data + """ + print("\n" + "=" * 100) + print("PRICE COMPARISON ACROSS COUNTRIES") + print("=" * 100) + + # Find the first successful result to get product count + successful_result = next((r for r in results if r.products), None) + if not successful_result: + print("No products found in any country.") + return + + # Display results for each product position + max_products = max(len(r.products) for r in results) + + for i in range(max_products): + print(f"\n--- Product {i + 1} ---") + + # Find the first available product name for this position + product_name = None + for r in results: + if i < len(r.products): + product_name = r.products[i].get("name") + break + + if product_name: + truncated_name = ( + product_name[:77] + "..." if len(product_name) > 80 else product_name + ) + print(f"Product: {truncated_name}") + + print("\nPrices by Country:") + print("-" * 70) + + for result in results: + country_pad = result.country.ljust(20) + if result.error: + print(f" {country_pad} | Error: {result.error}") + elif i < len(result.products): + product = result.products[i] + price = product.get("price", "N/A") + price_pad = price.ljust(18) + rating = product.get("rating", "N/A") + rating_short = rating.split(" out")[0] if " out" in rating else rating + rating_pad = rating_short.ljust(6) + reviews = product.get("reviews_count", "N/A") + print( + f" {country_pad} | {price_pad} | {rating_pad} stars | {reviews} reviews" + ) + else: + print(f" {country_pad} | Not available in this country") + + print("\n" + "=" * 100) + + +async def main(): + """ + Main application entry point. + + Orchestrates the entire price comparison automation process: + 1. Initializes configuration from environment variables + 2. Fetches products from Amazon for each country concurrently + 3. Displays formatted comparison table + 4. Outputs JSON results for programmatic use + """ + # Configure search parameters + search_query = "iPhone 15 Pro Max 256GB" + results_count = 3 + + print("=" * 60) + print("AMAZON PRICE COMPARISON - GEOLOCATION PROXY DEMO") + print("=" * 60) + print(f"Search Query: {search_query}") + print(f"Results per country: {results_count}") + print(f"Countries: {', '.join(c.code for c in COUNTRIES)}") + print("=" * 60) + + # Process all countries concurrently for faster execution + # Each country uses its own browser session, so they can run in parallel + print(f"\nFetching prices from {len(COUNTRIES)} countries concurrently...") + + results = await asyncio.gather( + *[ + get_products_for_country(search_query, country, results_count) + for country in COUNTRIES + ] + ) + + # Display formatted comparison table + display_comparison_table(list(results)) + + # Output JSON results for programmatic use + print("\n--- JSON OUTPUT ---") + json_results = [ + { + "country": r.country, + "countryCode": r.country_code, + "currency": r.currency, + "products": r.products, + "error": r.error, + } + for r in results + ] + print(json.dumps(json_results, indent=2)) + + print("\n=== Price comparison completed ===") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Application error: {err}") + print("\nCommon issues:") + print( + " - Check .env file has BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and MODEL_API_KEY" + ) + print( + " - Verify geolocation proxy locations are valid " + "(see https://docs.browserbase.com/features/proxies)" + ) + print(" - Ensure you have sufficient Browserbase credits") + print(" - Browserbase Developer plan or higher is required to use proxies") + print("Docs: https://docs.stagehand.dev/v3/sdk/python") + exit(1) diff --git a/python/amazon-global-price-comparison/pyproject.toml b/python/amazon-global-price-comparison/pyproject.toml new file mode 100644 index 00000000..3ed41f76 --- /dev/null +++ b/python/amazon-global-price-comparison/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "amazon-global-price-comparison" +version = "0.1.0" +description = "Compare Amazon product prices across multiple countries using geolocation proxies" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "browserbase", + "python-dotenv", + "pydantic>=2.0.0", + "stagehand", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/typescript/amazon-global-price-comparison/.env.example b/typescript/amazon-global-price-comparison/.env.example new file mode 100644 index 00000000..c0c758c0 --- /dev/null +++ b/typescript/amazon-global-price-comparison/.env.example @@ -0,0 +1,3 @@ +# Browserbase credentials - get these from https://www.browserbase.com/settings +BROWSERBASE_API_KEY=your_browserbase_api_key +BROWSERBASE_PROJECT_ID=your_browserbase_project_id diff --git a/typescript/amazon-global-price-comparison/README.md b/typescript/amazon-global-price-comparison/README.md new file mode 100644 index 00000000..23f589bc --- /dev/null +++ b/typescript/amazon-global-price-comparison/README.md @@ -0,0 +1,66 @@ +# Amazon Global Price Comparison + +## AT A GLANCE + +- Goal: compare Amazon product prices across multiple countries using geolocation proxies. +- Uses Browserbase's managed proxy infrastructure to route traffic through different geographic locations (US, UK, Germany, France, Italy, Spain). +- Extracts structured product data (name, price, rating, reviews) using Stagehand's extraction capabilities with Zod schema validation. +- Sequential processing shows how different proxy locations return different pricing from the same Amazon search. +- Docs → https://docs.browserbase.com/features/proxies + +## GLOSSARY + +- geolocation proxies: route traffic through specific geographic locations (city, country) to access location-specific content and pricing + Docs → https://docs.browserbase.com/features/proxies#set-proxy-geolocation +- extract: extract structured data from web pages using natural language instructions and Zod schemas + Docs → https://docs.stagehand.dev/basics/extract +- proxies: Browserbase's managed proxy infrastructure supporting 201+ countries for geolocation-based routing + Docs → https://docs.browserbase.com/features/proxies + +## QUICKSTART + +1. cd amazon-global-price-comparison +2. pnpm install +3. cp .env.example .env +4. Add your Browserbase API key and Project ID to .env +5. pnpm start + +## EXPECTED OUTPUT + +- Creates Browserbase sessions with geolocation proxies for each country (US, UK, DE, FR, IT, ES) +- Navigates to Amazon search results through location-specific proxies +- Extracts product name, price, rating, and review count for each location +- Displays formatted comparison table showing price differences across countries +- Outputs JSON results for programmatic use + +## COMMON PITFALLS + +- Browserbase Developer plan or higher is required to use proxies +- "Cannot find module": ensure all dependencies are installed (`pnpm install`) +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- Geolocation fields are case-insensitive (city, country can be any case) +- Amazon may show different products in different regions - comparison works best for globally available products +- ERR_TUNNEL_CONNECTION_FAILED: indicates either a temporary proxy hiccup or a site unsupported by built-in proxies + +## USE CASES + +• Price arbitrage: Find the best country to purchase products from for international shipping +• Market research: Compare pricing strategies across different Amazon regions +• Competitive analysis: Monitor how competitors price products globally +• Travel shopping: Check prices before international trips to plan purchases + +## NEXT STEPS + +• Add more countries: Extend the COUNTRIES array with additional regions (Japan, Australia, Canada, etc.) +• Currency conversion: Add real-time currency conversion to normalize prices for comparison +• Price tracking: Store results over time to track price changes across regions +• Email alerts: Send notifications when price drops below a threshold in any country + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/typescript/amazon-global-price-comparison/index.ts b/typescript/amazon-global-price-comparison/index.ts new file mode 100644 index 00000000..bf6ccb98 --- /dev/null +++ b/typescript/amazon-global-price-comparison/index.ts @@ -0,0 +1,266 @@ +// Amazon Global Price Comparison - See README.md for full documentation + +import "dotenv/config"; +import { Stagehand } from "@browserbasehq/stagehand"; +import { z } from "zod"; + +// Schema for a single product with structured extraction fields +const ProductSchema = z.object({ + name: z.string().describe("The full product title/name"), + price: z + .string() + .describe( + "The product price including currency symbol (e.g., '$29.99', '29,99 EUR', '29.99 GBP'). If no price is visible, return 'N/A'", + ), + rating: z.string().describe("The star rating (e.g., '4.5 out of 5 stars')"), + reviews_count: z.string().describe("The number of customer reviews (e.g., '1,234')"), + product_url: z + .string() + .describe("The full href URL link to the product detail page (starting with https:// or /dp/)"), +}); + +// Schema for extracting multiple products from search results +const ProductsSchema = z.object({ + products: z.array(ProductSchema).describe("Array of products from search results"), +}); + +type Product = z.infer; + +// Country configuration with geolocation proxy settings +// Each country routes traffic through its geographic location to see local pricing +interface CountryConfig { + name: string; + code: string; + city?: string; + currency: string; +} + +// Supported countries for price comparison +// Add or remove countries as needed - see https://docs.browserbase.com/features/proxies for available geolocations +const COUNTRIES: CountryConfig[] = [ + { name: "United States", code: "US", city: undefined, currency: "USD" }, + { name: "United Kingdom", code: "GB", city: "LONDON", currency: "GBP" }, + { name: "Germany", code: "DE", city: "BERLIN", currency: "EUR" }, + { name: "France", code: "FR", city: "PARIS", currency: "EUR" }, + { name: "Italy", code: "IT", city: "ROME", currency: "EUR" }, + { name: "Spain", code: "ES", city: "MADRID", currency: "EUR" }, +]; + +// Results structure for each country +interface CountryResult { + country: string; + countryCode: string; + currency: string; + products: Product[]; + error?: string; +} + +/** + * Fetches products from Amazon for a specific country using geolocation proxy + * Uses Browserbase's managed proxy infrastructure to route traffic through the target country + * This ensures Amazon shows location-specific pricing and availability + */ +async function getProductsForCountry( + searchQuery: string, + country: CountryConfig, + resultsCount: number = 3, +): Promise { + console.log(`\n=== Searching Amazon for "${searchQuery}" in ${country.name} ===`); + + // Build geolocation config for proxy routing + const geolocation: { country: string; city?: string } = { + country: country.code, + }; + if (country.city) { + geolocation.city = country.city; + } + + // Initialize Stagehand with geolocation proxy configuration + // This ensures all browser traffic routes through the specified geographic location + const stagehand = new Stagehand({ + env: "BROWSERBASE", + verbose: 0, + // 0 = errors only, 1 = info, 2 = debug + // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + // https://docs.stagehand.dev/configuration/logging + browserbaseSessionCreateParams: { + projectId: process.env.BROWSERBASE_PROJECT_ID!, + proxies: [ + { + type: "browserbase", // Use Browserbase's managed proxy infrastructure for reliable geolocation routing + geolocation, + }, + ], + }, + }); + + try { + console.log(`Initializing browser session with ${country.name} proxy...`); + await stagehand.init(); + + const page = stagehand.context.pages()[0]; + + // Alternative: Skip the search bar and go straight to results by building the search URL. + // Uncomment below to use direct navigation instead of stagehand.act() typing + clicking. + // const searchUrl = `https://www.amazon.com/s?k=${encodeURIComponent(searchQuery)}`; + // console.log(`Navigating to: ${searchUrl}`); + // await page.goto(searchUrl, { waitUntil: "domcontentloaded", timeout: 60000 }); + + // Navigate to Amazon homepage to begin search + console.log(`[${country.name}] Navigating to Amazon...`); + await page.goto("https://www.amazon.com", { + waitUntil: "domcontentloaded", + timeout: 60000, + }); + + // Perform search using natural language actions + console.log(`[${country.name}] Searching for: ${searchQuery}`); + await stagehand.act(`Type "${searchQuery}" into the search bar`); + await stagehand.act("Click the search button"); + + // Extract products from search results using Stagehand's structured extraction + console.log(`[${country.name}] Extracting top ${resultsCount} products...`); + + const extractionResult = await stagehand.extract( + `Extract the first ${resultsCount} product search results from this Amazon page. For each product, extract: + 1. name: the full product title + 2. price: the displayed price WITH currency symbol (like $599.99 or 599,99 EUR). If no price shown, use "N/A" + 3. rating: the star rating text (like "4.5 out of 5 stars") + 4. reviews_count: the number of reviews (like "2,508") + 5. product_url: the href link to the product page (starts with /dp/ or https://) + + Only extract actual product listings, skip sponsored ads or recommendations.`, + ProductsSchema, + ); + + // Clean up products - ensure price is never null and URLs are absolute + const cleanedProducts = extractionResult.products.map((p) => ({ + ...p, + price: p.price || "N/A", + product_url: p.product_url?.startsWith("http") + ? p.product_url + : p.product_url?.startsWith("/") + ? `https://www.amazon.com${p.product_url}` + : p.product_url || "N/A", + })); + + console.log(`Found ${cleanedProducts.length} products in ${country.name}`); + + await stagehand.close(); + + return { + country: country.name, + countryCode: country.code, + currency: country.currency, + products: cleanedProducts.slice(0, resultsCount), + }; + } catch (error) { + console.error(`Error fetching products from ${country.name}:`, error); + await stagehand.close(); + + return { + country: country.name, + countryCode: country.code, + currency: country.currency, + products: [], + error: error instanceof Error ? error.message : String(error), + }; + } +} + +/** + * Displays results in a formatted comparison table + * Shows product name, price, rating, and review count for each country + */ +function displayComparisonTable(results: CountryResult[]) { + console.log("\n" + "=".repeat(100)); + console.log("PRICE COMPARISON ACROSS COUNTRIES"); + console.log("=".repeat(100)); + + // Find the first successful result to get product count + const successfulResult = results.find((r) => r.products.length > 0); + if (!successfulResult) { + console.log("No products found in any country."); + return; + } + + // Display results for each product position + const maxProducts = Math.max(...results.map((r) => r.products.length)); + + for (let i = 0; i < maxProducts; i++) { + console.log(`\n--- Product ${i + 1} ---`); + + // Find the first available product name for this position + const productName = results.find((r) => r.products[i])?.products[i]?.name; + if (productName) { + const truncatedName = + productName.length > 80 ? productName.slice(0, 77) + "..." : productName; + console.log(`Product: ${truncatedName}`); + } + + console.log("\nPrices by Country:"); + console.log("-".repeat(70)); + + for (const result of results) { + const countryPad = result.country.padEnd(20); + if (result.error) { + console.log(` ${countryPad} | Error: ${result.error}`); + } else if (result.products[i]) { + const product = result.products[i]; + const price = product.price || "N/A"; + const pricePad = price.padEnd(18); + const ratingShort = product.rating?.split(" out")[0] || "N/A"; + const ratingPad = ratingShort.padEnd(6); + const reviews = product.reviews_count || "N/A"; + console.log(` ${countryPad} | ${pricePad} | ${ratingPad} stars | ${reviews} reviews`); + } else { + console.log(` ${countryPad} | Not available in this country`); + } + } + } + + console.log("\n" + "=".repeat(100)); +} + +async function main() { + // Configure search parameters + const searchQuery = "iPhone 15 Pro Max 256GB"; + const resultsCount = 3; + + console.log("=".repeat(60)); + console.log("AMAZON PRICE COMPARISON - GEOLOCATION PROXY DEMO"); + console.log("=".repeat(60)); + console.log(`Search Query: ${searchQuery}`); + console.log(`Results per country: ${resultsCount}`); + console.log(`Countries: ${COUNTRIES.map((c) => c.code).join(", ")}`); + console.log("=".repeat(60)); + + // Process all countries concurrently for faster execution + // Each country uses its own browser session, so they can run in parallel + console.log(`\nFetching prices from ${COUNTRIES.length} countries concurrently...`); + + const results = await Promise.all( + COUNTRIES.map((country) => getProductsForCountry(searchQuery, country, resultsCount)), + ); + + // Display formatted comparison table + displayComparisonTable(results); + + // Output JSON results for programmatic use + console.log("\n--- JSON OUTPUT ---"); + console.log(JSON.stringify(results, null, 2)); + + console.log("\n=== Price comparison completed ==="); +} + +main().catch((err) => { + console.error("Application error:", err); + console.error("\nCommon issues:"); + console.error(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY"); + console.error( + " - Verify geolocation proxy locations are valid (see https://docs.browserbase.com/features/proxies)", + ); + console.error(" - Ensure you have sufficient Browserbase credits"); + console.error("Docs: https://docs.stagehand.dev/v3/first-steps/introduction"); + process.exit(1); +}); diff --git a/typescript/amazon-global-price-comparison/package.json b/typescript/amazon-global-price-comparison/package.json new file mode 100644 index 00000000..c0e8a26c --- /dev/null +++ b/typescript/amazon-global-price-comparison/package.json @@ -0,0 +1,17 @@ +{ + "name": "amazon-global-price-comparison-template", + "type": "module", + "scripts": { + "build": "tsc", + "start": "tsx index.ts" + }, + "dependencies": { + "@browserbasehq/stagehand": "latest", + "dotenv": "^16.4.7", + "zod": "latest" + }, + "devDependencies": { + "tsx": "^4.19.2", + "typescript": "^5.0.0" + } +} From 98d2b64a4d4b7c5af1d7c2dc94a908c2256d7492 Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Mon, 2 Feb 2026 13:47:21 -0800 Subject: [PATCH 13/45] jaysahnan/gro-955-google-trends-template (#53) * init commit * addressing PR --- python/google-trends/.env.example | 6 + python/google-trends/README.md | 60 +++++++++ python/google-trends/main.py | 169 ++++++++++++++++++++++++++ python/google-trends/pyproject.toml | 33 +++++ typescript/google-trends/.env.example | 6 + typescript/google-trends/README.md | 61 ++++++++++ typescript/google-trends/index.ts | 112 +++++++++++++++++ typescript/google-trends/package.json | 31 +++++ 8 files changed, 478 insertions(+) create mode 100644 python/google-trends/.env.example create mode 100644 python/google-trends/README.md create mode 100644 python/google-trends/main.py create mode 100644 python/google-trends/pyproject.toml create mode 100644 typescript/google-trends/.env.example create mode 100644 typescript/google-trends/README.md create mode 100644 typescript/google-trends/index.ts create mode 100644 typescript/google-trends/package.json diff --git a/python/google-trends/.env.example b/python/google-trends/.env.example new file mode 100644 index 00000000..169cbbf3 --- /dev/null +++ b/python/google-trends/.env.example @@ -0,0 +1,6 @@ +# Browserbase credentials (required) +BROWSERBASE_PROJECT_ID=your-browserbase-project-id +BROWSERBASE_API_KEY=your-browserbase-api-key + +# Model API key (required) +GOOGLE_API_KEY=your-google-api-key diff --git a/python/google-trends/README.md b/python/google-trends/README.md new file mode 100644 index 00000000..7b1790db --- /dev/null +++ b/python/google-trends/README.md @@ -0,0 +1,60 @@ +# Stagehand + Browserbase: Google Trends Keywords Extractor + +## AT A GLANCE + +- Goal: Extract trending search keywords from Google Trends for any country with structured JSON output. +- Configurable by country code (US, GB, IN, DE, etc.) and language preference. +- Uses JSON schema validation for consistent, typed data extraction. +- Docs → https://docs.stagehand.dev/basics/extract + +## GLOSSARY + +- extract: extract structured data from web pages using natural language instructions and JSON schemas + Docs → https://docs.stagehand.dev/basics/extract +- act: perform UI actions from a prompt (click, type, dismiss dialogs) + Docs → https://docs.stagehand.dev/basics/act + +## QUICKSTART + +1. uv sync +2. cp .env.example .env # Add your API keys to .env +3. uv run python main.py + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase +- Displays live session link for monitoring +- Navigates to Google Trends trending page with configured country/language +- Dismisses any consent dialogs if present +- Extracts trending keywords with rank positions +- Outputs structured JSON with country code, language, timestamp, and keyword list +- Closes session cleanly + +## COMMON PITFALLS + +- "ModuleNotFoundError": ensure all dependencies are installed via `uv sync` +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY +- Invalid country code: ensure country code is a valid 2-letter ISO code (US, GB, IN, DE, FR, BR, etc.) +- Empty results: Google Trends may not have trending data for all country/language combinations +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +• Market research: Track trending topics across different regions to identify emerging interests and market opportunities. +• Content strategy: Discover popular search terms to inform blog posts, social media content, or SEO keyword targeting. +• Competitive intelligence: Monitor trending keywords in your industry to stay ahead of market shifts and consumer interests. + +## NEXT STEPS + +• Parameterize inputs: Accept country code and limit as command-line arguments or environment variables for flexible deployment. +• Historical tracking: Store extracted keywords with timestamps to build a trends database and analyze keyword momentum over time. +• Multi-region comparison: Extend to fetch trends from multiple countries in parallel and compare regional differences. + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/python/google-trends/main.py b/python/google-trends/main.py new file mode 100644 index 00000000..7144037c --- /dev/null +++ b/python/google-trends/main.py @@ -0,0 +1,169 @@ +# Stagehand + Browserbase: Google Trends Keywords Extractor - See README.md for full documentation + +import asyncio +import json +import os +from typing import List + +from dotenv import load_dotenv +from pydantic import BaseModel, Field +from stagehand import AsyncStagehand + + +class TrendingKeyword(BaseModel): + """Schema for a single trending keyword from Google Trends.""" + + rank: int = Field(description="Position in the trending list (1, 2, 3, etc.)") + keyword: str = Field(description="The main trending search term or keyword") + + +class TrendingKeywordsList(BaseModel): + """Schema for extracting a list of trending keywords.""" + + trending_keywords: List[TrendingKeyword] = Field( + description="List of trending keywords extracted from Google Trends" + ) + + +def dereference_schema(schema: dict) -> dict: + """Inline all $ref references in a JSON schema for Gemini compatibility.""" + defs = schema.pop("$defs", {}) + + def resolve_refs(obj): + if isinstance(obj, dict): + if "$ref" in obj: + ref_path = obj["$ref"].split("/")[-1] + return resolve_refs(defs.get(ref_path, {})) + return {k: resolve_refs(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [resolve_refs(item) for item in obj] + return obj + + return resolve_refs(schema) + + +# Load environment variables +load_dotenv() + +# Configuration variables +country_code = "US" # Two-letter ISO code (US, GB, IN, DE, FR, BR) +limit = 20 # Max keywords to return +language = "en-US" # Language code for results + + +async def main(): + """ + Extracts trending keywords from Google Trends for a specific country. + Uses Stagehand's structured extraction with JSON schema for type-safe data. + """ + print("Starting Google Trends Keywords Extractor...") + print(f"Country Code: {country_code}") + print(f"Language: {language}") + print(f"Limit: {limit} keywords") + + # Initialize AsyncStagehand client (v3 architecture) + client = AsyncStagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("GOOGLE_API_KEY"), + ) + + # Start a Stagehand session with Gemini model + start_response = await client.sessions.start(model_name="google/gemini-2.5-flash") + session_id = start_response.data.session_id + print(f"Stagehand session started: {session_id}") + + try: + # Provide live session URL for debugging and monitoring + print(f"Watch live: https://browserbase.com/sessions/{session_id}") + + # Build and navigate to Google Trends URL with country code and language + trends_url = f"https://trends.google.com/trending?geo={country_code.upper()}&hl={language}" + print(f"Navigating to: {trends_url}") + await client.sessions.navigate(id=session_id, url=trends_url) + print("Page loaded successfully") + + # Dismiss any consent/welcome dialogs that block content + try: + print("Checking for consent dialogs...") + await client.sessions.act( + id=session_id, + input='Click the "Got it" button if visible', + timeout=5.0, + ) + # Small delay to let the dialog close and content load + await asyncio.sleep(1.5) + except Exception: + # No dialog present, continue + print("No consent dialog found, continuing...") + + # Generate JSON schema from Pydantic model for structured extraction + schema = dereference_schema(TrendingKeywordsList.model_json_schema()) + + # Extract trending keywords using Stagehand's structured extraction + print("Extracting trending keywords from table...") + extract_response = await client.sessions.extract( + id=session_id, + instruction=( + f"Extract the trending search keywords from the Google Trends table. " + f"Each row has a trending topic/keyword shown as a button " + f"(like 'catherine ohara', 'don lemon arrested', 'fed chair', etc.). " + f"For each trend, extract the main keyword text and assign a rank " + f"starting from 1 for the first trend. Return up to {limit} items." + ), + schema=schema, + ) + + # Parse extraction results from wrapper model + extraction_result = extract_response.data.result + if isinstance(extraction_result, dict) and "trending_keywords" in extraction_result: + extracted_keywords = extraction_result["trending_keywords"] + limited_keywords = extracted_keywords[:limit] + else: + limited_keywords = [] + + print(f"Successfully extracted {len(limited_keywords)} trending keywords") + + # Build output structure with metadata + result = { + "country_code": country_code.upper(), + "language": language, + "extracted_at": __import__("datetime").datetime.now().isoformat(), + "trending_keywords": limited_keywords, + } + + # Display results in formatted JSON + print("\n=== Results ===") + print(json.dumps(result, indent=2)) + print(f"\nExtraction complete! Found {len(limited_keywords)} trending keywords.") + + except Exception as error: + print(f"Error extracting trending keywords: {error}") + + # Provide helpful troubleshooting information + print("\nCommon issues:") + print("1. Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") + print("2. Verify GOOGLE_API_KEY is set in environment") + print("3. Ensure country code is a valid 2-letter ISO code (US, GB, IN, DE, etc.)") + print("4. Verify Browserbase account has sufficient credits") + print("5. Check if Google Trends page structure has changed") + + raise + + finally: + # End the Stagehand session + print("Closing browser session...") + await client.sessions.end(id=session_id) + print("Session closed successfully") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Application error: {err}") + print("Common issues:") + print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") + print(" - Verify GOOGLE_API_KEY is set in environment") + print("Docs: https://docs.stagehand.dev/reference/python/overview") + exit(1) diff --git a/python/google-trends/pyproject.toml b/python/google-trends/pyproject.toml new file mode 100644 index 00000000..3a9b05cf --- /dev/null +++ b/python/google-trends/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "google-trends" +version = "0.1.0" +description = "Extract trending keywords from Google Trends using Stagehand and Browserbase" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "pydantic", + "python-dotenv", + "stagehand", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/typescript/google-trends/.env.example b/typescript/google-trends/.env.example new file mode 100644 index 00000000..169cbbf3 --- /dev/null +++ b/typescript/google-trends/.env.example @@ -0,0 +1,6 @@ +# Browserbase credentials (required) +BROWSERBASE_PROJECT_ID=your-browserbase-project-id +BROWSERBASE_API_KEY=your-browserbase-api-key + +# Model API key (required) +GOOGLE_API_KEY=your-google-api-key diff --git a/typescript/google-trends/README.md b/typescript/google-trends/README.md new file mode 100644 index 00000000..51240a94 --- /dev/null +++ b/typescript/google-trends/README.md @@ -0,0 +1,61 @@ +# Stagehand + Browserbase: Google Trends Keywords Extractor + +## AT A GLANCE + +- Goal: Extract trending search keywords from Google Trends for any country with structured JSON output. +- Configurable by country code (US, GB, IN, DE, etc.) and language preference. +- Uses Zod schema validation for consistent, typed data extraction. +- Docs → https://docs.stagehand.dev/basics/extract + +## GLOSSARY + +- extract: extract structured data from web pages using natural language instructions and Zod schemas + Docs → https://docs.stagehand.dev/basics/extract +- act: perform UI actions from a prompt (click, type, dismiss dialogs) + Docs → https://docs.stagehand.dev/basics/act + +## QUICKSTART + +1. pnpm install +2. cp .env.example .env +3. Add required API keys/IDs to .env +4. pnpm start + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase +- Displays live session link for monitoring +- Navigates to Google Trends trending page with configured country/language +- Dismisses any consent dialogs if present +- Extracts trending keywords with rank positions +- Outputs structured JSON with country code, language, timestamp, and keyword list +- Closes session cleanly + +## COMMON PITFALLS + +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY +- Invalid country code: ensure country code is a valid 2-letter ISO code (US, GB, IN, DE, FR, BR, etc.) +- Empty results: Google Trends may not have trending data for all country/language combinations +- Consent dialogs: script handles "Got it" dialogs automatically, but regional variations may require adjustment +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +• Market research: Track trending topics across different regions to identify emerging interests and market opportunities. +• Content strategy: Discover popular search terms to inform blog posts, social media content, or SEO keyword targeting. +• Competitive intelligence: Monitor trending keywords in your industry to stay ahead of market shifts and consumer interests. + +## NEXT STEPS + +• Parameterize inputs: Accept country code and limit as command-line arguments or environment variables for flexible deployment. +• Historical tracking: Store extracted keywords with timestamps to build a trends database and analyze keyword momentum over time. +• Multi-region comparison: Extend to fetch trends from multiple countries in parallel and compare regional differences. + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/typescript/google-trends/index.ts b/typescript/google-trends/index.ts new file mode 100644 index 00000000..307ad17c --- /dev/null +++ b/typescript/google-trends/index.ts @@ -0,0 +1,112 @@ +// Stagehand + Browserbase: Google Trends Keywords Extractor - See README.md for full documentation + +import "dotenv/config"; +import { Stagehand } from "@browserbasehq/stagehand"; +import { z } from "zod"; + +// Configuration variables +const countryCode = "US"; // Two-letter ISO code (US, GB, IN, DE, FR, BR) +const limit = 20; // Max keywords to return +const language = "en-US"; // Language code for results + +// Define Zod schema for structured data extraction +const TrendingKeywordSchema = z.object({ + rank: z.number().describe("Position in the trending list (1, 2, 3, etc.)"), + keyword: z.string().describe("The main trending search term or keyword"), +}); + +async function main() { + console.log("Starting Google Trends Keywords Extractor..."); + console.log(`Country Code: ${countryCode}`); + console.log(`Language: ${language}`); + console.log(`Limit: ${limit} keywords`); + + // Initialize Stagehand with Browserbase for cloud-based browser automation. + const stagehand = new Stagehand({ + env: "BROWSERBASE", + verbose: 1, + // 0 = errors only, 1 = info, 2 = debug + // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + // https://docs.stagehand.dev/configuration/logging + model: "google/gemini-2.5-flash", + }); + + try { + // Initialize browser session to start data extraction process. + await stagehand.init(); + console.log("Stagehand initialized successfully"); + + // Provide live session URL for debugging and monitoring extraction process. + console.log(`Watch live: https://browserbase.com/sessions/${stagehand.browserbaseSessionID}`); + + const page = stagehand.context.pages()[0]; + + // Build and navigate to Google Trends URL with country code and language. + const trendsUrl = `https://trends.google.com/trending?geo=${countryCode.toUpperCase()}&hl=${language}`; + console.log(`Navigating to: ${trendsUrl}`); + await page.goto(trendsUrl, { + waitUntil: "networkidle", + }); + console.log("Page loaded successfully"); + + // Dismiss any consent/welcome dialogs that block content. + try { + console.log("Checking for consent dialogs..."); + await stagehand.act('Click the "Got it" button if visible', { timeout: 5000 }); + // Small delay to let the dialog close and content load. + await new Promise((resolve) => setTimeout(resolve, 1500)); + } catch { + // No dialog present, continue. + console.log("No consent dialog found, continuing..."); + } + + // Extract trending keywords using Stagehand's structured extraction with Zod schema. + console.log("Extracting trending keywords from table..."); + const extractResult = await stagehand.extract( + `Extract the trending search keywords from the Google Trends table. Each row has a trending topic/keyword shown as a button (like "catherine ohara", "don lemon arrested", "fed chair", etc.). For each trend, extract the main keyword text and assign a rank starting from 1 for the first trend. Return up to ${limit} items.`, + z.array(TrendingKeywordSchema), + ); + + // Apply limit to results and build output structure. + const limitedKeywords = extractResult.slice(0, limit); + console.log(`Successfully extracted ${limitedKeywords.length} trending keywords`); + + // Output formatted results with metadata. + const result = { + country_code: countryCode.toUpperCase(), + language: language, + extracted_at: new Date().toISOString(), + trending_keywords: limitedKeywords, + }; + + console.log("\n=== Results ==="); + console.log(JSON.stringify(result, null, 2)); + console.log(`\nExtraction complete! Found ${limitedKeywords.length} trending keywords.`); + } catch (error) { + console.error("Error extracting trending keywords:", error); + + // Provide helpful troubleshooting information. + console.error("\nCommon issues:"); + console.error("1. Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY"); + console.error("2. Verify GOOGLE_API_KEY is set in environment"); + console.error("3. Ensure country code is a valid 2-letter ISO code (US, GB, IN, DE, etc.)"); + console.error("4. Verify Browserbase account has sufficient credits"); + console.error("5. Check if Google Trends page structure has changed"); + + throw error; + } finally { + // Always close session to release resources and clean up. + console.log("Closing browser session..."); + await stagehand.close(); + console.log("Session closed successfully"); + } +} + +main().catch((err) => { + console.error("Application error:", err); + console.error("Common issues:"); + console.error(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY"); + console.error(" - Verify country code is a valid 2-letter ISO code (US, GB, IN, DE, etc.)"); + console.error("Docs: https://docs.stagehand.dev/v3/first-steps/introduction"); + process.exit(1); +}); diff --git a/typescript/google-trends/package.json b/typescript/google-trends/package.json new file mode 100644 index 00000000..e8d3c53c --- /dev/null +++ b/typescript/google-trends/package.json @@ -0,0 +1,31 @@ +{ + "name": "google-trends-template", + "version": "1.0.0", + "description": "Stagehand + Browserbase: Google Trends Keywords Extractor", + "main": "index.ts", + "scripts": { + "start": "tsx index.ts", + "dev": "tsx watch index.ts" + }, + "keywords": [ + "stagehand", + "browserbase", + "automation", + "google-trends", + "keywords", + "seo", + "trending" + ], + "author": "", + "license": "MIT", + "dependencies": { + "@browserbasehq/stagehand": "^3.0.0", + "dotenv": "^16.0.0", + "zod": "^3.22.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "tsx": "^4.7.0", + "typescript": "^5.3.0" + } +} From 0dee55873136ef6a1a86991e01618f472f9cee46 Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Mon, 2 Feb 2026 14:06:38 -0800 Subject: [PATCH 14/45] jaysahnan/gro-951-sec-research-template (#51) * init commit * addressing PR comments --- python/sec-filing-research/.env.example | 9 + python/sec-filing-research/README.md | 93 +++++++++ python/sec-filing-research/main.py | 211 ++++++++++++++++++++ python/sec-filing-research/pyproject.toml | 33 +++ typescript/sec-filing-research/.env.example | 7 + typescript/sec-filing-research/README.md | 71 +++++++ typescript/sec-filing-research/index.ts | 180 +++++++++++++++++ typescript/sec-filing-research/package.json | 21 ++ 8 files changed, 625 insertions(+) create mode 100644 python/sec-filing-research/.env.example create mode 100644 python/sec-filing-research/README.md create mode 100644 python/sec-filing-research/main.py create mode 100644 python/sec-filing-research/pyproject.toml create mode 100644 typescript/sec-filing-research/.env.example create mode 100644 typescript/sec-filing-research/README.md create mode 100644 typescript/sec-filing-research/index.ts create mode 100644 typescript/sec-filing-research/package.json diff --git a/python/sec-filing-research/.env.example b/python/sec-filing-research/.env.example new file mode 100644 index 00000000..d0583656 --- /dev/null +++ b/python/sec-filing-research/.env.example @@ -0,0 +1,9 @@ +# Browserbase credentials (required) +# Get these from https://www.browserbase.com/settings +BROWSERBASE_PROJECT_ID=your-browserbase-project-id +BROWSERBASE_API_KEY=your-browserbase-api-key + +# Model API key (required) +# For Gemini: your Google API key +# For OpenAI: your OpenAI API key +MODEL_API_KEY=your-model-api-key diff --git a/python/sec-filing-research/README.md b/python/sec-filing-research/README.md new file mode 100644 index 00000000..7b44a8f3 --- /dev/null +++ b/python/sec-filing-research/README.md @@ -0,0 +1,93 @@ +# Stagehand + Browserbase: SEC Filing Research + +## AT A GLANCE + +- Goal: automate searching SEC EDGAR for a company and extracting recent filing metadata (type, date, description, accession number, file number). +- Search: supports company name, ticker symbol, or CIK number (e.g. "Apple Inc", "AAPL", "0000320193"). +- Data Extraction: uses Stagehand act/extract with Pydantic schemas to navigate SEC.gov and pull structured filing data. +- Output: company name, CIK, and a configurable number of most recent filings, printed as summary and JSON. + +## GLOSSARY + +- act: perform UI actions from a natural language prompt (click, type, submit). + Docs → https://docs.stagehand.dev/basics/act +- extract: pull structured data from web pages into validated objects using a JSON schema. + Docs → https://docs.stagehand.dev/basics/extract +- schema: JSON schema definition for filing and company info; enforces types and validation. +- SEC EDGAR: SEC’s company and filing search and filing system. + https://www.sec.gov/edgar/searchedgar/companysearch.html +- CIK: Central Index Key — unique numeric identifier for each company in EDGAR. + +## QUICKSTART + +1. cd python/sec-filing-research +2. Install dependencies with uv: + + ```bash + uv sync + ``` + + Or with pip in a venv: + + ```bash + python -m venv venv + source venv/bin/activate # On Windows: venv\Scripts\activate + pip install -e . + ``` + +3. cp .env.example .env +4. Add BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and MODEL_API_KEY to .env +5. (Optional) Edit SEARCH_QUERY and NUM_FILINGS in main.py +6. Run the script: + + ```bash + python main.py + ``` + + Or with uv: + + ```bash + uv run python main.py + ``` + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase and shows live view URL +- Navigates to SEC EDGAR company search +- Enters search query, submits, and selects the matching company +- Extracts company name and CIK from the filings page +- Extracts the N most recent filings (type, date, description, accession number, file number) +- Logs SEC FILING METADATA summary and per-filing details +- Outputs full result as JSON +- Closes session cleanly + +## COMMON PITFALLS + +- "ModuleNotFoundError": run `uv sync` or `pip install -e .` in sec-filing-research +- Missing credentials: ensure .env has BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and MODEL_API_KEY +- No company match: use a valid company name, ticker, or CIK; SEC search is case-sensitive for some queries +- Extraction errors: SEC page layout changes can break selectors; check live view and adjust act/extract prompts if needed +- Rate limiting: avoid excessive runs; SEC may throttle heavy or automated traffic + +## USE CASES + +• Compliance and due diligence: quickly pull recent 10-K, 10-Q, 8-K metadata for a list of companies. +• Research pipelines: feed accession numbers into downstream tools to fetch full filings or parse specific sections. +• Monitoring: periodically extract latest filings for watchlists and alert on new filings. +• Data enrichment: attach official company name and CIK to internal records using SEC as source of truth. + +## NEXT STEPS + +• Parameterize search: read SEARCH_QUERY and NUM_FILINGS from env or CLI for batch runs. +• Fetch full filings: use accession numbers with SEC’s full-text filing URLs or APIs to download documents. +• Multiple companies: loop over a list of tickers/names and aggregate results into a single report or JSON. +• Filter by type: restrict to 10-K/10-Q/8-K or other form types in the extract step or in post-processing. + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/python/sec-filing-research/main.py b/python/sec-filing-research/main.py new file mode 100644 index 00000000..a0d47c29 --- /dev/null +++ b/python/sec-filing-research/main.py @@ -0,0 +1,211 @@ +# Stagehand + Browserbase: SEC Filing Research - See README.md for full documentation + +import asyncio +import json +import os +from typing import List, Optional + +from dotenv import load_dotenv +from pydantic import BaseModel, Field +from stagehand import AsyncStagehand + + +class CompanyInfo(BaseModel): + """Schema for company information extraction.""" + + companyName: str = Field(description="Official company name") + cik: str = Field(description="Central Index Key (CIK) number") + + +class Filing(BaseModel): + """Schema for a single SEC filing.""" + + type: str = Field(description="Filing type (e.g., 10-K, 10-Q, 8-K)") + date: str = Field(description="Filing date in YYYY-MM-DD format") + description: str = Field(description="Full description of the filing") + accessionNumber: str = Field(description="SEC accession number") + fileNumber: Optional[str] = Field(default=None, description="File/Film number") + + +class FilingsList(BaseModel): + """Schema for extracting a list of SEC filings.""" + + filings: List[Filing] = Field(description="List of SEC filings") + + +def dereference_schema(schema: dict) -> dict: + """Inline all $ref references in a JSON schema for Gemini compatibility.""" + defs = schema.pop("$defs", {}) + + def resolve_refs(obj): + if isinstance(obj, dict): + if "$ref" in obj: + ref_path = obj["$ref"].split("/")[-1] + return resolve_refs(defs.get(ref_path, {})) + return {k: resolve_refs(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [resolve_refs(item) for item in obj] + return obj + + return resolve_refs(schema) + + +# Load environment variables from .env file +# Required: BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, MODEL_API_KEY +load_dotenv() + +# Search query - can be company name, ticker symbol, or CIK number +# Examples: "Apple Inc", "AAPL", "0000320193" +SEARCH_QUERY = "Apple Inc" + +# Number of filings to retrieve +NUM_FILINGS = 5 + + +async def main(): + """ + Searches SEC EDGAR for a company (by name, ticker, or CIK) and extracts + recent filing metadata: type, date, description, accession number, file number. + Uses Stagehand + Browserbase for AI-powered browser automation. + """ + print("Starting SEC Filing Research...") + print(f"Search query: {SEARCH_QUERY}") + print(f"Retrieving {NUM_FILINGS} most recent filings\n") + + # Initialize AsyncStagehand client (v3 architecture) + # Uses environment variables: BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, MODEL_API_KEY + client = AsyncStagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("MODEL_API_KEY") + or os.environ.get("GOOGLE_GENERATIVE_AI_API_KEY"), + ) + + # Start a new browser session + start_response = await client.sessions.start(model_name="google/gemini-2.5-flash") + session_id = start_response.data.session_id + print(f"Stagehand session started: {session_id}") + + try: + # Provide live session URL for debugging and monitoring + print(f"Live View: https://browserbase.com/sessions/{session_id}") + + # Navigate to modern SEC EDGAR company search page + print("\nNavigating to SEC EDGAR...") + await client.sessions.navigate( + id=session_id, + url="https://www.sec.gov/edgar/searchedgar/companysearch.html", + ) + + # Enter search query in the Company and Person Lookup search box + print(f"Searching for: {SEARCH_QUERY}") + await client.sessions.act( + id=session_id, + input="Click on the Company and Person Lookup search textbox", + ) + await client.sessions.act( + id=session_id, + input=f'Type "{SEARCH_QUERY}" in the search field', + ) + + # Submit search to load company results + await client.sessions.act(id=session_id, input="Click the search submit button") + + # Select the matching company from results to view their filings page + print("Selecting the correct company from results...") + await client.sessions.act( + id=session_id, + input=f'Click on "{SEARCH_QUERY}" in the search results to view their filings', + ) + + # Extract company information from the filings page + print("Extracting company information...") + company_info = {"companyName": SEARCH_QUERY, "cik": "Unknown"} + try: + extract_response = await client.sessions.extract( + id=session_id, + instruction="Extract the company name and CIK number from the page header or company information section. The CIK should be a numeric identifier.", + schema=dereference_schema(CompanyInfo.model_json_schema()), + ) + extracted = extract_response.data.result + if extracted and isinstance(extracted, dict) and extracted.get("companyName"): + company_info = extracted + except Exception as error: + print( + f"Could not extract company info, using search query as company name: {error}" + ) + + # Extract filing metadata from the filings table using structured schema + print(f"Extracting the {NUM_FILINGS} most recent filings...") + filings_response = await client.sessions.extract( + id=session_id, + instruction=f"Extract the {NUM_FILINGS} most recent SEC filings from the filings table. For each filing, get: the filing type (column: Filings, like 10-K, 10-Q, 8-K), the filing date (column: Filing Date), description, accession number (from the link or description), and file/film number if shown.", + schema=dereference_schema(FilingsList.model_json_schema()), + ) + filings_data = filings_response.data.result + + # Build result object with company info and normalized filing list + filings_list = ( + (filings_data.get("filings") or [])[:NUM_FILINGS] if filings_data else [] + ) + result = { + "company": company_info.get("companyName", SEARCH_QUERY), + "cik": company_info.get("cik", "Unknown"), + "searchQuery": SEARCH_QUERY, + "filings": [ + { + "type": f.get("type", ""), + "date": f.get("date", ""), + "description": f.get("description", ""), + "accessionNumber": f.get("accessionNumber", ""), + "fileNumber": f.get("fileNumber", ""), + } + for f in filings_list + ], + } + + # Log summary and per-filing details to console + print("\n" + "=" * 60) + print("SEC FILING METADATA") + print("=" * 60) + print(f"Company: {result['company']}") + print(f"CIK: {result['cik']}") + print(f"Search Query: {result['searchQuery']}") + print(f"Filings Retrieved: {len(result['filings'])}") + print("=" * 60) + + # Display each filing's type, date, description, accession number, file number + for index, filing in enumerate(result["filings"], start=1): + print(f"\nFiling {index}:") + print(f" Type: {filing['type']}") + print(f" Date: {filing['date']}") + desc = filing["description"] + print(f" Description: {desc[:80]}{'...' if len(desc) > 80 else ''}") + print(f" Accession Number: {filing['accessionNumber']}") + print(f" File Number: {filing['fileNumber']}") + + # Output full result as JSON for piping or integration + print("\n" + "=" * 60) + print("JSON OUTPUT:") + print("=" * 60) + print(json.dumps(result, indent=2)) + + finally: + # Always close session to release resources and clean up + await client.sessions.end(id=session_id) + print("\nSession closed successfully") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Application error: {err}") + # Provide helpful troubleshooting information + print("\nCommon issues:") + print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") + print(" - Verify MODEL_API_KEY is set (e.g. Google API key for Gemini)") + print(" - Verify internet connection and SEC website accessibility") + print(" - Ensure the search query is valid (company name, ticker, or CIK)") + print("Docs: https://docs.stagehand.dev/reference/python/overview") + exit(1) \ No newline at end of file diff --git a/python/sec-filing-research/pyproject.toml b/python/sec-filing-research/pyproject.toml new file mode 100644 index 00000000..df2d3f2c --- /dev/null +++ b/python/sec-filing-research/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "sec-filing-research" +version = "0.1.0" +description = "SEC Filing Research: Extract company and filing metadata from SEC EDGAR using Stagehand and Browserbase" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "pydantic", + "python-dotenv", + "stagehand", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/typescript/sec-filing-research/.env.example b/typescript/sec-filing-research/.env.example new file mode 100644 index 00000000..db189873 --- /dev/null +++ b/typescript/sec-filing-research/.env.example @@ -0,0 +1,7 @@ +# Browserbase credentials (required) +# Get these from https://www.browserbase.com/settings +BROWSERBASE_PROJECT_ID=your-browserbase-project-id +BROWSERBASE_API_KEY=your-browserbase-api-key + +# Google API key for Gemini model (required) +GOOGLE_API_KEY=your-google-api-key diff --git a/typescript/sec-filing-research/README.md b/typescript/sec-filing-research/README.md new file mode 100644 index 00000000..be1d7b8e --- /dev/null +++ b/typescript/sec-filing-research/README.md @@ -0,0 +1,71 @@ +# Stagehand + Browserbase: SEC Filing Research + +## AT A GLANCE + +- Goal: automate searching SEC EDGAR for a company and extracting recent filing metadata (type, date, description, accession number, file number). +- Search: supports company name, ticker symbol, or CIK number (e.g. "Apple Inc", "AAPL", "0000320193"). +- Data Extraction: uses Stagehand act/extract with Zod schemas to navigate SEC.gov and pull structured filing data. +- Output: company name, CIK, and a configurable number of most recent filings, printed as summary and JSON. + +## GLOSSARY + +- act: perform UI actions from a natural language prompt (click, type, submit). + Docs → https://docs.stagehand.dev/basics/act +- extract: pull structured data from web pages into validated objects using a Zod schema. + Docs → https://docs.stagehand.dev/basics/extract +- schema: Zod definition for filing and company info; enforces types and validation. + Docs → https://zod.dev/ +- SEC EDGAR: SEC’s company and filing search and filing system. + https://www.sec.gov/edgar/searchedgar/companysearch.html +- CIK: Central Index Key — unique numeric identifier for each company in EDGAR. + +## QUICKSTART + +1. cd sec-filing-research +2. npm install +3. cp .env.example .env +4. Add BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY to .env +5. (Optional) Edit SEARCH_QUERY and NUM_FILINGS in index.ts +6. npm start + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase and shows live view URL +- Navigates to SEC EDGAR company search +- Enters search query, submits, and selects the matching company +- Extracts company name and CIK from the filings page +- Extracts the N most recent filings (type, date, description, accession number, file number) +- Logs SEC FILING METADATA summary and per-filing details +- Outputs full result as JSON +- Closes session cleanly + +## COMMON PITFALLS + +- "Cannot find module": run npm install in sec-filing-research +- Missing credentials: ensure .env has BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY (for Gemini) +- No company match: use a valid company name, ticker, or CIK; SEC search is case-sensitive for some queries +- Extraction errors: SEC page layout changes can break selectors; check live view and adjust act/extract prompts if needed +- Rate limiting: avoid excessive runs; SEC may throttle heavy or automated traffic + +## USE CASES + +• Compliance and due diligence: quickly pull recent 10-K, 10-Q, 8-K metadata for a list of companies. +• Research pipelines: feed accession numbers into downstream tools to fetch full filings or parse specific sections. +• Monitoring: periodically extract latest filings for watchlists and alert on new filings. +• Data enrichment: attach official company name and CIK to internal records using SEC as source of truth. + +## NEXT STEPS + +• Parameterize search: read SEARCH_QUERY and NUM_FILINGS from env or CLI for batch runs. +• Fetch full filings: use accession numbers with SEC’s full-text filing URLs or APIs to download documents. +• Multiple companies: loop over a list of tickers/names and aggregate results into a single report or JSON. +• Filter by type: restrict to 10-K/10-Q/8-K or other form types in the extract step or in post-processing. + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/typescript/sec-filing-research/index.ts b/typescript/sec-filing-research/index.ts new file mode 100644 index 00000000..5cbf5fd8 --- /dev/null +++ b/typescript/sec-filing-research/index.ts @@ -0,0 +1,180 @@ +// Stagehand + Browserbase: SEC Filing Downloader - See README.md for full documentation + +import "dotenv/config"; +import { Stagehand } from "@browserbasehq/stagehand"; +import { z } from "zod"; + +// Search query - can be company name, ticker symbol, or CIK number +// Examples: "Apple Inc", "AAPL", "0000320193" +const SEARCH_QUERY = "Apple Inc"; + +// Number of filings to retrieve +const NUM_FILINGS = 5; + +// Schema for extracted filing data +const FilingSchema = z.object({ + filings: z.array( + z.object({ + type: z.string().describe("Filing type (e.g., 10-K, 10-Q, 8-K)"), + date: z.string().describe("Filing date in YYYY-MM-DD format"), + description: z.string().describe("Full description of the filing"), + accessionNumber: z.string().describe("SEC accession number"), + fileNumber: z.string().optional().describe("File/Film number"), + }), + ), +}); + +// Schema for company info extraction +const CompanyInfoSchema = z.object({ + companyName: z.string().describe("Official company name"), + cik: z.string().describe("Central Index Key (CIK) number"), +}); + +// Result shape returned after extracting company and filing metadata from SEC EDGAR +interface SECFilingResult { + company: string; + cik: string; + searchQuery: string; + filings: Array<{ + type: string; + date: string; + description: string; + accessionNumber: string; + fileNumber: string; + }>; +} + +/** + * Searches SEC EDGAR for a company (by name, ticker, or CIK) and extracts + * recent filing metadata: type, date, description, accession number, file number. + * Uses Stagehand + Browserbase for AI-powered browser automation. + */ +async function main(): Promise { + console.log("Starting SEC Filing Downloader..."); + console.log(`Search query: ${SEARCH_QUERY}`); + console.log(`Retrieving ${NUM_FILINGS} most recent filings\n`); + + // Initialize Stagehand with Browserbase for cloud-based browser automation + const stagehand = new Stagehand({ + env: "BROWSERBASE", + apiKey: process.env.BROWSERBASE_API_KEY, + projectId: process.env.BROWSERBASE_PROJECT_ID, + verbose: 1, + // 0 = errors only, 1 = info, 2 = debug + // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + // https://docs.stagehand.dev/configuration/logging + model: "google/gemini-2.5-flash", + }); + + try { + // Initialize browser session + await stagehand.init(); + console.log("Stagehand initialized successfully!"); + + const page = stagehand.context.pages()[0]; + + // Provide live session URL for debugging and monitoring + if (stagehand.browserbaseSessionId) { + console.log(`Live View: https://browserbase.com/sessions/${stagehand.browserbaseSessionId}`); + } + + // Navigate to modern SEC EDGAR company search page + console.log("\nNavigating to SEC EDGAR..."); + await page.goto("https://www.sec.gov/edgar/searchedgar/companysearch.html", { + waitUntil: "domcontentloaded", + }); + + // Enter search query in the Company and Person Lookup search box + console.log(`Searching for: ${SEARCH_QUERY}`); + await stagehand.act(`Click on the Company and Person Lookup search textbox`); + await stagehand.act(`Type "${SEARCH_QUERY}" in the search field`); + + // Submit search to load company results + await stagehand.act("Click the search submit button"); + + // Select the matching company from results to view their filings page + console.log("Selecting the correct company from results..."); + await stagehand.act(`Click on "${SEARCH_QUERY}" in the search results to view their filings`); + + // Extract company information from the filings page + console.log("Extracting company information..."); + let companyInfo = { companyName: SEARCH_QUERY, cik: "Unknown" }; + + try { + const extractedInfo = await stagehand.extract( + "Extract the company name and CIK number from the page header or company information section. The CIK should be a numeric identifier.", + CompanyInfoSchema, + ); + if (extractedInfo && extractedInfo.companyName) { + companyInfo = extractedInfo; + } + } catch (error) { + // Fallback to search query if extraction fails (e.g. page layout differs) + console.log("Could not extract company info, using search query as company name:", error); + } + + // Extract filing metadata from the filings table using structured schema + console.log(`Extracting the ${NUM_FILINGS} most recent filings...`); + const filingsData = await stagehand.extract( + `Extract the ${NUM_FILINGS} most recent SEC filings from the filings table. For each filing, get: the filing type (column: Filings, like 10-K, 10-Q, 8-K), the filing date (column: Filing Date), description, accession number (from the link or description), and file/film number if shown.`, + FilingSchema, + ); + + // Build result object with company info and normalized filing list + const result: SECFilingResult = { + company: companyInfo.companyName, + cik: companyInfo.cik, + searchQuery: SEARCH_QUERY, + filings: filingsData.filings.slice(0, NUM_FILINGS).map((f) => ({ + ...f, + fileNumber: f.fileNumber || "", + })), + }; + + // Log summary and per-filing details to console + console.log("\n" + "=".repeat(60)); + console.log("SEC FILING METADATA"); + console.log("=".repeat(60)); + console.log(`Company: ${result.company}`); + console.log(`CIK: ${result.cik}`); + console.log(`Search Query: ${result.searchQuery}`); + console.log(`Filings Retrieved: ${result.filings.length}`); + console.log("=".repeat(60)); + + // Display each filing's type, date, description, accession number, file number + result.filings.forEach((filing, index) => { + console.log(`\nFiling ${index + 1}:`); + console.log(` Type: ${filing.type}`); + console.log(` Date: ${filing.date}`); + console.log( + ` Description: ${filing.description.substring(0, 80)}${filing.description.length > 80 ? "..." : ""}`, + ); + console.log(` Accession Number: ${filing.accessionNumber}`); + console.log(` File Number: ${filing.fileNumber}`); + }); + + // Output full result as JSON for piping or integration + console.log("\n" + "=".repeat(60)); + console.log("JSON OUTPUT:"); + console.log("=".repeat(60)); + console.log(JSON.stringify(result, null, 2)); + } catch (error) { + console.error("Error during SEC filing extraction:", error); + throw error; + } finally { + // Always close session to release resources and clean up + await stagehand.close(); + console.log("\nSession closed successfully"); + } +} + +main().catch((err) => { + console.error("Application error:", err); + // Provide helpful troubleshooting information + console.error("\nCommon issues:"); + console.error(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY"); + console.error(" - Verify internet connection and SEC website accessibility"); + console.error(" - Ensure the search query is valid (company name, ticker, or CIK)"); + console.error("\nDocs: https://docs.stagehand.dev/v3/first-steps/introduction"); + process.exit(1); +}); diff --git a/typescript/sec-filing-research/package.json b/typescript/sec-filing-research/package.json new file mode 100644 index 00000000..3fd3528c --- /dev/null +++ b/typescript/sec-filing-research/package.json @@ -0,0 +1,21 @@ +{ + "name": "sec-filing-research", + "version": "1.0.0", + "description": "Stagehand + Browserbase: SEC Filing Research - Extract company and filing metadata from SEC EDGAR", + "type": "module", + "main": "index.ts", + "scripts": { + "start": "tsx index.ts", + "dev": "tsx watch index.ts" + }, + "dependencies": { + "@browserbasehq/stagehand": "^3.0.0", + "dotenv": "^16.4.5", + "zod": "^3.23.8" + }, + "devDependencies": { + "@types/node": "^20.14.0", + "tsx": "^4.16.0", + "typescript": "^5.5.0" + } +} From ce0d85cb88684478179aed7c04c9a11a3a7e50f3 Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Mon, 2 Feb 2026 14:14:10 -0800 Subject: [PATCH 15/45] jaysahnan/gro-950-amazon-product-fetcher-template (#50) * inti commit * formating * addressing PR comments --- python/amazon-product-scraping/.env.example | 10 ++ python/amazon-product-scraping/README.md | 66 +++++++++ python/amazon-product-scraping/main.py | 138 ++++++++++++++++++ python/amazon-product-scraping/pyproject.toml | 33 +++++ .../amazon-product-scraping/.env.example | 6 + typescript/amazon-product-scraping/README.md | 63 ++++++++ typescript/amazon-product-scraping/index.ts | 92 ++++++++++++ .../amazon-product-scraping/package.json | 21 +++ 8 files changed, 429 insertions(+) create mode 100644 python/amazon-product-scraping/.env.example create mode 100644 python/amazon-product-scraping/README.md create mode 100644 python/amazon-product-scraping/main.py create mode 100644 python/amazon-product-scraping/pyproject.toml create mode 100644 typescript/amazon-product-scraping/.env.example create mode 100644 typescript/amazon-product-scraping/README.md create mode 100644 typescript/amazon-product-scraping/index.ts create mode 100644 typescript/amazon-product-scraping/package.json diff --git a/python/amazon-product-scraping/.env.example b/python/amazon-product-scraping/.env.example new file mode 100644 index 00000000..f5cd8426 --- /dev/null +++ b/python/amazon-product-scraping/.env.example @@ -0,0 +1,10 @@ +# Browserbase credentials (required) +# Get these from https://www.browserbase.com/settings +BROWSERBASE_PROJECT_ID=your-browserbase-project-id +BROWSERBASE_API_KEY=your-browserbase-api-key + +# Model API key (required - use one of these) +# For Google Gemini models +MODEL_API_KEY=your-google-api-key +# Or alternatively: +# GOOGLE_API_KEY=your-google-api-key diff --git a/python/amazon-product-scraping/README.md b/python/amazon-product-scraping/README.md new file mode 100644 index 00000000..d81f8d4a --- /dev/null +++ b/python/amazon-product-scraping/README.md @@ -0,0 +1,66 @@ +# Stagehand + Browserbase: Amazon Product Scraping + +## AT A GLANCE + +- **Goal**: Scrape the first 3 Amazon search results for a given query and return structured product data. +- **AI-Powered Search**: Uses Stagehand `act` to type in the search bar and click search (or optionally navigate directly to the search URL). +- **Structured Extraction**: Uses `extract` with a JSON schema to get product name, price, rating, review count, and product URL. +- **Model**: Uses `google/gemini-2.5-flash` for fast, cost-effective automation. +- Docs → https://docs.stagehand.dev + +## GLOSSARY + +- **act**: Perform UI actions from a prompt (type in search bar, click search). + Docs → https://docs.stagehand.dev/basics/act +- **extract**: Pull structured data from pages using JSON schemas. + Docs → https://docs.stagehand.dev/basics/extract + +## QUICKSTART + +1. cd python/amazon-product-scraping +2. cp .env.example .env (or create .env with required keys) +3. Add `BROWSERBASE_PROJECT_ID`, `BROWSERBASE_API_KEY`, and `MODEL_API_KEY` (or `GOOGLE_API_KEY`) to .env +4. Optionally edit `SEARCH_QUERY` in main.py +5. Run the script: + ```bash + uv run python main.py + ``` + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase +- Displays live session link for monitoring +- Navigates to Amazon and performs search (or direct URL navigation if uncommented) +- Extracts the first 3 products with name, price, rating, reviews count, and product URL +- Outputs JSON to console +- Closes session cleanly + +## COMMON PITFALLS + +- **Import errors**: Ensure you're running with `uv run python main.py` so dependencies are installed +- **Missing credentials**: Verify .env contains `BROWSERBASE_PROJECT_ID`, `BROWSERBASE_API_KEY`, and `MODEL_API_KEY` (or `GOOGLE_API_KEY`) +- **Google API access**: Ensure you have access to the gemini-2.5-flash model +- **Amazon layout changes**: Extraction may need prompt/schema updates if Amazon changes their search results UI +- Find more information on your Browserbase dashboard → https://www.browserbase.com/sign-in + +## USE CASES + +• **Price monitoring**: Scrape top results for a product query to track prices and availability over time. +• **Competitor research**: Extract product titles, ratings, and review counts for comparison. +• **Catalog building**: Pull structured product data for feeds, dashboards, or internal tools. + +## NEXT STEPS + +• **Switch to direct URL**: Uncomment the URL-based search block in main.py for faster runs without LLM search actions. +• **Parameterize query**: Accept `SEARCH_QUERY` from CLI or env for different products without editing code. +• **Paginate**: Extend extraction to multiple pages or increase the number of products per run. + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev +📚 Python SDK: https://docs.stagehand.dev/reference/python/overview +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/python/amazon-product-scraping/main.py b/python/amazon-product-scraping/main.py new file mode 100644 index 00000000..576030ef --- /dev/null +++ b/python/amazon-product-scraping/main.py @@ -0,0 +1,138 @@ +# Stagehand + Browserbase: Amazon Product Scraping +# See README.md for full documentation + +import asyncio +import json +import os +from typing import List + +from dotenv import load_dotenv +from pydantic import BaseModel, Field +from stagehand import AsyncStagehand + + +class Product(BaseModel): + """Schema for a single Amazon product.""" + + name: str = Field(description="The full product title/name") + price: str = Field(description="The product price including currency symbol (e.g., '$29.99')") + rating: str = Field(description="The star rating (e.g., '4.5 out of 5 stars')") + reviews_count: str = Field(description="The number of customer reviews (e.g., '1,234')") + product_url: str = Field(description="The URL link to the product detail page on Amazon") + + +class ProductsList(BaseModel): + """Schema for extracting a list of Amazon products.""" + + products: List[Product] = Field(description="Array of the first 3 products from search results") + + +def dereference_schema(schema: dict) -> dict: + """Inline all $ref references in a JSON schema for Gemini compatibility.""" + defs = schema.pop("$defs", {}) + + def resolve_refs(obj): + if isinstance(obj, dict): + if "$ref" in obj: + ref_path = obj["$ref"].split("/")[-1] + return resolve_refs(defs.get(ref_path, {})) + return {k: resolve_refs(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [resolve_refs(item) for item in obj] + return obj + + return resolve_refs(schema) + + +# Load environment variables from .env file +# Required: BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, MODEL_API_KEY (or GOOGLE_API_KEY) +load_dotenv() + +# ============= CONFIGURATION ============= +# Update this value to search for different products +SEARCH_QUERY = "Seiko 5" +# ========================================= + + +async def main(): + """ + Main application entry point. + + Orchestrates Amazon product scraping automation: + 1. Initializes Stagehand with Browserbase for cloud browser automation + 2. Navigates to Amazon and performs a product search + 3. Extracts structured product data (name, price, rating, reviews, URL) + 4. Outputs results as JSON + """ + print("Starting Amazon Product Scraping...") + + # Initialize AsyncStagehand client (v3 BYOB architecture) + # Uses environment variables: BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, MODEL_API_KEY + client = AsyncStagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("MODEL_API_KEY") or os.environ.get("GOOGLE_API_KEY"), + ) + + # Start a Stagehand session with the specified model + start_response = await client.sessions.start(model_name="google/gemini-2.5-flash") + session_id = start_response.data.session_id + print("Stagehand initialized successfully!") + print(f"Live View Link: https://browserbase.com/sessions/{session_id}") + + try: + # Alternative: skip the search bar and go straight to results by building the search URL. + # Uncomment below to use direct navigation instead of stagehand act() typing + clicking. + # from urllib.parse import quote_plus + # encoded_query = quote_plus(SEARCH_QUERY) + # search_url = f"https://www.amazon.com/s?k={encoded_query}" + # print(f"Navigating to: {search_url}") + # await client.sessions.navigate(id=session_id, url=search_url) + + # Navigate to Amazon homepage to begin search + print("Navigating to Amazon...") + await client.sessions.navigate(id=session_id, url="https://www.amazon.com") + + # Perform search using natural language actions + print(f"Searching for: {SEARCH_QUERY}") + await client.sessions.act(id=session_id, input=f"Type {SEARCH_QUERY} into the search bar") + await client.sessions.act(id=session_id, input="Click the search button") + + # Extract structured product data using JSON schema for type safety + print("Extracting product data...") + extract_response = await client.sessions.extract( + id=session_id, + instruction=( + "Extract the details of the FIRST 3 products in the search results. " + "Get the product name, price, star rating, number of reviews, " + "and the URL link to the product page." + ), + schema=dereference_schema(ProductsList.model_json_schema()), + ) + + # Display extracted products as formatted JSON + products = extract_response.data.result + print("Products found:") + print(json.dumps(products, indent=2)) + + except Exception as error: + print(f"Error during product scraping: {error}") + raise + + finally: + # Always close session to release resources and clean up + await client.sessions.end(id=session_id) + print("Session closed successfully") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Error in Amazon product scraping: {err}") + print("Common issues:") + print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") + print(" - Verify MODEL_API_KEY or GOOGLE_API_KEY is set for the model") + print(" - Verify network connectivity") + print("Docs: https://docs.stagehand.dev") + exit(1) diff --git a/python/amazon-product-scraping/pyproject.toml b/python/amazon-product-scraping/pyproject.toml new file mode 100644 index 00000000..2927acb5 --- /dev/null +++ b/python/amazon-product-scraping/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "amazon-product-scraping" +version = "0.1.0" +description = "Scrape Amazon product search results using Stagehand and Browserbase" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "pydantic", + "python-dotenv", + "stagehand", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/typescript/amazon-product-scraping/.env.example b/typescript/amazon-product-scraping/.env.example new file mode 100644 index 00000000..bab7b243 --- /dev/null +++ b/typescript/amazon-product-scraping/.env.example @@ -0,0 +1,6 @@ +# Browserbase Configuration +BROWSERBASE_PROJECT_ID=your_browserbase_project_id +BROWSERBASE_API_KEY=your_browserbase_api_key + +# Google API key for gemini-2.5-flash model +GOOGLE_API_KEY=your_google_api_key diff --git a/typescript/amazon-product-scraping/README.md b/typescript/amazon-product-scraping/README.md new file mode 100644 index 00000000..1fc5c389 --- /dev/null +++ b/typescript/amazon-product-scraping/README.md @@ -0,0 +1,63 @@ +# Stagehand + Browserbase: Amazon Product Scraping + +## AT A GLANCE + +- Goal: scrape the first 3 Amazon search results for a given query and return structured product data. +- AI-Powered Search: uses Stagehand `act` to type in the search bar and click search (or optionally navigate directly to the search URL). +- Structured Extraction: uses `extract` with a Zod schema to get product name, price, rating, review count, and product URL. +- Model: uses `google/gemini-2.5-flash` for fast, cost-effective automation. + Docs → https://docs.stagehand.dev + +## GLOSSARY + +- act: perform UI actions from a prompt (type in search bar, click search) + Docs → https://docs.stagehand.dev/basics/act +- extract: pull structured data from pages using schemas + Docs → https://docs.stagehand.dev/basics/extract + +## QUICKSTART + +1. cd typescript/amazon-product-scraping +2. npm install +3. cp .env.example .env (or create .env with required keys) +4. Add BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY to .env +5. Optionally edit SEARCH_QUERY in index.ts +6. npm start + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase +- Displays live session link for monitoring +- Navigates to Amazon and performs search (or direct URL navigation if uncommented) +- Extracts the first 3 products with name, price, rating, reviews count, and product URL +- Outputs JSON to console +- Closes session cleanly + +## COMMON PITFALLS + +- "Cannot find module": ensure npm install completed +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY +- Google API access: ensure you have access to the gemini-2.5-flash model +- Amazon layout changes: extraction may need prompt/schema updates if Amazon changes their search results UI +- Find more information on your Browserbase dashboard → https://www.browserbase.com/sign-in + +## USE CASES + +• Price monitoring: Scrape top results for a product query to track prices and availability over time. +• Competitor research: Extract product titles, ratings, and review counts for comparison. +• Catalog building: Pull structured product data for feeds, dashboards, or internal tools. + +## NEXT STEPS + +• Switch to direct URL: Uncomment the URL-based search block in index.ts for faster runs without LLM search actions. +• Parameterize query: Accept SEARCH_QUERY from CLI or env for different products without editing code. +• Paginate: Extend extraction to multiple pages or increase the number of products per run. + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/typescript/amazon-product-scraping/index.ts b/typescript/amazon-product-scraping/index.ts new file mode 100644 index 00000000..3de964cb --- /dev/null +++ b/typescript/amazon-product-scraping/index.ts @@ -0,0 +1,92 @@ +// Stagehand + Browserbase: Amazon Product Scraping - See README.md for full documentation + +import "dotenv/config"; +import { Stagehand } from "@browserbasehq/stagehand"; +import { z } from "zod"; + +// ============= CONFIGURATION ============= +// Update this value to search for different products +const SEARCH_QUERY = "Seiko 5"; +// ========================================= + +// Schema for a single product with structured extraction fields +const ProductSchema = z.object({ + name: z.string().describe("The full product title/name"), + price: z.string().describe("The product price including currency symbol (e.g., '$29.99')"), + rating: z.string().describe("The star rating (e.g., '4.5 out of 5 stars')"), + reviews_count: z.string().describe("The number of customer reviews (e.g., '1,234')"), + product_url: z.string().url().describe("The URL link to the product detail page on Amazon"), +}); + +// Schema for extracting multiple products from search results +const ProductsSchema = z.object({ + products: z.array(ProductSchema).describe("Array of the first 3 products from search results"), +}); + +async function main(): Promise { + console.log("Starting Amazon Product Scraping..."); + + // Initialize Stagehand with Browserbase for cloud-based browser automation. + const stagehand = new Stagehand({ + env: "BROWSERBASE", + verbose: 1, + model: "google/gemini-2.5-flash", + }); + + try { + // Initialize browser session to start automation. + await stagehand.init(); + console.log("Stagehand initialized successfully!"); + console.log( + `Live View Link: https://browserbase.com/sessions/${stagehand.browserbaseSessionID}`, + ); + + const page = stagehand.context.pages()[0]; + + // Alternative: skip the search bar and go straight to results by building the search URL. + // Uncomment below to use direct navigation instead of stagehand.act() typing + clicking. + // // Build search URL + // const encodedQuery = encodeURIComponent(query).replace(/%20/g, "+"); + // const searchUrl = `https://www.amazon.com/s?k=${encodedQuery}`; + + // console.log(`Navigating to: ${searchUrl}`); + // await page.goto(searchUrl, { + // waitUntil: "domcontentloaded", + // }); + + // Navigate to Amazon homepage to begin search. + console.log("Navigating to Amazon..."); + await page.goto("https://www.amazon.com"); + + // Perform search using natural language actions. + console.log(`Searching for: ${SEARCH_QUERY}`); + await stagehand.act(`Type ${SEARCH_QUERY} into the search bar`); + await stagehand.act("Click the search button"); + + // Extract structured product data using Zod schema for type safety. + console.log("Extracting product data..."); + const products = await stagehand.extract( + "Extract the details of the FIRST 3 products in the search results. Get the product name, price, star rating, number of reviews, and the URL link to the product page.", + ProductsSchema, + ); + + console.log("Products found:"); + console.log(JSON.stringify(products, null, 2)); + } catch (error) { + console.error("Error during product scraping:", error); + } finally { + // Always close session to release resources and clean up. + await stagehand.close(); + console.log("Session closed successfully"); + } +} + +main().catch((err) => { + console.error("Error in Amazon product scraping:", err); + console.error("Common issues:"); + console.error(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY"); + console.error(" - Verify GOOGLE_API_KEY is set for the model"); + console.error(" - Verify network connectivity"); + console.error("Docs: https://docs.stagehand.dev"); + process.exit(1); +}); diff --git a/typescript/amazon-product-scraping/package.json b/typescript/amazon-product-scraping/package.json new file mode 100644 index 00000000..e3e97251 --- /dev/null +++ b/typescript/amazon-product-scraping/package.json @@ -0,0 +1,21 @@ +{ + "name": "amazon-product-scraping", + "version": "1.0.0", + "description": "Stagehand + Browserbase: Amazon Product Scraping", + "type": "module", + "main": "index.ts", + "scripts": { + "start": "tsx index.ts", + "dev": "tsx watch index.ts" + }, + "dependencies": { + "@browserbasehq/stagehand": "^3.0.0", + "dotenv": "^16.4.5", + "zod": "^3.23.8" + }, + "devDependencies": { + "@types/node": "^20.14.0", + "tsx": "^4.16.0", + "typescript": "^5.5.0" + } +} From e290fd3d03c744507bedc565987db50920b68c8e Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Tue, 3 Feb 2026 15:24:58 -0800 Subject: [PATCH 16/45] Exa-Browserbase-Template (#34) * init commit * added python version * Replace exa-browserbase-1 with exa-browserbase (TypeScript) * adding package.json, Exa import fix * addressing Pr comments --- python/exa-browserbase/.env.example | 12 + python/exa-browserbase/Dummy_CV.pdf | Bin 0 -> 2358 bytes python/exa-browserbase/README.md | 63 ++++ python/exa-browserbase/main.py | 443 ++++++++++++++++++++++++ python/exa-browserbase/pyproject.toml | 34 ++ typescript/exa-browserbase/Dummy_CV.pdf | Bin 0 -> 2358 bytes typescript/exa-browserbase/README.md | 62 ++++ typescript/exa-browserbase/index.ts | 367 ++++++++++++++++++++ typescript/exa-browserbase/package.json | 22 ++ 9 files changed, 1003 insertions(+) create mode 100644 python/exa-browserbase/.env.example create mode 100644 python/exa-browserbase/Dummy_CV.pdf create mode 100644 python/exa-browserbase/README.md create mode 100644 python/exa-browserbase/main.py create mode 100644 python/exa-browserbase/pyproject.toml create mode 100644 typescript/exa-browserbase/Dummy_CV.pdf create mode 100644 typescript/exa-browserbase/README.md create mode 100644 typescript/exa-browserbase/index.ts create mode 100644 typescript/exa-browserbase/package.json diff --git a/python/exa-browserbase/.env.example b/python/exa-browserbase/.env.example new file mode 100644 index 00000000..9524c689 --- /dev/null +++ b/python/exa-browserbase/.env.example @@ -0,0 +1,12 @@ +# Browserbase credentials +BROWSERBASE_PROJECT_ID=your-browserbase-project-id +BROWSERBASE_API_KEY=your-browserbase-api-key + +# LLM API key (any of these work) +MODEL_API_KEY=your-model-api-key +# Or use Google AI key aliases: +# GOOGLE_API_KEY=your-google-api-key +# GOOGLE_GENERATIVE_AI_API_KEY=your-google-api-key + +# Exa API key from https://dashboard.exa.ai/api-keys +EXA_API_KEY=your-exa-api-key diff --git a/python/exa-browserbase/Dummy_CV.pdf b/python/exa-browserbase/Dummy_CV.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fc455c206dabbeecb499c81d71c1d2fbe549dbb3 GIT binary patch literal 2358 zcma)8TX*6(5`O1bh+BYw@g3U)=wOI}aKD5Q%uTih7>sRfGu^!p`v-R4=6!za%5>7h zN$>8A4+lx5s;^3)N-8A>b%mhwTqODXzyI|I7$Mj3f)3O`1KG%f0WtvA0EVMqPRI_x zJ_y_*Nq+tMHSb+@EvV&n=MX%jvWcwUP+;m1kWaSm7^b}g3#e)g{$K$ZUGZ^c} zRp|QvxFTf7GX9P~=5PArZv547D3~xGh~fGH*+qs4W#=1E@t6X<#OE1-F94CF^P(t; zU9~L9zI~Pw!KDcrk(iD=6u?C)_k5Pc3$Z&JW3HpX9Y@Of^$?t9Ly={Rd zCIpZpC%~JD&B=)e?JHy`kAM&A2H5z9Iv%TjP!~>h9Ru_G;-j8p<4!2@KLY*1eGLZC za&|Wug&U0OCD=P2NH5aRcI@vb=i)Ci89>%N1OwBt>tS^1dXb@+0!1?-L(??JWGO0z ze;;~304}!gBHi4Y$hbnZ(f6;8XZRqE^>Cbj1%=r&E;_=&{F&_`jU7|-p!XdR;Fs*q zT#3RtcR+d&F5+_x6Em=I{GV5cxprNAgh&Ni{w=-X7JtxNb1v7dCEF%mgbKne8incb zgV!t7H>`&F^kJ%)R)80R^PoDewOS%yNBHc72f@O8z7H@`2idz|4mWkwbGSf02_%?~-X*)<&XhInwSXwORU(a{4c? zu~<63CK6IC+1}}m!-R+X5-ThHT=b~)xe9gP8MBr@B|9>mRGfXbxuJWItF??cJV%R_ z`TmKn*1KGlsW8JCaVbRAQ|)!0Mb-U!HOVk0pNm4fSQoP|+?sWfkUCGuJumkQYd;ahxr6fV z^XyA6+UY8sZ`lXhIW(H1t+;EQ-RWquD%Hhx=l)GAJtsyvSuUI!L=ka<*31a`Y2x{T z$XHLdwv@M0JvBSJvQbyn+4ylZZMx1>UGsynRrDKbz8fD%Z>i)yD$)D#Xt06;6Hw7+ zI|r+-Y_=Q2UcY>{h7uC8y>6^#AMUy;T1JVQ)}q|Oa--*AEgxhLI(JO9X6oGrC9I1` zG!}6)csdbT1@x^7q_C_!6)1WYej?^rb%9 zw4BwFs2IH$m~dvzG)vI}J=w;E=2^YJe;IO`#jfL%xAAhCyX$4`xp``Dmx5-Y<+Qsb zOt+oa839aFKjJp6})In}@a$3P=&^ z=LZTr&+=dke#!(~>B0}#KV|s({6)qvxPW}pW7%JAc@|&YpY#gUuXv(}@3BvM5_e-8 zc+j+vcYPGJ>L8+)AVu-42zjcY6t%gW%u9TDFguw z6W1twTmP>oOC}N_TlUrgVVxR9aEWvR5Y|21ADjO`nDgsp3BBOD3ADtsk!13zuSEU> DM)srX literal 0 HcmV?d00001 diff --git a/python/exa-browserbase/README.md b/python/exa-browserbase/README.md new file mode 100644 index 00000000..98c8142c --- /dev/null +++ b/python/exa-browserbase/README.md @@ -0,0 +1,63 @@ +# Stagehand + Browserbase + Exa: Intelligent Job Application Automation + +## AT A GLANCE + +- **Goal**: Automate job applications with AI that writes smart, tailored responses for each role. +- **Pattern Template**: Shows how to combine Exa (find companies & jobs) + Browserbase (control browser) + Stagehand Agent (fill forms smartly). +- **Workflow**: Exa finds companies you want, then finds their careers pages. Browserbase opens the page, Stagehand reads the job posting, and an AI agent fills out the application form with answers tailored to that specific job. +- **Plans**: Sequential mode works on all plans; concurrent applications and proxies require Startup or Developer plan or higher ([concurrency](https://docs.browserbase.com/guides/concurrency-rate-limits), [proxies](https://docs.browserbase.com/features/proxies)). +- Docs → [Stagehand Agent](https://docs.stagehand.dev/basics/agent) | [Exa Search](https://docs.exa.ai/reference/search) | [Stagehand Extract](https://docs.stagehand.dev/basics/extract) + +## THE 5-STEP FLOW + +1. **Search for companies** — Exa finds companies matching your criteria (e.g., "AI startups in SF") +2. **Find careers pages** — For each company, Exa searches for their careers/jobs page +3. **Extract job details** — Stagehand reads the job posting and extracts structured data (title, requirements, responsibilities) +4. **Smart form filling** — AI agent fills out application fields with tailored responses based on the job description +5. **Resume upload** — Playwright handles file uploads for resume/CV attachments + +## GLOSSARY + +- **agent**: An AI that can plan and do multi-step tasks on its own. It looks at the page and decides what to do next without needing step-by-step instructions. + Docs → https://docs.stagehand.dev/basics/agent +- **extract**: Pull structured data from web pages. You define what you want (job title, requirements, etc.) and it returns clean JSON. + Docs → https://docs.stagehand.dev/basics/extract +- **Exa Search**: AI search engine that finds relevant web content. Can search for companies, find similar pages, and filter by date. + Docs → https://docs.exa.ai/reference/search +- **Tailored responses**: The AI reads the job requirements and writes custom answers for cover letters and open-ended questions that highlight relevant skills. + +## QUICKSTART + +1. cd exa-browserbase +2. uv pip install -e . +3. playwright install chromium +4. cp .env.example .env +5. Add required API keys to .env: + - `BROWSERBASE_PROJECT_ID` — from Browserbase + - `BROWSERBASE_API_KEY` — from Browserbase + - `MODEL_API_KEY` — from your LLM provider (e.g., Google AI Studio) + - `EXA_API_KEY` — from https://dashboard.exa.ai/api-keys +6. Update `APPLICATION_DETAILS` dict in main.py with candidate information +7. Update `resume_path` to point to your PDF resume +8. uv run python main.py + +## EXPECTED OUTPUT + +- Uses your exact info for name, email, phone +- Writes custom answers for open-ended questions +- Creates a tailored cover letter based on the job +- Handles location and visa questions smartly +- Stops before submitting (for testing/review purposes) +- Closes session cleanly + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +📚 Stagehand Python SDK: https://docs.stagehand.dev/v3/sdk/python +📚 Stagehand Agent: https://docs.stagehand.dev/basics/agent +📚 Exa API Key: https://dashboard.exa.ai/api-keys +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/python/exa-browserbase/main.py b/python/exa-browserbase/main.py new file mode 100644 index 00000000..53c4cb53 --- /dev/null +++ b/python/exa-browserbase/main.py @@ -0,0 +1,443 @@ +# Stagehand + Browserbase + Exa: AI-Powered Job Search and Application +# See README.md for full documentation + +import asyncio +import json +import os +from pathlib import Path +from urllib.parse import urlparse + +from dotenv import load_dotenv +from exa_py import Exa +from playwright.async_api import async_playwright +from stagehand import AsyncStagehand + +# Load environment variables from .env file +# Required: BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, MODEL_API_KEY, EXA_API_KEY +load_dotenv() + +# Candidate application details - customize these for your job search +APPLICATION_DETAILS = { + "name": "John Doe", + "email": "john.doe@example.com", + "linkedin_url": "https://linkedin.com/in/johndoe", + "resume_path": "./Dummy_CV.pdf", + "current_location": "San Francisco, CA", + "willing_to_relocate": True, + "requires_sponsorship": False, + "visa_status": "", + "phone": "+1-555-123-4567", + "portfolio_url": "https://johndoe.dev", + "cover_letter": "I am excited to apply for this position...", +} + +# Search configuration - modify to target different companies +SEARCH_CONFIG = { + "company_query": "AI startups in SF", + "num_companies": 5, + # Concurrency: set to False for sequential (works on all plans); True = concurrent (requires Startup or Developer plan or higher) + "concurrent": True, + "max_concurrent_browsers": 5, # Max browsers when concurrent + # Proxies: requires Developer plan or higher; residential proxies help avoid bot detection (https://docs.browserbase.com/features/proxies) + "use_proxy": True, +} + +# JSON schema for extracting structured job description data +JOB_DESCRIPTION_SCHEMA = { + "type": "object", + "properties": { + "jobTitle": {"type": "string", "description": "The job title"}, + "companyName": {"type": "string", "description": "The company name"}, + "requirements": { + "type": "array", + "items": {"type": "string"}, + "description": "Job requirements", + }, + "responsibilities": { + "type": "array", + "items": {"type": "string"}, + "description": "Job responsibilities", + }, + "benefits": { + "type": "array", + "items": {"type": "string"}, + "description": "Job benefits", + }, + "location": {"type": "string", "description": "Job location"}, + "workType": {"type": "string", "description": "Remote, hybrid, or on-site"}, + "fullDescription": {"type": "string", "description": "Full job description text"}, + }, +} + +# System prompt for the job application agent +AGENT_SYSTEM_PROMPT = """You are an intelligent job application assistant with decision-making power. + +Your responsibilities: +- First, navigate to find a job posting and click through to its application page before filling out the form +- Analyze the job description to understand what the company is looking for +- Tailor responses to align with job requirements when available +- Craft thoughtful responses that highlight relevant experience/skills +- For cover letter or "why interested" fields, reference specific aspects of the job/company +- For location/relocation questions, use the willing_to_relocate flag to guide your answer +- For visa/sponsorship questions, answer honestly based on requires_sponsorship +- Skip resume/file upload fields - the resume will be uploaded automatically +- Use the provided application details as the source of truth for factual information +- IMPORTANT: Do NOT click the submit button - this is for testing purposes only + +Think critically about each field and present the candidate in the best professional light.""" + + +def build_agent_instruction(job_description: dict) -> str: + """ + Build the instruction prompt for the agent based on available job description. + + Args: + job_description: Extracted job description data + + Returns: + str: The instruction prompt for the agent + """ + has_job_description = job_description.get("jobTitle") or job_description.get("fullDescription") + + if has_job_description: + return f"""You are filling out a job application. Here is the job description that was found: + +JOB DESCRIPTION: +{json.dumps(job_description, indent=2)} + +CANDIDATE INFORMATION: +{json.dumps(APPLICATION_DETAILS, indent=2)} + +YOUR TASK: +- Fill out all text fields in the application form +- Reference specific aspects of the job description +- Highlight relevant skills/experience from the candidate's background +- Show alignment between candidate and role +- Skip file upload fields (resume will be handled separately) + +Remember: Your goal is to fill out this application in a way that maximizes the candidate's chances by showing strong alignment with this specific role.""" + + return f"""You are filling out a job application. No detailed job description was found on this page. + +CANDIDATE INFORMATION: +{json.dumps(APPLICATION_DETAILS, indent=2)} + +YOUR TASK: +- Fill out all text fields in the application form +- Write professional, thoughtful responses +- Highlight the candidate's general strengths and qualifications +- Express genuine interest and enthusiasm +- Skip file upload fields (resume will be handled separately) + +Remember: Even without a job description, present the candidate professionally and enthusiastically.""" + + +async def upload_resume(session_id: str, cdp_url: str, log_prefix: str = "") -> None: + """ + Upload resume file using Playwright, checking main page and iframes. + + Args: + session_id: The Stagehand session ID + cdp_url: The CDP URL to connect to + log_prefix: Optional prefix for log messages (e.g. per-application context) + """ + print(f"{log_prefix}Attempting to upload resume...") + + async with async_playwright() as p: + browser = await p.chromium.connect_over_cdp(cdp_url) + contexts = browser.contexts + if not contexts: + print(f"{log_prefix}No browser context found") + return + + pw_context = contexts[0] + pages = pw_context.pages + if not pages: + print(f"{log_prefix}No page found") + return + + pw_page = pages[0] + + # Check main page for file input + main_page_inputs = await pw_page.locator('input[type="file"]').count() + + if main_page_inputs > 0: + await pw_page.locator('input[type="file"]').first.set_input_files( + APPLICATION_DETAILS["resume_path"] + ) + print(f"{log_prefix}Resume uploaded successfully from main page!") + return + + # Check inside iframes for file input + frames = pw_page.frames + for frame in frames: + try: + frame_input_count = await frame.locator('input[type="file"]').count() + if frame_input_count > 0: + await frame.locator('input[type="file"]').first.set_input_files( + APPLICATION_DETAILS["resume_path"] + ) + print(f"{log_prefix}Resume uploaded successfully from iframe!") + return + except Exception: + # Frame not accessible, continue to next + pass + + print(f"{log_prefix}No file upload field found on page") + + +async def search_companies(exa: Exa) -> list[dict]: + """ + Search for companies matching the criteria using Exa. + + Args: + exa: Exa client instance + + Returns: + list: List of company results with title and url + """ + print(f'Searching for companies: "{SEARCH_CONFIG["company_query"]}"...') + + # Use asyncio.to_thread for synchronous Exa SDK calls + company_results = await asyncio.to_thread( + exa.search_and_contents, + SEARCH_CONFIG["company_query"], + category="company", + text=True, + type="auto", + livecrawl="fallback", + num_results=SEARCH_CONFIG["num_companies"], + ) + + print(f"Found {len(company_results.results)} companies:") + for i, company in enumerate(company_results.results): + print(f" {i + 1}. {company.title} - {company.url}") + + return company_results.results + + +async def find_careers_pages(exa: Exa, companies: list) -> list[dict]: + """ + Find careers pages for each discovered company. + + Args: + exa: Exa client instance + companies: List of company results from search + + Returns: + list: List of careers page data with company, url, and careersUrl + """ + print("\nSearching for careers pages...") + careers_pages = [] + + for company in companies: + # Extract domain from company URL for the careers search + parsed_url = urlparse(company.url) + company_domain = parsed_url.hostname.replace("www.", "") if parsed_url.hostname else "" + print(f" Looking for careers page: {company_domain}...") + + # Use asyncio.to_thread for synchronous Exa SDK calls + careers_result = await asyncio.to_thread( + exa.search_and_contents, + f"{company_domain} careers page", + context=True, + exclude_domains=["linkedin.com"], + num_results=5, + text=True, + type="deep", + livecrawl="fallback", + ) + + if careers_result.results: + careers_url = careers_result.results[0].url + print(f" Found: {careers_url}") + careers_pages.append( + { + "company": company.title or company_domain, + "url": company.url, + "careers_url": careers_url, + } + ) + else: + print(f" No careers page found for {company_domain}") + + return careers_pages + + +async def apply_to_job(careers_page: dict, index: int) -> dict: + """ + Apply to a single job posting: start session, extract job description, + run agent to fill form, upload resume. Returns result dict for summary. + + Args: + careers_page: Dict with company, url, careers_url + index: 0-based index for logging (e.g. [1/N] Company: ...) + + Returns: + dict: company, careers_url, success, message, session_url (optional) + """ + num_companies = SEARCH_CONFIG["num_companies"] + company_name = careers_page["company"] + log_prefix = f"[{index + 1}/{num_companies}] {company_name}: " + print(f"\n{log_prefix}Starting application...") + + model_api_key = ( + os.environ.get("MODEL_API_KEY") + or os.environ.get("GOOGLE_API_KEY") + or os.environ.get("GOOGLE_GENERATIVE_AI_API_KEY") + ) + client = AsyncStagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=model_api_key, + ) + + # Start session (proxies require Developer plan or higher) + start_response = await client.sessions.start(model_name="google/gemini-2.5-pro") + session_id = start_response.data.session_id + session_url = f"https://browserbase.com/sessions/{session_id}" + print(f"{log_prefix}Session started: {session_url}") + + try: + await client.sessions.navigate(id=session_id, url=careers_page["careers_url"]) + + extract_response = await client.sessions.extract( + id=session_id, + instruction="extract the full job description including title, requirements, responsibilities, and any important details about the role", + schema=JOB_DESCRIPTION_SCHEMA, + ) + job_description = extract_response.data.result or {} + + instruction = build_agent_instruction(job_description) + execute_response = await client.sessions.execute( + id=session_id, + execute_options={ + "instruction": instruction, + "max_steps": 50, + }, + agent_config={ + "model": "google/gemini-2.5-flash", + "system_prompt": AGENT_SYSTEM_PROMPT, + }, + timeout=300.0, + ) + result = execute_response.data.result + + try: + cdp_url = f"wss://connect.browserbase.com?apiKey={os.environ.get('BROWSERBASE_API_KEY')}&sessionId={session_id}" + await upload_resume(session_id, cdp_url, log_prefix) + except Exception as upload_error: + print(f"{log_prefix}Could not upload resume: {upload_error}") + + if result.success: + print(f"{log_prefix}Form filled successfully!") + else: + print(f"{log_prefix}Form filling may be incomplete") + + return { + "company": company_name, + "careers_url": careers_page["careers_url"], + "success": result.success, + "message": result.message, + "session_url": session_url, + } + except Exception as error: + print(f"{log_prefix}Error: {error}") + return { + "company": company_name, + "careers_url": careers_page["careers_url"], + "success": False, + "message": str(error), + "session_url": session_url, + } + finally: + await client.sessions.end(id=session_id) + print(f"{log_prefix}Session closed") + + +async def main(): + """ + Main application entry point. + + Orchestrates the job search and application automation: + 1. Uses Exa to find companies matching search criteria + 2. Finds careers pages for each company + 3. Navigates to careers page with Stagehand + 4. Extracts job description data + 5. Uses AI agent to fill out application form + 6. Uploads resume using Playwright + """ + print("Starting Exa + Browserbase Job Search and Application...") + + # Initialize Exa client for AI-powered company search + exa = Exa(api_key=os.environ.get("EXA_API_KEY")) + + # Search for companies matching the criteria using Exa + companies = await search_companies(exa) + + if not companies: + print("No companies found. Exiting.") + return + + # Find careers pages for each discovered company + careers_pages = await find_careers_pages(exa, companies) + + print(f"\nFound {len(careers_pages)} careers pages total.") + + if not careers_pages: + print("No careers pages found. Exiting.") + return + + # Apply to jobs either concurrently or sequentially based on config + concurrent = SEARCH_CONFIG["concurrent"] + max_browsers = SEARCH_CONFIG["max_concurrent_browsers"] + print("\n" + "=" * 50) + mode = f"concurrent, max {max_browsers} browsers" if concurrent else "sequential" + print(f"Starting applications ({mode})...") + print("=" * 50) + + if concurrent: + # Run applications concurrently with limited parallelism + results = [] + for i in range(0, len(careers_pages), max_browsers): + chunk = careers_pages[i : i + max_browsers] + chunk_results = await asyncio.gather( + *[apply_to_job(page, len(results) + j) for j, page in enumerate(chunk)] + ) + results.extend(chunk_results) + else: + # Run applications sequentially + results = [] + for i, careers_page in enumerate(careers_pages): + result = await apply_to_job(careers_page, i) + results.append(result) + + # Print summary + print("\n" + "=" * 50) + print("APPLICATION SUMMARY") + print("=" * 50) + successful = [r for r in results if r["success"]] + failed = [r for r in results if not r["success"]] + print(f"\nTotal: {len(results)} | Success: {len(successful)} | Failed: {len(failed)}\n") + for i, r in enumerate(results): + status = "[SUCCESS]" if r["success"] else "[FAILED]" + print(f"{i + 1}. {status} {r['company']}") + print(f" URL: {r['careers_url']}") + if r.get("session_url"): + print(f" Session: {r['session_url']}") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Error in Exa + Browserbase job application: {err}") + print("Common issues:") + print( + " - Check .env file has BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, " + "MODEL_API_KEY, and EXA_API_KEY" + ) + print(" - Verify companies exist for the search query") + print(" - Ensure careers pages are accessible") + print("Docs: https://docs.stagehand.dev/v3/sdk/python") + exit(1) diff --git a/python/exa-browserbase/pyproject.toml b/python/exa-browserbase/pyproject.toml new file mode 100644 index 00000000..8cc2a280 --- /dev/null +++ b/python/exa-browserbase/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "exa-browserbase" +version = "0.1.0" +description = "AI-Powered Job Search and Application using Exa, Stagehand, and Browserbase" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "exa-py", + "playwright", + "python-dotenv", + "stagehand", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/typescript/exa-browserbase/Dummy_CV.pdf b/typescript/exa-browserbase/Dummy_CV.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fc455c206dabbeecb499c81d71c1d2fbe549dbb3 GIT binary patch literal 2358 zcma)8TX*6(5`O1bh+BYw@g3U)=wOI}aKD5Q%uTih7>sRfGu^!p`v-R4=6!za%5>7h zN$>8A4+lx5s;^3)N-8A>b%mhwTqODXzyI|I7$Mj3f)3O`1KG%f0WtvA0EVMqPRI_x zJ_y_*Nq+tMHSb+@EvV&n=MX%jvWcwUP+;m1kWaSm7^b}g3#e)g{$K$ZUGZ^c} zRp|QvxFTf7GX9P~=5PArZv547D3~xGh~fGH*+qs4W#=1E@t6X<#OE1-F94CF^P(t; zU9~L9zI~Pw!KDcrk(iD=6u?C)_k5Pc3$Z&JW3HpX9Y@Of^$?t9Ly={Rd zCIpZpC%~JD&B=)e?JHy`kAM&A2H5z9Iv%TjP!~>h9Ru_G;-j8p<4!2@KLY*1eGLZC za&|Wug&U0OCD=P2NH5aRcI@vb=i)Ci89>%N1OwBt>tS^1dXb@+0!1?-L(??JWGO0z ze;;~304}!gBHi4Y$hbnZ(f6;8XZRqE^>Cbj1%=r&E;_=&{F&_`jU7|-p!XdR;Fs*q zT#3RtcR+d&F5+_x6Em=I{GV5cxprNAgh&Ni{w=-X7JtxNb1v7dCEF%mgbKne8incb zgV!t7H>`&F^kJ%)R)80R^PoDewOS%yNBHc72f@O8z7H@`2idz|4mWkwbGSf02_%?~-X*)<&XhInwSXwORU(a{4c? zu~<63CK6IC+1}}m!-R+X5-ThHT=b~)xe9gP8MBr@B|9>mRGfXbxuJWItF??cJV%R_ z`TmKn*1KGlsW8JCaVbRAQ|)!0Mb-U!HOVk0pNm4fSQoP|+?sWfkUCGuJumkQYd;ahxr6fV z^XyA6+UY8sZ`lXhIW(H1t+;EQ-RWquD%Hhx=l)GAJtsyvSuUI!L=ka<*31a`Y2x{T z$XHLdwv@M0JvBSJvQbyn+4ylZZMx1>UGsynRrDKbz8fD%Z>i)yD$)D#Xt06;6Hw7+ zI|r+-Y_=Q2UcY>{h7uC8y>6^#AMUy;T1JVQ)}q|Oa--*AEgxhLI(JO9X6oGrC9I1` zG!}6)csdbT1@x^7q_C_!6)1WYej?^rb%9 zw4BwFs2IH$m~dvzG)vI}J=w;E=2^YJe;IO`#jfL%xAAhCyX$4`xp``Dmx5-Y<+Qsb zOt+oa839aFKjJp6})In}@a$3P=&^ z=LZTr&+=dke#!(~>B0}#KV|s({6)qvxPW}pW7%JAc@|&YpY#gUuXv(}@3BvM5_e-8 zc+j+vcYPGJ>L8+)AVu-42zjcY6t%gW%u9TDFguw z6W1twTmP>oOC}N_TlUrgVVxR9aEWvR5Y|21ADjO`nDgsp3BBOD3ADtsk!13zuSEU> DM)srX literal 0 HcmV?d00001 diff --git a/typescript/exa-browserbase/README.md b/typescript/exa-browserbase/README.md new file mode 100644 index 00000000..e05004f3 --- /dev/null +++ b/typescript/exa-browserbase/README.md @@ -0,0 +1,62 @@ +# Stagehand + Browserbase + Exa: Intelligent Job Application Automation + +## AT A GLANCE + +- **Goal**: Automate job applications with AI that writes smart, tailored responses for each role. +- **Pattern Template**: Shows how to combine Exa (find companies & jobs) + Browserbase (control browser) + Stagehand Agent (fill forms smartly). +- **Workflow**: Exa finds companies you want, then finds their careers pages. Browserbase opens the page, Stagehand reads the job posting, and an AI agent fills out the application form with answers tailored to that specific job. +- **Plans**: Sequential mode works on all plans; concurrent applications and proxies require Startup or Developer plan or higher ([concurrency](https://docs.browserbase.com/guides/concurrency-rate-limits), [proxies](https://docs.browserbase.com/features/proxies)). +- Docs → [Stagehand Agent](https://docs.stagehand.dev/basics/agent) | [Exa Search](https://docs.exa.ai/reference/search) | [Stagehand Extract](https://docs.stagehand.dev/basics/extract) + +## THE 5-STEP FLOW + +1. **Search for companies** — Exa finds companies matching your criteria (e.g., "AI startups in SF") +2. **Find careers pages** — For each company, Exa searches for their careers/jobs page +3. **Extract job details** — Stagehand reads the job posting and extracts structured data (title, requirements, responsibilities) +4. **Smart form filling** — AI agent fills out application fields with tailored responses based on the job description +5. **Resume upload** — Playwright handles file uploads for resume/CV attachments + +## GLOSSARY + +- **agent**: An AI that can plan and do multi-step tasks on its own. It looks at the page and decides what to do next without needing step-by-step instructions. + Docs → https://docs.stagehand.dev/basics/agent +- **extract**: Pull structured data from web pages. You define what you want (job title, requirements, etc.) and it returns clean JSON. + Docs → https://docs.stagehand.dev/basics/extract +- **Exa Search**: AI search engine that finds relevant web content. Can search for companies, find similar pages, and filter by date. + Docs → https://docs.exa.ai/reference/search +- **Hybrid mode**: Agent mode that combines reading the page code (DOM) and looking at the page visually. Works better across different websites. +- **Tailored responses**: The AI reads the job requirements and writes custom answers for cover letters and open-ended questions that highlight relevant skills. + +## QUICKSTART + +1. cd exa-browserbase +2. pnpm install +3. cp .env.example .env +4. Add required API keys to .env: + - `BROWSERBASE_PROJECT_ID` — from Browserbase + - `BROWSERBASE_API_KEY` — from Browserbase + - `EXA_API_KEY` — from https://dashboard.exa.ai/api-keys + - Configure your Browserbase API key with OpenRouter/Anthropic +5. Update `applicationDetails` object with candidate information +6. Update `resumePath` to point to your PDF resume +7. pnpm start + +## EXPECTED OUTPUT + +- Uses your exact info for name, email, phone +- Writes custom answers for open-ended questions +- Creates a tailored cover letter based on the job +- Handles location and visa questions smartly +- Stops before submitting (for testing/review purposes) +- Closes session cleanly + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +📚 Stagehand Agent: https://docs.stagehand.dev/basics/agent +📚 Exa API Key: https://dashboard.exa.ai/api-keys +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/typescript/exa-browserbase/index.ts b/typescript/exa-browserbase/index.ts new file mode 100644 index 00000000..a449f9dc --- /dev/null +++ b/typescript/exa-browserbase/index.ts @@ -0,0 +1,367 @@ +// Stagehand + Browserbase + Exa: AI-Powered Job Search and Application - See README.md for full documentation + +import "dotenv/config"; +import { Stagehand } from "@browserbasehq/stagehand"; +import Exa from "exa-js"; +import { z } from "zod"; +import { chromium } from "playwright-core"; + +// Candidate application details - customize these for your job search +const applicationDetails = { + name: "John Doe", + email: "john.doe@example.com", + linkedInUrl: "https://linkedin.com/in/johndoe", + resumePath: "./Dummy_CV.pdf", + currentLocation: "San Francisco, CA", + willingToRelocate: true, + requiresSponsorship: false, + visaStatus: "", + phone: "+1-555-123-4567", + portfolioUrl: "https://johndoe.dev", + coverLetter: "I am excited to apply for this position...", +}; + +// Search configuration - modify to target different companies +const searchConfig = { + companyQuery: "AI startups in SF", + numCompanies: 5, + // Concurrency: set to false for sequential (works on all plans); true = concurrent (requires Startup or Developer plan or higher) + concurrent: true, + maxConcurrentBrowsers: 5, // Max browsers when concurrent + // Proxies: requires Developer plan or higher; residential proxies help avoid bot detection (https://docs.browserbase.com/features/proxies) + useProxy: true, +}; + +// Zod schema for extracting structured job description data +const jobDescriptionSchema = z.object({ + jobTitle: z.string().optional(), + companyName: z.string().optional(), + requirements: z.array(z.string()).optional(), + responsibilities: z.array(z.string()).optional(), + benefits: z.array(z.string()).optional(), + location: z.string().optional(), + workType: z.string().optional(), + fullDescription: z.string().optional(), +}); + +// Careers page data structure for tracking discovered job pages +interface CareersPage { + company: string; + url: string; + careersUrl: string; +} + +// System prompt for the job application agent +const agentSystemPrompt = `You are an intelligent job application assistant with decision-making power. + +Your responsibilities: +- First, navigate to find a job posting and click through to its application page before filling out the form +- Analyze the job description to understand what the company is looking for +- Tailor responses to align with job requirements when available +- Craft thoughtful responses that highlight relevant experience/skills +- For cover letter or "why interested" fields, reference specific aspects of the job/company +- For location/relocation questions, use the willingToRelocate flag to guide your answer +- For visa/sponsorship questions, answer honestly based on requiresSponsorship +- Skip resume/file upload fields - the resume will be uploaded automatically +- Use the provided application details as the source of truth for factual information +- IMPORTANT: Do NOT click the submit button - this is for testing purposes only + +Think critically about each field and present the candidate in the best professional light.`; + +// Builds the instruction prompt for the agent based on available job description +function buildAgentInstruction( + jobDescription: z.infer, +): string { + const hasJobDescription = jobDescription.jobTitle || jobDescription.fullDescription; + + if (hasJobDescription) { + return `You are filling out a job application. Here is the job description that was found: + +JOB DESCRIPTION: +${JSON.stringify(jobDescription, null, 2)} + +CANDIDATE INFORMATION: +${JSON.stringify(applicationDetails, null, 2)} + +YOUR TASK: +- Fill out all text fields in the application form +- Reference specific aspects of the job description +- Highlight relevant skills/experience from the candidate's background +- Show alignment between candidate and role +- Skip file upload fields (resume will be handled separately) + +Remember: Your goal is to fill out this application in a way that maximizes the candidate's chances by showing strong alignment with this specific role.`; + } + + return `You are filling out a job application. No detailed job description was found on this page. + +CANDIDATE INFORMATION: +${JSON.stringify(applicationDetails, null, 2)} + +YOUR TASK: +- Fill out all text fields in the application form +- Write professional, thoughtful responses +- Highlight the candidate's general strengths and qualifications +- Express genuine interest and enthusiasm +- Skip file upload fields (resume will be handled separately) + +Remember: Even without a job description, present the candidate professionally and enthusiastically.`; +} + +// Uploads resume file using Playwright, checking main page and iframes +async function uploadResume(stagehand: Stagehand, logPrefix: string = ""): Promise { + console.log(`${logPrefix}Attempting to upload resume...`); + + const browser = await chromium.connectOverCDP(stagehand.connectURL()); + const pwContext = browser.contexts()[0]; + const pwPage = pwContext.pages()[0]; + + // Check main page for file input + const mainPageInputs = await pwPage.locator('input[type="file"]').count(); + + if (mainPageInputs > 0) { + await pwPage + .locator('input[type="file"]') + .first() + .setInputFiles(applicationDetails.resumePath); + console.log(`${logPrefix}Resume uploaded successfully from main page!`); + return; + } + + // Check inside iframes for file input + const frames = pwPage.frames(); + for (const frame of frames) { + try { + const frameInputCount = await frame.locator('input[type="file"]').count(); + if (frameInputCount > 0) { + await frame + .locator('input[type="file"]') + .first() + .setInputFiles(applicationDetails.resumePath); + console.log(`${logPrefix}Resume uploaded successfully from iframe!`); + return; + } + } catch { + // Frame not accessible, continue to next + } + } + + console.log(`${logPrefix}No file upload field found on page`); +} + +// Result of a single job application attempt +interface ApplicationResult { + company: string; + careersUrl: string; + success: boolean; + message: string; + sessionUrl?: string; +} + +// Applies to a single job posting +async function applyToJob( + careersPage: CareersPage, + index: number, +): Promise { + const logPrefix = `[${index + 1}/${searchConfig.numCompanies}] ${careersPage.company}: `; + console.log(`\n${logPrefix}Starting application...`); + + const stagehand = new Stagehand({ + env: "BROWSERBASE", + verbose: 0, + experimental: true, + model: "google/gemini-2.5-pro", + // Browserbase session configuration (proxies require Developer plan or higher) + browserbaseSessionCreateParams: { + proxies: searchConfig.useProxy, + }, + }); + + try { + await stagehand.init(); + const sessionUrl = `https://browserbase.com/sessions/${stagehand.browserbaseSessionId}`; + console.log(`${logPrefix}Session started: ${sessionUrl}`); + + const page = stagehand.context.pages()[0]; + await page.goto(careersPage.careersUrl); + + // Extract job description + const jobDescription = await stagehand.extract( + "extract the full job description including title, requirements, responsibilities, and any important details about the role", + jobDescriptionSchema, + ); + + // Initialize and run agent + const agent = stagehand.agent({ + mode: "hybrid", + model: "google/gemini-3-flash-preview", + systemPrompt: agentSystemPrompt, + }); + + const instruction = buildAgentInstruction(jobDescription); + const result = await agent.execute({ + instruction, + maxSteps: 50, + }); + + // Upload resume after form filling + try { + await uploadResume(stagehand, logPrefix); + } catch (uploadError) { + console.log(`${logPrefix}Could not upload resume:`, uploadError); + } + + if (result.success) { + console.log(`${logPrefix}Form filled successfully!`); + } else { + console.log(`${logPrefix}Form filling may be incomplete`); + } + + return { + company: careersPage.company, + careersUrl: careersPage.careersUrl, + success: result.success, + message: result.message, + sessionUrl, + }; + } catch (error) { + console.error(`${logPrefix}Error:`, error); + return { + company: careersPage.company, + careersUrl: careersPage.careersUrl, + success: false, + message: error instanceof Error ? error.message : String(error), + }; + } finally { + await stagehand.close(); + console.log(`${logPrefix}Session closed`); + } +} + +async function main() { + console.log("Starting Exa + Browserbase Job Search and Application..."); + + // Initialize Exa client for AI-powered company search + const exa = new Exa(process.env.EXA_API_KEY); + + // Search for companies matching the criteria using Exa + console.log(`Searching for companies: "${searchConfig.companyQuery}"...`); + + const companyResults = await exa.searchAndContents(searchConfig.companyQuery, { + category: "company", + text: true, + type: "auto", + livecrawl: "fallback", + numResults: searchConfig.numCompanies, + }); + + console.log(`Found ${companyResults.results.length} companies:`); + companyResults.results.forEach((company, i) => { + console.log(` ${i + 1}. ${company.title} - ${company.url}`); + }); + + if (companyResults.results.length === 0) { + console.log("No companies found. Exiting."); + return; + } + + // Find careers pages for each discovered company + console.log("\nSearching for careers pages..."); + const careersPages: CareersPage[] = []; + + for (const company of companyResults.results) { + const companyDomain = new URL(company.url).hostname.replace("www.", ""); + console.log(` Looking for careers page: ${companyDomain}...`); + + const careersResult = await exa.searchAndContents(`${companyDomain} careers page`, { + context: true, + excludeDomains: ["linkedin.com"], + numResults: 5, + text: true, + type: "deep", + livecrawl: "fallback", + }); + + if (careersResult.results.length > 0) { + const careersUrl = careersResult.results[0].url; + console.log(` Found: ${careersUrl}`); + careersPages.push({ + company: company.title || companyDomain, + url: company.url, + careersUrl: careersUrl, + }); + } else { + console.log(` No careers page found for ${companyDomain}`); + } + } + + console.log(`\nFound ${careersPages.length} careers pages total.`); + + if (careersPages.length === 0) { + console.log("No careers pages found. Exiting."); + return; + } + + // Apply to jobs either concurrently or sequentially based on config + console.log(`\n${"=".repeat(50)}`); + console.log( + `Starting applications (${searchConfig.concurrent ? `concurrent, max ${searchConfig.maxConcurrentBrowsers} browsers` : "sequential"})...`, + ); + console.log(`${"=".repeat(50)}`); + + let results: ApplicationResult[]; + + if (searchConfig.concurrent) { + // Run applications concurrently with limited parallelism + const chunks: CareersPage[][] = []; + for (let i = 0; i < careersPages.length; i += searchConfig.maxConcurrentBrowsers) { + chunks.push(careersPages.slice(i, i + searchConfig.maxConcurrentBrowsers)); + } + + results = []; + for (const chunk of chunks) { + const chunkResults = await Promise.all( + chunk.map((page, idx) => applyToJob(page, results.length + idx)), + ); + results.push(...chunkResults); + } + } else { + // Run applications sequentially + results = []; + for (let i = 0; i < careersPages.length; i++) { + const result = await applyToJob(careersPages[i], i); + results.push(result); + } + } + + // Print summary + console.log(`\n${"=".repeat(50)}`); + console.log("APPLICATION SUMMARY"); + console.log(`${"=".repeat(50)}`); + + const successful = results.filter((r) => r.success); + const failed = results.filter((r) => !r.success); + + console.log(`\nTotal: ${results.length} | Success: ${successful.length} | Failed: ${failed.length}\n`); + + results.forEach((r, i) => { + const status = r.success ? "[SUCCESS]" : "[FAILED]"; + console.log(`${i + 1}. ${status} ${r.company}`); + console.log(` URL: ${r.careersUrl}`); + if (r.sessionUrl) { + console.log(` Session: ${r.sessionUrl}`); + } + }); +} + +main().catch((err) => { + console.error("Error in Exa + Browserbase job application:", err); + console.error("Common issues:"); + console.error( + " - Check .env file has BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, MODEL_API_KEY, and EXA_API_KEY", + ); + console.error(" - Verify companies exist for the search query"); + console.error(" - Ensure careers pages are accessible"); + console.error("Docs: https://docs.stagehand.dev/v3/first-steps/introduction"); + process.exit(1); +}); \ No newline at end of file diff --git a/typescript/exa-browserbase/package.json b/typescript/exa-browserbase/package.json new file mode 100644 index 00000000..46bcae87 --- /dev/null +++ b/typescript/exa-browserbase/package.json @@ -0,0 +1,22 @@ +{ + "name": "exa-browserbase", + "version": "1.0.0", + "description": "Stagehand + Browserbase + Exa: AI-Powered Job Search and Application", + "type": "module", + "main": "index.ts", + "scripts": { + "start": "tsx index.ts" + }, + "dependencies": { + "@browserbasehq/stagehand": "latest", + "dotenv": "latest", + "exa-js": "latest", + "playwright-core": "latest", + "zod": "latest" + }, + "devDependencies": { + "@types/node": "latest", + "tsx": "latest", + "typescript": "latest" + } +} From d7720fc67d377486dbd740ef7cee81f81977b04a Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Fri, 6 Feb 2026 01:45:13 -0500 Subject: [PATCH 17/45] Migrate Python templates from Stagehand SDK v2 to v3 (#41) * Restore descriptive comments in Python v3 templates Restore important descriptive comments across all 18 Python templates: - Replace "V3 BYOB architecture" with "cloud-based browser automation" - Add "Connect to the browser via CDP" comments before sync_playwright sections - Restore Pydantic model explanations and compatibility notes - Restore caching behavior and context management documentation - Restore login/navigation sequence explanations - Keep all Pydantic models using model_json_schema() for extract All templates tested for correctness where API keys available. Co-Authored-By: Claude Haiku 4.5 * Simplify Python templates to use pure Stagehand API Address PR review feedback: - Remove Playwright CDP connection from templates that only use act() and extract() - Use client.sessions.navigate() instead of page.goto() - Move session.end() to finally blocks to avoid duplicate calls - Remove unnecessary inline schema comment - Update READMEs with simplified quickstart instructions Affected templates: council-events, license-verification, form-filling, polymarket-research Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: Claude Haiku 4.5 --- python/basic-caching/main.py | 196 ++++---- python/basic-recaptcha/main.py | 100 ++-- python/company-value-prop-generator/main.py | 75 +-- python/context/main.py | 193 +++++--- python/council-events/README.md | 9 +- python/council-events/main.py | 129 ++--- python/download-financial-statements/main.py | 120 +++-- python/form-filling/README.md | 11 +- python/form-filling/main.py | 132 ++--- python/gift-finder/main.py | 283 ++++++----- python/job-application/main.py | 347 +++++++------ python/license-verification/README.md | 12 +- python/license-verification/main.py | 111 +++-- python/manual-mfa-with-contexts/main.py | 182 +++---- python/mfa-handling/main.py | 214 ++++---- python/nurse-verification/main.py | 133 +++-- python/pickleball/main.py | 492 ++++++++++--------- python/polymarket-research/README.md | 9 +- python/polymarket-research/main.py | 125 +++-- python/proxies-weather/main.py | 124 ++--- python/proxies/main.py | 129 +++-- python/website-link-tester/main.py | 232 +++++---- uv.lock | 3 + 23 files changed, 1834 insertions(+), 1527 deletions(-) create mode 100644 uv.lock diff --git a/python/basic-caching/main.py b/python/basic-caching/main.py index e7fa99f0..3013a6ec 100644 --- a/python/basic-caching/main.py +++ b/python/basic-caching/main.py @@ -1,15 +1,15 @@ # Stagehand + Browserbase: Basic Caching - See README.md for full documentation -import asyncio import json import os +import time from pathlib import Path from typing import Any -import aiofiles from dotenv import load_dotenv +from playwright.sync_api import sync_playwright -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables load_dotenv() @@ -18,12 +18,11 @@ CACHE_FILE = Path(__file__).parent / "cache.json" -async def get_cache(key: str) -> dict[str, Any] | None: +def get_cache(key: str) -> dict[str, Any] | None: """Get the cached value (None if it doesn't exist)""" try: - # Read cache file asynchronously for better performance - async with aiofiles.open(CACHE_FILE) as f: - cache_content = await f.read() + with open(CACHE_FILE) as f: + cache_content = f.read() parsed = json.loads(cache_content) return parsed.get(key) except (FileNotFoundError, json.JSONDecodeError): @@ -31,12 +30,12 @@ async def get_cache(key: str) -> dict[str, Any] | None: return None -async def set_cache(key: str, value: Any) -> None: +def set_cache(key: str, value: Any) -> None: """Set the cache value - converts ObserveResult (Pydantic model) to dict if needed""" try: # Read existing cache file to preserve other cached entries - async with aiofiles.open(CACHE_FILE) as f: - cache_content = await f.read() + with open(CACHE_FILE) as f: + cache_content = f.read() parsed = json.loads(cache_content) except (FileNotFoundError, json.JSONDecodeError): # Cache file doesn't exist - start with empty dict @@ -57,11 +56,11 @@ async def set_cache(key: str, value: Any) -> None: parsed[key] = dict(value) if hasattr(value, "__dict__") else value # Write updated cache back to file - async with aiofiles.open(CACHE_FILE, "w") as f: - await f.write(json.dumps(parsed, indent=2, default=str)) + with open(CACHE_FILE, "w") as f: + f.write(json.dumps(parsed, indent=2, default=str)) -async def act_with_cache(page, key: str, prompt: str, self_heal: bool = False): +def act_with_cache(client, session_id: str, key: str, prompt: str, self_heal: bool = False): """ Check the cache, get the action, and run it. If self_heal is true, we'll attempt to self-heal if the action fails. @@ -74,7 +73,7 @@ async def act_with_cache(page, key: str, prompt: str, self_heal: bool = False): """ try: # Check if action is already cached - cache_exists = await get_cache(key) + cache_exists = get_cache(key) if cache_exists: # Use the already-retrieved cached action - no LLM inference needed @@ -83,131 +82,143 @@ async def act_with_cache(page, key: str, prompt: str, self_heal: bool = False): else: # Get the observe result (the action) - this requires LLM inference print(f" → Observing: {prompt}") - actions = await page.observe(prompt) - action = actions[0] - + observe_response = client.sessions.observe( + id=session_id, + instruction=prompt, + ) + action = observe_response.data.results[0] if observe_response.data.results else {} # Cache the action for future use - await set_cache(key, action) + set_cache(key, action) print(f" ✓ Cached action for: {prompt}") # Run the action (no LLM inference when using cached action) - await page.act(action) + if isinstance(action, dict): + client.sessions.act(id=session_id, input=action) + else: + client.sessions.act(id=session_id, input=prompt) except Exception as e: print(f" ✗ Error: {e}") # In self_heal mode, retry the action with a fresh LLM call if self_heal: print(" → Attempting to self-heal...") - await page.act(prompt) + client.sessions.act(id=session_id, input=prompt) else: raise e -async def run_without_cache(): +def run_without_cache(): """Run workflow without caching (baseline) - demonstrates normal LLM usage""" print("RUN 1: WITHOUT CACHING") - start_time = asyncio.get_event_loop().time() + start_time = time.time() - # Initialize Stagehand with Browserbase for cloud-based browser automation. - # Note: set verbose: 0 to prevent API keys from appearing in logs when handling sensitive data. - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.getenv("BROWSERBASE_API_KEY"), - project_id=os.getenv("BROWSERBASE_PROJECT_ID"), - model_name="google/gemini-2.5-flash", + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.getenv("BROWSERBASE_API_KEY"), + browserbase_project_id=os.getenv("BROWSERBASE_PROJECT_ID"), model_api_key=os.getenv("GOOGLE_API_KEY"), - verbose=0, # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - browserbase_session_create_params={ - "project_id": os.getenv("BROWSERBASE_PROJECT_ID"), - }, ) - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - page = stagehand.page + start_response = client.sessions.start(model_name="google/gemini-2.5-flash") + session_id = start_response.data.session_id + + try: + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() - try: # Navigate to Stripe checkout demo page print("Navigating to Stripe checkout...") - await page.goto("https://checkout.stripe.dev/preview", wait_until="domcontentloaded") + page.goto("https://checkout.stripe.dev/preview", wait_until="domcontentloaded") # Each act() call requires LLM inference - no caching enabled - await page.act("Click on the View Demo button") - await page.act("Type 'test@example.com' into the email field") - await page.act("Type '4242424242424242' into the card number field") - await page.act("Type '12/34' into the expiration date field") + client.sessions.act(id=session_id, input="Click on the View Demo button") + client.sessions.act(id=session_id, input="Type 'test@example.com' into the email field") + client.sessions.act( + id=session_id, input="Type '4242424242424242' into the card number field" + ) + client.sessions.act(id=session_id, input="Type '12/34' into the expiration date field") - elapsed = f"{(asyncio.get_event_loop().time() - start_time):.2f}" + elapsed = f"{(time.time() - start_time):.2f}" print(f"Total time: {elapsed}s") print("Cost: ~$0.01-0.05 (4 LLM calls)") print("API calls: 4 (one per action)\n") - return {"elapsed": elapsed, "llm_calls": 4} - except Exception as error: - print(f"Error: {error}") - raise + browser.close() + client.sessions.end(id=session_id) + return {"elapsed": elapsed, "llm_calls": 4} -async def run_with_cache(): + except Exception as error: + print(f"Error: {error}") + client.sessions.end(id=session_id) + raise + + +def run_with_cache(): """Run workflow with caching enabled - demonstrates cost and latency savings""" print("RUN 2: WITH CACHING\n") - start_time = asyncio.get_event_loop().time() + start_time = time.time() - # Initialize Stagehand with Browserbase for cloud-based browser automation. - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.getenv("BROWSERBASE_API_KEY"), - project_id=os.getenv("BROWSERBASE_PROJECT_ID"), - model_name="google/gemini-2.5-flash", + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.getenv("BROWSERBASE_API_KEY"), + browserbase_project_id=os.getenv("BROWSERBASE_PROJECT_ID"), model_api_key=os.getenv("GOOGLE_API_KEY"), - verbose=0, # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - browserbase_session_create_params={ - "project_id": os.getenv("BROWSERBASE_PROJECT_ID"), - }, ) - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - page = stagehand.page + start_response = client.sessions.start(model_name="google/gemini-2.5-flash") + session_id = start_response.data.session_id + + try: + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() - try: # Navigate to Stripe checkout demo page print("Navigating to Stripe checkout...") - await page.goto("https://checkout.stripe.dev/preview", wait_until="domcontentloaded") + page.goto("https://checkout.stripe.dev/preview", wait_until="domcontentloaded") # Use cached actions - first run will observe and cache, subsequent runs use cache - await act_with_cache( - page, "Click on the View Demo button", "Click on the View Demo button" + act_with_cache( + client, session_id, "Click on the View Demo button", "Click on the View Demo button" ) - await act_with_cache( - page, + act_with_cache( + client, + session_id, "Type 'test@example.com' into the email field", "Type 'test@example.com' into the email field", ) - await act_with_cache( - page, + act_with_cache( + client, + session_id, "Type '4242424242424242' into the card number field", "Type '4242424242424242' into the card number field", ) - await act_with_cache( - page, + act_with_cache( + client, + session_id, "Type '12/34' into the expiration date field", "Type '12/34' into the expiration date field", ) - elapsed = f"{(asyncio.get_event_loop().time() - start_time):.2f}" + elapsed = f"{(time.time() - start_time):.2f}" cache_exists = CACHE_FILE.exists() # Count cache entries to determine if this was a cache hit or miss if cache_exists: - async with aiofiles.open(CACHE_FILE) as f: - cache_content = await f.read() + with open(CACHE_FILE) as f: + cache_content = f.read() cache_data = json.loads(cache_content) cache_count = len(cache_data) else: @@ -228,13 +239,18 @@ async def run_with_cache(): print("📂Cache created") print() - return {"elapsed": elapsed, "llm_calls": 0 if cache_count >= 4 else 4} - except Exception as error: - print(f"Error: {error}") - raise + browser.close() + + client.sessions.end(id=session_id) + return {"elapsed": elapsed, "llm_calls": 0 if cache_count >= 4 else 4} + + except Exception as error: + print(f"Error: {error}") + client.sessions.end(id=session_id) + raise -async def main(): +def main(): """Main function demonstrating caching benefits with side-by-side comparison""" print("\n╔═══════════════════════════════════════════════════════════╗") print("║ Caching Demo - Run This Script TWICE! ║") @@ -255,8 +271,8 @@ async def main(): if cache_exists: # Read cache file to count entries - async with aiofiles.open(CACHE_FILE) as f: - cache_content = await f.read() + with open(CACHE_FILE) as f: + cache_content = f.read() cache_data = json.loads(cache_content) cache_count = len(cache_data) print(f"📂 Cache found: {cache_count} entries") @@ -267,8 +283,8 @@ async def main(): print("\nRunning comparison: without cache vs with cache...\n") # Run both workflows for comparison - without_cache = await run_without_cache() - with_cache = await run_with_cache() + without_cache = run_without_cache() + with_cache = run_with_cache() # Display comparison results print("\n=== Comparison ===") @@ -286,11 +302,11 @@ async def main(): if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Error in caching demo: {err}") print("Common issues:") print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") print(" - Verify GOOGLE_API_KEY is set for the model") - print("Docs: https://docs.stagehand.dev/v2/first-steps/introduction") + print("Docs: https://docs.stagehand.dev/v3/first-steps/introduction") exit(1) diff --git a/python/basic-recaptcha/main.py b/python/basic-recaptcha/main.py index e8138a50..dc13bab5 100644 --- a/python/basic-recaptcha/main.py +++ b/python/basic-recaptcha/main.py @@ -1,11 +1,13 @@ # Basic reCAPTCHA Solving with Browserbase - See README.md for full documentation -import asyncio import os +import threading +from browserbase import Browserbase from dotenv import load_dotenv +from playwright.sync_api import sync_playwright -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables load_dotenv() @@ -14,52 +16,47 @@ solve_captchas = True -async def main(): - # Initialize Stagehand with Browserbase for cloud-based browser automation. - # Enable captcha solving in browser settings for automatic reCAPTCHA handling. - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), +def main(): + # Initialize Browserbase SDK for session creation with captcha solving + bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) + + # Create session with captcha solving enabled + session = bb.sessions.create( project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="google/gemini-2.5-pro", - model_api_key=os.environ.get("GOOGLE_API_KEY"), - verbose=0, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - "browser_settings": { - "solveCaptchas": solve_captchas, - }, + browser_settings={ + "solveCaptchas": solve_captchas, }, ) + session_id = session.id - try: - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - # Initialize browser session to start automation. - print("Stagehand initialized successfully!") - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("GOOGLE_API_KEY"), + ) - if session_id: - print(f"Live View Link: https://browserbase.com/sessions/{session_id}") + try: + print("Stagehand initialized successfully!") + print(f"Live View Link: https://browserbase.com/sessions/{session_id}") - page = stagehand.page + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() # Navigate to Google reCAPTCHA demo page to test captcha solving. print("Navigating to reCAPTCHA demo page...") - await page.goto("https://google.com/recaptcha/api2/demo") + page.goto("https://google.com/recaptcha/api2/demo") # Wait for Browserbase to solve the captcha automatically. # Listen for console messages indicating captcha solving progress. if solve_captchas: print("Waiting for captcha to be solved...") - captcha_solved = asyncio.Event() + captcha_solved = threading.Event() def handle_console(msg): if msg.text == "browserbase-solving-started": @@ -69,43 +66,54 @@ def handle_console(msg): captcha_solved.set() page.on("console", handle_console) - await captcha_solved.wait() + captcha_solved.wait() else: print("Captcha solving is disabled. Skipping wait...") # Click submit again after captcha is solved to complete the form submission. print("Clicking submit button after captcha is solved...") - await page.act("Click the Submit button") + client.sessions.act( + id=session_id, + input="Click the Submit button", + ) # Extract and display the page content to verify successful submission. print("Extracting page content...") - text = await page.extract("Extract all the text on this page") + extract_response = client.sessions.extract( + id=session_id, + instruction="Extract all the text on this page", + schema={ + "type": "object", + "properties": { + "text": {"type": "string", "description": "All text on the page"} + }, + "required": ["text"], + }, + ) + text = extract_response.data.result.get("text", "") print("Page content:") print(text) # Check if captcha was successfully solved by looking for success message. - if isinstance(text, dict) and "extraction" in text: - extraction_text = text["extraction"] - elif isinstance(text, str): - extraction_text = text - else: - extraction_text = str(text) - - if "Verification Success... Hooray!" in extraction_text: + if "Verification Success... Hooray!" in text: print("reCAPTCHA successfully solved!") else: print("Could not verify captcha success from page content") + browser.close() + + client.sessions.end(id=session_id) print("Session closed successfully") except Exception as error: print(f"Error during reCAPTCHA solving: {error}") + client.sessions.end(id=session_id) raise if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Error in reCAPTCHA solving example: {err}") print("Common issues:") diff --git a/python/company-value-prop-generator/main.py b/python/company-value-prop-generator/main.py index cc49e200..d6b3e09a 100644 --- a/python/company-value-prop-generator/main.py +++ b/python/company-value-prop-generator/main.py @@ -1,13 +1,13 @@ # Stagehand + Browserbase: Value Prop One-Liner Generator - See README.md for full documentation -import asyncio import os from dotenv import load_dotenv from openai import OpenAI +from playwright.sync_api import sync_playwright from pydantic import BaseModel, Field -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables load_dotenv() @@ -16,48 +16,45 @@ target_domain = "www.browserbase.com" # Or extract from email: email.split("@")[1] # Initialize OpenAI client -client = OpenAI() +openai_client = OpenAI() class ValueProp(BaseModel): value_prop: str = Field(..., description="the value proposition from the landing page") -async def generate_one_liner(domain: str) -> str: +def generate_one_liner(domain: str) -> str: """ Analyzes a website's landing page to generate a concise one-liner value proposition. Extracts the value prop using Stagehand, then uses an LLM to format it into a short phrase starting with "your". """ - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - }, - verbose=0, # 0 = errors only, 1 = info, 2 = debug ) - try: - async with Stagehand(config) as stagehand: - print("Stagehand initialized successfully!") - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id + # Start a new session + start_response = client.sessions.start(model_name="openai/gpt-4.1") + session_id = start_response.data.session_id - if session_id: - print(f"Live View Link: https://browserbase.com/sessions/{session_id}") + try: + print("Stagehand initialized successfully!") + print(f"Live View Link: https://browserbase.com/sessions/{session_id}") - page = stagehand.page + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() # Navigate to domain print(f"🌐 Navigating to https://{domain}...") # 5min timeout to handle slow-loading sites or network issues - await page.goto( + page.goto( f"https://{domain}/", wait_until="domcontentloaded", timeout=300000, @@ -67,12 +64,13 @@ async def generate_one_liner(domain: str) -> str: # Extract value proposition from landing page print(f"📝 Extracting value proposition for {domain}...") - value_prop_data = await page.extract( - "extract the value proposition from the landing page", - schema=ValueProp, + extract_response = client.sessions.extract( + id=session_id, + instruction="extract the value proposition from the landing page", + schema=ValueProp.model_json_schema(), ) - value_prop = value_prop_data.value_prop + value_prop = extract_response.data.result.get("value_prop", "") print(f"📊 Extracted value prop for {domain}: {value_prop}") # Validate extraction returned meaningful content @@ -85,8 +83,7 @@ async def generate_one_liner(domain: str) -> str: # System prompt enforces constraints (9 words max, no quotes, must start with "your") print(f"🤖 Generating email one-liner for {domain}...") - response = await asyncio.to_thread( - client.chat.completions.create, + response = openai_client.chat.completions.create( model="gpt-4.1", messages=[ { @@ -126,24 +123,28 @@ async def generate_one_liner(domain: str) -> str: ) print(f"✨ Generated one-liner for {domain}: {one_liner}") - return one_liner + + browser.close() + + client.sessions.end(id=session_id) + print("Session closed successfully") + return one_liner except Exception as error: error_message = str(error) if isinstance(error, Exception) else error print(f"❌ Generation failed for {domain}: {error_message}") + client.sessions.end(id=session_id) raise - finally: - print("Session closed successfully") -async def main(): +def main(): """ Main entry point: generates a one-liner value proposition for the target domain. """ print("Starting One-Liner Generator...") try: - one_liner = await generate_one_liner(target_domain) + one_liner = generate_one_liner(target_domain) print("\n✅ Success!") print(f"One-liner: {one_liner}") except Exception as error: @@ -162,7 +163,7 @@ async def main(): if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Fatal error: {err}") exit(1) diff --git a/python/context/main.py b/python/context/main.py index 156a2f76..eaf6fc40 100644 --- a/python/context/main.py +++ b/python/context/main.py @@ -1,20 +1,20 @@ # Stagehand + Browserbase: Context Authentication Example - See README.md for full documentation -import asyncio import os import requests from browserbase import Browserbase from dotenv import load_dotenv +from playwright.sync_api import sync_playwright from pydantic import BaseModel, Field -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables load_dotenv() -async def create_session_context_id(): +def create_session_context_id(): print("Creating new Browserbase context...") # First create a context using Browserbase SDK to get a context ID. bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) @@ -33,30 +33,33 @@ async def create_session_context_id(): } }, ) - print(f"Live view: https://browserbase.com/sessions/{session.id}") + session_id = session.id + print(f"Live view: https://browserbase.com/sessions/{session_id}") - # Connect Stagehand to the existing session (no new session created). - print("Connecting Stagehand to session...") - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_id=session.id, - verbose=1, # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging ) - async with Stagehand(config) as stagehand: - page = stagehand.page + # Connect Stagehand to the existing session (no new session created). + print("Connecting Stagehand to session...") + + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + ctx = browser.contexts[0] + page = ctx.pages[0] if ctx.pages else ctx.new_page() + email = os.environ.get("SF_REC_PARK_EMAIL") password = os.environ.get("SF_REC_PARK_PASSWORD") # Navigate to login page with extended timeout for slow-loading sites. print("Navigating to SF Rec & Park login page...") - await page.goto( + page.goto( "https://www.rec.us/organizations/san-francisco-rec-park", wait_until="domcontentloaded", timeout=60000, @@ -64,24 +67,42 @@ async def create_session_context_id(): # Perform login sequence: each step is atomic to handle dynamic page changes. print("Starting login sequence...") - await page.act("Click the Login button") - await page.act(f'Fill in the email or username field with "{email}"') - await page.act("Click the next, continue, or submit button to proceed") - await page.act(f'Fill in the password field with "{password}"') - await page.act("Click the login, sign in, or submit button") + client.sessions.act( + id=session_id, + input="Click the Login button", + ) + client.sessions.act( + id=session_id, + input=f'Fill in the email or username field with "{email}"', + ) + client.sessions.act( + id=session_id, + input="Click the next, continue, or submit button to proceed", + ) + client.sessions.act( + id=session_id, + input=f'Fill in the password field with "{password}"', + ) + client.sessions.act( + id=session_id, + input="Click the login, sign in, or submit button", + ) print("Login sequence completed!") + browser.close() + + client.sessions.end(id=session_id) print("Authentication state saved to context") # Return the context ID for reuse in future sessions. return {"id": context.id} -async def delete_context(context_id: str): +def delete_context(context_id: str): + """Delete context via Browserbase API to clean up stored authentication data. + This prevents accumulation of unused contexts and ensures security cleanup.""" try: print(f"Cleaning up context: {context_id}") - # Delete context via Browserbase API to clean up stored authentication data. - # This prevents accumulation of unused contexts and ensures security cleanup. response = requests.delete( f"https://api.browserbase.com/v1/contexts/{context_id}", headers={ @@ -94,75 +115,91 @@ async def delete_context(context_id: str): print(f"Error deleting context: {error_msg}") -async def main(): +def main(): print("Starting Context Authentication Example...") # Create context with login state for reuse in authenticated sessions. - context_id = await create_session_context_id() + context_id = create_session_context_id() # Initialize new session using existing context to inherit authentication state. # persist: true ensures any new changes (cookies, cache) are saved back to context. - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), + bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) + session = bb.sessions.create( project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - "browser_settings": { - "context": { - "id": context_id["id"], - "persist": True, - } - }, + browser_settings={ + "context": { + "id": context_id["id"], + "persist": True, + } }, - verbose=1, # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging ) + session_id = session.id - async with Stagehand(config) as stagehand: - print("Authenticated session ready!") - if hasattr(stagehand, "session_id"): - print(f"Live view: https://browserbase.com/sessions/{stagehand.session_id}") - elif hasattr(stagehand, "browserbase_session_id"): - print(f"Live view: https://browserbase.com/sessions/{stagehand.browserbase_session_id}") - - page = stagehand.page - - # Navigate to authenticated area - should skip login due to persisted cookies. - print("Navigating to authenticated area (should skip login)...") - await page.goto( - "https://www.rec.us/organizations/san-francisco-rec-park", - wait_until="domcontentloaded", - timeout=60000, - ) - - # Navigate to user-specific area to access personal data. - await page.act("Click on the reservations button") - - # Extract structured user data using Pydantic schema for type safety. - # Schema ensures consistent data format and validates extracted content. - print("Extracting user profile data...") - - # Define schema using Pydantic - class UserData(BaseModel): - full_name: str = Field(..., description="the user's full name") - address: str = Field(..., description="the user's address") - - user_data = await page.extract("Extract the user's full name and address", schema=UserData) + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("OPENAI_API_KEY"), + ) - print(f"Extracted user data: {user_data}") + try: + print("Authenticated session ready!") + print(f"Live view: https://browserbase.com/sessions/{session_id}") + + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() + + # Navigate to authenticated area - should skip login due to persisted cookies. + print("Navigating to authenticated area (should skip login)...") + page.goto( + "https://www.rec.us/organizations/san-francisco-rec-park", + wait_until="domcontentloaded", + timeout=60000, + ) + + # Navigate to user-specific area to access personal data. + client.sessions.act( + id=session_id, + input="Click on the reservations button", + ) + + # Extract structured user data using Pydantic schema for type safety. + # Schema ensures consistent data format and validates extracted content. + print("Extracting user profile data...") + + class UserData(BaseModel): + full_name: str = Field(..., description="the user's full name") + address: str = Field(..., description="the user's address") + + extract_response = client.sessions.extract( + id=session_id, + instruction="Extract the user's full name and address", + schema=UserData.model_json_schema(), + ) + + print(f"Extracted user data: {extract_response.data.result}") + + browser.close() + + client.sessions.end(id=session_id) + print("Session closed successfully") - print("Session closed successfully") + except Exception as error: + print(f"Error: {error}") + client.sessions.end(id=session_id) + raise # Clean up context to prevent accumulation and ensure security. - await delete_context(context_id["id"]) + delete_context(context_id["id"]) if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Error in context authentication example: {err}") print("Common issues:") diff --git a/python/council-events/README.md b/python/council-events/README.md index a047763f..7da0ab3b 100644 --- a/python/council-events/README.md +++ b/python/council-events/README.md @@ -18,11 +18,10 @@ ## QUICKSTART 1. cd council-events -2. uv venv venv -3. source venv/bin/activate # On Windows: venv\Scripts\activate -4. pip install stagehand python-dotenv pydantic -5. cp .env.example .env # Add your Browserbase API key, Project ID, and OpenAI API key to .env -6. python main.py +2. uv venv && source .venv/bin/activate # On Windows: .venv\Scripts\activate +3. pip install stagehand python-dotenv pydantic +4. cp .env.example .env # Add your Browserbase API key, Project ID, and OpenAI API key to .env +5. python main.py ## EXPECTED OUTPUT diff --git a/python/council-events/main.py b/python/council-events/main.py index 4e246959..6d8e0f39 100644 --- a/python/council-events/main.py +++ b/python/council-events/main.py @@ -1,12 +1,12 @@ # Stagehand + Browserbase: Philadelphia Council Events Scraper - See README.md for full documentation -import asyncio +import json import os from dotenv import load_dotenv from pydantic import BaseModel, Field -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables load_dotenv() @@ -26,7 +26,7 @@ class EventResults(BaseModel): results: list[Event] = Field(..., description="array of events") -async def main(): +def main(): """ Searches Philadelphia Council Events for 2025 and extracts event information. Uses AI-powered browser automation to navigate and interact with the site. @@ -34,64 +34,72 @@ async def main(): print("Starting Philadelphia Council Events automation...") # Initialize Stagehand with Browserbase for cloud-based browser automation - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), model_api_key=os.environ.get("OPENAI_API_KEY"), - verbose=1, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging ) - try: - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - print("Initializing browser session...") - print("Stagehand session started successfully") - - # Provide live session URL for debugging and monitoring - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id - - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") - - page = stagehand.page - - # Navigate to Philadelphia Council - print("Navigating to: https://phila.legistar.com/") - await page.goto("https://phila.legistar.com/") - print("Page loaded successfully") - - # Click calendar from the navigation menu - print("Clicking calendar from the navigation menu") - await page.act("click calendar from the navigation menu") - - # Select 2025 from the month dropdown - print("Selecting 2025 from the month dropdown") - await page.act("select 2025 from the month dropdown") - - # Extract event data using AI to parse the structured information - print("Extracting event information...") - results = await page.extract( - "Extract the table with the name, date and time of the events", schema=EventResults - ) + # Start a new session + start_response = client.sessions.start(model_name="openai/gpt-4.1") + session_id = start_response.data.session_id - print(f"Found {len(results.results)} events") - print("Event data extracted successfully:") - - # Display results in formatted JSON - import json - - print(json.dumps(results.model_dump(), indent=2)) - - print("Session closed successfully") + try: + print("Initializing browser session...") + print("Stagehand session started successfully") + # Provide live session URL for debugging and monitoring + print(f"Watch live: https://browserbase.com/sessions/{session_id}") + + # Navigate to Philadelphia Council + print("Navigating to: https://phila.legistar.com/") + client.sessions.navigate(id=session_id, url="https://phila.legistar.com/") + print("Page loaded successfully") + + # Click calendar from the navigation menu + print("Clicking calendar from the navigation menu") + client.sessions.act( + id=session_id, + input="click calendar from the navigation menu", + ) + + # Select 2025 from the month dropdown + print("Selecting 2025 from the month dropdown") + client.sessions.act( + id=session_id, + input="select 2025 from the month dropdown", + ) + + # Extract event data using AI to parse the structured information + print("Extracting event information...") + events_schema = { + "type": "object", + "properties": { + "results": { + "type": "array", + "description": "array of events", + "items": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "the name of the event"}, + "date": {"type": "string", "description": "the date of the event"}, + "time": {"type": "string", "description": "the time of the event"}, + }, + "required": ["name", "date", "time"], + }, + } + }, + "required": ["results"], + } + extract_response = client.sessions.extract( + id=session_id, + instruction="Extract the table with the name, date and time of the events", + schema=events_schema, + ) + + results = extract_response.data.result + print(f"Found {len(results.get('results', []))} events") + print("Event data extracted successfully:") + print(json.dumps(results, indent=2)) except Exception as error: print(f"Error during event extraction: {error}") @@ -103,13 +111,16 @@ async def main(): print("3. Ensure internet access and https://phila.legistar.com is accessible") print("4. Verify Browserbase account has sufficient credits") print("5. Check if the calendar page structure has changed") - raise + finally: + client.sessions.end(id=session_id) + print("Session closed successfully") + if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Application error: {err}") exit(1) diff --git a/python/download-financial-statements/main.py b/python/download-financial-statements/main.py index c32074ae..7cbca701 100644 --- a/python/download-financial-statements/main.py +++ b/python/download-financial-statements/main.py @@ -1,21 +1,20 @@ # Stagehand + Browserbase: Download Apple's Quarterly Financial Statements - See README.md for full documentation -import asyncio import os +import time from browserbase import Browserbase from dotenv import load_dotenv +from playwright.sync_api import sync_playwright -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables from .env file # Required: BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, GOOGLE_API_KEY load_dotenv() -async def save_downloads_with_retry( - bb: Browserbase, session_id: str, retry_for_seconds: int = 30 -) -> int: +def save_downloads_with_retry(bb: Browserbase, session_id: str, retry_for_seconds: int = 30) -> int: """ Polls Browserbase API for downloads with timeout handling. @@ -37,11 +36,11 @@ async def save_downloads_with_retry( print(f"Waiting up to {retry_for_seconds} seconds for downloads to complete...") # Track elapsed time to implement timeout without using threading timers - start_time = asyncio.get_event_loop().time() + start_time = time.time() timeout = retry_for_seconds while True: - elapsed = asyncio.get_event_loop().time() - start_time + elapsed = time.time() - start_time # Check if we've exceeded the timeout period if elapsed >= timeout: @@ -49,10 +48,8 @@ async def save_downloads_with_retry( try: print("Checking for downloads...") - # Use asyncio.to_thread for synchronous Browserbase SDK calls - # This prevents blocking the event loop while waiting for API responses - response = await asyncio.to_thread(bb.sessions.downloads.list, session_id) - download_buffer = await asyncio.to_thread(response.read) + response = bb.sessions.downloads.list(session_id) + download_buffer = response.read() # Check if downloads are ready (non-empty buffer indicates files are available) if len(download_buffer) > 0: @@ -71,10 +68,10 @@ async def save_downloads_with_retry( # Poll every 2 seconds to check if downloads are ready # This interval balances responsiveness with API rate limits - await asyncio.sleep(2) + time.sleep(2) -async def main(): +def main(): """ Main application entry point. @@ -92,77 +89,98 @@ async def main(): # Initialize Stagehand with Browserbase for cloud-based browser automation # Stagehand provides natural language browser control (act, extract, observe) - config = StagehandConfig( - env="BROWSERBASE", # Use Browserbase's cloud infrastructure instead of local browsers - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="google/gemini-2.5-flash-preview-05-20", # AI model for interpreting natural language actions + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), model_api_key=os.environ.get("GOOGLE_API_KEY"), - verbose=2, - # Logging levels: 0 = errors only, 1 = info, 2 = debug - # When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs - # https://docs.stagehand.dev/configuration/logging ) + # Start a new session + start_response = client.sessions.start( + model_name="google/gemini-2.5-flash-preview-05-20", + ) + session_id = start_response.data.session_id + try: - # Initialize browser session to start automation - # Using context manager ensures proper cleanup even if errors occur - async with Stagehand(config) as stagehand: - print("Stagehand initialized successfully!") - page = stagehand.page - - # Display live view URL for debugging and monitoring - # Live view allows real-time observation of browser automation - live_view_links = bb.sessions.debug(stagehand.session_id) - live_view_link = live_view_links.debuggerFullscreenUrl - print(f"🔍 Live View Link: {live_view_link}") + print("Stagehand initialized successfully!") + + # Display live view URL for debugging and monitoring + # Live view allows real-time observation of browser automation + live_view_links = bb.sessions.debug(session_id) + live_view_link = live_view_links.debuggerFullscreenUrl + print(f"🔍 Live View Link: {live_view_link}") + + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() # Navigate to Apple homepage with extended timeout for slow-loading sites print("Navigating to Apple.com...") - await page.goto("https://www.apple.com/", timeout=60000) + page.goto("https://www.apple.com/", timeout=60000) # Navigate to investor relations section using natural language actions # act() uses AI to interpret instructions and perform browser interactions print("Navigating to Investors section...") - await page.act("Click the 'Investors' button at the bottom of the page") - await page.act("Scroll down to the Financial Data section of the page") - await page.act("Under Quarterly Earnings Reports, click on '2025'") + client.sessions.act( + id=session_id, + input="Click the 'Investors' button at the bottom of the page", + ) + client.sessions.act( + id=session_id, + input="Scroll down to the Financial Data section of the page", + ) + client.sessions.act( + id=session_id, + input="Under Quarterly Earnings Reports, click on '2025'", + ) # Download all quarterly financial statements # When a URL of a PDF is opened, Browserbase automatically downloads and stores the PDF # Files are captured in the session and can be retrieved via the downloads API # See https://docs.browserbase.com/features/screenshots#pdfs for more info print("Downloading quarterly financial statements...") - await page.act("Click the 'Financial Statements' link under Q4") - await page.act("Click the 'Financial Statements' link under Q3") - await page.act("Click the 'Financial Statements' link under Q2") - await page.act("Click the 'Financial Statements' link under Q1") + client.sessions.act( + id=session_id, + input="Click the 'Financial Statements' link under Q4", + ) + client.sessions.act( + id=session_id, + input="Click the 'Financial Statements' link under Q3", + ) + client.sessions.act( + id=session_id, + input="Click the 'Financial Statements' link under Q2", + ) + client.sessions.act( + id=session_id, + input="Click the 'Financial Statements' link under Q1", + ) # Retrieve all downloads triggered during this session from Browserbase API # Files take time to process, so we poll with retry logic (45 second timeout) print("Retrieving downloads from Browserbase...") - await save_downloads_with_retry(bb, stagehand.session_id, 45) + save_downloads_with_retry(bb, session_id, 45) print("All downloads completed successfully!") - # Print history if available in Stagehand v2 - # History shows all actions taken during the session for debugging - if hasattr(stagehand, "history"): - print("\nStagehand History:") - print(stagehand.history) + browser.close() - # Context manager automatically closes the session and cleans up resources + client.sessions.end(id=session_id) print("Session closed successfully") except Exception as error: print(f"Error during automation: {error}") + client.sessions.end(id=session_id) raise if __name__ == "__main__": # Entry point for script execution - # asyncio.run() creates event loop and runs main() coroutine until completion try: - asyncio.run(main()) + main() except Exception as err: # Handle any uncaught exceptions and provide helpful debugging information print(f"Application error: {err}") @@ -170,5 +188,5 @@ async def main(): print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") print(" - Verify internet connection and Apple website accessibility") print(" - Ensure sufficient timeout for slow-loading pages") - print("Docs: https://docs.stagehand.dev/v2/first-steps/introduction") + print("Docs: https://docs.stagehand.dev/v3/first-steps/introduction") exit(1) diff --git a/python/form-filling/README.md b/python/form-filling/README.md index 5ac84aa7..e49077ca 100644 --- a/python/form-filling/README.md +++ b/python/form-filling/README.md @@ -18,12 +18,11 @@ ## QUICKSTART -1. cd form-fill-template -2. uv venv venv -3. source venv/bin/activate # On Windows: venv\Scripts\activate -4. pip install -r requirements.txt -5. cp .env.example .env # Add your Browserbase API key and Project ID to .env -6. python main.py +1. cd form-filling +2. uv venv && source .venv/bin/activate # On Windows: .venv\Scripts\activate +3. pip install stagehand python-dotenv +4. cp .env.example .env # Add your Browserbase API key, Project ID, and OpenAI API key to .env +5. python main.py ## EXPECTED OUTPUT diff --git a/python/form-filling/main.py b/python/form-filling/main.py index 2fd7d9e5..550a6c6b 100644 --- a/python/form-filling/main.py +++ b/python/form-filling/main.py @@ -1,11 +1,11 @@ # Stagehand + Browserbase: Form Filling Automation - See README.md for full documentation -import asyncio import os +import time from dotenv import load_dotenv -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables load_dotenv() @@ -22,79 +22,89 @@ ) -async def main(): +def main(): print("Starting Form Filling Example...") - # Initialize Stagehand with Browserbase for cloud-based browser automation. - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - }, - verbose=1, # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging ) - try: - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - print("Stagehand initialized successfully!") - if hasattr(stagehand, "session_id"): - print(f"Live View Link: https://browserbase.com/sessions/{stagehand.session_id}") - elif hasattr(stagehand, "browserbase_session_id"): - print( - f"Live View Link: https://browserbase.com/sessions/{stagehand.browserbase_session_id}" - ) - - page = stagehand.page - - # Navigate to contact page with extended timeout for slow-loading sites. - print("Navigating to Browserbase contact page...") - await page.goto( - "https://www.browserbase.com/contact", - wait_until="domcontentloaded", # Wait for DOM to be ready before proceeding. - timeout=60000, # Extended timeout for reliable page loading. - ) - - # Fill form using individual act() calls for reliability - print("Filling in contact form...") - - # Fill each field individually for better reliability - await page.act(f'Fill in the first name field with "{first_name}"') - await page.act(f'Fill in the last name field with "{last_name}"') - await page.act(f'Fill in the company field with "{company}"') - await page.act(f'Fill in the job title field with "{job_title}"') - await page.act(f'Fill in the email field with "{email}"') - await page.act(f'Fill in the message field with "{message}"') - - # Language choice in Stagehand act() is crucial for reliable automation. - # Use "click" for dropdown interactions rather than "select" - await page.act("Click on the How Can we help? dropdown") - await page.wait_for_timeout(500) - await page.act("Click on the first option from the dropdown") - # await page.act("Select the first option from the dropdown") # Less reliable than "click" - - # Uncomment the line below if you want to submit the form - # await page.act("Click the submit button") - - print("Form filled successfully! Waiting 3 seconds...") - await page.wait_for_timeout(30000) + # Start a new session + start_response = client.sessions.start( + model_name="openai/gpt-4.1", + ) + session_id = start_response.data.session_id + print(f"Live View Link: https://browserbase.com/sessions/{session_id}") - print("Session closed successfully") + try: + print("Stagehand initialized successfully!") + + # Navigate to contact page + print("Navigating to Browserbase contact page...") + client.sessions.navigate(id=session_id, url="https://www.browserbase.com/contact") + + # Fill form using individual act() calls for reliability + print("Filling in contact form...") + + # Fill each field individually for better reliability + client.sessions.act( + id=session_id, + input=f'Fill in the first name field with "{first_name}"', + ) + client.sessions.act( + id=session_id, + input=f'Fill in the last name field with "{last_name}"', + ) + client.sessions.act( + id=session_id, + input=f'Fill in the company field with "{company}"', + ) + client.sessions.act( + id=session_id, + input=f'Fill in the job title field with "{job_title}"', + ) + client.sessions.act( + id=session_id, + input=f'Fill in the email field with "{email}"', + ) + client.sessions.act( + id=session_id, + input=f'Fill in the message field with "{message}"', + ) + + # Language choice in Stagehand act() is crucial for reliable automation. + # Use "click" for dropdown interactions rather than "select" + client.sessions.act( + id=session_id, + input="Click on the How Can we help? dropdown", + ) + time.sleep(0.5) + client.sessions.act( + id=session_id, + input="Click on the first option from the dropdown", + ) + + # Uncomment the line below if you want to submit the form + # client.sessions.act(id=session_id, input="Click the submit button") + + print("Form filled successfully! Waiting 30 seconds...") + time.sleep(30) except Exception as error: print(f"Error during form filling: {error}") raise + finally: + client.sessions.end(id=session_id) + print("Session closed successfully") + if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Error in form filling example: {err}") print("Common issues:") diff --git a/python/gift-finder/main.py b/python/gift-finder/main.py index 980638bf..a889b522 100644 --- a/python/gift-finder/main.py +++ b/python/gift-finder/main.py @@ -1,17 +1,20 @@ # Stagehand + Browserbase: AI-Powered Gift Finder - See README.md for full documentation -import asyncio +import json import os +import time from dotenv import load_dotenv from openai import OpenAI +from playwright.sync_api import sync_playwright from pydantic import BaseModel, Field, HttpUrl -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables load_dotenv() + # ============= CONFIGURATION ============= # Update these values to customize your gift search CONFIG = { @@ -41,10 +44,28 @@ class SearchResult(BaseModel): products: list[Product] -client = OpenAI() +class ProductItem(BaseModel): + title: str = Field(..., description="the title/name of the product") + url: HttpUrl = Field(..., description="the full URL link to the product page") + price: str = Field(..., description="the price of the product (include currency symbol)") + rating: str = Field( + ..., + description="the star rating or number of reviews (e.g., '4.5 stars' or '123 reviews')", + ) + + +class ProductsData(BaseModel): + products: list[ProductItem] = Field( + ..., + max_length=3, + description="array of the first 3 products from search results", + ) + + +openai_client = OpenAI() -async def generate_search_queries(recipient: str, description: str) -> list[str]: +def generate_search_queries(recipient: str, description: str) -> list[str]: """ Generate intelligent search queries based on recipient profile. @@ -55,8 +76,7 @@ async def generate_search_queries(recipient: str, description: str) -> list[str] # Use AI to generate search terms based on recipient profile # This avoids generic searches and focuses on thoughtful, complementary gifts - response = await asyncio.to_thread( - client.chat.completions.create, + response = openai_client.chat.completions.create( model="gpt-4.1", messages=[ { @@ -89,7 +109,7 @@ async def generate_search_queries(recipient: str, description: str) -> list[str] return queries[:3] -async def score_products( +def score_products( products: list[Product], recipient: str, description: str, @@ -119,8 +139,7 @@ async def score_products( print(f"Scoring {len(all_products)} products...") - response = await asyncio.to_thread( - client.chat.completions.create, + response = openai_client.chat.completions.create( model="gpt-4.1", messages=[ { @@ -180,8 +199,6 @@ async def score_products( ) # Parse JSON response from AI scoring - import json - scores_data = json.loads(response_content) # Map AI scores back to products using index matching @@ -211,7 +228,7 @@ async def score_products( return all_products -async def get_user_input() -> GiftFinderAnswers: +def get_user_input() -> GiftFinderAnswers: """ Collect user input for gift recipient and description. @@ -231,21 +248,129 @@ async def get_user_input() -> GiftFinderAnswers: return GiftFinderAnswers(recipient=CONFIG["recipient"], description=CONFIG["description"]) -async def main() -> None: +def run_single_search(query: str, session_index: int) -> SearchResult: + """Run a single search session for a given query.""" + print(f'Starting search session {session_index + 1} for: "{query}"') + + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("OPENAI_API_KEY"), + ) + + # Start a new session + start_response = client.sessions.start( + model_name="openai/gpt-4.1", + ) + session_id = start_response.data.session_id + + try: + live_view_url = f"https://www.browserbase.com/sessions/{session_id}" + print(f"Session {session_index + 1} Live View: {live_view_url}") + + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() + + # Navigate to European gift site + print(f"Session {session_index + 1}: Navigating to Firebox.eu...") + page.goto("https://firebox.eu/") + + # Perform search using natural language actions + print(f'Session {session_index + 1}: Searching for "{query}"...') + client.sessions.act( + id=session_id, + input=f"Type {query} into the search bar", + ) + client.sessions.act( + id=session_id, + input="Click the search button", + ) + time.sleep(1) + + # Extract structured product data using inline schema (avoids $ref issues) + print(f"Session {session_index + 1}: Extracting product data...") + + products_schema = { + "type": "object", + "properties": { + "products": { + "type": "array", + "description": "array of the first 3 products from search results", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "the title/name of the product", + }, + "url": { + "type": "string", + "description": "the full URL link to the product page", + }, + "price": { + "type": "string", + "description": "the price of the product (include currency symbol)", + }, + "rating": { + "type": "string", + "description": "the star rating or number of reviews", + }, + }, + "required": ["title", "url", "price", "rating"], + }, + } + }, + "required": ["products"], + } + extract_response = client.sessions.extract( + id=session_id, + instruction="Extract the first 3 products from the search results", + schema=products_schema, + ) + products_data = extract_response.data.result + + print( + f'Session {session_index + 1}: Found {len(products_data.get("products", []))} products for "{query}"' + ) + + # Convert to Product objects + products = [ + Product(title=p["title"], url=str(p["url"]), price=p["price"], rating=p["rating"]) + for p in products_data.get("products", []) + ] + + browser.close() + + client.sessions.end(id=session_id) + return SearchResult(query=query, session_index=session_index + 1, products=products) + + except Exception as error: + print(f"Session {session_index + 1} failed: {error}") + client.sessions.end(id=session_id) + return SearchResult(query=query, session_index=session_index + 1, products=[]) + + +def main() -> None: """ Main application entry point. Orchestrates the entire gift finding process: 1. Collects user input 2. Generates intelligent search queries - 3. Runs concurrent browser searches + 3. Runs browser searches sequentially 4. Scores and ranks products with AI 5. Displays top recommendations """ print("Starting Gift Finder Application...") # Step 1: Collect user input - user_input = await get_user_input() + user_input = get_user_input() recipient = user_input.recipient description = user_input.description print(f"User input received: {recipient} - {description}") @@ -253,7 +378,7 @@ async def main() -> None: # Step 2: Generate search queries using AI print("\nGenerating intelligent search queries...") try: - search_queries = await generate_search_queries(recipient, description) + search_queries = generate_search_queries(recipient, description) print("\nGenerated Search Queries:") for index, query in enumerate(search_queries): @@ -265,125 +390,17 @@ async def main() -> None: search_queries = ["gifts", "accessories", "items"] print("Using fallback search queries") - # Step 3: Start concurrent browser searches - print("\nStarting concurrent browser searches...") - - async def run_single_search(query: str, session_index: int) -> SearchResult: - print(f'Starting search session {session_index + 1} for: "{query}"') - - # Create separate Stagehand instance for each search to run concurrently - # Each session searches independently to maximize speed and parallel processing - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - verbose=1, # Silent logging to avoid cluttering output - # Logging levels: 0 = errors only, 1 = info, 2 = debug - # When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs - # https://docs.stagehand.dev/configuration/logging - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - # Proxies require Developer Plan or higher - uncomment if you have a Developer Plan or higher - # "proxies": [ - # { - # "type": "browserbase", - # "geolocation": { - # "city": "LONDON", - # "country": "GB" - # } - # } - # ], - "region": "us-east-1", - "timeout": 900, - "browser_settings": { - "viewport": { - "width": 1920, - "height": 1080, - } - }, - }, - ) - - try: - # Initialize browser session with Stagehand - async with Stagehand(config) as session_stagehand: - session_page = session_stagehand.page - - # Display live view URL for debugging and monitoring - session_id = None - if hasattr(session_stagehand, "session_id"): - session_id = session_stagehand.session_id - elif hasattr(session_stagehand, "browserbase_session_id"): - session_id = session_stagehand.browserbase_session_id - - if session_id: - live_view_url = f"https://www.browserbase.com/sessions/{session_id}" - print(f"Session {session_index + 1} Live View: {live_view_url}") - - # Navigate to European gift site - print(f"Session {session_index + 1}: Navigating to Firebox.eu...") - await session_page.goto("https://firebox.eu/") - - # Perform search using natural language actions - print(f'Session {session_index + 1}: Searching for "{query}"...') - await session_page.act(f"Type {query} into the search bar") - await session_page.act("Click the search button") - await session_page.wait_for_timeout(1000) - - # Extract structured product data using Pydantic schema for type safety - print(f"Session {session_index + 1}: Extracting product data...") - - # Define Pydantic schemas for structured data extraction - class ProductItem(BaseModel): - title: str = Field(..., description="the title/name of the product") - url: HttpUrl = Field(..., description="the full URL link to the product page") - price: str = Field( - ..., description="the price of the product (include currency symbol)" - ) - rating: str = Field( - ..., - description="the star rating or number of reviews (e.g., '4.5 stars' or '123 reviews')", - ) - - class ProductsData(BaseModel): - products: list[ProductItem] = Field( - ..., - max_length=3, - description="array of the first 3 products from search results", - ) - - products_data = await session_page.extract( - "Extract the first 3 products from the search results", schema=ProductsData - ) - - print( - f'Session {session_index + 1}: Found {len(products_data.products)} products for "{query}"' - ) - - # Convert to Product objects - products = [ - Product(title=p.title, url=str(p.url), price=p.price, rating=p.rating) - for p in products_data.products - ] - - return SearchResult(query=query, session_index=session_index + 1, products=products) - except Exception as error: - print(f"Session {session_index + 1} failed: {error}") - - return SearchResult(query=query, session_index=session_index + 1, products=[]) - - # Create concurrent search tasks for all generated queries - search_promises = [ - run_single_search(query, index) for index, query in enumerate(search_queries) - ] + # Step 3: Start browser searches sequentially + print("\nStarting browser searches...") print("\nBrowser Sessions Starting...") print("Live view links will appear as each session initializes") - # Execute all searches concurrently using asyncio.gather() - all_results = await asyncio.gather(*search_promises) + # Execute searches sequentially + all_results = [] + for index, query in enumerate(search_queries): + result = run_single_search(query, index) + all_results.append(result) # Calculate total products found across all search sessions total_products = sum(len(result.products) for result in all_results) @@ -398,7 +415,7 @@ class ProductsData(BaseModel): if len(all_products_flat) > 0: try: # AI scores all products and ranks them by relevance to recipient - scored_products = await score_products(all_products_flat, recipient, description) + scored_products = score_products(all_products_flat, recipient, description) top3_products = scored_products[:3] print("\nTOP 3 RECOMMENDED GIFTS:") @@ -433,7 +450,7 @@ class ProductsData(BaseModel): if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Application error: {err}") print("Check your environment variables") diff --git a/python/job-application/main.py b/python/job-application/main.py index ee66a8c5..cc2315cf 100644 --- a/python/job-application/main.py +++ b/python/job-application/main.py @@ -1,6 +1,5 @@ # Stagehand + Browserbase: Job Application Automation - See README.md for full documentation -import asyncio import os import random import time @@ -8,9 +7,10 @@ import httpx from browserbase import Browserbase from dotenv import load_dotenv +from playwright.sync_api import sync_playwright from pydantic import BaseModel, Field, HttpUrl -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables load_dotenv() @@ -27,7 +27,7 @@ class JobsData(BaseModel): jobs: list[JobInfo] -async def get_project_concurrency() -> int: +def get_project_concurrency() -> int: """ Fetch project concurrency limit from Browserbase SDK. @@ -35,11 +35,7 @@ async def get_project_concurrency() -> int: capped at 5. """ bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) - - # Use asyncio.to_thread to run synchronous SDK call in thread pool - project = await asyncio.to_thread( - bb.projects.retrieve, os.environ.get("BROWSERBASE_PROJECT_ID") - ) + project = bb.projects.retrieve(os.environ.get("BROWSERBASE_PROJECT_ID")) return min(project.concurrency, 5) @@ -64,183 +60,238 @@ def generate_agent_id() -> str: return f"agent-{timestamp}-{random_string}" -async def apply_to_job(job_info: JobInfo, semaphore: asyncio.Semaphore): +def apply_to_job(job_info: JobInfo): """ Apply to a single job posting with automated form filling. Uses Stagehand to navigate to job page, fill out application form, upload resume, and submit the application. """ - # Semaphore ensures we don't exceed project concurrency limits - async with semaphore: - # Initialize Stagehand with Browserbase for cloud-based browser automation - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="google/gemini-2.5-flash", - model_api_key=os.environ.get("GOOGLE_GENERATIVE_AI_API_KEY"), - ) - - try: - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - print(f"[{job_info.title}] Session Started") - - # Get session ID for live viewing/debugging - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id - - if session_id: - print( - f"[{job_info.title}] Watch live: https://browserbase.com/sessions/{session_id}" - ) - - page = stagehand.page - - # Navigate to job URL - await page.goto(str(job_info.url)) - print(f"[{job_info.title}] Navigated to job page") - - # Click on the specific job listing to open application form - await page.act(f"click on {job_info.title}") - print(f"[{job_info.title}] Clicked on job") - - # Generate unique identifiers for this application - agent_id = generate_agent_id() - email = generate_random_email() - - print(f"[{job_info.title}] Agent ID: {agent_id}") - print(f"[{job_info.title}] Email: {email}") - - # Fill out application form fields using natural language actions - # Stagehand's act() method understands natural language instructions - await page.act(f"type '{agent_id}' into the agent identifier field") - - await page.act(f"type '{email}' into the contact endpoint field") + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("GOOGLE_GENERATIVE_AI_API_KEY"), + ) - await page.act("type 'us-west-2' into the deployment region field") + # Start a new session + start_response = client.sessions.start( + model_name="google/gemini-2.5-flash", + ) + session_id = start_response.data.session_id - # Upload agent profile/resume file - # Using observe() to find the upload button, then setting files programmatically - upload_actions = await page.observe("find the file upload button for agent profile") - if upload_actions and len(upload_actions) > 0: - upload_action = upload_actions[0] - upload_selector = str(upload_action.selector) - if upload_selector: - file_input = page.locator(upload_selector) + try: + print(f"[{job_info.title}] Session Started") + print(f"[{job_info.title}] Watch live: https://browserbase.com/sessions/{session_id}") + + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() + + # Navigate to job URL + page.goto(str(job_info.url)) + print(f"[{job_info.title}] Navigated to job page") + + # Click on the specific job listing to open application form + client.sessions.act( + id=session_id, + input=f"click on {job_info.title}", + ) + print(f"[{job_info.title}] Clicked on job") + + # Generate unique identifiers for this application + agent_id = generate_agent_id() + email = generate_random_email() + + print(f"[{job_info.title}] Agent ID: {agent_id}") + print(f"[{job_info.title}] Email: {email}") + + # Fill out application form fields using natural language actions + # Stagehand's act() method understands natural language instructions + client.sessions.act( + id=session_id, + input=f"type '{agent_id}' into the agent identifier field", + ) + + client.sessions.act( + id=session_id, + input=f"type '{email}' into the contact endpoint field", + ) + + client.sessions.act( + id=session_id, + input="type 'us-west-2' into the deployment region field", + ) + + # Upload agent profile/resume file + # Using observe() to find the upload button, then setting files programmatically + observe_response = client.sessions.observe( + id=session_id, + instruction="find the file upload button for agent profile", + ) + upload_actions = observe_response.data.results or [] + + if upload_actions and len(upload_actions) > 0: + upload_action = upload_actions[0] + upload_selector = ( + str(upload_action.selector) if hasattr(upload_action, "selector") else None + ) + if upload_selector: + file_input = page.locator(upload_selector) + + # Fetch resume PDF from remote URL + # Using httpx to download the file before uploading + resume_url = "https://agent-job-board.vercel.app/Agent%20Resume.pdf" + with httpx.Client() as http_client: + response = http_client.get(resume_url) + if response.status_code != 200: + raise Exception(f"Failed to fetch resume: {response.status_code}") + resume_buffer = response.content + + # Upload file using Playwright's set_input_files with buffer + file_input.set_input_files( + { + "name": "Agent Resume.pdf", + "mimeType": "application/pdf", + "buffer": resume_buffer, + } + ) + print(f"[{job_info.title}] Uploaded resume from {resume_url}") - # Fetch resume PDF from remote URL - # Using httpx to download the file before uploading - resume_url = "https://agent-job-board.vercel.app/Agent%20Resume.pdf" - async with httpx.AsyncClient() as client: - response = await client.get(resume_url) - if response.status_code != 200: - raise Exception(f"Failed to fetch resume: {response.status_code}") - resume_buffer = response.content + # Select multi-region deployment option + client.sessions.act( + id=session_id, + input="select 'Yes' for multi region deployment", + ) - # Upload file using Playwright's set_input_files with buffer - await file_input.set_input_files( - { - "name": "Agent Resume.pdf", - "mimeType": "application/pdf", - "buffer": resume_buffer, - } - ) - print(f"[{job_info.title}] Uploaded resume from {resume_url}") + # Submit the application form + client.sessions.act( + id=session_id, + input="click deploy agent button", + ) - # Select multi-region deployment option - await page.act("select 'Yes' for multi region deployment") + print(f"[{job_info.title}] Application submitted successfully!") - # Submit the application form - await page.act("click deploy agent button") + browser.close() - print(f"[{job_info.title}] Application submitted successfully!") + client.sessions.end(id=session_id) - except Exception as error: - print(f"[{job_info.title}] Error: {error}") - raise error + except Exception as error: + print(f"[{job_info.title}] Error: {error}") + client.sessions.end(id=session_id) + raise error -async def main(): +def main(): """ Main application entry point. Orchestrates the job application process: 1. Fetches project concurrency limits 2. Scrapes job listings from the job board - 3. Applies to all jobs in parallel with concurrency control + 3. Applies to all jobs sequentially """ print("Starting Job Application Automation...") - # Get project concurrency limit to control parallel execution - max_concurrency = await get_project_concurrency() - print(f"Executing with concurrency limit: {max_concurrency}") + # Get project concurrency limit + max_concurrency = get_project_concurrency() + print(f"Project concurrency limit: {max_concurrency}") - # Initialize Stagehand with Browserbase for cloud-based browser automation - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="google/gemini-2.5-flash", + # Initialize Stagehand with Browserbase for cloud-based browser automation (main session for job scraping) + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), model_api_key=os.environ.get("GOOGLE_GENERATIVE_AI_API_KEY"), ) - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - print("Main Stagehand Session Started") - - # Get session ID for live viewing/debugging - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id - - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") - - page = stagehand.page - - # Navigate to agent job board homepage - await page.goto("https://agent-job-board.vercel.app/") - print("Navigated to agent-job-board.vercel.app") - - # Click on "View Jobs" button to access job listings - await page.act("click on the view jobs button") - print("Clicked on view jobs button") - - # Extract all job listings with titles and URLs using structured schema - # Using extract() with Pydantic schema ensures consistent data extraction - jobs_result = await page.extract( - "extract all job listings with their titles and URLs", schema=JobsData - ) - - jobs_data = jobs_result.jobs - print(f"Found {len(jobs_data)} jobs") - - # Create semaphore with concurrency limit to control parallel job applications - # Semaphore ensures we don't exceed Browserbase project limits - semaphore = asyncio.Semaphore(max_concurrency) - - # Apply to all jobs in parallel with concurrency control - # Using asyncio.gather() to run all applications concurrently - print(f"Starting to apply to {len(jobs_data)} jobs with max concurrency of {max_concurrency}") + # Start a new session + start_response = client.sessions.start( + model_name="google/gemini-2.5-flash", + ) + session_id = start_response.data.session_id - application_tasks = [apply_to_job(job, semaphore) for job in jobs_data] + print("Main Stagehand Session Started") + print(f"Watch live: https://browserbase.com/sessions/{session_id}") - # Wait for all applications to complete - await asyncio.gather(*application_tasks) + try: + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() + + # Navigate to agent job board homepage + page.goto("https://agent-job-board.vercel.app/") + print("Navigated to agent-job-board.vercel.app") + + # Click on "View Jobs" button to access job listings + client.sessions.act( + id=session_id, + input="click on the view jobs button", + ) + print("Clicked on view jobs button") + + # Extract all job listings with titles and URLs using inline schema (avoids $ref issues) + jobs_schema = { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "url": {"type": "string", "description": "Job URL"}, + "title": {"type": "string", "description": "Job title"}, + }, + "required": ["url", "title"], + }, + } + }, + "required": ["jobs"], + } + extract_response = client.sessions.extract( + id=session_id, + instruction="extract all job listings with their titles and URLs", + schema=jobs_schema, + ) + jobs_result = extract_response.data.result + + jobs_data = [ + JobInfo(url=job["url"], title=job["title"]) for job in jobs_result.get("jobs", []) + ] + print(f"Found {len(jobs_data)} jobs") + + browser.close() + + client.sessions.end(id=session_id) + + except Exception as error: + print(f"Error during job scraping: {error}") + client.sessions.end(id=session_id) + raise error + + # Apply to all jobs sequentially + print(f"Starting to apply to {len(jobs_data)} jobs...") + + for job in jobs_data: + try: + apply_to_job(job) + except Exception as error: + print(f"Failed to apply to {job.title}: {error}") + continue print("All applications completed!") if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Error: {err}") exit(1) diff --git a/python/license-verification/README.md b/python/license-verification/README.md index 98cf9c24..ca374930 100644 --- a/python/license-verification/README.md +++ b/python/license-verification/README.md @@ -20,13 +20,11 @@ ## QUICKSTART -1. cd license-verification-template -2. uv venv venv -3. source venv/bin/activate # On Windows: venv\Scripts\activate -4. pip install -r requirements.txt -5. pip install pydantic -6. cp .env.example .env # Add your Browserbase API key and Project ID to .env -7. python main.py +1. cd license-verification +2. uv venv && source .venv/bin/activate # On Windows: .venv\Scripts\activate +3. pip install stagehand python-dotenv pydantic +4. cp .env.example .env # Add your Browserbase API key, Project ID, and OpenAI API key to .env +5. python main.py ## EXPECTED OUTPUT diff --git a/python/license-verification/main.py b/python/license-verification/main.py index 914f4cad..59524a99 100644 --- a/python/license-verification/main.py +++ b/python/license-verification/main.py @@ -1,12 +1,11 @@ # Stagehand + Browserbase: Data Extraction with Structured Schemas - See README.md for full documentation -import asyncio import os from dotenv import load_dotenv from pydantic import BaseModel, Field -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables load_dotenv() @@ -17,83 +16,83 @@ } -async def main(): - # Initialize Stagehand with Browserbase for cloud-based browser automation. - config = StagehandConfig( - env="BROWSERBASE", # Use Browserbase cloud browsers for reliable automation. - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - verbose=1, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - model_name="openai/gpt-4.1", +# Define schema using Pydantic +class LicenseData(BaseModel): + license_type: str | None = Field(None, description="Type of real estate license") + name: str | None = Field(None, description="License holder's full name") + mailing_address: str | None = Field(None, description="Current mailing address") + license_id: str | None = Field(None, description="Unique license identifier") + expiration_date: str | None = Field(None, description="License expiration date") + license_status: str | None = Field(None, description="Current status (active, expired, etc.)") + salesperson_license_issued: str | None = Field( + None, description="Date salesperson license was issued" + ) + former_names: str | None = Field(None, description="Any previous names used") + responsible_broker: str | None = Field(None, description="Associated broker name") + broker_license_id: str | None = Field(None, description="Broker's license ID") + broker_address: str | None = Field(None, description="Broker's business address") + disciplinary_action: str | None = Field(None, description="Any disciplinary actions taken") + other_comments: str | None = Field(None, description="Additional relevant information") + + +def main(): + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), model_api_key=os.environ.get("OPENAI_API_KEY"), ) - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - print("Stagehand Session Started") - - # Provide live session URL for debugging and monitoring extraction process. - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id - - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") + # Start a new session + start_response = client.sessions.start( + model_name="openai/gpt-4.1", + ) + session_id = start_response.data.session_id - page = stagehand.page + try: + print("Stagehand Session Started") + print(f"Watch live: https://browserbase.com/sessions/{session_id}") # Navigate to California DRE license verification website for data extraction. print("Navigating to: https://www2.dre.ca.gov/publicasp/pplinfo.asp") - await page.goto("https://www2.dre.ca.gov/publicasp/pplinfo.asp") + client.sessions.navigate(id=session_id, url="https://www2.dre.ca.gov/publicasp/pplinfo.asp") # Fill in license ID to search for specific real estate professional. print(f"Performing action: type {variables['input1']} into the License ID input field") - await page.act(f"type {variables['input1']} into the License ID input field") + client.sessions.act( + id=session_id, + input=f"type {variables['input1']} into the License ID input field", + ) # Submit search form to retrieve license verification data. print("Performing action: click the Find button") - await page.act("click the Find button") - - # Define schema using Pydantic - class LicenseData(BaseModel): - license_type: str | None = Field(None, description="Type of real estate license") - name: str | None = Field(None, description="License holder's full name") - mailing_address: str | None = Field(None, description="Current mailing address") - license_id: str | None = Field(None, description="Unique license identifier") - expiration_date: str | None = Field(None, description="License expiration date") - license_status: str | None = Field( - None, description="Current status (active, expired, etc.)" - ) - salesperson_license_issued: str | None = Field( - None, description="Date salesperson license was issued" - ) - former_names: str | None = Field(None, description="Any previous names used") - responsible_broker: str | None = Field(None, description="Associated broker name") - broker_license_id: str | None = Field(None, description="Broker's license ID") - broker_address: str | None = Field(None, description="Broker's business address") - disciplinary_action: str | None = Field( - None, description="Any disciplinary actions taken" - ) - other_comments: str | None = Field(None, description="Additional relevant information") + client.sessions.act( + id=session_id, + input="click the Find button", + ) # Extract structured license data using Pydantic schema for type safety and validation. print("Extracting: extract all the license verification details for DRE#02237476") - extracted_data = await page.extract( - "extract all the license verification details for DRE#02237476", schema=LicenseData + extract_response = client.sessions.extract( + id=session_id, + instruction="extract all the license verification details for DRE#02237476", + schema=LicenseData.model_json_schema(), ) + extracted_data = extract_response.data.result print(f"Extracted: {extracted_data}") - print("Session closed successfully") + except Exception as error: + print(f"Error: {error}") + raise + + finally: + client.sessions.end(id=session_id) + print("Session closed successfully") if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Error: {err}") exit(1) diff --git a/python/manual-mfa-with-contexts/main.py b/python/manual-mfa-with-contexts/main.py index c019c182..f44cad0c 100644 --- a/python/manual-mfa-with-contexts/main.py +++ b/python/manual-mfa-with-contexts/main.py @@ -1,15 +1,15 @@ # Manual MFA with Browserbase Contexts - See README.md for full documentation -import asyncio import os import time import requests from browserbase import Browserbase from dotenv import load_dotenv +from playwright.sync_api import sync_playwright from pydantic import BaseModel, Field -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables load_dotenv() @@ -17,7 +17,7 @@ bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) -async def create_session_with_context(): +def create_session_with_context(): """First session: Create context and login (with MFA)""" print("Creating new Browserbase context...") @@ -26,87 +26,90 @@ async def create_session_with_context(): print(f"Context created: {context.id}") print("First session: Performing login with MFA...") - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), + # Create session with context + session = bb.sessions.create( project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="google/gemini-2.5-flash", - model_api_key=os.environ.get("GOOGLE_GENERATIVE_AI_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - "browser_settings": { - "context": { - "id": context.id, - "persist": True, # Save authentication state including MFA - } - }, + browser_settings={ + "context": { + "id": context.id, + "persist": True, + } }, - verbose=0, # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging ) + session_id = session.id - async with Stagehand(config) as stagehand: - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("GOOGLE_GENERATIVE_AI_API_KEY"), + ) - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") + print(f"Watch live: https://browserbase.com/sessions/{session_id}") - page = stagehand.page + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + ctx = browser.contexts[0] + page = ctx.pages[0] if ctx.pages else ctx.new_page() # Navigate to GitHub login print("Navigating to GitHub login...") - await page.goto("https://github.com/login", wait_until="domcontentloaded") + page.goto("https://github.com/login", wait_until="domcontentloaded") # Fill in credentials print("Entering username...") - await page.act(f"Type '{os.environ.get('GITHUB_USERNAME')}' into the username field") + client.sessions.act( + id=session_id, + input=f"Type '{os.environ.get('GITHUB_USERNAME')}' into the username field", + ) print("Entering password...") - await page.act(f"Type '{os.environ.get('GITHUB_PASSWORD')}' into the password field") + client.sessions.act( + id=session_id, + input=f"Type '{os.environ.get('GITHUB_PASSWORD')}' into the password field", + ) print("Clicking Sign in...") - await page.act("Click the Sign in button") + client.sessions.act( + id=session_id, + input="Click the Sign in button", + ) - await page.wait_for_load_state("networkidle") + page.wait_for_load_state("networkidle") # Check if MFA is required class MFARequired(BaseModel): mfa_required: bool = Field(..., description="Whether MFA is required") - mfa_check = await page.extract( + mfa_response = client.sessions.extract( + id=session_id, instruction="Is there a two-factor authentication or verification code prompt on the page?", - schema=MFARequired, + schema=MFARequired.model_json_schema(), ) + mfa_check = mfa_response.data.result - if mfa_check.mfa_required: + if mfa_check.get("mfa_required"): print("MFA DETECTED!") print("═══════════════════════════════════════════════════════════") print("PAUSED: Please complete MFA in the browser") print("═══════════════════════════════════════════════════════════") - if session_id: - print( - f"1. Open the Browserbase session in your browser: https://browserbase.com/sessions/{session_id}" - ) - else: - print("1. Open the Browserbase session in your browser") + print( + f"1. Open the Browserbase session in your browser: https://browserbase.com/sessions/{session_id}" + ) print("2. Enter your 2FA code from authenticator app") print("3. Click 'Verify' or submit") print("4. Wait for login to complete") print("\nThe script will wait for you to complete MFA...\n") - # Wait for MFA completion (poll until we're no longer on login page) + # Wait for MFA completion login_complete = False start_time = time.time() timeout = 120 # 2 minutes while not login_complete and (time.time() - start_time) < timeout: - await asyncio.sleep(3) # Check every 3 seconds - + time.sleep(3) current_url = page.url if "/login" not in current_url and "/sessions/two-factor" not in current_url: login_complete = True @@ -123,73 +126,77 @@ class MFARequired(BaseModel): print(" - MFA trust/remember device state") print(" - All authentication data\n") + browser.close() + + client.sessions.end(id=session_id) return context.id -async def reuse_context(context_id: str): +def reuse_context(context_id: str): """Second session: Reuse context - NO MFA needed!""" print(f"Second session: Reusing context {context_id}") print(" (No login, no MFA required - auth state persisted)\n") - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), + # Create session with existing context + session = bb.sessions.create( project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="google/gemini-2.5-flash", - model_api_key=os.environ.get("GOOGLE_GENERATIVE_AI_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - "browser_settings": { - "context": { - "id": context_id, - "persist": True, - } - }, + browser_settings={ + "context": { + "id": context_id, + "persist": True, + } }, - verbose=0, # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging ) + session_id = session.id - async with Stagehand(config) as stagehand: - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("GOOGLE_GENERATIVE_AI_API_KEY"), + ) - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") + print(f"Watch live: https://browserbase.com/sessions/{session_id}") - page = stagehand.page + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + ctx = browser.contexts[0] + page = ctx.pages[0] if ctx.pages else ctx.new_page() # Navigate directly to GitHub (should already be logged in) print("Navigating to GitHub...") - await page.goto("https://github.com", wait_until="domcontentloaded") - await page.wait_for_load_state("networkidle") + page.goto("https://github.com", wait_until="domcontentloaded") + page.wait_for_load_state("networkidle") # Check if we're logged in class Username(BaseModel): username: str = Field(..., description="The logged-in username") - username_result = await page.extract( + username_response = client.sessions.extract( + id=session_id, instruction="Extract the logged-in username or check if we're authenticated", - schema=Username, + schema=Username.model_json_schema(), ) + username_result = username_response.data.result print("\nSUCCESS! Already logged in without MFA!") - print(f" Username: {username_result.username}") + print(f" Username: {username_result.get('username')}") print("\nThis is the power of Browserbase Contexts:") print(" - First session: User completes MFA once") print(" - Context saves trusted device state") print(" - All future sessions: No MFA required\n") + browser.close() + + client.sessions.end(id=session_id) -async def delete_context(context_id: str): + +def delete_context(context_id: str): """Clean up context""" print(f"Deleting context: {context_id}") try: - # Delete via API (SDK doesn't have delete method) response = requests.delete( f"https://api.browserbase.com/v1/contexts/{context_id}", headers={ @@ -207,10 +214,9 @@ async def delete_context(context_id: str): print(" Context will auto-expire after 30 days\n") -async def main(): +def main(): print("Starting Browserbase Context MFA Persistence Demo...") - # Check environment variables if not os.environ.get("BROWSERBASE_API_KEY") or not os.environ.get("BROWSERBASE_PROJECT_ID"): print("\nError: Missing Browserbase credentials") print(" Set BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID in .env") @@ -231,17 +237,13 @@ async def main(): print(" 2. Second session: No login, no MFA needed") print(" 3. Clean up context\n") - # First session: Create context and login with MFA - context_id = await create_session_with_context() + context_id = create_session_with_context() print("Waiting 5 seconds before reusing context...\n") - await asyncio.sleep(5) - - # Second session: Reuse context (NO MFA!) - await reuse_context(context_id) + time.sleep(5) - # Clean up - await delete_context(context_id) + reuse_context(context_id) + delete_context(context_id) print("═══════════════════════════════════════════════════════════") print("Key Takeaway:") @@ -261,11 +263,11 @@ async def main(): if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Application error: {err}") print("Common issues:") print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") print(" - Verify GOOGLE_GENERATIVE_AI_API_KEY is set in environment") - print("Docs: https://docs.stagehand.dev/v2/first-steps/introduction") + print("Docs: https://docs.stagehand.dev/v3/first-steps/introduction") exit(1) diff --git a/python/mfa-handling/main.py b/python/mfa-handling/main.py index 419feb78..1af669a3 100644 --- a/python/mfa-handling/main.py +++ b/python/mfa-handling/main.py @@ -1,12 +1,13 @@ # Stagehand + Browserbase: MFA Handling - TOTP Automation - See README.md for full documentation -import asyncio import hashlib import hmac import os import time +import traceback from dotenv import load_dotenv +from playwright.sync_api import sync_playwright from pydantic import BaseModel, Field from stagehand import Stagehand @@ -79,133 +80,156 @@ class RetryResult(BaseModel): success: bool = Field(..., description="Whether the retry login was successful") -async def main(): +def main(): print("Starting MFA Handling - TOTP Automation...") # Initialize Stagehand with Browserbase for cloud-based browser automation - # Note: set verbose: 0 to prevent API keys from appearing in logs when handling sensitive data. - stagehand = Stagehand( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model="openai/gpt-4.1", + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), model_api_key=os.environ.get("OPENAI_API_KEY"), - verbose=0, # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging ) + # Start a new session + start_response = client.sessions.start(model_name="openai/gpt-4.1") + session_id = start_response.data.session_id + try: - # Initialize browser session - await stagehand.init() print("Stagehand initialized successfully!") + print(f"Live View Link: https://browserbase.com/sessions/{session_id}") - # Get session ID if available - session_id = getattr(stagehand, "session_id", None) or getattr( - stagehand, "browserbase_session_id", None - ) - if session_id: - print(f"Live View Link: https://browserbase.com/sessions/{session_id}") - - # Get the page object - page = stagehand.page - - # Navigate to TOTP challenge demo page - print("Navigating to TOTP Challenge page...") - await page.goto(DEMO_URL, wait_until="domcontentloaded") - - # Extract test credentials and TOTP secret from the page - print("Extracting test credentials and TOTP secret...") - credentials = await page.extract( - instruction="Extract the test email, password, and TOTP secret key shown on the page", - schema=Credentials, - ) - - print(f"Credentials extracted - Email: {credentials.email}") - - # Generate TOTP code using RFC 6238 algorithm - totp_code = generate_totp(credentials.totp_secret) - seconds_left = 30 - (int(time.time()) % 30) - print(f"Generated TOTP code: {totp_code} (valid for {seconds_left} seconds)") - - # Fill in login form with email and password - print("Filling in email...") - await page.act(f"Type '{credentials.email}' into the email field") - - print("Filling in password...") - await page.act(f"Type '{credentials.password}' into the password field") - - # Fill in TOTP code - print("Filling in TOTP code...") - await page.act(f"Type '{totp_code}' into the TOTP code field") - - # Submit the form - print("Submitting form...") - await page.act("Click the submit or login button") - - # Wait for response - be tolerant of sites that never reach full "networkidle" - try: - print("Waiting for page to finish loading after submit...") - await page.wait_for_load_state("networkidle", timeout=15000) - except Exception: - print( - "Timed out waiting for 'networkidle' after submit; continuing because the login likely succeeded." + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() + + # Navigate to TOTP challenge demo page + print("Navigating to TOTP Challenge page...") + page.goto(DEMO_URL, wait_until="domcontentloaded") + + # Extract test credentials and TOTP secret from the page + print("Extracting test credentials and TOTP secret...") + extract_response = client.sessions.extract( + id=session_id, + instruction="Extract the test email, password, and TOTP secret key shown on the page", + schema=Credentials.model_json_schema(), ) + credentials_data = extract_response.data.result - # Check if login succeeded - print("Checking authentication result...") - result = await page.extract( - instruction="Check if the login was successful or if there's an error message", - schema=AuthResult, - ) + print(f"Credentials extracted - Email: {credentials_data.get('email')}") - if result.success: - print("SUCCESS! TOTP authentication completed automatically!") - print(f"Authentication Result: {result.message}") - else: - print(f"Authentication may have failed. Message: {result.message}") - print("Retrying with a fresh TOTP code...") + # Generate TOTP code using RFC 6238 algorithm + totp_code = generate_totp(credentials_data.get("totp_secret", "")) + seconds_left = 30 - (int(time.time()) % 30) + print(f"Generated TOTP code: {totp_code} (valid for {seconds_left} seconds)") - # Regenerate and retry with new code (handles time window edge cases) - new_code = generate_totp(credentials.totp_secret) - print(f"New TOTP code: {new_code}") + # Fill in login form with email and password + print("Filling in email...") + client.sessions.act( + id=session_id, + input=f"Type '{credentials_data.get('email')}' into the email field", + ) - await page.act("Clear the TOTP code field") - await page.act(f"Type '{new_code}' into the TOTP code field") - await page.act("Click the submit or login button") + print("Filling in password...") + client.sessions.act( + id=session_id, + input=f"Type '{credentials_data.get('password')}' into the password field", + ) + # Fill in TOTP code + print("Filling in TOTP code...") + client.sessions.act( + id=session_id, + input=f"Type '{totp_code}' into the TOTP code field", + ) + + # Submit the form + print("Submitting form...") + client.sessions.act( + id=session_id, + input="Click the submit or login button", + ) + + # Wait for response - be tolerant of sites that never reach full "networkidle" try: - print("Waiting for page to finish loading after retry submit...") - await page.wait_for_load_state("networkidle", timeout=15000) + print("Waiting for page to finish loading after submit...") + page.wait_for_load_state("networkidle", timeout=15000) except Exception: print( - "Timed out waiting for 'networkidle' after retry submit; continuing because the login likely succeeded." + "Timed out waiting for 'networkidle' after submit; continuing because the login likely succeeded." ) - retry_result = await page.extract( - instruction="Check if the login was successful", - schema=RetryResult, + # Check if login succeeded + print("Checking authentication result...") + result_response = client.sessions.extract( + id=session_id, + instruction="Check if the login was successful or if there's an error message", + schema=AuthResult.model_json_schema(), ) + result = result_response.data.result - if retry_result.success: - print("Success on retry!") + if result.get("success"): + print("SUCCESS! TOTP authentication completed automatically!") + print(f"Authentication Result: {result.get('message')}") else: - print("Authentication failed after retry") + print(f"Authentication may have failed. Message: {result.get('message')}") + print("Retrying with a fresh TOTP code...") + + # Regenerate and retry with new code (handles time window edge cases) + new_code = generate_totp(credentials_data.get("totp_secret", "")) + print(f"New TOTP code: {new_code}") - await stagehand.close() + client.sessions.act( + id=session_id, + input="Clear the TOTP code field", + ) + client.sessions.act( + id=session_id, + input=f"Type '{new_code}' into the TOTP code field", + ) + client.sessions.act( + id=session_id, + input="Click the submit or login button", + ) + + try: + print("Waiting for page to finish loading after retry submit...") + page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + print( + "Timed out waiting for 'networkidle' after retry submit; continuing because the login likely succeeded." + ) + + retry_response = client.sessions.extract( + id=session_id, + instruction="Check if the login was successful", + schema=RetryResult.model_json_schema(), + ) + retry_result = retry_response.data.result + + if retry_result.get("success"): + print("Success on retry!") + else: + print("Authentication failed after retry") + + browser.close() + + client.sessions.end(id=session_id) print("Session closed successfully") except Exception as error: print(f"Error during MFA handling: {error}") - import traceback - traceback.print_exc() + client.sessions.end(id=session_id) raise if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Error in MFA handling: {err}") print("Common issues:") @@ -213,5 +237,5 @@ async def main(): print(" - Check .env file has OPENAI_API_KEY (or set model_api_key for your chosen model)") print(" - TOTP code may have expired (try running again)") print(" - Page structure may have changed") - print("Docs: https://docs.stagehand.dev/v2/first-steps/introduction") + print("Docs: https://docs.stagehand.dev/v3/first-steps/introduction") exit(1) diff --git a/python/nurse-verification/main.py b/python/nurse-verification/main.py index 2bce22f0..cd3e7c4d 100644 --- a/python/nurse-verification/main.py +++ b/python/nurse-verification/main.py @@ -1,12 +1,13 @@ # Stagehand + Browserbase: Automated Nurse License Verification - See README.md for full documentation -import asyncio +import json import os from dotenv import load_dotenv +from playwright.sync_api import sync_playwright from pydantic import BaseModel, Field -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables load_dotenv() @@ -40,7 +41,7 @@ class LicenseResults(BaseModel): ] -async def main(): +def main(): """ Automated nurse license verification using AI-powered browser automation. Processes multiple license records and extracts verification results. @@ -48,35 +49,30 @@ async def main(): print("Starting Nurse License Verification Automation...") # Initialize Stagehand with Browserbase for cloud-based browser automation - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), model_api_key=os.environ.get("OPENAI_API_KEY"), - verbose=1, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging ) - try: - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - print("Initializing browser session...") - print("Stagehand session started successfully") - - # Provide live session URL for debugging and monitoring - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id - - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") + # Start a new session + start_response = client.sessions.start( + model_name="openai/gpt-4.1", + ) + session_id = start_response.data.session_id - page = stagehand.page + try: + print("Initializing browser session...") + print("Stagehand session started successfully") + print(f"Watch live: https://browserbase.com/sessions/{session_id}") + + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() # Process each license record sequentially for license_record in LICENSE_RECORDS: @@ -86,41 +82,83 @@ async def main(): # Navigate to license verification site print(f"Navigating to: {license_record['Site']}") - await page.goto(license_record["Site"]) - await page.wait_for_load_state("domcontentloaded") + page.goto(license_record["Site"]) + page.wait_for_load_state("domcontentloaded") # Brief timeout to ensure form fields are interactive - await page.wait_for_timeout(1000) + page.wait_for_timeout(1000) # Fill in form fields with license information print("Filling in license information...") - await page.act(f'Type "{license_record["FirstName"]}" into the first name field') - await page.act(f'Type "{license_record["LastName"]}" into the last name field') - await page.act( - f'Type "{license_record["LicenseNumber"]}" into the license number field' + client.sessions.act( + id=session_id, + input=f'Type "{license_record["FirstName"]}" into the first name field', + ) + client.sessions.act( + id=session_id, + input=f'Type "{license_record["LastName"]}" into the last name field', + ) + client.sessions.act( + id=session_id, + input=f'Type "{license_record["LicenseNumber"]}" into the license number field', ) # Submit search print("Clicking search button...") - await page.act("Click the search button") + client.sessions.act( + id=session_id, + input="Click the search button", + ) # Wait for search results to load - await page.wait_for_load_state("domcontentloaded") - await page.wait_for_timeout(1000) + page.wait_for_load_state("domcontentloaded") + page.wait_for_timeout(1000) - # Extract license verification results + # Extract license verification results using inline schema (avoids $ref issues) print("Extracting license verification results...") - results = await page.extract( - "Extract ALL the license verification results from the page, including name, license number and status", - schema=LicenseResults, + license_schema = { + "type": "object", + "properties": { + "list_of_licenses": { + "type": "array", + "description": "array of license verification results", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "the name of the license holder", + }, + "license_number": { + "type": "string", + "description": "the license number", + }, + "status": { + "type": "string", + "description": "the status of the license", + }, + "more_info_url": { + "type": "string", + "description": "URL for more information", + }, + }, + "required": ["name", "license_number", "status", "more_info_url"], + }, + } + }, + "required": ["list_of_licenses"], + } + extract_response = client.sessions.extract( + id=session_id, + instruction="Extract ALL the license verification results from the page, including name, license number and status", + schema=license_schema, ) print("License verification results extracted:") + print(json.dumps(extract_response.data.result, indent=2)) - # Display results in formatted JSON - import json - - print(json.dumps(results.model_dump(), indent=2)) + browser.close() + client.sessions.end(id=session_id) print("Session closed successfully") except Exception as error: @@ -133,12 +171,13 @@ async def main(): print("3. Ensure internet access and license verification site is accessible") print("4. Verify Browserbase account has sufficient credits") + client.sessions.end(id=session_id) raise if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Application error: {err}") exit(1) diff --git a/python/pickleball/main.py b/python/pickleball/main.py index 78ceea54..679317f7 100644 --- a/python/pickleball/main.py +++ b/python/pickleball/main.py @@ -1,41 +1,63 @@ # SF Court Booking Automation - See README.md for full documentation -import asyncio import os from datetime import datetime, timedelta from dotenv import load_dotenv from InquirerPy import inquirer +from playwright.sync_api import sync_playwright from pydantic import BaseModel, Field -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables load_dotenv() -async def login_to_site(page, email: str, password: str) -> None: +def login_to_site(client, session_id: str, email: str, password: str) -> None: print("Logging in...") - # Perform login sequence: each step is atomic to handle dynamic page changes. - await page.act("Click the Login button") - await page.act(f'Fill in the email or username field with "{email}"') - await page.act("Click the next, continue, or submit button to proceed") - await page.act(f'Fill in the password field with "{password}"') - await page.act("Click the login, sign in, or submit button") + client.sessions.act( + id=session_id, + input="Click the Login button", + ) + client.sessions.act( + id=session_id, + input=f'Fill in the email or username field with "{email}"', + ) + client.sessions.act( + id=session_id, + input="Click the next, continue, or submit button to proceed", + ) + client.sessions.act( + id=session_id, + input=f'Fill in the password field with "{password}"', + ) + client.sessions.act( + id=session_id, + input="Click the login, sign in, or submit button", + ) print("Logged in") -async def select_filters(page, activity: str, time_of_day: str, selected_date: str) -> None: +def select_filters( + client, session_id: str, activity: str, time_of_day: str, selected_date: str +) -> None: print("Selecting the activity") - # Filter by activity type first to narrow down available courts. - await page.act("Click the activites drop down menu") - await page.act(f"Select the {activity} activity") - await page.act("Click the Done button") + client.sessions.act( + id=session_id, + input="Click the activites drop down menu", + ) + client.sessions.act( + id=session_id, + input=f"Select the {activity} activity", + ) + client.sessions.act(id=session_id, input="Click the Done button") print(f"Selecting date: {selected_date}") - # Open calendar to select specific date for court booking. - await page.act("Click the date picker or calendar") + client.sessions.act( + id=session_id, + input="Click the date picker or calendar", + ) - # Parse date string to extract day number for calendar selection. date_parts = selected_date.split("-") if len(date_parts) != 3: raise ValueError(f"Invalid date format: {selected_date}. Expected YYYY-MM-DD") @@ -45,66 +67,103 @@ async def select_filters(page, activity: str, time_of_day: str, selected_date: s raise ValueError(f"Invalid day number: {day_number} from date: {selected_date}") print(f"Looking for day number: {day_number} in calendar") - # Click specific day number in calendar to select date. - await page.act(f"Click on the number {day_number} in the calendar") + client.sessions.act( + id=session_id, + input=f"Click on the number {day_number} in the calendar", + ) print(f"Selecting time of day: {time_of_day}") - # Filter by time period to find courts available during preferred hours. - await page.act("Click the time filter or time selection dropdown") - await page.act(f"Select {time_of_day} time period") - await page.act("Click the Done button") + client.sessions.act( + id=session_id, + input="Click the time filter or time selection dropdown", + ) + client.sessions.act( + id=session_id, + input=f"Select {time_of_day} time period", + ) + client.sessions.act(id=session_id, input="Click the Done button") - # Apply additional filters to show only available courts that accept reservations. - await page.act("Click Available Only button") - await page.act("Click All Facilities dropdown list") - await page.act("Select Accept Reservations checkbox") - await page.act("Click the Done button") + client.sessions.act( + id=session_id, + input="Click Available Only button", + ) + client.sessions.act( + id=session_id, + input="Click All Facilities dropdown list", + ) + client.sessions.act( + id=session_id, + input="Select Accept Reservations checkbox", + ) + client.sessions.act(id=session_id, input="Click the Done button") -async def check_and_extract_courts(page, time_of_day: str) -> None: +def check_and_extract_courts(client, session_id: str, time_of_day: str) -> None: print("Checking for available courts...") - # First observe the page to find all available court booking options. - available_courts = await page.observe( - "Find all available court booking slots, time slots, or court reservation options" + # Inline schema to avoid $ref issues with nested models + court_data_schema = { + "type": "object", + "properties": { + "courts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "the name or identifier of the court", + }, + "opening_times": { + "type": "string", + "description": "the opening hours or operating times of the court", + }, + "location": { + "type": "string", + "description": "the location or facility name", + }, + "availability": { + "type": "string", + "description": "availability status or any restrictions", + }, + "duration": { + "type": "string", + "description": "the duration of the court session in minutes", + }, + }, + "required": ["name", "opening_times", "location", "availability"], + }, + } + }, + "required": ["courts"], + } + + observe_response = client.sessions.observe( + id=session_id, + instruction="Find all available court booking slots, time slots, or court reservation options", ) + available_courts = observe_response.data.results or [] print(f"Found {len(available_courts)} available court options") - # Define schema using Pydantic - class Court(BaseModel): - name: str = Field(..., description="the name or identifier of the court") - opening_times: str = Field( - ..., description="the opening hours or operating times of the court" - ) - location: str = Field(..., description="the location or facility name") - availability: str = Field(..., description="availability status or any restrictions") - duration: str | None = Field( - None, description="the duration of the court session in minutes" - ) - - class CourtData(BaseModel): - courts: list[Court] - - # Extract structured court data using Pydantic schema for type safety and validation. - court_data = await page.extract( - "Extract all available court booking information including court names, time slots, locations, and any other relevant details", - schema=CourtData, + extract_response = client.sessions.extract( + id=session_id, + instruction="Extract all available court booking information including court names, time slots, locations, and any other relevant details", + schema=court_data_schema, ) + court_data = extract_response.data.result - # Check if any courts are actually available by filtering out unavailable status messages. + courts = court_data.get("courts", []) has_available_courts = any( - "no free spots" not in court.availability.lower() - and "unavailable" not in court.availability.lower() - and "next available" not in court.availability.lower() - and "the next available reservation" not in court.availability.lower() - for court in court_data.courts + "no free spots" not in court.get("availability", "").lower() + and "unavailable" not in court.get("availability", "").lower() + and "next available" not in court.get("availability", "").lower() + and "the next available reservation" not in court.get("availability", "").lower() + for court in courts ) - # If no courts available for selected time, try alternative time periods as fallback. if len(available_courts) == 0 or not has_available_courts: print("No courts available for selected time. Trying different time periods...") - # Generate alternative time periods to try if original selection has no availability. alternative_times = ( ["Afternoon", "Evening"] if time_of_day == "Morning" @@ -116,198 +175,203 @@ class CourtData(BaseModel): for alt_time in alternative_times: print(f"Trying {alt_time} time period...") - # Change time filter to alternative time period and check availability. - await page.act(f'Click the time filter dropdown that currently shows "{time_of_day}"') - await page.act(f"Select {alt_time} from the time period options") - await page.act("Click the Done button") + client.sessions.act( + id=session_id, + input=f'Click the time filter dropdown that currently shows "{time_of_day}"', + ) + client.sessions.act( + id=session_id, + input=f"Select {alt_time} from the time period options", + ) + client.sessions.act( + id=session_id, + input="Click the Done button", + ) - alt_available_courts = await page.observe( - "Find all available court booking slots, time slots, or court reservation options" + alt_observe_response = client.sessions.observe( + id=session_id, + instruction="Find all available court booking slots, time slots, or court reservation options", ) + alt_available_courts = alt_observe_response.data.results or [] print(f"Found {len(alt_available_courts)} available court options for {alt_time}") if len(alt_available_courts) > 0: - alt_court_data = await page.extract( - "Extract all available court booking information including court names, time slots, locations, and any other relevant details", - schema=CourtData, + alt_extract_response = client.sessions.extract( + id=session_id, + instruction="Extract all available court booking information including court names, time slots, locations, and any other relevant details", + schema=court_data_schema, ) + alt_court_data = alt_extract_response.data.result + alt_courts = alt_court_data.get("courts", []) has_alt_available_courts = any( - "no free spots" not in court.availability.lower() - and "unavailable" not in court.availability.lower() - and "next available" not in court.availability.lower() - and "the next available reservation" not in court.availability.lower() - for court in alt_court_data.courts + "no free spots" not in court.get("availability", "").lower() + and "unavailable" not in court.get("availability", "").lower() + and "next available" not in court.get("availability", "").lower() + and "the next available reservation" + not in court.get("availability", "").lower() + for court in alt_courts ) - # If alternative time has available courts, use that data and stop searching. if has_alt_available_courts: print(f"Found actually available courts for {alt_time}!") - court_data.courts = alt_court_data.courts + courts = alt_courts has_available_courts = True break - # If still no available courts found, extract final court data for display. if not has_available_courts: print("Extracting final court information...") - final_court_data = await page.extract( - "Extract all available court booking information including court names, time slots, locations, and any other relevant details", - schema=CourtData, + final_extract_response = client.sessions.extract( + id=session_id, + instruction="Extract all available court booking information including court names, time slots, locations, and any other relevant details", + schema=court_data_schema, ) - court_data.courts = final_court_data.courts + courts = final_extract_response.data.result.get("courts", []) - # Display all found court information to user for review and selection. print("Available Courts:") - if court_data.courts and len(court_data.courts) > 0: - for index, court in enumerate(court_data.courts): - print(f"{index + 1}. {court.name}") - print(f" Opening Times: {court.opening_times}") - print(f" Location: {court.location}") - print(f" Availability: {court.availability}") - if court.duration: - print(f" Duration: {court.duration} minutes") + if courts and len(courts) > 0: + for index, court in enumerate(courts): + print(f"{index + 1}. {court.get('name', 'Unknown')}") + print(f" Opening Times: {court.get('opening_times', 'N/A')}") + print(f" Location: {court.get('location', 'N/A')}") + print(f" Availability: {court.get('availability', 'N/A')}") + if court.get("duration"): + print(f" Duration: {court.get('duration')} minutes") print("") else: print("No court data available to display") -async def book_court(page) -> None: +def book_court(client, session_id: str) -> None: print("Starting court booking process...") + class Confirmation(BaseModel): + confirmation_message: str | None = Field( + None, description="any confirmation or success message" + ) + booking_details: str | None = Field( + None, description="booking details like time, court, etc." + ) + error_message: str | None = Field(None, description="any error message if booking failed") + try: - # Select the first available court time slot for booking. print("Clicking the top available time slot...") - await page.act("Click the first available time slot or court booking option") + client.sessions.act( + id=session_id, + input="Click the first available time slot or court booking option", + ) - # Select participant from dropdown - assumes only one participant is available. print("Opening participant dropdown...") - await page.act("Click the participant dropdown menu or select participant field") - await page.act("Click the only named participant in the dropdown!") + client.sessions.act( + id=session_id, + input="Click the participant dropdown menu or select participant field", + ) + client.sessions.act( + id=session_id, + input="Click the only named participant in the dropdown!", + ) - # Complete booking process and trigger verification code request. print("Clicking the book button to complete reservation...") - await page.act("Click the book, reserve, or confirm booking button") - await page.act("Click the Send Code Button") + client.sessions.act( + id=session_id, + input="Click the book, reserve, or confirm booking button", + ) + client.sessions.act( + id=session_id, + input="Click the Send Code Button", + ) - # Prompt user for verification code received via SMS/email for booking confirmation. def validate_code(text): if not text.strip(): raise ValueError("Please enter a verification code") return True - def get_verification_code(): - return inquirer.text( - message="Please enter the verification code you received:", - validate=validate_code, - ).execute() - - verification_code = await asyncio.to_thread(get_verification_code) + verification_code = inquirer.text( + message="Please enter the verification code you received:", + validate=validate_code, + ).execute() print(f"Verification code: {verification_code}") - # Enter verification code and confirm booking to complete reservation. - await page.act(f'Fill in the verification code field with "{verification_code}"') - await page.act("Click the confirm button") - - # Define schema using Pydantic - class Confirmation(BaseModel): - confirmation_message: str | None = Field( - None, description="any confirmation or success message" - ) - booking_details: str | None = Field( - None, description="booking details like time, court, etc." - ) - error_message: str | None = Field( - None, description="any error message if booking failed" - ) + client.sessions.act( + id=session_id, + input=f'Fill in the verification code field with "{verification_code}"', + ) + client.sessions.act( + id=session_id, + input="Click the confirm button", + ) - # Extract booking confirmation details to verify successful reservation. print("Checking for booking confirmation...") - confirmation = await page.extract( - "Extract any booking confirmation message, success notification, or reservation details", - schema=Confirmation, + confirm_response = client.sessions.extract( + id=session_id, + instruction="Extract any booking confirmation message, success notification, or reservation details", + schema=Confirmation.model_json_schema(), ) + confirmation = confirm_response.data.result - # Display confirmation details if booking was successful. - if confirmation.confirmation_message or confirmation.booking_details: + if confirmation.get("confirmation_message") or confirmation.get("booking_details"): print("Booking Confirmed!") - if confirmation.confirmation_message: - print(f"{confirmation.confirmation_message}") - if confirmation.booking_details: - print(f"{confirmation.booking_details}") + if confirmation.get("confirmation_message"): + print(f"{confirmation.get('confirmation_message')}") + if confirmation.get("booking_details"): + print(f"{confirmation.get('booking_details')}") - # Display error message if booking failed. - if confirmation.error_message: + if confirmation.get("error_message"): print("Booking Error:") - print(confirmation.error_message) + print(confirmation.get("error_message")) except Exception as error: print(f"Error during court booking: {error}") raise error -async def select_activity() -> str: - # Prompt user to select between Tennis and Pickleball activities. - def get_activity(): - return inquirer.select( - message="Please select an activity:", - choices=[ - {"name": "Tennis", "value": "Tennis"}, - {"name": "Pickleball", "value": "Pickleball"}, - ], - default="Tennis", - ).execute() - - activity = await asyncio.to_thread(get_activity) +def select_activity() -> str: + activity = inquirer.select( + message="Please select an activity:", + choices=[ + {"name": "Tennis", "value": "Tennis"}, + {"name": "Pickleball", "value": "Pickleball"}, + ], + default="Tennis", + ).execute() print(f"Selected: {activity}") return activity -async def select_time_of_day() -> str: - # Prompt user to select preferred time period for court booking. - def get_time_of_day(): - return inquirer.select( - message="Please select the time of day:", - choices=[ - {"name": "Morning (Before 12 PM)", "value": "Morning"}, - {"name": "Afternoon (After 12 PM)", "value": "Afternoon"}, - {"name": "Evening (After 5 PM)", "value": "Evening"}, - ], - default="Morning", - ).execute() - - time_of_day = await asyncio.to_thread(get_time_of_day) +def select_time_of_day() -> str: + time_of_day = inquirer.select( + message="Please select the time of day:", + choices=[ + {"name": "Morning (Before 12 PM)", "value": "Morning"}, + {"name": "Afternoon (After 12 PM)", "value": "Afternoon"}, + {"name": "Evening (After 5 PM)", "value": "Evening"}, + ], + default="Morning", + ).execute() print(f"Selected: {time_of_day}") return time_of_day -async def select_date() -> str: - # Generate date options for the next 7 days including today. +def select_date() -> str: today = datetime.now() date_options = [] for i in range(7): date = today + timedelta(days=i) - day_name = date.strftime("%A") month_day = date.strftime("%b %-d") full_date = date.strftime("%Y-%m-%d") display_name = f"{day_name}, {month_day} (Today)" if i == 0 else f"{day_name}, {month_day}" - date_options.append({"name": display_name, "value": full_date}) - # Prompt user to select from available date options. - def get_date(): - return inquirer.select( - message="Please select a date:", choices=date_options, default=date_options[0]["value"] - ).execute() + selected_date = inquirer.select( + message="Please select a date:", choices=date_options, default=date_options[0]["value"] + ).execute() - selected_date = await asyncio.to_thread(get_date) - - # Format selected date for display and return ISO date string. selected_date_obj = datetime.strptime(selected_date, "%Y-%m-%d") display_date = selected_date_obj.strftime("%A, %B %-d, %Y") @@ -315,81 +379,71 @@ def get_date(): return selected_date -async def book_tennis_paddle_court(): +def book_tennis_paddle_court(): print("Starting tennis/paddle court booking automation in SF...") - # Load credentials from environment variables for SF Rec & Parks login. email = os.environ.get("SF_REC_PARK_EMAIL") password = os.environ.get("SF_REC_PARK_PASSWORD") - # Validate that required credentials are available before proceeding. if not email or not password: raise ValueError("Missing SF_REC_PARK_EMAIL or SF_REC_PARK_PASSWORD environment variables") - # Collect user preferences for activity, date, and time selection. - activity = await select_activity() - selected_date = await select_date() - time_of_day = await select_time_of_day() + activity = select_activity() + selected_date = select_date() + time_of_day = select_time_of_day() print(f"Booking {activity} courts in San Francisco for {time_of_day} on {selected_date}...") - # Initialize Stagehand with Browserbase for AI-powered browser automation. + # Initialize Stagehand with Browserbase for cloud-based browser automation print("Initializing Stagehand with Browserbase") - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - verbose=1, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - model_name="openai/gpt-4.1", + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - "timeout": 900, - "region": "us-west-2", - }, ) - try: - # Start browser session and connect to SF Rec & Parks booking system. - async with Stagehand(config) as stagehand: - print("Browserbase Session Started") - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id + # Start a new session + start_response = client.sessions.start( + model_name="openai/gpt-4.1", + ) + session_id = start_response.data.session_id - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") + try: + print("Browserbase Session Started") + print(f"Watch live: https://browserbase.com/sessions/{session_id}") - page = stagehand.page + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() - # Navigate to SF Rec & Parks booking site with extended timeout for slow loading. print("Navigating to court booking site...") - await page.goto( + page.goto( "https://www.rec.us/organizations/san-francisco-rec-park", wait_until="domcontentloaded", timeout=60000, ) - # Execute booking workflow: login, filter, find courts, and complete booking. - await login_to_site(page, email, password) - await select_filters(page, activity, time_of_day, selected_date) - await check_and_extract_courts(page, time_of_day) - await book_court(page) + login_to_site(client, session_id, email, password) + select_filters(client, session_id, activity, time_of_day, selected_date) + check_and_extract_courts(client, session_id, time_of_day) + book_court(client, session_id) + + browser.close() + client.sessions.end(id=session_id) print("\nBrowser session closed") except Exception as error: print(f"Error during court booking: {error}") + client.sessions.end(id=session_id) raise error -async def main(): - # Display welcome message and explain the automation workflow to user. +def main(): print("Welcome to SF Court Booking Automation!") print("") print("This tool automates tennis and pickleball court bookings in San Francisco.") @@ -403,9 +457,7 @@ async def main(): print("") try: - # Execute the complete court booking automation workflow. - await book_tennis_paddle_court() - + book_tennis_paddle_court() print("Court booking completed successfully!") print("Your court has been reserved. Check your email for confirmation details.") except Exception as error: @@ -416,7 +468,7 @@ async def main(): if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Application error: {err}") exit(1) diff --git a/python/polymarket-research/README.md b/python/polymarket-research/README.md index beb08de4..ee3d60e7 100644 --- a/python/polymarket-research/README.md +++ b/python/polymarket-research/README.md @@ -18,11 +18,10 @@ ## QUICKSTART 1. cd polymarket-research -2. uv venv venv -3. source venv/bin/activate # On Windows: venv\Scripts\activate -4. pip install stagehand python-dotenv pydantic -5. cp .env.example .env # Add your Browserbase API key, Project ID, and OpenAI API key to .env -6. python main.py +2. uv venv && source .venv/bin/activate # On Windows: .venv\Scripts\activate +3. pip install stagehand python-dotenv pydantic +4. cp .env.example .env # Add your Browserbase API key, Project ID, and OpenAI API key to .env +5. python main.py ## EXPECTED OUTPUT diff --git a/python/polymarket-research/main.py b/python/polymarket-research/main.py index cb0bf91b..8ad858fe 100644 --- a/python/polymarket-research/main.py +++ b/python/polymarket-research/main.py @@ -1,12 +1,12 @@ # Stagehand + Browserbase: Polymarket prediction market research - See README.md for full documentation -import asyncio +import json import os from dotenv import load_dotenv from pydantic import BaseModel, Field -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables load_dotenv() @@ -23,7 +23,7 @@ class MarketData(BaseModel): priceChange: str | None = Field(None, description="the recent price change") -async def main(): +def main(): """ Searches Polymarket for a prediction market and extracts current odds, pricing, and volume data. Uses AI-powered browser automation to navigate and interact with the site. @@ -31,71 +31,61 @@ async def main(): print("Starting Polymarket research automation...") # Initialize Stagehand with Browserbase for cloud-based browser automation - # Using BROWSERBASE environment to run in cloud rather than locally - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="openai/gpt-4.1", + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), model_api_key=os.environ.get("OPENAI_API_KEY"), - verbose=1, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging ) - try: - # Use async context manager for automatic resource management - async with Stagehand(config) as stagehand: - print("Initializing browser session...") - print("Stagehand session started successfully") - - # Provide live session URL for debugging and monitoring - session_id = None - if hasattr(stagehand, "session_id"): - session_id = stagehand.session_id - elif hasattr(stagehand, "browserbase_session_id"): - session_id = stagehand.browserbase_session_id - - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") - - page = stagehand.page - - # Navigate to Polymarket - print("Navigating to: https://polymarket.com/") - await page.goto("https://polymarket.com/") - print("Page loaded successfully") - - # Click the search box to trigger search dropdown - print("Clicking the search box at the top of the page") - await page.act("click the search box at the top of the page") - - # Type search query - searchQuery = "Elon Musk unfollow Trump" - print(f"Typing '{searchQuery}' into the search box") - await page.act(f"type '{searchQuery}' into the search box") - - # Click the first market result from the search dropdown - print("Selecting first market result from search dropdown") - await page.act("click the first market result from the search dropdown") - print("Market page loaded") - - # Extract market data using AI to parse the structured information - print("Extracting market information...") - marketData = await page.extract( - "Extract the current odds and market information for the prediction market", - schema=MarketData, - ) - - print("Market data extracted successfully:") - - # Display results in formatted JSON - import json - - print(json.dumps(marketData.model_dump(), indent=2)) + # Start a new session + start_response = client.sessions.start( + model_name="openai/gpt-4.1", + ) + session_id = start_response.data.session_id - print("Session closed successfully") + try: + print("Initializing browser session...") + print("Stagehand session started successfully") + print(f"Watch live: https://browserbase.com/sessions/{session_id}") + + # Navigate to Polymarket + print("Navigating to: https://polymarket.com/") + client.sessions.navigate(id=session_id, url="https://polymarket.com/") + print("Page loaded successfully") + + # Click the search box to trigger search dropdown + print("Clicking the search box at the top of the page") + client.sessions.act( + id=session_id, + input="click the search box at the top of the page", + ) + + # Type search query + searchQuery = "Elon Musk unfollow Trump" + print(f"Typing '{searchQuery}' into the search box") + client.sessions.act( + id=session_id, + input=f"type '{searchQuery}' into the search box", + ) + + # Click the first market result from the search dropdown + print("Selecting first market result from search dropdown") + client.sessions.act( + id=session_id, + input="click the first market result from the search dropdown", + ) + print("Market page loaded") + + # Extract market data using AI to parse the structured information + print("Extracting market information...") + extract_response = client.sessions.extract( + id=session_id, + instruction="Extract the current odds and market information for the prediction market", + schema=MarketData.model_json_schema(), + ) + + print("Market data extracted successfully:") + print(json.dumps(extract_response.data.result, indent=2)) except Exception as error: print(f"Error during market research: {error}") @@ -106,13 +96,16 @@ async def main(): print("2. Verify OPENAI_API_KEY is set in environment") print("3. Ensure internet access and https://polymarket.com is accessible") print("4. Verify Browserbase account has sufficient credits") - raise + finally: + client.sessions.end(id=session_id) + print("Session closed successfully") + if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Error in polymarket research: {err}") print("Common issues:") diff --git a/python/proxies-weather/main.py b/python/proxies-weather/main.py index 18b3669d..1ad3e948 100644 --- a/python/proxies-weather/main.py +++ b/python/proxies-weather/main.py @@ -1,12 +1,14 @@ # Stagehand + Browserbase: Weather Proxy Demo - See README.md for full documentation -import asyncio import os +import time +from browserbase import Browserbase from dotenv import load_dotenv +from playwright.sync_api import sync_playwright from pydantic import BaseModel, Field -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand load_dotenv() @@ -36,7 +38,7 @@ class TemperatureData(BaseModel): unit: str = Field(..., description="The temperature unit") -async def get_weather_for_location(geolocation: GeolocationConfig) -> WeatherResult: +def get_weather_for_location(geolocation: GeolocationConfig) -> WeatherResult: """Fetch weather data for a specific location using geolocation proxies.""" city_name = geolocation.city.replace("_", " ") print(f"\n=== Getting weather for {city_name}, {geolocation.country} ===") @@ -50,72 +52,79 @@ async def get_weather_for_location(geolocation: GeolocationConfig) -> WeatherRes # Build proxy configuration for geolocation routing proxy_config = { - "type": "browserbase", # Use Browserbase's managed proxy infrastructure for reliable geolocation routing + "type": "browserbase", "geolocation": { - "city": geolocation.city, # City name (case-insensitive, e.g., "NEW_YORK", "new_york", "New York" all work) - "country": geolocation.country, # ISO country code (case-insensitive, e.g., "US", "us", "gb", "GB" all work) + "city": geolocation.city, + "country": geolocation.country, }, } - # Add state if provided (required for US locations to ensure accurate geolocation, case-insensitive) if geolocation.state: proxy_config["geolocation"]["state"] = geolocation.state - # Initialize Stagehand with geolocation proxy configuration - # This ensures all browser traffic routes through the specified geographic location - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), + # Initialize Browserbase SDK for session creation with proxy + bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) + + # Create session with geolocation proxy + session = bb.sessions.create( project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name=os.environ.get("MODEL_NAME", "google/gemini-2.5-flash"), - model_api_key=model_api_key, - verbose=0, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - "proxies": [proxy_config], - }, + proxies=[proxy_config], ) + session_id = session.id - stagehand = Stagehand(config) + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=model_api_key, + ) try: - # Initialize browser session to start automation print(f"Initializing Stagehand for {city_name}...") - await stagehand.init() - print(f"Stagehand initialized successfully for {city_name}") - - # Navigate to weather service - geolocation proxy ensures location-specific weather data - print(f"Navigating to weather service for {city_name}...") - await stagehand.page.goto("https://www.windy.com/", wait_until="networkidle") # Wait for network to be idle to ensure weather data is loaded - print(f"Page loaded for {city_name}") - - # Wait a bit for weather data to render - await asyncio.sleep(2) - - # Extract structured temperature data using Stagehand and Pydantic schema for type safety - print(f"Extracting temperature data for {city_name}...") - extract_result = await stagehand.page.extract( - instruction="Extract the current temperature and its unit", - schema=TemperatureData, - ) + print(f"Session URL: https://browserbase.com/sessions/{session_id}") - print( - f"Successfully extracted weather data for {city_name}: {extract_result.temperature} {extract_result.unit}" - ) + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() + + print(f"Stagehand initialized successfully for {city_name}") + + # Navigate to weather service + print(f"Navigating to weather service for {city_name}...") + page.goto("https://www.windy.com/", wait_until="networkidle") + print(f"Page loaded for {city_name}") + + # Wait a bit for weather data to render + time.sleep(2) + + # Extract structured temperature data + print(f"Extracting temperature data for {city_name}...") + extract_response = client.sessions.extract( + id=session_id, + instruction="Extract the current temperature and its unit", + schema=TemperatureData.model_json_schema(), + ) + + result_data = extract_response.data.result + print( + f"Successfully extracted weather data for {city_name}: {result_data.get('temperature')} {result_data.get('unit')}" + ) + + browser.close() - # Close Stagehand session to release resources - await stagehand.close() + client.sessions.end(id=session_id) return WeatherResult( city=city_name, country=geolocation.country, - temperature=extract_result.temperature, - unit=extract_result.unit, + temperature=result_data.get("temperature", 0.0), + unit=result_data.get("unit", ""), ) except Exception as error: - await stagehand.close() + client.sessions.end(id=session_id) print(f"Error getting weather for {city_name}: {error}") return WeatherResult( city=city_name, @@ -136,13 +145,10 @@ def display_results(results: list[WeatherResult]): print(f"{result.city}, {result.country}: {result.temperature} {result.unit}") -async def main(): +def main(): """Main orchestration function: processes multiple locations sequentially using geolocation proxies.""" - # Define locations to test - demonstrating the power of geolocation proxies - # Each location will route traffic through its respective geographic proxy to get location-specific weather - # Note: All geolocation fields (city, country, state) are case-insensitive locations = [ - GeolocationConfig(city="NEW_YORK", state="NY", country="US"), # State required for US locations (case-insensitive) + GeolocationConfig(city="NEW_YORK", state="NY", country="US"), GeolocationConfig(city="LONDON", country="GB"), GeolocationConfig(city="TOKYO", country="JP"), GeolocationConfig(city="SAO_PAULO", country="BR"), @@ -152,24 +158,20 @@ async def main(): print(f"Processing {len(locations)} locations with geolocation proxies...") print("Each location will use a different proxy to fetch location-specific weather data\n") - # Collect all results for final summary results: list[WeatherResult] = [] - # Run each location sequentially to show different weather based on proxy location - # Sequential processing ensures clear demonstration of proxy-based location differences for i, location in enumerate(locations, 1): print(f"\n[{i}/{len(locations)}] Processing {location.city}, {location.country}...") - result = await get_weather_for_location(location) + result = get_weather_for_location(location) results.append(result) - # Display all results in formatted summary display_results(results) print("\n=== All locations completed ===") if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print(f"Application error: {err}") print("Common issues:") @@ -179,4 +181,4 @@ async def main(): " - Verify geolocation proxy locations are valid (see https://docs.browserbase.com/features/proxies)" ) print("Docs: https://docs.stagehand.dev/v3/first-steps/introduction") - exit(1) \ No newline at end of file + exit(1) diff --git a/python/proxies/main.py b/python/proxies/main.py index 7ee29425..bac56b49 100644 --- a/python/proxies/main.py +++ b/python/proxies/main.py @@ -1,14 +1,14 @@ # Browserbase Proxy Testing Script - See README.md for full documentation -import asyncio +import json import os from browserbase import Browserbase from dotenv import load_dotenv -from playwright.async_api import async_playwright +from playwright.sync_api import sync_playwright from pydantic import BaseModel, Field -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand load_dotenv() @@ -29,27 +29,25 @@ class GeoInfo(BaseModel): hostname: str = Field(..., description="The hostname if available") -async def create_session_with_built_in_proxies(): +def create_session_with_built_in_proxies(): # Use Browserbase's default proxy rotation for enhanced privacy and IP diversity. - session = await asyncio.to_thread( - bb.sessions.create, + session = bb.sessions.create( project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - proxies=True, # Enables automatic proxy rotation across different IP addresses. + proxies=True, ) return session -async def create_session_with_geo_location(): +def create_session_with_geo_location(): # Route traffic through specific geographic location to test location-based restrictions. - session = await asyncio.to_thread( - bb.sessions.create, + session = bb.sessions.create( project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), proxies=[ { - "type": "browserbase", # Use Browserbase's managed proxy infrastructure. + "type": "browserbase", "geolocation": { - "city": "NEW_YORK", # Simulate traffic from New York for testing geo-specific content. - "state": "NY", # See https://docs.browserbase.com/features/proxies for more geolocation options. + "city": "NEW_YORK", + "state": "NY", "country": "US", }, } @@ -58,16 +56,15 @@ async def create_session_with_geo_location(): return session -async def create_session_with_custom_proxies(): +def create_session_with_custom_proxies(): # Use external proxy servers for custom routing or specific proxy requirements. - session = await asyncio.to_thread( - bb.sessions.create, + session = bb.sessions.create( project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), proxies=[ { - "type": "external", # Connect to your own proxy server infrastructure. - "server": "http://...", # Your proxy server endpoint. - "username": "user", # Authentication credentials for proxy access. + "type": "external", + "server": "http://...", + "username": "user", "password": "pass", } ], @@ -75,76 +72,68 @@ async def create_session_with_custom_proxies(): return session -async def test_session(session_function, session_name: str): +def test_session(session_function, session_name: str): print(f"\n=== Testing {session_name} ===") - # Create session with specific proxy configuration to test different routing scenarios. - session = await session_function() - print(f"Session URL: https://browserbase.com/sessions/{session.id}") - - # Connect to browser via CDP to control the session programmatically. - async with async_playwright() as p: - browser = await p.chromium.connect_over_cdp(session.connect_url) - default_context = browser.contexts[0] if browser.contexts else None - if not default_context: - raise Exception("No default context found") - - page = default_context.pages[0] if default_context.pages else None - if not page: - raise Exception("No page found in default context") - - # Initialize Stagehand for structured data extraction - stagehand_config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - verbose=1, - # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - model_name="openai/gpt-4.1", - model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_id=session.id, # Use the existing Browserbase session - ) - - stagehand = Stagehand(stagehand_config) - - try: - # Initialize Stagehand - await stagehand.init() + # Create session with specific proxy configuration + session = session_function() + session_id = session.id + print(f"Session URL: https://browserbase.com/sessions/{session_id}") + + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("OPENAI_API_KEY"), + ) + + try: + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" + ) + context = browser.contexts[0] if browser.contexts else None + if not context: + raise Exception("No default context found") + + page = context.pages[0] if context.pages else None + if not page: + raise Exception("No page found in default context") # Navigate to IP info service to verify proxy location and IP address. - await stagehand.page.goto("https://ipinfo.io/json", wait_until="domcontentloaded") + page.goto("https://ipinfo.io/json", wait_until="domcontentloaded") - # Extract structured IP and location data using Stagehand and Pydantic schema - geo_info = await stagehand.page.extract( + # Extract structured IP and location data using Stagehand + extract_response = client.sessions.extract( + id=session_id, instruction="Extract all IP information and geolocation data from the JSON response", - schema=GeoInfo, + schema=GeoInfo.model_json_schema(), ) - print("Geo Info:", geo_info.model_dump_json(indent=2)) + print("Geo Info:", json.dumps(extract_response.data.result, indent=2)) - # Close Stagehand session - await stagehand.close() - except Exception as error: - print(f"Error during Stagehand extraction: {error}") + browser.close() - # Close browser to release resources and end the test session. - await browser.close() + client.sessions.end(id=session_id) print(f"{session_name} test completed") + except Exception as error: + print(f"Error during Stagehand extraction: {error}") + client.sessions.end(id=session_id) + -async def main(): +def main(): # Test 1: Built-in proxies - Verify default proxy rotation works and shows different IPs. - await test_session(create_session_with_built_in_proxies, "Built-in Proxies") + test_session(create_session_with_built_in_proxies, "Built-in Proxies") # Test 2: Geolocation proxies - Confirm traffic routes through specified location (New York). - await test_session(create_session_with_geo_location, "Geolocation Proxies (New York)") + test_session(create_session_with_geo_location, "Geolocation Proxies (New York)") # Test 3: Custom external proxies - Enable if you have a custom proxy server set up. - # await test_session(create_session_with_custom_proxies, "Custom External Proxies") + # test_session(create_session_with_custom_proxies, "Custom External Proxies") print("\n=== All tests completed ===") if __name__ == "__main__": - asyncio.run(main()) + main() diff --git a/python/website-link-tester/main.py b/python/website-link-tester/main.py index 1349febe..4f48dd79 100644 --- a/python/website-link-tester/main.py +++ b/python/website-link-tester/main.py @@ -1,17 +1,18 @@ # Stagehand + Browserbase: Website Link Tester - See README.md for full documentation -import asyncio import json import os from dotenv import load_dotenv +from playwright.sync_api import sync_playwright from pydantic import BaseModel, Field, HttpUrl -from stagehand import Stagehand, StagehandConfig +from stagehand import Stagehand # Load environment variables load_dotenv() + # Base URL whose links we want to crawl and verify URL = "https://www.browserbase.com" @@ -71,33 +72,15 @@ class PageVerificationSummary(BaseModel): ] -def create_stagehand() -> Stagehand: - """Creates a preconfigured Stagehand instance for Browserbase sessions.""" - config = StagehandConfig( - env="BROWSERBASE", - api_key=os.environ.get("BROWSERBASE_API_KEY"), - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), - model_name="google/gemini-2.5-pro", - model_api_key=os.environ.get("GOOGLE_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - }, - verbose=0, # 0 = errors only, 1 = info, 2 = debug - # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) - # https://docs.stagehand.dev/configuration/logging - ) - return Stagehand(config) - - -def deduplicate_links(extracted_links: ExtractedLinks) -> list[ExtractedLink]: +def deduplicate_links(extracted_links: dict) -> list[dict]: """ Removes duplicate links by URL while preserving the first occurrence. """ seen_urls: set[str] = set() - unique_links: list[ExtractedLink] = [] + unique_links: list[dict] = [] - for link in extracted_links.links: - url = str(link.url) + for link in extracted_links.get("links", []): + url = str(link.get("url", "")) if url in seen_urls: continue seen_urls.add(url) @@ -106,88 +89,133 @@ def deduplicate_links(extracted_links: ExtractedLinks) -> list[ExtractedLink]: return unique_links -async def collect_links_from_homepage() -> list[ExtractedLink]: +def collect_links_from_homepage() -> list[dict]: """ Opens the homepage and uses Stagehand `extract()` to collect all links. Returns a de-duplicated list of link objects that we will later verify. """ print("Collecting links from homepage...") - stagehand: Stagehand | None = None + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("GOOGLE_API_KEY"), + ) + + # Start a new session + start_response = client.sessions.start( + model_name="google/gemini-2.5-pro", + ) + session_id = start_response.data.session_id try: - stagehand = create_stagehand() - async with stagehand: - await stagehand.init() + print(f"Watch live: https://browserbase.com/sessions/{session_id}") - session_id = getattr(stagehand, "session_id", None) or getattr( - stagehand, "browserbase_session_id", None + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" ) - if session_id: - print(f"Watch live: https://browserbase.com/sessions/{session_id}") - - page = stagehand.page + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() # Navigate to the base URL where we will harvest links print(f"Navigating to {URL}...") - await page.goto(URL, wait_until="domcontentloaded") + page.goto(URL, wait_until="domcontentloaded") print(f"Successfully loaded {URL}. Extracting links...") - extracted_links: ExtractedLinks = await page.extract( - "Extract all links on the page with their link text.", - schema=ExtractedLinks, + # Inline schema to avoid $ref issues + links_schema = { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "Destination URL of the link", + }, + "link_text": { + "type": "string", + "description": "Visible text of the link", + }, + }, + "required": ["url", "link_text"], + }, + } + }, + "required": ["links"], + } + extract_response = client.sessions.extract( + id=session_id, + instruction="Extract all links on the page with their link text.", + schema=links_schema, ) + extracted_links = extract_response.data.result # Remove duplicate URLs and log both raw and unique counts for visibility unique_links = deduplicate_links(extracted_links) print( - f"All links on the page ({len(extracted_links.links)} total, {len(unique_links)} unique):" - ) - print( - json.dumps( - {"links": [link.model_dump(mode="json") for link in unique_links]}, indent=2 - ) + f"All links on the page ({len(extracted_links.get('links', []))} total, {len(unique_links)} unique):" ) + print(json.dumps({"links": unique_links}, indent=2)) + + browser.close() - return unique_links + client.sessions.end(id=session_id) + return unique_links except Exception as error: print(f"Error while collecting links: {error}") + client.sessions.end(id=session_id) raise -async def verify_single_link(link: ExtractedLink) -> LinkVerificationResult: +def verify_single_link(link: dict) -> LinkVerificationResult: """ Verifies a single link by opening it in a dedicated browser session. - Confirms the page loads successfully. - For non-social links, uses `extract()` to check that the page content matches what the link text suggests. """ - print(f"\nChecking: {link.link_text} ({link.url})") + link_text = link.get("link_text", "Unknown") + link_url = link.get("url", "") - stagehand: Stagehand | None = None + print(f"\nChecking: {link_text} ({link_url})") + + # Initialize Stagehand with Browserbase for cloud-based browser automation + client = Stagehand( + browserbase_api_key=os.environ.get("BROWSERBASE_API_KEY"), + browserbase_project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), + model_api_key=os.environ.get("GOOGLE_API_KEY"), + ) + + # Start a new session + start_response = client.sessions.start( + model_name="google/gemini-2.5-pro", + ) + session_id = start_response.data.session_id try: - stagehand = create_stagehand() - async with stagehand: - await stagehand.init() + print(f"[{link_text}] Live View: https://browserbase.com/sessions/{session_id}") - session_id = getattr(stagehand, "session_id", None) or getattr( - stagehand, "browserbase_session_id", None + # Connect to the browser via CDP + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp( + f"wss://connect.browserbase.com?apiKey={os.environ['BROWSERBASE_API_KEY']}&sessionId={session_id}" ) - if session_id: - print( - f"[{link.link_text}] Live View: https://browserbase.com/sessions/{session_id}" - ) - - page = stagehand.page + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() # Detect if this is a social link (we treat those differently) - is_social_link = any(domain in str(link.url) for domain in SOCIAL_DOMAINS) + is_social_link = any(domain in str(link_url) for domain in SOCIAL_DOMAINS) - await page.goto(str(link.url), wait_until="domcontentloaded", timeout=30000) + page.goto(str(link_url), wait_until="domcontentloaded", timeout=30000) current_url = page.url @@ -195,14 +223,16 @@ async def verify_single_link(link: ExtractedLink) -> LinkVerificationResult: if not current_url or current_url == "about:blank": raise Exception("Page failed to load - invalid URL detected") - print(f"Link opened successfully: {link.link_text}") + print(f"Link opened successfully: {link_text}") # For social links, we consider a successful load good enough if is_social_link: - print(f"[{link.link_text}] Social media link - skipping content verification") + print(f"[{link_text}] Social media link - skipping content verification") + browser.close() + client.sessions.end(id=session_id) return LinkVerificationResult( - link_text=link.link_text, - url=link.url, + link_text=link_text, + url=link_url, success=True, page_title="Social Media Link", content_matches=True, @@ -210,50 +240,58 @@ async def verify_single_link(link: ExtractedLink) -> LinkVerificationResult: ) # Ask the model to read the page and decide whether it matches the link text - print(f"[{link.link_text}] Verifying page content against link text...") - verification: PageVerificationSummary = await page.extract( - f'Does the page content match what the link text "{link.link_text}" suggests? Extract the page title and provide a brief assessment (maximum 8 words).', - schema=PageVerificationSummary, + print(f"[{link_text}] Verifying page content against link text...") + extract_response = client.sessions.extract( + id=session_id, + instruction=f'Does the page content match what the link text "{link_text}" suggests? Extract the page title and provide a brief assessment (maximum 8 words).', + schema=PageVerificationSummary.model_json_schema(), ) + verification = extract_response.data.result - print(f"[{link.link_text}] Page Title: {verification.page_title}") + print(f"[{link_text}] Page Title: {verification.get('page_title')}") print( - f"[{link.link_text}] Content Matches: {'YES' if verification.content_matches else 'NO'}" - ) - print(f"[{link.link_text}] Assessment: {verification.assessment}") - - return LinkVerificationResult( - link_text=link.link_text, - url=link.url, - success=True, - page_title=verification.page_title, - content_matches=verification.content_matches, - assessment=verification.assessment, + f"[{link_text}] Content Matches: {'YES' if verification.get('content_matches') else 'NO'}" ) + print(f"[{link_text}] Assessment: {verification.get('assessment')}") + + browser.close() + + client.sessions.end(id=session_id) + + return LinkVerificationResult( + link_text=link_text, + url=link_url, + success=True, + page_title=verification.get("page_title"), + content_matches=verification.get("content_matches"), + assessment=verification.get("assessment"), + ) except Exception as error: error_message = str(error) - print(f'Failed to verify link "{link.link_text}": {error_message}') + print(f'Failed to verify link "{link_text}": {error_message}') + + client.sessions.end(id=session_id) # On failure, return a structured result capturing the error message return LinkVerificationResult( - link_text=link.link_text, - url=link.url, + link_text=link_text, + url=link_url, success=False, error=error_message, ) -async def verify_links_in_batches( - links: list[ExtractedLink], +def verify_links_in_batches( + links: list[dict], ) -> list[LinkVerificationResult]: """ - Verifies all links in batches to avoid opening too many concurrent sessions. + Verifies all links sequentially. Returns a list of LinkVerificationResult objects for all processed links. """ max_concurrent = max(1, MAX_CONCURRENT_LINKS) - print(f"\nVerifying links in batches of {max_concurrent}...") + print(f"\nVerifying links (batch size: {max_concurrent})...") results: list[LinkVerificationResult] = [] @@ -264,8 +302,10 @@ async def verify_links_in_batches( print(f"\n=== Processing batch {batch_number}/{total_batches} ({len(batch)} links) ===") - batch_results = await asyncio.gather(*[verify_single_link(link) for link in batch]) - results.extend(batch_results) + # Process links sequentially + for link in batch: + result = verify_single_link(link) + results.append(result) print(f"\nBatch {batch_number}/{total_batches} complete ({len(results)} total verified)") @@ -300,7 +340,7 @@ def output_results(results: list[LinkVerificationResult], label: str = "FINAL RE print("\n" + "=" * 80) -async def main(): +def main(): """ Orchestrates the full flow: 1. Collect all links from the homepage. @@ -312,12 +352,12 @@ async def main(): results: list[LinkVerificationResult] = [] try: - links = await collect_links_from_homepage() + links = collect_links_from_homepage() print(f"Collected {len(links)} links, starting verification...") - results = await verify_links_in_batches(links) + results = verify_links_in_batches(links) - print("\n✓ All links verified!") + print("\nAll links verified!") print(f"Results array length: {len(results)}") output_results(results) @@ -337,7 +377,7 @@ async def main(): if __name__ == "__main__": try: - asyncio.run(main()) + main() except Exception as err: print("Application error:", err) print("Common issues:") diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..bda02073 --- /dev/null +++ b/uv.lock @@ -0,0 +1,3 @@ +version = 1 +revision = 3 +requires-python = ">=3.13" From 78b30224ca3ed9ab0ac714288ea727e93bf99314 Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Thu, 5 Feb 2026 22:54:47 -0800 Subject: [PATCH 18/45] init commit (#54) --- python/amazon-global-price-comparison/main.py | 49 ++++++++++--------- python/amazon-product-scraping/main.py | 4 +- python/browserbase-reducto/main.py | 1 + .../form_filling_node.py | 48 +++++++++++------- python/cartesia-form-filling/main.py | 5 +- .../stagehand_form_filler.py | 16 +++--- python/exa-browserbase/main.py | 2 +- python/google-trends/main.py | 4 +- python/sec-filing-research/main.py | 16 +++--- 9 files changed, 82 insertions(+), 63 deletions(-) diff --git a/python/amazon-global-price-comparison/main.py b/python/amazon-global-price-comparison/main.py index 908df56a..30c6168a 100644 --- a/python/amazon-global-price-comparison/main.py +++ b/python/amazon-global-price-comparison/main.py @@ -4,7 +4,6 @@ import json import os from dataclasses import dataclass -from typing import Optional from browserbase import Browserbase from dotenv import load_dotenv @@ -29,9 +28,7 @@ class Product(BaseModel): "'29.99 GBP'). If no price is visible, return 'N/A'" ), ) - rating: str = Field( - default="N/A", description="The star rating (e.g., '4.5 out of 5 stars')" - ) + rating: str = Field(default="N/A", description="The star rating (e.g., '4.5 out of 5 stars')") reviews_count: str = Field( default="N/A", description="The number of customer reviews (e.g., '1,234')" ) @@ -56,7 +53,7 @@ class ProductsResult(BaseModel): class CountryConfig: name: str code: str - city: Optional[str] + city: str | None currency: str @@ -79,7 +76,7 @@ class CountryResult: country_code: str currency: str products: list[dict] - error: Optional[str] = None + error: str | None = None # Initialize Browserbase SDK for session management with proxies @@ -168,11 +165,26 @@ async def get_products_for_country( "items": { "type": "object", "properties": { - "name": {"type": "string", "description": "The full product title/name"}, - "price": {"type": "string", "description": "The product price including currency symbol (e.g., '$29.99'). If no price is visible, return 'N/A'"}, - "rating": {"type": "string", "description": "The star rating (e.g., '4.5 out of 5 stars')"}, - "reviews_count": {"type": "string", "description": "The number of customer reviews (e.g., '1,234')"}, - "product_url": {"type": "string", "description": "The full href URL link to the product detail page"}, + "name": { + "type": "string", + "description": "The full product title/name", + }, + "price": { + "type": "string", + "description": "The product price including currency symbol (e.g., '$29.99'). If no price is visible, return 'N/A'", + }, + "rating": { + "type": "string", + "description": "The star rating (e.g., '4.5 out of 5 stars')", + }, + "reviews_count": { + "type": "string", + "description": "The number of customer reviews (e.g., '1,234')", + }, + "product_url": { + "type": "string", + "description": "The full href URL link to the product detail page", + }, }, "required": ["name"], }, @@ -188,7 +200,7 @@ async def get_products_for_country( 3. rating: the star rating text (like "4.5 out of 5 stars") 4. reviews_count: the number of reviews (like "2,508") 5. product_url: the href link to the product page (starts with /dp/ or https://) - + Only extract actual product listings, skip sponsored ads or recommendations.""", schema=products_schema, ) @@ -276,9 +288,7 @@ def display_comparison_table(results: list[CountryResult]) -> None: break if product_name: - truncated_name = ( - product_name[:77] + "..." if len(product_name) > 80 else product_name - ) + truncated_name = product_name[:77] + "..." if len(product_name) > 80 else product_name print(f"Product: {truncated_name}") print("\nPrices by Country:") @@ -296,9 +306,7 @@ def display_comparison_table(results: list[CountryResult]) -> None: rating_short = rating.split(" out")[0] if " out" in rating else rating rating_pad = rating_short.ljust(6) reviews = product.get("reviews_count", "N/A") - print( - f" {country_pad} | {price_pad} | {rating_pad} stars | {reviews} reviews" - ) + print(f" {country_pad} | {price_pad} | {rating_pad} stars | {reviews} reviews") else: print(f" {country_pad} | Not available in this country") @@ -332,10 +340,7 @@ async def main(): print(f"\nFetching prices from {len(COUNTRIES)} countries concurrently...") results = await asyncio.gather( - *[ - get_products_for_country(search_query, country, results_count) - for country in COUNTRIES - ] + *[get_products_for_country(search_query, country, results_count) for country in COUNTRIES] ) # Display formatted comparison table diff --git a/python/amazon-product-scraping/main.py b/python/amazon-product-scraping/main.py index 576030ef..f7db7032 100644 --- a/python/amazon-product-scraping/main.py +++ b/python/amazon-product-scraping/main.py @@ -4,10 +4,10 @@ import asyncio import json import os -from typing import List from dotenv import load_dotenv from pydantic import BaseModel, Field + from stagehand import AsyncStagehand @@ -24,7 +24,7 @@ class Product(BaseModel): class ProductsList(BaseModel): """Schema for extracting a list of Amazon products.""" - products: List[Product] = Field(description="Array of the first 3 products from search results") + products: list[Product] = Field(description="Array of the first 3 products from search results") def dereference_schema(schema: dict) -> dict: diff --git a/python/browserbase-reducto/main.py b/python/browserbase-reducto/main.py index e2eb0896..abe1e1c9 100644 --- a/python/browserbase-reducto/main.py +++ b/python/browserbase-reducto/main.py @@ -9,6 +9,7 @@ from browserbase import Browserbase from dotenv import load_dotenv from reducto import Reducto + from stagehand import AsyncStagehand # Load environment variables from .env file diff --git a/python/cartesia-form-filling/form_filling_node.py b/python/cartesia-form-filling/form_filling_node.py index 73b9f818..d8864513 100644 --- a/python/cartesia-form-filling/form_filling_node.py +++ b/python/cartesia-form-filling/form_filling_node.py @@ -7,20 +7,19 @@ """ import asyncio +from collections.abc import AsyncGenerator from dataclasses import dataclass -from typing import AsyncGenerator, Dict, List, Optional, Union from config import DEFAULT_MODEL_ID, DEFAULT_TEMPERATURE from google.genai import types as gemini_types -from loguru import logger -from pydantic import BaseModel, Field -from stagehand_form_filler import StagehandFormFiller - from line.events import AgentResponse, EndCall, ToolResult from line.nodes.conversation_context import ConversationContext from line.nodes.reasoning import ReasoningNode from line.tools.system_tools import EndCallArgs, end_call from line.utils.gemini_utils import convert_messages_to_gemini +from loguru import logger +from pydantic import BaseModel, Field +from stagehand_form_filler import StagehandFormFiller class RecordFormFieldArgs(BaseModel): @@ -42,7 +41,7 @@ def description() -> str: return "Record a value for a form field that needs to be filled" @staticmethod - def parameters() -> Dict: + def parameters() -> dict: return RecordFormFieldArgs.model_json_schema() @staticmethod @@ -110,12 +109,12 @@ def __init__( # Browser automation self.form_url = form_url - self.stagehand_filler: Optional[StagehandFormFiller] = None + self.stagehand_filler: StagehandFormFiller | None = None # Form state - self.collected_data: Dict[str, str] = {} + self.collected_data: dict[str, str] = {} # Pre-initialize questions so conversation can start immediately - self.questions: List[FormQuestion] = self._create_questions() + self.questions: list[FormQuestion] = self._create_questions() self.current_question_index = 0 # Browser initialization @@ -194,7 +193,7 @@ async def _initialize_browser(self) -> None: finally: self.browser_initializing = False - def _create_questions(self) -> List[FormQuestion]: + def _create_questions(self) -> list[FormQuestion]: """Create questions for the form. Returns: @@ -204,13 +203,22 @@ def _create_questions(self) -> List[FormQuestion]: # This matches form at https://forms.fillout.com/t/34ccsqafUFus form_questions = [ FormQuestion( - field_name="full_name", question="What is your full name?", field_type="text", required=True + field_name="full_name", + question="What is your full name?", + field_type="text", + required=True, ), FormQuestion( - field_name="email", question="What is your email address?", field_type="email", required=True + field_name="email", + question="What is your email address?", + field_type="email", + required=True, ), FormQuestion( - field_name="phone", question="What is your phone number?", field_type="phone", required=False + field_name="phone", + question="What is your phone number?", + field_type="phone", + required=False, ), FormQuestion( field_name="work_eligibility", @@ -324,7 +332,7 @@ async def _submit_form(self) -> bool: logger.error(f"Error submitting form: {e}") return False - def get_current_question(self) -> Optional[FormQuestion]: + def get_current_question(self) -> FormQuestion | None: """Get the current question to ask. Returns: @@ -336,7 +344,7 @@ def get_current_question(self) -> Optional[FormQuestion]: async def process_context( self, context: ConversationContext - ) -> AsyncGenerator[Union[AgentResponse, EndCall], None]: + ) -> AsyncGenerator[AgentResponse | EndCall, None]: """Process conversation context with real-time form filling. Args: @@ -385,7 +393,11 @@ async def process_context( # Check if all questions have been answered # Only submit if we've actually collected data - if not current_question and self.current_question_index > 0 and len(self.collected_data) > 0: + if ( + not current_question + and self.current_question_index > 0 + and len(self.collected_data) > 0 + ): # All questions answered - submit the form logger.info(f"All {self.current_question_index} questions answered") logger.info(f"Collected data for {len(self.collected_data)} fields") @@ -454,7 +466,9 @@ async def process_context( if msg.function_calls: for function_call in msg.function_calls: if function_call.name == RecordFormFieldTool.name(): - field_name = function_call.args.get("field_name", current_question.field_name) + field_name = function_call.args.get( + "field_name", current_question.field_name + ) value = function_call.args.get("value", "") logger.info(f"Recording: {field_name} = {value}") diff --git a/python/cartesia-form-filling/main.py b/python/cartesia-form-filling/main.py index 4d93c92c..553d1445 100644 --- a/python/cartesia-form-filling/main.py +++ b/python/cartesia-form-filling/main.py @@ -10,7 +10,6 @@ from config import SYSTEM_PROMPT from form_filling_node import FormFillingNode from google import genai - from line import Bridge, CallRequest, VoiceAgentApp, VoiceAgentSystem from line.events import UserStartedSpeaking, UserStoppedSpeaking, UserTranscriptionReceived @@ -35,7 +34,9 @@ async def handle_new_call(system: VoiceAgentSystem, call_request: CallRequest) - """ # Create form filling node with browser automation - form_node = FormFillingNode(system_prompt=SYSTEM_PROMPT, gemini_client=gemini_client, form_url=FORM_URL) + form_node = FormFillingNode( + system_prompt=SYSTEM_PROMPT, gemini_client=gemini_client, form_url=FORM_URL + ) # Set up bridge for event handling form_bridge = Bridge(form_node) diff --git a/python/cartesia-form-filling/stagehand_form_filler.py b/python/cartesia-form-filling/stagehand_form_filler.py index 9a775af8..b3d9e65c 100644 --- a/python/cartesia-form-filling/stagehand_form_filler.py +++ b/python/cartesia-form-filling/stagehand_form_filler.py @@ -6,12 +6,12 @@ """ import asyncio +import os from dataclasses import dataclass from enum import Enum -import os -from typing import Dict, List, Optional from loguru import logger + from stagehand import AsyncStagehand @@ -33,7 +33,7 @@ class FormField: field_type: FieldType label: str required: bool = False - options: Optional[List[str]] = None + options: list[str] | None = None class FormFieldMapping: @@ -110,7 +110,7 @@ def __init__(self): ), } - def get_form_field(self, question_id: str) -> Optional[FormField]: + def get_form_field(self, question_id: str) -> FormField | None: """Get the form field mapping for a question ID. Args: @@ -127,11 +127,11 @@ class StagehandFormFiller: def __init__(self, form_url: str): self.form_url = form_url - self.client: Optional[AsyncStagehand] = None + self.client: AsyncStagehand | None = None self.session = None self.is_initialized = False self.field_mapper = FormFieldMapping() - self.collected_data: Dict[str, str] = {} + self.collected_data: dict[str, str] = {} async def initialize(self) -> None: """Initialize Stagehand and open the form. @@ -151,7 +151,9 @@ async def initialize(self) -> None: model_api_key=os.environ.get("GEMINI_API_KEY"), ) - self.session = await self.client.sessions.create(model_name="google/gemini-3-flash-preview") + self.session = await self.client.sessions.create( + model_name="google/gemini-3-flash-preview" + ) logger.info(f"Session started: {self.session.id}") diff --git a/python/exa-browserbase/main.py b/python/exa-browserbase/main.py index 53c4cb53..63764b35 100644 --- a/python/exa-browserbase/main.py +++ b/python/exa-browserbase/main.py @@ -4,12 +4,12 @@ import asyncio import json import os -from pathlib import Path from urllib.parse import urlparse from dotenv import load_dotenv from exa_py import Exa from playwright.async_api import async_playwright + from stagehand import AsyncStagehand # Load environment variables from .env file diff --git a/python/google-trends/main.py b/python/google-trends/main.py index 7144037c..a1f02086 100644 --- a/python/google-trends/main.py +++ b/python/google-trends/main.py @@ -3,10 +3,10 @@ import asyncio import json import os -from typing import List from dotenv import load_dotenv from pydantic import BaseModel, Field + from stagehand import AsyncStagehand @@ -20,7 +20,7 @@ class TrendingKeyword(BaseModel): class TrendingKeywordsList(BaseModel): """Schema for extracting a list of trending keywords.""" - trending_keywords: List[TrendingKeyword] = Field( + trending_keywords: list[TrendingKeyword] = Field( description="List of trending keywords extracted from Google Trends" ) diff --git a/python/sec-filing-research/main.py b/python/sec-filing-research/main.py index a0d47c29..bdafddf2 100644 --- a/python/sec-filing-research/main.py +++ b/python/sec-filing-research/main.py @@ -3,10 +3,10 @@ import asyncio import json import os -from typing import List, Optional from dotenv import load_dotenv from pydantic import BaseModel, Field + from stagehand import AsyncStagehand @@ -24,13 +24,13 @@ class Filing(BaseModel): date: str = Field(description="Filing date in YYYY-MM-DD format") description: str = Field(description="Full description of the filing") accessionNumber: str = Field(description="SEC accession number") - fileNumber: Optional[str] = Field(default=None, description="File/Film number") + fileNumber: str | None = Field(default=None, description="File/Film number") class FilingsList(BaseModel): """Schema for extracting a list of SEC filings.""" - filings: List[Filing] = Field(description="List of SEC filings") + filings: list[Filing] = Field(description="List of SEC filings") def dereference_schema(schema: dict) -> dict: @@ -131,9 +131,7 @@ async def main(): if extracted and isinstance(extracted, dict) and extracted.get("companyName"): company_info = extracted except Exception as error: - print( - f"Could not extract company info, using search query as company name: {error}" - ) + print(f"Could not extract company info, using search query as company name: {error}") # Extract filing metadata from the filings table using structured schema print(f"Extracting the {NUM_FILINGS} most recent filings...") @@ -145,9 +143,7 @@ async def main(): filings_data = filings_response.data.result # Build result object with company info and normalized filing list - filings_list = ( - (filings_data.get("filings") or [])[:NUM_FILINGS] if filings_data else [] - ) + filings_list = (filings_data.get("filings") or [])[:NUM_FILINGS] if filings_data else [] result = { "company": company_info.get("companyName", SEARCH_QUERY), "cik": company_info.get("cik", "Unknown"), @@ -208,4 +204,4 @@ async def main(): print(" - Verify internet connection and SEC website accessibility") print(" - Ensure the search query is valid (company name, ticker, or CIK)") print("Docs: https://docs.stagehand.dev/reference/python/overview") - exit(1) \ No newline at end of file + exit(1) From 69cf92f094fb1f504eb2c068d9b32b7e8d4a277c Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Fri, 6 Feb 2026 10:40:23 -0800 Subject: [PATCH 19/45] init commit (#55) --- .husky/pre-commit | 1 + package.json | 20 +++- pnpm-lock.yaml | 282 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 301 insertions(+), 2 deletions(-) create mode 100644 .husky/pre-commit diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100644 index 00000000..5ee7abd8 --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1 @@ +pnpm exec lint-staged diff --git a/package.json b/package.json index ee1334cb..45ed7ac4 100644 --- a/package.json +++ b/package.json @@ -13,15 +13,31 @@ "lint:python:fix": "uvx ruff check --fix python/", "format:python": "uvx ruff format python/", "format:python:check": "uvx ruff format --check python/", - "check": "npm run format:check && npm run lint && npm run lint:python && npm run format:python:check" + "check": "npm run format:check && npm run lint && npm run lint:python && npm run format:python:check", + "prepare": "husky" }, "devDependencies": { "@typescript-eslint/eslint-plugin": "^8.50.1", "@typescript-eslint/parser": "^8.50.1", "eslint": "^9.39.2", "eslint-config-prettier": "^10.1.8", + "husky": "^9.1.7", + "lint-staged": "^16.2.7", "prettier": "^3.2.5", "typescript-eslint": "^8.50.1" }, - "packageManager": "pnpm@9.0.0" + "packageManager": "pnpm@9.0.0", + "lint-staged": { + "*.{ts,tsx,js,jsx}": [ + "eslint --fix", + "prettier --write" + ], + "*.{json,md,yml,yaml}": [ + "prettier --write" + ], + "*.py": [ + "uvx ruff check --fix", + "uvx ruff format" + ] + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0dc80696..c2e1df4d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,12 @@ importers: eslint-config-prettier: specifier: ^10.1.8 version: 10.1.8(eslint@9.39.2) + husky: + specifier: ^9.1.7 + version: 9.1.7 + lint-staged: + specifier: ^16.2.7 + version: 16.2.7 prettier: specifier: ^3.2.5 version: 3.7.4 @@ -161,10 +167,22 @@ packages: ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} + ansi-escapes@7.3.0: + resolution: {integrity: sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==} + engines: {node: '>=18'} + + ansi-regex@6.2.2: + resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==} + engines: {node: '>=12'} + ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} + ansi-styles@6.2.3: + resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} + engines: {node: '>=12'} + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -177,6 +195,10 @@ packages: brace-expansion@2.0.2: resolution: {integrity: sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==} + braces@3.0.3: + resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} + engines: {node: '>=8'} + callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} @@ -185,6 +207,14 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + cli-cursor@5.0.0: + resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} + engines: {node: '>=18'} + + cli-truncate@5.1.1: + resolution: {integrity: sha512-SroPvNHxUnk+vIW/dOSfNqdy1sPEFkrTk6TUtqLCnBlo3N7TNYYkzzN7uSD6+jVjrdO4+p8nH7JzH6cIvUem6A==} + engines: {node: '>=20'} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -192,6 +222,13 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + + commander@14.0.3: + resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} + engines: {node: '>=20'} + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -211,6 +248,13 @@ packages: deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + emoji-regex@10.6.0: + resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} + + environment@1.1.0: + resolution: {integrity: sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==} + engines: {node: '>=18'} + escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} @@ -263,6 +307,9 @@ packages: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} + eventemitter3@5.0.4: + resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -285,6 +332,10 @@ packages: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} + fill-range@7.1.1: + resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} + engines: {node: '>=8'} + find-up@5.0.0: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} engines: {node: '>=10'} @@ -296,6 +347,10 @@ packages: flatted@3.3.3: resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==} + get-east-asian-width@1.4.0: + resolution: {integrity: sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q==} + engines: {node: '>=18'} + glob-parent@6.0.2: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} @@ -308,6 +363,11 @@ packages: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} + husky@9.1.7: + resolution: {integrity: sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==} + engines: {node: '>=18'} + hasBin: true + ignore@5.3.2: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} @@ -328,10 +388,18 @@ packages: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} + is-fullwidth-code-point@5.1.0: + resolution: {integrity: sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==} + engines: {node: '>=18'} + is-glob@4.0.3: resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} engines: {node: '>=0.10.0'} + is-number@7.0.0: + resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} + engines: {node: '>=0.12.0'} + isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} @@ -355,6 +423,15 @@ packages: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} + lint-staged@16.2.7: + resolution: {integrity: sha512-lDIj4RnYmK7/kXMya+qJsmkRFkGolciXjrsZ6PC25GdTfWOAWetR0ZbsNXRAj1EHHImRSalc+whZFg56F5DVow==} + engines: {node: '>=20.17'} + hasBin: true + + listr2@9.0.5: + resolution: {integrity: sha512-ME4Fb83LgEgwNw96RKNvKV4VTLuXfoKudAmm2lP8Kk87KaMK0/Xrx/aAkMWmT8mDb+3MlFDspfbCs7adjRxA2g==} + engines: {node: '>=20.0.0'} + locate-path@6.0.0: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} @@ -362,6 +439,18 @@ packages: lodash.merge@4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} + log-update@6.1.0: + resolution: {integrity: sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==} + engines: {node: '>=18'} + + micromatch@4.0.8: + resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} + engines: {node: '>=8.6'} + + mimic-function@5.0.1: + resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==} + engines: {node: '>=18'} + minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} @@ -372,9 +461,17 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + nano-spawn@2.0.0: + resolution: {integrity: sha512-tacvGzUY5o2D8CBh2rrwxyNojUsZNU2zjNTzKQrkgGJQTbGAfArVWXSKMBokBeeg6C7OLRGUEyoFlYbfeWQIqw==} + engines: {node: '>=20.17'} + natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + onetime@7.0.0: + resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} + engines: {node: '>=18'} + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -399,10 +496,19 @@ packages: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} + picomatch@2.3.1: + resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} + engines: {node: '>=8.6'} + picomatch@4.0.3: resolution: {integrity: sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==} engines: {node: '>=12'} + pidtree@0.6.0: + resolution: {integrity: sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g==} + engines: {node: '>=0.10'} + hasBin: true + prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} @@ -420,6 +526,13 @@ packages: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} + restore-cursor@5.1.0: + resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} + engines: {node: '>=18'} + + rfdc@1.4.1: + resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + semver@7.7.3: resolution: {integrity: sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==} engines: {node: '>=10'} @@ -433,6 +546,30 @@ packages: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + + slice-ansi@7.1.2: + resolution: {integrity: sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w==} + engines: {node: '>=18'} + + string-argv@0.3.2: + resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==} + engines: {node: '>=0.6.19'} + + string-width@7.2.0: + resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} + engines: {node: '>=18'} + + string-width@8.1.1: + resolution: {integrity: sha512-KpqHIdDL9KwYk22wEOg/VIqYbrnLeSApsKT/bSj6Ez7pn3CftUiLAv2Lccpq1ALcpLV9UX1Ppn92npZWu2w/aw==} + engines: {node: '>=20'} + + strip-ansi@7.1.2: + resolution: {integrity: sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==} + engines: {node: '>=12'} + strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} @@ -445,6 +582,10 @@ packages: resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} engines: {node: '>=12.0.0'} + to-regex-range@5.0.1: + resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} + engines: {node: '>=8.0'} + ts-api-utils@2.1.0: resolution: {integrity: sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ==} engines: {node: '>=18.12'} @@ -479,6 +620,15 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} + wrap-ansi@9.0.2: + resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} + engines: {node: '>=18'} + + yaml@2.8.2: + resolution: {integrity: sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==} + engines: {node: '>= 14.6'} + hasBin: true + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -650,10 +800,18 @@ snapshots: json-schema-traverse: 0.4.1 uri-js: 4.4.1 + ansi-escapes@7.3.0: + dependencies: + environment: 1.1.0 + + ansi-regex@6.2.2: {} + ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 + ansi-styles@6.2.3: {} + argparse@2.0.1: {} balanced-match@1.0.2: {} @@ -667,6 +825,10 @@ snapshots: dependencies: balanced-match: 1.0.2 + braces@3.0.3: + dependencies: + fill-range: 7.1.1 + callsites@3.1.0: {} chalk@4.1.2: @@ -674,12 +836,25 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 + cli-cursor@5.0.0: + dependencies: + restore-cursor: 5.1.0 + + cli-truncate@5.1.1: + dependencies: + slice-ansi: 7.1.2 + string-width: 8.1.1 + color-convert@2.0.1: dependencies: color-name: 1.1.4 color-name@1.1.4: {} + colorette@2.0.20: {} + + commander@14.0.3: {} + concat-map@0.0.1: {} cross-spawn@7.0.6: @@ -694,6 +869,10 @@ snapshots: deep-is@0.1.4: {} + emoji-regex@10.6.0: {} + + environment@1.1.0: {} + escape-string-regexp@4.0.0: {} eslint-config-prettier@10.1.8(eslint@9.39.2): @@ -766,6 +945,8 @@ snapshots: esutils@2.0.3: {} + eventemitter3@5.0.4: {} + fast-deep-equal@3.1.3: {} fast-json-stable-stringify@2.1.0: {} @@ -780,6 +961,10 @@ snapshots: dependencies: flat-cache: 4.0.1 + fill-range@7.1.1: + dependencies: + to-regex-range: 5.0.1 + find-up@5.0.0: dependencies: locate-path: 6.0.0 @@ -792,6 +977,8 @@ snapshots: flatted@3.3.3: {} + get-east-asian-width@1.4.0: {} + glob-parent@6.0.2: dependencies: is-glob: 4.0.3 @@ -800,6 +987,8 @@ snapshots: has-flag@4.0.0: {} + husky@9.1.7: {} + ignore@5.3.2: {} ignore@7.0.5: {} @@ -813,10 +1002,16 @@ snapshots: is-extglob@2.1.1: {} + is-fullwidth-code-point@5.1.0: + dependencies: + get-east-asian-width: 1.4.0 + is-glob@4.0.3: dependencies: is-extglob: 2.1.1 + is-number@7.0.0: {} + isexe@2.0.0: {} js-yaml@4.1.1: @@ -838,12 +1033,46 @@ snapshots: prelude-ls: 1.2.1 type-check: 0.4.0 + lint-staged@16.2.7: + dependencies: + commander: 14.0.3 + listr2: 9.0.5 + micromatch: 4.0.8 + nano-spawn: 2.0.0 + pidtree: 0.6.0 + string-argv: 0.3.2 + yaml: 2.8.2 + + listr2@9.0.5: + dependencies: + cli-truncate: 5.1.1 + colorette: 2.0.20 + eventemitter3: 5.0.4 + log-update: 6.1.0 + rfdc: 1.4.1 + wrap-ansi: 9.0.2 + locate-path@6.0.0: dependencies: p-locate: 5.0.0 lodash.merge@4.6.2: {} + log-update@6.1.0: + dependencies: + ansi-escapes: 7.3.0 + cli-cursor: 5.0.0 + slice-ansi: 7.1.2 + strip-ansi: 7.1.2 + wrap-ansi: 9.0.2 + + micromatch@4.0.8: + dependencies: + braces: 3.0.3 + picomatch: 2.3.1 + + mimic-function@5.0.1: {} + minimatch@3.1.2: dependencies: brace-expansion: 1.1.12 @@ -854,8 +1083,14 @@ snapshots: ms@2.1.3: {} + nano-spawn@2.0.0: {} + natural-compare@1.4.0: {} + onetime@7.0.0: + dependencies: + mimic-function: 5.0.1 + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -881,8 +1116,12 @@ snapshots: path-key@3.1.1: {} + picomatch@2.3.1: {} + picomatch@4.0.3: {} + pidtree@0.6.0: {} + prelude-ls@1.2.1: {} prettier@3.7.4: {} @@ -891,6 +1130,13 @@ snapshots: resolve-from@4.0.0: {} + restore-cursor@5.1.0: + dependencies: + onetime: 7.0.0 + signal-exit: 4.1.0 + + rfdc@1.4.1: {} + semver@7.7.3: {} shebang-command@2.0.0: @@ -899,6 +1145,30 @@ snapshots: shebang-regex@3.0.0: {} + signal-exit@4.1.0: {} + + slice-ansi@7.1.2: + dependencies: + ansi-styles: 6.2.3 + is-fullwidth-code-point: 5.1.0 + + string-argv@0.3.2: {} + + string-width@7.2.0: + dependencies: + emoji-regex: 10.6.0 + get-east-asian-width: 1.4.0 + strip-ansi: 7.1.2 + + string-width@8.1.1: + dependencies: + get-east-asian-width: 1.4.0 + strip-ansi: 7.1.2 + + strip-ansi@7.1.2: + dependencies: + ansi-regex: 6.2.2 + strip-json-comments@3.1.1: {} supports-color@7.2.0: @@ -910,6 +1180,10 @@ snapshots: fdir: 6.5.0(picomatch@4.0.3) picomatch: 4.0.3 + to-regex-range@5.0.1: + dependencies: + is-number: 7.0.0 + ts-api-utils@2.1.0(typescript@5.9.3): dependencies: typescript: 5.9.3 @@ -941,4 +1215,12 @@ snapshots: word-wrap@1.2.5: {} + wrap-ansi@9.0.2: + dependencies: + ansi-styles: 6.2.3 + string-width: 7.2.0 + strip-ansi: 7.1.2 + + yaml@2.8.2: {} + yocto-queue@0.1.0: {} From d636d28fc91ba57b2c526eb2f573c9860b405d28 Mon Sep 17 00:00:00 2001 From: Jay Sahnan <79011389+jay-sahnan@users.noreply.github.com> Date: Tue, 10 Feb 2026 08:14:37 -0800 Subject: [PATCH 20/45] jaysahnan/gro-982-extend-browserbase-template (#56) * init commit * addressing PR comments * pin versions --- python/extend-browserbase/.env.example | 13 + python/extend-browserbase/README.md | 81 +++ python/extend-browserbase/main.py | 641 +++++++++++++++++++++ python/extend-browserbase/pyproject.toml | 34 ++ typescript/extend-browserbase/.env.example | 12 + typescript/extend-browserbase/README.md | 77 +++ typescript/extend-browserbase/index.ts | 534 +++++++++++++++++ typescript/extend-browserbase/package.json | 24 + 8 files changed, 1416 insertions(+) create mode 100644 python/extend-browserbase/.env.example create mode 100644 python/extend-browserbase/README.md create mode 100644 python/extend-browserbase/main.py create mode 100644 python/extend-browserbase/pyproject.toml create mode 100644 typescript/extend-browserbase/.env.example create mode 100644 typescript/extend-browserbase/README.md create mode 100644 typescript/extend-browserbase/index.ts create mode 100644 typescript/extend-browserbase/package.json diff --git a/python/extend-browserbase/.env.example b/python/extend-browserbase/.env.example new file mode 100644 index 00000000..fad95662 --- /dev/null +++ b/python/extend-browserbase/.env.example @@ -0,0 +1,13 @@ +# Browserbase credentials (required) +# Get these from https://www.browserbase.com/settings +BROWSERBASE_PROJECT_ID=your_browserbase_project_id +BROWSERBASE_API_KEY=your_browserbase_api_key + +# Google API key for Gemini model (required for Stagehand) +GOOGLE_API_KEY=your_google_api_key + +# Extend AI (optional – enables receipt parsing; omit to only download receipts) +EXTEND_API_KEY=your_extend_api_key + +# Optional: set after first run to reuse the created processor +# EXTEND_PROCESSOR_ID=your_processor_id diff --git a/python/extend-browserbase/README.md b/python/extend-browserbase/README.md new file mode 100644 index 00000000..15d558fb --- /dev/null +++ b/python/extend-browserbase/README.md @@ -0,0 +1,81 @@ +# Stagehand + Browserbase + Extend: Download Expense Receipts and Parse with Extend AI + +## AT A GLANCE + +- **Goal**: Automate downloading receipts from an expense portal and extract structured receipt data using AI-powered document parsing. +- **Pattern Template**: Demonstrates the integration pattern of Browserbase (browser automation + download capture) + Extend AI (schema-based document extraction). +- **Workflow**: Stagehand navigates the expense portal and clicks each receipt's download link; Browserbase captures downloads. The script polls for the session's download ZIP, extracts files, then optionally sends them to Extend for structured extraction (vendor, date, totals, line items, etc.). +- **Download Handling**: Implements retry/polling around Browserbase's Session Downloads API until the ZIP is available. +- **Structured Extraction**: Extend EXTRACT processor with a receipt JSON schema; results written to `output/results/receipts.json` and `receipts.csv`. +- Docs → [Browserbase Downloads](https://docs.browserbase.com/features/downloads) | [Extend AI](https://docs.extend.app) + +## GLOSSARY + +- **act**: perform UI actions from natural language prompts (click, scroll, navigate) + Docs → https://docs.stagehand.dev/basics/act +- **observe**: find and return interactive elements on the page matching a description, without performing actions. Used here to locate all individual download buttons before clicking them. + Docs → https://docs.stagehand.dev/basics/observe +- **Browserbase Downloads**: When files are downloaded during a browser session, Browserbase captures and stores them. Files are retrieved via the Session Downloads API as a ZIP archive. + Docs → https://docs.browserbase.com/features/downloads +- **Extend EXTRACT processor**: A configurable document extraction pipeline that parses files against a JSON schema and returns structured data. Processors are reusable and persist across runs. + Docs → https://docs.extend.app +- **Download polling**: Browserbase syncs downloads in real-time; the script retries every 2 seconds until the ZIP is available or a timeout is reached. + +## QUICKSTART + +1. cd python/extend-browserbase +2. cp .env.example .env +3. Add required API keys to .env: + - `BROWSERBASE_PROJECT_ID` + - `BROWSERBASE_API_KEY` + - `GOOGLE_API_KEY` + - `EXTEND_API_KEY` (optional — enables receipt parsing) +4. Run the script: + + ```bash + uv run python main.py + ``` + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase and opens the live view link +- Navigates to the expense portal and finds all per-receipt download links via observe +- Clicks each download button; Browserbase captures files +- After closing the session, polls for the session's download ZIP and extracts to `output/documents/` +- If `EXTEND_API_KEY` is set: creates/uses an Extend "Receipt Extractor" processor, uploads each file, runs extraction, writes `output/results/receipts.json` and `receipts.csv` +- Opens the Extend dashboard runs page in your browser for review +- Closes session cleanly + +## COMMON PITFALLS + +- "ModuleNotFoundError": ensure you're running with `uv run python main.py` so dependencies are installed automatically from pyproject.toml +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY +- Download timeout: increase `retry_for_seconds` parameter in `save_downloads_with_retry` if downloads take longer than 60 seconds +- Empty ZIP file: ensure downloads were actually triggered (check live view link to debug) +- Rate limiting on Extend: the script retries with exponential backoff on 429 errors, but very large batches may need the batch size reduced from 9 +- Find more information on your Browserbase dashboard → https://www.browserbase.com/sign-in + +## USE CASES + +• Expense automation: Download receipts from expense portal and extract vendor, date, totals, and line items for accounting systems. +• Document batch processing: Collect files from web portals and run structured extraction across all of them with a single script. +• Receipt digitization: Convert paper/PDF receipts into structured JSON and CSV for import into ERP, bookkeeping, or reimbursement tools. + +## NEXT STEPS + +• Parameterize the portal URL: Accept the expense portal URL from env or CLI to support different receipt sources. +• Custom schemas: Modify `RECEIPT_EXTRACTION_CONFIG` to extract different document types (invoices, W-2s, contracts) by changing the JSON schema. +• Add validation: Compare extracted totals against line item sums to flag discrepancies or incomplete extractions. +• Scheduled runs: Deploy on cron/Lambda to periodically check for new receipts and process them automatically. + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +📚 Python SDK: https://docs.stagehand.dev/v3/sdk/python +📚 Browserbase Downloads: https://docs.browserbase.com/features/downloads +📚 Extend AI: https://docs.extend.app +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/python/extend-browserbase/main.py b/python/extend-browserbase/main.py new file mode 100644 index 00000000..8caea1ae --- /dev/null +++ b/python/extend-browserbase/main.py @@ -0,0 +1,641 @@ +# Stagehand + Browserbase + Extend: Download Expense Receipts and Parse with Extend AI +# See README.md for full documentation + +import asyncio +import csv +import json +import os +import re +import webbrowser +import zipfile +from pathlib import Path + +from browserbase import APIStatusError, Browserbase +from dotenv import load_dotenv +from extend_ai import Extend +from stagehand import AsyncStagehand + +# Load environment variables from .env file +# Required: BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, GOOGLE_API_KEY +# Optional: EXTEND_API_KEY, EXTEND_PROCESSOR_ID +load_dotenv() + + +# Receipt extraction config for Extend AI +# Uses extraction_light base processor with parse_performance engine for low latency +RECEIPT_EXTRACTION_CONFIG = { + "type": "EXTRACT", + "baseProcessor": "extraction_light", + "baseVersion": "3.4.0", + "parser": { + "engine": "parse_performance", + "target": "markdown", + "blockOptions": { + "text": { + "agentic": {"enabled": False}, + "signatureDetectionEnabled": False, + }, + "tables": { + "agentic": {"enabled": False}, + "targetFormat": "markdown", + "cellBlocksEnabled": False, + "tableHeaderContinuationEnabled": False, + }, + "figures": { + "enabled": False, + "figureImageClippingEnabled": False, + }, + }, + "engineVersion": "1.0.1", + "advancedOptions": { + "engine": "parse_performance", + "agenticOcrEnabled": False, + "pageBreaksEnabled": True, + "pageRotationEnabled": False, + "verticalGroupingThreshold": 1, + }, + "chunkingStrategy": {"type": "document"}, + }, + "schema": { + "type": "object", + "required": [ + "vendor_name", + "receipt_date", + "receipt_number", + "total_amount", + "subtotal_amount", + "tax_amount", + "line_items", + "payment_method", + ], + "properties": { + "vendor_name": { + "type": ["string", "null"], + "description": "The name of the merchant or vendor on the receipt.", + }, + "receipt_date": { + "type": ["string", "null"], + "description": "The date of the transaction shown on the receipt.", + "extend:type": "date", + }, + "receipt_number": { + "type": ["string", "null"], + "description": "The receipt or transaction number, if present.", + }, + "total_amount": { + "type": "object", + "required": ["amount", "iso_4217_currency_code"], + "properties": { + "amount": {"type": ["number", "null"]}, + "iso_4217_currency_code": {"type": ["string", "null"]}, + }, + "description": "The total amount paid on the receipt.", + "extend:type": "currency", + "additionalProperties": False, + }, + "subtotal_amount": { + "type": "object", + "required": ["amount", "iso_4217_currency_code"], + "properties": { + "amount": {"type": ["number", "null"]}, + "iso_4217_currency_code": {"type": ["string", "null"]}, + }, + "description": "The subtotal before tax, if shown.", + "extend:type": "currency", + "additionalProperties": False, + }, + "tax_amount": { + "type": "object", + "required": ["amount", "iso_4217_currency_code"], + "properties": { + "amount": {"type": ["number", "null"]}, + "iso_4217_currency_code": {"type": ["string", "null"]}, + }, + "description": "The tax amount on the receipt.", + "extend:type": "currency", + "additionalProperties": False, + }, + "line_items": { + "type": "array", + "items": { + "type": "object", + "required": ["description", "quantity", "unit_price", "amount"], + "properties": { + "description": { + "type": ["string", "null"], + "description": "Description of the item purchased.", + }, + "quantity": { + "type": ["number", "null"], + "description": "Quantity of the item, if shown.", + }, + "unit_price": { + "type": ["number", "null"], + "description": "Price per unit, if shown.", + }, + "amount": { + "type": ["number", "null"], + "description": "Total amount for this line item.", + }, + }, + "additionalProperties": False, + }, + "description": "Individual items on the receipt.", + }, + "payment_method": { + "type": ["string", "null"], + "description": "The payment method used (e.g., cash, credit card, etc.).", + }, + }, + "additionalProperties": False, + }, + "advancedOptions": { + "advancedMultimodalEnabled": False, + "citationsEnabled": True, + "arrayCitationStrategy": "item", + "pageRanges": [], + "chunkingOptions": {}, + "advancedFigureParsingEnabled": True, + }, +} + + +def open_in_browser(url: str) -> None: + """Opens a URL in the default browser for live view and dashboard links.""" + try: + webbrowser.open(url) + except Exception: + print(f"Could not auto-open: {url}") + + +# Polls Browserbase API for completed downloads with retry logic +async def save_downloads_with_retry( + bb: Browserbase, session_id: str, retry_for_seconds: int = 60 +) -> int: + """ + Polls Browserbase API for downloads with timeout handling. + + Browserbase stores downloaded files during a session and makes them available + via API. Files may take a few seconds to process, so this function implements + retry logic to wait for downloads to be ready before retrieving them. + + Args: + bb: Browserbase client instance for API calls + session_id: The Browserbase session ID to retrieve downloads from + retry_for_seconds: Maximum time to wait for downloads (default: 60 seconds) + + Returns: + int: The size of the downloaded ZIP file in bytes + + Raises: + TimeoutError: If downloads aren't ready within the specified timeout + """ + print(f"Waiting up to {retry_for_seconds} seconds for downloads to complete...") + + # Track elapsed time to implement timeout without using threading timers + start_time = asyncio.get_event_loop().time() + timeout = retry_for_seconds + + while True: + elapsed = asyncio.get_event_loop().time() - start_time + + # Check if we've exceeded the timeout period + if elapsed >= timeout: + raise TimeoutError("Download timeout exceeded") + + try: + print("Checking for downloads...") + # Fetch downloads from Browserbase API and save to disk when ready + # Use asyncio.to_thread for synchronous Browserbase SDK calls + # This prevents blocking the event loop while waiting for API responses + response = await asyncio.to_thread(bb.sessions.downloads.list, session_id) + download_buffer = await asyncio.to_thread(response.read) + + # Save downloads to disk when file size indicates content is available + # Empty zip files are ~22 bytes, so require at least 100 bytes for real content + if len(download_buffer) > 100: + print(f"Downloads ready! File size: {len(download_buffer)} bytes") + # Save the ZIP file containing all downloaded receipts to disk + with open("downloaded_files.zip", "wb") as f: + f.write(download_buffer) + print("Files saved as: downloaded_files.zip") + return len(download_buffer) + else: + print("Downloads not ready yet, retrying...") + except APIStatusError as e: + # Handle 404 (session not found) gracefully + if e.status_code == 404: + print("Session not found, returning empty result") + return 0 + print(f"Error fetching downloads: {e}") + raise + except Exception as e: + # HTML error response - session may not be ready yet, keep retrying + error_message = str(e) + if "Unexpected token '<'" in error_message or " list[str]: + """ + Extract receipt files from a ZIP archive. + + Args: + zip_path: Path to the ZIP file containing receipts + output_dir: Directory to extract files to (default: "output/documents") + + Returns: + list[str]: Paths to all extracted files + + Raises: + ValueError: If no files are found in the ZIP + """ + print(f"Extracting files from {zip_path}...") + + # Create output directories for documents and results if they don't exist + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + Path("output/results").mkdir(parents=True, exist_ok=True) + + extracted_files: list[str] = [] + + with zipfile.ZipFile(zip_path, "r") as zip_ref: + # Open zip file and iterate over entries + entries = [e for e in zip_ref.namelist() if not e.endswith("/")] + + if len(entries) == 0: + raise ValueError("No files found in the downloaded zip") + + # Extract all non-directory entries and collect file paths + for entry in entries: + zip_ref.extract(entry, output_dir) + extracted_path = output_path / entry + print(f"Extracted: {extracted_path}") + extracted_files.append(str(extracted_path)) + + print(f"\nTotal files extracted: {len(extracted_files)}") + return extracted_files + + +# Gets existing Extend processor or creates a new one, saving the ID to .env for reuse. +# This avoids recreating the processor on every run. +async def get_or_create_processor(extend_client: Extend) -> str: + """ + Get an existing Extend processor or create a new one. + + Checks for EXTEND_PROCESSOR_ID in environment. If not found, creates a new + 'Receipt Extractor' processor and saves the ID to .env for future runs. + + Args: + extend_client: Extend client instance for API calls + + Returns: + str: The processor ID + """ + # Check if a processor ID already exists in environment + existing_id = os.environ.get("EXTEND_PROCESSOR_ID") + if existing_id and existing_id != "YOUR_EXTEND_PROCESSOR_ID_HERE": + print(f"Using existing processor: {existing_id}") + return existing_id + + # Create a new Receipt Extractor processor via the Extend API + print("No EXTEND_PROCESSOR_ID found. Creating 'Receipt Extractor' processor...") + + response = await asyncio.to_thread( + extend_client.processor.create, + name="Receipt Extractor", + type="EXTRACT", + config=RECEIPT_EXTRACTION_CONFIG, + ) + + processor_id = response.processor.id + print(f"Created processor: {processor_id}") + print(f" View in dashboard: https://dashboard.extend.app/studio/processors/{processor_id}") + + # Persist the processor ID to .env so we don't recreate on next run + env_path = Path(".env") + if env_path.exists(): + env_content = env_path.read_text() + if "EXTEND_PROCESSOR_ID=" in env_content: + env_content = re.sub( + r"EXTEND_PROCESSOR_ID=.*", + f"EXTEND_PROCESSOR_ID={processor_id}", + env_content, + ) + else: + env_content += f"\nEXTEND_PROCESSOR_ID={processor_id}\n" + env_path.write_text(env_content) + else: + with open(env_path, "a") as f: + f.write(f"EXTEND_PROCESSOR_ID={processor_id}\n") + print(" Saved EXTEND_PROCESSOR_ID to .env for future runs.") + + return processor_id + + +# Uploads receipt files to Extend AI, runs extraction, and saves results as JSON and CSV +async def parse_receipts_with_extend(file_paths: list[str]) -> None: + """ + Upload receipt files to Extend AI, run extraction, and save results. + + Initializes the Extend client, gets or creates a processor, uploads each file, + runs synchronous extraction, and saves results as JSON and CSV. + + Args: + file_paths: List of file paths to receipt documents + """ + # Skip parsing if Extend API key is not configured + extend_api_key = os.environ.get("EXTEND_API_KEY") + if not extend_api_key or extend_api_key == "YOUR_EXTEND_API_KEY_HERE": + print("\nWARNING: EXTEND_API_KEY not configured. Skipping receipt parsing.") + print(" Add your Extend API key to .env to enable automatic receipt parsing.") + return + + print("\n=== Parsing Receipts with Extend AI ===\n") + + # Initialize Extend AI client and get or create the receipt processor + extend_client = Extend(token=extend_api_key) + processor_id = await get_or_create_processor(extend_client) + + print(f"Processing {len(file_paths)} receipts...") + runs_url = f"https://dashboard.extend.app/studio/processors/{processor_id}?tab=Runs" + print(f"View all runs: {runs_url}\n") + open_in_browser(runs_url) + + # Process all files with retry on rate limiting (429 errors) + results: list[dict] = [] + + # Uploads a single file to Extend and runs extraction with exponential backoff retry + async def process_with_retry(file_path: str, max_retries: int = 3) -> dict: + file_name = Path(file_path).name + + for attempt in range(1, max_retries + 1): + try: + # Upload the file to Extend (multipart form upload) + # Pass as (filename, bytes) tuple so Extend knows the file name + with open(file_path, "rb") as f: + file_bytes = f.read() + upload_response = await asyncio.to_thread( + extend_client.file.upload, + file=(file_name, file_bytes), + ) + file_id = upload_response.file.id + + # Run extraction on the uploaded file + extraction_response = await asyncio.to_thread( + extend_client.processor_run.create, + processor_id=processor_id, + file={"fileId": file_id}, + sync=True, + config=RECEIPT_EXTRACTION_CONFIG, + ) + + parsed_data = extraction_response.processor_run + run_id = parsed_data.id + run_url = ( + f"https://dashboard.extend.app/studio/processors/{processor_id}/runs/{run_id}" + ) + print(f" Parsed {file_name}") + print(f" -> {run_url}") + + # Convert to dict for JSON serialization + data = parsed_data + if hasattr(parsed_data, "to_dict"): + data = parsed_data.to_dict() + elif hasattr(parsed_data, "dict"): + data = parsed_data.dict() + + return { + "file": file_name, + "runId": run_id, + "runUrl": run_url, + "data": data, + } + except Exception as error: + error_msg = str(error) + is_retryable = ( + "429" in error_msg or "rate" in error_msg or "disturbed or locked" in error_msg + ) + + if is_retryable and attempt < max_retries: + delay = 2**attempt # Exponential backoff: 2s, 4s, 8s + print( + f" Rate limited on {file_name}, retrying in {delay}s " + f"(attempt {attempt}/{max_retries})" + ) + await asyncio.sleep(delay) + else: + print(f" Failed to parse {file_name}: {error_msg}") + return {"file": file_name, "data": {"error": error_msg}} + + return {"file": file_name, "data": {"error": "Max retries exceeded"}} + + # Process in batches of 9 to balance speed and reliability + for i in range(0, len(file_paths), 9): + batch = file_paths[i : i + 9] + batch_results = await asyncio.gather(*[process_with_retry(fp) for fp in batch]) + results.extend(batch_results) + + # Save results to JSON + json_path = "output/results/receipts.json" + with open(json_path, "w") as f: + json.dump(results, f, indent=2, default=str) + print(f"\nSaved JSON: {json_path}") + + # Convert results to CSV for easy viewing in spreadsheet tools + csv_path = "output/results/receipts.csv" + with open(csv_path, "w", newline="") as f: + writer = csv.writer(f, quoting=csv.QUOTE_ALL) + writer.writerow( + [ + "file", + "run_url", + "vendor_name", + "receipt_date", + "receipt_number", + "total_amount", + "currency", + "subtotal", + "tax", + "payment_method", + "line_items_count", + ] + ) + + # Build CSV rows from extraction results + for result in results: + data = result.get("data", {}) + output = {} + if isinstance(data, dict): + output = data.get("output", {}).get("value", {}) or {} + + writer.writerow( + [ + result.get("file", ""), + result.get("runUrl", ""), + output.get("vendor_name", ""), + output.get("receipt_date", ""), + output.get("receipt_number", ""), + (output.get("total_amount") or {}).get("amount", ""), + (output.get("total_amount") or {}).get("iso_4217_currency_code", ""), + (output.get("subtotal_amount") or {}).get("amount", ""), + (output.get("tax_amount") or {}).get("amount", ""), + output.get("payment_method", ""), + len(output.get("line_items", [])) + if isinstance(output.get("line_items"), list) + else 0, + ] + ) + + print(f"Saved CSV: {csv_path}") + print(f"\nView all runs: {runs_url}") + + +async def main() -> None: + """ + Main application entry point. + + Orchestrates the entire receipt download and extraction automation process: + 1. Initializes Browserbase and Stagehand clients + 2. Navigates to the expense portal + 3. Finds and clicks all individual receipt download buttons + 4. Retrieves downloads from Browserbase and extracts files + 5. Optionally parses receipts with Extend AI for structured data extraction + """ + print("Starting Expense Receipt Downloader...\n") + + browserbase_api_key = os.environ.get("BROWSERBASE_API_KEY") + browserbase_project_id = os.environ.get("BROWSERBASE_PROJECT_ID") + google_api_key = os.environ.get("GOOGLE_API_KEY") + + if not browserbase_api_key or not browserbase_project_id: + raise ValueError("BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID are required") + + # Initialize Browserbase SDK for session management and download retrieval + bb = Browserbase(api_key=browserbase_api_key) + + # Initialize AsyncStagehand client (v3 BYOB architecture) + client = AsyncStagehand( + browserbase_api_key=browserbase_api_key, + browserbase_project_id=browserbase_project_id, + model_api_key=google_api_key, + ) + + # Start a Stagehand session (returns a response with session_id) + start_response = await client.sessions.start( + model_name="google/gemini-2.5-flash", + ) + session_id = start_response.data.session_id + print(f"Stagehand session started: {session_id}") + + try: + # Get live view URL for monitoring browser session in real-time + # Use asyncio.to_thread for synchronous Browserbase SDK calls + live_view_links = await asyncio.to_thread(bb.sessions.debug, session_id) + live_view_link = live_view_links.debuggerFullscreenUrl + print(f"Live View Link: {live_view_link}") + open_in_browser(live_view_link) + + # Navigate to the expense portal where receipts are hosted + print("\nNavigating to expense portal...") + await client.sessions.navigate( + id=session_id, + url="https://v0-reimburse-me-expense-portal.vercel.app/", + ) + + # Use observe to find all individual download buttons (not the Download All button) + print("\nFinding all individual download buttons...") + observe_response = await client.sessions.observe( + id=session_id, + instruction="Find all the small Download links on individual receipt cards.", + ) + download_buttons = observe_response.data.result + + # Click each download button using observe -> act pattern + # Pass the observed action directly to act for precise element targeting + success_count = 0 + for i, action in enumerate(download_buttons): + print(f"Downloading receipt {i + 1}/{len(download_buttons)}...") + + # Convert observed action to dict for passing to act + action_dict = ( + action.to_dict(exclude_none=True) if hasattr(action, "to_dict") else action + ) + + try: + await client.sessions.act(id=session_id, input=action_dict) + success_count += 1 + except Exception: + # If click fails, scroll element into view and retry + print(f" Could not click download button {i + 1}, trying to scroll and retry...") + try: + await client.sessions.act(id=session_id, input="Scroll down slightly") + await client.sessions.act(id=session_id, input=action_dict) + success_count += 1 + except Exception: + print(f" Skipping receipt {i + 1}") + + # Scroll down periodically to ensure elements are in view + if (i + 1) % 4 == 0 and (i + 1) < len(download_buttons): + await client.sessions.act(id=session_id, input="Scroll down slightly") + + print(f"\nDownload clicks completed! ({success_count}/{len(download_buttons)} successful)") + + # End the Stagehand session before fetching downloads + await client.sessions.end(id=session_id) + print("Session closed successfully") + + # Wait for session to finalize downloads before polling + await asyncio.sleep(2) + + # Retrieve all downloads triggered during this session from Browserbase API + print("\nRetrieving downloads from Browserbase...") + download_size = await save_downloads_with_retry(bb, session_id, 60) + + if download_size > 0: + # Extract receipt files from downloaded zip archive + extracted_files = extract_files_from_zip("downloaded_files.zip") + + print("\n=== Download Summary ===") + print(f"Total files downloaded: {len(extracted_files)}") + print("Files saved to: ./output/documents/") + + # Parse downloaded receipts with Extend AI for structured data extraction + await parse_receipts_with_extend(extracted_files) + else: + print("No downloads were captured") + + print("\nExpense receipt download complete!") + + except Exception as error: + print(f"Error during automation: {error}") + try: + await client.sessions.end(id=session_id) + except Exception: + # Ignore close errors during cleanup + pass + raise + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Application error: {err}") + print("Common issues:") + print( + " - Check .env file has BROWSERBASE_PROJECT_ID, " + "BROWSERBASE_API_KEY, and GOOGLE_API_KEY" + ) + print(" - Add EXTEND_API_KEY to .env to enable receipt parsing with Extend AI") + print(" - Verify internet connection and expense portal accessibility") + print("Docs: https://docs.stagehand.dev/v3/first-steps/introduction") + exit(1) diff --git a/python/extend-browserbase/pyproject.toml b/python/extend-browserbase/pyproject.toml new file mode 100644 index 00000000..29b4f903 --- /dev/null +++ b/python/extend-browserbase/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "extend-browserbase" +version = "0.1.0" +description = "Download expense receipts and parse with Extend AI using Stagehand and Browserbase" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "browserbase>=1.4.0", + "extend-ai>=0.0.21", + "python-dotenv>=1.2.1", + "stagehand>=3.5.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] diff --git a/typescript/extend-browserbase/.env.example b/typescript/extend-browserbase/.env.example new file mode 100644 index 00000000..b249aafa --- /dev/null +++ b/typescript/extend-browserbase/.env.example @@ -0,0 +1,12 @@ +# Browserbase Configuration +BROWSERBASE_PROJECT_ID=your_browserbase_project_id +BROWSERBASE_API_KEY=your_browserbase_api_key + +# Google API key for Gemini model (Stagehand) +GOOGLE_API_KEY=your_google_api_key + +# Extend AI (optional – enables receipt parsing; omit to only download receipts) +EXTEND_API_KEY=your_extend_api_key + +# Optional: set after first run to reuse the created processor +# EXTEND_PROCESSOR_ID=your_processor_id diff --git a/typescript/extend-browserbase/README.md b/typescript/extend-browserbase/README.md new file mode 100644 index 00000000..3ec84395 --- /dev/null +++ b/typescript/extend-browserbase/README.md @@ -0,0 +1,77 @@ +# Stagehand + Browserbase + Extend: Download Expense Receipts and Parse with Extend AI + +## AT A GLANCE + +- **Goal**: Automate downloading receipts from an expense portal and extract structured receipt data using AI-powered document parsing. +- **Pattern Template**: Demonstrates the integration pattern of Browserbase (browser automation + download capture) + Extend AI (schema-based document extraction). +- **Workflow**: Stagehand navigates the expense portal and clicks each receipt's download link; Browserbase captures downloads. The script polls for the session's download ZIP, extracts files, then optionally sends them to Extend for structured extraction (vendor, date, totals, line items, etc.). +- **Download Handling**: Implements retry/polling around Browserbase's Session Downloads API until the ZIP is available. +- **Structured Extraction**: Extend EXTRACT processor with a receipt JSON schema; results written to `output/results/receipts.json` and `receipts.csv`. +- Docs → [Browserbase Downloads](https://docs.browserbase.com/features/downloads) | [Extend AI](https://docs.extend.app) + +## GLOSSARY + +- **act**: perform UI actions from natural language prompts (click, scroll, navigate) + Docs → https://docs.stagehand.dev/basics/act +- **observe**: find and return interactive elements on the page matching a description, without performing actions. Used here to locate all individual download buttons before clicking them. + Docs → https://docs.stagehand.dev/basics/observe +- **Browserbase Downloads**: When files are downloaded during a browser session, Browserbase captures and stores them. Files are retrieved via the Session Downloads API as a ZIP archive. + Docs → https://docs.browserbase.com/features/downloads +- **Extend EXTRACT processor**: A configurable document extraction pipeline that parses files against a JSON schema and returns structured data. Processors are reusable and persist across runs. + Docs → https://docs.extend.app +- **Download polling**: Browserbase syncs downloads in real-time; the script retries every 2 seconds until the ZIP is available or a timeout is reached. + +## QUICKSTART + +1. cd typescript/extend-browserbase +2. pnpm install +3. cp .env.example .env +4. Add required API keys to .env: + - `BROWSERBASE_PROJECT_ID` + - `BROWSERBASE_API_KEY` + - `GOOGLE_API_KEY` + - `EXTEND_API_KEY` (optional — enables receipt parsing) +5. pnpm start + +## EXPECTED OUTPUT + +- Initializes Stagehand session with Browserbase and opens the live view link +- Navigates to the expense portal and finds all per-receipt download links via observe +- Clicks each download button; Browserbase captures files +- After closing the session, polls for the session's download ZIP and extracts to `output/documents/` +- If `EXTEND_API_KEY` is set: creates/uses an Extend "Receipt Extractor" processor, uploads each file, runs extraction, writes `output/results/receipts.json` and `receipts.csv` +- Opens the Extend dashboard runs page in your browser for review +- Closes session cleanly + +## COMMON PITFALLS + +- "Cannot find module": ensure pnpm install completed in the extend-browserbase directory +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY +- Download timeout: increase `retryForSeconds` parameter in `saveDownloadsWithRetry` if downloads take longer than 60 seconds +- Empty ZIP file: ensure downloads were actually triggered (check live view link to debug) +- Rate limiting on Extend: the script retries with exponential backoff on 429 errors, but very large batches may need the batch size reduced from 9 +- Find more information on your Browserbase dashboard → https://www.browserbase.com/sign-in + +## USE CASES + +• Expense automation: Download receipts from expense portals and extract vendor, date, totals, and line items for accounting systems. +• Document batch processing: Collect files from web portals and run structured extraction across all of them with a single script. +• Receipt digitization: Convert paper/PDF receipts into structured JSON and CSV for import into ERP, bookkeeping, or reimbursement tools. + +## NEXT STEPS + +• Parameterize the portal URL: Accept the expense portal URL from env or CLI to support different receipt sources. +• Custom schemas: Modify `receiptExtractionConfig` to extract different document types (invoices, W-2s, contracts) by changing the JSON schema. +• Add validation: Compare extracted totals against line item sums to flag discrepancies or incomplete extractions. +• Scheduled runs: Deploy on cron/Lambda to periodically check for new receipts and process them automatically. + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +📚 Browserbase Downloads: https://docs.browserbase.com/features/downloads +📚 Extend AI: https://docs.extend.app +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/typescript/extend-browserbase/index.ts b/typescript/extend-browserbase/index.ts new file mode 100644 index 00000000..679c52d5 --- /dev/null +++ b/typescript/extend-browserbase/index.ts @@ -0,0 +1,534 @@ +// Stagehand + Browserbase + Extend: Download Expense Receipts and Parse with Extend AI - See README.md for full documentation + +import "dotenv/config"; +import { Browserbase } from "@browserbasehq/sdk"; +import { Stagehand } from "@browserbasehq/stagehand"; +import fs from "fs"; +import path from "path"; +import AdmZip from "adm-zip"; +import { ExtendClient } from "extend-ai"; +import open from "open"; + +// Opens a URL in the default browser (cross-platform) +function openInBrowser(url: string): void { + open(url).catch(() => { + console.log(`Could not auto-open: ${url}`); + }); +} + +// Polls Browserbase API for completed downloads with retry logic. +// Retries every 2 seconds until downloads are ready or timeout is reached. +async function saveDownloadsWithRetry( + bb: Browserbase, + sessionId: string, + timeoutSecs: number = 60, +): Promise { + console.log(`Waiting up to ${timeoutSecs} seconds for downloads to complete...`); + const deadline = Date.now() + timeoutSecs * 1000; + + while (Date.now() < deadline) { + try { + console.log("Checking for downloads..."); + const response = await bb.sessions.downloads.list(sessionId); + const buf = Buffer.from(await response.arrayBuffer()); + + if (buf.byteLength > 100) { + console.log(`Downloads ready! File size: ${buf.byteLength} bytes`); + fs.writeFileSync("downloaded_files.zip", buf); + console.log("Files saved as: downloaded_files.zip"); + return buf.byteLength; + } + console.log("Downloads not ready yet, retrying..."); + } catch (e: unknown) { + const errorMessage = e instanceof Error ? e.message : String(e); + // HTML error response - session may not be ready yet, keep retrying + if (errorMessage.includes("Unexpected token '<'") || errorMessage.includes(" setTimeout(r, 2000)); + } + throw new Error("Download timeout exceeded"); +} + +// Extracts receipt files from downloaded zip archive into output directories +function extractFilesFromZip(zipPath: string, outputDir: string = "output/documents"): string[] { + console.log(`Extracting files from ${zipPath}...`); + + // Create output directories for documents and results if they don't exist + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + if (!fs.existsSync("output/results")) { + fs.mkdirSync("output/results", { recursive: true }); + } + + // Open zip file and iterate over entries + const zip = new AdmZip(zipPath); + const entries = zip.getEntries(); + + if (entries.length === 0) { + throw new Error("No files found in the downloaded zip"); + } + + // Extract all non-directory entries and collect file paths + const extractedFiles: string[] = []; + + for (const entry of entries) { + if (!entry.isDirectory) { + const outputPath = path.join(outputDir, entry.entryName); + zip.extractEntryTo(entry, outputDir, false, true); + console.log(`Extracted: ${outputPath}`); + extractedFiles.push(outputPath); + } + } + + console.log(`\nTotal files extracted: ${extractedFiles.length}`); + return extractedFiles; +} + +// Receipt extraction config for Extend AI +// Uses extraction_light base processor with parse_performance engine for low latency +const receiptExtractionConfig = { + type: "EXTRACT", + baseProcessor: "extraction_light", + baseVersion: "3.4.0", + parser: { + engine: "parse_performance", + target: "markdown", + blockOptions: { + text: { + agentic: { enabled: false }, + signatureDetectionEnabled: false, + }, + tables: { + agentic: { enabled: false }, + targetFormat: "markdown", + cellBlocksEnabled: false, + tableHeaderContinuationEnabled: false, + }, + figures: { + enabled: false, + figureImageClippingEnabled: false, + }, + }, + engineVersion: "1.0.1", + advancedOptions: { + engine: "parse_performance", + agenticOcrEnabled: false, + pageBreaksEnabled: true, + pageRotationEnabled: false, + verticalGroupingThreshold: 1, + }, + chunkingStrategy: { type: "document" }, + }, + schema: { + type: "object", + required: [ + "vendor_name", + "receipt_date", + "receipt_number", + "total_amount", + "subtotal_amount", + "tax_amount", + "line_items", + "payment_method", + ], + properties: { + vendor_name: { + type: ["string", "null"], + description: "The name of the merchant or vendor on the receipt.", + }, + receipt_date: { + type: ["string", "null"], + description: "The date of the transaction shown on the receipt.", + "extend:type": "date", + }, + receipt_number: { + type: ["string", "null"], + description: "The receipt or transaction number, if present.", + }, + total_amount: { + type: "object", + required: ["amount", "iso_4217_currency_code"], + properties: { + amount: { type: ["number", "null"] }, + iso_4217_currency_code: { type: ["string", "null"] }, + }, + description: "The total amount paid on the receipt.", + "extend:type": "currency", + additionalProperties: false, + }, + subtotal_amount: { + type: "object", + required: ["amount", "iso_4217_currency_code"], + properties: { + amount: { type: ["number", "null"] }, + iso_4217_currency_code: { type: ["string", "null"] }, + }, + description: "The subtotal before tax, if shown.", + "extend:type": "currency", + additionalProperties: false, + }, + tax_amount: { + type: "object", + required: ["amount", "iso_4217_currency_code"], + properties: { + amount: { type: ["number", "null"] }, + iso_4217_currency_code: { type: ["string", "null"] }, + }, + description: "The tax amount on the receipt.", + "extend:type": "currency", + additionalProperties: false, + }, + line_items: { + type: "array", + items: { + type: "object", + required: ["description", "quantity", "unit_price", "amount"], + properties: { + description: { + type: ["string", "null"], + description: "Description of the item purchased.", + }, + quantity: { + type: ["number", "null"], + description: "Quantity of the item, if shown.", + }, + unit_price: { + type: ["number", "null"], + description: "Price per unit, if shown.", + }, + amount: { + type: ["number", "null"], + description: "Total amount for this line item.", + }, + }, + additionalProperties: false, + }, + description: "Individual items on the receipt.", + }, + payment_method: { + type: ["string", "null"], + description: "The payment method used (e.g., cash, credit card, etc.).", + }, + }, + additionalProperties: false, + }, + advancedOptions: { + advancedMultimodalEnabled: false, + citationsEnabled: true, + arrayCitationStrategy: "item", + pageRanges: [], + chunkingOptions: {}, + advancedFigureParsingEnabled: true, + }, +}; + +// Gets existing Extend processor or creates a new one, saving the ID to .env for reuse. +// This avoids recreating the processor on every run. +async function getOrCreateProcessor(client: ExtendClient): Promise { + // Check if a processor ID already exists in environment + const existingId = process.env.EXTEND_PROCESSOR_ID; + if (existingId && existingId !== "YOUR_EXTEND_PROCESSOR_ID_HERE") { + console.log(`Using existing processor: ${existingId}`); + return existingId; + } + + // Create a new Receipt Extractor processor via the Extend API + console.log("No EXTEND_PROCESSOR_ID found. Creating 'Receipt Extractor' processor..."); + + const response = await client.processor.create({ + name: "Receipt Extractor", + type: "EXTRACT", + config: receiptExtractionConfig as Parameters[0]["config"], + }); + + const processorId = response.processor.id; + console.log(`Created processor: ${processorId}`); + console.log(` View in dashboard: https://dashboard.extend.app/studio/processors/${processorId}`); + + // Persist the processor ID to .env so we don't recreate on next run + const envPath = path.resolve(".env"); + if (fs.existsSync(envPath)) { + let envContent = fs.readFileSync(envPath, "utf-8"); + if (envContent.includes("EXTEND_PROCESSOR_ID=")) { + envContent = envContent.replace( + /EXTEND_PROCESSOR_ID=.*/, + `EXTEND_PROCESSOR_ID=${processorId}`, + ); + } else { + envContent += `\nEXTEND_PROCESSOR_ID=${processorId}\n`; + } + fs.writeFileSync(envPath, envContent); + } else { + fs.writeFileSync(envPath, `EXTEND_PROCESSOR_ID=${processorId}\n`, { flag: "a" }); + } + console.log(" Saved EXTEND_PROCESSOR_ID to .env for future runs."); + + return processorId; +} + +// Uploads receipt files to Extend AI, runs extraction, and saves results as JSON and CSV +async function parseReceiptsWithExtend(filePaths: string[]): Promise { + // Skip parsing if Extend API key is not configured + if (!process.env.EXTEND_API_KEY || process.env.EXTEND_API_KEY === "YOUR_EXTEND_API_KEY_HERE") { + console.log("\nWARNING: EXTEND_API_KEY not configured. Skipping receipt parsing."); + console.log(" Add your Extend API key to .env to enable automatic receipt parsing."); + return; + } + + console.log("\n=== Parsing Receipts with Extend AI ===\n"); + + // Initialize Extend AI client and get or create the receipt processor + // SDK auto-retries 429s and 5xx errors with exponential backoff + const client = new ExtendClient({ token: process.env.EXTEND_API_KEY }); + const processorId = await getOrCreateProcessor(client); + + console.log(`Processing ${filePaths.length} receipts...`); + const runsUrl = `https://dashboard.extend.app/studio/processors/${processorId}?tab=Runs`; + console.log(`View all runs: ${runsUrl}\n`); + openInBrowser(runsUrl); + + // Process all files - SDK handles retries automatically with exponential backoff + const results: { file: string; runId?: string; runUrl?: string; data: unknown }[] = []; + + // Process in batches of 9 to balance speed and reliability + for (let i = 0; i < filePaths.length; i += 9) { + const batch = filePaths.slice(i, i + 9); + const batchResults = await Promise.all( + batch.map(async (filePath) => { + const fileName = path.basename(filePath); + try { + // Upload the file to Extend + const fileBuffer = fs.readFileSync(filePath); + const blob = new Blob([fileBuffer]); + const uploadResponse = await client.file.upload( + blob as Parameters[0], + { maxRetries: 4 }, + ); + const fileId = uploadResponse.file.id; + + // Run extraction on the uploaded file + const extractionResponse = await client.processorRun.create( + { + processorId, + file: { fileId }, + sync: true, + config: receiptExtractionConfig as Parameters< + typeof client.processorRun.create + >[0]["config"], + }, + { maxRetries: 4 }, + ); + + const parsedData = extractionResponse.processorRun; + const runId = parsedData.id; + const runUrl = `https://dashboard.extend.app/studio/processors/${processorId}/runs/${runId}`; + console.log(` Parsed ${fileName}`); + console.log(` → ${runUrl}`); + return { file: fileName, runId, runUrl, data: parsedData }; + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + console.error(` Failed to parse ${fileName}:`, errorMsg); + return { file: fileName, data: { error: errorMsg } }; + } + }), + ); + results.push(...batchResults); + } + + // Save results to JSON + const jsonPath = "output/results/receipts.json"; + fs.writeFileSync(jsonPath, JSON.stringify(results, null, 2)); + console.log(`\nSaved JSON: ${jsonPath}`); + + // Convert results to CSV for easy viewing in spreadsheet tools + const csvRows: string[] = []; + csvRows.push( + "file,run_url,vendor_name,receipt_date,receipt_number,total_amount,currency,subtotal,tax,payment_method,line_items_count", + ); + + // Shape of the extracted receipt data from Extend processor runs + type ReceiptOutput = { + vendor_name?: string; + receipt_date?: string; + receipt_number?: string; + total_amount?: { amount?: string; iso_4217_currency_code?: string }; + subtotal_amount?: { amount?: string }; + tax_amount?: { amount?: string }; + payment_method?: string; + line_items?: unknown[]; + }; + + // Build CSV rows from extraction results + for (const result of results) { + const data = result.data as { output?: { value?: ReceiptOutput } } | undefined; + const output: ReceiptOutput = data?.output?.value || {}; + const row = [ + result.file, + result.runUrl || "", + output.vendor_name || "", + output.receipt_date || "", + output.receipt_number || "", + output.total_amount?.amount ?? "", + output.total_amount?.iso_4217_currency_code || "", + output.subtotal_amount?.amount ?? "", + output.tax_amount?.amount ?? "", + output.payment_method || "", + Array.isArray(output.line_items) ? output.line_items.length : 0, + ] + .map((v) => `"${String(v).replace(/"/g, '""')}"`) + .join(","); + csvRows.push(row); + } + + const csvPath = "output/results/receipts.csv"; + fs.writeFileSync(csvPath, csvRows.join("\n")); + console.log(`Saved CSV: ${csvPath}`); + + console.log(`\nView all runs: ${runsUrl}`); +} + +async function main(): Promise { + console.log("Starting Expense Receipt Downloader...\n"); + + if (!process.env.BROWSERBASE_API_KEY || !process.env.BROWSERBASE_PROJECT_ID) { + throw new Error("BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID are required"); + } + + // Initialize Browserbase SDK for session management and download retrieval + const bb = new Browserbase({ + apiKey: process.env.BROWSERBASE_API_KEY as string, + }); + + // Initialize Stagehand with Browserbase for cloud-based browser automation + const stagehand = new Stagehand({ + env: "BROWSERBASE", + verbose: 1, + // 0 = errors only, 1 = info, 2 = debug + // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) + // https://docs.stagehand.dev/configuration/logging + model: { + modelName: "google/gemini-2.5-flash", + apiKey: process.env.GOOGLE_API_KEY, + }, + }); + + let sessionId: string | undefined; + + try { + // Initialize browser session to start automation + await stagehand.init(); + console.log("Stagehand initialized successfully!"); + const page = stagehand.context.pages()[0]; + sessionId = stagehand.browserbaseSessionId; + + // Get live view URL for monitoring browser session in real-time + if (sessionId) { + const liveViewLinks = await bb.sessions.debug(sessionId); + console.log(`Live View Link: ${liveViewLinks.debuggerFullscreenUrl}`); + openInBrowser(liveViewLinks.debuggerFullscreenUrl); + } + + // Navigate to the expense portal where receipts are hosted + console.log("\nNavigating to expense portal..."); + await page.goto("https://v0-reimburse-me-expense-portal.vercel.app/", { + waitUntil: "domcontentloaded", + }); + + // Use observe to find all individual download buttons (not the Download All button) + console.log("\nFinding all individual download buttons..."); + const downloadButtons = await stagehand.observe( + "Find all the small Download links on individual receipt cards.", + ); + + // Click each download button using observe → act pattern + // Pass the observed action directly to act for precise element targeting + let successCount = 0; + for (let i = 0; i < downloadButtons.length; i++) { + const action = downloadButtons[i]; + console.log(`Downloading receipt ${i + 1}/${downloadButtons.length}...`); + + try { + await stagehand.act(action, { page }); + successCount++; + } catch (clickError) { + // If click fails, scroll element into view and retry + console.log(` Could not click download button ${i + 1}, trying to scroll and retry...`); + try { + await page.evaluate(() => window.scrollBy(0, 200)); + await stagehand.act(action, { page }); + successCount++; + } catch { + console.log(` Skipping receipt ${i + 1}`); + } + } + + // Scroll down periodically to ensure elements are in view + if ((i + 1) % 4 === 0 && i + 1 < downloadButtons.length) { + await page.evaluate(() => window.scrollBy(0, 300)); + } + } + + console.log( + `\nDownload clicks completed! (${successCount}/${downloadButtons.length} successful)`, + ); + + // Retrieve all downloads triggered during this session from Browserbase API + if (sessionId) { + console.log("\nRetrieving downloads from Browserbase..."); + + // Close the browser session before fetching downloads + await stagehand.close(); + + // Wait for session to finalize downloads before polling + await new Promise((resolve) => setTimeout(resolve, 2000)); + + try { + const downloadSize = await saveDownloadsWithRetry(bb, sessionId, 60); + + if (downloadSize > 0) { + // Extract receipt files from downloaded zip archive + const extractedFiles = extractFilesFromZip("downloaded_files.zip"); + + console.log("\n=== Download Summary ==="); + console.log(`Total files downloaded: ${extractedFiles.length}`); + console.log("Files saved to: ./output/documents/"); + + // Parse downloaded receipts with Extend AI for structured data extraction + await parseReceiptsWithExtend(extractedFiles); + } else { + console.log("No downloads were captured"); + } + } catch (downloadError) { + console.error("Download retrieval failed:", downloadError); + } + } + + console.log("\nExpense receipt download complete!"); + } catch (error) { + console.error("Error during automation:", error); + try { + await stagehand.close(); + } catch { + // Ignore close errors during cleanup + } + throw error; + } +} + +main().catch((err) => { + console.error("Application error:", err); + console.error("Common issues:"); + console.error( + " - Check .env file has BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and GOOGLE_API_KEY", + ); + console.error(" - Add EXTEND_API_KEY to .env to enable receipt parsing with Extend AI"); + console.error(" - Verify internet connection and expense portal accessibility"); + console.error("Docs: https://docs.stagehand.dev/v3/first-steps/introduction"); + process.exit(1); +}); diff --git a/typescript/extend-browserbase/package.json b/typescript/extend-browserbase/package.json new file mode 100644 index 00000000..a45260df --- /dev/null +++ b/typescript/extend-browserbase/package.json @@ -0,0 +1,24 @@ +{ + "name": "extend-browserbase-partnership-template", + "version": "1.0.0", + "description": "Stagehand + Browserbase + Extend: Download receipts from expense portal and extract structured data with Extend AI", + "type": "module", + "main": "index.ts", + "scripts": { + "start": "tsx index.ts" + }, + "dependencies": { + "@browserbasehq/sdk": "^2.6.0", + "@browserbasehq/stagehand": "^3.0.8", + "adm-zip": "^0.5.16", + "dotenv": "^17.2.4", + "extend-ai": "^0.0.18", + "open": "^11.0.0" + }, + "devDependencies": { + "@types/node": "^25.2.2", + "tsx": "^4.21.0", + "typescript": "^5.9.3" + }, + "packageManager": "pnpm@9.0.0" +} From 3d8bfe47e5ba5581d29fdf06741443f903060fa0 Mon Sep 17 00:00:00 2001 From: Addison Huff Date: Tue, 10 Feb 2026 17:57:32 -0500 Subject: [PATCH 21/45] Migrate extend-browserbase templates to Extend API v2026-02-09 (#58) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace processor/processorRun pattern with inline config via client.extract() - Remove getOrCreateProcessor/get_or_create_processor and EXTEND_PROCESSOR_ID - Rename config.parser to config.parseConfig, remove config.type - Update file upload: client.file.upload() → client.files.upload() - Unwrap API responses (response.id instead of response.processor.id) - Bump extend-ai SDK: TS ^0.0.18 → ^1.0.2, Python >=0.0.21 → >=1.0.0 - Update READMEs and .env.example files Co-authored-by: Claude (claude-opus-4-6) --- python/extend-browserbase/.env.example | 3 - python/extend-browserbase/README.md | 7 +- python/extend-browserbase/main.py | 116 ++++----------------- python/extend-browserbase/pyproject.toml | 2 +- typescript/extend-browserbase/.env.example | 3 - typescript/extend-browserbase/README.md | 7 +- typescript/extend-browserbase/index.ts | 93 +++-------------- typescript/extend-browserbase/package.json | 2 +- 8 files changed, 47 insertions(+), 186 deletions(-) diff --git a/python/extend-browserbase/.env.example b/python/extend-browserbase/.env.example index fad95662..0942dfa0 100644 --- a/python/extend-browserbase/.env.example +++ b/python/extend-browserbase/.env.example @@ -8,6 +8,3 @@ GOOGLE_API_KEY=your_google_api_key # Extend AI (optional – enables receipt parsing; omit to only download receipts) EXTEND_API_KEY=your_extend_api_key - -# Optional: set after first run to reuse the created processor -# EXTEND_PROCESSOR_ID=your_processor_id diff --git a/python/extend-browserbase/README.md b/python/extend-browserbase/README.md index 15d558fb..5d93c91d 100644 --- a/python/extend-browserbase/README.md +++ b/python/extend-browserbase/README.md @@ -6,7 +6,7 @@ - **Pattern Template**: Demonstrates the integration pattern of Browserbase (browser automation + download capture) + Extend AI (schema-based document extraction). - **Workflow**: Stagehand navigates the expense portal and clicks each receipt's download link; Browserbase captures downloads. The script polls for the session's download ZIP, extracts files, then optionally sends them to Extend for structured extraction (vendor, date, totals, line items, etc.). - **Download Handling**: Implements retry/polling around Browserbase's Session Downloads API until the ZIP is available. -- **Structured Extraction**: Extend EXTRACT processor with a receipt JSON schema; results written to `output/results/receipts.json` and `receipts.csv`. +- **Structured Extraction**: Extend AI extraction with inline receipt JSON schema config; results written to `output/results/receipts.json` and `receipts.csv`. - Docs → [Browserbase Downloads](https://docs.browserbase.com/features/downloads) | [Extend AI](https://docs.extend.app) ## GLOSSARY @@ -17,7 +17,7 @@ Docs → https://docs.stagehand.dev/basics/observe - **Browserbase Downloads**: When files are downloaded during a browser session, Browserbase captures and stores them. Files are retrieved via the Session Downloads API as a ZIP archive. Docs → https://docs.browserbase.com/features/downloads -- **Extend EXTRACT processor**: A configurable document extraction pipeline that parses files against a JSON schema and returns structured data. Processors are reusable and persist across runs. +- **Extend AI extraction**: A configurable document extraction pipeline that parses files against a JSON schema and returns structured data. Config can be passed inline or via a saved extractor resource. Docs → https://docs.extend.app - **Download polling**: Browserbase syncs downloads in real-time; the script retries every 2 seconds until the ZIP is available or a timeout is reached. @@ -42,8 +42,7 @@ - Navigates to the expense portal and finds all per-receipt download links via observe - Clicks each download button; Browserbase captures files - After closing the session, polls for the session's download ZIP and extracts to `output/documents/` -- If `EXTEND_API_KEY` is set: creates/uses an Extend "Receipt Extractor" processor, uploads each file, runs extraction, writes `output/results/receipts.json` and `receipts.csv` -- Opens the Extend dashboard runs page in your browser for review +- If `EXTEND_API_KEY` is set: uploads each file to Extend and runs extraction with inline config, writes `output/results/receipts.json` and `receipts.csv` - Closes session cleanly ## COMMON PITFALLS diff --git a/python/extend-browserbase/main.py b/python/extend-browserbase/main.py index 8caea1ae..b9c5a5ef 100644 --- a/python/extend-browserbase/main.py +++ b/python/extend-browserbase/main.py @@ -5,7 +5,6 @@ import csv import json import os -import re import webbrowser import zipfile from pathlib import Path @@ -17,17 +16,16 @@ # Load environment variables from .env file # Required: BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, GOOGLE_API_KEY -# Optional: EXTEND_API_KEY, EXTEND_PROCESSOR_ID +# Optional: EXTEND_API_KEY load_dotenv() # Receipt extraction config for Extend AI -# Uses extraction_light base processor with parse_performance engine for low latency +# Uses extraction_light base extractor with parse_performance engine for low latency RECEIPT_EXTRACTION_CONFIG = { - "type": "EXTRACT", "baseProcessor": "extraction_light", "baseVersion": "3.4.0", - "parser": { + "parseConfig": { "engine": "parse_performance", "target": "markdown", "blockOptions": { @@ -285,69 +283,14 @@ def extract_files_from_zip(zip_path: str, output_dir: str = "output/documents") return extracted_files -# Gets existing Extend processor or creates a new one, saving the ID to .env for reuse. -# This avoids recreating the processor on every run. -async def get_or_create_processor(extend_client: Extend) -> str: - """ - Get an existing Extend processor or create a new one. - - Checks for EXTEND_PROCESSOR_ID in environment. If not found, creates a new - 'Receipt Extractor' processor and saves the ID to .env for future runs. - - Args: - extend_client: Extend client instance for API calls - - Returns: - str: The processor ID - """ - # Check if a processor ID already exists in environment - existing_id = os.environ.get("EXTEND_PROCESSOR_ID") - if existing_id and existing_id != "YOUR_EXTEND_PROCESSOR_ID_HERE": - print(f"Using existing processor: {existing_id}") - return existing_id - - # Create a new Receipt Extractor processor via the Extend API - print("No EXTEND_PROCESSOR_ID found. Creating 'Receipt Extractor' processor...") - - response = await asyncio.to_thread( - extend_client.processor.create, - name="Receipt Extractor", - type="EXTRACT", - config=RECEIPT_EXTRACTION_CONFIG, - ) - - processor_id = response.processor.id - print(f"Created processor: {processor_id}") - print(f" View in dashboard: https://dashboard.extend.app/studio/processors/{processor_id}") - - # Persist the processor ID to .env so we don't recreate on next run - env_path = Path(".env") - if env_path.exists(): - env_content = env_path.read_text() - if "EXTEND_PROCESSOR_ID=" in env_content: - env_content = re.sub( - r"EXTEND_PROCESSOR_ID=.*", - f"EXTEND_PROCESSOR_ID={processor_id}", - env_content, - ) - else: - env_content += f"\nEXTEND_PROCESSOR_ID={processor_id}\n" - env_path.write_text(env_content) - else: - with open(env_path, "a") as f: - f.write(f"EXTEND_PROCESSOR_ID={processor_id}\n") - print(" Saved EXTEND_PROCESSOR_ID to .env for future runs.") - - return processor_id - - # Uploads receipt files to Extend AI, runs extraction, and saves results as JSON and CSV async def parse_receipts_with_extend(file_paths: list[str]) -> None: """ Upload receipt files to Extend AI, run extraction, and save results. - Initializes the Extend client, gets or creates a processor, uploads each file, - runs synchronous extraction, and saves results as JSON and CSV. + Initializes the Extend client, uploads each file, runs synchronous extraction + using inline config (no need to pre-create an extractor resource), and saves + results as JSON and CSV. Args: file_paths: List of file paths to receipt documents @@ -361,14 +304,10 @@ async def parse_receipts_with_extend(file_paths: list[str]) -> None: print("\n=== Parsing Receipts with Extend AI ===\n") - # Initialize Extend AI client and get or create the receipt processor + # Initialize Extend AI client extend_client = Extend(token=extend_api_key) - processor_id = await get_or_create_processor(extend_client) - print(f"Processing {len(file_paths)} receipts...") - runs_url = f"https://dashboard.extend.app/studio/processors/{processor_id}?tab=Runs" - print(f"View all runs: {runs_url}\n") - open_in_browser(runs_url) + print(f"Processing {len(file_paths)} receipts with inline config...\n") # Process all files with retry on rate limiting (429 errors) results: list[dict] = [] @@ -384,39 +323,31 @@ async def process_with_retry(file_path: str, max_retries: int = 3) -> dict: with open(file_path, "rb") as f: file_bytes = f.read() upload_response = await asyncio.to_thread( - extend_client.file.upload, + extend_client.files.upload, file=(file_name, file_bytes), ) - file_id = upload_response.file.id - - # Run extraction on the uploaded file - extraction_response = await asyncio.to_thread( - extend_client.processor_run.create, - processor_id=processor_id, - file={"fileId": file_id}, - sync=True, + file_id = upload_response.id + + # Run extraction using inline config — no need to pre-create an extractor + result = await asyncio.to_thread( + extend_client.extract, config=RECEIPT_EXTRACTION_CONFIG, + file={"id": file_id}, ) - parsed_data = extraction_response.processor_run - run_id = parsed_data.id - run_url = ( - f"https://dashboard.extend.app/studio/processors/{processor_id}/runs/{run_id}" - ) - print(f" Parsed {file_name}") - print(f" -> {run_url}") + run_id = result.id + print(f" Parsed {file_name} (run: {run_id})") # Convert to dict for JSON serialization - data = parsed_data - if hasattr(parsed_data, "to_dict"): - data = parsed_data.to_dict() - elif hasattr(parsed_data, "dict"): - data = parsed_data.dict() + data = result + if hasattr(result, "to_dict"): + data = result.to_dict() + elif hasattr(result, "dict"): + data = result.dict() return { "file": file_name, "runId": run_id, - "runUrl": run_url, "data": data, } except Exception as error: @@ -457,7 +388,6 @@ async def process_with_retry(file_path: str, max_retries: int = 3) -> dict: writer.writerow( [ "file", - "run_url", "vendor_name", "receipt_date", "receipt_number", @@ -480,7 +410,6 @@ async def process_with_retry(file_path: str, max_retries: int = 3) -> dict: writer.writerow( [ result.get("file", ""), - result.get("runUrl", ""), output.get("vendor_name", ""), output.get("receipt_date", ""), output.get("receipt_number", ""), @@ -496,7 +425,6 @@ async def process_with_retry(file_path: str, max_retries: int = 3) -> dict: ) print(f"Saved CSV: {csv_path}") - print(f"\nView all runs: {runs_url}") async def main() -> None: diff --git a/python/extend-browserbase/pyproject.toml b/python/extend-browserbase/pyproject.toml index 29b4f903..bc789e20 100644 --- a/python/extend-browserbase/pyproject.toml +++ b/python/extend-browserbase/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" requires-python = ">=3.9" dependencies = [ "browserbase>=1.4.0", - "extend-ai>=0.0.21", + "extend-ai>=1.0.0", "python-dotenv>=1.2.1", "stagehand>=3.5.0", ] diff --git a/typescript/extend-browserbase/.env.example b/typescript/extend-browserbase/.env.example index b249aafa..5431be57 100644 --- a/typescript/extend-browserbase/.env.example +++ b/typescript/extend-browserbase/.env.example @@ -7,6 +7,3 @@ GOOGLE_API_KEY=your_google_api_key # Extend AI (optional – enables receipt parsing; omit to only download receipts) EXTEND_API_KEY=your_extend_api_key - -# Optional: set after first run to reuse the created processor -# EXTEND_PROCESSOR_ID=your_processor_id diff --git a/typescript/extend-browserbase/README.md b/typescript/extend-browserbase/README.md index 3ec84395..d05ac8f8 100644 --- a/typescript/extend-browserbase/README.md +++ b/typescript/extend-browserbase/README.md @@ -6,7 +6,7 @@ - **Pattern Template**: Demonstrates the integration pattern of Browserbase (browser automation + download capture) + Extend AI (schema-based document extraction). - **Workflow**: Stagehand navigates the expense portal and clicks each receipt's download link; Browserbase captures downloads. The script polls for the session's download ZIP, extracts files, then optionally sends them to Extend for structured extraction (vendor, date, totals, line items, etc.). - **Download Handling**: Implements retry/polling around Browserbase's Session Downloads API until the ZIP is available. -- **Structured Extraction**: Extend EXTRACT processor with a receipt JSON schema; results written to `output/results/receipts.json` and `receipts.csv`. +- **Structured Extraction**: Extend AI extraction with inline receipt JSON schema config; results written to `output/results/receipts.json` and `receipts.csv`. - Docs → [Browserbase Downloads](https://docs.browserbase.com/features/downloads) | [Extend AI](https://docs.extend.app) ## GLOSSARY @@ -17,7 +17,7 @@ Docs → https://docs.stagehand.dev/basics/observe - **Browserbase Downloads**: When files are downloaded during a browser session, Browserbase captures and stores them. Files are retrieved via the Session Downloads API as a ZIP archive. Docs → https://docs.browserbase.com/features/downloads -- **Extend EXTRACT processor**: A configurable document extraction pipeline that parses files against a JSON schema and returns structured data. Processors are reusable and persist across runs. +- **Extend AI extraction**: A configurable document extraction pipeline that parses files against a JSON schema and returns structured data. Config can be passed inline or via a saved extractor resource. Docs → https://docs.extend.app - **Download polling**: Browserbase syncs downloads in real-time; the script retries every 2 seconds until the ZIP is available or a timeout is reached. @@ -39,8 +39,7 @@ - Navigates to the expense portal and finds all per-receipt download links via observe - Clicks each download button; Browserbase captures files - After closing the session, polls for the session's download ZIP and extracts to `output/documents/` -- If `EXTEND_API_KEY` is set: creates/uses an Extend "Receipt Extractor" processor, uploads each file, runs extraction, writes `output/results/receipts.json` and `receipts.csv` -- Opens the Extend dashboard runs page in your browser for review +- If `EXTEND_API_KEY` is set: uploads each file to Extend and runs extraction with inline config, writes `output/results/receipts.json` and `receipts.csv` - Closes session cleanly ## COMMON PITFALLS diff --git a/typescript/extend-browserbase/index.ts b/typescript/extend-browserbase/index.ts index 679c52d5..94321b19 100644 --- a/typescript/extend-browserbase/index.ts +++ b/typescript/extend-browserbase/index.ts @@ -91,12 +91,11 @@ function extractFilesFromZip(zipPath: string, outputDir: string = "output/docume } // Receipt extraction config for Extend AI -// Uses extraction_light base processor with parse_performance engine for low latency +// Uses extraction_light base extractor with parse_performance engine for low latency const receiptExtractionConfig = { - type: "EXTRACT", baseProcessor: "extraction_light", baseVersion: "3.4.0", - parser: { + parseConfig: { engine: "parse_performance", target: "markdown", blockOptions: { @@ -228,50 +227,6 @@ const receiptExtractionConfig = { }, }; -// Gets existing Extend processor or creates a new one, saving the ID to .env for reuse. -// This avoids recreating the processor on every run. -async function getOrCreateProcessor(client: ExtendClient): Promise { - // Check if a processor ID already exists in environment - const existingId = process.env.EXTEND_PROCESSOR_ID; - if (existingId && existingId !== "YOUR_EXTEND_PROCESSOR_ID_HERE") { - console.log(`Using existing processor: ${existingId}`); - return existingId; - } - - // Create a new Receipt Extractor processor via the Extend API - console.log("No EXTEND_PROCESSOR_ID found. Creating 'Receipt Extractor' processor..."); - - const response = await client.processor.create({ - name: "Receipt Extractor", - type: "EXTRACT", - config: receiptExtractionConfig as Parameters[0]["config"], - }); - - const processorId = response.processor.id; - console.log(`Created processor: ${processorId}`); - console.log(` View in dashboard: https://dashboard.extend.app/studio/processors/${processorId}`); - - // Persist the processor ID to .env so we don't recreate on next run - const envPath = path.resolve(".env"); - if (fs.existsSync(envPath)) { - let envContent = fs.readFileSync(envPath, "utf-8"); - if (envContent.includes("EXTEND_PROCESSOR_ID=")) { - envContent = envContent.replace( - /EXTEND_PROCESSOR_ID=.*/, - `EXTEND_PROCESSOR_ID=${processorId}`, - ); - } else { - envContent += `\nEXTEND_PROCESSOR_ID=${processorId}\n`; - } - fs.writeFileSync(envPath, envContent); - } else { - fs.writeFileSync(envPath, `EXTEND_PROCESSOR_ID=${processorId}\n`, { flag: "a" }); - } - console.log(" Saved EXTEND_PROCESSOR_ID to .env for future runs."); - - return processorId; -} - // Uploads receipt files to Extend AI, runs extraction, and saves results as JSON and CSV async function parseReceiptsWithExtend(filePaths: string[]): Promise { // Skip parsing if Extend API key is not configured @@ -283,18 +238,14 @@ async function parseReceiptsWithExtend(filePaths: string[]): Promise { console.log("\n=== Parsing Receipts with Extend AI ===\n"); - // Initialize Extend AI client and get or create the receipt processor + // Initialize Extend AI client // SDK auto-retries 429s and 5xx errors with exponential backoff const client = new ExtendClient({ token: process.env.EXTEND_API_KEY }); - const processorId = await getOrCreateProcessor(client); - console.log(`Processing ${filePaths.length} receipts...`); - const runsUrl = `https://dashboard.extend.app/studio/processors/${processorId}?tab=Runs`; - console.log(`View all runs: ${runsUrl}\n`); - openInBrowser(runsUrl); + console.log(`Processing ${filePaths.length} receipts with inline config...\n`); // Process all files - SDK handles retries automatically with exponential backoff - const results: { file: string; runId?: string; runUrl?: string; data: unknown }[] = []; + const results: { file: string; runId?: string; data: unknown }[] = []; // Process in batches of 9 to balance speed and reliability for (let i = 0; i < filePaths.length; i += 9) { @@ -306,31 +257,24 @@ async function parseReceiptsWithExtend(filePaths: string[]): Promise { // Upload the file to Extend const fileBuffer = fs.readFileSync(filePath); const blob = new Blob([fileBuffer]); - const uploadResponse = await client.file.upload( - blob as Parameters[0], + const uploadResponse = await client.files.upload( + blob as Parameters[0], { maxRetries: 4 }, ); - const fileId = uploadResponse.file.id; + const fileId = uploadResponse.id; - // Run extraction on the uploaded file - const extractionResponse = await client.processorRun.create( + // Run extraction using inline config — no need to pre-create an extractor resource + const result = await client.extract( { - processorId, - file: { fileId }, - sync: true, - config: receiptExtractionConfig as Parameters< - typeof client.processorRun.create - >[0]["config"], + config: receiptExtractionConfig as Parameters[0]["config"], + file: { id: fileId }, }, { maxRetries: 4 }, ); - const parsedData = extractionResponse.processorRun; - const runId = parsedData.id; - const runUrl = `https://dashboard.extend.app/studio/processors/${processorId}/runs/${runId}`; - console.log(` Parsed ${fileName}`); - console.log(` → ${runUrl}`); - return { file: fileName, runId, runUrl, data: parsedData }; + const runId = result.id; + console.log(` Parsed ${fileName} (run: ${runId})`); + return { file: fileName, runId, data: result }; } catch (error) { const errorMsg = error instanceof Error ? error.message : String(error); console.error(` Failed to parse ${fileName}:`, errorMsg); @@ -349,10 +293,10 @@ async function parseReceiptsWithExtend(filePaths: string[]): Promise { // Convert results to CSV for easy viewing in spreadsheet tools const csvRows: string[] = []; csvRows.push( - "file,run_url,vendor_name,receipt_date,receipt_number,total_amount,currency,subtotal,tax,payment_method,line_items_count", + "file,vendor_name,receipt_date,receipt_number,total_amount,currency,subtotal,tax,payment_method,line_items_count", ); - // Shape of the extracted receipt data from Extend processor runs + // Shape of the extracted receipt data from Extend extract runs type ReceiptOutput = { vendor_name?: string; receipt_date?: string; @@ -370,7 +314,6 @@ async function parseReceiptsWithExtend(filePaths: string[]): Promise { const output: ReceiptOutput = data?.output?.value || {}; const row = [ result.file, - result.runUrl || "", output.vendor_name || "", output.receipt_date || "", output.receipt_number || "", @@ -389,8 +332,6 @@ async function parseReceiptsWithExtend(filePaths: string[]): Promise { const csvPath = "output/results/receipts.csv"; fs.writeFileSync(csvPath, csvRows.join("\n")); console.log(`Saved CSV: ${csvPath}`); - - console.log(`\nView all runs: ${runsUrl}`); } async function main(): Promise { diff --git a/typescript/extend-browserbase/package.json b/typescript/extend-browserbase/package.json index a45260df..9cf1c6d3 100644 --- a/typescript/extend-browserbase/package.json +++ b/typescript/extend-browserbase/package.json @@ -12,7 +12,7 @@ "@browserbasehq/stagehand": "^3.0.8", "adm-zip": "^0.5.16", "dotenv": "^17.2.4", - "extend-ai": "^0.0.18", + "extend-ai": "^1.0.2", "open": "^11.0.0" }, "devDependencies": { From 436d0dd6bb5299d3642a48c03ab6daef563cfdfd Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Fri, 13 Feb 2026 14:37:57 -0500 Subject: [PATCH 22/45] Add Cerebras docs checker template (#60) * Add Cerebras docs checker Python template 4-phase documentation checker: crawl docs via parallel Stagehand BYOB workers, discover the source GitHub repo, clone it, and verify docs accuracy against source code using Cerebras tool-calling agent. Co-Authored-By: Claude Opus 4.6 * Add inline comments to match other Python template conventions Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- python/cerebras-docs-checker/.env.example | 9 + python/cerebras-docs-checker/README.md | 62 ++ python/cerebras-docs-checker/main.py | 986 ++++++++++++++++++++ python/cerebras-docs-checker/pyproject.toml | 36 + 4 files changed, 1093 insertions(+) create mode 100644 python/cerebras-docs-checker/.env.example create mode 100644 python/cerebras-docs-checker/README.md create mode 100644 python/cerebras-docs-checker/main.py create mode 100644 python/cerebras-docs-checker/pyproject.toml diff --git a/python/cerebras-docs-checker/.env.example b/python/cerebras-docs-checker/.env.example new file mode 100644 index 00000000..4caa2946 --- /dev/null +++ b/python/cerebras-docs-checker/.env.example @@ -0,0 +1,9 @@ +# Browserbase credentials (required) +BROWSERBASE_API_KEY=your-browserbase-api-key +BROWSERBASE_PROJECT_ID=your-browserbase-project-id + +# Cerebras API key (required) +CEREBRAS_API_KEY=your-cerebras-api-key + +# Model selection (optional, defaults to llama-3.3-70b) +# CEREBRAS_MODEL=llama-3.3-70b diff --git a/python/cerebras-docs-checker/README.md b/python/cerebras-docs-checker/README.md new file mode 100644 index 00000000..7c6fbbe6 --- /dev/null +++ b/python/cerebras-docs-checker/README.md @@ -0,0 +1,62 @@ +# Stagehand + Browserbase: Cerebras Documentation Checker + +## AT A GLANCE + +- Goal: Crawl any documentation site, discover its source repo, and verify docs accuracy against the actual codebase using Cerebras LLMs. +- Parallel browser workers crawl docs pages and extract accessibility trees via Stagehand BYOB (Bring Your Own Browser). +- Cerebras-powered verification agent uses tool calling to grep and read source code, cross-referencing every function, parameter, and code example. +- Falls back to content-only analysis when no source repository is found. +- Docs → https://docs.stagehand.dev + +## GLOSSARY + +- extract: pull structured data or the accessibility tree from a page without LLM cost + Docs → https://docs.stagehand.dev/basics/extract +- execute: run a multi-step Stagehand agent with an instruction and step limit + Docs → https://docs.stagehand.dev/basics/agent +- BYOB (Bring Your Own Browser): connect Playwright directly to a Browserbase session for low-level control + Docs → https://docs.browserbase.com + +## QUICKSTART + +1. uv sync && playwright install chromium +2. cp .env.example .env # Add your CEREBRAS_API_KEY, BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID +3. uv run python main.py https://your-docs-site.com + +## EXPECTED OUTPUT + +- Spins up parallel crawl workers with live Browserbase session links +- BFS-crawls the docs site, extracting aria trees and checking for broken links/anchors +- Discovers the GitHub source repository from crawled page content +- Clones the repo and runs a Cerebras verification agent on each page +- Prints a summary table with issue counts by severity and type +- Saves a detailed Markdown report to `docs_report_YYYYMMDD_HHMM.md` + +## COMMON PITFALLS + +- "Missing required API keys": verify .env contains CEREBRAS_API_KEY, BROWSERBASE_API_KEY, and BROWSERBASE_PROJECT_ID +- Playwright not installed: run `playwright install chromium` after `uv sync` +- Cerebras 422 errors: the model may rate-limit under heavy load — reduce MAX_PAGES or MAX_CRAWL_WORKERS in main.py +- Clone failures: ensure the target docs site links to a public GitHub repo +- Find more information on your Browserbase dashboard -> https://www.browserbase.com/sign-in + +## USE CASES + +- Documentation audits: Automatically verify that API docs match the actual source code before a release. +- Broken link detection: Crawl a docs site and surface all broken external links and internal anchors. +- CI/CD integration: Run as a scheduled check to catch documentation drift as the codebase evolves. + +## NEXT STEPS + +- Add JSON export: Extend the output to include a machine-readable JSON issues file for downstream tooling. +- Configurable models: Support switching between Cerebras models (llama-3.3-70b for speed, qwen-3-235b for precision) via CLI flags. +- Incremental checks: Cache previously verified pages and only re-check pages whose content has changed. + +## HELPFUL RESOURCES + +📚 Stagehand Docs: https://docs.stagehand.dev/v3/first-steps/introduction +🎮 Browserbase: https://www.browserbase.com +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/python/cerebras-docs-checker/main.py b/python/cerebras-docs-checker/main.py new file mode 100644 index 00000000..dd629a03 --- /dev/null +++ b/python/cerebras-docs-checker/main.py @@ -0,0 +1,986 @@ +# Stagehand + Browserbase: Cerebras Documentation Checker - See README.md for full documentation + +import asyncio +import json +import os +import re +import subprocess +import sys +import tempfile +from collections import Counter +from datetime import datetime +from pathlib import Path +from typing import Optional +from urllib.parse import urlparse + +import httpx +from dotenv import load_dotenv +from openai import AsyncOpenAI, OpenAI +from playwright.async_api import async_playwright +from pydantic import BaseModel, Field +from stagehand import Stagehand + +load_dotenv(override=True) + +# API keys (loaded from .env file) +CEREBRAS_API_KEY = os.getenv("CEREBRAS_API_KEY") +BROWSERBASE_API_KEY = os.getenv("BROWSERBASE_API_KEY") +BROWSERBASE_PROJECT_ID = os.getenv("BROWSERBASE_PROJECT_ID") + +if not all([CEREBRAS_API_KEY, BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID]): + raise ValueError("Missing required API keys. Check .env file.") + +# Cerebras model to use for verification and analysis (override via CEREBRAS_MODEL env var) +CEREBRAS_MODEL = os.getenv("CEREBRAS_MODEL", "llama-3.3-70b") + +# Crawl configuration (modify these to control scope and speed) +DEFAULT_URL = "https://docs.stagehand.dev" # Target docs site (can also be passed as CLI arg) +MAX_PAGES = 20 # Maximum number of pages to crawl +MAX_DEPTH = 2 # Maximum link depth from the root page +MAX_CRAWL_WORKERS = 5 # Number of parallel browser sessions for crawling + + +# ── Data Models ──────────────────────────────────────────────────── + + +class Page(BaseModel): + url: str + title: str = "" + content: str = "" + last_updated: Optional[datetime] = None + depth: int = 0 + links: list[dict] = Field(default_factory=list) + broken_links: list[dict] = Field(default_factory=list) + broken_anchors: list[str] = Field(default_factory=list) + + +class Issue(BaseModel): + url: str + type: str # broken_link, outdated, unclear, grammar, missing_context, broken_anchor, error + severity: str # critical, high, medium, low + description: str + suggestion: str + context: str = "" + + +# ── Phase 1: Crawl ──────────────────────────────────────────────── + + +async def _crawl_worker( + worker_id: int, + queue: asyncio.Queue, + visited: dict, + visited_lock: asyncio.Lock, + base_domain: str, + max_pages: int, + max_depth: int, + stagehand: Stagehand, + counter: list, +): + """Worker coroutine: pops URLs from the shared queue, navigates via Stagehand, and extracts + the accessibility tree (aria tree) for each page. Each worker runs its own browser session.""" + + session_id = None + browser = None + pw = None + + try: + # Start a new Browserbase session via the Stagehand REST API + start_response = stagehand.sessions.start(model_name="cerebras/llama-3.3-70b") + session_id = start_response.data.session_id + live_url = f"https://www.browserbase.com/sessions/{session_id}" + print(f" [Worker {worker_id}] Live session: {live_url}") + + # Connect Playwright to the remote browser via Chrome DevTools Protocol (BYOB pattern) + cdp_url = ( + f"wss://connect.browserbase.com?apiKey={BROWSERBASE_API_KEY}&sessionId={session_id}" + ) + + pw = await async_playwright().start() + browser = await pw.chromium.connect_over_cdp(cdp_url) + context = browser.contexts[0] + page = context.pages[0] if context.pages else await context.new_page() + + # HTTP client for checking broken links (HEAD requests) + http = httpx.AsyncClient() + + while True: + async with visited_lock: + if len(visited) >= max_pages: + break + + try: + url, depth = queue.get_nowait() + except asyncio.QueueEmpty: + await asyncio.sleep(0.2) + try: + url, depth = queue.get_nowait() + except asyncio.QueueEmpty: + break + + url = url.split("#")[0] + + async with visited_lock: + if url in visited or len(visited) >= max_pages: + queue.task_done() + continue + visited[url] = None + + if depth > max_depth: + async with visited_lock: + del visited[url] + queue.task_done() + continue + + async with visited_lock: + counter[0] += 1 + page_num = counter[0] + short_url = url.split("/")[-1] or url.split("/")[-2] or "index" + start_time = asyncio.get_event_loop().time() + + try: + # Navigate to the URL using Stagehand's server-side navigation + stagehand.sessions.navigate(id=session_id, url=url) + title = await page.title() + + # Extract the accessibility tree (aria tree) — this is free, no LLM call needed. + # The aria tree gives us the page's text content in a structured format. + extract_response = stagehand.sessions.extract(id=session_id) + result = extract_response.data.result + if isinstance(result, dict): + aria_tree = result.get("pageText", str(result)) + elif isinstance(result, str): + aria_tree = result + else: + aria_tree = str(result) + + # Collect all links on the page for BFS crawling and broken link detection + links = await page.eval_on_selector_all( + "a[href]", + "els => els.map(e => ({href: e.href, text: e.textContent.trim().slice(0,30)}))", + ) + + # Check first 15 links for broken URLs using HEAD requests + broken = [] + for link in links[:15]: + href = link["href"] + if href.startswith("http"): + try: + r = await http.head(href, follow_redirects=True, timeout=8) + if r.status_code >= 400: + broken.append( + {"url": href, "text": link["text"], "status": r.status_code} + ) + except Exception: + broken.append({"url": href, "text": link["text"], "status": 0}) + + # Find broken anchor links (e.g. #section-name that doesn't exist on the page) + broken_anchors = await page.evaluate("""() => { + return Array.from(document.querySelectorAll('a[href^="#"]')) + .map(a => a.href.split('#')[1]) + .filter(id => { + if (!id || id.length <= 1 || id === 'top') return false; + return !document.getElementById(id); + }); + }""") + + elapsed = asyncio.get_event_loop().time() - start_time + content_kb = len(aria_tree) / 1024 + print( + f" [{page_num}/{max_pages}] {short_url}" + f" — aria: {content_kb:.1f}KB, {len(broken)} broken links" + f" ({elapsed:.1f}s)" + ) + + page_obj = Page( + url=url, + title=title, + content=aria_tree, + depth=depth, + links=links, + broken_links=broken, + broken_anchors=broken_anchors, + ) + async with visited_lock: + visited[url] = page_obj + + # Add newly discovered same-domain links to the BFS queue + for link in links: + href = link["href"].split("#")[0] + parsed = urlparse(href) + async with visited_lock: + if parsed.netloc == base_domain and href not in visited: + queue.put_nowait((href, depth + 1)) + + except Exception as e: + elapsed = asyncio.get_event_loop().time() - start_time + print(f" [{page_num}/{max_pages}] {short_url} — ERROR: {e!s:.80} ({elapsed:.1f}s)") + page_obj = Page(url=url, title="[Error]", content=str(e), depth=depth) + async with visited_lock: + visited[url] = page_obj + + queue.task_done() + + await http.aclose() + + finally: + if browser: + await browser.close() + if pw: + await pw.stop() + if session_id: + try: + stagehand.sessions.end(id=session_id) + except Exception: + pass + + +async def crawl( + root_url: str, max_pages: int = 30, max_depth: int = 2, max_workers: int = 5 +) -> list[Page]: + """Parallel BFS crawl using an async work queue with multiple Stagehand sessions. + Each worker gets its own browser session and pulls URLs from a shared queue.""" + parsed_root = urlparse(root_url) + base_domain = parsed_root.netloc + + # Shared state protected by asyncio.Lock (needed because multiple workers access concurrently) + visited: dict[str, Page] = {} + visited_lock = asyncio.Lock() + counter = [0] # Mutable list used as a counter so workers can increment it + + queue = asyncio.Queue() + queue.put_nowait((root_url, 0)) + + # Initialize the Stagehand REST client (used to create and manage browser sessions) + stagehand = Stagehand( + browserbase_api_key=BROWSERBASE_API_KEY, + browserbase_project_id=BROWSERBASE_PROJECT_ID, + model_api_key=CEREBRAS_API_KEY, + ) + + print(f"Starting {max_workers} crawl workers...") + + workers = [ + asyncio.create_task( + _crawl_worker( + i, queue, visited, visited_lock, base_domain, + max_pages, max_depth, stagehand, counter, + ) + ) + for i in range(max_workers) + ] + + await asyncio.gather(*workers) + + result = {url: pg for url, pg in visited.items() if pg is not None} + print(f"Crawled {len(result)} pages\n") + return list(result.values()) + + +# ── Phase 2: Discover GitHub Repository ─────────────────────────── + + +def discover_repo_from_pages(pages: list[Page]) -> Optional[str]: + """Search crawled page content and links for a GitHub repository URL. + This is the fast path — no browser session needed, just regex over existing data.""" + github_pattern = re.compile(r"https?://github\.com/[\w\-]+/[\w\-]+") + + # Check both aria tree text and extracted links for GitHub URLs + for pg in pages: + matches = github_pattern.findall(pg.content) + for m in matches: + parts = m.rstrip("/").split("/") + if len(parts) >= 5: + print(f" Found repo in aria tree: {m}") + return m + + for link in pg.links: + href = link.get("href", "") + matches = github_pattern.findall(href) + for m in matches: + parts = m.rstrip("/").split("/") + if len(parts) >= 5: + print(f" Found repo in page links: {m}") + return m + + print(" No GitHub link found in aria trees or links") + return None + + +async def discover_repo_with_agent(root_url: str, stagehand: Stagehand) -> Optional[str]: + """Fallback: use a Stagehand agent to find the GitHub repo link on a dynamic page. + Some sites render the GitHub link via JavaScript, so regex alone won't find it.""" + print(" Using Stagehand agent to find GitHub repo link...") + + session_id = None + browser = None + pw = None + + try: + # Start a new browser session for the agent + start_response = stagehand.sessions.start(model_name="cerebras/llama-3.3-70b") + session_id = start_response.data.session_id + live_url = f"https://www.browserbase.com/sessions/{session_id}" + print(f" Live session (agent): {live_url}") + + # Connect Playwright via CDP (same BYOB pattern as crawl workers) + cdp_url = ( + f"wss://connect.browserbase.com?apiKey={BROWSERBASE_API_KEY}&sessionId={session_id}" + ) + + pw = await async_playwright().start() + browser = await pw.chromium.connect_over_cdp(cdp_url) + context = browser.contexts[0] + page = context.pages[0] if context.pages else await context.new_page() + + # Navigate and wait for JavaScript to render + stagehand.sessions.navigate(id=session_id, url=root_url) + await page.wait_for_timeout(3000) + + # Use the Stagehand agent to autonomously click around and find the GitHub link + agent_result = stagehand.sessions.execute( + id=session_id, + agent_config={ + "model": { + "modelName": "cerebras/llama-3.3-70b", + "apiKey": CEREBRAS_API_KEY, + }, + }, + execute_options={ + "instruction": ( + "Find the GitHub repository link on this page. It may be in a button, " + "footer, navigation, or generated by JavaScript. Click any button that " + "might reveal it (like a 'Source' or 'GitHub' button). " + "Return the full GitHub URL." + ), + "max_steps": 10, + }, + ) + + # Parse the agent's response for a GitHub URL + result_text = str(agent_result.data) if agent_result.data else "" + github_pattern = re.compile(r"https?://github\.com/[\w\-]+/[\w\-]+") + matches = github_pattern.findall(result_text) + + if matches: + print(f" Agent found repo: {matches[0]}") + return matches[0] + + # Fallback: check the aria tree after the agent has interacted with the page + extract_response = stagehand.sessions.extract(id=session_id) + result = extract_response.data.result + matches = github_pattern.findall(str(result)) + if matches: + print(f" Found repo in post-agent aria tree: {matches[0]}") + return matches[0] + + print(" Agent could not find GitHub repo link") + return None + + except Exception as e: + print(f" Agent error: {e}") + return None + finally: + if browser: + await browser.close() + if pw: + await pw.stop() + if session_id: + try: + stagehand.sessions.end(id=session_id) + except Exception: + pass + + +# ── Phase 3: Clone Repository ───────────────────────────────────── + + +def clone_repo(repo_url: str) -> Optional[Path]: + """Clone a GitHub repo to a temp directory. Returns the path.""" + repo_url = repo_url.rstrip("/") + clone_url = repo_url + ".git" if not repo_url.endswith(".git") else repo_url + repo_name = repo_url.split("/")[-1] + clone_dir = Path(tempfile.mkdtemp(prefix="docs-verify-")) / repo_name + + print(f" Cloning {repo_url} -> {clone_dir}") + try: + subprocess.run( + ["git", "clone", clone_url, str(clone_dir)], + capture_output=True, text=True, timeout=60, check=True, + ) + print(f" Cloned successfully") + return clone_dir + except subprocess.CalledProcessError as e: + print(f" Clone failed: {e.stderr.strip()}") + return None + except Exception as e: + print(f" Clone error: {e}") + return None + + +# ── Phase 4: Verification Agent (Cerebras tool calling) ────────── + +# Tools the Cerebras verification agent can call to inspect the cloned codebase. +# Each tool maps to a function in _execute_verification_tool(). +VERIFICATION_TOOLS = [ + { + "type": "function", + "function": { + "name": "grep_codebase", + "description": "Search for a pattern (regex) across all source files in the codebase.", + "parameters": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "A function name, class name, variable, or regex pattern", + } + }, + "required": ["pattern"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "read_file", + "description": "Read the full contents of a source file from the codebase.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Relative path to the file, e.g. 'src/auth.py'", + } + }, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "list_files", + "description": "List all files in the codebase.", + "parameters": {"type": "object", "properties": {}}, + }, + }, +] + +VERIFICATION_SYSTEM_PROMPT = """You are a documentation accuracy verifier. Your job is to check whether \ +a documentation page is accurate against the actual source code of the project. + +You have access to tools that let you search and read the codebase. Use them to verify every claim \ +in the documentation: +- Function signatures (names, parameters, defaults, return types) +- Class constructors and their parameters +- Method names and their existence +- Code examples (do they use the correct API?) +- Endpoint paths and constants +- Data model fields + +For each issue you find, report it with: +- type: one of "outdated", "wrong_signature", "removed_function", "missing_parameter", \ +"renamed_method", "wrong_endpoint", "missing_function" +- severity: "critical" for removed/renamed APIs, "high" for wrong signatures/params, \ +"medium" for other inaccuracies +- description: what is wrong +- suggestion: what the correct version should be (based on source code) +- context: the specific text in the docs that is wrong + +Be thorough. Check EVERY function, parameter, code example, and data model mentioned in the docs. +Also check if the codebase has new functions/methods NOT mentioned in the docs. + +Use one tool at a time. Return your findings as a JSON object: {"issues": [...]}""" + + +def _execute_verification_tool(name: str, args: dict, codebase_path: Path) -> str: + """Execute a verification tool call against the cloned codebase.""" + if name == "grep_codebase": + pattern = args.get("pattern", "") + try: + result = subprocess.run( + ["grep", "-rn", pattern, str(codebase_path)], + capture_output=True, text=True, timeout=10, + ) + output = result.stdout.strip() + if not output: + return f"No matches found for pattern: {pattern}" + output = output.replace(str(codebase_path) + "/", "") + return output[:4000] + except Exception as e: + return f"Error: {e}" + + elif name == "read_file": + path = args.get("path", "") + full_path = codebase_path / path + if not full_path.exists(): + return f"File not found: {path}" + try: + return full_path.read_text()[:8000] + except Exception as e: + return f"Error reading file: {e}" + + elif name == "list_files": + files = [] + for p in sorted(codebase_path.rglob("*")): + if p.is_file() and not any( + part.startswith(".") for part in p.relative_to(codebase_path).parts + ): + files.append(str(p.relative_to(codebase_path))) + return "\n".join(files) + + return f"Unknown tool: {name}" + + +def _get_codebase_listing(codebase_path: Path) -> str: + """Get a file listing for the codebase.""" + files = [] + for p in sorted(codebase_path.rglob("*")): + if p.is_file() and not any( + part.startswith(".") for part in p.relative_to(codebase_path).parts + ): + files.append(str(p.relative_to(codebase_path))) + return "\n".join(files) + + +def verify_page( + pg: Page, + llm: OpenAI, + codebase_path: Path, + file_listing: str, + page_num: int, + total: int, + max_turns: int = 10, +) -> list[Issue]: + """Run the Cerebras verification agent on a single page's documentation content. + The agent uses tool calling to grep and read the source code, then reports inaccuracies.""" + short_url = pg.url.split("/")[-1] or pg.url.split("/")[-2] or "index" + + if pg.title == "[Error]": + print(f" [{page_num}/{total}] Skip: {short_url} (crawl error)") + return [ + Issue( + url=pg.url, type="error", severity="high", + description="Page failed to load during crawl", + suggestion="Check URL accessibility", context=pg.content[:100], + ) + ] + + # Build the conversation with system prompt + page content for the agent + messages = [ + {"role": "system", "content": VERIFICATION_SYSTEM_PROMPT}, + { + "role": "user", + "content": ( + f"Here are the files in the codebase:\n{file_listing}\n\n" + f"Now verify this documentation page against the codebase. " + f"The docs may be outdated — check every function, parameter, and code example. " + f"Use one tool at a time.\n\n" + f"Page URL: {pg.url}\nPage title: {pg.title}\n\n" + f"Page content (aria tree):\n{pg.content[:12000]}" + ), + }, + ] + + # Multi-turn tool calling loop: the agent greps/reads code, then returns findings as JSON + print(f" [{page_num}/{total}] Verifying {short_url}...", end="", flush=True) + turns_used = 0 + + for turn in range(max_turns): + try: + # Ask the model to verify the docs, allowing it to call tools + resp = llm.chat.completions.create( + model=CEREBRAS_MODEL, messages=messages, + tools=VERIFICATION_TOOLS, tool_choice="auto", + ) + except Exception as e: + # Some models return tool_use_failed when tool calling isn't supported — retry without tools + if "tool_use_failed" in str(e): + resp = llm.chat.completions.create(model=CEREBRAS_MODEL, messages=messages) + else: + raise + + msg = resp.choices[0].message + messages.append(msg) + turns_used += 1 + + # If no tool calls, the agent is done — parse its JSON response + if not msg.tool_calls: + content = msg.content or "" + issues = _parse_verification_response(content, pg.url) + break + else: + # Execute each tool call and feed results back to the agent + for tc in msg.tool_calls: + args = json.loads(tc.function.arguments) + print(f" {tc.function.name}", end="", flush=True) + result = _execute_verification_tool(tc.function.name, args, codebase_path) + messages.append({"role": "tool", "tool_call_id": tc.id, "content": result}) + else: + # Force the agent to return its findings if it used all turns without finishing + messages.append( + {"role": "user", "content": "Provide your final JSON response now with all issues found."} + ) + resp = llm.chat.completions.create(model=CEREBRAS_MODEL, messages=messages) + issues = _parse_verification_response(resp.choices[0].message.content or "", pg.url) + + # Append broken links and anchors detected during crawl (Phase 1) + for link in pg.broken_links: + issues.append( + Issue( + url=pg.url, type="broken_link", severity="high", + description=f"Broken link: {link.get('text', '?')} -> status {link.get('status', '?')}", + suggestion="Fix or remove the link", context=link.get("url", ""), + ) + ) + for anchor in pg.broken_anchors: + issues.append( + Issue( + url=pg.url, type="broken_anchor", severity="medium", + description=f"Broken anchor: #{anchor}", + suggestion="Add missing ID or fix link", context=anchor, + ) + ) + + print(f" {len(issues)} issues ({turns_used} turns)") + return issues + + +def _parse_verification_response(content: str, url: str) -> list[Issue]: + """Parse the verification agent's JSON response into Issue objects.""" + json_match = re.search(r'\{[\s\S]*"issues"[\s\S]*\}', content) + if not json_match: + return [] + + try: + data = json.loads(json_match.group()) + issues = [] + for item in data.get("issues", []): + issues.append( + Issue( + url=url, + type=item.get("type", "outdated"), + severity=item.get("severity", "medium"), + description=item.get("description", ""), + suggestion=item.get("suggestion", ""), + context=item.get("context", ""), + ) + ) + return issues + except json.JSONDecodeError: + return [] + + +def verify_all(pages: list[Page], codebase_path: Path) -> list[Issue]: + """Verify all pages against the cloned codebase using Cerebras tool calling. + Pages are processed sequentially because each verification runs a multi-turn conversation.""" + all_issues = [] + file_listing = _get_codebase_listing(codebase_path) + + print(f"Verifying {len(pages)} pages against {codebase_path}") + print(f"Codebase files:\n {file_listing.replace(chr(10), chr(10) + ' ')}\n") + + # Cerebras uses an OpenAI-compatible API, so we use the OpenAI client with a custom base URL + llm = OpenAI( + base_url="https://api.cerebras.ai/v1", + api_key=CEREBRAS_API_KEY, + timeout=30.0, + ) + + for i, pg in enumerate(pages): + issues = verify_page(pg, llm, codebase_path, file_listing, i + 1, len(pages)) + all_issues.extend(issues) + + print(f"\nDone: {len(all_issues)} issues across {len(pages)} pages") + return all_issues + + +# ── Fallback Analysis (when no source repo is found) ───────────── +# When we can't find or clone the GitHub repo, we fall back to content-only analysis. +# This checks for outdated info, unclear writing, and missing context without source code. + +ANALYSIS_PROMPT = """Current date and time: {current_datetime} + +You are a documentation quality checker. Analyze the following accessibility tree +from a documentation page and identify content issues. + +Look for: outdated information (check copyright years, old version numbers), +unclear writing, grammar errors, missing context. + +When checking dates and years, remember that today is {current_datetime}. +A copyright showing the current year is CORRECT, not outdated. + +Return your findings as a JSON object with an "issues" array. Each issue has: +- type: one of "outdated", "unclear", "grammar", "missing_context" +- severity: one of "critical", "high", "medium", "low" +- description: what the issue is +- suggestion: how to fix it +- context: the relevant text snippet + +If no issues are found, return {{"issues": []}}. + +Page URL: {url} +Page title: {title} + +Accessibility tree: +{content}""" + + +async def analyze_page( + pg: Page, llm: AsyncOpenAI, page_num: int, total: int, current_datetime: str, +) -> list[Issue]: + """Analyze a single page by sending its aria tree to Cerebras.""" + short_url = pg.url.split("/")[-1] or pg.url.split("/")[-2] or "index" + start = asyncio.get_event_loop().time() + + if pg.title == "[Error]": + print(f" [{page_num}/{total}] Skip: {short_url} (crawl error)") + return [ + Issue( + url=pg.url, type="error", severity="high", + description="Page failed to load during crawl", + suggestion="Check URL accessibility", context=pg.content[:100], + ) + ] + + try: + prompt = ANALYSIS_PROMPT.format( + current_datetime=current_datetime, url=pg.url, + title=pg.title, content=pg.content[:12000], + ) + + resp = await llm.chat.completions.create( + model=CEREBRAS_MODEL, + messages=[{"role": "user", "content": prompt}], + response_format={"type": "json_object"}, + ) + + raw = resp.choices[0].message.content + data = json.loads(raw) + issues = [] + for item in data.get("issues", []): + issues.append( + Issue( + url=pg.url, + type=item.get("type", "unclear"), + severity=item.get("severity", "medium"), + description=item.get("description", ""), + suggestion=item.get("suggestion", ""), + context=item.get("context", ""), + ) + ) + except Exception as e: + elapsed = asyncio.get_event_loop().time() - start + print(f" [{page_num}/{total}] Error: {short_url} — {e!s:.80} ({elapsed:.1f}s)") + issues = [] + + for link in pg.broken_links: + issues.append( + Issue( + url=pg.url, type="broken_link", severity="high", + description=f"Broken link: {link.get('text', '?')} -> status {link.get('status', '?')}", + suggestion="Fix or remove the link", context=link.get("url", ""), + ) + ) + for anchor in pg.broken_anchors: + issues.append( + Issue( + url=pg.url, type="broken_anchor", severity="medium", + description=f"Broken anchor: #{anchor}", + suggestion="Add missing ID or fix link", context=anchor, + ) + ) + + elapsed = asyncio.get_event_loop().time() - start + print(f" [{page_num}/{total}] Done: {short_url} — {len(issues)} issues ({elapsed:.1f}s)") + return issues + + +async def analyze_all_parallel(pages: list[Page], max_workers: int = 10) -> list[Issue]: + """Analyze all pages via direct Cerebras API calls (no browser sessions needed). + Pages are processed in batches for concurrency without overwhelming the API.""" + all_issues = [] + current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + print(f"Current date/time sent to LLM: {current_datetime}") + print(f"Analyzing {len(pages)} pages (max_workers={max_workers})\n") + + llm = AsyncOpenAI( + base_url="https://api.cerebras.ai/v1", + api_key=CEREBRAS_API_KEY, + ) + + for i in range(0, len(pages), max_workers): + batch = pages[i : i + max_workers] + batch_end = min(i + max_workers, len(pages)) + print(f"Batch [{i + 1}-{batch_end}/{len(pages)}]") + + tasks = [ + analyze_page(pg, llm, i + j + 1, len(pages), current_datetime) + for j, pg in enumerate(batch) + ] + batch_results = await asyncio.gather(*tasks, return_exceptions=True) + + for j, result in enumerate(batch_results): + if isinstance(result, list): + all_issues.extend(result) + elif isinstance(result, Exception): + page_url = batch[j].url.split("/")[-1] or "index" + print(f" Error: {page_url} — {result}") + + print() + + print(f"Done: {len(all_issues)} issues across {len(pages)} pages") + return all_issues + + +# ── Display and Export ──────────────────────────────────────────── + + +def print_summary(pages: list[Page], issues: list[Issue], root_url: str): + """Print summary statistics to the terminal.""" + severity_counts = Counter(i.severity for i in issues) + type_counts = Counter(i.type for i in issues) + + print(f"\n{'=' * 60}") + print(f" Documentation Analysis Summary") + print(f"{'=' * 60}") + print(f" Site: {root_url}") + print(f" Pages crawled: {len(pages)}") + print(f" Total issues: {len(issues)}") + print(f" Critical: {severity_counts.get('critical', 0)}") + print(f" High: {severity_counts.get('high', 0)}") + print(f" Medium: {severity_counts.get('medium', 0)}") + print(f" Low: {severity_counts.get('low', 0)}") + print(f" By type: {', '.join(f'{t}: {c}' for t, c in type_counts.most_common())}") + print(f"{'=' * 60}\n") + + +def print_issues(issues: list[Issue], severity_filter: str = None): + """Print issues to the terminal.""" + filtered = [i for i in issues if severity_filter is None or i.severity == severity_filter] + if not filtered: + print("No issues found.") + return + + for issue in filtered: + page_name = issue.url.split("/")[-1] or "index" + print(f" [{issue.severity.upper()}] [{issue.type}] {page_name}") + print(f" {issue.description}") + print(f" Suggestion: {issue.suggestion}") + if issue.context: + print(f" Context: {issue.context[:80]}") + print() + + +def export_markdown(pages: list[Page], issues: list[Issue], root_url: str, filename: str = None) -> str: + """Generate and save a markdown report.""" + if filename is None: + filename = f"docs_report_{datetime.now().strftime('%Y%m%d_%H%M')}.md" + + severity_counts = Counter(i.severity for i in issues) + + report = f"""# Documentation Analysis Report + +**Site:** {root_url} +**Date:** {datetime.now().strftime("%Y-%m-%d %H:%M")} +**Pages:** {len(pages)} | **Issues:** {len(issues)} + +## Summary + +| Severity | Count | +|----------|-------| +| Critical | {severity_counts.get("critical", 0)} | +| High | {severity_counts.get("high", 0)} | +| Medium | {severity_counts.get("medium", 0)} | +| Low | {severity_counts.get("low", 0)} | + +## Issues + +""" + for sev in ["critical", "high", "medium", "low"]: + sev_issues = [i for i in issues if i.severity == sev] + if sev_issues: + report += f"### {sev.title()} ({len(sev_issues)})\n\n" + for i in sev_issues: + page_name = i.url.split("/")[-1] or "index" + report += f"- **[{page_name}]** {i.type}: {i.description}\n" + report += f" - *{i.suggestion}*\n" + report += "\n" + + with open(filename, "w") as f: + f.write(report) + + print(f"Saved report to: {filename}") + return report + + +# ── Main ────────────────────────────────────────────────────────── + + +async def main(): + """ + 4-phase documentation checker: + 1. Crawl docs site via parallel Stagehand BYOB workers + 2. Discover the source GitHub repository from crawled pages + 3. Clone the repo locally + 4. Verify docs accuracy against source code using Cerebras tool-calling agent + """ + # Accept target URL as CLI argument or use default + docs_url = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_URL + + print(f"Target: {docs_url}") + print(f"Model: {CEREBRAS_MODEL}") + print(f"Max pages: {MAX_PAGES}, Max depth: {MAX_DEPTH}\n") + + # Phase 1: Crawl the docs site using parallel browser sessions + print("Phase 1: Crawling documentation site...") + pages = await crawl(docs_url, max_pages=MAX_PAGES, max_depth=MAX_DEPTH, max_workers=MAX_CRAWL_WORKERS) + + # Phase 2: Try to find the source GitHub repo (regex first, then agent fallback) + print("Phase 2: Discovering source repository...") + repo_url = discover_repo_from_pages(pages) + + if not repo_url: + # Regex didn't find a repo — try using a Stagehand agent to click around and find it + stagehand = Stagehand( + browserbase_api_key=BROWSERBASE_API_KEY, + browserbase_project_id=BROWSERBASE_PROJECT_ID, + model_api_key=CEREBRAS_API_KEY, + ) + repo_url = await discover_repo_with_agent(docs_url, stagehand) + + if repo_url: + # Phase 3: Clone the repo so the verification agent can inspect source code + print(f"\nPhase 3: Cloning repository...") + codebase_path = clone_repo(repo_url) + + if codebase_path: + # Phase 4: Run the Cerebras tool-calling agent to verify docs against source + print(f"\nPhase 4: Verification agent...") + issues = verify_all(pages, codebase_path) + else: + print("\nClone failed, falling back to basic analysis...") + issues = await analyze_all_parallel(pages, max_workers=10) + else: + # No repo found — fall back to content-only analysis (no source code verification) + print("\nNo source repo found, falling back to basic analysis...") + issues = await analyze_all_parallel(pages, max_workers=10) + + # Display results and save markdown report + print_summary(pages, issues, docs_url) + print_issues(issues) + export_markdown(pages, issues, docs_url) + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as err: + print(f"Application error: {err}") + print("\nCommon issues:") + print(" - Check .env file has CEREBRAS_API_KEY, BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID") + print(" - Ensure playwright is installed: playwright install chromium") + print("Docs: https://docs.stagehand.dev/reference/python/overview") + exit(1) diff --git a/python/cerebras-docs-checker/pyproject.toml b/python/cerebras-docs-checker/pyproject.toml new file mode 100644 index 00000000..5f73ddfc --- /dev/null +++ b/python/cerebras-docs-checker/pyproject.toml @@ -0,0 +1,36 @@ +[project] +name = "cerebras-docs-checker" +version = "0.1.0" +description = "AI-powered documentation checker using Cerebras LLMs, Stagehand, and Browserbase" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "httpx", + "openai", + "playwright", + "pydantic", + "python-dotenv", + "stagehand", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "ruff>=0.1.0", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W"] From a4cf6fb2cfe5cb6480b4057c8dcfe74fb0b01fcf Mon Sep 17 00:00:00 2001 From: Alex Qiu Date: Fri, 27 Feb 2026 11:42:06 -0800 Subject: [PATCH 23/45] Update typescript example for manual-mfa-with-contexts template (#63) --- typescript/manual-mfa-with-contexts/index.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/typescript/manual-mfa-with-contexts/index.ts b/typescript/manual-mfa-with-contexts/index.ts index 785f8dac..13813c60 100644 --- a/typescript/manual-mfa-with-contexts/index.ts +++ b/typescript/manual-mfa-with-contexts/index.ts @@ -2,7 +2,7 @@ import "dotenv/config"; import { Stagehand } from "@browserbasehq/stagehand"; -import Browserbase from "@browserbasehq/sdk"; +import { Browserbase } from "@browserbasehq/sdk"; import { z } from "zod"; const bb = new Browserbase({ @@ -15,9 +15,7 @@ const bb = new Browserbase({ async function createSessionWithContext() { console.log("Creating new Browserbase context..."); - const context = await bb.contexts.create({ - projectId: process.env.BROWSERBASE_PROJECT_ID!, - }); + const context = await bb.createContext() console.log(`Context created: ${context.id}`); console.log("First session: Performing login with MFA..."); From 4d8f10cfb4086caca3a48fcdf72a6a5659880f93 Mon Sep 17 00:00:00 2001 From: Derek <75138022+derekmeegan@users.noreply.github.com> Date: Wed, 11 Mar 2026 15:09:19 -0700 Subject: [PATCH 24/45] Bump JS SDK to v2.7 and Python SDK to v1.5, remove project_id from API calls (#65) * Bump JS SDK to v2.7 and Python SDK to v1.5, remove project_id from API calls - Bump @browserbasehq/sdk to ^2.7.0 in all TypeScript templates - Bump browserbase to >=1.5.0 in all Python templates - Remove projectId/project_id from bb.sessions.create(), bb.contexts.create(), and browserbaseSessionCreateParams/browserbase_session_create_params since it no longer needs to be explicitly set Co-Authored-By: Claude Opus 4.6 * Remove empty object args from sessions.create() and contexts.create() Co-Authored-By: Claude Opus 4.6 * readmes and env examples --------- Co-authored-by: Claude Opus 4.6 --- python/amazon-global-price-comparison/main.py | 1 - .../amazon-global-price-comparison/pyproject.toml | 2 +- python/basic-recaptcha/main.py | 1 - python/browserbase-reducto/pyproject.toml | 2 +- python/business-lookup/main.py | 3 --- python/company-address-finder/main.py | 1 - python/context/main.py | 4 +--- python/extend-browserbase/pyproject.toml | 2 +- python/gemini-cua/main.py | 1 - python/manual-mfa-with-contexts/main.py | 4 +--- python/manual-mfa-with-contexts/requirements.txt | 2 +- python/playwright-mfa-handling/.env.example | 1 - python/playwright-mfa-handling/README.md | 4 ++-- python/playwright-mfa-handling/main.py | 7 ++----- python/playwright-mfa-handling/pyproject.toml | 2 +- python/playwright/basic-recaptcha/.env.example | 1 - python/playwright/basic-recaptcha/README.md | 7 ++----- python/playwright/basic-recaptcha/main.py | 14 +++++--------- python/playwright/basic-recaptcha/pyproject.toml | 2 +- python/proxies-weather/main.py | 1 - python/proxies/main.py | 3 --- typescript/amazon-global-price-comparison/index.ts | 1 - typescript/basic-caching/index.ts | 6 ------ typescript/browserbase-reducto/package.json | 2 +- typescript/company-value-prop-generator/index.ts | 3 --- typescript/context/index.ts | 6 +----- typescript/extend-browserbase/package.json | 2 +- typescript/form-filling/index.ts | 3 --- typescript/gemini-3-flash/index.ts | 1 - typescript/gemini-cua/index.ts | 1 - typescript/gift-finder/index.ts | 1 - typescript/manual-mfa-with-contexts/index.ts | 2 -- typescript/manual-mfa-with-contexts/package.json | 2 +- typescript/microsoft-cua/index.ts | 1 - typescript/pickleball/index.ts | 1 - typescript/playwright-mfa-handling/.env.example | 1 - typescript/playwright-mfa-handling/README.md | 4 ++-- typescript/playwright-mfa-handling/index.ts | 10 ++-------- typescript/playwright-mfa-handling/package.json | 2 +- typescript/playwright/basic-recaptcha/.env.example | 1 - typescript/playwright/basic-recaptcha/README.md | 4 ++-- typescript/playwright/basic-recaptcha/index.ts | 13 ++++--------- typescript/playwright/basic-recaptcha/package.json | 2 +- typescript/proxies-weather/index.ts | 1 - typescript/proxies/index.ts | 3 --- 45 files changed, 35 insertions(+), 103 deletions(-) diff --git a/python/amazon-global-price-comparison/main.py b/python/amazon-global-price-comparison/main.py index 30c6168a..516459f7 100644 --- a/python/amazon-global-price-comparison/main.py +++ b/python/amazon-global-price-comparison/main.py @@ -114,7 +114,6 @@ async def get_products_for_country( print(f"Creating Browserbase session with {country.name} proxy...") session = await asyncio.to_thread( bb.sessions.create, - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), proxies=[ { "type": "browserbase", # Use Browserbase's managed proxy infrastructure diff --git a/python/amazon-global-price-comparison/pyproject.toml b/python/amazon-global-price-comparison/pyproject.toml index 3ed41f76..dcd57d87 100644 --- a/python/amazon-global-price-comparison/pyproject.toml +++ b/python/amazon-global-price-comparison/pyproject.toml @@ -5,7 +5,7 @@ description = "Compare Amazon product prices across multiple countries using geo readme = "README.md" requires-python = ">=3.9" dependencies = [ - "browserbase", + "browserbase>=1.5.0", "python-dotenv", "pydantic>=2.0.0", "stagehand", diff --git a/python/basic-recaptcha/main.py b/python/basic-recaptcha/main.py index dc13bab5..d6274d08 100644 --- a/python/basic-recaptcha/main.py +++ b/python/basic-recaptcha/main.py @@ -22,7 +22,6 @@ def main(): # Create session with captcha solving enabled session = bb.sessions.create( - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), browser_settings={ "solveCaptchas": solve_captchas, }, diff --git a/python/browserbase-reducto/pyproject.toml b/python/browserbase-reducto/pyproject.toml index c6ccd25a..be572df2 100644 --- a/python/browserbase-reducto/pyproject.toml +++ b/python/browserbase-reducto/pyproject.toml @@ -5,7 +5,7 @@ description = "Download Apple's Q4 Financial Statement and Parse with Reducto us readme = "README.md" requires-python = ">=3.9" dependencies = [ - "browserbase", + "browserbase>=1.5.0", "python-dotenv", "reductoai", "stagehand", diff --git a/python/business-lookup/main.py b/python/business-lookup/main.py index 74358685..423f9adf 100644 --- a/python/business-lookup/main.py +++ b/python/business-lookup/main.py @@ -27,9 +27,6 @@ async def main(): project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), model_name="openai/gpt-4.1", model_api_key=os.environ.get("OPENAI_API_KEY"), - browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), - }, verbose=1, # 0 = errors only, 1 = info, 2 = debug # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) # https://docs.stagehand.dev/configuration/logging diff --git a/python/company-address-finder/main.py b/python/company-address-finder/main.py index f2764923..a3291b90 100644 --- a/python/company-address-finder/main.py +++ b/python/company-address-finder/main.py @@ -80,7 +80,6 @@ async def process_company(company_name: str) -> CompanyData: # (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) # https://docs.stagehand.dev/configuration/logging browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), "region": "us-east-1", "timeout": 900, "browser_settings": { diff --git a/python/context/main.py b/python/context/main.py index eaf6fc40..fe1f99c0 100644 --- a/python/context/main.py +++ b/python/context/main.py @@ -18,14 +18,13 @@ def create_session_context_id(): print("Creating new Browserbase context...") # First create a context using Browserbase SDK to get a context ID. bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) - context = bb.contexts.create(project_id=os.environ.get("BROWSERBASE_PROJECT_ID")) + context = bb.contexts.create() print(f"Created context ID: {context.id}") # Create a single session using the context ID to perform initial login. print("Creating session for initial login...") session = bb.sessions.create( - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), browser_settings={ "context": { "id": context.id, @@ -124,7 +123,6 @@ def main(): # persist: true ensures any new changes (cookies, cache) are saved back to context. bb = Browserbase(api_key=os.environ.get("BROWSERBASE_API_KEY")) session = bb.sessions.create( - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), browser_settings={ "context": { "id": context_id["id"], diff --git a/python/extend-browserbase/pyproject.toml b/python/extend-browserbase/pyproject.toml index bc789e20..c1a3593e 100644 --- a/python/extend-browserbase/pyproject.toml +++ b/python/extend-browserbase/pyproject.toml @@ -5,7 +5,7 @@ description = "Download expense receipts and parse with Extend AI using Stagehan readme = "README.md" requires-python = ">=3.9" dependencies = [ - "browserbase>=1.4.0", + "browserbase>=1.5.0", "extend-ai>=1.0.0", "python-dotenv>=1.2.1", "stagehand>=3.5.0", diff --git a/python/gemini-cua/main.py b/python/gemini-cua/main.py index b5491bac..89539a88 100644 --- a/python/gemini-cua/main.py +++ b/python/gemini-cua/main.py @@ -41,7 +41,6 @@ async def main(): "GOOGLE_API_KEY" ), # this is the model stagehand uses in act, observe, extract (not agent) browserbase_session_create_params={ - "project_id": os.environ.get("BROWSERBASE_PROJECT_ID"), "proxies": True, # Using proxies will give the agent a better chance of success - requires Developer Plan or higher, comment out if you don't have access "region": "us-west-2", "browser_settings": {"block_ads": True, "viewport": {"width": 1288, "height": 711}}, diff --git a/python/manual-mfa-with-contexts/main.py b/python/manual-mfa-with-contexts/main.py index f44cad0c..136338ee 100644 --- a/python/manual-mfa-with-contexts/main.py +++ b/python/manual-mfa-with-contexts/main.py @@ -21,14 +21,13 @@ def create_session_with_context(): """First session: Create context and login (with MFA)""" print("Creating new Browserbase context...") - context = bb.contexts.create(project_id=os.environ.get("BROWSERBASE_PROJECT_ID")) + context = bb.contexts.create() print(f"Context created: {context.id}") print("First session: Performing login with MFA...") # Create session with context session = bb.sessions.create( - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), browser_settings={ "context": { "id": context.id, @@ -139,7 +138,6 @@ def reuse_context(context_id: str): # Create session with existing context session = bb.sessions.create( - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), browser_settings={ "context": { "id": context_id, diff --git a/python/manual-mfa-with-contexts/requirements.txt b/python/manual-mfa-with-contexts/requirements.txt index e180bbee..15596bbf 100644 --- a/python/manual-mfa-with-contexts/requirements.txt +++ b/python/manual-mfa-with-contexts/requirements.txt @@ -1,4 +1,4 @@ -browserbase +browserbase>=1.5.0 python-dotenv pydantic stagehand diff --git a/python/playwright-mfa-handling/.env.example b/python/playwright-mfa-handling/.env.example index 36c24713..8c3f7702 100644 --- a/python/playwright-mfa-handling/.env.example +++ b/python/playwright-mfa-handling/.env.example @@ -1,3 +1,2 @@ # Browserbase Configuration -BROWSERBASE_PROJECT_ID=your_browserbase_project_id BROWSERBASE_API_KEY=your_browserbase_api_key diff --git a/python/playwright-mfa-handling/README.md b/python/playwright-mfa-handling/README.md index 7980e10e..88216824 100644 --- a/python/playwright-mfa-handling/README.md +++ b/python/playwright-mfa-handling/README.md @@ -75,7 +75,7 @@ has_failure = await page.locator('text="Login Failure"').is_visible() 3. pip install . 4. playwright install chromium 5. cp .env.example .env -6. Add required API keys/IDs to .env (BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY) +6. Add required API keys/IDs to .env (BROWSERBASE_API_KEY) 7. python main.py ## EXPECTED OUTPUT @@ -94,7 +94,7 @@ has_failure = await page.locator('text="Login Failure"').is_visible() - Dependency install errors: ensure pip install . completed - Missing Playwright browsers: run `playwright install chromium` after installing dependencies -- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- Missing credentials: verify .env contains BROWSERBASE_API_KEY - TOTP code expiration: codes are valid for 30 seconds - if authentication fails, the script automatically retries with a fresh code - CDP connection issues: ensure stable internet connection for reliable Browserbase connection - Selector changes: if the demo site structure changes, Playwright selectors may need updating diff --git a/python/playwright-mfa-handling/main.py b/python/playwright-mfa-handling/main.py index 761802b0..fc6edf64 100644 --- a/python/playwright-mfa-handling/main.py +++ b/python/playwright-mfa-handling/main.py @@ -72,15 +72,12 @@ def generate_totp(secret: str, window: int = 0) -> str: async def create_browserbase_session() -> tuple[Browser, BrowserContext, Page, str]: """Create a Browserbase session and connect via CDP.""" api_key = os.environ.get("BROWSERBASE_API_KEY") - project_id = os.environ.get("BROWSERBASE_PROJECT_ID") if not api_key: raise ValueError("BROWSERBASE_API_KEY environment variable is required") - if not project_id: - raise ValueError("BROWSERBASE_PROJECT_ID environment variable is required") bb = Browserbase(api_key=api_key) - session = bb.sessions.create(project_id=project_id) + session = bb.sessions.create() print(f"Session created: https://browserbase.com/sessions/{session.id}") @@ -244,7 +241,7 @@ async def main(): except Exception as err: print(f"\nError in MFA handling: {err}") print("\nCommon issues:") - print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") + print(" - Check .env file has BROWSERBASE_API_KEY") print(" - TOTP code may have expired (try running again)") print(" - Page structure may have changed") exit(1) diff --git a/python/playwright-mfa-handling/pyproject.toml b/python/playwright-mfa-handling/pyproject.toml index ae919862..34e6773e 100644 --- a/python/playwright-mfa-handling/pyproject.toml +++ b/python/playwright-mfa-handling/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" requires-python = ">=3.9" dependencies = [ "playwright", - "browserbase", + "browserbase>=1.5.0", "python-dotenv", ] diff --git a/python/playwright/basic-recaptcha/.env.example b/python/playwright/basic-recaptcha/.env.example index dfaaf699..9e87821b 100644 --- a/python/playwright/basic-recaptcha/.env.example +++ b/python/playwright/basic-recaptcha/.env.example @@ -1,3 +1,2 @@ # Browserbase credentials - Get these from https://www.browserbase.com/overview BROWSERBASE_API_KEY=your_browserbase_api_key_here -BROWSERBASE_PROJECT_ID=your_browserbase_project_id_here diff --git a/python/playwright/basic-recaptcha/README.md b/python/playwright/basic-recaptcha/README.md index 51f708be..1b930716 100644 --- a/python/playwright/basic-recaptcha/README.md +++ b/python/playwright/basic-recaptcha/README.md @@ -43,7 +43,6 @@ For non-standard or custom captcha providers, you can specify CSS selectors to g ```python session = bb.sessions.create( - project_id=project_id, browser_settings={ "solveCaptchas": True, "captchaImageSelector": "#custom-captcha-image-id", @@ -64,7 +63,6 @@ If you want to disable automatic captcha solving, set `solveCaptchas: False` in ```python session = bb.sessions.create( - project_id=project_id, browser_settings={"solveCaptchas": False}, ) ``` @@ -75,7 +73,6 @@ Enable proxies alongside CAPTCHA solving for better success rates: ```python session = bb.sessions.create( - project_id=project_id, browser_settings={"solveCaptchas": True}, proxies=True, # Enable Browserbase managed proxies ) @@ -88,7 +85,7 @@ session = bb.sessions.create( 3. `source venv/bin/activate` # On Windows: `venv\Scripts\activate` 4. `uv pip install .` # Install dependencies from pyproject.toml 5. `playwright install chromium` # Install browser binaries -6. `cp .env.example .env` # Add your Browserbase API key and Project ID to .env +6. `cp .env.example .env` # Add your Browserbase API key to .env 7. `python main.py` ## EXPECTED OUTPUT @@ -106,7 +103,7 @@ session = bb.sessions.create( ## COMMON PITFALLS -- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- Missing credentials: verify .env contains BROWSERBASE_API_KEY - Browser not installed: run `playwright install chromium` to install browser binaries - Captcha solving timeout: the template uses a 60-second timeout; increase if needed for complex CAPTCHAs - No browser context: ensure the Browserbase session was created successfully before connecting diff --git a/python/playwright/basic-recaptcha/main.py b/python/playwright/basic-recaptcha/main.py index 2a03264a..912fad87 100644 --- a/python/playwright/basic-recaptcha/main.py +++ b/python/playwright/basic-recaptcha/main.py @@ -16,32 +16,29 @@ SOLVE_CAPTCHAS = True # Set to False to disable automatic captcha solving -def validate_env() -> tuple[str, str]: +def validate_env() -> str: """Validate required environment variables before starting. Browserbase credentials are required to create browser sessions. Returns: - Tuple of (api_key, project_id) + The api_key string Raises: ValueError: If required environment variables are missing """ api_key = os.environ.get("BROWSERBASE_API_KEY") - project_id = os.environ.get("BROWSERBASE_PROJECT_ID") if not api_key: raise ValueError("BROWSERBASE_API_KEY environment variable is required") - if not project_id: - raise ValueError("BROWSERBASE_PROJECT_ID environment variable is required") - return api_key, project_id + return api_key async def main(): print("Starting Playwright + Browserbase reCAPTCHA Example...") - api_key, project_id = validate_env() + api_key = validate_env() # Initialize the Browserbase SDK client. bb = Browserbase(api_key=api_key) @@ -52,7 +49,6 @@ async def main(): # Docs: https://docs.browserbase.com/features/stealth-mode#captcha-solving print("Creating Browserbase session with captcha solving enabled...") session = bb.sessions.create( - project_id=project_id, browser_settings={"solveCaptchas": SOLVE_CAPTCHAS}, ) @@ -145,7 +141,7 @@ def handle_console(msg): except Exception as err: print(f"Application error: {err}") print("\nCommon issues:") - print(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY") + print(" - Check .env file has BROWSERBASE_API_KEY") print(" - Verify solveCaptchas is enabled (True by default)") print(" - Allow up to 60 seconds for CAPTCHA solving to complete") print(" - Enable proxies for higher success rates") diff --git a/python/playwright/basic-recaptcha/pyproject.toml b/python/playwright/basic-recaptcha/pyproject.toml index 9ea927b8..1f6db4cd 100644 --- a/python/playwright/basic-recaptcha/pyproject.toml +++ b/python/playwright/basic-recaptcha/pyproject.toml @@ -5,7 +5,7 @@ description = "Basic reCAPTCHA solving with Playwright and Browserbase" readme = "README.md" requires-python = ">=3.9" dependencies = [ - "browserbase", + "browserbase>=1.5.0", "playwright", "python-dotenv", ] diff --git a/python/proxies-weather/main.py b/python/proxies-weather/main.py index 1ad3e948..f87f33a3 100644 --- a/python/proxies-weather/main.py +++ b/python/proxies-weather/main.py @@ -66,7 +66,6 @@ def get_weather_for_location(geolocation: GeolocationConfig) -> WeatherResult: # Create session with geolocation proxy session = bb.sessions.create( - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), proxies=[proxy_config], ) session_id = session.id diff --git a/python/proxies/main.py b/python/proxies/main.py index bac56b49..134ce83a 100644 --- a/python/proxies/main.py +++ b/python/proxies/main.py @@ -32,7 +32,6 @@ class GeoInfo(BaseModel): def create_session_with_built_in_proxies(): # Use Browserbase's default proxy rotation for enhanced privacy and IP diversity. session = bb.sessions.create( - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), proxies=True, ) return session @@ -41,7 +40,6 @@ def create_session_with_built_in_proxies(): def create_session_with_geo_location(): # Route traffic through specific geographic location to test location-based restrictions. session = bb.sessions.create( - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), proxies=[ { "type": "browserbase", @@ -59,7 +57,6 @@ def create_session_with_geo_location(): def create_session_with_custom_proxies(): # Use external proxy servers for custom routing or specific proxy requirements. session = bb.sessions.create( - project_id=os.environ.get("BROWSERBASE_PROJECT_ID"), proxies=[ { "type": "external", diff --git a/typescript/amazon-global-price-comparison/index.ts b/typescript/amazon-global-price-comparison/index.ts index bf6ccb98..eda3af1e 100644 --- a/typescript/amazon-global-price-comparison/index.ts +++ b/typescript/amazon-global-price-comparison/index.ts @@ -84,7 +84,6 @@ async function getProductsForCountry( // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) // https://docs.stagehand.dev/configuration/logging browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID!, proxies: [ { type: "browserbase", // Use Browserbase's managed proxy infrastructure for reliable geolocation routing diff --git a/typescript/basic-caching/index.ts b/typescript/basic-caching/index.ts index 8fb9ccbc..4a71c25c 100644 --- a/typescript/basic-caching/index.ts +++ b/typescript/basic-caching/index.ts @@ -20,9 +20,6 @@ async function runWithoutCache() { // https://docs.stagehand.dev/configuration/logging model: "google/gemini-2.5-flash", enableCaching: false, - browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID!, - }, }); await stagehand.init(); @@ -69,9 +66,6 @@ async function runWithCache() { model: "google/gemini-2.5-flash", enableCaching: true, cacheDir: CACHE_DIR, - browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID!, - }, }); await stagehand.init(); diff --git a/typescript/browserbase-reducto/package.json b/typescript/browserbase-reducto/package.json index 78f0b4aa..0fdb6b8a 100644 --- a/typescript/browserbase-reducto/package.json +++ b/typescript/browserbase-reducto/package.json @@ -8,7 +8,7 @@ "start": "tsx index.ts" }, "dependencies": { - "@browserbasehq/sdk": "latest", + "@browserbasehq/sdk": "^2.7.0", "@browserbasehq/stagehand": "latest", "adm-zip": "latest", "dotenv": "latest", diff --git a/typescript/company-value-prop-generator/index.ts b/typescript/company-value-prop-generator/index.ts index 1cc3392e..201a8301 100644 --- a/typescript/company-value-prop-generator/index.ts +++ b/typescript/company-value-prop-generator/index.ts @@ -16,9 +16,6 @@ async function generateOneLiner(domain: string): Promise { env: "BROWSERBASE", model: "openai/gpt-4.1", verbose: 0, // 0 = errors only, 1 = info, 2 = debug - browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID!, - }, }); try { diff --git a/typescript/context/index.ts b/typescript/context/index.ts index 6c5977e2..9948b98a 100644 --- a/typescript/context/index.ts +++ b/typescript/context/index.ts @@ -10,16 +10,13 @@ async function createSessionContextID() { console.log("Creating new Browserbase context..."); // First create a context using Browserbase SDK to get a context ID. const bb = new Browserbase({ apiKey: process.env.BROWSERBASE_API_KEY! }); - const context = await bb.contexts.create({ - projectId: process.env.BROWSERBASE_PROJECT_ID!, - }); + const context = await bb.contexts.create(); console.log("Created context ID:", context.id); // Create a single session using the context ID to perform initial login. console.log("Creating session for initial login..."); const session = await bb.sessions.create({ - projectId: process.env.BROWSERBASE_PROJECT_ID!, browserSettings: { context: { id: context.id, @@ -96,7 +93,6 @@ async function main() { model: "openai/gpt-4.1", verbose: 1, browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID!, browserSettings: { context: { id: contextId.id, diff --git a/typescript/extend-browserbase/package.json b/typescript/extend-browserbase/package.json index 9cf1c6d3..de824b93 100644 --- a/typescript/extend-browserbase/package.json +++ b/typescript/extend-browserbase/package.json @@ -8,7 +8,7 @@ "start": "tsx index.ts" }, "dependencies": { - "@browserbasehq/sdk": "^2.6.0", + "@browserbasehq/sdk": "^2.7.0", "@browserbasehq/stagehand": "^3.0.8", "adm-zip": "^0.5.16", "dotenv": "^17.2.4", diff --git a/typescript/form-filling/index.ts b/typescript/form-filling/index.ts index 7329b1b1..5c2ff1a9 100644 --- a/typescript/form-filling/index.ts +++ b/typescript/form-filling/index.ts @@ -21,9 +21,6 @@ async function main() { env: "BROWSERBASE", model: "openai/gpt-4.1", verbose: 1, - browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID!, - }, }); try { diff --git a/typescript/gemini-3-flash/index.ts b/typescript/gemini-3-flash/index.ts index 802cee77..0ec1cb54 100644 --- a/typescript/gemini-3-flash/index.ts +++ b/typescript/gemini-3-flash/index.ts @@ -29,7 +29,6 @@ async function main() { // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) // https://docs.stagehand.dev/configuration/logging browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID!, proxies: true, // Using proxies will give the agent a better chance of success - requires Developer Plan or higher, comment out if you don't have access region: "us-west-2", browserSettings: { diff --git a/typescript/gemini-cua/index.ts b/typescript/gemini-cua/index.ts index 868467d6..7b1b11a7 100644 --- a/typescript/gemini-cua/index.ts +++ b/typescript/gemini-cua/index.ts @@ -29,7 +29,6 @@ async function main() { // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) // https://docs.stagehand.dev/configuration/logging browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID!, proxies: true, // Using proxies will give the agent a better chance of success - requires Developer Plan or higher, comment out if you don't have access region: "us-west-2", browserSettings: { diff --git a/typescript/gift-finder/index.ts b/typescript/gift-finder/index.ts index 56e160cc..d90673e8 100644 --- a/typescript/gift-finder/index.ts +++ b/typescript/gift-finder/index.ts @@ -236,7 +236,6 @@ async function main(): Promise { // https://docs.stagehand.dev/configuration/logging model: "openai/gpt-4.1", browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID!, // Proxies require Developer Plan or higher - comment in if you have a Developer Plan or higher // proxies: [ // { diff --git a/typescript/manual-mfa-with-contexts/index.ts b/typescript/manual-mfa-with-contexts/index.ts index 13813c60..4f5ca5e4 100644 --- a/typescript/manual-mfa-with-contexts/index.ts +++ b/typescript/manual-mfa-with-contexts/index.ts @@ -28,7 +28,6 @@ async function createSessionWithContext() { // https://docs.stagehand.dev/configuration/logging model: "openai/gpt-4.1-mini", browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID!, browserSettings: { context: { id: context.id, @@ -127,7 +126,6 @@ async function reuseContext(contextId: string) { // https://docs.stagehand.dev/configuration/logging model: "openai/gpt-4.1-mini", browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID!, browserSettings: { context: { id: contextId, diff --git a/typescript/manual-mfa-with-contexts/package.json b/typescript/manual-mfa-with-contexts/package.json index 4d63fd4e..e5ab4e5a 100644 --- a/typescript/manual-mfa-with-contexts/package.json +++ b/typescript/manual-mfa-with-contexts/package.json @@ -9,7 +9,7 @@ "dev": "tsx watch index.ts" }, "dependencies": { - "@browserbasehq/sdk": "^1.0.0", + "@browserbasehq/sdk": "^2.7.0", "@browserbasehq/stagehand": "^3.0.0", "dotenv": "^16.4.5", "zod": "^3.23.8" diff --git a/typescript/microsoft-cua/index.ts b/typescript/microsoft-cua/index.ts index 01ad7949..c2864808 100644 --- a/typescript/microsoft-cua/index.ts +++ b/typescript/microsoft-cua/index.ts @@ -29,7 +29,6 @@ async function main() { // (When handling sensitive data like passwords or API keys, set verbose: 0 to prevent secrets from appearing in logs.) // https://docs.stagehand.dev/configuration/logging browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID!, proxies: true, // Using proxies will give the agent a better chance of success - requires Developer Plan or higher, comment out if you don't have access region: "us-west-2", browserSettings: { diff --git a/typescript/pickleball/index.ts b/typescript/pickleball/index.ts index 2c5a54e2..db58ff44 100644 --- a/typescript/pickleball/index.ts +++ b/typescript/pickleball/index.ts @@ -401,7 +401,6 @@ async function bookTennisPaddleCourt() { // https://docs.stagehand.dev/configuration/logging model: "openai/gpt-4.1", browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID!, timeout: 900, region: "us-west-2", }, diff --git a/typescript/playwright-mfa-handling/.env.example b/typescript/playwright-mfa-handling/.env.example index 36c24713..8c3f7702 100644 --- a/typescript/playwright-mfa-handling/.env.example +++ b/typescript/playwright-mfa-handling/.env.example @@ -1,3 +1,2 @@ # Browserbase Configuration -BROWSERBASE_PROJECT_ID=your_browserbase_project_id BROWSERBASE_API_KEY=your_browserbase_api_key diff --git a/typescript/playwright-mfa-handling/README.md b/typescript/playwright-mfa-handling/README.md index 2b4cfd8e..5e18e615 100644 --- a/typescript/playwright-mfa-handling/README.md +++ b/typescript/playwright-mfa-handling/README.md @@ -61,7 +61,7 @@ const hasFailure = await page.locator('text="Login Failure"').isVisible(); 1. cd playwright-mfa-handling 2. pnpm install 3. cp .env.example .env -4. Add required API keys/IDs to .env (BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY) +4. Add required API keys/IDs to .env (BROWSERBASE_API_KEY) 5. pnpm start ## EXPECTED OUTPUT @@ -79,7 +79,7 @@ const hasFailure = await page.locator('text="Login Failure"').isVisible(); ## COMMON PITFALLS - Dependency install errors: ensure pnpm install completed -- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- Missing credentials: verify .env contains BROWSERBASE_API_KEY - TOTP code expiration: codes are valid for 30 seconds - if authentication fails, the script automatically retries with a fresh code - CDP connection issues: ensure stable internet connection for reliable Browserbase connection - Selector changes: if the demo site structure changes, Playwright selectors may need updating diff --git a/typescript/playwright-mfa-handling/index.ts b/typescript/playwright-mfa-handling/index.ts index a730c53d..d3fefe05 100644 --- a/typescript/playwright-mfa-handling/index.ts +++ b/typescript/playwright-mfa-handling/index.ts @@ -73,15 +73,9 @@ async function createBrowserbaseSession(): Promise { if (!process.env.BROWSERBASE_API_KEY) { throw new Error("BROWSERBASE_API_KEY environment variable is required"); } - if (!process.env.BROWSERBASE_PROJECT_ID) { - throw new Error("BROWSERBASE_PROJECT_ID environment variable is required"); - } - const bb = new Browserbase({ apiKey: process.env.BROWSERBASE_API_KEY }); - const session = await bb.sessions.create({ - projectId: process.env.BROWSERBASE_PROJECT_ID, - }); + const session = await bb.sessions.create(); console.log(`Session created: https://browserbase.com/sessions/${session.id}`); @@ -236,7 +230,7 @@ async function main() { main().catch((err) => { console.error("\nError in MFA handling:", err); console.error("\nCommon issues:"); - console.error(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY"); + console.error(" - Check .env file has BROWSERBASE_API_KEY"); console.error(" - TOTP code may have expired (try running again)"); console.error(" - Page structure may have changed"); process.exit(1); diff --git a/typescript/playwright-mfa-handling/package.json b/typescript/playwright-mfa-handling/package.json index 17d5a8c4..858576ea 100644 --- a/typescript/playwright-mfa-handling/package.json +++ b/typescript/playwright-mfa-handling/package.json @@ -19,7 +19,7 @@ "author": "", "license": "MIT", "dependencies": { - "@browserbasehq/sdk": "^2.0.0", + "@browserbasehq/sdk": "^2.7.0", "dotenv": "^16.0.0", "playwright-core": "^1.40.0" }, diff --git a/typescript/playwright/basic-recaptcha/.env.example b/typescript/playwright/basic-recaptcha/.env.example index 36c24713..8c3f7702 100644 --- a/typescript/playwright/basic-recaptcha/.env.example +++ b/typescript/playwright/basic-recaptcha/.env.example @@ -1,3 +1,2 @@ # Browserbase Configuration -BROWSERBASE_PROJECT_ID=your_browserbase_project_id BROWSERBASE_API_KEY=your_browserbase_api_key diff --git a/typescript/playwright/basic-recaptcha/README.md b/typescript/playwright/basic-recaptcha/README.md index 8e7a01d9..c91dd184 100644 --- a/typescript/playwright/basic-recaptcha/README.md +++ b/typescript/playwright/basic-recaptcha/README.md @@ -70,7 +70,7 @@ browserSettings: { 1. cd typescript/playwright/basic-recaptcha 2. pnpm install 3. cp .env.example .env -4. Add your Browserbase API key and Project ID to .env +4. Add your Browserbase API key to .env 5. pnpm start ## EXPECTED OUTPUT @@ -88,7 +88,7 @@ browserSettings: { ## COMMON PITFALLS -- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- Missing credentials: verify .env contains BROWSERBASE_API_KEY - Captcha solving timeout: the template uses a 60-second timeout; increase if needed for complex CAPTCHAs - No browser context: ensure the Browserbase session was created successfully before connecting - Proxies not enabled: enable proxies in browserSettings for higher CAPTCHA solving success rates diff --git a/typescript/playwright/basic-recaptcha/index.ts b/typescript/playwright/basic-recaptcha/index.ts index 93fe05ac..5ff675f4 100644 --- a/typescript/playwright/basic-recaptcha/index.ts +++ b/typescript/playwright/basic-recaptcha/index.ts @@ -12,24 +12,20 @@ const SOLVE_CAPTCHAS = true; // Set to false to disable automatic captcha solvin // Validate required environment variables before starting. // Browserbase credentials are required to create browser sessions. -function validateEnv(): { apiKey: string; projectId: string } { +function validateEnv(): { apiKey: string } { const apiKey = process.env.BROWSERBASE_API_KEY; - const projectId = process.env.BROWSERBASE_PROJECT_ID; if (!apiKey) { throw new Error("BROWSERBASE_API_KEY environment variable is required"); } - if (!projectId) { - throw new Error("BROWSERBASE_PROJECT_ID environment variable is required"); - } - return { apiKey, projectId }; + return { apiKey }; } async function main() { console.log("Starting Playwright + Browserbase reCAPTCHA Example..."); - const { apiKey, projectId } = validateEnv(); + const { apiKey } = validateEnv(); // Initialize the Browserbase SDK client. const bb = new Browserbase({ apiKey }); @@ -39,7 +35,6 @@ async function main() { // Docs: https://docs.browserbase.com/features/stealth-mode#captcha-solving console.log("Creating Browserbase session with captcha solving enabled..."); const session = await bb.sessions.create({ - projectId, browserSettings: { solveCaptchas: SOLVE_CAPTCHAS, }, @@ -143,7 +138,7 @@ async function main() { main().catch((err) => { console.error("Application error:", err); console.error("\nCommon issues:"); - console.error(" - Check .env file has BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY"); + console.error(" - Check .env file has BROWSERBASE_API_KEY"); console.error(" - Verify solveCaptchas is enabled (true by default)"); console.error(" - Allow up to 60 seconds for CAPTCHA solving to complete"); console.error(" - Enable proxies for higher success rates"); diff --git a/typescript/playwright/basic-recaptcha/package.json b/typescript/playwright/basic-recaptcha/package.json index 547782eb..a1dad25a 100644 --- a/typescript/playwright/basic-recaptcha/package.json +++ b/typescript/playwright/basic-recaptcha/package.json @@ -6,7 +6,7 @@ "start": "tsx index.ts" }, "dependencies": { - "@browserbasehq/sdk": "latest", + "@browserbasehq/sdk": "^2.7.0", "playwright-core": "latest", "dotenv": "^16.4.7" }, diff --git a/typescript/proxies-weather/index.ts b/typescript/proxies-weather/index.ts index b8ef16aa..aff7256a 100644 --- a/typescript/proxies-weather/index.ts +++ b/typescript/proxies-weather/index.ts @@ -34,7 +34,6 @@ async function getWeatherForLocation(geolocation: GeolocationConfig): Promise Date: Thu, 12 Mar 2026 09:33:01 -0700 Subject: [PATCH 25/45] feat: add smart-fetch-scraper template (Fetch API + browser fallback) (#67) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add smart-fetch-scraper template — Fetch API + browser fallback * feat: add package.json for smart-fetch-scraper * feat: add .env.example for smart-fetch-scraper * feat: add README for smart-fetch-scraper * add more robust browser fallback checks * refactor code * refactor readme * update code + add example sites in readme * chore: remove package-lock.json from tracking and add .gitignore Co-Authored-By: Claude Sonnet 4.6 * link directly to fetch * fix relative cd path command in quickstart * remove stagehand, just use playwright page.content when falling back to browser * Revert "remove stagehand, just use playwright page.content when falling back to browser" This reverts commit 76c0db5abf03c8421723d5bbc1d3a9dccf8b08b2. * adding python example --------- Co-authored-by: Alex Qiu Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Jay Sahnan --- python/smart-fetch-scraper/.env.example | 8 + python/smart-fetch-scraper/README.md | 65 +++++ python/smart-fetch-scraper/main.py | 253 ++++++++++++++++++++ python/smart-fetch-scraper/pyproject.toml | 34 +++ typescript/smart-fetch-scraper/.env.example | 6 + typescript/smart-fetch-scraper/.gitignore | 3 + typescript/smart-fetch-scraper/README.md | 77 ++++++ typescript/smart-fetch-scraper/index.ts | 216 +++++++++++++++++ typescript/smart-fetch-scraper/package.json | 22 ++ 9 files changed, 684 insertions(+) create mode 100644 python/smart-fetch-scraper/.env.example create mode 100644 python/smart-fetch-scraper/README.md create mode 100644 python/smart-fetch-scraper/main.py create mode 100644 python/smart-fetch-scraper/pyproject.toml create mode 100644 typescript/smart-fetch-scraper/.env.example create mode 100644 typescript/smart-fetch-scraper/.gitignore create mode 100644 typescript/smart-fetch-scraper/README.md create mode 100644 typescript/smart-fetch-scraper/index.ts create mode 100644 typescript/smart-fetch-scraper/package.json diff --git a/python/smart-fetch-scraper/.env.example b/python/smart-fetch-scraper/.env.example new file mode 100644 index 00000000..a4e38f2d --- /dev/null +++ b/python/smart-fetch-scraper/.env.example @@ -0,0 +1,8 @@ +# Browserbase Configuration +BROWSERBASE_PROJECT_ID=your_browserbase_project_id +BROWSERBASE_API_KEY=your_browserbase_api_key + +# Google API key for gemini-2.5-flash model (used in browser fallback) +MODEL_API_KEY=your_google_api_key +# Or alternatively: +# GOOGLE_API_KEY=your_google_api_key diff --git a/python/smart-fetch-scraper/README.md b/python/smart-fetch-scraper/README.md new file mode 100644 index 00000000..f30060c8 --- /dev/null +++ b/python/smart-fetch-scraper/README.md @@ -0,0 +1,65 @@ +# Stagehand + Browserbase: Smart Fetch Scraper + +## AT A GLANCE + +- Goal: scrape a webpage using the fastest method available — Fetch API first, full browser session as fallback. +- Fetch API fast-path: sends a lightweight HTTP request via `POST /v1/fetch` — no browser session, no AI credits. Returns raw HTML in milliseconds. +- Browser fallback: when the Fetch API returns insufficient or JS-rendered content, automatically falls back to a Stagehand browser session with AI-powered `extract()`. +- Fallback detection: checks status code, content length, JS-challenge patterns, and text density to decide whether the Fetch API result is usable. +- Model: uses `google/gemini-2.5-flash` for the browser fallback path. + Docs → https://docs.stagehand.dev + +## GLOSSARY + +- Fetch API: Browserbase's lightweight HTTP fetching endpoint — fetches page content through Browserbase infrastructure without spinning up a browser. + Docs → https://docs.browserbase.com/features/fetch +- extract: pull structured data from pages using schemas and AI. + Docs → https://docs.stagehand.dev/basics/extract +- Stagehand: AI browser automation framework. + Docs → https://docs.stagehand.dev + +## QUICKSTART + +1. cd python/smart-fetch-scraper +2. uv pip install -e . +3. cp .env.example .env +4. Add BROWSERBASE_PROJECT_ID, BROWSERBASE_API_KEY, and MODEL_API_KEY to .env +5. uv run python main.py \ — e.g. `uv run python main.py https://news.ycombinator.com` + +## EXAMPLE URLS + +Fetch API fast-path (server-rendered, returns usable HTML directly): + +- `uv run python main.py https://news.ycombinator.com` — server-rendered, lightweight HTML +- `uv run python main.py https://en.wikipedia.org/wiki/Web_scraping` — static content, no JS required +- `uv run python main.py https://www.bbc.com/news` — server-rendered news page + +Browser fallback (JS-rendered, blocked, or low text density): + +- `uv run python main.py https://www.reddit.com` — returns a 403, triggers fallback +- `uv run python main.py https://x.com` — returns an "Enable JavaScript" shell page +- `uv run python main.py https://github.com/trending` — HTML is mostly inline scripts (3.6% text density), triggers fallback + +## EXPECTED OUTPUT + +- Logs the strategy being used (Fetch API vs browser) +- On Fetch API success: prints page title, link count, status code, content length, and a 500-char preview +- On browser fallback: prints Stagehand live view link, then structured JSON with page title and extracted items + +## COMMON PITFALLS + +- Missing credentials: verify .env contains BROWSERBASE_PROJECT_ID and BROWSERBASE_API_KEY +- Fetch API access: the Fetch API may require enablement on your account — contact support if you get a 404 +- Google API access: MODEL_API_KEY (or GOOGLE_API_KEY) is only needed for the browser fallback path +- Content threshold: adjust MIN_CONTENT_LENGTH if server-rendered pages are incorrectly triggering the browser fallback +- Text density: adjust MIN_TEXT_DENSITY if pages with lots of inline scripts/styles are incorrectly triggering the browser fallback +- JS-challenge detection: JS_REQUIRED_PATTERNS covers common bot-detection pages (Cloudflare, etc.) — extend the list for other patterns you encounter +- Timeout: the Fetch API has a 10-second timeout and 1MB max response — very large or slow pages will need the browser path + +## HELPFUL RESOURCES + +📚 Browserbase Fetch Docs: https://docs.browserbase.com/features/fetch +💡 Try it out: https://www.browserbase.com/playground +🔧 Templates: https://www.browserbase.com/templates +📧 Need help? support@browserbase.com +💬 Discord: http://stagehand.dev/discord diff --git a/python/smart-fetch-scraper/main.py b/python/smart-fetch-scraper/main.py new file mode 100644 index 00000000..bfa5d918 --- /dev/null +++ b/python/smart-fetch-scraper/main.py @@ -0,0 +1,253 @@ +# Stagehand + Browserbase: Smart Fetch Scraper - See README.md for full documentation +# +# Tries the Browserbase Fetch API first (fast, no browser session needed). +# If the page is JS-rendered or the content is insufficient, falls back to +# a full Stagehand browser session with AI-powered extraction. + +import asyncio +import json +import os +import re +import sys + +from browserbase import Browserbase +from dotenv import load_dotenv +from pydantic import BaseModel, Field +from stagehand import AsyncStagehand + +# Load environment variables from .env file +load_dotenv() + +# ============= CONFIGURATION ============= + +# Minimum character threshold — if Fetch API returns less than this, +# the page is likely JS-rendered and we fall back to a browser session. +MIN_CONTENT_LENGTH = 500 + +# Minimum ratio of visible text to raw HTML — pages below this are likely shells. +MIN_TEXT_DENSITY = 0.05 + +# Patterns that indicate the page requires JavaScript to render real content. +JS_REQUIRED_PATTERNS = [ + re.compile(r"enable javascript", re.IGNORECASE), + re.compile(r"javascript is (required|disabled|not enabled)", re.IGNORECASE), + re.compile(r"please enable javascript", re.IGNORECASE), + re.compile(r"this (site|page|app) requires javascript", re.IGNORECASE), + re.compile(r"checking your browser", re.IGNORECASE), # Cloudflare challenge + re.compile(r"