diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..2bcd70e --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 88 diff --git a/.github/workflows/pr-review-codex.yml b/.github/workflows/pr-review-codex.yml new file mode 100644 index 0000000..0a94628 --- /dev/null +++ b/.github/workflows/pr-review-codex.yml @@ -0,0 +1,67 @@ +name: Perform a code review when a pull request is created. +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review, labeled] + +jobs: + codex: + runs-on: ubuntu-24.04 + + permissions: + contents: read + pull-requests: write + issues: write + + outputs: + final_message: ${{ steps.run_codex.outputs.final-message }} + steps: + - uses: actions/checkout@v5 + with: + ref: refs/pull/${{ github.event.pull_request.number }}/merge + + - name: Pre-fetch base and head refs for the PR + run: | + git fetch --no-tags origin \ + ${{ github.event.pull_request.base.ref }} \ + +refs/pull/${{ github.event.pull_request.number }}/head + + - name: Run Codex + id: run_codex + uses: openai/codex-action@v1 + with: + openai-api-key: ${{ secrets.OPENAI_API_KEY }} + prompt: | + This is PR #${{ github.event.pull_request.number }} for ${{ github.repository }}. + Base SHA: ${{ github.event.pull_request.base.sha }} + Head SHA: ${{ github.event.pull_request.head.sha }} + + Review ONLY the changes introduced by the PR. + Suggest any improvements, potential bugs, security issues, or issues. + Be concise and specific in your feedback. + + Pull request title and body: + ---- + ${{ github.event.pull_request.title }} + ${{ github.event.pull_request.body }} + + post_feedback: + runs-on: ubuntu-latest + needs: codex + if: needs.codex.outputs.final_message != '' + permissions: + issues: write + pull-requests: write + steps: + - name: Report Codex feedback + uses: actions/github-script@v7 + env: + CODEX_FINAL_MESSAGE: ${{ needs.codex.outputs.final_message }} + with: + github-token: ${{ github.token }} + script: | + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + body: process.env.CODEX_FINAL_MESSAGE, + }); diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..1f962b4 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,32 @@ +name: Test + +on: + pull_request: + push: + branches: + - main + +jobs: + pytest: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install package and test dependencies + run: python -m pip install --upgrade pip && python -m pip install -e ".[dev]" + + - name: Run tests + run: python -m pytest diff --git a/.gitignore b/.gitignore index b765a90..d6b7b16 100644 --- a/.gitignore +++ b/.gitignore @@ -106,4 +106,7 @@ dmypy.json # Example files temp directories tmp/ -.DS_Store \ No newline at end of file +.DS_Store + +# Claude +.claude/ \ No newline at end of file diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..2efde2f --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,160 @@ +# Development Guide + +This guide covers how to set up and test the package locally before publishing. + +## Setup + +### Create a Virtual Environment + +```bash +# Create a virtual environment +python -m venv venv + +# Activate the virtual environment +source venv/bin/activate # On macOS/Linux +# venv\Scripts\activate # On Windows +``` + +### Install in Development Mode + +With the virtual environment activated, install the package with dev dependencies: + +```bash +# Install in editable/development mode with dev dependencies +pip install -e ".[dev]" +``` + +This installs the package along with `python-dotenv` (for `.env` file support), `pytest`, `mypy`, `black`, and `flake8`. + +### Environment Variables + +Set the following environment variables for testing: + +```bash +export FABRICATE_API_KEY="your-api-key" +export FABRICATE_API_URL="https://fabricate.tonic.ai/api/v1" # or your local instance +``` + +Or create a `.env` file (requires `python-dotenv`): + +``` +FABRICATE_API_KEY=your-api-key +FABRICATE_API_URL=https://fabricate.tonic.ai/api/v1 +``` + +## Testing + +### Quick Import Test + +Verify the module imports correctly: + +```bash +python -c "from tonic_fabricate import generate, run_workflow, AgentEvalsClient; print('All imports work!')" +``` + +### Run Unit Tests + +The Agent Evals client tests use a local HTTP server and do not require +Fabricate credentials: + +```bash +python -m pytest +``` + +### Run the Examples + +The examples test the actual API calls: + +```bash +# Test the generate function +python examples/download.py + +# Test the workflow function +python examples/workflow.py + +# Test Agent Evals against a configured Fabricate project and suite +python examples/agent_evals.py +``` + +### Interactive Testing + +```python +from tonic_fabricate import run_workflow +import os + +result = run_workflow( + database='your_database', + workspace='your_workspace', + workflow='your_workflow', + api_url=os.environ.get('FABRICATE_API_URL'), + on_progress=lambda p: print(p) +) +print(result.result) +``` + +## Code Quality + +### Install Dev Dependencies + +```bash +pip install -r requirements-dev.txt +``` + +### Type Checking + +```bash +mypy tonic_fabricate/ +``` + +### Linting + +```bash +flake8 tonic_fabricate/ +``` + +### Formatting + +```bash +# Check formatting +black --check tonic_fabricate/ + +# Auto-format +black tonic_fabricate/ +``` + +## Pre-Publish Testing + +Before publishing to production PyPI, test with TestPyPI: + +```bash +# Publish to TestPyPI +./publish-test.sh + +# Install from TestPyPI to verify +pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ tonic-fabricate==1.2.0 + +# Test the installed package +python -c "from tonic_fabricate import generate, run_workflow, AgentEvalsClient; print('Package works!')" +``` + +## Publishing + +See [publishing.md](publishing.md) for detailed publishing instructions. + +### Quick Reference + +```bash +# Test publish +./publish-test.sh + +# Production publish (after testing) +./publish.sh +``` + +## Deactivating the Virtual Environment + +When you're done developing: + +```bash +deactivate +``` diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f2fc0f2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Tonic AI, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 889cd56..0960a9f 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,31 @@ pip install tonic-fabricate ## Usage +### Report-only Agent Evals + +Client-owned tasks and grader prompts can be reported without creating +Fabricate definitions: + +```python +run = client.create_run(project_id, {"suite_name": "Git-defined evals", "model": model}) +client.report_trial( + run["id"], + { + "task": {"key": "users", "input": "Generate 100 users", "tags": ["users"]}, + "grader_definitions": [ + {"name": "quality", "prompt": "Inspect the attached data.", "tags": ["users"]} + ], + "transcript": {"messages": spans}, + "attachment_upload_ids": upload_ids, + "grade": True, + }, +) +``` + +Trial token metrics include `cache_read_tokens`, `cache_write_5m_tokens`, and +`cache_write_1h_tokens`. Fabricate can derive them from equivalent +`llm.token_count.prompt_details.*` OpenInference attributes. + To generate and download data from Fabricate: ```python @@ -109,12 +134,213 @@ The client will automatically use the following environment variables if they ar - `FABRICATE_API_KEY`: Your Fabricate API key - `FABRICATE_API_URL`: The Fabricate API URL (defaults to https://fabricate.tonic.ai/api/v1) +## Workflows + +Fabricate supports workflows that can perform custom operations and generate files. To run a workflow: + +```python +from tonic_fabricate import run_workflow + +result = run_workflow( + # The workspace to use + workspace='Default', + + # The name of the database + database='my_database', + + # The name of the workflow to run + workflow='my_workflow', + + # Optional: Parameters to pass to the workflow + params={ + 'message': 'Hello, world!', + }, +) + +# Access the workflow result +print(f"Result: {result.result}") + +# Download generated files if any +if result.task.files: + for file in result.task.files: + print(f"File: {file.name} ({file.size} bytes)") + result.download_file(file.id, f"./output/{file.name}") + + # Or download all files at once + result.download_all_files('./output') +``` + +### Workflow Progress Tracking + +```python +from tonic_fabricate import run_workflow + +def on_progress(data): + status = data.get('status', '') + message = data.get('message', '') + print(f"[{status}] {message}") + +result = run_workflow( + workspace='Default', + database='my_database', + workflow='my_workflow', + on_progress=on_progress +) +``` + +### Workflow File Downloads + +You can also download workflow files directly using `download_workflow_file`: + +```python +from tonic_fabricate import download_workflow_file + +download_workflow_file( + task_id='your-task-id', + file_id=123, + dest_path='./output/file.txt' +) +``` + +## Agent Evals + +`AgentEvalsClient` runs any Python agent against evaluation suites managed in +Fabricate. It is framework-independent: invoke your agent however you prefer, +then report its execution as flattened +[OpenInference](https://github.com/Arize-ai/openinference) spans. + +```python +from tonic_fabricate import AgentEvalsClient + +client = AgentEvalsClient() + +# Fabricate is the source of truth for the suite and its tasks. +suite = client.find_suite( + project_id="your-project-id", + name="Agent Eval Tasks", +) +if suite is None: + raise RuntimeError("Suite not found") + +tasks = client.list_tasks(suite["id"]) +run = client.create_run( + "your-project-id", + { + "suite_id": suite["id"], + "model": "gpt-5-mini", + "git_branch": "feature/my-agent", + }, +) + +for task in tasks: + # Replace this span with the OpenInference spans captured from your agent. + transcript = [ + { + "name": "my-agent", + "attributes": { + "openinference.span.kind": "AGENT", + "input.value": task["input"], + }, + } + ] + reported = client.report_trial( + run["id"], + { + "task_key": task["key"], + "transcript": {"messages": transcript}, + "grade": True, + }, + ) + graded = client.wait_for_grading(reported["id"]) + print(task["key"], "PASS" if graded["passed"] else "FAIL") + +client.update_run(run["id"], {"status": "completed"}) +``` + +The client reads `FABRICATE_API_KEY` and `FABRICATE_API_URL` by default. +`wait_for_grading` polls every three seconds and times out after two minutes; +both values are configurable. + +`find_suite(project_id, name="Agent Eval Tasks")` selects the latest version. +Pin a CI run with `version=2`, or pass a suite **version** UUID with `id=...`. +Use that version UUID (`suite["id"]`) for `list_tasks` and `create_run`; do +not use the stable suite UUID in `suite["suite_id"]`. To snapshot a version's +metadata and tasks into the next version, call +`client.create_suite_version(suite["id"])`. + +### Attachments + +Upload a file before reporting a trial, then reference its upload ID: + +```python +upload_id = client.upload_attachment( + "Default", + filename="payments.csv", + content_type="text/csv", + data=csv_bytes, +) + +client.report_trial( + run["id"], + { + "task_key": "payments-csv", + "transcript": {"messages": transcript}, + "attachment_upload_ids": [upload_id], + "grade": True, + }, +) +``` + +### Fixture Versions, Grader Versions, and management APIs + +Tasks expose their resolved Fixture Version as `effective_fixture`. Database +Fixture Version entries can be materialized as SQLite bytes: + +```python +sqlite_bytes = client.download_fixture_database( + fixture_id=task["effective_fixture"]["id"], + database_id=database_entry["value"]["database_id"], +) +``` + +Fixtures and Graders each have independently numbered versions. The +`list_fixtures` and `list_graders` APIs return the latest version by default; +use `create_fixture_version(fixture_version_id)` or +`create_grader_version(grader_version_id)` to copy a mutable version into the +next version. Pin a run's selected inputs with structured override lists: + +```python +client.create_run( + project_id, + { + "suite_id": suite["id"], + "fixture_overrides": [ + { + "fixture_id": "fixture-uuid", + "fixture_version_id": "fixture-version-uuid", + } + ], + "grader_overrides": [ + { + "grader_id": "grader-uuid", + "grader_version_id": "grader-version-uuid", + } + ], + }, +) +``` + +The run snapshots its resolved Fixture Version manifests and Grader Version +rubrics at creation, so subsequent edits do not change history. +`list_grader_definitions` and `find_or_create_grader_definition` remain +deprecated aliases for endpoint compatibility. + ## Error Handling The client raises appropriate exceptions for various error conditions: ```python -from tonic_fabricate import generate +from tonic_fabricate import generate, run_workflow try: generate( @@ -127,4 +353,15 @@ except ValueError as e: print(f"Invalid parameters: {e}") except Exception as e: print(f"Generation failed: {e}") + +try: + result = run_workflow( + workspace='Default', + database='my_database', + workflow='my_workflow' + ) +except ValueError as e: + print(f"Invalid parameters: {e}") +except Exception as e: + print(f"Workflow failed: {e}") ``` diff --git a/examples/agent_evals.py b/examples/agent_evals.py new file mode 100644 index 0000000..db820c9 --- /dev/null +++ b/examples/agent_evals.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Run a Fabricate Agent Evals suite with a framework-agnostic agent.""" + +import os +from typing import List + +from dotenv import load_dotenv + +from tonic_fabricate import ( + AgentEvalsClient, + CreateRunInput, + OpenInferenceSpan, +) + +load_dotenv() + + +def run_agent(task_input: str) -> List[OpenInferenceSpan]: + """Replace this example span with spans captured from your agent.""" + return [ + { + "name": "example-agent", + "attributes": { + "openinference.span.kind": "LLM", + "llm.input_messages.0.message.role": "user", + "llm.input_messages.0.message.content": task_input, + "llm.output_messages.0.message.role": "assistant", + "llm.output_messages.0.message.content": "Replace this with your agent's response.", + "llm.token_count.prompt": 100, + "llm.token_count.completion": 20, + "llm.token_count.prompt_details.cache_read": 40, + "llm.token_count.prompt_details.cache_write": 30, + "llm.token_count.prompt_details.cache_write_5m": 10, + "llm.token_count.prompt_details.cache_write_1h": 20, + }, + } + ] + + +def required_environment(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required") + return value + + +def main() -> None: + project_id = required_environment("FABRICATE_PROJECT_ID") + suite_name = os.environ.get("FABRICATE_SUITE_NAME", "Agent Eval Tasks") + suite_id = os.environ.get("FABRICATE_SUITE_ID") + suite_version = os.environ.get("FABRICATE_SUITE_VERSION") + client = AgentEvalsClient() + + suite = client.find_suite( + project_id, + id=suite_id, + name=None if suite_id else suite_name, + version=int(suite_version) if suite_version else None, + ) + if suite is None: + version_hint = f" version {suite_version}" if suite_version else "" + raise RuntimeError(f"Suite {suite_name!r}{version_hint} was not found") + + tasks = client.list_tasks(suite["id"]) + run_input: CreateRunInput = { + "suite_id": suite["id"], + "model": os.environ.get("AGENT_MODEL", "example-agent"), + } + if os.environ.get("GIT_BRANCH"): + run_input["git_branch"] = os.environ["GIT_BRANCH"] + if os.environ.get("GIT_SHA"): + run_input["git_sha"] = os.environ["GIT_SHA"] + + run = client.create_run(project_id, run_input) + try: + for task in tasks: + reported = client.report_trial( + run["id"], + { + "task_key": task["key"], + "transcript": {"messages": run_agent(task["input"])}, + "grade": True, + }, + ) + graded = client.wait_for_grading(reported["id"], timeout_seconds=300) + result = "PASS" if graded["passed"] else "FAIL" + print(f"{task['key']}: {result}") + except Exception: + client.update_run(run["id"], {"status": "failed"}) + raise + else: + client.update_run(run["id"], {"status": "completed"}) + + +if __name__ == "__main__": + main() diff --git a/examples/download.py b/examples/download.py index a9279b5..cbcbdc4 100644 --- a/examples/download.py +++ b/examples/download.py @@ -7,14 +7,10 @@ import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) -from tonic_fabricate import generate +from dotenv import load_dotenv +load_dotenv() -# Load environment variables if python-dotenv is available -try: - from dotenv import load_dotenv - load_dotenv() -except ImportError: - pass +from tonic_fabricate import generate def on_progress(data): """Progress callback function.""" diff --git a/examples/workflow.py b/examples/workflow.py new file mode 100644 index 0000000..74090db --- /dev/null +++ b/examples/workflow.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +""" +Example script for running a workflow in Fabricate. +""" + +import os +import sys +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from dotenv import load_dotenv +load_dotenv() + +from tonic_fabricate import run_workflow + +def on_progress(data): + """Progress callback function.""" + status = data.get('status', '') + message = data.get('message', '') + print(f"[{status}] {message}") + +if __name__ == "__main__": + print("Starting workflow...") + + # Prepare arguments + kwargs = { + 'database': 'agent_api_test', + 'workspace': 'API', + 'workflow': 'file', + 'params': { + 'message': 'Hello, world!', + }, + 'on_progress': on_progress, + } + + # Add api_url if available + api_url = os.environ.get('FABRICATE_API_URL') + if api_url: + kwargs['api_url'] = api_url + + # Run the workflow + workflow_result = run_workflow(**kwargs) + + print(f"Workflow result: {workflow_result.result}") + + # List and download files if any were generated + if workflow_result.task.files: + print(f"\nWorkflow generated {len(workflow_result.task.files)} file(s):") + + dest_dir = './tmp/workflow_output' + + for file in workflow_result.task.files: + print(f" - {file.name} ({file.content_type}, {file.size} bytes, id: {file.id})") + + # Download a specific file by id + workflow_result.download_file(file.id, f"{dest_dir}/{file.name}") + + # Or, download all files to a directory + # print('\nDownloading all files...') + # workflow_result.download_all_files(dest_dir) + # print(f'Files downloaded to {dest_dir}/') + + print("Done.") diff --git a/publish-test.sh b/publish-test.sh index 39f8b0c..e9f5847 100755 --- a/publish-test.sh +++ b/publish-test.sh @@ -30,7 +30,7 @@ pip install --upgrade pip build twine # Clean previous builds echo "๐Ÿงน Cleaning previous builds..." -rm -rf dist/ build/ *.egg-info fabricate_client.egg-info/ +rm -rf dist/ build/ *.egg-info tonic_fabricate.egg-info/ # Build the package echo "๐Ÿ”จ Building package..." @@ -64,7 +64,7 @@ if [ $? -eq 0 ]; then echo " pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ tonic-fabricate" echo "" echo "2. Test the package:" - echo " python -c \"from fabricate_client import generate; print('Package works!')\"" + echo " python -c \"from tonic_fabricate import generate, run_workflow, AgentEvalsClient; print('Package works!')\"" echo "" echo "3. View on TestPyPI:" echo " https://test.pypi.org/project/tonic-fabricate/" diff --git a/publish.sh b/publish.sh index 00438e9..0fe9c14 100755 --- a/publish.sh +++ b/publish.sh @@ -36,7 +36,7 @@ if [ "$CURRENT_VERSION" = "0.0.0" ]; then read -p "Do you want to continue with version 0.0.0? (y/N): " -n 1 -r echo if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "โŒ Cancelled. Please update the version in pyproject.toml and fabricate_client/__init__.py" + echo "โŒ Cancelled. Please update the version in pyproject.toml and tonic_fabricate/__init__.py" exit 1 fi else @@ -82,7 +82,7 @@ pip install --upgrade pip build twine # Clean previous builds echo "๐Ÿงน Cleaning previous builds..." -rm -rf dist/ build/ *.egg-info fabricate_client.egg-info/ +rm -rf dist/ build/ *.egg-info tonic_fabricate.egg-info/ # Build the package echo "๐Ÿ”จ Building package..." @@ -147,7 +147,7 @@ if [ $? -eq 0 ]; then echo " pip install tonic-fabricate" echo "" echo "2. Test the package:" - echo " python -c \"from fabricate_client import generate; print('Package works!')\"" + echo " python -c \"from tonic_fabricate import generate, run_workflow, AgentEvalsClient; print('Package works!')\"" echo "" echo "3. View on PyPI:" echo " https://pypi.org/project/tonic-fabricate/" diff --git a/publishing.md b/publishing.md index 79678d7..3b1aa49 100644 --- a/publishing.md +++ b/publishing.md @@ -6,12 +6,12 @@ Your package is **ready for publication**! The package structure includes: - Modern `pyproject.toml` configuration - Legacy `setup.py` for compatibility -- Proper package structure with `fabricate_client/` +- Proper package structure with `tonic_fabricate/` - README.md with usage documentation - MANIFEST.in for including additional files -- Version management (currently v1.0.0) +- Version management (currently v1.2.0) - Package name: `tonic-fabricate` -- Import name: `fabricate_client` +- Import name: `tonic_fabricate` ## ๐Ÿš€ Publishing Process @@ -45,32 +45,36 @@ You'll need accounts on: #### **3. Update Version Number (if needed)** -Simply update the version in **one place** - `pyproject.toml`: +The version lives in **two places** that must be kept in sync: ```toml # In pyproject.toml (line 7) version = "x.y.z" ``` +```python +# In tonic_fabricate/__init__.py +__version__ = "x.y.z" +``` + **Quick update command:** ```bash -# Example: Update from 1.0.0 to 1.0.1 -sed -i '' 's/version = "1.0.0"/version = "1.0.1"/' pyproject.toml +# Example: Update from 1.2.0 to 1.2.1 +sed -i '' 's/version = "1.2.0"/version = "1.2.1"/' pyproject.toml +sed -i '' 's/__version__ = "1.2.0"/__version__ = "1.2.1"/' tonic_fabricate/__init__.py # Verify the change -grep "version =" pyproject.toml +grep '^version = ' pyproject.toml +grep '^__version__ = ' tonic_fabricate/__init__.py ``` -โœ… **Single Source of Truth:** The other files (`setup.py` and `__init__.py`) automatically read the version from `pyproject.toml` at build time and runtime using the `tomllib`/`tomli` library. +โš ๏ธ **Two places, not one:** `setup.py` reads the version from `pyproject.toml` at build time via `tomllib`/`tomli`, so it needs no edit. But `tonic_fabricate/__init__.py` hardcodes `__version__` and will silently drift if you forget it. The published artifact takes its version from `pyproject.toml`; `__version__` is what users see at runtime. #### **4. Test Publishing** ```bash -# Navigate to the client directory -cd /Users/mark/Code/fabricate/clients/python - -# Run the test publishing script +# From the root of this repository ./publish-test.sh ``` @@ -86,7 +90,7 @@ You'll be prompted for: pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ tonic-fabricate # Test the package -python -c "from fabricate_client import generate; print('Package works!')" +python -c "from tonic_fabricate import generate, run_workflow, AgentEvalsClient; print('Package works!')" ``` #### **6. Production Publishing** @@ -117,7 +121,7 @@ You'll be prompted for: pip install tonic-fabricate # Test the package -python -c "from fabricate_client import generate; print('Package installed successfully!')" +python -c "from tonic_fabricate import generate, run_workflow, AgentEvalsClient; print('Package installed successfully!')" ``` ### **Option 2: Automated Publishing with GitHub Actions (Recommended)** @@ -149,8 +153,8 @@ First, you need to configure your PyPI API token as a GitHub secret: ```bash # Tag the current commit - git tag v1.0.2 - git push origin v1.0.2 + git tag vx.y.z + git push origin vx.y.z ``` 2. Go to GitHub โ†’ **Releases** โ†’ **Create a new release** @@ -205,21 +209,21 @@ This ensures that publishing requires manual approval even when automated. - **MINOR** (y): New features, backward compatible (e.g., 1.0.0 โ†’ 1.1.0) - **MAJOR** (x): Breaking changes (e.g., 1.0.0 โ†’ 2.0.0) -### **Single Source Management:** +### **Where the version lives:** -- โœ… **Only edit:** `pyproject.toml` -- โœ… **Automatic sync:** `setup.py` and `__init__.py` read from `pyproject.toml` -- โœ… **No version conflicts:** Impossible for versions to get out of sync -- โœ… **Modern approach:** Uses `tomllib` (Python 3.11+) or `tomli` (Python 3.8-3.10) +- โœ… **Edit both:** `pyproject.toml` (`version`) and `tonic_fabricate/__init__.py` (`__version__`) +- โœ… **Reads automatically:** `setup.py` pulls the version from `pyproject.toml` at build time +- โš ๏ธ **Can drift:** `__version__` is hardcoded โ€” nothing enforces that it matches `pyproject.toml` +- โœ… **Modern approach:** Uses `tomllib` (Python 3.11+) or `tomli` (Python 3.9-3.10) ## ๐ŸŽฏ Quick Publishing Workflow ### **GitHub Actions (Recommended)** ```bash -# 1. Update version in pyproject.toml (if needed) +# 1. Update version in pyproject.toml and tonic_fabricate/__init__.py (if needed) # 2. Commit and push changes -git add pyproject.toml +git add pyproject.toml tonic_fabricate/__init__.py git commit -m "Bump version to x.y.z" git push @@ -234,11 +238,10 @@ git push origin vx.y.z ### **Manual Publishing** ```bash -# 1. Navigate to directory -cd /Users/mark/Code/tonic-fabricate-python +# 1. Start from the root of this repository # 2. Update version (if needed) -# Edit only pyproject.toml - other files read it automatically +# Edit pyproject.toml and tonic_fabricate/__init__.py # 3. Test publish ./publish-test.sh @@ -253,8 +256,8 @@ pip install --index-url https://test.pypi.org/simple/ --extra-index-url https:// ## ๐Ÿšจ Important Notes 1. **Package Name**: Installable as `tonic-fabricate` -2. **Import Name**: Import as `fabricate_client` -3. **Version Update**: Only update version in `pyproject.toml` +2. **Import Name**: Import as `tonic_fabricate` +3. **Version Update**: Update the version in `pyproject.toml` **and** `tonic_fabricate/__init__.py` 4. **GitHub Actions**: Requires `PYPI_API_TOKEN` secret to be configured 5. **Always test first**: Use TestPyPI before production (manual) or test releases (GitHub Actions) 6. **No duplicates**: You cannot upload the same version twice @@ -278,4 +281,4 @@ pip install --index-url https://test.pypi.org/simple/ --extra-index-url https:// **Import errors after installation:** -- Verify package name (`tonic-fabricate`) vs import name (`fabricate_client`) +- Verify package name (`tonic-fabricate`) vs import name (`tonic_fabricate`) diff --git a/pyproject.toml b/pyproject.toml index 7756280..80fedac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,22 +1,22 @@ [build-system] -requires = ["setuptools>=45", "wheel"] +requires = ["setuptools>=77", "wheel"] build-backend = "setuptools.build_meta" [project] name = "tonic-fabricate" -version = "1.0.2" +version = "1.2.0" description = "The official Fabricate client for Python" authors = [{name = "Fabricate Tools"}] readme = "README.md" license = "MIT" -requires-python = ">=3.8" +license-files = ["LICENSE"] +requires-python = ">=3.9" classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -37,16 +37,16 @@ dev = [ ] [project.urls] -Homepage = "https://github.com/fabricate-tools/client-python" -Repository = "https://github.com/fabricate-tools/client-python" -Documentation = "https://github.com/fabricate-tools/client-python#readme" +Homepage = "https://github.com/TonicAI/tonic-fabricate-python" +Repository = "https://github.com/TonicAI/tonic-fabricate-python" +Documentation = "https://github.com/TonicAI/tonic-fabricate-python#readme" [tool.black] line-length = 88 -target-version = ['py38'] +target-version = ['py39'] [tool.mypy] -python_version = "3.8" +python_version = "3.9" warn_return_any = true warn_unused_configs = true disallow_untyped_defs = true \ No newline at end of file diff --git a/setup.py b/setup.py index a2c0548..a39dabf 100644 --- a/setup.py +++ b/setup.py @@ -18,21 +18,19 @@ description="The official Fabricate client for Python", long_description=long_description, long_description_content_type="text/markdown", - url="https://github.com/fabricate-tools/client-python", + url="https://github.com/TonicAI/tonic-fabricate-python", packages=find_packages(), classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ], - python_requires=">=3.8", + python_requires=">=3.9", install_requires=[ "requests>=2.25.0", ], diff --git a/tests/test_agent_evals.py b/tests/test_agent_evals.py new file mode 100644 index 0000000..3c6fb80 --- /dev/null +++ b/tests/test_agent_evals.py @@ -0,0 +1,581 @@ +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, List, Optional + +import pytest + +from tonic_fabricate import AgentEvalsClient, AgentEvalsError + + +class ExpectedRequest: + def __init__( + self, + method: str, + path: str, + *, + response_json: Optional[Any] = None, + response_body: bytes = b"", + status: int = 200, + response_content_type: str = "application/json", + ): + self.method = method + self.path = path + self.response_json = response_json + self.response_body = response_body + self.status = status + self.response_content_type = response_content_type + + +class StubServer(ThreadingHTTPServer): + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), StubHandler) + self.expected: List[ExpectedRequest] = [] + self.requests: List[Dict[str, Any]] = [] + + @property + def url(self) -> str: + host, port = self.server_address + return f"http://{host}:{port}" + + def enqueue( + self, + method: str, + path: str, + *, + response_json: Optional[Any] = None, + response_body: bytes = b"", + status: int = 200, + response_content_type: str = "application/json", + ) -> None: + self.expected.append( + ExpectedRequest( + method, + path, + response_json=response_json, + response_body=response_body, + status=status, + response_content_type=response_content_type, + ) + ) + + +class StubHandler(BaseHTTPRequestHandler): + server: StubServer + + def do_GET(self) -> None: + self._handle() + + def do_POST(self) -> None: + self._handle() + + def do_PATCH(self) -> None: + self._handle() + + def do_DELETE(self) -> None: + self._handle() + + def do_PUT(self) -> None: + self._handle() + + def log_message(self, format: str, *args: Any) -> None: + return + + def _handle(self) -> None: + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) if length else b"" + self.server.requests.append( + { + "method": self.command, + "path": self.path, + "headers": dict(self.headers), + "body": body, + } + ) + + if not self.server.expected: + self.send_error(500, "Unexpected request") + return + + expected = self.server.expected.pop(0) + if self.command != expected.method or self.path != expected.path: + self.send_error( + 500, + f"Expected {expected.method} {expected.path}; " + f"got {self.command} {self.path}", + ) + return + + if expected.response_json is not None: + response_body = json.dumps(expected.response_json).encode("utf-8") + else: + response_body = expected.response_body + + self.send_response(expected.status) + self.send_header("Content-Type", expected.response_content_type) + self.send_header("Content-Length", str(len(response_body))) + self.end_headers() + self.wfile.write(response_body) + + +@pytest.fixture +def stub_server() -> Any: + server = StubServer() + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + assert not server.expected + finally: + server.shutdown() + thread.join() + server.server_close() + + +@pytest.fixture +def client(stub_server: StubServer) -> AgentEvalsClient: + return AgentEvalsClient( + api_key="test-key", api_url=f"{stub_server.url}/api/v1///" + ) + + +def request_json(request: Dict[str, Any]) -> Any: + return json.loads(request["body"].decode("utf-8")) + + +def test_report_only_payloads( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "POST", + "/api/v1/projects/project/runs", + response_json={"id": "run", "status": "in_progress"}, + status=201, + ) + stub_server.enqueue( + "POST", + "/api/v1/runs/run/trials", + response_json={"id": "trial", "status": "completed"}, + status=202, + ) + + client.create_run("project", {"suite_name": "Local suite", "model": "gpt-x"}) + client.report_trial( + "run", + { + "task": {"key": "users", "input": "Generate users", "tags": ["users"]}, + "grader_definitions": [ + { + "name": "quality", + "prompt": "Check the output.", + "tags": ["users"], + } + ], + "transcript": {"messages": []}, + "cache_read_tokens": 10, + "cache_write_5m_tokens": 20, + "cache_write_1h_tokens": 30, + "grade": True, + }, + ) + + assert request_json(stub_server.requests[0]) == { + "suite_name": "Local suite", + "model": "gpt-x", + } + assert request_json(stub_server.requests[1])["task"]["key"] == "users" + assert request_json(stub_server.requests[1])["grader_definitions"][0][ + "prompt" + ] == "Check the output." + assert request_json(stub_server.requests[1])["cache_read_tokens"] == 10 + assert request_json(stub_server.requests[1])["cache_write_5m_tokens"] == 20 + assert request_json(stub_server.requests[1])["cache_write_1h_tokens"] == 30 + + +def test_constructor_uses_environment_and_normalizes_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("FABRICATE_API_KEY", "env-key") + monkeypatch.setenv("FABRICATE_API_URL", "https://example.test/api/v1///") + + client = AgentEvalsClient() + + assert client.api_key == "env-key" + assert client.api_url == "https://example.test/api/v1" + + +def test_constructor_requires_api_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FABRICATE_API_KEY", raising=False) + + with pytest.raises(ValueError, match="api_key is required"): + AgentEvalsClient() + + +def test_project_suite_and_task_methods( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "GET", "/api/v1/workspaces/My%20Workspace/projects", response_json=[] + ) + stub_server.enqueue( + "POST", + "/api/v1/workspaces/My%20Workspace/projects", + response_json={"id": "project-1"}, + ) + stub_server.enqueue("GET", "/api/v1/projects/project-1/suites", response_json=[]) + stub_server.enqueue( + "POST", + "/api/v1/projects/project-1/suites", + response_json={"id": "suite-1"}, + ) + stub_server.enqueue("GET", "/api/v1/suites/suite-1/tasks", response_json=[]) + stub_server.enqueue( + "POST", + "/api/v1/suites/suite-1/tasks", + response_json={"id": "task-1"}, + ) + + assert client.list_projects("My Workspace") == [] + assert ( + client.find_or_create_project("My Workspace", {"name": "Agent project"})["id"] + == "project-1" + ) + assert client.list_suites("project-1") == [] + assert ( + client.find_or_create_suite("project-1", {"name": "Suite", "tags": ["smoke"]})[ + "id" + ] + == "suite-1" + ) + assert client.list_tasks("suite-1") == [] + assert ( + client.upsert_task("suite-1", {"key": "task", "input": "Do the task"})["id"] + == "task-1" + ) + + assert stub_server.requests[0]["headers"]["Authorization"] == "Bearer test-key" + assert request_json(stub_server.requests[1]) == {"name": "Agent project"} + assert request_json(stub_server.requests[3]) == { + "name": "Suite", + "tags": ["smoke"], + } + assert request_json(stub_server.requests[5]) == { + "key": "task", + "input": "Do the task", + } + + +def test_find_suite_matches_id_or_case_insensitive_name_and_version( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + suites = [ + {"id": "one", "suite_id": "p", "name": "Regression", "version": 1}, + {"id": "two", "suite_id": "p", "name": "Regression", "version": 2}, + ] + for _ in range(3): + stub_server.enqueue( + "GET", "/api/v1/projects/project/suites", response_json=suites + ) + + assert client.find_suite("project", id="two") == suites[1] + # Without a version selector, the highest-numbered matching version wins. + assert client.find_suite("project", name="regression") == suites[1] + assert ( + client.find_suite("project", name="REGRESSION", version=1) == suites[0] + ) + + +def test_create_suite_version_uses_the_source_version_endpoint( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "POST", + "/api/v1/suites/source-version/versions", + response_json={ + "id": "next-version", + "suite_id": "parent-suite", + "name": "Regression", + "version": 2, + }, + status=201, + ) + + suite = client.create_suite_version("source-version") + + assert suite["id"] == "next-version" + assert suite["version"] == 2 + assert stub_server.requests[0]["headers"]["Authorization"] == "Bearer test-key" + assert stub_server.requests[0]["body"] == b"" + + +def test_fixture_methods_and_database_download( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue("GET", "/api/v1/projects/project/fixtures", response_json=[]) + stub_server.enqueue( + "POST", + "/api/v1/projects/project/fixtures", + response_json={"id": "fixture"}, + ) + stub_server.enqueue( + "GET", "/api/v1/fixtures/fixture", response_json={"id": "fixture"} + ) + stub_server.enqueue( + "PATCH", + "/api/v1/fixtures/fixture", + response_json={"id": "fixture", "name": "Updated"}, + ) + stub_server.enqueue( + "GET", + "/api/v1/fixtures/fixture/databases/database", + response_body=b"SQLite format 3\x00", + response_content_type="application/vnd.sqlite3", + ) + stub_server.enqueue("DELETE", "/api/v1/fixtures/fixture", status=204) + + assert client.list_fixtures("project") == [] + assert ( + client.find_or_create_fixture("project", {"name": "Fixture", "entries": []})[ + "id" + ] + == "fixture" + ) + assert client.get_fixture("fixture")["id"] == "fixture" + assert client.update_fixture("fixture", {"name": "Updated"})["name"] == "Updated" + assert client.download_fixture_database("fixture", "database").startswith(b"SQLite") + assert client.delete_fixture("fixture") is None + assert stub_server.requests[4]["headers"]["Accept"] == "application/vnd.sqlite3" + + +def test_grader_run_and_trial_methods( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "GET", "/api/v1/projects/project/grader_definitions", response_json=[] + ) + stub_server.enqueue( + "POST", + "/api/v1/projects/project/grader_definitions", + response_json={"id": "grader"}, + ) + stub_server.enqueue( + "GET", + "/api/v1/projects/project/runs?branch=feature%2Fagent", + response_json=[], + ) + stub_server.enqueue( + "POST", + "/api/v1/projects/project/runs", + response_json={"id": "run"}, + ) + stub_server.enqueue( + "GET", + "/api/v1/runs/run", + response_json={"id": "run", "trials": []}, + ) + stub_server.enqueue("PATCH", "/api/v1/runs/run", response_json={"id": "run"}) + stub_server.enqueue( + "POST", + "/api/v1/runs/run/trials", + response_json={"id": "trial", "task_key": "task"}, + status=202, + ) + stub_server.enqueue( + "GET", + "/api/v1/trials/trial", + response_json={"id": "trial", "task_key": "task"}, + ) + + assert client.list_grader_definitions("project") == [] + assert ( + client.find_or_create_grader_definition( + "project", {"name": "Judge", "kind": "llm_judge"} + )["id"] + == "grader" + ) + assert client.list_runs("project", branch="feature/agent") == [] + assert client.create_run("project", {"suite_id": "suite"})["id"] == "run" + assert client.get_run("run")["trials"] == [] + assert client.update_run("run", {"status": "completed"})["id"] == "run" + assert ( + client.report_trial( + "run", + { + "task_key": "task", + "transcript": { + "messages": [ + { + "name": "agent", + "attributes": {"openinference.span.kind": "AGENT"}, + } + ] + }, + "grade": True, + }, + )["id"] + == "trial" + ) + assert client.get_trial("trial")["id"] == "trial" + + assert request_json(stub_server.requests[5]) == {"status": "completed"} + assert request_json(stub_server.requests[6])["grade"] is True + + +def test_list_runs_omits_empty_branch( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue("GET", "/api/v1/projects/project/runs", response_json=[]) + + assert client.list_runs("project", branch="") == [] + + +def test_http_errors_include_request_context( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "GET", + "/api/v1/trials/missing", + response_json={"error": "Not found"}, + status=404, + ) + + with pytest.raises(AgentEvalsError) as raised: + client.get_trial("missing") + + error = raised.value + assert error.status == 404 + assert error.method == "GET" + assert error.path == "/trials/missing" + assert '"error": "Not found"' in error.body + + +def test_wait_for_grading_polls_until_completed( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "GET", + "/api/v1/trials/trial", + response_json={"id": "trial", "grading": {"status": "pending"}}, + ) + stub_server.enqueue( + "GET", + "/api/v1/trials/trial", + response_json={"id": "trial", "grading": {"status": "in_progress"}}, + ) + stub_server.enqueue( + "GET", + "/api/v1/trials/trial", + response_json={"id": "trial", "grading": {"status": "completed"}}, + ) + + trial = client.wait_for_grading("trial", interval_seconds=0, timeout_seconds=1) + + assert trial["grading"]["status"] == "completed" + + +@pytest.mark.parametrize("grading", [None, {"status": "failed"}]) +def test_wait_for_grading_returns_terminal_or_ungraded_trial( + client: AgentEvalsClient, + stub_server: StubServer, + grading: Optional[Dict[str, str]], +) -> None: + response = {"id": "trial"} + if grading is not None: + response["grading"] = grading + stub_server.enqueue("GET", "/api/v1/trials/trial", response_json=response) + + assert ( + client.wait_for_grading("trial", interval_seconds=0, timeout_seconds=1)["id"] + == "trial" + ) + + +def test_wait_for_grading_times_out( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "GET", + "/api/v1/trials/trial", + response_json={"id": "trial", "grading": {"status": "in_progress"}}, + ) + + with pytest.raises(TimeoutError, match="Timed out after 0s"): + client.wait_for_grading("trial", interval_seconds=0, timeout_seconds=0) + + +def test_wait_for_grading_can_be_cancelled( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "GET", + "/api/v1/trials/trial", + response_json={"id": "trial", "grading": {"status": "in_progress"}}, + ) + cancel_event = threading.Event() + cancel_event.set() + + with pytest.raises(RuntimeError, match="Aborted"): + client.wait_for_grading( + "trial", + interval_seconds=0, + timeout_seconds=1, + cancel_event=cancel_event, + ) + + +def test_upload_attachment_mints_and_puts_bytes( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + upload_url = f"{stub_server.url}/api/v1/agent_evals/uploads/upload" + stub_server.enqueue( + "POST", + "/api/v1/workspaces/My%20Workspace/agent_evals/uploads", + response_json={ + "upload_id": "upload", + "upload_url": upload_url, + "method": "PUT", + "max_bytes": 100, + "expires_at": "later", + "instructions": "PUT bytes", + }, + status=201, + ) + stub_server.enqueue( + "PUT", + "/api/v1/agent_evals/uploads/upload", + response_json={"upload_id": "upload", "status": "uploaded"}, + ) + + upload_id = client.upload_attachment( + "My Workspace", + filename="result.csv", + content_type="text/csv", + data=b"a,b\n1,2\n", + ) + + assert upload_id == "upload" + assert request_json(stub_server.requests[0]) == { + "filename": "result.csv", + "content_type": "text/csv", + } + assert stub_server.requests[1]["body"] == b"a,b\n1,2\n" + assert stub_server.requests[1]["headers"]["Content-Type"] == "text/csv" + assert stub_server.requests[1]["headers"]["Authorization"] == "Bearer test-key" + + +def test_upload_errors_use_absolute_url( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + upload_url = f"{stub_server.url}/upload" + stub_server.enqueue( + "PUT", + "/upload", + response_json={"error": "expired"}, + status=409, + ) + + with pytest.raises(AgentEvalsError) as raised: + client.put_upload_bytes(upload_url, b"data") + + assert raised.value.method == "PUT" + assert raised.value.path == upload_url diff --git a/tonic_fabricate/__init__.py b/tonic_fabricate/__init__.py index 211d230..c925c43 100644 --- a/tonic_fabricate/__init__.py +++ b/tonic_fabricate/__init__.py @@ -2,8 +2,102 @@ Tonic Fabricate - The official Fabricate client for Python. """ -from .client import generate +from .client import ( + generate, + run_workflow, + download_workflow_file, + WorkflowFile, + WorkflowTask, + WorkflowResult, +) +from .agent_evals import ( + AgentEvalsAssertion, + AgentEvalsAttachment, + AgentEvalsClient, + AgentEvalsError, + AgentEvalsFixture, + AgentEvalsFixtureVersion, + AgentEvalsFixtureEntry, + AgentEvalsFixtureEntryInput, + AgentEvalsFixtureEntryType, + AgentEvalsFixtureVersionOverride, + AgentEvalsGrader, + AgentEvalsGraderDefinition, + AgentEvalsGraderKind, + AgentEvalsGraderVersion, + AgentEvalsGraderVersionOverride, + AgentEvalsGrading, + AgentEvalsGradingStatus, + AgentEvalsProject, + AgentEvalsClientRunStatus, + AgentEvalsRun, + AgentEvalsRunStatus, + AgentEvalsRunWithTrials, + AgentEvalsSuite, + AgentEvalsTask, + AgentEvalsTrial, + AgentEvalsTrialStatus, + AgentEvalsTrialSummary, + AgentEvalsUpload, + CreateUploadInput, + CreateRunInput, + FindOrCreateGraderDefinitionInput, + FindOrCreateProjectInput, + FindOrCreateSuiteInput, + FixtureInput, + OpenInferenceSpan, + ReportTrialInput, + UpdateFixtureInput, + UpdateRunInput, + UpsertTaskInput, +) -__version__ = "1.0.1" +__version__ = "1.2.0" -__all__ = ["generate"] \ No newline at end of file +__all__ = [ + "generate", + "run_workflow", + "download_workflow_file", + "WorkflowFile", + "WorkflowTask", + "WorkflowResult", + "AgentEvalsAssertion", + "AgentEvalsAttachment", + "AgentEvalsClient", + "AgentEvalsError", + "AgentEvalsFixture", + "AgentEvalsFixtureVersion", + "AgentEvalsFixtureEntry", + "AgentEvalsFixtureEntryInput", + "AgentEvalsFixtureEntryType", + "AgentEvalsFixtureVersionOverride", + "AgentEvalsGrader", + "AgentEvalsGraderDefinition", + "AgentEvalsGraderKind", + "AgentEvalsGraderVersion", + "AgentEvalsGraderVersionOverride", + "AgentEvalsGrading", + "AgentEvalsGradingStatus", + "AgentEvalsProject", + "AgentEvalsClientRunStatus", + "AgentEvalsRun", + "AgentEvalsRunStatus", + "AgentEvalsRunWithTrials", + "AgentEvalsSuite", + "AgentEvalsTask", + "AgentEvalsTrial", + "AgentEvalsTrialStatus", + "AgentEvalsTrialSummary", + "AgentEvalsUpload", + "CreateUploadInput", + "CreateRunInput", + "FindOrCreateGraderDefinitionInput", + "FindOrCreateProjectInput", + "FindOrCreateSuiteInput", + "FixtureInput", + "OpenInferenceSpan", + "ReportTrialInput", + "UpdateFixtureInput", + "UpdateRunInput", + "UpsertTaskInput", +] diff --git a/tonic_fabricate/agent_evals.py b/tonic_fabricate/agent_evals.py new file mode 100644 index 0000000..1d19863 --- /dev/null +++ b/tonic_fabricate/agent_evals.py @@ -0,0 +1,854 @@ +"""Framework-agnostic client for Fabricate's Agent Evals API.""" + +from __future__ import annotations + +import os +import time +from threading import Event +from typing import ( + Any, + Dict, + List, + Literal, + Mapping, + Optional, + Sequence, + TypedDict, + Union, + cast, +) +from urllib.parse import quote + +import requests + +DEFAULT_API_URL = "https://fabricate.tonic.ai/api/v1" +DEFAULT_GRADING_INTERVAL_SECONDS = 3.0 +DEFAULT_GRADING_TIMEOUT_SECONDS = 120.0 + +AgentEvalsRunStatus = Literal["in_progress", "completed", "failed", "timed_out"] +AgentEvalsClientRunStatus = Literal["in_progress", "completed", "failed"] +AgentEvalsTrialStatus = Literal["completed", "failed", "error"] +AgentEvalsGraderKind = Literal["llm_judge", "script"] +AgentEvalsFixtureEntryType = Literal["database", "table", "workflow", "mock_api"] + + +class OpenInferenceSpan(TypedDict): + """One flattened OpenInference span in a trial transcript.""" + + name: str + attributes: Dict[str, Any] + + +class AgentEvalsProject(TypedDict): + id: str + name: str + description: Optional[str] + workspace_id: str + + +class AgentEvalsFixtureEntryDiagnostic(TypedDict): + code: str + message: str + + +class AgentEvalsFixtureEntry(TypedDict): + id: str + key: str + type: AgentEvalsFixtureEntryType + value: Dict[str, Any] + position: int + resource: Optional[Dict[str, Any]] + diagnostic: Optional[AgentEvalsFixtureEntryDiagnostic] + + +class _AgentEvalsFixtureVersionRequired(TypedDict): + id: str + project_id: str + fixture_id: str + name: str + version: int + description: Optional[str] + entries: List[AgentEvalsFixtureEntry] + + +class AgentEvalsFixtureVersion(_AgentEvalsFixtureVersionRequired, total=False): + created_at: str + updated_at: str + + +AgentEvalsFixture = AgentEvalsFixtureVersion + + +class AgentEvalsSuite(TypedDict): + """A suite version; ``id`` is its version UUID and ``suite_id`` is its stable suite UUID.""" + + id: str + project_id: str + suite_id: str + name: str + version: int + description: Optional[str] + tags: List[str] + task_count: int + default_fixture_id: Optional[str] + default_fixture: Optional[AgentEvalsFixture] + + +class AgentEvalsTask(TypedDict): + """A task scoped to one suite version via ``suite_id`` (not the stable suite UUID).""" + + id: str + suite_id: str + key: str + input: str + expected_output: Optional[str] + tags: List[str] + input_token_limit: Optional[int] + output_token_limit: Optional[int] + fixture_id: Optional[str] + effective_fixture: Optional[AgentEvalsFixture] + + +class AgentEvalsGraderVersion(TypedDict): + id: str + project_id: str + grader_id: str + name: str + version: int + description: Optional[str] + kind: AgentEvalsGraderKind + prompt: Optional[str] + code: Optional[str] + model: Optional[str] + tags: List[str] + + +AgentEvalsGraderDefinition = AgentEvalsGraderVersion + + +class AgentEvalsRun(TypedDict): + """A run whose ``suite_id`` is the suite version UUID, not the stable suite UUID.""" + + id: str + project_id: str + run_number: int + suite_id: Optional[str] + suite_linked: bool + suite_name: Optional[str] + suite_version: Optional[int] + git_branch: Optional[str] + git_sha: Optional[str] + git_repo_url: Optional[str] + model: Optional[str] + name: Optional[str] + status: AgentEvalsRunStatus + started_at: Optional[str] + completed_at: Optional[str] + last_client_update_at: str + aggregate_metrics: Optional[Dict[str, Any]] + metadata: Optional[Dict[str, Any]] + fixture_overrides: Sequence["AgentEvalsFixtureVersionOverride"] + grader_overrides: Sequence["AgentEvalsGraderVersionOverride"] + created_at: str + + +class _AgentEvalsAssertionRequired(TypedDict): + assertion_name: str + passed: Optional[bool] + score: Optional[float] + reasoning: Optional[str] + + +class AgentEvalsAssertion(_AgentEvalsAssertionRequired, total=False): + id: str + + +class _AgentEvalsGraderRequired(TypedDict): + grader_name: str + passed: Optional[bool] + assertions: List[AgentEvalsAssertion] + + +class AgentEvalsGrader(_AgentEvalsGraderRequired, total=False): + id: str + + +AgentEvalsGradingStatus = Literal["pending", "in_progress", "completed", "failed"] + + +class AgentEvalsGrading(TypedDict): + id: str + status: AgentEvalsGradingStatus + graders_total: int + graders_completed: int + error: Optional[str] + started_at: Optional[str] + completed_at: Optional[str] + + +class AgentEvalsAttachment(TypedDict): + id: str + filename: str + content_type: Optional[str] + byte_size: int + + +class AgentEvalsTrialSummary(TypedDict): + id: str + run_id: str + task_id: Optional[str] + task_linked: bool + task_key: str + task_name: Optional[str] + trial_number: int + status: str + passed: Optional[bool] + cost: Optional[float] + latency_ms: Optional[int] + input_tokens: Optional[int] + output_tokens: Optional[int] + cache_read_tokens: Optional[int] + cached_tokens: Optional[int] + cache_write_5m_tokens: Optional[int] + cache_write_1h_tokens: Optional[int] + + +class AgentEvalsTranscript(TypedDict): + id: str + messages: List[OpenInferenceSpan] + + +class AgentEvalsTrial(AgentEvalsTrialSummary, total=False): + task_input: str + task_expected_output: Optional[str] + task_tags: List[str] + grader_definitions_snapshot: List[AgentEvalsGraderVersion] + transcript: Optional[AgentEvalsTranscript] + prompt_context: Any + fixture: Optional[AgentEvalsFixture] + attachments: List[AgentEvalsAttachment] + graders: List[AgentEvalsGrader] + grading: Optional[AgentEvalsGrading] + + +class AgentEvalsRunWithTrials(AgentEvalsRun): + trials: List[AgentEvalsTrialSummary] + + +class AgentEvalsFixtureEntryInput(TypedDict): + key: str + type: AgentEvalsFixtureEntryType + value: Dict[str, Any] + + +class _FindOrCreateProjectInputRequired(TypedDict): + name: str + + +class FindOrCreateProjectInput(_FindOrCreateProjectInputRequired, total=False): + description: str + + +class _FindOrCreateSuiteInputRequired(TypedDict): + name: str + + +class FindOrCreateSuiteInput(_FindOrCreateSuiteInputRequired, total=False): + description: str + tags: Sequence[str] + default_fixture_id: str + + +class _UpsertTaskInputRequired(TypedDict): + key: str + input: str + + +class UpsertTaskInput(_UpsertTaskInputRequired, total=False): + expected_output: str + tags: Sequence[str] + fixture_id: str + input_token_limit: int + output_token_limit: int + + +class _FixtureInputRequired(TypedDict): + name: str + + +class FixtureInput(_FixtureInputRequired, total=False): + description: str + entries: Sequence[AgentEvalsFixtureEntryInput] + + +class AgentEvalsFixtureVersionOverride(TypedDict): + """Select a Fixture Version for one Fixture when creating a run.""" + + fixture_id: str + fixture_version_id: str + + +class UpdateFixtureInput(TypedDict, total=False): + name: str + description: str + entries: Sequence[AgentEvalsFixtureEntryInput] + + +class _FindOrCreateGraderDefinitionInputRequired(TypedDict): + name: str + + +class FindOrCreateGraderDefinitionInput( + _FindOrCreateGraderDefinitionInputRequired, total=False +): + description: str + kind: AgentEvalsGraderKind + prompt: str + code: str + model: str + tags: Sequence[str] + + +class AgentEvalsGraderVersionOverride(TypedDict): + """Select a Grader Version for one Grader when creating a run.""" + + grader_id: str + grader_version_id: str + + +class CreateRunInput(TypedDict, total=False): + """Choose ``suite_id`` for server-driven mode or ``suite_name`` for report-only mode.""" + + suite_id: str + suite_name: str + git_branch: str + git_sha: str + git_repo_url: str + model: str + name: str + status: AgentEvalsClientRunStatus + metadata: Dict[str, Any] + fixture_overrides: Sequence[AgentEvalsFixtureVersionOverride] + grader_overrides: Sequence[AgentEvalsGraderVersionOverride] + + +class _ReportTrialAssertionInputRequired(TypedDict): + assertion_name: str + + +class ReportTrialAssertionInput(_ReportTrialAssertionInputRequired, total=False): + passed: bool + score: float + reasoning: str + + +class _ReportTrialGraderInputRequired(TypedDict): + grader_name: str + assertions: Sequence[ReportTrialAssertionInput] + + +class ReportTrialGraderInput(_ReportTrialGraderInputRequired): + pass + + +class _ReportOnlyTaskInputRequired(TypedDict): + key: str + input: str + + +class ReportOnlyTaskInput(_ReportOnlyTaskInputRequired, total=False): + name: str + expected_output: str + tags: Sequence[str] + + +class _ReportTrialInputRequired(TypedDict): + transcript: Mapping[str, Sequence[OpenInferenceSpan]] + + +class ReportTrialInput(_ReportTrialInputRequired, total=False): + task_key: str + task: ReportOnlyTaskInput + trial_number: int + status: AgentEvalsTrialStatus + cost: float + latency_ms: int + input_tokens: int + output_tokens: int + cache_read_tokens: int + cached_tokens: int + cache_write_5m_tokens: int + cache_write_1h_tokens: int + attachment_upload_ids: Sequence[str] + grader_definitions: Sequence[FindOrCreateGraderDefinitionInput] + graders: Sequence[ReportTrialGraderInput] + grade: bool + + +class AgentEvalsUpload(TypedDict): + upload_id: str + upload_url: str + method: str + max_bytes: int + expires_at: str + instructions: str + + +class UpdateRunInput(TypedDict, total=False): + status: AgentEvalsClientRunStatus + metadata: Dict[str, Any] + + +class _CreateUploadInputRequired(TypedDict): + filename: str + + +class CreateUploadInput(_CreateUploadInputRequired, total=False): + content_type: str + + +JsonObject = Mapping[str, Any] +BytesLike = Union[bytes, bytearray, memoryview] + + +class AgentEvalsError(Exception): + """A non-successful response from the Agent Evals API.""" + + def __init__(self, status: int, method: str, path: str, body: str): + self.status = status + self.method = method + self.path = path + self.body = body + super().__init__( + f"Fabricate agent evals request {method} {path} " + f"failed with {status}: {body}" + ) + + +def _encode(value: str) -> str: + return quote(value, safe="") + + +class AgentEvalsClient: + """Client for Fabricate's framework-agnostic Agent Evals API.""" + + def __init__( + self, + api_key: Optional[str] = None, + api_url: Optional[str] = None, + session: Optional[requests.Session] = None, + ): + resolved_api_key = api_key or os.environ.get("FABRICATE_API_KEY") + if not resolved_api_key: + raise ValueError( + "api_key is required (set it explicitly or via FABRICATE_API_KEY)" + ) + + self.api_key = resolved_api_key + self.api_url = ( + api_url or os.environ.get("FABRICATE_API_URL") or DEFAULT_API_URL + ).rstrip("/") + self.session = session or requests.Session() + self.session.headers.update( + { + "Authorization": f"Bearer {self.api_key}", + "Accept": "application/json", + } + ) + + def _request( + self, + method: str, + path: str, + *, + json: Optional[JsonObject] = None, + params: Optional[Mapping[str, Optional[str]]] = None, + ) -> Any: + search_params = { + key: value + for key, value in (params or {}).items() + if value is not None and value != "" + } + response = self.session.request( + method, + f"{self.api_url}{path}", + json=json, + params=search_params or None, + ) + if not response.ok: + raise AgentEvalsError( + response.status_code, method.upper(), path, response.text + ) + if not response.content: + return None + return response.json() + + # Projects + + def list_projects(self, workspace: str) -> List[AgentEvalsProject]: + return cast( + List[AgentEvalsProject], + self._request("GET", f"/workspaces/{_encode(workspace)}/projects"), + ) + + def find_or_create_project( + self, workspace: str, input: FindOrCreateProjectInput + ) -> AgentEvalsProject: + return cast( + AgentEvalsProject, + self._request( + "POST", + f"/workspaces/{_encode(workspace)}/projects", + json=input, + ), + ) + + # Suites + + def list_suites(self, project_id: str) -> List[AgentEvalsSuite]: + return cast( + List[AgentEvalsSuite], + self._request("GET", f"/projects/{_encode(project_id)}/suites"), + ) + + def find_or_create_suite( + self, project_id: str, input: FindOrCreateSuiteInput + ) -> AgentEvalsSuite: + return cast( + AgentEvalsSuite, + self._request( + "POST", + f"/projects/{_encode(project_id)}/suites", + json=input, + ), + ) + + def find_suite( + self, + project_id: str, + *, + id: Optional[str] = None, + name: Optional[str] = None, + version: Optional[int] = None, + ) -> Optional[AgentEvalsSuite]: + """Find a suite version without creating it. + + When ``version`` is omitted the highest-numbered version of the matching + suite is returned. + """ + suites = self.list_suites(project_id) + if id: + return next((suite for suite in suites if suite["id"] == id), None) + if name: + wanted_name = name.casefold() + named = [suite for suite in suites if suite["name"].casefold() == wanted_name] + if version is not None: + return next((suite for suite in named if suite.get("version") == version), None) + return max(named, key=lambda suite: suite.get("version", 0), default=None) + return None + + def create_suite_version(self, suite_id: str) -> AgentEvalsSuite: + """Create the next version of a suite by copying the given version.""" + return cast( + AgentEvalsSuite, + self._request("POST", f"/suites/{_encode(suite_id)}/versions"), + ) + + # Tasks + + def list_tasks(self, suite_id: str) -> List[AgentEvalsTask]: + return cast( + List[AgentEvalsTask], + self._request("GET", f"/suites/{_encode(suite_id)}/tasks"), + ) + + def upsert_task(self, suite_id: str, input: UpsertTaskInput) -> AgentEvalsTask: + return cast( + AgentEvalsTask, + self._request("POST", f"/suites/{_encode(suite_id)}/tasks", json=input), + ) + + # Fixtures + + def list_fixtures(self, project_id: str) -> List[AgentEvalsFixtureVersion]: + """List Fixture Versions; the API returns the latest version by default.""" + return cast( + List[AgentEvalsFixtureVersion], + self._request("GET", f"/projects/{_encode(project_id)}/fixtures"), + ) + + def find_or_create_fixture( + self, project_id: str, input: FixtureInput + ) -> AgentEvalsFixtureVersion: + """Find or create a Fixture, returning its latest Fixture Version.""" + return cast( + AgentEvalsFixtureVersion, + self._request( + "POST", + f"/projects/{_encode(project_id)}/fixtures", + json=input, + ), + ) + + def get_fixture_version(self, fixture_version_id: str) -> AgentEvalsFixtureVersion: + return cast( + AgentEvalsFixtureVersion, + self._request("GET", f"/fixtures/{_encode(fixture_version_id)}"), + ) + + def get_fixture(self, fixture_version_id: str) -> AgentEvalsFixtureVersion: + """Deprecated alias for :meth:`get_fixture_version`.""" + return self.get_fixture_version(fixture_version_id) + + def create_fixture_version( + self, fixture_version_id: str + ) -> AgentEvalsFixtureVersion: + """Create the next Fixture Version by copying the given version.""" + return cast( + AgentEvalsFixtureVersion, + self._request("POST", f"/fixtures/{_encode(fixture_version_id)}/versions"), + ) + + def update_fixture( + self, fixture_id: str, input: UpdateFixtureInput + ) -> AgentEvalsFixtureVersion: + return cast( + AgentEvalsFixtureVersion, + self._request("PATCH", f"/fixtures/{_encode(fixture_id)}", json=input), + ) + + def delete_fixture(self, fixture_id: str) -> None: + self._request("DELETE", f"/fixtures/{_encode(fixture_id)}") + + def download_fixture_database(self, fixture_id: str, database_id: str) -> bytes: + """Download SQLite bytes for a database referenced by a fixture.""" + path = f"/fixtures/{_encode(fixture_id)}/databases/" f"{_encode(database_id)}" + response = self.session.get( + f"{self.api_url}{path}", + headers={"Accept": "application/vnd.sqlite3"}, + ) + if not response.ok: + raise AgentEvalsError(response.status_code, "GET", path, response.text) + return cast(bytes, response.content) + + # Graders + + def list_graders( + self, project_id: str + ) -> List[AgentEvalsGraderVersion]: + return cast( + List[AgentEvalsGraderVersion], + self._request( + "GET", + f"/projects/{_encode(project_id)}/grader_definitions", + ), + ) + + def list_grader_definitions( + self, project_id: str + ) -> List[AgentEvalsGraderVersion]: + """Deprecated alias for :meth:`list_graders`.""" + return self.list_graders(project_id) + + def find_or_create_grader( + self, + project_id: str, + input: FindOrCreateGraderDefinitionInput, + ) -> AgentEvalsGraderVersion: + return cast( + AgentEvalsGraderVersion, + self._request( + "POST", + f"/projects/{_encode(project_id)}/grader_definitions", + json=input, + ), + ) + + def find_or_create_grader_definition( + self, + project_id: str, + input: FindOrCreateGraderDefinitionInput, + ) -> AgentEvalsGraderVersion: + """Deprecated alias for :meth:`find_or_create_grader`.""" + return self.find_or_create_grader(project_id, input) + + def create_grader_version( + self, grader_version_id: str + ) -> AgentEvalsGraderVersion: + """Create the next Grader Version by copying the given version.""" + return cast( + AgentEvalsGraderVersion, + self._request( + "POST", + f"/grader_definitions/{_encode(grader_version_id)}/versions", + ), + ) + + # Runs + + def list_runs( + self, project_id: str, *, branch: Optional[str] = None + ) -> List[AgentEvalsRun]: + return cast( + List[AgentEvalsRun], + self._request( + "GET", + f"/projects/{_encode(project_id)}/runs", + params={"branch": branch}, + ), + ) + + def create_run( + self, project_id: str, input: CreateRunInput + ) -> AgentEvalsRun: + return cast( + AgentEvalsRun, + self._request( + "POST", + f"/projects/{_encode(project_id)}/runs", + json=input, + ), + ) + + def get_run(self, run_id: str) -> AgentEvalsRunWithTrials: + return cast( + AgentEvalsRunWithTrials, + self._request("GET", f"/runs/{_encode(run_id)}"), + ) + + def update_run(self, run_id: str, input: UpdateRunInput) -> AgentEvalsRun: + return cast( + AgentEvalsRun, + self._request("PATCH", f"/runs/{_encode(run_id)}", json=input), + ) + + # Trials + + def report_trial(self, run_id: str, input: ReportTrialInput) -> AgentEvalsTrial: + return cast( + AgentEvalsTrial, + self._request("POST", f"/runs/{_encode(run_id)}/trials", json=input), + ) + + def get_trial(self, trial_id: str) -> AgentEvalsTrial: + return cast( + AgentEvalsTrial, + self._request("GET", f"/trials/{_encode(trial_id)}"), + ) + + def wait_for_grading( + self, + trial_id: str, + *, + interval_seconds: float = DEFAULT_GRADING_INTERVAL_SECONDS, + timeout_seconds: float = DEFAULT_GRADING_TIMEOUT_SECONDS, + cancel_event: Optional[Event] = None, + ) -> AgentEvalsTrial: + """Poll until asynchronous grading completes, fails, or times out.""" + deadline = time.monotonic() + timeout_seconds + trial = self.get_trial(trial_id) + grading = trial.get("grading") + + while grading and grading["status"] not in ("completed", "failed"): + if time.monotonic() >= deadline: + raise TimeoutError( + f"Timed out after {timeout_seconds:g}s waiting for " + f"trial {trial_id} grading." + ) + + if cancel_event: + if cancel_event.wait(interval_seconds): + raise RuntimeError("Aborted") + else: + time.sleep(interval_seconds) + + trial = self.get_trial(trial_id) + grading = trial.get("grading") + + return trial + + # Attachments + + def create_upload( + self, workspace: str, input: CreateUploadInput + ) -> AgentEvalsUpload: + return cast( + AgentEvalsUpload, + self._request( + "POST", + f"/workspaces/{_encode(workspace)}/agent_evals/uploads", + json=input, + ), + ) + + def put_upload_bytes( + self, + upload_url: str, + data: BytesLike, + content_type: Optional[str] = None, + ) -> None: + headers = {"Content-Type": content_type} if content_type else None + response = self.session.put(upload_url, data=bytes(data), headers=headers) + if not response.ok: + raise AgentEvalsError( + response.status_code, "PUT", upload_url, response.text + ) + + def upload_attachment( + self, + workspace: str, + *, + filename: str, + data: BytesLike, + content_type: Optional[str] = None, + ) -> str: + upload_input: CreateUploadInput = {"filename": filename} + if content_type: + upload_input["content_type"] = content_type + upload = self.create_upload(workspace, upload_input) + self.put_upload_bytes(upload["upload_url"], data, content_type) + return upload["upload_id"] + + +__all__ = [ + "AgentEvalsAssertion", + "AgentEvalsAttachment", + "AgentEvalsClient", + "AgentEvalsError", + "AgentEvalsFixture", + "AgentEvalsFixtureVersion", + "AgentEvalsFixtureEntry", + "AgentEvalsFixtureEntryInput", + "AgentEvalsFixtureEntryType", + "AgentEvalsFixtureVersionOverride", + "AgentEvalsGrader", + "AgentEvalsGraderDefinition", + "AgentEvalsGraderKind", + "AgentEvalsGraderVersion", + "AgentEvalsGraderVersionOverride", + "AgentEvalsGrading", + "AgentEvalsGradingStatus", + "AgentEvalsProject", + "AgentEvalsClientRunStatus", + "AgentEvalsRun", + "AgentEvalsRunStatus", + "AgentEvalsRunWithTrials", + "AgentEvalsSuite", + "AgentEvalsTask", + "AgentEvalsTrial", + "AgentEvalsTrialStatus", + "AgentEvalsTrialSummary", + "AgentEvalsUpload", + "CreateUploadInput", + "CreateRunInput", + "FindOrCreateProjectInput", + "FindOrCreateGraderDefinitionInput", + "FindOrCreateSuiteInput", + "FixtureInput", + "OpenInferenceSpan", + "ReportTrialInput", + "ReportOnlyTaskInput", + "UpdateFixtureInput", + "UpdateRunInput", + "UpsertTaskInput", +] diff --git a/tonic_fabricate/client.py b/tonic_fabricate/client.py index 9bcf730..bf98569 100644 --- a/tonic_fabricate/client.py +++ b/tonic_fabricate/client.py @@ -3,10 +3,302 @@ import zipfile import shutil from pathlib import Path -from typing import Optional, Callable, Dict, Any, Union +from typing import Optional, Callable, Dict, Any, Union, List +from dataclasses import dataclass, field +from urllib.parse import quote import requests +@dataclass +class WorkflowFile: + """Represents a file generated by a workflow.""" + id: int + name: str + size: int + content_type: str + + +@dataclass +class WorkflowTask: + """Represents a workflow task.""" + id: str + workflow_id: str + status: str # 'in_progress', 'completed', 'failed', 'canceled' + result: Optional[Any] = None + files: Optional[List[WorkflowFile]] = None + error: Optional[str] = None + started_at: Optional[str] = None + completed_at: Optional[str] = None + created_at: Optional[str] = None + + +@dataclass +class WorkflowResult: + """Result of running a workflow, with helpers to download generated files.""" + result: Any + task: WorkflowTask + _api_key: str = field(repr=False) + _api_url: str = field(repr=False) + + def download_file(self, file_id: int, dest_path: str) -> None: + """ + Downloads a file from the workflow task. + + Args: + file_id: The ID of the file to download (from task.files[].id) + dest_path: The destination path to save the file + """ + download_workflow_file( + api_key=self._api_key, + api_url=self._api_url, + task_id=self.task.id, + file_id=file_id, + dest_path=dest_path + ) + + def download_all_files(self, dest_dir: str) -> None: + """ + Downloads all files from the workflow task to a directory. + + Args: + dest_dir: The destination directory to save the files + """ + if not self.task.files: + return + + dest_path = Path(dest_dir) + dest_path.mkdir(parents=True, exist_ok=True) + + for file in self.task.files: + # Sanitize filename to prevent path traversal attacks + safe_name = Path(file.name).name + file_dest = dest_path / safe_name + self.download_file(file.id, str(file_dest)) + + +def run_workflow( + database: str, + workspace: str, + workflow: str, + api_key: Optional[str] = None, + api_url: str = "https://fabricate.tonic.ai/api/v1", + params: Optional[Dict[str, Any]] = None, + on_progress: Optional[Callable[[Dict[str, str]], None]] = None, +) -> WorkflowResult: + """ + Runs a workflow and waits for the result. + + Args: + database: The name of the database (required) + workspace: The workspace to use (required) + workflow: The name of the workflow to run (required) + api_key: The API key for authentication. Defaults to FABRICATE_API_KEY env var. + api_url: The API URL. Defaults to https://fabricate.tonic.ai/api/v1 + params: Optional parameters to pass to the workflow + on_progress: Optional progress callback function receiving {'status': str, 'message': str} + + Returns: + WorkflowResult containing the result, task, and file download methods + + Raises: + ValueError: If required parameters are missing + Exception: If workflow execution fails + """ + # Get API key from parameter or environment + if api_key is None: + api_key = os.environ.get('FABRICATE_API_KEY') + + # Validate required parameters + if not api_key: + raise ValueError('api_key is required') + + if not database: + raise ValueError('database is required') + + if not workspace: + raise ValueError('workspace is required') + + if not workflow: + raise ValueError('workflow is required') + + if params is None: + params = {} + + headers = {'Authorization': f'Bearer {api_key}'} + url = ( + f'{api_url}/workspaces/{quote(workspace, safe="")}' + f'/databases/{quote(database, safe="")}' + f'/workflows/{quote(workflow, safe="")}' + ) + + try: + response = requests.post(url, json=params, headers=headers) + response.raise_for_status() + data = response.json() + except requests.exceptions.HTTPError as e: + if e.response is not None and 'application/json' in e.response.headers.get('content-type', ''): + error_data = e.response.json() + raise Exception(error_data.get('error', str(e))) + else: + raise + + task_id = data.get('task_id') + status = data.get('status', '') + + if not task_id: + raise Exception('No task_id returned from API') + + if on_progress: + on_progress({'status': status, 'message': 'Workflow started'}) + + # Poll for completion + task = _poll_workflow_task(task_id, api_url, api_key, on_progress) + + if task.error: + raise Exception(task.error) + + return WorkflowResult( + result=task.result, + task=task, + _api_key=api_key, + _api_url=api_url + ) + + +def _poll_workflow_task( + task_id: str, + api_url: str, + api_key: str, + on_progress: Optional[Callable[[Dict[str, str]], None]] = None +) -> WorkflowTask: + """ + Polls the workflow task API until the task is completed. + + Args: + task_id: The ID of the task to poll + api_url: The API URL + api_key: The API key + on_progress: Optional progress callback + + Returns: + The completed WorkflowTask + + Raises: + Exception: If task fails or is canceled + """ + headers = {'Authorization': f'Bearer {api_key}'} + + while True: + response = requests.get( + f'{api_url}/workflow_tasks/{task_id}', + headers=headers + ) + response.raise_for_status() + data = response.json() + + task = _parse_workflow_task(data) + + if task.status == 'completed': + if on_progress: + on_progress({'status': 'completed', 'message': 'Workflow completed'}) + return task + elif task.status == 'failed': + raise Exception(task.error or 'Workflow failed') + elif task.status == 'canceled': + raise Exception('Workflow was canceled') + else: + if on_progress: + on_progress({'status': task.status, 'message': 'Workflow running...'}) + time.sleep(1) + + +def _parse_workflow_task(data: Dict[str, Any]) -> WorkflowTask: + """Parses a workflow task from API response data.""" + files = None + if data.get('files'): + files = [ + WorkflowFile( + id=f['id'], + name=f['name'], + size=f['size'], + content_type=f['content_type'] + ) + for f in data['files'] + ] + + return WorkflowTask( + id=data['id'], + workflow_id=data.get('workflow_id', ''), + status=data['status'], + result=data.get('result'), + files=files, + error=data.get('error'), + started_at=data.get('started_at'), + completed_at=data.get('completed_at'), + created_at=data.get('created_at') + ) + + +def download_workflow_file( + task_id: str, + file_id: int, + dest_path: str, + api_key: Optional[str] = None, + api_url: str = "https://fabricate.tonic.ai/api/v1", +) -> None: + """ + Downloads a file from a workflow task. + + Args: + task_id: The ID of the workflow task + file_id: The ID of the file to download + dest_path: The destination path to save the file + api_key: The API key for authentication. Defaults to FABRICATE_API_KEY env var. + api_url: The API URL. Defaults to https://fabricate.tonic.ai/api/v1 + + Raises: + ValueError: If required parameters are missing + requests.HTTPError: If download fails + """ + # Get API key from parameter or environment + if api_key is None: + api_key = os.environ.get('FABRICATE_API_KEY') + + if not api_key: + raise ValueError('api_key is required') + + if not task_id: + raise ValueError('task_id is required') + + if not file_id: + raise ValueError('file_id is required') + + if not dest_path: + raise ValueError('dest_path is required') + + # Ensure the directory exists + dest_file = Path(dest_path) + dest_dir = dest_file.parent + + if dest_dir != Path('.') and not dest_dir.exists(): + dest_dir.mkdir(parents=True, exist_ok=True) + + url = f'{api_url}/workflow_tasks/{task_id}/{file_id}/download' + headers = {'Authorization': f'Bearer {api_key}'} + + try: + with requests.get(url, headers=headers, stream=True) as response: + response.raise_for_status() + with open(dest_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + except Exception: + # Clean up partial file on error + if dest_file.exists(): + dest_file.unlink() + raise + + def generate( api_key: Optional[str] = None, api_url: str = "https://fabricate.tonic.ai/api/v1",