From 74861beb6fc0a5709204636da2f78c77f7f9d3fc Mon Sep 17 00:00:00 2001 From: Mark Brocato Date: Fri, 23 Jan 2026 17:06:59 +0200 Subject: [PATCH 1/9] Update version to 1.1.0 and enhance README with workflow examples; add workflow-related classes and methods for improved functionality. --- DEVELOPMENT.md | 148 ++++++++++++++++++ README.md | 81 +++++++++- examples/download.py | 10 +- examples/workflow.py | 62 ++++++++ pyproject.toml | 2 +- tonic_fabricate/__init__.py | 20 ++- tonic_fabricate/client.py | 294 +++++++++++++++++++++++++++++++++++- 7 files changed, 604 insertions(+), 13 deletions(-) create mode 100644 DEVELOPMENT.md create mode 100644 examples/workflow.py diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..b3989e5 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,148 @@ +# Development Guide + +This guide covers how to set up and test the package locally before publishing. + +## Setup + +### Create a Virtual Environment + +```bash +# Create a virtual environment +python -m venv venv + +# Activate the virtual environment +source venv/bin/activate # On macOS/Linux +# venv\Scripts\activate # On Windows +``` + +### Install in Development Mode + +With the virtual environment activated, install the package with dev dependencies: + +```bash +# Install in editable/development mode with dev dependencies +pip install -e ".[dev]" +``` + +This installs the package along with `python-dotenv` (for `.env` file support), `pytest`, `mypy`, `black`, and `flake8`. + +### Environment Variables + +Set the following environment variables for testing: + +```bash +export FABRICATE_API_KEY="your-api-key" +export FABRICATE_API_URL="https://fabricate.tonic.ai/api/v1" # or your local instance +``` + +Or create a `.env` file (requires `python-dotenv`): + +``` +FABRICATE_API_KEY=your-api-key +FABRICATE_API_URL=https://fabricate.tonic.ai/api/v1 +``` + +## Testing + +### Quick Import Test + +Verify the module imports correctly: + +```bash +python -c "from tonic_fabricate import generate, run_workflow, WorkflowResult; print('All imports work!')" +``` + +### Run the Examples + +The examples test the actual API calls: + +```bash +# Test the generate function +python examples/download.py + +# Test the workflow function +python examples/workflow.py +``` + +### Interactive Testing + +```python +from tonic_fabricate import run_workflow +import os + +result = run_workflow( + database='your_database', + workspace='your_workspace', + workflow='your_workflow', + api_url=os.environ.get('FABRICATE_API_URL'), + on_progress=lambda p: print(p) +) +print(result.result) +``` + +## Code Quality + +### Install Dev Dependencies + +```bash +pip install -r requirements-dev.txt +``` + +### Type Checking + +```bash +mypy tonic_fabricate/ +``` + +### Linting + +```bash +flake8 tonic_fabricate/ +``` + +### Formatting + +```bash +# Check formatting +black --check tonic_fabricate/ + +# Auto-format +black tonic_fabricate/ +``` + +## Pre-Publish Testing + +Before publishing to production PyPI, test with TestPyPI: + +```bash +# Publish to TestPyPI +./publish-test.sh + +# Install from TestPyPI to verify +pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ tonic-fabricate==1.1.0 + +# Test the installed package +python -c "from tonic_fabricate import generate, run_workflow; print('Package works!')" +``` + +## Publishing + +See [publishing.md](publishing.md) for detailed publishing instructions. + +### Quick Reference + +```bash +# Test publish +./publish-test.sh + +# Production publish (after testing) +./publish.sh +``` + +## Deactivating the Virtual Environment + +When you're done developing: + +```bash +deactivate +``` diff --git a/README.md b/README.md index 889cd56..6e64837 100644 --- a/README.md +++ b/README.md @@ -109,12 +109,80 @@ The client will automatically use the following environment variables if they ar - `FABRICATE_API_KEY`: Your Fabricate API key - `FABRICATE_API_URL`: The Fabricate API URL (defaults to https://fabricate.tonic.ai/api/v1) +## Workflows + +Fabricate supports workflows that can perform custom operations and generate files. To run a workflow: + +```python +from tonic_fabricate import run_workflow + +result = run_workflow( + # The workspace to use + workspace='Default', + + # The name of the database + database='my_database', + + # The name of the workflow to run + workflow='my_workflow', + + # Optional: Parameters to pass to the workflow + params={ + 'message': 'Hello, world!', + }, +) + +# Access the workflow result +print(f"Result: {result.result}") + +# Download generated files if any +if result.task.files: + for file in result.task.files: + print(f"File: {file.name} ({file.size} bytes)") + result.download_file(file.id, f"./output/{file.name}") + + # Or download all files at once + result.download_all_files('./output') +``` + +### Workflow Progress Tracking + +```python +from tonic_fabricate import run_workflow + +def on_progress(data): + status = data.get('status', '') + message = data.get('message', '') + print(f"[{status}] {message}") + +result = run_workflow( + workspace='Default', + database='my_database', + workflow='my_workflow', + on_progress=on_progress +) +``` + +### Workflow File Downloads + +You can also download workflow files directly using `download_workflow_file`: + +```python +from tonic_fabricate import download_workflow_file + +download_workflow_file( + task_id='your-task-id', + file_id=123, + dest_path='./output/file.txt' +) +``` + ## Error Handling The client raises appropriate exceptions for various error conditions: ```python -from tonic_fabricate import generate +from tonic_fabricate import generate, run_workflow try: generate( @@ -127,4 +195,15 @@ except ValueError as e: print(f"Invalid parameters: {e}") except Exception as e: print(f"Generation failed: {e}") + +try: + result = run_workflow( + workspace='Default', + database='my_database', + workflow='my_workflow' + ) +except ValueError as e: + print(f"Invalid parameters: {e}") +except Exception as e: + print(f"Workflow failed: {e}") ``` diff --git a/examples/download.py b/examples/download.py index a9279b5..cbcbdc4 100644 --- a/examples/download.py +++ b/examples/download.py @@ -7,14 +7,10 @@ import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) -from tonic_fabricate import generate +from dotenv import load_dotenv +load_dotenv() -# Load environment variables if python-dotenv is available -try: - from dotenv import load_dotenv - load_dotenv() -except ImportError: - pass +from tonic_fabricate import generate def on_progress(data): """Progress callback function.""" diff --git a/examples/workflow.py b/examples/workflow.py new file mode 100644 index 0000000..74090db --- /dev/null +++ b/examples/workflow.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +""" +Example script for running a workflow in Fabricate. +""" + +import os +import sys +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from dotenv import load_dotenv +load_dotenv() + +from tonic_fabricate import run_workflow + +def on_progress(data): + """Progress callback function.""" + status = data.get('status', '') + message = data.get('message', '') + print(f"[{status}] {message}") + +if __name__ == "__main__": + print("Starting workflow...") + + # Prepare arguments + kwargs = { + 'database': 'agent_api_test', + 'workspace': 'API', + 'workflow': 'file', + 'params': { + 'message': 'Hello, world!', + }, + 'on_progress': on_progress, + } + + # Add api_url if available + api_url = os.environ.get('FABRICATE_API_URL') + if api_url: + kwargs['api_url'] = api_url + + # Run the workflow + workflow_result = run_workflow(**kwargs) + + print(f"Workflow result: {workflow_result.result}") + + # List and download files if any were generated + if workflow_result.task.files: + print(f"\nWorkflow generated {len(workflow_result.task.files)} file(s):") + + dest_dir = './tmp/workflow_output' + + for file in workflow_result.task.files: + print(f" - {file.name} ({file.content_type}, {file.size} bytes, id: {file.id})") + + # Download a specific file by id + workflow_result.download_file(file.id, f"{dest_dir}/{file.name}") + + # Or, download all files to a directory + # print('\nDownloading all files...') + # workflow_result.download_all_files(dest_dir) + # print(f'Files downloaded to {dest_dir}/') + + print("Done.") diff --git a/pyproject.toml b/pyproject.toml index 7756280..53760f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "tonic-fabricate" -version = "1.0.2" +version = "1.1.0" description = "The official Fabricate client for Python" authors = [{name = "Fabricate Tools"}] readme = "README.md" diff --git a/tonic_fabricate/__init__.py b/tonic_fabricate/__init__.py index 211d230..2ce1808 100644 --- a/tonic_fabricate/__init__.py +++ b/tonic_fabricate/__init__.py @@ -2,8 +2,22 @@ Tonic Fabricate - The official Fabricate client for Python. """ -from .client import generate +from .client import ( + generate, + run_workflow, + download_workflow_file, + WorkflowFile, + WorkflowTask, + WorkflowResult, +) -__version__ = "1.0.1" +__version__ = "1.1.0" -__all__ = ["generate"] \ No newline at end of file +__all__ = [ + "generate", + "run_workflow", + "download_workflow_file", + "WorkflowFile", + "WorkflowTask", + "WorkflowResult", +] \ No newline at end of file diff --git a/tonic_fabricate/client.py b/tonic_fabricate/client.py index 9bcf730..7e77575 100644 --- a/tonic_fabricate/client.py +++ b/tonic_fabricate/client.py @@ -3,10 +3,302 @@ import zipfile import shutil from pathlib import Path -from typing import Optional, Callable, Dict, Any, Union +from typing import Optional, Callable, Dict, Any, Union, List +from dataclasses import dataclass +from urllib.parse import quote import requests +@dataclass +class WorkflowFile: + """Represents a file generated by a workflow.""" + id: int + name: str + size: int + content_type: str + + +@dataclass +class WorkflowTask: + """Represents a workflow task.""" + id: str + workflow_id: str + status: str # 'in_progress', 'completed', 'failed', 'canceled' + result: Optional[Any] = None + files: Optional[List[WorkflowFile]] = None + error: Optional[str] = None + started_at: Optional[str] = None + completed_at: Optional[str] = None + created_at: Optional[str] = None + + +@dataclass +class WorkflowResult: + """Result of running a workflow, with helpers to download generated files.""" + result: Any + task: WorkflowTask + _api_key: str + _api_url: str + + def download_file(self, file_id: int, dest_path: str) -> None: + """ + Downloads a file from the workflow task. + + Args: + file_id: The ID of the file to download (from task.files[].id) + dest_path: The destination path to save the file + """ + download_workflow_file( + api_key=self._api_key, + api_url=self._api_url, + task_id=self.task.id, + file_id=file_id, + dest_path=dest_path + ) + + def download_all_files(self, dest_dir: str) -> None: + """ + Downloads all files from the workflow task to a directory. + + Args: + dest_dir: The destination directory to save the files + """ + if not self.task.files: + return + + dest_path = Path(dest_dir) + dest_path.mkdir(parents=True, exist_ok=True) + + for file in self.task.files: + # Sanitize filename to prevent path traversal attacks + safe_name = Path(file.name).name + file_dest = dest_path / safe_name + self.download_file(file.id, str(file_dest)) + + +def run_workflow( + database: str, + workspace: str, + workflow: str, + api_key: Optional[str] = None, + api_url: str = "https://fabricate.tonic.ai/api/v1", + params: Optional[Dict[str, Any]] = None, + on_progress: Optional[Callable[[Dict[str, str]], None]] = None, +) -> WorkflowResult: + """ + Runs a workflow and waits for the result. + + Args: + database: The name of the database (required) + workspace: The workspace to use (required) + workflow: The name of the workflow to run (required) + api_key: The API key for authentication. Defaults to FABRICATE_API_KEY env var. + api_url: The API URL. Defaults to https://fabricate.tonic.ai/api/v1 + params: Optional parameters to pass to the workflow + on_progress: Optional progress callback function receiving {'status': str, 'message': str} + + Returns: + WorkflowResult containing the result, task, and file download methods + + Raises: + ValueError: If required parameters are missing + Exception: If workflow execution fails + """ + # Get API key from parameter or environment + if api_key is None: + api_key = os.environ.get('FABRICATE_API_KEY') + + # Validate required parameters + if not api_key: + raise ValueError('api_key is required') + + if not database: + raise ValueError('database is required') + + if not workspace: + raise ValueError('workspace is required') + + if not workflow: + raise ValueError('workflow is required') + + if params is None: + params = {} + + headers = {'Authorization': f'Bearer {api_key}'} + url = ( + f'{api_url}/workspaces/{quote(workspace, safe="")}' + f'/databases/{quote(database, safe="")}' + f'/workflows/{quote(workflow, safe="")}' + ) + + try: + response = requests.post(url, json=params, headers=headers) + response.raise_for_status() + data = response.json() + except requests.exceptions.HTTPError as e: + if e.response is not None and 'application/json' in e.response.headers.get('content-type', ''): + error_data = e.response.json() + raise Exception(error_data.get('error', str(e))) + else: + raise + + task_id = data.get('task_id') + status = data.get('status', '') + + if not task_id: + raise Exception('No task_id returned from API') + + if on_progress: + on_progress({'status': status, 'message': 'Workflow started'}) + + # Poll for completion + task = _poll_workflow_task(task_id, api_url, api_key, on_progress) + + if task.error: + raise Exception(task.error) + + return WorkflowResult( + result=task.result, + task=task, + _api_key=api_key, + _api_url=api_url + ) + + +def _poll_workflow_task( + task_id: str, + api_url: str, + api_key: str, + on_progress: Optional[Callable[[Dict[str, str]], None]] = None +) -> WorkflowTask: + """ + Polls the workflow task API until the task is completed. + + Args: + task_id: The ID of the task to poll + api_url: The API URL + api_key: The API key + on_progress: Optional progress callback + + Returns: + The completed WorkflowTask + + Raises: + Exception: If task fails or is canceled + """ + headers = {'Authorization': f'Bearer {api_key}'} + + while True: + response = requests.get( + f'{api_url}/workflow_tasks/{task_id}', + headers=headers + ) + response.raise_for_status() + data = response.json() + + task = _parse_workflow_task(data) + + if task.status == 'completed': + if on_progress: + on_progress({'status': 'completed', 'message': 'Workflow completed'}) + return task + elif task.status == 'failed': + raise Exception(task.error or 'Workflow failed') + elif task.status == 'canceled': + raise Exception('Workflow was canceled') + else: + if on_progress: + on_progress({'status': task.status, 'message': 'Workflow running...'}) + time.sleep(1) + + +def _parse_workflow_task(data: Dict[str, Any]) -> WorkflowTask: + """Parses a workflow task from API response data.""" + files = None + if data.get('files'): + files = [ + WorkflowFile( + id=f['id'], + name=f['name'], + size=f['size'], + content_type=f['content_type'] + ) + for f in data['files'] + ] + + return WorkflowTask( + id=data['id'], + workflow_id=data.get('workflow_id', ''), + status=data['status'], + result=data.get('result'), + files=files, + error=data.get('error'), + started_at=data.get('started_at'), + completed_at=data.get('completed_at'), + created_at=data.get('created_at') + ) + + +def download_workflow_file( + task_id: str, + file_id: int, + dest_path: str, + api_key: Optional[str] = None, + api_url: str = "https://fabricate.tonic.ai/api/v1", +) -> None: + """ + Downloads a file from a workflow task. + + Args: + task_id: The ID of the workflow task + file_id: The ID of the file to download + dest_path: The destination path to save the file + api_key: The API key for authentication. Defaults to FABRICATE_API_KEY env var. + api_url: The API URL. Defaults to https://fabricate.tonic.ai/api/v1 + + Raises: + ValueError: If required parameters are missing + requests.HTTPError: If download fails + """ + # Get API key from parameter or environment + if api_key is None: + api_key = os.environ.get('FABRICATE_API_KEY') + + if not api_key: + raise ValueError('api_key is required') + + if not task_id: + raise ValueError('task_id is required') + + if not file_id: + raise ValueError('file_id is required') + + if not dest_path: + raise ValueError('dest_path is required') + + # Ensure the directory exists + dest_file = Path(dest_path) + dest_dir = dest_file.parent + + if dest_dir != Path('.') and not dest_dir.exists(): + dest_dir.mkdir(parents=True, exist_ok=True) + + url = f'{api_url}/workflow_tasks/{task_id}/{file_id}/download' + headers = {'Authorization': f'Bearer {api_key}'} + + try: + with requests.get(url, headers=headers, stream=True) as response: + response.raise_for_status() + with open(dest_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + except Exception: + # Clean up partial file on error + if dest_file.exists(): + dest_file.unlink() + raise + + def generate( api_key: Optional[str] = None, api_url: str = "https://fabricate.tonic.ai/api/v1", From f98abfdb0f4ff886199b2ef064d12953f6b22f34 Mon Sep 17 00:00:00 2001 From: Mark Brocato Date: Fri, 23 Jan 2026 17:08:48 +0200 Subject: [PATCH 2/9] Add GitHub Actions workflow for automated code review using Codex on pull requests labeled with 'ai_review'. --- .github/workflows/pr-review-codex.yml | 68 +++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/workflows/pr-review-codex.yml diff --git a/.github/workflows/pr-review-codex.yml b/.github/workflows/pr-review-codex.yml new file mode 100644 index 0000000..4574223 --- /dev/null +++ b/.github/workflows/pr-review-codex.yml @@ -0,0 +1,68 @@ +name: Perform a code review when a pull request is created. +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review, labeled] + +jobs: + codex: + runs-on: ubuntu-24.04 + if: contains(github.event.pull_request.labels.*.name, 'ai_review') + + permissions: + contents: read + pull-requests: write + issues: write + + outputs: + final_message: ${{ steps.run_codex.outputs.final-message }} + steps: + - uses: actions/checkout@v5 + with: + ref: refs/pull/${{ github.event.pull_request.number }}/merge + + - name: Pre-fetch base and head refs for the PR + run: | + git fetch --no-tags origin \ + ${{ github.event.pull_request.base.ref }} \ + +refs/pull/${{ github.event.pull_request.number }}/head + + - name: Run Codex + id: run_codex + uses: openai/codex-action@v1 + with: + openai-api-key: ${{ secrets.OPENAI_API_KEY }} + prompt: | + This is PR #${{ github.event.pull_request.number }} for ${{ github.repository }}. + Base SHA: ${{ github.event.pull_request.base.sha }} + Head SHA: ${{ github.event.pull_request.head.sha }} + + Review ONLY the changes introduced by the PR. + Suggest any improvements, potential bugs, security issues, or issues. + Be concise and specific in your feedback. + + Pull request title and body: + ---- + ${{ github.event.pull_request.title }} + ${{ github.event.pull_request.body }} + + post_feedback: + runs-on: ubuntu-latest + needs: codex + if: needs.codex.outputs.final_message != '' + permissions: + issues: write + pull-requests: write + steps: + - name: Report Codex feedback + uses: actions/github-script@v7 + env: + CODEX_FINAL_MESSAGE: ${{ needs.codex.outputs.final_message }} + with: + github-token: ${{ github.token }} + script: | + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + body: process.env.CODEX_FINAL_MESSAGE, + }); From 6026669ada3af4565d46bcde18d128374cde5801 Mon Sep 17 00:00:00 2001 From: Mark Brocato Date: Fri, 23 Jan 2026 17:11:05 +0200 Subject: [PATCH 3/9] Remove conditional check for 'ai_review' label in Codex workflow to allow broader execution on pull requests. --- .github/workflows/pr-review-codex.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr-review-codex.yml b/.github/workflows/pr-review-codex.yml index 4574223..0a94628 100644 --- a/.github/workflows/pr-review-codex.yml +++ b/.github/workflows/pr-review-codex.yml @@ -6,7 +6,6 @@ on: jobs: codex: runs-on: ubuntu-24.04 - if: contains(github.event.pull_request.labels.*.name, 'ai_review') permissions: contents: read From a67df7f73bdb88d68e1a7e41032f8aa445461b2b Mon Sep 17 00:00:00 2001 From: Mark Brocato Date: Fri, 23 Jan 2026 17:15:05 +0200 Subject: [PATCH 4/9] Update .gitignore to include .claude directory and ensure .DS_Store is listed; fix formatting in __init__.py by adding newline at end of file. --- .gitignore | 5 ++++- tonic_fabricate/__init__.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index b765a90..d6b7b16 100644 --- a/.gitignore +++ b/.gitignore @@ -106,4 +106,7 @@ dmypy.json # Example files temp directories tmp/ -.DS_Store \ No newline at end of file +.DS_Store + +# Claude +.claude/ \ No newline at end of file diff --git a/tonic_fabricate/__init__.py b/tonic_fabricate/__init__.py index 2ce1808..624760e 100644 --- a/tonic_fabricate/__init__.py +++ b/tonic_fabricate/__init__.py @@ -20,4 +20,4 @@ "WorkflowFile", "WorkflowTask", "WorkflowResult", -] \ No newline at end of file +] From 4c43819f474502dfdb13fe1227240cec2681737f Mon Sep 17 00:00:00 2001 From: Mark Brocato Date: Thu, 27 Aug 2026 09:33:28 +0300 Subject: [PATCH 5/9] Add the Agent Evals client and bump the package to 1.2.0. Expose AgentEvalsClient with suite, run, trial, fixture, grader, and attachment APIs, plus tests, an example, and docs. Co-authored-by: Cursor --- .flake8 | 2 + .github/workflows/test.yml | 33 ++ DEVELOPMENT.md | 18 +- README.md | 158 ++++++ examples/agent_evals.py | 96 ++++ pyproject.toml | 2 +- tests/test_agent_evals.py | 581 ++++++++++++++++++++++ tonic_fabricate/__init__.py | 82 +++- tonic_fabricate/agent_evals.py | 854 +++++++++++++++++++++++++++++++++ 9 files changed, 1821 insertions(+), 5 deletions(-) create mode 100644 .flake8 create mode 100644 .github/workflows/test.yml create mode 100644 examples/agent_evals.py create mode 100644 tests/test_agent_evals.py create mode 100644 tonic_fabricate/agent_evals.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..2bcd70e --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 88 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..d9b6022 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,33 @@ +name: Test + +on: + pull_request: + push: + branches: + - main + +jobs: + pytest: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install package and test dependencies + run: python -m pip install --upgrade pip && python -m pip install -e ".[dev]" + + - name: Run tests + run: python -m pytest diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index b3989e5..2efde2f 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -49,7 +49,16 @@ FABRICATE_API_URL=https://fabricate.tonic.ai/api/v1 Verify the module imports correctly: ```bash -python -c "from tonic_fabricate import generate, run_workflow, WorkflowResult; print('All imports work!')" +python -c "from tonic_fabricate import generate, run_workflow, AgentEvalsClient; print('All imports work!')" +``` + +### Run Unit Tests + +The Agent Evals client tests use a local HTTP server and do not require +Fabricate credentials: + +```bash +python -m pytest ``` ### Run the Examples @@ -62,6 +71,9 @@ python examples/download.py # Test the workflow function python examples/workflow.py + +# Test Agent Evals against a configured Fabricate project and suite +python examples/agent_evals.py ``` ### Interactive Testing @@ -119,10 +131,10 @@ Before publishing to production PyPI, test with TestPyPI: ./publish-test.sh # Install from TestPyPI to verify -pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ tonic-fabricate==1.1.0 +pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ tonic-fabricate==1.2.0 # Test the installed package -python -c "from tonic_fabricate import generate, run_workflow; print('Package works!')" +python -c "from tonic_fabricate import generate, run_workflow, AgentEvalsClient; print('Package works!')" ``` ## Publishing diff --git a/README.md b/README.md index 6e64837..0960a9f 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,31 @@ pip install tonic-fabricate ## Usage +### Report-only Agent Evals + +Client-owned tasks and grader prompts can be reported without creating +Fabricate definitions: + +```python +run = client.create_run(project_id, {"suite_name": "Git-defined evals", "model": model}) +client.report_trial( + run["id"], + { + "task": {"key": "users", "input": "Generate 100 users", "tags": ["users"]}, + "grader_definitions": [ + {"name": "quality", "prompt": "Inspect the attached data.", "tags": ["users"]} + ], + "transcript": {"messages": spans}, + "attachment_upload_ids": upload_ids, + "grade": True, + }, +) +``` + +Trial token metrics include `cache_read_tokens`, `cache_write_5m_tokens`, and +`cache_write_1h_tokens`. Fabricate can derive them from equivalent +`llm.token_count.prompt_details.*` OpenInference attributes. + To generate and download data from Fabricate: ```python @@ -177,6 +202,139 @@ download_workflow_file( ) ``` +## Agent Evals + +`AgentEvalsClient` runs any Python agent against evaluation suites managed in +Fabricate. It is framework-independent: invoke your agent however you prefer, +then report its execution as flattened +[OpenInference](https://github.com/Arize-ai/openinference) spans. + +```python +from tonic_fabricate import AgentEvalsClient + +client = AgentEvalsClient() + +# Fabricate is the source of truth for the suite and its tasks. +suite = client.find_suite( + project_id="your-project-id", + name="Agent Eval Tasks", +) +if suite is None: + raise RuntimeError("Suite not found") + +tasks = client.list_tasks(suite["id"]) +run = client.create_run( + "your-project-id", + { + "suite_id": suite["id"], + "model": "gpt-5-mini", + "git_branch": "feature/my-agent", + }, +) + +for task in tasks: + # Replace this span with the OpenInference spans captured from your agent. + transcript = [ + { + "name": "my-agent", + "attributes": { + "openinference.span.kind": "AGENT", + "input.value": task["input"], + }, + } + ] + reported = client.report_trial( + run["id"], + { + "task_key": task["key"], + "transcript": {"messages": transcript}, + "grade": True, + }, + ) + graded = client.wait_for_grading(reported["id"]) + print(task["key"], "PASS" if graded["passed"] else "FAIL") + +client.update_run(run["id"], {"status": "completed"}) +``` + +The client reads `FABRICATE_API_KEY` and `FABRICATE_API_URL` by default. +`wait_for_grading` polls every three seconds and times out after two minutes; +both values are configurable. + +`find_suite(project_id, name="Agent Eval Tasks")` selects the latest version. +Pin a CI run with `version=2`, or pass a suite **version** UUID with `id=...`. +Use that version UUID (`suite["id"]`) for `list_tasks` and `create_run`; do +not use the stable suite UUID in `suite["suite_id"]`. To snapshot a version's +metadata and tasks into the next version, call +`client.create_suite_version(suite["id"])`. + +### Attachments + +Upload a file before reporting a trial, then reference its upload ID: + +```python +upload_id = client.upload_attachment( + "Default", + filename="payments.csv", + content_type="text/csv", + data=csv_bytes, +) + +client.report_trial( + run["id"], + { + "task_key": "payments-csv", + "transcript": {"messages": transcript}, + "attachment_upload_ids": [upload_id], + "grade": True, + }, +) +``` + +### Fixture Versions, Grader Versions, and management APIs + +Tasks expose their resolved Fixture Version as `effective_fixture`. Database +Fixture Version entries can be materialized as SQLite bytes: + +```python +sqlite_bytes = client.download_fixture_database( + fixture_id=task["effective_fixture"]["id"], + database_id=database_entry["value"]["database_id"], +) +``` + +Fixtures and Graders each have independently numbered versions. The +`list_fixtures` and `list_graders` APIs return the latest version by default; +use `create_fixture_version(fixture_version_id)` or +`create_grader_version(grader_version_id)` to copy a mutable version into the +next version. Pin a run's selected inputs with structured override lists: + +```python +client.create_run( + project_id, + { + "suite_id": suite["id"], + "fixture_overrides": [ + { + "fixture_id": "fixture-uuid", + "fixture_version_id": "fixture-version-uuid", + } + ], + "grader_overrides": [ + { + "grader_id": "grader-uuid", + "grader_version_id": "grader-version-uuid", + } + ], + }, +) +``` + +The run snapshots its resolved Fixture Version manifests and Grader Version +rubrics at creation, so subsequent edits do not change history. +`list_grader_definitions` and `find_or_create_grader_definition` remain +deprecated aliases for endpoint compatibility. + ## Error Handling The client raises appropriate exceptions for various error conditions: diff --git a/examples/agent_evals.py b/examples/agent_evals.py new file mode 100644 index 0000000..db820c9 --- /dev/null +++ b/examples/agent_evals.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Run a Fabricate Agent Evals suite with a framework-agnostic agent.""" + +import os +from typing import List + +from dotenv import load_dotenv + +from tonic_fabricate import ( + AgentEvalsClient, + CreateRunInput, + OpenInferenceSpan, +) + +load_dotenv() + + +def run_agent(task_input: str) -> List[OpenInferenceSpan]: + """Replace this example span with spans captured from your agent.""" + return [ + { + "name": "example-agent", + "attributes": { + "openinference.span.kind": "LLM", + "llm.input_messages.0.message.role": "user", + "llm.input_messages.0.message.content": task_input, + "llm.output_messages.0.message.role": "assistant", + "llm.output_messages.0.message.content": "Replace this with your agent's response.", + "llm.token_count.prompt": 100, + "llm.token_count.completion": 20, + "llm.token_count.prompt_details.cache_read": 40, + "llm.token_count.prompt_details.cache_write": 30, + "llm.token_count.prompt_details.cache_write_5m": 10, + "llm.token_count.prompt_details.cache_write_1h": 20, + }, + } + ] + + +def required_environment(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} is required") + return value + + +def main() -> None: + project_id = required_environment("FABRICATE_PROJECT_ID") + suite_name = os.environ.get("FABRICATE_SUITE_NAME", "Agent Eval Tasks") + suite_id = os.environ.get("FABRICATE_SUITE_ID") + suite_version = os.environ.get("FABRICATE_SUITE_VERSION") + client = AgentEvalsClient() + + suite = client.find_suite( + project_id, + id=suite_id, + name=None if suite_id else suite_name, + version=int(suite_version) if suite_version else None, + ) + if suite is None: + version_hint = f" version {suite_version}" if suite_version else "" + raise RuntimeError(f"Suite {suite_name!r}{version_hint} was not found") + + tasks = client.list_tasks(suite["id"]) + run_input: CreateRunInput = { + "suite_id": suite["id"], + "model": os.environ.get("AGENT_MODEL", "example-agent"), + } + if os.environ.get("GIT_BRANCH"): + run_input["git_branch"] = os.environ["GIT_BRANCH"] + if os.environ.get("GIT_SHA"): + run_input["git_sha"] = os.environ["GIT_SHA"] + + run = client.create_run(project_id, run_input) + try: + for task in tasks: + reported = client.report_trial( + run["id"], + { + "task_key": task["key"], + "transcript": {"messages": run_agent(task["input"])}, + "grade": True, + }, + ) + graded = client.wait_for_grading(reported["id"], timeout_seconds=300) + result = "PASS" if graded["passed"] else "FAIL" + print(f"{task['key']}: {result}") + except Exception: + client.update_run(run["id"], {"status": "failed"}) + raise + else: + client.update_run(run["id"], {"status": "completed"}) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 53760f3..8c0356b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "tonic-fabricate" -version = "1.1.0" +version = "1.2.0" description = "The official Fabricate client for Python" authors = [{name = "Fabricate Tools"}] readme = "README.md" diff --git a/tests/test_agent_evals.py b/tests/test_agent_evals.py new file mode 100644 index 0000000..3c6fb80 --- /dev/null +++ b/tests/test_agent_evals.py @@ -0,0 +1,581 @@ +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, List, Optional + +import pytest + +from tonic_fabricate import AgentEvalsClient, AgentEvalsError + + +class ExpectedRequest: + def __init__( + self, + method: str, + path: str, + *, + response_json: Optional[Any] = None, + response_body: bytes = b"", + status: int = 200, + response_content_type: str = "application/json", + ): + self.method = method + self.path = path + self.response_json = response_json + self.response_body = response_body + self.status = status + self.response_content_type = response_content_type + + +class StubServer(ThreadingHTTPServer): + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), StubHandler) + self.expected: List[ExpectedRequest] = [] + self.requests: List[Dict[str, Any]] = [] + + @property + def url(self) -> str: + host, port = self.server_address + return f"http://{host}:{port}" + + def enqueue( + self, + method: str, + path: str, + *, + response_json: Optional[Any] = None, + response_body: bytes = b"", + status: int = 200, + response_content_type: str = "application/json", + ) -> None: + self.expected.append( + ExpectedRequest( + method, + path, + response_json=response_json, + response_body=response_body, + status=status, + response_content_type=response_content_type, + ) + ) + + +class StubHandler(BaseHTTPRequestHandler): + server: StubServer + + def do_GET(self) -> None: + self._handle() + + def do_POST(self) -> None: + self._handle() + + def do_PATCH(self) -> None: + self._handle() + + def do_DELETE(self) -> None: + self._handle() + + def do_PUT(self) -> None: + self._handle() + + def log_message(self, format: str, *args: Any) -> None: + return + + def _handle(self) -> None: + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) if length else b"" + self.server.requests.append( + { + "method": self.command, + "path": self.path, + "headers": dict(self.headers), + "body": body, + } + ) + + if not self.server.expected: + self.send_error(500, "Unexpected request") + return + + expected = self.server.expected.pop(0) + if self.command != expected.method or self.path != expected.path: + self.send_error( + 500, + f"Expected {expected.method} {expected.path}; " + f"got {self.command} {self.path}", + ) + return + + if expected.response_json is not None: + response_body = json.dumps(expected.response_json).encode("utf-8") + else: + response_body = expected.response_body + + self.send_response(expected.status) + self.send_header("Content-Type", expected.response_content_type) + self.send_header("Content-Length", str(len(response_body))) + self.end_headers() + self.wfile.write(response_body) + + +@pytest.fixture +def stub_server() -> Any: + server = StubServer() + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + assert not server.expected + finally: + server.shutdown() + thread.join() + server.server_close() + + +@pytest.fixture +def client(stub_server: StubServer) -> AgentEvalsClient: + return AgentEvalsClient( + api_key="test-key", api_url=f"{stub_server.url}/api/v1///" + ) + + +def request_json(request: Dict[str, Any]) -> Any: + return json.loads(request["body"].decode("utf-8")) + + +def test_report_only_payloads( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "POST", + "/api/v1/projects/project/runs", + response_json={"id": "run", "status": "in_progress"}, + status=201, + ) + stub_server.enqueue( + "POST", + "/api/v1/runs/run/trials", + response_json={"id": "trial", "status": "completed"}, + status=202, + ) + + client.create_run("project", {"suite_name": "Local suite", "model": "gpt-x"}) + client.report_trial( + "run", + { + "task": {"key": "users", "input": "Generate users", "tags": ["users"]}, + "grader_definitions": [ + { + "name": "quality", + "prompt": "Check the output.", + "tags": ["users"], + } + ], + "transcript": {"messages": []}, + "cache_read_tokens": 10, + "cache_write_5m_tokens": 20, + "cache_write_1h_tokens": 30, + "grade": True, + }, + ) + + assert request_json(stub_server.requests[0]) == { + "suite_name": "Local suite", + "model": "gpt-x", + } + assert request_json(stub_server.requests[1])["task"]["key"] == "users" + assert request_json(stub_server.requests[1])["grader_definitions"][0][ + "prompt" + ] == "Check the output." + assert request_json(stub_server.requests[1])["cache_read_tokens"] == 10 + assert request_json(stub_server.requests[1])["cache_write_5m_tokens"] == 20 + assert request_json(stub_server.requests[1])["cache_write_1h_tokens"] == 30 + + +def test_constructor_uses_environment_and_normalizes_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("FABRICATE_API_KEY", "env-key") + monkeypatch.setenv("FABRICATE_API_URL", "https://example.test/api/v1///") + + client = AgentEvalsClient() + + assert client.api_key == "env-key" + assert client.api_url == "https://example.test/api/v1" + + +def test_constructor_requires_api_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FABRICATE_API_KEY", raising=False) + + with pytest.raises(ValueError, match="api_key is required"): + AgentEvalsClient() + + +def test_project_suite_and_task_methods( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "GET", "/api/v1/workspaces/My%20Workspace/projects", response_json=[] + ) + stub_server.enqueue( + "POST", + "/api/v1/workspaces/My%20Workspace/projects", + response_json={"id": "project-1"}, + ) + stub_server.enqueue("GET", "/api/v1/projects/project-1/suites", response_json=[]) + stub_server.enqueue( + "POST", + "/api/v1/projects/project-1/suites", + response_json={"id": "suite-1"}, + ) + stub_server.enqueue("GET", "/api/v1/suites/suite-1/tasks", response_json=[]) + stub_server.enqueue( + "POST", + "/api/v1/suites/suite-1/tasks", + response_json={"id": "task-1"}, + ) + + assert client.list_projects("My Workspace") == [] + assert ( + client.find_or_create_project("My Workspace", {"name": "Agent project"})["id"] + == "project-1" + ) + assert client.list_suites("project-1") == [] + assert ( + client.find_or_create_suite("project-1", {"name": "Suite", "tags": ["smoke"]})[ + "id" + ] + == "suite-1" + ) + assert client.list_tasks("suite-1") == [] + assert ( + client.upsert_task("suite-1", {"key": "task", "input": "Do the task"})["id"] + == "task-1" + ) + + assert stub_server.requests[0]["headers"]["Authorization"] == "Bearer test-key" + assert request_json(stub_server.requests[1]) == {"name": "Agent project"} + assert request_json(stub_server.requests[3]) == { + "name": "Suite", + "tags": ["smoke"], + } + assert request_json(stub_server.requests[5]) == { + "key": "task", + "input": "Do the task", + } + + +def test_find_suite_matches_id_or_case_insensitive_name_and_version( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + suites = [ + {"id": "one", "suite_id": "p", "name": "Regression", "version": 1}, + {"id": "two", "suite_id": "p", "name": "Regression", "version": 2}, + ] + for _ in range(3): + stub_server.enqueue( + "GET", "/api/v1/projects/project/suites", response_json=suites + ) + + assert client.find_suite("project", id="two") == suites[1] + # Without a version selector, the highest-numbered matching version wins. + assert client.find_suite("project", name="regression") == suites[1] + assert ( + client.find_suite("project", name="REGRESSION", version=1) == suites[0] + ) + + +def test_create_suite_version_uses_the_source_version_endpoint( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "POST", + "/api/v1/suites/source-version/versions", + response_json={ + "id": "next-version", + "suite_id": "parent-suite", + "name": "Regression", + "version": 2, + }, + status=201, + ) + + suite = client.create_suite_version("source-version") + + assert suite["id"] == "next-version" + assert suite["version"] == 2 + assert stub_server.requests[0]["headers"]["Authorization"] == "Bearer test-key" + assert stub_server.requests[0]["body"] == b"" + + +def test_fixture_methods_and_database_download( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue("GET", "/api/v1/projects/project/fixtures", response_json=[]) + stub_server.enqueue( + "POST", + "/api/v1/projects/project/fixtures", + response_json={"id": "fixture"}, + ) + stub_server.enqueue( + "GET", "/api/v1/fixtures/fixture", response_json={"id": "fixture"} + ) + stub_server.enqueue( + "PATCH", + "/api/v1/fixtures/fixture", + response_json={"id": "fixture", "name": "Updated"}, + ) + stub_server.enqueue( + "GET", + "/api/v1/fixtures/fixture/databases/database", + response_body=b"SQLite format 3\x00", + response_content_type="application/vnd.sqlite3", + ) + stub_server.enqueue("DELETE", "/api/v1/fixtures/fixture", status=204) + + assert client.list_fixtures("project") == [] + assert ( + client.find_or_create_fixture("project", {"name": "Fixture", "entries": []})[ + "id" + ] + == "fixture" + ) + assert client.get_fixture("fixture")["id"] == "fixture" + assert client.update_fixture("fixture", {"name": "Updated"})["name"] == "Updated" + assert client.download_fixture_database("fixture", "database").startswith(b"SQLite") + assert client.delete_fixture("fixture") is None + assert stub_server.requests[4]["headers"]["Accept"] == "application/vnd.sqlite3" + + +def test_grader_run_and_trial_methods( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "GET", "/api/v1/projects/project/grader_definitions", response_json=[] + ) + stub_server.enqueue( + "POST", + "/api/v1/projects/project/grader_definitions", + response_json={"id": "grader"}, + ) + stub_server.enqueue( + "GET", + "/api/v1/projects/project/runs?branch=feature%2Fagent", + response_json=[], + ) + stub_server.enqueue( + "POST", + "/api/v1/projects/project/runs", + response_json={"id": "run"}, + ) + stub_server.enqueue( + "GET", + "/api/v1/runs/run", + response_json={"id": "run", "trials": []}, + ) + stub_server.enqueue("PATCH", "/api/v1/runs/run", response_json={"id": "run"}) + stub_server.enqueue( + "POST", + "/api/v1/runs/run/trials", + response_json={"id": "trial", "task_key": "task"}, + status=202, + ) + stub_server.enqueue( + "GET", + "/api/v1/trials/trial", + response_json={"id": "trial", "task_key": "task"}, + ) + + assert client.list_grader_definitions("project") == [] + assert ( + client.find_or_create_grader_definition( + "project", {"name": "Judge", "kind": "llm_judge"} + )["id"] + == "grader" + ) + assert client.list_runs("project", branch="feature/agent") == [] + assert client.create_run("project", {"suite_id": "suite"})["id"] == "run" + assert client.get_run("run")["trials"] == [] + assert client.update_run("run", {"status": "completed"})["id"] == "run" + assert ( + client.report_trial( + "run", + { + "task_key": "task", + "transcript": { + "messages": [ + { + "name": "agent", + "attributes": {"openinference.span.kind": "AGENT"}, + } + ] + }, + "grade": True, + }, + )["id"] + == "trial" + ) + assert client.get_trial("trial")["id"] == "trial" + + assert request_json(stub_server.requests[5]) == {"status": "completed"} + assert request_json(stub_server.requests[6])["grade"] is True + + +def test_list_runs_omits_empty_branch( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue("GET", "/api/v1/projects/project/runs", response_json=[]) + + assert client.list_runs("project", branch="") == [] + + +def test_http_errors_include_request_context( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "GET", + "/api/v1/trials/missing", + response_json={"error": "Not found"}, + status=404, + ) + + with pytest.raises(AgentEvalsError) as raised: + client.get_trial("missing") + + error = raised.value + assert error.status == 404 + assert error.method == "GET" + assert error.path == "/trials/missing" + assert '"error": "Not found"' in error.body + + +def test_wait_for_grading_polls_until_completed( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "GET", + "/api/v1/trials/trial", + response_json={"id": "trial", "grading": {"status": "pending"}}, + ) + stub_server.enqueue( + "GET", + "/api/v1/trials/trial", + response_json={"id": "trial", "grading": {"status": "in_progress"}}, + ) + stub_server.enqueue( + "GET", + "/api/v1/trials/trial", + response_json={"id": "trial", "grading": {"status": "completed"}}, + ) + + trial = client.wait_for_grading("trial", interval_seconds=0, timeout_seconds=1) + + assert trial["grading"]["status"] == "completed" + + +@pytest.mark.parametrize("grading", [None, {"status": "failed"}]) +def test_wait_for_grading_returns_terminal_or_ungraded_trial( + client: AgentEvalsClient, + stub_server: StubServer, + grading: Optional[Dict[str, str]], +) -> None: + response = {"id": "trial"} + if grading is not None: + response["grading"] = grading + stub_server.enqueue("GET", "/api/v1/trials/trial", response_json=response) + + assert ( + client.wait_for_grading("trial", interval_seconds=0, timeout_seconds=1)["id"] + == "trial" + ) + + +def test_wait_for_grading_times_out( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "GET", + "/api/v1/trials/trial", + response_json={"id": "trial", "grading": {"status": "in_progress"}}, + ) + + with pytest.raises(TimeoutError, match="Timed out after 0s"): + client.wait_for_grading("trial", interval_seconds=0, timeout_seconds=0) + + +def test_wait_for_grading_can_be_cancelled( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + stub_server.enqueue( + "GET", + "/api/v1/trials/trial", + response_json={"id": "trial", "grading": {"status": "in_progress"}}, + ) + cancel_event = threading.Event() + cancel_event.set() + + with pytest.raises(RuntimeError, match="Aborted"): + client.wait_for_grading( + "trial", + interval_seconds=0, + timeout_seconds=1, + cancel_event=cancel_event, + ) + + +def test_upload_attachment_mints_and_puts_bytes( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + upload_url = f"{stub_server.url}/api/v1/agent_evals/uploads/upload" + stub_server.enqueue( + "POST", + "/api/v1/workspaces/My%20Workspace/agent_evals/uploads", + response_json={ + "upload_id": "upload", + "upload_url": upload_url, + "method": "PUT", + "max_bytes": 100, + "expires_at": "later", + "instructions": "PUT bytes", + }, + status=201, + ) + stub_server.enqueue( + "PUT", + "/api/v1/agent_evals/uploads/upload", + response_json={"upload_id": "upload", "status": "uploaded"}, + ) + + upload_id = client.upload_attachment( + "My Workspace", + filename="result.csv", + content_type="text/csv", + data=b"a,b\n1,2\n", + ) + + assert upload_id == "upload" + assert request_json(stub_server.requests[0]) == { + "filename": "result.csv", + "content_type": "text/csv", + } + assert stub_server.requests[1]["body"] == b"a,b\n1,2\n" + assert stub_server.requests[1]["headers"]["Content-Type"] == "text/csv" + assert stub_server.requests[1]["headers"]["Authorization"] == "Bearer test-key" + + +def test_upload_errors_use_absolute_url( + client: AgentEvalsClient, stub_server: StubServer +) -> None: + upload_url = f"{stub_server.url}/upload" + stub_server.enqueue( + "PUT", + "/upload", + response_json={"error": "expired"}, + status=409, + ) + + with pytest.raises(AgentEvalsError) as raised: + client.put_upload_bytes(upload_url, b"data") + + assert raised.value.method == "PUT" + assert raised.value.path == upload_url diff --git a/tonic_fabricate/__init__.py b/tonic_fabricate/__init__.py index 624760e..c925c43 100644 --- a/tonic_fabricate/__init__.py +++ b/tonic_fabricate/__init__.py @@ -10,8 +10,49 @@ WorkflowTask, WorkflowResult, ) +from .agent_evals import ( + AgentEvalsAssertion, + AgentEvalsAttachment, + AgentEvalsClient, + AgentEvalsError, + AgentEvalsFixture, + AgentEvalsFixtureVersion, + AgentEvalsFixtureEntry, + AgentEvalsFixtureEntryInput, + AgentEvalsFixtureEntryType, + AgentEvalsFixtureVersionOverride, + AgentEvalsGrader, + AgentEvalsGraderDefinition, + AgentEvalsGraderKind, + AgentEvalsGraderVersion, + AgentEvalsGraderVersionOverride, + AgentEvalsGrading, + AgentEvalsGradingStatus, + AgentEvalsProject, + AgentEvalsClientRunStatus, + AgentEvalsRun, + AgentEvalsRunStatus, + AgentEvalsRunWithTrials, + AgentEvalsSuite, + AgentEvalsTask, + AgentEvalsTrial, + AgentEvalsTrialStatus, + AgentEvalsTrialSummary, + AgentEvalsUpload, + CreateUploadInput, + CreateRunInput, + FindOrCreateGraderDefinitionInput, + FindOrCreateProjectInput, + FindOrCreateSuiteInput, + FixtureInput, + OpenInferenceSpan, + ReportTrialInput, + UpdateFixtureInput, + UpdateRunInput, + UpsertTaskInput, +) -__version__ = "1.1.0" +__version__ = "1.2.0" __all__ = [ "generate", @@ -20,4 +61,43 @@ "WorkflowFile", "WorkflowTask", "WorkflowResult", + "AgentEvalsAssertion", + "AgentEvalsAttachment", + "AgentEvalsClient", + "AgentEvalsError", + "AgentEvalsFixture", + "AgentEvalsFixtureVersion", + "AgentEvalsFixtureEntry", + "AgentEvalsFixtureEntryInput", + "AgentEvalsFixtureEntryType", + "AgentEvalsFixtureVersionOverride", + "AgentEvalsGrader", + "AgentEvalsGraderDefinition", + "AgentEvalsGraderKind", + "AgentEvalsGraderVersion", + "AgentEvalsGraderVersionOverride", + "AgentEvalsGrading", + "AgentEvalsGradingStatus", + "AgentEvalsProject", + "AgentEvalsClientRunStatus", + "AgentEvalsRun", + "AgentEvalsRunStatus", + "AgentEvalsRunWithTrials", + "AgentEvalsSuite", + "AgentEvalsTask", + "AgentEvalsTrial", + "AgentEvalsTrialStatus", + "AgentEvalsTrialSummary", + "AgentEvalsUpload", + "CreateUploadInput", + "CreateRunInput", + "FindOrCreateGraderDefinitionInput", + "FindOrCreateProjectInput", + "FindOrCreateSuiteInput", + "FixtureInput", + "OpenInferenceSpan", + "ReportTrialInput", + "UpdateFixtureInput", + "UpdateRunInput", + "UpsertTaskInput", ] diff --git a/tonic_fabricate/agent_evals.py b/tonic_fabricate/agent_evals.py new file mode 100644 index 0000000..1d19863 --- /dev/null +++ b/tonic_fabricate/agent_evals.py @@ -0,0 +1,854 @@ +"""Framework-agnostic client for Fabricate's Agent Evals API.""" + +from __future__ import annotations + +import os +import time +from threading import Event +from typing import ( + Any, + Dict, + List, + Literal, + Mapping, + Optional, + Sequence, + TypedDict, + Union, + cast, +) +from urllib.parse import quote + +import requests + +DEFAULT_API_URL = "https://fabricate.tonic.ai/api/v1" +DEFAULT_GRADING_INTERVAL_SECONDS = 3.0 +DEFAULT_GRADING_TIMEOUT_SECONDS = 120.0 + +AgentEvalsRunStatus = Literal["in_progress", "completed", "failed", "timed_out"] +AgentEvalsClientRunStatus = Literal["in_progress", "completed", "failed"] +AgentEvalsTrialStatus = Literal["completed", "failed", "error"] +AgentEvalsGraderKind = Literal["llm_judge", "script"] +AgentEvalsFixtureEntryType = Literal["database", "table", "workflow", "mock_api"] + + +class OpenInferenceSpan(TypedDict): + """One flattened OpenInference span in a trial transcript.""" + + name: str + attributes: Dict[str, Any] + + +class AgentEvalsProject(TypedDict): + id: str + name: str + description: Optional[str] + workspace_id: str + + +class AgentEvalsFixtureEntryDiagnostic(TypedDict): + code: str + message: str + + +class AgentEvalsFixtureEntry(TypedDict): + id: str + key: str + type: AgentEvalsFixtureEntryType + value: Dict[str, Any] + position: int + resource: Optional[Dict[str, Any]] + diagnostic: Optional[AgentEvalsFixtureEntryDiagnostic] + + +class _AgentEvalsFixtureVersionRequired(TypedDict): + id: str + project_id: str + fixture_id: str + name: str + version: int + description: Optional[str] + entries: List[AgentEvalsFixtureEntry] + + +class AgentEvalsFixtureVersion(_AgentEvalsFixtureVersionRequired, total=False): + created_at: str + updated_at: str + + +AgentEvalsFixture = AgentEvalsFixtureVersion + + +class AgentEvalsSuite(TypedDict): + """A suite version; ``id`` is its version UUID and ``suite_id`` is its stable suite UUID.""" + + id: str + project_id: str + suite_id: str + name: str + version: int + description: Optional[str] + tags: List[str] + task_count: int + default_fixture_id: Optional[str] + default_fixture: Optional[AgentEvalsFixture] + + +class AgentEvalsTask(TypedDict): + """A task scoped to one suite version via ``suite_id`` (not the stable suite UUID).""" + + id: str + suite_id: str + key: str + input: str + expected_output: Optional[str] + tags: List[str] + input_token_limit: Optional[int] + output_token_limit: Optional[int] + fixture_id: Optional[str] + effective_fixture: Optional[AgentEvalsFixture] + + +class AgentEvalsGraderVersion(TypedDict): + id: str + project_id: str + grader_id: str + name: str + version: int + description: Optional[str] + kind: AgentEvalsGraderKind + prompt: Optional[str] + code: Optional[str] + model: Optional[str] + tags: List[str] + + +AgentEvalsGraderDefinition = AgentEvalsGraderVersion + + +class AgentEvalsRun(TypedDict): + """A run whose ``suite_id`` is the suite version UUID, not the stable suite UUID.""" + + id: str + project_id: str + run_number: int + suite_id: Optional[str] + suite_linked: bool + suite_name: Optional[str] + suite_version: Optional[int] + git_branch: Optional[str] + git_sha: Optional[str] + git_repo_url: Optional[str] + model: Optional[str] + name: Optional[str] + status: AgentEvalsRunStatus + started_at: Optional[str] + completed_at: Optional[str] + last_client_update_at: str + aggregate_metrics: Optional[Dict[str, Any]] + metadata: Optional[Dict[str, Any]] + fixture_overrides: Sequence["AgentEvalsFixtureVersionOverride"] + grader_overrides: Sequence["AgentEvalsGraderVersionOverride"] + created_at: str + + +class _AgentEvalsAssertionRequired(TypedDict): + assertion_name: str + passed: Optional[bool] + score: Optional[float] + reasoning: Optional[str] + + +class AgentEvalsAssertion(_AgentEvalsAssertionRequired, total=False): + id: str + + +class _AgentEvalsGraderRequired(TypedDict): + grader_name: str + passed: Optional[bool] + assertions: List[AgentEvalsAssertion] + + +class AgentEvalsGrader(_AgentEvalsGraderRequired, total=False): + id: str + + +AgentEvalsGradingStatus = Literal["pending", "in_progress", "completed", "failed"] + + +class AgentEvalsGrading(TypedDict): + id: str + status: AgentEvalsGradingStatus + graders_total: int + graders_completed: int + error: Optional[str] + started_at: Optional[str] + completed_at: Optional[str] + + +class AgentEvalsAttachment(TypedDict): + id: str + filename: str + content_type: Optional[str] + byte_size: int + + +class AgentEvalsTrialSummary(TypedDict): + id: str + run_id: str + task_id: Optional[str] + task_linked: bool + task_key: str + task_name: Optional[str] + trial_number: int + status: str + passed: Optional[bool] + cost: Optional[float] + latency_ms: Optional[int] + input_tokens: Optional[int] + output_tokens: Optional[int] + cache_read_tokens: Optional[int] + cached_tokens: Optional[int] + cache_write_5m_tokens: Optional[int] + cache_write_1h_tokens: Optional[int] + + +class AgentEvalsTranscript(TypedDict): + id: str + messages: List[OpenInferenceSpan] + + +class AgentEvalsTrial(AgentEvalsTrialSummary, total=False): + task_input: str + task_expected_output: Optional[str] + task_tags: List[str] + grader_definitions_snapshot: List[AgentEvalsGraderVersion] + transcript: Optional[AgentEvalsTranscript] + prompt_context: Any + fixture: Optional[AgentEvalsFixture] + attachments: List[AgentEvalsAttachment] + graders: List[AgentEvalsGrader] + grading: Optional[AgentEvalsGrading] + + +class AgentEvalsRunWithTrials(AgentEvalsRun): + trials: List[AgentEvalsTrialSummary] + + +class AgentEvalsFixtureEntryInput(TypedDict): + key: str + type: AgentEvalsFixtureEntryType + value: Dict[str, Any] + + +class _FindOrCreateProjectInputRequired(TypedDict): + name: str + + +class FindOrCreateProjectInput(_FindOrCreateProjectInputRequired, total=False): + description: str + + +class _FindOrCreateSuiteInputRequired(TypedDict): + name: str + + +class FindOrCreateSuiteInput(_FindOrCreateSuiteInputRequired, total=False): + description: str + tags: Sequence[str] + default_fixture_id: str + + +class _UpsertTaskInputRequired(TypedDict): + key: str + input: str + + +class UpsertTaskInput(_UpsertTaskInputRequired, total=False): + expected_output: str + tags: Sequence[str] + fixture_id: str + input_token_limit: int + output_token_limit: int + + +class _FixtureInputRequired(TypedDict): + name: str + + +class FixtureInput(_FixtureInputRequired, total=False): + description: str + entries: Sequence[AgentEvalsFixtureEntryInput] + + +class AgentEvalsFixtureVersionOverride(TypedDict): + """Select a Fixture Version for one Fixture when creating a run.""" + + fixture_id: str + fixture_version_id: str + + +class UpdateFixtureInput(TypedDict, total=False): + name: str + description: str + entries: Sequence[AgentEvalsFixtureEntryInput] + + +class _FindOrCreateGraderDefinitionInputRequired(TypedDict): + name: str + + +class FindOrCreateGraderDefinitionInput( + _FindOrCreateGraderDefinitionInputRequired, total=False +): + description: str + kind: AgentEvalsGraderKind + prompt: str + code: str + model: str + tags: Sequence[str] + + +class AgentEvalsGraderVersionOverride(TypedDict): + """Select a Grader Version for one Grader when creating a run.""" + + grader_id: str + grader_version_id: str + + +class CreateRunInput(TypedDict, total=False): + """Choose ``suite_id`` for server-driven mode or ``suite_name`` for report-only mode.""" + + suite_id: str + suite_name: str + git_branch: str + git_sha: str + git_repo_url: str + model: str + name: str + status: AgentEvalsClientRunStatus + metadata: Dict[str, Any] + fixture_overrides: Sequence[AgentEvalsFixtureVersionOverride] + grader_overrides: Sequence[AgentEvalsGraderVersionOverride] + + +class _ReportTrialAssertionInputRequired(TypedDict): + assertion_name: str + + +class ReportTrialAssertionInput(_ReportTrialAssertionInputRequired, total=False): + passed: bool + score: float + reasoning: str + + +class _ReportTrialGraderInputRequired(TypedDict): + grader_name: str + assertions: Sequence[ReportTrialAssertionInput] + + +class ReportTrialGraderInput(_ReportTrialGraderInputRequired): + pass + + +class _ReportOnlyTaskInputRequired(TypedDict): + key: str + input: str + + +class ReportOnlyTaskInput(_ReportOnlyTaskInputRequired, total=False): + name: str + expected_output: str + tags: Sequence[str] + + +class _ReportTrialInputRequired(TypedDict): + transcript: Mapping[str, Sequence[OpenInferenceSpan]] + + +class ReportTrialInput(_ReportTrialInputRequired, total=False): + task_key: str + task: ReportOnlyTaskInput + trial_number: int + status: AgentEvalsTrialStatus + cost: float + latency_ms: int + input_tokens: int + output_tokens: int + cache_read_tokens: int + cached_tokens: int + cache_write_5m_tokens: int + cache_write_1h_tokens: int + attachment_upload_ids: Sequence[str] + grader_definitions: Sequence[FindOrCreateGraderDefinitionInput] + graders: Sequence[ReportTrialGraderInput] + grade: bool + + +class AgentEvalsUpload(TypedDict): + upload_id: str + upload_url: str + method: str + max_bytes: int + expires_at: str + instructions: str + + +class UpdateRunInput(TypedDict, total=False): + status: AgentEvalsClientRunStatus + metadata: Dict[str, Any] + + +class _CreateUploadInputRequired(TypedDict): + filename: str + + +class CreateUploadInput(_CreateUploadInputRequired, total=False): + content_type: str + + +JsonObject = Mapping[str, Any] +BytesLike = Union[bytes, bytearray, memoryview] + + +class AgentEvalsError(Exception): + """A non-successful response from the Agent Evals API.""" + + def __init__(self, status: int, method: str, path: str, body: str): + self.status = status + self.method = method + self.path = path + self.body = body + super().__init__( + f"Fabricate agent evals request {method} {path} " + f"failed with {status}: {body}" + ) + + +def _encode(value: str) -> str: + return quote(value, safe="") + + +class AgentEvalsClient: + """Client for Fabricate's framework-agnostic Agent Evals API.""" + + def __init__( + self, + api_key: Optional[str] = None, + api_url: Optional[str] = None, + session: Optional[requests.Session] = None, + ): + resolved_api_key = api_key or os.environ.get("FABRICATE_API_KEY") + if not resolved_api_key: + raise ValueError( + "api_key is required (set it explicitly or via FABRICATE_API_KEY)" + ) + + self.api_key = resolved_api_key + self.api_url = ( + api_url or os.environ.get("FABRICATE_API_URL") or DEFAULT_API_URL + ).rstrip("/") + self.session = session or requests.Session() + self.session.headers.update( + { + "Authorization": f"Bearer {self.api_key}", + "Accept": "application/json", + } + ) + + def _request( + self, + method: str, + path: str, + *, + json: Optional[JsonObject] = None, + params: Optional[Mapping[str, Optional[str]]] = None, + ) -> Any: + search_params = { + key: value + for key, value in (params or {}).items() + if value is not None and value != "" + } + response = self.session.request( + method, + f"{self.api_url}{path}", + json=json, + params=search_params or None, + ) + if not response.ok: + raise AgentEvalsError( + response.status_code, method.upper(), path, response.text + ) + if not response.content: + return None + return response.json() + + # Projects + + def list_projects(self, workspace: str) -> List[AgentEvalsProject]: + return cast( + List[AgentEvalsProject], + self._request("GET", f"/workspaces/{_encode(workspace)}/projects"), + ) + + def find_or_create_project( + self, workspace: str, input: FindOrCreateProjectInput + ) -> AgentEvalsProject: + return cast( + AgentEvalsProject, + self._request( + "POST", + f"/workspaces/{_encode(workspace)}/projects", + json=input, + ), + ) + + # Suites + + def list_suites(self, project_id: str) -> List[AgentEvalsSuite]: + return cast( + List[AgentEvalsSuite], + self._request("GET", f"/projects/{_encode(project_id)}/suites"), + ) + + def find_or_create_suite( + self, project_id: str, input: FindOrCreateSuiteInput + ) -> AgentEvalsSuite: + return cast( + AgentEvalsSuite, + self._request( + "POST", + f"/projects/{_encode(project_id)}/suites", + json=input, + ), + ) + + def find_suite( + self, + project_id: str, + *, + id: Optional[str] = None, + name: Optional[str] = None, + version: Optional[int] = None, + ) -> Optional[AgentEvalsSuite]: + """Find a suite version without creating it. + + When ``version`` is omitted the highest-numbered version of the matching + suite is returned. + """ + suites = self.list_suites(project_id) + if id: + return next((suite for suite in suites if suite["id"] == id), None) + if name: + wanted_name = name.casefold() + named = [suite for suite in suites if suite["name"].casefold() == wanted_name] + if version is not None: + return next((suite for suite in named if suite.get("version") == version), None) + return max(named, key=lambda suite: suite.get("version", 0), default=None) + return None + + def create_suite_version(self, suite_id: str) -> AgentEvalsSuite: + """Create the next version of a suite by copying the given version.""" + return cast( + AgentEvalsSuite, + self._request("POST", f"/suites/{_encode(suite_id)}/versions"), + ) + + # Tasks + + def list_tasks(self, suite_id: str) -> List[AgentEvalsTask]: + return cast( + List[AgentEvalsTask], + self._request("GET", f"/suites/{_encode(suite_id)}/tasks"), + ) + + def upsert_task(self, suite_id: str, input: UpsertTaskInput) -> AgentEvalsTask: + return cast( + AgentEvalsTask, + self._request("POST", f"/suites/{_encode(suite_id)}/tasks", json=input), + ) + + # Fixtures + + def list_fixtures(self, project_id: str) -> List[AgentEvalsFixtureVersion]: + """List Fixture Versions; the API returns the latest version by default.""" + return cast( + List[AgentEvalsFixtureVersion], + self._request("GET", f"/projects/{_encode(project_id)}/fixtures"), + ) + + def find_or_create_fixture( + self, project_id: str, input: FixtureInput + ) -> AgentEvalsFixtureVersion: + """Find or create a Fixture, returning its latest Fixture Version.""" + return cast( + AgentEvalsFixtureVersion, + self._request( + "POST", + f"/projects/{_encode(project_id)}/fixtures", + json=input, + ), + ) + + def get_fixture_version(self, fixture_version_id: str) -> AgentEvalsFixtureVersion: + return cast( + AgentEvalsFixtureVersion, + self._request("GET", f"/fixtures/{_encode(fixture_version_id)}"), + ) + + def get_fixture(self, fixture_version_id: str) -> AgentEvalsFixtureVersion: + """Deprecated alias for :meth:`get_fixture_version`.""" + return self.get_fixture_version(fixture_version_id) + + def create_fixture_version( + self, fixture_version_id: str + ) -> AgentEvalsFixtureVersion: + """Create the next Fixture Version by copying the given version.""" + return cast( + AgentEvalsFixtureVersion, + self._request("POST", f"/fixtures/{_encode(fixture_version_id)}/versions"), + ) + + def update_fixture( + self, fixture_id: str, input: UpdateFixtureInput + ) -> AgentEvalsFixtureVersion: + return cast( + AgentEvalsFixtureVersion, + self._request("PATCH", f"/fixtures/{_encode(fixture_id)}", json=input), + ) + + def delete_fixture(self, fixture_id: str) -> None: + self._request("DELETE", f"/fixtures/{_encode(fixture_id)}") + + def download_fixture_database(self, fixture_id: str, database_id: str) -> bytes: + """Download SQLite bytes for a database referenced by a fixture.""" + path = f"/fixtures/{_encode(fixture_id)}/databases/" f"{_encode(database_id)}" + response = self.session.get( + f"{self.api_url}{path}", + headers={"Accept": "application/vnd.sqlite3"}, + ) + if not response.ok: + raise AgentEvalsError(response.status_code, "GET", path, response.text) + return cast(bytes, response.content) + + # Graders + + def list_graders( + self, project_id: str + ) -> List[AgentEvalsGraderVersion]: + return cast( + List[AgentEvalsGraderVersion], + self._request( + "GET", + f"/projects/{_encode(project_id)}/grader_definitions", + ), + ) + + def list_grader_definitions( + self, project_id: str + ) -> List[AgentEvalsGraderVersion]: + """Deprecated alias for :meth:`list_graders`.""" + return self.list_graders(project_id) + + def find_or_create_grader( + self, + project_id: str, + input: FindOrCreateGraderDefinitionInput, + ) -> AgentEvalsGraderVersion: + return cast( + AgentEvalsGraderVersion, + self._request( + "POST", + f"/projects/{_encode(project_id)}/grader_definitions", + json=input, + ), + ) + + def find_or_create_grader_definition( + self, + project_id: str, + input: FindOrCreateGraderDefinitionInput, + ) -> AgentEvalsGraderVersion: + """Deprecated alias for :meth:`find_or_create_grader`.""" + return self.find_or_create_grader(project_id, input) + + def create_grader_version( + self, grader_version_id: str + ) -> AgentEvalsGraderVersion: + """Create the next Grader Version by copying the given version.""" + return cast( + AgentEvalsGraderVersion, + self._request( + "POST", + f"/grader_definitions/{_encode(grader_version_id)}/versions", + ), + ) + + # Runs + + def list_runs( + self, project_id: str, *, branch: Optional[str] = None + ) -> List[AgentEvalsRun]: + return cast( + List[AgentEvalsRun], + self._request( + "GET", + f"/projects/{_encode(project_id)}/runs", + params={"branch": branch}, + ), + ) + + def create_run( + self, project_id: str, input: CreateRunInput + ) -> AgentEvalsRun: + return cast( + AgentEvalsRun, + self._request( + "POST", + f"/projects/{_encode(project_id)}/runs", + json=input, + ), + ) + + def get_run(self, run_id: str) -> AgentEvalsRunWithTrials: + return cast( + AgentEvalsRunWithTrials, + self._request("GET", f"/runs/{_encode(run_id)}"), + ) + + def update_run(self, run_id: str, input: UpdateRunInput) -> AgentEvalsRun: + return cast( + AgentEvalsRun, + self._request("PATCH", f"/runs/{_encode(run_id)}", json=input), + ) + + # Trials + + def report_trial(self, run_id: str, input: ReportTrialInput) -> AgentEvalsTrial: + return cast( + AgentEvalsTrial, + self._request("POST", f"/runs/{_encode(run_id)}/trials", json=input), + ) + + def get_trial(self, trial_id: str) -> AgentEvalsTrial: + return cast( + AgentEvalsTrial, + self._request("GET", f"/trials/{_encode(trial_id)}"), + ) + + def wait_for_grading( + self, + trial_id: str, + *, + interval_seconds: float = DEFAULT_GRADING_INTERVAL_SECONDS, + timeout_seconds: float = DEFAULT_GRADING_TIMEOUT_SECONDS, + cancel_event: Optional[Event] = None, + ) -> AgentEvalsTrial: + """Poll until asynchronous grading completes, fails, or times out.""" + deadline = time.monotonic() + timeout_seconds + trial = self.get_trial(trial_id) + grading = trial.get("grading") + + while grading and grading["status"] not in ("completed", "failed"): + if time.monotonic() >= deadline: + raise TimeoutError( + f"Timed out after {timeout_seconds:g}s waiting for " + f"trial {trial_id} grading." + ) + + if cancel_event: + if cancel_event.wait(interval_seconds): + raise RuntimeError("Aborted") + else: + time.sleep(interval_seconds) + + trial = self.get_trial(trial_id) + grading = trial.get("grading") + + return trial + + # Attachments + + def create_upload( + self, workspace: str, input: CreateUploadInput + ) -> AgentEvalsUpload: + return cast( + AgentEvalsUpload, + self._request( + "POST", + f"/workspaces/{_encode(workspace)}/agent_evals/uploads", + json=input, + ), + ) + + def put_upload_bytes( + self, + upload_url: str, + data: BytesLike, + content_type: Optional[str] = None, + ) -> None: + headers = {"Content-Type": content_type} if content_type else None + response = self.session.put(upload_url, data=bytes(data), headers=headers) + if not response.ok: + raise AgentEvalsError( + response.status_code, "PUT", upload_url, response.text + ) + + def upload_attachment( + self, + workspace: str, + *, + filename: str, + data: BytesLike, + content_type: Optional[str] = None, + ) -> str: + upload_input: CreateUploadInput = {"filename": filename} + if content_type: + upload_input["content_type"] = content_type + upload = self.create_upload(workspace, upload_input) + self.put_upload_bytes(upload["upload_url"], data, content_type) + return upload["upload_id"] + + +__all__ = [ + "AgentEvalsAssertion", + "AgentEvalsAttachment", + "AgentEvalsClient", + "AgentEvalsError", + "AgentEvalsFixture", + "AgentEvalsFixtureVersion", + "AgentEvalsFixtureEntry", + "AgentEvalsFixtureEntryInput", + "AgentEvalsFixtureEntryType", + "AgentEvalsFixtureVersionOverride", + "AgentEvalsGrader", + "AgentEvalsGraderDefinition", + "AgentEvalsGraderKind", + "AgentEvalsGraderVersion", + "AgentEvalsGraderVersionOverride", + "AgentEvalsGrading", + "AgentEvalsGradingStatus", + "AgentEvalsProject", + "AgentEvalsClientRunStatus", + "AgentEvalsRun", + "AgentEvalsRunStatus", + "AgentEvalsRunWithTrials", + "AgentEvalsSuite", + "AgentEvalsTask", + "AgentEvalsTrial", + "AgentEvalsTrialStatus", + "AgentEvalsTrialSummary", + "AgentEvalsUpload", + "CreateUploadInput", + "CreateRunInput", + "FindOrCreateProjectInput", + "FindOrCreateGraderDefinitionInput", + "FindOrCreateSuiteInput", + "FixtureInput", + "OpenInferenceSpan", + "ReportTrialInput", + "ReportOnlyTaskInput", + "UpdateFixtureInput", + "UpdateRunInput", + "UpsertTaskInput", +] From f87fe8dea0cf0f773b470f23fdab36a69f2c111c Mon Sep 17 00:00:00 2001 From: Yuliia Korabelska Date: Thu, 27 Aug 2026 15:20:42 -0400 Subject: [PATCH 6/9] Correct release metadata and docs before the 1.2.0 publish The project URLs pointed at fabricate-tools/client-python, which would have become the sidebar links on the PyPI page for this release. Point them at TonicAI/tonic-fabricate-python. publishing.md described the import name as fabricate_client rather than tonic_fabricate, so every verification command in it failed as written. It also claimed __init__.py derives its version from pyproject.toml, when __version__ is hardcoded and can silently drift, and it contained local developer paths in a public repo. The publish scripts printed the same broken import on success and cleaned an egg-info directory that never existed. Co-authored-by: Cursor --- publish-test.sh | 4 ++-- publish.sh | 6 ++--- publishing.md | 59 ++++++++++++++++++++++++++----------------------- pyproject.toml | 6 ++--- setup.py | 2 +- 5 files changed, 40 insertions(+), 37 deletions(-) diff --git a/publish-test.sh b/publish-test.sh index 39f8b0c..e9f5847 100755 --- a/publish-test.sh +++ b/publish-test.sh @@ -30,7 +30,7 @@ pip install --upgrade pip build twine # Clean previous builds echo "๐Ÿงน Cleaning previous builds..." -rm -rf dist/ build/ *.egg-info fabricate_client.egg-info/ +rm -rf dist/ build/ *.egg-info tonic_fabricate.egg-info/ # Build the package echo "๐Ÿ”จ Building package..." @@ -64,7 +64,7 @@ if [ $? -eq 0 ]; then echo " pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ tonic-fabricate" echo "" echo "2. Test the package:" - echo " python -c \"from fabricate_client import generate; print('Package works!')\"" + echo " python -c \"from tonic_fabricate import generate, run_workflow, AgentEvalsClient; print('Package works!')\"" echo "" echo "3. View on TestPyPI:" echo " https://test.pypi.org/project/tonic-fabricate/" diff --git a/publish.sh b/publish.sh index 00438e9..0fe9c14 100755 --- a/publish.sh +++ b/publish.sh @@ -36,7 +36,7 @@ if [ "$CURRENT_VERSION" = "0.0.0" ]; then read -p "Do you want to continue with version 0.0.0? (y/N): " -n 1 -r echo if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "โŒ Cancelled. Please update the version in pyproject.toml and fabricate_client/__init__.py" + echo "โŒ Cancelled. Please update the version in pyproject.toml and tonic_fabricate/__init__.py" exit 1 fi else @@ -82,7 +82,7 @@ pip install --upgrade pip build twine # Clean previous builds echo "๐Ÿงน Cleaning previous builds..." -rm -rf dist/ build/ *.egg-info fabricate_client.egg-info/ +rm -rf dist/ build/ *.egg-info tonic_fabricate.egg-info/ # Build the package echo "๐Ÿ”จ Building package..." @@ -147,7 +147,7 @@ if [ $? -eq 0 ]; then echo " pip install tonic-fabricate" echo "" echo "2. Test the package:" - echo " python -c \"from fabricate_client import generate; print('Package works!')\"" + echo " python -c \"from tonic_fabricate import generate, run_workflow, AgentEvalsClient; print('Package works!')\"" echo "" echo "3. View on PyPI:" echo " https://pypi.org/project/tonic-fabricate/" diff --git a/publishing.md b/publishing.md index 79678d7..a2480cd 100644 --- a/publishing.md +++ b/publishing.md @@ -6,12 +6,12 @@ Your package is **ready for publication**! The package structure includes: - Modern `pyproject.toml` configuration - Legacy `setup.py` for compatibility -- Proper package structure with `fabricate_client/` +- Proper package structure with `tonic_fabricate/` - README.md with usage documentation - MANIFEST.in for including additional files -- Version management (currently v1.0.0) +- Version management (currently v1.2.0) - Package name: `tonic-fabricate` -- Import name: `fabricate_client` +- Import name: `tonic_fabricate` ## ๐Ÿš€ Publishing Process @@ -45,32 +45,36 @@ You'll need accounts on: #### **3. Update Version Number (if needed)** -Simply update the version in **one place** - `pyproject.toml`: +The version lives in **two places** that must be kept in sync: ```toml # In pyproject.toml (line 7) version = "x.y.z" ``` +```python +# In tonic_fabricate/__init__.py +__version__ = "x.y.z" +``` + **Quick update command:** ```bash -# Example: Update from 1.0.0 to 1.0.1 -sed -i '' 's/version = "1.0.0"/version = "1.0.1"/' pyproject.toml +# Example: Update from 1.2.0 to 1.2.1 +sed -i '' 's/version = "1.2.0"/version = "1.2.1"/' pyproject.toml +sed -i '' 's/__version__ = "1.2.0"/__version__ = "1.2.1"/' tonic_fabricate/__init__.py # Verify the change -grep "version =" pyproject.toml +grep '^version = ' pyproject.toml +grep '^__version__ = ' tonic_fabricate/__init__.py ``` -โœ… **Single Source of Truth:** The other files (`setup.py` and `__init__.py`) automatically read the version from `pyproject.toml` at build time and runtime using the `tomllib`/`tomli` library. +โš ๏ธ **Two places, not one:** `setup.py` reads the version from `pyproject.toml` at build time via `tomllib`/`tomli`, so it needs no edit. But `tonic_fabricate/__init__.py` hardcodes `__version__` and will silently drift if you forget it. The published artifact takes its version from `pyproject.toml`; `__version__` is what users see at runtime. #### **4. Test Publishing** ```bash -# Navigate to the client directory -cd /Users/mark/Code/fabricate/clients/python - -# Run the test publishing script +# From the root of this repository ./publish-test.sh ``` @@ -86,7 +90,7 @@ You'll be prompted for: pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ tonic-fabricate # Test the package -python -c "from fabricate_client import generate; print('Package works!')" +python -c "from tonic_fabricate import generate, run_workflow, AgentEvalsClient; print('Package works!')" ``` #### **6. Production Publishing** @@ -117,7 +121,7 @@ You'll be prompted for: pip install tonic-fabricate # Test the package -python -c "from fabricate_client import generate; print('Package installed successfully!')" +python -c "from tonic_fabricate import generate, run_workflow, AgentEvalsClient; print('Package installed successfully!')" ``` ### **Option 2: Automated Publishing with GitHub Actions (Recommended)** @@ -149,8 +153,8 @@ First, you need to configure your PyPI API token as a GitHub secret: ```bash # Tag the current commit - git tag v1.0.2 - git push origin v1.0.2 + git tag vx.y.z + git push origin vx.y.z ``` 2. Go to GitHub โ†’ **Releases** โ†’ **Create a new release** @@ -205,11 +209,11 @@ This ensures that publishing requires manual approval even when automated. - **MINOR** (y): New features, backward compatible (e.g., 1.0.0 โ†’ 1.1.0) - **MAJOR** (x): Breaking changes (e.g., 1.0.0 โ†’ 2.0.0) -### **Single Source Management:** +### **Where the version lives:** -- โœ… **Only edit:** `pyproject.toml` -- โœ… **Automatic sync:** `setup.py` and `__init__.py` read from `pyproject.toml` -- โœ… **No version conflicts:** Impossible for versions to get out of sync +- โœ… **Edit both:** `pyproject.toml` (`version`) and `tonic_fabricate/__init__.py` (`__version__`) +- โœ… **Reads automatically:** `setup.py` pulls the version from `pyproject.toml` at build time +- โš ๏ธ **Can drift:** `__version__` is hardcoded โ€” nothing enforces that it matches `pyproject.toml` - โœ… **Modern approach:** Uses `tomllib` (Python 3.11+) or `tomli` (Python 3.8-3.10) ## ๐ŸŽฏ Quick Publishing Workflow @@ -217,9 +221,9 @@ This ensures that publishing requires manual approval even when automated. ### **GitHub Actions (Recommended)** ```bash -# 1. Update version in pyproject.toml (if needed) +# 1. Update version in pyproject.toml and tonic_fabricate/__init__.py (if needed) # 2. Commit and push changes -git add pyproject.toml +git add pyproject.toml tonic_fabricate/__init__.py git commit -m "Bump version to x.y.z" git push @@ -234,11 +238,10 @@ git push origin vx.y.z ### **Manual Publishing** ```bash -# 1. Navigate to directory -cd /Users/mark/Code/tonic-fabricate-python +# 1. Start from the root of this repository # 2. Update version (if needed) -# Edit only pyproject.toml - other files read it automatically +# Edit pyproject.toml and tonic_fabricate/__init__.py # 3. Test publish ./publish-test.sh @@ -253,8 +256,8 @@ pip install --index-url https://test.pypi.org/simple/ --extra-index-url https:// ## ๐Ÿšจ Important Notes 1. **Package Name**: Installable as `tonic-fabricate` -2. **Import Name**: Import as `fabricate_client` -3. **Version Update**: Only update version in `pyproject.toml` +2. **Import Name**: Import as `tonic_fabricate` +3. **Version Update**: Update the version in `pyproject.toml` **and** `tonic_fabricate/__init__.py` 4. **GitHub Actions**: Requires `PYPI_API_TOKEN` secret to be configured 5. **Always test first**: Use TestPyPI before production (manual) or test releases (GitHub Actions) 6. **No duplicates**: You cannot upload the same version twice @@ -278,4 +281,4 @@ pip install --index-url https://test.pypi.org/simple/ --extra-index-url https:// **Import errors after installation:** -- Verify package name (`tonic-fabricate`) vs import name (`fabricate_client`) +- Verify package name (`tonic-fabricate`) vs import name (`tonic_fabricate`) diff --git a/pyproject.toml b/pyproject.toml index 8c0356b..7f7f222 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,9 +37,9 @@ dev = [ ] [project.urls] -Homepage = "https://github.com/fabricate-tools/client-python" -Repository = "https://github.com/fabricate-tools/client-python" -Documentation = "https://github.com/fabricate-tools/client-python#readme" +Homepage = "https://github.com/TonicAI/tonic-fabricate-python" +Repository = "https://github.com/TonicAI/tonic-fabricate-python" +Documentation = "https://github.com/TonicAI/tonic-fabricate-python#readme" [tool.black] line-length = 88 diff --git a/setup.py b/setup.py index a2c0548..e0c03f8 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ description="The official Fabricate client for Python", long_description=long_description, long_description_content_type="text/markdown", - url="https://github.com/fabricate-tools/client-python", + url="https://github.com/TonicAI/tonic-fabricate-python", packages=find_packages(), classifiers=[ "Development Status :: 4 - Beta", From 87ca9dc0c85638f995ca874f7cf5fe0ba2d47e75 Mon Sep 17 00:00:00 2001 From: Yuliia Korabelska Date: Tue, 1 Sep 2026 14:20:00 -0400 Subject: [PATCH 7/9] Drop Python 3.8 and add the missing MIT LICENSE file The 3.8 CI leg could not install the package at all: setuptools caps at 75.x on 3.8, which predates PEP 639 and rejects the SPDX `license = "MIT"` string in pyproject.toml. 3.8 has also been end-of-life since Oct 2024, so raise the floor to 3.9 rather than downgrade the license declaration. The repo claimed MIT in its metadata without ever shipping the license text. Add LICENSE and reference it via `license-files` so it is bundled in the sdist and wheel. Co-authored-by: Cursor --- .github/workflows/test.yml | 1 - LICENSE | 20 ++++++++++++++++++++ publishing.md | 2 +- pyproject.toml | 10 +++++----- setup.py | 4 +--- 5 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 LICENSE diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d9b6022..1f962b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,6 @@ jobs: fail-fast: false matrix: python-version: - - "3.8" - "3.9" - "3.10" - "3.11" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..77d3c39 --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +MIT License + +Copyright (c) 2025 Tonic AI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/publishing.md b/publishing.md index a2480cd..3b1aa49 100644 --- a/publishing.md +++ b/publishing.md @@ -214,7 +214,7 @@ This ensures that publishing requires manual approval even when automated. - โœ… **Edit both:** `pyproject.toml` (`version`) and `tonic_fabricate/__init__.py` (`__version__`) - โœ… **Reads automatically:** `setup.py` pulls the version from `pyproject.toml` at build time - โš ๏ธ **Can drift:** `__version__` is hardcoded โ€” nothing enforces that it matches `pyproject.toml` -- โœ… **Modern approach:** Uses `tomllib` (Python 3.11+) or `tomli` (Python 3.8-3.10) +- โœ… **Modern approach:** Uses `tomllib` (Python 3.11+) or `tomli` (Python 3.9-3.10) ## ๐ŸŽฏ Quick Publishing Workflow diff --git a/pyproject.toml b/pyproject.toml index 7f7f222..80fedac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=45", "wheel"] +requires = ["setuptools>=77", "wheel"] build-backend = "setuptools.build_meta" [project] @@ -9,14 +9,14 @@ description = "The official Fabricate client for Python" authors = [{name = "Fabricate Tools"}] readme = "README.md" license = "MIT" -requires-python = ">=3.8" +license-files = ["LICENSE"] +requires-python = ">=3.9" classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -43,10 +43,10 @@ Documentation = "https://github.com/TonicAI/tonic-fabricate-python#readme" [tool.black] line-length = 88 -target-version = ['py38'] +target-version = ['py39'] [tool.mypy] -python_version = "3.8" +python_version = "3.9" warn_return_any = true warn_unused_configs = true disallow_untyped_defs = true \ No newline at end of file diff --git a/setup.py b/setup.py index e0c03f8..a39dabf 100644 --- a/setup.py +++ b/setup.py @@ -23,16 +23,14 @@ classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ], - python_requires=">=3.8", + python_requires=">=3.9", install_requires=[ "requests>=2.25.0", ], From 50d8114101e5faee2af0c96b5d5ff67c3545fdb6 Mon Sep 17 00:00:00 2001 From: Yuliia Korabelska Date: Tue, 1 Sep 2026 14:27:04 -0400 Subject: [PATCH 8/9] Match the LICENSE to Tonic's standard MIT text Use the same text as other Tonic repos: the "Tonic AI, Inc." entity name and the current year. The previous version also dropped "THE USE OR" from the final clause, narrowing the warranty disclaimer so it no longer explicitly covered claims arising from use of the software. Co-authored-by: Cursor --- LICENSE | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 77d3c39..f2fc0f2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2025 Tonic AI +Copyright (c) 2026 Tonic AI, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -17,4 +17,5 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR OTHER DEALINGS IN THE SOFTWARE. +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From f2d4e7ada2ddf54b5f672bc1c108c72fad2a7da4 Mon Sep 17 00:00:00 2001 From: Yuliia Korabelska Date: Tue, 1 Sep 2026 15:10:03 -0400 Subject: [PATCH 9/9] prevent api key from being printed out --- tonic_fabricate/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tonic_fabricate/client.py b/tonic_fabricate/client.py index 7e77575..bf98569 100644 --- a/tonic_fabricate/client.py +++ b/tonic_fabricate/client.py @@ -4,7 +4,7 @@ import shutil from pathlib import Path from typing import Optional, Callable, Dict, Any, Union, List -from dataclasses import dataclass +from dataclasses import dataclass, field from urllib.parse import quote import requests @@ -37,8 +37,8 @@ class WorkflowResult: """Result of running a workflow, with helpers to download generated files.""" result: Any task: WorkflowTask - _api_key: str - _api_url: str + _api_key: str = field(repr=False) + _api_url: str = field(repr=False) def download_file(self, file_id: int, dest_path: str) -> None: """