Agent bench

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 88

.github/workflows/pr-review-codex.yml

@@ -0,0 +1,67 @@
+name: Perform a code review when a pull request is created.
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review, labeled]
+
+jobs:
+  codex:
+    runs-on: ubuntu-24.04
+
+    permissions:
+      contents: read
+      pull-requests: write
+      issues: write
+
+    outputs:
+      final_message: ${{ steps.run_codex.outputs.final-message }}
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          ref: refs/pull/${{ github.event.pull_request.number }}/merge
+
+      - name: Pre-fetch base and head refs for the PR
+        run: |
+          git fetch --no-tags origin \
+            ${{ github.event.pull_request.base.ref }} \
+            +refs/pull/${{ github.event.pull_request.number }}/head
+
+      - name: Run Codex
+        id: run_codex
+        uses: openai/codex-action@v1
+        with:
+          openai-api-key: ${{ secrets.OPENAI_API_KEY }}
+          prompt: |
+            This is PR #${{ github.event.pull_request.number }} for ${{ github.repository }}.
+            Base SHA: ${{ github.event.pull_request.base.sha }}
+            Head SHA: ${{ github.event.pull_request.head.sha }}
+
+            Review ONLY the changes introduced by the PR.
+            Suggest any improvements, potential bugs, security issues, or issues.
+            Be concise and specific in your feedback.
+
+            Pull request title and body:
+            ----
+            ${{ github.event.pull_request.title }}
+            ${{ github.event.pull_request.body }}
+
+  post_feedback:
+    runs-on: ubuntu-latest
+    needs: codex
+    if: needs.codex.outputs.final_message != ''
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - name: Report Codex feedback
+        uses: actions/github-script@v7
+        env:
+          CODEX_FINAL_MESSAGE: ${{ needs.codex.outputs.final_message }}
+        with:
+          github-token: ${{ github.token }}
+          script: |
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.payload.pull_request.number,
+              body: process.env.CODEX_FINAL_MESSAGE,
+            });

.github/workflows/test.yml

@@ -0,0 +1,32 @@
+name: Test
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package and test dependencies
+        run: python -m pip install --upgrade pip && python -m pip install -e ".[dev]"
+
+      - name: Run tests
+        run: python -m pytest

.gitignore

@@ -106,4 +106,7 @@ dmypy.json
 # Example files temp directories
 tmp/ 
 
-.DS_Store
+.DS_Store
+
+# Claude
+.claude/

DEVELOPMENT.md

@@ -0,0 +1,160 @@
+# Development Guide
+
+This guide covers how to set up and test the package locally before publishing.
+
+## Setup
+
+### Create a Virtual Environment
+
+```bash
+# Create a virtual environment
+python -m venv venv
+
+# Activate the virtual environment
+source venv/bin/activate  # On macOS/Linux
+# venv\Scripts\activate   # On Windows
+```
+
+### Install in Development Mode
+
+With the virtual environment activated, install the package with dev dependencies:
+
+```bash
+# Install in editable/development mode with dev dependencies
+pip install -e ".[dev]"
+```
+
+This installs the package along with `python-dotenv` (for `.env` file support), `pytest`, `mypy`, `black`, and `flake8`.
+
+### Environment Variables
+
+Set the following environment variables for testing:
+
+```bash
+export FABRICATE_API_KEY="your-api-key"
+export FABRICATE_API_URL="https://fabricate.tonic.ai/api/v1"  # or your local instance
+```
+
+Or create a `.env` file (requires `python-dotenv`):
+
+```
+FABRICATE_API_KEY=your-api-key
+FABRICATE_API_URL=https://fabricate.tonic.ai/api/v1
+```
+
+## Testing
+
+### Quick Import Test
+
+Verify the module imports correctly:
+
+```bash
+python -c "from tonic_fabricate import generate, run_workflow, AgentEvalsClient; print('All imports work!')"
+```
+
+### Run Unit Tests
+
+The Agent Evals client tests use a local HTTP server and do not require
+Fabricate credentials:
+
+```bash
+python -m pytest
+```
+
+### Run the Examples
+
+The examples test the actual API calls:
+
+```bash
+# Test the generate function
+python examples/download.py
+
+# Test the workflow function
+python examples/workflow.py
+
+# Test Agent Evals against a configured Fabricate project and suite
+python examples/agent_evals.py
+```
+
+### Interactive Testing
+
+```python
+from tonic_fabricate import run_workflow
+import os
+
+result = run_workflow(
+    database='your_database',
+    workspace='your_workspace',
+    workflow='your_workflow',
+    api_url=os.environ.get('FABRICATE_API_URL'),
+    on_progress=lambda p: print(p)
+)
+print(result.result)
+```
+
+## Code Quality
+
+### Install Dev Dependencies
+
+```bash
+pip install -r requirements-dev.txt
+```
+
+### Type Checking
+
+```bash
+mypy tonic_fabricate/
+```
+
+### Linting
+
+```bash
+flake8 tonic_fabricate/
+```
+
+### Formatting
+
+```bash
+# Check formatting
+black --check tonic_fabricate/
+
+# Auto-format
+black tonic_fabricate/
+```
+
+## Pre-Publish Testing
+
+Before publishing to production PyPI, test with TestPyPI:
+
+```bash
+# Publish to TestPyPI
+./publish-test.sh
+
+# Install from TestPyPI to verify
+pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ tonic-fabricate==1.2.0
+
+# Test the installed package
+python -c "from tonic_fabricate import generate, run_workflow, AgentEvalsClient; print('Package works!')"
+```
+
+## Publishing
+
+See [publishing.md](publishing.md) for detailed publishing instructions.
+
+### Quick Reference
+
+```bash
+# Test publish
+./publish-test.sh
+
+# Production publish (after testing)
+./publish.sh
+```
+
+## Deactivating the Virtual Environment
+
+When you're done developing:
+
+```bash
+deactivate
+```

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Tonic AI, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.