feat(project): add project add evaluator code-based

src/assets/evaluators/autoevals-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[autoevals](https://github.com/braintrustdata/autoevals) — scores each session
+with autoevals' `{{ EvaluatorClass }}` scorer.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in an `AutoEvalsAdapter`
+  behind the standard `@custom_code_based_evaluator()` handler. With a Bedrock
+  judge model set, autoevals grades via a LiteLLM client → Bedrock.
+- `pyproject.toml` — autoevals + judge dependencies, managed with
+  [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+  judge model.
+
+## Customize
+
+- Change the scorer or its arguments in `lambda_function.py`.
+- Scorers like `Factuality` / `ClosedQA` / `SQL` need an expected/reference
+  output — provide it when you invoke the evaluator.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/autoevals-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["bedrock:InvokeModel"],
+      "Resource": "*"
+    }
+  ]
+}

src/assets/evaluators/autoevals-lambda/lambda_function.py

@@ -0,0 +1,37 @@
+{{#if ModelProviderBedrock}}
+import os
+
+# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
+os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
+
+from autoevals import {{ EvaluatorClass }}, init
+from autoevals.litellm import LiteLLMClient
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+client = LiteLLMClient()
+init(client=client, default_model="bedrock/{{ Model }}")
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{else}}
+from autoevals import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+    return adapter(evaluator_input, context)

src/assets/evaluators/autoevals-lambda/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (Autoevals)"
+requires-python = ">=3.10"
+dependencies = [
+    # 1.20.0 is the first release shipping third_party.autoevals.AutoEvalsAdapter;
+    # the extra owns the autoevals version, so pinning it here only conflicts.
+    "bedrock-agentcore[autoevals]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+    # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
+    "litellm>=1.60,<1.85",
+{{else}}
+    "openai>=1.0.0,<2.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/deepeval-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[DeepEval](https://docs.confident-ai.com/) — scores each session with DeepEval's
+`{{ EvaluatorClass }}` metric, judged by Amazon Bedrock.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in a `DeepEvalAdapter`
+  behind the standard `@custom_code_based_evaluator()` handler.
+- `pyproject.toml` — DeepEval + Bedrock dependencies, managed with
+  [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+  judge model; add more if your metric needs it.
+
+## Customize
+
+- Swap the metric or tune its threshold in `lambda_function.py`.
+- Some DeepEval metrics need retrieval context or a reference/expected output —
+  supply those when you invoke the evaluator, or the metric returns
+  `MISSING_REQUIRED_FIELD`.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

src/assets/evaluators/deepeval-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["bedrock:InvokeModel"],
+      "Resource": "*"
+    }
+  ]
+}

src/assets/evaluators/deepeval-lambda/lambda_function.py

@@ -0,0 +1,29 @@
+import os
+
+os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
+os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
+os.chdir("/tmp")
+
+{{#if ModelProviderBedrock}}
+from deepeval.models import AmazonBedrockModel
+{{/if}}
+from deepeval.metrics import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+{{#if ModelProviderBedrock}}
+model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
+{{else}}
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+    return adapter(evaluator_input, context)

src/assets/evaluators/deepeval-lambda/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (DeepEval)"
+requires-python = ">=3.10"
+dependencies = [
+    # 1.20.0 is the first release shipping third_party.deepeval.DeepEvalAdapter;
+    # the extra owns the deepeval version, so pinning it here only conflicts.
+    "bedrock-agentcore[deepeval]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+    # deepeval's AmazonBedrockModel imports aiobotocore at runtime; only 3.x
+    # allows the botocore that bedrock-agentcore requires.
+    "aiobotocore>=3.0.0,<4.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/python-lambda/README.md

@@ -0,0 +1,28 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** — a Lambda that scores an agent session
+with your own logic.
+
+## What's here
+
+- `lambda_function.py` — the evaluator. The `@custom_code_based_evaluator()`
+  handler receives an `EvaluatorInput` (the session / trace / tool-call to
+  grade) and returns an `EvaluatorOutput` (`value` + `label`, or an error). It
+  ships as a stub that returns `Pass` for everything — replace the `TODO` with
+  your scoring logic.
+- `pyproject.toml` — Python dependencies, managed with
+  [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — extra IAM the evaluator Lambda gets at runtime.
+  Add statements here for anything your logic calls (DynamoDB, S3, …).
+
+## Write your evaluator
+
+```python
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+    # inspect input.session_spans / input.target_trace_id / input.target_span_id
+    return EvaluatorOutput(value=1.0, label="Pass", explanation="…")
+```
+
+Then `agentcore project deploy` packages this directory into the evaluator
+Lambda and registers the evaluator.

src/assets/evaluators/python-lambda/execution-role-policy.json

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+    }
+  ]
+}

src/assets/evaluators/python-lambda/lambda_function.py

@@ -0,0 +1,19 @@
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    custom_code_based_evaluator,
+    EvaluatorInput,
+    EvaluatorOutput,
+)
+
+
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+    """Evaluate agent behavior with custom logic.
+
+    Args:
+        input: Contains evaluation_level, session_spans, target_trace_id, target_span_id
+
+    Returns:
+        EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure.
+    """
+    # TODO: Replace with your evaluation logic
+    return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed")

src/assets/evaluators/python-lambda/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator"
+requires-python = ">=3.10"
+dependencies = [
+    "bedrock-agentcore>=1.6.0,<2.0.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/core/project/fsUtils.ts

@@ -20,3 +20,10 @@ export function enclosingProjectRoot(directory: string): string | undefined {
     }
   }
 }
+
+export function toPythonPackageName(name: string): string {
+  return name
+    .replace(/[^a-zA-Z0-9._-]/g, "-")
+    .replace(/^[^a-zA-Z0-9]+/, "")
+    .replace(/[^a-zA-Z0-9]+$/, "");
+}

src/core/project/manager.tsx

@@ -42,6 +42,7 @@ import {
 } from "./templates/export";
 import { HarnessSpecSchema } from "../../projectSchemas/harness";
 import { FsTreeNode } from "./templates/fsTree";
+import { getEvaluatorTemplateResolver } from "./templates/evaluator";
 import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project";
 import { ConfigBundleSchema } from "../../projectSchemas/config-bundle";
 import {
@@ -336,7 +337,23 @@ export class FsProjectManager implements ProjectManager {
         break;
       }
       case "evaluator": {
-        projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+        if (input.scaffold) {
+          yield { message: "Scaffolding evaluator in project" };
+          const outputPath = join(project.rootPath, "app", input.scaffold.name);
+          if (existsSync(outputPath))
+            throw new InputValidationError(
+              `cannot scaffold evaluator '${input.scaffold.name}': 'app/${input.scaffold.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`,
+            );
+          scaffoldedPaths.push(outputPath);
+          const result = await getEvaluatorTemplateResolver({
+            assetSource: this.assetSource,
+            templateRenderer: this.templateRenderer,
+          }).resolve(input.scaffold);
+          await result.tree.write(dirname(outputPath));
+          projectSpec.evaluators.push(...(result.spec.evaluators ?? []));
+        } else {
+          projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+        }
         break;
       }
       case "gateway":

src/core/project/templates/evaluator.ts

@@ -0,0 +1,81 @@
+import { FsTreeNode } from "./fsTree";
+import type { AssetSource } from "../source";
+import type { Evaluator } from "../../../projectSchemas/evaluator";
+import type { TemplateRenderer, TemplateResolver } from "./types";
+import { toPythonPackageName } from "../fsUtils";
+import type {
+  EvaluatorLibrary,
+  ManagedEvaluatorScaffoldInput,
+} from "../../../handlers/project/types";
+
+const DEFAULT_TIMEOUT = 60;
+
+const EVALUATOR_ASSETS: Record<
+  EvaluatorLibrary,
+  { assetDir: string; defaultTimeoutSeconds: number }
+> = {
+  deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 },
+  autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT },
+};
+
+const EMPTY_ASSET_DIR = "evaluators/python-lambda";
+
+function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator {
+  const timeoutSeconds =
+    input.timeoutSeconds ??
+    (input.metric ? EVALUATOR_ASSETS[input.metric.library].defaultTimeoutSeconds : DEFAULT_TIMEOUT);
+  return {
+    name: input.name,
+    level: input.level,
+    ...(input.description && { description: input.description }),
+    config: {
+      codeBased: {
+        managed: {
+          codeLocation: `app/${input.name}`,
+          entrypoint: "lambda_function.handler",
+          timeoutSeconds,
+          additionalPolicies: ["execution-role-policy.json"],
+        },
+      },
+    },
+    ...(input.kmsKeyArn && { kmsKeyArn: input.kmsKeyArn }),
+    ...(input.tags && { tags: input.tags }),
+  };
+}
+
+function buildRenderContext(input: ManagedEvaluatorScaffoldInput): Record<string, unknown> {
+  const context: Record<string, unknown> = { Name: toPythonPackageName(input.name) };
+  if (input.metric) {
+    context["EvaluatorClass"] = input.metric.metricClass;
+    context["Model"] = input.model ?? "";
+    context["ModelProviderBedrock"] = input.model !== undefined;
+    context["EvaluatorParams"] = "";
+  }
+  return context;
+}
+
+type GetEvaluatorTemplateResolverConfig = {
+  assetSource: AssetSource;
+  templateRenderer: TemplateRenderer;
+};
+
+export function getEvaluatorTemplateResolver(
+  config: GetEvaluatorTemplateResolverConfig,
+): TemplateResolver<ManagedEvaluatorScaffoldInput> {
+  return {
+    async resolve(input) {
+      const assetDir = input.metric
+        ? EVALUATOR_ASSETS[input.metric.library].assetDir
+        : EMPTY_ASSET_DIR;
+      const tree = await FsTreeNode.fromAssetSource(
+        { assetSource: config.assetSource },
+        { assetDir },
+        {
+          rootDirName: input.name,
+          transformContent: (raw) => config.templateRenderer.render(raw, buildRenderContext(input)),
+        },
+      );
+      return { tree, spec: { evaluators: [buildManagedEvaluatorSpec(input)] } };
+    },
+  };
+}