Stop overriding Vally telemetry settings

.github/workflows/ci.yml

@@ -41,6 +41,9 @@ jobs:
     name: Action smoke test
     runs-on: ubuntu-latest
     timeout-minutes: 5
+    env:
+      VALLY_TELEMETRY_OPTOUT: '1'
+      DO_NOT_TRACK: '1'
     steps:
       - uses: actions/checkout@v7
         with:

README.md

@@ -349,6 +349,28 @@ service tokens and process-injection variables are not forwarded. An explicit
 `copilot-token` or `credential-env` replaces automatic model authentication in
 all; no stored personal credential fallback is used.
 
+Vally collects pseudonymous usage telemetry by default; the Action does not
+force telemetry on or off. See [Vally's telemetry reference](https://microsoft.github.io/vally/reference/telemetry/)
+for what it collects and how to opt out.
+
+To opt out, set `VALLY_TELEMETRY_OPTOUT: '1'` (or `DO_NOT_TRACK: '1'`) in your
+workflow's `env`:
+
+```yaml
+- uses: github/evals-action@v0
+  env:
+    VALLY_TELEMETRY_OPTOUT: '1'
+  with:
+    working-directory: evals
+    suite: smoke
+```
+
+Caller-provided `VALLY_TELEMETRY_OPTOUT`, `DO_NOT_TRACK`, and `OTEL_SDK_DISABLED`
+environment values are preserved unchanged during evals, lint and version
+checks, including explicit opt-outs, without listing them in `credential-env`.
+Other environment filtering remains unchanged;
+telemetry endpoints, headers and arbitrary ambient variables are not forwarded.
+
 See [Copilot in Actions](https://docs.github.com/en/copilot/how-tos/copilot-cli/use-copilot-cli-in-actions)
 and [SDK server authentication](https://docs.github.com/en/copilot/how-tos/copilot-sdk/auth/server-to-server-tokens).
 Live entitlement, organization policy, subject/judge compatibility and billing

src/native.ts

@@ -48,9 +48,6 @@ export function nativeEnvironment(
 ): NodeJS.ProcessEnv {
   const env = inertEnvironment(source);
   env.HOME = home;
-  env.VALLY_TELEMETRY_OPTOUT = '1';
-  env.DO_NOT_TRACK = '1';
-  env.OTEL_SDK_DISABLED = 'true';
   if (inputs.command !== 'run') return env;
   const forbidden = /^(GITHUB_|GH_|ACTIONS_|RUNNER_|INPUT_|NODE_|NPM_|PNPM_|GIT_|LD_|DYLD_)|^(PATH|HOME|ENV|BASH_ENV|SHELLOPTS|BASHOPTS|COMSPEC|SYSTEMROOT|WINDIR|TMP|TEMP|TMPDIR|PYTHONPATH|PYTHONHOME|RUBYOPT|PERL5OPT)$/;
   const reportingSecrets = [reportToken, source.GH_TOKEN, source.ACTIONS_RUNTIME_TOKEN].filter(Boolean);
@@ -253,7 +250,7 @@ export async function runNative(
   const runId = randomUUID();
   const commands = nativeArguments(inputs, cwd, directory, runId);
   if (sha256(await readFile(engine.entry)) !== engine.sha256) throw new NativeError('engine-executable-changed', 'dependency');
-  const version = await invoke(engine, ['--version'], cwd, { ...inertEnvironment(env), HOME: env.HOME, VALLY_TELEMETRY_OPTOUT: '1', DO_NOT_TRACK: '1' }, directory);
+  const version = await invoke(engine, ['--version'], cwd, { ...inertEnvironment(env), HOME: env.HOME }, directory);
   if (version.cancelled) return { result: { outcome: 'cancelled', completeness: 'partial', category: 'execution', code: 'native-cancelled' }, files: [version.stdout, version.stderr] };
   if (version.code !== 0 || (await readFile(version.stdout, 'utf8')).trim() !== engine.version) throw new NativeError('native-version-mismatch', 'dependency');
   const files: string[] = [];

src/policy.ts

@@ -53,7 +53,10 @@ export async function containedFile(directory: string, input: string, maximumByt
 // This is process hygiene, not isolation from other processes under the same runner user.
 export function inertEnvironment(source: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
   const result: NodeJS.ProcessEnv = {};
-  for (const key of ['PATH', 'SystemRoot', 'WINDIR', 'TMP', 'TEMP', 'TMPDIR', 'LANG', 'LC_ALL']) {
+  for (const key of [
+    'PATH', 'SystemRoot', 'WINDIR', 'TMP', 'TEMP', 'TMPDIR', 'LANG', 'LC_ALL',
+    'VALLY_TELEMETRY_OPTOUT', 'DO_NOT_TRACK', 'OTEL_SDK_DISABLED',
+  ]) {
     if (source[key] !== undefined) result[key] = source[key];
   }
   result.CI = 'true';

tests/action.test.ts

@@ -545,6 +545,22 @@ it('defaults to lint then run and posts once using the automatic shared job toke
   expect(JSON.stringify(mocks.setOutput.mock.calls)).not.toContain('automatic-token');
 });
 
+it.each([undefined, '1', '0', ''])('preserves caller telemetry settings in both default-all phases: %s', async (value) => {
+  delete mocks.inputs.command;
+  const names = ['VALLY_TELEMETRY_OPTOUT', 'DO_NOT_TRACK', 'OTEL_SDK_DISABLED'];
+  for (const name of names) vi.stubEnv(name, value);
+  vi.stubEnv('OTEL_EXPORTER_OTLP_HEADERS', 'private-header');
+  await runAction();
+  expect(mocks.nativeCommands.mock.calls.map(([input]) => input.command)).toEqual(['lint', 'run']);
+  for (const [env] of mocks.nativeEnv.mock.calls) {
+    for (const name of names) {
+      if (value === undefined) expect(env).not.toHaveProperty(name);
+      else expect(env).toHaveProperty(name, value);
+    }
+    expect(env).not.toHaveProperty('OTEL_EXPORTER_OTLP_HEADERS');
+  }
+});
+
 it('supports BYOK in all without also injecting the automatic Copilot token', async () => {
   mocks.inputs.command = 'all';
   mocks.inputs['github-token'] = 'automatic-report-token';

tests/example.test.ts

@@ -9,6 +9,37 @@ import { assertSerializedReporter } from '../src/reporter.js';
 
 const execute = promisify(execFile);
 
+describe('telemetry workflow contracts', () => {
+  it('opts out packaged acceptance fixtures without changing other workflow defaults', async () => {
+    const workflow = parse(await readFile('.github/workflows/ci.yml', 'utf8')) as {
+      env?: Record<string, string>;
+      jobs: Record<string, {
+        env?: Record<string, string>;
+        steps: Array<{ uses?: string; env?: Record<string, string>; with?: Record<string, string> }>;
+      }>;
+    };
+    expect(workflow.env).toBeUndefined();
+    expect(workflow.jobs.check?.env).toBeUndefined();
+    const packaged = workflow.jobs.packaged!;
+    expect(packaged.env).toEqual({ VALLY_TELEMETRY_OPTOUT: '1', DO_NOT_TRACK: '1' });
+    const acceptanceSteps = packaged.steps.filter((step) => step.uses === './');
+    expect(acceptanceSteps.map((step) => step.with?.command)).toEqual(['run', 'run', 'lint']);
+    for (const step of acceptanceSteps) expect(step.env).toBeUndefined();
+  });
+
+  it('documents the native telemetry default and a concrete caller opt-out', async () => {
+    const readme = await readFile('README.md', 'utf8');
+    expect(readme).toContain('Vally collects pseudonymous usage telemetry by default');
+    expect(readme).toContain('https://microsoft.github.io/vally/reference/telemetry/');
+    const source = [...readme.matchAll(/```yaml\n([\s\S]*?)```/g)].map((match) => match[1]!)
+      .find((yaml) => yaml.includes('VALLY_TELEMETRY_OPTOUT:'))!;
+    const steps = parse(source) as Array<{ uses: string; env: Record<string, string>; with: Record<string, string> }>;
+    expect(steps).toHaveLength(1);
+    expect(steps[0]).toMatchObject({ uses: 'github/evals-action@v0', env: { VALLY_TELEMETRY_OPTOUT: '1' } });
+    expect(steps[0]!.with).not.toHaveProperty('credential-env');
+  });
+});
+
 describe('real Copilot example contract', () => {
   it('documents separate jobs with one automatic reporter and no artifact wiring', async () => {
     const readme = await readFile('README.md', 'utf8');

tests/native.test.ts

@@ -17,6 +17,13 @@ async function temporary() {
 const inputs = (overrides: Record<string, string> = {}) => parseInputs((key) => ({
   command: 'run', 'working-directory': '.', 'eval-paths': 'pass.eval.yaml', ...overrides,
 })[key] ?? '');
+const telemetryControls = ['VALLY_TELEMETRY_OPTOUT', 'DO_NOT_TRACK', 'OTEL_SDK_DISABLED'] as const;
+const telemetryCases: NodeJS.ProcessEnv[] = [
+  {},
+  ...telemetryControls.flatMap((name) => ['1', '0', 'true', 'false', ''].map((value) => ({ [name]: value }))),
+  { VALLY_TELEMETRY_OPTOUT: '1', DO_NOT_TRACK: '1', OTEL_SDK_DISABLED: 'true' },
+  { VALLY_TELEMETRY_OPTOUT: '0', DO_NOT_TRACK: '', OTEL_SDK_DISABLED: 'false' },
+];
 
 describe('direct CLI translation', () => {
   it('passes explicit argv without shell interpolation and preserves native override semantics', () => {
@@ -67,14 +74,35 @@ describe('direct CLI translation', () => {
       { command: 'lint', runs: '2' }, { suite: 'smoke' },
     ]) expect(() => nativeArguments(inputs(values), '.', '.', 'id')).toThrow();
   });
+  it.each(['run', 'lint'])('preserves Vally defaults and exact caller telemetry controls for %s', (command) => {
+    for (const controls of telemetryCases) {
+      const source = {
+        ...controls, PATH: '/bin', HOME: '/caller-home', COPILOT_GITHUB_TOKEN: 'model',
+        GITHUB_TOKEN: 'report', GH_TOKEN: 'gh', ACTIONS_RUNTIME_TOKEN: 'artifact', NODE_OPTIONS: 'unsafe',
+        OTEL_EXPORTER_OTLP_ENDPOINT: 'https://untrusted.invalid', OTEL_EXPORTER_OTLP_HEADERS: 'private-header',
+      };
+      const original = { ...source };
+      const env = nativeEnvironment(inputs({ command }), source, '/isolated-home', 'report');
+      for (const name of telemetryControls) {
+        if (controls[name] === undefined) expect(env).not.toHaveProperty(name);
+        else expect(env).toHaveProperty(name, controls[name]);
+      }
+      expect(env.HOME).toBe('/isolated-home');
+      for (const name of [
+        'COPILOT_GITHUB_TOKEN', 'GITHUB_TOKEN', 'GH_TOKEN', 'ACTIONS_RUNTIME_TOKEN', 'NODE_OPTIONS',
+        'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_HEADERS',
+      ]) expect(env).not.toHaveProperty(name);
+      expect(source).toEqual(original);
+    }
+  });
   it('separates model credentials from installation, preview, reporting and process injection', () => {
     const source = { COPILOT_GITHUB_TOKEN: 'model', OPENAI_API_KEY: 'byok', GITHUB_TOKEN: 'report', ACTIONS_RUNTIME_TOKEN: 'artifact', PATH: '/bin', NODE_OPTIONS: 'unsafe' };
     const unlisted = nativeEnvironment(inputs(), source, '/home', 'report');
     for (const name of ['COPILOT_GITHUB_TOKEN', 'OPENAI_API_KEY', 'GITHUB_TOKEN', 'ACTIONS_RUNTIME_TOKEN', 'NODE_OPTIONS']) {
       expect(unlisted).not.toHaveProperty(name);
     }
     const byok = nativeEnvironment(inputs({ 'credential-env': 'OPENAI_API_KEY' }), source, '/home', 'report');
-    expect(byok).toMatchObject({ OPENAI_API_KEY: 'byok', VALLY_TELEMETRY_OPTOUT: '1' });
+    expect(byok).toHaveProperty('OPENAI_API_KEY', 'byok');
     expect(byok).not.toHaveProperty('COPILOT_GITHUB_TOKEN');
     expect(nativeEnvironment(inputs({ 'credential-env': 'COPILOT_GITHUB_TOKEN' }), source, '/home', 'report'))
       .toHaveProperty('COPILOT_GITHUB_TOKEN', 'model');
@@ -150,6 +178,39 @@ describe('direct CLI translation', () => {
     expect(actual.result).toMatchObject({ outcome: 'not-run', code: 'native-empty-selection' });
     for (const path of actual.files) expect(await readFile(path, 'utf8')).not.toContain(token);
   });
+  it.each(['run', 'lint'])('transports caller telemetry settings through --version and %s without forced opt-outs', async (command) => {
+    for (const controls of [telemetryCases[0]!, ...telemetryCases.slice(-2)]) {
+      const root = await temporary();
+      const entry = join(root, 'telemetry-control.cjs');
+      await writeFile(entry, [
+        "const assert = require('node:assert/strict');",
+        `const expected = ${JSON.stringify(controls)};`,
+        `for (const name of ${JSON.stringify(telemetryControls)}) assert.equal(process.env[name], expected[name]);`,
+        "for (const name of ['GITHUB_TOKEN', 'GH_TOKEN', 'ACTIONS_RUNTIME_TOKEN', 'NODE_OPTIONS', 'OTEL_EXPORTER_OTLP_HEADERS']) assert.equal(process.env[name], undefined);",
+        "assert.ok(!JSON.stringify(process.argv).includes('synthetic-model-token'));",
+        "if (process.argv[2] === '--version') {",
+        "  assert.equal(process.env.COPILOT_GITHUB_TOKEN, undefined); console.log('0.19.0');",
+        "} else if (process.argv[2] === 'lint') {",
+        "  assert.equal(process.env.COPILOT_GITHUB_TOKEN, undefined);",
+        "} else {",
+        "  assert.equal(process.env.COPILOT_GITHUB_TOKEN, 'synthetic-model-token');",
+        "  const runId = process.argv.find(arg => arg.startsWith('--run-id=')).slice('--run-id='.length);",
+        "  console.log(JSON.stringify({type: 'run-summary', runId, passed: true, hadExecutionErrors: false, evals: []}));",
+        "}",
+      ].join('\n'));
+      const engine: EngineIdentity = { entry, version: '0.19.0', integrity: 'synthetic', sha256: sha256(await readFile(entry)), dependencySha256: '', lockSha256: '' };
+      const selected = inputs({ command, 'credential-env': 'COPILOT_GITHUB_TOKEN' });
+      const env = nativeEnvironment(selected, {
+        ...inertEnvironment(process.env), ...Object.fromEntries(telemetryControls.map((name) => [name, undefined])),
+        ...controls, COPILOT_GITHUB_TOKEN: 'synthetic-model-token', GITHUB_TOKEN: 'synthetic-report-token',
+        NODE_OPTIONS: 'unsafe', OTEL_EXPORTER_OTLP_HEADERS: 'private-header',
+      }, root, 'synthetic-report-token');
+      const original = { ...env };
+      const actual = await runNative(selected, engine, root, root, env);
+      expect(actual.result.outcome).toBe(command === 'run' ? 'not-run' : 'validated');
+      expect(env).toEqual(original);
+    }
+  });
 });
 
 function evidence(passed = true) {
@@ -218,14 +279,18 @@ async function nativeFixture() {
     entry, version: '0.19.0', sha256: sha256(await readFile(entry)),
     integrity: 'pinned-development-fixture', dependencySha256: 'development-lockfile', lockSha256: 'development-lockfile',
   };
-  return { root, project, output, engine };
+  const source = {
+    PATH: process.env.PATH, SystemRoot: process.env.SystemRoot,
+    VALLY_TELEMETRY_OPTOUT: '1', DO_NOT_TRACK: '1', OTEL_SDK_DISABLED: 'true',
+  };
+  return { root, project, output, engine, source };
 }
 
 describe('real Vally CLI, built-in mock executor, no models', () => {
   it('lints the entire project without selectors and catches invalid nested evals outside supplied selections', async () => {
     const fixture = await nativeFixture();
     const selected = inputs({ command: 'lint', 'eval-paths': '' });
-    const env = nativeEnvironment(selected, { PATH: process.env.PATH, SystemRoot: process.env.SystemRoot }, fixture.root, '');
+    const env = nativeEnvironment(selected, fixture.source, fixture.root, '');
     const valid = await runNative(selected, fixture.engine, fixture.project, fixture.output, env);
     expect(valid.result).toMatchObject({ outcome: 'validated', executed: 0 });
     await mkdir(join(fixture.project, 'nested'));
@@ -236,7 +301,7 @@ describe('real Vally CLI, built-in mock executor, no models', () => {
   it('runs an actual suite and respects native weighted thresholds instead of all-graders-pass rules', async () => {
     const fixture = await nativeFixture();
     const selected = inputs({ 'eval-paths': '', suite: 'smoke', runs: '2', model: 'one,two' });
-    const env = nativeEnvironment(selected, { PATH: process.env.PATH, SystemRoot: process.env.SystemRoot }, fixture.root, '');
+    const env = nativeEnvironment(selected, fixture.source, fixture.root, '');
     const actual = await runNative(selected, fixture.engine, fixture.project, fixture.output, env);
     expect(actual.result).toMatchObject({ outcome: 'passed', executed: 4, passed: 2, failed: 0 });
     expect(actual.result.groups).toHaveLength(2);
@@ -246,35 +311,35 @@ describe('real Vally CLI, built-in mock executor, no models', () => {
   it('preserves native include, exclude and comma-separated tag filters in real CLI execution', async () => {
     const fixture = await nativeFixture();
     const selected = inputs({ tags: 'cost=free,low\ncost!=paid' });
-    const env = nativeEnvironment(selected, { PATH: process.env.PATH, SystemRoot: process.env.SystemRoot }, fixture.root, '');
+    const env = nativeEnvironment(selected, fixture.source, fixture.root, '');
     const actual = await runNative(selected, fixture.engine, fixture.project, fixture.output, env);
     expect(actual.result).toMatchObject({ outcome: 'passed', executed: 1, passed: 1, failed: 0 });
   }, NATIVE_TEST_TIMEOUT_MS);
   it('fails an actual native verdict and reports a fully skipped selection as not-run', async () => {
     const fixture = await nativeFixture();
-    const env = nativeEnvironment(inputs(), { PATH: process.env.PATH, SystemRoot: process.env.SystemRoot }, fixture.root, '');
+    const env = nativeEnvironment(inputs(), fixture.source, fixture.root, '');
     const failed = await runNative(inputs({ 'eval-paths': 'fail.eval.yaml' }), fixture.engine, fixture.project, fixture.output, env);
     expect(failed.result).toMatchObject({ outcome: 'failed', executed: 1, failed: 1 });
     const skipped = await runNative(inputs({ 'eval-paths': 'skipped.eval.yaml' }), fixture.engine, fixture.project, fixture.output, env);
     expect(skipped.result).toMatchObject({ outcome: 'not-run', executed: 0 });
   }, NATIVE_TEST_TIMEOUT_MS);
   it('does not turn missing suites or evidence-free discovery into success', async () => {
     const fixture = await nativeFixture();
-    const env = nativeEnvironment(inputs(), { PATH: process.env.PATH, SystemRoot: process.env.SystemRoot }, fixture.root, '');
+    const env = nativeEnvironment(inputs(), fixture.source, fixture.root, '');
     for (const suite of ['missing', 'empty']) {
       const actual = await runNative(inputs({ 'eval-paths': '', suite }), fixture.engine, fixture.project, fixture.output, env);
       expect(actual.result).toMatchObject({ outcome: 'error', code: 'native-run-summary-missing' });
     }
   }, NATIVE_TEST_TIMEOUT_MS);
   it('refuses changed executable identity and mismatched native versions', async () => {
     const fixture = await nativeFixture();
-    const env = nativeEnvironment(inputs(), { PATH: process.env.PATH, SystemRoot: process.env.SystemRoot }, fixture.root, '');
+    const env = nativeEnvironment(inputs(), fixture.source, fixture.root, '');
     await expect(runNative(inputs(), { ...fixture.engine, sha256: 'wrong' }, fixture.project, fixture.output, env)).rejects.toThrow(/executable-changed/);
     await expect(runNative(inputs(), { ...fixture.engine, version: '0.18.0' }, fixture.project, fixture.output, env)).rejects.toThrow(/version-mismatch/);
   }, NATIVE_TEST_TIMEOUT_MS);
   it('preserves actual ungraded runs and native grader operational errors', async () => {
     const fixture = await nativeFixture();
-    const env = nativeEnvironment(inputs(), { PATH: process.env.PATH, SystemRoot: process.env.SystemRoot }, fixture.root, '');
+    const env = nativeEnvironment(inputs(), fixture.source, fixture.root, '');
     const ungraded = await runNative(inputs({ 'eval-paths': 'ungraded.eval.yaml' }), fixture.engine, fixture.project, fixture.output, env);
     expect(ungraded.result).toMatchObject({ outcome: 'passed', executed: 1, passed: 1 });
     const brokenGrader = await runNative(inputs({ 'eval-paths': 'grader-error.eval.yaml' }), fixture.engine, fixture.project, fixture.output, env);

tests/policy.test.ts

@@ -29,6 +29,17 @@ describe('trust and inert operations', () => {
       npm_config_userconfig: 'malicious', GIT_CONFIG_COUNT: '1', ACTIONS_RUNTIME_TOKEN: 'artifact',
     })).toEqual({ PATH: '/usr/bin', LANG: 'C', CI: 'true' });
   });
+  it('preserves only the named caller telemetry controls without interpreting their values', () => {
+    for (const value of ['1', '0', 'true', 'false', '']) {
+      const controls = { VALLY_TELEMETRY_OPTOUT: value, DO_NOT_TRACK: value, OTEL_SDK_DISABLED: value };
+      expect(inertEnvironment({
+        ...controls, GITHUB_TOKEN: 'report', COPILOT_GITHUB_TOKEN: 'model', NODE_OPTIONS: 'unsafe',
+        OTEL_EXPORTER_OTLP_ENDPOINT: 'https://untrusted.invalid', OTEL_EXPORTER_OTLP_HEADERS: 'private-header',
+        OTEL_RESOURCE_ATTRIBUTES: 'private-attributes',
+      })).toEqual({ ...controls, CI: 'true' });
+    }
+    expect(inertEnvironment({})).toEqual({ CI: 'true' });
+  });
   it('checks immutable checkout identity without claiming authorization', () => {
     assertRevision(`${'a'.repeat(40)}\n`, 'a'.repeat(40));
     expect(() => assertRevision('a'.repeat(40), 'b'.repeat(40))).toThrow();