Support vitest 5

.github/workflows/codspeed.yml

@@ -82,8 +82,11 @@ jobs:
       - name: Run benchmarks
         # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2`
         uses: CodSpeedHQ/action@main
+        env:
+          CODSPEED_WALLTIME_PROFILER: samply
         with:
           mode: walltime
+          runner-version: v4.17.7-alpha.2
           run: |
             pnpm turbo run bench --filter=@codspeed/tinybench-plugin
             pnpm turbo run bench --filter=@codspeed/vitest-plugin
@@ -173,3 +176,4 @@ jobs:
           working-directory: examples/with-electron-and-walltime
           run: xvfb-run -a pnpm bench:electron
           mode: walltime
+          runner-version: v4.17.7-alpha.2

packages/core/package.json

@@ -32,7 +32,6 @@
   "homepage": "https://codspeed.io",
   "license": "Apache-2.0",
   "devDependencies": {
-    "@types/find-up": "^4.0.0",
     "@types/stack-trace": "^0.0.30",
     "node-addon-api": "^5.1.0",
     "node-gyp": "^12.2.0",
@@ -41,7 +40,6 @@
   },
   "dependencies": {
     "axios": "^1.4.0",
-    "find-up": "^6.3.0",
     "form-data": "^4.0.4",
     "node-gyp-build": "^4.6.0",
     "stack-trace": "1.0.0-pre2"

packages/core/src/utils.ts

@@ -1,14 +1,36 @@
-import { Options as FindupOptions, findUpSync } from "find-up";
-import path, { dirname } from "path";
+import { existsSync, statSync } from "fs";
+import path, { dirname, join } from "path";
 import { get as getStackTrace } from "stack-trace";
 import { fileURLToPath } from "url";
 
-export function getGitDir(path: string): string | undefined {
-  const dotGitPath = findUpSync(".git", {
-    cwd: path,
-    type: "directory",
-  } as FindupOptions);
-  return dotGitPath ? dirname(dotGitPath) : undefined;
+export function getGitDir(fromPath: string): string | undefined {
+  // Walk up from the starting path looking for the first ancestor that contains
+  // a `.git` entry, accepting it whether it's a directory (regular checkout) or
+  // a file (a gitlink, as used by git worktrees and submodules). Restricting to
+  // directories would miss those layouts and either fail or wrongly resolve to a
+  // parent repository.
+  let current = directoryOf(fromPath);
+  for (;;) {
+    if (existsSync(join(current, ".git"))) {
+      return current;
+    }
+    const parent = dirname(current);
+    if (parent === current) {
+      return undefined;
+    }
+    current = parent;
+  }
+}
+
+/** The starting directory for the walk: `p` itself if it's a directory, else its parent. */
+function directoryOf(p: string): string {
+  try {
+    return statSync(p).isDirectory() ? p : dirname(p);
+  } catch {
+    // Path doesn't exist (e.g. a not-yet-written file): treat it as a file and
+    // start from its containing directory.
+    return dirname(p);
+  }
 }
 
 /**

packages/vitest-plugin/package.json

@@ -41,13 +41,13 @@
   "peerDependencies": {
     "tinybench": ">=2.9.0",
     "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0",
-    "vitest": "^3.2 || ^4"
+    "vitest": "^3.2 || ^4 || ^5.0.0-beta"
   },
   "devDependencies": {
     "@total-typescript/shoehorn": "^0.1.1",
     "execa": "^8.0.1",
     "tinybench": "^2.9.0",
     "vite": "^7.0.0",
-    "vitest": "^4.0.18"
+    "vitest": "5.0.0-beta.5"
   }
 }

packages/vitest-plugin/rollup.config.ts

@@ -32,4 +32,10 @@ export default defineConfig([
     plugins: jsPlugins(pkg.version),
     external: ["@codspeed/core", /^vitest/],
   },
+  {
+    input: "src/v5/runner.ts",
+    output: { file: "dist/v5-runner.mjs", format: "es" },
+    plugins: jsPlugins(pkg.version),
+    external: ["@codspeed/core", /^vitest/, "tinybench"],
+  },
 ]);

packages/vitest-plugin/src/__tests__/index.test.ts

@@ -43,6 +43,19 @@ vi.mock("fs", () => {
 
 console.warn = vi.fn();
 
+const EXPECTED_EXEC_ARGV = [
+  "--interpreted-frames-native-stack",
+  "--allow-natives-syntax",
+  "--hash-seed=1",
+  "--random-seed=1",
+  "--no-opt",
+  "--predictable",
+  "--predictable-gc-schedule",
+  "--expose-gc",
+  "--no-concurrent-sweeping",
+  "--max-old-space-size=4096",
+];
+
 describe("codSpeedPlugin", () => {
   beforeAll(() => {
     // Set environment variables to trigger instrumented mode
@@ -54,6 +67,7 @@ describe("codSpeedPlugin", () => {
     // Clean up environment variables
     delete process.env.CODSPEED_ENV;
     delete process.env.CODSPEED_RUNNER_MODE;
+    fsMocks.setMockVersion("4.0.18");
   });
 
   it("should have a name", async () => {
@@ -65,7 +79,9 @@ describe("codSpeedPlugin", () => {
   });
 
   describe("apply", () => {
-    it("should not apply the plugin when the mode is not benchmark", async () => {
+    it("should not apply the plugin when the mode is not benchmark (v3/v4)", async () => {
+      fsMocks.setMockVersion("4.0.18");
+
       const applyPlugin = applyPluginFunction(
         {},
         fromPartial({ mode: "test" }),
@@ -74,7 +90,8 @@ describe("codSpeedPlugin", () => {
       expect(applyPlugin).toBe(false);
     });
 
-    it("should apply the plugin when there is no instrumentation", async () => {
+    it("should apply the plugin when there is no instrumentation (v3/v4)", async () => {
+      fsMocks.setMockVersion("4.0.18");
       coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(false);
 
       const applyPlugin = applyPluginFunction(
@@ -88,7 +105,8 @@ describe("codSpeedPlugin", () => {
       expect(applyPlugin).toBe(true);
     });
 
-    it("should apply the plugin when there is instrumentation", async () => {
+    it("should apply the plugin when there is instrumentation (v3/v4)", async () => {
+      fsMocks.setMockVersion("4.0.18");
       coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(true);
 
       const applyPlugin = applyPluginFunction(
@@ -98,33 +116,40 @@ describe("codSpeedPlugin", () => {
 
       expect(applyPlugin).toBe(true);
     });
+
+    it("should stay active regardless of mode on v5 (benchmark gating happens in config)", async () => {
+      fsMocks.setMockVersion("5.0.0-beta.5");
+      coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(true);
+
+      const applyPlugin = applyPluginFunction(
+        {},
+        fromPartial({ mode: "test" }),
+      );
+
+      expect(applyPlugin).toBe(true);
+      fsMocks.setMockVersion("4.0.18");
+    });
   });
 
   it("should apply the codspeed config for v4", () => {
+    fsMocks.setMockVersion("4.0.18");
     const config = resolvedCodSpeedPlugin.config;
     if (typeof config !== "function")
       throw new Error("config is not a function");
 
-    const result = config.call({} as never, {}, fromPartial({}));
+    const result = config.call(
+      {} as never,
+      {},
+      fromPartial({ mode: "benchmark" }),
+    );
 
     expect(result).toStrictEqual({
       test: {
         globalSetup: [
           expect.stringContaining("packages/vitest-plugin/src/globalSetup.ts"),
         ],
         pool: "forks",
-        execArgv: [
-          "--interpreted-frames-native-stack",
-          "--allow-natives-syntax",
-          "--hash-seed=1",
-          "--random-seed=1",
-          "--no-opt",
-          "--predictable",
-          "--predictable-gc-schedule",
-          "--expose-gc",
-          "--no-concurrent-sweeping",
-          "--max-old-space-size=4096",
-        ],
+        execArgv: EXPECTED_EXEC_ARGV,
         runner: expect.stringContaining(
           "packages/vitest-plugin/src/analysis.ts",
         ),
@@ -133,16 +158,18 @@ describe("codSpeedPlugin", () => {
   });
 
   it("should apply the codspeed config for v3 with poolOptions", () => {
-    // Set mock version to v3
     fsMocks.setMockVersion("3.2.0");
 
-    // Create a new plugin instance to pick up the mocked version
     const v3Plugin = codspeedPlugin();
     const config = v3Plugin.config;
     if (typeof config !== "function")
       throw new Error("config is not a function");
 
-    const result = config.call({} as never, {}, fromPartial({}));
+    const result = config.call(
+      {} as never,
+      {},
+      fromPartial({ mode: "benchmark" }),
+    );
 
     expect(result).toStrictEqual({
       test: {
@@ -152,18 +179,7 @@ describe("codSpeedPlugin", () => {
         pool: "forks",
         poolOptions: {
           forks: {
-            execArgv: [
-              "--interpreted-frames-native-stack",
-              "--allow-natives-syntax",
-              "--hash-seed=1",
-              "--random-seed=1",
-              "--no-opt",
-              "--predictable",
-              "--predictable-gc-schedule",
-              "--expose-gc",
-              "--no-concurrent-sweeping",
-              "--max-old-space-size=4096",
-            ],
+            execArgv: EXPECTED_EXEC_ARGV,
           },
         },
         runner: expect.stringContaining(
@@ -172,7 +188,51 @@ describe("codSpeedPlugin", () => {
       },
     });
 
-    // Reset mock version back to v4
     fsMocks.setMockVersion("4.0.18");
   });
+
+  describe("v5 config", () => {
+    it("should not inject config when benchmarks are not enabled", () => {
+      fsMocks.setMockVersion("5.0.0-beta.5");
+      const v5Plugin = codspeedPlugin();
+      const config = v5Plugin.config;
+      if (typeof config !== "function")
+        throw new Error("config is not a function");
+
+      const result = config.call({} as never, {}, fromPartial({ mode: "test" }));
+
+      expect(result).toBeUndefined();
+      fsMocks.setMockVersion("4.0.18");
+    });
+
+    it("should inject the v5 runner when benchmarks are enabled", () => {
+      fsMocks.setMockVersion("5.0.0-beta.5");
+      const v5Plugin = codspeedPlugin();
+      const config = v5Plugin.config;
+      if (typeof config !== "function")
+        throw new Error("config is not a function");
+
+      const result = config.call(
+        {} as never,
+        // `benchmark.enabled` is a Vitest 5 config field the v3/4 typings (which
+        // this file may be compiled against) don't expose.
+        { test: { benchmark: { enabled: true } } } as never,
+        fromPartial({ mode: "test" }),
+      );
+
+      expect(result).toStrictEqual({
+        test: {
+          globalSetup: [
+            expect.stringContaining(
+              "packages/vitest-plugin/src/globalSetup.ts",
+            ),
+          ],
+          pool: "forks",
+          execArgv: EXPECTED_EXEC_ARGV,
+          runner: expect.stringContaining("packages/vitest-plugin/src/v5"),
+        },
+      });
+      fsMocks.setMockVersion("4.0.18");
+    });
+  });
 });

packages/vitest-plugin/src/__tests__/instrumented.test.ts

@@ -1,8 +1,16 @@
 import { fromPartial } from "@total-typescript/shoehorn";
 import { describe, expect, it, vi, type RunnerTestSuite } from "vitest";
+// `vitest/suite` only exists on Vitest 3/4; this file is excluded from the test
+// run under v5+ (see vitest.config.ts).
+// eslint-disable-next-line import/no-unresolved
 import { getBenchFn } from "vitest/suite";
 import { AnalysisRunner as CodSpeedRunner } from "../analysis";
 
+// The legacy AnalysisRunner targets the Vitest 3/4 benchmark backend
+// (`NodeBenchmarkRunner`, `vitest/suite`), which Vitest 5 removed. This whole
+// file is excluded from the test run under v5+ (see vitest.config.ts); the v5
+// path is covered separately.
+
 const coreMocks = vi.hoisted(() => {
   return {
     InstrumentHooks: {

packages/vitest-plugin/src/analysis.ts

@@ -8,7 +8,11 @@ import {
   wrapWithRootFrame,
 } from "@codspeed/core";
 import { Benchmark, type RunnerTestSuite } from "vitest";
+// `vitest/runners` and `vitest/suite` only exist on Vitest 3/4; this runner is
+// loaded only there.
+// eslint-disable-next-line import/no-unresolved
 import { NodeBenchmarkRunner } from "vitest/runners";
+// eslint-disable-next-line import/no-unresolved
 import { getBenchFn } from "vitest/suite";
 import {
   callSuiteHook,
@@ -36,11 +40,14 @@ async function runAnalysisBench(
   currentSuiteName: string,
 ) {
   const uri = `${currentSuiteName}::${benchmark.name}`;
-  const fn = getBenchFn(benchmark);
+  // tinybench's bench fn carries a `this: Bench` requirement on Vitest 3/4 that
+  // we don't need (the work under test is self-contained); call it as a plain
+  // parameterless function. The cast also smooths over the typing differences
+  // across supported Vitest versions.
+  const fn = getBenchFn(benchmark) as () => unknown;
 
   await optimizeFunction(async () => {
     await callSuiteHook(suite, benchmark, "beforeEach");
-    // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench
     await fn();
     await callSuiteHook(suite, benchmark, "afterEach");
   });
@@ -50,7 +57,6 @@ async function runAnalysisBench(
   global.gc?.();
   await wrapWithRootFrame(async () => {
     InstrumentHooks.startBenchmark();
-    // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench
     await fn();
     InstrumentHooks.stopBenchmark();
     InstrumentHooks.setExecutedBenchmark(process.pid, uri);

packages/vitest-plugin/src/common.ts

@@ -1,6 +1,8 @@
 import { getGitDir } from "@codspeed/core";
 import path from "path";
 import { Benchmark, type RunnerTask, type RunnerTestSuite } from "vitest";
+// `vitest/suite` only exists on Vitest 3/4; this module is used only there.
+// eslint-disable-next-line import/no-unresolved
 import { getHooks } from "vitest/suite";
 type SuiteHooks = ReturnType<typeof getHooks>;
 
@@ -19,8 +21,11 @@ export async function callSuiteHook<T extends keyof SuiteHooks>(
 
   const hooks = getSuiteHooks(suite, name);
 
-  // @ts-expect-error TODO: add support for hooks parameters
-  await Promise.all(hooks.map((fn) => fn()));
+  // TODO: add support for hook parameters. The hook signature differs across
+  // supported Vitest versions, so we call them through a parameterless cast.
+  await Promise.all(
+    (hooks as Array<() => unknown>).map((fn) => fn()),
+  );
 
   if (name === "afterEach" && suite?.suite) {
     await callSuiteHook(suite.suite, currentTask, name);