From 05fe5296d3b7bdf5bc08b4ec32a65c1158d0c4a5 Mon Sep 17 00:00:00 2001 From: Anish Sinha Date: Fri, 17 Jul 2026 17:33:57 +0530 Subject: [PATCH] feat(browserstack-service): test-level custom metadata hook handling (beforeEach/afterEach) for WDIO+Mocha MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds correct beforeEach/afterEach hook-timing handling for the setCustomTags feature already on main. WDIO's Mocha runner fires user hooks OUTSIDE the SDK's per-test tracking span: beforeEach runs before beforeTest tracks the test instance, and afterEach runs after afterTest finishes it — so tags set in those hooks previously either landed on the wrong test or missed the payload. - Buffer tags set before a per-test context exists (e.g. from beforeEach) and flush them into the test at test start (a new TEST/PRE observer in CustomTagsModule minting an onBeforeTest flush). - The service layer records the open Mocha hook window (beforeHook/afterHook) so CustomTagsModule can route a setCustomTags call to the right test. - Defer the TestRunFinished (TEST/POST) send past the after-each hook window so tags set in afterEach still make the payload; flushed at the next test's boundary, or from service.after() for the worker's last test. Ports webdriverio PR anish353/webdriverio#3 into the migrated standalone repo, reconciling against pre-existing drift in testHubModule.ts and service.ts. Standalone on main, independent of the title-based-TC work. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../test-level-custom-metadata-hooks.md | 7 ++ .../src/cli/modules/customTagsModule.ts | 69 +++++++++++++++++-- .../src/cli/modules/testHubModule.ts | 59 ++++++++++++++-- .../browserstack-service/src/customTags.ts | 53 ++++++++++++++ packages/browserstack-service/src/service.ts | 20 ++++++ 5 files changed, 197 insertions(+), 11 deletions(-) create mode 100644 .changeset/test-level-custom-metadata-hooks.md diff --git a/.changeset/test-level-custom-metadata-hooks.md b/.changeset/test-level-custom-metadata-hooks.md new file mode 100644 index 0000000..cb4968d --- /dev/null +++ b/.changeset/test-level-custom-metadata-hooks.md @@ -0,0 +1,7 @@ +--- +"@wdio/browserstack-service": minor +--- + +Custom tags (`browser.setCustomTags`) set inside Mocha `beforeEach` / `afterEach` hooks now reliably land on the intended test's custom metadata. WDIO's Mocha runner fires user hooks outside the SDK's per-test tracking span — `beforeEach` runs before the test instance is tracked and `afterEach` runs after it is finished — so tags set in those hooks previously either attached to the wrong test or were dropped from the payload. + +Tags set in `beforeEach` are now buffered and flushed onto the test at its start, and the test-finished event is deferred past the `afterEach` window so late tags still make the payload. Values set across `beforeEach`, the test body, and `afterEach` union and dedupe onto the test (parity with Java `@Before` / `@After`). diff --git a/packages/browserstack-service/src/cli/modules/customTagsModule.ts b/packages/browserstack-service/src/cli/modules/customTagsModule.ts index 34b47f7..ce24ae6 100644 --- a/packages/browserstack-service/src/cli/modules/customTagsModule.ts +++ b/packages/browserstack-service/src/cli/modules/customTagsModule.ts @@ -7,10 +7,11 @@ import type AutomationFrameworkInstance from '../instances/automationFrameworkIn import type TestFrameworkInstance from '../instances/testFrameworkInstance.js' import { AutomationFrameworkState } from '../states/automationFrameworkState.js' import { HookState } from '../states/hookState.js' +import { TestFrameworkState } from '../states/testFrameworkState.js' import { TestFrameworkConstants } from '../frameworks/constants/testFrameworkConstants.js' import { CLIUtils } from '../cliUtils.js' import WdioMochaTestFramework from '../frameworks/wdioMochaTestFramework.js' -import { mergeIntoTags, parseCommaSeparatedValues } from '../../customTags.js' +import { mergeIntoTags, parseCommaSeparatedValues, getCurrentMochaHookWindow } from '../../customTags.js' import type { CustomMetadata } from '../../customTags.js' /** @@ -33,10 +34,20 @@ export default class CustomTagsModule extends BaseModule { name: string static MODULE_NAME = 'CustomTagsModule' + /** + * Tags set before a per-test context exists — e.g. from a `beforeEach` hook, which + * WDIO runs BEFORE `beforeTest` tracks the test instance. Buffered here and flushed + * into the test at test start (onBeforeTest), then reset per-test, so hook-set tags + * land on the test (parity with Java @Before). Process-local (one module per worker) + * and Mocha runs tests serially, so a plain object is safe. + */ + private pendingTestLevelTags: CustomMetadata = {} + constructor() { super() this.name = 'CustomTagsModule' AutomationFramework.registerObserver(AutomationFrameworkState.CREATE, HookState.POST, this.onBeforeExecute.bind(this)) + TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this)) } getModuleName() { @@ -64,18 +75,29 @@ export default class CustomTagsModule extends BaseModule { return } - const testInstance: TestFrameworkInstance = TestFramework.getTrackedInstance() - if (!testInstance) { - this.logger.warn('setCustomTags called outside of a resolvable test context; ignoring') - return - } - const values = parseCommaSeparatedValues(value) if (values.length === 0) { this.logger.warn(`setCustomTags: no usable values parsed from "${value}"; ignoring call`) return } + // Route by the open Mocha hook window (recorded by the service layer — + // the CLI framework never sees Mocha's beforeEach/afterEach): + // - before-each / before-all → the tag belongs to the UPCOMING test. + // The tracked instance is absent (first test) or still the PREVIOUS + // test here, so buffer and flush at test start (onBeforeTest). + // - after-each / after-all / in-test → the CURRENT tracked test. Its + // TestRunFinished send is deferred past the after-each window + // (TestHubModule), so late merges still make the payload. + const hookWindow = getCurrentMochaHookWindow() + const isPreTestWindow = hookWindow === 'before_each' || hookWindow === 'before_all' + const testInstance: TestFrameworkInstance = TestFramework.getTrackedInstance() + if (!testInstance || isPreTestWindow) { + mergeIntoTags(this.pendingTestLevelTags, key, values) + this.logger.debug(`setCustomTags: buffered pre-test tag key=${key} values=${JSON.stringify(values)} (window=${hookWindow ?? 'none'}; will flush at test start)`) + return + } + // Merge into the per-test instance map (test-scoped accumulator). const existing = (TestFramework.getState(testInstance, TestFrameworkConstants.KEY_CUSTOM_TAGS) as CustomMetadata) || {} mergeIntoTags(existing, key, values) @@ -97,6 +119,39 @@ export default class CustomTagsModule extends BaseModule { } } + /** + * At each test start (TEST/PRE — after beforeTest tracks the instance), flush any + * tags buffered before the test had a context (e.g. set in a `beforeEach` hook — see + * the setCustomTags closure) into the tracked instance's custom_metadata via the SAME + * merge path as explicit setCustomTags, then reset the buffer for the next test — so + * hook-set tags union onto the test (parity with Java @Before). + */ + async onBeforeTest(args: Record) { + try { + // Prefer the tracked instance (the SAME one the explicit setCustomTags closure + // reads/writes) so buffered and programmatic tags all union. + const testInstance = TestFramework.getTrackedInstance() || (args.instance as TestFrameworkInstance) + if (!testInstance) { + return + } + + // Flush tags buffered before this test had a context (e.g. from beforeEach). + if (Object.keys(this.pendingTestLevelTags).length > 0) { + const buffered = (TestFramework.getState(testInstance, TestFrameworkConstants.KEY_CUSTOM_TAGS) as CustomMetadata) || {} + for (const [k, entry] of Object.entries(this.pendingTestLevelTags)) { + mergeIntoTags(buffered, k, entry.values) + } + testInstance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_CUSTOM_TAGS]: buffered + }) + this.logger.debug(`setCustomTags: flushed buffered pre-test tags ${JSON.stringify(Object.keys(this.pendingTestLevelTags))} into test`) + this.pendingTestLevelTags = {} + } + } catch (error) { + this.logger.debug(`setCustomTags: error flushing buffered pre-test tags: ${error}`) + } + } + /** * If the test framework is currently inside a hook, merge custom_metadata onto * the last-started hook record so the binary receives hook.custom_metadata. diff --git a/packages/browserstack-service/src/cli/modules/testHubModule.ts b/packages/browserstack-service/src/cli/modules/testHubModule.ts index d10fb90..1d8d33d 100644 --- a/packages/browserstack-service/src/cli/modules/testHubModule.ts +++ b/packages/browserstack-service/src/cli/modules/testHubModule.ts @@ -27,6 +27,20 @@ export default class TestHubModule extends BaseModule { name: string static MODULE_NAME = 'TestHubModule' + /** + * Mocha-only: the TEST/POST (TestRunFinished) send deferred past the after-each hook + * window. WDIO fires `afterTest` (which triggers TEST/POST) BEFORE the user's + * `afterEach` hooks run, so custom tags set in `afterEach` would otherwise miss the + * event. The payload (`eventJson`) is serialized from the instance's live data map at + * SEND time, so deferring the send picks up those late merges; uuid / started_at / + * ended_at are already stamped in the data and do not drift. The stashed + * `args.instance` is the finished test's OWN object — INIT_TEST replaces the tracked + * slot with a fresh instance for the next test — so it stays valid across tests. + * Flushed at the next test's first event (INIT_TEST / TEST PRE) or, for the worker's + * last test, from service.after() via flushPendingTestFinishEvent(). + */ + private pendingTestFinish: { args: Record } | null = null + /** * Create a new TestHubModule */ @@ -66,6 +80,13 @@ export default class TestHubModule extends BaseModule { const testState = instance.getCurrentTestState() const hookState = instance.getCurrentHookState() const keyTestDeferred = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_DEFERRED) + + // A NEW test is starting (INIT_TEST minted a fresh instance) — the previous test's + // after-each hook window is definitively over, so flush its deferred finish first + // (payload build is synchronous, so gRPC send order is preserved). + if (this.pendingTestFinish && (testState === TestFrameworkState.INIT_TEST || (testState === TestFrameworkState.TEST && hookState === HookState.PRE))) { + this.flushPendingTestFinishEvent() + } if (testState === TestFrameworkState.LOG) { this.logger.debug(`onAllTestEvents: TestFrameworkState.LOG - ${testState}`) const logEntries = WdioMochaTestFramework.getLogEntries(instance, testState, hookState) @@ -94,11 +115,41 @@ export default class TestHubModule extends BaseModule { } if (testState === TestFrameworkState.TEST || CLIUtils.matchHookRegex(testState.toString().split('.')[1])) { - this.sendTestFrameworkEvent(args) + const frameworkName = String(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') + if (testState === TestFrameworkState.TEST && hookState === HookState.POST && frameworkName.toLowerCase().includes('mocha')) { + // Defer the TestRunFinished send past the Mocha after-each hook window so + // custom tags set in `afterEach` still make the payload (see field docs). + // If a previous finish is somehow still pending for a DIFFERENT test, flush + // it first; a re-stash for the same instance just replaces the stash. + if (this.pendingTestFinish && (this.pendingTestFinish.args.instance as TestFrameworkInstance) !== instance) { + this.flushPendingTestFinishEvent() + } + this.pendingTestFinish = { args } + this.logger.debug('onAllTestEvents: deferred TEST/POST send past the after-each hook window') + } else { + this.sendTestFrameworkEvent(args) + } } } - async sendTestFrameworkEvent(args: Record) { + /** + * Send a deferred TEST/POST (TestRunFinished) event, if one is pending. The instance's + * CURRENT state may have moved on (e.g. LOG events during afterEach), so the send uses + * an explicit TEST/POST state override; the payload data itself is read fresh from the + * instance so late custom-tag merges are included. Called from onAllTestEvents at the + * next test's boundary and from service.after() at worker end. + */ + flushPendingTestFinishEvent(): Promise | undefined { + if (!this.pendingTestFinish) { + return undefined + } + const { args } = this.pendingTestFinish + this.pendingTestFinish = null + this.logger.debug('flushPendingTestFinishEvent: sending deferred TEST/POST event') + return this.sendTestFrameworkEvent(args, { testFrameworkState: 'TEST', testHookState: 'POST' }) + } + + async sendTestFrameworkEvent(args: Record, stateOverride?: { testFrameworkState: string, testHookState: string }) { try { const testArgs = args as { test: Frameworks.Test, instance: TestFrameworkInstance } const instance = testArgs.instance as TestFrameworkInstance @@ -108,8 +159,8 @@ export default class TestHubModule extends BaseModule { const testFrameworkVersion = testData.get(TestFrameworkConstants.KEY_TEST_FRAMEWORK_VERSION) || '' const startedAt = testData.get(TestFrameworkConstants.KEY_TEST_STARTED_AT) || '' const endedAt = testData.get(TestFrameworkConstants.KEY_TEST_ENDED_AT) || '' - const testFrameworkState = instance.getCurrentTestState().toString().split('.')[1] - const testHookState = instance.getCurrentHookState().toString().split('.')[1] + const testFrameworkState = stateOverride?.testFrameworkState || instance.getCurrentTestState().toString().split('.')[1] + const testHookState = stateOverride?.testHookState || instance.getCurrentHookState().toString().split('.')[1] this.logger.debug(`sendTestFrameworkEvent for testState: ${testFrameworkState} hookState: ${testHookState}`) const platformIndex = process.env.WDIO_WORKER_ID ? parseInt(process.env.WDIO_WORKER_ID.split('-')[0]) : 0 diff --git a/packages/browserstack-service/src/customTags.ts b/packages/browserstack-service/src/customTags.ts index 199251d..b0ef3f8 100644 --- a/packages/browserstack-service/src/customTags.ts +++ b/packages/browserstack-service/src/customTags.ts @@ -131,3 +131,56 @@ export class CustomTagAccumulator { } } } + +/* -------------------------------------------------------------------------- + * Mocha hook-window tracking. + * + * WDIO's Mocha runner fires user hooks OUTSIDE the SDK's test-tracking span: + * `beforeEach` runs before `beforeTest` (which creates/tracks the per-test + * instance), and `afterEach` runs after `afterTest` (which finishes the test). + * The CLI framework never sees these hooks, so a `setCustomTags` call made + * inside one cannot be routed by instance state alone — during the NEXT test's + * `beforeEach` the tracked instance is still the PREVIOUS test. + * + * The service layer (which does see WDIO's beforeHook/afterHook callbacks) + * records which Mocha hook window is currently open; the CLI CustomTagsModule + * reads it to route tags: before-each / before-all → buffer for the upcoming + * test; after-each / after-all → the current (tracked) test, whose + * TestRunFinished send is deferred past the hook window (see TestHubModule). + * Process-local; Mocha runs tests serially within a worker. + * ------------------------------------------------------------------------ */ + +export type MochaHookWindow = 'before_each' | 'after_each' | 'before_all' | 'after_all' | null + +let currentMochaHookWindow: MochaHookWindow = null + +/** Classify a Mocha hook title (e.g. '"before each" hook: setup') into a window type. */ +export function classifyMochaHookTitle(title: string | undefined | null): MochaHookWindow { + if (!title) { + return null + } + const t = title.toLowerCase() + if (t.includes('before each')) { + return 'before_each' + } + if (t.includes('after each')) { + return 'after_each' + } + if (t.includes('before all')) { + return 'before_all' + } + if (t.includes('after all')) { + return 'after_all' + } + return null +} + +/** Set by the service's beforeHook (mocha); cleared by afterHook. */ +export function setCurrentMochaHookWindow(window: MochaHookWindow): void { + currentMochaHookWindow = window +} + +/** Read by the CLI CustomTagsModule to route setCustomTags calls made inside hooks. */ +export function getCurrentMochaHookWindow(): MochaHookWindow { + return currentMochaHookWindow +} diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index c5f21d1..4f3346b 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -19,6 +19,8 @@ import { DEFAULT_OPTIONS, NOT_ALLOWED_KEYS_IN_CAPS, PERF_MEASUREMENT_ENV } from import CrashReporter from './crash-reporter.js' import AccessibilityHandler from './accessibility-handler.js' import CustomTagsHandler from './custom-tags-handler.js' +import { classifyMochaHookTitle, setCurrentMochaHookWindow } from './customTags.js' +import type TestHubModule from './cli/modules/testHubModule.js' import { BStackLogger } from './bstackLogger.js' import PercyHandler from './Percy/Percy-Handler.js' import Listener from './testOps/listener.js' @@ -389,6 +391,11 @@ export default class BrowserstackService implements Services.ServiceInstance { if (this._config.framework !== 'cucumber') { this._currentTest = test as Frameworks.Test // not update currentTest when this is called for cucumber step } + // Record which Mocha hook window is open so custom-tag calls made inside user + // hooks route to the right test (the CLI framework never sees these hooks). + if (this._config.framework === 'mocha') { + setCurrentMochaHookWindow(classifyMochaHookTitle((test as Frameworks.Test).title)) + } // CLI flow: route hook lifecycle to the binary via the TestFramework tracker (gRPC), // mirroring beforeTest/afterTest. Without this, hook events fall through to the legacy @@ -414,6 +421,10 @@ export default class BrowserstackService implements Services.ServiceInstance { @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'afterHook' }) async afterHook(test: Frameworks.Test | CucumberHook, context: unknown, result: Frameworks.TestResult) { + // The Mocha hook window is closed — clear the tracker (see beforeHook). + if (this._config.framework === 'mocha') { + setCurrentMochaHookWindow(null) + } // Track hook failures separately if (result && !result.passed) { const hookError = (result.error && result.error.message) || 'Hook failed' @@ -537,6 +548,15 @@ export default class BrowserstackService implements Services.ServiceInstance { } if (BrowserstackCLI.getInstance().isRunning()) { + // Flush a test-finish event deferred past the after-each hook window — the last + // test of the worker has no next-test boundary to trigger the flush. Must run + // before worker teardown so the event isn't dropped. + try { + const testHubModule = BrowserstackCLI.getInstance().modules.TestHubModule as TestHubModule | undefined + await testHubModule?.flushPendingTestFinishEvent() + } catch (flushErr) { + BStackLogger.debug(`Exception flushing deferred test finish in after(): ${util.format(flushErr)}`) + } await BrowserstackCLI.getInstance().getAutomationFramework()!.trackEvent(AutomationFrameworkState.EXECUTE, HookState.POST, {}) }