e2e tests: collect unstable tests statistics

apps/demos/utils/visual-tests/testcafe-runner.ts

@@ -91,7 +91,7 @@ async function main() {
     .browsers(process.env.BROWSERS || 'chrome --no-sandbox --disable-dev-shm-usage --disable-partial-raster --disable-skia-runtime-opts --run-all-compositor-stages-before-draw --disable-new-content-rendering-timeout --disable-threaded-animation --disable-threaded-scrolling --disable-checker-imaging --disable-image-animation-resync --use-gl=swiftshader --disable-features=PaintHolding --js-flags=--random-seed=2147483647 --font-render-hinting=none --disable-font-subpixel-positioning')
     .concurrency(concurrency || 1)
     .run({
-      quarantineMode: { successThreshold: 1, attemptLimit: 2 },
+      quarantineMode: { successThreshold: 1, attemptLimit: 3 },
       // @ts-expect-error ts-error
       hooks: {
         test: {

e2e/testcafe-devextreme/runner.ts

@@ -1,6 +1,7 @@
 /* eslint-disable spellcheck/spell-checker */
 import createTestCafe, { ClientFunction } from 'testcafe';
 import * as fs from 'fs';
+import * as path from 'path';
 import * as process from 'process';
 import parseArgs from 'minimist';
 import { DEFAULT_BROWSER_SIZE } from './helpers/const';
@@ -53,9 +54,40 @@ interface ParsedArgs {
   platform: string;
   theme: string;
   shadowDom: boolean;
-  skipUnstable: boolean;
+  skipQuarantined: boolean;
+  onlyQuarantined: boolean;
   disableScreenshots: boolean;
   retryFailed: boolean;
+  testsFile: string;
+  reportFlaky: string;
+  reportFailures: string;
+  retryAttempts: number;
+  forcePageReloads: boolean;
+}
+
+const QUARANTINE_FILE = path.join(__dirname, 'quarantine.json');
+
+interface QuarantineEntry {
+  test: string;
+  file?: string;
+}
+
+// test name -> fixture files ('' = match any file, for entries without "file")
+function readQuarantinedTests(): Map<string, string[]> {
+  if (!fs.existsSync(QUARANTINE_FILE)) {
+    return new Map<string, string[]>();
+  }
+
+  const { tests } = JSON.parse(fs.readFileSync(QUARANTINE_FILE, 'utf8'));
+  const result = new Map<string, string[]>();
+
+  (tests as QuarantineEntry[]).forEach(({ test, file }) => {
+    const files = result.get(test) ?? [];
+    files.push(file ?? '');
+    result.set(test, files);
+  });
+
+  return result;
 }
 
 const getTestCafeConfig = (cache: boolean): Partial<TestCafeConfigurationOptions> => ({
@@ -112,13 +144,27 @@ function getArgs(): ParsedArgs {
       platform: '',
       theme: '',
       shadowDom: false,
-      skipUnstable: true,
+      skipQuarantined: true,
+      onlyQuarantined: false,
       disableScreenshots: false,
       retryFailed: true,
+      testsFile: '',
+      reportFlaky: '',
+      reportFailures: '',
+      retryAttempts: FAILED_TESTS_RETRY_ATTEMPTS,
+      forcePageReloads: false,
     },
   }) as ParsedArgs;
 }
 
+function writeTestsReport(reportPath: string, tests: Set<string>, failedCount?: number): void {
+  fs.mkdirSync(path.dirname(reportPath), { recursive: true });
+  fs.writeFileSync(
+    reportPath,
+    JSON.stringify({ tests: [...tests], failedCount: failedCount ?? tests.size }, null, 2),
+  );
+}
+
 async function main() {
   let testCafe: Awaited<ReturnType<typeof createTestCafe>> | null = null;
 
@@ -138,6 +184,26 @@ async function main() {
     const componentFolderArg = typeof args.componentFolder === 'string' ? args.componentFolder.trim() : '';
     const componentFolder = componentFolderArg ? `${componentFolderArg}/**` : '**';
 
+    const onlyQuarantined = `${args.onlyQuarantined}` === 'true';
+    const retryAttempts = Number(args.retryAttempts) || 0;
+    const forcePageReloads = `${args.forcePageReloads}` === 'true';
+    const testsFromFile = new Set<string>(
+      args.testsFile ? JSON.parse(fs.readFileSync(args.testsFile, 'utf8')).tests : [],
+    );
+
+    const quarantinedTests = readQuarantinedTests();
+    const isQuarantined = (testNameArg: string, fixturePath: string, testMeta?: any): boolean => {
+      if (testMeta?.unstable) {
+        return true;
+      }
+
+      const files = quarantinedTests.get(testNameArg);
+
+      return !!files?.some(
+        (fixtureFile) => !fixtureFile || fixturePath.replace(/\\/g, '/').endsWith(fixtureFile),
+      );
+    };
+
     if (fs.existsSync('./screenshots')) {
       fs.rmSync('./screenshots', { recursive: true });
     }
@@ -190,17 +256,28 @@ async function main() {
         filters.push((name: string) => name === testName);
       }
 
+      if (testsFromFile.size > 0) {
+        filters.push((name: string) => testsFromFile.has(name));
+      }
+
       if (filterByFailedTests && testsToFilter && testsToFilter.size > 0) {
         filters.push((name: string) => testsToFilter.has(name));
       }
 
-      if (args.skipUnstable) {
+      if (onlyQuarantined) {
         filters.push((
-          _testName: string,
+          filterTestName: string,
           _fixtureName: string,
-          _fixturePath: string,
+          fixturePath: string,
           testMeta?: any,
-        ) => !(testMeta)?.unstable);
+        ) => isQuarantined(filterTestName, fixturePath, testMeta));
+      } else if (args.skipQuarantined) {
+        filters.push((
+          filterTestName: string,
+          _fixtureName: string,
+          fixturePath: string,
+          testMeta?: any,
+        ) => !isQuarantined(filterTestName, fixturePath, testMeta));
       }
 
       filters.push((
@@ -255,6 +332,14 @@ async function main() {
       hooks: {
         test: {
           before: async (t: TestController) => {
+            if (forcePageReloads) {
+              const href = await ClientFunction(
+                () => window.location.href,
+              ).with({ boundTestRun: t })();
+
+              await t.navigateTo(href);
+            }
+
             if (args.shadowDom) {
               await loadShadowDomExtension(t);
               await addShadowRootTree(t);
@@ -290,12 +375,10 @@ async function main() {
           after: async (t: TestController) => {
             await clearTestPage(t);
 
-            if (args.retryFailed) {
-              // @ts-expect-error ts-errors
-              const { test, errs } = t.testRun;
-              if (errs && errs.length > 0) {
-                failedTests.add(test.name);
-              }
+            // @ts-expect-error ts-errors
+            const { test, errs } = t.testRun;
+            if (errs && errs.length > 0) {
+              failedTests.add(test.name);
             }
           },
         },
@@ -306,22 +389,26 @@ async function main() {
       runOptions.disableScreenshots = true;
     }
 
-    // First run - all tests
     const runner = createRunner(false);
     let failedCount = await retry(() => runner.run(runOptions), LAUNCH_RETRY_ATTEMPTS);
 
-    // Retry failed tests multiple times if enabled and there are failures
     if (args.retryFailed && failedTests.size > 0 && failedCount > 0) {
       const initialFailedCount = failedTests.size;
-      let attemptsLeft = FAILED_TESTS_RETRY_ATTEMPTS;
+      let attemptsLeft = retryAttempts;
 
       while (attemptsLeft > 0 && failedCount > 0) {
-        const attemptNumber = FAILED_TESTS_RETRY_ATTEMPTS - attemptsLeft + 1;
+        if (failedTests.size === 0) {
+          // eslint-disable-next-line no-console
+          console.info('Failed tests could not be identified (hook failures) - skipping retries.');
+          break;
+        }
+
+        const attemptNumber = retryAttempts - attemptsLeft + 1;
 
         /* eslint-disable no-console */
         console.info('\n');
         console.info('='.repeat(60));
-        console.info(`RETRY ATTEMPT ${attemptNumber}/${FAILED_TESTS_RETRY_ATTEMPTS}`);
+        console.info(`RETRY ATTEMPT ${attemptNumber}/${retryAttempts}`);
         console.info(`Retrying ${failedTests.size} failed test(s)`);
         console.info('='.repeat(60));
         console.info('Failed tests:');
@@ -369,14 +456,33 @@ async function main() {
       console.info('FINAL RETRY RESULTS');
       console.info('='.repeat(60));
       console.info(`Initially failed: ${initialFailedCount}`);
-      console.info(`Total retry attempts used: ${FAILED_TESTS_RETRY_ATTEMPTS - attemptsLeft}`);
+      console.info(`Total retry attempts used: ${retryAttempts - attemptsLeft}`);
       console.info(`Final failing count: ${failedCount}`);
       console.info(`Successfully recovered: ${initialFailedCount - failedCount}`);
       console.info('='.repeat(60));
       console.info('\n');
       /* eslint-enable no-console */
     }
 
+    if (args.reportFailures) {
+      writeTestsReport(args.reportFailures, failedTests, failedCount);
+    }
+
+    if (args.reportFlaky && failedCount === 1 && failedTests.size === 1) {
+      writeTestsReport(args.reportFlaky, failedTests);
+
+      /* eslint-disable no-console */
+      console.info('\n');
+      console.info('='.repeat(60));
+      console.info(`Single failed test "${[...failedTests][0]}" is treated as flaky.`);
+      console.info('It will be rerun in the "quarantine" job.');
+      console.info('='.repeat(60));
+      console.info('\n');
+      /* eslint-enable no-console */
+
+      failedCount = 0;
+    }
+
     await testCafe.close();
 
     process.exit(failedCount);

e2e/testcafe-devextreme/tests/common/pivotGrid/fieldPanel/T1283238_OLAP_drag_and_drop_field.ts

@@ -1,17 +1,23 @@
 import { createScreenshotsComparer } from 'devextreme-screenshot-comparer';
 import PivotGrid from 'devextreme-testcafe-models/pivotGrid';
+import { ClientFunction } from 'testcafe';
 import { createWidget } from '../../../../helpers/createWidget';
 import url from '../../../../helpers/getPageUrl';
 import { testScreenshot } from '../../../../helpers/themeUtils';
 import { OLAPApiMock } from '../apiMocks/OLAP_api.mock';
 
+const blurActiveElement = ClientFunction(() => {
+  (document.activeElement as HTMLElement | null)?.blur();
+});
+
 fixture.disablePageReloads`pivotGrid_olap_drag-n-drop`
-  .page(url(__dirname, '../../container.html'));
+  .page(url(__dirname, '../../../container.html'))
+  .requestHooks(OLAPApiMock);
 
 const PIVOT_GRID_SELECTOR = '#container';
 
 [true, false].forEach((showRowGrandTotals) => {
-  test.meta({ unstable: true })(`Empty table has one ${showRowGrandTotals ? 'total' : 'empty'} row after drag-n-drop for paginated data`, async (t) => {
+  test(`Empty table has one ${showRowGrandTotals ? 'total' : 'empty'} row after drag-n-drop for paginated data`, async (t) => {
     const { takeScreenshot, compareResults } = createScreenshotsComparer(t);
 
     const pivotGrid = new PivotGrid(PIVOT_GRID_SELECTOR);
@@ -28,6 +34,8 @@ const PIVOT_GRID_SELECTOR = '#container';
     );
     await t.expect(loadPanel.isInvisible()).ok();
 
+    await blurActiveElement();
+
     await testScreenshot(
       t,
       takeScreenshot,
@@ -37,8 +45,7 @@ const PIVOT_GRID_SELECTOR = '#container';
 
     await t.expect(compareResults.isValid())
       .ok(compareResults.errorMessages());
-  }).before(async (t) => {
-    await t.addRequestHooks(OLAPApiMock);
+  }).before(async () => {
     await createWidget('dxPivotGrid', {
       height: 500,
       fieldPanel: { visible: true },

e2e/testcafe-devextreme/tests/dataGrid/common/aiColumn/virtualScrolling.functional.ts

@@ -8,6 +8,10 @@ fixture`Ai Column.Virtual Scrolling.Functional`
 
 const DATA_GRID_SELECTOR = '#container';
 
+const CLASS = {
+  loadPanelContent: 'dx-loadpanel-content',
+};
+
 const checkAIColumnTexts = async (
   t: TestController,
   component: DataGrid,
@@ -41,13 +45,13 @@ const deleteGlobalVariables = ClientFunction((): void => {
   delete (window as any).aiResolve;
 });
 
-test.meta({ unstable: true })('DataGrid should send an AI request for rendered rows after scrolling without changing the page index', async (t) => {
+test('DataGrid should send an AI request for rendered rows after scrolling without changing the page index', async (t) => {
   // arrange
   const dataGrid = new DataGrid(DATA_GRID_SELECTOR);
 
   // assert
   await t
-    .expect(dataGrid.getLoadPanel().isVisible())
+    .expect(dataGrid.element.find(`.${CLASS.loadPanelContent}`).visible)
     .ok();
 
   // act
@@ -57,7 +61,7 @@ test.meta({ unstable: true })('DataGrid should send an AI request for rendered r
   await t
     .expect(dataGrid.isReady())
     .ok()
-    .expect(dataGrid.getLoadPanel().isVisible())
+    .expect(dataGrid.element.find(`.${CLASS.loadPanelContent}`).visible)
     .notOk();
   await checkAIColumnTexts(t, dataGrid, 11);
 
@@ -72,7 +76,7 @@ test.meta({ unstable: true })('DataGrid should send an AI request for rendered r
     .eql(0)
     .expect(dataGrid.getDataCell(20, 0).element.textContent)
     .eql('21')
-    .expect(dataGrid.getLoadPanel().isVisible())
+    .expect(dataGrid.element.find(`.${CLASS.loadPanelContent}`).visible)
     .ok();
 
   // act
@@ -82,7 +86,7 @@ test.meta({ unstable: true })('DataGrid should send an AI request for rendered r
   await t
     .expect(dataGrid.isReady())
     .ok()
-    .expect(dataGrid.getLoadPanel().isVisible())
+    .expect(dataGrid.element.find(`.${CLASS.loadPanelContent}`).visible)
     .notOk();
   await checkAIColumnTexts(t, dataGrid, 12);
 })
@@ -144,13 +148,13 @@ test.meta({ unstable: true })('DataGrid should send an AI request for rendered r
     await deleteGlobalVariables();
   });
 
-test.meta({ unstable: true })('DataGrid should send an AI request for rendered rows after scrolling with changing the page index', async (t) => {
+test('DataGrid should send an AI request for rendered rows after scrolling with changing the page index', async (t) => {
   // arrange
   const dataGrid = new DataGrid(DATA_GRID_SELECTOR);
 
   // assert
   await t
-    .expect(dataGrid.getLoadPanel().isVisible())
+    .expect(dataGrid.element.find(`.${CLASS.loadPanelContent}`).visible)
     .ok();
 
   // act
@@ -160,7 +164,7 @@ test.meta({ unstable: true })('DataGrid should send an AI request for rendered r
   await t
     .expect(dataGrid.isReady())
     .ok()
-    .expect(dataGrid.getLoadPanel().isVisible())
+    .expect(dataGrid.element.find(`.${CLASS.loadPanelContent}`).visible)
     .notOk();
   await checkAIColumnTexts(t, dataGrid, 11);
 
@@ -175,7 +179,7 @@ test.meta({ unstable: true })('DataGrid should send an AI request for rendered r
     .eql(1)
     .expect(dataGrid.getDataCell(20, 0).element.textContent)
     .eql('21')
-    .expect(dataGrid.getLoadPanel().isVisible())
+    .expect(dataGrid.element.find(`.${CLASS.loadPanelContent}`).visible)
     .ok();
 
   // act
@@ -185,7 +189,7 @@ test.meta({ unstable: true })('DataGrid should send an AI request for rendered r
   await t
     .expect(dataGrid.isReady())
     .ok()
-    .expect(dataGrid.getLoadPanel().isVisible())
+    .expect(dataGrid.element.find(`.${CLASS.loadPanelContent}`).visible)
     .notOk();
   await checkAIColumnTexts(t, dataGrid, 12);
 })