test: make integration suites assert deterministic outcomes

src/methods.ts

@@ -1,6 +1,6 @@
 import { ux } from './utils/progress';
 import { createHash } from 'node:crypto';
-import { createReadStream, readdirSync, writeFileSync } from 'node:fs';
+import { createReadStream, mkdirSync, readdirSync, writeFileSync } from 'node:fs';
 import { access, readFile } from 'node:fs/promises';
 import * as path from 'node:path';
 import * as StreamZip from 'node-stream-zip';
@@ -1064,6 +1064,10 @@ export const writeJSONFile = (
   logger: { log: (message: string) => void; warn: (message: string) => void },
 ) => {
   try {
+    const directory = path.dirname(filePath);
+    if (directory !== '.') {
+      mkdirSync(directory, { recursive: true });
+    }
     writeFileSync(filePath, JSON.stringify(data, null, 2));
     logger.log(colors.dim('JSON output written to: ') + colors.highlight(path.resolve(filePath)));
   } catch (error) {

test/integration/artifacts.integration.test.ts

@@ -1,147 +1,156 @@
 import { expect } from 'chai';
-import { exec as execCallback } from 'node:child_process';
-import { promisify } from 'node:util';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+
+import {
+  CLI,
+  DEAD_API_URL,
+  MOCK_API_KEY,
+  MOCK_API_URL,
+  exec,
+  runExpectingFailure,
+} from './helpers';
 
-const exec = promisify(execCallback);
+describe('Artifacts Command Integration Tests', () => {
+  const mockApiUrl = MOCK_API_URL;
+  const mockApiKey = MOCK_API_KEY;
+  const mockUploadId = '123e4567-e89b-12d3-a456-426614174000';
+  // Downloads write into cwd, so keep them out of the repo tree.
+  let tempDir: string;
 
-const run = (args: string, env?: Record<string, string>) =>
-  exec(`./dist/index.js artifacts ${args}`, {
-    env: { ...process.env, ...env },
-    timeout: 15_000,
+  before(() => {
+    tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dcd-artifacts-test-'));
   });
 
-const errorOutput = (error: unknown): string => {
-  if (error && typeof error === 'object') {
-    if ('stderr' in error && typeof error.stderr === 'string' && error.stderr) return error.stderr;
-    if ('stdout' in error && typeof error.stdout === 'string' && error.stdout) return error.stdout;
-  }
+  after(() => {
+    if (fs.existsSync(tempDir)) {
+      fs.rmSync(tempDir, { force: true, recursive: true });
+    }
+  });
 
-  return '';
-};
+  const run = (args: string, env?: Record<string, string>) =>
+    exec(`${CLI} artifacts ${args}`, {
+      cwd: tempDir,
+      env: { ...process.env, ...env },
+      timeout: 15_000,
+    });
 
-describe('Artifacts Command Integration Tests', () => {
-  const mockApiUrl = 'http://localhost:3001';
-  const mockApiKey = 'test-api-key-123';
-  const mockUploadId = '123e4567-e89b-12d3-a456-426614174000';
+  const runFailing = (args: string, env?: Record<string, string>) =>
+    runExpectingFailure(`${CLI} artifacts ${args}`, {
+      cwd: tempDir,
+      env: { ...process.env, ...env },
+    });
 
   describe('flag validation', () => {
     it('should require --upload-id', async () => {
-      try {
-        await run(`--download-artifacts FAILED --api-key ${mockApiKey} --api-url ${mockApiUrl}`);
-        expect.fail('Should have thrown');
-      } catch (error) {
-        expect(errorOutput(error)).to.match(/upload-id/i);
-      }
+      const { output } = await runFailing(
+        `--download-artifacts FAILED --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
+      );
+      expect(output).to.match(/upload-id/i);
     });
 
     it('should require either --download-artifacts or --report', async () => {
-      try {
-        await run(`--upload-id ${mockUploadId} --api-key ${mockApiKey} --api-url ${mockApiUrl}`);
-        expect.fail('Should have thrown');
-      } catch (error) {
-        expect(errorOutput(error)).to.match(/download-artifacts|report/i);
-      }
+      const { output } = await runFailing(
+        `--upload-id ${mockUploadId} --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
+      );
+      expect(output).to.match(/download-artifacts|report/i);
     });
 
     it('should reject --download-artifacts and --report together', async () => {
-      try {
-        await run(
-          `--upload-id ${mockUploadId} --download-artifacts FAILED --report junit --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
-        );
-        expect.fail('Should have thrown');
-      } catch (error) {
-        expect(errorOutput(error)).to.match(/cannot also be provided/i);
-      }
+      const { output } = await runFailing(
+        `--upload-id ${mockUploadId} --download-artifacts FAILED --report junit --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
+      );
+      expect(output).to.match(/cannot also be provided/i);
     });
 
     it('should reject --artifacts-path without --download-artifacts', async () => {
-      try {
-        await run(
-          `--upload-id ${mockUploadId} --artifacts-path ./out.zip --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
-        );
-        expect.fail('Should have thrown');
-      } catch (error) {
-        expect(errorOutput(error)).to.match(/artifacts-path|download-artifacts/i);
-      }
+      const { output } = await runFailing(
+        `--upload-id ${mockUploadId} --artifacts-path ./out.zip --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
+      );
+      expect(output).to.match(/artifacts-path|download-artifacts/i);
     });
 
     it('should reject --junit-path without --report', async () => {
-      try {
-        await run(
-          `--upload-id ${mockUploadId} --junit-path ./report.xml --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
-        );
-        expect.fail('Should have thrown');
-      } catch (error) {
-        expect(errorOutput(error)).to.match(/junit-path|report/i);
-      }
+      const { output } = await runFailing(
+        `--upload-id ${mockUploadId} --junit-path ./report.xml --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
+      );
+      expect(output).to.match(/junit-path|report/i);
     });
 
     it('should only accept ALL or FAILED for --download-artifacts', async () => {
-      try {
-        await run(
-          `--upload-id ${mockUploadId} --download-artifacts SOME --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
-        );
-        expect.fail('Should have thrown');
-      } catch (error) {
-        expect(errorOutput(error)).to.match(/all|failed|expected.*to be one of/i);
-      }
+      const { output } = await runFailing(
+        `--upload-id ${mockUploadId} --download-artifacts SOME --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
+      );
+      expect(output).to.match(/all|failed|expected.*to be one of/i);
     });
   });
 
   describe('authentication', () => {
     it('should require an API key', async () => {
-      try {
-        await run(
-          `--upload-id ${mockUploadId} --download-artifacts FAILED --api-url ${mockApiUrl}`,
-          { DEVICE_CLOUD_API_KEY: '' },
-        );
-        expect.fail('Should have thrown');
-      } catch (error) {
-        expect(errorOutput(error)).to.match(/api key/i);
-      }
+      const { output } = await runFailing(
+        `--upload-id ${mockUploadId} --download-artifacts FAILED --api-url ${mockApiUrl}`,
+        { DEVICE_CLOUD_API_KEY: '' },
+      );
+      expect(output).to.match(/api key/i);
     });
 
     it('should accept API key from environment variable', async () => {
-      try {
-        await run(
-          `--upload-id ${mockUploadId} --download-artifacts FAILED --api-url ${mockApiUrl}`,
-          { DEVICE_CLOUD_API_KEY: mockApiKey },
-        );
-      } catch (error) {
-        // API unreachable is fine — the key was accepted if we don't see the key error
-        expect(errorOutput(error)).to.not.match(/api key is required/i);
-      }
+      // Download failures against the mock are warnings, not errors — the
+      // command exits 0 once the key is accepted (see download behaviour below).
+      const { stderr } = await run(
+        `--upload-id ${mockUploadId} --download-artifacts FAILED --api-url ${mockApiUrl}`,
+        { DEVICE_CLOUD_API_KEY: mockApiKey },
+      );
+      expect(stderr).to.not.match(/api key is required/i);
+    });
+  });
+
+  describe('download behaviour against the mock API', () => {
+    // Prism can't serve the binary download endpoints, so the deterministic
+    // outcome is a warning and exit 0 — download failures must not fail the
+    // command or crash.
+    it('should warn and exit 0 when artifacts download fails', async () => {
+      const { stderr } = await run(
+        `--upload-id ${mockUploadId} --download-artifacts FAILED --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
+      );
+      expect(stderr).to.include('Failed to download artifacts');
+    });
+
+    it('should download the junit report', async () => {
+      // Unlike the artifacts zip, Prism can serve the junit report endpoint.
+      const { stdout } = await run(
+        `--upload-id ${mockUploadId} --report junit --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
+      );
+      expect(stdout).to.include('JUNIT test report has been downloaded');
+      expect(fs.existsSync(path.join(tempDir, 'report.xml'))).to.be.true;
     });
   });
 
   describe('network error handling', () => {
     it('should handle unreachable API gracefully for --download-artifacts', async () => {
-      try {
-        await run(
-          `--upload-id ${mockUploadId} --download-artifacts FAILED --api-key ${mockApiKey} --api-url http://localhost:9999`,
-        );
-      } catch (error) {
-        // Should warn, not crash with an uncaught exception
-        const out = errorOutput(error);
-        expect(out).to.not.match(/typeerror|unhandledpromiserejection/i);
-      }
+      // Should warn, not crash with an uncaught exception
+      const { stderr, stdout } = await run(
+        `--upload-id ${mockUploadId} --download-artifacts FAILED --api-key ${mockApiKey} --api-url ${DEAD_API_URL}`,
+      );
+      expect(stderr + stdout).to.not.match(/typeerror|unhandledpromiserejection/i);
+      expect(stderr).to.include('Failed to download artifacts');
     });
 
     it('should handle unreachable API gracefully for --report', async () => {
-      try {
-        await run(
-          `--upload-id ${mockUploadId} --report junit --api-key ${mockApiKey} --api-url http://localhost:9999`,
-        );
-      } catch (error) {
-        const out = errorOutput(error);
-        expect(out).to.not.match(/typeerror|unhandledpromiserejection/i);
-      }
+      const { stderr, stdout } = await run(
+        `--upload-id ${mockUploadId} --report junit --api-key ${mockApiKey} --api-url ${DEAD_API_URL}`,
+      );
+      expect(stderr + stdout).to.not.match(/typeerror|unhandledpromiserejection/i);
+      expect(stderr).to.match(/failed to download/i);
     });
   });
 
   describe('help', () => {
     it('should display help with all expected flags', async () => {
-      const { stdout } = await exec('./dist/index.js artifacts --help', { timeout: 10_000 });
+      const { stdout } = await exec(`${CLI} artifacts --help`, {
+        timeout: 10_000,
+      });
       expect(stdout).to.include('--upload-id');
       expect(stdout).to.include('--download-artifacts');
       expect(stdout).to.include('--artifacts-path');