Add App Doctor scan command

.changeset/bright-doctors-scan.md

@@ -0,0 +1,5 @@
+---
+'@shopify/app': minor
+---
+
+Add `shopify app doctor scan` for Shopify-specific security reviews.

packages/app/package.json

@@ -42,6 +42,7 @@
   "scripts": {
     "build": "nx build",
     "clean": "nx clean",
+    "generate:app-doctor-checks": "node src/cli/services/app-doctor-engine/embed-checks.mjs",
     "lint": "nx lint",
     "lint:fix": "nx lint:fix",
     "prepack": "NODE_ENV=production pnpm nx build && cp ../../README.md README.md",
@@ -55,6 +56,7 @@
   },
   "dependencies": {
     "@graphql-typed-document-node/core": "3.2.0",
+    "@iarna/toml": "2.2.5",
     "@luckycatfactory/esbuild-graphql-loader": "3.8.1",
     "@oclif/core": "4.8.3",
     "@shopify/cli-kit": "4.7.0",
@@ -63,9 +65,12 @@
     "@shopify/theme": "4.7.0",
     "@shopify/theme-check-node": "3.29.0",
     "@shopify/toml-patch": "0.3.0",
+    "acorn": "8.17.0",
+    "acorn-walk": "8.3.5",
     "chokidar": "3.6.0",
     "diff": "5.2.2",
     "esbuild": "0.28.1",
+    "fast-glob": "3.3.3",
     "graphql-request": "6.1.0",
     "h3": "1.15.11",
     "http-proxy-node16": "1.0.6",

packages/app/src/cli/commands/app/doctor/scan.test.ts

@@ -0,0 +1,70 @@
+import DoctorScan from './scan.js'
+import doctor from '../../../services/doctor.js'
+import AppLinkedCommand from '../../../utilities/app-linked-command.js'
+import BaseCommand from '@shopify/cli-kit/node/base-command'
+import {resolvePath} from '@shopify/cli-kit/node/path'
+import {describe, expect, test, vi} from 'vitest'
+
+vi.mock('../../../services/doctor.js')
+
+describe('app doctor scan command', () => {
+  test('does not require linked app context', () => {
+    expect(DoctorScan.prototype).toBeInstanceOf(BaseCommand)
+    expect(DoctorScan.prototype).not.toBeInstanceOf(AppLinkedCommand)
+  })
+
+  test('forwards the directory and flags to the service', async () => {
+    await DoctorScan.run(
+      ['./fixtures/unlinked-app', '--json', '--verbose', '--blocking', 'high', '--skip-skill'],
+      import.meta.url,
+    )
+
+    expect(doctor).toHaveBeenCalledWith({
+      directory: resolvePath('./fixtures/unlinked-app'),
+      json: true,
+      verbose: true,
+      blocking: 'high',
+      yes: false,
+      skipSkill: true,
+      findingsPath: undefined,
+    })
+  })
+
+  test('forwards --yes without requiring an app configuration', async () => {
+    await DoctorScan.run(['/tmp/directory-without-shopify-toml', '--yes'], import.meta.url)
+
+    expect(doctor).toHaveBeenCalledWith({
+      directory: '/tmp/directory-without-shopify-toml',
+      json: false,
+      verbose: false,
+      blocking: 'none',
+      yes: true,
+      skipSkill: false,
+      findingsPath: undefined,
+    })
+  })
+
+  test('resolves and forwards an agent findings file', async () => {
+    await DoctorScan.run(['.', '--findings', './findings.json', '--skip-skill'], import.meta.url)
+
+    expect(doctor).toHaveBeenCalledWith(expect.objectContaining({findingsPath: resolvePath('./findings.json')}))
+  })
+
+  test('describes --yes as showing instructions and keeps it mutually exclusive with --skip-skill', () => {
+    expect(DoctorScan.flags.yes.description).toBe(
+      'Show optional App Doctor skill setup instructions without prompting.',
+    )
+    expect(DoctorScan.flags['skip-skill'].description).toBe("Don't offer App Doctor skill setup instructions.")
+    expect(DoctorScan.flags.yes.exclusive).toEqual(['skip-skill'])
+    expect(DoctorScan.flags['skip-skill'].exclusive).toEqual(['yes'])
+    expect(DoctorScan.descriptionWithMarkdown).toContain(
+      "Shopify CLI only shows instructions; it doesn't install or configure the skill.",
+    )
+  })
+
+  test('allows --yes in JSON mode while preserving non-interactive output behavior', async () => {
+    await DoctorScan.run(['--json', '--yes'], import.meta.url)
+
+    expect(doctor).toHaveBeenCalledWith(expect.objectContaining({json: true, yes: true}))
+  })
+})

packages/app/src/cli/commands/app/doctor/scan.ts

@@ -0,0 +1,67 @@
+import doctor from '../../../services/doctor.js'
+import {Args, Flags} from '@oclif/core'
+import BaseCommand from '@shopify/cli-kit/node/base-command'
+import {globalFlags, jsonFlag} from '@shopify/cli-kit/node/cli'
+import {cwd, resolvePath} from '@shopify/cli-kit/node/path'
+import type {AppDoctorBlockingLevel} from '../../../services/app-doctor-api.js'
+
+const blockingLevels: AppDoctorBlockingLevel[] = ['critical', 'high', 'medium', 'low', 'none']
+
+export default class DoctorScan extends BaseCommand {
+  static summary = 'Check an app for Shopify-specific security issues.'
+
+  static descriptionWithMarkdown = `Runs Shopify App Doctor locally and creates its review pack and trace.
+
+Pass \`--findings\` after completing the review pack to validate agent findings and compile them into the trace. In CI and other non-interactive environments, skill setup instructions aren't offered unless you pass \`--yes\`. JSON output never prompts or prints those instructions. Shopify CLI only shows instructions; it doesn't install or configure the skill.`
+
+  static description = this.descriptionWithoutMarkdown()
+
+  static args = {
+    directory: Args.string({
+      description: 'The app directory to check. Defaults to the current directory.',
+      parse: async (input) => resolvePath(input),
+    }),
+  }
+
+  static flags = {
+    ...globalFlags,
+    ...jsonFlag,
+    findings: Flags.string({
+      description: 'Validate agent findings from a JSON file and compile them into the trace.',
+      parse: async (input) => resolvePath(input),
+      env: 'SHOPIFY_FLAG_APP_DOCTOR_FINDINGS',
+    }),
+    blocking: Flags.string({
+      description: 'The minimum finding severity that causes a non-zero exit code.',
+      options: blockingLevels,
+      default: 'none',
+      env: 'SHOPIFY_FLAG_APP_DOCTOR_BLOCKING',
+    }),
+    yes: Flags.boolean({
+      description: 'Show optional App Doctor skill setup instructions without prompting.',
+      default: false,
+      exclusive: ['skip-skill'],
+      env: 'SHOPIFY_FLAG_YES',
+    }),
+    'skip-skill': Flags.boolean({
+      description: "Don't offer App Doctor skill setup instructions.",
+      default: false,
+      exclusive: ['yes'],
+      env: 'SHOPIFY_FLAG_SKIP_SKILL',
+    }),
+  }
+
+  public async run(): Promise<void> {
+    const {args, flags} = await this.parse(DoctorScan)
+
+    await doctor({
+      directory: args.directory ?? cwd(),
+      json: flags.json,
+      verbose: Boolean(flags.verbose),
+      blocking: flags.blocking as AppDoctorBlockingLevel,
+      yes: flags.yes,
+      skipSkill: flags['skip-skill'],
+      findingsPath: flags.findings,
+    })
+  }
+}

packages/app/src/cli/index.test.ts

@@ -0,0 +1,9 @@
+import {commands} from './index.js'
+import DoctorScan from './commands/app/doctor/scan.js'
+import {describe, expect, test} from 'vitest'
+
+describe('@shopify/app command registration', () => {
+  test('registers app:doctor:scan', () => {
+    expect(commands['app:doctor:scan']).toBe(DoctorScan)
+  })
+})

packages/app/src/cli/index.ts

@@ -7,6 +7,7 @@ import ConfigPull from './commands/app/config/pull.js'
 import DemoWatcher from './commands/app/demo/watcher.js'
 import Deploy from './commands/app/deploy.js'
 import Dev from './commands/app/dev.js'
+import DoctorScan from './commands/app/doctor/scan.js'
 import Logs from './commands/app/logs.js'
 import Sources from './commands/app/app-logs/sources.js'
 import EnvPull from './commands/app/env/pull.js'
@@ -48,6 +49,7 @@ export const commands: {[key: string]: typeof AppLinkedCommand | typeof AppUnlin
   'app:deploy': Deploy,
   'app:dev': Dev,
   'app:dev:clean': DevClean,
+  'app:doctor:scan': DoctorScan,
   'app:logs': Logs,
   'app:logs:sources': Sources,
   'app:import-custom-data-definitions': ImportCustomDataDefinitions,

packages/app/src/cli/services/app-doctor-api.test.ts

@@ -0,0 +1,91 @@
+import {runAppDoctor} from './app-doctor-api.js'
+import {loadChecks} from './app-doctor-engine/index.js'
+import {inTemporaryDirectory, mkdir, readFile, writeFile} from '@shopify/cli-kit/node/fs'
+import {joinPath} from '@shopify/cli-kit/node/path'
+import {describe, expect, test} from 'vitest'
+
+async function createApp(directory: string, source = 'export const loader = () => ({ok: true})'): Promise<string> {
+  const sourceDirectory = joinPath(directory, 'app', 'routes')
+  const sourcePath = joinPath(sourceDirectory, 'index.ts')
+  await mkdir(sourceDirectory)
+  await writeFile(joinPath(directory, 'shopify.app.toml'), 'name = "Test app"\nclient_id = "test"\n')
+  await writeFile(joinPath(directory, 'package.json'), '{"name":"test-app"}\n')
+  await writeFile(sourcePath, source)
+  return sourcePath
+}
+
+describe('App Doctor CLI integration', () => {
+  test('runs the in-tree engine and writes the review pack and trace', async () => {
+    await inTemporaryDirectory(async (directory) => {
+      await createApp(directory)
+
+      const result = await runAppDoctor({directory, format: 'human', verbose: true, blocking: 'none'})
+      const review = JSON.parse(await readFile(joinPath(directory, 'app-doctor-review.json')))
+      const trace = JSON.parse(await readFile(joinPath(directory, 'app-doctor-trace.json')))
+
+      expect(review.checks).toHaveLength(16)
+      expect(review.checks.every((check: {prompt: string}) => check.prompt.length > 0)).toBe(true)
+      expect(trace.schema_version).toBe(1)
+      expect(trace.engine.name).toBe('shopify-app-doctor')
+      expect(result.engine).toEqual(trace.engine)
+      expect(result.output).toContain('shopify app doctor scan --findings <findings.json>')
+      expect(result.exitCode).toBe(0)
+    })
+  })
+
+  test('preserves JSON output and applies the requested blocking severity', async () => {
+    await inTemporaryDirectory(async (directory) => {
+      const testToken = ['shpat', '0123456789abcdef0123456789abcdef'].join('_')
+      await createApp(directory, `const access_token = "${testToken}"`)
+
+      const result = await runAppDoctor({directory, format: 'json', verbose: false, blocking: 'high'})
+
+      expect(() => JSON.parse(result.output)).not.toThrow()
+      expect(result.output).not.toContain(testToken)
+      expect(result.exitCode).toBe(1)
+    })
+  })
+
+  test('validates agent findings and compiles them into the trace', async () => {
+    await inTemporaryDirectory(async (directory) => {
+      await createApp(directory)
+      const check = loadChecks().get('MISSING_TENANT_ISOLATION')!
+      const findingsPath = joinPath(directory, 'findings.json')
+      await writeFile(
+        findingsPath,
+        `${JSON.stringify({
+          checks_executed: [{check_id: check.id, check_version: check.version, prompt_hash: check.prompt_hash}],
+          findings: [
+            {
+              check_id: check.id,
+              check_version: check.version,
+              prompt_hash: check.prompt_hash,
+              file: 'app/routes/index.ts',
+              line: 1,
+              message: 'The query is not scoped to the current shop.',
+              evidence: [{file: 'app/routes/index.ts', line: 1, quote: 'loader'}],
+            },
+          ],
+        })}\n`,
+      )
+
+      const result = await runAppDoctor({
+        directory,
+        findingsPath,
+        format: 'json',
+        verbose: false,
+        blocking: 'none',
+      })
+      const trace = JSON.parse(result.output)
+
+      expect(trace.findings).toEqual(
+        expect.arrayContaining([expect.objectContaining({source: 'agent', check_id: 'MISSING_TENANT_ISOLATION'})]),
+      )
+      expect(trace.checks_executed).toEqual(
+        expect.arrayContaining([expect.objectContaining({id: 'MISSING_TENANT_ISOLATION', status: 'executed'})]),
+      )
+      expect(JSON.parse(await readFile(joinPath(directory, 'app-doctor-trace.json')))).toEqual(trace)
+      expect(result.exitCode).toBe(0)
+    })
+  })
+})