diff --git a/docs/adr/003-runner-storage-provider-boundary.md b/docs/adr/003-runner-storage-provider-boundary.md new file mode 100644 index 0000000000..15e18918fa --- /dev/null +++ b/docs/adr/003-runner-storage-provider-boundary.md @@ -0,0 +1,124 @@ +# ADR-003: Runner Storage Provider Boundary + +## Status + +Proposed + +## Date + +2026-08-24 + +## Context + +The runner control plane uses AWS Systems Manager Parameter Store for several +unrelated purposes: + +- durable GitHub App credentials and webhook secrets; +- durable webhook matcher and runner configuration; +- short-lived registration tokens and just-in-time runner configuration; +- a rebuildable runner-group ID cache; +- compute-provider-specific values such as an EC2 AMI ID; and +- EC2 Systems Manager access, which is not a storage concern. + +Treating all of these uses as one generic key/value provider would erase their +different confidentiality, ownership, retention, consistency, and cleanup +requirements. It would also make a future backend inherit operations that it +does not need. For example, a runner-bootstrap backend must write a sensitive +payload for one runner, while the runner-group cache stores a non-secret value +that can be rebuilt from GitHub. + +The existing control-plane implementation calls the shared SSM utility +directly for both runtime-created runner payloads and runner-group cache +entries. That couples orchestration logic to Parameter Store and its write-rate +behavior. + +## Decision + +Storage is divided by usage capability. This change introduces two initial +contracts: + +- `RunnerBootstrapStore` writes the short-lived, sensitive registration or JIT + payload consumed by one runner. +- `RunnerGroupCacheStore` reads and writes the rebuildable GitHub runner-group + ID cache. + +The contracts expose only the operations needed by their consumers. They do +not expose a generic `get`, `put`, or `delete` API shared across all storage +uses. Provider implementations own backend-specific path construction, +serialization, encryption selection, tags, throughput guidance, SDK calls, and +errors. + +The first registered implementation is `aws_ssm`. Scale-up and pool select it +independently through +`RUNNER_BOOTSTRAP_STORAGE_PROVIDER_TYPE` and +`RUNNER_GROUP_CACHE_STORAGE_PROVIDER_TYPE`. Independent selection prevents a +future cache or bootstrap migration from silently moving both data classes. + +The SSM implementation preserves the existing behavior: + +- runner payloads remain `SecureString` parameters at + `/`; +- runner-group IDs remain `String` parameters at + `/runner-group/`; +- the existing parameter tags and per-runner metadata tags are preserved; and +- the existing SSM write-rate guidance continues to pace burst writes. + +No Terraform-managed SSM resource, resource address, path, IAM policy, KMS +behavior, runner bootstrap reader, or housekeeper is moved by this decision. +The stable and experimental Terraform paths only add explicit provider +selection to the existing Lambda environments. + +### Package ownership + +The runtime implementation is organized as follows: + +```text +lambdas/libs/storage-providers/ +├── core/ # usage contracts and provider registry +├── provider-types.ts # supported provider identifiers and default resolution +└── aws/ssm.ts # Parameter Store implementation +``` + +Shared orchestration consumes the usage contracts. The SSM package owns calls +to `aws-ssm-util`; compute- and orchestration-provider packages do not acquire +new SSM parsing or IAM behavior. + +### Deferred storage uses + +The following SSM uses remain unchanged and are deliberately not forced behind +the initial contracts: + +- GitHub credentials and webhook secrets need a secret-resolution contract. +- Matcher configuration, persistent runner configuration, and controller + manifests need owner-specific durable configuration contracts. +- EC2 AMI Parameter Store resolution remains compute-provider-specific because + EC2 consumes the SSM reference directly. +- Systems Manager access to runner instances remains an EC2 capability rather + than a storage provider. +- Reading and deleting the bootstrap payload remains in the existing runner + bootstrap implementation until that protocol can be migrated without + changing images or runtime compatibility. + +A later backend, including DynamoDB, must implement and test only the usage +capabilities it supports. It must not broaden these contracts into an +unrestricted storage API. + +## Consequences + +### Positive + +- Runtime-created runner payloads and rebuildable cache entries have separate, + testable semantics. +- Existing SSM resources and operational behavior remain stable. +- A later backend can be introduced for one usage without changing unrelated + credentials, configuration, or compute-provider behavior. +- Sensitive runner payloads remain encrypted and are not added to logs or + provider configuration. + +### Negative + +- The initial provider boundary covers only the control-plane writer and cache; + runner-side consumption is still SSM-specific. +- Two provider selectors are more verbose than one global storage selector. +- Additional usage contracts will be needed before other SSM responsibilities + can migrate. diff --git a/lambdas/functions/control-plane/package.json b/lambdas/functions/control-plane/package.json index dc74e770a9..0f443fc849 100644 --- a/lambdas/functions/control-plane/package.json +++ b/lambdas/functions/control-plane/package.json @@ -33,6 +33,7 @@ "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", "@aws-github-runner/compute-providers": "*", + "@aws-github-runner/storage-providers": "*", "@aws-lambda-powertools/parameters": "^2.31.0", "@aws-sdk/client-ec2": "^3.1009.0", "@aws-sdk/client-sqs": "^3.1009.0", diff --git a/lambdas/functions/control-plane/src/modules.d.ts b/lambdas/functions/control-plane/src/modules.d.ts index d5157ccb37..801a8997da 100644 --- a/lambdas/functions/control-plane/src/modules.d.ts +++ b/lambdas/functions/control-plane/src/modules.d.ts @@ -16,6 +16,8 @@ declare namespace NodeJS { PARAMETER_GITHUB_APP_ID_NAME: string; PARAMETER_GITHUB_APP_KEY_BASE64_NAME: string; RUNNER_OWNER: string; + RUNNER_BOOTSTRAP_STORAGE_PROVIDER_TYPE?: string; + RUNNER_GROUP_CACHE_STORAGE_PROVIDER_TYPE?: string; COMPUTE_PROVIDER_TYPE?: string; SCALE_DOWN_CONFIG: string; SSM_TOKEN_PATH: string; diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts index 745c66770a..f1ca1c9ace 100644 --- a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts @@ -1,5 +1,9 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util'; +import { + createRunnerBootstrapStoreFromEnvironment, + createRunnerGroupCacheStoreFromEnvironment, +} from '@aws-github-runner/storage-providers'; +import type { RunnerBootstrapStore, RunnerGroupCacheStore } from '@aws-github-runner/storage-providers'; import { Octokit } from '@octokit/rest'; import { getStoredInstallationId } from '../github/auth'; @@ -14,7 +18,7 @@ export interface GitHubRunnerMetadata { } export interface StartRunnerConfigOptions { - getSsmParameterTags?: (runnerId: string) => { Key: string; Value: string }[]; + getStorageMetadataTags?: (runnerId: string) => { key: string; value: string }[]; onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise; } @@ -182,39 +186,27 @@ export async function isJobQueued( export async function getRunnerGroupId( githubRunnerConfig: CreateGitHubRunnerConfig, ghClient: Octokit, + runnerGroupCacheStore: RunnerGroupCacheStore = createRunnerGroupCacheStore(githubRunnerConfig), ): Promise { // if the runnerType is Repo, then runnerGroupId is default to 1 let runnerGroupId: number | undefined = 1; if (githubRunnerConfig.runnerType === 'Org' && githubRunnerConfig.runnerGroup !== undefined) { let runnerGroup: string | undefined; - // check if runner group id is already stored in SSM Parameter Store and - // use it if it exists to avoid API call to GitHub + const identity = { kind: 'runner_group_cache' as const, groupName: githubRunnerConfig.runnerGroup }; + // Use the cached runner-group ID when available to avoid a GitHub API call. try { - runnerGroup = await getParameter( - `${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}`, - ); + runnerGroup = (await runnerGroupCacheStore.get(identity))?.payload; } catch (err) { logger.debug('Handling error:', err as Error); - logger.warn( - `SSM Parameter "${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}" - for Runner group ${githubRunnerConfig.runnerGroup} does not exist`, - ); + logger.warn(`Cached ID for runner group ${githubRunnerConfig.runnerGroup} does not exist`); } if (runnerGroup === undefined) { // get runner group id from GitHub runnerGroupId = await getRunnerGroupByName(ghClient, githubRunnerConfig); - // store runner group id in SSM try { - await putParameter( - `${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}`, - runnerGroupId.toString(), - false, - { - tags: githubRunnerConfig.ssmParameterStoreTags, - }, - ); + await runnerGroupCacheStore.put({ identity, payload: runnerGroupId.toString() }); } catch (err) { - logger.debug('Error storing runner group id in SSM Parameter Store', err as Error); + logger.debug('Error storing runner group ID in the cache', err as Error); throw err; } } else { @@ -250,18 +242,45 @@ export async function createStartRunnerConfig( ghClient: Octokit, options: StartRunnerConfigOptions = {}, ): Promise { + const runnerBootstrapStore = createRunnerBootstrapStore(githubRunnerConfig); if (githubRunnerConfig.enableJitConfig && githubRunnerConfig.ephemeral) { - return await createJitConfig(githubRunnerConfig, runnerIds, ghClient, options); + return await createJitConfig( + githubRunnerConfig, + runnerIds, + ghClient, + options, + runnerBootstrapStore, + createRunnerGroupCacheStore(githubRunnerConfig), + ); } else { - return await createRegistrationTokenConfig(githubRunnerConfig, runnerIds, ghClient, options); + return await createRegistrationTokenConfig(githubRunnerConfig, runnerIds, ghClient, options, runnerBootstrapStore); } } -function addDelay(runnerIds: string[]) { +function createRunnerBootstrapStore(githubRunnerConfig: CreateGitHubRunnerConfig): RunnerBootstrapStore { + return createRunnerBootstrapStoreFromEnvironment({ + locator: githubRunnerConfig.ssmTokenPath, + metadataTags: githubRunnerConfig.ssmParameterStoreTags.map(({ Key, Value }) => ({ key: Key, value: Value })), + }); +} + +function createRunnerGroupCacheStore(githubRunnerConfig: CreateGitHubRunnerConfig): RunnerGroupCacheStore { + return createRunnerGroupCacheStoreFromEnvironment({ + locator: githubRunnerConfig.ssmConfigPath, + metadataTags: githubRunnerConfig.ssmParameterStoreTags.map(({ Key, Value }) => ({ key: Key, value: Value })), + }); +} + +function addDelay(runnerIds: string[], runnerBootstrapStore: RunnerBootstrapStore) { const delay = async (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - const ssmParameterStoreMaxThroughput = 40; - const isDelay = runnerIds.length >= ssmParameterStoreMaxThroughput; - return { isDelay, delay }; + const maxWritesPerSecond = runnerBootstrapStore.maxWritesPerSecond; + const isDelay = maxWritesPerSecond !== undefined && runnerIds.length >= maxWritesPerSecond; + const delayMs = maxWritesPerSecond === undefined ? 0 : 1000 / maxWritesPerSecond; + return { isDelay, delay, delayMs }; +} + +function getRunnerBootstrapMetadataTags(options: StartRunnerConfigOptions, runnerId: string) { + return options.getStorageMetadataTags?.(runnerId) ?? []; } /** @@ -274,8 +293,9 @@ async function createRegistrationTokenConfig( runnerIds: string[], ghClient: Octokit, options: StartRunnerConfigOptions, + runnerBootstrapStore: RunnerBootstrapStore, ): Promise { - const { isDelay, delay } = addDelay(runnerIds); + const { isDelay, delay, delayMs } = addDelay(runnerIds, runnerBootstrapStore); const token = await getGithubRunnerRegistrationToken(githubRunnerConfig, ghClient); const runnerServiceConfig = generateRunnerServiceConfig(githubRunnerConfig, token); @@ -284,12 +304,13 @@ async function createRegistrationTokenConfig( }); for (const runnerId of runnerIds) { - await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerServiceConfig.join(' '), true, { - tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags], - }); + await runnerBootstrapStore.put( + { identity: { kind: 'runner_bootstrap', runnerId }, payload: runnerServiceConfig.join(' ') }, + { metadataTags: getRunnerBootstrapMetadataTags(options, runnerId) }, + ); if (isDelay) { // Delay to prevent AWS ssm rate limits by being within the max throughput limit - await delay(25); + await delay(delayMs); } } @@ -307,9 +328,11 @@ async function createJitConfig( runnerIds: string[], ghClient: Octokit, options: StartRunnerConfigOptions, + runnerBootstrapStore: RunnerBootstrapStore, + runnerGroupCacheStore: RunnerGroupCacheStore, ): Promise { - const runnerGroupId = await getRunnerGroupId(githubRunnerConfig, ghClient); - const { isDelay, delay } = addDelay(runnerIds); + const runnerGroupId = await getRunnerGroupId(githubRunnerConfig, ghClient, runnerGroupCacheStore); + const { isDelay, delay, delayMs } = addDelay(runnerIds, runnerBootstrapStore); const runnerLabels = githubRunnerConfig.runnerLabels.split(','); const failedRunnerIds: string[] = []; @@ -347,16 +370,16 @@ async function createJitConfig( runnerLabels, }); - // store jit config in ssm parameter store logger.debug('Runner JIT config for ephemeral runner generated.', { instance: runnerId, }); - await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerConfig.data.encoded_jit_config, true, { - tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags], - }); + await runnerBootstrapStore.put( + { identity: { kind: 'runner_bootstrap', runnerId }, payload: runnerConfig.data.encoded_jit_config }, + { metadataTags: getRunnerBootstrapMetadataTags(options, runnerId) }, + ); if (isDelay) { // Delay to prevent AWS ssm rate limits by being within the max throughput limit - await delay(25); + await delay(delayMs); } } catch (error) { failedRunnerIds.push(runnerId); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts index eb6899ff79..7863aca43e 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts @@ -168,7 +168,7 @@ async function createTestProviderRunners(input: CreateScaleUpRunnersInput [{ Key: 'RunnerId', Value: runnerId }], + getStorageMetadataTags: (runnerId) => [{ key: 'RunnerId', value: runnerId }], }, ); } catch { diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-config.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-config.ts index bdc46b8c44..9cc97fa416 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-config.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-config.ts @@ -141,7 +141,7 @@ async function terminateFailedInstances(instanceIds: string[]): Promise { function createEc2StartRunnerConfigOptions(): StartRunnerConfigOptions { return { - getSsmParameterTags: (instanceId) => [{ Key: 'InstanceId', Value: instanceId }], + getStorageMetadataTags: (instanceId) => [{ key: 'InstanceId', value: instanceId }], onJitConfigCreated: async (instanceId, metadata) => await tagEc2RunnerMetadata(instanceId, metadata), }; } diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts index 0c6bac69f4..6fa9c386e3 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts @@ -182,7 +182,7 @@ describe('scaleUp with GHES', () => { { Key: 'ghr:runner_labels', Value: 'label1,label2' }, ]); const [, , , options] = mockCreateStartRunnerConfig.mock.calls[0]; - expect(options?.getSsmParameterTags?.('i-12345')).toEqual([{ Key: 'InstanceId', Value: 'i-12345' }]); + expect(options?.getStorageMetadataTags?.('i-12345')).toEqual([{ key: 'InstanceId', value: 'i-12345' }]); }); it('chunks comma-joined GitHub runner labels by the EC2 tag value max length', async () => { diff --git a/lambdas/libs/compute-providers/core/index.ts b/lambdas/libs/compute-providers/core/index.ts index 2b5f937f36..f85bad64d5 100644 --- a/lambdas/libs/compute-providers/core/index.ts +++ b/lambdas/libs/compute-providers/core/index.ts @@ -32,7 +32,7 @@ export interface GitHubRunnerMetadata { } export interface StartRunnerConfigOptions { - getSsmParameterTags?: (runnerId: string) => { Key: string; Value: string }[]; + getStorageMetadataTags?: (runnerId: string) => { key: string; value: string }[]; onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise; } diff --git a/lambdas/libs/storage-providers/aws/ssm.test.ts b/lambdas/libs/storage-providers/aws/ssm.test.ts new file mode 100644 index 0000000000..448b0def1e --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm.test.ts @@ -0,0 +1,90 @@ +import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + createRunnerBootstrapStoreFromEnvironment, + createRunnerGroupCacheStoreFromEnvironment, + storageProviderRegistry, +} from '..'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + getParameter: vi.fn(), + putParameter: vi.fn(), +})); + +const getParameterMock = vi.mocked(getParameter); +const putParameterMock = vi.mocked(putParameter); + +describe('aws_ssm storage provider', () => { + beforeEach(() => { + vi.clearAllMocks(); + delete process.env.RUNNER_BOOTSTRAP_STORAGE_PROVIDER_TYPE; + delete process.env.RUNNER_GROUP_CACHE_STORAGE_PROVIDER_TYPE; + }); + + it('preserves the sensitive runner-bootstrap path, encryption, and tags', async () => { + const store = createRunnerBootstrapStoreFromEnvironment({ + locator: '/runner/tokens', + metadataTags: [{ key: 'Environment', value: 'test' }], + }); + + await store.put( + { identity: { kind: 'runner_bootstrap', runnerId: 'i-123' }, payload: 'jit-payload' }, + { metadataTags: [{ key: 'InstanceId', value: 'i-123' }] }, + ); + + expect(store.provider).toBe('aws_ssm'); + expect(store.maxWritesPerSecond).toBe(40); + expect(putParameterMock).toHaveBeenCalledWith('/runner/tokens/i-123', 'jit-payload', true, { + tags: [ + { Key: 'InstanceId', Value: 'i-123' }, + { Key: 'Environment', Value: 'test' }, + ], + }); + }); + + it('preserves the rebuildable runner-group cache path and non-secret value type', async () => { + const store = createRunnerGroupCacheStoreFromEnvironment({ + locator: '/runner/config', + metadataTags: [{ key: 'Environment', value: 'test' }], + }); + const identity = { kind: 'runner_group_cache' as const, groupName: 'Default' }; + getParameterMock.mockResolvedValue('1'); + + await expect(store.get(identity)).resolves.toEqual({ identity, payload: '1' }); + await store.put({ identity, payload: '2' }); + + expect(store.provider).toBe('aws_ssm'); + expect(getParameterMock).toHaveBeenCalledWith('/runner/config/runner-group/Default'); + expect(putParameterMock).toHaveBeenCalledWith('/runner/config/runner-group/Default', '2', false, { + tags: [{ Key: 'Environment', Value: 'test' }], + }); + }); + + it('selects providers independently for each storage usage', () => { + process.env.RUNNER_BOOTSTRAP_STORAGE_PROVIDER_TYPE = 'aws_ssm'; + process.env.RUNNER_GROUP_CACHE_STORAGE_PROVIDER_TYPE = 'not-registered'; + + expect( + createRunnerBootstrapStoreFromEnvironment({ + locator: '/runner/tokens', + metadataTags: [], + }).provider, + ).toBe('aws_ssm'); + expect(() => + createRunnerGroupCacheStoreFromEnvironment({ + locator: '/runner/config', + metadataTags: [], + }), + ).toThrow("Unsupported storage provider type 'not-registered'"); + }); + + it('rejects a capability request for a provider missing from the registry', () => { + expect(() => + storageProviderRegistry.createRunnerBootstrapStore('missing' as 'aws_ssm', { + locator: '/runner/tokens', + metadataTags: [], + }), + ).toThrow("No storage provider registered for 'missing'"); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/ssm.ts b/lambdas/libs/storage-providers/aws/ssm.ts new file mode 100644 index 0000000000..877fe797b4 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm.ts @@ -0,0 +1,54 @@ +import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util'; + +import type { + RunnerBootstrapRecord, + RunnerBootstrapStore, + RunnerBootstrapStoreContext, + RunnerBootstrapWriteOptions, + RunnerGroupCacheIdentity, + RunnerGroupCacheRecord, + RunnerGroupCacheStore, + RunnerGroupCacheStoreContext, + StorageMetadataTag, + StorageProvider, +} from '../core'; + +export const awsSsmStorageProvider: StorageProvider = { + type: 'aws_ssm', + createRunnerBootstrapStore: (context) => new AwsSsmRunnerBootstrapStore(context), + createRunnerGroupCacheStore: (context) => new AwsSsmRunnerGroupCacheStore(context), +}; + +function toSsmTags(tags: StorageMetadataTag[]) { + return tags.map(({ key, value }) => ({ Key: key, Value: value })); +} + +class AwsSsmRunnerBootstrapStore implements RunnerBootstrapStore { + readonly provider = 'aws_ssm'; + readonly maxWritesPerSecond = 40; + + constructor(private readonly context: RunnerBootstrapStoreContext) {} + + async put(record: RunnerBootstrapRecord, options: RunnerBootstrapWriteOptions = {}): Promise { + await putParameter(`${this.context.locator}/${record.identity.runnerId}`, record.payload, true, { + tags: toSsmTags([...(options.metadataTags ?? []), ...this.context.metadataTags]), + }); + } +} + +class AwsSsmRunnerGroupCacheStore implements RunnerGroupCacheStore { + readonly provider = 'aws_ssm'; + + constructor(private readonly context: RunnerGroupCacheStoreContext) {} + + async get(identity: RunnerGroupCacheIdentity): Promise { + const payload = await getParameter(`${this.context.locator}/runner-group/${identity.groupName}`); + return payload === undefined ? undefined : { identity, payload }; + } + + async put(record: RunnerGroupCacheRecord): Promise { + await putParameter(`${this.context.locator}/runner-group/${record.identity.groupName}`, record.payload, false, { + tags: toSsmTags(this.context.metadataTags), + }); + } +} diff --git a/lambdas/libs/storage-providers/core/index.ts b/lambdas/libs/storage-providers/core/index.ts new file mode 100644 index 0000000000..7e62c99a72 --- /dev/null +++ b/lambdas/libs/storage-providers/core/index.ts @@ -0,0 +1,85 @@ +import type { StorageProviderType } from '../provider-types'; + +export interface StorageMetadataTag { + key: string; + value: string; +} + +export interface RunnerBootstrapIdentity { + kind: 'runner_bootstrap'; + runnerId: string; +} + +export interface RunnerBootstrapRecord { + identity: RunnerBootstrapIdentity; + payload: string; +} + +export interface RunnerBootstrapWriteOptions { + metadataTags?: StorageMetadataTag[]; +} + +/** + * Short-lived, sensitive handoff from orchestration to one runner. + * + * Reading and deletion happen in the runner bootstrap implementation. This + * writer deliberately exposes no generic key/value operations. + */ +export interface RunnerBootstrapStore { + readonly provider: StorageProviderType; + readonly maxWritesPerSecond?: number; + put(record: RunnerBootstrapRecord, options?: RunnerBootstrapWriteOptions): Promise; +} + +export interface RunnerGroupCacheIdentity { + kind: 'runner_group_cache'; + groupName: string; +} + +export interface RunnerGroupCacheRecord { + identity: RunnerGroupCacheIdentity; + payload: string; +} + +/** Rebuildable cache for GitHub runner-group IDs. */ +export interface RunnerGroupCacheStore { + readonly provider: StorageProviderType; + get(identity: RunnerGroupCacheIdentity): Promise; + put(record: RunnerGroupCacheRecord): Promise; +} + +export interface RunnerBootstrapStoreContext { + locator: string; + metadataTags: StorageMetadataTag[]; +} + +export interface RunnerGroupCacheStoreContext { + locator: string; + metadataTags: StorageMetadataTag[]; +} + +export interface StorageProvider { + readonly type: StorageProviderType; + createRunnerBootstrapStore(context: RunnerBootstrapStoreContext): RunnerBootstrapStore; + createRunnerGroupCacheStore(context: RunnerGroupCacheStoreContext): RunnerGroupCacheStore; +} + +export function createStorageProviderRegistry(providers: readonly StorageProvider[]) { + const providersByType = new Map(providers.map((provider) => [provider.type, provider])); + + function get(type: StorageProviderType): StorageProvider { + const provider = providersByType.get(type); + if (!provider) { + throw new Error(`No storage provider registered for '${type}'`); + } + return provider; + } + + return { + get, + createRunnerBootstrapStore: (type: StorageProviderType, context: RunnerBootstrapStoreContext) => + get(type).createRunnerBootstrapStore(context), + createRunnerGroupCacheStore: (type: StorageProviderType, context: RunnerGroupCacheStoreContext) => + get(type).createRunnerGroupCacheStore(context), + }; +} diff --git a/lambdas/libs/storage-providers/index.ts b/lambdas/libs/storage-providers/index.ts new file mode 100644 index 0000000000..cd08741950 --- /dev/null +++ b/lambdas/libs/storage-providers/index.ts @@ -0,0 +1,22 @@ +import { awsSsmStorageProvider } from './aws/ssm'; +import { + createStorageProviderRegistry, + type RunnerBootstrapStoreContext, + type RunnerGroupCacheStoreContext, +} from './core'; +import { resolveStorageProviderType } from './provider-types'; + +export type * from './core'; +export type * from './provider-types'; + +export const storageProviderRegistry = createStorageProviderRegistry([awsSsmStorageProvider]); + +export function createRunnerBootstrapStoreFromEnvironment(context: RunnerBootstrapStoreContext) { + const provider = resolveStorageProviderType(process.env.RUNNER_BOOTSTRAP_STORAGE_PROVIDER_TYPE); + return storageProviderRegistry.createRunnerBootstrapStore(provider, context); +} + +export function createRunnerGroupCacheStoreFromEnvironment(context: RunnerGroupCacheStoreContext) { + const provider = resolveStorageProviderType(process.env.RUNNER_GROUP_CACHE_STORAGE_PROVIDER_TYPE); + return storageProviderRegistry.createRunnerGroupCacheStore(provider, context); +} diff --git a/lambdas/libs/storage-providers/package.json b/lambdas/libs/storage-providers/package.json new file mode 100644 index 0000000000..22cd461fe7 --- /dev/null +++ b/lambdas/libs/storage-providers/package.json @@ -0,0 +1,32 @@ +{ + "name": "@aws-github-runner/storage-providers", + "version": "1.0.0", + "main": "index.ts", + "exports": { + ".": "./index.ts", + "./core": "./core/index.ts", + "./provider-types": "./provider-types.ts", + "./aws/ssm": "./aws/ssm.ts" + }, + "type": "module", + "license": "MIT", + "scripts": { + "test": "NODE_ENV=test nx test", + "test:watch": "NODE_ENV=test nx test --watch", + "lint": "eslint .", + "format": "prettier --write \"**/*.ts\"", + "format-check": "prettier --check \"**/*.ts\"", + "all": "yarn format && yarn lint && yarn test" + }, + "dependencies": { + "@aws-github-runner/aws-ssm-util": "*" + }, + "nx": { + "includedScripts": [ + "format", + "format-check", + "lint", + "all" + ] + } +} diff --git a/lambdas/libs/storage-providers/provider-types.ts b/lambdas/libs/storage-providers/provider-types.ts new file mode 100644 index 0000000000..cf6926103c --- /dev/null +++ b/lambdas/libs/storage-providers/provider-types.ts @@ -0,0 +1,24 @@ +export const storageProviderTypes = ['aws_ssm'] as const; + +export type StorageProviderType = (typeof storageProviderTypes)[number]; + +export const defaultStorageProvider = 'aws_ssm' satisfies StorageProviderType; + +export function normalizeStorageProviderType(type: unknown): StorageProviderType | undefined { + if (type === undefined) return defaultStorageProvider; + if (typeof type !== 'string') return undefined; + + const normalizedType = type.trim().toLowerCase(); + if (!normalizedType) return defaultStorageProvider; + + return storageProviderTypes.find((storageProviderType) => storageProviderType === normalizedType); +} + +export function resolveStorageProviderType(type: unknown): StorageProviderType { + const normalizedType = normalizeStorageProviderType(type); + if (!normalizedType) { + throw new Error(`Unsupported storage provider type '${String(type)}'`); + } + + return normalizedType; +} diff --git a/lambdas/libs/storage-providers/tsconfig.json b/lambdas/libs/storage-providers/tsconfig.json new file mode 100644 index 0000000000..139069a7cf --- /dev/null +++ b/lambdas/libs/storage-providers/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.json", + "include": ["**/*.ts"], + "exclude": ["**/*.test.ts", "vitest.config.ts"] +} diff --git a/lambdas/libs/storage-providers/vitest.config.ts b/lambdas/libs/storage-providers/vitest.config.ts new file mode 100644 index 0000000000..d02f7af396 --- /dev/null +++ b/lambdas/libs/storage-providers/vitest.config.ts @@ -0,0 +1,14 @@ +import { resolve } from 'path'; + +import { mergeConfig } from 'vitest/config'; +import defaultConfig from '../../vitest.base.config'; + +export default mergeConfig(defaultConfig, { + test: { + setupFiles: [resolve(__dirname, '../../aws-vitest-setup.ts')], + coverage: { + include: ['index.ts', 'provider-types.ts', 'core/**/*.ts', 'aws/**/*.ts'], + exclude: ['**/*.test.ts'], + }, + }, +}); diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 56ae435c2c..84a1a7069c 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -164,6 +164,7 @@ __metadata: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-github-runner/compute-providers": "npm:*" + "@aws-github-runner/storage-providers": "npm:*" "@aws-lambda-powertools/parameters": "npm:^2.31.0" "@aws-sdk/client-ec2": "npm:^3.1009.0" "@aws-sdk/client-sqs": "npm:^3.1009.0" @@ -209,6 +210,14 @@ __metadata: languageName: unknown linkType: soft +"@aws-github-runner/storage-providers@npm:*, @aws-github-runner/storage-providers@workspace:libs/storage-providers": + version: 0.0.0-use.local + resolution: "@aws-github-runner/storage-providers@workspace:libs/storage-providers" + dependencies: + "@aws-github-runner/aws-ssm-util": "npm:*" + languageName: unknown + linkType: soft + "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher": version: 0.0.0-use.local resolution: "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher" diff --git a/mkdocs.yaml b/mkdocs.yaml index d974019b1b..a4e73e9110 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -59,6 +59,7 @@ nav: - Security: security.md - Architecture decisions: - ADR-002 Runner orchestration provider boundary: adr/002-runner-orchestration-provider-boundary.md + - ADR-003 Runner storage provider boundary: adr/003-runner-storage-provider-boundary.md - Modules: - Runners (main): modules/runners.md - Submodules (public): diff --git a/modules/orchestration-providers/webhook/pool/pool.tf b/modules/orchestration-providers/webhook/pool/pool.tf index cff2776e90..269c7dc65f 100644 --- a/modules/orchestration-providers/webhook/pool/pool.tf +++ b/modules/orchestration-providers/webhook/pool/pool.tf @@ -22,6 +22,8 @@ locals { RUNNER_LABELS = lower(join(",", var.config.runner.labels)) RUNNER_GROUP_NAME = var.config.runner.group_name RUNNER_NAME_PREFIX = var.config.runner.name_prefix + RUNNER_BOOTSTRAP_STORAGE_PROVIDER_TYPE = "aws_ssm" + RUNNER_GROUP_CACHE_STORAGE_PROVIDER_TYPE = "aws_ssm" RUNNER_OWNER = var.config.runner.pool_owner RUNNER_BOOT_TIME_IN_MINUTES = var.config.runner.boot_time_in_minutes RUNNERS_MAXIMUM_COUNT = var.config.runners_maximum_count diff --git a/modules/orchestration-providers/webhook/pool/tests/provider.tftest.hcl b/modules/orchestration-providers/webhook/pool/tests/provider.tftest.hcl index c04f435024..f1c0c17a3c 100644 --- a/modules/orchestration-providers/webhook/pool/tests/provider.tftest.hcl +++ b/modules/orchestration-providers/webhook/pool/tests/provider.tftest.hcl @@ -4,6 +4,18 @@ mock_provider "aws" { json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"logs:CreateLogStream\",\"Resource\":\"*\"}]}" } } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/pool-test" + } + } + + mock_resource "aws_lambda_function" { + defaults = { + arn = "arn:aws:lambda:eu-west-1:123456789012:function:pool-test" + } + } } variables { @@ -140,8 +152,10 @@ run "provider_supplies_only_compute_specific_pool_configuration" { aws_lambda_function.pool.environment[0].variables["RUNNER_OWNER"] == "example" && aws_lambda_function.pool.environment[0].variables["RUNNERS_MAXIMUM_COUNT"] == "10" && aws_lambda_function.pool.environment[0].variables["RUNNER_BOOT_TIME_IN_MINUTES"] == "13" + && aws_lambda_function.pool.environment[0].variables["RUNNER_BOOTSTRAP_STORAGE_PROVIDER_TYPE"] == "aws_ssm" + && aws_lambda_function.pool.environment[0].variables["RUNNER_GROUP_CACHE_STORAGE_PROVIDER_TYPE"] == "aws_ssm" ) - error_message = "The pool module must assemble common runner registration values and webhook-provider capacity and boot-time settings." + error_message = "The pool module must assemble runner settings and independently select SSM for bootstrap handoff and runner-group caching." } assert { diff --git a/modules/orchestration-providers/webhook/scale-runners/scale-up.tf b/modules/orchestration-providers/webhook/scale-runners/scale-up.tf index 92dfa4267a..c5aa0e36ed 100644 --- a/modules/orchestration-providers/webhook/scale-runners/scale-up.tf +++ b/modules/orchestration-providers/webhook/scale-runners/scale-up.tf @@ -39,6 +39,8 @@ resource "aws_lambda_function" "scale_up" { RUNNER_LABELS = lower(join(",", var.config.runner.labels)) RUNNER_GROUP_NAME = var.config.runner.group_name RUNNER_NAME_PREFIX = var.config.runner.name_prefix + RUNNER_BOOTSTRAP_STORAGE_PROVIDER_TYPE = "aws_ssm" + RUNNER_GROUP_CACHE_STORAGE_PROVIDER_TYPE = "aws_ssm" COMPUTE_PROVIDER_TYPE = var.runner_provider.type RUNNERS_MAXIMUM_COUNT = var.config.runner.maximum_count POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-scale-up" diff --git a/modules/orchestration-providers/webhook/scale-runners/tests/scale-runners.tftest.hcl b/modules/orchestration-providers/webhook/scale-runners/tests/scale-runners.tftest.hcl index 53fb829eb6..cd9590a936 100644 --- a/modules/orchestration-providers/webhook/scale-runners/tests/scale-runners.tftest.hcl +++ b/modules/orchestration-providers/webhook/scale-runners/tests/scale-runners.tftest.hcl @@ -10,6 +10,18 @@ mock_provider "aws" { arn = "arn:aws:iam::123456789012:role/scale-runners-test" } } + + mock_resource "aws_lambda_function" { + defaults = { + arn = "arn:aws-us-gov:lambda:us-gov-west-1:123456789012:function:scale-runners-test" + } + } + + mock_resource "aws_cloudwatch_event_rule" { + defaults = { + arn = "arn:aws-us-gov:events:us-gov-west-1:123456789012:rule/scale-runners-test" + } + } } variables { @@ -257,6 +269,14 @@ run "assembles_provider_neutral_scaling_control_plane" { error_message = "Scale runners must assemble shared runner, logging, TLS, lifetime, and Parameter Store environment variables." } + assert { + condition = ( + aws_lambda_function.scale_up.environment[0].variables["RUNNER_BOOTSTRAP_STORAGE_PROVIDER_TYPE"] == "aws_ssm" + && aws_lambda_function.scale_up.environment[0].variables["RUNNER_GROUP_CACHE_STORAGE_PROVIDER_TYPE"] == "aws_ssm" + ) + error_message = "Scale-up must select SSM independently for runner-bootstrap handoff and runner-group caching." + } + assert { condition = ( aws_lambda_function.scale_up.environment[0].variables["PARAMETER_GITHUB_APP_ID_NAME"] == "/github-runner/app-id:/github-runner/app-id-2" diff --git a/modules/runners/pool/main.tf b/modules/runners/pool/main.tf index 3be3fe41ef..f367f8246a 100644 --- a/modules/runners/pool/main.tf +++ b/modules/runners/pool/main.tf @@ -48,6 +48,8 @@ resource "aws_lambda_function" "pool" { RUNNER_LABELS = lower(join(",", var.config.runner.labels)) RUNNER_GROUP_NAME = var.config.runner.group_name RUNNER_NAME_PREFIX = var.config.runner.name_prefix + RUNNER_BOOTSTRAP_STORAGE_PROVIDER_TYPE = "aws_ssm" + RUNNER_GROUP_CACHE_STORAGE_PROVIDER_TYPE = "aws_ssm" RUNNER_OWNER = var.config.runner.pool_owner RUNNERS_MAXIMUM_COUNT = var.config.runners_maximum_count SSM_TOKEN_PATH = var.config.ssm_token_path diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf index 2e045345a6..68586a169a 100644 --- a/modules/runners/scale-up.tf +++ b/modules/runners/scale-up.tf @@ -55,6 +55,8 @@ resource "aws_lambda_function" "scale_up" { RUNNER_LABELS = lower(join(",", var.runner_labels)) RUNNER_GROUP_NAME = var.runner_group_name RUNNER_NAME_PREFIX = var.runner_name_prefix + RUNNER_BOOTSTRAP_STORAGE_PROVIDER_TYPE = "aws_ssm" + RUNNER_GROUP_CACHE_STORAGE_PROVIDER_TYPE = "aws_ssm" COMPUTE_PROVIDER_TYPE = "ec2" RUNNERS_MAXIMUM_COUNT = var.runners_maximum_count POWERTOOLS_SERVICE_NAME = "${var.prefix}-scale-up" diff --git a/modules/runners/tests/pool.tftest.hcl b/modules/runners/tests/pool.tftest.hcl index d6d327c598..117e4f37d6 100644 --- a/modules/runners/tests/pool.tftest.hcl +++ b/modules/runners/tests/pool.tftest.hcl @@ -4,6 +4,24 @@ mock_provider "aws" { json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" } } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/runners-test" + } + } + + mock_resource "aws_lambda_function" { + defaults = { + arn = "arn:aws:lambda:eu-west-1:123456789012:function:runners-test" + } + } + + mock_resource "aws_cloudwatch_event_rule" { + defaults = { + arn = "arn:aws:events:eu-west-1:123456789012:rule/runners-test" + } + } } variables { @@ -58,4 +76,12 @@ run "plan_with_pool_enabled" { condition = length(module.pool) == 1 error_message = "Pool module should be enabled when pool_config is non-empty" } + + assert { + condition = ( + module.pool[0].lambda.environment[0].variables["RUNNER_BOOTSTRAP_STORAGE_PROVIDER_TYPE"] == "aws_ssm" + && module.pool[0].lambda.environment[0].variables["RUNNER_GROUP_CACHE_STORAGE_PROVIDER_TYPE"] == "aws_ssm" + ) + error_message = "Stable pool must preserve SSM while selecting bootstrap handoff and runner-group cache independently." + } }