Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .codacy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Codacy: paths that don't ship to users (demo tooling).
exclude_paths:
- "scripts/demo-gif.mjs"
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

Lambda MicroVMs · EC2 (on-demand and Spot) · ECS Fargate · Lambda functions · one `runs-on` line · MIT

[![CI](https://github.com/mintwork/runway/actions/workflows/ci.yml/badge.svg)](https://github.com/mintwork/runway/actions/workflows/ci.yml) [![npm](https://img.shields.io/npm/v/@mintwork/runway?color=cb3837&logo=npm&label=npm)](https://www.npmjs.com/package/@mintwork/runway) [![npm downloads](https://img.shields.io/npm/dm/@mintwork/runway?color=cb3837&logo=npm&label=downloads)](https://www.npmjs.com/package/@mintwork/runway) [![npm provenance](https://img.shields.io/badge/provenance-signed-7EE2B8?labelColor=0D1714&logo=sigstore&logoColor=white)](https://www.npmjs.com/package/@mintwork/runway#provenance) [![Node 22+](https://img.shields.io/badge/node-%E2%89%A522-339933?logo=nodedotjs&logoColor=white)](package.json) [![license: MIT](https://img.shields.io/badge/license-MIT-blue)](LICENSE) [![GitHub stars](https://img.shields.io/github/stars/mintwork/runway?style=social)](https://github.com/mintwork/runway/stargazers)
[![CI](https://github.com/mintwork/runway/actions/workflows/ci.yml/badge.svg)](https://github.com/mintwork/runway/actions/workflows/ci.yml) [![Codacy grade](https://app.codacy.com/project/badge/Grade/dd55188dca4c423cbcebcb0362c33f2b)](https://app.codacy.com/gh/mintwork/runway/dashboard) [![npm](https://img.shields.io/npm/v/@mintwork/runway?color=cb3837&logo=npm&label=npm)](https://www.npmjs.com/package/@mintwork/runway) [![npm downloads](https://img.shields.io/npm/dm/@mintwork/runway?color=cb3837&logo=npm&label=downloads)](https://www.npmjs.com/package/@mintwork/runway) [![npm provenance](https://img.shields.io/badge/provenance-signed-7EE2B8?labelColor=0D1714&logo=sigstore&logoColor=white)](https://www.npmjs.com/package/@mintwork/runway#provenance) [![Node 22+](https://img.shields.io/badge/node-%E2%89%A522-339933?logo=nodedotjs&logoColor=white)](package.json) [![license: MIT](https://img.shields.io/badge/license-MIT-blue)](LICENSE) [![GitHub stars](https://img.shields.io/github/stars/mintwork/runway?style=social)](https://github.com/mintwork/runway/stargazers)

<img src="docs/demo.gif" alt="runway setup deploying the stack, creating the GitHub App in the browser and building images (a real install, sped up), then a workflow using four compute types, then runway status listing a runner on each" width="900">

Expand Down
14 changes: 8 additions & 6 deletions agent/runway_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import os
import shutil
import signal
import subprocess
import subprocess # nosec B404 - the agent exists to run the runner and the container tools
import sys
import threading
import time
Expand Down Expand Up @@ -141,7 +141,8 @@ def _wait_for(check, timeout, interval=0.5, clock=time.monotonic):


def _quiet(cmd):
return subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0
# cmd is a fixed argument list built in this file, never a shell string.
return subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0 # nosec B603 # nosemgrep


def start_docker(storage=None, containerd_timeout=120, popen=subprocess.Popen):
Expand Down Expand Up @@ -233,7 +234,7 @@ class HookServer(BaseHTTPRequestHandler):
warmup = None
lock = threading.Lock()

def log_message(self, fmt, *args): # keep stdout for structured logs
def log_message(self, format, *args): # pylint: disable=redefined-builtin # keep stdout for structured logs
pass

def _reply(self, status):
Expand Down Expand Up @@ -303,7 +304,8 @@ def _run_job(payload):

def serve():
port = env_int("RUNWAY_HOOK_PORT", 9000)
server = ThreadingHTTPServer(("0.0.0.0", port), HookServer)
# Lambda MicroVMs call the hooks over the VM's network interface, so loopback isn't enough.
server = ThreadingHTTPServer(("0.0.0.0", port), HookServer) # nosec B104
log("hook server listening", port=port)
server.serve_forever()

Expand All @@ -324,7 +326,7 @@ def oneshot():
return job.run()


def prepare_lambda_runner(source="/opt/actions-runner", target="/tmp/actions-runner"):
def prepare_lambda_runner(source="/opt/actions-runner", target="/tmp/actions-runner"): # nosec B108 - /tmp is Lambda's only writable path
"""The Lambda filesystem is read-only outside /tmp, and the runner writes next to itself."""
if not os.path.exists(os.path.join(target, "run.sh")):
started = time.monotonic()
Expand All @@ -338,7 +340,7 @@ def lambda_handler(event, context):
if not jit:
log("invocation has no jitConfig")
return {"ok": False}
os.environ.setdefault("HOME", "/tmp")
os.environ.setdefault("HOME", "/tmp") # nosec B108 - /tmp is Lambda's only writable path
runner_dir = prepare_lambda_runner()
remaining = int(context.get_remaining_time_in_millis() / 1000) - 20 if context else 880
job = Job(runner_dir, jit, max_seconds=max(60, remaining), idle_seconds=min(env_int("RUNWAY_IDLE_SECONDS", 300), max(60, remaining)))
Expand Down
6 changes: 3 additions & 3 deletions agent/test_runway_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import json
import os
import threading
import time
import unittest
import urllib.error
import urllib.request
Expand Down Expand Up @@ -185,7 +184,7 @@ def tearDown(self):
def post(self, hook, body=None):
req = urllib.request.Request(self.base + agent.HOOK_PREFIX + hook, data=json.dumps(body or {}).encode(), method="POST")
try:
return urllib.request.urlopen(req).status
return urllib.request.urlopen(req).status # nosec B310 - a local http:// test server
except urllib.error.HTTPError as e:
return e.code

Expand Down Expand Up @@ -248,7 +247,8 @@ class LambdaTest(unittest.TestCase):
def test_runs_the_job_then_discards_the_environment(self):
ctx = mock.Mock()
ctx.get_remaining_time_in_millis.return_value = 900_000
with mock.patch.object(agent, "prepare_lambda_runner", return_value="/tmp/actions-runner"), \
LAMBDA_RUNNER_DIR = "/tmp/actions-runner" # nosec B108 - Lambda's only writable path
with mock.patch.object(agent, "prepare_lambda_runner", return_value=LAMBDA_RUNNER_DIR), \
mock.patch.object(agent.Job, "run", return_value=0) as run, \
mock.patch.object(agent.os, "_exit") as exit_:
agent.lambda_handler({"jitConfig": "J", "runnerName": "runway-a"}, ctx)
Expand Down
3 changes: 3 additions & 0 deletions images/microvm/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,7 @@ ENV HOME=/root \
DOTNET_NOLOGO=1

EXPOSE 9000
# Runs as root on purpose: the agent starts containerd and dockerd, and each MicroVM runs one job
# and is then discarded.
# nosemgrep: dockerfile.security.missing-user.missing-user
CMD ["python3", "/opt/runway/runway_agent.py", "serve"]
1 change: 1 addition & 0 deletions src/cli/aws.ts
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ async function waitForStack(
since: Date,
): Promise<Stack> {
const p = progress(label);
// eslint-disable-next-line security-node/detect-unhandled-async-errors -- errors propagate to the caller, which reports them
for (;;) {
const stack = await describeStack(cf, name);
const status = stack?.StackStatus ?? 'DELETE_COMPLETE';
Expand Down
11 changes: 8 additions & 3 deletions src/cli/create-app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
// GitHub sends the browser back to it with a one-time code, which the CLI redeems for the App's
// credentials. The key goes from GitHub to this process to SSM, and nowhere else.
import { randomBytes, timingSafeEqual } from 'node:crypto';
import { createServer, type Server } from 'node:http';
import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http';
import type { AddressInfo } from 'node:net';
import {
convertManifestCode,
Expand Down Expand Up @@ -64,7 +64,12 @@ export function createApp(opts: CreateAppOptions): Promise<StoredApp> {
else resolve(app as StoredApp);
};

server = createServer(async (req, res) => {
// A plain (non-async) listener, so an unexpected error ends the flow instead of becoming an
// unhandled rejection.
server = createServer((req, res) => {
handle(req, res).catch((e: unknown) => finish(e instanceof Error ? e : new Error(String(e))));
});
const handle = async (req: IncomingMessage, res: ServerResponse): Promise<void> => {
const url = new URL(req.url ?? '/', base);
const send = (status: number, html: string, headers: Record<string, string> = {}) => {
res.writeHead(status, {
Expand Down Expand Up @@ -116,7 +121,7 @@ export function createApp(opts: CreateAppOptions): Promise<StoredApp> {
return;
}
send(404, page('Not found', '<h1>Not found</h1>'));
});
};
server.on('error', (e) => finish(e));
// Loopback only: nothing else on the network can reach it.
server.listen(0, '127.0.0.1', () => {
Expand Down
1 change: 1 addition & 0 deletions src/cli/deploy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ async function startBuilds(t: Target, maintainer: string): Promise<void> {
Payload: Buffer.from(JSON.stringify({ force: true })),
}),
);
// eslint-disable-next-line security-node/detect-unhandled-async-errors -- errors propagate to the caller, which reports them
if (res.FunctionError) {
const detail = Buffer.from(res.Payload ?? []).toString();
throw new CliError(
Expand Down
5 changes: 3 additions & 2 deletions src/cli/github-app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import { spawn } from 'node:child_process';
import type { AppCredentials } from '../core/github.js';
import { appJwt } from '../core/github.js';
import { CliError } from './ui.js';
import { CliError, info } from './ui.js';

const API = 'https://api.github.com';
const headers = {
Expand All @@ -17,7 +17,7 @@ export function openBrowser(url: string): void {
const args = process.platform === 'win32' ? ['/c', 'start', '', url] : [url];
try {
spawn(cmd, args, { stdio: 'ignore', detached: true })
.on('error', () => {})
.on('error', () => info(`Couldn't open a browser; open the link above.`))
.unref();
} catch {
// The URL is printed too.
Expand Down Expand Up @@ -68,6 +68,7 @@ export async function waitForInstallation(
timeoutMs = 15 * 60_000,
): Promise<Installation> {
const start = Date.now();
// eslint-disable-next-line security-node/detect-unhandled-async-errors -- errors propagate to the caller, which reports them
for (;;) {
const found = (await installations(creds)).find(
(i) => i.account.login.toLowerCase() === owner.toLowerCase(),
Expand Down
1 change: 1 addition & 0 deletions src/core/images.ts
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,7 @@ export async function waitForSettledImage(
sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)),
): Promise<void> {
const deadline = Date.now() + timeoutMs;
// eslint-disable-next-line security-node/detect-unhandled-async-errors -- errors propagate to the caller, which reports them
for (;;) {
const { state } = await microvmImageState(client, arn);
if (state && SETTLED.has(state)) return;
Expand Down
4 changes: 4 additions & 0 deletions src/lambda/bootstrap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ async function exists(name: string): Promise<boolean> {
/** EC2 Spot and ECS create these on first use, but only for callers allowed to create IAM roles. */
async function ensureServiceLinkedRoles(): Promise<void> {
const iam = new IAMClient({});
// eslint-disable-next-line security-node/detect-unhandled-async-errors -- errors propagate to the caller, which reports them
for (const AWSServiceName of ['spot.amazonaws.com', 'ecs.amazonaws.com']) {
await iam.send(new CreateServiceLinkedRoleCommand({ AWSServiceName })).catch((e: Error) => {
if (!/has been taken|already exists/i.test(e.message)) throw e;
Expand Down Expand Up @@ -142,6 +143,7 @@ async function cleanUp(): Promise<void> {
for (const Name of [...n.params, n.legacySetupToken])
await ssm.send(new DeleteParameterCommand({ Name })).catch(() => {});
let ContinuationToken: string | undefined;
// eslint-disable-next-line security-node/detect-unhandled-async-errors -- errors propagate to the caller, which reports them
do {
const page = await s3.send(new ListObjectsV2Command({ Bucket: n.bucket, ContinuationToken }));
const keys = (page.Contents ?? []).map((o) => ({ Key: o.Key as string }));
Expand Down Expand Up @@ -170,6 +172,8 @@ async function respond(
LogicalResourceId: event.LogicalResourceId,
Data: data,
});
// CloudFormation's presigned response URL for this custom resource; it's meant to be called.
// nosemgrep
await fetch(event.ResponseURL, { method: 'PUT', body, headers: { 'content-type': '' } });
}

Expand Down
1 change: 1 addition & 0 deletions src/lambda/sweeper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { log, runtimeDeps } from './runtime.js';

export async function handler(): Promise<void> {
const result = await sweep(await runtimeDeps());
// eslint-disable-next-line security-node/detect-unhandled-async-errors -- errors propagate to the caller, which reports them
if (result.terminated.length || result.orphans.length || result.relaunched.length)
log('sweep', { ...result });
}
3 changes: 2 additions & 1 deletion src/providers/ec2.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// EC2: one instance per job, on-demand or Spot, any instance type. The user-data script
// installs Docker and the runner on Amazon Linux 2023, runs the agent for one job, and
// shuts down; InstanceInitiatedShutdownBehavior=terminate turns that into termination.
import { randomInt } from 'node:crypto';
import {
DescribeInstancesCommand,
EC2Client,
Expand Down Expand Up @@ -201,7 +202,7 @@ export class Ec2Provider implements Provider {
function shuffle<T>(items: T[]): T[] {
const a = [...items];
for (let i = a.length - 1; i > 0; i--) {
const j = Math.floor(Math.random() * (i + 1));
const j = randomInt(i + 1);
[a[i], a[j]] = [a[j], a[i]];
}
return a;
Expand Down