test(e2e): add playwright e2e harness
Process and API-only Playwright harness: binary and image setup, server/CLI runners, fake Docker Engine socket, read-only database inspection, secret redaction in reports, seeded fuzzing, leak detection and harness self-checks.
This commit is contained in:
bryanpedini committed
1 parent
a02eef13a7
commit
fe0cd4bf6d
33 files changed
+6370
No files matched your search
@@ -0,0 +1,354 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
// Copyright (C) 2026 Bryan Joshua Pedini
|
||||
|
||||
/**
|
||||
* Container-image helpers (project `image`): run the final image as a uniquely named,
|
||||
* labelled container with the Docker socket and a data directory mounted, drive the CLI
|
||||
* through `docker exec`, read logs/exit codes, and inspect the root-owned database.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { ALPINE_IMAGE, imageRef, LABEL_RUN, LABEL_TEST, RESOURCE_PREFIX, runId } from './env';
|
||||
import { HarnessError } from './errors';
|
||||
import { FakeDocker } from './fake-docker';
|
||||
import { buildRawRequest, rawHttp } from './http';
|
||||
import { parseLogs, type LogSet } from './logs';
|
||||
import { runProcess, sleep, toolEnv, type ProcResult } from './proc';
|
||||
import { dockerCli } from './docker';
|
||||
import { forceRemoveDir, type Workspace } from './workspace';
|
||||
import { printsTokens, type CliResult } from './cli';
|
||||
import type { Recorder } from './recorder';
|
||||
import type { SecretRegistry } from './secrets';
|
||||
|
||||
/** Socket mounted at /var/run/docker.sock in the container. */
|
||||
export type ImageDockerBackend = 'real' | 'none' | FakeDocker | { socketPath: string };
|
||||
|
||||
export interface ImageStartOptions {
|
||||
/** LOG_LEVEL (default 'INFO'; null = unset). */
|
||||
logLevel?: string | null;
|
||||
/** HTTP_ADDR inside the container (default unset → 0.0.0.0). */
|
||||
addr?: string | null;
|
||||
/** HTTP_PORT inside the container (default unset → 80); the published port follows it. */
|
||||
containerPort?: number;
|
||||
/** DB_PATH inside the container (default '/data/dpsapi.db'; null = unset). */
|
||||
dbPath?: string | null;
|
||||
/** Extra env (undefined removes). */
|
||||
env?: Record<string, string | undefined>;
|
||||
/** Default: a new FakeDocker (empty) — pass 'real' for the host Engine. */
|
||||
docker?: ImageDockerBackend;
|
||||
/** Host directory mounted at /data (default: a new workspace dir). Reuse it to replace a container. */
|
||||
dataDir?: string;
|
||||
/** docker run --user. */
|
||||
user?: string;
|
||||
/** Publish the container port on 127.0.0.1:<random> (default true). */
|
||||
publish?: boolean;
|
||||
/** Extra `docker run` arguments (before the image). */
|
||||
extraArgs?: string[];
|
||||
/** Arguments after the image (default none → server mode). */
|
||||
args?: string[];
|
||||
/** Wait for HTTP readiness (default true when publishing and no args). */
|
||||
waitReady?: boolean;
|
||||
readyTimeoutMs?: number;
|
||||
/** Name suffix (default img<n>). */
|
||||
suffix?: string;
|
||||
}
|
||||
|
||||
export interface ImageExecOptions {
|
||||
/** `docker exec -e K=V` (undefined values are skipped). */
|
||||
env?: Record<string, string | undefined>;
|
||||
user?: string;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface ContainerExit {
|
||||
exitCode: number;
|
||||
oomKilled: boolean;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
export interface ImageContext {
|
||||
workspace: Workspace;
|
||||
recorder: Recorder;
|
||||
secrets: SecretRegistry;
|
||||
testKey: string;
|
||||
newFakeDocker: () => Promise<FakeDocker>;
|
||||
containers: ImageContainer[];
|
||||
}
|
||||
|
||||
let seq = 0;
|
||||
|
||||
export class ImageContainer {
|
||||
/** Published host port (0 when not published). */
|
||||
hostPort = 0;
|
||||
/** http://127.0.0.1:<hostPort> */
|
||||
baseURL = '';
|
||||
id = '';
|
||||
|
||||
constructor(
|
||||
private readonly ctx: ImageContext,
|
||||
readonly name: string,
|
||||
readonly image: string,
|
||||
/** Host directory mounted at /data (files inside are root-owned). */
|
||||
readonly dataDir: string,
|
||||
readonly options: ImageStartOptions,
|
||||
readonly fakeDocker?: FakeDocker,
|
||||
) {}
|
||||
|
||||
/** DB_PATH inside the container. */
|
||||
get containerDbPath(): string | undefined {
|
||||
return this.options.dbPath === null ? undefined : (this.options.dbPath ?? '/data/dpsapi.db');
|
||||
}
|
||||
|
||||
url(p = '/api/v1/containers'): string {
|
||||
return this.baseURL + p;
|
||||
}
|
||||
|
||||
/** Polls `OPTIONS *` through the published port until the app answers (docker-proxy accepts early, so a real HTTP response is required). */
|
||||
async waitReady(timeoutMs = this.options.readyTimeoutMs ?? 20_000): Promise<void> {
|
||||
if (!this.hostPort) throw new HarnessError('infrastructure', `container ${this.name} has no published port`);
|
||||
const deadline = performance.now() + timeoutMs;
|
||||
let n = 0;
|
||||
while (true) {
|
||||
const r = await rawHttp({ host: '127.0.0.1', port: this.hostPort, data: buildRawRequest({ method: 'OPTIONS', target: '*' }), timeoutMs: 2000, allowTimeout: true }).catch(() => undefined);
|
||||
if (r?.outcome === 'response') return;
|
||||
if (++n % 10 === 0 && !(await this.isRunning())) {
|
||||
const l = await this.logs();
|
||||
throw new Error(`container ${this.name} exited during startup\n--- stderr\n${l.stderr}`);
|
||||
}
|
||||
if (performance.now() > deadline) {
|
||||
const l = await this.logs();
|
||||
throw new HarnessError('timeout', `container ${this.name} not ready on ${this.baseURL} within ${timeoutMs} ms\n--- stderr\n${l.stderr}`);
|
||||
}
|
||||
await sleep(50);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* `docker exec [-e ..] <name> /app ...args` with the same result shape as CliRunner.run.
|
||||
* Exit codes >= 125 (docker exec failures, signal deaths reported as 128+n) and daemon-refused execs are HarnessErrors.
|
||||
*/
|
||||
async exec(args: string[], opts: ImageExecOptions = {}): Promise<CliResult> {
|
||||
const r = await this.execRaw(['/app', ...args], opts);
|
||||
const res: CliResult = { ...r, args: [...args], via: 'docker-exec' };
|
||||
if (printsTokens(args)) this.ctx.secrets.registerAllIn(r.stdout);
|
||||
this.ctx.recorder.addCli({ via: 'docker-exec', command: `docker exec ${this.name} /app`, args, env: execEnv(opts.env), exitCode: r.exitCode, signal: r.signal, timedOut: r.timedOut, durationMs: Math.round(r.durationMs), stdout: r.stdout, stderr: r.stderr });
|
||||
// The daemon can refuse an exec with exit 1 (e.g. unknown --user), which must not pass for an app failure.
|
||||
const daemonRefused = r.exitCode !== 0 && /^(Error response from daemon:|OCI runtime exec failed)/.test(r.stderr + r.stdout);
|
||||
if ((r.exitCode !== null && r.exitCode >= 125) || daemonRefused) {
|
||||
throw new HarnessError('infrastructure', `docker exec ${this.name} /app ${args.slice(0, 2).join(' ')} failed at the docker level (exit ${r.exitCode}): ${(r.stderr + r.stdout).trim()}`, r);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
/** `docker exec` of an arbitrary argv (e.g. ['sh'] to prove there is no shell). Never throws for exit codes. */
|
||||
async execRaw(argv: string[], opts: ImageExecOptions = {}): Promise<ProcResult> {
|
||||
const a = ['exec'];
|
||||
if (opts.user) a.push('--user', opts.user);
|
||||
for (const [k, v] of Object.entries(opts.env ?? {})) if (v !== undefined) a.push('-e', `${k}=${v}`);
|
||||
a.push(this.name, ...argv);
|
||||
return runProcess('docker', a, { env: toolEnv(), timeoutMs: opts.timeoutMs ?? 30_000, label: `docker exec ${argv[0]}` });
|
||||
}
|
||||
|
||||
/** `docker logs`: the container's stdout and stderr separately. */
|
||||
async logs(): Promise<{ stdout: string; stderr: string }> {
|
||||
const r = await dockerCli(['logs', this.name], { allowFailure: true });
|
||||
return { stdout: r.stdout, stderr: r.stderr };
|
||||
}
|
||||
|
||||
/** Parsed application logs (container stderr). */
|
||||
async parsedLogs(): Promise<LogSet> {
|
||||
return parseLogs((await this.logs()).stderr);
|
||||
}
|
||||
|
||||
async inspect(): Promise<Record<string, unknown>> {
|
||||
const r = await dockerCli(['inspect', this.name]);
|
||||
return (JSON.parse(r.stdout) as Array<Record<string, unknown>>)[0];
|
||||
}
|
||||
|
||||
async isRunning(): Promise<boolean> {
|
||||
const r = await dockerCli(['inspect', '-f', '{{.State.Running}}', this.name], { allowFailure: true });
|
||||
return r.stdout.trim() === 'true';
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops the container and returns the application's exit code.
|
||||
* Default: `docker stop -t <timeoutSec>` (SIGTERM, then SIGKILL after the grace period).
|
||||
* With `signal`: `docker kill -s <signal>` and wait up to timeoutSec.
|
||||
* A SIGKILL by docker after the grace period shows as exitCode 137.
|
||||
*/
|
||||
async stop({ signal, timeoutSec = 20 }: { signal?: NodeJS.Signals; timeoutSec?: number } = {}): Promise<ContainerExit> {
|
||||
if (signal) await dockerCli(['kill', '-s', signal, this.name]);
|
||||
else await dockerCli(['stop', '-t', String(timeoutSec), this.name], { timeoutMs: (timeoutSec + 30) * 1000 });
|
||||
return this.waitForExit((timeoutSec + 10) * 1000);
|
||||
}
|
||||
|
||||
/** Waits for the container to exit (`docker wait`), then returns exit code and logs. */
|
||||
async waitForExit(timeoutMs = 30_000): Promise<ContainerExit> {
|
||||
const r = await runProcess('docker', ['wait', this.name], { env: toolEnv(), timeoutMs, label: 'docker wait' });
|
||||
if (r.exitCode !== 0) throw new HarnessError('infrastructure', `docker wait ${this.name}: ${r.stderr}`);
|
||||
const st = (await this.inspect()).State as { ExitCode: number; OOMKilled: boolean };
|
||||
const l = await this.logs();
|
||||
return { exitCode: st.ExitCode, oomKilled: st.OOMKilled, stdout: l.stdout, stderr: l.stderr };
|
||||
}
|
||||
|
||||
/** docker rm -f -v. */
|
||||
async remove(): Promise<void> {
|
||||
await dockerCli(['rm', '-f', '-v', this.name], { allowFailure: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies the data directory (db + -wal, not -shm) into a new host-owned workspace directory and
|
||||
* returns the path of the copied database file — inspect it with readTokens/assertDbInvariants.
|
||||
* Works whether the app container is running, stopped or removed.
|
||||
*/
|
||||
async snapshotDb(): Promise<string> {
|
||||
return snapshotDataDir(this.ctx, this.dataDir, path.basename(this.containerDbPath ?? '/data/dpsapi.db'));
|
||||
}
|
||||
}
|
||||
|
||||
function execEnv(env: Record<string, string | undefined> = {}): Record<string, string> {
|
||||
const o: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(env)) if (v !== undefined) o[k] = v;
|
||||
return o;
|
||||
}
|
||||
|
||||
let snapSeq = 0;
|
||||
|
||||
/** Copies a (root-owned) data dir into a host-owned directory using a throwaway alpine container. Returns the copied db path. */
|
||||
export async function snapshotDataDir(ctx: Pick<ImageContext, 'workspace' | 'testKey'>, dataDir: string, dbFile = 'dpsapi.db'): Promise<string> {
|
||||
const out = ctx.workspace.mkdir(`db-snapshot-${++snapSeq}`);
|
||||
const uid = process.getuid!();
|
||||
const gid = process.getgid!();
|
||||
const name = `${RESOURCE_PREFIX}${runId()}-${ctx.testKey}-snap${snapSeq}`;
|
||||
await dockerCli([
|
||||
'run', '--rm', '--name', name, '--label', `${LABEL_RUN}=${runId()}`, '--label', `${LABEL_TEST}=${ctx.testKey}`, '--network', 'none',
|
||||
'-v', `${dataDir}:/data:ro`, '-v', `${out}:/out`, ALPINE_IMAGE,
|
||||
'sh', '-c', `cp -a /data/. /out/ && rm -f /out/*-shm && chown -R ${uid}:${gid} /out && chmod -R u+rwX /out`,
|
||||
]);
|
||||
const p = path.join(out, dbFile);
|
||||
if (!fs.existsSync(p)) throw new HarnessError('infrastructure', `no ${dbFile} in ${dataDir}`);
|
||||
return p;
|
||||
}
|
||||
|
||||
export class ImageRunner {
|
||||
constructor(private readonly ctx: ImageContext) {}
|
||||
|
||||
/** The image under test ($DPSAPI_IMAGE). */
|
||||
get image(): string {
|
||||
const i = imageRef();
|
||||
if (!i) throw new HarnessError('prerequisite', 'DPSAPI_IMAGE is not set (use `make docker-e2e` or export DPSAPI_IMAGE=<ref>)');
|
||||
return i;
|
||||
}
|
||||
|
||||
/** Name for a suite-owned container: dpsapi-e2e-<runId>-<testKey>-<suffix>. */
|
||||
containerName(suffix: string): string {
|
||||
return `${RESOURCE_PREFIX}${runId()}-${this.ctx.testKey}-${suffix}`;
|
||||
}
|
||||
|
||||
/** docker run -d the image (server mode by default) and wait for readiness. */
|
||||
async start(opts: ImageStartOptions = {}): Promise<ImageContainer> {
|
||||
const image = this.image;
|
||||
const suffix = opts.suffix ?? `img${++seq}`;
|
||||
const name = this.containerName(suffix);
|
||||
const dataDir = opts.dataDir ?? this.ctx.workspace.mkdir(`data-${suffix}`);
|
||||
let fake: FakeDocker | undefined;
|
||||
let sock: string | undefined;
|
||||
const backend = opts.docker ?? (fake = await this.ctx.newFakeDocker());
|
||||
if (backend === 'real') sock = '/var/run/docker.sock';
|
||||
else if (backend instanceof FakeDocker) {
|
||||
fake = backend;
|
||||
sock = backend.socketPath;
|
||||
} else if (backend !== 'none') sock = backend.socketPath;
|
||||
if (sock && !fs.existsSync(sock)) throw new HarnessError('infrastructure', `socket ${sock} does not exist (docker would create a root-owned directory there)`);
|
||||
const c = new ImageContainer(this.ctx, name, image, dataDir, opts, fake);
|
||||
const containerPort = opts.containerPort ?? 80;
|
||||
const args = ['run', '-d', '--name', name, '--label', `${LABEL_RUN}=${runId()}`, '--label', `${LABEL_TEST}=${this.ctx.testKey}`];
|
||||
if (opts.publish ?? true) args.push('-p', `127.0.0.1::${containerPort}`);
|
||||
if (sock) args.push('-v', `${sock}:/var/run/docker.sock`);
|
||||
args.push('-v', `${dataDir}:/data`);
|
||||
const env: Record<string, string | undefined> = {
|
||||
DB_PATH: opts.dbPath === null ? undefined : (opts.dbPath ?? '/data/dpsapi.db'),
|
||||
LOG_LEVEL: opts.logLevel === null ? undefined : (opts.logLevel ?? 'INFO'),
|
||||
HTTP_ADDR: opts.addr ?? undefined,
|
||||
HTTP_PORT: opts.containerPort !== undefined ? String(opts.containerPort) : undefined,
|
||||
...(opts.env ?? {}),
|
||||
};
|
||||
for (const [k, v] of Object.entries(env)) if (v !== undefined) args.push('-e', `${k}=${v}`);
|
||||
if (opts.user) args.push('--user', opts.user);
|
||||
args.push(...(opts.extraArgs ?? []), image, ...(opts.args ?? []));
|
||||
this.ctx.containers.push(c);
|
||||
const r = await dockerCli(args);
|
||||
c.id = r.stdout.trim();
|
||||
this.ctx.recorder.note(`image container ${name} started: docker ${JSON.stringify(args)}`);
|
||||
if (opts.publish ?? true) {
|
||||
const p = await dockerCli(['port', name, `${containerPort}/tcp`], { allowFailure: true });
|
||||
const m = /127\.0\.0\.1:(\d+)/.exec(p.stdout);
|
||||
if (m) {
|
||||
c.hostPort = Number(m[1]);
|
||||
c.baseURL = `http://127.0.0.1:${c.hostPort}`;
|
||||
} else if (opts.waitReady ?? !opts.args?.length) {
|
||||
const l = await c.logs();
|
||||
throw new Error(`container ${name} has no published port (exited?): ${p.stderr}\n--- stderr\n${l.stderr}`);
|
||||
}
|
||||
}
|
||||
if ((opts.waitReady ?? !opts.args?.length) && (opts.publish ?? true)) await c.waitReady();
|
||||
return c;
|
||||
}
|
||||
|
||||
/**
|
||||
* One-shot `docker run --rm` of the image with args (CLI mode / startup-failure checks).
|
||||
* Returns the CliResult shape (via 'docker-run'); exit codes 125–127 are HarnessErrors.
|
||||
*/
|
||||
async run(
|
||||
args: string[],
|
||||
opts: { env?: Record<string, string | undefined>; dataDir?: string; docker?: ImageDockerBackend; user?: string; timeoutMs?: number; extraArgs?: string[]; suffix?: string } = {},
|
||||
): Promise<CliResult> {
|
||||
const name = this.containerName(opts.suffix ?? `run${++seq}`);
|
||||
const dataDir = opts.dataDir ?? this.ctx.workspace.mkdir(`data-${name.slice(-12)}`);
|
||||
const a = ['run', '--rm', '--name', name, '--label', `${LABEL_RUN}=${runId()}`, '--label', `${LABEL_TEST}=${this.ctx.testKey}`, '-v', `${dataDir}:/data`];
|
||||
const backend = opts.docker ?? 'none';
|
||||
if (backend === 'real') a.push('-v', '/var/run/docker.sock:/var/run/docker.sock');
|
||||
else if (backend instanceof FakeDocker) a.push('-v', `${backend.socketPath}:/var/run/docker.sock`);
|
||||
else if (backend !== 'none') a.push('-v', `${backend.socketPath}:/var/run/docker.sock`);
|
||||
const env = { DB_PATH: '/data/dpsapi.db', ...(opts.env ?? {}) };
|
||||
for (const [k, v] of Object.entries(env)) if (v !== undefined) a.push('-e', `${k}=${v}`);
|
||||
if (opts.user) a.push('--user', opts.user);
|
||||
a.push(...(opts.extraArgs ?? []), this.image, ...args);
|
||||
const r = await runProcess('docker', a, { env: toolEnv(), timeoutMs: opts.timeoutMs ?? 60_000, label: `docker run ${args.slice(0, 2).join(' ')}` });
|
||||
if (printsTokens(args)) this.ctx.secrets.registerAllIn(r.stdout);
|
||||
this.ctx.recorder.addCli({ via: 'docker-run', command: `docker run ${this.image}`, args, env: execEnv(env), exitCode: r.exitCode, signal: r.signal, timedOut: r.timedOut, durationMs: Math.round(r.durationMs), stdout: r.stdout, stderr: r.stderr });
|
||||
if (r.exitCode !== null && r.exitCode >= 125 && r.exitCode <= 127) {
|
||||
throw new HarnessError('infrastructure', `docker run failed at the docker level (exit ${r.exitCode}): ${r.stderr.trim()}`, r);
|
||||
}
|
||||
return { ...r, args: [...args], via: 'docker-run' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies /app out of the image (docker create + docker cp, no container is started) into the
|
||||
* workspace and returns its host path — the static binary can then be driven on the host with
|
||||
* cli.withBinary(path) / startServer({ binary: path }) for multi-transport comparisons.
|
||||
*/
|
||||
async extractBinary(): Promise<string> {
|
||||
const name = this.containerName(`extract${++seq}`);
|
||||
const dest = this.ctx.workspace.path(`app-from-image-${seq}`);
|
||||
await dockerCli(['create', '--name', name, '--label', `${LABEL_RUN}=${runId()}`, '--label', `${LABEL_TEST}=${this.ctx.testKey}`, this.image]);
|
||||
try {
|
||||
await dockerCli(['cp', `${name}:/app`, dest]);
|
||||
} finally {
|
||||
await dockerCli(['rm', '-f', name], { allowFailure: true });
|
||||
}
|
||||
fs.chmodSync(dest, 0o755);
|
||||
return dest;
|
||||
}
|
||||
|
||||
/** Copies a data dir to a readable location; returns the db path (see ImageContainer.snapshotDb). */
|
||||
snapshotDataDir(dataDir: string, dbFile = 'dpsapi.db'): Promise<string> {
|
||||
return snapshotDataDir(this.ctx, dataDir, dbFile);
|
||||
}
|
||||
|
||||
/** Deletes a (root-owned) data directory via a throwaway container. */
|
||||
removeDataDir(dataDir: string): Promise<void> {
|
||||
return forceRemoveDir(dataDir);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user