test(e2e): add playwright e2e harness
Process and API-only Playwright harness: binary and image setup, server/CLI runners, fake Docker Engine socket, read-only database inspection, secret redaction in reports, seeded fuzzing, leak detection and harness self-checks.
This commit is contained in:
bryanpedini committed
1 parent
a02eef13a7
commit
fe0cd4bf6d
33 files changed
+6370
No files matched your search
@@ -0,0 +1,354 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
// Copyright (C) 2026 Bryan Joshua Pedini
|
||||
|
||||
/**
|
||||
* Running the compiled binary in server mode with a controlled Docker socket.
|
||||
*/
|
||||
import { spawn, type ChildProcess } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import { binaryPath } from './env';
|
||||
import { HarnessError } from './errors';
|
||||
import { FakeDocker } from './fake-docker';
|
||||
import { buildRawRequest, rawHttp } from './http';
|
||||
import { parseLogs, type LogQuery, type LogRecord, type LogSet } from './logs';
|
||||
import { killGroup, sleep, trackProcess } from './proc';
|
||||
import { baseAppEnv, mergeEnv, recordableEnv } from './cli';
|
||||
import type { Recorder } from './recorder';
|
||||
import type { Workspace } from './workspace';
|
||||
|
||||
/**
|
||||
* What the server sees at /var/run/docker.sock:
|
||||
* - 'real': the host socket (binary runs directly, no namespaces);
|
||||
* - FakeDocker: the fake's socket bind-mounted over the path (private mount namespace);
|
||||
* - 'inaccessible': a root-owned unreadable file (/etc/shadow) bind-mounted over it → EACCES;
|
||||
* - 'missing': an empty tmpfs over /run, so the path does not exist → ENOENT;
|
||||
* - {bindSource}: any host path bind-mounted over it.
|
||||
*/
|
||||
export type DockerBackend = 'real' | 'inaccessible' | 'missing' | FakeDocker | { bindSource: string };
|
||||
|
||||
export interface StartServerOptions {
|
||||
/** LOG_LEVEL; default 'INFO'; null leaves it unset (app default). */
|
||||
logLevel?: string | null;
|
||||
/** HTTP_ADDR; default '127.0.0.1'; null leaves it unset (app default 0.0.0.0). */
|
||||
addr?: string | null;
|
||||
/** HTTP_PORT; default: a freshly allocated free port; null leaves it unset (app default 80 — only usable with privateNetwork). */
|
||||
port?: number | string | null;
|
||||
/** DB_PATH; default the workspace's dpsapi.db. */
|
||||
dbPath?: string;
|
||||
/** Default: a new FakeDocker in empty mode (exposed as handle.fakeDocker). */
|
||||
docker?: DockerBackend;
|
||||
/** Extra environment overrides (undefined removes a variable). Applied last. */
|
||||
env?: Record<string, string | undefined>;
|
||||
/** Run in a private network namespace (loopback down; only log-based readiness). */
|
||||
privateNetwork?: boolean;
|
||||
/** Wait for HTTP readiness before returning (default true unless privateNetwork). */
|
||||
waitReady?: boolean;
|
||||
/** Default 10 s. */
|
||||
readyTimeoutMs?: number;
|
||||
/** Name used in artifacts. */
|
||||
label?: string;
|
||||
/** Binary to run (default $DPSAPI_BIN / ../dpsapi), e.g. /app extracted with image.extractBinary(). */
|
||||
binary?: string;
|
||||
}
|
||||
|
||||
export interface ExitResult {
|
||||
exitCode: number | null;
|
||||
signal: NodeJS.Signals | null;
|
||||
/** Time from the stop request (or start, for waitForExit) to exit. */
|
||||
durationMs: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
export interface ServerContext {
|
||||
workspace: Workspace;
|
||||
recorder: Recorder;
|
||||
allocatePort: () => Promise<number>;
|
||||
newFakeDocker: () => Promise<FakeDocker>;
|
||||
/** Registry of every handle started in the test (for teardown/artifacts). */
|
||||
servers: ServerHandle[];
|
||||
}
|
||||
|
||||
interface Generation {
|
||||
child: ChildProcess;
|
||||
pid: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
startedAt: number;
|
||||
exit?: { exitCode: number | null; signal: NodeJS.Signals | null; at: number };
|
||||
closed: Promise<void>;
|
||||
}
|
||||
|
||||
export class ServerHandle {
|
||||
readonly label: string;
|
||||
/** Effective HTTP_ADDR value (undefined when unset → app default 0.0.0.0). */
|
||||
readonly addr: string | undefined;
|
||||
/** Effective port (80 when HTTP_PORT is unset). */
|
||||
readonly port: number;
|
||||
/** Address the harness connects to (0.0.0.0 → 127.0.0.1, :: → ::1). */
|
||||
readonly connectHost: string;
|
||||
/** e.g. http://127.0.0.1:21003 or http://[::1]:21003 (no trailing slash). */
|
||||
readonly baseURL: string;
|
||||
readonly dbPath: string;
|
||||
/** Present when the backend is a FakeDocker. */
|
||||
readonly fakeDocker?: FakeDocker;
|
||||
/** Environment of the server process. */
|
||||
readonly env: Record<string, string>;
|
||||
private readonly gens: Generation[] = [];
|
||||
|
||||
constructor(
|
||||
private readonly ctx: ServerContext,
|
||||
readonly options: StartServerOptions & { docker: DockerBackend },
|
||||
env: Record<string, string>,
|
||||
port: number,
|
||||
) {
|
||||
this.label = options.label ?? `server${ctx.servers.length + 1}`;
|
||||
this.env = env;
|
||||
this.addr = env.HTTP_ADDR;
|
||||
this.port = port;
|
||||
const a = this.addr ?? '0.0.0.0';
|
||||
this.connectHost = a === '0.0.0.0' ? '127.0.0.1' : a === '::' ? '::1' : a;
|
||||
this.baseURL = `http://${this.connectHost.includes(':') ? `[${this.connectHost}]` : this.connectHost}:${port}`;
|
||||
this.dbPath = env.DB_PATH;
|
||||
if (options.docker instanceof FakeDocker) this.fakeDocker = options.docker;
|
||||
}
|
||||
|
||||
/** baseURL + path, path used verbatim (must start with '/'). */
|
||||
url(path = '/api/v1/containers'): string {
|
||||
return this.baseURL + path;
|
||||
}
|
||||
|
||||
/** Process id of the current generation (the binary itself, also under unshare). */
|
||||
get pid(): number {
|
||||
return this.current().pid;
|
||||
}
|
||||
|
||||
/** How many times the process was (re)started. */
|
||||
get generation(): number {
|
||||
return this.gens.length;
|
||||
}
|
||||
|
||||
/** stdout of the current process generation. */
|
||||
get stdout(): string {
|
||||
return this.current().stdout;
|
||||
}
|
||||
|
||||
/** stderr (logs) of the current process generation. */
|
||||
get stderr(): string {
|
||||
return this.current().stderr;
|
||||
}
|
||||
|
||||
/** stderr of every generation, concatenated with separators (artifacts / secret scans). */
|
||||
allOutput(): Array<{ generation: number; pid: number; stdout: string; stderr: string; exit?: Generation['exit'] }> {
|
||||
return this.gens.map((g, i) => ({ generation: i + 1, pid: g.pid, stdout: g.stdout, stderr: g.stderr, exit: g.exit }));
|
||||
}
|
||||
|
||||
/** Parsed logs of the current generation. */
|
||||
logs(): LogSet {
|
||||
return parseLogs(this.stderr);
|
||||
}
|
||||
|
||||
isRunning(): boolean {
|
||||
const g = this.gens[this.gens.length - 1];
|
||||
return !!g && !g.exit;
|
||||
}
|
||||
|
||||
/** Exit status of the current generation once it exited. */
|
||||
get exit(): ExitResult | undefined {
|
||||
const g = this.current();
|
||||
return g.exit ? { exitCode: g.exit.exitCode, signal: g.exit.signal, durationMs: g.exit.at - g.startedAt, stdout: g.stdout, stderr: g.stderr } : undefined;
|
||||
}
|
||||
|
||||
/** @internal spawns a new generation. */
|
||||
async spawn(): Promise<void> {
|
||||
const opts = this.options;
|
||||
const bin = opts.binary ?? binaryPath();
|
||||
let command: string;
|
||||
let args: string[];
|
||||
const backend = opts.docker;
|
||||
if (backend === 'real' && !opts.privateNetwork) {
|
||||
command = bin;
|
||||
args = [];
|
||||
} else {
|
||||
const flags = opts.privateNetwork ? '-Urmn' : '-Urm';
|
||||
let script: string;
|
||||
let src = '';
|
||||
if (backend === 'real') script = 'exec "$2"';
|
||||
else if (backend === 'missing') script = 'mount -t tmpfs none /run && exec "$2"';
|
||||
else {
|
||||
src = backend === 'inaccessible' ? '/etc/shadow' : backend instanceof FakeDocker ? backend.socketPath : backend.bindSource;
|
||||
script = 'mount --bind "$1" /var/run/docker.sock && exec "$2"';
|
||||
}
|
||||
command = 'unshare';
|
||||
args = [flags, 'sh', '-c', script, 'sh', src, bin];
|
||||
}
|
||||
const child = spawn(command, args, { env: this.env, cwd: this.ctx.workspace.dir, stdio: ['ignore', 'pipe', 'pipe'], detached: true });
|
||||
const spawnErr = await new Promise<Error | undefined>((resolve) => {
|
||||
child.once('spawn', () => resolve(undefined));
|
||||
child.once('error', (e) => resolve(e));
|
||||
});
|
||||
if (spawnErr || child.pid === undefined) throw new HarnessError('spawn', `cannot start server ${this.label}: ${spawnErr?.message}`);
|
||||
trackProcess(child, `server ${this.label}`);
|
||||
const gen: Generation = { child, pid: child.pid, stdout: '', stderr: '', startedAt: performance.now(), closed: Promise.resolve() };
|
||||
const logFile = this.ctx.workspace.path(`${this.label}.gen${this.gens.length + 1}.stderr.log`);
|
||||
child.stdout!.setEncoding('utf8');
|
||||
child.stderr!.setEncoding('utf8');
|
||||
child.stdout!.on('data', (d: string) => (gen.stdout += d));
|
||||
child.stderr!.on('data', (d: string) => {
|
||||
gen.stderr += d;
|
||||
try {
|
||||
fs.appendFileSync(logFile, d);
|
||||
} catch {
|
||||
/* workspace removed during teardown */
|
||||
}
|
||||
});
|
||||
child.once('exit', (exitCode, signal) => {
|
||||
gen.exit = { exitCode, signal, at: performance.now() };
|
||||
});
|
||||
gen.closed = new Promise<void>((resolve) => child.once('close', () => resolve()));
|
||||
this.gens.push(gen);
|
||||
this.ctx.recorder.note(`server ${this.label} gen ${this.gens.length} started pid=${child.pid} cmd=${command} ${JSON.stringify(args)} env=${JSON.stringify(recordableEnv(this.env))}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits until the server answers HTTP: polls `OPTIONS * HTTP/1.1` over a raw socket (answered by
|
||||
* Go's server itself, so it creates no application log record and no audit row). Throws if the
|
||||
* process exits first (with its stderr) or HarnessError('timeout').
|
||||
*/
|
||||
async waitReady(timeoutMs = this.options.readyTimeoutMs ?? 10_000): Promise<void> {
|
||||
const deadline = performance.now() + timeoutMs;
|
||||
while (true) {
|
||||
const g = this.current();
|
||||
if (g.exit) {
|
||||
await g.closed;
|
||||
throw new Error(`server ${this.label} exited during startup (code=${g.exit.exitCode}, signal=${g.exit.signal})\n--- stderr\n${g.stderr}`);
|
||||
}
|
||||
const r = await rawHttp({
|
||||
host: this.connectHost,
|
||||
port: this.port,
|
||||
data: buildRawRequest({ method: 'OPTIONS', target: '*' }),
|
||||
timeoutMs: 2000,
|
||||
allowTimeout: true,
|
||||
}).catch(() => undefined);
|
||||
if (r?.outcome === 'response') return;
|
||||
if (performance.now() > deadline) {
|
||||
throw new HarnessError('timeout', `server ${this.label} not ready via HTTP on ${this.baseURL} within ${timeoutMs} ms\n--- stderr\n${g.stderr}`);
|
||||
}
|
||||
await sleep(20);
|
||||
}
|
||||
}
|
||||
|
||||
/** Waits until a log record matching q appears in the current generation's stderr. */
|
||||
async waitForLog(q: LogQuery, timeoutMs = 10_000): Promise<LogRecord> {
|
||||
const deadline = performance.now() + timeoutMs;
|
||||
while (true) {
|
||||
const r = this.logs().find(q);
|
||||
if (r) return r;
|
||||
if (this.current().exit) {
|
||||
await this.current().closed;
|
||||
const again = this.logs().find(q);
|
||||
if (again) return again;
|
||||
throw new Error(`server ${this.label} exited before logging ${JSON.stringify(q)}\n--- stderr\n${this.stderr}`);
|
||||
}
|
||||
if (performance.now() > deadline) throw new HarnessError('timeout', `log record ${JSON.stringify(q)} not seen within ${timeoutMs} ms`);
|
||||
await sleep(20);
|
||||
}
|
||||
}
|
||||
|
||||
/** Sends a signal to the server process (does not wait). */
|
||||
signal(sig: NodeJS.Signals): void {
|
||||
const g = this.current();
|
||||
if (g.exit) throw new Error(`server ${this.label} is not running`);
|
||||
process.kill(g.pid, sig);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends `sig` (default SIGTERM) and waits for the process to exit and its output to be flushed.
|
||||
* Throws HarnessError('hang') (after SIGKILL) if it does not exit within timeoutMs, and
|
||||
* HarnessError('signal') if it died from a signal (dpsapi handles SIGTERM/SIGINT itself) unless
|
||||
* allowSignalExit. Returns the exit status; assert `exitCode` explicitly.
|
||||
*/
|
||||
async stop(sig: NodeJS.Signals = 'SIGTERM', { timeoutMs = 20_000, allowSignalExit = false }: { timeoutMs?: number; allowSignalExit?: boolean } = {}): Promise<ExitResult> {
|
||||
const g = this.current();
|
||||
const t0 = performance.now();
|
||||
if (!g.exit) process.kill(g.pid, sig);
|
||||
const exited = await Promise.race([g.closed.then(() => true), sleep(timeoutMs).then(() => false)]);
|
||||
if (!exited) {
|
||||
killGroup(g.pid, 'SIGKILL');
|
||||
await Promise.race([g.closed, sleep(5000)]);
|
||||
throw new HarnessError('hang', `server ${this.label} (pid ${g.pid}) did not exit within ${timeoutMs} ms after ${sig}; killed\n--- stderr (tail)\n${g.stderr.slice(-3000)}`);
|
||||
}
|
||||
const res: ExitResult = { exitCode: g.exit!.exitCode, signal: g.exit!.signal, durationMs: g.exit!.at - t0, stdout: g.stdout, stderr: g.stderr };
|
||||
this.ctx.recorder.note(`server ${this.label} gen ${this.gens.length} stopped with ${sig}: exitCode=${res.exitCode} signal=${res.signal}`);
|
||||
if (res.signal !== null && !allowSignalExit) {
|
||||
throw new HarnessError('signal', `server ${this.label} was terminated by signal ${res.signal} after ${sig} instead of exiting\n--- stderr (tail)\n${g.stderr.slice(-3000)}`);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
/** SIGKILLs the process group and waits; never throws for the signal exit. */
|
||||
async kill(): Promise<ExitResult> {
|
||||
const g = this.current();
|
||||
const t0 = performance.now();
|
||||
if (!g.exit) killGroup(g.pid, 'SIGKILL');
|
||||
await Promise.race([g.closed, sleep(10_000)]);
|
||||
if (!g.exit) throw new HarnessError('hang', `server ${this.label} (pid ${g.pid}) survived SIGKILL`);
|
||||
return { exitCode: g.exit.exitCode, signal: g.exit.signal, durationMs: g.exit.at - t0, stdout: g.stdout, stderr: g.stderr };
|
||||
}
|
||||
|
||||
/** Waits for the process to exit on its own (e.g. startup failure). HarnessError('timeout') otherwise. */
|
||||
async waitForExit(timeoutMs = 15_000): Promise<ExitResult> {
|
||||
const g = this.current();
|
||||
const exited = await Promise.race([g.closed.then(() => true), sleep(timeoutMs).then(() => false)]);
|
||||
if (!exited) throw new HarnessError('timeout', `server ${this.label} still running after ${timeoutMs} ms\n--- stderr (tail)\n${g.stderr.slice(-3000)}`);
|
||||
if (g.exit!.signal !== null) throw new HarnessError('signal', `server ${this.label} terminated by signal ${g.exit!.signal}`);
|
||||
return { exitCode: g.exit!.exitCode, signal: g.exit!.signal, durationMs: g.exit!.at - g.startedAt, stdout: g.stdout, stderr: g.stderr };
|
||||
}
|
||||
|
||||
/**
|
||||
* Graceful restart against the same DB/port/backend: SIGTERM, require exit code 0, start a new
|
||||
* process and wait for readiness. stdout/stderr/logs() then refer to the new generation.
|
||||
*/
|
||||
async restart(): Promise<void> {
|
||||
if (this.isRunning()) {
|
||||
const r = await this.stop('SIGTERM');
|
||||
if (r.exitCode !== 0) throw new Error(`server ${this.label} exited with ${r.exitCode} on restart\n${r.stderr}`);
|
||||
}
|
||||
await this.spawn();
|
||||
if (this.options.waitReady ?? !this.options.privateNetwork) await this.waitReady();
|
||||
}
|
||||
|
||||
private current(): Generation {
|
||||
const g = this.gens[this.gens.length - 1];
|
||||
if (!g) throw new Error(`server ${this.label} was never started`);
|
||||
return g;
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolves options, spawns the server and (by default) waits for HTTP readiness. */
|
||||
export async function startServer(ctx: ServerContext, opts: StartServerOptions = {}): Promise<ServerHandle> {
|
||||
const docker = opts.docker ?? (await ctx.newFakeDocker());
|
||||
let port: number;
|
||||
let portEnv: string | undefined;
|
||||
if (opts.port === null) {
|
||||
port = 80;
|
||||
portEnv = undefined;
|
||||
} else if (opts.port === undefined) {
|
||||
port = await ctx.allocatePort();
|
||||
portEnv = String(port);
|
||||
} else {
|
||||
portEnv = String(opts.port);
|
||||
port = Number(opts.port);
|
||||
}
|
||||
const env = mergeEnv(baseAppEnv(opts.dbPath ?? ctx.workspace.dbPath), {
|
||||
HTTP_ADDR: opts.addr === null ? undefined : (opts.addr ?? '127.0.0.1'),
|
||||
HTTP_PORT: portEnv,
|
||||
LOG_LEVEL: opts.logLevel === null ? undefined : (opts.logLevel ?? 'INFO'),
|
||||
...(opts.env ?? {}),
|
||||
});
|
||||
const handle = new ServerHandle(ctx, { ...opts, docker }, env, port);
|
||||
ctx.servers.push(handle);
|
||||
await handle.spawn();
|
||||
if (opts.waitReady ?? !opts.privateNetwork) await handle.waitReady();
|
||||
return handle;
|
||||
}
|
||||
Reference in new issue
Block a user