test(e2e): add playwright e2e harness
Process and API-only Playwright harness: binary and image setup, server/CLI runners, fake Docker Engine socket, read-only database inspection, secret redaction in reports, seeded fuzzing, leak detection and harness self-checks.
This commit is contained in:
bryanpedini committed
1 parent
a02eef13a7
commit
fe0cd4bf6d
33 files changed
+6370
No files matched your search
@@ -0,0 +1,323 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
// Copyright (C) 2026 Bryan Joshua Pedini
|
||||
|
||||
/**
|
||||
* Playwright fixtures: per-test isolation, automatic cleanup, leak detection, secret scanning
|
||||
* and failure artifacts. Import `test`/`expect` from 'harness' (index.ts), never from
|
||||
* '@playwright/test', in spec files.
|
||||
*/
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { test as base, expect, type TestInfo } from '@playwright/test';
|
||||
import { CliRunner } from './cli';
|
||||
import { dbSnapshot } from './db';
|
||||
import { listByLabel, RealDocker, dockerCli } from './docker';
|
||||
import { globalSeed, LABEL_TEST, secretsDir } from './env';
|
||||
import { HarnessError } from './errors';
|
||||
import { FakeDocker } from './fake-docker';
|
||||
import { HttpClient } from './http';
|
||||
import { ImageRunner, type ImageContainer } from './image';
|
||||
import { killGroup, liveProcesses } from './proc';
|
||||
import { jsonSafe, Recorder } from './recorder';
|
||||
import { Rng, testSeed } from './rng';
|
||||
import { redactText, SecretRegistry } from './secrets';
|
||||
import { startServer, type ServerHandle, type StartServerOptions } from './server';
|
||||
import { allocatePort, forceRemoveDir, Workspace } from './workspace';
|
||||
|
||||
/** Everything the harness tracks for one test. */
|
||||
export interface E2EContext {
|
||||
/** Short unique key of this test execution (8 hex); used in Docker names/labels and workspace names. */
|
||||
testKey: string;
|
||||
workspace: Workspace;
|
||||
secrets: SecretRegistry;
|
||||
recorder: Recorder;
|
||||
servers: ServerHandle[];
|
||||
fakes: FakeDocker[];
|
||||
imageContainers: ImageContainer[];
|
||||
/** Extra database files to snapshot on failure (workspace db and server dbs are automatic). */
|
||||
trackedDbs: Set<string>;
|
||||
/** Set when the `docker`/`image` fixtures were used (enables label-based container cleanup). */
|
||||
usedDocker: boolean;
|
||||
/** Seed of this test's rng/fuzz streams. */
|
||||
seed: string;
|
||||
/** Allocates a free TCP port (parallel-safe). */
|
||||
allocatePort(): Promise<number>;
|
||||
/** Starts a FakeDocker on a new socket in the workspace; closed automatically. */
|
||||
newFakeDocker(): Promise<FakeDocker>;
|
||||
}
|
||||
|
||||
/** Records fuzz cases; on failure they are attached (redacted) as fuzz-cases.json together with the seed. */
|
||||
export class FuzzRecorder {
|
||||
constructor(
|
||||
/** Seed of this test's stream (reproduce with DPSAPI_E2E_SEED=<run seed>). */
|
||||
readonly seed: string,
|
||||
/** Deterministic generator for this test. */
|
||||
readonly rng: Rng,
|
||||
private readonly recorder: Recorder,
|
||||
private readonly testInfo: TestInfo,
|
||||
private readonly secrets: SecretRegistry,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Records one generated case and returns it unchanged. What is stored is a JSON-safe snapshot
|
||||
* taken now (BigInts as decimal strings), so the fuzz-cases.json attachment always serializes;
|
||||
* fields added to the case object later are not recorded — record the finished case, or use
|
||||
* `recordLive` for a case object that is filled in while the step runs.
|
||||
*/
|
||||
record<T>(c: T): T {
|
||||
this.recorder.fuzzCases.push(jsonSafe(c));
|
||||
return c;
|
||||
}
|
||||
|
||||
/** Records a mutable case object by reference (serialized with BigInt support when attached). */
|
||||
recordLive<T extends object>(c: T): T {
|
||||
this.recorder.fuzzCases.push(c);
|
||||
return c;
|
||||
}
|
||||
|
||||
get cases(): readonly unknown[] {
|
||||
return this.recorder.fuzzCases;
|
||||
}
|
||||
|
||||
/** Attaches the cases now (normally only done automatically on failure). */
|
||||
async attach(name = 'fuzz-cases.json'): Promise<void> {
|
||||
await this.testInfo.attach(name, {
|
||||
body: JSON.stringify(this.secrets.redactValue({ seed: this.seed, runSeed: globalSeed(), cases: this.recorder.fuzzCases }), null, 2),
|
||||
contentType: 'application/json',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export interface E2EOptions {
|
||||
/** Fail the test if a registered token appears in any captured server log / CLI stderr / container log (default true). */
|
||||
scanLogsForSecrets: boolean;
|
||||
}
|
||||
|
||||
export interface E2EFixtures {
|
||||
/** Per-test harness context (auto fixture; owns teardown, leak checks and artifacts). */
|
||||
e2e: E2EContext;
|
||||
/** Short unique key of this test execution. */
|
||||
testKey: string;
|
||||
/** Isolated temp directory; `ws.dbPath` is the default DB_PATH. Removed after the test. */
|
||||
ws: Workspace;
|
||||
secrets: SecretRegistry;
|
||||
recorder: Recorder;
|
||||
/** Host binary as CLI (no shell; `cli.shell()` for shell-quoting tests). */
|
||||
cli: CliRunner;
|
||||
/** Starts the host binary in server mode; must be stopped by the test (leak = failure). */
|
||||
startServer: (opts?: StartServerOptions) => Promise<ServerHandle>;
|
||||
/** Starts an additional FakeDocker (closed automatically). */
|
||||
newFakeDocker: () => Promise<FakeDocker>;
|
||||
/** Playwright request / curl / raw-socket HTTP clients, all recorded. */
|
||||
http: HttpClient;
|
||||
/** Real Docker Engine helpers scoped to this test (cleanup guaranteed). */
|
||||
docker: RealDocker;
|
||||
/** Image helpers ($DPSAPI_IMAGE); containers removed automatically. */
|
||||
image: ImageRunner;
|
||||
/** Deterministic PRNG for this test (seed annotated on the test). */
|
||||
rng: Rng;
|
||||
fuzz: FuzzRecorder;
|
||||
}
|
||||
|
||||
/** Registry of the test currently running in this worker (tests run one at a time per worker). */
|
||||
let currentSecrets: SecretRegistry | undefined;
|
||||
let stdioWrapped = false;
|
||||
|
||||
/**
|
||||
* Redacts registered secrets and token-shaped strings from everything the test process writes to
|
||||
* stdout/stderr (console.log in tests), which reporters would otherwise stream verbatim.
|
||||
*/
|
||||
function installStdioRedaction(): void {
|
||||
if (stdioWrapped) return;
|
||||
stdioWrapped = true;
|
||||
for (const stream of [process.stdout, process.stderr]) {
|
||||
const original = stream.write.bind(stream) as (...a: unknown[]) => boolean;
|
||||
stream.write = ((chunk: unknown, ...rest: unknown[]) => {
|
||||
const text = typeof chunk === 'string' ? chunk : chunk instanceof Uint8Array ? Buffer.from(chunk).toString('utf8') : undefined;
|
||||
return original(text === undefined ? chunk : redactText(text, currentSecrets?.all() ?? []), ...rest);
|
||||
}) as typeof stream.write;
|
||||
}
|
||||
}
|
||||
|
||||
function keyFor(info: TestInfo): string {
|
||||
return crypto.createHash('sha256').update(`${info.testId}:${info.repeatEachIndex}:${info.retry}:${process.pid}`).digest('hex').slice(0, 8);
|
||||
}
|
||||
|
||||
export const test = base.extend<E2EFixtures & E2EOptions>({
|
||||
scanLogsForSecrets: [true, { option: true }],
|
||||
|
||||
e2e: [
|
||||
async ({ scanLogsForSecrets }, use, testInfo) => {
|
||||
const testKey = keyFor(testInfo);
|
||||
const workspace = Workspace.create(testKey);
|
||||
const secrets = new SecretRegistry(path.join(secretsDir(), `${testKey}.jsonl`));
|
||||
installStdioRedaction();
|
||||
currentSecrets = secrets;
|
||||
const recorder = new Recorder();
|
||||
const seed = testSeed(globalSeed(), testInfo.titlePath);
|
||||
let fakeSeq = 0;
|
||||
const ctx: E2EContext = {
|
||||
testKey,
|
||||
workspace,
|
||||
secrets,
|
||||
recorder,
|
||||
servers: [],
|
||||
fakes: [],
|
||||
imageContainers: [],
|
||||
trackedDbs: new Set(),
|
||||
usedDocker: false,
|
||||
seed,
|
||||
allocatePort: () => allocatePort(testInfo.parallelIndex),
|
||||
newFakeDocker: async () => {
|
||||
const f = await FakeDocker.start(workspace.path(`fd${++fakeSeq}.sock`));
|
||||
ctx.fakes.push(f);
|
||||
return f;
|
||||
},
|
||||
};
|
||||
|
||||
await use(ctx);
|
||||
|
||||
const problems: string[] = [];
|
||||
const attachments: Array<{ name: string; body: string; contentType: string }> = [];
|
||||
const red = (s: string) => secrets.redact(s);
|
||||
|
||||
// 1. Server processes must have been stopped by the test.
|
||||
for (const s of ctx.servers) {
|
||||
if (s.isRunning()) {
|
||||
problems.push(`server ${s.label} (pid ${s.pid}) was still running at teardown — tests must stop every server they start`);
|
||||
await s.kill().catch((e) => problems.push(`could not kill server ${s.label}: ${(e as Error).message}`));
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Image containers: collect logs, then remove.
|
||||
const containerLogs: Array<{ name: string; stdout: string; stderr: string }> = [];
|
||||
for (const c of ctx.imageContainers) {
|
||||
const l = await c.logs().catch(() => ({ stdout: '', stderr: '' }));
|
||||
containerLogs.push({ name: c.name, ...l });
|
||||
await c.remove().catch(() => undefined);
|
||||
}
|
||||
|
||||
// 3. Fake Docker sockets.
|
||||
for (const f of ctx.fakes) await f.close().catch(() => undefined);
|
||||
|
||||
// 4. Every container labelled for this test (test containers, image containers, helpers).
|
||||
if (ctx.usedDocker) {
|
||||
try {
|
||||
const left = await listByLabel(`${LABEL_TEST}=${testKey}`, true);
|
||||
if (left.length) await dockerCli(['rm', '-f', '-v', ...left.map((c) => c.name)], { allowFailure: true, timeoutMs: 120_000 });
|
||||
const still = await listByLabel(`${LABEL_TEST}=${testKey}`, true);
|
||||
if (still.length) problems.push(`containers survived cleanup: ${still.map((c) => c.name).join(', ')}`);
|
||||
} catch (e) {
|
||||
problems.push(`docker cleanup failed: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Secret scan of everything the application wrote to logs.
|
||||
if (scanLogsForSecrets) {
|
||||
const scan = (text: string, where: string) => {
|
||||
const leaked = secrets.findIn(text);
|
||||
if (leaked.length) problems.push(`token secret(s) ${leaked.join(', ')} found in ${where}`);
|
||||
};
|
||||
for (const s of ctx.servers) for (const g of s.allOutput()) {
|
||||
scan(g.stderr, `server ${s.label} gen ${g.generation} stderr`);
|
||||
scan(g.stdout, `server ${s.label} gen ${g.generation} stdout`);
|
||||
}
|
||||
for (const [i, c] of recorder.cli.entries()) scan(c.stderr, `stderr of CLI invocation #${i + 1} (${c.via} ${c.args.slice(0, 2).join(' ')})`);
|
||||
for (const c of containerLogs) scan(c.stderr + c.stdout, `container ${c.name} logs`);
|
||||
}
|
||||
|
||||
// 6. Failure artifacts (all redacted).
|
||||
const failed = problems.length > 0 || testInfo.status !== testInfo.expectedStatus || testInfo.errors.length > 0;
|
||||
// Building artifacts must never skip the cleanup below (a throw here used to leak the
|
||||
// workspace): an artifact failure is reported as a teardown problem instead.
|
||||
if (failed) {
|
||||
try {
|
||||
for (const s of ctx.servers) for (const g of s.allOutput()) {
|
||||
attachments.push({ name: `server-${s.label}-gen${g.generation}.stderr.log`, body: red(g.stderr), contentType: 'text/plain' });
|
||||
if (g.stdout) attachments.push({ name: `server-${s.label}-gen${g.generation}.stdout.log`, body: red(g.stdout), contentType: 'text/plain' });
|
||||
attachments.push({ name: `server-${s.label}-gen${g.generation}.exit.json`, body: JSON.stringify({ pid: g.pid, exit: g.exit ?? 'still running at teardown', env: s.env.LOG_LEVEL ? { LOG_LEVEL: s.env.LOG_LEVEL, HTTP_ADDR: s.env.HTTP_ADDR, HTTP_PORT: s.env.HTTP_PORT } : {} }), contentType: 'application/json' });
|
||||
}
|
||||
for (const c of containerLogs) attachments.push({ name: `container-${c.name}.log`, body: red(`--- stdout\n${c.stdout}\n--- stderr\n${c.stderr}`), contentType: 'text/plain' });
|
||||
if (recorder.cli.length) attachments.push({ name: 'cli-invocations.json', body: JSON.stringify(secrets.redactValue({ dropped: recorder.droppedCli, invocations: recorder.cli }), null, 2), contentType: 'application/json' });
|
||||
if (recorder.http.length) attachments.push({ name: 'http-exchanges.json', body: JSON.stringify(secrets.redactValue({ dropped: recorder.droppedHttp, exchanges: recorder.http }), null, 2), contentType: 'application/json' });
|
||||
if (recorder.fuzzCases.length) attachments.push({ name: 'fuzz-cases.json', body: JSON.stringify(secrets.redactValue({ seed, runSeed: globalSeed(), cases: recorder.fuzzCases }), null, 2), contentType: 'application/json' });
|
||||
for (const [i, f] of ctx.fakes.entries()) attachments.push({ name: `fake-docker-${i + 1}.json`, body: JSON.stringify({ socket: f.socketPath, mode: f.mode, stats: f.stats, requests: f.requests.slice(-200) }, null, 2), contentType: 'application/json' });
|
||||
const dbs = new Set<string>([workspace.dbPath, ...ctx.servers.map((s) => s.dbPath), ...ctx.trackedDbs]);
|
||||
let n = 0;
|
||||
for (const db of dbs) if (fs.existsSync(db)) attachments.push({ name: `db-snapshot-${++n}.txt`, body: dbSnapshot(db, red), contentType: 'text/plain' });
|
||||
if (problems.length) attachments.push({ name: 'harness-problems.txt', body: red(problems.join('\n')), contentType: 'text/plain' });
|
||||
if (recorder.notes.length) attachments.push({ name: 'harness-notes.txt', body: red(recorder.notes.join('\n')), contentType: 'text/plain' });
|
||||
for (const a of attachments) await testInfo.attach(a.name, { body: a.body, contentType: a.contentType });
|
||||
} catch (e) {
|
||||
problems.push(`building the failure artifacts failed: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Workspace removal (root-owned files handled through a container).
|
||||
try {
|
||||
await forceRemoveDir(workspace.dir);
|
||||
if (fs.existsSync(workspace.dir)) problems.push(`workspace ${workspace.dir} survived removal`);
|
||||
} catch (e) {
|
||||
problems.push(`workspace removal failed: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
// 8. Any process this worker spawned that is still alive is a leak.
|
||||
for (const p of liveProcesses()) {
|
||||
problems.push(`process ${p.label} (pid ${p.pid}) still alive at teardown`);
|
||||
killGroup(p.pid, 'SIGKILL');
|
||||
}
|
||||
|
||||
if (problems.length) throw new HarnessError('leak', `teardown problems:\n- ${problems.map(red).join('\n- ')}`);
|
||||
},
|
||||
{ auto: true },
|
||||
],
|
||||
|
||||
testKey: async ({ e2e }, use) => use(e2e.testKey),
|
||||
ws: async ({ e2e }, use) => use(e2e.workspace),
|
||||
secrets: async ({ e2e }, use) => use(e2e.secrets),
|
||||
recorder: async ({ e2e }, use) => use(e2e.recorder),
|
||||
cli: async ({ e2e }, use) => use(new CliRunner({ workspace: e2e.workspace, secrets: e2e.secrets, recorder: e2e.recorder })),
|
||||
|
||||
startServer: async ({ e2e }, use) => {
|
||||
await use((opts?: StartServerOptions) =>
|
||||
startServer({ workspace: e2e.workspace, recorder: e2e.recorder, allocatePort: e2e.allocatePort, newFakeDocker: e2e.newFakeDocker, servers: e2e.servers }, opts),
|
||||
);
|
||||
},
|
||||
|
||||
newFakeDocker: async ({ e2e }, use) => use(e2e.newFakeDocker),
|
||||
|
||||
http: async ({ e2e, request }, use) => use(new HttpClient(request, e2e.workspace, e2e.recorder)),
|
||||
|
||||
docker: async ({ e2e }, use) => {
|
||||
e2e.usedDocker = true;
|
||||
const d = new RealDocker(e2e.testKey);
|
||||
await use(d);
|
||||
// The e2e teardown removes everything carrying this test's label.
|
||||
},
|
||||
|
||||
image: async ({ e2e }, use) => {
|
||||
e2e.usedDocker = true;
|
||||
await use(
|
||||
new ImageRunner({
|
||||
workspace: e2e.workspace,
|
||||
recorder: e2e.recorder,
|
||||
secrets: e2e.secrets,
|
||||
testKey: e2e.testKey,
|
||||
newFakeDocker: e2e.newFakeDocker,
|
||||
containers: e2e.imageContainers,
|
||||
}),
|
||||
);
|
||||
},
|
||||
|
||||
rng: async ({ e2e }, use, testInfo) => {
|
||||
testInfo.annotations.push({ type: 'seed', description: `${e2e.seed} (run seed: ${globalSeed()})` });
|
||||
await use(new Rng(e2e.seed));
|
||||
},
|
||||
|
||||
fuzz: async ({ e2e, rng, secrets }, use, testInfo) => {
|
||||
await use(new FuzzRecorder(e2e.seed, rng, e2e.recorder, testInfo, secrets));
|
||||
},
|
||||
});
|
||||
|
||||
export { expect };
|
||||
Reference in new issue
Block a user