// SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2026 Bryan Joshua Pedini /** * Playwright fixtures: per-test isolation, automatic cleanup, leak detection, secret scanning * and failure artifacts. Import `test`/`expect` from 'harness' (index.ts), never from * '@playwright/test', in spec files. */ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import { test as base, expect, type TestInfo } from '@playwright/test'; import { CliRunner } from './cli'; import { dbSnapshot } from './db'; import { listByLabel, RealDocker, dockerCli } from './docker'; import { globalSeed, LABEL_TEST, secretsDir } from './env'; import { HarnessError } from './errors'; import { FakeDocker } from './fake-docker'; import { HttpClient } from './http'; import { ImageRunner, type ImageContainer } from './image'; import { killGroup, liveProcesses } from './proc'; import { jsonSafe, Recorder } from './recorder'; import { Rng, testSeed } from './rng'; import { redactText, SecretRegistry } from './secrets'; import { startServer, type ServerHandle, type StartServerOptions } from './server'; import { allocatePort, forceRemoveDir, Workspace } from './workspace'; /** Everything the harness tracks for one test. */ export interface E2EContext { /** Short unique key of this test execution (8 hex); used in Docker names/labels and workspace names. */ testKey: string; workspace: Workspace; secrets: SecretRegistry; recorder: Recorder; servers: ServerHandle[]; fakes: FakeDocker[]; imageContainers: ImageContainer[]; /** Extra database files to snapshot on failure (workspace db and server dbs are automatic). */ trackedDbs: Set; /** Set when the `docker`/`image` fixtures were used (enables label-based container cleanup). */ usedDocker: boolean; /** Seed of this test's rng/fuzz streams. */ seed: string; /** Allocates a free TCP port (parallel-safe). */ allocatePort(): Promise; /** Starts a FakeDocker on a new socket in the workspace; closed automatically. */ newFakeDocker(): Promise; } /** Records fuzz cases; on failure they are attached (redacted) as fuzz-cases.json together with the seed. */ export class FuzzRecorder { constructor( /** Seed of this test's stream (reproduce with DPSAPI_E2E_SEED=). */ readonly seed: string, /** Deterministic generator for this test. */ readonly rng: Rng, private readonly recorder: Recorder, private readonly testInfo: TestInfo, private readonly secrets: SecretRegistry, ) {} /** * Records one generated case and returns it unchanged. What is stored is a JSON-safe snapshot * taken now (BigInts as decimal strings), so the fuzz-cases.json attachment always serializes; * fields added to the case object later are not recorded — record the finished case, or use * `recordLive` for a case object that is filled in while the step runs. */ record(c: T): T { this.recorder.fuzzCases.push(jsonSafe(c)); return c; } /** Records a mutable case object by reference (serialized with BigInt support when attached). */ recordLive(c: T): T { this.recorder.fuzzCases.push(c); return c; } get cases(): readonly unknown[] { return this.recorder.fuzzCases; } /** Attaches the cases now (normally only done automatically on failure). */ async attach(name = 'fuzz-cases.json'): Promise { await this.testInfo.attach(name, { body: JSON.stringify(this.secrets.redactValue({ seed: this.seed, runSeed: globalSeed(), cases: this.recorder.fuzzCases }), null, 2), contentType: 'application/json', }); } } export interface E2EOptions { /** Fail the test if a registered token appears in any captured server log / CLI stderr / container log (default true). */ scanLogsForSecrets: boolean; } export interface E2EFixtures { /** Per-test harness context (auto fixture; owns teardown, leak checks and artifacts). */ e2e: E2EContext; /** Short unique key of this test execution. */ testKey: string; /** Isolated temp directory; `ws.dbPath` is the default DB_PATH. Removed after the test. */ ws: Workspace; secrets: SecretRegistry; recorder: Recorder; /** Host binary as CLI (no shell; `cli.shell()` for shell-quoting tests). */ cli: CliRunner; /** Starts the host binary in server mode; must be stopped by the test (leak = failure). */ startServer: (opts?: StartServerOptions) => Promise; /** Starts an additional FakeDocker (closed automatically). */ newFakeDocker: () => Promise; /** Playwright request / curl / raw-socket HTTP clients, all recorded. */ http: HttpClient; /** Real Docker Engine helpers scoped to this test (cleanup guaranteed). */ docker: RealDocker; /** Image helpers ($DPSAPI_IMAGE); containers removed automatically. */ image: ImageRunner; /** Deterministic PRNG for this test (seed annotated on the test). */ rng: Rng; fuzz: FuzzRecorder; } /** Registry of the test currently running in this worker (tests run one at a time per worker). */ let currentSecrets: SecretRegistry | undefined; let stdioWrapped = false; /** * Redacts registered secrets and token-shaped strings from everything the test process writes to * stdout/stderr (console.log in tests), which reporters would otherwise stream verbatim. */ function installStdioRedaction(): void { if (stdioWrapped) return; stdioWrapped = true; for (const stream of [process.stdout, process.stderr]) { const original = stream.write.bind(stream) as (...a: unknown[]) => boolean; stream.write = ((chunk: unknown, ...rest: unknown[]) => { const text = typeof chunk === 'string' ? chunk : chunk instanceof Uint8Array ? Buffer.from(chunk).toString('utf8') : undefined; return original(text === undefined ? chunk : redactText(text, currentSecrets?.all() ?? []), ...rest); }) as typeof stream.write; } } function keyFor(info: TestInfo): string { return crypto.createHash('sha256').update(`${info.testId}:${info.repeatEachIndex}:${info.retry}:${process.pid}`).digest('hex').slice(0, 8); } export const test = base.extend({ scanLogsForSecrets: [true, { option: true }], e2e: [ async ({ scanLogsForSecrets }, use, testInfo) => { const testKey = keyFor(testInfo); const workspace = Workspace.create(testKey); const secrets = new SecretRegistry(path.join(secretsDir(), `${testKey}.jsonl`)); installStdioRedaction(); currentSecrets = secrets; const recorder = new Recorder(); const seed = testSeed(globalSeed(), testInfo.titlePath); let fakeSeq = 0; const ctx: E2EContext = { testKey, workspace, secrets, recorder, servers: [], fakes: [], imageContainers: [], trackedDbs: new Set(), usedDocker: false, seed, allocatePort: () => allocatePort(testInfo.parallelIndex), newFakeDocker: async () => { const f = await FakeDocker.start(workspace.path(`fd${++fakeSeq}.sock`)); ctx.fakes.push(f); return f; }, }; await use(ctx); const problems: string[] = []; const attachments: Array<{ name: string; body: string; contentType: string }> = []; const red = (s: string) => secrets.redact(s); // 1. Server processes must have been stopped by the test. for (const s of ctx.servers) { if (s.isRunning()) { problems.push(`server ${s.label} (pid ${s.pid}) was still running at teardown — tests must stop every server they start`); await s.kill().catch((e) => problems.push(`could not kill server ${s.label}: ${(e as Error).message}`)); } } // 2. Image containers: collect logs, then remove. const containerLogs: Array<{ name: string; stdout: string; stderr: string }> = []; for (const c of ctx.imageContainers) { const l = await c.logs().catch(() => ({ stdout: '', stderr: '' })); containerLogs.push({ name: c.name, ...l }); await c.remove().catch(() => undefined); } // 3. Fake Docker sockets. for (const f of ctx.fakes) await f.close().catch(() => undefined); // 4. Every container labelled for this test (test containers, image containers, helpers). if (ctx.usedDocker) { try { const left = await listByLabel(`${LABEL_TEST}=${testKey}`, true); if (left.length) await dockerCli(['rm', '-f', '-v', ...left.map((c) => c.name)], { allowFailure: true, timeoutMs: 120_000 }); const still = await listByLabel(`${LABEL_TEST}=${testKey}`, true); if (still.length) problems.push(`containers survived cleanup: ${still.map((c) => c.name).join(', ')}`); } catch (e) { problems.push(`docker cleanup failed: ${(e as Error).message}`); } } // 5. Secret scan of everything the application wrote to logs. if (scanLogsForSecrets) { const scan = (text: string, where: string) => { const leaked = secrets.findIn(text); if (leaked.length) problems.push(`token secret(s) ${leaked.join(', ')} found in ${where}`); }; for (const s of ctx.servers) for (const g of s.allOutput()) { scan(g.stderr, `server ${s.label} gen ${g.generation} stderr`); scan(g.stdout, `server ${s.label} gen ${g.generation} stdout`); } for (const [i, c] of recorder.cli.entries()) scan(c.stderr, `stderr of CLI invocation #${i + 1} (${c.via} ${c.args.slice(0, 2).join(' ')})`); for (const c of containerLogs) scan(c.stderr + c.stdout, `container ${c.name} logs`); } // 6. Failure artifacts (all redacted). const failed = problems.length > 0 || testInfo.status !== testInfo.expectedStatus || testInfo.errors.length > 0; // Building artifacts must never skip the cleanup below (a throw here used to leak the // workspace): an artifact failure is reported as a teardown problem instead. if (failed) { try { for (const s of ctx.servers) for (const g of s.allOutput()) { attachments.push({ name: `server-${s.label}-gen${g.generation}.stderr.log`, body: red(g.stderr), contentType: 'text/plain' }); if (g.stdout) attachments.push({ name: `server-${s.label}-gen${g.generation}.stdout.log`, body: red(g.stdout), contentType: 'text/plain' }); attachments.push({ name: `server-${s.label}-gen${g.generation}.exit.json`, body: JSON.stringify({ pid: g.pid, exit: g.exit ?? 'still running at teardown', env: s.env.LOG_LEVEL ? { LOG_LEVEL: s.env.LOG_LEVEL, HTTP_ADDR: s.env.HTTP_ADDR, HTTP_PORT: s.env.HTTP_PORT } : {} }), contentType: 'application/json' }); } for (const c of containerLogs) attachments.push({ name: `container-${c.name}.log`, body: red(`--- stdout\n${c.stdout}\n--- stderr\n${c.stderr}`), contentType: 'text/plain' }); if (recorder.cli.length) attachments.push({ name: 'cli-invocations.json', body: JSON.stringify(secrets.redactValue({ dropped: recorder.droppedCli, invocations: recorder.cli }), null, 2), contentType: 'application/json' }); if (recorder.http.length) attachments.push({ name: 'http-exchanges.json', body: JSON.stringify(secrets.redactValue({ dropped: recorder.droppedHttp, exchanges: recorder.http }), null, 2), contentType: 'application/json' }); if (recorder.fuzzCases.length) attachments.push({ name: 'fuzz-cases.json', body: JSON.stringify(secrets.redactValue({ seed, runSeed: globalSeed(), cases: recorder.fuzzCases }), null, 2), contentType: 'application/json' }); for (const [i, f] of ctx.fakes.entries()) attachments.push({ name: `fake-docker-${i + 1}.json`, body: JSON.stringify({ socket: f.socketPath, mode: f.mode, stats: f.stats, requests: f.requests.slice(-200) }, null, 2), contentType: 'application/json' }); const dbs = new Set([workspace.dbPath, ...ctx.servers.map((s) => s.dbPath), ...ctx.trackedDbs]); let n = 0; for (const db of dbs) if (fs.existsSync(db)) attachments.push({ name: `db-snapshot-${++n}.txt`, body: dbSnapshot(db, red), contentType: 'text/plain' }); if (problems.length) attachments.push({ name: 'harness-problems.txt', body: red(problems.join('\n')), contentType: 'text/plain' }); if (recorder.notes.length) attachments.push({ name: 'harness-notes.txt', body: red(recorder.notes.join('\n')), contentType: 'text/plain' }); for (const a of attachments) await testInfo.attach(a.name, { body: a.body, contentType: a.contentType }); } catch (e) { problems.push(`building the failure artifacts failed: ${(e as Error).message}`); } } // 7. Workspace removal (root-owned files handled through a container). try { await forceRemoveDir(workspace.dir); if (fs.existsSync(workspace.dir)) problems.push(`workspace ${workspace.dir} survived removal`); } catch (e) { problems.push(`workspace removal failed: ${(e as Error).message}`); } // 8. Any process this worker spawned that is still alive is a leak. for (const p of liveProcesses()) { problems.push(`process ${p.label} (pid ${p.pid}) still alive at teardown`); killGroup(p.pid, 'SIGKILL'); } if (problems.length) throw new HarnessError('leak', `teardown problems:\n- ${problems.map(red).join('\n- ')}`); }, { auto: true }, ], testKey: async ({ e2e }, use) => use(e2e.testKey), ws: async ({ e2e }, use) => use(e2e.workspace), secrets: async ({ e2e }, use) => use(e2e.secrets), recorder: async ({ e2e }, use) => use(e2e.recorder), cli: async ({ e2e }, use) => use(new CliRunner({ workspace: e2e.workspace, secrets: e2e.secrets, recorder: e2e.recorder })), startServer: async ({ e2e }, use) => { await use((opts?: StartServerOptions) => startServer({ workspace: e2e.workspace, recorder: e2e.recorder, allocatePort: e2e.allocatePort, newFakeDocker: e2e.newFakeDocker, servers: e2e.servers }, opts), ); }, newFakeDocker: async ({ e2e }, use) => use(e2e.newFakeDocker), http: async ({ e2e, request }, use) => use(new HttpClient(request, e2e.workspace, e2e.recorder)), docker: async ({ e2e }, use) => { e2e.usedDocker = true; const d = new RealDocker(e2e.testKey); await use(d); // The e2e teardown removes everything carrying this test's label. }, image: async ({ e2e }, use) => { e2e.usedDocker = true; await use( new ImageRunner({ workspace: e2e.workspace, recorder: e2e.recorder, secrets: e2e.secrets, testKey: e2e.testKey, newFakeDocker: e2e.newFakeDocker, containers: e2e.imageContainers, }), ); }, rng: async ({ e2e }, use, testInfo) => { testInfo.annotations.push({ type: 'seed', description: `${e2e.seed} (run seed: ${globalSeed()})` }); await use(new Rng(e2e.seed)); }, fuzz: async ({ e2e, rng, secrets }, use, testInfo) => { await use(new FuzzRecorder(e2e.seed, rng, e2e.recorder, testInfo, secrets)); }, }); export { expect };