dpsapi/e2e/harness/fixtures.ts

// SPDX-License-Identifier: GPL-2.0-or-later
// Copyright (C) 2026 Bryan Joshua Pedini

/**
 * Playwright fixtures: per-test isolation, automatic cleanup, leak detection, secret scanning
 * and failure artifacts. Import `test`/`expect` from 'harness' (index.ts), never from
 * '@playwright/test', in spec files.
 */
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { test as base, expect, type TestInfo } from '@playwright/test';
import { CliRunner } from './cli';
import { dbSnapshot } from './db';
import { listByLabel, RealDocker, dockerCli } from './docker';
import { globalSeed, LABEL_TEST, secretsDir } from './env';
import { HarnessError } from './errors';
import { FakeDocker } from './fake-docker';
import { HttpClient } from './http';
import { ImageRunner, type ImageContainer } from './image';
import { killGroup, liveProcesses } from './proc';
import { jsonSafe, Recorder } from './recorder';
import { Rng, testSeed } from './rng';
import { redactText, SecretRegistry } from './secrets';
import { startServer, type ServerHandle, type StartServerOptions } from './server';
import { allocatePort, forceRemoveDir, Workspace } from './workspace';

/** Everything the harness tracks for one test. */
export interface E2EContext {
  /** Short unique key of this test execution (8 hex); used in Docker names/labels and workspace names. */
  testKey: string;
  workspace: Workspace;
  secrets: SecretRegistry;
  recorder: Recorder;
  servers: ServerHandle[];
  fakes: FakeDocker[];
  imageContainers: ImageContainer[];
  /** Extra database files to snapshot on failure (workspace db and server dbs are automatic). */
  trackedDbs: Set<string>;
  /** Set when the `docker`/`image` fixtures were used (enables label-based container cleanup). */
  usedDocker: boolean;
  /** Seed of this test's rng/fuzz streams. */
  seed: string;
  /** Allocates a free TCP port (parallel-safe). */
  allocatePort(): Promise<number>;
  /** Starts a FakeDocker on a new socket in the workspace; closed automatically. */
  newFakeDocker(): Promise<FakeDocker>;
}

/** Records fuzz cases; on failure they are attached (redacted) as fuzz-cases.json together with the seed. */
export class FuzzRecorder {
  constructor(
    /** Seed of this test's stream (reproduce with DPSAPI_E2E_SEED=<run seed>). */
    readonly seed: string,
    /** Deterministic generator for this test. */
    readonly rng: Rng,
    private readonly recorder: Recorder,
    private readonly testInfo: TestInfo,
    private readonly secrets: SecretRegistry,
  ) {}

  /**
   * Records one generated case and returns it unchanged. What is stored is a JSON-safe snapshot
   * taken now (BigInts as decimal strings), so the fuzz-cases.json attachment always serializes;
   * fields added to the case object later are not recorded — record the finished case, or use
   * `recordLive` for a case object that is filled in while the step runs.
   */
  record<T>(c: T): T {
    this.recorder.fuzzCases.push(jsonSafe(c));
    return c;
  }

  /** Records a mutable case object by reference (serialized with BigInt support when attached). */
  recordLive<T extends object>(c: T): T {
    this.recorder.fuzzCases.push(c);
    return c;
  }

  get cases(): readonly unknown[] {
    return this.recorder.fuzzCases;
  }

  /** Attaches the cases now (normally only done automatically on failure). */
  async attach(name = 'fuzz-cases.json'): Promise<void> {
    await this.testInfo.attach(name, {
      body: JSON.stringify(this.secrets.redactValue({ seed: this.seed, runSeed: globalSeed(), cases: this.recorder.fuzzCases }), null, 2),
      contentType: 'application/json',
    });
  }
}

export interface E2EOptions {
  /** Fail the test if a registered token appears in any captured server log / CLI stderr / container log (default true). */
  scanLogsForSecrets: boolean;
}

export interface E2EFixtures {
  /** Per-test harness context (auto fixture; owns teardown, leak checks and artifacts). */
  e2e: E2EContext;
  /** Short unique key of this test execution. */
  testKey: string;
  /** Isolated temp directory; `ws.dbPath` is the default DB_PATH. Removed after the test. */
  ws: Workspace;
  secrets: SecretRegistry;
  recorder: Recorder;
  /** Host binary as CLI (no shell; `cli.shell()` for shell-quoting tests). */
  cli: CliRunner;
  /** Starts the host binary in server mode; must be stopped by the test (leak = failure). */
  startServer: (opts?: StartServerOptions) => Promise<ServerHandle>;
  /** Starts an additional FakeDocker (closed automatically). */
  newFakeDocker: () => Promise<FakeDocker>;
  /** Playwright request / curl / raw-socket HTTP clients, all recorded. */
  http: HttpClient;
  /** Real Docker Engine helpers scoped to this test (cleanup guaranteed). */
  docker: RealDocker;
  /** Image helpers ($DPSAPI_IMAGE); containers removed automatically. */
  image: ImageRunner;
  /** Deterministic PRNG for this test (seed annotated on the test). */
  rng: Rng;
  fuzz: FuzzRecorder;
}

/** Registry of the test currently running in this worker (tests run one at a time per worker). */
let currentSecrets: SecretRegistry | undefined;
let stdioWrapped = false;

/**
 * Redacts registered secrets and token-shaped strings from everything the test process writes to
 * stdout/stderr (console.log in tests), which reporters would otherwise stream verbatim.
 */
function installStdioRedaction(): void {
  if (stdioWrapped) return;
  stdioWrapped = true;
  for (const stream of [process.stdout, process.stderr]) {
    const original = stream.write.bind(stream) as (...a: unknown[]) => boolean;
    stream.write = ((chunk: unknown, ...rest: unknown[]) => {
      const text = typeof chunk === 'string' ? chunk : chunk instanceof Uint8Array ? Buffer.from(chunk).toString('utf8') : undefined;
      return original(text === undefined ? chunk : redactText(text, currentSecrets?.all() ?? []), ...rest);
    }) as typeof stream.write;
  }
}

function keyFor(info: TestInfo): string {
  return crypto.createHash('sha256').update(`${info.testId}:${info.repeatEachIndex}:${info.retry}:${process.pid}`).digest('hex').slice(0, 8);
}

export const test = base.extend<E2EFixtures & E2EOptions>({
  scanLogsForSecrets: [true, { option: true }],

  e2e: [
    async ({ scanLogsForSecrets }, use, testInfo) => {
      const testKey = keyFor(testInfo);
      const workspace = Workspace.create(testKey);
      const secrets = new SecretRegistry(path.join(secretsDir(), `${testKey}.jsonl`));
      installStdioRedaction();
      currentSecrets = secrets;
      const recorder = new Recorder();
      const seed = testSeed(globalSeed(), testInfo.titlePath);
      let fakeSeq = 0;
      const ctx: E2EContext = {
        testKey,
        workspace,
        secrets,
        recorder,
        servers: [],
        fakes: [],
        imageContainers: [],
        trackedDbs: new Set(),
        usedDocker: false,
        seed,
        allocatePort: () => allocatePort(testInfo.parallelIndex),
        newFakeDocker: async () => {
          const f = await FakeDocker.start(workspace.path(`fd${++fakeSeq}.sock`));
          ctx.fakes.push(f);
          return f;
        },
      };

      await use(ctx);

      const problems: string[] = [];
      const attachments: Array<{ name: string; body: string; contentType: string }> = [];
      const red = (s: string) => secrets.redact(s);

      // 1. Server processes must have been stopped by the test.
      for (const s of ctx.servers) {
        if (s.isRunning()) {
          problems.push(`server ${s.label} (pid ${s.pid}) was still running at teardown — tests must stop every server they start`);
          await s.kill().catch((e) => problems.push(`could not kill server ${s.label}: ${(e as Error).message}`));
        }
      }

      // 2. Image containers: collect logs, then remove.
      const containerLogs: Array<{ name: string; stdout: string; stderr: string }> = [];
      for (const c of ctx.imageContainers) {
        const l = await c.logs().catch(() => ({ stdout: '', stderr: '' }));
        containerLogs.push({ name: c.name, ...l });
        await c.remove().catch(() => undefined);
      }

      // 3. Fake Docker sockets.
      for (const f of ctx.fakes) await f.close().catch(() => undefined);

      // 4. Every container labelled for this test (test containers, image containers, helpers).
      if (ctx.usedDocker) {
        try {
          const left = await listByLabel(`${LABEL_TEST}=${testKey}`, true);
          if (left.length) await dockerCli(['rm', '-f', '-v', ...left.map((c) => c.name)], { allowFailure: true, timeoutMs: 120_000 });
          const still = await listByLabel(`${LABEL_TEST}=${testKey}`, true);
          if (still.length) problems.push(`containers survived cleanup: ${still.map((c) => c.name).join(', ')}`);
        } catch (e) {
          problems.push(`docker cleanup failed: ${(e as Error).message}`);
        }
      }

      // 5. Secret scan of everything the application wrote to logs.
      if (scanLogsForSecrets) {
        const scan = (text: string, where: string) => {
          const leaked = secrets.findIn(text);
          if (leaked.length) problems.push(`token secret(s) ${leaked.join(', ')} found in ${where}`);
        };
        for (const s of ctx.servers) for (const g of s.allOutput()) {
          scan(g.stderr, `server ${s.label} gen ${g.generation} stderr`);
          scan(g.stdout, `server ${s.label} gen ${g.generation} stdout`);
        }
        for (const [i, c] of recorder.cli.entries()) scan(c.stderr, `stderr of CLI invocation #${i + 1} (${c.via} ${c.args.slice(0, 2).join(' ')})`);
        for (const c of containerLogs) scan(c.stderr + c.stdout, `container ${c.name} logs`);
      }

      // 6. Failure artifacts (all redacted).
      const failed = problems.length > 0 || testInfo.status !== testInfo.expectedStatus || testInfo.errors.length > 0;
      // Building artifacts must never skip the cleanup below (a throw here used to leak the
      // workspace): an artifact failure is reported as a teardown problem instead.
      if (failed) {
        try {
          for (const s of ctx.servers) for (const g of s.allOutput()) {
            attachments.push({ name: `server-${s.label}-gen${g.generation}.stderr.log`, body: red(g.stderr), contentType: 'text/plain' });
            if (g.stdout) attachments.push({ name: `server-${s.label}-gen${g.generation}.stdout.log`, body: red(g.stdout), contentType: 'text/plain' });
            attachments.push({ name: `server-${s.label}-gen${g.generation}.exit.json`, body: JSON.stringify({ pid: g.pid, exit: g.exit ?? 'still running at teardown', env: s.env.LOG_LEVEL ? { LOG_LEVEL: s.env.LOG_LEVEL, HTTP_ADDR: s.env.HTTP_ADDR, HTTP_PORT: s.env.HTTP_PORT } : {} }), contentType: 'application/json' });
          }
          for (const c of containerLogs) attachments.push({ name: `container-${c.name}.log`, body: red(`--- stdout\n${c.stdout}\n--- stderr\n${c.stderr}`), contentType: 'text/plain' });
          if (recorder.cli.length) attachments.push({ name: 'cli-invocations.json', body: JSON.stringify(secrets.redactValue({ dropped: recorder.droppedCli, invocations: recorder.cli }), null, 2), contentType: 'application/json' });
          if (recorder.http.length) attachments.push({ name: 'http-exchanges.json', body: JSON.stringify(secrets.redactValue({ dropped: recorder.droppedHttp, exchanges: recorder.http }), null, 2), contentType: 'application/json' });
          if (recorder.fuzzCases.length) attachments.push({ name: 'fuzz-cases.json', body: JSON.stringify(secrets.redactValue({ seed, runSeed: globalSeed(), cases: recorder.fuzzCases }), null, 2), contentType: 'application/json' });
          for (const [i, f] of ctx.fakes.entries()) attachments.push({ name: `fake-docker-${i + 1}.json`, body: JSON.stringify({ socket: f.socketPath, mode: f.mode, stats: f.stats, requests: f.requests.slice(-200) }, null, 2), contentType: 'application/json' });
          const dbs = new Set<string>([workspace.dbPath, ...ctx.servers.map((s) => s.dbPath), ...ctx.trackedDbs]);
          let n = 0;
          for (const db of dbs) if (fs.existsSync(db)) attachments.push({ name: `db-snapshot-${++n}.txt`, body: dbSnapshot(db, red), contentType: 'text/plain' });
          if (problems.length) attachments.push({ name: 'harness-problems.txt', body: red(problems.join('\n')), contentType: 'text/plain' });
          if (recorder.notes.length) attachments.push({ name: 'harness-notes.txt', body: red(recorder.notes.join('\n')), contentType: 'text/plain' });
          for (const a of attachments) await testInfo.attach(a.name, { body: a.body, contentType: a.contentType });
        } catch (e) {
          problems.push(`building the failure artifacts failed: ${(e as Error).message}`);
        }
      }

      // 7. Workspace removal (root-owned files handled through a container).
      try {
        await forceRemoveDir(workspace.dir);
        if (fs.existsSync(workspace.dir)) problems.push(`workspace ${workspace.dir} survived removal`);
      } catch (e) {
        problems.push(`workspace removal failed: ${(e as Error).message}`);
      }

      // 8. Any process this worker spawned that is still alive is a leak.
      for (const p of liveProcesses()) {
        problems.push(`process ${p.label} (pid ${p.pid}) still alive at teardown`);
        killGroup(p.pid, 'SIGKILL');
      }

      if (problems.length) throw new HarnessError('leak', `teardown problems:\n- ${problems.map(red).join('\n- ')}`);
    },
    { auto: true },
  ],

  testKey: async ({ e2e }, use) => use(e2e.testKey),
  ws: async ({ e2e }, use) => use(e2e.workspace),
  secrets: async ({ e2e }, use) => use(e2e.secrets),
  recorder: async ({ e2e }, use) => use(e2e.recorder),
  cli: async ({ e2e }, use) => use(new CliRunner({ workspace: e2e.workspace, secrets: e2e.secrets, recorder: e2e.recorder })),

  startServer: async ({ e2e }, use) => {
    await use((opts?: StartServerOptions) =>
      startServer({ workspace: e2e.workspace, recorder: e2e.recorder, allocatePort: e2e.allocatePort, newFakeDocker: e2e.newFakeDocker, servers: e2e.servers }, opts),
    );
  },

  newFakeDocker: async ({ e2e }, use) => use(e2e.newFakeDocker),

  http: async ({ e2e, request }, use) => use(new HttpClient(request, e2e.workspace, e2e.recorder)),

  docker: async ({ e2e }, use) => {
    e2e.usedDocker = true;
    const d = new RealDocker(e2e.testKey);
    await use(d);
    // The e2e teardown removes everything carrying this test's label.
  },

  image: async ({ e2e }, use) => {
    e2e.usedDocker = true;
    await use(
      new ImageRunner({
        workspace: e2e.workspace,
        recorder: e2e.recorder,
        secrets: e2e.secrets,
        testKey: e2e.testKey,
        newFakeDocker: e2e.newFakeDocker,
        containers: e2e.imageContainers,
      }),
    );
  },

  rng: async ({ e2e }, use, testInfo) => {
    testInfo.annotations.push({ type: 'seed', description: `${e2e.seed} (run seed: ${globalSeed()})` });
    await use(new Rng(e2e.seed));
  },

  fuzz: async ({ e2e, rng, secrets }, use, testInfo) => {
    await use(new FuzzRecorder(e2e.seed, rng, e2e.recorder, testInfo, secrets));
  },
});

export { expect };