Files
dpsapi/e2e/harness/cli.ts
T
bryanpedini fe0cd4bf6d test(e2e): add playwright e2e harness
Process and API-only Playwright harness: binary and image setup, server/CLI runners, fake Docker Engine socket, read-only database inspection, secret redaction in reports, seeded fuzzing, leak detection and harness self-checks.
2026-10-06 10:52:25 +02:00

519 lines
23 KiB
TypeScript

// SPDX-License-Identifier: GPL-2.0-or-later
// Copyright (C) 2026 Bryan Joshua Pedini
/**
* Running the compiled binary as a CLI, parsing its output, and asserting outcomes.
*/
import { expect } from '@playwright/test';
import { RFC3339NANO_UTC_RE } from './db';
import { binaryPath, PROC_MARKER_ENV, runId } from './env';
import { HarnessError } from './errors';
import { goQuote, goUnquoteBytes } from './gostrings';
import { runProcess, type ProcResult } from './proc';
import type { CliVia, Recorder } from './recorder';
import { redactText, TOKEN_RE, type SecretRegistry } from './secrets';
import type { Workspace } from './workspace';
export interface CliResult extends ProcResult {
/** Application arguments (after the binary / `docker exec <c> /app`), or [script] for shell runs. */
args: string[];
via: CliVia;
}
export interface CliOptions {
/** Environment overrides on top of the default app env; `undefined` removes a variable. */
env?: Record<string, string | undefined>;
cwd?: string;
/** Default 30 s. Server-mode invocations expected to fail fast should pass ~10 s. */
timeoutMs?: number;
input?: string | Buffer;
/** Do not throw HarnessError on signal termination (only for tests that deliver signals). */
allowSignal?: boolean;
}
/** Usage text: printed to stderr after every usage error, and to stdout by `help`. */
export const USAGE_TEXT =
'usage:\n' +
' dpsapi start the HTTP server\n' +
' dpsapi help print this usage text\n' +
' dpsapi token list [all] list the active tokens, or every token with "all"\n' +
' dpsapi token new [description] create a token and print it\n' +
' dpsapi token delete <token> deactivate a token\n' +
' dpsapi audit print the latest 10 audit rows\n' +
' dpsapi audit <n> print the latest n audit rows\n' +
' dpsapi audit <n><unit> print the audit rows of the last n units (s, m, h, d, w)\n' +
' dpsapi audit all print every audit row\n';
/** Every usage error message (text after `dpsapi: usage error: `) the CLI can print. */
export const USAGE_ERROR_MESSAGES = [
'unknown command',
'help takes no arguments',
'missing token subcommand',
'unknown token subcommand',
'token list takes at most one argument',
'unknown token list argument',
'token new takes at most one argument',
'token delete requires exactly one argument',
'audit takes at most one argument',
'invalid audit argument',
] as const;
// ---------------------------------------------------------------------------------------------
// Padded tables (Go text/tabwriter: minwidth 0, tabwidth 0, padding 2, padchar ' ', flags 0)
// ---------------------------------------------------------------------------------------------
/** Length in Unicode code points (what tabwriter counts: runes, not bytes, not display width). */
export function runeLength(s: string): number {
return Array.from(s).length;
}
/**
* Renders lines of cells exactly like the app's tabwriter: every column but the last is padded
* with spaces to (2 + the longest cell of that column over all lines, in runes); the last cell is
* written as-is; every line ends with '\n'. All lines must have the same number of cells.
*/
export function padTable(lines: ReadonlyArray<readonly string[]>): string {
if (lines.length === 0) return '';
const n = lines[0].length;
for (const l of lines) if (l.length !== n) throw new Error(`padTable: every line needs ${n} cells, got ${l.length}`);
const widths = Array.from({ length: Math.max(0, n - 1) }, (_, c) => 2 + Math.max(...lines.map((l) => runeLength(l[c]))));
return lines.map((l) => l.map((cell, c) => (c < n - 1 ? cell + ' '.repeat(widths[c] - runeLength(cell)) : cell)).join('') + '\n').join('');
}
/** Header cells of `token list` / `token list all`. */
export const TOKEN_LIST_COLUMNS = ['TOKEN', 'DESCRIPTION', 'ACTIVE', 'CREATED_AT', 'DELETED_AT'] as const;
/** Header cells of `audit`. */
export const AUDIT_COLUMNS = ['TIME', 'TOKEN', 'ORIGIN', 'OPERATION'] as const;
/** Exact stdout of `token list [all]` with no rows (the header padded by its own widths). */
export const EMPTY_TOKEN_LIST = padTable([TOKEN_LIST_COLUMNS]);
/** Exact stdout of `audit` with no rows. */
export const EMPTY_AUDIT = padTable([AUDIT_COLUMNS]);
/**
* True for commands whose successful stdout carries full token values (`token new`,
* `token list [all]`, `audit [all | n | n<unit>]`): the runners register every token-shaped string of their stdout
* as a secret. (`help` prints only the usage text.)
*/
export function printsTokens(args: readonly string[]): boolean {
return (args[0] === 'token' && (args[1] === 'new' || args[1] === 'list')) || args[0] === 'audit';
}
/** Error-message rendering of (part of) an output: clipped, token-shaped strings redacted. */
function show(s: string | undefined): string {
const v = s ?? '<missing>';
return JSON.stringify(redactText(v.length > 200 ? v.slice(0, 200) + '…' : v, []));
}
/** Splits table stdout into lines (must end with a newline; at least the header). */
function tableLines(stdout: string, what: string): string[] {
if (!stdout.endsWith('\n')) throw new Error(`${what} output does not end with a newline: ${show(stdout.slice(-120))}`);
return stdout.slice(0, -1).split('\n');
}
/** The re-rendered table must equal the output byte for byte (padding, widths, no trailing blanks). */
function expectRerenders(stdout: string, cells: string[][], what: string): void {
const again = padTable(cells);
if (again === stdout) return;
const a = stdout.split('\n');
const b = again.split('\n');
const i = a.findIndex((l, k) => l !== b[k]);
throw new Error(`${what} output is not the padded table of its own cells (first differing line ${i}): got ${show(a[i])}, expected ${show(b[i])}`);
}
export interface AuditRow {
/** TIME exactly as printed (= as stored). */
time: string;
/** Full token value. */
token: string;
origin: string;
operation: string;
/** The raw (padded) line. */
line: string;
}
/**
* Parses `audit` stdout strictly: trailing newline, the header, rows of TIME / TOKEN / ORIGIN
* (no blanks inside, separated by runs of spaces) and OPERATION (the rest of the line), TIME in
* strict UTC RFC3339Nano, TOKEN 43-char base64url; finally the whole output must re-render byte
* for byte as the padded table of the parsed cells. Throws on any deviation (malformed output is
* a test failure). Token values are redacted in the error message.
*/
export function parseAuditOutput(stdout: string): AuditRow[] {
const lines = tableLines(stdout, 'audit');
const header = lines[0].split(/ +/);
if (header.join('\t') !== AUDIT_COLUMNS.join('\t')) throw new Error(`unexpected audit header: ${show(lines[0])}`);
const rows = lines.slice(1).map((line, i) => {
const m = /^(\S+) +(\S+) +(\S+) +(\S.*)$/.exec(line);
if (!m) throw new Error(`audit row ${i + 1} does not have 4 space-separated cells: ${show(line)}`);
const [, time, token, origin, operation] = m;
if (!RFC3339NANO_UTC_RE.test(time)) throw new Error(`audit row ${i + 1}: TIME ${show(time)} is not strict UTC RFC3339Nano`);
if (!TOKEN_RE.test(token)) throw new Error(`audit row ${i + 1}: TOKEN is not a 43-character base64url token: ${show(token)}`);
return { time, token, origin, operation, line };
});
expectRerenders(stdout, [[...AUDIT_COLUMNS], ...rows.map((r) => [r.time, r.token, r.origin, r.operation])], 'audit');
return rows;
}
/** The exact `audit` stdout for the given rows (padded header + one line per row, in the given order). */
export function formatAuditOutput(rows: ReadonlyArray<{ time: string; token: string; origin: string; operation: string }>): string {
return padTable([[...AUDIT_COLUMNS], ...rows.map((r) => [r.time, r.token, r.origin, r.operation])]);
}
export interface TokenListRow {
token: string;
/** DESCRIPTION column exactly as printed (Go-quoted). */
descriptionQuoted: string;
/** Unquoted description bytes. */
descriptionBytes: Buffer;
/** Unquoted description as UTF-8 text. */
description: string;
active: boolean;
createdAt: string;
/** null when the column is `-`. */
deletedAt: string | null;
/** The raw (padded) line. */
line: string;
}
/** Index just after the closing quote of the Go double-quoted literal starting at s[start] (-1 if unterminated). */
function endOfQuoted(s: string, start: number): number {
if (s[start] !== '"') return -1;
for (let i = start + 1; i < s.length; i++) {
if (s[i] === '\\') i++;
else if (s[i] === '"') return i + 1;
}
return -1;
}
/**
* Parses `token list` / `token list all` stdout strictly: trailing newline, the header, rows of
* TOKEN (the first 43 characters, base64url), spaces, a Go-quoted DESCRIPTION (parsed as a quoted
* literal: it may contain spaces and escaped quotes), then ACTIVE (`true`/`false`), CREATED_AT
* (strict RFC3339Nano UTC) and DELETED_AT (`-` or a timestamp) separated by runs of spaces;
* finally the whole output must re-render byte for byte as the padded table of the parsed cells.
* Throws on any deviation: malformed output is a test failure.
*/
export function parseTokenList(stdout: string): TokenListRow[] {
const lines = tableLines(stdout, 'token list');
const header = lines[0].split(/ +/);
if (header.join('\t') !== TOKEN_LIST_COLUMNS.join('\t')) throw new Error(`unexpected token list header: ${show(lines[0])}`);
const rows = lines.slice(1).map((line, i): TokenListRow => {
const fail = (why: string): never => {
throw new Error(`token list row ${i + 1}: ${why}: ${show(line)}`);
};
const token = line.slice(0, 43);
if (!TOKEN_RE.test(token)) fail('TOKEN is not 43 base64url characters');
const q = /^ +/.exec(line.slice(43));
if (!q) fail('no spaces after TOKEN');
const qStart = 43 + q![0].length;
const qEnd = endOfQuoted(line, qStart);
if (qEnd < 0) fail('DESCRIPTION is not a terminated Go-quoted string');
const descriptionQuoted = line.slice(qStart, qEnd);
let descriptionBytes: Buffer;
try {
descriptionBytes = goUnquoteBytes(descriptionQuoted);
} catch (e) {
return fail(`DESCRIPTION is not a valid Go-quoted string (${(e as Error).message})`);
}
const rest = /^ +(true|false) +(\S+) +(\S+)$/.exec(line.slice(qEnd));
if (!rest) fail('expected ACTIVE, CREATED_AT and DELETED_AT after DESCRIPTION');
const [, active, createdAt, deletedAt] = rest!;
if (!RFC3339NANO_UTC_RE.test(createdAt)) fail(`CREATED_AT ${show(createdAt)} is not strict UTC RFC3339Nano`);
if (deletedAt !== '-' && !RFC3339NANO_UTC_RE.test(deletedAt)) fail(`DELETED_AT ${show(deletedAt)} is neither "-" nor strict UTC RFC3339Nano`);
return {
token,
descriptionQuoted,
descriptionBytes,
description: descriptionBytes.toString('utf8'),
active: active === 'true',
createdAt,
deletedAt: deletedAt === '-' ? null : deletedAt,
line,
};
});
expectRerenders(stdout, [[...TOKEN_LIST_COLUMNS], ...rows.map((r) => [r.token, r.descriptionQuoted, String(r.active), r.createdAt, r.deletedAt ?? '-'])], 'token list');
return rows;
}
/** One row as `token list` prints it; `description` is the exact stored text/bytes (Go-quoted here). */
export interface TokenListEntry {
token: string;
description: string | Buffer;
active: boolean;
createdAt: string;
deletedAt: string | null;
}
/** The exact `token list [all]` stdout for the given rows (padded header + one line per row, in the given order). */
export function formatTokenList(rows: ReadonlyArray<TokenListEntry>): string {
return padTable([[...TOKEN_LIST_COLUMNS], ...rows.map((r) => [r.token, goQuote(r.description), String(r.active), r.createdAt, r.deletedAt ?? '-'])]);
}
/** Throws HarnessError unless the process exited on its own with an exit status. */
export function expectExitedNormally(r: ProcResult, label = r.command): void {
if (r.timedOut) throw new HarnessError('timeout', `${label} timed out (killed by the harness)`);
if (r.signal !== null || r.exitCode === null) throw new HarnessError('signal', `${label} terminated by signal ${r.signal}`);
}
export interface CliSuccessExpectation {
/** Exact stdout or pattern. */
stdout?: string | RegExp;
/** Exact stderr or pattern ('' = must be empty). */
stderr?: string | RegExp;
}
/** Asserts a normal exit with status 0 (and optional exact stdout/stderr). */
export function expectCliSuccess(r: ProcResult, exp: CliSuccessExpectation = {}): void {
expectExitedNormally(r);
expect(r.exitCode, `exit code (stderr: ${r.stderr.slice(-500)})`).toBe(0);
if (exp.stdout !== undefined) {
if (typeof exp.stdout === 'string') expect(r.stdout, 'stdout').toBe(exp.stdout);
else expect(r.stdout, 'stdout').toMatch(exp.stdout);
}
if (exp.stderr !== undefined) {
if (typeof exp.stderr === 'string') expect(r.stderr, 'stderr').toBe(exp.stderr);
else expect(r.stderr, 'stderr').toMatch(exp.stderr);
}
}
export interface CliFailureExpectation {
/** Required: the exact expected exit status (1 runtime failure, 2 usage/configuration). */
exitCode: number;
/** Text after `dpsapi: ` on the error line (exact string or pattern). */
message?: string | RegExp;
/** Usage text expected after the error line. Default: message is a string starting with 'usage error:'. */
usage?: boolean;
/**
* true (default): stderr must be exactly the error line (+ usage text) — right for the default
* INFO level. false: only the error line (+ usage) at the end is checked (DEBUG/TRACE runs).
*/
exactStderr?: boolean;
/** Expected stdout (default ''). */
stdout?: string;
}
/**
* Asserts an expected application failure. First requires a normal exit (a signal/timeout is a
* HarnessError, never an expected failure), then exit status, stdout, and the stderr error line.
*/
export function expectCliFailure(r: ProcResult, exp: CliFailureExpectation): void {
expectExitedNormally(r);
expect(r.exitCode, `exit code (stderr: ${r.stderr.slice(-500)})`).toBe(exp.exitCode);
expect(r.stdout, 'stdout').toBe(exp.stdout ?? '');
const usage = exp.usage ?? (typeof exp.message === 'string' && exp.message.startsWith('usage error:'));
const suffix = usage ? USAGE_TEXT : '';
expect(r.stderr.endsWith('\n' + suffix) || r.stderr.endsWith(suffix), 'stderr ends with usage text').toBe(true);
const head = suffix ? r.stderr.slice(0, r.stderr.length - suffix.length) : r.stderr;
const lines = head.split('\n');
if (lines[lines.length - 1] === '') lines.pop();
const errorLine = lines[lines.length - 1] ?? '';
expect(errorLine.startsWith('dpsapi: '), `error line ${JSON.stringify(errorLine)} starts with "dpsapi: "`).toBe(true);
const message = errorLine.slice('dpsapi: '.length);
if (exp.message !== undefined) {
if (typeof exp.message === 'string') expect(message, 'error message').toBe(exp.message);
else expect(message, 'error message').toMatch(exp.message);
}
if (exp.exactStderr ?? true) expect(lines.length, `stderr has only the error line:\n${r.stderr}`).toBe(1);
}
/** Last line of stderr starting with `dpsapi: `, without the prefix (undefined when none). */
export function errorMessage(r: ProcResult): string | undefined {
const line = r.stderr
.split('\n')
.filter((l) => l.startsWith('dpsapi: '))
.pop();
return line?.slice('dpsapi: '.length);
}
export interface CliContext {
workspace: Workspace;
secrets: SecretRegistry;
recorder: Recorder;
}
/** Default environment of every app process: PATH, DB_PATH (workspace default) and the leak marker. */
export function baseAppEnv(dbPath: string): Record<string, string> {
return { PATH: process.env.PATH || '/usr/local/bin:/usr/bin:/bin', DB_PATH: dbPath, [PROC_MARKER_ENV]: runId() };
}
/** Applies overrides; `undefined` deletes. */
export function mergeEnv(base: Record<string, string>, over: Record<string, string | undefined> = {}): Record<string, string> {
const env = { ...base };
for (const [k, v] of Object.entries(over)) {
if (v === undefined) delete env[k];
else env[k] = v;
}
return env;
}
/** Env subset recorded in artifacts (no PATH / marker). */
export function recordableEnv(env: Record<string, string | undefined>): Record<string, string> {
const out: Record<string, string> = {};
for (const [k, v] of Object.entries(env)) if (k !== 'PATH' && k !== PROC_MARKER_ENV && v !== undefined) out[k] = v;
return out;
}
/** Runs the host binary as a CLI. Bound to a test's workspace/secrets/recorder by the `cli` fixture. */
export class CliRunner {
/** Absolute path of the binary this runner executes. */
readonly binary: string;
constructor(
private readonly ctx: CliContext,
binary: string = binaryPath(),
) {
this.binary = binary;
}
/** Same workspace/secrets/recorder, different binary (e.g. /app extracted from the image). */
withBinary(binary: string): CliRunner {
return new CliRunner(this.ctx, binary);
}
/** Default environment (PATH, DB_PATH=<workspace>/dpsapi.db, leak marker) with overrides applied. */
env(over?: Record<string, string | undefined>): Record<string, string> {
return mergeEnv(baseAppEnv(this.ctx.workspace.dbPath), over);
}
/**
* Runs `<binary> ...args` without a shell: every argument reaches the program byte-exact.
* Throws HarnessError on timeout/signal/spawn failure. Never throws for non-zero exit codes.
*/
async run(args: string[], opts: CliOptions = {}): Promise<CliResult> {
const env = this.env(opts.env);
const r = await runProcess(this.binary, args, {
env,
cwd: opts.cwd ?? this.ctx.workspace.dir,
timeoutMs: opts.timeoutMs,
input: opts.input,
allowSignal: opts.allowSignal,
label: `dpsapi ${args.slice(0, 2).join(' ')}`,
}).catch((e) => {
this.recordError('argv', this.binary, args, env, e);
throw e;
});
const res: CliResult = { ...r, args: [...args], via: 'argv' };
if (printsTokens(args)) this.ctx.secrets.registerAllIn(r.stdout);
this.record(res, env);
return res;
}
/**
* Runs `script` with /bin/sh -c (for tests that exercise shell quoting). The environment has
* DPSAPI_BIN (the binary path) plus the default app env, so scripts use `"$DPSAPI_BIN" token new ...`.
* Token-shaped strings printed on stdout are registered as secrets.
*/
async shell(script: string, opts: CliOptions = {}): Promise<CliResult> {
const env = { ...this.env(opts.env), DPSAPI_BIN: this.binary };
const r = await runProcess('/bin/sh', ['-c', script], {
env,
cwd: opts.cwd ?? this.ctx.workspace.dir,
timeoutMs: opts.timeoutMs,
input: opts.input,
allowSignal: opts.allowSignal,
label: 'sh -c <script>',
}).catch((e) => {
this.recordError('shell', '/bin/sh', [script], env, e);
throw e;
});
const res: CliResult = { ...r, args: [script], via: 'shell' };
this.record(res, env);
this.ctx.secrets.registerAllIn(r.stdout);
return res;
}
/**
* `token new [description]`: asserts success, a single 43-char base64url line on stdout and
* (unless LOG_LEVEL is overridden) empty stderr. Returns the (registered) token.
*/
async tokenNew(description?: string, opts: CliOptions = {}): Promise<string> {
const r = await this.run(description === undefined ? ['token', 'new'] : ['token', 'new', description], opts);
expectCliSuccess(r);
expect(r.stdout, 'token new stdout').toMatch(/^[A-Za-z0-9_-]{43}\n$/);
if (!opts.env?.LOG_LEVEL) expect(r.stderr, 'token new stderr').toBe('');
const token = r.stdout.slice(0, -1);
expect(TOKEN_RE.test(token)).toBe(true);
return this.ctx.secrets.register(token);
}
/**
* `token list` (active tokens only) or, with 'all', `token list all` (every token): asserts
* success (and empty stderr unless LOG_LEVEL is overridden) and returns the strictly parsed rows.
* The printed tokens are registered as secrets.
*/
async tokenList(scope: 'active' | 'all' = 'active', opts: CliOptions = {}): Promise<TokenListRow[]> {
const args = scope === 'all' ? ['token', 'list', 'all'] : ['token', 'list'];
const r = await this.run(args, opts);
expectCliSuccess(r);
if (!opts.env?.LOG_LEVEL) expect(r.stderr, `${args.join(' ')} stderr`).toBe('');
const rows = parseTokenList(r.stdout);
if (scope === 'active') expect(rows.filter((x) => !x.active || x.deletedAt !== null).map((x) => x.line.slice(43)), 'token list prints only active tokens').toEqual([]);
return rows;
}
/** `token list all`: every token, active and inactive (see tokenList). */
async tokenListAll(opts: CliOptions = {}): Promise<TokenListRow[]> {
return this.tokenList('all', opts);
}
/** `help`: asserts exit 0, stdout exactly the usage text and (unless LOG_LEVEL is overridden) empty stderr. */
async help(opts: CliOptions = {}): Promise<CliResult> {
const r = await this.run(['help'], opts);
expectCliSuccess(r, { stdout: USAGE_TEXT });
if (!opts.env?.LOG_LEVEL) expect(r.stderr, 'help stderr').toBe('');
return r;
}
/**
* `audit [arg]` (a string = one argument, e.g. `'all'`, `'10'`, `'1h'`; an array = the arguments verbatim): asserts success
* (and empty stderr unless LOG_LEVEL is overridden) and returns the strictly parsed rows.
* The printed tokens are registered as secrets.
*/
async audit(arg?: string | string[], opts: CliOptions = {}): Promise<AuditRow[]> {
const r = await this.run(['audit', ...(arg === undefined ? [] : typeof arg === 'string' ? [arg] : arg)], opts);
expectCliSuccess(r);
if (!opts.env?.LOG_LEVEL) expect(r.stderr, 'audit stderr').toBe('');
return parseAuditOutput(r.stdout);
}
/** `token delete <token>`: asserts success with empty stdout (and stderr unless LOG_LEVEL is overridden). */
async tokenDelete(token: string, opts: CliOptions = {}): Promise<CliResult> {
const r = await this.run(['token', 'delete', token], opts);
expectCliSuccess(r, { stdout: '' });
if (!opts.env?.LOG_LEVEL) expect(r.stderr, 'token delete stderr').toBe('');
return r;
}
private record(r: CliResult, env: Record<string, string>): void {
this.ctx.recorder.addCli({
via: r.via,
command: r.command,
args: r.args,
env: recordableEnv(env),
exitCode: r.exitCode,
signal: r.signal,
timedOut: r.timedOut,
durationMs: Math.round(r.durationMs),
stdout: r.stdout,
stderr: r.stderr,
});
}
private recordError(via: CliVia, command: string, args: string[], env: Record<string, string>, e: unknown): void {
const d = (e as HarnessError).details as ProcResult | undefined;
if (d?.stdout && (via === 'shell' || printsTokens(args))) this.ctx.secrets.registerAllIn(d.stdout);
this.ctx.recorder.addCli({
via,
command,
args,
env: recordableEnv(env),
exitCode: d?.exitCode ?? null,
signal: d?.signal ?? null,
timedOut: d?.timedOut ?? false,
durationMs: Math.round(d?.durationMs ?? 0),
stdout: d?.stdout ?? '',
stderr: `${d?.stderr ?? ''}\n[harness] ${(e as Error).message}`,
});
}
}