diff --git a/e2e/.gitignore b/e2e/.gitignore new file mode 100644 index 0000000..2ac1fff --- /dev/null +++ b/e2e/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +test-results/ +reports/ diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 0000000..1ee7706 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,120 @@ +# dpsapi E2E suite (Playwright) + +API/process-only Playwright suite (no browsers) exercising the compiled `dpsapi` binary and the final image. + +## Running + +```sh +make test-e2e # make build, then --project=host +make docker-e2e # make docker, then --project=image with DPSAPI_IMAGE set +cd e2e && npm ci +npx playwright test --project=host +DPSAPI_IMAGE=git.bjphoster.com/source/dpsapi:dev npx playwright test --project=image +npx playwright test --project=host tests/host/auth.spec.ts -g 'duplicated' +``` + +| Variable | Meaning | +|---|---| +| `DPSAPI_BIN` | Binary under test (relative paths resolve against `e2e/`). Default `../dpsapi`; only the default is (re)built by `host-setup` via `make build`. | +| `DPSAPI_E2E_SKIP_BUILD=1` | Skip that build (set by `make test-e2e`, which already built). | +| `DPSAPI_IMAGE` | Image under test for `--project=image` (required there). | +| `DPSAPI_E2E_SEED` | Run seed for all PRNG streams (default `dpsapi-e2e-default-seed-1`, printed at start). | +| `DPSAPI_E2E_WORKERS` | Worker count (default 4). | +| `DPSAPI_E2E_ALPINE` | Image for test/helper containers (default `alpine:3.22`, must exist locally). | + +Prerequisites (checked by the setup projects, failing fast with `[harness:prerequisite]`): `curl`, unprivileged `unshare -Urm` with bind/tmpfs mounts, a reachable Docker daemon, `alpine:3.22`, and the binary (host) or the image (image). + +Projects: `host-setup` → `host` (`tests/host/`), `image-setup` → `image` (`tests/image/`). Global setup assigns a run id and scratch dir `/tmp/dpsapi-e2e-/`; global teardown fails the run (non-zero exit) if containers/networks/volumes labelled `dpsapi.e2e=`, processes carrying `DPSAPI_E2E_PROC=`, or workspaces remain, after cleaning them up. + +Reports: `reports/html/` (HTML), `reports/results.json`, per-test files in `test-results/`. + +## Writing tests + +Import everything from the harness, never directly from `@playwright/test`: + +```ts +import { test, expect, expectErrorResponse, assertDbInvariants } from '../../harness'; + +test('inactive token → 401', async ({ cli, startServer, http }) => { + const token = await cli.tokenNew('demo'); + await cli.tokenDelete(token); + const server = await startServer({ logLevel: 'DEBUG' }); // FakeDocker (empty) by default + expectErrorResponse(await http.get(server.url(), { token }), 401); + expect((await server.stop()).exitCode).toBe(0); // every server must be stopped + assertDbInvariants(server.dbPath, { operationCount: 0 }); +}); +``` + +### Fixtures (`harness/fixtures.ts`) + +| Fixture | Type | Notes | +|---|---|---| +| `ws` | `Workspace` | Fresh temp dir; `ws.dbPath` (`/dpsapi.db`, not created) is the default `DB_PATH`. `path()`, `mkdir()`, `writeFile()`. Removed after the test (root-owned files via an alpine container). | +| `cli` | `CliRunner` | Host binary, **no shell**, argv byte-exact. `run(args, {env, cwd, timeoutMs, input, allowSignal})` → `CliResult`; `shell(script, opts)` runs `/bin/sh -c` with `$DPSAPI_BIN`; `tokenNew(desc?)` → token, `tokenList(scope = 'active' | 'all', opts?)` → `TokenListRow[]` (`token list`: active tokens only, asserted; `'all'` runs `token list all`), `tokenListAll(opts?)`, `tokenDelete(tok)`, `audit(arg? \| args[], opts?)` → `AuditRow[]` (`audit()` = latest 10, `audit('all')` = every row, `audit('25')`, `audit('1h')`), `help(opts?)` (stdout exactly `USAGE_TEXT`) (all assert exit 0 and, unless `LOG_LEVEL` is overridden, empty stderr); `env(over)`; `withBinary(path)`. Env overrides: `undefined` removes a variable. Default env is only `PATH`, `DB_PATH`, leak marker. | +| `startServer` | `(opts?) => Promise` | See below. | +| `newFakeDocker` | `() => Promise` | Extra fake Engine on a workspace socket; closed automatically. | +| `http` | `HttpClient` | `fetch(url, {method, headers, token, data, timeoutMs})` / `get()` (Playwright request, no redirects) → `HttpResult`; `curl(CurlOptions)` → `CurlResult`; `raw(RawHttpOptions)` → `RawResult`. All recorded. | +| `docker` | `RealDocker` | Real Engine containers named `dpsapi-e2e---`, labelled `dpsapi.e2e=`, `dpsapi.e2e.test=`: `run({suffix, image, command, labels, env, init, network, extraArgs})`, `runMany(n, optsFor, concurrency)`, `stop(c, sec)`, `start`, `remove`, `inspect`, `listOwn({all})`, `runningIds()`, `cleanup()`. Everything labelled for the test is force-removed at teardown. | +| `image` | `ImageRunner` | `$DPSAPI_IMAGE`: `start(ImageStartOptions)` → `ImageContainer`; `run(args, {env, dataDir, docker, user, timeoutMs, extraArgs})` (one-shot `docker run --rm`, `CliResult` with `via: 'docker-run'`); `extractBinary()` → host path of `/app`; `snapshotDataDir(dir)`; `removeDataDir(dir)`; `containerName(suffix)`. | +| `secrets` | `SecretRegistry` | `register`, `registerAllIn`, `redact`, `findIn`, `assertNoSecretsIn(text, where)`. Tokens printed by `token new`, `token list [all]` and `audit [all \| n \| n]` via `cli`/`image` (`printsTokens(args)`) are registered automatically; `help` prints none. | +| `rng` | `Rng` | Deterministic per test (seed = run seed + title path, annotated as `seed`). `next/int/bool/pick/shuffle/bytes/string/fork`. | +| `fuzz` | `FuzzRecorder` | `fuzz.record(case)` stores a JSON-safe snapshot taken at that moment (BigInts, e.g. audit selections, become decimal strings) and returns the case unchanged; `fuzz.recordLive(obj)` for a case object filled in afterwards; `fuzz.seed`, `fuzz.rng`; cases attached as `fuzz-cases.json` on failure (or `fuzz.attach()`). Use `safeStringify(v)` (not `JSON.stringify`) for labels built from cases. | +| `recorder`, `testKey`, `e2e` | internals | `e2e.allocatePort()` (parallel-safe free port), `e2e.trackedDbs.add(path)` (snapshot on failure). | +| `scanLogsForSecrets` | option, default `true` | `test.use({ scanLogsForSecrets: false })` for tests that put registered tokens into descriptions on purpose. | + +### `startServer(opts)` / `ServerHandle` (`harness/server.ts`) + +Options: `logLevel` (default `'INFO'`, `null` = unset), `addr` (default `'127.0.0.1'`, `null` = unset), `port` (default allocated, `null` = unset → 80, only with `privateNetwork`), `dbPath`, `docker` (`FakeDocker` | `'real'` | `'inaccessible'` (EACCES) | `'missing'` (ENOENT) | `{bindSource}`; default a new empty FakeDocker), `env`, `privateNetwork`, `waitReady` (default true), `readyTimeoutMs`, `label`, `binary`. + +Non-`real` backends run the binary as `unshare -Urm sh -c 'mount --bind /var/run/docker.sock && exec '` (same pid, host network). Readiness polls `OPTIONS *` over a raw socket: answered by Go itself, so it creates no app log record and no audit row. + +Handle: `baseURL`, `url(path = '/api/v1/containers')` (path verbatim), `port`, `addr`, `connectHost`, `dbPath`, `env`, `fakeDocker`, `pid`, `generation`, `stdout`, `stderr`, `logs()` → `LogSet`, `allOutput()`, `isRunning()`, `exit`, `waitReady()`, `waitForLog(query, ms)`, `signal(sig)`, `stop(sig = 'SIGTERM', {timeoutMs, allowSignalExit})` → `ExitResult {exitCode, signal, durationMs, stdout, stderr}` (hang → `HarnessError('hang')` after SIGKILL; death by signal → `HarnessError('signal')`), `kill()`, `waitForExit(ms)`, `restart()` (SIGTERM, requires exit 0, same port/DB/backend). + +### `FakeDocker` (`harness/fake-docker.ts`) + +Fake Engine API on a unix socket: `HEAD/GET /_ping`, `GET [/vX]/containers/json`. Live switches: `empty()`, `many(n, rng?)`, `setContainers(list)`, `error(status, message)`/`error500()`, `flaky(k)` (next k list calls → 500), `destroy()`, `hang()`, `badJson(body)`, `passthrough(upstream)`, `delay(ms)`. Counters: `stats {ping, list, other, total}`, `requests[] {method, url, path, query, at}`, `listCalls()`, `resetStats()`. Summaries: `makeContainerSummary(i, rng?, overrides)`, `makeContainerSummaries(n, rng?)`. Never unlink the socket while a server uses it (bind mounts pin the inode) — switch modes instead. + +### `ImageContainer` (`harness/image.ts`) + +`image.start({logLevel, addr, containerPort, dbPath, env, docker: 'real' | 'none' | FakeDocker | {socketPath} (default new FakeDocker), dataDir, user, publish, extraArgs, args, waitReady, readyTimeoutMs, suffix})`. Mounts the socket at `/var/run/docker.sock`, `dataDir` at `/data` (`DB_PATH=/data/dpsapi.db`), publishes `127.0.0.1:`. Handle: `name`, `id`, `hostPort`, `baseURL`, `url()`, `dataDir`, `containerDbPath`, `fakeDocker`, `waitReady()`, `exec(args, {env, user, timeoutMs})` → `CliResult` (`via: 'docker-exec'`; exit ≥125 → `HarnessError`), `execRaw(argv)` → `ProcResult`, `logs()` → `{stdout, stderr}`, `parsedLogs()`, `inspect()`, `isRunning()`, `stop({signal?, timeoutSec})` → `{exitCode, oomKilled, stdout, stderr}` (137 = killed after grace period), `waitForExit()`, `remove()`, `snapshotDb()` → host path of a host-owned copy of the (root-owned) DB + WAL. Reuse `dataDir` to replace a container on the same data. + +### HTTP helpers (`harness/http.ts`, `harness/assertions.ts`) + +- `HttpResult` / `CurlResult`: `status`, `headers` (lower-case, repeats joined by `, `), `headersArray`, `body`, `bodyBuffer`, `json()`, `durationMs`; curl adds `exitCode` (7 refused, 52 empty reply…), `stderr`, `writeOut`, `argv`. Curl options: `url, method, headers: string[]` (raw `-H` lines, repeatable; `'Name;'` = empty value), `token, data, pathAsIs, head, httpVersion, timeoutMs, allowTimeout, extraArgs`. +- `RawResult.outcome`: `'response' | 'closed' | 'reset' | 'refused' | 'timeout'` (timeout only with `allowTimeout`, otherwise `HarnessError`); `responses[]`/`response` (`status`, `headers` pairs, `header(n)`, `headerAll(n)`, `body`, `bodyText`), `raw`, `closedByServer`. Options: `host, port, data (string | Buffer | chunks), chunkDelayMs, timeoutMs, expectResponses, waitForClose, endAfterWrite, head`. `buildRawRequest({method, target, headers, body, host, version, close})`. +- `expectErrorResponse(res, 401|404|405|500, {head})` (exact body, Content-Type/Length, `WWW-Authenticate: Bearer` only on 401, `Allow: GET` only on 405), `expectContainersResponse(res)` → array, `expectNoInternalDetails(body)`, `view(res)`, `ERROR_BODIES`. + +### CLI / DB / logs + +- `CliResult` = `ProcResult` (`stdout, stderr, stdoutBuffer, stderrBuffer, exitCode, signal, timedOut, durationMs, command, argv`) + `args`, `via`. +- `expectCliSuccess(r, {stdout?, stderr?})`; `expectCliFailure(r, {exitCode, message?, usage?, exactStderr = true, stdout = ''})` — first throws `HarnessError` on signal/timeout; `message` is the text after `dpsapi: `; usage text expected for `usage error:` messages. `expectExitedNormally(r)`, `errorMessage(r)`, `USAGE_TEXT` (printed on stderr after every usage error and on stdout by `help`), `USAGE_ERROR_MESSAGES` (the complete list). +- Tables (`token list [all]`, `audit`): `padTable(lines: string[][])` reproduces the app's `text/tabwriter` layout exactly (every column but the last padded to 2 + its widest cell in runes — `runeLength(s)` = `Array.from(s).length` — over header and rows of that invocation; last cell unpadded; no trailing blanks; `\n` after every line; header only when there are no rows). `TOKEN_LIST_COLUMNS`, `AUDIT_COLUMNS`, `EMPTY_TOKEN_LIST`, `EMPTY_AUDIT` (header-only outputs), `formatTokenList(entries)` / `formatAuditOutput(rows)` (exact expected stdout for known rows; prefer `expect(r.stdout).toBe(...)` with them), and the strict parsers `parseTokenList(stdout)` (TOKEN = first 43 chars, then a Go-quoted DESCRIPTION parsed as a quoted literal — it may contain spaces and `\"` — then ACTIVE, CREATED_AT, DELETED_AT) / `parseAuditOutput(stdout)` (TIME, TOKEN, ORIGIN, then OPERATION = rest of the line), which also require the output to re-render byte-exactly through `padTable`. Rows carry the raw padded `line`; widths depend on the printed rows, so compare rows by cells, not lines, across invocations. +- Oracles (`harness/personas.ts`): `cliOutcome(args, model)` → `help` (exit 0, usage text on stdout) | `list` with `scope: 'active' | 'all'` (`token list` / `token list all`; any other single argument → `unknown token list argument`, two or more → `token list takes at most one argument`) | `new` | `delete` | `audit` (`selection`: `{all: true}` for `audit all` — exact, lowercase, one argument —, `{limit}` for no argument / `n`, `{windowNs}` for `n`) | `error`; `auditArgsOutcome(args)`, `selectAuditRows(ops, selection, nowNs?)` / `expectedAuditStdout(ops, selection, nowNs?)` (`all`: every row, oldest first — parsed time, then rowid; `nowNs` only for windows), `expectAuditWindow(...)`, `expectedTokenListStdout(dbPath, scope)` (exact stdout from the stored rows), `StateModel.verifyList(rows, label, scope = 'active')`, `HELP_NEAR_MISSES` / `LIST_ALL_NEAR_MISSES` / `AUDIT_ALL_ARGV_NEAR_MISSES` (full argv lists) and `AUDIT_ALL_NEAR_MISSES` / `AUDIT_ALL_EXTRA_ARGS` (`audit` arguments: `ALL`, `All`, `all1`, `1all`, ` all`, `alls`, `--all`… → `invalid audit argument`; `all x`… → `audit takes at most one argument`) for argv generators; `fuzzAuditArgs(rng)` also yields `all` and its near misses. +- `goQuote`, `goUnquote`, `goUnquoteBytes` (Go `strconv` compatible), `shellQuote`. +- DB (read-only `node:sqlite`): `readTokens(db)` (`descriptionHex` = exact bytes), `readToken`, `readOperations`, `countRows`, `readSchema`, `withDb`, `fileDigest`, `dbSnapshot`, `RFC3339NANO_UTC_RE`, `rfc3339ToNanos`, `msToNanos`, and `assertDbInvariants(db, {tokenCount?, operationCount?, auditBeforeDeletion?})` → `{tokens, activeTokens, inactiveTokens, operations}`: integrity_check, foreign_key_check, exact schema, storage classes, token format, active∈{0,1}, deleted_at NULL iff active, strict RFC3339Nano UTC timestamps, deleted_at ≥ created_at, operation = `GET containers`, origin is an IP, audit time ≥ token created_at. +- Logs: `parseLogs(text)` → `LogSet` (`records`, `unparsed`, `filter/find/count/has(query)`, `levels()`, `messages(level?)`, `requests()`); `LogQuery {level, msg (string|RegExp), attrs {k: string|number|boolean|RegExp|fn}}`; `LOG_LEVELS`, `visibleLevels(min)`, `levelRank`, `LOG_TIME_RE`; `tokenId(token)` = the `token_id` the app logs. +- Misc: `runProcess(cmd, argv, {env, cwd, timeoutMs, input, allowTimeout, allowSignal})`, `toolEnv()`, `dockerCli(args, {allowFailure})`, `listByLabel`, `mapLimit`, `pollUntil`, `sleep`, `occupyPort`, `isPortFree`, `HarnessError`/`isHarnessError`. + +## Conventions + +- **Files**: `tests/host/.spec.ts`, `tests/image/.spec.ts`; persona suites `tests/host/persona-.spec.ts`. One concern per `test`; use `test.step` for case tables. +- **Exit codes are always explicit**: `expect(r.exitCode).toBe(0)` / `expectCliFailure(r, {exitCode: 1|2, ...})`; every server is stopped by the test and its exit code asserted. +- **Infrastructure vs application failures**: timeouts, unexpected signals, hangs, spawn/docker failures, unparseable output throw `HarnessError` (`[harness:]`) — never catch them to satisfy a negative assertion. +- **Isolation**: everything mutable lives in the test's workspace, FakeDocker, ports and Docker resources keyed by `testKey`. Never assert on global container counts — only on the suite's names/labels. Never touch resources without the `dpsapi-e2e-` prefix. +- **Cleanup/leaks**: teardown kills+fails on running servers or any live process spawned by the worker, removes labelled containers and the workspace; global teardown fails the run on any leftover. +- **Secrets**: never `console.log` tokens (worker stdout is redacted anyway). Artifacts and the report redact registered tokens and any token-shaped 43-char string as `[REDACTED:]`. Server logs, CLI stderr and container logs are scanned for registered tokens at teardown. +- **Artifacts on failure**: per server generation `stderr`/`stdout`/exit, container logs, `cli-invocations.json`, `http-exchanges.json`, `fake-docker-N.json`, `db-snapshot-N.txt` (integrity + rows), `fuzz-cases.json` (with seeds), `harness-problems.txt`, `harness-notes.txt`. +- **DB**: read-only inspection only; write to the app DB from tests only when a test explicitly simulates lock contention (open your own `DatabaseSync` for that). +- **Timing**: wait on observable conditions (`waitReady`, `waitForLog`, `pollUntil`, `fake.listCalls()`), never fixed sleeps. + +## CLI command coverage + +| Command | Main specs | +|---|---| +| `help` (stdout = usage text, exit 0, no DB; config errors win; DEBUG/TRACE log records on stderr only) | `tests/host/config-usage.spec.ts`, `logging.spec.ts`, `tests/image/image-contract.spec.ts` (`docker exec … /app help`, `docker run … help`) | +| `token list` (active only) / `token list all` (every token), padded tables, `unknown token list argument` / `token list takes at most one argument` | `tests/host/cli-tokens.spec.ts`, `config-usage.spec.ts`, personas, `tests/image/image-contract.spec.ts`, `multi-transport.spec.ts`, `image-lifecycle.spec.ts` | +| `audit [all \| n \| n]` (selection forms, units, limits, ordering, window boundary, read-only, usage errors, log levels, unparseable time) | `tests/host/cli-audit.spec.ts`, `population.spec.ts`, personas, `tests/image/image-contract.spec.ts` (`docker exec … /app audit`, `/app audit all`), `multi-transport.spec.ts`, `image-lifecycle.spec.ts` | +| `audit all` (every row oldest first — parsed time, ties in insertion order —; header only when empty; exact padded table with more than 10 rows; read-only; `ALL`/`All`/`all1`/`1all`/` all`/`alls`/`--all` → `invalid audit argument`, `all x` → `audit takes at most one argument`, never echoed, no database; DEBUG `cli dispatch command=audit all=true`) | `tests/host/cli-audit.spec.ts` (`audit all` describe; also in the read-only, log-level, unparseable-time, population and usage tables), `config-usage.spec.ts`, `harness-selfcheck.spec.ts`, personas (fuzzer, operator session, cross abuse, stupid user), `tests/image/image-contract.spec.ts`, `multi-transport.spec.ts` | +| Usage-error table (every message, exact stderr, no DB) | `tests/host/config-usage.spec.ts` (checked against `USAGE_ERROR_MESSAGES` and `cliOutcome`) | + +The argv fuzzers (`persona-fuzzer-user`, `persona-cross-abuse`, `persona-operator-session`) generate `help`, `token list all`, `audit all` and their near misses (`token list ALL`, `help help`, `token list all all`, `audit ALL`, `audit all x`, `audit --all`, …), check every outcome with `cliOutcome`, and require minimum counts of each (e.g. the fuzzer forces `audit all` / an `audit all` near miss every 30th case; the operator session has an `audit-all` action). diff --git a/e2e/harness/assertions.ts b/e2e/harness/assertions.ts new file mode 100644 index 0000000..4dc1576 --- /dev/null +++ b/e2e/harness/assertions.ts @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// Copyright (C) 2026 Bryan Joshua Pedini + +/** + * Shared assertions on the HTTP contract of dpsapi. + */ +import { expect } from '@playwright/test'; +import type { CurlResult, HttpResult, RawHttpResponse } from './http'; + +/** Exact error bodies of the application. */ +export const ERROR_BODIES = { + 401: '{"error":"unauthorized"}', + 404: '{"error":"not found"}', + 405: '{"error":"method not allowed"}', + 500: '{"error":"internal server error"}', +} as const; + +export type AppErrorStatus = keyof typeof ERROR_BODIES; + +/** Client-independent response view. */ +export interface ResponseView { + status: number; + /** Lower-cased names; repeated headers joined with ', '. */ + headers: Record; + body: string; +} + +/** Normalizes Playwright / curl / raw responses into a ResponseView. */ +export function view(r: HttpResult | CurlResult | RawHttpResponse | ResponseView): ResponseView { + if ('bodyText' in r) { + const headers: Record = {}; + for (const [k, v] of r.headers) { + const l = k.toLowerCase(); + headers[l] = l in headers ? `${headers[l]}, ${v}` : v; + } + return { status: r.status, headers, body: r.bodyText }; + } + return { status: r.status, headers: r.headers, body: r.body }; +} + +/** Substrings that must never appear in an HTTP response body. */ +export const INTERNAL_DETAIL_MARKERS = [ + 'sql', + 'sqlite', + 'docker.sock', + '/var/run', + 'unix', + 'dial', + 'goroutine', + 'panic', + 'runtime error', + 'fake engine', + 'connection refused', + 'permission denied', + 'no such file', + 'context deadline', + 'list containers', + 'token_id', +]; + +/** Fails if the body contains any internal-detail marker (case-insensitive). */ +export function expectNoInternalDetails(body: string): void { + const low = body.toLowerCase(); + expect(INTERNAL_DETAIL_MARKERS.filter((m) => low.includes(m)), `internal details leaked in body ${JSON.stringify(body.slice(0, 300))}`).toEqual([]); +} + +/** + * Asserts one of the application's JSON error responses exactly: status, body, Content-Type, + * Content-Length, WWW-Authenticate (only on 401: `Bearer`) and Allow (only on 405: `GET`). + */ +export function expectErrorResponse(res: HttpResult | CurlResult | RawHttpResponse | ResponseView, status: AppErrorStatus, { head = false }: { head?: boolean } = {}): void { + const r = view(res); + const body = ERROR_BODIES[status]; + expect(r.status, `status (body ${JSON.stringify(r.body.slice(0, 200))})`).toBe(status); + expect(r.body, 'error body').toBe(head ? '' : body); + expect(r.headers['content-type'], 'content-type').toBe('application/json'); + expect(r.headers['content-length'], 'content-length').toBe(String(body.length)); + if (status === 401) expect(r.headers['www-authenticate'], 'www-authenticate').toBe('Bearer'); + else expect(r.headers['www-authenticate'], 'www-authenticate').toBeUndefined(); + if (status === 405) expect(r.headers['allow'], 'allow').toBe('GET'); + else expect(r.headers['allow'], 'allow').toBeUndefined(); + expect(r.headers['location'], 'no redirect').toBeUndefined(); +} + +/** + * Asserts a successful container listing: 200, application/json, exact Content-Length, a JSON + * array without trailing newline. Returns the parsed array. + */ +export function expectContainersResponse>(res: HttpResult | CurlResult | RawHttpResponse | ResponseView): T[] { + const r = view(res); + expect(r.status, `status (body ${JSON.stringify(r.body.slice(0, 200))})`).toBe(200); + expect(r.headers['content-type'], 'content-type').toBe('application/json'); + expect(r.headers['content-length'], 'content-length').toBe(String(Buffer.byteLength(r.body))); + expect(r.body.endsWith('\n'), 'no trailing newline').toBe(false); + const parsed = JSON.parse(r.body) as unknown; + expect(Array.isArray(parsed), 'body is a JSON array').toBe(true); + return parsed as T[]; +} diff --git a/e2e/harness/cli.ts b/e2e/harness/cli.ts new file mode 100644 index 0000000..484906d --- /dev/null +++ b/e2e/harness/cli.ts @@ -0,0 +1,518 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// Copyright (C) 2026 Bryan Joshua Pedini + +/** + * Running the compiled binary as a CLI, parsing its output, and asserting outcomes. + */ +import { expect } from '@playwright/test'; +import { RFC3339NANO_UTC_RE } from './db'; +import { binaryPath, PROC_MARKER_ENV, runId } from './env'; +import { HarnessError } from './errors'; +import { goQuote, goUnquoteBytes } from './gostrings'; +import { runProcess, type ProcResult } from './proc'; +import type { CliVia, Recorder } from './recorder'; +import { redactText, TOKEN_RE, type SecretRegistry } from './secrets'; +import type { Workspace } from './workspace'; + +export interface CliResult extends ProcResult { + /** Application arguments (after the binary / `docker exec /app`), or [script] for shell runs. */ + args: string[]; + via: CliVia; +} + +export interface CliOptions { + /** Environment overrides on top of the default app env; `undefined` removes a variable. */ + env?: Record; + cwd?: string; + /** Default 30 s. Server-mode invocations expected to fail fast should pass ~10 s. */ + timeoutMs?: number; + input?: string | Buffer; + /** Do not throw HarnessError on signal termination (only for tests that deliver signals). */ + allowSignal?: boolean; +} + +/** Usage text: printed to stderr after every usage error, and to stdout by `help`. */ +export const USAGE_TEXT = + 'usage:\n' + + ' dpsapi start the HTTP server\n' + + ' dpsapi help print this usage text\n' + + ' dpsapi token list [all] list the active tokens, or every token with "all"\n' + + ' dpsapi token new [description] create a token and print it\n' + + ' dpsapi token delete deactivate a token\n' + + ' dpsapi audit print the latest 10 audit rows\n' + + ' dpsapi audit print the latest n audit rows\n' + + ' dpsapi audit print the audit rows of the last n units (s, m, h, d, w)\n' + + ' dpsapi audit all print every audit row\n'; + +/** Every usage error message (text after `dpsapi: usage error: `) the CLI can print. */ +export const USAGE_ERROR_MESSAGES = [ + 'unknown command', + 'help takes no arguments', + 'missing token subcommand', + 'unknown token subcommand', + 'token list takes at most one argument', + 'unknown token list argument', + 'token new takes at most one argument', + 'token delete requires exactly one argument', + 'audit takes at most one argument', + 'invalid audit argument', +] as const; + +// --------------------------------------------------------------------------------------------- +// Padded tables (Go text/tabwriter: minwidth 0, tabwidth 0, padding 2, padchar ' ', flags 0) +// --------------------------------------------------------------------------------------------- + +/** Length in Unicode code points (what tabwriter counts: runes, not bytes, not display width). */ +export function runeLength(s: string): number { + return Array.from(s).length; +} + +/** + * Renders lines of cells exactly like the app's tabwriter: every column but the last is padded + * with spaces to (2 + the longest cell of that column over all lines, in runes); the last cell is + * written as-is; every line ends with '\n'. All lines must have the same number of cells. + */ +export function padTable(lines: ReadonlyArray): string { + if (lines.length === 0) return ''; + const n = lines[0].length; + for (const l of lines) if (l.length !== n) throw new Error(`padTable: every line needs ${n} cells, got ${l.length}`); + const widths = Array.from({ length: Math.max(0, n - 1) }, (_, c) => 2 + Math.max(...lines.map((l) => runeLength(l[c])))); + return lines.map((l) => l.map((cell, c) => (c < n - 1 ? cell + ' '.repeat(widths[c] - runeLength(cell)) : cell)).join('') + '\n').join(''); +} + +/** Header cells of `token list` / `token list all`. */ +export const TOKEN_LIST_COLUMNS = ['TOKEN', 'DESCRIPTION', 'ACTIVE', 'CREATED_AT', 'DELETED_AT'] as const; +/** Header cells of `audit`. */ +export const AUDIT_COLUMNS = ['TIME', 'TOKEN', 'ORIGIN', 'OPERATION'] as const; +/** Exact stdout of `token list [all]` with no rows (the header padded by its own widths). */ +export const EMPTY_TOKEN_LIST = padTable([TOKEN_LIST_COLUMNS]); +/** Exact stdout of `audit` with no rows. */ +export const EMPTY_AUDIT = padTable([AUDIT_COLUMNS]); + +/** + * True for commands whose successful stdout carries full token values (`token new`, + * `token list [all]`, `audit [all | n | n]`): the runners register every token-shaped string of their stdout + * as a secret. (`help` prints only the usage text.) + */ +export function printsTokens(args: readonly string[]): boolean { + return (args[0] === 'token' && (args[1] === 'new' || args[1] === 'list')) || args[0] === 'audit'; +} + +/** Error-message rendering of (part of) an output: clipped, token-shaped strings redacted. */ +function show(s: string | undefined): string { + const v = s ?? ''; + return JSON.stringify(redactText(v.length > 200 ? v.slice(0, 200) + '…' : v, [])); +} + +/** Splits table stdout into lines (must end with a newline; at least the header). */ +function tableLines(stdout: string, what: string): string[] { + if (!stdout.endsWith('\n')) throw new Error(`${what} output does not end with a newline: ${show(stdout.slice(-120))}`); + return stdout.slice(0, -1).split('\n'); +} + +/** The re-rendered table must equal the output byte for byte (padding, widths, no trailing blanks). */ +function expectRerenders(stdout: string, cells: string[][], what: string): void { + const again = padTable(cells); + if (again === stdout) return; + const a = stdout.split('\n'); + const b = again.split('\n'); + const i = a.findIndex((l, k) => l !== b[k]); + throw new Error(`${what} output is not the padded table of its own cells (first differing line ${i}): got ${show(a[i])}, expected ${show(b[i])}`); +} + +export interface AuditRow { + /** TIME exactly as printed (= as stored). */ + time: string; + /** Full token value. */ + token: string; + origin: string; + operation: string; + /** The raw (padded) line. */ + line: string; +} + +/** + * Parses `audit` stdout strictly: trailing newline, the header, rows of TIME / TOKEN / ORIGIN + * (no blanks inside, separated by runs of spaces) and OPERATION (the rest of the line), TIME in + * strict UTC RFC3339Nano, TOKEN 43-char base64url; finally the whole output must re-render byte + * for byte as the padded table of the parsed cells. Throws on any deviation (malformed output is + * a test failure). Token values are redacted in the error message. + */ +export function parseAuditOutput(stdout: string): AuditRow[] { + const lines = tableLines(stdout, 'audit'); + const header = lines[0].split(/ +/); + if (header.join('\t') !== AUDIT_COLUMNS.join('\t')) throw new Error(`unexpected audit header: ${show(lines[0])}`); + const rows = lines.slice(1).map((line, i) => { + const m = /^(\S+) +(\S+) +(\S+) +(\S.*)$/.exec(line); + if (!m) throw new Error(`audit row ${i + 1} does not have 4 space-separated cells: ${show(line)}`); + const [, time, token, origin, operation] = m; + if (!RFC3339NANO_UTC_RE.test(time)) throw new Error(`audit row ${i + 1}: TIME ${show(time)} is not strict UTC RFC3339Nano`); + if (!TOKEN_RE.test(token)) throw new Error(`audit row ${i + 1}: TOKEN is not a 43-character base64url token: ${show(token)}`); + return { time, token, origin, operation, line }; + }); + expectRerenders(stdout, [[...AUDIT_COLUMNS], ...rows.map((r) => [r.time, r.token, r.origin, r.operation])], 'audit'); + return rows; +} + +/** The exact `audit` stdout for the given rows (padded header + one line per row, in the given order). */ +export function formatAuditOutput(rows: ReadonlyArray<{ time: string; token: string; origin: string; operation: string }>): string { + return padTable([[...AUDIT_COLUMNS], ...rows.map((r) => [r.time, r.token, r.origin, r.operation])]); +} + +export interface TokenListRow { + token: string; + /** DESCRIPTION column exactly as printed (Go-quoted). */ + descriptionQuoted: string; + /** Unquoted description bytes. */ + descriptionBytes: Buffer; + /** Unquoted description as UTF-8 text. */ + description: string; + active: boolean; + createdAt: string; + /** null when the column is `-`. */ + deletedAt: string | null; + /** The raw (padded) line. */ + line: string; +} + +/** Index just after the closing quote of the Go double-quoted literal starting at s[start] (-1 if unterminated). */ +function endOfQuoted(s: string, start: number): number { + if (s[start] !== '"') return -1; + for (let i = start + 1; i < s.length; i++) { + if (s[i] === '\\') i++; + else if (s[i] === '"') return i + 1; + } + return -1; +} + +/** + * Parses `token list` / `token list all` stdout strictly: trailing newline, the header, rows of + * TOKEN (the first 43 characters, base64url), spaces, a Go-quoted DESCRIPTION (parsed as a quoted + * literal: it may contain spaces and escaped quotes), then ACTIVE (`true`/`false`), CREATED_AT + * (strict RFC3339Nano UTC) and DELETED_AT (`-` or a timestamp) separated by runs of spaces; + * finally the whole output must re-render byte for byte as the padded table of the parsed cells. + * Throws on any deviation: malformed output is a test failure. + */ +export function parseTokenList(stdout: string): TokenListRow[] { + const lines = tableLines(stdout, 'token list'); + const header = lines[0].split(/ +/); + if (header.join('\t') !== TOKEN_LIST_COLUMNS.join('\t')) throw new Error(`unexpected token list header: ${show(lines[0])}`); + const rows = lines.slice(1).map((line, i): TokenListRow => { + const fail = (why: string): never => { + throw new Error(`token list row ${i + 1}: ${why}: ${show(line)}`); + }; + const token = line.slice(0, 43); + if (!TOKEN_RE.test(token)) fail('TOKEN is not 43 base64url characters'); + const q = /^ +/.exec(line.slice(43)); + if (!q) fail('no spaces after TOKEN'); + const qStart = 43 + q![0].length; + const qEnd = endOfQuoted(line, qStart); + if (qEnd < 0) fail('DESCRIPTION is not a terminated Go-quoted string'); + const descriptionQuoted = line.slice(qStart, qEnd); + let descriptionBytes: Buffer; + try { + descriptionBytes = goUnquoteBytes(descriptionQuoted); + } catch (e) { + return fail(`DESCRIPTION is not a valid Go-quoted string (${(e as Error).message})`); + } + const rest = /^ +(true|false) +(\S+) +(\S+)$/.exec(line.slice(qEnd)); + if (!rest) fail('expected ACTIVE, CREATED_AT and DELETED_AT after DESCRIPTION'); + const [, active, createdAt, deletedAt] = rest!; + if (!RFC3339NANO_UTC_RE.test(createdAt)) fail(`CREATED_AT ${show(createdAt)} is not strict UTC RFC3339Nano`); + if (deletedAt !== '-' && !RFC3339NANO_UTC_RE.test(deletedAt)) fail(`DELETED_AT ${show(deletedAt)} is neither "-" nor strict UTC RFC3339Nano`); + return { + token, + descriptionQuoted, + descriptionBytes, + description: descriptionBytes.toString('utf8'), + active: active === 'true', + createdAt, + deletedAt: deletedAt === '-' ? null : deletedAt, + line, + }; + }); + expectRerenders(stdout, [[...TOKEN_LIST_COLUMNS], ...rows.map((r) => [r.token, r.descriptionQuoted, String(r.active), r.createdAt, r.deletedAt ?? '-'])], 'token list'); + return rows; +} + +/** One row as `token list` prints it; `description` is the exact stored text/bytes (Go-quoted here). */ +export interface TokenListEntry { + token: string; + description: string | Buffer; + active: boolean; + createdAt: string; + deletedAt: string | null; +} + +/** The exact `token list [all]` stdout for the given rows (padded header + one line per row, in the given order). */ +export function formatTokenList(rows: ReadonlyArray): string { + return padTable([[...TOKEN_LIST_COLUMNS], ...rows.map((r) => [r.token, goQuote(r.description), String(r.active), r.createdAt, r.deletedAt ?? '-'])]); +} + +/** Throws HarnessError unless the process exited on its own with an exit status. */ +export function expectExitedNormally(r: ProcResult, label = r.command): void { + if (r.timedOut) throw new HarnessError('timeout', `${label} timed out (killed by the harness)`); + if (r.signal !== null || r.exitCode === null) throw new HarnessError('signal', `${label} terminated by signal ${r.signal}`); +} + +export interface CliSuccessExpectation { + /** Exact stdout or pattern. */ + stdout?: string | RegExp; + /** Exact stderr or pattern ('' = must be empty). */ + stderr?: string | RegExp; +} + +/** Asserts a normal exit with status 0 (and optional exact stdout/stderr). */ +export function expectCliSuccess(r: ProcResult, exp: CliSuccessExpectation = {}): void { + expectExitedNormally(r); + expect(r.exitCode, `exit code (stderr: ${r.stderr.slice(-500)})`).toBe(0); + if (exp.stdout !== undefined) { + if (typeof exp.stdout === 'string') expect(r.stdout, 'stdout').toBe(exp.stdout); + else expect(r.stdout, 'stdout').toMatch(exp.stdout); + } + if (exp.stderr !== undefined) { + if (typeof exp.stderr === 'string') expect(r.stderr, 'stderr').toBe(exp.stderr); + else expect(r.stderr, 'stderr').toMatch(exp.stderr); + } +} + +export interface CliFailureExpectation { + /** Required: the exact expected exit status (1 runtime failure, 2 usage/configuration). */ + exitCode: number; + /** Text after `dpsapi: ` on the error line (exact string or pattern). */ + message?: string | RegExp; + /** Usage text expected after the error line. Default: message is a string starting with 'usage error:'. */ + usage?: boolean; + /** + * true (default): stderr must be exactly the error line (+ usage text) — right for the default + * INFO level. false: only the error line (+ usage) at the end is checked (DEBUG/TRACE runs). + */ + exactStderr?: boolean; + /** Expected stdout (default ''). */ + stdout?: string; +} + +/** + * Asserts an expected application failure. First requires a normal exit (a signal/timeout is a + * HarnessError, never an expected failure), then exit status, stdout, and the stderr error line. + */ +export function expectCliFailure(r: ProcResult, exp: CliFailureExpectation): void { + expectExitedNormally(r); + expect(r.exitCode, `exit code (stderr: ${r.stderr.slice(-500)})`).toBe(exp.exitCode); + expect(r.stdout, 'stdout').toBe(exp.stdout ?? ''); + const usage = exp.usage ?? (typeof exp.message === 'string' && exp.message.startsWith('usage error:')); + const suffix = usage ? USAGE_TEXT : ''; + expect(r.stderr.endsWith('\n' + suffix) || r.stderr.endsWith(suffix), 'stderr ends with usage text').toBe(true); + const head = suffix ? r.stderr.slice(0, r.stderr.length - suffix.length) : r.stderr; + const lines = head.split('\n'); + if (lines[lines.length - 1] === '') lines.pop(); + const errorLine = lines[lines.length - 1] ?? ''; + expect(errorLine.startsWith('dpsapi: '), `error line ${JSON.stringify(errorLine)} starts with "dpsapi: "`).toBe(true); + const message = errorLine.slice('dpsapi: '.length); + if (exp.message !== undefined) { + if (typeof exp.message === 'string') expect(message, 'error message').toBe(exp.message); + else expect(message, 'error message').toMatch(exp.message); + } + if (exp.exactStderr ?? true) expect(lines.length, `stderr has only the error line:\n${r.stderr}`).toBe(1); +} + +/** Last line of stderr starting with `dpsapi: `, without the prefix (undefined when none). */ +export function errorMessage(r: ProcResult): string | undefined { + const line = r.stderr + .split('\n') + .filter((l) => l.startsWith('dpsapi: ')) + .pop(); + return line?.slice('dpsapi: '.length); +} + +export interface CliContext { + workspace: Workspace; + secrets: SecretRegistry; + recorder: Recorder; +} + +/** Default environment of every app process: PATH, DB_PATH (workspace default) and the leak marker. */ +export function baseAppEnv(dbPath: string): Record { + return { PATH: process.env.PATH || '/usr/local/bin:/usr/bin:/bin', DB_PATH: dbPath, [PROC_MARKER_ENV]: runId() }; +} + +/** Applies overrides; `undefined` deletes. */ +export function mergeEnv(base: Record, over: Record = {}): Record { + const env = { ...base }; + for (const [k, v] of Object.entries(over)) { + if (v === undefined) delete env[k]; + else env[k] = v; + } + return env; +} + +/** Env subset recorded in artifacts (no PATH / marker). */ +export function recordableEnv(env: Record): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(env)) if (k !== 'PATH' && k !== PROC_MARKER_ENV && v !== undefined) out[k] = v; + return out; +} + +/** Runs the host binary as a CLI. Bound to a test's workspace/secrets/recorder by the `cli` fixture. */ +export class CliRunner { + /** Absolute path of the binary this runner executes. */ + readonly binary: string; + + constructor( + private readonly ctx: CliContext, + binary: string = binaryPath(), + ) { + this.binary = binary; + } + + /** Same workspace/secrets/recorder, different binary (e.g. /app extracted from the image). */ + withBinary(binary: string): CliRunner { + return new CliRunner(this.ctx, binary); + } + + /** Default environment (PATH, DB_PATH=/dpsapi.db, leak marker) with overrides applied. */ + env(over?: Record): Record { + return mergeEnv(baseAppEnv(this.ctx.workspace.dbPath), over); + } + + /** + * Runs ` ...args` without a shell: every argument reaches the program byte-exact. + * Throws HarnessError on timeout/signal/spawn failure. Never throws for non-zero exit codes. + */ + async run(args: string[], opts: CliOptions = {}): Promise { + const env = this.env(opts.env); + const r = await runProcess(this.binary, args, { + env, + cwd: opts.cwd ?? this.ctx.workspace.dir, + timeoutMs: opts.timeoutMs, + input: opts.input, + allowSignal: opts.allowSignal, + label: `dpsapi ${args.slice(0, 2).join(' ')}`, + }).catch((e) => { + this.recordError('argv', this.binary, args, env, e); + throw e; + }); + const res: CliResult = { ...r, args: [...args], via: 'argv' }; + if (printsTokens(args)) this.ctx.secrets.registerAllIn(r.stdout); + this.record(res, env); + return res; + } + + /** + * Runs `script` with /bin/sh -c (for tests that exercise shell quoting). The environment has + * DPSAPI_BIN (the binary path) plus the default app env, so scripts use `"$DPSAPI_BIN" token new ...`. + * Token-shaped strings printed on stdout are registered as secrets. + */ + async shell(script: string, opts: CliOptions = {}): Promise { + const env = { ...this.env(opts.env), DPSAPI_BIN: this.binary }; + const r = await runProcess('/bin/sh', ['-c', script], { + env, + cwd: opts.cwd ?? this.ctx.workspace.dir, + timeoutMs: opts.timeoutMs, + input: opts.input, + allowSignal: opts.allowSignal, + label: 'sh -c