test(e2e): add persona e2e suites
Power, careless, malicious, fuzzing, cross-abuse, operator-session and concurrency personas exercising the whole application with model-checked database state.
This commit is contained in:
bryanpedini committed
1 parent
581a672fcf
commit
a139f6793d
7 files changed
+2983
No files matched your search
@@ -0,0 +1,540 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
// Copyright (C) 2026 Bryan Joshua Pedini
|
||||
|
||||
/**
|
||||
* PLAN §12 persona "Fuzzer user": seeded, reproducible high-cardinality input across CLI argv,
|
||||
* descriptions, tokens, configuration variables (CLI and server mode), methods, targets, query
|
||||
* strings, headers, Authorization values, token populations, container populations and
|
||||
* concurrent/pipelined/fragmented request patterns.
|
||||
*
|
||||
* Every case is generated from the test's seed (run seed DPSAPI_E2E_SEED + title path, annotated
|
||||
* as `seed`) and recorded with `fuzz.record`; on failure the cases are attached as fuzz-cases.json.
|
||||
* Outcomes are checked against exact oracles (harness/personas.ts) derived from the README
|
||||
* contract: exit code + error line for the CLI, status + exact body for HTTP, audit rows per token,
|
||||
* logged request/auth-class multisets. Malformed-by-construction requests have explicit allowed
|
||||
* Go-level status sets. After every batch the database is checked against the model and the
|
||||
* structural invariants, so the process must survive, stay bounded and keep serving valid traffic.
|
||||
*/
|
||||
import {
|
||||
test,
|
||||
expect,
|
||||
errorMessage,
|
||||
expectCliFailure,
|
||||
expectCliSuccess,
|
||||
expectContainersResponse,
|
||||
goUnquoteBytes,
|
||||
makeContainerSummary,
|
||||
mapLimit,
|
||||
msToNanos,
|
||||
parseAuditOutput,
|
||||
parseLogs,
|
||||
parseTokenList,
|
||||
readOperations,
|
||||
readToken,
|
||||
safeStringify,
|
||||
USAGE_TEXT,
|
||||
tokenId,
|
||||
type CliResult,
|
||||
type ContainerSummary,
|
||||
type CurlResult,
|
||||
type HttpResult,
|
||||
type RawHttpResponse,
|
||||
type Rng,
|
||||
} from '../../harness';
|
||||
import {
|
||||
StateModel,
|
||||
TOKEN_CHARS,
|
||||
checkResponse,
|
||||
cliOutcome,
|
||||
configOutcome,
|
||||
expectAuditWindow,
|
||||
expectHttp,
|
||||
expectedAuditStdout,
|
||||
expectedTokenListStdout,
|
||||
fuzzAddr,
|
||||
fuzzAuditArgs,
|
||||
fuzzLogLevel,
|
||||
fuzzPort,
|
||||
fuzzRequest,
|
||||
fuzzText,
|
||||
isTokenChars,
|
||||
isValidLogLevel,
|
||||
isValidPort,
|
||||
malformedRequest,
|
||||
matchesConfigError,
|
||||
nearMatches,
|
||||
onlyResponse,
|
||||
rawBytes,
|
||||
routeOf,
|
||||
stateDigest,
|
||||
type AppEnv,
|
||||
type HttpCase,
|
||||
type HttpExpectation,
|
||||
AUDIT_ALL_ARGV_NEAR_MISSES,
|
||||
HELP_NEAR_MISSES,
|
||||
LIST_ALL_NEAR_MISSES,
|
||||
} from '../../harness/personas';
|
||||
|
||||
/** Compact, artifact-friendly rendering of a fuzz value (long values are summarized). */
|
||||
function show(v: string): string | { length: number; head: string } {
|
||||
return v.length <= 160 ? v : { length: v.length, head: v.slice(0, 80) };
|
||||
}
|
||||
|
||||
const MISSPELLED_COMMANDS = ['Token', 'tokens', 'TOKEN', 'tokn', 'list', 'new', 'delete', 'help', '-h', '--help', '', 'token ', ' token', 'tok\u0435n', 'token\n', 'AUDIT', 'Audit', 'audits', 'audit ', ' audit', 'aud\u0456t', 'log'];
|
||||
const MISSPELLED_SUBCOMMANDS = ['lst', 'ls', 'neww', 'nwe', 'delet', 'del', 'rm', 'remove', 'LIST', 'New', 'DELETE', '', ' list', 'list ', 'new\t', 'create', 'revoke'];
|
||||
|
||||
/** Seeded argv for the CLI fuzzer. */
|
||||
function genArgs(r: Rng, model: StateModel): string[] {
|
||||
if (r.bool(0.15)) return ['audit', ...fuzzAuditArgs(r).args];
|
||||
// `help`, `token list all` and `audit all`, their near misses (case, blanks, look-alikes, extra arguments)
|
||||
if (r.bool(0.14)) {
|
||||
const q = r.next();
|
||||
if (q < 0.3) return ['help'];
|
||||
if (q < 0.45) return ['token', 'list', 'all'];
|
||||
if (q < 0.6) return ['audit', 'all'];
|
||||
return [...r.pick([...HELP_NEAR_MISSES, ...LIST_ALL_NEAR_MISSES, ...AUDIT_ALL_ARGV_NEAR_MISSES])];
|
||||
}
|
||||
const p = r.next();
|
||||
const tail = () => Array.from({ length: r.int(0, 2) }, () => fuzzText(r, { maxLen: 512 }).value);
|
||||
if (p < 0.07) return [r.pick(MISSPELLED_COMMANDS), ...tail()];
|
||||
if (p < 0.1) return [fuzzText(r, { maxLen: 512 }).value, ...tail()];
|
||||
if (p < 0.13) return ['token'];
|
||||
if (p < 0.2) return ['token', r.bool(0.6) ? r.pick(MISSPELLED_SUBCOMMANDS) : fuzzText(r, { maxLen: 256 }).value, ...tail()];
|
||||
const sub = r.pick(['list', 'new', 'new', 'new', 'delete', 'delete', 'delete']);
|
||||
if (sub === 'list') {
|
||||
const q = r.next();
|
||||
if (q < 0.6) return ['token', 'list'];
|
||||
if (q < 0.8) return ['token', 'list', 'all'];
|
||||
return ['token', 'list', ...Array.from({ length: r.int(1, 2) }, () => fuzzText(r, { maxLen: 64 }).value)];
|
||||
}
|
||||
if (sub === 'new') {
|
||||
const q = r.next();
|
||||
if (q < 0.1) return ['token', 'new'];
|
||||
if (q < 0.9) return ['token', 'new', fuzzText(r, { maxLen: r.bool(0.05) ? 131_071 : 4096 }).value];
|
||||
return ['token', 'new', ...Array.from({ length: r.int(2, 3) }, () => fuzzText(r, { maxLen: 64 }).value)];
|
||||
}
|
||||
const q = r.next();
|
||||
if (q < 0.08) return ['token', 'delete'];
|
||||
if (q < 0.15) return ['token', 'delete', ...Array.from({ length: r.int(2, 3) }, () => (model.active().length && r.bool() ? r.pick(model.active()) : fuzzText(r, { maxLen: 64 }).value))];
|
||||
const active = model.active();
|
||||
const inactive = model.inactive();
|
||||
const v = r.next();
|
||||
if (v < 0.35 && active.length) return ['token', 'delete', r.pick(active)];
|
||||
if (v < 0.5 && inactive.length) return ['token', 'delete', r.pick(inactive)];
|
||||
if (v < 0.75 && model.all().length) return ['token', 'delete', nearMatches(r.pick(model.all()), r, 1)[0]?.value ?? 'x'];
|
||||
return ['token', 'delete', r.bool(0.3) ? r.string(43, TOKEN_CHARS) : fuzzText(r, { maxLen: 256 }).value];
|
||||
}
|
||||
|
||||
/** argv mentioning the word `all` in any case / look-alike form (`audit ALL`, `token list all x`...). */
|
||||
const mentionsAll = (args: string[]) => args.some((a) => /[a\u0430]ll/i.test(a));
|
||||
|
||||
/** Seeded configuration variables (each independently set with some probability). */
|
||||
function genEnv(r: Rng): { env: AppEnv; addrValid?: boolean } {
|
||||
if (r.bool(0.65)) return { env: {} };
|
||||
const env: AppEnv = {};
|
||||
let addrValid: boolean | undefined;
|
||||
if (r.bool(0.4)) {
|
||||
const a = fuzzAddr(r);
|
||||
env.HTTP_ADDR = a.value;
|
||||
addrValid = a.valid;
|
||||
}
|
||||
if (r.bool(0.5)) env.HTTP_PORT = fuzzPort(r);
|
||||
if (r.bool(0.6)) env.LOG_LEVEL = fuzzLogLevel(r);
|
||||
return { env, addrValid };
|
||||
}
|
||||
|
||||
/** Stderr of a successful verbose run must consist of well-formed slog records only. */
|
||||
function expectOnlyLogRecords(r: CliResult, label: string): void {
|
||||
const logs = parseLogs(r.stderr);
|
||||
expect(logs.unparsed.map((u) => u.text.slice(0, 200)), `${label}: every stderr line is a slog record`).toEqual([]);
|
||||
}
|
||||
|
||||
test('CLI: 300 seeded argv/environment cases with an exact outcome oracle; database always matches the model', async ({ cli, ws, fuzz, secrets, startServer, http }) => {
|
||||
test.setTimeout(150_000);
|
||||
const r = fuzz.rng.fork('cli');
|
||||
const model = new StateModel();
|
||||
// starting population so delete/near-match cases have targets
|
||||
for (let i = 0; i < 6; i++) {
|
||||
const d = fuzzText(r, { maxLen: 256 }).value;
|
||||
model.add(await cli.tokenNew(d), d);
|
||||
}
|
||||
// audit history (real authenticated requests) so `audit` selections are non-trivial
|
||||
const server = await startServer();
|
||||
for (let i = 0; i < 24; i++) {
|
||||
const t = r.pick(model.active());
|
||||
expectContainersResponse(await http.get(server.url(), { token: t }));
|
||||
model.audit(t);
|
||||
}
|
||||
expect((await server.stop()).exitCode).toBe(0);
|
||||
let auditRuns = 0;
|
||||
let auditUsageErrors = 0;
|
||||
let helpRuns = 0;
|
||||
let helpListUsageErrors = 0;
|
||||
let auditAllRuns = 0;
|
||||
let auditAllUsageErrors = 0;
|
||||
const listRuns = { active: 0, all: 0 };
|
||||
const N = 300;
|
||||
for (let i = 0; i < N; i++) {
|
||||
// every 30th case is `audit all` or one of its near misses, so both are always exercised
|
||||
const forced = i % 30 === 5 ? ['audit', 'all'] : i % 30 === 20 ? [...r.pick(AUDIT_ALL_ARGV_NEAR_MISSES.filter((a) => a[0] === 'audit'))] : undefined;
|
||||
const viaShell = !forced && r.bool(0.06);
|
||||
if (viaShell) {
|
||||
// Invalid UTF-8 can only reach argv/env through raw bytes: a real shell with printf octal escapes.
|
||||
const bytes = Buffer.from(r.bytes(r.int(1, 24)).map((b) => (b === 0 ? 0xff : b)));
|
||||
if (bytes[bytes.length - 1] === 0x0a) bytes[bytes.length - 1] = 0xc0; // $(...) strips trailing newlines
|
||||
const fmt = [...bytes].map((b) => '\\' + b.toString(8).padStart(3, '0')).join('');
|
||||
const intoEnv = r.bool(0.4);
|
||||
const c = fuzz.record({ i, via: 'shell', intoEnv, bytesHex: bytes.toString('hex') });
|
||||
const before = stateDigest(ws.dbPath);
|
||||
if (intoEnv) {
|
||||
const res = await cli.shell(`HTTP_PORT="$(printf '${fmt}')" "$DPSAPI_BIN" token list`);
|
||||
expectCliFailure(res, { exitCode: 2, usage: false });
|
||||
const msg = errorMessage(res)!;
|
||||
const m = /^configuration error: invalid HTTP_PORT (".*"): must be an integer between 1 and 65535$/.exec(msg);
|
||||
expect(m, `case ${i}: ${JSON.stringify(c)} → ${msg}`).not.toBeNull();
|
||||
expect(goUnquoteBytes(m![1]).equals(bytes), `case ${i}: value quoted byte-exact`).toBe(true);
|
||||
expect(stateDigest(ws.dbPath)).toBe(before);
|
||||
} else {
|
||||
const res = await cli.shell(`"$DPSAPI_BIN" token new "$(printf '${fmt}')"`);
|
||||
expectCliSuccess(res, { stderr: '' });
|
||||
expect(res.stdout).toMatch(/^[A-Za-z0-9_-]{43}\n$/);
|
||||
const tok = res.stdout.trim();
|
||||
model.add(tok, bytes);
|
||||
expect(readToken(ws.dbPath, tok)!.descriptionHex, `case ${i}: invalid UTF-8 stored byte-exact`).toBe(bytes.toString('hex').toUpperCase());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const args = forced ?? genArgs(r, model);
|
||||
const { env, addrValid } = genEnv(r);
|
||||
const cfg = configOutcome(env, addrValid);
|
||||
const outcome = cfg.ok ? cliOutcome(args, model) : undefined;
|
||||
const c = fuzz.record({ i, args: args.map(show), env: Object.fromEntries(Object.entries(env).map(([k, v]) => [k, show(v!)])), expected: cfg.ok ? outcome : cfg });
|
||||
const label = `case ${i} ${safeStringify(c).slice(0, 400)}`;
|
||||
const verbose = cfg.ok && env.LOG_LEVEL !== undefined && /^(debug|trace)$/i.test(env.LOG_LEVEL);
|
||||
const before = stateDigest(ws.dbPath);
|
||||
const t0 = msToNanos(Date.now() - 1);
|
||||
const res = await cli.run(args, { env });
|
||||
const t1 = msToNanos(Date.now() + 1);
|
||||
|
||||
if (!cfg.ok) {
|
||||
// configuration errors are reported before the logger exists: exactly one stderr line
|
||||
expectCliFailure(res, { exitCode: 2, usage: false });
|
||||
expect(matchesConfigError(errorMessage(res), cfg), `${label}: got ${JSON.stringify(errorMessage(res))}`).toBe(true);
|
||||
expect(stateDigest(ws.dbPath), `${label}: state unchanged`).toBe(before);
|
||||
continue;
|
||||
}
|
||||
switch (outcome!.kind) {
|
||||
case 'list': {
|
||||
expectCliSuccess(res);
|
||||
if (!verbose) expect(res.stderr, label).toBe('');
|
||||
else expectOnlyLogRecords(res, label);
|
||||
model.verifyList(parseTokenList(res.stdout), label, outcome!.scope);
|
||||
expect(res.stdout, `${label}: exact padded table of the stored rows`).toBe(expectedTokenListStdout(ws.dbPath, outcome!.scope));
|
||||
expect(stateDigest(ws.dbPath), `${label}: list is read-only`).toBe(before);
|
||||
listRuns[outcome!.scope]++;
|
||||
break;
|
||||
}
|
||||
case 'help': {
|
||||
expectCliSuccess(res, { stdout: USAGE_TEXT });
|
||||
if (!verbose) expect(res.stderr, label).toBe('');
|
||||
else expectOnlyLogRecords(res, label);
|
||||
expect(stateDigest(ws.dbPath), `${label}: help is read-only`).toBe(before);
|
||||
helpRuns++;
|
||||
break;
|
||||
}
|
||||
case 'new': {
|
||||
expectCliSuccess(res);
|
||||
expect(res.stdout, label).toMatch(/^[A-Za-z0-9_-]{43}\n$/);
|
||||
if (!verbose) expect(res.stderr, label).toBe('');
|
||||
else expectOnlyLogRecords(res, label);
|
||||
const tok = secrets.register(res.stdout.trim());
|
||||
model.add(tok, outcome!.description);
|
||||
break;
|
||||
}
|
||||
case 'delete': {
|
||||
expectCliSuccess(res, { stdout: '' });
|
||||
if (!verbose) expect(res.stderr, label).toBe('');
|
||||
else expectOnlyLogRecords(res, label);
|
||||
model.deactivate(outcome!.token);
|
||||
break;
|
||||
}
|
||||
case 'audit': {
|
||||
expectCliSuccess(res);
|
||||
if (!verbose) expect(res.stderr, label).toBe('');
|
||||
else expectOnlyLogRecords(res, label);
|
||||
const sel = outcome!.selection;
|
||||
const ops = readOperations(ws.dbPath);
|
||||
if ('windowNs' in sel) expectAuditWindow(parseAuditOutput(res.stdout), ops, sel.windowNs, t0, t1, label);
|
||||
else expect(res.stdout, label).toBe(expectedAuditStdout(ops, sel));
|
||||
expect(stateDigest(ws.dbPath), `${label}: audit is read-only`).toBe(before);
|
||||
auditRuns++;
|
||||
if ('all' in sel) {
|
||||
expect(parseAuditOutput(res.stdout), `${label}: audit all prints every row`).toHaveLength(model.auditTotal);
|
||||
auditAllRuns++;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'error': {
|
||||
expectCliFailure(res, { exitCode: outcome!.exitCode, message: outcome!.message, usage: outcome!.usage, exactStderr: !verbose });
|
||||
if (verbose) {
|
||||
// nothing but log records before the error line (no echo of any argument)
|
||||
const head = outcome!.usage ? res.stderr.slice(0, res.stderr.length - USAGE_TEXT.length) : res.stderr;
|
||||
expect(parseLogs(head).unparsed.map((u) => u.text.slice(0, 120)), `${label}: only the error line is not a log record`).toEqual([`dpsapi: ${outcome!.message}`]);
|
||||
}
|
||||
if (args[0] === 'audit') auditUsageErrors++;
|
||||
if (args[0] === 'help' || (args[0] === 'token' && args[1] === 'list')) helpListUsageErrors++;
|
||||
if (args[0] === 'audit' && mentionsAll(args.slice(1))) auditAllUsageErrors++;
|
||||
expect(stateDigest(ws.dbPath), `${label}: failed command left the state unchanged`).toBe(before);
|
||||
const deleted = args[1] === 'delete' ? args[2] : undefined;
|
||||
if (deleted && /^[A-Za-z0-9_-]{40,}$/.test(deleted)) expect(res.stderr.includes(deleted), `${label}: supplied token never echoed`).toBe(false);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (i % 20 === 19) model.verifyDb(ws.dbPath, `after case ${i}`);
|
||||
}
|
||||
model.verifyDb(ws.dbPath, 'final state');
|
||||
model.verifyList(await cli.tokenList(), 'final token list');
|
||||
model.verifyList(await cli.tokenListAll(), 'final token list all', 'all');
|
||||
expect(model.tokens.size, 'the fuzzer actually created tokens').toBeGreaterThan(20);
|
||||
expect(model.inactive().length, 'the fuzzer actually deleted tokens').toBeGreaterThan(3);
|
||||
expect(auditRuns, 'the fuzzer ran valid audit selections').toBeGreaterThan(8);
|
||||
expect(auditUsageErrors, 'the fuzzer produced invalid audit arguments').toBeGreaterThan(8);
|
||||
expect(helpRuns, 'the fuzzer ran help').toBeGreaterThan(2);
|
||||
expect(listRuns.active, 'the fuzzer ran token list').toBeGreaterThan(5);
|
||||
expect(listRuns.all, 'the fuzzer ran token list all').toBeGreaterThan(2);
|
||||
expect(helpListUsageErrors, 'the fuzzer produced help / token list usage errors').toBeGreaterThan(5);
|
||||
expect(auditAllRuns, 'the fuzzer ran audit all').toBeGreaterThan(3);
|
||||
expect(auditAllUsageErrors, 'the fuzzer produced audit all near misses').toBeGreaterThan(3);
|
||||
test.info().annotations.push({ type: 'audit fuzz', description: `${auditRuns} valid audit invocations (${auditAllRuns} audit all), ${auditUsageErrors} audit usage errors (${auditAllUsageErrors} near misses of audit all)` });
|
||||
test.info().annotations.push({ type: 'help/list fuzz', description: `${helpRuns} help, ${listRuns.active} token list, ${listRuns.all} token list all, ${helpListUsageErrors} help/list usage errors` });
|
||||
});
|
||||
|
||||
type Transport = 'raw' | 'raw-fragmented' | 'playwright' | 'curl';
|
||||
|
||||
function playwrightEligible(c: HttpCase, base: string): boolean {
|
||||
if (!['GET', 'POST', 'PUT', 'DELETE', 'PATCH'].includes(c.method) || c.rawLines || c.goReject || routeOf(c.target) === 'bad') return false;
|
||||
const names = c.headers.map(([n]) => n.toLowerCase());
|
||||
if (new Set(names).size !== names.length) return false;
|
||||
// eslint-disable-next-line no-control-regex
|
||||
if (!c.headers.every(([, v]) => /^[\x20-\x7e]*$/.test(v) && v === v.trim())) return false;
|
||||
try {
|
||||
const u = new URL(base + c.target);
|
||||
return u.pathname + u.search === c.target; // WHATWG URL parsing must not rewrite the target
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function curlEligible(c: HttpCase): boolean {
|
||||
if (c.rawLines || c.goReject || c.target.length > 2000) return false;
|
||||
// curl cannot combine -I with a body; Go-rejected targets close the connection with the body unread (RST)
|
||||
if ((c.method === 'HEAD' && c.body !== undefined) || routeOf(c.target) === 'bad' || !isTokenChars(c.method)) return false;
|
||||
if (!/^[A-Za-z0-9\-._~%/?=&;:@!$'()*+,]+$/.test(c.target)) return false;
|
||||
return c.headers.every(([n, v]) => /^[\x21-\x7e]([\x20-\x7e]*[\x21-\x7e])?$/.test(v) && !/^(accept|user-agent)$/i.test(n));
|
||||
}
|
||||
|
||||
test('HTTP: 1000+ seeded requests across methods, targets, headers and Authorization with an exact oracle, evolving token and container populations', async ({ cli, startServer, http, fuzz, secrets }) => {
|
||||
test.setTimeout(240_000);
|
||||
const r = fuzz.rng.fork('http');
|
||||
const model = new StateModel();
|
||||
const descs = Array.from({ length: 24 }, () => fuzzText(r, { maxLen: 512 }).value);
|
||||
const toks = await mapLimit(descs, 8, (d) => cli.tokenNew(d));
|
||||
toks.forEach((t, i) => model.add(t, descs[i]));
|
||||
for (const t of r.shuffle(toks).slice(0, 7)) {
|
||||
await cli.tokenDelete(t);
|
||||
model.deactivate(t);
|
||||
}
|
||||
const server = await startServer({ logLevel: 'DEBUG' });
|
||||
const fake = server.fakeDocker!;
|
||||
const host = { host: server.connectHost, port: server.port };
|
||||
|
||||
let reached = 0;
|
||||
let dockerErrors = 0;
|
||||
const expectedClasses: string[] = [];
|
||||
const statusesReached: number[] = [];
|
||||
let sent = 0;
|
||||
const seen = new Map<number, number>();
|
||||
|
||||
const BATCHES = 20;
|
||||
for (let b = 0; b < BATCHES; b++) {
|
||||
// container population and backend health for this batch
|
||||
const n = r.pick([0, 0, 1, 2, r.int(3, 50), r.int(50, 400)]);
|
||||
const containers: ContainerSummary[] = Array.from({ length: n }, (_, i) =>
|
||||
makeContainerSummary(i, r, { Labels: { 'fuzz.batch': String(b), 'fuzz.text': fuzzText(r, { maxLen: 64 }).value }, Names: [`/fuzz-${b}-${i}`] }),
|
||||
);
|
||||
const mode = r.pick(['ok', 'ok', 'ok', 'ok', 'ok', 'error', 'destroy', 'badjson'] as const);
|
||||
fake.setContainers(containers);
|
||||
if (mode === 'error') fake.error500();
|
||||
else if (mode === 'destroy') fake.destroy();
|
||||
else if (mode === 'badjson') fake.badJson(r.pick(['{', '{"a":1}', '[{"Id":', '<html>', '"text"', '42']));
|
||||
const backendOk = mode === 'ok';
|
||||
const concurrency = r.int(1, 16);
|
||||
|
||||
// the batch: well-formed singles, malformed singles, pipelines
|
||||
type Unit = { kind: 'single'; c: HttpCase; t: Transport } | { kind: 'pipeline'; cs: HttpCase[] };
|
||||
const units: Unit[] = [];
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const p = r.next();
|
||||
if (p < 0.06) {
|
||||
const cs: HttpCase[] = [];
|
||||
while (cs.length < r.int(2, 5)) {
|
||||
const c = fuzzRequest(r, model);
|
||||
const e = expectHttp(c, (t) => model.lookup(t), backendOk);
|
||||
if (e.reachesApp && c.method !== 'HEAD') cs.push(c);
|
||||
}
|
||||
units.push({ kind: 'pipeline', cs });
|
||||
continue;
|
||||
}
|
||||
const c = p < 0.16 ? malformedRequest(r, model) : fuzzRequest(r, model);
|
||||
const options: Transport[] = ['raw', 'raw', 'raw', 'raw-fragmented'];
|
||||
if (playwrightEligible(c, server.baseURL)) options.push('playwright', 'playwright');
|
||||
if (curlEligible(c)) options.push('curl');
|
||||
units.push({ kind: 'single', c, t: r.pick(options) });
|
||||
}
|
||||
|
||||
const audits: string[] = [];
|
||||
const tally = (exp: HttpExpectation, status: number): void => {
|
||||
seen.set(status, (seen.get(status) ?? 0) + 1);
|
||||
if (exp.reachesApp) {
|
||||
reached++;
|
||||
statusesReached.push(status);
|
||||
}
|
||||
if (exp.authClass) expectedClasses.push(exp.authClass);
|
||||
if (exp.audited && (status === 200 || status === 500)) audits.push(exp.audited);
|
||||
if (status === 500) dockerErrors++;
|
||||
};
|
||||
await mapLimit(units, concurrency, async (u, ui) => {
|
||||
const label = `batch ${b} unit ${ui}`;
|
||||
if (u.kind === 'pipeline') {
|
||||
const exps = u.cs.map((c) => expectHttp(c, (t) => model.lookup(t), backendOk));
|
||||
fuzz.record({ b, ui, transport: 'pipeline', cases: u.cs.map((c) => ({ method: c.method, target: show(c.target), headers: c.headers.map(([k, v]) => [k, show(v)]) })), expected: exps });
|
||||
const data = Buffer.concat(u.cs.map((c, i) => rawBytes(c, { keepAlive: i < u.cs.length - 1 })));
|
||||
const res = await http.raw({ ...host, data, expectResponses: u.cs.length, timeoutMs: 20_000 });
|
||||
expect(res.responses.length, `${label}: one response per pipelined request`).toBe(u.cs.length);
|
||||
res.responses.forEach((resp, i) => {
|
||||
checkResponse(resp, exps[i], { containers: n, secrets, label: `${label}#${i}` });
|
||||
tally(exps[i], resp.status);
|
||||
});
|
||||
sent += u.cs.length;
|
||||
return;
|
||||
}
|
||||
const { c, t } = u;
|
||||
const exp = expectHttp(c, (tok) => model.lookup(tok), backendOk);
|
||||
fuzz.record({ b, ui, transport: t, cls: c.cls, method: show(c.method), target: show(c.target), headers: c.headers.map(([k, v]) => [k, show(v)]), rawLines: c.rawLines?.slice(0, 3).map(show), body: c.body === undefined ? undefined : show(c.body), expected: exp });
|
||||
let res: RawHttpResponse | HttpResult | CurlResult;
|
||||
const head = c.method === 'HEAD';
|
||||
if (t === 'raw' || t === 'raw-fragmented') {
|
||||
// Requests with a body are sent keep-alive: with `Connection: close` Go answers before
|
||||
// reading the body and closes with unread bytes (RST), which can destroy the response in
|
||||
// flight on the client side — a transport artefact, not an application outcome.
|
||||
const bytes = rawBytes(c, { keepAlive: c.body !== undefined });
|
||||
let data: Buffer | Buffer[] = bytes;
|
||||
if (t === 'raw-fragmented') {
|
||||
const cuts = [...new Set(Array.from({ length: r.int(1, 6) }, () => r.int(1, bytes.length - 1)))].sort((a, z) => a - z);
|
||||
data = [0, ...cuts].map((s, k) => bytes.subarray(s, cuts[k] ?? bytes.length));
|
||||
}
|
||||
res = onlyResponse(await http.raw({ ...host, data, chunkDelayMs: t === 'raw-fragmented' ? 3 : undefined, head, timeoutMs: 20_000 }), label);
|
||||
} else if (t === 'playwright') {
|
||||
res = await http.fetch(server.url(c.target), { method: c.method, headers: Object.fromEntries(c.headers), data: c.body });
|
||||
} else {
|
||||
res = await http.curl({ url: server.url(c.target), method: head ? undefined : c.method, head, headers: c.headers.map(([k, v]) => `${k}: ${v}`), data: c.body, pathAsIs: true });
|
||||
expect((res as CurlResult).exitCode, `${label}: curl exit`).toBe(0);
|
||||
}
|
||||
checkResponse(res, exp, { head, containers: n, secrets, label });
|
||||
tally(exp, res.status);
|
||||
sent++;
|
||||
});
|
||||
|
||||
for (const t of audits) model.audit(t);
|
||||
model.verifyDb(server.dbPath, `after batch ${b} (${mode}, ${n} containers, concurrency ${concurrency})`);
|
||||
|
||||
// token population evolves between batches
|
||||
if (r.bool(0.6) && model.active().length > 3) {
|
||||
const victim = r.pick(model.active());
|
||||
await cli.tokenDelete(victim);
|
||||
model.deactivate(victim);
|
||||
}
|
||||
if (r.bool(0.6)) {
|
||||
const d = fuzzText(r, { maxLen: 256 }).value;
|
||||
model.add(await cli.tokenNew(d), d);
|
||||
}
|
||||
}
|
||||
|
||||
// the server is still fully functional
|
||||
fake.many(3);
|
||||
const last = model.active()[0];
|
||||
expect(expectContainersResponse(await http.get(server.url(), { token: last }))).toHaveLength(3);
|
||||
model.audit(last);
|
||||
expect((await server.stop()).exitCode).toBe(0);
|
||||
|
||||
const logs = server.logs();
|
||||
expect(sent, 'requests sent').toBeGreaterThanOrEqual(1000);
|
||||
for (const st of [200, 400, 401, 404, 405, 500]) expect(seen.get(st) ?? 0, `the fuzzer produced status ${st}`).toBeGreaterThan(0);
|
||||
test.info().annotations.push({ type: 'status distribution', description: JSON.stringify(Object.fromEntries([...seen].sort((a, z) => a[0] - z[0]))) });
|
||||
expect(logs.requests().length, 'every request that reached the app was logged exactly once').toBe(reached + 1);
|
||||
expect(
|
||||
logs.requests().map((x) => Number(x.attrs.status)).sort(),
|
||||
'logged statuses equal the observed ones',
|
||||
).toEqual([...statusesReached, 200].sort());
|
||||
expect(logs.filter({ msg: 'authentication result' }).map((x) => x.attrs.result).sort(), 'authentication classes equal the oracle').toEqual([...expectedClasses, 'ok'].sort());
|
||||
expect(logs.count({ level: 'ERROR', msg: 'docker request failed' }), 'one ERROR per Docker failure').toBe(dockerErrors);
|
||||
expect(logs.count({ level: 'ERROR' })).toBe(dockerErrors);
|
||||
expect(logs.count({ level: 'WARN' })).toBe(0);
|
||||
model.verifyDb(server.dbPath, 'final state');
|
||||
// every token id in the auth log belongs to a known token: nothing unknown was ever authenticated
|
||||
const known = new Set(model.all().map(tokenId));
|
||||
expect(logs.filter({ msg: 'authentication result', attrs: { result: 'ok' } }).every((x) => known.has(x.attrs.token_id))).toBe(true);
|
||||
});
|
||||
|
||||
test('environment: 60 seeded HTTP_ADDR/HTTP_PORT/LOG_LEVEL combinations in server mode — exit 2 with the exact message, or a working server', async ({ cli, ws, startServer, http, e2e, fuzz }) => {
|
||||
test.setTimeout(150_000);
|
||||
const r = fuzz.rng.fork('env');
|
||||
const token = await cli.tokenNew('env fuzz');
|
||||
const loopbacks = ['127.0.0.1', '::1', '127.0.0.2', '127.255.255.254', '0:0:0:0:0:0:0:1', '::ffff:127.0.0.1'];
|
||||
let started = 0;
|
||||
let rejected = 0;
|
||||
for (let i = 0; i < 60; i++) {
|
||||
const digest = stateDigest(ws.dbPath);
|
||||
const port = await e2e.allocatePort();
|
||||
const env: AppEnv = {};
|
||||
let addrValid = true;
|
||||
// addresses: valid loopbacks only when they would be bound; otherwise invalid values
|
||||
if (r.bool(0.55)) env.HTTP_ADDR = r.pick(loopbacks);
|
||||
else {
|
||||
const a = fuzzAddr(r);
|
||||
env.HTTP_ADDR = a.valid ? r.pick(loopbacks) : a.value;
|
||||
addrValid = a.valid;
|
||||
}
|
||||
env.HTTP_PORT = r.bool(0.55) ? '0'.repeat(r.pick([0, 0, 1, 5])) + String(port) : fuzzPort(r);
|
||||
if (isValidPort(env.HTTP_PORT) && Number(env.HTTP_PORT) !== port) env.HTTP_PORT = String(port); // never bind a port we did not allocate
|
||||
if (r.bool(0.8)) env.LOG_LEVEL = r.bool(0.6) ? r.pick(['error', 'Warn', 'INFO', 'debug', 'TrAcE']) : fuzzLogLevel(r);
|
||||
const cfg = configOutcome(env, addrValid);
|
||||
fuzz.record({ i, env: Object.fromEntries(Object.entries(env).map(([k, v]) => [k, show(v!)])), expected: cfg });
|
||||
const label = `case ${i} ${JSON.stringify(env).slice(0, 300)}`;
|
||||
if (!cfg.ok) {
|
||||
const res = await cli.run([], { env, timeoutMs: 10_000 });
|
||||
expectCliFailure(res, { exitCode: 2, usage: false });
|
||||
expect(matchesConfigError(errorMessage(res), cfg), `${label}: got ${JSON.stringify(errorMessage(res))}`).toBe(true);
|
||||
expect(stateDigest(ws.dbPath), `${label}: database untouched`).toBe(digest);
|
||||
rejected++;
|
||||
continue;
|
||||
}
|
||||
const server = await startServer({ addr: env.HTTP_ADDR, port: env.HTTP_PORT, logLevel: env.LOG_LEVEL ?? null, label: `env${i}` });
|
||||
const res = await http.get(server.url(), { token });
|
||||
expectContainersResponse(res);
|
||||
const exit = await server.stop();
|
||||
expect(exit.exitCode, label).toBe(0);
|
||||
const level = (env.LOG_LEVEL ?? 'INFO').toUpperCase();
|
||||
expect(isValidLogLevel(level)).toBe(true);
|
||||
const logs = server.logs();
|
||||
expect(logs.unparsed, `${label}: stderr is only slog records`).toEqual([]);
|
||||
const rank = ['TRACE', 'DEBUG', 'INFO', 'WARN', 'ERROR'].indexOf(level);
|
||||
expect(logs.requests().length, `${label}: request record visible iff level <= INFO`).toBe(rank <= 2 ? 1 : 0);
|
||||
expect(logs.levels().every((l) => ['TRACE', 'DEBUG', 'INFO', 'WARN', 'ERROR'].indexOf(l) >= rank), `${label}: nothing below ${level}`).toBe(true);
|
||||
if (rank <= 2) expect(logs.find({ msg: 'server listening' })!.attrs.addr).toMatch(new RegExp(`:${port}$`));
|
||||
started++;
|
||||
}
|
||||
expect(started, 'some combinations were valid').toBeGreaterThan(5);
|
||||
expect(rejected, 'some combinations were invalid').toBeGreaterThan(5);
|
||||
});
|
||||
Reference in new issue
Block a user