test(e2e): add persona e2e suites
Power, careless, malicious, fuzzing, cross-abuse, operator-session and concurrency personas exercising the whole application with model-checked database state.
This commit is contained in:
bryanpedini committed
1 parent
581a672fcf
commit
a139f6793d
7 files changed
+2983
No files matched your search
@@ -0,0 +1,360 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
// Copyright (C) 2026 Bryan Joshua Pedini
|
||||
|
||||
/**
|
||||
* PLAN §12 "long-running simulated operator session": one live instance, a large seeded sequence
|
||||
* of valid, mistaken, adversarial and randomized actions (CLI incl. `audit`, Playwright, curl, raw
|
||||
* socket, Engine faults, restarts, crashes, short foreign write locks), an in-test model of the expected
|
||||
* state (tokens active/inactive, audit rows per token), and invariant checks every 50 steps and at
|
||||
* the end.
|
||||
*
|
||||
* The session never stops at an expected failure: every step's outcome is checked, application
|
||||
* deviations are collected (with the step number and seed) and the session goes on, so a bad
|
||||
* request or command cannot mask later corruption. Periodic invariant checks, and any harness
|
||||
* error (timeout, signal, hang, transport failure), stop the session immediately.
|
||||
*/
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import {
|
||||
test,
|
||||
expect,
|
||||
errorMessage,
|
||||
expectCliFailure,
|
||||
expectCliSuccess,
|
||||
expectContainersResponse,
|
||||
expectErrorResponse,
|
||||
formatAuditOutput,
|
||||
isHarnessError,
|
||||
msToNanos,
|
||||
parseAuditOutput,
|
||||
parseLogs,
|
||||
parseTokenList,
|
||||
readOperations,
|
||||
tokenId,
|
||||
type CurlResult,
|
||||
type HttpResult,
|
||||
type RawHttpResponse,
|
||||
type Rng,
|
||||
} from '../../harness';
|
||||
import {
|
||||
StateModel,
|
||||
TOKEN_CHARS,
|
||||
checkResponse,
|
||||
cliOutcome,
|
||||
configOutcome,
|
||||
expectAuditWindow,
|
||||
expectHttp,
|
||||
expectNoTokenFragments,
|
||||
expectedAuditStdout,
|
||||
fuzzAddr,
|
||||
fuzzAuditArgs,
|
||||
fuzzLogLevel,
|
||||
fuzzPort,
|
||||
fuzzRequest,
|
||||
fuzzText,
|
||||
malformedRequest,
|
||||
matchesConfigError,
|
||||
nearMatches,
|
||||
onlyResponse,
|
||||
rawBytes,
|
||||
stateDigest,
|
||||
AUDIT_ALL_ARGV_NEAR_MISSES,
|
||||
type AppEnv,
|
||||
} from '../../harness/personas';
|
||||
|
||||
type Action =
|
||||
| 'create'
|
||||
| 'use'
|
||||
| 'use-revoked'
|
||||
| 'delete'
|
||||
| 'delete-again'
|
||||
| 'delete-unknown'
|
||||
| 'list'
|
||||
| 'bad-auth'
|
||||
| 'wrong-route'
|
||||
| 'cli-mistake'
|
||||
| 'config-mistake'
|
||||
| 'fuzz-request'
|
||||
| 'malformed-request'
|
||||
| 'backend'
|
||||
| 'containers'
|
||||
| 'restart'
|
||||
| 'crash'
|
||||
| 'short-lock'
|
||||
| 'trace-list'
|
||||
| 'audit'
|
||||
| 'audit-all';
|
||||
|
||||
const WEIGHTS: Array<[Action, number]> = [
|
||||
['create', 9],
|
||||
['use', 24],
|
||||
['use-revoked', 5],
|
||||
['delete', 5],
|
||||
['delete-again', 3],
|
||||
['delete-unknown', 3],
|
||||
['list', 4],
|
||||
['bad-auth', 6],
|
||||
['wrong-route', 4],
|
||||
['cli-mistake', 5],
|
||||
['config-mistake', 3],
|
||||
['fuzz-request', 12],
|
||||
['malformed-request', 4],
|
||||
['backend', 3],
|
||||
['containers', 2],
|
||||
['restart', 1],
|
||||
['crash', 0.7],
|
||||
['short-lock', 1],
|
||||
['trace-list', 1],
|
||||
['audit', 3],
|
||||
['audit-all', 1.5],
|
||||
];
|
||||
|
||||
function pickAction(r: Rng): Action {
|
||||
const total = WEIGHTS.reduce((n, [, w]) => n + w, 0);
|
||||
let x = r.next() * total;
|
||||
for (const [a, w] of WEIGHTS) if ((x -= w) < 0) return a;
|
||||
return 'use';
|
||||
}
|
||||
|
||||
test('operator session: 1000 seeded steps of valid, mistaken, adversarial and random actions on one live instance; invariants every 50 steps', async ({ cli, ws, startServer, http, fuzz, secrets }) => {
|
||||
test.setTimeout(240_000);
|
||||
const r = fuzz.rng.fork('session');
|
||||
const model = new StateModel();
|
||||
const server = await startServer({ logLevel: 'INFO' });
|
||||
const fake = server.fakeDocker!.many(5, r.fork('initial'));
|
||||
let backendOk = true;
|
||||
const deviations: string[] = [];
|
||||
const counts = new Map<Action, number>();
|
||||
let auditAllMistakes = 0;
|
||||
const STEPS = 1000;
|
||||
|
||||
const use = async (token: string, label: string): Promise<void> => {
|
||||
const kind = r.pick(['playwright', 'playwright', 'curl', 'raw']);
|
||||
let res: HttpResult | CurlResult | RawHttpResponse;
|
||||
if (kind === 'playwright') res = await http.get(server.url(), { token });
|
||||
else if (kind === 'curl') res = await http.curl({ url: server.url(), token });
|
||||
else res = onlyResponse(await http.raw({ host: server.connectHost, port: server.port, data: rawBytes({ cls: 'use', method: 'GET', target: '/api/v1/containers', headers: [['Authorization', `Bearer ${token}`]] }) }), label);
|
||||
const exp = expectHttp({ cls: 'use', method: 'GET', target: '/api/v1/containers', headers: [['Authorization', `Bearer ${token}`]] }, (t) => model.lookup(t), backendOk);
|
||||
checkResponse(res, exp, { containers: fake.containers.length, secrets, label });
|
||||
if (exp.audited) model.audit(exp.audited);
|
||||
};
|
||||
|
||||
for (let i = 0; i < STEPS; i++) {
|
||||
let action = pickAction(r);
|
||||
if ((action === 'use' || action === 'delete') && model.active().length === 0) action = 'create';
|
||||
if ((action === 'use-revoked' || action === 'delete-again') && model.inactive().length === 0) action = 'create';
|
||||
counts.set(action, (counts.get(action) ?? 0) + 1);
|
||||
const label = `step ${i} (${action})`;
|
||||
const rec: Record<string, unknown> = { i, action };
|
||||
fuzz.recordLive(rec); // filled in by the step below
|
||||
try {
|
||||
switch (action) {
|
||||
case 'create': {
|
||||
const d = fuzzText(r, { maxLen: r.bool(0.05) ? 60_000 : 512 }).value;
|
||||
rec.description = d.slice(0, 100);
|
||||
model.add(await cli.tokenNew(d), d);
|
||||
break;
|
||||
}
|
||||
case 'use':
|
||||
await use(r.pick(model.active()), label);
|
||||
break;
|
||||
case 'use-revoked':
|
||||
await use(r.pick(model.inactive()), label);
|
||||
break;
|
||||
case 'delete': {
|
||||
const t = r.pick(model.active());
|
||||
await cli.tokenDelete(t);
|
||||
model.deactivate(t);
|
||||
break;
|
||||
}
|
||||
case 'delete-again':
|
||||
case 'delete-unknown': {
|
||||
const arg = action === 'delete-again' ? r.pick(model.inactive()) : r.bool() && model.all().length ? nearMatches(r.pick(model.all()), r, 1)[0]?.value ?? 'x' : r.string(43, TOKEN_CHARS);
|
||||
const before = stateDigest(ws.dbPath);
|
||||
const o = cliOutcome(['token', 'delete', arg], model);
|
||||
const res = await cli.run(['token', 'delete', arg]);
|
||||
if (o.kind === 'error') expectCliFailure(res, { exitCode: o.exitCode, message: o.message });
|
||||
else throw new Error(`generator produced a valid deletion at ${label}`);
|
||||
expect(stateDigest(ws.dbPath), `${label}: state unchanged`).toBe(before);
|
||||
break;
|
||||
}
|
||||
case 'list':
|
||||
if (r.bool(0.5)) model.verifyList(await cli.tokenList(), label);
|
||||
else model.verifyList(await cli.tokenListAll(), `${label} (all)`, 'all');
|
||||
if (r.bool(0.1)) await cli.help();
|
||||
break;
|
||||
case 'bad-auth': {
|
||||
const t = model.all().length ? r.pick(model.all()) : 'x';
|
||||
const header = r.pick([`Bearer ${r.string(43, TOKEN_CHARS)}`, `Basic ${Buffer.from(`u:${t}`).toString('base64')}`, t, `Bearer ${t}`, `Bearer "${t}"`, 'Bearer', '', `Bearer ${t.toUpperCase()}x`]);
|
||||
rec.header = header.slice(0, 60);
|
||||
const res = onlyResponse(await http.raw({ host: server.connectHost, port: server.port, data: Buffer.from(`GET /api/v1/containers HTTP/1.1\r\nHost: x\r\nAuthorization: ${header}\r\nConnection: close\r\n\r\n`) }), label);
|
||||
expectErrorResponse(res, 401);
|
||||
break;
|
||||
}
|
||||
case 'wrong-route': {
|
||||
const t = model.active()[0] ?? 'x';
|
||||
if (r.bool()) expectErrorResponse(await http.get(server.url(r.pick(['/', '/api/v1/containers/', '/api/v1/container', '/api/v1/containers/x', '//api/v1/containers'])), { token: t }), 404);
|
||||
else expectErrorResponse(await http.fetch(server.url(), { method: r.pick(['POST', 'PUT', 'DELETE', 'PATCH']), token: t, data: '{}' }), 405);
|
||||
break;
|
||||
}
|
||||
case 'cli-mistake': {
|
||||
const allMiss = r.bool(0.3); // near misses of `audit all`: `audit ALL`, `audit all x`, `audit --all`...
|
||||
const args = allMiss
|
||||
? [...r.pick(AUDIT_ALL_ARGV_NEAR_MISSES)]
|
||||
: r.pick([['token'], ['token', 'lst'], ['tokens'], ['token', 'new', 'a', 'b'], ['token', 'list', 'x'], ['token', 'list', 'ALL'], ['token', 'list', 'all', 'all'], ['help', 'token'], ['Help'], ['token', 'delete'], ['--help'], ['token', 'delete', 'a', 'b'], ['audit', '0'], ['audit', '1', 'h'], ['audit', '1H'], ['audits'], ['audit', 'last']]);
|
||||
rec.args = args;
|
||||
if (allMiss) auditAllMistakes++;
|
||||
const o = cliOutcome(args, model);
|
||||
if (o.kind !== 'error') throw new Error('cli-mistake must fail');
|
||||
const before = stateDigest(ws.dbPath);
|
||||
expectCliFailure(await cli.run(args), { exitCode: o.exitCode, message: o.message, usage: o.usage });
|
||||
expect(stateDigest(ws.dbPath)).toBe(before);
|
||||
break;
|
||||
}
|
||||
case 'config-mistake': {
|
||||
let env: AppEnv = {};
|
||||
let addrValid: boolean | undefined;
|
||||
for (;;) {
|
||||
const a = fuzzAddr(r);
|
||||
env = r.bool() ? { HTTP_ADDR: a.value } : r.bool() ? { HTTP_PORT: fuzzPort(r) } : { LOG_LEVEL: fuzzLogLevel(r) };
|
||||
addrValid = env.HTTP_ADDR === undefined ? undefined : a.valid;
|
||||
if (!configOutcome(env, addrValid).ok) break;
|
||||
}
|
||||
rec.env = env;
|
||||
const cfg = configOutcome(env, addrValid);
|
||||
const res = await cli.run(r.bool() ? [] : ['token', 'new', 'never'], { env, timeoutMs: 10_000 });
|
||||
expectCliFailure(res, { exitCode: 2, usage: false });
|
||||
expect(matchesConfigError(errorMessage(res), cfg as Exclude<typeof cfg, { ok: true }>), `${label}: ${errorMessage(res)}`).toBe(true);
|
||||
break;
|
||||
}
|
||||
case 'fuzz-request':
|
||||
case 'malformed-request': {
|
||||
const c = action === 'fuzz-request' ? fuzzRequest(r, model) : malformedRequest(r, model);
|
||||
const exp = expectHttp(c, (t) => model.lookup(t), backendOk);
|
||||
Object.assign(rec, { method: c.method, target: c.target.slice(0, 200), headers: c.headers.map(([k, v]) => [k, v.slice(0, 100)]), expected: exp });
|
||||
const res = onlyResponse(await http.raw({ host: server.connectHost, port: server.port, data: rawBytes(c, { keepAlive: c.body !== undefined }), head: c.method === 'HEAD', timeoutMs: 20_000 }), label);
|
||||
checkResponse(res, exp, { head: c.method === 'HEAD', containers: fake.containers.length, secrets, label });
|
||||
if (exp.audited && (res.status === 200 || res.status === 500)) model.audit(exp.audited);
|
||||
break;
|
||||
}
|
||||
case 'backend': {
|
||||
backendOk = !backendOk || r.bool(0.3);
|
||||
rec.backendOk = backendOk;
|
||||
if (backendOk) fake.setContainers(fake.containers);
|
||||
else r.pick([() => fake.error500(), () => fake.destroy(), () => fake.badJson('{')])();
|
||||
break;
|
||||
}
|
||||
case 'containers':
|
||||
if (backendOk) fake.many(r.pick([0, 1, 3, 50, 400]), r);
|
||||
break;
|
||||
case 'restart': {
|
||||
const listing = (await cli.run(['token', 'list', 'all'])).stdout;
|
||||
await server.restart();
|
||||
expect((await cli.run(['token', 'list', 'all'])).stdout, `${label}: listing survives`).toBe(listing);
|
||||
break;
|
||||
}
|
||||
case 'crash': {
|
||||
const before = stateDigest(ws.dbPath);
|
||||
const k = await server.kill();
|
||||
expect(k.signal).toBe('SIGKILL');
|
||||
await server.restart();
|
||||
expect(stateDigest(ws.dbPath), `${label}: committed state survives SIGKILL`).toBe(before);
|
||||
break;
|
||||
}
|
||||
case 'short-lock': {
|
||||
const db = new DatabaseSync(ws.dbPath, { timeout: 3000 });
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
const release = new Promise<void>((res) =>
|
||||
setTimeout(() => {
|
||||
db.exec('ROLLBACK');
|
||||
db.close();
|
||||
res();
|
||||
}, 300),
|
||||
);
|
||||
try {
|
||||
if (model.active().length) await use(r.pick(model.active()), label);
|
||||
} finally {
|
||||
await release;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'audit': {
|
||||
// mostly the forms an operator types, sometimes fuzzed arguments; compared with SQLite and the model
|
||||
const args = r.bool(0.7) ? r.pick([[], ['all'], ['1'], ['5'], ['10'], ['25'], ['100000'], ['0010'], ['1h'], ['1d'], ['1w']]) : fuzzAuditArgs(r).args;
|
||||
rec.args = args.map((a) => a.slice(0, 60));
|
||||
const o = cliOutcome(['audit', ...args], model);
|
||||
const before = stateDigest(ws.dbPath);
|
||||
const t0 = msToNanos(Date.now() - 1);
|
||||
const res = await cli.run(['audit', ...args]);
|
||||
const t1 = msToNanos(Date.now() + 1);
|
||||
if (o.kind === 'audit') {
|
||||
expectCliSuccess(res, { stderr: '' });
|
||||
const ops = readOperations(ws.dbPath);
|
||||
const rows = parseAuditOutput(res.stdout);
|
||||
if ('windowNs' in o.selection) expectAuditWindow(rows, ops, o.selection.windowNs, t0, t1, label);
|
||||
else expect(res.stdout, label).toBe(expectedAuditStdout(ops, o.selection));
|
||||
if (rows.length === ops.length) {
|
||||
// a complete listing: rows per token equal the model's audit counts
|
||||
const per = new Map<string, number>();
|
||||
for (const x of rows) per.set(x.token, (per.get(x.token) ?? 0) + 1);
|
||||
for (const t of model.tokens.values()) expect(per.get(t.token) ?? 0, `${label}: rows of ${tokenId(t.token)}`).toBe(t.audits);
|
||||
}
|
||||
} else if (o.kind === 'error') expectCliFailure(res, { exitCode: o.exitCode, message: o.message, usage: o.usage });
|
||||
else throw new Error(`unexpected audit outcome ${o.kind}`);
|
||||
expect(stateDigest(ws.dbPath), `${label}: audit is read-only`).toBe(before);
|
||||
break;
|
||||
}
|
||||
case 'audit-all': {
|
||||
// `audit all`: the whole history, oldest first, byte-exact against SQLite and the model's per-token counts
|
||||
const o = cliOutcome(['audit', 'all'], model);
|
||||
if (o.kind !== 'audit' || !('all' in o.selection)) throw new Error('audit all must select every row');
|
||||
const before = stateDigest(ws.dbPath);
|
||||
const res = await cli.run(['audit', 'all']);
|
||||
expectCliSuccess(res, { stderr: '' });
|
||||
const ops = readOperations(ws.dbPath);
|
||||
expect(res.stdout, `${label}: every stored row`).toBe(expectedAuditStdout(ops, o.selection));
|
||||
const rows = parseAuditOutput(res.stdout);
|
||||
expect(rows.length, `${label}: one row per audited request`).toBe(model.auditTotal);
|
||||
const per = new Map<string, number>();
|
||||
for (const x of rows) per.set(x.token, (per.get(x.token) ?? 0) + 1);
|
||||
for (const t of model.tokens.values()) expect(per.get(t.token) ?? 0, `${label}: rows of ${tokenId(t.token)}`).toBe(t.audits);
|
||||
expect(stateDigest(ws.dbPath), `${label}: audit all is read-only`).toBe(before);
|
||||
break;
|
||||
}
|
||||
case 'trace-list': {
|
||||
const res = await cli.run(['token', 'list'], { env: { LOG_LEVEL: 'TRACE' } });
|
||||
expectCliSuccess(res);
|
||||
expect(parseLogs(res.stderr).unparsed, `${label}: TRACE output is well-formed`).toEqual([]);
|
||||
model.verifyList(parseTokenList(res.stdout), label);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
if (isHarnessError(e)) throw e; // infrastructure problems are never "expected"
|
||||
deviations.push(`${label}: ${(e as Error).message.split('\n').slice(0, 8).join(' | ')}`);
|
||||
if (deviations.length > 25) break;
|
||||
}
|
||||
if (i % 50 === 49) {
|
||||
// hard checks: corruption must stop the session immediately
|
||||
model.verifyDb(ws.dbPath, `invariants after step ${i}`, { auditBeforeDeletion: true });
|
||||
expect(server.isRunning(), `server alive after step ${i}`).toBe(true);
|
||||
}
|
||||
}
|
||||
|
||||
expect(deviations, 'steps whose outcome deviated from the model').toEqual([]);
|
||||
if (!backendOk) fake.setContainers(fake.containers);
|
||||
for (const t of model.active().slice(0, 5)) {
|
||||
expectContainersResponse(await http.get(server.url(), { token: t }));
|
||||
model.audit(t);
|
||||
}
|
||||
expect((await server.stop()).exitCode).toBe(0);
|
||||
model.verifyDb(ws.dbPath, 'end of session', { auditBeforeDeletion: true });
|
||||
expect(formatAuditOutput(await cli.audit('100000')), 'end of session: audit lists the whole history').toBe(expectedAuditStdout(readOperations(ws.dbPath), { limit: 100000n }));
|
||||
expect(formatAuditOutput(await cli.audit('all')), 'end of session: audit all lists the whole history').toBe(expectedAuditStdout(readOperations(ws.dbPath), { all: true }));
|
||||
model.verifyList(await cli.tokenList(), 'end of session');
|
||||
model.verifyList(await cli.tokenListAll(), 'end of session (all)', 'all');
|
||||
for (const g of server.allOutput()) expectNoTokenFragments(g.stderr, secrets, `server generation ${g.generation}`);
|
||||
test.info().annotations.push({ type: 'session', description: JSON.stringify({ steps: STEPS, generations: server.generation, tokens: model.tokens.size, inactive: model.inactive().length, audits: model.auditTotal, actions: Object.fromEntries(counts) }) });
|
||||
for (const a of ['create', 'use', 'delete', 'fuzz-request', 'restart', 'crash', 'short-lock', 'audit'] as const) expect(counts.get(a) ?? 0, `the session exercised ${a}`).toBeGreaterThan(0);
|
||||
expect(counts.get('audit-all') ?? 0, 'the session ran audit all').toBeGreaterThan(3);
|
||||
expect(auditAllMistakes, 'the session produced near misses of audit all').toBeGreaterThan(2);
|
||||
});
|
||||
Reference in new issue
Block a user