// SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2026 Bryan Joshua Pedini /** * PLAN ยง12 "long-running simulated operator session": one live instance, a large seeded sequence * of valid, mistaken, adversarial and randomized actions (CLI incl. `audit`, Playwright, curl, raw * socket, Engine faults, restarts, crashes, short foreign write locks), an in-test model of the expected * state (tokens active/inactive, audit rows per token), and invariant checks every 50 steps and at * the end. * * The session never stops at an expected failure: every step's outcome is checked, application * deviations are collected (with the step number and seed) and the session goes on, so a bad * request or command cannot mask later corruption. Periodic invariant checks, and any harness * error (timeout, signal, hang, transport failure), stop the session immediately. */ import { DatabaseSync } from 'node:sqlite'; import { test, expect, errorMessage, expectCliFailure, expectCliSuccess, expectContainersResponse, expectErrorResponse, formatAuditOutput, isHarnessError, msToNanos, parseAuditOutput, parseLogs, parseTokenList, readOperations, tokenId, type CurlResult, type HttpResult, type RawHttpResponse, type Rng, } from '../../harness'; import { StateModel, TOKEN_CHARS, checkResponse, cliOutcome, configOutcome, expectAuditWindow, expectHttp, expectNoTokenFragments, expectedAuditStdout, fuzzAddr, fuzzAuditArgs, fuzzLogLevel, fuzzPort, fuzzRequest, fuzzText, malformedRequest, matchesConfigError, nearMatches, onlyResponse, rawBytes, stateDigest, AUDIT_ALL_ARGV_NEAR_MISSES, type AppEnv, } from '../../harness/personas'; type Action = | 'create' | 'use' | 'use-revoked' | 'delete' | 'delete-again' | 'delete-unknown' | 'list' | 'bad-auth' | 'wrong-route' | 'cli-mistake' | 'config-mistake' | 'fuzz-request' | 'malformed-request' | 'backend' | 'containers' | 'restart' | 'crash' | 'short-lock' | 'trace-list' | 'audit' | 'audit-all'; const WEIGHTS: Array<[Action, number]> = [ ['create', 9], ['use', 24], ['use-revoked', 5], ['delete', 5], ['delete-again', 3], ['delete-unknown', 3], ['list', 4], ['bad-auth', 6], ['wrong-route', 4], ['cli-mistake', 5], ['config-mistake', 3], ['fuzz-request', 12], ['malformed-request', 4], ['backend', 3], ['containers', 2], ['restart', 1], ['crash', 0.7], ['short-lock', 1], ['trace-list', 1], ['audit', 3], ['audit-all', 1.5], ]; function pickAction(r: Rng): Action { const total = WEIGHTS.reduce((n, [, w]) => n + w, 0); let x = r.next() * total; for (const [a, w] of WEIGHTS) if ((x -= w) < 0) return a; return 'use'; } test('operator session: 1000 seeded steps of valid, mistaken, adversarial and random actions on one live instance; invariants every 50 steps', async ({ cli, ws, startServer, http, fuzz, secrets }) => { test.setTimeout(240_000); const r = fuzz.rng.fork('session'); const model = new StateModel(); const server = await startServer({ logLevel: 'INFO' }); const fake = server.fakeDocker!.many(5, r.fork('initial')); let backendOk = true; const deviations: string[] = []; const counts = new Map(); let auditAllMistakes = 0; const STEPS = 1000; const use = async (token: string, label: string): Promise => { const kind = r.pick(['playwright', 'playwright', 'curl', 'raw']); let res: HttpResult | CurlResult | RawHttpResponse; if (kind === 'playwright') res = await http.get(server.url(), { token }); else if (kind === 'curl') res = await http.curl({ url: server.url(), token }); else res = onlyResponse(await http.raw({ host: server.connectHost, port: server.port, data: rawBytes({ cls: 'use', method: 'GET', target: '/api/v1/containers', headers: [['Authorization', `Bearer ${token}`]] }) }), label); const exp = expectHttp({ cls: 'use', method: 'GET', target: '/api/v1/containers', headers: [['Authorization', `Bearer ${token}`]] }, (t) => model.lookup(t), backendOk); checkResponse(res, exp, { containers: fake.containers.length, secrets, label }); if (exp.audited) model.audit(exp.audited); }; for (let i = 0; i < STEPS; i++) { let action = pickAction(r); if ((action === 'use' || action === 'delete') && model.active().length === 0) action = 'create'; if ((action === 'use-revoked' || action === 'delete-again') && model.inactive().length === 0) action = 'create'; counts.set(action, (counts.get(action) ?? 0) + 1); const label = `step ${i} (${action})`; const rec: Record = { i, action }; fuzz.recordLive(rec); // filled in by the step below try { switch (action) { case 'create': { const d = fuzzText(r, { maxLen: r.bool(0.05) ? 60_000 : 512 }).value; rec.description = d.slice(0, 100); model.add(await cli.tokenNew(d), d); break; } case 'use': await use(r.pick(model.active()), label); break; case 'use-revoked': await use(r.pick(model.inactive()), label); break; case 'delete': { const t = r.pick(model.active()); await cli.tokenDelete(t); model.deactivate(t); break; } case 'delete-again': case 'delete-unknown': { const arg = action === 'delete-again' ? r.pick(model.inactive()) : r.bool() && model.all().length ? nearMatches(r.pick(model.all()), r, 1)[0]?.value ?? 'x' : r.string(43, TOKEN_CHARS); const before = stateDigest(ws.dbPath); const o = cliOutcome(['token', 'delete', arg], model); const res = await cli.run(['token', 'delete', arg]); if (o.kind === 'error') expectCliFailure(res, { exitCode: o.exitCode, message: o.message }); else throw new Error(`generator produced a valid deletion at ${label}`); expect(stateDigest(ws.dbPath), `${label}: state unchanged`).toBe(before); break; } case 'list': if (r.bool(0.5)) model.verifyList(await cli.tokenList(), label); else model.verifyList(await cli.tokenListAll(), `${label} (all)`, 'all'); if (r.bool(0.1)) await cli.help(); break; case 'bad-auth': { const t = model.all().length ? r.pick(model.all()) : 'x'; const header = r.pick([`Bearer ${r.string(43, TOKEN_CHARS)}`, `Basic ${Buffer.from(`u:${t}`).toString('base64')}`, t, `Bearer ${t}`, `Bearer "${t}"`, 'Bearer', '', `Bearer ${t.toUpperCase()}x`]); rec.header = header.slice(0, 60); const res = onlyResponse(await http.raw({ host: server.connectHost, port: server.port, data: Buffer.from(`GET /api/v1/containers HTTP/1.1\r\nHost: x\r\nAuthorization: ${header}\r\nConnection: close\r\n\r\n`) }), label); expectErrorResponse(res, 401); break; } case 'wrong-route': { const t = model.active()[0] ?? 'x'; if (r.bool()) expectErrorResponse(await http.get(server.url(r.pick(['/', '/api/v1/containers/', '/api/v1/container', '/api/v1/containers/x', '//api/v1/containers'])), { token: t }), 404); else expectErrorResponse(await http.fetch(server.url(), { method: r.pick(['POST', 'PUT', 'DELETE', 'PATCH']), token: t, data: '{}' }), 405); break; } case 'cli-mistake': { const allMiss = r.bool(0.3); // near misses of `audit all`: `audit ALL`, `audit all x`, `audit --all`... const args = allMiss ? [...r.pick(AUDIT_ALL_ARGV_NEAR_MISSES)] : r.pick([['token'], ['token', 'lst'], ['tokens'], ['token', 'new', 'a', 'b'], ['token', 'list', 'x'], ['token', 'list', 'ALL'], ['token', 'list', 'all', 'all'], ['help', 'token'], ['Help'], ['token', 'delete'], ['--help'], ['token', 'delete', 'a', 'b'], ['audit', '0'], ['audit', '1', 'h'], ['audit', '1H'], ['audits'], ['audit', 'last']]); rec.args = args; if (allMiss) auditAllMistakes++; const o = cliOutcome(args, model); if (o.kind !== 'error') throw new Error('cli-mistake must fail'); const before = stateDigest(ws.dbPath); expectCliFailure(await cli.run(args), { exitCode: o.exitCode, message: o.message, usage: o.usage }); expect(stateDigest(ws.dbPath)).toBe(before); break; } case 'config-mistake': { let env: AppEnv = {}; let addrValid: boolean | undefined; for (;;) { const a = fuzzAddr(r); env = r.bool() ? { HTTP_ADDR: a.value } : r.bool() ? { HTTP_PORT: fuzzPort(r) } : { LOG_LEVEL: fuzzLogLevel(r) }; addrValid = env.HTTP_ADDR === undefined ? undefined : a.valid; if (!configOutcome(env, addrValid).ok) break; } rec.env = env; const cfg = configOutcome(env, addrValid); const res = await cli.run(r.bool() ? [] : ['token', 'new', 'never'], { env, timeoutMs: 10_000 }); expectCliFailure(res, { exitCode: 2, usage: false }); expect(matchesConfigError(errorMessage(res), cfg as Exclude), `${label}: ${errorMessage(res)}`).toBe(true); break; } case 'fuzz-request': case 'malformed-request': { const c = action === 'fuzz-request' ? fuzzRequest(r, model) : malformedRequest(r, model); const exp = expectHttp(c, (t) => model.lookup(t), backendOk); Object.assign(rec, { method: c.method, target: c.target.slice(0, 200), headers: c.headers.map(([k, v]) => [k, v.slice(0, 100)]), expected: exp }); const res = onlyResponse(await http.raw({ host: server.connectHost, port: server.port, data: rawBytes(c, { keepAlive: c.body !== undefined }), head: c.method === 'HEAD', timeoutMs: 20_000 }), label); checkResponse(res, exp, { head: c.method === 'HEAD', containers: fake.containers.length, secrets, label }); if (exp.audited && (res.status === 200 || res.status === 500)) model.audit(exp.audited); break; } case 'backend': { backendOk = !backendOk || r.bool(0.3); rec.backendOk = backendOk; if (backendOk) fake.setContainers(fake.containers); else r.pick([() => fake.error500(), () => fake.destroy(), () => fake.badJson('{')])(); break; } case 'containers': if (backendOk) fake.many(r.pick([0, 1, 3, 50, 400]), r); break; case 'restart': { const listing = (await cli.run(['token', 'list', 'all'])).stdout; await server.restart(); expect((await cli.run(['token', 'list', 'all'])).stdout, `${label}: listing survives`).toBe(listing); break; } case 'crash': { const before = stateDigest(ws.dbPath); const k = await server.kill(); expect(k.signal).toBe('SIGKILL'); await server.restart(); expect(stateDigest(ws.dbPath), `${label}: committed state survives SIGKILL`).toBe(before); break; } case 'short-lock': { const db = new DatabaseSync(ws.dbPath, { timeout: 3000 }); db.exec('BEGIN IMMEDIATE'); const release = new Promise((res) => setTimeout(() => { db.exec('ROLLBACK'); db.close(); res(); }, 300), ); try { if (model.active().length) await use(r.pick(model.active()), label); } finally { await release; } break; } case 'audit': { // mostly the forms an operator types, sometimes fuzzed arguments; compared with SQLite and the model const args = r.bool(0.7) ? r.pick([[], ['all'], ['1'], ['5'], ['10'], ['25'], ['100000'], ['0010'], ['1h'], ['1d'], ['1w']]) : fuzzAuditArgs(r).args; rec.args = args.map((a) => a.slice(0, 60)); const o = cliOutcome(['audit', ...args], model); const before = stateDigest(ws.dbPath); const t0 = msToNanos(Date.now() - 1); const res = await cli.run(['audit', ...args]); const t1 = msToNanos(Date.now() + 1); if (o.kind === 'audit') { expectCliSuccess(res, { stderr: '' }); const ops = readOperations(ws.dbPath); const rows = parseAuditOutput(res.stdout); if ('windowNs' in o.selection) expectAuditWindow(rows, ops, o.selection.windowNs, t0, t1, label); else expect(res.stdout, label).toBe(expectedAuditStdout(ops, o.selection)); if (rows.length === ops.length) { // a complete listing: rows per token equal the model's audit counts const per = new Map(); for (const x of rows) per.set(x.token, (per.get(x.token) ?? 0) + 1); for (const t of model.tokens.values()) expect(per.get(t.token) ?? 0, `${label}: rows of ${tokenId(t.token)}`).toBe(t.audits); } } else if (o.kind === 'error') expectCliFailure(res, { exitCode: o.exitCode, message: o.message, usage: o.usage }); else throw new Error(`unexpected audit outcome ${o.kind}`); expect(stateDigest(ws.dbPath), `${label}: audit is read-only`).toBe(before); break; } case 'audit-all': { // `audit all`: the whole history, oldest first, byte-exact against SQLite and the model's per-token counts const o = cliOutcome(['audit', 'all'], model); if (o.kind !== 'audit' || !('all' in o.selection)) throw new Error('audit all must select every row'); const before = stateDigest(ws.dbPath); const res = await cli.run(['audit', 'all']); expectCliSuccess(res, { stderr: '' }); const ops = readOperations(ws.dbPath); expect(res.stdout, `${label}: every stored row`).toBe(expectedAuditStdout(ops, o.selection)); const rows = parseAuditOutput(res.stdout); expect(rows.length, `${label}: one row per audited request`).toBe(model.auditTotal); const per = new Map(); for (const x of rows) per.set(x.token, (per.get(x.token) ?? 0) + 1); for (const t of model.tokens.values()) expect(per.get(t.token) ?? 0, `${label}: rows of ${tokenId(t.token)}`).toBe(t.audits); expect(stateDigest(ws.dbPath), `${label}: audit all is read-only`).toBe(before); break; } case 'trace-list': { const res = await cli.run(['token', 'list'], { env: { LOG_LEVEL: 'TRACE' } }); expectCliSuccess(res); expect(parseLogs(res.stderr).unparsed, `${label}: TRACE output is well-formed`).toEqual([]); model.verifyList(parseTokenList(res.stdout), label); break; } } } catch (e) { if (isHarnessError(e)) throw e; // infrastructure problems are never "expected" deviations.push(`${label}: ${(e as Error).message.split('\n').slice(0, 8).join(' | ')}`); if (deviations.length > 25) break; } if (i % 50 === 49) { // hard checks: corruption must stop the session immediately model.verifyDb(ws.dbPath, `invariants after step ${i}`, { auditBeforeDeletion: true }); expect(server.isRunning(), `server alive after step ${i}`).toBe(true); } } expect(deviations, 'steps whose outcome deviated from the model').toEqual([]); if (!backendOk) fake.setContainers(fake.containers); for (const t of model.active().slice(0, 5)) { expectContainersResponse(await http.get(server.url(), { token: t })); model.audit(t); } expect((await server.stop()).exitCode).toBe(0); model.verifyDb(ws.dbPath, 'end of session', { auditBeforeDeletion: true }); expect(formatAuditOutput(await cli.audit('100000')), 'end of session: audit lists the whole history').toBe(expectedAuditStdout(readOperations(ws.dbPath), { limit: 100000n })); expect(formatAuditOutput(await cli.audit('all')), 'end of session: audit all lists the whole history').toBe(expectedAuditStdout(readOperations(ws.dbPath), { all: true })); model.verifyList(await cli.tokenList(), 'end of session'); model.verifyList(await cli.tokenListAll(), 'end of session (all)', 'all'); for (const g of server.allOutput()) expectNoTokenFragments(g.stderr, secrets, `server generation ${g.generation}`); test.info().annotations.push({ type: 'session', description: JSON.stringify({ steps: STEPS, generations: server.generation, tokens: model.tokens.size, inactive: model.inactive().length, audits: model.auditTotal, actions: Object.fromEntries(counts) }) }); for (const a of ['create', 'use', 'delete', 'fuzz-request', 'restart', 'crash', 'short-lock', 'audit'] as const) expect(counts.get(a) ?? 0, `the session exercised ${a}`).toBeGreaterThan(0); expect(counts.get('audit-all') ?? 0, 'the session ran audit all').toBeGreaterThan(3); expect(auditAllMistakes, 'the session produced near misses of audit all').toBeGreaterThan(2); });