// SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2026 Bryan Joshua Pedini /** * PLAN §12 persona "Power user": realistic administrative missions at high volume with * deliberate state transitions — many tokens used simultaneously, repeated list/delete, * credential rotation, thousands of authenticated requests, real Docker containers with varied * names/labels/commands, large FakeDocker result sets, concurrent API traffic, repeated server * restarts (graceful and with a different log level) against the same long-lived database. * * After every transition the database is compared with the test's model (StateModel: tokens, * active flags, exact description bytes, audit rows per token) and the structural invariants, * `token list all` is compared byte-for-byte across restarts, `token list` / `token list all` with * the model and the stored rows, and audit history must be append-only. */ import { test, expect, assertDbInvariants, expectCliFailure, expectContainersResponse, expectErrorResponse, EMPTY_TOKEN_LIST, formatAuditOutput, mapLimit, readOperations, type ContainerSummary, type CurlResult, type HttpResult, type HttpClient, type RealDocker, type ServerHandle, type TestContainer, } from '../../harness'; import { StateModel, expectedAuditStdout, expectedTokenListStdout, stateDigest } from '../../harness/personas'; interface ApiContainer { Id: string; Names: string[]; Image: string; Command: string; Labels: Record; State: string; } /** n authenticated GETs spread over `tokens`, `concurrency` at a time, half Playwright / some curl; returns per-token counts. */ async function useSimultaneously(http: HttpClient, server: ServerHandle, tokens: string[], n: number, concurrency: number, expectCount: number): Promise> { const plan = Array.from({ length: n }, (_, i) => tokens[i % tokens.length]); const counts = new Map(); await mapLimit(plan, concurrency, async (t, i) => { const r = i % 10 === 9 ? await http.curl({ url: server.url(), token: t }) : await http.get(server.url(), { token: t }); const list = expectContainersResponse(r); expect(list.length).toBe(expectCount); counts.set(t, (counts.get(t) ?? 0) + 1); }); return counts; } test('fleet mission: 60 tokens, simultaneous use, repeated list/delete, rotations and 6 restarts against one long-lived database', async ({ cli, ws, startServer, http, rng }) => { test.setTimeout(180_000); const model = new StateModel(); const boundary: Array = [undefined, '', 'x', ' ', 'm'.repeat(131_071), '\u{65e5}'.repeat(20_000), 'ci \u{2192} prod \u{1f680}', 'tab\tnew\nline', `quote ' " \\ end`]; // boundary descriptions first, then a realistic population; half before the server starts for (const d of boundary) model.add(await cli.tokenNew(d), d ?? ''); const descs = Array.from({ length: 51 }, (_, i) => `svc-${String(i).padStart(2, '0')} ${rng.pick(['ci', 'deploy', 'monitoring', 'backup', 'dashboard'])}`); for (const [i, t] of (await mapLimit(descs.slice(0, 25), 8, (d) => cli.tokenNew(d))).entries()) model.add(t, descs[i]); model.verifyDb(ws.dbPath, 'initial population'); let server = await startServer({ logLevel: 'INFO', label: 'gen' }); const fake = server.fakeDocker!; for (const [i, t] of (await mapLimit(descs.slice(25), 8, (d) => cli.tokenNew(d))).entries()) model.add(t, descs[25 + i]); expect(model.tokens.size).toBe(60); const levels = ['DEBUG', 'TRACE', 'WARN', 'ERROR', 'INFO', 'DEBUG']; let rotation = 0; let listing = (await cli.run(['token', 'list', 'all'])).stdout; let opsBefore = readOperations(ws.dbPath); for (let phase = 0; phase < 6; phase++) { await test.step(`phase ${phase}`, async () => { // container population of this phase: empty, one, many, large const sizes = [0, 1, 40, 1500, 3000, 7]; fake.many(sizes[phase], rng.fork(`containers-${phase}`)); const ids = fake.containers.map((c: ContainerSummary) => c.Id); // 1. simultaneous use of 20 tokens: 400 requests, 20 concurrent const users = rng.shuffle(model.active()).slice(0, 20); const counts = await useSimultaneously(http, server, users, phase === 3 || phase === 4 ? 60 : 400, 20, ids.length); for (const [t, n] of counts) model.audit(t, n); if (ids.length) { const sample = expectContainersResponse(await http.get(server.url(), { token: users[0] })); expect(sample.map((c) => c.Id), 'complete and in Engine order').toEqual(ids); model.audit(users[0]); } model.verifyDb(ws.dbPath, `phase ${phase}: after simultaneous use`); // the operator's view of the history equals the stored rows (full listing and the default 10) const ops = readOperations(ws.dbPath); expect(formatAuditOutput(await cli.audit('100000')), `phase ${phase}: audit 100000`).toBe(expectedAuditStdout(ops, { limit: 100000n })); expect(formatAuditOutput(await cli.audit()), `phase ${phase}: audit`).toBe(expectedAuditStdout(ops, { limit: 10n })); // 2. repeated listings are byte-identical and match the model and the stored rows for (const scope of ['active', 'all'] as const) { const args = scope === 'all' ? ['token', 'list', 'all'] : ['token', 'list']; const a = await cli.run(args); const b = await cli.run(args); expect(b.stdout).toBe(a.stdout); expect(a.stdout, `phase ${phase}: ${args.join(' ')} equals the stored rows`).toBe(expectedTokenListStdout(ws.dbPath, scope)); model.verifyList(await cli.tokenList(scope), `phase ${phase}: ${args.join(' ')}`, scope); } // 3. credential rotation: replacement first, then revoke the old one for (let k = 0; k < 3; k++) { const old = rng.pick(model.active()); const desc = `${model.tokens.get(old)!.description.toString('utf8').slice(0, 40)} (rotation ${++rotation})`; const next = await cli.tokenNew(desc); model.add(next, desc); expectContainersResponse(await http.get(server.url(), { token: next })); model.audit(next); await cli.tokenDelete(old); model.deactivate(old); expectErrorResponse(await http.get(server.url(), { token: old }), 401); expectContainersResponse(await http.get(server.url(), { token: next })); model.audit(next); } // 4. repeated deletions of revoked tokens fail and change nothing const before = stateDigest(ws.dbPath); for (const t of rng.shuffle(model.inactive()).slice(0, 4)) expectCliFailure(await cli.run(['token', 'delete', t]), { exitCode: 1, message: 'token already inactive' }); expect(stateDigest(ws.dbPath)).toBe(before); model.verifyDb(ws.dbPath, `phase ${phase}: after rotation`, { auditBeforeDeletion: true }); // 5. restart against the same database (graceful restart, or a new process with another log level) listing = (await cli.run(['token', 'list', 'all'])).stdout; opsBefore = readOperations(ws.dbPath); if (phase % 2 === 0) { await server.restart(); } else { const exit = await server.stop(); expect(exit.exitCode).toBe(0); server = await startServer({ logLevel: levels[phase], docker: fake, dbPath: ws.dbPath, label: `gen-${levels[phase]}` }); } expect((await cli.run(['token', 'list', 'all'])).stdout, 'listing byte-identical across the restart').toBe(listing); expect(readOperations(ws.dbPath), 'audit history unchanged by the restart').toEqual(opsBefore); for (const t of model.inactive().slice(0, 3)) expectErrorResponse(await http.get(server.url(), { token: t }), 401); for (const t of model.active().slice(0, 3)) { expectContainersResponse(await http.get(server.url(), { token: t })); model.audit(t); } model.verifyDb(ws.dbPath, `phase ${phase}: after restart`, { auditBeforeDeletion: true }); }); } expect((await server.stop()).exitCode).toBe(0); const ops = readOperations(ws.dbPath); expect(ops.slice(0, opsBefore.length), 'audit history is append-only').toEqual(opsBefore); expect(ops.length).toBeGreaterThan(1500); const report = model.verifyDb(ws.dbPath, 'final', { auditBeforeDeletion: true }); expect(report).toEqual({ tokens: 60 + rotation, activeTokens: 60, inactiveTokens: rotation, operations: model.auditTotal }); }); /** Our containers in a container listing. */ const own = (list: ApiContainer[], docker: RealDocker) => list.filter((c) => c.Labels?.['dpsapi.e2e.test'] === docker.testKey); test('real Docker fleet: varied containers, concurrent clients with several tokens, lifecycle transitions and restarts — API equals docker ps after every transition', async ({ cli, ws, startServer, http, docker, rng }) => { test.setTimeout(180_000); const model = new StateModel(); for (const d of ['ops', 'ci', 'grafana', 'backup']) model.add(await cli.tokenNew(d), d); const commands = [['sleep', '3600'], ['tail', '-f', '/dev/null'], ['sh', '-c', 'trap "exit 0" TERM; while :; do sleep 1; done'], ['top', '-b', '-d', '30'], ['sh', '-c', 'echo "quoted \\"arg\\" $HOME"; exec sleep 3600']]; const spec = (i: number, gen: string) => ({ suffix: `${gen}${String(i).padStart(2, '0')}${['', '.dot', '_under', '-dash'][i % 4]}`, command: commands[i % commands.length], labels: { 'com.example.index': String(i), 'com.example.gen': gen, [`com.example.only-${gen}-${i}`]: i % 3 === 0 ? '' : i % 3 === 1 ? 'caf\u{e9} \u{2603} \u{65e5}\u{672c}' : 'x'.repeat(500), }, env: { ROLE: `role-${i}` }, }); const created = new Map(); for (const c of await docker.runMany(12, (i) => spec(i, 'a'), 6)) created.set(c.id, c); let server = await startServer({ docker: 'real', logLevel: 'DEBUG' }); const checkpoint = async (label: string) => { const running = (await docker.runningIds()).sort(); // every token, concurrently, through Playwright (2 each) and curl (1 each) const jobs: Array<() => Promise> = model.active().flatMap((t) => [() => http.get(server.url(), { token: t }), () => http.get(server.url(), { token: t }), () => http.curl({ url: server.url(), token: t })]); const results = await mapLimit(jobs, 8, (j) => j()); for (const r of results) { const list = expectContainersResponse(r); expect(list.every((c) => c.State === 'running'), `${label}: running containers only`).toBe(true); const mine = own(list, docker); expect(mine.map((c) => c.Id).sort(), `${label}: API equals docker ps for our label`).toEqual(running); for (const c of mine) { const exp = created.get(c.Id)!; expect(c.Names).toEqual([`/${exp.name}`]); expect(c.Image).toBe('alpine:3.22'); for (const [k, v] of Object.entries(exp.labels)) expect(c.Labels[k], `${label}: label ${k}`).toBe(v); } } for (const t of model.active()) model.audit(t, 3); model.verifyDb(ws.dbPath, label); return running.length; }; expect(await checkpoint('12 started')).toBe(12); const ids = [...created.keys()]; const stopped = rng.shuffle(ids).slice(0, 3); await Promise.all(stopped.map((id) => docker.stop(created.get(id)!, 1))); expect(await checkpoint('3 stopped')).toBe(9); const removable = ids.filter((id) => !stopped.includes(id)).slice(0, 2); await Promise.all(removable.map((id) => docker.remove(created.get(id)!))); expect(await checkpoint('2 removed')).toBe(7); await docker.start(created.get(stopped[0])!); expect(await checkpoint('1 restarted')).toBe(8); for (const c of await docker.runMany(3, (i) => spec(i, 'b'), 3)) created.set(c.id, c); expect(await checkpoint('3 more started')).toBe(11); // rotate a credential and restart the server twice against the same database const old = model.active()[0]; const next = await cli.tokenNew('ops v2'); model.add(next, 'ops v2'); await cli.tokenDelete(old); model.deactivate(old); expectErrorResponse(await http.get(server.url(), { token: old }), 401); const opsBefore = readOperations(ws.dbPath); await server.restart(); expect(await checkpoint('after graceful restart')).toBe(11); expect((await server.stop()).exitCode).toBe(0); server = await startServer({ docker: 'real', logLevel: 'INFO', dbPath: ws.dbPath }); expect(await checkpoint('after a new server process')).toBe(11); expect(readOperations(ws.dbPath).slice(0, opsBefore.length)).toEqual(opsBefore); await docker.cleanup(); expect(await checkpoint('all removed')).toBe(0); expect((await server.stop()).exitCode).toBe(0); model.verifyDb(ws.dbPath, 'final', { auditBeforeDeletion: true }); }); test('boundary populations on one database: zero tokens, one token, 1000 tokens from 16 concurrent CLI processes, 3000 audited requests, bulk deletion', async ({ cli, ws, startServer, http, rng }) => { test.setTimeout(180_000); const model = new StateModel(); const server = await startServer({ logLevel: 'WARN' }); server.fakeDocker!.many(3); await test.step('zero tokens: header-only listing, every credential refused', async () => { expect((await cli.run(['token', 'list'])).stdout).toBe(EMPTY_TOKEN_LIST); expect((await cli.run(['token', 'list', 'all'])).stdout).toBe(EMPTY_TOKEN_LIST); for (const t of ['', 'x', rng.string(43, 'abcdefghijklmnopqrstuvwxyz')]) expectErrorResponse(await http.get(server.url(), { headers: { Authorization: `Bearer ${t}` } }), 401); assertDbInvariants(ws.dbPath, { tokenCount: 0, operationCount: 0 }); }); await test.step('one token: usable immediately, by a running server', async () => { const t = model.add(await cli.tokenNew('the one'), 'the one').token; expectContainersResponse(await http.get(server.url(), { token: t })); model.audit(t); model.verifyDb(ws.dbPath); }); await test.step('1000 tokens created by 16 concurrent CLI processes while the server serves traffic', async () => { const descs = Array.from({ length: 999 }, (_, i) => `bulk ${i}`); const traffic = mapLimit(Array.from({ length: 200 }, () => model.active()[0]), 8, async (t) => { expectContainersResponse(await http.get(server.url(), { token: t })); }); const toks = await mapLimit(descs, 16, (d) => cli.tokenNew(d)); await traffic; model.audit(model.active()[0], 200); toks.forEach((t, i) => model.add(t, descs[i])); expect(new Set(toks).size).toBe(999); model.verifyDb(ws.dbPath, '1000 tokens'); model.verifyList(await cli.tokenList()); }); await test.step('3000 authenticated requests spread over 300 tokens, 32 concurrent', async () => { const users = rng.shuffle(model.active()).slice(0, 300); const plan = Array.from({ length: 3000 }, (_, i) => users[i % users.length]); await mapLimit(plan, 32, async (t) => { const r = await http.get(server.url(), { token: t }); expect(r.status).toBe(200); }); for (const t of plan) model.audit(t); model.verifyDb(ws.dbPath, '3000 requests'); const ops = readOperations(ws.dbPath); expect(formatAuditOutput(await cli.audit('5000')), 'audit 5000 lists all 3201 rows').toBe(expectedAuditStdout(ops, { limit: 5000n })); expect(formatAuditOutput(await cli.audit('1h')), 'audit 1h: the whole mission').toBe(expectedAuditStdout(ops, { limit: 5000n })); expect(formatAuditOutput(await cli.audit('300')), 'audit 300').toBe(expectedAuditStdout(ops, { limit: 300n })); }); await test.step('bulk deletion of 500 tokens (12 concurrent) and a second pass that must fail for each', async () => { const victims = rng.shuffle(model.active()).slice(0, 500); await mapLimit(victims, 12, (t) => cli.tokenDelete(t)); for (const t of victims) model.deactivate(t); const before = stateDigest(ws.dbPath); const again = await mapLimit(victims.slice(0, 100), 12, (t) => cli.run(['token', 'delete', t])); for (const r of again) expectCliFailure(r, { exitCode: 1, message: 'token already inactive' }); expect(stateDigest(ws.dbPath)).toBe(before); const probes = rng.shuffle([...victims.slice(0, 100), ...rng.shuffle(model.active()).slice(0, 100)]); await mapLimit(probes, 16, async (t) => { const r = await http.get(server.url(), { token: t }); if (model.lookup(t) === 'active') { expectContainersResponse(r); } else expectErrorResponse(r, 401); }); for (const t of probes) if (model.lookup(t) === 'active') model.audit(t); model.verifyList(await cli.tokenList(), 'after bulk deletion: 500 active'); model.verifyList(await cli.tokenListAll(), 'after bulk deletion: all 1000', 'all'); expect((await cli.run(['token', 'list'])).stdout, 'token list equals the stored active rows').toBe(expectedTokenListStdout(ws.dbPath, 'active')); }); expect((await server.stop()).exitCode).toBe(0); expect(server.logs().records, 'nothing at WARN or above during the whole mission').toEqual([]); const report = model.verifyDb(ws.dbPath, 'final', { auditBeforeDeletion: true }); expect(report.tokens).toBe(1000); expect(report.inactiveTokens).toBe(500); });