bryanpedini
a139f6793d
Power, careless, malicious, fuzzing, cross-abuse, operator-session and concurrency personas exercising the whole application with model-checked database state.
312 lines
17 KiB
TypeScript
312 lines
17 KiB
TypeScript
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
// Copyright (C) 2026 Bryan Joshua Pedini
|
|
|
|
/**
|
|
* PLAN §12 persona "Power user": realistic administrative missions at high volume with
|
|
* deliberate state transitions — many tokens used simultaneously, repeated list/delete,
|
|
* credential rotation, thousands of authenticated requests, real Docker containers with varied
|
|
* names/labels/commands, large FakeDocker result sets, concurrent API traffic, repeated server
|
|
* restarts (graceful and with a different log level) against the same long-lived database.
|
|
*
|
|
* After every transition the database is compared with the test's model (StateModel: tokens,
|
|
* active flags, exact description bytes, audit rows per token) and the structural invariants,
|
|
* `token list all` is compared byte-for-byte across restarts, `token list` / `token list all` with
|
|
* the model and the stored rows, and audit history must be append-only.
|
|
*/
|
|
import {
|
|
test,
|
|
expect,
|
|
assertDbInvariants,
|
|
expectCliFailure,
|
|
expectContainersResponse,
|
|
expectErrorResponse,
|
|
EMPTY_TOKEN_LIST,
|
|
formatAuditOutput,
|
|
mapLimit,
|
|
readOperations,
|
|
type ContainerSummary,
|
|
type CurlResult,
|
|
type HttpResult,
|
|
type HttpClient,
|
|
type RealDocker,
|
|
type ServerHandle,
|
|
type TestContainer,
|
|
} from '../../harness';
|
|
import { StateModel, expectedAuditStdout, expectedTokenListStdout, stateDigest } from '../../harness/personas';
|
|
|
|
interface ApiContainer {
|
|
Id: string;
|
|
Names: string[];
|
|
Image: string;
|
|
Command: string;
|
|
Labels: Record<string, string>;
|
|
State: string;
|
|
}
|
|
|
|
/** n authenticated GETs spread over `tokens`, `concurrency` at a time, half Playwright / some curl; returns per-token counts. */
|
|
async function useSimultaneously(http: HttpClient, server: ServerHandle, tokens: string[], n: number, concurrency: number, expectCount: number): Promise<Map<string, number>> {
|
|
const plan = Array.from({ length: n }, (_, i) => tokens[i % tokens.length]);
|
|
const counts = new Map<string, number>();
|
|
await mapLimit(plan, concurrency, async (t, i) => {
|
|
const r = i % 10 === 9 ? await http.curl({ url: server.url(), token: t }) : await http.get(server.url(), { token: t });
|
|
const list = expectContainersResponse(r);
|
|
expect(list.length).toBe(expectCount);
|
|
counts.set(t, (counts.get(t) ?? 0) + 1);
|
|
});
|
|
return counts;
|
|
}
|
|
|
|
test('fleet mission: 60 tokens, simultaneous use, repeated list/delete, rotations and 6 restarts against one long-lived database', async ({ cli, ws, startServer, http, rng }) => {
|
|
test.setTimeout(180_000);
|
|
const model = new StateModel();
|
|
const boundary: Array<string | undefined> = [undefined, '', 'x', ' ', 'm'.repeat(131_071), '\u{65e5}'.repeat(20_000), 'ci \u{2192} prod \u{1f680}', 'tab\tnew\nline', `quote ' " \\ end`];
|
|
// boundary descriptions first, then a realistic population; half before the server starts
|
|
for (const d of boundary) model.add(await cli.tokenNew(d), d ?? '');
|
|
const descs = Array.from({ length: 51 }, (_, i) => `svc-${String(i).padStart(2, '0')} ${rng.pick(['ci', 'deploy', 'monitoring', 'backup', 'dashboard'])}`);
|
|
for (const [i, t] of (await mapLimit(descs.slice(0, 25), 8, (d) => cli.tokenNew(d))).entries()) model.add(t, descs[i]);
|
|
model.verifyDb(ws.dbPath, 'initial population');
|
|
|
|
let server = await startServer({ logLevel: 'INFO', label: 'gen' });
|
|
const fake = server.fakeDocker!;
|
|
for (const [i, t] of (await mapLimit(descs.slice(25), 8, (d) => cli.tokenNew(d))).entries()) model.add(t, descs[25 + i]);
|
|
expect(model.tokens.size).toBe(60);
|
|
|
|
const levels = ['DEBUG', 'TRACE', 'WARN', 'ERROR', 'INFO', 'DEBUG'];
|
|
let rotation = 0;
|
|
let listing = (await cli.run(['token', 'list', 'all'])).stdout;
|
|
let opsBefore = readOperations(ws.dbPath);
|
|
for (let phase = 0; phase < 6; phase++) {
|
|
await test.step(`phase ${phase}`, async () => {
|
|
// container population of this phase: empty, one, many, large
|
|
const sizes = [0, 1, 40, 1500, 3000, 7];
|
|
fake.many(sizes[phase], rng.fork(`containers-${phase}`));
|
|
const ids = fake.containers.map((c: ContainerSummary) => c.Id);
|
|
|
|
// 1. simultaneous use of 20 tokens: 400 requests, 20 concurrent
|
|
const users = rng.shuffle(model.active()).slice(0, 20);
|
|
const counts = await useSimultaneously(http, server, users, phase === 3 || phase === 4 ? 60 : 400, 20, ids.length);
|
|
for (const [t, n] of counts) model.audit(t, n);
|
|
if (ids.length) {
|
|
const sample = expectContainersResponse<ContainerSummary>(await http.get(server.url(), { token: users[0] }));
|
|
expect(sample.map((c) => c.Id), 'complete and in Engine order').toEqual(ids);
|
|
model.audit(users[0]);
|
|
}
|
|
model.verifyDb(ws.dbPath, `phase ${phase}: after simultaneous use`);
|
|
// the operator's view of the history equals the stored rows (full listing and the default 10)
|
|
const ops = readOperations(ws.dbPath);
|
|
expect(formatAuditOutput(await cli.audit('100000')), `phase ${phase}: audit 100000`).toBe(expectedAuditStdout(ops, { limit: 100000n }));
|
|
expect(formatAuditOutput(await cli.audit()), `phase ${phase}: audit`).toBe(expectedAuditStdout(ops, { limit: 10n }));
|
|
|
|
// 2. repeated listings are byte-identical and match the model and the stored rows
|
|
for (const scope of ['active', 'all'] as const) {
|
|
const args = scope === 'all' ? ['token', 'list', 'all'] : ['token', 'list'];
|
|
const a = await cli.run(args);
|
|
const b = await cli.run(args);
|
|
expect(b.stdout).toBe(a.stdout);
|
|
expect(a.stdout, `phase ${phase}: ${args.join(' ')} equals the stored rows`).toBe(expectedTokenListStdout(ws.dbPath, scope));
|
|
model.verifyList(await cli.tokenList(scope), `phase ${phase}: ${args.join(' ')}`, scope);
|
|
}
|
|
|
|
// 3. credential rotation: replacement first, then revoke the old one
|
|
for (let k = 0; k < 3; k++) {
|
|
const old = rng.pick(model.active());
|
|
const desc = `${model.tokens.get(old)!.description.toString('utf8').slice(0, 40)} (rotation ${++rotation})`;
|
|
const next = await cli.tokenNew(desc);
|
|
model.add(next, desc);
|
|
expectContainersResponse(await http.get(server.url(), { token: next }));
|
|
model.audit(next);
|
|
await cli.tokenDelete(old);
|
|
model.deactivate(old);
|
|
expectErrorResponse(await http.get(server.url(), { token: old }), 401);
|
|
expectContainersResponse(await http.get(server.url(), { token: next }));
|
|
model.audit(next);
|
|
}
|
|
|
|
// 4. repeated deletions of revoked tokens fail and change nothing
|
|
const before = stateDigest(ws.dbPath);
|
|
for (const t of rng.shuffle(model.inactive()).slice(0, 4)) expectCliFailure(await cli.run(['token', 'delete', t]), { exitCode: 1, message: 'token already inactive' });
|
|
expect(stateDigest(ws.dbPath)).toBe(before);
|
|
model.verifyDb(ws.dbPath, `phase ${phase}: after rotation`, { auditBeforeDeletion: true });
|
|
|
|
// 5. restart against the same database (graceful restart, or a new process with another log level)
|
|
listing = (await cli.run(['token', 'list', 'all'])).stdout;
|
|
opsBefore = readOperations(ws.dbPath);
|
|
if (phase % 2 === 0) {
|
|
await server.restart();
|
|
} else {
|
|
const exit = await server.stop();
|
|
expect(exit.exitCode).toBe(0);
|
|
server = await startServer({ logLevel: levels[phase], docker: fake, dbPath: ws.dbPath, label: `gen-${levels[phase]}` });
|
|
}
|
|
expect((await cli.run(['token', 'list', 'all'])).stdout, 'listing byte-identical across the restart').toBe(listing);
|
|
expect(readOperations(ws.dbPath), 'audit history unchanged by the restart').toEqual(opsBefore);
|
|
for (const t of model.inactive().slice(0, 3)) expectErrorResponse(await http.get(server.url(), { token: t }), 401);
|
|
for (const t of model.active().slice(0, 3)) {
|
|
expectContainersResponse(await http.get(server.url(), { token: t }));
|
|
model.audit(t);
|
|
}
|
|
model.verifyDb(ws.dbPath, `phase ${phase}: after restart`, { auditBeforeDeletion: true });
|
|
});
|
|
}
|
|
expect((await server.stop()).exitCode).toBe(0);
|
|
const ops = readOperations(ws.dbPath);
|
|
expect(ops.slice(0, opsBefore.length), 'audit history is append-only').toEqual(opsBefore);
|
|
expect(ops.length).toBeGreaterThan(1500);
|
|
const report = model.verifyDb(ws.dbPath, 'final', { auditBeforeDeletion: true });
|
|
expect(report).toEqual({ tokens: 60 + rotation, activeTokens: 60, inactiveTokens: rotation, operations: model.auditTotal });
|
|
});
|
|
|
|
/** Our containers in a container listing. */
|
|
const own = (list: ApiContainer[], docker: RealDocker) => list.filter((c) => c.Labels?.['dpsapi.e2e.test'] === docker.testKey);
|
|
|
|
test('real Docker fleet: varied containers, concurrent clients with several tokens, lifecycle transitions and restarts — API equals docker ps after every transition', async ({ cli, ws, startServer, http, docker, rng }) => {
|
|
test.setTimeout(180_000);
|
|
const model = new StateModel();
|
|
for (const d of ['ops', 'ci', 'grafana', 'backup']) model.add(await cli.tokenNew(d), d);
|
|
const commands = [['sleep', '3600'], ['tail', '-f', '/dev/null'], ['sh', '-c', 'trap "exit 0" TERM; while :; do sleep 1; done'], ['top', '-b', '-d', '30'], ['sh', '-c', 'echo "quoted \\"arg\\" $HOME"; exec sleep 3600']];
|
|
const spec = (i: number, gen: string) => ({
|
|
suffix: `${gen}${String(i).padStart(2, '0')}${['', '.dot', '_under', '-dash'][i % 4]}`,
|
|
command: commands[i % commands.length],
|
|
labels: {
|
|
'com.example.index': String(i),
|
|
'com.example.gen': gen,
|
|
[`com.example.only-${gen}-${i}`]: i % 3 === 0 ? '' : i % 3 === 1 ? 'caf\u{e9} \u{2603} \u{65e5}\u{672c}' : 'x'.repeat(500),
|
|
},
|
|
env: { ROLE: `role-${i}` },
|
|
});
|
|
const created = new Map<string, TestContainer>();
|
|
for (const c of await docker.runMany(12, (i) => spec(i, 'a'), 6)) created.set(c.id, c);
|
|
let server = await startServer({ docker: 'real', logLevel: 'DEBUG' });
|
|
|
|
const checkpoint = async (label: string) => {
|
|
const running = (await docker.runningIds()).sort();
|
|
// every token, concurrently, through Playwright (2 each) and curl (1 each)
|
|
const jobs: Array<() => Promise<HttpResult | CurlResult>> = model.active().flatMap((t) => [() => http.get(server.url(), { token: t }), () => http.get(server.url(), { token: t }), () => http.curl({ url: server.url(), token: t })]);
|
|
const results = await mapLimit(jobs, 8, (j) => j());
|
|
for (const r of results) {
|
|
const list = expectContainersResponse<ApiContainer>(r);
|
|
expect(list.every((c) => c.State === 'running'), `${label}: running containers only`).toBe(true);
|
|
const mine = own(list, docker);
|
|
expect(mine.map((c) => c.Id).sort(), `${label}: API equals docker ps for our label`).toEqual(running);
|
|
for (const c of mine) {
|
|
const exp = created.get(c.Id)!;
|
|
expect(c.Names).toEqual([`/${exp.name}`]);
|
|
expect(c.Image).toBe('alpine:3.22');
|
|
for (const [k, v] of Object.entries(exp.labels)) expect(c.Labels[k], `${label}: label ${k}`).toBe(v);
|
|
}
|
|
}
|
|
for (const t of model.active()) model.audit(t, 3);
|
|
model.verifyDb(ws.dbPath, label);
|
|
return running.length;
|
|
};
|
|
|
|
expect(await checkpoint('12 started')).toBe(12);
|
|
const ids = [...created.keys()];
|
|
const stopped = rng.shuffle(ids).slice(0, 3);
|
|
await Promise.all(stopped.map((id) => docker.stop(created.get(id)!, 1)));
|
|
expect(await checkpoint('3 stopped')).toBe(9);
|
|
const removable = ids.filter((id) => !stopped.includes(id)).slice(0, 2);
|
|
await Promise.all(removable.map((id) => docker.remove(created.get(id)!)));
|
|
expect(await checkpoint('2 removed')).toBe(7);
|
|
await docker.start(created.get(stopped[0])!);
|
|
expect(await checkpoint('1 restarted')).toBe(8);
|
|
for (const c of await docker.runMany(3, (i) => spec(i, 'b'), 3)) created.set(c.id, c);
|
|
expect(await checkpoint('3 more started')).toBe(11);
|
|
|
|
// rotate a credential and restart the server twice against the same database
|
|
const old = model.active()[0];
|
|
const next = await cli.tokenNew('ops v2');
|
|
model.add(next, 'ops v2');
|
|
await cli.tokenDelete(old);
|
|
model.deactivate(old);
|
|
expectErrorResponse(await http.get(server.url(), { token: old }), 401);
|
|
const opsBefore = readOperations(ws.dbPath);
|
|
await server.restart();
|
|
expect(await checkpoint('after graceful restart')).toBe(11);
|
|
expect((await server.stop()).exitCode).toBe(0);
|
|
server = await startServer({ docker: 'real', logLevel: 'INFO', dbPath: ws.dbPath });
|
|
expect(await checkpoint('after a new server process')).toBe(11);
|
|
expect(readOperations(ws.dbPath).slice(0, opsBefore.length)).toEqual(opsBefore);
|
|
|
|
await docker.cleanup();
|
|
expect(await checkpoint('all removed')).toBe(0);
|
|
expect((await server.stop()).exitCode).toBe(0);
|
|
model.verifyDb(ws.dbPath, 'final', { auditBeforeDeletion: true });
|
|
});
|
|
|
|
test('boundary populations on one database: zero tokens, one token, 1000 tokens from 16 concurrent CLI processes, 3000 audited requests, bulk deletion', async ({ cli, ws, startServer, http, rng }) => {
|
|
test.setTimeout(180_000);
|
|
const model = new StateModel();
|
|
const server = await startServer({ logLevel: 'WARN' });
|
|
server.fakeDocker!.many(3);
|
|
|
|
await test.step('zero tokens: header-only listing, every credential refused', async () => {
|
|
expect((await cli.run(['token', 'list'])).stdout).toBe(EMPTY_TOKEN_LIST);
|
|
expect((await cli.run(['token', 'list', 'all'])).stdout).toBe(EMPTY_TOKEN_LIST);
|
|
for (const t of ['', 'x', rng.string(43, 'abcdefghijklmnopqrstuvwxyz')]) expectErrorResponse(await http.get(server.url(), { headers: { Authorization: `Bearer ${t}` } }), 401);
|
|
assertDbInvariants(ws.dbPath, { tokenCount: 0, operationCount: 0 });
|
|
});
|
|
|
|
await test.step('one token: usable immediately, by a running server', async () => {
|
|
const t = model.add(await cli.tokenNew('the one'), 'the one').token;
|
|
expectContainersResponse(await http.get(server.url(), { token: t }));
|
|
model.audit(t);
|
|
model.verifyDb(ws.dbPath);
|
|
});
|
|
|
|
await test.step('1000 tokens created by 16 concurrent CLI processes while the server serves traffic', async () => {
|
|
const descs = Array.from({ length: 999 }, (_, i) => `bulk ${i}`);
|
|
const traffic = mapLimit(Array.from({ length: 200 }, () => model.active()[0]), 8, async (t) => {
|
|
expectContainersResponse(await http.get(server.url(), { token: t }));
|
|
});
|
|
const toks = await mapLimit(descs, 16, (d) => cli.tokenNew(d));
|
|
await traffic;
|
|
model.audit(model.active()[0], 200);
|
|
toks.forEach((t, i) => model.add(t, descs[i]));
|
|
expect(new Set(toks).size).toBe(999);
|
|
model.verifyDb(ws.dbPath, '1000 tokens');
|
|
model.verifyList(await cli.tokenList());
|
|
});
|
|
|
|
await test.step('3000 authenticated requests spread over 300 tokens, 32 concurrent', async () => {
|
|
const users = rng.shuffle(model.active()).slice(0, 300);
|
|
const plan = Array.from({ length: 3000 }, (_, i) => users[i % users.length]);
|
|
await mapLimit(plan, 32, async (t) => {
|
|
const r = await http.get(server.url(), { token: t });
|
|
expect(r.status).toBe(200);
|
|
});
|
|
for (const t of plan) model.audit(t);
|
|
model.verifyDb(ws.dbPath, '3000 requests');
|
|
const ops = readOperations(ws.dbPath);
|
|
expect(formatAuditOutput(await cli.audit('5000')), 'audit 5000 lists all 3201 rows').toBe(expectedAuditStdout(ops, { limit: 5000n }));
|
|
expect(formatAuditOutput(await cli.audit('1h')), 'audit 1h: the whole mission').toBe(expectedAuditStdout(ops, { limit: 5000n }));
|
|
expect(formatAuditOutput(await cli.audit('300')), 'audit 300').toBe(expectedAuditStdout(ops, { limit: 300n }));
|
|
});
|
|
|
|
await test.step('bulk deletion of 500 tokens (12 concurrent) and a second pass that must fail for each', async () => {
|
|
const victims = rng.shuffle(model.active()).slice(0, 500);
|
|
await mapLimit(victims, 12, (t) => cli.tokenDelete(t));
|
|
for (const t of victims) model.deactivate(t);
|
|
const before = stateDigest(ws.dbPath);
|
|
const again = await mapLimit(victims.slice(0, 100), 12, (t) => cli.run(['token', 'delete', t]));
|
|
for (const r of again) expectCliFailure(r, { exitCode: 1, message: 'token already inactive' });
|
|
expect(stateDigest(ws.dbPath)).toBe(before);
|
|
const probes = rng.shuffle([...victims.slice(0, 100), ...rng.shuffle(model.active()).slice(0, 100)]);
|
|
await mapLimit(probes, 16, async (t) => {
|
|
const r = await http.get(server.url(), { token: t });
|
|
if (model.lookup(t) === 'active') {
|
|
expectContainersResponse(r);
|
|
} else expectErrorResponse(r, 401);
|
|
});
|
|
for (const t of probes) if (model.lookup(t) === 'active') model.audit(t);
|
|
model.verifyList(await cli.tokenList(), 'after bulk deletion: 500 active');
|
|
model.verifyList(await cli.tokenListAll(), 'after bulk deletion: all 1000', 'all');
|
|
expect((await cli.run(['token', 'list'])).stdout, 'token list equals the stored active rows').toBe(expectedTokenListStdout(ws.dbPath, 'active'));
|
|
});
|
|
expect((await server.stop()).exitCode).toBe(0);
|
|
expect(server.logs().records, 'nothing at WARN or above during the whole mission').toEqual([]);
|
|
const report = model.verifyDb(ws.dbPath, 'final', { auditBeforeDeletion: true });
|
|
expect(report.tokens).toBe(1000);
|
|
expect(report.inactiveTokens).toBe(500);
|
|
});
|