bryanpedini
a139f6793d
Power, careless, malicious, fuzzing, cross-abuse, operator-session and concurrency personas exercising the whole application with model-checked database state.
230 lines
12 KiB
TypeScript
230 lines
12 KiB
TypeScript
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
// Copyright (C) 2026 Bryan Joshua Pedini
|
|
|
|
/**
|
|
* PLAN §12 "at least one Playwright test must deliberately mix concurrent authenticated requests,
|
|
* invalid authentication attempts, token creation/deletion, database reads, and Docker container
|
|
* lifecycle operations, then verify the final database and API state against the operations
|
|
* actually performed".
|
|
*
|
|
* Everything below runs at the same time against one server on the real Docker Engine:
|
|
* - steady request loops (Playwright + curl) for long-lived tokens;
|
|
* - request loops for "doomed" tokens that get deleted mid-flight;
|
|
* - an invalid-authentication loop (random, near-match, revoked, malformed);
|
|
* - a CLI creator (new tokens are used immediately) and a CLI deleter;
|
|
* - a DB reader taking consistent snapshots (and `token list`, `audit`) while everyone writes;
|
|
* - real container lifecycle operations (run / stop / remove / start) on test-owned containers.
|
|
*
|
|
* Exact invariant for requests racing a deletion (justified by the code: authenticate() reads the
|
|
* token row, then RecordOperation inserts the audit row, with no common transaction): a request
|
|
* may be answered 200 only if it started before `token delete` returned; it is audited iff it was
|
|
* answered 200; an audit row may carry a time after deleted_at only for such a racing request, and
|
|
* never after the end of the token's last 200 response.
|
|
*/
|
|
import {
|
|
test,
|
|
expect,
|
|
assertDbInvariants,
|
|
expectContainersResponse,
|
|
expectErrorResponse,
|
|
msToNanos,
|
|
parseAuditOutput,
|
|
parseTokenList,
|
|
pollUntil,
|
|
readOperations,
|
|
readToken,
|
|
rfc3339ToNanos,
|
|
tokenId,
|
|
type TestContainer,
|
|
} from '../../harness';
|
|
import { StateModel, TOKEN_CHARS, authOf, consistentSnapshot, nearMatches, snapshotProblems } from '../../harness/personas';
|
|
|
|
interface ApiContainer {
|
|
Id: string;
|
|
State: string;
|
|
Labels: Record<string, string>;
|
|
}
|
|
|
|
interface Sample {
|
|
token: string;
|
|
start: number;
|
|
end: number;
|
|
endWall: number;
|
|
status: number;
|
|
own?: string[];
|
|
}
|
|
|
|
test('concurrency mix: authenticated + invalid requests, token creation/deletion, DB reads and real container lifecycle at once — final DB and API state equal the operations performed', async ({ cli, ws, startServer, http, docker, rng }) => {
|
|
test.setTimeout(180_000);
|
|
const model = new StateModel();
|
|
const steady = await Promise.all(Array.from({ length: 4 }, (_, i) => cli.tokenNew(`steady ${i}`)));
|
|
steady.forEach((t, i) => model.add(t, `steady ${i}`));
|
|
const doomed = await Promise.all(Array.from({ length: 4 }, (_, i) => cli.tokenNew(`doomed ${i}`)));
|
|
doomed.forEach((t, i) => model.add(t, `doomed ${i}`));
|
|
const revokedEarly = model.add(await cli.tokenNew('revoked early'), 'revoked early').token;
|
|
await cli.tokenDelete(revokedEarly);
|
|
model.deactivate(revokedEarly);
|
|
|
|
const everCreated = new Map<string, TestContainer>();
|
|
for (const c of await docker.runMany(4, (i) => ({ suffix: `init${i}`, labels: { 'mix.phase': 'initial' } }), 4)) everCreated.set(c.id, c);
|
|
const server = await startServer({ docker: 'real', logLevel: 'INFO' });
|
|
|
|
const samples: Sample[] = [];
|
|
const invalid: number[] = [];
|
|
const deletedAt = new Map<string, number>();
|
|
const createdTokens: string[] = [];
|
|
let snapshots = 0;
|
|
let auditRows = 0;
|
|
let stopping = false;
|
|
|
|
const request = async (token: string, viaCurl: boolean): Promise<void> => {
|
|
const start = performance.now();
|
|
const r = viaCurl ? await http.curl({ url: server.url(), token }) : await http.get(server.url(), { token });
|
|
const end = performance.now();
|
|
const s: Sample = { token, start, end, endWall: Date.now(), status: r.status };
|
|
if (r.status === 200) {
|
|
const list = expectContainersResponse<ApiContainer>(r);
|
|
expect(list.every((c) => c.State === 'running'), 'only running containers, ever').toBe(true);
|
|
s.own = list.filter((c) => c.Labels?.['dpsapi.e2e.test'] === docker.testKey).map((c) => c.Id);
|
|
} else expectErrorResponse(r, 401);
|
|
samples.push(s);
|
|
};
|
|
|
|
const steadyLoop = async (t: string, k: number) => {
|
|
for (let n = 0; !stopping; n++) await request(t, (n + k) % 5 === 0);
|
|
};
|
|
const doomedLoop = async (t: string) => {
|
|
let after = 0;
|
|
for (let n = 0; !stopping; n++) {
|
|
const start = performance.now();
|
|
await request(t, n % 4 === 0);
|
|
if (deletedAt.has(t) && start > deletedAt.get(t)! && ++after >= 4) return;
|
|
}
|
|
};
|
|
const invalidLoop = async () => {
|
|
while (!stopping) {
|
|
const base = rng.pick(model.all());
|
|
const near = nearMatches(base, rng, 1)[0]?.value ?? 'x';
|
|
const header = rng.pick([`Bearer ${rng.string(43, TOKEN_CHARS)}`, `Bearer ${near}`, `Bearer ${revokedEarly}`, `Basic ${base}`, 'Bearer ']);
|
|
// Playwright carries printable ASCII only; skip values the oracle says would authenticate (OWS trimming)
|
|
if (!/^[\x20-\x7e]*$/.test(header) || authOf([header], (t) => model.lookup(t)).cls === 'ok') continue;
|
|
const r = await http.get(server.url(), { headers: { Authorization: header } });
|
|
expectErrorResponse(r, 401);
|
|
invalid.push(r.status);
|
|
}
|
|
};
|
|
|
|
const creator = async () => {
|
|
for (let i = 0; i < 6 && !stopping; i++) {
|
|
const t = await cli.tokenNew(`created mid-flight ${i}`);
|
|
model.add(t, `created mid-flight ${i}`);
|
|
createdTokens.push(t);
|
|
for (let n = 0; n < 5; n++) await request(t, n === 4);
|
|
}
|
|
};
|
|
const deleter = async () => {
|
|
for (const t of rng.shuffle(doomed)) {
|
|
const warm = rng.int(5, 25);
|
|
await pollUntil('traffic for the doomed token', () => (stopping || samples.filter((s) => s.token === t).length >= warm ? true : undefined), { timeoutMs: 30_000 });
|
|
if (stopping) return;
|
|
await cli.tokenDelete(t);
|
|
deletedAt.set(t, performance.now());
|
|
model.deactivate(t);
|
|
}
|
|
};
|
|
const reader = async () => {
|
|
while (!stopping) {
|
|
const snap = consistentSnapshot(ws.dbPath);
|
|
expect(snapshotProblems(snap), 'consistent snapshot taken mid-flight is structurally valid').toEqual([]);
|
|
// alternately `token list` (active only: no inactive row may slip through) and `token list all`
|
|
const all = snapshots % 2 === 1;
|
|
const list = await cli.run(all ? ['token', 'list', 'all'] : ['token', 'list']);
|
|
expect(list.exitCode).toBe(0);
|
|
const rows = parseTokenList(list.stdout);
|
|
if (!all) expect(rows.filter((r) => !r.active || r.deletedAt !== null).map((r) => tokenId(r.token)), 'token list prints active tokens only').toEqual([]);
|
|
const known = new Set(model.all());
|
|
// a token being created right now is in the database before `token new` has returned
|
|
expect(rows.every((r) => known.has(r.token) || /^created mid-flight \d$/.test(r.description)), 'no token appears that was not created by the test').toBe(true);
|
|
// `audit` while everyone writes: a complete chronological table, append-only, only known tokens
|
|
const audit = await cli.run(['audit', '1000000']);
|
|
expect(audit.exitCode).toBe(0);
|
|
const arows = parseAuditOutput(audit.stdout);
|
|
const times = arows.map((r) => rfc3339ToNanos(r.time));
|
|
expect(times.every((t, k) => k === 0 || times[k - 1] <= t), 'audit output is chronological').toBe(true);
|
|
expect(arows.length, 'audit history only grows').toBeGreaterThanOrEqual(auditRows);
|
|
auditRows = arows.length;
|
|
const users = new Set(model.all());
|
|
expect(arows.every((r) => users.has(r.token) && r.operation === 'GET containers'), 'audit rows belong to tokens the test created').toBe(true);
|
|
snapshots++;
|
|
}
|
|
};
|
|
const lifecycle = async () => {
|
|
const extra = await docker.runMany(4, (i) => ({ suffix: `mid${i}`, command: i % 2 ? ['tail', '-f', '/dev/null'] : ['sleep', '3600'], labels: { 'mix.phase': 'mid', 'mix.i': String(i) } }), 2);
|
|
for (const c of extra) everCreated.set(c.id, c);
|
|
const all = [...everCreated.values()];
|
|
await Promise.all(all.slice(0, 2).map((c) => docker.stop(c, 1)));
|
|
await Promise.all(all.slice(2, 4).map((c) => docker.remove(c)));
|
|
await docker.start(all[0]);
|
|
for (const c of await docker.runMany(2, (i) => ({ suffix: `late${i}`, labels: { 'mix.phase': 'late' } }), 2)) everCreated.set(c.id, c);
|
|
};
|
|
|
|
// any failure stops every loop (a failed worker must not turn into a hang)
|
|
const guard = <T,>(p: Promise<T>) => p.catch((e) => {
|
|
stopping = true;
|
|
throw e;
|
|
});
|
|
const orchestration = (async () => {
|
|
const work = await Promise.allSettled([creator(), deleter(), lifecycle()].map(guard));
|
|
const bad = work.find((x) => x.status === 'rejected');
|
|
if (bad) throw (bad as PromiseRejectedResult).reason;
|
|
await pollUntil('doomed loops finished', () => (doomed.every((t) => samples.filter((s) => s.token === t && s.start > deletedAt.get(t)!).length >= 8) ? true : undefined), { timeoutMs: 30_000 });
|
|
stopping = true;
|
|
})();
|
|
// wait for every worker to wind down (no stray CLI/docker processes), then surface the first failure
|
|
const settled = await Promise.allSettled([guard(orchestration), ...[...steady.map((t, k) => steadyLoop(t, k)), ...doomed.flatMap((t) => [doomedLoop(t), doomedLoop(t)]), invalidLoop(), reader()].map(guard)]);
|
|
const failed = settled.find((x) => x.status === 'rejected');
|
|
if (failed) throw (failed as PromiseRejectedResult).reason;
|
|
|
|
// ---- final API state equals docker ps for our containers
|
|
const running = (await docker.runningIds()).sort();
|
|
const finalList = expectContainersResponse<ApiContainer>(await http.get(server.url(), { token: steady[0] }));
|
|
expect(finalList.filter((c) => c.Labels?.['dpsapi.e2e.test'] === docker.testKey).map((c) => c.Id).sort(), 'final API state equals docker ps').toEqual(running);
|
|
expect(running.length).toBe(7); // 4 initial + 4 mid + 2 late \u2212 2 removed \u2212 1 still stopped
|
|
samples.push({ token: steady[0], start: performance.now(), end: performance.now(), endWall: Date.now(), status: 200 });
|
|
expect((await server.stop()).exitCode).toBe(0);
|
|
|
|
// every container ever seen in a 200 belongs to the set this test created
|
|
for (const s of samples) for (const id of s.own ?? []) expect(everCreated.has(id), 'only containers created by this test carry its label').toBe(true);
|
|
|
|
// ---- database equals the operations actually performed
|
|
const ops = readOperations(ws.dbPath);
|
|
for (const t of model.all()) {
|
|
const mine = samples.filter((s) => s.token === t);
|
|
const oks = mine.filter((s) => s.status === 200);
|
|
model.audit(t, oks.length);
|
|
expect(ops.filter((o) => o.token === t).length, `${tokenId(t)}: audit rows equal the 200 responses`).toBe(oks.length);
|
|
}
|
|
for (const t of [...steady, ...createdTokens]) expect(samples.filter((s) => s.token === t).every((s) => s.status === 200), `${tokenId(t)}: never refused`).toBe(true);
|
|
let racing = 0;
|
|
for (const t of doomed) {
|
|
const del = deletedAt.get(t)!;
|
|
const mine = samples.filter((s) => s.token === t);
|
|
expect(mine.filter((s) => s.start > del && s.status !== 401), `${tokenId(t)}: everything started after the deletion is 401`).toEqual([]);
|
|
expect(mine.filter((s) => s.status === 200).every((s) => s.start < del), `${tokenId(t)}: every 200 started before the deletion returned`).toBe(true);
|
|
const row = readToken(ws.dbPath, t)!;
|
|
const lastOk = Math.max(0, ...mine.filter((s) => s.status === 200).map((s) => s.endWall));
|
|
for (const o of ops.filter((x) => x.token === t)) {
|
|
expect(rfc3339ToNanos(o.time) <= msToNanos(lastOk + 1), 'audit row precedes the end of the last successful response').toBe(true);
|
|
if (rfc3339ToNanos(o.time) > rfc3339ToNanos(row.deleted_at!)) racing++;
|
|
}
|
|
}
|
|
expect(ops.filter((o) => o.token === revokedEarly)).toEqual([]);
|
|
expect(invalid.length).toBeGreaterThan(20);
|
|
expect(snapshots).toBeGreaterThan(3);
|
|
model.verifyDb(ws.dbPath, 'final database equals the model');
|
|
model.verifyList(await cli.tokenList(), 'final token list');
|
|
model.verifyList(await cli.tokenListAll(), 'final token list all', 'all');
|
|
assertDbInvariants(ws.dbPath, { tokenCount: 4 + 4 + 1 + 6, operationCount: samples.filter((s) => s.status === 200).length });
|
|
test.info().annotations.push({ type: 'mix', description: JSON.stringify({ requests: samples.length, invalid: invalid.length, snapshots, racingAuditRows: racing, containers: everCreated.size }) });
|
|
});
|