// SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2026 Bryan Joshua Pedini /** * PLAN §12 "at least one Playwright test must deliberately mix concurrent authenticated requests, * invalid authentication attempts, token creation/deletion, database reads, and Docker container * lifecycle operations, then verify the final database and API state against the operations * actually performed". * * Everything below runs at the same time against one server on the real Docker Engine: * - steady request loops (Playwright + curl) for long-lived tokens; * - request loops for "doomed" tokens that get deleted mid-flight; * - an invalid-authentication loop (random, near-match, revoked, malformed); * - a CLI creator (new tokens are used immediately) and a CLI deleter; * - a DB reader taking consistent snapshots (and `token list`, `audit`) while everyone writes; * - real container lifecycle operations (run / stop / remove / start) on test-owned containers. * * Exact invariant for requests racing a deletion (justified by the code: authenticate() reads the * token row, then RecordOperation inserts the audit row, with no common transaction): a request * may be answered 200 only if it started before `token delete` returned; it is audited iff it was * answered 200; an audit row may carry a time after deleted_at only for such a racing request, and * never after the end of the token's last 200 response. */ import { test, expect, assertDbInvariants, expectContainersResponse, expectErrorResponse, msToNanos, parseAuditOutput, parseTokenList, pollUntil, readOperations, readToken, rfc3339ToNanos, tokenId, type TestContainer, } from '../../harness'; import { StateModel, TOKEN_CHARS, authOf, consistentSnapshot, nearMatches, snapshotProblems } from '../../harness/personas'; interface ApiContainer { Id: string; State: string; Labels: Record; } interface Sample { token: string; start: number; end: number; endWall: number; status: number; own?: string[]; } test('concurrency mix: authenticated + invalid requests, token creation/deletion, DB reads and real container lifecycle at once — final DB and API state equal the operations performed', async ({ cli, ws, startServer, http, docker, rng }) => { test.setTimeout(180_000); const model = new StateModel(); const steady = await Promise.all(Array.from({ length: 4 }, (_, i) => cli.tokenNew(`steady ${i}`))); steady.forEach((t, i) => model.add(t, `steady ${i}`)); const doomed = await Promise.all(Array.from({ length: 4 }, (_, i) => cli.tokenNew(`doomed ${i}`))); doomed.forEach((t, i) => model.add(t, `doomed ${i}`)); const revokedEarly = model.add(await cli.tokenNew('revoked early'), 'revoked early').token; await cli.tokenDelete(revokedEarly); model.deactivate(revokedEarly); const everCreated = new Map(); for (const c of await docker.runMany(4, (i) => ({ suffix: `init${i}`, labels: { 'mix.phase': 'initial' } }), 4)) everCreated.set(c.id, c); const server = await startServer({ docker: 'real', logLevel: 'INFO' }); const samples: Sample[] = []; const invalid: number[] = []; const deletedAt = new Map(); const createdTokens: string[] = []; let snapshots = 0; let auditRows = 0; let stopping = false; const request = async (token: string, viaCurl: boolean): Promise => { const start = performance.now(); const r = viaCurl ? await http.curl({ url: server.url(), token }) : await http.get(server.url(), { token }); const end = performance.now(); const s: Sample = { token, start, end, endWall: Date.now(), status: r.status }; if (r.status === 200) { const list = expectContainersResponse(r); expect(list.every((c) => c.State === 'running'), 'only running containers, ever').toBe(true); s.own = list.filter((c) => c.Labels?.['dpsapi.e2e.test'] === docker.testKey).map((c) => c.Id); } else expectErrorResponse(r, 401); samples.push(s); }; const steadyLoop = async (t: string, k: number) => { for (let n = 0; !stopping; n++) await request(t, (n + k) % 5 === 0); }; const doomedLoop = async (t: string) => { let after = 0; for (let n = 0; !stopping; n++) { const start = performance.now(); await request(t, n % 4 === 0); if (deletedAt.has(t) && start > deletedAt.get(t)! && ++after >= 4) return; } }; const invalidLoop = async () => { while (!stopping) { const base = rng.pick(model.all()); const near = nearMatches(base, rng, 1)[0]?.value ?? 'x'; const header = rng.pick([`Bearer ${rng.string(43, TOKEN_CHARS)}`, `Bearer ${near}`, `Bearer ${revokedEarly}`, `Basic ${base}`, 'Bearer ']); // Playwright carries printable ASCII only; skip values the oracle says would authenticate (OWS trimming) if (!/^[\x20-\x7e]*$/.test(header) || authOf([header], (t) => model.lookup(t)).cls === 'ok') continue; const r = await http.get(server.url(), { headers: { Authorization: header } }); expectErrorResponse(r, 401); invalid.push(r.status); } }; const creator = async () => { for (let i = 0; i < 6 && !stopping; i++) { const t = await cli.tokenNew(`created mid-flight ${i}`); model.add(t, `created mid-flight ${i}`); createdTokens.push(t); for (let n = 0; n < 5; n++) await request(t, n === 4); } }; const deleter = async () => { for (const t of rng.shuffle(doomed)) { const warm = rng.int(5, 25); await pollUntil('traffic for the doomed token', () => (stopping || samples.filter((s) => s.token === t).length >= warm ? true : undefined), { timeoutMs: 30_000 }); if (stopping) return; await cli.tokenDelete(t); deletedAt.set(t, performance.now()); model.deactivate(t); } }; const reader = async () => { while (!stopping) { const snap = consistentSnapshot(ws.dbPath); expect(snapshotProblems(snap), 'consistent snapshot taken mid-flight is structurally valid').toEqual([]); // alternately `token list` (active only: no inactive row may slip through) and `token list all` const all = snapshots % 2 === 1; const list = await cli.run(all ? ['token', 'list', 'all'] : ['token', 'list']); expect(list.exitCode).toBe(0); const rows = parseTokenList(list.stdout); if (!all) expect(rows.filter((r) => !r.active || r.deletedAt !== null).map((r) => tokenId(r.token)), 'token list prints active tokens only').toEqual([]); const known = new Set(model.all()); // a token being created right now is in the database before `token new` has returned expect(rows.every((r) => known.has(r.token) || /^created mid-flight \d$/.test(r.description)), 'no token appears that was not created by the test').toBe(true); // `audit` while everyone writes: a complete chronological table, append-only, only known tokens const audit = await cli.run(['audit', '1000000']); expect(audit.exitCode).toBe(0); const arows = parseAuditOutput(audit.stdout); const times = arows.map((r) => rfc3339ToNanos(r.time)); expect(times.every((t, k) => k === 0 || times[k - 1] <= t), 'audit output is chronological').toBe(true); expect(arows.length, 'audit history only grows').toBeGreaterThanOrEqual(auditRows); auditRows = arows.length; const users = new Set(model.all()); expect(arows.every((r) => users.has(r.token) && r.operation === 'GET containers'), 'audit rows belong to tokens the test created').toBe(true); snapshots++; } }; const lifecycle = async () => { const extra = await docker.runMany(4, (i) => ({ suffix: `mid${i}`, command: i % 2 ? ['tail', '-f', '/dev/null'] : ['sleep', '3600'], labels: { 'mix.phase': 'mid', 'mix.i': String(i) } }), 2); for (const c of extra) everCreated.set(c.id, c); const all = [...everCreated.values()]; await Promise.all(all.slice(0, 2).map((c) => docker.stop(c, 1))); await Promise.all(all.slice(2, 4).map((c) => docker.remove(c))); await docker.start(all[0]); for (const c of await docker.runMany(2, (i) => ({ suffix: `late${i}`, labels: { 'mix.phase': 'late' } }), 2)) everCreated.set(c.id, c); }; // any failure stops every loop (a failed worker must not turn into a hang) const guard = (p: Promise) => p.catch((e) => { stopping = true; throw e; }); const orchestration = (async () => { const work = await Promise.allSettled([creator(), deleter(), lifecycle()].map(guard)); const bad = work.find((x) => x.status === 'rejected'); if (bad) throw (bad as PromiseRejectedResult).reason; await pollUntil('doomed loops finished', () => (doomed.every((t) => samples.filter((s) => s.token === t && s.start > deletedAt.get(t)!).length >= 8) ? true : undefined), { timeoutMs: 30_000 }); stopping = true; })(); // wait for every worker to wind down (no stray CLI/docker processes), then surface the first failure const settled = await Promise.allSettled([guard(orchestration), ...[...steady.map((t, k) => steadyLoop(t, k)), ...doomed.flatMap((t) => [doomedLoop(t), doomedLoop(t)]), invalidLoop(), reader()].map(guard)]); const failed = settled.find((x) => x.status === 'rejected'); if (failed) throw (failed as PromiseRejectedResult).reason; // ---- final API state equals docker ps for our containers const running = (await docker.runningIds()).sort(); const finalList = expectContainersResponse(await http.get(server.url(), { token: steady[0] })); expect(finalList.filter((c) => c.Labels?.['dpsapi.e2e.test'] === docker.testKey).map((c) => c.Id).sort(), 'final API state equals docker ps').toEqual(running); expect(running.length).toBe(7); // 4 initial + 4 mid + 2 late \u2212 2 removed \u2212 1 still stopped samples.push({ token: steady[0], start: performance.now(), end: performance.now(), endWall: Date.now(), status: 200 }); expect((await server.stop()).exitCode).toBe(0); // every container ever seen in a 200 belongs to the set this test created for (const s of samples) for (const id of s.own ?? []) expect(everCreated.has(id), 'only containers created by this test carry its label').toBe(true); // ---- database equals the operations actually performed const ops = readOperations(ws.dbPath); for (const t of model.all()) { const mine = samples.filter((s) => s.token === t); const oks = mine.filter((s) => s.status === 200); model.audit(t, oks.length); expect(ops.filter((o) => o.token === t).length, `${tokenId(t)}: audit rows equal the 200 responses`).toBe(oks.length); } for (const t of [...steady, ...createdTokens]) expect(samples.filter((s) => s.token === t).every((s) => s.status === 200), `${tokenId(t)}: never refused`).toBe(true); let racing = 0; for (const t of doomed) { const del = deletedAt.get(t)!; const mine = samples.filter((s) => s.token === t); expect(mine.filter((s) => s.start > del && s.status !== 401), `${tokenId(t)}: everything started after the deletion is 401`).toEqual([]); expect(mine.filter((s) => s.status === 200).every((s) => s.start < del), `${tokenId(t)}: every 200 started before the deletion returned`).toBe(true); const row = readToken(ws.dbPath, t)!; const lastOk = Math.max(0, ...mine.filter((s) => s.status === 200).map((s) => s.endWall)); for (const o of ops.filter((x) => x.token === t)) { expect(rfc3339ToNanos(o.time) <= msToNanos(lastOk + 1), 'audit row precedes the end of the last successful response').toBe(true); if (rfc3339ToNanos(o.time) > rfc3339ToNanos(row.deleted_at!)) racing++; } } expect(ops.filter((o) => o.token === revokedEarly)).toEqual([]); expect(invalid.length).toBeGreaterThan(20); expect(snapshots).toBeGreaterThan(3); model.verifyDb(ws.dbPath, 'final database equals the model'); model.verifyList(await cli.tokenList(), 'final token list'); model.verifyList(await cli.tokenListAll(), 'final token list all', 'all'); assertDbInvariants(ws.dbPath, { tokenCount: 4 + 4 + 1 + 6, operationCount: samples.filter((s) => s.status === 200).length }); test.info().annotations.push({ type: 'mix', description: JSON.stringify({ requests: samples.length, invalid: invalid.length, snapshots, racingAuditRows: racing, containers: everCreated.size }) }); });