bryanpedini
a139f6793d
Power, careless, malicious, fuzzing, cross-abuse, operator-session and concurrency personas exercising the whole application with model-checked database state.
224 lines
12 KiB
TypeScript
224 lines
12 KiB
TypeScript
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
// Copyright (C) 2026 Bryan Joshua Pedini
|
|
|
|
/**
|
|
* PLAN §12 cross-persona abuse sequence: a power-user state (tokens, revocations, audit history
|
|
* across restarts, a large container set) is attacked by malicious and fuzzer traffic — HTTP
|
|
* enumeration, near-matches, oracle-checked fuzz requests, malformed requests, failing CLI
|
|
* garbage and a foreign write lock, all at the same time — then the server is killed (SIGKILL)
|
|
* in the middle of a burst, restarted on the same database, and finally administered normally.
|
|
* Recovery and integrity are verified against the model after every phase.
|
|
*/
|
|
import {
|
|
test,
|
|
expect,
|
|
expectCliFailure,
|
|
errorMessage,
|
|
expectContainersResponse,
|
|
expectErrorResponse,
|
|
mapLimit,
|
|
pollUntil,
|
|
formatAuditOutput,
|
|
readOperations,
|
|
type CliResult,
|
|
type Rng,
|
|
} from '../../harness';
|
|
import { DatabaseSync } from 'node:sqlite';
|
|
import {
|
|
StateModel,
|
|
TOKEN_CHARS,
|
|
authOf,
|
|
checkResponse,
|
|
cliOutcome,
|
|
configOutcome,
|
|
expectHttp,
|
|
expectNoTokenFragments,
|
|
fuzzAddr,
|
|
fuzzLogLevel,
|
|
fuzzPort,
|
|
fuzzRequest,
|
|
fuzzText,
|
|
malformedRequest,
|
|
matchesConfigError,
|
|
nearMatches,
|
|
onlyResponse,
|
|
rawBytes,
|
|
stateDigest,
|
|
AUDIT_ALL_ARGV_NEAR_MISSES,
|
|
HELP_NEAR_MISSES,
|
|
LIST_ALL_NEAR_MISSES,
|
|
expectedAuditStdout,
|
|
type AppEnv,
|
|
} from '../../harness/personas';
|
|
|
|
/** Failing CLI invocations only (the attacker cannot change state through the CLI without a valid token argument). */
|
|
function garbageCli(r: Rng, model: StateModel): { args: string[]; env: AppEnv; addrValid?: boolean } {
|
|
for (;;) {
|
|
const kind = r.pick(['usage', 'usage', 'delete', 'delete', 'config']);
|
|
if (kind === 'config') {
|
|
const a = fuzzAddr(r);
|
|
const env: AppEnv = r.bool() ? { HTTP_ADDR: a.value } : r.bool() ? { HTTP_PORT: fuzzPort(r) } : { LOG_LEVEL: fuzzLogLevel(r) };
|
|
const cfg = configOutcome(env, env.HTTP_ADDR === undefined ? undefined : a.valid);
|
|
if (!cfg.ok) return { args: ['token', 'list'], env, addrValid: env.HTTP_ADDR === undefined ? undefined : a.valid };
|
|
continue;
|
|
}
|
|
const args =
|
|
kind === 'usage'
|
|
? r.pick([['token'], ['tokens'], ['token', 'nwe'], ['token', 'list', fuzzText(r, { maxLen: 64 }).value], ['token', 'new', 'a', fuzzText(r, { maxLen: 64 }).value], ['token', 'delete'], [fuzzText(r, { maxLen: 64 }).value], [...r.pick(HELP_NEAR_MISSES)], [...r.pick(LIST_ALL_NEAR_MISSES)], [...r.pick(AUDIT_ALL_ARGV_NEAR_MISSES)]])
|
|
: ['token', 'delete', r.pick([...nearMatches(r.pick(model.all()), r, 2).map((m) => m.value), r.pick(model.inactive().length ? model.inactive() : ['x']), fuzzText(r, { maxLen: 128 }).value, r.string(43, TOKEN_CHARS)])];
|
|
const o = cliOutcome(args, model);
|
|
if (o.kind === 'error') return { args, env: {} };
|
|
}
|
|
}
|
|
|
|
test('power-user state attacked by malicious + fuzzer traffic, a write lock and a SIGKILL mid-burst, then normal administration: full recovery and integrity', async ({ cli, ws, startServer, http, fuzz, secrets, e2e }) => {
|
|
test.setTimeout(180_000);
|
|
const r = fuzz.rng.fork('cross');
|
|
const model = new StateModel();
|
|
e2e.trackedDbs.add(ws.dbPath);
|
|
|
|
// ---- 1. power-user state
|
|
const descs = Array.from({ length: 40 }, (_, i) => (i % 5 === 0 ? fuzzText(r, { maxLen: 512 }).value : `team-${i} ${r.pick(['ci', 'ops', 'dash'])}`));
|
|
(await mapLimit(descs, 8, (d) => cli.tokenNew(d))).forEach((t, i) => model.add(t, descs[i]));
|
|
for (const t of r.shuffle(model.all()).slice(0, 10)) {
|
|
await cli.tokenDelete(t);
|
|
model.deactivate(t);
|
|
}
|
|
const server = await startServer({ logLevel: 'DEBUG' });
|
|
const fake = server.fakeDocker!.many(300, r.fork('containers'));
|
|
for (let round = 0; round < 3; round++) {
|
|
const users = r.shuffle(model.active()).slice(0, 15);
|
|
const plan = Array.from({ length: 100 }, (_, i) => users[i % users.length]);
|
|
await mapLimit(plan, 16, async (t) => {
|
|
expect(expectContainersResponse(await http.get(server.url(), { token: t }))).toHaveLength(300);
|
|
});
|
|
for (const t of plan) model.audit(t);
|
|
if (round < 2) await server.restart();
|
|
}
|
|
model.verifyDb(ws.dbPath, 'power-user state');
|
|
const lookup = (t: string) => model.lookup(t);
|
|
|
|
// ---- 2. attack: everything at once
|
|
await test.step('concurrent attack: enumeration, near-matches, fuzz, malformed requests, CLI garbage and a 2 s foreign write lock', async () => {
|
|
const audits: string[] = [];
|
|
const host = { host: server.connectHost, port: server.port };
|
|
const enumeration = mapLimit(Array.from({ length: 400 }, () => r.string(43, TOKEN_CHARS)), 8, async (g) => {
|
|
expectErrorResponse(await http.get(server.url(), { token: g }), 401);
|
|
});
|
|
const nearCases = model.all().flatMap((t) => nearMatches(t, r, 6)).filter((m) => !/[\r\n]/.test(m.value));
|
|
const near = mapLimit(nearCases, 6, async (m) => {
|
|
const header = `Bearer ${m.value}`;
|
|
const exp = authOf([header], lookup);
|
|
const res = onlyResponse(await http.raw({ ...host, data: Buffer.from(`GET /api/v1/containers HTTP/1.1\r\nHost: x\r\nAuthorization: ${header}\r\nConnection: close\r\n\r\n`, 'utf8'), timeoutMs: 20_000 }), m.kind);
|
|
if (exp.cls === 'ok') {
|
|
expectContainersResponse(res);
|
|
audits.push(exp.token!);
|
|
} else expectErrorResponse(res, 401);
|
|
});
|
|
const fuzzCases = Array.from({ length: 300 }, (_, i) => (i % 8 === 0 ? malformedRequest(r, model) : fuzzRequest(r, model)));
|
|
const fuzzing = mapLimit(fuzzCases, 6, async (c, i) => {
|
|
const exp = expectHttp(c, lookup, true);
|
|
fuzz.record({ phase: 'attack', i, method: c.method, target: c.target.slice(0, 200), headers: c.headers.map(([k, v]) => [k, v.slice(0, 200)]), expected: exp });
|
|
const res = onlyResponse(await http.raw({ ...host, data: rawBytes(c, { keepAlive: c.body !== undefined }), head: c.method === 'HEAD', timeoutMs: 20_000 }), `fuzz ${i}`);
|
|
checkResponse(res, exp, { head: c.method === 'HEAD', containers: 300, secrets, label: `fuzz ${i}` });
|
|
if (exp.audited && res.status === 200) audits.push(exp.audited);
|
|
});
|
|
// every 10th garbage invocation is a near miss of `audit all` (`audit ALL`, `audit all x`, ...)
|
|
const garbageCases = Array.from({ length: 60 }, (_, i) => (i % 10 === 3 ? { args: [...r.pick(AUDIT_ALL_ARGV_NEAR_MISSES.filter((a) => /^audit$/i.test(a[0])))], env: {} as AppEnv, addrValid: undefined } : garbageCli(r, model)));
|
|
const auditAllMisses = garbageCases.filter((g) => !Object.keys(g.env).length && g.args.some((a) => /[a\u0430]ll/i.test(a)) && /^audit$/i.test(g.args[0])).length;
|
|
expect(auditAllMisses, 'the garbage includes near misses of audit all').toBeGreaterThanOrEqual(6);
|
|
const garbage = mapLimit(garbageCases, 4, async (g, i) => {
|
|
fuzz.record({ phase: 'cli-garbage', i, args: g.args.map((a) => a.slice(0, 120)), env: g.env });
|
|
const res: CliResult = await cli.run(g.args, { env: g.env });
|
|
if (Object.keys(g.env).length) {
|
|
const cfg = configOutcome(g.env, g.addrValid);
|
|
expect(cfg.ok).toBe(false);
|
|
expectCliFailure(res, { exitCode: 2, usage: false });
|
|
expect(matchesConfigError(errorMessage(res), cfg as Exclude<typeof cfg, { ok: true }>), `cli garbage ${i}: ${errorMessage(res)}`).toBe(true);
|
|
} else {
|
|
const o = cliOutcome(g.args, model);
|
|
if (o.kind !== 'error') throw new Error('garbage generator produced a successful command');
|
|
expectCliFailure(res, { exitCode: o.exitCode, message: o.message, usage: o.usage });
|
|
}
|
|
});
|
|
const lock = (async () => {
|
|
await pollUntil('attack traffic under way', () => (fuzz.cases.length > 100 ? true : undefined), { timeoutMs: 30_000, intervalMs: 5 });
|
|
const db = new DatabaseSync(ws.dbPath, { timeout: 3000 });
|
|
db.exec('BEGIN IMMEDIATE');
|
|
await new Promise((res) => setTimeout(res, 2000));
|
|
db.exec('ROLLBACK');
|
|
db.close();
|
|
})();
|
|
await Promise.all([enumeration, near, fuzzing, garbage, lock]);
|
|
for (const t of audits) model.audit(t);
|
|
model.verifyDb(ws.dbPath, 'after the attack: only legitimately authenticated requests left a trace');
|
|
});
|
|
|
|
// ---- 3. crash in the middle of a burst, restart on the same database
|
|
await test.step('SIGKILL during a burst of 80 authenticated requests, then restart: committed rows survive, nothing is half-written', async () => {
|
|
const victim = model.active()[0];
|
|
const before = readOperations(ws.dbPath).filter((o) => o.token === victim).length;
|
|
fake.delay(5); // keep requests in flight
|
|
let done = 0;
|
|
const burst = mapLimit(Array.from({ length: 80 }, () => 0), 10, async () => {
|
|
const res = await http.raw({ host: server.connectHost, port: server.port, data: rawBytes({ cls: 'burst', method: 'GET', target: '/api/v1/containers', headers: [['Authorization', `Bearer ${victim}`]] }), timeoutMs: 20_000 });
|
|
done++;
|
|
return res;
|
|
});
|
|
await pollUntil('a third of the burst answered', () => (done >= 25 ? true : undefined), { timeoutMs: 30_000, intervalMs: 2 });
|
|
const killed = await server.kill();
|
|
expect(killed.signal).toBe('SIGKILL');
|
|
const results = await burst;
|
|
const ok = results.filter((x) => x.outcome === 'response' && x.response!.status === 200).length;
|
|
const lost = results.filter((x) => x.outcome !== 'response').length;
|
|
expect(results.every((x) => x.outcome !== 'response' || x.response!.status === 200), 'answered requests all succeeded').toBe(true);
|
|
expect(ok + lost).toBe(80);
|
|
expect(lost, 'the kill really interrupted the burst').toBeGreaterThan(0);
|
|
fake.delay(0);
|
|
await server.restart(); // new process, same DB/port/backend: WAL recovery
|
|
const after = readOperations(ws.dbPath).filter((o) => o.token === victim).length - before;
|
|
// every answered request is durably audited; interrupted ones may or may not have committed their row
|
|
expect(after).toBeGreaterThanOrEqual(ok);
|
|
expect(after).toBeLessThanOrEqual(ok + lost);
|
|
model.audit(victim, after);
|
|
model.verifyDb(ws.dbPath, 'after crash recovery');
|
|
// `audit all` on the recovered database: every committed row, oldest first, byte-exact
|
|
const all = await cli.audit('all');
|
|
expect(all, 'audit all after recovery: every row').toHaveLength(model.auditTotal);
|
|
expect(formatAuditOutput(all)).toBe(expectedAuditStdout(readOperations(ws.dbPath), { all: true }));
|
|
test.info().annotations.push({ type: 'crash', description: `${ok} answered, ${lost} interrupted, ${after - ok} interrupted request(s) had committed their audit row` });
|
|
});
|
|
|
|
// ---- 4. normal administration on the recovered instance
|
|
await test.step('normal administration: list, create, rotate, revoke, use with every client, graceful restart', async () => {
|
|
model.verifyList(await cli.tokenList());
|
|
model.verifyList(await cli.tokenListAll(), 'token list all after the incident', 'all');
|
|
for (let i = 0; i < 5; i++) model.add(await cli.tokenNew(`after incident ${i}`), `after incident ${i}`);
|
|
for (let i = 0; i < 3; i++) {
|
|
const old = r.pick(model.active());
|
|
const next = model.add(await cli.tokenNew(`rotated ${i}`), `rotated ${i}`).token;
|
|
await cli.tokenDelete(old);
|
|
model.deactivate(old);
|
|
expectErrorResponse(await http.get(server.url(), { token: old }), 401);
|
|
expectContainersResponse(await http.curl({ url: server.url(), token: next }));
|
|
model.audit(next);
|
|
}
|
|
for (const t of model.active()) {
|
|
expectContainersResponse(await http.get(server.url(), { token: t }));
|
|
model.audit(t);
|
|
}
|
|
const listing = (await cli.run(['token', 'list', 'all'])).stdout;
|
|
const history = (await cli.run(['audit', 'all'])).stdout;
|
|
expect(history, 'audit all: the whole history').toBe(expectedAuditStdout(readOperations(ws.dbPath), { all: true }));
|
|
await server.restart();
|
|
expect((await cli.run(['token', 'list', 'all'])).stdout).toBe(listing);
|
|
expect((await cli.run(['audit', 'all'])).stdout, 'audit all survives a graceful restart').toBe(history);
|
|
for (const t of model.inactive()) expectErrorResponse(await http.get(server.url(), { token: t }), 401);
|
|
});
|
|
expect((await server.stop()).exitCode).toBe(0);
|
|
for (const g of server.allOutput()) expectNoTokenFragments(g.stderr, secrets, `server generation ${g.generation}`);
|
|
expect(server.logs().count({ level: ['WARN', 'ERROR'] }), 'the final generation saw no errors').toBe(0);
|
|
model.verifyDb(ws.dbPath, 'final');
|
|
expect(stateDigest(ws.dbPath)).not.toBe('absent');
|
|
});
|