feat(workflows): crash recovery — resume runs orphaned mid-flow

Closes the crash-safety gap: a run left in running/pending by a crash had
nothing to resume it (the scheduler only wakes 'waiting'). Adds a heartbeat
(workflow_runs.updated_at, stamped on every node advance + start/resume) and a
recoverStaleRuns() sweep that re-enters runs whose heartbeat has gone stale
(>10 min) from their persisted node. Runs on the scheduler tick AND the boot
tick, so a restart catches anything stranded during downtime.

Re-entry is at-least-once (the current node may re-execute) — loop counters +
the late-fee math are idempotent, so the only residual risk is a duplicate
reminder email. An attempts counter (migration 145, cap 5) marks a run failed
instead of recovering a node that reliably crashes the process (crash-loop
backstop). Flag-gated. Tests: orphan-resume + crash-loop cap.
This commit is contained in:
Luca
2026-06-23 13:39:17 +02:00
parent 83dc95a62b
commit 192d2cbc06
4 changed files with 133 additions and 4 deletions
@@ -279,4 +279,40 @@ describe('workflow engine', () => {
const after = await db('workflows').where({ id: wf.id }).first();
expect(after.version).toBe(before.version); // unchanged
});
test('recoverStaleRuns resumes a run orphaned mid-flow (crash recovery)', async () => {
const wfId = await makeWorkflow({
trigger: 'recover.event',
nodes: [{ key: 'r1', type: 'trigger' }, { key: 'r2', type: 'action', config: { action: 'noop' } }],
edges: [{ from: 'r1', to: 'r2' }],
});
// Simulate a run left 'running' at r2 with a stale heartbeat (crash mid-flow).
await db('workflow_runs').insert({
workflow_id: wfId, version: 1, trigger_event: 'recover.event', status: 'running', current_node: 'r2',
context: JSON.stringify({ vars: {} }), dedup_key: 'recover-1',
updated_at: new Date(Date.now() - 3600000).toISOString(),
});
const run0 = await db('workflow_runs').where({ dedup_key: 'recover-1' }).first();
const n = await engine.recoverStaleRuns({ staleMs: 1000 });
expect(n).toBeGreaterThanOrEqual(1);
const run = await db('workflow_runs').where({ id: run0.id }).first();
expect(run.status).toBe('done');
});
test('recoverStaleRuns abandons a crash-looping run after the attempts cap', async () => {
const wfId = await makeWorkflow({
trigger: 'crashloop.event',
nodes: [{ key: 'c1', type: 'trigger' }, { key: 'c2', type: 'action', config: { action: 'noop' } }],
edges: [{ from: 'c1', to: 'c2' }],
});
await db('workflow_runs').insert({
workflow_id: wfId, version: 1, trigger_event: 'crashloop.event', status: 'running', current_node: 'c2',
context: JSON.stringify({ vars: {} }), dedup_key: 'crash-1', attempts: 5,
updated_at: new Date(Date.now() - 3600000).toISOString(),
});
const run0 = await db('workflow_runs').where({ dedup_key: 'crash-1' }).first();
await engine.recoverStaleRuns({ staleMs: 1000 });
const run = await db('workflow_runs').where({ id: run0.id }).first();
expect(run.status).toBe('failed');
});
});