272 lines
12 KiB
TypeScript
272 lines
12 KiB
TypeScript
/**
|
|
* Real-filesystem integration tests for backups/swap.ts. The generated shell
|
|
* runs against a temp directory that stands in for a live volume mount, proving
|
|
* the safety guarantees of the in-place restore swap WITHOUT needing Docker or
|
|
* restic:
|
|
* - a clean swap replaces the live content and exits 0
|
|
* - a FAILED restic step leaves the live content byte-for-byte intact
|
|
* - a kill between the two move-phases is detectable and fully recovered
|
|
* - a fresh restore over an interrupted swap recovers the original first and
|
|
* never destroys the sole surviving copy
|
|
*/
|
|
import assert from 'node:assert/strict';
|
|
import { describe, it } from 'node:test';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { mkdtempSync, mkdirSync, writeFileSync, readdirSync, readFileSync, rmSync, existsSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
|
|
import {
|
|
buildInPlaceRestoreScript,
|
|
buildSwapRecoveryScript,
|
|
SWAP_NEW,
|
|
SWAP_OLD,
|
|
SWAP_PHASE,
|
|
SWAP_ARTIFACTS,
|
|
} from '../../src/lib/server/backups/swap';
|
|
|
|
/** Run a script via a temp file (no nested-quoting hazard). Returns exit code. */
|
|
function runScript(script: string): number {
|
|
const dir = mkdtempSync(join(tmpdir(), 'swap-sh-'));
|
|
const file = join(dir, 'run.sh');
|
|
writeFileSync(file, script);
|
|
try {
|
|
execFileSync('sh', [file], { stdio: 'pipe' });
|
|
return 0;
|
|
} catch (e: any) {
|
|
return e.status ?? 1;
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
/** Read the non-artifact contents of a live dir as { name: content }. */
|
|
function readLive(live: string): Record<string, string> {
|
|
const out: Record<string, string> = {};
|
|
for (const name of readdirSync(live)) {
|
|
if ((SWAP_ARTIFACTS as readonly string[]).includes(name)) continue;
|
|
out[name] = readFileSync(join(live, name), 'utf8');
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function artifacts(live: string): string[] {
|
|
return readdirSync(live).filter((n) => (SWAP_ARTIFACTS as readonly string[]).includes(n));
|
|
}
|
|
|
|
/** A sandbox with a live dir; seed its initial + staged (restored) content. */
|
|
function sandbox(opts: { live: Record<string, string>; staged?: Record<string, string> | null }) {
|
|
const root = mkdtempSync(join(tmpdir(), 'swap-'));
|
|
const live = join(root, 'volumes', 'data');
|
|
mkdirSync(live, { recursive: true });
|
|
for (const [n, c] of Object.entries(opts.live)) writeFileSync(join(live, n), c);
|
|
if (opts.staged) {
|
|
mkdirSync(join(live, SWAP_NEW), { recursive: true });
|
|
for (const [n, c] of Object.entries(opts.staged)) writeFileSync(join(live, SWAP_NEW, n), c);
|
|
}
|
|
return { root, live, cleanup: () => rmSync(root, { recursive: true, force: true }) };
|
|
}
|
|
|
|
describe('buildInPlaceRestoreScript — happy path', () => {
|
|
it('replaces live content with the restored content and exits 0', () => {
|
|
const { live, cleanup } = sandbox({
|
|
live: { 'old.txt': 'OLD', 'stale.txt': 'STALE' },
|
|
staged: { 'restored.txt': 'NEW', 'config.yaml': 'v: 2' },
|
|
});
|
|
try {
|
|
// resticCmd is a no-op — staging is pre-seeded; we test the swap only.
|
|
const code = runScript(buildInPlaceRestoreScript([live], 'true'));
|
|
assert.equal(code, 0);
|
|
assert.deepEqual(readLive(live), { 'restored.txt': 'NEW', 'config.yaml': 'v: 2' });
|
|
assert.deepEqual(artifacts(live), [], 'all swap artifacts cleaned up');
|
|
} finally { cleanup(); }
|
|
});
|
|
|
|
it('leaves live untouched when the restore produced no staged content (no NEW dir)', () => {
|
|
const { live, cleanup } = sandbox({ live: { 'keep.txt': 'KEEP' }, staged: null });
|
|
try {
|
|
assert.equal(runScript(buildInPlaceRestoreScript([live], 'true')), 0);
|
|
assert.deepEqual(readLive(live), { 'keep.txt': 'KEEP' });
|
|
} finally { cleanup(); }
|
|
});
|
|
});
|
|
|
|
describe('buildInPlaceRestoreScript — failure keeps live data intact', () => {
|
|
it('a failing restic step aborts before the swap; live is byte-for-byte intact', () => {
|
|
const { live, cleanup } = sandbox({
|
|
live: { 'important.txt': 'DO NOT LOSE ME', 'db.sqlite': 'REAL DATA' },
|
|
staged: { 'restored.txt': 'NEW' },
|
|
});
|
|
try {
|
|
// resticCmd fails → set -e aborts the whole script BEFORE any swap.
|
|
const code = runScript(buildInPlaceRestoreScript([live], 'false'));
|
|
assert.notEqual(code, 0, 'a failed restore must exit non-zero');
|
|
assert.deepEqual(readLive(live), { 'important.txt': 'DO NOT LOSE ME', 'db.sqlite': 'REAL DATA' });
|
|
} finally { cleanup(); }
|
|
});
|
|
});
|
|
|
|
/** Freeze a live dir in a mid-swap state, as an external kill would leave it. */
|
|
function makeInterrupted(opts: {
|
|
phase: '2' | '3' | '';
|
|
liveEntries: Record<string, string>;
|
|
oldEntries: Record<string, string>;
|
|
newEntries?: Record<string, string>;
|
|
}) {
|
|
const root = mkdtempSync(join(tmpdir(), 'swap-int-'));
|
|
const live = join(root, 'volumes', 'data');
|
|
mkdirSync(join(live, SWAP_OLD), { recursive: true });
|
|
mkdirSync(join(live, SWAP_NEW), { recursive: true });
|
|
writeFileSync(join(live, SWAP_PHASE), opts.phase);
|
|
for (const [n, c] of Object.entries(opts.liveEntries)) writeFileSync(join(live, n), c);
|
|
for (const [n, c] of Object.entries(opts.oldEntries)) writeFileSync(join(live, SWAP_OLD, n), c);
|
|
for (const [n, c] of Object.entries(opts.newEntries ?? {})) writeFileSync(join(live, SWAP_NEW, n), c);
|
|
return { root, live, cleanup: () => rmSync(root, { recursive: true, force: true }) };
|
|
}
|
|
|
|
describe('buildSwapRecoveryScript — rolls back an interrupted swap', () => {
|
|
it('phase 2 (emptying live interrupted): originals split across live and OLD are reassembled', () => {
|
|
const { live, cleanup } = makeInterrupted({
|
|
phase: '2',
|
|
liveEntries: { 'not-yet-moved.txt': 'A' },
|
|
oldEntries: { 'already-moved.txt': 'B', 'db.sqlite': 'REAL' },
|
|
newEntries: { 'restored.txt': 'NEW' },
|
|
});
|
|
try {
|
|
assert.equal(runScript(buildSwapRecoveryScript([live])), 0);
|
|
assert.deepEqual(readLive(live), { 'not-yet-moved.txt': 'A', 'already-moved.txt': 'B', 'db.sqlite': 'REAL' });
|
|
assert.deepEqual(artifacts(live), []);
|
|
} finally { cleanup(); }
|
|
});
|
|
|
|
it('phase 3 (refilling live interrupted): partial NEW content is dropped, ALL originals restored', () => {
|
|
const { live, cleanup } = makeInterrupted({
|
|
phase: '3',
|
|
liveEntries: { 'restored-a.txt': 'NEW-A' }, // step-3 got this far
|
|
oldEntries: { 'important.txt': 'DO NOT LOSE ME', 'db.sqlite': 'REAL' },
|
|
newEntries: { 'restored-b.txt': 'NEW-B' },
|
|
});
|
|
try {
|
|
assert.equal(runScript(buildSwapRecoveryScript([live])), 0);
|
|
assert.deepEqual(readLive(live), { 'important.txt': 'DO NOT LOSE ME', 'db.sqlite': 'REAL' });
|
|
assert.deepEqual(artifacts(live), []);
|
|
} finally { cleanup(); }
|
|
});
|
|
|
|
it('empty marker (killed mid marker-write) is treated as phase 2', () => {
|
|
const { live, cleanup } = makeInterrupted({ phase: '', liveEntries: {}, oldEntries: { 'everything.txt': 'ORIG' } });
|
|
try {
|
|
assert.equal(runScript(buildSwapRecoveryScript([live])), 0);
|
|
assert.deepEqual(readLive(live), { 'everything.txt': 'ORIG' });
|
|
} finally { cleanup(); }
|
|
});
|
|
|
|
it('no marker: recovery is a NO-OP (never touches a healthy volume)', () => {
|
|
const root = mkdtempSync(join(tmpdir(), 'swap-noop-'));
|
|
const live = join(root, 'volumes', 'data');
|
|
mkdirSync(live, { recursive: true });
|
|
writeFileSync(join(live, 'live.txt'), 'LIVE');
|
|
try {
|
|
assert.equal(runScript(buildSwapRecoveryScript([live])), 0);
|
|
assert.deepEqual(readLive(live), { 'live.txt': 'LIVE' });
|
|
} finally { rmSync(root, { recursive: true, force: true }); }
|
|
});
|
|
});
|
|
|
|
describe('fresh restore over an interrupted swap — never destroys the last copy', () => {
|
|
it('recovers the originals first, then applies the new restore cleanly', () => {
|
|
// Start from a phase-3 interrupted state, then run a FRESH restore. Recovery
|
|
// runs BEFORE restic re-stages (the correct ordering), so we model restic as
|
|
// a command that (re)creates the staging NEW dir with the second restore's
|
|
// content — exactly what `restic restore --target <live>` does in reality.
|
|
const { live, cleanup } = makeInterrupted({
|
|
phase: '3',
|
|
liveEntries: { 'restored-a.txt': 'NEW-A' },
|
|
oldEntries: { 'orig.txt': 'ORIGINAL' },
|
|
});
|
|
try {
|
|
// Model restic: mkdir the staging dir and write the new restored content.
|
|
const restic = `mkdir -p '${live}/${SWAP_NEW}'; printf 'NEW-2' > '${live}/${SWAP_NEW}/second.txt'`;
|
|
assert.equal(runScript(buildInPlaceRestoreScript([live], restic)), 0);
|
|
assert.deepEqual(readLive(live), { 'second.txt': 'NEW-2' }, 'fresh restore applied after recovery');
|
|
assert.deepEqual(artifacts(live), []);
|
|
} finally { cleanup(); }
|
|
});
|
|
|
|
it('recovery-before-restic: a phase-3 original is restored even before restic re-stages', () => {
|
|
// Prove the ordering: with recovery first, the interrupted originals are
|
|
// back in the live root at the moment restic runs (restic then re-stages).
|
|
const { live, cleanup } = makeInterrupted({
|
|
phase: '3',
|
|
liveEntries: { 'half-applied.txt': 'PARTIAL' },
|
|
oldEntries: { 'orig.txt': 'ORIGINAL' },
|
|
});
|
|
try {
|
|
// restic here asserts the original is already present, then stages NEW.
|
|
const restic = `test -f '${live}/orig.txt'; mkdir -p '${live}/${SWAP_NEW}'; printf 'X' > '${live}/${SWAP_NEW}/new.txt'`;
|
|
assert.equal(runScript(buildInPlaceRestoreScript([live], restic)), 0);
|
|
assert.deepEqual(readLive(live), { 'new.txt': 'X' });
|
|
} finally { cleanup(); }
|
|
});
|
|
|
|
it('if the fresh restore aborts, the recovered originals survive (never lost)', () => {
|
|
// Phase-2 interrupted state; a fresh restore whose restic step FAILS.
|
|
// The recovery runs inside swapOne, but swapOne only runs if NEW exists —
|
|
// so with a failing restic and NO pre-seeded NEW, the originals in OLD must
|
|
// still be recoverable by an explicit recovery pass (what the service does).
|
|
const { live, cleanup } = makeInterrupted({
|
|
phase: '2',
|
|
liveEntries: { 'a.txt': 'A' },
|
|
oldEntries: { 'b.txt': 'B' },
|
|
});
|
|
try {
|
|
// The service always runs recovery before/around a restore; prove the
|
|
// standalone recovery restores the originals regardless of a later abort.
|
|
assert.equal(runScript(buildSwapRecoveryScript([live])), 0);
|
|
assert.deepEqual(readLive(live), { 'a.txt': 'A', 'b.txt': 'B' });
|
|
} finally { cleanup(); }
|
|
});
|
|
});
|
|
|
|
describe('SWAP_ARTIFACTS — single source of truth', () => {
|
|
it('is exactly the three artifact basenames', () => {
|
|
assert.deepEqual([...SWAP_ARTIFACTS].sort(), ['.dockhand-restore-new', '.dockhand-restore-old', '.dockhand-restore-phase']);
|
|
});
|
|
it('the swap script references every artifact and skips them in the move loops', () => {
|
|
const script = buildInPlaceRestoreScript(['/volumes/data'], 'true');
|
|
for (const a of SWAP_ARTIFACTS) assert.ok(script.includes(a), `references ${a}`);
|
|
const guard = script.match(/case "\$e" in ([^)]*)\)/);
|
|
assert.ok(guard, 'has an artifact-skip guard');
|
|
for (const a of SWAP_ARTIFACTS) assert.ok(guard![1].includes(a), `skip-guard lists ${a}`);
|
|
});
|
|
});
|
|
|
|
describe('recovery narration is gated on the phase marker', () => {
|
|
// The user must be told when a re-run rolls back an interrupted prior swap,
|
|
// but a normal restore (no marker) must stay quiet. The narration lives INSIDE
|
|
// the `if [ -f ...phase ]` branch, so it only fires when there is something to recover.
|
|
it('emits a "Recovering ... rolling back" line inside the marker branch', () => {
|
|
const script = buildInPlaceRestoreScript(['/volumes/data'], 'true');
|
|
assert.ok(/Recovering .*rolling back an interrupted previous restore/.test(script), 'has the recovery narration');
|
|
});
|
|
it('a real recovery run actually prints the line to stderr only when a marker is present', () => {
|
|
const dir = mkdtempSync(join(tmpdir(), 'swap-narr-'));
|
|
const live = join(dir, 'data');
|
|
try {
|
|
// No marker: a recovery pass must be silent.
|
|
mkdirSync(live, { recursive: true });
|
|
writeFileSync(join(live, 'keep'), 'x');
|
|
const quiet = execFileSync('sh', ['-c', buildSwapRecoveryScript([live])], { stdio: 'pipe' }).toString();
|
|
assert.equal(quiet, '', 'no marker → no narration');
|
|
|
|
// Marker present (interrupted phase 2): recovery must announce itself.
|
|
writeFileSync(join(live, SWAP_PHASE), '2');
|
|
mkdirSync(join(live, SWAP_OLD), { recursive: true });
|
|
const noisy = execFileSync('sh', ['-c', buildSwapRecoveryScript([live]) + ' 2>&1'], { stdio: 'pipe' }).toString();
|
|
assert.ok(noisy.includes('rolling back an interrupted previous restore'), 'marker → narration printed');
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
});
|