read-side backup snapshot endpoints use job polling

src/lib/server/sse.ts

@@ -68,3 +68,21 @@ export function createJobResponse(
 
 	return json({ jobId: job.id });
 }
+
+/**
+ * Job-polling wrapper for a read endpoint that just returns DATA (no progress lines):
+ * run `fn`, deliver its value as the single `result`, and turn a thrown error into a
+ * `{ error }` result (HTTP 200 + a failed-shaped payload the client reads back via
+ * readJobResponse). Collapses the identical createJobResponse+try/catch boilerplate every
+ * backup read endpoint (browse/preview/metadata/diff) had. The proxy sees {jobId} at once,
+ * so a slow restic op can't be aborted mid-flight at the reverse-proxy's ~15s cap.
+ */
+export function jobResult<T>(request: Request | undefined, fn: () => Promise<T>): Response {
+	return createJobResponse(async (send) => {
+		try {
+			send('result', await fn());
+		} catch (error) {
+			send('result', { error: error instanceof Error ? error.message : String(error) });
+		}
+	}, request);
+}

src/routes/api/backup/instance/+server.ts

@@ -0,0 +1,18 @@
+import { json } from '@sveltejs/kit';
+import type { RequestHandler } from './$types';
+import { authorize } from '$lib/server/authorize';
+import { requireBackups } from '$lib/server/backups/route-guards';
+import { getInstanceId } from '$lib/server/backups/identity';
+
+// This install's stable instance id. The snapshots list used to ride it on an
+// X-Dockhand-Instance response header, but that list is now job-polled (a proxy would
+// abort the sync `restic snapshots` at ~15s) and headers don't survive polling. The
+// client fetches the id here once and matches it against each snapshot's
+// dockhand:instance= tag to tell own snapshots from foreign orphans (#1351).
+export const GET: RequestHandler = async ({ cookies }) => {
+	const auth = await authorize(cookies);
+	const denied = await requireBackups(auth, 'view');
+	if (denied) return denied;
+
+	return json({ instanceId: await getInstanceId() });
+};

src/routes/api/backup/restore/preview/+server.ts

@@ -5,6 +5,7 @@ import { requireBackups } from '$lib/server/backups/route-guards';
 import { previewSnapshot, previewRestoreTargets } from '$lib/server/backups';
 import { validateSnapshotId } from '$lib/server/docker-validation';
 import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards';
+import { jobResult } from '$lib/server/sse';
 
 export const POST: RequestHandler = async ({ request, cookies }) => {
 	const auth = await authorize(cookies);
@@ -26,7 +27,11 @@ export const POST: RequestHandler = async ({ request, cookies }) => {
 	const denied = await guardSnapshotEnvAccess(auth, body.destinationId, body.snapshotId);
 	if (denied) return denied;
 
-	try {
+	// Job-polling: the preview runs restic (verify-ownership + ls/dump) and, with a mode,
+	// per-volume probe helpers - all slow enough that a reverse proxy would abort the sync
+	// request at ~15s and SIGTERM the restic mid-op (the "wrong password / signal terminated"
+	// red herrings). Returning {jobId} immediately keeps the connection alive; the client polls.
+	return jobResult(request, async () => {
 		const access = { isEnterprise: auth.isEnterprise, canAccessEnvironment: (id: number) => auth.canAccessEnvironment(id) };
 		const preview = await previewSnapshot(body.destinationId, body.snapshotId, access);
 		// When the caller supplies a restore mode, ALSO resolve the exact on-disk targets and probe
@@ -44,11 +49,8 @@ export const POST: RequestHandler = async ({ request, cookies }) => {
 				mergeStackFiles: body.mergeStackFiles,
 				volumes: body.volumes,
 			}, access);
-			return json({ ...preview, targets });
+			return { ...preview, targets };
 		}
-		return json(preview);
-	} catch (error) {
-		const errorMsg = error instanceof Error ? error.message : String(error);
-		return json({ error: errorMsg }, { status: 500 });
-	}
+		return preview;
+	});
 };

src/routes/api/backup/snapshots/+server.ts

@@ -5,8 +5,9 @@ import { requireBackups } from '$lib/server/backups/route-guards';
 import { getBackupConfig, getBackupDestinations } from '$lib/server/db';
 import { listSnapshots } from '$lib/server/backups';
 import { filterSnapshotsByEnvAccess } from '$lib/server/backups/route-guards';
+import { jobResult } from '$lib/server/sse';
 
-export const GET: RequestHandler = async ({ url, cookies }) => {
+export const GET: RequestHandler = async ({ url, cookies, request }) => {
 	const auth = await authorize(cookies);
 	const denied = await requireBackups(auth, 'view');
 	if (denied) return denied;
@@ -30,13 +31,16 @@ export const GET: RequestHandler = async ({ url, cookies }) => {
 		// Narrow to THIS config's identity by its stable config-id tag so a sibling
 		// config sharing the same destination + targetName isn't mixed in.
 		if (allDests) {
-			// Search ALL destinations for this config's snapshots
-			try {
+			// Search ALL destinations for this config's snapshots.
+			// Job-polling: `restic snapshots` across every destination is the slowest read here
+			// and a reverse proxy would abort the sync request at ~15s. The result stays a bare
+			// snapshot array (unchanged contract - many callers/tests treat it as an array); the
+			// old X-Incomplete-Destinations header is now log-only.
+			return jobResult(request, async () => {
 				const destinations = await getBackupDestinations();
 				// Bound each destination's listing so one slow or unreachable repo
 				// can't stall the whole response for minutes (restic's own timeout is
-				// several minutes). A timed-out destination is treated as a failed
-				// one — surfaced as a partial result via X-Incomplete-Destinations.
+				// several minutes). A timed-out destination is treated as a failed one.
 				const PER_DEST_MS = Number(process.env.SNAPSHOT_LIST_TIMEOUT_MS ?? 30_000);
 				const withTimeout = <T>(p: Promise<T>, name: string): Promise<T> =>
 					Promise.race([
@@ -50,10 +54,8 @@ export const GET: RequestHandler = async ({ url, cookies }) => {
 				);
 				const allSnapshots: any[] = [];
 				const seen = new Set<string>();
-				// (audit #29) Don't silently drop destinations that errored. Collect
-				// the failures and surface them via a response header (keeping the body
-				// a plain array for compatibility), plus a server log — so a partial
-				// result doesn't look complete.
+				// (audit #29) Don't silently drop destinations that errored. Collect the
+				// failures and surface a count so a partial result doesn't look complete.
 				const failed: string[] = [];
 				results.forEach((result, i) => {
 					if (result.status === 'fulfilled') {
@@ -72,48 +74,33 @@ export const GET: RequestHandler = async ({ url, cookies }) => {
 				allSnapshots.sort((a, b) => new Date(b.time).getTime() - new Date(a.time).getTime());
 				if (failed.length > 0) {
 					console.warn(`[Backup] Snapshot search incomplete — ${failed.length} destination(s) failed: ${failed.join('; ')}`);
-					return json(allSnapshots, { headers: { 'X-Incomplete-Destinations': String(failed.length) } });
 				}
-				return json(allSnapshots);
-			} catch (error) {
-				const errorMsg = error instanceof Error ? error.message : String(error);
-				return json({ error: errorMsg }, { status: 500 });
-			}
+				return allSnapshots;
+			});
 		}
 
-		// Default: only search the config's current destination
-		try {
-			const snapshots = await listSnapshots(config.destinationId, config.id);
-			return json(snapshots);
-		} catch (error) {
-			const errorMsg = error instanceof Error ? error.message : String(error);
-			return json({ error: errorMsg }, { status: 500 });
-		}
+		// Default: only search the config's current destination.
+		return jobResult(request, () => listSnapshots(config.destinationId, config.id));
 	} else if (destIdParam) {
 		const destinationId = parseInt(destIdParam);
 		if (isNaN(destinationId)) return json({ error: 'Invalid destinationId' }, { status: 400 });
 
-		try {
+		// Job-polling: `restic snapshots` would otherwise be aborted at the proxy's ~15s cap.
+		// Returns a bare snapshot array (unchanged contract). The per-snapshot dockhand:instance=
+		// tag identifies the writing install; the client learns its OWN id from
+		// GET /api/backup/instance (the old X-Dockhand-Instance header can't ride job polling).
+		return jobResult(request, async () => {
 			// No target filter — return all snapshots in this destination (incl. snapshots
-			// from OTHER Dockhand instances sharing/copied into the repo). The response carries
-			// this install's instance id so the client can tell own snapshots from foreign
-			// orphans without relying on configid (which collides across instances - #1351).
-			const { getInstanceId } = await import('$lib/server/backups/identity');
-			const thisInstanceId = await getInstanceId();
+			// from OTHER Dockhand instances sharing/copied into the repo).
 			const snapshots = await listSnapshots(destinationId);
 			// (HIGH #8) On enterprise, a non-admin must not see snapshots belonging to
 			// environments they can't access — the ids/paths alone are disclosure. Drop
 			// any snapshot whose owning env (from its dockhand:env tag) isn't accessible.
-			const headers = { 'X-Dockhand-Instance': thisInstanceId };
 			if (auth.isEnterprise) {
-				const filtered = await filterSnapshotsByEnvAccess(auth, snapshots);
-				return json(filtered, { headers });
+				return await filterSnapshotsByEnvAccess(auth, snapshots);
 			}
-			return json(snapshots, { headers });
-		} catch (error) {
-			const errorMsg = error instanceof Error ? error.message : String(error);
-			return json({ error: errorMsg }, { status: 500 });
-		}
+			return snapshots;
+		});
 	} else {
 		return json({ error: 'configId or destinationId parameter is required' }, { status: 400 });
 	}

src/routes/api/backup/snapshots/[id]/browse/+server.ts

@@ -5,8 +5,9 @@ import { authorize } from '$lib/server/authorize';
 import { requireBackups } from '$lib/server/backups/route-guards';
 import { browseSnapshot } from '$lib/server/backups';
 import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards';
+import { jobResult } from '$lib/server/sse';
 
-export const GET: RequestHandler = async ({ params, url, cookies }) => {
+export const GET: RequestHandler = async ({ params, url, cookies, request }) => {
 	const auth = await authorize(cookies);
 	const denied = await requireBackups(auth, 'view');
 	if (denied) return denied;
@@ -38,11 +39,10 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => {
 		return json({ error: 'Environment access denied' }, { status: 403 });
 	}
 
-	try {
+	// Job-polling: `restic ls` behind a reverse proxy would abort at ~15s and SIGTERM restic
+	// (surfacing a misleading "wrong password / signal terminated"). Return {jobId} at once.
+	return jobResult(request, async () => {
 		const entries = await browseSnapshot(destinationId, snapshotId, path);
-		return json({ entries, path });
-	} catch (error) {
-		const errorMsg = error instanceof Error ? error.message : String(error);
-		return json({ error: errorMsg }, { status: 500 });
-	}
+		return { entries, path };
+	});
 };

src/routes/api/backup/snapshots/[id]/dump/+server.ts

@@ -6,8 +6,9 @@ import { requireBackups } from '$lib/server/backups/route-guards';
 import { dumpSnapshotFile, dumpSnapshotFileBytes, dumpSnapshotArchive } from '$lib/server/backups';
 import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards';
 import { parseSnapshotLayout, redactSnapshotLayout } from '$lib/server/backups/snapshot-layout';
+import { jobResult } from '$lib/server/sse';
 
-export const GET: RequestHandler = async ({ params, url, cookies }) => {
+export const GET: RequestHandler = async ({ params, url, cookies, request }) => {
 	const auth = await authorize(cookies);
 	const denied = await requireBackups(auth, 'view');
 	if (denied) return denied;
@@ -58,35 +59,35 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => {
 		if (download) {
 			return json({ error: 'metadata.json cannot be downloaded raw; use the snapshot metadata endpoint (secrets are redacted there)' }, { status: 403 });
 		}
-		try {
+		// Job-polling: `restic dump` behind a reverse proxy would abort at ~15s.
+		return jobResult(request, async () => {
 			const raw = await dumpSnapshotFile(destinationId, snapshotId, '/metadata/metadata.json');
 			const layout = parseSnapshotLayout(raw);
-			if (!layout) return json({ error: 'metadata unreadable' }, { status: 404 });
-			return json({ content: JSON.stringify(redactSnapshotLayout(layout), null, 2) });
-		} catch (error) {
-			const errorMsg = error instanceof Error ? error.message : String(error);
-			return json({ error: errorMsg }, { status: 500 });
-		}
+			if (!layout) return { error: 'metadata unreadable' };
+			return { content: JSON.stringify(redactSnapshotLayout(layout), null, 2) };
+		});
 	}
 
 	// Sanitize filename for Content-Disposition (strip quotes, backslashes, control chars)
 	const sanitizeFilename = (name: string) => name.replace(/["\\\x00-\x1f]/g, '_');
 
-	try {
-		if (download && isDir) {
-			// Binary tar stream — serve the raw bytes untouched (a UTF-8 round-trip
-			// would corrupt any non-ASCII byte in the archive).
-			const tarData = await dumpSnapshotArchive(destinationId, snapshotId, path);
-			const filename = sanitizeFilename((path.split('/').filter(Boolean).pop() || 'archive') + '.tar');
-			return new Response(new Uint8Array(tarData), {
-				headers: {
-					'Content-Type': 'application/x-tar',
-					'Content-Disposition': `attachment; filename="${filename}"`
-				}
-			});
-		}
-
-		if (download) {
+	// Binary downloads are window.open navigations, not fetch()es — they can't poll a job,
+	// so they stay synchronous. (A raw byte/tar stream also can't be JSON-wrapped.) These
+	// stream restic's output, so the proxy sees bytes flowing and won't idle-abort them.
+	if (download) {
+		try {
+			if (isDir) {
+				// Binary tar stream — serve the raw bytes untouched (a UTF-8 round-trip
+				// would corrupt any non-ASCII byte in the archive).
+				const tarData = await dumpSnapshotArchive(destinationId, snapshotId, path);
+				const filename = sanitizeFilename((path.split('/').filter(Boolean).pop() || 'archive') + '.tar');
+				return new Response(new Uint8Array(tarData), {
+					headers: {
+						'Content-Type': 'application/x-tar',
+						'Content-Disposition': `attachment; filename="${filename}"`
+					}
+				});
+			}
 			// A file download may be binary — serve raw bytes, not a decoded string.
 			const bytes = await dumpSnapshotFileBytes(destinationId, snapshotId, path);
 			const filename = sanitizeFilename(path.split('/').pop() || 'file');
@@ -96,13 +97,16 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => {
 					'Content-Disposition': `attachment; filename="${filename}"`
 				}
 			});
+		} catch (error) {
+			const errorMsg = error instanceof Error ? error.message : String(error);
+			return json({ error: errorMsg }, { status: 500 });
 		}
-
-		// Inline preview — text only.
-		const content = await dumpSnapshotFile(destinationId, snapshotId, path);
-		return json({ content });
-	} catch (error) {
-		const errorMsg = error instanceof Error ? error.message : String(error);
-		return json({ error: errorMsg }, { status: 500 });
 	}
+
+	// Inline preview (text only) — job-polling: `restic dump` would otherwise be aborted
+	// at the reverse-proxy's ~15s cap.
+	return jobResult(request, async () => {
+		const content = await dumpSnapshotFile(destinationId, snapshotId, path);
+		return { content };
+	});
 };

src/routes/api/backup/snapshots/[id]/metadata/+server.ts

@@ -6,8 +6,9 @@ import { requireBackups } from '$lib/server/backups/route-guards';
 import { getSnapshotMetadata } from '$lib/server/backups';
 import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards';
 import { redactSnapshotLayout } from '$lib/server/backups/snapshot-layout';
+import { jobResult } from '$lib/server/sse';
 
-export const GET: RequestHandler = async ({ params, url, cookies }) => {
+export const GET: RequestHandler = async ({ params, url, cookies, request }) => {
 	const auth = await authorize(cookies);
 	const denied = await requireBackups(auth, 'view');
 	if (denied) return denied;
@@ -26,14 +27,12 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => {
 	const envDenied = await guardSnapshotEnvAccess(auth, destinationId, snapshotId);
 	if (envDenied) return envDenied;
 
-	try {
+	// Job-polling: reading metadata runs `restic dump` which a proxy would abort at ~15s.
+	return jobResult(request, async () => {
 		const metadata = await getSnapshotMetadata(destinationId, snapshotId);
-		if (!metadata) return json({ error: 'No metadata available' }, { status: 404 });
+		if (!metadata) return { error: 'No metadata available' };
 		// Single redaction point: strips stack.secrets values AND container inspect
 		// Config.Env/Labels (plaintext secrets) before anything reaches the client.
-		return json(redactSnapshotLayout(metadata));
-	} catch (error) {
-		const msg = error instanceof Error ? error.message : String(error);
-		return json({ error: msg }, { status: 500 });
-	}
+		return redactSnapshotLayout(metadata);
+	});
 };

src/routes/api/backup/snapshots/diff/+server.ts

@@ -5,8 +5,9 @@ import { requireBackups } from '$lib/server/backups/route-guards';
 import { diffSnapshots } from '$lib/server/backups';
 import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards';
 import { validateSnapshotId } from '$lib/server/docker-validation';
+import { jobResult } from '$lib/server/sse';
 
-export const GET: RequestHandler = async ({ url, cookies }) => {
+export const GET: RequestHandler = async ({ url, cookies, request }) => {
 	const auth = await authorize(cookies);
 	const denied = await requireBackups(auth, 'view');
 	if (denied) return denied;
@@ -31,11 +32,6 @@ export const GET: RequestHandler = async ({ url, cookies }) => {
 	const deniedB = await guardSnapshotEnvAccess(auth, destinationId, snapB);
 	if (deniedB) return deniedB;
 
-	try {
-		const result = await diffSnapshots(destinationId, snapA, snapB);
-		return json(result);
-	} catch (error) {
-		const msg = error instanceof Error ? error.message : String(error);
-		return json({ error: msg }, { status: 500 });
-	}
+	// Job-polling: `restic diff` runs two restic reads a proxy would abort at ~15s.
+	return jobResult(request, () => diffSnapshots(destinationId, snapA, snapB));
 };

src/routes/backups/+page.svelte

@@ -25,7 +25,7 @@
 	import RotateCwFadingClock from '$lib/components/icons/RotateCwFadingClock.svelte';
 	import { formatDateTime, formatRelativeTime } from '$lib/stores/settings';
 	import { currentEnvironment } from '$lib/stores/environment';
-	import { watchJob } from '$lib/utils/sse-fetch';
+	import { watchJob, readJobResponse } from '$lib/utils/sse-fetch';
 	import { getRepoTypeIcon, formatCron, retentionSummary, classifyJobResult, tagLogLine } from '$lib/utils/backup';
 	import SnapshotBrowser from '../containers/SnapshotBrowser.svelte';
 	import RestoreModal from '../containers/RestoreModal.svelte';
@@ -328,6 +328,14 @@
 		// one of these belongs to a shown config; anything else is orphaned.
 		const liveConfigIds = new Set(configs.map(c => c.id));
 
+		// This install's id, fetched once: a snapshot is a LOCAL config's only when THIS
+		// instance wrote it (configid collides across installs - #1351). Formerly read from
+		// the snapshots list's X-Dockhand-Instance header, which job polling can't carry.
+		const thisInstance = await fetch('/api/backup/instance')
+			.then(r => r.ok ? r.json() : { instanceId: '' })
+			.then(d => d.instanceId || '')
+			.catch(() => '');
+
 		// Query each destination once, then split its snapshots two ways: PER CONFIG
 		// (by dockhand:configid) for the shown rows, and PER NAME for the orphan rows
 		// (snapshots whose config was deleted). The right-column count must match the
@@ -340,13 +348,12 @@
 		// rest keep filling in counts progressively behind the scenes.
 		await Promise.allSettled(destinations.map(async (dest) => {
 			try {
-				const res = await fetch(`/api/backup/snapshots?destinationId=${dest.id}`);
-				if (!res.ok) return;
-				// This install id: a snapshot belongs to a LOCAL config only when THIS instance wrote it.
-				// configid collides across instances (a fresh install reuses 1,2,3...), so a foreign
-				// snapshot with configid=1 must be an orphan, not mis-bucketed into local config #1 (#1351).
-				const thisInstance = res.headers.get("X-Dockhand-Instance") || "";
-				const d = await res.json();
+				// Job-polling so a slow `restic snapshots` behind a proxy isn't aborted at ~15s.
+				const res = await fetch(`/api/backup/snapshots?destinationId=${dest.id}`, {
+					headers: { Accept: 'text/event-stream' }
+				});
+				const d = await readJobResponse(res);
+				if (!d || d.error) return;
 				const snaps: any[] = d.snapshots ?? d;
 				if (!Array.isArray(snaps)) return;
 
@@ -406,16 +413,17 @@
 		loadingSnapshots = newLoading;
 
 		try {
-			// Query snapshots for this config's destination only (not allDestinations)
+			// Query snapshots for this config's destination only (not allDestinations).
+			// Job-polling so a slow `restic snapshots` behind a proxy isn't aborted at ~15s.
 			const snapUrl = config.isOrphan
 				? `/api/backup/snapshots?destinationId=${config.destinationId}`
 				: `/api/backup/snapshots?configId=${config.id}`;
-			const [snapRes, execRes] = await Promise.all([
-				fetch(snapUrl),
+			const [snapJson, execRes] = await Promise.all([
+				fetch(snapUrl, { headers: { Accept: 'text/event-stream' } }).then(readJobResponse),
 				config.isOrphan ? Promise.resolve(new Response('{"executions":[]}')) : fetch(`/api/schedules/executions?scheduleType=backup&scheduleId=${config.id}&limit=50`)
 			]);
-			const snapJson = await snapRes.json();
-			let snapData = snapJson.snapshots ?? snapJson;
+			const snapError = snapJson?.error;
+			let snapData = snapJson?.snapshots ?? snapJson;
 			// For orphans, filter by target name (they can span destinations).
 			if (config.isOrphan && Array.isArray(snapData)) {
 				snapData = snapData.filter((s: any) => (s.tags || []).some((t: string) => t === `dockhand:name=${config.targetName}`));
@@ -433,11 +441,11 @@
 				}));
 			}
 			const newMap = new Map(snapshotsMap);
-			if (snapRes.ok) {
+			if (!snapError) {
 				newMap.set(key, Array.isArray(snapData) ? snapData : []);
 			} else {
 				newMap.set(key, []);
-				toast.error(cleanErrorMessage(snapData.error || 'Failed to load snapshots'));
+				toast.error(cleanErrorMessage(snapError || 'Failed to load snapshots'));
 			}
 			snapshotsMap = newMap;
 
@@ -784,13 +792,17 @@
 			<p class="text-sm text-muted-foreground">Configure backups on individual containers or stacks via their edit modal.</p>
 		</div>
 	{:else}
+		<!-- virtualScroll is OFF here on purpose: it assumes a fixed rowHeight, but an
+		     expanded config renders a variable-height snapshot list, so the fixed-height
+		     spacers mis-size the scroll area (flicker + endless scroll). Backup configs
+		     number in the dozens, so plain rendering - the browser measures real heights -
+		     is correct and fast. -->
 		<DataGrid
 			data={filteredConfigs}
 			keyField="key"
 			gridId="backups"
 			loading={loading}
-			virtualScroll={true}
-			rowHeight={33}
+			virtualScroll={false}
 			sortState={{ field: sortField, direction: sortDirection }}
 			onSortChange={(state) => { sortField = state.field; sortDirection = state.direction; }}
 			bind:expandedKeys={expandedKeys}

src/routes/backups/SnapshotDiffModal.svelte

@@ -3,6 +3,7 @@
 	import { Badge } from '$lib/components/ui/badge';
 	import { Button } from '$lib/components/ui/button';
 	import { Plus, Minus, FileEdit, FileCheck, Loader2, ArrowRight, FolderOpen, Container, Info } from 'lucide-svelte';
+	import { readJobResponse } from '$lib/utils/sse-fetch';
 	import { formatDateTime } from '$lib/stores/settings';
 
 	interface Props {
@@ -43,13 +44,11 @@
 		metaA = null;
 		metaB = null;
 		try {
-			const res = await fetch(`/api/backup/snapshots/diff?destinationId=${destinationId}&snapshotA=${snapshotA.id}&snapshotB=${snapshotB.id}`);
-			if (!res.ok) {
-				const data = await res.json();
-				error = data.error || `HTTP ${res.status}`;
-				return;
-			}
-			diff = await res.json();
+			// Job-polling so the two `restic diff` reads behind a proxy aren't aborted at ~15s.
+			const res = await fetch(`/api/backup/snapshots/diff?destinationId=${destinationId}&snapshotA=${snapshotA.id}&snapshotB=${snapshotB.id}`, { headers: { Accept: 'text/event-stream' } });
+			const data = await readJobResponse(res);
+			if (data?.error) { error = data.error; return; }
+			diff = data;
 		} catch (err) {
 			error = err instanceof Error ? err.message : String(err);
 		} finally {
@@ -61,12 +60,13 @@
 		if (metaA && metaB) return;
 		metaLoading = true;
 		try {
-			const [resA, resB] = await Promise.all([
-				fetch(`/api/backup/snapshots/${snapshotA.id}/metadata?destinationId=${destinationId}`),
-				fetch(`/api/backup/snapshots/${snapshotB.id}/metadata?destinationId=${destinationId}`)
+			// Job-polling for both metadata dumps (slow `restic dump` behind a proxy).
+			const [dataA, dataB] = await Promise.all([
+				fetch(`/api/backup/snapshots/${snapshotA.id}/metadata?destinationId=${destinationId}`, { headers: { Accept: 'text/event-stream' } }).then(readJobResponse),
+				fetch(`/api/backup/snapshots/${snapshotB.id}/metadata?destinationId=${destinationId}`, { headers: { Accept: 'text/event-stream' } }).then(readJobResponse)
 			]);
-			metaA = resA.ok ? await resA.json() : null;
-			metaB = resB.ok ? await resB.json() : null;
+			metaA = dataA?.error ? null : dataA;
+			metaB = dataB?.error ? null : dataB;
 		} catch {}
 		metaLoading = false;
 	}

src/routes/containers/FileBrowserPanel.svelte

@@ -1,5 +1,6 @@
 <script lang="ts">
 	import { onMount, onDestroy } from 'svelte';
+	import { readJobResponse } from '$lib/utils/sse-fetch';
 	import { Button } from '$lib/components/ui/button';
 	import * as Table from '$lib/components/ui/table';
 	import * as Dialog from '$lib/components/ui/dialog';
@@ -400,9 +401,15 @@
 
 		try {
 			let res: Response;
+			let data: any;
 			if (isSnapshotMode) {
 				const params = new URLSearchParams({ destinationId: String(destinationId), path: filePath });
-				res = await fetch(`/api/backup/snapshots/${snapshotId}/dump?${params}`);
+				// Job-polling (readJobResponse) so a slow `restic dump` behind a proxy isn't aborted at ~15s.
+				res = await fetch(`/api/backup/snapshots/${snapshotId}/dump?${params}`, {
+					headers: { Accept: 'text/event-stream' }
+				});
+				data = await readJobResponse(res);
+				if (data?.error) throw new Error(data.error);
 			} else {
 				const params = new URLSearchParams({ path: filePath });
 				if (envId) params.set('env', envId.toString());
@@ -411,11 +418,10 @@
 				} else {
 					res = await fetch(`/api/containers/${effectiveContainerId}/files/content?${params}`);
 				}
-			}
-			const data = await res.json();
-
-			if (!res.ok) {
-				throw new Error(data.error || 'Failed to read file');
+				data = await res.json();
+				if (!res.ok) {
+					throw new Error(data.error || 'Failed to read file');
+				}
 			}
 
 			viewingFile = {
@@ -740,8 +746,12 @@
 
 			if (isSnapshotMode) {
 				const snapshotParams = new URLSearchParams({ destinationId: String(destinationId), path });
-				res = await fetch(`/api/backup/snapshots/${snapshotId}/browse?${snapshotParams}`);
-				data = await res.json();
+				// Job-polling (readJobResponse) so a slow `restic ls` behind a proxy isn't aborted at ~15s.
+				res = await fetch(`/api/backup/snapshots/${snapshotId}/browse?${snapshotParams}`, { headers: { Accept: 'text/event-stream' } });
+				data = await readJobResponse(res);
+				// A jobified error comes back as { error } with HTTP 200, so surface it explicitly
+				// (the shared `if (!res.ok)` gate below can't see it).
+				if (data?.error) throw new Error(data.error);
 			} else if (isVolumeMode) {
 				res = await fetch(`/api/volumes/${encodeURIComponent(volumeName!)}/browse?${params}`);
 				data = await res.json();

src/routes/containers/RestoreModal.svelte

@@ -9,7 +9,7 @@
 	import { RotateCcw, AlertTriangle, Loader2, HardDrive, Folder, Clock, Play, CheckCircle2, XCircle, Server, PackagePlus, Ban, Rocket, Box, Layers, HelpCircle, Info, KeyRound, FileX } from 'lucide-svelte';
 	import * as Tooltip from '$lib/components/ui/tooltip';
 	import { toast } from 'svelte-sonner';
-	import { watchJob } from '$lib/utils/sse-fetch';
+	import { watchJob, readJobResponse } from '$lib/utils/sse-fetch';
 	import { environments } from '$lib/stores/environment';
 	import { formatDateTime } from '$lib/stores/settings';
 	import EnvironmentIcon from '$lib/components/EnvironmentIcon.svelte';
@@ -346,16 +346,18 @@
 		loading = true;
 		error = '';
 		try {
+			// Job-polling (server returns {jobId}; readJobResponse polls to the result) so a slow
+			// restic verify/read behind a reverse proxy can't be aborted at ~15s.
 			const res = await fetch('/api/backup/restore/preview', {
 				method: 'POST',
-				headers: { 'Content-Type': 'application/json' },
+				headers: { 'Content-Type': 'application/json', Accept: 'text/event-stream' },
 				body: JSON.stringify({ destinationId, snapshotId, environmentId })
 			});
-			if (!res.ok) {
-				error = (await res.json()).error || 'Failed to read the snapshot';
+			const data = await readJobResponse(res);
+			if (data?.error) {
+				error = data.error || 'Failed to read the snapshot';
 				return;
 			}
-			const data = await res.json();
 			const types: Record<string, 'volume' | 'bind'> = data.volumeTypes || {};
 			const sources: Record<string, string> = data.volumeSources || {};
 			// Pre-fill each destination 1:1 from the snapshot's original location: a
@@ -398,9 +400,11 @@
 			const volumeDestinations = mode === 'new-location'
 				? selectedRows.map((v) => ({ volume: v.name, kind: v.destKind, target: v.dest.trim() }))
 				: undefined;
+			// Job-polling: the target probe runs a helper container per volume on the target host
+			// (slowest read path) - a sync request would abort at the proxy's ~15s.
 			const res = await fetch('/api/backup/restore/preview', {
 				method: 'POST',
-				headers: { 'Content-Type': 'application/json' },
+				headers: { 'Content-Type': 'application/json', Accept: 'text/event-stream' },
 				body: JSON.stringify({
 					destinationId, snapshotId, mode, environmentId: envId,
 					targetType: targetIsStack ? 'stack' : 'container',
@@ -409,14 +413,13 @@
 					volumes: selectedRows.map((v) => v.name),
 				})
 			});
-			if (seq !== targetPreviewSeq) return; // a newer request superseded this one
-			if (!res.ok) {
+			const data = await readJobResponse(res);
+			if (seq !== targetPreviewSeq) return; // a newer request superseded this one (checked AFTER the poll)
+			if (data?.error) {
 				targetPreview = null;
-				const msg = await res.json().then((d) => d?.error).catch(() => null);
-				targetPreviewError = msg || 'Could not check the target paths on the environment.';
+				targetPreviewError = data.error || 'Could not check the target paths on the environment.';
 				return;
 			}
-			const data = await res.json();
 			targetPreview = data.targets ?? null;
 		} catch {
 			if (seq === targetPreviewSeq) { targetPreview = null; targetPreviewError = 'Could not reach the server to check the target paths.'; }

src/routes/containers/SnapshotBrowser.svelte

@@ -1,5 +1,6 @@
 <script lang="ts">
 	import * as Dialog from '$lib/components/ui/dialog';
+	import { readJobResponse } from '$lib/utils/sse-fetch';
 	import { Badge } from '$lib/components/ui/badge';
 	import {
 		FolderOpen, Box, Layers, Network, HardDrive, Tag, Variable,
@@ -68,9 +69,12 @@
 				destinationRepo = dest.repository || '';
 			}
 			// Get snapshot time + owning environment from the snapshot's tags.
-			const snapRes = await fetch(`/api/backup/snapshots?destinationId=${destinationId}`);
-			if (snapRes.ok) {
-				const d = await snapRes.json();
+			// Job-polling so a slow `restic snapshots` behind a proxy isn't aborted at ~15s.
+			const snapRes = await fetch(`/api/backup/snapshots?destinationId=${destinationId}`, {
+				headers: { Accept: 'text/event-stream' }
+			});
+			const d = await readJobResponse(snapRes);
+			if (d && !d.error) {
 				const snaps = d.snapshots ?? d;
 				const snap = (Array.isArray(snaps) ? snaps : []).find((s: any) => s.id === snapshotId || s.shortId === snapshotId?.slice(0, 8));
 				if (snap) {
@@ -91,13 +95,11 @@
 		metadataLoading = true;
 		metadataError = '';
 		try {
-			const res = await fetch(`/api/backup/snapshots/${snapshotId}/metadata?destinationId=${destinationId}`);
-			if (res.ok) {
-				metadata = await res.json();
-			} else {
-				try { metadataError = (await res.json()).error || 'No metadata available'; }
-				catch { metadataError = 'No metadata available'; }
-			}
+			// Job-polling so a slow `restic dump` behind a proxy isn't aborted at ~15s.
+			const res = await fetch(`/api/backup/snapshots/${snapshotId}/metadata?destinationId=${destinationId}`, { headers: { Accept: 'text/event-stream' } });
+			const data = await readJobResponse(res);
+			if (data?.error) metadataError = data.error || 'No metadata available';
+			else metadata = data;
 		} catch { metadataError = 'Failed to load metadata'; }
 		finally { metadataLoading = false; }
 	}

src/routes/containers/SnapshotsPanel.svelte

@@ -6,6 +6,7 @@
 	import { LoadingState } from '$lib/components/ui/loading-state';
 	import { formatDateTime, formatRelativeTime } from '$lib/stores/settings';
 	import { formatBytes } from '$lib/utils/format';
+	import { readJobResponse } from '$lib/utils/sse-fetch';
 	import { getRepoTypeIcon } from '$lib/utils/backup';
 	import SnapshotBrowser from './SnapshotBrowser.svelte';
 	import RestoreModal from './RestoreModal.svelte';
@@ -111,9 +112,12 @@
 			// many (or one unreachable) repos the list fills in live rather than after a long wait.
 			await Promise.all(destinations.map(async (dest) => {
 				try {
-					const res = await fetch(`/api/backup/snapshots?destinationId=${dest.id}`);
-					if (!res.ok) return;
-					const data = await res.json();
+					// Job-polling so a slow `restic snapshots` behind a proxy isn't aborted at ~15s.
+					const res = await fetch(`/api/backup/snapshots?destinationId=${dest.id}`, {
+						headers: { Accept: 'text/event-stream' }
+					});
+					const data = await readJobResponse(res);
+					if (data?.error) return;
 					const snaps: any[] = data.snapshots ?? data;
 					const mine = snaps
 						.filter((s) => (s.tags || []).some((t: string) => t === nameTag))

src/routes/settings/backups/BackupsTab.svelte

@@ -26,7 +26,7 @@
 	import { canAccess } from '$lib/stores/auth';
 	import DestinationModal from './DestinationModal.svelte';
 	import BackupLogModal from '../../backups/BackupLogModal.svelte';
-	import { watchJob } from '$lib/utils/sse-fetch';
+	import { watchJob, readJobResponse } from '$lib/utils/sse-fetch';
 	import RotatePasswordModal from './RotatePasswordModal.svelte';
 	import { EmptyState } from '$lib/components/ui/empty-state';
 
@@ -370,13 +370,17 @@
 		browseOpen = true;
 		browseLoading = true;
 		try {
-			// List all snapshots in this destination (no configId filter)
-			const res = await fetch(`/api/backup/snapshots?destinationId=${dest.id}`);
-			if (res.ok) { const d = await res.json(); browseSnapshots = d.snapshots ?? d; }
-			else {
-				const data = await res.json();
-				const errMsg = data.error || 'Failed to list snapshots';
+			// List all snapshots in this destination (no configId filter). Job-polling so a
+			// slow `restic snapshots` behind a proxy isn't aborted at ~15s.
+			const res = await fetch(`/api/backup/snapshots?destinationId=${dest.id}`, {
+				headers: { Accept: 'text/event-stream' }
+			});
+			const d = await readJobResponse(res);
+			if (d?.error) {
+				const errMsg = d.error || 'Failed to list snapshots';
 				try { const p = JSON.parse(errMsg); toast.error(p.message || errMsg); } catch { toast.error(errMsg); }
+			} else {
+				browseSnapshots = d.snapshots ?? d;
 			}
 		} catch { toast.error('Failed to list snapshots'); }
 		finally { browseLoading = false; }