Google Cloud backup destinations now use a service-account JSON key

src/lib/server/backups/restic-script.ts

@@ -5,6 +5,8 @@
  * exits 0 so docker doesn't treat a restic non-zero as a crash.
  */
 
+import { isLocalRepo } from './models';
+
 /** The marker line the helper prints carrying restic's real exit code. */
 export const EXIT_MARKER = 'DOCKHAND_RESTIC_EXIT=';
 
@@ -28,13 +30,30 @@ export function finishScript(command: string): string {
 	return `${command}; __rc=$?; echo "${EXIT_MARKER}\${__rc}"`;
 }
 
+/** Where the helper writes the GCS service-account JSON before restic reads it. */
+export const GCS_CRED_FILE = '/tmp/dockhand-gcs-sa.json';
+
+/**
+ * restic's GCS backend authenticates with a service-account JSON file whose path
+ * is in GOOGLE_APPLICATION_CREDENTIALS (it auto-refreshes the token). We pass the
+ * JSON CONTENT as the GOOGLE_APPLICATION_CREDENTIALS_JSON env var; this preamble
+ * (prepended to the helper's `sh -c` script, in the SAME shell as restic) writes
+ * it to a 0600 file and exports GOOGLE_APPLICATION_CREDENTIALS at that path.
+ *
+ * A no-op when the var is empty/unset, so S3/B2/Azure/local repos are untouched.
+ * `printf %s` avoids the JSON being mangled by echo's escape handling.
+ */
+export function gcsCredentialPreamble(): string {
+	return `if [ -n "\${GOOGLE_APPLICATION_CREDENTIALS_JSON:-}" ]; then umask 077; printf '%s' "$GOOGLE_APPLICATION_CREDENTIALS_JSON" > ${GCS_CRED_FILE}; export GOOGLE_APPLICATION_CREDENTIALS=${GCS_CRED_FILE}; fi; `;
+}
+
 /**
  * Fail loud if a bind-mounted local repo isn't visible on the target daemon's
  * host: Docker would auto-create the missing path empty, silently backing up to
  * the wrong host. '' for non-local repos (they never bind a path).
  */
 export function localRepoGuard(repository: string): string {
-	if (!(repository.startsWith('/') || repository.startsWith('./'))) return '';
+	if (!isLocalRepo(repository)) return '';
 	const msg = `restic repository not found at $RESTIC_REPOSITORY on this environment's Docker host. A local-path repository only works when the environment's Docker daemon runs on the same host as Dockhand (e.g. a co-located socket-proxy). For a remote host, use an S3 or REST destination.`;
 	return `test -f "$RESTIC_REPOSITORY/config" || { echo ${shellQuote(msg)} >&2; exit 1; }; `;
 }
@@ -52,7 +71,7 @@ export function localRepoGuard(repository: string): string {
  */
 export function localRepoChown(repository: string, ownerSpec: string): string {
 	if (!ownerSpec) return '';
-	if (!(repository.startsWith('/') || repository.startsWith('./'))) return '';
+	if (!isLocalRepo(repository)) return '';
 	return `; chown -R ${ownerSpec} "$RESTIC_REPOSITORY" 2>/dev/null || true`;
 }
 
@@ -134,7 +153,7 @@ export function buildHelperBinds(
 	resolveLocalRepoHostPath: (repoPath: string) => string
 ): string[] {
 	const binds = [...volumeBinds];
-	if (repository.startsWith('/')) {
+	if (isLocalRepo(repository)) {
 		binds.push(`${resolveLocalRepoHostPath(repository)}:${repository}`);
 	}
 	return binds;

src/lib/server/backups/restic.ts

@@ -1,31 +1,26 @@
 /**
- * backups/restic.ts — the ONE way this codebase runs restic.
+ * The two ways restic runs, and the only place that knows the difference:
+ *   runLocal()    - restic on the Dockhand host, straight to the repo. Repo-only
+ *                   ops (snapshots, ls, dump, forget, check, init, unlock, stats,
+ *                   diff, key): no container, they don't touch a target's volumes.
+ *   runInHelper() - restic inside a single throwaway helper container with the
+ *                   target's volumes bind-mounted. Backup + restore only.
  *
- * There are exactly two ways restic has to run, and this class is the only place
- * that knows the difference:
+ * Both return a `ResticRun` and NEVER throw for an operational outcome; the caller
+ * inspects `exitCode`. `exitCode === undefined` = outcome could not be determined
+ * (killed helper, daemon hiccup) and the caller treats it as failure, so an
+ * unknown outcome is never a silent success.
  *
- *   runLocal()    — restic on the Dockhand host, talking straight to the repo.
- *                   Used for every repo-only operation: snapshots, ls, dump,
- *                   forget, check, init, unlock, stats, diff, key. No container
- *                   needed because these don't touch a target's volumes.
- *
- *   runInHelper() — restic inside a single throwaway helper container that has
- *                   the target's volumes bind-mounted. Used for backup and
- *                   restore, which must read/write the actual volume data. This
- *                   is the "one session container, do the work, close it" path.
- *
- * Both return a `ResticRun` and NEVER throw for an operational outcome — the
- * caller inspects `exitCode` and decides. `exitCode === undefined` means we
- * genuinely could not determine the outcome (a killed helper, a daemon hiccup);
- * callers treat that as failure so an unknown outcome is never a silent success.
- *
- * The security-sensitive pieces (which env vars, which flags, which binds) live
- * in one place here, built from the hardened allowlists in ./security — never
- * from the raw process environment or unfiltered user input.
+ * Env vars / flags / binds are built from the hardened allowlists in ./security,
+ * never from the raw process environment or unfiltered user input.
  */
 import { spawn } from 'child_process';
+import { mkdtempSync, writeFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import { runContainerWithStreaming, inspectImage, pullImage, putContainerArchive } from '../docker';
-import { buildTar } from './tar';
+import { buildTar, buildTarStream, type TarFileSource } from './tar';
+import { putContainerArchiveStreaming } from './put-archive-stream';
 import type { MetadataFile } from './backup-script';
 import { getInstanceId } from './identity';
 import { decryptBackupDestination, getSetting } from '../db';
@@ -36,10 +31,39 @@ import {
 	sanitizeResticFlags,
 } from './security';
 import { TIMEOUTS, type ResticRun, type TimeoutTier } from './models';
-import { withTimeout } from './helpers';
-import { resticCommand, finishScript, readExitMarker, buildHelperEnv, buildHelperBinds, localRepoGuard, localRepoChown, classifyProcClose, classifyProcError } from './restic-script';
+import { withTimeout, parseBackupFlags } from './helpers';
+import { resticCommand, finishScript, readExitMarker, buildHelperEnv, buildHelperBinds, localRepoGuard, localRepoChown, classifyProcClose, classifyProcError, gcsCredentialPreamble } from './restic-script';
 import { translateContainerPathViaMount } from '../host-path';
 
+/** The destination's BACKUP/global flags for a given restic subcommand. Restore has its
+ * OWN flags (threaded into the restore args by the restore builders), so this returns
+ * NOTHING for a `restore` command - backup flags must never reach `restic restore`. */
+function backupFlagsForCommand(destination: BackupDestination, command: string | undefined): string[] {
+	if (command === 'restore') return [];
+	return sanitizeResticFlags(parseBackupFlags(destination.flags).backup);
+}
+
+/**
+ * For a LOCAL restic run (Dockhand host), restic's GCS backend needs a service-account
+ * JSON FILE at GOOGLE_APPLICATION_CREDENTIALS. We carry the JSON as the env var
+ * GOOGLE_APPLICATION_CREDENTIALS_JSON; this writes it to a private 0600 temp file,
+ * points GOOGLE_APPLICATION_CREDENTIALS at it for the duration of `fn`, and deletes it
+ * after. A no-op (fn(env) unchanged) when the var is absent, so non-GCS runs are untouched.
+ * The helper-container path handles this itself in the shell script (gcsCredentialPreamble).
+ */
+export async function withGcsCredFile<T>(env: Record<string, string>, fn: (env: Record<string, string>) => Promise<T>): Promise<T> {
+	const json = env.GOOGLE_APPLICATION_CREDENTIALS_JSON;
+	if (!json) return fn(env);
+	const dir = mkdtempSync(join(tmpdir(), 'dockhand-gcs-'));
+	const file = join(dir, 'sa.json');
+	writeFileSync(file, json, { mode: 0o600 });
+	try {
+		return await fn({ ...env, GOOGLE_APPLICATION_CREDENTIALS: file });
+	} finally {
+		rmSync(dir, { recursive: true, force: true });
+	}
+}
+
 /** The label key stamped on every helper container so a reaper only removes
  * containers THIS installation created (never a co-located install's helper on a
  * shared Docker daemon). */
@@ -54,9 +78,8 @@ const INSTANCE_LABEL = 'dockhand.instance';
  * DOCKHAND_VARIANT=baseline; its Wolfi helper would crash with SIGILL / "requires
  * v2 microarchitecture", so it uses the `-baseline` (Alpine/musl) helper instead.
  *
- * This is ONLY the default: the `default_backup_image` setting always wins (see
- * helperImage()), so a user who points the helper at their own registry/tag is never
- * overridden. */
+ * Only the default: the `default_backup_image` setting always wins (see
+ * ensureHelperImage()), so a user's own registry/tag is never overridden. */
 const APP_VERSION = typeof __APP_VERSION__ !== 'undefined' ? __APP_VERSION__ : null;
 const HELPER_VARIANT_SUFFIX = process.env.DOCKHAND_VARIANT === 'baseline' ? '-baseline' : '';
 export const DEFAULT_HELPER_IMAGE = APP_VERSION
@@ -64,13 +87,14 @@ export const DEFAULT_HELPER_IMAGE = APP_VERSION
 	: `fnsys/dockhand-backup:latest${HELPER_VARIANT_SUFFIX}`;
 
 export interface HelperRunSpec {
-	/** The restic argv (without the leading `restic`). */
-	args: string[];
+	/** The restic argv (without the leading `restic`). Optional: a script-mode spec
+	 * (`script` set) ignores it. runInHelper defaults it to []. */
+	args?: string[];
 	/** Volume/bind mounts for the helper, e.g. `myvol:/volumes/myvol:ro`. */
 	binds: string[];
 	/** Target environment id (null/undefined = local Docker daemon). */
 	envId?: number | null;
-	/** Container name prefix — one name per operation, for cancel + reap. */
+	/** Container name prefix - one name per operation, for cancel + reap. */
 	name: string;
 	/** Streamed stderr callback for live progress parsing. */
 	onStderr?: (line: string) => void;
@@ -84,22 +108,30 @@ export interface HelperRunSpec {
 	 * `args` is ignored and the script runs via `sh -c`. The script MUST print
 	 * the exit marker as its final line (use finishScript() to append it). */
 	script?: string;
-	/** Metadata/stack files to stream into the container's /metadata directory via
-	 * the Docker put-archive API BEFORE it starts. Kept out of the Cmd so large
-	 * stack files don't blow ARG_MAX. `path` is relative to /metadata. */
+	/** Small in-RAM files (e.g. metadata.json) put-archived into the container BEFORE it
+	 * starts. `path` is the full path from the container root. */
 	metadataFiles?: MetadataFile[];
+	/** tar-mode ONLY (direct-remote, no remote_stacks_dir): the stack dir's files, streamed
+	 * from disk into the helper (O(1) RAM, no cap) alongside metadataFiles in the same
+	 * put-archive. Each source's archivePath is the full container path
+	 * (volumes/__dockhand_stackdir__/<rel>). Empty/absent = no streaming. */
+	stackDirStreamSources?: TarFileSource[];
 }
 
 export class Restic {
 	/**
-	 * Run restic on the host. Resolves a ResticRun for EVERY outcome — a non-zero
+	 * Run restic on the host. Resolves a ResticRun for EVERY outcome - a non-zero
 	 * exit, a timeout (exitCode undefined + a signal), or a spawn error all come
 	 * back as data, never as a thrown exception.
 	 */
 	async runLocal(
 		destination: BackupDestination,
 		args: string[],
-		tier: TimeoutTier = 'interactive'
+		tier: TimeoutTier = 'interactive',
+		/** Opt-in live output callbacks (chunk-level). restic writes `check`/`prune`
+		 * progress to stderr; without these the output only arrives buffered after
+		 * exit. */
+		stream?: { onStdout?: (chunk: string) => void; onStderr?: (chunk: string) => void }
 	): Promise<ResticRun> {
 		const decrypted = decryptBackupDestination(destination);
 		const env = buildResticEnv(process.env, {
@@ -107,27 +139,27 @@ export class Restic {
 			password: decrypted.decryptedPassword,
 			envVars: decrypted.decryptedEnvVars,
 		});
-		const fullArgs = [...args, ...sanitizeResticFlags(destination.flags)];
+		const fullArgs = [...args, ...backupFlagsForCommand(destination, args[0])];
 
-		return new Promise<ResticRun>((resolve) => {
+		return withGcsCredFile(env, (runEnv) => new Promise<ResticRun>((resolve) => {
 			let stdout = '';
 			let stderr = '';
 			let settled = false;
 			const done = (run: ResticRun) => { if (!settled) { settled = true; resolve(run); } };
 
-			const proc = spawn('restic', fullArgs, { env, timeout: TIMEOUTS[tier] });
-			proc.stdout.on('data', (d) => { stdout += d; });
-			proc.stderr.on('data', (d) => { stderr += d; });
+			const proc = spawn('restic', fullArgs, { env: runEnv, timeout: TIMEOUTS[tier] });
+			proc.stdout.on('data', (d) => { const s = String(d); stdout += s; stream?.onStdout?.(s); });
+			proc.stderr.on('data', (d) => { const s = String(d); stderr += s; stream?.onStderr?.(s); });
 			proc.on('close', (code, signal) => {
 				const { exitCode, stderr: stderrOut } = classifyProcClose(code, signal, stderr);
 				done({ exitCode, stdout, stderr: stderrOut });
 			});
 			proc.on('error', (err) => {
-				// restic missing / not spawnable — a real failure, surfaced as data.
+				// restic missing / not spawnable - a real failure, surfaced as data.
 				const { exitCode, stderr: stderrOut } = classifyProcError(err, stderr);
 				done({ exitCode, stdout, stderr: stderrOut });
 			});
-		});
+		}));
 	}
 
 	/**
@@ -147,9 +179,9 @@ export class Restic {
 			password: decrypted.decryptedPassword,
 			envVars: decrypted.decryptedEnvVars,
 		});
-		const fullArgs = [...args, ...sanitizeResticFlags(destination.flags)];
+		const fullArgs = [...args, ...backupFlagsForCommand(destination, args[0])];
 
-		return new Promise((resolve) => {
+		return withGcsCredFile(env, (runEnv) => new Promise((resolve) => {
 			const chunks: Buffer[] = [];
 			let stderr = '';
 			let settled = false;
@@ -157,7 +189,7 @@ export class Restic {
 				if (!settled) { settled = true; resolve(run); }
 			};
 
-			const proc = spawn('restic', fullArgs, { env, timeout: TIMEOUTS[tier] });
+			const proc = spawn('restic', fullArgs, { env: runEnv, timeout: TIMEOUTS[tier] });
 			proc.stdout.on('data', (d: Buffer) => { chunks.push(Buffer.from(d)); });
 			proc.stderr.on('data', (d) => { stderr += d; });
 			proc.on('close', (code, signal) => {
@@ -168,7 +200,7 @@ export class Restic {
 				const { exitCode, stderr: stderrOut } = classifyProcError(err, stderr);
 				done({ exitCode, stdout: Buffer.concat(chunks), stderr: stderrOut });
 			});
-		});
+		}));
 	}
 
 	/**
@@ -187,38 +219,51 @@ export class Restic {
 		const binds = buildHelperBinds(destination.repository, spec.binds, (repo) =>
 			destination.hostPath || translateLocalRepoPath(repo)
 		);
-		const image = await this.helperImage(spec.envId);
+		const image = await ensureHelperImage(spec.envId);
 		// Both paths append the exit marker so readExitMarker() sees the real exit
 		// code. A caller-supplied script (e.g. the in-place restore swap) may use
-		// `set -e`, so capture its exit in a subshell — the marker line must still
+		// `set -e`, so capture its exit in a subshell - the marker line must still
 		// print on success, otherwise a clean run reads back as undefined (failure).
 		// The local-repo guard runs FIRST (inside the marked command, so its exit 1
 		// surfaces via the marker): a local repo whose `config` isn't visible on the
-		// target daemon's host is a wrong-host mount — fail loud, not silently empty.
+		// target daemon's host is a wrong-host mount - fail loud, not silently empty.
 		const guard = localRepoGuard(destination.repository);
 		// The helper runs as root; re-own a local repo to the main process's uid:gid
 		// after the op so the main process (su-exec'd to that uid) can read it back.
 		// Appended AFTER the exit marker so it never changes restic's exit code.
 		const ownerSpec = `${process.getuid?.() ?? 0}:${process.getgid?.() ?? 0}`;
 		const chown = localRepoChown(destination.repository, ownerSpec);
+		const argv = spec.args ?? [];
+		// GCS: materialize the service-account JSON to a file + export
+		// GOOGLE_APPLICATION_CREDENTIALS before restic runs (no-op for other backends).
+		// Prepended inside the marked command so it shares restic's shell; covers every
+		// helper op (backup/restore/swap) in one place.
+		const gcs = gcsCredentialPreamble();
 		const cmd = spec.script
-			? ['sh', '-c', finishScript(`( ${guard}${spec.script} )`) + chown]
-			: ['sh', '-c', finishScript(`${guard}${resticCommand([...spec.args, ...sanitizeResticFlags(destination.flags)])}`) + chown];
+			? ['sh', '-c', finishScript(`( ${gcs}${guard}${spec.script} )`) + chown]
+			: ['sh', '-c', finishScript(`${gcs}${guard}${resticCommand([...argv, ...backupFlagsForCommand(destination, argv[0])])}`) + chown];
 
-		// Large metadata/stack files go into the container via put-archive (docker
-		// cp), NOT the Cmd, so they can't blow ARG_MAX. Build the tar once; stream
-		// it into /metadata after create, before start.
+		// The small metadata files (metadata.json + the light stack-dir listing) go into
+		// the container via put-archive (docker cp), NOT the Cmd, so they can't blow
+		// ARG_MAX. The stack dir's bytes ride a read-only host bind mount, not these files.
 		const metadataFiles = spec.metadataFiles ?? [];
-		const beforeStart = metadataFiles.length > 0
-			? async (containerId: string) => {
-				const tar = await buildTar(metadataFiles.map((f) => ({
-					path: `metadata/${f.path.replace(/^\/+/, '')}`,
-					content: Buffer.from(f.contentBase64, 'base64'),
-				})));
-				// Extract at / — the archive entries are already rooted at metadata/.
-				await putContainerArchive(containerId, '/', tar, spec.envId);
-			}
-			: undefined;
+			const streamSources = spec.stackDirStreamSources ?? [];
+			const inlineEntries = metadataFiles.map((f) => ({ path: f.path.replace(/^\/+/, ''), content: Buffer.from(f.contentBase64, 'base64') }));
+			const beforeStart = (metadataFiles.length > 0 || streamSources.length > 0)
+				? async (containerId: string) => {
+					// Each file's `path` is the full path from the container root (e.g.
+					// `metadata/metadata.json`), placing it under the right root.
+					if (streamSources.length > 0) {
+						// tar-mode: STREAM the stack dir files from disk (O(1) RAM, no cap) into the
+						// helper, with metadata.json as inline entries in the same tar. All under `/`.
+						const tar = buildTarStream(streamSources, inlineEntries);
+						const { streamed } = await putContainerArchiveStreaming(containerId, '/', tar, spec.envId);
+						if (!streamed) throw new Error(`tar-mode: transport for env ${spec.envId} cannot stream the stack dir`);
+					} else {
+						await putContainerArchive(containerId, '/', await buildTar(inlineEntries), spec.envId);
+					}
+				}
+				: undefined;
 
 		let stdout = '';
 		let stderr = '';
@@ -236,10 +281,9 @@ export class Restic {
 					stderr += data;
 					if (spec.onStderr) for (const line of data.split('\n')) if (line.trim()) spec.onStderr(line);
 				},
-				// Opt-in: forward live stdout lines to the spec's onStdout. The buffered
-				// `stdout` above stays the source of truth for summary parsing (post-exit);
-				// this only mirrors the same lines LIVE to the caller. Only wired when the
-				// caller asked for it, so non-backup callers are unaffected.
+				// Forward live stdout lines to the spec's onStdout. The buffered `stdout`
+				// above stays the source of truth for summary parsing (post-exit); this
+				// only mirrors the same lines LIVE to the caller.
 				onStdout: spec.onStdout
 					? (data) => { for (const line of data.split('\n')) if (line.trim()) spec.onStdout!(line); }
 					: undefined,
@@ -251,41 +295,13 @@ export class Restic {
 		} catch (err) {
 			// runContainerWithStreaming throws on a non-zero/indeterminate CONTAINER
 			// exit or a timeout. That means the helper itself failed (not restic
-			// reporting a code) — an unknown outcome. Surface it as undefined-exit
+			// reporting a code) - an unknown outcome. Surface it as undefined-exit
 			// data, never as a thrown error, so the caller fails closed.
 			const message = err instanceof Error ? err.message : String(err);
 			return { exitCode: undefined, stdout, stderr: `${stderr}\n${message}` };
 		}
 	}
 
-	// --- internals -----------------------------------------------------------
-
-	/** Resolve the helper image, pulling it only if genuinely absent (a 404 on
-	 * inspect). Any other daemon error propagates rather than triggering an
-	 * unnecessary pull. */
-	private async helperImage(envId?: number | null): Promise<string> {
-		const image = (await getSetting('default_backup_image')) || DEFAULT_HELPER_IMAGE;
-		// FAIL FAST: resolving the helper image must never hang. inspectImage /
-		// pullImage go through dockerFetch with `streaming: true`, which on a TCP/mTLS
-		// env carries NO request timeout — a stalled pull (registry unreachable, dead
-		// TCP) would otherwise wait out the whole backup op timeout (data tier = 6h)
-		// with the operation stuck before it ever starts. Bound both steps so a
-		// helper-image problem surfaces in seconds/minutes as a clear error instead.
-		let exists = true;
-		try {
-			await withTimeout(inspectImage(image, envId ?? undefined), HELPER_INSPECT_TIMEOUT_MS,
-				`timed out inspecting helper image "${image}"`);
-		} catch (err: any) {
-			if (err?.statusCode === 404) exists = false;
-			else throw err; // a real inspect error or a timeout BackupError — surface it
-		}
-		if (!exists) {
-			console.log(`[Backups] Pulling helper image: ${image}`);
-			await withTimeout(pullImage(image, undefined, envId ?? undefined), HELPER_PULL_TIMEOUT_MS,
-				`timed out pulling helper image "${image}" — check that it is available and the registry is reachable`);
-		}
-		return image;
-	}
 }
 
 /** Helper-image resolution timeouts (fail-fast so a stalled pull never hangs a
@@ -293,6 +309,42 @@ export class Restic {
 const HELPER_INSPECT_TIMEOUT_MS = Number(process.env.BACKUP_HELPER_INSPECT_TIMEOUT ?? 20_000);
 const HELPER_PULL_TIMEOUT_MS = Number(process.env.BACKUP_HELPER_PULL_TIMEOUT ?? 300_000);
 
+/**
+ * Resolve the helper image and ENSURE it is present on the target daemon (inspect-or-pull), with
+ * fail-fast timeouts. Shared by the backup runner AND the config-time stack-dir probe - the probe
+ * runs the same helper on the target daemon, so on an env that hasn't pulled the image yet (e.g. a
+ * hawser agent) it must pull it too, or the probe fails with "No such image".
+ */
+// Short-TTL memo of "image is present on this env" so a burst of ensureHelperImage calls in one
+// request (a restore preview probes N volumes) does ONE inspect, not N. Only a confirmed-present
+// result is cached; a miss/pull is never cached, so a later-disappearing image is re-pulled after
+// the TTL. Keyed by (image, envId).
+const helperImagePresentUntil = new Map<string, number>();
+const HELPER_IMAGE_MEMO_MS = 30_000;
+
+export async function ensureHelperImage(envId?: number | null): Promise<string> {
+	const image = (await getSetting('default_backup_image')) || DEFAULT_HELPER_IMAGE;
+	const memoKey = `${image}${envId ?? ''}`;
+	const until = helperImagePresentUntil.get(memoKey);
+	if (until && until > Date.now()) return image; // recently confirmed present on this env
+
+	let exists = true;
+	try {
+		await withTimeout(inspectImage(image, envId ?? undefined), HELPER_INSPECT_TIMEOUT_MS,
+			`timed out inspecting helper image "${image}"`);
+	} catch (err: any) {
+		if (err?.statusCode === 404) exists = false;
+		else throw err;
+	}
+	if (!exists) {
+		console.log(`[Backups] Pulling helper image: ${image}`);
+		await withTimeout(pullImage(image, undefined, envId ?? undefined), HELPER_PULL_TIMEOUT_MS,
+			`timed out pulling helper image "${image}" - check that it is available and the registry is reachable`);
+	}
+	helperImagePresentUntil.set(memoKey, Date.now() + HELPER_IMAGE_MEMO_MS);
+	return image;
+}
+
 /** Translate an in-container repo path to its host path for the bind mount. */
 function translateLocalRepoPath(repoPath: string): string {
 	return translateContainerPathViaMount(repoPath) || repoPath;

src/routes/api/backup/destinations/test/+server.ts

@@ -1,9 +1,11 @@
 import { json } from '@sveltejs/kit';
 import type { RequestHandler } from './$types';
 import { authorize } from '$lib/server/authorize';
+import { requireBackups } from '$lib/server/backups/route-guards';
 import { getBackupDestination, decryptBackupDestination, updateBackupDestinationTestStatus } from '$lib/server/db';
 import { spawn } from 'child_process';
 import { buildResticEnv, cleanErrorMsg, isRepoNotInitializedError, RESTIC_EXIT_REPO_NOT_FOUND, validateRepositoryForSave } from '$lib/server/backups/helpers';
+import { withGcsCredFile } from '$lib/server/backups/restic';
 
 /**
  * Test a backup destination configuration.
@@ -12,9 +14,8 @@ import { buildResticEnv, cleanErrorMsg, isRepoNotInitializedError, RESTIC_EXIT_R
  */
 export const POST: RequestHandler = async ({ request, cookies }) => {
 	const auth = await authorize(cookies);
-	if (auth.authEnabled && !await auth.can('backups', 'manage')) {
-		return json({ error: 'Permission denied' }, { status: 403 });
-	}
+	const denied = await requireBackups(auth, 'manage');
+	if (denied) return denied;
 
 	const body = await request.json();
 
@@ -54,15 +55,15 @@ export const POST: RequestHandler = async ({ request, cookies }) => {
 	const env = buildResticEnv(process.env, { repository, password, envVars });
 
 	try {
-		const result = await new Promise<{ code: number; stdout: string; stderr: string }>((resolve) => {
-			const proc = spawn('restic', ['snapshots', '--json', '--no-lock', '--latest', '1'], { env, timeout: 30000 });
+		const result = await withGcsCredFile(env, (runEnv) => new Promise<{ code: number; stdout: string; stderr: string }>((resolve) => {
+			const proc = spawn('restic', ['snapshots', '--json', '--no-lock', '--latest', '1'], { env: runEnv, timeout: 30000 });
 			let stdout = '';
 			let stderr = '';
 			proc.stdout.on('data', (d) => { stdout += d; });
 			proc.stderr.on('data', (d) => { stderr += d; });
 			proc.on('close', (code) => resolve({ code: code ?? 1, stdout, stderr }));
 			proc.on('error', (err) => resolve({ code: 1, stdout: '', stderr: err.message }));
-		});
+		}));
 
 		if (result.code === 0) {
 			if (body.destinationId) await updateBackupDestinationTestStatus(body.destinationId, 'success');