diff --git a/src/lib/components/backup/StackFilesPicker.svelte b/src/lib/components/backup/StackFilesPicker.svelte new file mode 100644 index 0000000..199b4c5 --- /dev/null +++ b/src/lib/components/backup/StackFilesPicker.svelte @@ -0,0 +1,254 @@ + + +
+
Stack files on the host
+ + {#if loading} +
+ + Probing the host for the stack folder... +
+ {:else if listing?.kind === 'listed' || listing?.kind === 'tar'} + {#if listing.kind === 'listed'} + +
+ +
+
+ Captured from the host at + + + + + +
+

Where this path comes from

+ {#if envKind === 'socket'} +

+ {#if envIcon && envId != null}{/if} + + {envLabel} is a socket + environment. Backup reads the stack folder from Dockhand's own stack + directory on this host. +

+

+ If this path looks wrong, set HOST_DATA_DIR + when Dockhand runs in a container, so it points to where the data volume + lives on the host. +

+ {:else if envKind === 'direct'} +

+ {#if envIcon && envId != null}{/if} + + {envLabel} is a direct + environment. Backup reads the stack folder from this path on the remote + Docker host. +

+ {#if configuredStackPath} +

+ Remote stack path set on this environment: + {configuredStackPath} +

+ {:else} +

+ It is derived from the stack's bind mounts. To set it explicitly, use + Environments > (edit) > Remote stack path (for backup) + and re-open this dialog. +

+ {/if} + {:else} +

+ {#if envIcon && envId != null}{/if} + {#if connectionType === 'hawser-edge'} + + {:else} + + {/if} + {envLabel} is a Hawser + environment. Backup reads the stack folder from the Hawser agent's stack + directory on its host. +

+ {#if configuredStackPath} +

+ Remote stack path set on this environment: + {configuredStackPath} +

+ {:else} +

+ It defaults to /data/stacks. If the + agent uses a custom STACKS_DIR, set it + under Environments > (edit) > Remote stack path (for backup) + and re-open this dialog. +

+ {/if} + {/if} +
+ + +
+
{listing.hostPath}
+
+
+ {:else} + +
+ +
+
Only these compose/config files will be captured
+
This environment has no stack path set, so the folder can't be read on the remote host. Dockhand backs up its own copy of the files below. Bind-mount data on the remote host is NOT included. Set a stack path on this environment for a full backup.
+
+
+ {/if} + +
+
+ + toggleAll(!allFiles)} /> +
+
+ {#each entries as entry} + {@const locked = isLocked(entry)} + + {/each} +
+
+ {:else if listing?.kind === 'helper-failed'} + +
+ +
+
The backup helper can't run on this environment
+
{listing.reason}
+
A backup can't be created until this is fixed - the same helper container captures the stack files.
+
+
+ {:else if listing?.kind === 'unknown'} +
+ +
+
No stack folder found on the host
+
{listing.reason}
+ {#if envKind === 'socket'} +
On a socket environment this usually means HOST_DATA_DIR does not match where the data volume is mounted on the host. Fix it, then cancel and re-configure this backup.
+ {:else if envKind === 'direct'} +
Set the environment's stack path (Environments > edit > Remote stack path for backup), then cancel and re-configure this backup.
+ {:else} +
On Hawser this usually means the agent uses a custom STACKS_DIR (not the default /data/stacks). Set the environment's stack path (Environments > edit > Remote stack path for backup), then cancel and re-configure this backup.
+ {/if} +
+
+ {/if} +
diff --git a/src/lib/server/backups/backup-script.ts b/src/lib/server/backups/backup-script.ts index f8025d8..3b0d0d8 100644 --- a/src/lib/server/backups/backup-script.ts +++ b/src/lib/server/backups/backup-script.ts @@ -27,9 +27,9 @@ export interface MetadataFile { * The metadata files (metadata.json, and for stacks the compose/.env and the * whole stackfiles/ tree) are NOT written by this script — they are streamed into * the container's /metadata directory via the Docker put-archive API BEFORE the - * container starts (see restic.ts runInHelper). Embedding their base64 inline in - * the Cmd used to blow the kernel ARG_MAX for large stack files ("argument list - * too long"); put-archive has no such limit. + * container starts (see restic.ts runInHelper). Inlining their base64 in the Cmd + * would hit the kernel ARG_MAX for large stack files ("argument list too long"); + * put-archive has no such limit. * * So the script only: * 1. ensures /metadata exists (put-archive already created it, but be defensive @@ -41,7 +41,7 @@ export interface MetadataFile { * @param resticArgs the full restic argv (starting with 'backup') — already * tag/exclude/flag-assembled by the caller. */ -export function buildBackupScript(resticArgs: string[]): string { +export function buildBackupScript(resticArgs: string[], stackDirProbe?: { volumeKey: string; composeFileName: string; label?: string; hostPath?: string }): string { // restic runs backgrounded so the SIGINT-forwarding trap can reach it on cancel // (SIGINT lets restic release its repo lock; a SIGKILL orphans it and hangs the // next backup on --retry-lock). The kill-0 re-wait loop only fires when a signal @@ -49,7 +49,30 @@ export function buildBackupScript(resticArgs: string[]): string { // real code (0/3) for the marker. const restic = `restic ${resticArgs.map(shellQuote).join(' ')}`; const step = `${restic} & __rpid=$!; trap 'kill -INT $__rpid 2>/dev/null' INT TERM; wait $__rpid; __rc=$?; while kill -0 $__rpid 2>/dev/null; do wait $__rpid; __rc=$?; done; ( exit $__rc )`; - return `mkdir -p /metadata; set +e; ${finishScript(step)}`; + + // STACK-DIR VOLUME PROBE (100% runtime guard for the 'volume' capture mode). The plan + // decided the target daemon is local and bind-mounted the stack dir at + // /volumes/. If that mount is a phantom (a remote daemon Docker + // auto-created an EMPTY dir for), the compose file will be MISSING under it — backing + // that up would be a silent-empty snapshot. So before restic runs, assert the compose + // is visible; if not, exit non-zero (the finishScript marker turns this into a hard + // backup failure), never a silent success. + let probe = ''; + if (stackDirProbe) { + const composePath = shellQuote(`/volumes/${stackDirProbe.volumeKey}/${stackDirProbe.composeFileName}`); + const label = stackDirProbe.label ? ` ${stackDirProbe.label}` : ''; + // Name the REAL host path we mounted (/) so the operator knows + // exactly where to look, and what to do: redeploy the stack to stage its files there. + const hostHint = stackDirProbe.hostPath + ? ` The stack folder is empty or missing at ${stackDirProbe.hostPath} on the Docker host - redeploy the stack to stage its files there.` + : ''; + probe = + `if [ ! -f ${composePath} ]; then ` + + `echo "[backup] $(date -u +%Y-%m-%dT%H:%M:%SZ)${label} STACKDIR PROBE FAILED: the stack's compose (${stackDirProbe.composeFileName}) is not present under the mounted stack dir.${hostHint} Refusing to write a silent-empty snapshot." 1>&2; ` + + `exit 1; fi; `; + } + + return `mkdir -p /metadata; set +e; ${probe}${finishScript(step)}`; } /** diff --git a/src/lib/server/backups/backup-service.ts b/src/lib/server/backups/backup-service.ts index 02e6d32..ca51526 100644 --- a/src/lib/server/backups/backup-service.ts +++ b/src/lib/server/backups/backup-service.ts @@ -42,18 +42,8 @@ import { buildBackupScript, buildBackupArgs, type MetadataFile } from './backup- import { EXIT_MARKER } from './restic-script'; import { parseRetention, buildForgetArgs, checkWouldWipe, retentionActive } from './retention'; import { liveTargetKey } from './locks'; -import { stopIntentKey as stopIntentKeyFor } from './stop-recovery-core'; import type { DiscoveredVolume } from './discovery-core'; - -/** What a fresh process needs to restart a target stopped for a consistent - * backup (or an in-place restore) after a crash between stop and restart. */ -export interface StopIntent { - type: BackupTargetType; - targetName: string; - envId: number | null; - /** For a container target: the ids/names that were running and got stopped. */ - containers: Array<{ id: string; name: string }>; -} +import { STACKDIR_VOLUME_KEY } from './stackdir-plan'; /** What the service needs from the outside world. All injected for testability. */ export interface BackupPorts { @@ -63,34 +53,45 @@ export interface BackupPorts { /** Discover volumes; MUST throw (fail closed) on any inspect failure. */ discoverVolumes(containers: Array<{ id: string; name: string }>, envId: number | null | undefined, selected: string[] | null): Promise<{ volumes: DiscoveredVolume[]; skipped: Array<{ type: string; destination: string; reason: string }> }>; + /** Decide how a STACK's directory is captured. Normally: locate the HOST folder and build the + * synthetic __dockhand_stackdir__ volume the helper bind-mounts (`kind: 'candidate'`). + * `kind: 'tar'` is the direct-remote-with-no-remote_stacks_dir case: nothing is staged on the + * host, so the compose/config is tarred from Dockhand's local copy instead (no host bind). + * `composeFileName` feeds the in-helper probe (assert the compose is + * visible under the mount). `kind: 'unknown'` means the stack is not compose-managed (no + * working_dir label) -> the caller HARD-FAILS the backup. Logged inside this port. */ + planStackDirVolume(targetName: string, envId: number | null | undefined, excludedStackFiles?: string[]): + Promise< + | { kind: 'unknown'; reason: string } + | { kind: 'candidate'; syntheticVolume: DiscoveredVolume; volumeKey: string; composeFileName: string; excludePaths: string[]; bindSources: string[] } + | { kind: 'tar'; localStackDir: string; composeFileName: string } + >; /** Stop the target for a consistent backup; returns a restart closure. */ stopForBackup(type: BackupTargetType, targetName: string, containers: Array<{ id: string; name: string; state: string }>, envId: number | null | undefined): Promise<{ restart: () => Promise<{ failed: Array<{ name: string; error: string }> }> }>; - /** Persist a durable "this target was stopped for backup" intent BEFORE the - * stop, so a process death between stop and restart is recoverable on startup. - * Keyed on a stable per-target id (see stopIntentKey). */ - recordStopIntent(key: string, intent: StopIntent): Promise; - /** Clear the durable stop intent once the in-process restart has succeeded. */ - clearStopIntent(key: string): Promise; - /** Run the backup session container; returns the restic run. metadataFiles are - * streamed into /metadata via put-archive (not the Cmd) so large stack files - * don't blow ARG_MAX. */ - runInHelper(spec: { args?: string[]; script: string; binds: string[]; envId: number | null | undefined; name: string; metadataFiles?: MetadataFile[]; onStderr?: (line: string) => void; onStdout?: (line: string) => void }): + /** Run the backup session container; returns the restic run. metadataFiles (metadata.json + * + the light stack-dir listing) are put-archived into /metadata (not the Cmd) so they + * can't blow ARG_MAX. The stack dir's bytes ride a host bind mount, not these files. */ + runInHelper(spec: { args?: string[]; script: string; binds: string[]; envId: number | null | undefined; name: string; metadataFiles?: MetadataFile[]; stackDirStreamSources?: Array<{ diskPath: string; archivePath: string }>; onStderr?: (line: string) => void; onStdout?: (line: string) => void }): Promise; /** Run a repo-only restic command on the host (verify / retention). */ runLocal(args: string[], tier?: 'interactive' | 'data'): Promise; - /** Collect the restore metadata files (metadata.json + stack files). - * `stackFilesTruncated` is true when a stack dir exceeded the capture cap, so - * the snapshot is an incomplete stack-file capture — the run is downgraded to - * a warning. */ - collectMetadata(type: BackupTargetType, targetName: string, envId: number | null | undefined, volumes: DiscoveredVolume[]): - Promise<{ files: MetadataFile[]; stackFilesTruncated: boolean }>; + /** Collect the restore metadata files: metadata.json plus, for a stack, a light file + * listing. The stack dir's bytes normally ride the helper's host bind mount, not these + * files. EXCEPTION - `stackDirTar`: for a direct-remote stack with no remote_stacks_dir + * there is no host bind, so the collector reads localStackDir and tars its files into the + * snapshot at /volumes/__dockhand_stackdir__ (the same path restore reads). */ + collectMetadata(type: BackupTargetType, targetName: string, envId: number | null | undefined, volumes: DiscoveredVolume[], stackDirTar?: { localStackDir: string; composeFileName: string } | null): + Promise<{ files: MetadataFile[]; stackDirStreamSources?: Array<{ diskPath: string; archivePath: string }> }>; /** The Dockhand host name for restic --host. */ host(): string; /** This installation's stable instance id. */ instanceId(): Promise; /** The live-target mutex: returns a release fn, or null if already held. */ acquireLiveTarget(key: string): (() => void) | null; + /** True if the user asked to cancel this config's in-flight backup — used to + * report a killed helper's non-zero exit as "cancelled", not a restic failure. */ + isCancelling(configId: number): boolean; /** Serialize on a destination; runs fn behind the per-destination queue. */ serializeDestination(destinationId: number, fn: () => Promise): Promise; /** Open the operation record; returns handles to update + close it. */ @@ -128,7 +129,7 @@ export interface BackupJob { selectedVolumes: string[] | null; stopBeforeBackup: boolean; retention: string | null | Record; - options: { excludePatterns?: string[]; excludeCaches?: boolean; compression?: string; limitUpload?: number; limitDownload?: number; webhookSuccess?: string; webhookFailure?: string }; + options: { excludePatterns?: string[]; excludeCaches?: boolean; compression?: string; limitUpload?: number; limitDownload?: number; webhookSuccess?: string; webhookFailure?: string; excludedStackFiles?: string[] }; helperName: string; } @@ -146,6 +147,17 @@ export class BackupService { // --- discover targets first (need the volume identity for the lock key) --- let containers: Array<{ id: string; name: string; state: string }>; let volumes: DiscoveredVolume[]; + // The stack dir is captured by the helper's HOST bind mount (a synthetic + // __dockhand_stackdir__ volume). `stackDirProbe` carries what the helper needs to + // assert that mount is real (test -f the compose) before restic runs. Null for + // containers (no stack dir). + let stackDirProbe: { volumeKey: string; composeFileName: string; hostPath?: string } | null = null; + // restic --exclude paths for compose bind dirs inside the stackdir volume (they are + // captured separately as their own /volumes/, honoring volume selection). + let stackDirExcludes: string[] = []; + // Set (instead of a host bind) for a direct-remote stack with no remote_stacks_dir: the + // compose/config is tarred into /metadata from Dockhand's local copy, not the host. + let stackDirTar: { localStackDir: string; composeFileName: string } | null = null; try { const t = await this.ports.resolveTargets(job.type, job.targetName, envId); containers = t.containers; @@ -161,12 +173,37 @@ export class BackupService { // A target with no volumes/binds is fine: metadata.json (the container/stack // config, and for stacks the compose files) is always captured, so the // snapshot is a config-only backup — restorable via recreate/redeploy. - } catch (err) { - return this.errorResult(err); - } + + // For a STACK, always capture the whole stack dir by bind-mounting it from the + // TARGET daemon's HOST (__dockhand_stackdir__). The plan locates the host folder + // from the compose working_dir label; a stack with no such label is not + // compose-managed, so we HARD-FAIL rather than silently skip its files (option A). + // The synthetic volume rides the same helper-bind + lock machinery as any volume. + if (job.type === 'stack') { + const p = await this.ports.planStackDirVolume(job.targetName, envId, job.options.excludedStackFiles); + if (p.kind === 'unknown') { + return this.earlySkipOrError(job, triggeredBy, + `Cannot locate the stack folder on the host for "${job.targetName}" (${p.reason}). ` + + `For remote environments, set a Remote stacks directory or use matching paths so the stack files exist on the Docker host. ` + + `Refusing to write a backup missing the stack's compose and config.`); + } else if (p.kind === 'tar') { + // direct-remote, no remote_stacks_dir: no host bind (nothing staged), so the + // compose/config is captured from Dockhand's local copy into /metadata via a tar. + // No synthetic volume, no probe - the metadata collector reads localStackDir. + stackDirTar = { localStackDir: p.localStackDir, composeFileName: p.composeFileName }; + } else { + volumes = [...volumes, p.syntheticVolume]; + stackDirProbe = { volumeKey: p.volumeKey, composeFileName: p.composeFileName, hostPath: p.syntheticVolume.source ?? undefined }; + stackDirExcludes = p.excludePaths; + } + } + } catch (err) { + return this.errorResult(err); + } // --- claim the live-target mutex (reject a concurrent op on this data) --- - const key = liveTargetKey(envId, volumes.map((v) => v.bind)); + // fallback identity keeps config-only targets (no volumes) from colliding on ${env}:: + const key = liveTargetKey(envId, volumes.map((v) => v.bind), `${job.type}:${job.targetName}`); const release = this.ports.acquireLiveTarget(key); if (!release) { return { status: 'skipped', reason: `Another backup or restore is already running for "${job.targetName}".` }; @@ -184,7 +221,7 @@ export class BackupService { try { // --- serialize on the destination (share the repo, one at a time) --- return await this.ports.serializeDestination(job.destinationId, () => - this.runLocked(job, triggeredBy, containers, volumes, startTime, releaseOnce)); + this.runLocked(job, triggeredBy, containers, volumes, stackDirProbe, stackDirExcludes, stackDirTar, startTime, releaseOnce)); } finally { releaseOnce(); } @@ -200,6 +237,9 @@ export class BackupService { triggeredBy: 'cron' | 'manual' | 'webhook', containers: Array<{ id: string; name: string; state: string }>, volumes: DiscoveredVolume[], + stackDirProbe: { volumeKey: string; composeFileName: string } | null, + stackDirExcludes: string[], + stackDirTar: { localStackDir: string; composeFileName: string } | null, startTime: number, releaseLiveTarget: () => void, ): Promise { @@ -207,26 +247,25 @@ export class BackupService { const op = await this.ports.openOperation(job.targetName, job.configId, envId, triggeredBy); let restart: (() => Promise<{ failed: Array<{ name: string; error: string }> }>) | null = null; let restarted = false; - let stopIntentKey: string | null = null; - // ========================= @BACKUP-DIAG-TEMP ========================= - // TEMPORARY CI backup-flake diagnostics. Remove this whole block (and the other - // `@BACKUP-DIAG-TEMP` markers below) once the flake is fixed: - // grep -rn '@BACKUP-DIAG-TEMP' src/lib/server/backups/backup-service.ts - // Every log line is prefixed `[BACKUP-DIAG-TEMP]` for a one-shot log filter. - // Gated behind BACKUP_DIAG=1 so it's silent unless CI turns it on. - // Emits per-phase timing (backup/verify/restart/retention/close) + a repo - // pre-probe (does the repo exist? is it locked?) to settle the "destination - // points at a wiped/uninit repo" and "stale lock" hypotheses. - const diagOn = process.env.BACKUP_DIAG === '1'; + // Two log tiers, both stamped with a wall-clock ISO time and the {env,target} + // key so any line can be lined up with the shard test log by time and target: + // log() - always on. The lines an operator wants: started / snapshot created / + // completed / failed. Plain and self-explanatory, no scary internals. + // diag() - behind BACKUP_DIAG=1 (CI turns it on). Deep internals: repo/lock + // pre-probes, instance identity, micro-phase timings. Noise for a user, + // gold for a CI flake. + const key = `${job.targetName} env=${envId}`; const t0 = Date.now(); + const log = (msg: string) => console.log(`[backup] ${new Date().toISOString()} ${key} | ${msg}`); + const diagOn = process.env.BACKUP_DIAG === '1'; const diag = (phase: string, extra?: string) => { if (!diagOn) return; - const ms = Date.now() - t0; - console.log(`[BACKUP-DIAG-TEMP] ${job.targetName} cfg=${job.configId} op=${op.id} env=${envId} | ${phase} @${ms}ms${extra ? ' | ' + extra : ''}`); + console.log(`[backup] ${new Date().toISOString()} ${key} cfg=${job.configId} op=${op.id} | ${phase} @${Date.now() - t0}ms${extra ? ' | ' + extra : ''}`); }; - diag('START', `triggeredBy=${triggeredBy} stopBeforeBackup=${job.stopBeforeBackup} retention=${JSON.stringify(job.retention)}`); + log(`started (${triggeredBy})`); if (diagOn) { + diag('START', `stopBeforeBackup=${job.stopBeforeBackup} retention=${JSON.stringify(job.retention)}`); try { const cfg = await this.ports.runLocal(['cat', 'config', '--no-lock'], 'interactive'); diag('REPO_PROBE', `cat-config exit=${cfg.exitCode} (${cfg.exitCode === 10 ? 'NOT-INITIALISED' : cfg.exitCode === 0 ? 'exists' : 'error'})`); @@ -237,49 +276,47 @@ export class BackupService { diag('LOCK_PROBE', `list-locks exit=${locks.exitCode} heldLocks=${lockLines.length}${lockLines.length ? ' [' + lockLines.join(',').slice(0, 120) + ']' : ''}`); } catch (e) { diag('LOCK_PROBE', `list-locks THREW ${e instanceof Error ? e.message : String(e)}`); } } - // ======================= end @BACKUP-DIAG-TEMP ======================= try { // --- optionally stop the target for a consistent snapshot --- if (job.stopBeforeBackup) { op.progress('stopping', `Stopping ${job.targetName} for a consistent backup...`); - // Record a durable stop intent BEFORE the stop, so a process death - // between the stop and the restart leaves a row a fresh startup can - // replay to bring the target back up (see reconcileOnStartup). - const running = containers.filter((c) => c.state === 'running'); - if (running.length > 0) { - stopIntentKey = stopIntentKeyFor(envId, job.type, job.targetName); - await this.ports.recordStopIntent(stopIntentKey, { - type: job.type, targetName: job.targetName, envId, - containers: running.map((c) => ({ id: c.id, name: c.name })), - }); - } const stopped = await this.ports.stopForBackup(job.type, job.targetName, containers, envId); restart = stopped.restart; } // --- metadata + backup args + session script --- op.progress('metadata', 'Collecting metadata...'); - const { files: metadata, stackFilesTruncated } = await this.ports.collectMetadata(job.type, job.targetName, envId, volumes); + const { files: metadata, stackDirStreamSources } = await this.ports.collectMetadata(job.type, job.targetName, envId, volumes, stackDirTar); const instanceId = await this.ports.instanceId(); const tags = buildSnapshotTags({ instanceId, configId: job.configId, environmentId: envId, targetName: job.targetName, type: job.type }); - // @BACKUP-DIAG-TEMP: cross-shard identity. If two shards log the SAME - // instanceId + configId (they share a cloned DB) AND the same repo, their - // snapshots/retention collide. host() + destinationId show which repo this - // run targets, so a CI log across shards reveals any overlap. + // Cross-shard identity. If two shards log the SAME instanceId + configId + // (they share a cloned DB) AND the same repo, their snapshots/retention + // collide. host() + destinationId show which repo this run targets, so a + // CI log across shards reveals any overlap. diag('IDENTITY', `instanceId=${instanceId} host=${this.ports.host()} destId=${job.destinationId} tags=[${tags.join(',')}]`); + // restic must back up /volumes/ when there's anything to capture there: real bind/named + // volumes (volumes.length), OR a tar-mode stack whose compose/config was put-archived + // into /volumes/__dockhand_stackdir__ (no synthetic volume in `volumes`, so it wouldn't + // count otherwise - and /metadata-only would silently drop the tarred stack files). const args = buildBackupArgs({ host: this.ports.host(), tags, - hasVolumes: volumes.length > 0, - excludePatterns: job.options.excludePatterns, + hasVolumes: volumes.length > 0 || stackDirTar !== null, + // User excludes PLUS the compose bind dirs inside the stackdir volume (those + // ride their own /volumes/, so excluding them here avoids a duplicate and + // keeps volume DESELECTION meaningful for in-folder binds). + excludePatterns: [...(job.options.excludePatterns ?? []), ...stackDirExcludes], excludeCaches: job.options.excludeCaches, compression: job.options.compression, limitUpload: job.options.limitUpload, limitDownload: job.options.limitDownload, swapArtifacts: SWAP_ARTIFACTS, }); - const script = buildBackupScript(args); + const script = buildBackupScript( + args, + stackDirProbe ? { ...stackDirProbe, label: `env=${envId} target=${job.targetName}` } : undefined + ); const binds = volumes.map((v) => v.bind); // Echo the options actually applied to the restic run, so the execution @@ -291,8 +328,7 @@ export class BackupService { if (o.excludeCaches !== false) op.log('Exclude caches: yes'); if (o.excludePatterns && o.excludePatterns.length) op.log(`Exclude patterns: ${o.excludePatterns.join(', ')}`); // Record WHICH volumes were actually selected (not just the count), so a - // regression in the selectedVolumes filter is visible in the execution - // log rather than silent (matches the legacy engine's log line). + // regression in the selectedVolumes filter is visible in the execution log. op.log(`Volumes to backup: ${volumes.map((v) => v.name).join(', ') || 'none'}`); // --- clear any STALE repo lock before backing up --- @@ -326,18 +362,26 @@ export class BackupService { } else { op.progress('backing-up', `Backing up ${volumes.length} ${volumes.length === 1 ? 'volume' : 'volumes'}:`); for (const v of volumes) { - op.progress('backing-up', ` • [${v.type === 'bind' ? 'BIND' : 'VOL'}] ${v.name}`); + // A bind's `name` is only its container destination, so two binds to the same + // path (e.g. /data) read identically. Show `host-source -> destination` so each + // is distinguishable in the log (#1373). + const label = v.name === STACKDIR_VOLUME_KEY + ? 'Stack data' + : v.type === 'bind' + ? `${v.source} → ${v.name}` + : v.name; + op.progress('backing-up', ` • [${v.type === 'bind' ? 'BIND' : 'VOL'}] ${label}`); } } // Did the LIVE stdout stream deliver anything? restic writes its --json - // progress to STDOUT, which we forward via onStdout. But live stdout only + // progress to STDOUT (forwarded via onStdout); but live stdout only // works on transports that stream it (local socket/http); if nothing arrives // live, we MUST still emit restic's output post-exit from run.stdout — else // the log shows only our own stage markers and no restic lines at all (a - // regression). This flag drives that fallback. + // This flag drives that fallback. let liveStdoutSeen = false; const run = await this.ports.runInHelper({ - script, binds, envId, name: job.helperName, metadataFiles: metadata, + script, binds, envId, name: job.helperName, metadataFiles: metadata, stackDirStreamSources, // Stream restic's own output to the UI LIVE: restic writes its --json // progress (per-file %) to STDOUT and its status messages to STDERR. Wire // BOTH so the log fills in as the backup runs, not all at once at the end. @@ -349,7 +393,7 @@ export class BackupService { // Fallback: if live stdout streaming produced nothing (transport didn't // stream it), emit restic's buffered stdout now so the log always shows what - // restic actually did — the pre-live-streaming behaviour, kept as a safety net. + // restic actually did. if (!liveStdoutSeen) { for (const l of formatResticLines(run.stdout)) { if (!l.includes(EXIT_MARKER)) op.progress('progress', `[restic] ${l}`); @@ -365,13 +409,14 @@ export class BackupService { // No summary ⇒ unknown outcome ⇒ NOT a success (never record a phantom). throw new BackupError('INTEGRITY', 'backup produced no snapshot summary — treating as failed'); } - // A partial is either restic's own "couldn't read all files" (exit 3) OR - // a stack dir that exceeded the capture cap (snapshot is redeploy-incomplete). - const partial = resticPartial(run) || stackFilesTruncated; + // A partial is restic's own "couldn't read all files" (exit 3). A stack dir + // over the capture cap is no longer a partial — it hard-fails in collectMetadata. + const partial = resticPartial(run); // --- VERIFY the snapshot is actually readable before declaring success --- + log(`snapshot ${summary.snapshotId.slice(0, 8)} created, verifying`); op.progress('verifying', 'Verifying the snapshot is readable...'); - diag("BEFORE_VERIFY", "snapshot=" + summary.snapshotId); // @BACKUP-DIAG-TEMP + diag("BEFORE_VERIFY", "snapshot=" + summary.snapshotId); await this.verifySnapshot(summary.snapshotId, instanceId); // --- commit: record success + config status --- @@ -382,14 +427,14 @@ export class BackupService { // retention, then release the live-target lock: retention is a repo-only // forget/prune (already serialized per-destination) and a concurrent // restore neither reads nor writes the live volume during it. - diag("BEFORE_RESTART"); // @BACKUP-DIAG-TEMP - await this.restartAndSurface(restart, job, op, stopIntentKey); + diag("BEFORE_RESTART"); + await this.restartAndSurface(restart, job, op); restarted = true; releaseLiveTarget(); // --- retention: ONLY now, after a confirmed + verified snapshot AND // after the live target is back up + its lock freed --- - diag("BEFORE_RETENTION"); // @BACKUP-DIAG-TEMP + diag("BEFORE_RETENTION"); const retentionOutcome = await this.applyRetention(job, instanceId, op); const details = { @@ -401,21 +446,36 @@ export class BackupService { retention: retentionOutcome, }; if (partial) { - const warnMsg = stackFilesTruncated - ? 'Backup completed with warnings: the stack directory exceeded the capture cap, so the snapshot is an incomplete stack-file capture (redeploy may be incomplete).' - : 'Backup completed with warnings (some files could not be read).'; - await op.close({ kind: 'warning', message: warnMsg }, details); + log(`completed with warnings (partial read) in ${((Date.now() - t0) / 1000).toFixed(1)}s`); + await op.close({ kind: 'warning', message: 'Backup completed with warnings (some files could not be read).' }, details); await this.notifySuccess(job, summary, envId, true, op.id, startTime); - return { status: 'warning', executionId: op.id, snapshotId: summary.snapshotId, summary, warning: stackFilesTruncated ? 'stack files truncated' : 'partial read', retention: retentionOutcome }; + return { status: 'warning', executionId: op.id, snapshotId: summary.snapshotId, summary, warning: 'partial read', retention: retentionOutcome }; } - diag("BEFORE_CLOSE"); // @BACKUP-DIAG-TEMP + log(`completed in ${((Date.now() - t0) / 1000).toFixed(1)}s`); + diag("BEFORE_CLOSE"); await op.close({ kind: 'ok' }, details); await this.notifySuccess(job, summary, envId, false, op.id, startTime); return { status: 'success', executionId: op.id, snapshotId: summary.snapshotId, summary, retention: retentionOutcome }; } catch (err) { + // If the user cancelled THIS backup, its helper was SIGINT/SIGKILL'd — the + // resulting non-zero exit (137) is not a real failure. Report it as cancelled + // with a clean message instead of the raw restic exit spew. + if (this.ports.isCancelling(job.configId)) { + log('cancelled by user'); + if (!restarted) await this.restartAndSurface(restart, job, op); + try { await this.ports.runLocal(['unlock']); } catch { /* best-effort */ } + try { await this.ports.setConfigStatus(job.configId, 'failed'); } catch { /* non-fatal */ } + await op.close({ kind: 'cancelled', message: 'Backup cancelled' }); + return { status: 'skipped', executionId: op.id, reason: 'Backup cancelled' }; + } + // Log the FULL raw error to the server log (docker logs dockhand) before it's + // cleaned for the UI — the raw text (restic JSON spew, exit code, stderr) is + // what we actually need to diagnose a failure, and cleanErrorMsg strips it. + log(`failed after ${((Date.now() - t0) / 1000).toFixed(1)}s`); + console.error(`[backup] ${new Date().toISOString()} ${key} | raw error:`, err instanceof Error ? (err.stack ?? err.message) : String(err)); // Always try to restart what we stopped, even on failure — unless the // success path already restarted it (avoid a double start). - if (!restarted) await this.restartAndSurface(restart, job, op, stopIntentKey); + if (!restarted) await this.restartAndSurface(restart, job, op); // Clean up the repo lock this failed/killed backup may have orphaned, so // the NEXT op to this repo doesn't wait out `--retry-lock` on a dead // owner. Still behind the per-destination serializer here. Best-effort. @@ -424,7 +484,7 @@ export class BackupService { try { await this.ports.setConfigStatus(job.configId, 'failed'); } catch { /* non-fatal */ } await op.close({ kind: 'error', code, message }, { errorCode: code }); try { await this.ports.notify('backup_failed', { target: job.targetName, kind: 'backup', errorCode: code, message }, envId); } catch { /* non-fatal */ } - // Per-config failure webhook — same payload shape as the legacy engine. + // Per-config failure webhook. if (job.options.webhookFailure) { try { this.ports.fireWebhook(job.options.webhookFailure, { @@ -472,24 +532,26 @@ export class BackupService { // can see the wait, not just the eventual result. if (dryMs > 5000) op.log(`Retention: dry-run took ${(dryMs / 1000).toFixed(1)}s (waited on the repo lock).`); if (dryRun.exitCode !== 0) { - op.progress('warning', `Retention dry-run failed (exit ${dryRun.exitCode}): ${dryRun.stderr.trim() || 'restic failed'} — skipping prune, backup is still safe.`); + op.progress('warning', `Retention dry-run failed (exit ${dryRun.exitCode}): ${dryRun.stderr.trim() || 'restic failed'} — skipping retention, backup is still safe.`); return 'failed'; } if (checkWouldWipe(dryRun.stdout).wouldWipe) { - op.progress('warning', 'Retention would delete every snapshot for this config — skipping prune.'); + op.progress('warning', 'Retention would delete every snapshot for this config — skipping retention.'); return 'skipped-would-wipe'; } + // forget only removes snapshot references (fast, metadata) — NO --prune here, so a + // backup never hangs reclaiming space. The destination's prune schedule reclaims it. op.progress('pruning', 'Applying retention...'); const forget = buildForgetArgs(policy, filter)!; const t1 = Date.now(); const run = await this.ports.runLocal(forget, 'data'); const forgetMs = Date.now() - t1; if (run.exitCode !== 0) { - // A prune failure is non-fatal (the backup itself succeeded) but surfaced. + // A forget failure is non-fatal (the backup itself succeeded) but surfaced. // The common non-fatal case: --retry-lock elapsed because another op (an // orphaned lock, or a second Dockhand instance sharing this repo) held the // lock — retention just runs next schedule; no snapshot is lost. - op.progress('warning', `Retention prune failed after ${(forgetMs / 1000).toFixed(1)}s (exit ${run.exitCode}): ${run.stderr.trim() || 'restic failed'} — backup is safe, retention will retry next run.`); + op.progress('warning', `Retention failed after ${(forgetMs / 1000).toFixed(1)}s (exit ${run.exitCode}): ${run.stderr.trim() || 'restic failed'} — backup is safe, retention will retry next run.`); return 'failed'; } op.log(`Retention: applied in ${(forgetMs / 1000).toFixed(1)}s.`); @@ -503,7 +565,6 @@ export class BackupService { restart: (() => Promise<{ failed: Array<{ name: string; error: string }> }>) | null, job: BackupJob, op: OperationHandle, - stopIntentKey: string | null, ): Promise { if (!restart) return; let failed: Array<{ name: string; error: string }> = []; @@ -513,14 +574,11 @@ export class BackupService { failed = [{ name: job.targetName, error: err instanceof Error ? err.message : String(err) }]; } if (failed.length > 0) { - // Restart failed — leave the durable intent for startup retry. const names = failed.map((f) => f.name).join(', '); const msg = `Backup finished but ${names} could not be restarted and is STILL STOPPED — start it manually.`; op.progress('warning', msg); try { await this.ports.notify('backup_failed', { target: job.targetName, kind: 'backup', errorCode: 'STOPPED_RESTART_FAILED', message: msg }, job.environmentId); } catch { /* non-fatal */ } } else { - // Restart succeeded — drop the durable intent so no stale row remains. - if (stopIntentKey) { try { await this.ports.clearStopIntent(stopIntentKey); } catch { /* non-fatal */ } } op.progress('restarted', `${job.targetName} restarted.`); } } @@ -539,7 +597,7 @@ export class BackupService { dataAdded: summary.dataAdded, partial, }, envId); } catch { /* notification failure never changes the outcome */ } - // Per-config success webhook — same payload shape as the legacy engine. + // Per-config success webhook. if (job.options.webhookSuccess) { try { this.ports.fireWebhook(job.options.webhookSuccess, { @@ -558,11 +616,15 @@ export class BackupService { } /** The backup target (container/stack) doesn't exist — record the op and fail - * with a validation error (there is genuinely nothing to back up). */ + * with a validation error (there is genuinely nothing to back up). Close the durable + * execution row as a FAILED/VALIDATION error, NOT op.skip() — skip() hardcodes + * CONCURRENCY + skipped:true, which would log a benign-looking "skipped" row while the + * API returns VALIDATION. An operator filtering for failed backups would then never see + * that this config has been backing up nothing (its target is gone). */ private async earlySkipOrError(job: BackupJob, triggeredBy: 'cron' | 'manual' | 'webhook', reason: string): Promise { try { const op = await this.ports.openOperation(job.targetName, job.configId, job.environmentId, triggeredBy); - await op.skip(reason); + await op.close({ kind: 'error', code: 'VALIDATION', message: reason }); return { status: 'error', executionId: op.id, code: 'VALIDATION', error: reason }; } catch { return { status: 'error', code: 'VALIDATION', error: reason }; diff --git a/src/lib/server/backups/index.ts b/src/lib/server/backups/index.ts index 963833b..db0b5ac 100644 --- a/src/lib/server/backups/index.ts +++ b/src/lib/server/backups/index.ts @@ -1,38 +1,35 @@ /** * backups/index.ts — the composition root. Wires the pure BackupService / * RestoreService to the real world (db, docker, restic, stacks, notifications) - * by implementing their injected port interfaces, and exposes the single startup - * reconciler. This is the only place in the domain that touches everything, so - * the seams stay visible. + * by implementing their injected port interfaces. This is the only place in the + * domain that touches everything, so the seams stay visible. * * Public surface: * runBackup(configId, trigger) → run a backup for a config * runRestore(request) → run a restore - * reconcileOnStartup() → recover interrupted operations * the read/repo helpers re-exported for the API routes */ -import { Restic } from './restic'; -import { BackupService, type BackupPorts, type BackupJob, type OperationHandle, type StopIntent } from './backup-service'; -import { planStopRecovery, isTargetGoneError, type RestartAction } from './stop-recovery-core'; +import { Restic, type HelperRunSpec } from './restic'; +import { BackupService, formatResticLines, type BackupPorts, type BackupJob, type OperationHandle } from './backup-service'; import { RestoreService, type RestorePorts, type RestoreJob } from './restore-service'; -import { openOperation, scrubInterruptedOperations } from './operations'; +import { openOperation } from './operations'; import { LiveTargetLocks, DestinationSerializer } from './locks'; import { resolveTargets, discoverVolumes, stopForBackup } from './docker'; import { guardSnapshotAccess, listSnapshots as listSnapshotsCore, resolveSnapshotEnvId as resolveSnapshotEnvIdCore, filterSnapshotsByAccessibleEnv } from './snapshots'; import { initRepository as initRepoCore, testRepository as testRepoCore, checkRepository, pruneRepository, unlockRepository, repoStats, rotateDestinationPassword as rotateCore } from './repo'; import { parseRetention, buildForgetArgs, checkWouldWipe } from './retention'; -import { instanceTagFilter, parseSnapshots, retentionTagFilter } from './models'; -import { buildSwapRecoveryScript } from './swap'; -import { parseOptionsJson, fireWebhook, parseResticDiff, type SnapshotDiff } from './helpers'; +import { buildSnapshotLayout, serializeLayout, parseSnapshotLayout, type SnapshotLayout, type SnapshotStack, type SnapshotSecret } from './snapshot-layout'; +import { instanceTagFilter, parseSnapshots, retentionTagFilter, BackupError } from './models'; +import { parseOptionsJson, buildJobOptions, parseSelectedVolumes, parseBackupFlags, sanitizeRestoreFlags, fireWebhook, parseResticDiff, type SnapshotDiff } from './helpers'; import { getHostname } from '../license'; import { getBackupConfig, getBackupConfigs, getBackupDestination, updateBackupConfig, updateBackupDestination, decryptBackupDestination } from '../db'; import { getInstanceId } from './identity'; import { inspectContainer } from '../docker'; import { sendEventNotification } from '../notifications'; -import { recordOperation, clearOperation, listOperations } from './journal'; import type { BackupResult, RestoreResult, ResticRun, BackupTargetType } from './models'; import type { MetadataFile } from './backup-script'; import type { DiscoveredVolume } from './discovery-core'; +import { volumeBind } from './discovery-core'; import { resolveBindFromMetadata } from './restore-core'; import { parseSnapshotLsEntries } from './browse-core'; @@ -43,6 +40,10 @@ const destSerializer = new DestinationSerializer(); /** Config ids with a backup currently in flight — so a config edit/delete can * be blocked mid-run. Maintained by runBackup. */ const runningBackupConfigs = new Set(); +/** Config ids whose in-flight backup the user asked to cancel. Set by cancelBackup, + * read by the engine so a helper it SIGKILLs (exit 137) is reported as "cancelled", + * not as a restic failure. Cleared with runningBackupConfigs when the run ends. */ +const cancellingBackupConfigs = new Set(); // ============================================================================= // Operation-handle adapter — bridges the service's OperationHandle to our record @@ -65,6 +66,464 @@ async function makeOperationHandle( }; } +// ============================================================================= +// Stack-dir capture plan — the ONE place that locates a stack's HOST folder +// ============================================================================= +/** + * A stack's directory is ALWAYS captured by bind-mounting it from the TARGET daemon's HOST at + * /volumes/__dockhand_stackdir__:ro (restic reads it from disk); the helper is a sibling + * container on that daemon, so a host path on it is always mountable. The host path is resolved + * by resolveHostStackDir. A stack whose folder can't be located resolves to `unknown` -> the + * caller HARD-FAILS (never silently skip files the user expects). Compose bind DIRECTORIES + * inside the stack dir ride their own /volumes/, so they are --exclude'd here (keeps + * volume deselection meaningful; an in-place whole-dir restore can't clobber a deselected data dir). + */ + +/** + * List a stack's containers (retrying while empty, since a just-deployed stack takes a moment + * to become listable) and, in ONE inspect pass, harvest the two host-path signals the planner + * needs: every bind mount `Source`, and the compose `working_dir` label. Both from the SAME + * container list so they can't disagree on which containers are visible. + */ +async function collectStackContainerHostInfo( + targetName: string, + envId: number | null | undefined, + attempts = 7, +): Promise<{ containers: Array<{ id: string; name: string; state: string }>; bindSources: string[]; workingDirLabel: string | null }> { + // A just-deployed stack can take a few seconds to become listable (slower under CI load) - + // missing it leaves a bind-less stack with no working_dir label -> UNKNOWN. Retry over ~15s + // (attempts=7). The config-time PROBE passes attempts=1: there the stack already exists, so + // a non-listable container is legitimately stopped/absent and waiting 15s just hangs the picker. + let containers: Array<{ id: string; name: string; state: string }> = []; + const ATTEMPTS = Math.max(1, attempts); + for (let i = 0; i < ATTEMPTS; i++) { + try { containers = (await resolveTargets('stack', targetName, envId)).containers; } catch { containers = []; } + if (containers.length > 0) break; + if (i < ATTEMPTS - 1) await new Promise((r) => setTimeout(r, 2500)); + } + if (containers.length === 0 && ATTEMPTS > 1) console.warn(`[Backup] collectStackContainerHostInfo "${targetName}": no containers listed after ${ATTEMPTS} tries (~${(ATTEMPTS - 1) * 2.5}s) - a bind-less stack will resolve to UNKNOWN`); + + const bindSources: string[] = []; + let workingDirLabel: string | null = null; + for (const c of containers) { + try { + const insp = await inspectContainer(c.id, envId ?? undefined) as { Mounts?: Array<{ Type?: string; Source?: string }>; Config?: { Labels?: Record } }; + for (const m of insp.Mounts ?? []) if (m.Type === 'bind' && m.Source) bindSources.push(m.Source); + if (!workingDirLabel) workingDirLabel = insp.Config?.Labels?.['com.docker.compose.project.working_dir'] || null; + } catch { /* one container's inspect failing just means fewer bind sources to match */ } + } + return { containers, bindSources, workingDirLabel }; +} + +async function planStackDirVolume( + targetName: string, + envId: number | null | undefined, + excludedStackFiles?: string[], + attempts = 7, +): Promise< + | { kind: 'unknown'; reason: string } + | { kind: 'candidate'; syntheticVolume: DiscoveredVolume; volumeKey: string; composeFileName: string; excludePaths: string[]; bindSources: string[] } + | { kind: 'tar'; localStackDir: string; composeFileName: string } +> { + const { getStackComposeFile } = await import('../stacks'); + const { dirname, join, basename } = await import('path'); + const { lstatSync } = await import('fs'); + const { translateToHostPath, translateContainerPathViaMount, getOwnDockerHost, getAutoDetectedDockerHost } = await import('../host-path'); + const { resolveHostStackDir, deriveStackDirFromBinds, trustBindDerivedForEnv, isLocalDaemon, STACKDIR_VOLUME_KEY } = await import('./stackdir-plan'); + const { relativeBindDirsFromCompose, relativeBindsFromCompose } = await import('./stackfile-filter'); + + // The stack dir as DOCKHAND sees it (compose file's parent). This is the authoritative + // source for BOTH the compose filename (not the config_files label, which is `-` for stdin + // deploys) and the host-path translations below. + const compose = await getStackComposeFile(targetName, envId ?? undefined); + const dockhandStackDir = compose.success && compose.composePath ? dirname(compose.composePath) : null; + const composeFileName0 = compose.success && compose.composePath ? basename(compose.composePath) : null; + + // Inspect the stack's containers ONCE and harvest BOTH host-path candidates (bind mount + // sources + the working_dir label) from that single pass, so they agree on which containers + // are visible. + const { containers: stackContainers, bindSources, workingDirLabel } = + await collectStackContainerHostInfo(targetName, envId, attempts); + + // PRIMARY, authoritative host-path source: match the compose's relative bind dirs to the host + // paths the DAEMON reports for them (mount.Source). Needs no DATA_DIR/HOST_DATA_DIR config and + // works on socket/direct-local/adopted/hawser alike. + let bindDerivedHostPath: string | null = null; + try { + // ALL relative binds (files AND dirs), not just dirs: a single-file bind (./config.yaml) + // also pins the stack dir via its daemon-reported mount.Source, so it derives the host + // path without the working_dir label (which needs the container listable - a timing race + // under load that made bind-less/single-file stacks resolve to UNKNOWN in CI). + const relBinds = compose.content && dockhandStackDir ? relativeBindsFromCompose(compose.content, dockhandStackDir) : []; + if (relBinds.length > 0) bindDerivedHostPath = deriveStackDirFromBinds(relBinds, bindSources); + } catch { /* best-effort: fall through to translation/label candidates below */ } + + // The DATA_DIR / mount translations map a container path to DOCKHAND'S OWN host - valid ONLY + // when the target daemon IS Dockhand's host. For a REMOTE env (hawser, or a direct env whose + // tcp host != Dockhand's) the stack lives on a different machine, so those translations would + // hand the helper a path that only exists on Dockhand's host (the rambo bug). Gate them on + // isLocalDaemon; bind-derived (from the target daemon's own mount.Source) and the working_dir + // label stay valid for remote. + // Default agent STACKS_DIR (compose.go: getEnvString("STACKS_DIR", "/data/stacks")). A hawser + // stack is deployed by the AGENT into / on the agent's HOST, so this + // is where the helper finds the compose/config for a host-mount backup. The user can override + // it per-env via the "Remote stack path (for backup)" setting when the agent uses a custom + // STACKS_DIR; the in-helper probe validates the guess and errors clearly if it's wrong. + const HAWSER_DEFAULT_STACKS_DIR = '/data/stacks'; + let envConnType: string | null = null; + let envTcpHost: string | null = null; + let remoteStacksDir: string | null = null; + let isHawser = false; + if (envId != null) { + try { + const { getEnvironment, getEnvSetting } = await import('../db'); + const env = await getEnvironment(envId); + envConnType = env?.connectionType ?? null; + envTcpHost = env?.host && env?.port ? `tcp://${env.host}:${env.port}` : null; + isHawser = envConnType === 'hawser-standard' || envConnType === 'hawser-edge'; + if (envConnType === 'direct') { + const rsd = await getEnvSetting('remote_stacks_dir', envId); + remoteStacksDir = typeof rsd === 'string' && rsd.trim() ? rsd.trim().replace(/\/+$/, '') : null; + } else if (isHawser) { + // Same `remote_stacks_dir` setting as direct, but for hawser it is a BACKUP-ONLY + // declaration of where the AGENT keeps stack files on its host - it does NOT steer + // deploy (the agent hardcodes its STACKS_DIR); it only tells the backup helper where + // to bind-mount. Falls back to the agent's default STACKS_DIR (/data/stacks) so a + // standard agent needs no configuration. + const rsd = await getEnvSetting('remote_stacks_dir', envId); + remoteStacksDir = typeof rsd === 'string' && rsd.trim() + ? rsd.trim().replace(/\/+$/, '') + : HAWSER_DEFAULT_STACKS_DIR; + } + } catch { /* treat as unknown -> non-local (safe: skips the wrong-host translation) */ } + } + const ownHost = getOwnDockerHost() ?? getAutoDetectedDockerHost(); + const localDaemon = isLocalDaemon(envConnType, envTcpHost, ownHost); + + // A direct-REMOTE daemon (not Dockhand's host) shares no filesystem with Dockhand. Its + // working_dir label is Dockhand's OWN path (the stack was deployed via stdin), NOT a path on + // the remote host - trusting it mounts a phantom empty dir (the rambo bug). So for a remote + // direct env the decision is explicit, not guessed: + // - remote_stacks_dir SET -> the deploy staged the files at / on the host, so + // that IS the host path (bind-derived can still win below if a bind pins it more exactly). + // - remote_stacks_dir UNSET -> nothing was staged; capture the compose/config from + // Dockhand's OWN copy via a tar into the snapshot (no host bind, so no :ro mask). The + // caller signals this with kind: 'tar'. + const directRemote = envConnType === 'direct' && !localDaemon; + // The working_dir label is only trustworthy for hawser (agent ran compose on its host) or a + // LOCAL daemon. For direct-remote it's Dockhand's path, so drop it from the candidates. + const trustedWorkingDirLabel = directRemote ? null : workingDirLabel; + // The user-declared (or defaulted) host path where the stack folder lives, for direct-remote + // AND hawser. For hawser this is the agent's STACKS_DIR (default /data/stacks) - the DETERMINISTIC + // host location, so it wins over the flaky working_dir label (which is "/" for a start-from-stdin + // hawser stack). The in-helper probe confirms the compose is actually there. + const remoteStacksDirHostPath = (directRemote || isHawser) && remoteStacksDir ? `${remoteStacksDir}/${targetName}` : null; + + // bind-derived is a PHANTOM empty dir for a direct-remote stdin deploy with no remote_stacks_dir + // (see trustBindDerivedForEnv) - distrust it there so resolution falls through to kind:'tar'. + const trustedBindDerived = trustBindDerivedForEnv(bindDerivedHostPath, { directRemote, hasRemoteStacksDir: !!remoteStacksDir }); + + // Fallback host-path candidates (LOCAL daemon only). dataDirHostPath: DATA_DIR -> HOST_DATA_DIR + // (returns the input unchanged when NOT under DATA_DIR, so we null it in that case). + // mountHostPath: via a container bind mount (adopted/external stacks outside DATA_DIR). + // workingDirLabel: for hawser/matching-paths where the label already IS the host path. + const viaDataRaw = localDaemon && dockhandStackDir ? translateToHostPath(dockhandStackDir) : null; + const dataDirHostPath = viaDataRaw && viaDataRaw !== dockhandStackDir ? viaDataRaw : null; + const mountHostPath = localDaemon && dockhandStackDir ? translateContainerPathViaMount(dockhandStackDir) : null; + + const resolution = resolveHostStackDir({ + composeFileName: composeFileName0, + // remote_stacks_dir is the explicit, user-declared host location for a direct-remote env; + // bindDerived can still be more exact when a bind pins the dir, so it's tried first inside + // the resolver, then this. Null for socket/hawser/local. + remoteStacksDirHostPath, + bindDerivedHostPath: trustedBindDerived, + dataDirHostPath, + mountHostPath, + workingDirLabel: trustedWorkingDirLabel, + }); + if (resolution.kind === 'unknown') { + // A direct-REMOTE env with NO remote_stacks_dir has nothing staged on the host, but + // Dockhand HAS the stack's compose/config locally - capture it via a tar into the + // snapshot instead of hard-failing. (Only compose/config; volume data rides its own + // /volumes/. Secrets are never on disk - carried as ciphertext in metadata.) + if (directRemote && !remoteStacksDir && dockhandStackDir) { + console.log(`[Backup] stackdir plan for "${targetName}": TAR (direct-remote, no remote_stacks_dir) - capturing compose/config from Dockhand's local copy in folder ${dockhandStackDir}`); + return { kind: 'tar', localStackDir: dockhandStackDir, composeFileName: composeFileName0 ?? 'docker-compose.yml' }; + } + // Report every candidate we tried so an UNKNOWN is diagnosable without SSH: which + // inputs were null tells us WHY (e.g. workingDir null = 0 containers listed). + console.log(`[Backup] stackdir plan for "${targetName}": UNKNOWN reason="${resolution.reason}" dockhandStackDir=${dockhandStackDir ?? 'none'} | connType=${envConnType} localDaemon=${localDaemon} remoteStacksDir=${remoteStacksDir ?? 'null'} containers=${stackContainers.length} bindDerived=${bindDerivedHostPath ?? 'null'} dataDir=${dataDirHostPath ?? 'null'} mount=${mountHostPath ?? 'null'} workingDirLabel=${workingDirLabel ?? 'null'}`); + return { kind: 'unknown', reason: resolution.reason }; + } + const hostPath = resolution.hostPath; + const composeFileName = resolution.composeFile; + console.log(`[Backup] stackdir plan for "${targetName}": host bind hostPath=${hostPath} composeFile=${composeFileName} source="${resolution.source}"`); + + // Compose bind DIRECTORIES inside the stack dir are captured separately as their own + // /volumes/, so exclude them from the whole-dir stackdir capture. We derive the dirs + // by PARSING THE COMPOSE (Dockhand has it locally) - keyed on the compose-relative source, + // the same namespace as the on-disk stack dir. isDirRel checks against Dockhand's own copy + // of the stack dir (compose file's parent), which is a faithful mirror of the host layout. + const isDirRel = (rel: string): boolean => { + if (!dockhandStackDir) return false; + try { return lstatSync(join(dockhandStackDir, rel)).isDirectory(); } catch { return false; } + }; + const bindDirs = compose.content ? relativeBindDirsFromCompose(compose.content, dockhandStackDir ?? '', isDirRel) : []; + const excludePaths = bindDirs.map((rel) => `/volumes/${STACKDIR_VOLUME_KEY}/${rel}`); + if (excludePaths.length > 0) { + console.log(`[Backup] stackdir "${targetName}": excluding ${excludePaths.length} compose bind dir(s) from the stackdir volume (captured separately as their own volumes): [${bindDirs.join(', ')}]`); + } + + // User deselections from the "Stack files on the host" picker: exclude each named top-level + // entry from the stackdir capture. LOAD-BEARING files (compose/.env) are re-guarded here on + // the SERVER - a stale/tampered config can never drop them, regardless of the client. + if (excludedStackFiles && excludedStackFiles.length > 0) { + const { isLoadBearingStackFile } = await import('./stackfile-filter'); + const dropped: string[] = []; + for (const name of excludedStackFiles) { + // Only a plain top-level entry name (no path separators / traversal) is honored. + if (!name || name.includes('/') || name === '.' || name === '..') continue; + if (isLoadBearingStackFile(name)) continue; // never exclude compose/.env + const p = `/volumes/${STACKDIR_VOLUME_KEY}/${name}`; + if (!excludePaths.includes(p)) { excludePaths.push(p); dropped.push(name); } + } + if (dropped.length > 0) { + console.log(`[Backup] stackdir "${targetName}": user deselected ${dropped.length} stack file(s) from the capture: [${dropped.join(', ')}]`); + } + } + + return { + kind: 'candidate', + syntheticVolume: { + key: STACKDIR_VOLUME_KEY, + bind: volumeBind(hostPath, STACKDIR_VOLUME_KEY, 'ro'), + name: STACKDIR_VOLUME_KEY, + type: 'bind', + source: hostPath, + }, + volumeKey: STACKDIR_VOLUME_KEY, + composeFileName, + excludePaths, + bindSources, + }; +} + +/** + * UI preview of WHERE a stack's directory will be captured from on the host, WITHOUT running + * a backup. Lets the create-schedule dialog show the resolved host path up front (and warn if + * it's `unknown` before the user schedules a backup that would hard-fail). Thin wrapper over + * planStackDirVolume - returns only the display-relevant fields (never the synthetic volume). + */ +export async function previewStackBackupPath( + targetName: string, + envId: number | null | undefined, +): Promise<{ kind: 'candidate'; hostPath: string; composeFile: string } | { kind: 'tar'; composeFile: string } | { kind: 'unknown'; reason: string }> { + const plan = await planStackDirVolume(targetName, envId); + if (plan.kind === 'unknown') return { kind: 'unknown', reason: plan.reason }; + // TAR mode (direct-remote, no remote_stacks_dir): the stack folder is captured from + // Dockhand's own copy into the snapshot, not from a host path. Surface that to the UI so the + // preview says "captured from Dockhand" rather than showing a misleading host path. + if (plan.kind === 'tar') return { kind: 'tar', composeFile: plan.composeFileName }; + return { kind: 'candidate', hostPath: plan.syntheticVolume.source ?? '', composeFile: plan.composeFileName }; +} + +/** + * Probe the TARGET host at backup-config creation time and list the ACTUAL contents of the stack + * directory, so the UI can show the resolved HOST path and let the user pick what to back up. + * The host path comes from the SAME resolver the backup uses (`planStackDirVolume` -> + * `resolveHostStackDir`), so a helper container on the target daemon mounts read-only exactly + * what a backup would capture - validating the path empirically (an unmountable/empty path + * resolves to `unknown`, and the UI tells the user to set the env's stack path). + * Never throws for an operational failure - every miss returns { kind:'unknown', reason }. + */ +export async function probeStackDir( + targetName: string, + envId: number | null | undefined, +): Promise< + | { kind: 'listed'; hostPath: string; entries: import('./stackdir-plan').StackDirEntry[] } + | { kind: 'tar'; localStackDir: string; entries: import('./stackdir-plan').StackDirEntry[] } + | { kind: 'helper-failed'; reason: string } + | { kind: 'unknown'; reason: string } +> { + const { parseProbeListing, tagCapturedEntries, STACKDIR_VOLUME_KEY } = await import('./stackdir-plan'); + const { runContainerWithStreaming } = await import('../docker'); + const { ensureHelperImage } = await import('./restic'); + const { INSTANCE_LABEL } = await import('./reap-core'); + + // Ask the real capture planner how this stack resolves. TAR mode (direct-remote, no + // remote_stacks_dir) has no host folder to mount - the backup streams Dockhand's OWN local + // copy (compose/config only; bind data lives on the remote host, out of reach). Surface THAT + // to the picker so the user knows exactly what a backup would contain, instead of a bare + // "unknown". A `candidate`/`unknown` plan falls through to the host-mount probe below. + // attempts=1: config-time probe. The stack already exists (the user is editing its backup), + // so a not-listable container is legitimately stopped/absent - the 15s just-deployed retry + // would only hang the picker. (planStackDirVolume calls collectStackContainerHostInfo ONCE and + // returns its bindSources, so the probe does NOT inspect the containers a second time.) + const plan = await planStackDirVolume(targetName, envId, undefined, 1); + if (plan.kind === 'tar') { + const { readdirSync, statSync } = await import('fs'); + const { join } = await import('path'); + let entries: import('./stackdir-plan').StackDirEntry[] = []; + try { + entries = readdirSync(plan.localStackDir).map((name) => { + let type: 'dir' | 'file' = 'file'; let size = 0; + try { const st = statSync(join(plan.localStackDir, name)); type = st.isDirectory() ? 'dir' : 'file'; size = st.isFile() ? st.size : 0; } catch { /* keep defaults */ } + return { name, type, size }; + }).sort((a, b) => (a.type === b.type ? a.name.localeCompare(b.name) : a.type === 'dir' ? -1 : 1)); + } catch { /* empty local copy -> empty list, still tar kind */ } + return { kind: 'tar', localStackDir: plan.localStackDir, entries }; + } + + if (plan.kind === 'unknown') { + return { kind: 'unknown', reason: plan.reason }; + } + // plan.kind === 'candidate': the resolver already produced the TRUE host path (via bind-derive + // / DATA_DIR->HOST_DATA_DIR translation / remote_stacks_dir / working_dir label, in priority). + // Mount THAT - never the raw working_dir label, which for a socket env is Dockhand's CONTAINER + // path (/app/data/...) and does not exist on the host (the resolver translates it). + const hostPath = plan.syntheticVolume.source; + if (!hostPath) { + return { kind: 'unknown', reason: 'No stack folder could be located on the host. Set the environment\'s stack path and re-configure.' }; + } + const bindSources = plan.bindSources; + + // Ensure the helper image is present on the target daemon (inspect-or-pull). If it CAN'T run + // (image pull/run error) neither a backup nor a restore can happen on this env, so surface that + // as a distinct `helper-failed` (the caller hard-fails) instead of folding it into `unknown`. + let image: string; + try { + image = await ensureHelperImage(envId); + } catch (e) { + return { kind: 'helper-failed', reason: e instanceof Error ? e.message : String(e) }; + } + try { + // Portable listing: the helper is busybox, whose `find` has NO `-printf` (GNU-only). Loop + // the top-level entries (incl. dotfiles) and emit `\t\t` per line - the + // same tab-delimited form parseProbeListing reads. + const listScript = + `cd /probe 2>/dev/null || exit 1; for f in * .*; do ` + + `[ "$f" = "." ] || [ "$f" = ".." ] && continue; [ -e "$f" ] || continue; ` + + `if [ -d "$f" ]; then t=d; else t=f; fi; s=$(stat -c %s "$f" 2>/dev/null || echo 0); ` + + `printf '%s\\t%s\\t%s\\n' "$t" "$s" "$f"; done`; + const stdout = await runContainerWithStreaming({ + image, + cmd: ['sh', '-c', listScript], + binds: [`${hostPath}:/probe:ro`], + name: `dockhand-probe-${Date.now()}`, + labels: { [INSTANCE_LABEL]: await getInstanceId() }, + envId, + timeout: 60_000, + }); + const entries = parseProbeListing(stdout); + if (entries.length === 0) { + return { kind: 'unknown', reason: `The stack folder at ${hostPath} is empty or does not exist on the Docker host. Set the environment's stack path and re-configure.` }; + } + // Tag entries that are ALSO a bind mount so the picker shows them as non-deselectable + // (captured via the Volumes section, not here). Two sources: daemon bind sources (runtime, + // only for a RUNNING stack) AND the compose bind-dir names (always available) — the latter + // so a stopped stack's `./memos` bind still shows as captured-by-bind, not a plain file. + const bindDirNames = plan.excludePaths + .map((p) => p.replace(`/volumes/${STACKDIR_VOLUME_KEY}/`, '').split('/')[0]) + .filter(Boolean); + return { kind: 'listed', hostPath, entries: tagCapturedEntries(entries, hostPath, bindSources, bindDirNames) }; + } catch (e) { + return { kind: 'unknown', reason: `Could not read the stack folder at ${hostPath} on the host: ${e instanceof Error ? e.message : String(e)}` }; + } +} + +/** The busybox list-script for a data-presence probe: print the missing sentinel if the mount + * point does not exist, otherwise emit one `\t\t` line per top-level entry. */ +function probeListScript(missingSentinel: string): string { + return ( + `if [ ! -e /probe ]; then printf '%s' '${missingSentinel}'; exit 0; fi; ` + + `cd /probe 2>/dev/null || { printf '%s' '${missingSentinel}'; exit 0; }; ` + + `for f in * .*; do ` + + `[ "$f" = "." ] || [ "$f" = ".." ] && continue; [ -e "$f" ] || continue; ` + + `if [ -d "$f" ]; then t=d; else t=f; fi; s=$(stat -c %s "$f" 2>/dev/null || echo 0); ` + + `printf '%s\\t%s\\t%s\\n' "$t" "$s" "$f"; done` + ); +} + +/** + * Does an arbitrary host path have data on the TARGET daemon? Runs the same helper-container + * mechanism as probeStackDir but for a caller-supplied path, and - unlike probeStackDir which + * folds every failure into `unknown` - DISTINGUISHES a helper that could not run (`helper-failed`, + * so callers can hard-fail up front) from a helper that ran and found the dir empty / missing / + * populated. Used by the restore preview to warn before overwriting existing data. + */ +export async function probeHostPath( + hostPath: string, + envId: number | null | undefined, +): Promise<{ kind: import('./stackdir-plan').ProbeDataKind; reason?: string }> { + const { classifyProbeListing, PROBE_MISSING_SENTINEL } = await import('./stackdir-plan'); + const { runContainerWithStreaming } = await import('../docker'); + const { ensureHelperImage } = await import('./restic'); + const { INSTANCE_LABEL } = await import('./reap-core'); + let image: string; + try { + image = await ensureHelperImage(envId); + } catch (e) { + return { kind: 'helper-failed', reason: e instanceof Error ? e.message : String(e) }; + } + try { + const stdout = await runContainerWithStreaming({ + image, + cmd: ['sh', '-c', probeListScript(PROBE_MISSING_SENTINEL)], + binds: [`${hostPath}:/probe:ro`], + name: `dockhand-probe-${Date.now()}`, + labels: { [INSTANCE_LABEL]: await getInstanceId() }, + envId, + timeout: 60_000, + }); + return { kind: classifyProbeListing(stdout) }; + } catch (e) { + return { kind: 'helper-failed', reason: e instanceof Error ? e.message : String(e) }; + } +} + +/** + * Does a NAMED volume exist and hold data on the TARGET daemon? A named volume has no host path to + * bind, so this inspects it (missing -> `missing`) then mounts it by name into the helper and lists + * it. Same helper-failed distinction as probeHostPath. + */ +export async function probeVolumeData( + volumeName: string, + envId: number | null | undefined, +): Promise<{ kind: import('./stackdir-plan').ProbeDataKind; reason?: string }> { + const { classifyProbeListing, PROBE_MISSING_SENTINEL } = await import('./stackdir-plan'); + const { runContainerWithStreaming, inspectVolume } = await import('../docker'); + const { ensureHelperImage } = await import('./restic'); + const { INSTANCE_LABEL } = await import('./reap-core'); + try { + await inspectVolume(volumeName, envId); + } catch { + return { kind: 'missing' }; + } + let image: string; + try { + image = await ensureHelperImage(envId); + } catch (e) { + return { kind: 'helper-failed', reason: e instanceof Error ? e.message : String(e) }; + } + try { + const stdout = await runContainerWithStreaming({ + image, + cmd: ['sh', '-c', probeListScript(PROBE_MISSING_SENTINEL)], + binds: [`${volumeName}:/probe:ro`], + name: `dockhand-probe-${Date.now()}`, + labels: { [INSTANCE_LABEL]: await getInstanceId() }, + envId, + timeout: 60_000, + }); + // A just-inspected volume exists, so an empty listing means empty, never missing. + const k = classifyProbeListing(stdout); + return { kind: k === 'missing' ? 'empty' : k }; + } catch (e) { + return { kind: 'helper-failed', reason: e instanceof Error ? e.message : String(e) }; + } +} + // ============================================================================= // Metadata collector — reads restore context into base64 files for the snapshot // ============================================================================= @@ -73,108 +532,121 @@ async function collectMetadata( targetName: string, envId: number | null | undefined, volumes: DiscoveredVolume[], -): Promise<{ files: MetadataFile[]; stackFilesTruncated: boolean }> { + stackDirTar?: { localStackDir: string; composeFileName: string } | null, +): Promise<{ files: MetadataFile[]; stackDirStreamSources?: Array<{ diskPath: string; archivePath: string }> }> { const files: MetadataFile[] = []; - // True when a stack directory exceeded the capture cap → the snapshot is an - // incomplete stack-file capture. The caller downgrades the result to a warning. - let stackFilesTruncated = false; - const metadata: Record = { - type, targetName, environmentId: envId ?? null, - backupTime: new Date().toISOString(), - // Store the full identity restore needs to bind each volume back to its - // ORIGINAL location: for a bind, `source` is the absolute host path (NOT - // the slugged key). Without this an in-place bind restore can't recover - // the host path and would write to an orphan named volume. - volumes: volumes.map((v) => ({ key: v.key, name: v.name, source: v.source, type: v.type })), - }; + // tar-mode ONLY: disk paths of the stack dir files to stream into the helper (no base64, no + // cap). Filled in the stack block below when stackDirTar is set; empty otherwise. + const stackDirStreamSources: Array<{ diskPath: string; archivePath: string }> = []; + // Writes metadata.json plus, for a stack, a light file listing (path + size, no content) + // of the stack dir. The stack dir's bytes are captured by the helper's HOST bind mount + // (the synthetic __dockhand_stackdir__ volume from planStackDirVolume), not here. + // + // metadata.json is the typed SnapshotLayout (snapshot-layout.ts): capture builds it, + // restore parses it, drift = compile error. Accumulate the stack half + secrets + + // container inspect into locals, then buildSnapshotLayout via writeLayout(). `source` + // carries each volume's ORIGINAL location (bind -> absolute host path, not the slug) so + // an in-place bind restore can rebind it. + const layoutVolumes = volumes.map((v) => ({ key: v.key, name: v.name, source: v.source, type: v.type })); + const backupTime = new Date().toISOString(); // Capture each target container's config for reference (recreate-as-is). + let containerInspect: unknown; if (type === 'container') { try { const [c] = (await resolveTargets(type, targetName, envId)).containers; - if (c) metadata.container = await inspectContainer(c.id, envId ?? undefined); + if (c) containerInspect = await inspectContainer(c.id, envId ?? undefined); } catch { /* metadata best-effort; the volume data is what matters */ } } - files.push({ path: 'metadata.json', contentBase64: Buffer.from(JSON.stringify(metadata, null, 2)).toString('base64') }); + // The single metadata.json writer. Called now (so the file is files[0], present even if + // the stack walk below fails) and again at the end once the stack half is filled in. + let stackInfo: SnapshotStack | undefined; + let stackSecrets: SnapshotSecret[] = []; + const writeLayout = () => { + const layout = buildSnapshotLayout({ + type, targetName, environmentId: envId ?? null, backupTime, + volumes: layoutVolumes, + container: containerInspect, + stack: stackInfo ? { ...stackInfo, secrets: stackSecrets } : undefined, + }); + const entry = { path: 'metadata/metadata.json', contentBase64: Buffer.from(serializeLayout(layout)).toString('base64') }; + if (files.length === 0) files.push(entry); else files[0] = entry; + }; + writeLayout(); - // For a stack, capture its files so it can be redeployed as-is. compose.yaml - // and .env live at stacks// (restore/redeploy read them from there); the - // ENTIRE stack directory is also captured under stacks//stackfiles/ so - // sibling config a stack depends on (nginx.conf, html/, certs, …) survives — - // not just the compose file. + // For a stack, record its stack half in metadata (composeFileName, file listing, + // secrets). The whole stack directory's bytes - compose, .env, and sibling config + // (nginx.conf, html/, certs, ...) - are captured by the helper's host bind mount at + // /volumes/__dockhand_stackdir__, not here; this block only writes the metadata. if (type === 'stack') { const { getStackComposeFile } = await import('../stacks'); - const { readFileSync, readdirSync, lstatSync } = await import('fs'); - const { dirname, join, relative, basename } = await import('path'); + const { readFileSync, readdirSync, lstatSync, statSync } = await import('fs'); + const { dirname, join, relative, basename, sep } = await import('path'); const compose = await getStackComposeFile(targetName, envId ?? undefined); if (compose.success && compose.composePath) { - metadata.hasStackFiles = true; - // Record the ORIGINAL compose filename (e.g. immich.yaml / docker-compose.yml) - // so restore redeploys from the real file inside stackfiles/ — reproducing the - // stack dir 1:1. The whole dir (compose + include:d files + sidecars + .env) is - // captured under stackfiles/ below; we do NOT also store a normalized - // compose.yaml/.env copy — that was a duplicate the redeploy never reads. - metadata.composeFileName = basename(compose.composePath); - // Copy the whole stack source directory under stackfiles/. A bounded - // recursive walk (skips symlinks; total capped) so a huge or looping - // tree can't blow up the metadata payload. The payload is base64'd and - // held in heap (~2.4x the bytes) before being tar'd into the helper, so - // the cap is a memory guard, not an arbitrary limit. When it IS hit the - // backup is INCOMPLETE — record that in metadata.json and surface a - // warning so a truncated stack dir never silently looks fully captured. - try { - const stackDir = dirname(compose.composePath); - const MAX_FILES = 5000; - const MAX_TOTAL_BYTES = 250 * 1024 * 1024; - // Cap the file LISTING recorded in metadata.json (relPath + size) so the - // Metadata tab can show the captured files without a browse roundtrip. - // Independent of the capture cap: a stack can be fully captured yet have - // more entries than we want to inline in metadata. - const MAX_LIST = 500; - const fileList: Array<{ path: string; bytes: number }> = []; - let listTruncated = false; - let count = 0; - let bytes = 0; - let truncated = false; - const walk = (dir: string): void => { - for (const entry of readdirSync(dir)) { - if (count >= MAX_FILES || bytes >= MAX_TOTAL_BYTES) { truncated = true; return; } - const abs = join(dir, entry); - let st; - try { st = lstatSync(abs); } catch { continue; } - if (st.isSymbolicLink()) continue; - if (st.isDirectory()) { walk(abs); continue; } - if (!st.isFile()) continue; - try { - const content = readFileSync(abs); + // Presence of `stackInfo` IS hasStackFiles (derived, not a stored flag that could + // disagree with reality). composeFileName is the ORIGINAL name (immich.yaml / + // docker-compose.yml) so restore redeploys from the real file, reproducing the dir + // 1:1. fileList/excludedBindDirs are filled by the LIGHT listing walk below. + stackInfo = { + composeFileName: basename(compose.composePath), + fileList: [], + excludedBindDirs: [], + secrets: [], + }; + + // List the stack dir's files (path + size) for the browse/restore UI. The bytes + // themselves ride the helper's host bind mount at /volumes/__dockhand_stackdir__; + // this walk over Dockhand's local mirror only records the listing (lstat, no content + // read), skipping compose bind dirs that are captured as their own /volumes/. + const { STACKDIR_VOLUME_KEY } = await import('./stackdir-plan'); + { + try { + const { relativeBindDirsFromCompose, isUnderRelDir, isLoadBearingStackFile } = await import('./stackfile-filter'); + const stackDir = dirname(compose.composePath); + const isDirRel = (rel: string): boolean => { try { return lstatSync(join(stackDir, rel)).isDirectory(); } catch { return false; } }; + const excludeRelDirs = compose.content ? relativeBindDirsFromCompose(compose.content, stackDir, isDirRel) : []; + const listed: Array<{ path: string; bytes: number }> = []; + const walkList = (dir: string) => { + let names: string[]; + try { names = readdirSync(dir); } catch { return; } + for (const entry of names) { + if (listed.length >= 5000) return; // runaway guard + const abs = join(dir, entry); const relPath = relative(stackDir, abs); - files.push({ path: `stacks/${targetName}/stackfiles/${relPath}`, contentBase64: content.toString('base64') }); - count++; - bytes += content.length; - if (fileList.length < MAX_LIST) fileList.push({ path: relPath, bytes: content.length }); - else listTruncated = true; - } catch { /* skip an unreadable file; the rest still back up */ } + if (excludeRelDirs.length > 0 && !isLoadBearingStackFile(relPath) && isUnderRelDir(relPath, excludeRelDirs)) continue; + let st; + try { st = lstatSync(abs); } catch { continue; } + if (st.isSymbolicLink()) { if (isLoadBearingStackFile(relPath)) { try { const r = statSync(abs); if (r.isFile()) listed.push({ path: relPath, bytes: r.size }); } catch { /* skip */ } } continue; } + if (st.isDirectory()) { walkList(abs); continue; } + if (st.isFile()) listed.push({ path: relPath, bytes: st.size }); + } + }; + walkList(stackDir); + listed.sort((a, b) => a.path.localeCompare(b.path)); + if (stackInfo) { stackInfo.fileList = listed; stackInfo.excludedBindDirs = excludeRelDirs; } + + if (stackDirTar) { + // TAR mode (direct-remote, no remote_stacks_dir): no host bind exists, so the + // listed files are STREAMED from disk into the helper at + // /volumes/__dockhand_stackdir__/ (O(1) RAM, no cap; restore reads the same + // path via stackDirSource). Bind data dirs are excluded (same as listed) - their + // runtime bytes live on the target host, out of reach; compose/config only. + for (const { path: rel } of listed) { + stackDirStreamSources.push({ + diskPath: join(stackDir, rel), + archivePath: `volumes/${STACKDIR_VOLUME_KEY}/${rel.split(sep).join('/')}`, + }); + } + console.log(`[Backup] stackfiles "${targetName}": TAR mode - streaming ${stackDirStreamSources.length} file(s) into /volumes/${STACKDIR_VOLUME_KEY} from folder ${stackDir}`); + } else { + console.log(`[Backup] stackfiles "${targetName}": captured via HOST bind mount at /volumes/${STACKDIR_VOLUME_KEY} - listed ${listed.length} file(s), ${excludeRelDirs.length} bind dir(s) excluded`); } - }; - walk(stackDir); - // Record the captured file listing (sorted for a stable UI order) so the - // Metadata tab can enumerate exactly what's in the snapshot's stack dir. - if (fileList.length > 0) { - fileList.sort((a, b) => a.path.localeCompare(b.path)); - metadata.stackFilesList = fileList; - if (listTruncated) metadata.stackFilesListTruncated = true; + } catch (e) { + console.warn(`[Backup] stackfiles "${targetName}": listing failed (files still captured via bind):`, e instanceof Error ? e.message : e); } - if (truncated) { - stackFilesTruncated = true; - // Record it durably IN the snapshot (metadata.json) so a truncated - // capture is visible on browse/restore, and log it loudly. An - // over-cap stack dir therefore never silently looks fully captured. - metadata.stackFilesTruncated = true; - metadata.stackFilesCaptured = { files: count, bytes, maxFiles: MAX_FILES, maxBytes: MAX_TOTAL_BYTES }; - console.warn(`[Backup] Stack directory for "${targetName}" exceeded the backup cap (${MAX_FILES} files / ${Math.round(MAX_TOTAL_BYTES / 1024 / 1024)} MB) — only ${count} file(s) / ${bytes} byte(s) captured. The snapshot is INCOMPLETE for redeploy; reduce the stack dir or exclude large sibling files.`); - } - } catch { /* directory walk best-effort — compose/.env already captured */ } + } // Carry the stack's secret env vars IN the snapshot so a restore reproduces a // WORKING stack — secrets and all — even after the source stack (and its DB // rows) are gone. Stored as their at-rest ciphertext (enc:v1:...), encrypted @@ -184,23 +656,25 @@ async function collectMetadata( // Values are NEVER stored plaintext (getStackSecretCiphertexts guarantees it). try { const { getStackSecretCiphertexts } = await import('../db'); - const secrets = await getStackSecretCiphertexts(targetName, envId ?? null); - if (secrets.length > 0) metadata.secrets = secrets; + stackSecrets = await getStackSecretCiphertexts(targetName, envId ?? null); } catch { /* best-effort — a lookup failure just omits secrets from the snapshot */ } - // Rewrite metadata.json with hasStackFiles set. - files[0] = { path: 'metadata.json', contentBase64: Buffer.from(JSON.stringify(metadata, null, 2)).toString('base64') }; + // Rewrite metadata.json now the stack half (files, excludes, secrets) is filled in. + writeLayout(); } } - return { files, stackFilesTruncated }; + return { files, stackDirStreamSources: stackDirStreamSources.length > 0 ? stackDirStreamSources : undefined }; } // A restic helper runner bound to one destination (so no shared mutable state — // each run closes over its own destination). Used by both port factories. function boundRunner(destination: any) { return { - runInHelper: (spec: { script?: string; args?: string[]; binds: string[]; envId: number | null | undefined; name: string; metadataFiles?: MetadataFile[]; onStderr?: (line: string) => void; onStdout?: (line: string) => void }): Promise => - restic.runInHelper(destination, { script: spec.script, args: spec.args ?? [], binds: spec.binds, envId: spec.envId, name: spec.name, metadataFiles: spec.metadataFiles, onStderr: spec.onStderr, onStdout: spec.onStdout, timeout: 'data' }), + // Spread the spec through unchanged (adding only the timeout tier) - do NOT re-list + // fields by hand, or a spec field silently gets dropped here while the caller still + // passes it. + runInHelper: (spec: HelperRunSpec): Promise => + restic.runInHelper(destination, { ...spec, timeout: 'data' }), runLocal: (args: string[], tier?: 'interactive' | 'data'): Promise => restic.runLocal(destination, args, tier), }; } @@ -213,15 +687,15 @@ function backupPorts(destination: any, onProgress?: (status: string, message: st return { resolveTargets: (type, targetName, envId) => resolveTargets(type, targetName, envId), discoverVolumes: (containers, envId, selected) => discoverVolumes(containers, envId, selected), + planStackDirVolume: (targetName, envId, excludedStackFiles) => planStackDirVolume(targetName, envId, excludedStackFiles), stopForBackup: (type, targetName, containers, envId) => stopForBackup(type, targetName, containers, envId), - recordStopIntent: async (key, intent) => { await recordStopIntent(key, intent); }, - clearStopIntent: (key) => clearOperation(key), runInHelper: (spec) => run.runInHelper(spec), runLocal: (args, tier) => run.runLocal(args, tier), collectMetadata, host: () => getHostname(), instanceId: () => getInstanceId(), acquireLiveTarget: (key) => liveLocks.tryAcquire(key), + isCancelling: (configId) => cancellingBackupConfigs.has(configId), serializeDestination: (id, fn) => serializeByRepo(id, fn), // Forward the run's live-progress callback into the operation record so the // SSE route streams it (the record also throttles the durable log writes). @@ -237,7 +711,7 @@ function backupPorts(destination: any, onProgress?: (status: string, message: st // RestorePorts — the real implementation, bound to one destination + access ctx // ============================================================================= /** - * Extract a snapshot's captured stack files (/metadata/stacks//stackfiles/) into + * Extract a snapshot's captured stack files (/volumes/__dockhand_stackdir__) into * `targetPath` under the stacks root, with a path-traversal guard, a data-loss guard * (an empty extraction never wipes the live dir), and an atomic staging-swap so a * mid-copy failure can't leave the dir half-written. Returns false (no-op) when the @@ -251,19 +725,28 @@ async function materialiseStackFiles(destination: any, snapId: string, stackName const { join, dirname, resolve, sep } = await import('path'); const { randomUUID } = await import('crypto'); const { getStacksDir } = await import('../stacks'); - // Use `restic restore --include` (not dump→tar): restore writes a - // DETERMINISTIC path under the target — /metadata/stacks// - // stackfiles/ — so there's no tar-prefix guessing. + const { stackDirSource } = await import('./stackdir-plan'); + // The stack dir is ALWAYS at /volumes/__dockhand_stackdir__ in the snapshot (local bind + // and remote tar both write there), so restore reads ONE deterministic path via + // `restic restore --include` regardless of how the backup captured it. + const { include, extractSub } = stackDirSource(); + // The AUTHORITATIVE compose filename recorded at capture (immich.yaml, etc.). Restore + // must locate the extracted compose by THIS name, not re-guess it by regex - a wrong + // guess yields wrong bind dirs and the overwrite swap then deletes just-restored volume + // data. Best-effort read; older snapshots without it fall back + // to the regex heuristic below. + let recordedComposeName: string | null = null; + try { recordedComposeName = (await getSnapshotMetadata(destination.id, snapId))?.stack?.composeFileName ?? null; } + catch { /* fall back to regex if metadata is unreadable */ } const tmp = mkdtempSync(join(tmpdir(), 'dh-stackfiles-')); - console.log(`[Backup] materialiseStackFiles: "${stackName}" snapshot=${snapId} → extract to ${tmp}, then swap into ${targetPath} (overwrite=${overwrite})`); + console.log(`[Backup] materialiseStackFiles: "${stackName}" snapshot=${snapId} include=${include} compose=${recordedComposeName ?? '(regex fallback)'} -> extract to ${tmp}, then swap into ${targetPath} (overwrite=${overwrite})`); try { - const include = `/metadata/stacks/${stackName}/stackfiles`; const run = await restic.runLocal(destination, ['restore', snapId, '--target', tmp, '--include', include], 'data'); if (run.exitCode !== 0) { console.log(`[Backup] materialiseStackFiles: restic restore failed for "${stackName}": ${run.stderr.trim() || `exit ${run.exitCode}`}`); return false; } - const extractedDir = join(tmp, 'metadata', 'stacks', stackName, 'stackfiles'); + const extractedDir = join(tmp, ...extractSub.split('/')); // DATA-LOSS guard: only proceed if extraction actually produced files. const extracted = existsSync(extractedDir) ? readdirSync(extractedDir) : []; if (extracted.length === 0) { @@ -292,6 +775,33 @@ async function materialiseStackFiles(destination: any, snapId: string, stackName for (const existing of readdirSync(resolvedTarget)) { cpSync(join(resolvedTarget, existing), join(staging, existing), { recursive: true, force: true }); } + } else { + // OVERWRITE (in-place restore): the stackdir capture EXCLUDES compose bind dirs + // (./data, ./config) — they ride their own /volumes/ and the volume swap + // (which ran BEFORE this) already restored them into the live stack dir. So the + // whole-dir replace must PRESERVE those live bind dirs, or it would delete the + // data the volume swap just restored. Parse the EXTRACTED compose for bind dirs + // and carry the matching live subdirs into staging. + try { + const { relativeBindDirsFromCompose } = await import('./stackfile-filter'); + const { readFileSync, lstatSync } = await import('fs'); + // Locate the compose by its RECORDED name (authoritative); only fall back to + // the regex heuristic for older snapshots that didn't store composeFileName. + const composeFile = (recordedComposeName && extracted.includes(recordedComposeName)) + ? recordedComposeName + : (extracted.find((e) => /^(docker-)?compose\.ya?ml$/i.test(e)) ?? extracted.find((e) => /\.ya?ml$/i.test(e))); + const composeText = composeFile ? readFileSync(join(extractedDir, composeFile), 'utf8') : ''; + const isDirRelLive = (rel: string): boolean => { try { return lstatSync(join(resolvedTarget, rel)).isDirectory(); } catch { return false; } }; + const bindDirs = composeText ? relativeBindDirsFromCompose(composeText, resolvedTarget, isDirRelLive) : []; + for (const rel of bindDirs) { + const liveBind = join(resolvedTarget, rel); + if (existsSync(liveBind)) { + mkdirSync(dirname(join(staging, rel)), { recursive: true }); + cpSync(liveBind, join(staging, rel), { recursive: true, force: true }); + } + } + if (bindDirs.length > 0) console.log(`[Backup] materialiseStackFiles: preserved ${bindDirs.length} bind dir(s) across the stackdir swap: [${bindDirs.join(', ')}]`); + } catch (e) { console.warn(`[Backup] materialiseStackFiles: bind-dir preservation skipped:`, e instanceof Error ? e.message : e); } } for (const entry of extracted) { cpSync(join(extractedDir, entry), join(staging, entry), { recursive: true, force: true }); @@ -344,23 +854,31 @@ function restorePorts(destination: any, access: { isEnterprise: boolean; canAcce return { restart: async () => { await startStack(stackName, envId ?? undefined).catch(() => {}); } }; }, runInHelper: (spec) => run.runInHelper(spec), - resolveVolumeBind: async (destinationId, snapshotId, _envId, volumeName) => { - // Read the snapshot metadata and resolve the volume's ORIGINAL live - // location. For a bind this recovers the absolute host path; if it - // can't, resolveBindFromMetadata throws rather than slug-binding into - // an orphan named volume (silent data loss). - const metadata = await getSnapshotMetadata(destinationId, snapshotId); - return resolveBindFromMetadata(metadata, volumeName); + resolveVolumeBind: async (metadata, _envId, volumeName) => { + // Resolve the volume's ORIGINAL live location through the SAME resolver the restore + // dialog uses, from the metadata the caller ALREADY read (no second dump). For a bind + // this recovers the absolute host path; an unrecoverable bind is surfaced as + // `unresolved` -> we re-throw rather than slug-bind into an orphan named volume (data loss). + const { resolveRestoreTargets } = await import('./restore-targets'); + const resolved = resolveRestoreTargets({ job: { mode: 'in-place' }, volumes: [volumeName], metadata: metadata as any, stackDir: null }); + const t = resolved.volumes[0]; + // Unresolved (bind source unrecoverable): resolveRestoreTargets caught the throw into + // `unresolved`; re-run the leaf directly to surface the SAME BackupError to the swap. + if (!t) return resolveBindFromMetadata(metadata as any, volumeName); + return { bind: t.bind, include: t.include, source: t.target, type: t.type }; + }, + resolveCloneTargets: async (job, volumes) => { + // Same resolver as the dialog preview - clone destinations resolve to the exact bind + // strings shown to the user. The caller does the volume-creation side effects. + const { resolveRestoreTargets } = await import('./restore-targets'); + const resolved = resolveRestoreTargets({ job, volumes, metadata: null, stackDir: null }); + return resolved.volumes.map((t) => ({ key: t.key, type: t.type, target: t.target, include: t.include, bind: t.bind })); }, acquireLiveTarget: (key) => liveLocks.tryAcquire(key), serializeDestination: (id, fn) => serializeByRepo(id, fn), openOperation: (entityName, environmentId, triggeredBy, cb) => makeOperationHandle('restore', entityName, 0, environmentId, triggeredBy === 'cron' ? 'cron' : 'manual', cb ?? onProgress), notify: (event, payload, envId) => notify(event, payload, envId), - recordSwapIntent: async (key, data) => { await recordOperation({ id: key, kind: 'swap', instanceId: await getInstanceId(), envId: data.envId, startedAt: new Date().toISOString(), snapshotId: data.snapshotId, includes: data.includes, volumeBinds: data.binds }); }, - clearSwapIntent: (key) => clearOperation(key), - recordStopIntent: async (key, intent) => { await recordStopIntent(key, intent); }, - clearStopIntent: (key) => clearOperation(key), helperName: (snapshotId) => `dockhand-restore-${snapshotId.slice(0, 12)}`, // --- post-restore actions ------------------------------------------------- @@ -403,86 +921,102 @@ function restorePorts(destination: any, access: { isEnterprise: boolean; canAcce ); }, // Redeploy from the files captured in the snapshot (restore-as-is), reproducing - // the stack dir 1:1: extract the WHOLE stackfiles/ tree and run `docker compose - // up -d` against the ORIGINAL compose filename — so include:, override files, and - // sibling configs referenced by relative paths resolve, and the file keeps its - // real name (e.g. immich.yaml). The backup always captures the full dir plus the - // recorded compose filename, so there is no normalized-compose.yaml fallback. + // the stack dir 1:1: materialise the captured stack dir (/volumes/__dockhand_stackdir__) + // and run `docker compose up -d` against the ORIGINAL compose filename — so include:, + // override files, and sibling configs referenced by relative paths resolve, and the + // file keeps its real name (e.g. immich.yaml). The backup always captures the full dir + // plus the recorded compose filename, so there is no normalized-compose.yaml fallback. redeployStack: async (name, envId, destId, snapId, restoreSecrets) => { const { redeployStackFromDir, getStackDir } = await import('../stacks'); const { existsSync } = await import('fs'); const { join } = await import('path'); const { upsertStackSource } = await import('../db'); + const { runRedeployStack } = await import('./redeploy-stack-core'); const log = (msg: string) => console.log(`[Restore:redeploy ${name}] ${msg}`); log(`start: env=${envId ?? 'local'} destination=${destId} snapshot=${snapId}`); const meta = await getSnapshotMetadata(destId, snapId); - const composeFileName = meta && typeof meta.composeFileName === 'string' ? meta.composeFileName : ''; + const composeFileName = meta?.stack?.composeFileName ?? ''; if (!composeFileName) { throw new Error('snapshot has no recorded compose filename; cannot redeploy this stack'); } - // Restore the stack's secrets FROM THE SNAPSHOT (opt-in, default on). They are - // stored as at-rest ciphertext encrypted under this instance's key; writing - // them into the TARGET env's DB before redeploy lets redeployStackFromDir's - // normal secret injection pick them up. setStackEnvVars re-runs encrypt(), - // which is a pass-through for an already-encrypted value, so the ciphertext is - // stored verbatim (no double-encryption). On a different instance without the - // matching key these values won't decrypt — the restore UI warns about that. - if (restoreSecrets && Array.isArray(meta?.secrets) && meta.secrets.length > 0) { - const { setStackEnvVars } = await import('../db'); - const secrets = meta.secrets - .filter((s: any) => s && typeof s.key === 'string' && typeof s.value === 'string') - .map((s: any) => ({ key: s.key, value: s.value, isSecret: true })); - if (secrets.length > 0) { - await setStackEnvVars(name, envId ?? null, secrets); - log(`restored ${secrets.length} secret(s) from snapshot → env ${envId ?? 'local'}`); - } - } - - // Materialise the snapshot's stack files into the CANONICAL stack dir - // (stacks///) — where every managed stack lives — and - // bring the stack up FROM THERE, so after a restore (even one that deleted - // the stack completely) it is a normal managed stack on disk, not a - // throwaway /tmp copy (#1329). writeLocalStackFiles carries the path- - // traversal + data-loss + atomic-swap guards; overwrite=true because an - // in-place restore is destructive by design. const stackDir = await getStackDir(name, envId ?? undefined); const destination = await loadDest(destId); - log(`materialising snapshot stackfiles → canonical dir: ${stackDir} (compose: ${composeFileName})`); - const wrote = await materialiseStackFiles(destination, snapId, name, stackDir, true); - if (!wrote || !existsSync(join(stackDir, composeFileName))) { - throw new Error(`could not materialise stack files for "${name}" into ${stackDir}; cannot redeploy`); - } - log(`materialised OK → ${stackDir}`); - - log(`redeploying (docker compose up) from: ${stackDir}`); - const r = await redeployStackFromDir(name, stackDir, composeFileName, envId ?? undefined); - if (!r.success) throw new Error(r.error || 'docker compose up failed'); - log(`redeploy OK`); - - // Register the restored stack as managed (internal) so the UI can - // edit/redeploy it — a complete-delete-then-restore must come back managed. const composePath = join(stackDir, composeFileName); - const envPath = existsSync(join(stackDir, '.env')) ? join(stackDir, '.env') : null; - await upsertStackSource({ - stackName: name, - environmentId: envId ?? null, - sourceType: 'internal', - composePath, - envPath + + // Order matters: materialise -> register(internal) -> deploy. Registration happens + // BEFORE the deploy so a failed `docker compose up` still leaves an editable managed + // stack. See redeploy-stack-core.ts for the invariant + the reasoning. + await runRedeployStack({ + log, + // Restore the stack's secrets FROM THE SNAPSHOT (opt-in, default on). Stored as + // at-rest ciphertext under this instance's key; writing them into the TARGET + // env's DB before redeploy lets redeployStackFromDir's normal secret injection + // pick them up. setStackEnvVars re-runs encrypt(), a pass-through for an already- + // encrypted value (no double-encryption). On a different instance without the key + // these won't decrypt - the restore UI warns about that. + restoreSecrets: async () => { + const snapSecrets = meta?.stack?.secrets ?? []; + if (restoreSecrets && snapSecrets.length > 0) { + const { setStackEnvVars } = await import('../db'); + const secrets = snapSecrets.map((s) => ({ key: s.key, value: s.value, isSecret: true })); + await setStackEnvVars(name, envId ?? null, secrets); + log(`restored ${secrets.length} secret(s) from snapshot → env ${envId ?? 'local'}`); + } + }, + // Materialise into the CANONICAL stack dir (stacks///) so the + // restored stack is a normal managed stack on disk, not a throwaway /tmp copy + // (#1329). Carries the path-traversal + data-loss + atomic-swap guards; + // overwrite=true because an in-place restore is destructive by design. + materialise: async () => { + log(`materialising snapshot stackfiles → canonical dir: ${stackDir} (compose: ${composeFileName})`); + const wrote = await materialiseStackFiles(destination, snapId, name, stackDir, true); + return wrote && existsSync(composePath); + }, + register: async () => { + const envPath = existsSync(join(stackDir, '.env')) ? join(stackDir, '.env') : null; + await upsertStackSource({ stackName: name, environmentId: envId ?? null, sourceType: 'internal', composePath, envPath }); + log(`registered: composePath=${composePath} envPath=${envPath ?? '(none)'}`); + }, + deploy: async () => { + const r = await redeployStackFromDir(name, stackDir, composeFileName, envId ?? undefined); + if (!r.success) throw new Error(r.error || 'docker compose up failed'); + } }); - log(`registered as managed (internal): composePath=${composePath} envPath=${envPath ?? '(none)'}`); }, readSnapshotMetadata: (destId, snapId) => getSnapshotMetadata(destId, snapId), // Materialise the snapshot's captured stack files into Dockhand's LOCAL data - // dir (targetPath) so Dockhand can manage/redeploy the restored stack. The - // files were backed up under /metadata/stacks//stackfiles/; dump that - // subtree as a tar and extract it flat into targetPath. - writeLocalStackFiles: async (destId, snapId, stackName, targetPath, overwrite) => { + // dir (targetPath) so Dockhand can manage/redeploy the restored stack. The stack + // dir is always at /volumes/__dockhand_stackdir__ in the snapshot; materialise + // reads it from there and swaps it into targetPath. + writeLocalStackFiles: async (destId, snapId, stackName, targetPath, overwrite, envId) => { const destination = await loadDest(destId); - return materialiseStackFiles(destination, snapId, stackName, targetPath, overwrite); + const wrote = await materialiseStackFiles(destination, snapId, stackName, targetPath, overwrite); + // Register the materialised stack as managed (internal) so the UI can EDIT and + // redeploy it. Without this the restored stack is EXTERNAL - compose sits on disk + // but the UI offers only start/stop/remove, so "update the compose file... then + // redeploy" is impossible. Only when we know the canonical dir + compose name. + if (wrote && envId !== undefined) { + try { + const { join } = await import('path'); + const { existsSync } = await import('fs'); + const { upsertStackSource } = await import('../db'); + const composeFileName = (await getSnapshotMetadata(destId, snapId))?.stack?.composeFileName; + if (composeFileName && existsSync(join(targetPath, composeFileName))) { + const envPath = existsSync(join(targetPath, '.env')) ? join(targetPath, '.env') : null; + await upsertStackSource({ stackName, environmentId: envId ?? null, sourceType: 'internal', composePath: join(targetPath, composeFileName), envPath }); + } + } catch (e) { + console.log(`[Restore] register-managed failed for "${stackName}": ${e instanceof Error ? e.message : String(e)}`); + } + } + return wrote; + }, + stackDirFor: async (stackName, envId) => { + const { getStackDir } = await import('../stacks'); + return getStackDir(stackName, envId ?? undefined); }, // --- clone (cross-env restore) ----------------------------------------------- volumeExists: async (name, envId) => { @@ -499,20 +1033,29 @@ function restorePorts(destination: any, access: { isEnterprise: boolean; canAcce /** A ResticReader adapter over the shared Restic instance (for snapshots.ts). */ function reader() { - return { runLocal: (dest: any, args: string[], tier?: 'interactive' | 'data') => restic.runLocal(dest, args, tier) }; + return { + runLocal: (dest: any, args: string[], tier?: 'interactive' | 'data', stream?: { onStdout?: (chunk: string) => void; onStderr?: (chunk: string) => void }) => + restic.runLocal(dest, args, tier, stream) + }; } -/** List the volume names present in a snapshot (from `restic ls /volumes/`). */ +/** List the volume names present in a snapshot (from `restic ls /volumes/`). The reserved + * __dockhand_stackdir__ key (the stack dir captured as a volume) is filtered OUT here — it + * is NOT a user data volume, so it must never reach the restore volume picker, "restore + * all", or the in-place swap. The stack dir is restored only via the redeploy path. */ async function listSnapshotVolumes(destination: any, snapshotId: string): Promise { + const { isReservedVolumeKey } = await import('./stackdir-plan'); const run = await restic.runLocal(destination, ['ls', '--json', '--no-lock', snapshotId, '/volumes/']); if (run.exitCode !== 0) return []; const names = new Set(); for (const line of run.stdout.split('\n')) { try { const e = JSON.parse(line.trim()); - if (e?.struct_type === 'node' && typeof e.path === 'string' && e.path.startsWith('/volumes/')) { + // Accept EITHER node discriminator: message_type (current restic) or struct_type + // (deprecated) - a restic bump that drops the old field must not silently return []. + if ((e?.message_type === 'node' || e?.struct_type === 'node') && typeof e.path === 'string' && e.path.startsWith('/volumes/')) { const parts = e.path.split('/'); - if (parts.length >= 3 && parts[2]) names.add(parts[2]); + if (parts.length >= 3 && parts[2] && !isReservedVolumeKey(parts[2])) names.add(parts[2]); } } catch { /* skip non-JSON */ } } @@ -523,22 +1066,6 @@ async function notify(event: string, payload: Record, envId: nu try { await sendEventNotification(event as any, payload as any, envId ?? undefined); } catch { /* never changes the outcome */ } } -/** Persist a durable 'stop' intent so a fresh process can restart a target that - * was stopped for a consistent backup / in-place restore but never brought back - * up (process death between the stop and the in-process restart). */ -async function recordStopIntent(key: string, intent: StopIntent): Promise { - await recordOperation({ - id: key, - kind: 'stop', - instanceId: await getInstanceId(), - envId: intent.envId, - startedAt: new Date().toISOString(), - stopType: intent.type === 'stack' ? 'stack' : 'container', - targetName: intent.targetName, - containers: intent.containers, - }); -} - // ============================================================================= // Public entry points // ============================================================================= @@ -562,52 +1089,119 @@ export async function previewSnapshot( // Cheap metadata presence check (ls the /metadata dir, depth-1). const meta = await restic.runLocal(destination, ['ls', '--json', '--no-lock', snapshotId, '/metadata']); const hasMetadata = meta.exitCode === 0 && meta.stdout.includes('/metadata/metadata.json'); - const hasStackFiles = meta.exitCode === 0 && meta.stdout.includes('/metadata/stacks'); - // Pull the per-volume type (volume|bind) and the backup time from metadata.json so - // the restore UI can show the same volume/bind icons as the backup picker. Best- - // effort: an older snapshot without typed metadata just yields no types. + // Per-volume type (volume|bind) for the restore UI's icons, and the ORIGINAL source per + // volume (named volume -> its name; bind -> its absolute host path) so the clone UI can + // pre-fill a 1:1 destination. hasStackFiles is DERIVED from the typed layout (stack != + // undefined), not a stored flag. sourceSecretKeys is NAMES only (values stay ciphertext). const volumeTypes: Record = {}; - // The ORIGINAL location per volume (named volume → its name; bind → its - // absolute host path), used by the clone UI to pre-fill a 1:1 destination. const volumeSources: Record = {}; + let hasStackFiles = false; let backupTime: string | null = null; let sourceEnvironmentId: number | null = null; - // Names (never values) of the secret env vars carried IN the snapshot for this stack. - // The restore UI lists these under the "restore secrets from this backup" toggle so - // the operator sees exactly which secrets come back. The VALUES live in the snapshot - // as ciphertext (never exposed here) and are written to the DB by redeployStack. let sourceSecretKeys: string[] = []; let mdType: string | null = null; let mdTargetName: string | null = null; if (hasMetadata) { const dump = await restic.runLocal(destination, ['dump', '--no-lock', snapshotId, '/metadata/metadata.json']); - if (dump.exitCode === 0) { - try { - const md = JSON.parse(dump.stdout); - backupTime = md?.backupTime ?? null; - sourceEnvironmentId = typeof md?.environmentId === 'number' ? md.environmentId : null; - mdType = typeof md?.type === 'string' ? md.type : null; - mdTargetName = typeof md?.targetName === 'string' ? md.targetName : null; - if (Array.isArray(md?.secrets)) { - sourceSecretKeys = md.secrets - .map((s: any) => s?.key) - .filter((k: unknown): k is string => typeof k === 'string'); - } - for (const v of md?.volumes ?? []) { - // Key by v.key — that's what listSnapshotVolumes returns (the /volumes/ - // segment) and what the restore UI indexes on. For a bind, key !== name - // (name is the host source), so keying on name left the UI unable to find - // the bind's type/source → it fell back to 'volume' and mislabelled binds. - const k = v?.key ?? v?.name; - if (k && (v.type === 'volume' || v.type === 'bind')) volumeTypes[k] = v.type; - if (k && typeof v.source === 'string') volumeSources[k] = v.source; - } - } catch { /* leave types empty on malformed metadata */ } + const layout = dump.exitCode === 0 ? parseSnapshotLayout(dump.stdout) : null; + if (layout) { + backupTime = layout.backupTime || null; + sourceEnvironmentId = layout.environmentId; + mdType = layout.type; + mdTargetName = layout.targetName; + hasStackFiles = layout.stack !== undefined; + sourceSecretKeys = (layout.stack?.secrets ?? []).map((s) => s.key); + for (const v of layout.volumes) { + // Key by v.key — the /volumes/ segment listSnapshotVolumes returns and the + // restore UI indexes on. For a bind key !== name (name is the host source). + volumeTypes[v.key] = v.type; + if (v.source) volumeSources[v.key] = v.source; + } } } return { snapshotId, volumes, volumeTypes, volumeSources, backupTime, sourceEnvironmentId, hasMetadata, hasStackFiles, sourceSecretKeys }; } +/** Per-target resolved destination + whether the host already holds data there. */ +export interface RestoreTargetPreview { + volumes: Array<{ key: string; type: 'bind' | 'volume'; target: string; origin: string; hasData: import('./stackdir-plan').ProbeDataKind }>; + stackFiles: { targetDir: string; willWrite: boolean; hasData: import('./stackdir-plan').ProbeDataKind } | null; + unresolved: Array<{ key: string; reason: string }>; + helperOk: boolean; + helperError?: string; +} + +/** + * Resolve the EXACT on-disk targets a restore would write (via the shared resolveRestoreTargets - + * the same computation the real restore uses) and probe each one on the TARGET host to report + * whether it already holds data. Only host paths / volume names cross the boundary here - never + * Config.Env or secrets - so no metadata redaction is needed. `helperOk:false` means the probe + * container can't run at all (image pull/run failed); callers hard-fail because a restore can't + * succeed either. + */ +export async function previewRestoreTargets( + destinationId: number, + snapshotId: string, + jobLike: { + mode: 'in-place' | 'new-location'; + environmentId: number | null; + targetType?: 'container' | 'stack'; + targetName?: string | null; + targetPath?: string | null; + volumeDestinations?: Array<{ volume: string; kind: 'volume' | 'path'; target: string }>; + skipStackFiles?: boolean; + mergeStackFiles?: boolean; + volumes?: string[]; + }, + access: { isEnterprise: boolean; canAccessEnvironment: (id: number) => Promise } = { isEnterprise: false, canAccessEnvironment: async () => true }, +): Promise { + const { resolveRestoreTargets } = await import('./restore-targets'); + const destination = await getBackupDestination(destinationId); + if (!destination) throw new Error('backup destination not found'); + const instanceId = await getInstanceId(); + await guardSnapshotAccess(reader(), destination, instanceId, snapshotId, access); + + const envId = jobLike.environmentId; + const metadata = await getSnapshotMetadata(destinationId, snapshotId); + const allVolumes = await listSnapshotVolumes(destination, snapshotId); + const requested = jobLike.volumes && jobLike.volumes.length > 0 ? jobLike.volumes : allVolumes; + const stackDir = jobLike.targetType === 'stack' && jobLike.targetName + ? await (async () => { const { getStackDir } = await import('../stacks'); return getStackDir(jobLike.targetName!, envId ?? undefined); })() + : null; + + const resolved = resolveRestoreTargets({ job: jobLike, volumes: requested, metadata: metadata as any, stackDir }); + + // Probe each resolved target on the target host. A helper-failed on ANY probe flips helperOk + // off (the helper is per-env; if it can't run once it won't run for the rest). + // Probe every target in PARALLEL - the probes are independent, read-only helper runs, and + // ensureHelperImage is memoized per env so the burst does one inspect not N. Serial per-volume + // spawns were the dominant latency (the sticky modal), especially on hawser where each spawn + // crosses the WS transport. + const volProbes = await Promise.all(resolved.volumes.map(async (v) => ({ + v, probe: v.type === 'volume' ? await probeVolumeData(v.target, envId) : await probeHostPath(v.target, envId), + }))); + const stackProbe = resolved.stackFiles + ? (resolved.stackFiles.willWrite ? await probeHostPath(resolved.stackFiles.targetDir, envId) : { kind: 'empty' as const }) + : null; + + // Collect helperOk AFTER all probes: a helper-failed on ANY probe flips it off. + let helperOk = true; + let helperError: string | undefined; + const noteHelper = (r: { kind: import('./stackdir-plan').ProbeDataKind; reason?: string }) => { + if (r.kind === 'helper-failed') { helperOk = false; if (!helperError) helperError = r.reason; } + return r.kind; + }; + + const volumes: RestoreTargetPreview['volumes'] = volProbes.map(({ v, probe }) => ({ + key: v.key, type: v.type, target: v.target, origin: v.origin, hasData: noteHelper(probe), + })); + const stackFiles: RestoreTargetPreview['stackFiles'] = resolved.stackFiles + ? { targetDir: resolved.stackFiles.targetDir, willWrite: resolved.stackFiles.willWrite, hasData: noteHelper(stackProbe!) } + : null; + + return { volumes, stackFiles, unresolved: resolved.unresolved, helperOk, helperError }; +} + // --- snapshot reads (routes already apply the env guard; we add instance own.) --- async function loadDest(destinationId: number): Promise { @@ -679,13 +1273,17 @@ export async function dumpSnapshotArchive(destinationId: number, snapshotId: str } /** Read the restore metadata.json embedded in a snapshot, or null. */ -export async function getSnapshotMetadata(destinationId: number, snapshotId: string): Promise { +export async function getSnapshotMetadata(destinationId: number, snapshotId: string): Promise { const destination = await loadDest(destinationId); const instanceId = await getInstanceId(); await guardSnapshotAccess(reader(), destination, instanceId, snapshotId, { isEnterprise: false, canAccessEnvironment: async () => true }); const run = await restic.runLocal(destination, ['dump', '--no-lock', snapshotId, '/metadata/metadata.json']); if (run.exitCode !== 0) return null; - try { return JSON.parse(run.stdout); } catch { return null; } + // parseSnapshotLayout returns null on bad JSON / wrong version / bad shape - an explicit + // "unreadable metadata" outcome, not a cascade of undefineds. Callers read TYPED fields. + const layout = parseSnapshotLayout(run.stdout); + if (!layout) console.warn(`[Backup] snapshot ${snapshotId.slice(0, 8)}: metadata.json present but unreadable (bad shape/version)`); + return layout; } /** Diff two snapshots (restic diff — text output). */ @@ -758,7 +1356,7 @@ export async function verifyBackup( opts?: { dataSubset?: string; onProgress?: (message: string) => void }, ): Promise<{ success: boolean; output?: string; error?: string }> { const destination = await loadDest(destinationId); - const r = await checkRepository(reader(), destination, opts?.dataSubset ?? '5%'); + const r = await checkRepository(reader(), destination, opts?.dataSubset ?? '5%', opts?.onProgress); logRepoOp(destination.name, `verify (${opts?.dataSubset ?? '5%'})`, r.ok, r.ok ? { output: r.output } : { error: r.error }); return r.ok ? { success: true, output: r.output } : { success: false, error: r.error }; } @@ -770,14 +1368,19 @@ export type RepoTask = 'unlock' | 'check' | 'prune' | 'stats' | 'repair-index' | /** Run a repo-maintenance task. Returns { success } for the route + activity log. */ export interface RepoStats { totalSize: number; totalFiles: number; snapshots: number } -export async function runRepoTask(destinationId: number, task: RepoTask, opts?: { maxUnused?: string; dataSubset?: string; staleOnly?: boolean }): Promise<{ success: boolean; output?: string; error?: string; stats?: RepoStats }> { +export async function runRepoTask(destinationId: number, task: RepoTask, opts?: { maxUnused?: string; dataSubset?: string; staleOnly?: boolean; onProgress?: (line: string) => void }): Promise<{ success: boolean; output?: string; error?: string; stats?: RepoStats }> { const destination = await loadDest(destinationId); + const onProgress = opts?.onProgress; if (task === 'repair-index' || task === 'repair-snapshots') { const sub = task === 'repair-index' ? 'index' : 'snapshots'; // repair takes the repo lock — serialize against a concurrent backup to the same // repo, and --retry-lock so a legitimately-running cross-instance backup doesn't // make repair fail REPO_LOCKED (parity with prune/forget). - const run = await serializeByRepo(destinationId, () => restic.runLocal(destination, ['repair', sub, '--retry-lock', '5m'], 'data')); + const stream = onProgress && { + onStdout: (c: string) => { for (const l of formatResticLines(c)) onProgress(l); }, + onStderr: (c: string) => { for (const l of formatResticLines(c)) onProgress(l); } + }; + const run = await serializeByRepo(destinationId, () => restic.runLocal(destination, ['repair', sub, '--retry-lock', '5m'], 'data', stream || undefined)); logRepoOp(destination.name, `repair ${sub}`, run.exitCode === 0, { output: run.stdout, error: run.stderr }); return run.exitCode === 0 ? { success: true, output: run.stdout.trim() } : { success: false, error: run.stderr.trim() || 'repair failed' }; } @@ -786,9 +1389,9 @@ export async function runRepoTask(destinationId: number, task: RepoTask, opts?: // unlock: the EXPLICIT user action uses `--remove-all`; an AUTOMATIC caller passes // staleOnly (plain unlock) so it never wipes a live foreign lock on a shared repo. // stats (--no-lock) is read-safe and stays unserialized. - const r = task === 'prune' ? await serializeByRepo(destinationId, () => pruneRepository(reader(), destination, opts?.maxUnused)) - : task === 'check' ? await serializeByRepo(destinationId, () => checkRepository(reader(), destination, opts?.dataSubset)) - : task === 'unlock' ? await serializeByRepo(destinationId, () => unlockRepository(reader(), destination, !opts?.staleOnly)) + const r = task === 'prune' ? await serializeByRepo(destinationId, () => pruneRepository(reader(), destination, opts?.maxUnused, onProgress)) + : task === 'check' ? await serializeByRepo(destinationId, () => checkRepository(reader(), destination, opts?.dataSubset, onProgress)) + : task === 'unlock' ? await serializeByRepo(destinationId, () => unlockRepository(reader(), destination, !opts?.staleOnly, onProgress)) : await repoStats(reader(), destination); logRepoOp(destination.name, task, r.ok, r.ok ? { output: r.output } : { error: r.error }); if (!r.ok) return { success: false, error: r.error }; @@ -838,6 +1441,11 @@ export async function cancelBackup(configId: number): Promise { const prefix = `dockhand-backup-${configId}`; const signalled = await killHelpers(prefix, envId, 'SIGINT'); if (signalled.length === 0) return false; + // Mark this config as user-cancelled so the engine reports the killed helper's + // non-zero exit as "cancelled" instead of a restic failure. Cleared in runBackup's + // finally. (Set even if the run finishes cleanly before the kill lands — a stale + // flag is harmless: it's only read on the failure path of THIS run and cleared at end.) + cancellingBackupConfigs.add(configId); // Force-kill any helper that ignored SIGINT (pathological restic hang), so the // user's cancel resolves regardless. Best-effort, out of band. void (async () => { @@ -846,12 +1454,24 @@ export async function cancelBackup(configId: number): Promise { await new Promise((r) => setTimeout(r, 1000)); try { const alive = (await listContainers(true, envId)).some((c) => c.state === 'running' && signalled.includes(c.id)); - if (!alive) return; + if (!alive) return; // SIGINT worked: restic released its lock cleanly, nothing to clear } catch { return; } } + // SIGINT was ignored -> SIGKILL. That orphans the restic repo lock (its hostname is the + // dead container id, so restic won't age it out as stale for 30 min, and the next backup + // eats ~5 min of --retry-lock). We just killed OUR OWN helper for this repo under the + // per-repo serializer, so a scoped --remove-all is safe (no other op of ours is running, + // and a foreign instance's live lock is protected because it can't be proven ours here - + // but we only reach this after killing our helper, so the lock we clear is the one we orphaned). for (const id of signalled) { try { await dockerFetch(`/containers/${id}/kill`, { method: 'POST' }, envId); } catch { /* gone */ } } + if (config?.destinationId != null) { + try { + const destination = await getBackupDestination(config.destinationId); + if (destination) await serializeByRepo(config.destinationId, () => unlockRepository(reader(), destination, true)); + } catch { /* best-effort: the pre-backup plain unlock + retry-lock is still the fallback */ } + } })(); return true; } @@ -955,6 +1575,7 @@ export async function runBackup(configId: number, triggeredBy: 'cron' | 'manual' return await runBackupRegistered(configId, triggeredBy, onProgress); } finally { runningBackupConfigs.delete(configId); + cancellingBackupConfigs.delete(configId); } } @@ -972,18 +1593,10 @@ async function runBackupRegistered(configId: number, triggeredBy: 'cron' | 'manu environmentId: config.environmentId ?? null, destinationId: config.destinationId, allVolumes: config.allVolumes ?? true, - selectedVolumes: config.selectedVolumes ? JSON.parse(config.selectedVolumes) : null, + selectedVolumes: parseSelectedVolumes(config.selectedVolumes), stopBeforeBackup: config.stopBeforeBackup ?? false, retention: config.retention ?? null, - options: { - excludePatterns: typeof opts.excludePatterns === 'string' ? opts.excludePatterns.split(',').map((s: string) => s.trim()).filter(Boolean) : undefined, - excludeCaches: opts.excludeCaches, - compression: opts.compression, - limitUpload: opts.limitUpload, - limitDownload: opts.limitDownload, - webhookSuccess: typeof opts.webhookSuccess === 'string' ? opts.webhookSuccess : undefined, - webhookFailure: typeof opts.webhookFailure === 'string' ? opts.webhookFailure : undefined, - }, + options: buildJobOptions(opts), helperName: `dockhand-backup-${config.id}`, }; @@ -1000,119 +1613,14 @@ export async function runRestore( const destination = await getBackupDestination(job.destinationId); if (!destination) return { status: 'error', code: 'VALIDATION', error: 'backup destination not found' }; + // Thread the destination's RESTORE flags (parsed from the split flags column, validated + // against the restore allowlist) into the restore command. This reaches ALL restore + // modes (in-place/clone/new-location) via the build*Restore flags param. Backup flags stay OUT of restore (restic.ts's backupFlagsForCommand + // returns nothing for the restore command). + const restoreFlags = sanitizeRestoreFlags(parseBackupFlags(destination.flags).restore); + // A local-path repo is allowed on any env: the restore helper (on the target // daemon) fails loud via the localRepoGuard if the repo isn't visible there // (wrong-host mount), so no silent bad restore — see restic-script.ts. - return new RestoreService(restorePorts(destination, access, onProgress)).run(job, 'manual'); -} - -// ============================================================================= -// Startup reconciliation — ONE pass, fixed order -// ============================================================================= - -/** - * On startup, recover interrupted operations, in a fixed order so a container is - * never restarted onto a half-swapped volume: - * 1. roll back any interrupted volume swap (from the durable journal); - * 2. restart any target left stopped for a backup / in-place restore whose - * in-process restart never ran (process died between stop and restart); - * 3. mark any operation left `running` by a dead process as terminal (scrub). - * - * Step 2 MUST run after step 1 so a container is never restarted onto a - * half-swapped volume. Returns the counts for each recovery step. - */ -export async function reconcileOnStartup(): Promise<{ swapsRolledBack: number; containersRestarted: number; opsScrubbed: number }> { - const instanceId = await getInstanceId(); - let swapsRolledBack = 0; - - // 1. interrupted restore swaps (kind 'swap' in the journal). The recovery is a - // pure shell rollback inside the volume — it does NOT touch the restic repo, - // so it needs no real destination (empty repo/password; only the volume binds - // and env matter). - const swaps = await listOperations('swap', instanceId).catch(() => []); - const recoveryDest = { repository: '', password: '', envVars: null, flags: null, hostPath: null } as any; - for (const rec of swaps) { - try { - const includes = rec.includes ?? []; - const binds = rec.volumeBinds ?? []; - if (includes.length === 0) { await clearOperation(rec.id); continue; } - await restic.runInHelper(recoveryDest, { - script: buildSwapRecoveryScript(includes), - args: [], - binds, - envId: rec.envId ?? undefined, - name: `dockhand-restore-recover-${rec.id.slice(-8)}`, - timeout: 'data', - }); - await clearOperation(rec.id); - swapsRolledBack++; - } catch { - // Leave the record for a later startup to retry. - } - } - - // 2. restart targets stopped for a backup / in-place restore whose restart - // never ran. AFTER the swap rollback above (never restart onto a half-swapped - // volume). Each intent is best-effort: a failure leaves the row for the next - // startup and never aborts the pass. - let containersRestarted = 0; - const stopIntents = await listOperations('stop', instanceId).catch(() => []); - const { startContainer, inspectContainer } = await import('../docker'); - const { startStack, listComposeStacks } = await import('../stacks'); - - // Whether the intent's target still exists. A stop intent for a target that has - // since been DELETED (user removed the container, or a test tore it down) can - // never be satisfied by a restart — startContainer/startStack would 404 forever - // and the row would be retried on every startup, accumulating stale intents - // indefinitely (observed: a 4-day-old intent for a long-gone container). We only - // treat a 404 (target genuinely absent) as "gone"; any other error (env offline, - // daemon down) is transient, so the intent is kept for a later retry. - async function targetExists(action: RestartAction): Promise { - try { - if (action.type === 'stack') { - const stacks = await listComposeStacks(action.envId ?? undefined); - return stacks.some((s) => s.name === action.targetName); - } - // container: exists if ANY recorded container id still inspects. - for (const c of action.containers) { - try { - await inspectContainer(c.id, action.envId ?? undefined); - return true; - } catch (err) { - if (!isTargetGoneError(err)) throw err; // transient — bubble up, keep intent - // 404 → this id is gone; check the next recorded container - } - } - return false; // all recorded ids 404'd → target gone - } catch { - // Couldn't determine existence (env unreachable etc.) — assume it may - // still exist so we don't drop a live intent; leave it for a later pass. - return true; - } - } - - for (const action of planStopRecovery(stopIntents)) { - // Drop intents whose target no longer exists — a restart can never succeed. - if (!(await targetExists(action))) { - await clearOperation(action.intentId).catch(() => {}); - continue; - } - try { - if (action.type === 'stack') { - const r = await startStack(action.targetName, action.envId ?? undefined); - if (!r.success) throw new Error(r.error || 'stack start failed'); - } else { - for (const c of action.containers) await startContainer(c.id, action.envId ?? undefined); - } - await clearOperation(action.intentId); - containersRestarted++; - } catch { - // Leave the record for a later startup to retry. - } - } - - // 3. scrub stale running operations. - const opsScrubbed = await scrubInterruptedOperations().catch(() => 0); - - return { swapsRolledBack, containersRestarted, opsScrubbed }; + return new RestoreService(restorePorts(destination, access, onProgress)).run({ ...job, restoreFlags }, 'manual'); } diff --git a/src/lib/server/backups/put-archive-stream.ts b/src/lib/server/backups/put-archive-stream.ts new file mode 100644 index 0000000..06e0355 --- /dev/null +++ b/src/lib/server/backups/put-archive-stream.ts @@ -0,0 +1,102 @@ +/** + * put-archive-stream.ts — a BACKUP-ONLY streaming upload to a container's PUT /archive. + * + * WHY a separate function (not the shared putContainerArchive): the shared one has three + * callers (file-browser upload, the files-upload API, and backup). It routes through + * `dockerFetch`, whose plain-HTTP path uses global `fetch` (undici) which BUFFERS the whole + * request body in RAM (verified: 200 MB file -> 239 MB RSS). For the backup helper we want + * to stream a large stack-dir tar with O(1) RAM, so we go straight to node http/https core + * (verified: 200 MB -> 43 MB RSS). Keeping this backup-only means the file-upload paths are + * completely untouched. + * + * TRANSPORTS: + * - socket / plain-http (direct, hawser-standard) -> node http.request, pipe the tar in. + * - https / mTLS (direct, hawser-standard TLS) -> httpsAgentRequest (its ReadableStream + * body branch streams via req.write). + * - hawser-edge -> CANNOT stream (its WS protocol frames + * the whole base64 tar). canStream() + * returns false; the caller falls back + * to the buffered putContainerArchive. + */ +import http from 'node:http'; +import { Readable } from 'node:stream'; +import type { ReadableStream as NodeReadableStream } from 'node:stream/web'; +import { getDockerConfig, httpsAgentRequest, putContainerArchive, type DockerClientConfig } from '../docker'; + +// Pure transport predicates + the idle watchdog live in an import-light module so they can be +// unit-tested without pulling docker.ts (and thus better-sqlite3). +import { transportCanStream, sanitizeArchivePath, withIdleWatchdog } from './put-archive-transport'; +export { transportCanStream, sanitizeArchivePath, withIdleWatchdog }; + +/** PUT a streaming tar into a container over a unix socket or plain HTTP, via node + * http.request (streams the piped body — unlike undici fetch, which buffers). */ +function putViaNodeHttp(config: DockerClientConfig, containerId: string, archivePath: string, tar: Readable, extraHeaders: Record): Promise { + return new Promise((resolve, reject) => { + const path = `/containers/${containerId}/archive?path=${encodeURIComponent(sanitizeArchivePath(archivePath))}`; + const reqOptions: http.RequestOptions = config.type === 'socket' + ? { socketPath: config.socketPath, path, method: 'PUT', headers: { 'Content-Type': 'application/x-tar', ...extraHeaders } } + : { host: config.host, port: config.port, path, method: 'PUT', headers: { 'Content-Type': 'application/x-tar', ...extraHeaders } }; + + const req = http.request(reqOptions, (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => { + const status = res.statusCode ?? 0; + if (status >= 200 && status < 300) resolve(); + else reject(new Error(`put-archive-stream: PUT /archive -> ${status} ${Buffer.concat(chunks).toString().slice(0, 200)}`)); + }); + res.on('error', reject); + }); + req.on('error', reject); + // Idle watchdog: abort if the target stops reading (backpressure stalls the tar) so a + // half-dead agent can't hang the backup up to the 6h helper ceiling. + const guarded = withIdleWatchdog(tar, 'put-archive-stream'); + guarded.on('error', (e) => req.destroy(e)); + guarded.pipe(req); // streams: node writes chunks as the tar produces them, O(1) RAM + }); +} + +/** + * Stream a tar (a web ReadableStream, e.g. from modern-tar's createTarPacker) into a + * container's PUT /archive with O(1) RAM. Returns false WITHOUT uploading if the env's + * transport can't stream (hawser-edge) — the caller must then use the buffered path. + */ +export async function putContainerArchiveStreaming( + containerId: string, + archivePath: string, + tar: ReadableStream, + envId: number | null | undefined, +): Promise<{ streamed: boolean }> { + const config = await getDockerConfig(envId ?? undefined); + if (!transportCanStream(config)) { + console.log(`[Backup] put-archive-stream: env ${envId} is ${config.connectionType} (cannot stream) — caller should use the buffered path`); + return { streamed: false }; + } + + const extraHeaders: Record = {}; + if (config.connectionType === 'hawser-standard' && config.hawserToken) { + extraHeaders['X-Hawser-Token'] = config.hawserToken; + } + + console.log(`[Backup] put-archive-stream: streaming tar to container ${containerId.slice(0, 12)} via ${config.type} (${config.connectionType ?? 'local'})`); + + if (config.type === 'https') { + // httpsAgentRequest streams a ReadableStream body (req.write per chunk) and, seeing a + // stream body, skips the 30s idle timeout meant for small requests - a multi-GB tar PUT + // can legitimately stall >30s (slow disk, mTLS backpressure, GC) without being dead. Run it + // through the idle watchdog (node Readable) so a truly-stalled target still aborts. + const guardedWeb = Readable.toWeb(withIdleWatchdog(Readable.fromWeb(tar as unknown as NodeReadableStream), 'put-archive-stream')) as unknown as ReadableStream; + const res = await httpsAgentRequest(config, `/containers/${containerId}/archive?path=${encodeURIComponent(sanitizeArchivePath(archivePath))}`, + { method: 'PUT', headers: { 'Content-Type': 'application/x-tar', ...extraHeaders }, body: guardedWeb }, false, extraHeaders); + if (!res.ok) throw new Error(`put-archive-stream: PUT /archive -> ${res.status} ${(await res.text()).slice(0, 200)}`); + await res.body?.cancel().catch(() => {}); + return { streamed: true }; + } + + // socket or plain http: node http.request, pipe the tar (converted to a node Readable). + await putViaNodeHttp(config, containerId, archivePath, Readable.fromWeb(tar as unknown as NodeReadableStream), extraHeaders); + return { streamed: true }; +} + +/** Re-exported for the caller's buffered fallback (edge). */ +export { putContainerArchive }; diff --git a/src/lib/server/backups/redeploy-stack-core.ts b/src/lib/server/backups/redeploy-stack-core.ts new file mode 100644 index 0000000..2b0906b --- /dev/null +++ b/src/lib/server/backups/redeploy-stack-core.ts @@ -0,0 +1,35 @@ +// Restore-redeploy orchestration, extracted so the STEP ORDER is unit-testable without the +// real port's IO. Invariant: register(internal) runs BEFORE deploy and is NOT gated on the +// deploy succeeding - a failed `docker compose up` must still leave an editable managed stack. + +export interface RedeployStackSteps { + /** Restore snapshot secrets into the target env's DB (no-op when none / opted out). */ + restoreSecrets(): Promise; + /** Materialise compose/.env into the canonical stack dir. false -> hard error (nothing to deploy). */ + materialise(): Promise; + /** Register the stack as managed (internal) so the UI can edit/redeploy it. */ + register(): Promise; + /** `docker compose up` from the canonical dir. Throwing means the deploy failed. */ + deploy(): Promise; + log?(msg: string): void; +} + +/** Order: restoreSecrets -> materialise -> register(internal) -> deploy. materialise false is a + * hard error; a deploy throw propagates but only AFTER register(), so it never costs the stack. */ +export async function runRedeployStack(steps: RedeployStackSteps): Promise { + const log = steps.log ?? (() => {}); + + await steps.restoreSecrets(); + + const wrote = await steps.materialise(); + if (!wrote) throw new Error('could not materialise stack files; cannot redeploy'); + log('materialised OK'); + + // Register BEFORE deploy - see the module header. This line order is the whole point. + await steps.register(); + log('registered as managed (internal)'); + + log('redeploying (docker compose up)'); + await steps.deploy(); + log('redeploy OK'); +} diff --git a/src/lib/server/backups/stackfile-filter.ts b/src/lib/server/backups/stackfile-filter.ts new file mode 100644 index 0000000..19cbf74 --- /dev/null +++ b/src/lib/server/backups/stackfile-filter.ts @@ -0,0 +1,181 @@ +/** + * stackfile-filter.ts — decide which entries under a stack directory belong in the + * metadata "stackfiles/" capture, and which are bind-mounted DATA that restic already + * backs up separately as a volume. + * + * A common stack layout keeps the bind data inside the stack dir: + * + * stack/ + * ├── compose.yaml <- config, MUST stay in stackfiles/ (redeploy reads it) + * ├── .env <- config, MUST stay + * ├── config/ <- bind-mounted -> restic volume, exclude + * └── data/ <- bind-mounted -> restic volume, exclude + * + * We skip any DIRECTORY the compose file bind-mounts from inside the stack dir: it is already + * backed up as its own restic volume, so capturing it here too would double-back-up and blow the + * capture cap, marking the snapshot INCOMPLETE. + * + * WHY WE PARSE THE COMPOSE, NOT the docker Mount.Source: + * The walk reads the stack dir in DOCKHAND's namespace (`/app/data/stacks/...`), but a + * bind's `Mount.Source` is the DAEMON HOST path (`/docker/data/...` or `/tmp/...`) which + * a containerized Dockhand cannot see or stat — so keying the exclusion on Mount.Source + * silently no-ops for every Dockerized install. The compose file, which Dockhand always + * has locally, declares binds RELATIVE to the stack dir (`./data`) — the same namespace + * the walk already computes `relative(stackDir, abs)` in. So we derive the excluded dirs + * from the compose text and compare on relative paths: namespace-agnostic, works + * containerized / on-host / remote alike. + * + * Hard rules (adversarial review): + * - NEVER exclude compose/`.yaml`/`.yml`/`.env` (`isLoadBearingStackFile`) — a whole-dir + * bind `./:/app` must not strand them or restore can't redeploy. + * - Exclude only DIRECTORY binds — a single-FILE bind (`./cfg.yaml:/x`) is config kept. + * - Named volumes (`type: volume`, or a non-relative source) are NEVER excluded. + * - A `${VAR}` in a bind path is left CAPTURED (conservative over-capture, never drops + * a needed file). + * + * Pure and dependency-light (js-yaml is already a repo dep); the directory check is + * injected so it stays unit-testable. + */ +import yaml from 'js-yaml'; + +/** Strip a trailing slash and a leading `./`; keep the rest. */ +function normRel(p: string): string { + let s = p.trim().replace(/\/+$/, ''); + if (s.startsWith('./')) s = s.slice(2); + return s; +} + +/** A load-bearing stack file that must NEVER be excluded, regardless of a bind that + * happens to cover it: compose/`.yaml`/`.yml` (the compose graph) and `.env*` at any + * level. Data dirs are excluded by the bind rule, not by extension. */ +export function isLoadBearingStackFile(relPath: string): boolean { + const base = relPath.split('/').pop() ?? relPath; + if (base === '.env' || base.startsWith('.env.')) return true; + return /\.ya?ml$/i.test(base); +} + +/** The bind `source` from one compose `volumes:` entry, short or long syntax, or null + * if it isn't a bind we care about (named volume, no source). */ +function bindSourceOf(entry: unknown): string | null { + if (typeof entry === 'string') { + // "src:target[:mode]" — a bind if src is a path (starts with . or /). A bare + // name ("db_data:/x") is a NAMED VOLUME — not a bind, skip. + const src = entry.split(':')[0]; + if (src.startsWith('.') || src.startsWith('/')) return src; + return null; + } + if (entry && typeof entry === 'object') { + const e = entry as { type?: string; source?: string }; + if (e.type === 'volume') return null; // named volume + if (typeof e.source === 'string') { + if (e.type === 'bind' || e.source.startsWith('.') || e.source.startsWith('/')) return e.source; + } + } + return null; +} + +/** + * Directory bind sources declared in the compose file, expressed as paths RELATIVE to + * the stack dir — e.g. `['data', 'config']`. `stackDir` is used only to relativize an + * absolute bind that points into the stack dir. `isDirRel(rel)` decides whether a given + * stack-dir-relative path is a directory (in the walk's own namespace) — a single-file + * bind is NOT returned. `${VAR}` sources are dropped (left to be captured). + */ +export function relativeBindDirsFromCompose( + composeContent: string, + stackDir: string, + isDirRel: (rel: string) => boolean +): string[] { + let doc: unknown; + try { doc = yaml.load(composeContent); } catch { return []; } + const services = (doc as { services?: Record } | null)?.services; + if (!services || typeof services !== 'object') return []; + + const rels = new Set(); + for (const svc of Object.values(services)) { + const vols = (svc as { volumes?: unknown[] } | null)?.volumes; + if (!Array.isArray(vols)) continue; + for (const entry of vols) { + const src = bindSourceOf(entry); + if (!src) continue; + if (src.includes('${')) continue; // unresolved var -> capture, don't risk dropping + let rel: string; + if (src.startsWith('/')) { + // Absolute bind: keep only if it resolves INSIDE the stack dir. + const r = relativePosix(stackDir, src); + if (r === '' || r.startsWith('..')) continue; + rel = r; + } else { + rel = normRel(src); + if (rel === '' || rel.startsWith('..')) continue; // escapes stack dir + } + // A whole-dir bind (`.`, `./`, or an absolute source == the stack dir) would + // mean "exclude everything" — refuse it: it would strip config sidecars that + // aren't load-bearing (nginx.conf, certs/). Over-capture the whole dir instead + // (safe: nothing lost, compose kept). isLoadBearingStackFile can't rescue a + // non-yaml sidecar, so the only safe move here is to not exclude at all. + if (rel === '.' || rel === '') continue; + if (isDirRel(rel)) rels.add(rel); + } + } + return [...rels]; +} + +/** + * Every RELATIVE bind's path below the stack dir - FILES and DIRS alike (`./config.yaml`, + * `./data`). Unlike relativeBindDirsFromCompose (dirs only, for EXCLUDE), this feeds host-path + * DERIVATION: a single-file bind's daemon-reported mount.Source still pins the stack dir (strip + * the tail), so a bind-less-but-single-file stack resolves without depending on the working_dir + * label (which needs the container to be listable - a timing race under load). Whole-dir binds + * (`.`/`./`) are skipped (the source IS the stack dir; the bind-derived path adds nothing). + */ +export function relativeBindsFromCompose(composeContent: string, stackDir: string): string[] { + let doc: unknown; + try { doc = yaml.load(composeContent); } catch { return []; } + const services = (doc as { services?: Record } | null)?.services; + if (!services || typeof services !== 'object') return []; + + const rels = new Set(); + for (const svc of Object.values(services)) { + const vols = (svc as { volumes?: unknown[] } | null)?.volumes; + if (!Array.isArray(vols)) continue; + for (const entry of vols) { + const src = bindSourceOf(entry); + if (!src || src.includes('${')) continue; + let rel: string; + if (src.startsWith('/')) { + const r = relativePosix(stackDir, src); + if (r === '' || r.startsWith('..')) continue; + rel = r; + } else { + rel = normRel(src); + if (rel === '' || rel.startsWith('..')) continue; + } + if (rel === '.' || rel === '') continue; + rels.add(rel); + } + } + return [...rels]; +} + +/** posix `relative(from, to)` on already-clean absolute paths (no realpath). */ +function relativePosix(from: string, to: string): string { + const a = from.replace(/\/+$/, '').split('/'); + const b = to.replace(/\/+$/, '').split('/'); + let i = 0; + while (i < a.length && i < b.length && a[i] === b[i]) i++; + const up = a.slice(i).map(() => '..'); + return [...up, ...b.slice(i)].join('/'); +} + +/** True when `relPath` IS one of the excluded relative dirs, or lives inside one. + * Segment-aware so `data-backup` doesn't match `data`. */ +export function isUnderRelDir(relPath: string, relDirs: string[]): boolean { + const t = normRel(relPath); + for (const raw of relDirs) { + const d = normRel(raw); + if (t === d) return true; + if (t.startsWith(d + '/')) return true; + } + return false; +} diff --git a/src/lib/server/backups/tar.ts b/src/lib/server/backups/tar.ts index ad59154..364e846 100644 --- a/src/lib/server/backups/tar.ts +++ b/src/lib/server/backups/tar.ts @@ -1,14 +1,15 @@ /** - * Build the tar we put-archive (docker cp) into the backup helper, replacing the - * base64-in-Cmd approach that blew ARG_MAX on large stack files. + * Build the tar we put-archive (docker cp) into the backup helper. * * `modern-tar` is zero-dependency and emits PAX headers, so paths of ANY length - * round-trip (the old hand-rolled USTAR writer threw on names >100 bytes, breaking - * deep stack trees). Zero deps also matters for CI: it installs into a symlinked - * node_modules where libs with transitive deps fail to resolve them. + * round-trip (USTAR caps names at 100 bytes, breaking deep stack trees). Zero deps + * also matters for CI: it installs into a symlinked node_modules where libs with + * transitive deps fail to resolve them. */ -import { packTar } from 'modern-tar'; +import { packTar, createTarPacker } from 'modern-tar'; +import { createReadStream, statSync } from 'node:fs'; +import { Readable } from 'node:stream'; export interface TarEntry { /** Path inside the archive, e.g. "metadata/stacks/foo/compose.yaml". A leading slash is stripped. */ @@ -16,6 +17,53 @@ export interface TarEntry { content: Uint8Array; } +/** A file to stream into the tar lazily from disk (no in-RAM contents). */ +export interface TarFileSource { + /** Absolute path on disk to read from. */ + diskPath: string; + /** Path inside the archive (leading slash stripped). */ + archivePath: string; +} + +/** + * Build a tar as a streaming web ReadableStream, reading each file from disk LAZILY (O(1) + * RAM regardless of total size). Used by the backup helper's streaming put-archive so a + * huge stack dir never buffers in memory. Entries are produced in the given order; the + * consumer (the HTTP request) drives the pace via backpressure. + * + * `inlineEntries` (e.g. the small metadata.json) are written first from memory; then each + * file source is streamed from disk. NOTE: the header size is fixed from statSync but the + * body is read lazily via createReadStream, so a file that GROWS or SHRINKS between stat and + * read makes modern-tar's packer throw ("exceeds given size" / "Size mismatch") and errors + * the whole stream - the run FAILS loudly (it is NOT silently tolerated). This is acceptable + * because the backup helper's stack dir is quiescent (the stack is stopped or the files are + * config); a mid-capture size change is a real problem the caller should see, not hide. + */ +export function buildTarStream(sources: TarFileSource[], inlineEntries: TarEntry[] = [], mtimeSecs = Math.floor(Date.now() / 1000)): ReadableStream { + const packer = createTarPacker(); + const mtime = new Date(mtimeSecs * 1000); + + (async () => { + try { + for (const e of inlineEntries) { + const w = packer.controller.add({ name: e.path.replace(/^\/+/, ''), size: e.content.length, mode: 0o644, mtime, uid: 0, gid: 0, uname: 'root', gname: 'root', type: 'file' }); + await Readable.toWeb(Readable.from(Buffer.from(e.content))).pipeTo(w); + } + for (const s of sources) { + const size = statSync(s.diskPath).size; + const w = packer.controller.add({ name: s.archivePath.replace(/^\/+/, ''), size, mode: 0o644, mtime, uid: 0, gid: 0, uname: 'root', gname: 'root', type: 'file' }); + await Readable.toWeb(createReadStream(s.diskPath)).pipeTo(w); + } + packer.controller.finalize(); + } catch (err) { + // Surface the failure to the consumer by erroring the stream. + packer.controller.error(err); + } + })(); + + return packer.readable; +} + /** Build a tar (any path length via PAX). `mtimeSecs` injectable for deterministic tests. */ export function buildTar(entries: TarEntry[], mtimeSecs = Math.floor(Date.now() / 1000)): Promise { return packTar( diff --git a/src/lib/server/remote-staging-plan.ts b/src/lib/server/remote-staging-plan.ts new file mode 100644 index 0000000..06b8d62 --- /dev/null +++ b/src/lib/server/remote-staging-plan.ts @@ -0,0 +1,40 @@ +/** + * Pure decision logic for staging a stack's files onto a `direct` remote host (no server + * deps - safe to unit test). The IO (helper container + put-archive) lives in + * stage-remote-stackfiles.ts; this module only decides IF and WHERE. + */ + +export interface RemoteStagingPlan { + stage: boolean; + /** Absolute project dir on the remote host (/), when stage is true. */ + projectDir?: string; + reason: string; +} + +/** + * Decide whether a `direct` deploy should stage files onto the remote host, and where. + * + * Staging happens when ALL hold: + * - operation is 'up' (down/stop/start/restart don't re-stage), + * - the env has a non-empty remote_stacks_dir configured, + * - there are stack files to stage. + * Setting remote_stacks_dir is the user's explicit "keep my stacks here on the host", so we + * stage the WHOLE stack dir (compose + .env + sibling config) - not only when the compose has + * relative binds. Relative binds still need it to resolve, but a bind-less stack needs its + * compose on the host just as much: to be manageable and BACKUPABLE (the helper bind-mounts the + * host dir; without the compose there, a backup can't capture it). + * Any miss => stage:false => the deploy takes the exact current code path (no --project-directory). + */ +export function planRemoteStaging(input: { + operation: string; + remoteStacksDir: string | null | undefined; + stackName: string; + composeContent: string; + hasStackFiles: boolean; +}): RemoteStagingPlan { + if (input.operation !== 'up') return { stage: false, reason: `operation ${input.operation} does not stage` }; + if (!input.hasStackFiles) return { stage: false, reason: 'no stack files to stage' }; + const base = typeof input.remoteStacksDir === 'string' ? input.remoteStacksDir.trim().replace(/\/+$/, '') : ''; + if (!base) return { stage: false, reason: 'no remote_stacks_dir configured on the env' }; + return { stage: true, projectDir: `${base}/${input.stackName}`, reason: 'direct env with remote_stacks_dir' }; +} diff --git a/src/lib/server/stacks.ts b/src/lib/server/stacks.ts index 640590e..d494410 100644 --- a/src/lib/server/stacks.ts +++ b/src/lib/server/stacks.ts @@ -41,6 +41,7 @@ import { import { unregisterSchedule } from './scheduler'; import { sendEventNotification } from './notifications'; import { deleteGitStackFiles, parseEnvFileContent } from './git'; +import { isDeletableStackDir } from './stack-delete-guard'; import { cleanPem } from '$lib/utils/pem'; import { rewriteComposeVolumePaths, getHostDataDir } from './host-path'; import { getOrderValue } from './container-labels'; @@ -988,7 +989,8 @@ async function executeLocalCompose( serviceName?: string, build?: boolean, noBuildCache?: boolean, - pullPolicy?: string + pullPolicy?: string, + remoteProjectDir?: string ): Promise { const logPrefix = `[Stack:${stackName}]`; @@ -1142,6 +1144,15 @@ async function executeLocalCompose( const useStdin = finalComposeContent !== composeContent; const args = ['docker', 'compose', '-p', stackName]; + // For a `direct` remote daemon we can't share Dockhand's filesystem, so relative binds + // (./config, ./data) are staged onto the remote host under `remoteProjectDir` and the + // compose is pointed there with --project-directory: compose resolves `./x` against that + // remote path (which need not exist locally) instead of the client cwd, so the daemon + // binds the staged files. undefined = current behavior (no staging, no flag). + if (remoteProjectDir) { + args.push('--project-directory', remoteProjectDir); + } + // Temp file for path-translated override content (cleaned up in finally block) let tempOverridePath: string | undefined; @@ -1169,6 +1180,12 @@ async function executeLocalCompose( args.push('-f', overrideFile); console.log(`${logPrefix} Including override file: ${basename(overrideFile)}`); } + } else if (remoteProjectDir) { + // Internal stack + remote staging: --project-directory repoints compose's base to the + // remote path, which breaks the usual cwd auto-discovery of compose.yaml (that file is + // on Dockhand's host, not under the remote project dir). Point -f at the local file + // explicitly so compose reads it locally while resolving ./binds against the remote dir. + args.push('-f', composeFile); } // else: internal stack without path translation - no -f needed. // Docker Compose auto-discovers compose.yaml + compose.override.yaml from cwd. @@ -1683,6 +1700,34 @@ async function executeComposeCommand( skipVerify: env.tlsSkipVerify ?? false } : undefined; + // A direct daemon shares no filesystem with Dockhand, so relative binds + // (./config, ./data) would resolve to a path the remote daemon can't see and + // Docker would auto-create empty dirs. When the env has a `remote_stacks_dir` + // set AND the compose has relative binds, stage the stack files onto the remote + // host under / and point compose there with --project-directory. + // The plan() decides IF/WHERE (pure, unit-tested); anything it declines takes the + // exact current path (remoteProjectDir stays undefined = no flag, no staging). + let remoteProjectDir: string | undefined; + { + const { getEnvSetting } = await import('./db'); + const { planRemoteStaging } = await import('./remote-staging-plan'); + const remoteStacksDir = await getEnvSetting('remote_stacks_dir', envId ?? undefined); + // Stage the WHOLE local stack dir (compose + relative-bind files like nginx.conf, + // ./data). The local stack dir is config-only - runtime data written by the remote + // container never comes back here - so this can't clobber live data. The tar is + // STREAMED from disk (O(1) RAM), so a large ./data doesn't buffer in memory. + const hasLocalDir = !!(operation === 'up' && workingDir && existsSync(workingDir)); + const plan = planRemoteStaging({ + operation, remoteStacksDir, stackName, composeContent, hasStackFiles: hasLocalDir, + }); + if (plan.stage && plan.projectDir && workingDir) { + const { stageStackDirOnRemote } = await import('./stage-remote-stackfiles'); + const { staged } = await stageStackDirOnRemote(envId!, plan.projectDir, workingDir); + console.log(`[Stack:${stackName}] direct env: staged ${staged} file(s) to ${plan.projectDir} on the remote host (${plan.reason})`); + remoteProjectDir = plan.projectDir; + } + } + return executeLocalCompose( operation, stackName, @@ -1701,7 +1746,8 @@ async function executeComposeCommand( serviceName, build, noBuildCache, - pullPolicy + pullPolicy, + remoteProjectDir ); } @@ -2331,11 +2377,66 @@ export async function downStack( * Remove a stack completely (compose down + delete files + cleanup database) * Uses stack locking to prevent concurrent operations. */ +/** + * Compute exactly which on-disk directories a `removeStack(..., deleteFiles)` would delete, + * WITHOUT deleting anything. The delete-preview endpoint uses this so the confirm modal + * shows the user the same paths the backend will actually remove — one source of truth, no + * frontend/backend drift (the class of bug behind #675). Adopted stacks whose files live + * outside DATA_DIR are reported as `null` (Dockhand never deletes those). + */ +export async function computeStackDeletionPaths( + stackName: string, + envId?: number | null +): Promise<{ stackDir: string | null; gitDir: string | null; sourceType: string | null; namedVolumes: string[] }> { + const stackSource = await getStackSource(stackName, envId); + const stacksDir = getStacksDir(); + + // Named volumes compose created for this stack — exactly what `down --volumes` would + // remove (compose-managed, labeled with the project). Best-effort: a docker/API hiccup + // just yields an empty list (the delete still works; the modal just won't preview them). + let namedVolumes: string[] = []; + try { + const { listVolumes } = await import('./docker.js'); + const vols = await listVolumes(envId); + namedVolumes = vols + .filter((v) => v.labels?.['com.docker.compose.project'] === stackName) + .map((v) => v.name) + .sort(); + } catch { /* best-effort */ } + + let stackDir: string | null = null; + if (stackSource?.composePath) { + const customDir = dirname(stackSource.composePath); + // SAME strict guard as removeStack (#675): strict subdir + basename match. + if (isDeletableStackDir(customDir, stacksDir, stackName) && existsSync(customDir)) { + stackDir = customDir; + } + } + if (!stackDir && !stackSource?.composePath) { + const defaultDir = await findStackDir(stackName, envId) || await getStackDir(stackName, envId); + if (existsSync(defaultDir)) stackDir = defaultDir; + } + + // Git stacks additionally have a cloned repo dir that removeStack deletes. + let gitDir: string | null = null; + const gitStack = await getGitStackByName(stackName, envId); + if (gitStack) { + try { + const { getStackRepoPath } = await import('./git'); + const repoPath = await getStackRepoPath(gitStack.id, gitStack.stackName, gitStack.environmentId); + if (repoPath && existsSync(repoPath)) gitDir = repoPath; + } catch { /* best-effort: no git dir shown if we can't resolve it */ } + } + + return { stackDir, gitDir, sourceType: stackSource?.sourceType ?? null, namedVolumes }; +} + export async function removeStack( stackName: string, envId?: number | null, force = false, - removeVolumes = false + removeVolumes = false, + deleteFiles = true ): Promise { return withStackLock(stackName, async () => { // Get compose file (may not exist for external stacks) @@ -2395,6 +2496,23 @@ export async function removeStack( // Remove any dynamically-spawned child containers not handled by compose await cleanupOrphanStackContainers(stackName, envId, 'remove'); + + // Local stack files ARE deleted below, but only under the DATA_DIR strict guard + // (#675) - Dockhand owns that dir. A direct env's REMOTE staged dir has no such + // guard: remote_stacks_dir is a user path that can hold co-located user data, and + // nothing distinguishes a staged file from the user's own there. So we never delete + // it - a stale compose is safe residue; an rm -rf could wipe user data. + if (deleteFiles && envId != null) { + try { + const { getEnvironment, getEnvSetting } = await import('./db'); + const env = await getEnvironment(envId); + if (env?.connectionType === 'direct') { + const remoteStacksDir = await getEnvSetting('remote_stacks_dir', envId); + const base = typeof remoteStacksDir === 'string' ? remoteStacksDir.trim().replace(/\/+$/, '') : ''; + if (base) console.log(`[Stack:${stackName}] leaving staged files at ${base}/${stackName} on the remote host (not deleting - may hold user data)`); + } + } catch { /* log-only, never blocks removal */ } + } } else { // External stack - remove containers directly in parallel const { removeContainer } = await import('./docker.js'); @@ -2461,16 +2579,10 @@ export async function removeStack( let stackDir: string | null = null; if (stackSource?.composePath) { - // Check if the compose path is within Dockhand's stacks directory + // Only delete if the compose dir is a STRICT SUBDIR of DATA_DIR/stacks/ whose + // basename matches the stack name (#675 guard; see stack-delete-guard.ts). const customDir = dirname(stackSource.composePath); - const resolvedCustomDir = resolve(customDir); - const resolvedStacksDir = resolve(stacksDir); - - // Only delete if the directory is within DATA_DIR/stacks/ (files we created) - // AND the directory basename matches the stack name exactly (for safety) - if (resolvedCustomDir.startsWith(resolvedStacksDir) && - basename(resolvedCustomDir) === stackName && - existsSync(customDir)) { + if (isDeletableStackDir(customDir, stacksDir, stackName) && existsSync(customDir)) { stackDir = customDir; } } @@ -2484,8 +2596,10 @@ export async function removeStack( } } - // Delete the directory if found - if (stackDir) { + // Delete the directory if found — but ONLY when the caller asked to remove files. + // "Remove stack" (deleteFiles=false) leaves the compose/.env/data on disk; "Remove + // stack + files" (default) deletes them. + if (stackDir && deleteFiles) { try { rmSync(stackDir, { recursive: true, force: true }); } catch (err: any) { @@ -2512,19 +2626,20 @@ export async function removeStack( cleanupErrors.push(`env vars: ${err.message}`); } - // If git stack, clean up git stack record + // If git stack, clean up git stack record. The DB record always goes (the stack is + // gone); the cloned repo FILES on disk go only when deleteFiles is set. try { const gitStack = await getGitStackByName(stackName, envId); if (gitStack) { await deleteGitStack(gitStack.id); - await deleteGitStackFiles(gitStack.id, gitStack.stackName, gitStack.environmentId); + if (deleteFiles) await deleteGitStackFiles(gitStack.id, gitStack.stackName, gitStack.environmentId); } // Also cleanup any orphaned git stacks with NULL environment_id for this stack name if (envId !== undefined && envId !== null) { const orphanedGitStack = await getGitStackByName(stackName, null); if (orphanedGitStack) { await deleteGitStack(orphanedGitStack.id); - await deleteGitStackFiles(orphanedGitStack.id, orphanedGitStack.stackName, orphanedGitStack.environmentId); + if (deleteFiles) await deleteGitStackFiles(orphanedGitStack.id, orphanedGitStack.stackName, orphanedGitStack.environmentId); } } } catch (err: any) { diff --git a/src/lib/server/stage-remote-stackfiles.ts b/src/lib/server/stage-remote-stackfiles.ts new file mode 100644 index 0000000..1f4573e --- /dev/null +++ b/src/lib/server/stage-remote-stackfiles.ts @@ -0,0 +1,91 @@ +/** + * Stage a stack's files onto a REMOTE `direct` daemon's host filesystem. + * + * A `direct` env has no agent and no shared FS with Dockhand, so a `docker compose up` + * over TCP resolves relative binds (`./config.yaml`, `./data`) against a path the remote + * daemon can't see - it auto-creates empty dirs. This function puts the stack dir onto the + * remote host under `remoteDir` (compose points there via --project-directory), using a + * throwaway helper container + the Docker put-archive API - the same mechanism the backup + * helper uses to write to a remote host. After this runs, the relative binds resolve. + * + * The tar is STREAMED from disk (O(1) RAM) via the same primitive the backup helper uses: + * a stack dir with a large relative bind (a few-GB ./data) must not buffer in memory. + * DIRECT streams (transportCanStream is true for everything but hawser-edge, which never + * reaches this code - staging is direct-only). + * + * DIRECT ONLY. Socket shares the FS; Hawser has an agent that writes files itself. + */ +import { runContainerWithStreaming } from './docker'; +import { buildTarStream, type TarFileSource } from './backups/tar'; +import { putContainerArchiveStreaming } from './backups/put-archive-stream'; +import { readdirSync } from 'node:fs'; +import { join, relative, sep } from 'node:path'; + +const STAGER_IMAGE = 'alpine:latest'; + +export interface StageResult { + staged: number; + remoteDir: string; +} + +/** Walk a stack dir into TarFileSource[] (disk paths, no in-RAM contents), so the tar + * streams lazily. `.git` is skipped. There is deliberately NO per-file size cap: the tar + * streams from disk with O(1) RAM, so a large relative-bind file (a big ./data config) + * must stage in full - a cap would defeat the whole point of streaming. */ +function collectSources(rootDir: string): TarFileSource[] { + const sources: TarFileSource[] = []; + const walk = (dir: string) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git') continue; + const full = join(dir, entry.name); + if (entry.isDirectory()) { walk(full); continue; } + if (!entry.isFile()) continue; + // archivePath is POSIX-relative to rootDir (tar entries under remoteDir). + sources.push({ diskPath: full, archivePath: relative(rootDir, full).split(sep).join('/') }); + } + }; + walk(rootDir); + return sources; +} + +/** + * Stream the stack dir at `localDir` onto the remote host at `remoteDir` via a helper + * container. Throws on failure so the caller fails loudly instead of deploying against + * empty dirs (a silent no-op deploy). + * + * @param envId the direct env whose daemon hosts the files + * @param remoteDir absolute path on the remote host (validated by the API layer) + * @param localDir the local stack dir to stage (its files land under remoteDir 1:1) + */ +export async function stageStackDirOnRemote( + envId: number, + remoteDir: string, + localDir: string +): Promise { + const sources = collectSources(localDir); + if (sources.length === 0) return { staged: 0, remoteDir }; + + const name = `dockhand-stage-${envId}-${Math.floor(Date.now() / 1000)}`; + + await runContainerWithStreaming({ + image: STAGER_IMAGE, + // The container only needs to exist for the put-archive in beforeStart; the bind mount + // itself creates remoteDir on the remote host, then it exits immediately. + cmd: ['sh', '-c', `mkdir -p '${remoteDir.replace(/'/g, "'\\''")}'`], + binds: [`${remoteDir}:${remoteDir}:rw`], + name, + envId, + timeout: 300000, + labels: { 'io.dockhand.role': 'stackfile-stager' }, + // After create, before start: the bind mount already exists, so a streaming put-archive + // lands the files on the remote host FS under remoteDir with O(1) RAM. + beforeStart: async (containerId: string) => { + const tar = buildTarStream(sources); + const { streamed } = await putContainerArchiveStreaming(containerId, remoteDir, tar, envId); + // direct always streams; a false here would mean a transport we don't stage on. + if (!streamed) throw new Error(`remote staging: transport for env ${envId} cannot stream the stack dir`); + }, + }); + + return { staged: sources.length, remoteDir }; +} diff --git a/src/lib/utils/backup.ts b/src/lib/utils/backup.ts index be6a448..e3cb951 100644 --- a/src/lib/utils/backup.ts +++ b/src/lib/utils/backup.ts @@ -95,6 +95,9 @@ export interface BackupOptions { excludeCaches?: boolean; webhookSuccess?: string; webhookFailure?: string; + /** Stack-dir entries the user DEselected in the "Stack files on the host" picker; excluded + * from the stack-dir capture (never load-bearing compose/.env - the server re-guards). */ + excludedStackFiles?: string[]; } export function parseRetention(retention: string | RetentionPolicy | null | undefined): RetentionPolicy { diff --git a/src/routes/api/backup/restore/+server.ts b/src/routes/api/backup/restore/+server.ts index a55f5f9..0f24d33 100644 --- a/src/routes/api/backup/restore/+server.ts +++ b/src/routes/api/backup/restore/+server.ts @@ -1,6 +1,7 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; import { authorize } from '$lib/server/authorize'; +import { requireBackups } from '$lib/server/backups/route-guards'; import { runRestore } from '$lib/server/backups'; import { validateRestoreRequest, type RestoreMode } from '$lib/server/backups/validate'; import { createJobResponse } from '$lib/server/sse'; @@ -10,9 +11,8 @@ import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards'; export const POST: RequestHandler = async (event) => { const { request, cookies } = event; const auth = await authorize(cookies); - if (auth.authEnabled && !await auth.can('backups', 'manage')) { - return json({ error: 'Permission denied' }, { status: 403 }); - } + const rbacDenied = await requireBackups(auth, 'manage'); + if (rbacDenied) return rbacDenied; const body = await request.json(); const mode: RestoreMode = body.mode === 'in-place' ? 'in-place' : 'new-location'; @@ -25,6 +25,9 @@ export const POST: RequestHandler = async (event) => { // The client sends false to bring the stack up without them. Only a stack redeploy // consumes this; container restores ignore it. const restoreSecrets = body.restoreSecrets !== false; + // New-location STACK restore: skip the captured compose/.env (data-only restore), + // and/or ADOPT the restored stack into Dockhand afterwards (opt-in). Both default off. + const skipStackFiles = body.skipStackFiles === true; // Authorization gates run BEFORE request validation, so an unauthorized caller // never learns anything about the request shape. @@ -76,6 +79,7 @@ export const POST: RequestHandler = async (event) => { postRestore, volumeDestinations, restoreSecrets, + skipStackFiles, }, access, // Stream progress to the client (the restore modal's log). Without this the diff --git a/src/routes/api/backup/stack-dir-listing/+server.ts b/src/routes/api/backup/stack-dir-listing/+server.ts new file mode 100644 index 0000000..18728ad --- /dev/null +++ b/src/routes/api/backup/stack-dir-listing/+server.ts @@ -0,0 +1,32 @@ +import { json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { authorize } from '$lib/server/authorize'; +import { requireBackups } from '$lib/server/backups/route-guards'; +import { probeStackDir } from '$lib/server/backups'; + +// GET /api/backup/stack-dir-listing?target=&env= +// Probe the TARGET host and list the actual contents of the stack directory, so the +// create/edit backup dialog can show the resolved host path + let the user pick which +// stack-dir entries to back up. Returns { kind:'listed', hostPath, entries } or +// { kind:'unknown', reason } (never throws for an operational failure). +export const GET: RequestHandler = async ({ url, cookies }) => { + const auth = await authorize(cookies); + const denied = await requireBackups(auth, 'view'); + if (denied) return denied; + + const targetName = url.searchParams.get('target'); + if (!targetName) return json({ error: 'target is required' }, { status: 400 }); + const envParam = url.searchParams.get('env'); + const environmentId = envParam ? parseInt(envParam) : null; + + if (environmentId != null && auth.isEnterprise && !await auth.canAccessEnvironment(environmentId)) { + return json({ error: 'Environment access denied' }, { status: 403 }); + } + + try { + const result = await probeStackDir(targetName, environmentId); + return json(result); + } catch (e) { + return json({ kind: 'unknown', reason: e instanceof Error ? e.message : 'failed to probe stack directory' }); + } +}; diff --git a/src/routes/api/backup/stack-path/+server.ts b/src/routes/api/backup/stack-path/+server.ts new file mode 100644 index 0000000..4da5c4e --- /dev/null +++ b/src/routes/api/backup/stack-path/+server.ts @@ -0,0 +1,31 @@ +import { json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { authorize } from '$lib/server/authorize'; +import { requireBackups } from '$lib/server/backups/route-guards'; +import { previewStackBackupPath } from '$lib/server/backups'; + +// GET /api/backup/stack-path?target=&env= +// Preview WHERE a stack's directory will be captured from on the host (the resolved host +// bind path), or `unknown` with a reason - so the create-schedule dialog can show it up front +// and warn before scheduling a backup that would hard-fail the stack-dir probe. +export const GET: RequestHandler = async ({ url, cookies }) => { + const auth = await authorize(cookies); + const denied = await requireBackups(auth, 'view'); + if (denied) return denied; + + const targetName = url.searchParams.get('target'); + if (!targetName) return json({ error: 'target is required' }, { status: 400 }); + const envParam = url.searchParams.get('env'); + const environmentId = envParam ? parseInt(envParam) : null; + + if (environmentId != null && auth.isEnterprise && !await auth.canAccessEnvironment(environmentId)) { + return json({ error: 'Environment access denied' }, { status: 403 }); + } + + try { + const result = await previewStackBackupPath(targetName, environmentId); + return json(result); + } catch (e) { + return json({ kind: 'unknown', reason: e instanceof Error ? e.message : 'failed to resolve stack path' }); + } +}; diff --git a/src/routes/api/environments/[id]/remote-stacks-dir/+server.ts b/src/routes/api/environments/[id]/remote-stacks-dir/+server.ts new file mode 100644 index 0000000..d06729c --- /dev/null +++ b/src/routes/api/environments/[id]/remote-stacks-dir/+server.ts @@ -0,0 +1,53 @@ +import { json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { authorize } from '$lib/server/authorize'; +import { getEnvironment, getEnvSetting, setEnvSetting } from '$lib/server/db'; + +// Per-env "remote stacks directory" for `direct` environments. A direct daemon has no +// agent and no shared filesystem with Dockhand, so relative bind files (./config.yaml, +// ./data) never reach it. When this is set, Dockhand stages the stack files onto the +// remote host under this dir before `docker compose up`. Empty/unset = current behavior. +// Stored via the generic per-env settings store (no schema column). + +export const GET: RequestHandler = async ({ params, cookies }) => { + const auth = await authorize(cookies); + if (auth.authEnabled && !(await auth.can('environments', 'view'))) { + return json({ error: 'Permission denied' }, { status: 403 }); + } + const id = parseInt(params.id); + const env = await getEnvironment(id); + if (!env) return json({ error: 'Environment not found' }, { status: 404 }); + + const stored = await getEnvSetting('remote_stacks_dir', id); + const remoteStacksDir = stored && typeof stored === 'string' && stored.trim() !== '' ? stored : null; + return json({ remoteStacksDir }); +}; + +export const POST: RequestHandler = async ({ params, request, cookies }) => { + const auth = await authorize(cookies); + if (auth.authEnabled && !(await auth.can('environments', 'edit'))) { + return json({ error: 'Permission denied' }, { status: 403 }); + } + const id = parseInt(params.id); + const env = await getEnvironment(id); + if (!env) return json({ error: 'Environment not found' }, { status: 404 }); + + const data = await request.json().catch(() => ({})); + const raw = data?.remoteStacksDir; + + // null / '' clears the setting (reverts to current behavior). + if (raw === null || raw === undefined || raw === '') { + await setEnvSetting('remote_stacks_dir', '', id); + return json({ success: true, remoteStacksDir: null }); + } + if (typeof raw !== 'string') { + return json({ error: 'remoteStacksDir must be a string or null' }, { status: 400 }); + } + // Must be an absolute path on the remote host; reject relative or traversal-y input. + const value = raw.trim(); + if (!value.startsWith('/') || value.includes('..')) { + return json({ error: 'remoteStacksDir must be an absolute path with no ".."' }, { status: 400 }); + } + await setEnvSetting('remote_stacks_dir', value, id); + return json({ success: true, remoteStacksDir: value }); +}; diff --git a/src/routes/backups/CreateBackupModal.svelte b/src/routes/backups/CreateBackupModal.svelte index c4a79a7..66fc55f 100644 --- a/src/routes/backups/CreateBackupModal.svelte +++ b/src/routes/backups/CreateBackupModal.svelte @@ -13,6 +13,7 @@ import { getRepoTypeIcon, formatCron, runBackupAction, type BackupFormState } from '$lib/utils/backup'; import { toast } from 'svelte-sonner'; import VolumePicker from '$lib/components/backup/VolumePicker.svelte'; + import StackFilesPicker from '$lib/components/backup/StackFilesPicker.svelte'; import DestinationPicker from '$lib/components/backup/DestinationPicker.svelte'; import { normalizeMounts, volumesForStack, type VolumeInfo } from '$lib/utils/mounts'; @@ -47,6 +48,45 @@ let allVolumes = $state(true); let selectedVolumes = $state([]); + // Stack files on the host (probe listing) — stack targets only. + type StackListing = + | { kind: 'listed'; hostPath: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] } + | { kind: 'tar'; localStackDir: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] } + | { kind: 'helper-failed'; reason: string } + | { kind: 'unknown'; reason: string } + | null; + let stackListing = $state(null); + // The probe helper couldn't run on the target - a backup can't run either, so block Save. + const stackHelperFailed = $derived(stackListing?.kind === 'helper-failed'); + let loadingStackListing = $state(false); + let excludedStackFiles = $state([]); + // The user-set "Remote stack path (for backup)" for direct/hawser envs, shown in the picker + // tooltip so the user sees THEIR configured path (empty when not set / socket env). + let stackRemoteDir = $state(''); + + async function fetchStackListing(item: ContainerItem) { + if (item.type !== 'stack') { stackListing = null; return; } + loadingStackListing = true; + stackListing = null; + excludedStackFiles = []; + stackRemoteDir = ''; + if (item.envId != null) { + fetch(`/api/environments/${item.envId}/remote-stacks-dir`) + .then((r) => (r.ok ? r.json() : null)) + .then((d) => { stackRemoteDir = d?.remoteStacksDir ?? ''; }) + .catch(() => { /* non-fatal - tooltip just omits the configured path */ }); + } + try { + const envQ = item.envId != null ? `&env=${item.envId}` : ''; + const res = await fetch(`/api/backup/stack-dir-listing?target=${encodeURIComponent(item.name)}${envQ}`); + stackListing = await res.json(); + } catch (e) { + stackListing = { kind: 'unknown', reason: e instanceof Error ? e.message : 'probe failed' }; + } finally { + loadingStackListing = false; + } + } + // Step 3: Schedule & Run let saveSchedule = $state(false); let schedule = $state('0 2 * * *'); @@ -126,6 +166,7 @@ function selectSource(item: ContainerItem) { selectedItem = item; step = 2; + fetchStackListing(item); } // Save the config, then (when run=true) hand it off to the parent's shared @@ -141,7 +182,8 @@ destinationId: selectedDestId, stopBeforeBackup, allVolumes, - selectedVolumes + selectedVolumes, + options: excludedStackFiles.length > 0 ? { excludedStackFiles } : undefined }; try { const result = await runBackupAction({ @@ -182,7 +224,7 @@ { if (!isOpen && saving) return; open = isOpen; }}> - + Create backup @@ -316,6 +358,19 @@ + {#if selectedItem?.type === 'stack'} + + {/if} + {#if selectedItem} step = 2} disabled={saving}>Back
{#if saveSchedule} - {/if} - diff --git a/src/routes/containers/BackupPanel.svelte b/src/routes/containers/BackupPanel.svelte index 3dac67d..c794c27 100644 --- a/src/routes/containers/BackupPanel.svelte +++ b/src/routes/containers/BackupPanel.svelte @@ -7,9 +7,10 @@ import { Badge } from '$lib/components/ui/badge'; import { TogglePill } from '$lib/components/ui/toggle-pill'; import CronEditor from '$lib/components/cron-editor.svelte'; - import { Save, Play, Pause, RefreshCw, CheckCircle, XCircle, ChevronDown, Loader2, Plus, Trash2, Pencil, X, Check, HardDrive, Box, Layers, FileText } from 'lucide-svelte'; + import { Save, Play, Pause, RefreshCw, CheckCircle, XCircle, ChevronDown, Loader2, Plus, Trash2, Pencil, X, Check, HardDrive, Box, Layers, FileText, FolderOpen, AlertTriangle } from 'lucide-svelte'; import RotateCwFadingClock from '$lib/components/icons/RotateCwFadingClock.svelte'; import VolumePicker from '$lib/components/backup/VolumePicker.svelte'; + import StackFilesPicker from '$lib/components/backup/StackFilesPicker.svelte'; import DestinationPicker from '$lib/components/backup/DestinationPicker.svelte'; import LogConsole from '$lib/components/LogConsole.svelte'; import SnapshotsPanel from './SnapshotsPanel.svelte'; @@ -23,10 +24,11 @@ import { watchJob } from '$lib/utils/sse-fetch'; import ConfirmPopover from '$lib/components/ConfirmPopover.svelte'; import { getRepoTypeIcon, parseRetention, parseOptions, retentionSummary as getRetentionSummary, formatCron, runBackupAction, classifyJobResult, tagLogLine, fetchBackupExecutions, type BackupAction, type BackupFormState } from '$lib/utils/backup'; + import { reconcileSelectedVolumeKeys } from '$lib/utils/mounts'; interface Props { containerName: string; - volumes: { name: string; mountPoint: string; mountType?: 'volume' | 'bind' }[]; + volumes: { key: string; name: string; mountPoint: string; mountType?: 'volume' | 'bind' }[]; type?: 'container' | 'stack'; environmentId?: number; onConfigSaved?: () => void; @@ -238,9 +240,12 @@ editLimitDownload = opts.limitDownload || ''; editWebhookSuccess = opts.webhookSuccess || ''; editWebhookFailure = opts.webhookFailure || ''; + excludedStackFiles = Array.isArray(opts.excludedStackFiles) ? opts.excludedStackFiles : []; editStopBefore = cfg.stopBeforeBackup; editAllVolumes = cfg.selectedVolumes === null; - editSelectedVolumes = cfg.selectedVolumes ? (typeof cfg.selectedVolumes === 'string' ? JSON.parse(cfg.selectedVolumes) : cfg.selectedVolumes) : []; + const storedSel = cfg.selectedVolumes ? (typeof cfg.selectedVolumes === 'string' ? JSON.parse(cfg.selectedVolumes) : cfg.selectedVolumes) : []; + // Reconcile a stored selection (may hold legacy names) to the current volume keys. + editSelectedVolumes = reconcileSelectedVolumeKeys(volumes, storedSel); showAdvanced = false; } @@ -283,9 +288,11 @@ if (editWebhookFailure !== (opts.webhookFailure || '')) return true; if (editStopBefore !== cfg.stopBeforeBackup) return true; if (editAllVolumes !== (cfg.selectedVolumes === null)) return true; - const origVolumes = cfg.selectedVolumes + const storedSel = cfg.selectedVolumes ? (typeof cfg.selectedVolumes === 'string' ? JSON.parse(cfg.selectedVolumes) : cfg.selectedVolumes) : []; + // Reconcile the baseline to keys too, so a name->key format shift alone isn't seen as dirty. + const origVolumes = reconcileSelectedVolumeKeys(volumes, storedSel); if (JSON.stringify([...editSelectedVolumes].sort()) !== JSON.stringify([...origVolumes].sort())) return true; return false; } @@ -314,7 +321,8 @@ limitUpload: editLimitUpload || undefined, limitDownload: editLimitDownload || undefined, webhookSuccess: editWebhookSuccess || undefined, - webhookFailure: editWebhookFailure || undefined + webhookFailure: editWebhookFailure || undefined, + excludedStackFiles: excludedStackFiles.length > 0 ? excludedStackFiles : undefined } }; } @@ -468,7 +476,53 @@ return `${sel.length} of ${volumes.length} volumes`; } + // Resolved host stack-dir path (stacks only): WHERE the helper will read the stack folder + // from on the target host. Shown in the create form so the user sees what gets captured - + // and a clear warning up front when it can't be located (would hard-fail the backup). + let stackPath = $state<{ kind: 'candidate'; hostPath: string; composeFile: string } | { kind: 'unknown'; reason: string } | null>(null); + async function fetchStackPath() { + if (type !== 'stack') return; + try { + const params = new URLSearchParams({ target: containerName }); + // Use the derived envId (prop OR the currently selected environment), the SAME + // value the rest of the panel uses for save/run - the bare `environmentId` prop is + // often undefined (the modal doesn't pass it), which made the server report + // "No environment specified". + if (envId != null) params.set('env', String(envId)); + const res = await fetch(`/api/backup/stack-path?${params}`); + if (res.ok) stackPath = await res.json(); + } catch { /* non-fatal: the path preview is informational */ } + } + + // Stack files on the host (probe listing) — supersedes the plain stackPath callout for the + // picker, but stackPath is still fetched for the compose-file name shown elsewhere. + type StackListing = + | { kind: 'listed'; hostPath: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] } + | { kind: 'tar'; localStackDir: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] } + | { kind: 'unknown'; reason: string } + | null; + let stackListing = $state(null); + let loadingStackListing = $state(false); + let excludedStackFiles = $state([]); + async function fetchStackListing() { + if (type !== 'stack') { stackListing = null; return; } + loadingStackListing = true; + try { + const params = new URLSearchParams({ target: containerName }); + if (envId != null) params.set('env', String(envId)); + const res = await fetch(`/api/backup/stack-dir-listing?${params}`); + stackListing = await res.json(); + } catch (e) { + stackListing = { kind: 'unknown', reason: e instanceof Error ? e.message : 'probe failed' }; + } finally { + loadingStackListing = false; + } + } + onMount(() => { fetchDestinations(); fetchConfigs(); }); + // Resolve the stack path once envId is known (it's derived from $currentEnvironment, which + // may not be ready at mount). Re-runs if the environment changes. + $effect(() => { if (type === 'stack' && envId != null) { void fetchStackPath(); void fetchStackListing(); } }); + {#if type === 'stack'} + + {/if} + ('socket'); + // Envs that keep stack files on a REMOTE host, so backup needs a declared stack path: + // direct (Dockhand stages there, drives deploy + backup) and hawser (the agent keeps files + // at its STACKS_DIR; the path is backup-only, does not steer deploy). + const usesStackPath = (ct: ConnectionType) => ct === 'direct' || ct === 'hawser-standard' || ct === 'hawser-edge'; + const isHawserConn = (ct: ConnectionType) => ct === 'hawser-standard' || ct === 'hawser-edge'; let formHawserToken = $state(''); let formLabels = $state([]); let newLabelInput = $state(''); @@ -598,6 +606,7 @@ loadImagePruneSettings(environment.id); loadTimezone(environment.id); loadDiskWarningSettings(environment.id); + if (usesStackPath(formConnectionType)) loadRemoteStacksDir(environment.id); // Load Hawser token if edge mode if (formConnectionType === 'hawser-edge') { loadHawserToken(environment.id); @@ -607,6 +616,7 @@ formHost = ''; formPort = 2375; formProtocol = 'http'; + formRemoteStacksDir = ''; formTlsCa = ''; formTlsCert = ''; formTlsKey = ''; @@ -883,6 +893,7 @@ if (newEnv?.id) { await saveTimezone(newEnv.id); await saveDiskWarningSettings(newEnv.id); + if (usesStackPath(formConnectionType)) await saveRemoteStacksDir(newEnv.id); } onSaved(); onClose(); @@ -1019,6 +1030,7 @@ await saveImagePruneSettings(environment.id); await saveTimezone(environment.id); await saveDiskWarningSettings(environment.id); + if (usesStackPath(formConnectionType)) await saveRemoteStacksDir(environment.id); toast.success(`Updated environment: ${formName}`); onSaved(); onClose(); @@ -1049,6 +1061,30 @@ } } + async function loadRemoteStacksDir(envId: number) { + try { + const response = await fetch(`/api/environments/${envId}/remote-stacks-dir`); + if (response.ok) { + const data = await response.json(); + formRemoteStacksDir = data.remoteStacksDir ?? ''; + } + } catch (error) { + console.error('Failed to load remote stacks dir:', error); + } + } + + async function saveRemoteStacksDir(envId: number) { + try { + await fetch(`/api/environments/${envId}/remote-stacks-dir`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ remoteStacksDir: formRemoteStacksDir.trim() || null }) + }); + } catch (error) { + console.error('Failed to save remote stacks dir:', error); + } + } + async function saveDiskWarningSettings(envId: number) { try { await fetch(`/api/environments/${envId}/disk-warning`, { @@ -1607,7 +1643,7 @@ -
+
@@ -1911,6 +1947,75 @@
{/if} + + {#if usesStackPath(formConnectionType)} +
+
+ + + + + + + {#if isHawserConn(formConnectionType)} +
+

Where the agent keeps stack files

+

+ The Hawser agent stores each stack's folder at + <this path>/<stack> + on its own host. Backup reads + the compose and config from there. This does not + change where deploy or restore write - the agent always uses its own + STACKS_DIR; this only tells backup where to look. +

+

+ So set it to MATCH the agent's STACKS_DIR + (default /data/stacks). Leave empty for + the default; set it only if the agent runs with a custom one. +

+
+ {:else} +
+

How this path is used

+

+ A direct daemon shares no filesystem with Dockhand. When set, Dockhand + materialises each stack's whole folder - compose, .env, + and relative-bind files (./config, + ./data) - into + <this path>/<stack> + on the remote host, and runs compose there + via --project-directory. +

+

+ This applies to normal stacks and git sync alike, so relative binds resolve + and every stack is backupable. Leave empty to use only named volumes or + absolute paths. +

+
+ {/if} + + +
+ +

+ {#if isHawserConn(formConnectionType)} + Absolute path on the agent's host where it keeps stack folders. Used only to back up + each stack's compose and config. Leave empty for the default /data/stacks. + {:else} + Absolute path on the remote host where Dockhand stages each stack's files before deploying. + Set it for stacks with relative binds, or to keep stacks backupable on a remote direct daemon. + {/if} +

+
+ {/if} + {#if formConnectionType === 'direct'}