direct remote environments - remote stacks dir configurable, for putting the whole stack onto the remote

This commit is contained in:
jarek committed 2026-08-09 10:31:27 +02:00
1 parent 86ef3f743b
commit bd06b340c5
19 files changed
+2316 -506

No files matched your search

@@ -0,0 +1,254 @@
<script lang="ts">
/**
* "Stack files on the host" — probes the target host at backup-config time, shows the
* resolved HOST path of the stack folder, and lets the user pick which entries to back up.
* Load-bearing files (compose, .env) are always kept (non-deselectable). When the folder
* can't be located on the host, an info callout tells the user to set the env's stack path
* and re-configure. Mirrors VolumePicker's list + BackupPanel's host-path callout.
*/
import { Label } from '$lib/components/ui/label';
import { Checkbox } from '$lib/components/ui/checkbox';
import { TogglePill } from '$lib/components/ui/toggle-pill';
import { FolderOpen, Folder, FileText, Info, Loader2, AlertTriangle, HelpCircle, Unplug, Route, UndoDot, Icon } from 'lucide-svelte';
import { whale } from '@lucide/lab';
import EnvironmentIcon from '$lib/components/EnvironmentIcon.svelte';
import * as Tooltip from '$lib/components/ui/tooltip';
type Entry = { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' };
type Listing =
| { kind: 'listed'; hostPath: string; entries: Entry[] }
| { kind: 'tar'; localStackDir: string; entries: Entry[] }
| { kind: 'helper-failed'; reason: string }
| { kind: 'unknown'; reason: string }
| null;
interface Props {
listing: Listing;
loading?: boolean;
/** Names of stack-dir entries the user has DEselected (excluded from the backup). */
excludedStackFiles: string[];
/** Environment connection type - the tooltip explains the host path differently per type. */
connectionType?: string;
/** Environment name, shown in the tooltip ("<name> is a socket environment"). */
envName?: string;
/** Environment icon (Lucide name or custom) + its id, for the env glyph in the tooltip. */
envIcon?: string;
envId?: number;
/** The user-set "Remote stack path (for backup)" for direct/hawser envs (empty if unset). */
configuredStackPath?: string;
}
let { listing, loading = false, excludedStackFiles = $bindable(), connectionType, envName, envIcon, envId, configuredStackPath = '' }: Props = $props();
// Falls back to a generic subject when the env name isn't available.
const envLabel = $derived(envName || 'This environment');
// The stack folder resolves to a host path by a DIFFERENT route per environment type, so the
// "where this path comes from" help must differ too (undefined = socket, the default).
const envKind = $derived(
connectionType === 'direct' ? 'direct'
: connectionType === 'hawser-standard' || connectionType === 'hawser-edge' ? 'hawser'
: 'socket'
);
// A stack always needs its compose + env files - never deselectable (matches the server
// isLoadBearingStackFile guard, which refuses to exclude them even if the config lists them).
function isLoadBearing(name: string): boolean {
const n = name.toLowerCase();
return (
n === 'compose.yaml' || n === 'compose.yml' ||
n === 'docker-compose.yml' || n === 'docker-compose.yaml' ||
n === '.env' || n.startsWith('.env.')
);
}
// An entry the user cannot deselect HERE: either load-bearing (always kept) or captured via
// its own bind/volume channel (controlled in the Volumes section, not here).
function isLocked(e: Entry): boolean {
return isLoadBearing(e.name) || e.capturedAs != null;
}
const entries = $derived(listing?.kind === 'listed' || listing?.kind === 'tar' ? listing.entries : []);
const selectable = $derived(entries.filter((e) => !isLocked(e)));
// "All files" is on when nothing selectable is excluded.
const allFiles = $derived(excludedStackFiles.length === 0);
function toggleAll(on: boolean) {
excludedStackFiles = on ? [] : selectable.map((e) => e.name);
}
function toggleEntry(e: Entry) {
if (isLocked(e)) return;
excludedStackFiles = excludedStackFiles.includes(e.name)
? excludedStackFiles.filter((n) => n !== e.name)
: [...excludedStackFiles, e.name];
}
function fmtSize(bytes: number): string {
if (bytes < 1024) return `${bytes} B`;
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)} KB`;
return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
}
</script>
<div class="space-y-2">
<div class="text-sm font-medium">Stack files on the host</div>
{#if loading}
<div class="flex items-center gap-2 rounded-md border bg-muted/30 p-2.5 text-xs text-muted-foreground">
<Loader2 class="h-4 w-4 shrink-0 animate-spin" />
Probing the host for the stack folder...
</div>
{:else if listing?.kind === 'listed' || listing?.kind === 'tar'}
{#if listing.kind === 'listed'}
<!-- Resolved host path callout (copied from BackupPanel's stack-path candidate). -->
<div class="flex items-start gap-2 rounded-md border bg-muted/30 p-2.5 text-xs">
<FolderOpen class="mt-0.5 h-4 w-4 shrink-0 text-sky-500" />
<div class="min-w-0 flex-1">
<div class="flex items-center gap-1.5">
<span class="font-medium text-foreground">Captured from the host at</span>
<Tooltip.Root>
<Tooltip.Trigger type="button" class="text-muted-foreground hover:text-foreground">
<HelpCircle class="h-3.5 w-3.5" />
</Tooltip.Trigger>
<Tooltip.Content class="w-80 z-[200]" side="right">
<div class="space-y-2">
<p class="font-medium">Where this path comes from</p>
{#if envKind === 'socket'}
<p class="flex items-center gap-1.5 text-muted-foreground">
{#if envIcon && envId != null}<EnvironmentIcon icon={envIcon} {envId} class="h-3.5 w-3.5 shrink-0" />{/if}
<Unplug class="h-3.5 w-3.5 shrink-0 text-cyan-500" />
<span><span class="font-medium text-foreground">{envLabel}</span> is a socket
environment. Backup reads the stack folder from Dockhand's own stack
directory on this host.</span>
</p>
<p class="text-muted-foreground">
If this path looks wrong, set <code class="bg-muted px-1 rounded">HOST_DATA_DIR</code>
when Dockhand runs in a container, so it points to where the data volume
lives on the host.
</p>
{:else if envKind === 'direct'}
<p class="flex items-center gap-1.5 text-muted-foreground">
{#if envIcon && envId != null}<EnvironmentIcon icon={envIcon} {envId} class="h-3.5 w-3.5 shrink-0" />{/if}
<Icon iconNode={whale} class="h-3.5 w-3.5 shrink-0 text-blue-500" />
<span><span class="font-medium text-foreground">{envLabel}</span> is a direct
environment. Backup reads the stack folder from this path on the remote
Docker host.</span>
</p>
{#if configuredStackPath}
<p class="text-muted-foreground">
Remote stack path set on this environment:
<code class="bg-muted px-1 rounded break-all">{configuredStackPath}</code>
</p>
{:else}
<p class="text-muted-foreground">
It is derived from the stack's bind mounts. To set it explicitly, use
<span class="font-medium text-foreground">Environments &gt; (edit) &gt; Remote stack path (for backup)</span>
and re-open this dialog.
</p>
{/if}
{:else}
<p class="flex items-center gap-1.5 text-muted-foreground">
{#if envIcon && envId != null}<EnvironmentIcon icon={envIcon} {envId} class="h-3.5 w-3.5 shrink-0" />{/if}
{#if connectionType === 'hawser-edge'}
<UndoDot class="h-3.5 w-3.5 shrink-0 text-green-500" />
{:else}
<Route class="h-3.5 w-3.5 shrink-0 text-purple-500" />
{/if}
<span><span class="font-medium text-foreground">{envLabel}</span> is a Hawser
environment. Backup reads the stack folder from the Hawser agent's stack
directory on its host.</span>
</p>
{#if configuredStackPath}
<p class="text-muted-foreground">
Remote stack path set on this environment:
<code class="bg-muted px-1 rounded break-all">{configuredStackPath}</code>
</p>
{:else}
<p class="text-muted-foreground">
It defaults to <code class="bg-muted px-1 rounded">/data/stacks</code>. If the
agent uses a custom <code class="bg-muted px-1 rounded">STACKS_DIR</code>, set it
under <span class="font-medium text-foreground">Environments &gt; (edit) &gt; Remote stack path (for backup)</span>
and re-open this dialog.
</p>
{/if}
{/if}
</div>
</Tooltip.Content>
</Tooltip.Root>
</div>
<div class="mt-0.5 break-all font-mono text-muted-foreground">{listing.hostPath}</div>
</div>
</div>
{:else}
<!-- TAR mode: no host folder to read (direct-remote, no stack path set). Only Dockhand's
own copy of the compose/config is captured; bind-mount DATA on the remote host is NOT. -->
<div class="flex items-start gap-2 rounded-md border border-amber-500/40 bg-amber-500/10 p-2.5 text-xs">
<AlertTriangle class="mt-0.5 h-4 w-4 shrink-0 text-amber-500" />
<div class="min-w-0">
<div class="font-medium text-amber-600 dark:text-amber-400">Only these compose/config files will be captured</div>
<div class="mt-0.5 text-muted-foreground">This environment has no stack path set, so the folder can't be read on the remote host. Dockhand backs up its own copy of the files below. <strong>Bind-mount data on the remote host is NOT included.</strong> Set a stack path on this environment for a full backup.</div>
</div>
</div>
{/if}
<div class="border rounded-md overflow-hidden">
<div class="flex items-center gap-3 px-3 py-2 bg-muted/30 border-b">
<Label class="text-xs">Backup all files ({selectable.length})</Label>
<TogglePill checked={allFiles} onLabel="Yes" offLabel="No" onchange={() => toggleAll(!allFiles)} />
</div>
<div class="divide-y max-h-40 overflow-y-auto">
{#each entries as entry}
{@const locked = isLocked(entry)}
<label class="flex items-center gap-2 px-3 py-1.5 text-xs" class:cursor-pointer={!allFiles && !locked} class:opacity-60={locked}>
<Checkbox
checked={!excludedStackFiles.includes(entry.name)}
disabled={allFiles || locked}
onCheckedChange={() => toggleEntry(entry)}
class="h-3.5 w-3.5"
/>
{#if entry.type === 'dir'}
<Folder class="h-3.5 w-3.5 shrink-0 text-sky-500" />
{:else}
<FileText class="h-3.5 w-3.5 shrink-0 text-muted-foreground" />
{/if}
<span class="font-mono truncate">{entry.name}</span>
{#if entry.capturedAs}
<span class="text-muted-foreground ml-auto shrink-0 italic">captured as a {entry.capturedAs} below</span>
{:else if isLoadBearing(entry.name)}
<span class="text-muted-foreground ml-auto shrink-0 italic">always kept</span>
{:else if entry.type === 'file'}
<span class="text-muted-foreground ml-auto shrink-0">{fmtSize(entry.size)}</span>
{/if}
</label>
{/each}
</div>
</div>
{:else if listing?.kind === 'helper-failed'}
<!-- The probe helper container couldn't run on the target. A backup can't run either (it
uses the same helper), so this is a hard block, not a soft warning. -->
<div class="flex items-start gap-2 rounded-md border border-destructive/50 bg-destructive/10 p-2.5 text-xs">
<AlertTriangle class="mt-0.5 h-4 w-4 shrink-0 text-destructive" />
<div class="min-w-0">
<div class="font-medium text-destructive">The backup helper can't run on this environment</div>
<div class="mt-0.5 break-all text-muted-foreground">{listing.reason}</div>
<div class="mt-1 text-muted-foreground">A backup can't be created until this is fixed - the same helper container captures the stack files.</div>
</div>
</div>
{:else if listing?.kind === 'unknown'}
<div class="flex items-start gap-2 rounded-md border bg-muted/30 p-2.5 text-xs">
<Info class="mt-0.5 h-4 w-4 shrink-0 text-sky-500" />
<div class="min-w-0">
<div class="font-medium text-foreground">No stack folder found on the host</div>
<div class="mt-0.5 text-muted-foreground">{listing.reason}</div>
{#if envKind === 'socket'}
<div class="mt-1 text-muted-foreground">On a socket environment this usually means <code class="bg-muted px-1 rounded">HOST_DATA_DIR</code> does not match where the data volume is mounted on the host. Fix it, then cancel and re-configure this backup.</div>
{:else if envKind === 'direct'}
<div class="mt-1 text-muted-foreground">Set the environment's stack path (Environments &gt; edit &gt; Remote stack path for backup), then cancel and re-configure this backup.</div>
{:else}
<div class="mt-1 text-muted-foreground">On Hawser this usually means the agent uses a custom <code class="bg-muted px-1 rounded">STACKS_DIR</code> (not the default <code class="bg-muted px-1 rounded">/data/stacks</code>). Set the environment's stack path (Environments &gt; edit &gt; Remote stack path for backup), then cancel and re-configure this backup.</div>
{/if}
</div>
</div>
{/if}
</div>
+28 -5
View File
@@ -27,9 +27,9 @@ export interface MetadataFile {
* The metadata files (metadata.json, and for stacks the compose/.env and the
* whole stackfiles/ tree) are NOT written by this script — they are streamed into
* the container's /metadata directory via the Docker put-archive API BEFORE the
* container starts (see restic.ts runInHelper). Embedding their base64 inline in
* the Cmd used to blow the kernel ARG_MAX for large stack files ("argument list
* too long"); put-archive has no such limit.
* container starts (see restic.ts runInHelper). Inlining their base64 in the Cmd
* would hit the kernel ARG_MAX for large stack files ("argument list too long");
* put-archive has no such limit.
*
* So the script only:
* 1. ensures /metadata exists (put-archive already created it, but be defensive
@@ -41,7 +41,7 @@ export interface MetadataFile {
* @param resticArgs the full restic argv (starting with 'backup') — already
* tag/exclude/flag-assembled by the caller.
*/
export function buildBackupScript(resticArgs: string[]): string {
export function buildBackupScript(resticArgs: string[], stackDirProbe?: { volumeKey: string; composeFileName: string; label?: string; hostPath?: string }): string {
// restic runs backgrounded so the SIGINT-forwarding trap can reach it on cancel
// (SIGINT lets restic release its repo lock; a SIGKILL orphans it and hangs the
// next backup on --retry-lock). The kill-0 re-wait loop only fires when a signal
@@ -49,7 +49,30 @@ export function buildBackupScript(resticArgs: string[]): string {
// real code (0/3) for the marker.
const restic = `restic ${resticArgs.map(shellQuote).join(' ')}`;
const step = `${restic} & __rpid=$!; trap 'kill -INT $__rpid 2>/dev/null' INT TERM; wait $__rpid; __rc=$?; while kill -0 $__rpid 2>/dev/null; do wait $__rpid; __rc=$?; done; ( exit $__rc )`;
return `mkdir -p /metadata; set +e; ${finishScript(step)}`;
// STACK-DIR VOLUME PROBE (100% runtime guard for the 'volume' capture mode). The plan
// decided the target daemon is local and bind-mounted the stack dir at
// /volumes/<volumeKey>. If that mount is a phantom (a remote daemon Docker
// auto-created an EMPTY dir for), the compose file will be MISSING under it — backing
// that up would be a silent-empty snapshot. So before restic runs, assert the compose
// is visible; if not, exit non-zero (the finishScript marker turns this into a hard
// backup failure), never a silent success.
let probe = '';
if (stackDirProbe) {
const composePath = shellQuote(`/volumes/${stackDirProbe.volumeKey}/${stackDirProbe.composeFileName}`);
const label = stackDirProbe.label ? ` ${stackDirProbe.label}` : '';
// Name the REAL host path we mounted (<remote_stacks_dir>/<stack>) so the operator knows
// exactly where to look, and what to do: redeploy the stack to stage its files there.
const hostHint = stackDirProbe.hostPath
? ` The stack folder is empty or missing at ${stackDirProbe.hostPath} on the Docker host - redeploy the stack to stage its files there.`
: '';
probe =
`if [ ! -f ${composePath} ]; then ` +
`echo "[backup] $(date -u +%Y-%m-%dT%H:%M:%SZ)${label} STACKDIR PROBE FAILED: the stack's compose (${stackDirProbe.composeFileName}) is not present under the mounted stack dir.${hostHint} Refusing to write a silent-empty snapshot." 1>&2; ` +
`exit 1; fi; `;
}
return `mkdir -p /metadata; set +e; ${probe}${finishScript(step)}`;
}
/**
+162 -100
View File
@@ -42,18 +42,8 @@ import { buildBackupScript, buildBackupArgs, type MetadataFile } from './backup-
import { EXIT_MARKER } from './restic-script';
import { parseRetention, buildForgetArgs, checkWouldWipe, retentionActive } from './retention';
import { liveTargetKey } from './locks';
import { stopIntentKey as stopIntentKeyFor } from './stop-recovery-core';
import type { DiscoveredVolume } from './discovery-core';
/** What a fresh process needs to restart a target stopped for a consistent
* backup (or an in-place restore) after a crash between stop and restart. */
export interface StopIntent {
type: BackupTargetType;
targetName: string;
envId: number | null;
/** For a container target: the ids/names that were running and got stopped. */
containers: Array<{ id: string; name: string }>;
}
import { STACKDIR_VOLUME_KEY } from './stackdir-plan';
/** What the service needs from the outside world. All injected for testability. */
export interface BackupPorts {
@@ -63,34 +53,45 @@ export interface BackupPorts {
/** Discover volumes; MUST throw (fail closed) on any inspect failure. */
discoverVolumes(containers: Array<{ id: string; name: string }>, envId: number | null | undefined, selected: string[] | null):
Promise<{ volumes: DiscoveredVolume[]; skipped: Array<{ type: string; destination: string; reason: string }> }>;
/** Decide how a STACK's directory is captured. Normally: locate the HOST folder and build the
* synthetic __dockhand_stackdir__ volume the helper bind-mounts (`kind: 'candidate'`).
* `kind: 'tar'` is the direct-remote-with-no-remote_stacks_dir case: nothing is staged on the
* host, so the compose/config is tarred from Dockhand's local copy instead (no host bind).
* `composeFileName` feeds the in-helper probe (assert the compose is
* visible under the mount). `kind: 'unknown'` means the stack is not compose-managed (no
* working_dir label) -> the caller HARD-FAILS the backup. Logged inside this port. */
planStackDirVolume(targetName: string, envId: number | null | undefined, excludedStackFiles?: string[]):
Promise<
| { kind: 'unknown'; reason: string }
| { kind: 'candidate'; syntheticVolume: DiscoveredVolume; volumeKey: string; composeFileName: string; excludePaths: string[]; bindSources: string[] }
| { kind: 'tar'; localStackDir: string; composeFileName: string }
>;
/** Stop the target for a consistent backup; returns a restart closure. */
stopForBackup(type: BackupTargetType, targetName: string, containers: Array<{ id: string; name: string; state: string }>, envId: number | null | undefined):
Promise<{ restart: () => Promise<{ failed: Array<{ name: string; error: string }> }> }>;
/** Persist a durable "this target was stopped for backup" intent BEFORE the
* stop, so a process death between stop and restart is recoverable on startup.
* Keyed on a stable per-target id (see stopIntentKey). */
recordStopIntent(key: string, intent: StopIntent): Promise<void>;
/** Clear the durable stop intent once the in-process restart has succeeded. */
clearStopIntent(key: string): Promise<void>;
/** Run the backup session container; returns the restic run. metadataFiles are
* streamed into /metadata via put-archive (not the Cmd) so large stack files
* don't blow ARG_MAX. */
runInHelper(spec: { args?: string[]; script: string; binds: string[]; envId: number | null | undefined; name: string; metadataFiles?: MetadataFile[]; onStderr?: (line: string) => void; onStdout?: (line: string) => void }):
/** Run the backup session container; returns the restic run. metadataFiles (metadata.json
* + the light stack-dir listing) are put-archived into /metadata (not the Cmd) so they
* can't blow ARG_MAX. The stack dir's bytes ride a host bind mount, not these files. */
runInHelper(spec: { args?: string[]; script: string; binds: string[]; envId: number | null | undefined; name: string; metadataFiles?: MetadataFile[]; stackDirStreamSources?: Array<{ diskPath: string; archivePath: string }>; onStderr?: (line: string) => void; onStdout?: (line: string) => void }):
Promise<ResticRun>;
/** Run a repo-only restic command on the host (verify / retention). */
runLocal(args: string[], tier?: 'interactive' | 'data'): Promise<ResticRun>;
/** Collect the restore metadata files (metadata.json + stack files).
* `stackFilesTruncated` is true when a stack dir exceeded the capture cap, so
* the snapshot is an incomplete stack-file capture — the run is downgraded to
* a warning. */
collectMetadata(type: BackupTargetType, targetName: string, envId: number | null | undefined, volumes: DiscoveredVolume[]):
Promise<{ files: MetadataFile[]; stackFilesTruncated: boolean }>;
/** Collect the restore metadata files: metadata.json plus, for a stack, a light file
* listing. The stack dir's bytes normally ride the helper's host bind mount, not these
* files. EXCEPTION - `stackDirTar`: for a direct-remote stack with no remote_stacks_dir
* there is no host bind, so the collector reads localStackDir and tars its files into the
* snapshot at /volumes/__dockhand_stackdir__ (the same path restore reads). */
collectMetadata(type: BackupTargetType, targetName: string, envId: number | null | undefined, volumes: DiscoveredVolume[], stackDirTar?: { localStackDir: string; composeFileName: string } | null):
Promise<{ files: MetadataFile[]; stackDirStreamSources?: Array<{ diskPath: string; archivePath: string }> }>;
/** The Dockhand host name for restic --host. */
host(): string;
/** This installation's stable instance id. */
instanceId(): Promise<string>;
/** The live-target mutex: returns a release fn, or null if already held. */
acquireLiveTarget(key: string): (() => void) | null;
/** True if the user asked to cancel this config's in-flight backup — used to
* report a killed helper's non-zero exit as "cancelled", not a restic failure. */
isCancelling(configId: number): boolean;
/** Serialize on a destination; runs fn behind the per-destination queue. */
serializeDestination<T>(destinationId: number, fn: () => Promise<T>): Promise<T>;
/** Open the operation record; returns handles to update + close it. */
@@ -128,7 +129,7 @@ export interface BackupJob {
selectedVolumes: string[] | null;
stopBeforeBackup: boolean;
retention: string | null | Record<string, unknown>;
options: { excludePatterns?: string[]; excludeCaches?: boolean; compression?: string; limitUpload?: number; limitDownload?: number; webhookSuccess?: string; webhookFailure?: string };
options: { excludePatterns?: string[]; excludeCaches?: boolean; compression?: string; limitUpload?: number; limitDownload?: number; webhookSuccess?: string; webhookFailure?: string; excludedStackFiles?: string[] };
helperName: string;
}
@@ -146,6 +147,17 @@ export class BackupService {
// --- discover targets first (need the volume identity for the lock key) ---
let containers: Array<{ id: string; name: string; state: string }>;
let volumes: DiscoveredVolume[];
// The stack dir is captured by the helper's HOST bind mount (a synthetic
// __dockhand_stackdir__ volume). `stackDirProbe` carries what the helper needs to
// assert that mount is real (test -f the compose) before restic runs. Null for
// containers (no stack dir).
let stackDirProbe: { volumeKey: string; composeFileName: string; hostPath?: string } | null = null;
// restic --exclude paths for compose bind dirs inside the stackdir volume (they are
// captured separately as their own /volumes/<key>, honoring volume selection).
let stackDirExcludes: string[] = [];
// Set (instead of a host bind) for a direct-remote stack with no remote_stacks_dir: the
// compose/config is tarred into /metadata from Dockhand's local copy, not the host.
let stackDirTar: { localStackDir: string; composeFileName: string } | null = null;
try {
const t = await this.ports.resolveTargets(job.type, job.targetName, envId);
containers = t.containers;
@@ -161,12 +173,37 @@ export class BackupService {
// A target with no volumes/binds is fine: metadata.json (the container/stack
// config, and for stacks the compose files) is always captured, so the
// snapshot is a config-only backup — restorable via recreate/redeploy.
} catch (err) {
return this.errorResult(err);
}
// For a STACK, always capture the whole stack dir by bind-mounting it from the
// TARGET daemon's HOST (__dockhand_stackdir__). The plan locates the host folder
// from the compose working_dir label; a stack with no such label is not
// compose-managed, so we HARD-FAIL rather than silently skip its files (option A).
// The synthetic volume rides the same helper-bind + lock machinery as any volume.
if (job.type === 'stack') {
const p = await this.ports.planStackDirVolume(job.targetName, envId, job.options.excludedStackFiles);
if (p.kind === 'unknown') {
return this.earlySkipOrError(job, triggeredBy,
`Cannot locate the stack folder on the host for "${job.targetName}" (${p.reason}). ` +
`For remote environments, set a Remote stacks directory or use matching paths so the stack files exist on the Docker host. ` +
`Refusing to write a backup missing the stack's compose and config.`);
} else if (p.kind === 'tar') {
// direct-remote, no remote_stacks_dir: no host bind (nothing staged), so the
// compose/config is captured from Dockhand's local copy into /metadata via a tar.
// No synthetic volume, no probe - the metadata collector reads localStackDir.
stackDirTar = { localStackDir: p.localStackDir, composeFileName: p.composeFileName };
} else {
volumes = [...volumes, p.syntheticVolume];
stackDirProbe = { volumeKey: p.volumeKey, composeFileName: p.composeFileName, hostPath: p.syntheticVolume.source ?? undefined };
stackDirExcludes = p.excludePaths;
}
}
} catch (err) {
return this.errorResult(err);
}
// --- claim the live-target mutex (reject a concurrent op on this data) ---
const key = liveTargetKey(envId, volumes.map((v) => v.bind));
// fallback identity keeps config-only targets (no volumes) from colliding on ${env}::
const key = liveTargetKey(envId, volumes.map((v) => v.bind), `${job.type}:${job.targetName}`);
const release = this.ports.acquireLiveTarget(key);
if (!release) {
return { status: 'skipped', reason: `Another backup or restore is already running for "${job.targetName}".` };
@@ -184,7 +221,7 @@ export class BackupService {
try {
// --- serialize on the destination (share the repo, one at a time) ---
return await this.ports.serializeDestination(job.destinationId, () =>
this.runLocked(job, triggeredBy, containers, volumes, startTime, releaseOnce));
this.runLocked(job, triggeredBy, containers, volumes, stackDirProbe, stackDirExcludes, stackDirTar, startTime, releaseOnce));
} finally {
releaseOnce();
}
@@ -200,6 +237,9 @@ export class BackupService {
triggeredBy: 'cron' | 'manual' | 'webhook',
containers: Array<{ id: string; name: string; state: string }>,
volumes: DiscoveredVolume[],
stackDirProbe: { volumeKey: string; composeFileName: string } | null,
stackDirExcludes: string[],
stackDirTar: { localStackDir: string; composeFileName: string } | null,
startTime: number,
releaseLiveTarget: () => void,
): Promise<BackupResult> {
@@ -207,26 +247,25 @@ export class BackupService {
const op = await this.ports.openOperation(job.targetName, job.configId, envId, triggeredBy);
let restart: (() => Promise<{ failed: Array<{ name: string; error: string }> }>) | null = null;
let restarted = false;
let stopIntentKey: string | null = null;
// ========================= @BACKUP-DIAG-TEMP =========================
// TEMPORARY CI backup-flake diagnostics. Remove this whole block (and the other
// `@BACKUP-DIAG-TEMP` markers below) once the flake is fixed:
// grep -rn '@BACKUP-DIAG-TEMP' src/lib/server/backups/backup-service.ts
// Every log line is prefixed `[BACKUP-DIAG-TEMP]` for a one-shot log filter.
// Gated behind BACKUP_DIAG=1 so it's silent unless CI turns it on.
// Emits per-phase timing (backup/verify/restart/retention/close) + a repo
// pre-probe (does the repo exist? is it locked?) to settle the "destination
// points at a wiped/uninit repo" and "stale lock" hypotheses.
const diagOn = process.env.BACKUP_DIAG === '1';
// Two log tiers, both stamped with a wall-clock ISO time and the {env,target}
// key so any line can be lined up with the shard test log by time and target:
// log() - always on. The lines an operator wants: started / snapshot created /
// completed / failed. Plain and self-explanatory, no scary internals.
// diag() - behind BACKUP_DIAG=1 (CI turns it on). Deep internals: repo/lock
// pre-probes, instance identity, micro-phase timings. Noise for a user,
// gold for a CI flake.
const key = `${job.targetName} env=${envId}`;
const t0 = Date.now();
const log = (msg: string) => console.log(`[backup] ${new Date().toISOString()} ${key} | ${msg}`);
const diagOn = process.env.BACKUP_DIAG === '1';
const diag = (phase: string, extra?: string) => {
if (!diagOn) return;
const ms = Date.now() - t0;
console.log(`[BACKUP-DIAG-TEMP] ${job.targetName} cfg=${job.configId} op=${op.id} env=${envId} | ${phase} @${ms}ms${extra ? ' | ' + extra : ''}`);
console.log(`[backup] ${new Date().toISOString()} ${key} cfg=${job.configId} op=${op.id} | ${phase} @${Date.now() - t0}ms${extra ? ' | ' + extra : ''}`);
};
diag('START', `triggeredBy=${triggeredBy} stopBeforeBackup=${job.stopBeforeBackup} retention=${JSON.stringify(job.retention)}`);
log(`started (${triggeredBy})`);
if (diagOn) {
diag('START', `stopBeforeBackup=${job.stopBeforeBackup} retention=${JSON.stringify(job.retention)}`);
try {
const cfg = await this.ports.runLocal(['cat', 'config', '--no-lock'], 'interactive');
diag('REPO_PROBE', `cat-config exit=${cfg.exitCode} (${cfg.exitCode === 10 ? 'NOT-INITIALISED' : cfg.exitCode === 0 ? 'exists' : 'error'})`);
@@ -237,49 +276,47 @@ export class BackupService {
diag('LOCK_PROBE', `list-locks exit=${locks.exitCode} heldLocks=${lockLines.length}${lockLines.length ? ' [' + lockLines.join(',').slice(0, 120) + ']' : ''}`);
} catch (e) { diag('LOCK_PROBE', `list-locks THREW ${e instanceof Error ? e.message : String(e)}`); }
}
// ======================= end @BACKUP-DIAG-TEMP =======================
try {
// --- optionally stop the target for a consistent snapshot ---
if (job.stopBeforeBackup) {
op.progress('stopping', `Stopping ${job.targetName} for a consistent backup...`);
// Record a durable stop intent BEFORE the stop, so a process death
// between the stop and the restart leaves a row a fresh startup can
// replay to bring the target back up (see reconcileOnStartup).
const running = containers.filter((c) => c.state === 'running');
if (running.length > 0) {
stopIntentKey = stopIntentKeyFor(envId, job.type, job.targetName);
await this.ports.recordStopIntent(stopIntentKey, {
type: job.type, targetName: job.targetName, envId,
containers: running.map((c) => ({ id: c.id, name: c.name })),
});
}
const stopped = await this.ports.stopForBackup(job.type, job.targetName, containers, envId);
restart = stopped.restart;
}
// --- metadata + backup args + session script ---
op.progress('metadata', 'Collecting metadata...');
const { files: metadata, stackFilesTruncated } = await this.ports.collectMetadata(job.type, job.targetName, envId, volumes);
const { files: metadata, stackDirStreamSources } = await this.ports.collectMetadata(job.type, job.targetName, envId, volumes, stackDirTar);
const instanceId = await this.ports.instanceId();
const tags = buildSnapshotTags({ instanceId, configId: job.configId, environmentId: envId, targetName: job.targetName, type: job.type });
// @BACKUP-DIAG-TEMP: cross-shard identity. If two shards log the SAME
// instanceId + configId (they share a cloned DB) AND the same repo, their
// snapshots/retention collide. host() + destinationId show which repo this
// run targets, so a CI log across shards reveals any overlap.
// Cross-shard identity. If two shards log the SAME instanceId + configId
// (they share a cloned DB) AND the same repo, their snapshots/retention
// collide. host() + destinationId show which repo this run targets, so a
// CI log across shards reveals any overlap.
diag('IDENTITY', `instanceId=${instanceId} host=${this.ports.host()} destId=${job.destinationId} tags=[${tags.join(',')}]`);
// restic must back up /volumes/ when there's anything to capture there: real bind/named
// volumes (volumes.length), OR a tar-mode stack whose compose/config was put-archived
// into /volumes/__dockhand_stackdir__ (no synthetic volume in `volumes`, so it wouldn't
// count otherwise - and /metadata-only would silently drop the tarred stack files).
const args = buildBackupArgs({
host: this.ports.host(),
tags,
hasVolumes: volumes.length > 0,
excludePatterns: job.options.excludePatterns,
hasVolumes: volumes.length > 0 || stackDirTar !== null,
// User excludes PLUS the compose bind dirs inside the stackdir volume (those
// ride their own /volumes/<key>, so excluding them here avoids a duplicate and
// keeps volume DESELECTION meaningful for in-folder binds).
excludePatterns: [...(job.options.excludePatterns ?? []), ...stackDirExcludes],
excludeCaches: job.options.excludeCaches,
compression: job.options.compression,
limitUpload: job.options.limitUpload,
limitDownload: job.options.limitDownload,
swapArtifacts: SWAP_ARTIFACTS,
});
const script = buildBackupScript(args);
const script = buildBackupScript(
args,
stackDirProbe ? { ...stackDirProbe, label: `env=${envId} target=${job.targetName}` } : undefined
);
const binds = volumes.map((v) => v.bind);
// Echo the options actually applied to the restic run, so the execution
@@ -291,8 +328,7 @@ export class BackupService {
if (o.excludeCaches !== false) op.log('Exclude caches: yes');
if (o.excludePatterns && o.excludePatterns.length) op.log(`Exclude patterns: ${o.excludePatterns.join(', ')}`);
// Record WHICH volumes were actually selected (not just the count), so a
// regression in the selectedVolumes filter is visible in the execution
// log rather than silent (matches the legacy engine's log line).
// regression in the selectedVolumes filter is visible in the execution log.
op.log(`Volumes to backup: ${volumes.map((v) => v.name).join(', ') || 'none'}`);
// --- clear any STALE repo lock before backing up ---
@@ -326,18 +362,26 @@ export class BackupService {
} else {
op.progress('backing-up', `Backing up ${volumes.length} ${volumes.length === 1 ? 'volume' : 'volumes'}:`);
for (const v of volumes) {
op.progress('backing-up', ` • [${v.type === 'bind' ? 'BIND' : 'VOL'}] ${v.name}`);
// A bind's `name` is only its container destination, so two binds to the same
// path (e.g. /data) read identically. Show `host-source -> destination` so each
// is distinguishable in the log (#1373).
const label = v.name === STACKDIR_VOLUME_KEY
? 'Stack data'
: v.type === 'bind'
? `${v.source} → ${v.name}`
: v.name;
op.progress('backing-up', ` • [${v.type === 'bind' ? 'BIND' : 'VOL'}] ${label}`);
}
}
// Did the LIVE stdout stream deliver anything? restic writes its --json
// progress to STDOUT, which we forward via onStdout. But live stdout only
// progress to STDOUT (forwarded via onStdout); but live stdout only
// works on transports that stream it (local socket/http); if nothing arrives
// live, we MUST still emit restic's output post-exit from run.stdout — else
// the log shows only our own stage markers and no restic lines at all (a
// regression). This flag drives that fallback.
// This flag drives that fallback.
let liveStdoutSeen = false;
const run = await this.ports.runInHelper({
script, binds, envId, name: job.helperName, metadataFiles: metadata,
script, binds, envId, name: job.helperName, metadataFiles: metadata, stackDirStreamSources,
// Stream restic's own output to the UI LIVE: restic writes its --json
// progress (per-file %) to STDOUT and its status messages to STDERR. Wire
// BOTH so the log fills in as the backup runs, not all at once at the end.
@@ -349,7 +393,7 @@ export class BackupService {
// Fallback: if live stdout streaming produced nothing (transport didn't
// stream it), emit restic's buffered stdout now so the log always shows what
// restic actually did — the pre-live-streaming behaviour, kept as a safety net.
// restic actually did.
if (!liveStdoutSeen) {
for (const l of formatResticLines(run.stdout)) {
if (!l.includes(EXIT_MARKER)) op.progress('progress', `[restic] ${l}`);
@@ -365,13 +409,14 @@ export class BackupService {
// No summary ⇒ unknown outcome ⇒ NOT a success (never record a phantom).
throw new BackupError('INTEGRITY', 'backup produced no snapshot summary — treating as failed');
}
// A partial is either restic's own "couldn't read all files" (exit 3) OR
// a stack dir that exceeded the capture cap (snapshot is redeploy-incomplete).
const partial = resticPartial(run) || stackFilesTruncated;
// A partial is restic's own "couldn't read all files" (exit 3). A stack dir
// over the capture cap is no longer a partial — it hard-fails in collectMetadata.
const partial = resticPartial(run);
// --- VERIFY the snapshot is actually readable before declaring success ---
log(`snapshot ${summary.snapshotId.slice(0, 8)} created, verifying`);
op.progress('verifying', 'Verifying the snapshot is readable...');
diag("BEFORE_VERIFY", "snapshot=" + summary.snapshotId); // @BACKUP-DIAG-TEMP
diag("BEFORE_VERIFY", "snapshot=" + summary.snapshotId);
await this.verifySnapshot(summary.snapshotId, instanceId);
// --- commit: record success + config status ---
@@ -382,14 +427,14 @@ export class BackupService {
// retention, then release the live-target lock: retention is a repo-only
// forget/prune (already serialized per-destination) and a concurrent
// restore neither reads nor writes the live volume during it.
diag("BEFORE_RESTART"); // @BACKUP-DIAG-TEMP
await this.restartAndSurface(restart, job, op, stopIntentKey);
diag("BEFORE_RESTART");
await this.restartAndSurface(restart, job, op);
restarted = true;
releaseLiveTarget();
// --- retention: ONLY now, after a confirmed + verified snapshot AND
// after the live target is back up + its lock freed ---
diag("BEFORE_RETENTION"); // @BACKUP-DIAG-TEMP
diag("BEFORE_RETENTION");
const retentionOutcome = await this.applyRetention(job, instanceId, op);
const details = {
@@ -401,21 +446,36 @@ export class BackupService {
retention: retentionOutcome,
};
if (partial) {
const warnMsg = stackFilesTruncated
? 'Backup completed with warnings: the stack directory exceeded the capture cap, so the snapshot is an incomplete stack-file capture (redeploy may be incomplete).'
: 'Backup completed with warnings (some files could not be read).';
await op.close({ kind: 'warning', message: warnMsg }, details);
log(`completed with warnings (partial read) in ${((Date.now() - t0) / 1000).toFixed(1)}s`);
await op.close({ kind: 'warning', message: 'Backup completed with warnings (some files could not be read).' }, details);
await this.notifySuccess(job, summary, envId, true, op.id, startTime);
return { status: 'warning', executionId: op.id, snapshotId: summary.snapshotId, summary, warning: stackFilesTruncated ? 'stack files truncated' : 'partial read', retention: retentionOutcome };
return { status: 'warning', executionId: op.id, snapshotId: summary.snapshotId, summary, warning: 'partial read', retention: retentionOutcome };
}
diag("BEFORE_CLOSE"); // @BACKUP-DIAG-TEMP
log(`completed in ${((Date.now() - t0) / 1000).toFixed(1)}s`);
diag("BEFORE_CLOSE");
await op.close({ kind: 'ok' }, details);
await this.notifySuccess(job, summary, envId, false, op.id, startTime);
return { status: 'success', executionId: op.id, snapshotId: summary.snapshotId, summary, retention: retentionOutcome };
} catch (err) {
// If the user cancelled THIS backup, its helper was SIGINT/SIGKILL'd — the
// resulting non-zero exit (137) is not a real failure. Report it as cancelled
// with a clean message instead of the raw restic exit spew.
if (this.ports.isCancelling(job.configId)) {
log('cancelled by user');
if (!restarted) await this.restartAndSurface(restart, job, op);
try { await this.ports.runLocal(['unlock']); } catch { /* best-effort */ }
try { await this.ports.setConfigStatus(job.configId, 'failed'); } catch { /* non-fatal */ }
await op.close({ kind: 'cancelled', message: 'Backup cancelled' });
return { status: 'skipped', executionId: op.id, reason: 'Backup cancelled' };
}
// Log the FULL raw error to the server log (docker logs dockhand) before it's
// cleaned for the UI — the raw text (restic JSON spew, exit code, stderr) is
// what we actually need to diagnose a failure, and cleanErrorMsg strips it.
log(`failed after ${((Date.now() - t0) / 1000).toFixed(1)}s`);
console.error(`[backup] ${new Date().toISOString()} ${key} | raw error:`, err instanceof Error ? (err.stack ?? err.message) : String(err));
// Always try to restart what we stopped, even on failure — unless the
// success path already restarted it (avoid a double start).
if (!restarted) await this.restartAndSurface(restart, job, op, stopIntentKey);
if (!restarted) await this.restartAndSurface(restart, job, op);
// Clean up the repo lock this failed/killed backup may have orphaned, so
// the NEXT op to this repo doesn't wait out `--retry-lock` on a dead
// owner. Still behind the per-destination serializer here. Best-effort.
@@ -424,7 +484,7 @@ export class BackupService {
try { await this.ports.setConfigStatus(job.configId, 'failed'); } catch { /* non-fatal */ }
await op.close({ kind: 'error', code, message }, { errorCode: code });
try { await this.ports.notify('backup_failed', { target: job.targetName, kind: 'backup', errorCode: code, message }, envId); } catch { /* non-fatal */ }
// Per-config failure webhook — same payload shape as the legacy engine.
// Per-config failure webhook.
if (job.options.webhookFailure) {
try {
this.ports.fireWebhook(job.options.webhookFailure, {
@@ -472,24 +532,26 @@ export class BackupService {
// can see the wait, not just the eventual result.
if (dryMs > 5000) op.log(`Retention: dry-run took ${(dryMs / 1000).toFixed(1)}s (waited on the repo lock).`);
if (dryRun.exitCode !== 0) {
op.progress('warning', `Retention dry-run failed (exit ${dryRun.exitCode}): ${dryRun.stderr.trim() || 'restic failed'} — skipping prune, backup is still safe.`);
op.progress('warning', `Retention dry-run failed (exit ${dryRun.exitCode}): ${dryRun.stderr.trim() || 'restic failed'} — skipping retention, backup is still safe.`);
return 'failed';
}
if (checkWouldWipe(dryRun.stdout).wouldWipe) {
op.progress('warning', 'Retention would delete every snapshot for this config — skipping prune.');
op.progress('warning', 'Retention would delete every snapshot for this config — skipping retention.');
return 'skipped-would-wipe';
}
// forget only removes snapshot references (fast, metadata) — NO --prune here, so a
// backup never hangs reclaiming space. The destination's prune schedule reclaims it.
op.progress('pruning', 'Applying retention...');
const forget = buildForgetArgs(policy, filter)!;
const t1 = Date.now();
const run = await this.ports.runLocal(forget, 'data');
const forgetMs = Date.now() - t1;
if (run.exitCode !== 0) {
// A prune failure is non-fatal (the backup itself succeeded) but surfaced.
// A forget failure is non-fatal (the backup itself succeeded) but surfaced.
// The common non-fatal case: --retry-lock elapsed because another op (an
// orphaned lock, or a second Dockhand instance sharing this repo) held the
// lock — retention just runs next schedule; no snapshot is lost.
op.progress('warning', `Retention prune failed after ${(forgetMs / 1000).toFixed(1)}s (exit ${run.exitCode}): ${run.stderr.trim() || 'restic failed'} — backup is safe, retention will retry next run.`);
op.progress('warning', `Retention failed after ${(forgetMs / 1000).toFixed(1)}s (exit ${run.exitCode}): ${run.stderr.trim() || 'restic failed'} — backup is safe, retention will retry next run.`);
return 'failed';
}
op.log(`Retention: applied in ${(forgetMs / 1000).toFixed(1)}s.`);
@@ -503,7 +565,6 @@ export class BackupService {
restart: (() => Promise<{ failed: Array<{ name: string; error: string }> }>) | null,
job: BackupJob,
op: OperationHandle,
stopIntentKey: string | null,
): Promise<void> {
if (!restart) return;
let failed: Array<{ name: string; error: string }> = [];
@@ -513,14 +574,11 @@ export class BackupService {
failed = [{ name: job.targetName, error: err instanceof Error ? err.message : String(err) }];
}
if (failed.length > 0) {
// Restart failed — leave the durable intent for startup retry.
const names = failed.map((f) => f.name).join(', ');
const msg = `Backup finished but ${names} could not be restarted and is STILL STOPPED — start it manually.`;
op.progress('warning', msg);
try { await this.ports.notify('backup_failed', { target: job.targetName, kind: 'backup', errorCode: 'STOPPED_RESTART_FAILED', message: msg }, job.environmentId); } catch { /* non-fatal */ }
} else {
// Restart succeeded — drop the durable intent so no stale row remains.
if (stopIntentKey) { try { await this.ports.clearStopIntent(stopIntentKey); } catch { /* non-fatal */ } }
op.progress('restarted', `${job.targetName} restarted.`);
}
}
@@ -539,7 +597,7 @@ export class BackupService {
dataAdded: summary.dataAdded, partial,
}, envId);
} catch { /* notification failure never changes the outcome */ }
// Per-config success webhook — same payload shape as the legacy engine.
// Per-config success webhook.
if (job.options.webhookSuccess) {
try {
this.ports.fireWebhook(job.options.webhookSuccess, {
@@ -558,11 +616,15 @@ export class BackupService {
}
/** The backup target (container/stack) doesn't exist — record the op and fail
* with a validation error (there is genuinely nothing to back up). */
* with a validation error (there is genuinely nothing to back up). Close the durable
* execution row as a FAILED/VALIDATION error, NOT op.skip() — skip() hardcodes
* CONCURRENCY + skipped:true, which would log a benign-looking "skipped" row while the
* API returns VALIDATION. An operator filtering for failed backups would then never see
* that this config has been backing up nothing (its target is gone). */
private async earlySkipOrError(job: BackupJob, triggeredBy: 'cron' | 'manual' | 'webhook', reason: string): Promise<BackupResult> {
try {
const op = await this.ports.openOperation(job.targetName, job.configId, job.environmentId, triggeredBy);
await op.skip(reason);
await op.close({ kind: 'error', code: 'VALIDATION', message: reason });
return { status: 'error', executionId: op.id, code: 'VALIDATION', error: reason };
} catch {
return { status: 'error', code: 'VALIDATION', error: reason };
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,102 @@
/**
* put-archive-stream.ts — a BACKUP-ONLY streaming upload to a container's PUT /archive.
*
* WHY a separate function (not the shared putContainerArchive): the shared one has three
* callers (file-browser upload, the files-upload API, and backup). It routes through
* `dockerFetch`, whose plain-HTTP path uses global `fetch` (undici) which BUFFERS the whole
* request body in RAM (verified: 200 MB file -> 239 MB RSS). For the backup helper we want
* to stream a large stack-dir tar with O(1) RAM, so we go straight to node http/https core
* (verified: 200 MB -> 43 MB RSS). Keeping this backup-only means the file-upload paths are
* completely untouched.
*
* TRANSPORTS:
* - socket / plain-http (direct, hawser-standard) -> node http.request, pipe the tar in.
* - https / mTLS (direct, hawser-standard TLS) -> httpsAgentRequest (its ReadableStream
* body branch streams via req.write).
* - hawser-edge -> CANNOT stream (its WS protocol frames
* the whole base64 tar). canStream()
* returns false; the caller falls back
* to the buffered putContainerArchive.
*/
import http from 'node:http';
import { Readable } from 'node:stream';
import type { ReadableStream as NodeReadableStream } from 'node:stream/web';
import { getDockerConfig, httpsAgentRequest, putContainerArchive, type DockerClientConfig } from '../docker';
// Pure transport predicates + the idle watchdog live in an import-light module so they can be
// unit-tested without pulling docker.ts (and thus better-sqlite3).
import { transportCanStream, sanitizeArchivePath, withIdleWatchdog } from './put-archive-transport';
export { transportCanStream, sanitizeArchivePath, withIdleWatchdog };
/** PUT a streaming tar into a container over a unix socket or plain HTTP, via node
* http.request (streams the piped body — unlike undici fetch, which buffers). */
function putViaNodeHttp(config: DockerClientConfig, containerId: string, archivePath: string, tar: Readable, extraHeaders: Record<string, string>): Promise<void> {
return new Promise((resolve, reject) => {
const path = `/containers/${containerId}/archive?path=${encodeURIComponent(sanitizeArchivePath(archivePath))}`;
const reqOptions: http.RequestOptions = config.type === 'socket'
? { socketPath: config.socketPath, path, method: 'PUT', headers: { 'Content-Type': 'application/x-tar', ...extraHeaders } }
: { host: config.host, port: config.port, path, method: 'PUT', headers: { 'Content-Type': 'application/x-tar', ...extraHeaders } };
const req = http.request(reqOptions, (res) => {
const chunks: Buffer[] = [];
res.on('data', (c: Buffer) => chunks.push(c));
res.on('end', () => {
const status = res.statusCode ?? 0;
if (status >= 200 && status < 300) resolve();
else reject(new Error(`put-archive-stream: PUT /archive -> ${status} ${Buffer.concat(chunks).toString().slice(0, 200)}`));
});
res.on('error', reject);
});
req.on('error', reject);
// Idle watchdog: abort if the target stops reading (backpressure stalls the tar) so a
// half-dead agent can't hang the backup up to the 6h helper ceiling.
const guarded = withIdleWatchdog(tar, 'put-archive-stream');
guarded.on('error', (e) => req.destroy(e));
guarded.pipe(req); // streams: node writes chunks as the tar produces them, O(1) RAM
});
}
/**
* Stream a tar (a web ReadableStream, e.g. from modern-tar's createTarPacker) into a
* container's PUT /archive with O(1) RAM. Returns false WITHOUT uploading if the env's
* transport can't stream (hawser-edge) — the caller must then use the buffered path.
*/
export async function putContainerArchiveStreaming(
containerId: string,
archivePath: string,
tar: ReadableStream<Uint8Array>,
envId: number | null | undefined,
): Promise<{ streamed: boolean }> {
const config = await getDockerConfig(envId ?? undefined);
if (!transportCanStream(config)) {
console.log(`[Backup] put-archive-stream: env ${envId} is ${config.connectionType} (cannot stream) — caller should use the buffered path`);
return { streamed: false };
}
const extraHeaders: Record<string, string> = {};
if (config.connectionType === 'hawser-standard' && config.hawserToken) {
extraHeaders['X-Hawser-Token'] = config.hawserToken;
}
console.log(`[Backup] put-archive-stream: streaming tar to container ${containerId.slice(0, 12)} via ${config.type} (${config.connectionType ?? 'local'})`);
if (config.type === 'https') {
// httpsAgentRequest streams a ReadableStream body (req.write per chunk) and, seeing a
// stream body, skips the 30s idle timeout meant for small requests - a multi-GB tar PUT
// can legitimately stall >30s (slow disk, mTLS backpressure, GC) without being dead. Run it
// through the idle watchdog (node Readable) so a truly-stalled target still aborts.
const guardedWeb = Readable.toWeb(withIdleWatchdog(Readable.fromWeb(tar as unknown as NodeReadableStream), 'put-archive-stream')) as unknown as ReadableStream<Uint8Array>;
const res = await httpsAgentRequest(config, `/containers/${containerId}/archive?path=${encodeURIComponent(sanitizeArchivePath(archivePath))}`,
{ method: 'PUT', headers: { 'Content-Type': 'application/x-tar', ...extraHeaders }, body: guardedWeb }, false, extraHeaders);
if (!res.ok) throw new Error(`put-archive-stream: PUT /archive -> ${res.status} ${(await res.text()).slice(0, 200)}`);
await res.body?.cancel().catch(() => {});
return { streamed: true };
}
// socket or plain http: node http.request, pipe the tar (converted to a node Readable).
await putViaNodeHttp(config, containerId, archivePath, Readable.fromWeb(tar as unknown as NodeReadableStream), extraHeaders);
return { streamed: true };
}
/** Re-exported for the caller's buffered fallback (edge). */
export { putContainerArchive };
@@ -0,0 +1,35 @@
// Restore-redeploy orchestration, extracted so the STEP ORDER is unit-testable without the
// real port's IO. Invariant: register(internal) runs BEFORE deploy and is NOT gated on the
// deploy succeeding - a failed `docker compose up` must still leave an editable managed stack.
export interface RedeployStackSteps {
/** Restore snapshot secrets into the target env's DB (no-op when none / opted out). */
restoreSecrets(): Promise<void>;
/** Materialise compose/.env into the canonical stack dir. false -> hard error (nothing to deploy). */
materialise(): Promise<boolean>;
/** Register the stack as managed (internal) so the UI can edit/redeploy it. */
register(): Promise<void>;
/** `docker compose up` from the canonical dir. Throwing means the deploy failed. */
deploy(): Promise<void>;
log?(msg: string): void;
}
/** Order: restoreSecrets -> materialise -> register(internal) -> deploy. materialise false is a
* hard error; a deploy throw propagates but only AFTER register(), so it never costs the stack. */
export async function runRedeployStack(steps: RedeployStackSteps): Promise<void> {
const log = steps.log ?? (() => {});
await steps.restoreSecrets();
const wrote = await steps.materialise();
if (!wrote) throw new Error('could not materialise stack files; cannot redeploy');
log('materialised OK');
// Register BEFORE deploy - see the module header. This line order is the whole point.
await steps.register();
log('registered as managed (internal)');
log('redeploying (docker compose up)');
await steps.deploy();
log('redeploy OK');
}
+181
View File
@@ -0,0 +1,181 @@
/**
* stackfile-filter.ts — decide which entries under a stack directory belong in the
* metadata "stackfiles/" capture, and which are bind-mounted DATA that restic already
* backs up separately as a volume.
*
* A common stack layout keeps the bind data inside the stack dir:
*
* stack/
* ├── compose.yaml <- config, MUST stay in stackfiles/ (redeploy reads it)
* ├── .env <- config, MUST stay
* ├── config/ <- bind-mounted -> restic volume, exclude
* └── data/ <- bind-mounted -> restic volume, exclude
*
* We skip any DIRECTORY the compose file bind-mounts from inside the stack dir: it is already
* backed up as its own restic volume, so capturing it here too would double-back-up and blow the
* capture cap, marking the snapshot INCOMPLETE.
*
* WHY WE PARSE THE COMPOSE, NOT the docker Mount.Source:
* The walk reads the stack dir in DOCKHAND's namespace (`/app/data/stacks/...`), but a
* bind's `Mount.Source` is the DAEMON HOST path (`/docker/data/...` or `/tmp/...`) which
* a containerized Dockhand cannot see or stat — so keying the exclusion on Mount.Source
* silently no-ops for every Dockerized install. The compose file, which Dockhand always
* has locally, declares binds RELATIVE to the stack dir (`./data`) — the same namespace
* the walk already computes `relative(stackDir, abs)` in. So we derive the excluded dirs
* from the compose text and compare on relative paths: namespace-agnostic, works
* containerized / on-host / remote alike.
*
* Hard rules (adversarial review):
* - NEVER exclude compose/`.yaml`/`.yml`/`.env` (`isLoadBearingStackFile`) — a whole-dir
* bind `./:/app` must not strand them or restore can't redeploy.
* - Exclude only DIRECTORY binds — a single-FILE bind (`./cfg.yaml:/x`) is config kept.
* - Named volumes (`type: volume`, or a non-relative source) are NEVER excluded.
* - A `${VAR}` in a bind path is left CAPTURED (conservative over-capture, never drops
* a needed file).
*
* Pure and dependency-light (js-yaml is already a repo dep); the directory check is
* injected so it stays unit-testable.
*/
import yaml from 'js-yaml';
/** Strip a trailing slash and a leading `./`; keep the rest. */
function normRel(p: string): string {
let s = p.trim().replace(/\/+$/, '');
if (s.startsWith('./')) s = s.slice(2);
return s;
}
/** A load-bearing stack file that must NEVER be excluded, regardless of a bind that
* happens to cover it: compose/`.yaml`/`.yml` (the compose graph) and `.env*` at any
* level. Data dirs are excluded by the bind rule, not by extension. */
export function isLoadBearingStackFile(relPath: string): boolean {
const base = relPath.split('/').pop() ?? relPath;
if (base === '.env' || base.startsWith('.env.')) return true;
return /\.ya?ml$/i.test(base);
}
/** The bind `source` from one compose `volumes:` entry, short or long syntax, or null
* if it isn't a bind we care about (named volume, no source). */
function bindSourceOf(entry: unknown): string | null {
if (typeof entry === 'string') {
// "src:target[:mode]" — a bind if src is a path (starts with . or /). A bare
// name ("db_data:/x") is a NAMED VOLUME — not a bind, skip.
const src = entry.split(':')[0];
if (src.startsWith('.') || src.startsWith('/')) return src;
return null;
}
if (entry && typeof entry === 'object') {
const e = entry as { type?: string; source?: string };
if (e.type === 'volume') return null; // named volume
if (typeof e.source === 'string') {
if (e.type === 'bind' || e.source.startsWith('.') || e.source.startsWith('/')) return e.source;
}
}
return null;
}
/**
* Directory bind sources declared in the compose file, expressed as paths RELATIVE to
* the stack dir — e.g. `['data', 'config']`. `stackDir` is used only to relativize an
* absolute bind that points into the stack dir. `isDirRel(rel)` decides whether a given
* stack-dir-relative path is a directory (in the walk's own namespace) — a single-file
* bind is NOT returned. `${VAR}` sources are dropped (left to be captured).
*/
export function relativeBindDirsFromCompose(
composeContent: string,
stackDir: string,
isDirRel: (rel: string) => boolean
): string[] {
let doc: unknown;
try { doc = yaml.load(composeContent); } catch { return []; }
const services = (doc as { services?: Record<string, unknown> } | null)?.services;
if (!services || typeof services !== 'object') return [];
const rels = new Set<string>();
for (const svc of Object.values(services)) {
const vols = (svc as { volumes?: unknown[] } | null)?.volumes;
if (!Array.isArray(vols)) continue;
for (const entry of vols) {
const src = bindSourceOf(entry);
if (!src) continue;
if (src.includes('${')) continue; // unresolved var -> capture, don't risk dropping
let rel: string;
if (src.startsWith('/')) {
// Absolute bind: keep only if it resolves INSIDE the stack dir.
const r = relativePosix(stackDir, src);
if (r === '' || r.startsWith('..')) continue;
rel = r;
} else {
rel = normRel(src);
if (rel === '' || rel.startsWith('..')) continue; // escapes stack dir
}
// A whole-dir bind (`.`, `./`, or an absolute source == the stack dir) would
// mean "exclude everything" — refuse it: it would strip config sidecars that
// aren't load-bearing (nginx.conf, certs/). Over-capture the whole dir instead
// (safe: nothing lost, compose kept). isLoadBearingStackFile can't rescue a
// non-yaml sidecar, so the only safe move here is to not exclude at all.
if (rel === '.' || rel === '') continue;
if (isDirRel(rel)) rels.add(rel);
}
}
return [...rels];
}
/**
* Every RELATIVE bind's path below the stack dir - FILES and DIRS alike (`./config.yaml`,
* `./data`). Unlike relativeBindDirsFromCompose (dirs only, for EXCLUDE), this feeds host-path
* DERIVATION: a single-file bind's daemon-reported mount.Source still pins the stack dir (strip
* the tail), so a bind-less-but-single-file stack resolves without depending on the working_dir
* label (which needs the container to be listable - a timing race under load). Whole-dir binds
* (`.`/`./`) are skipped (the source IS the stack dir; the bind-derived path adds nothing).
*/
export function relativeBindsFromCompose(composeContent: string, stackDir: string): string[] {
let doc: unknown;
try { doc = yaml.load(composeContent); } catch { return []; }
const services = (doc as { services?: Record<string, unknown> } | null)?.services;
if (!services || typeof services !== 'object') return [];
const rels = new Set<string>();
for (const svc of Object.values(services)) {
const vols = (svc as { volumes?: unknown[] } | null)?.volumes;
if (!Array.isArray(vols)) continue;
for (const entry of vols) {
const src = bindSourceOf(entry);
if (!src || src.includes('${')) continue;
let rel: string;
if (src.startsWith('/')) {
const r = relativePosix(stackDir, src);
if (r === '' || r.startsWith('..')) continue;
rel = r;
} else {
rel = normRel(src);
if (rel === '' || rel.startsWith('..')) continue;
}
if (rel === '.' || rel === '') continue;
rels.add(rel);
}
}
return [...rels];
}
/** posix `relative(from, to)` on already-clean absolute paths (no realpath). */
function relativePosix(from: string, to: string): string {
const a = from.replace(/\/+$/, '').split('/');
const b = to.replace(/\/+$/, '').split('/');
let i = 0;
while (i < a.length && i < b.length && a[i] === b[i]) i++;
const up = a.slice(i).map(() => '..');
return [...up, ...b.slice(i)].join('/');
}
/** True when `relPath` IS one of the excluded relative dirs, or lives inside one.
* Segment-aware so `data-backup` doesn't match `data`. */
export function isUnderRelDir(relPath: string, relDirs: string[]): boolean {
const t = normRel(relPath);
for (const raw of relDirs) {
const d = normRel(raw);
if (t === d) return true;
if (t.startsWith(d + '/')) return true;
}
return false;
}
+54 -6
View File
@@ -1,14 +1,15 @@
/**
* Build the tar we put-archive (docker cp) into the backup helper, replacing the
* base64-in-Cmd approach that blew ARG_MAX on large stack files.
* Build the tar we put-archive (docker cp) into the backup helper.
*
* `modern-tar` is zero-dependency and emits PAX headers, so paths of ANY length
* round-trip (the old hand-rolled USTAR writer threw on names >100 bytes, breaking
* deep stack trees). Zero deps also matters for CI: it installs into a symlinked
* node_modules where libs with transitive deps fail to resolve them.
* round-trip (USTAR caps names at 100 bytes, breaking deep stack trees). Zero deps
* also matters for CI: it installs into a symlinked node_modules where libs with
* transitive deps fail to resolve them.
*/
import { packTar } from 'modern-tar';
import { packTar, createTarPacker } from 'modern-tar';
import { createReadStream, statSync } from 'node:fs';
import { Readable } from 'node:stream';
export interface TarEntry {
/** Path inside the archive, e.g. "metadata/stacks/foo/compose.yaml". A leading slash is stripped. */
@@ -16,6 +17,53 @@ export interface TarEntry {
content: Uint8Array;
}
/** A file to stream into the tar lazily from disk (no in-RAM contents). */
export interface TarFileSource {
/** Absolute path on disk to read from. */
diskPath: string;
/** Path inside the archive (leading slash stripped). */
archivePath: string;
}
/**
* Build a tar as a streaming web ReadableStream, reading each file from disk LAZILY (O(1)
* RAM regardless of total size). Used by the backup helper's streaming put-archive so a
* huge stack dir never buffers in memory. Entries are produced in the given order; the
* consumer (the HTTP request) drives the pace via backpressure.
*
* `inlineEntries` (e.g. the small metadata.json) are written first from memory; then each
* file source is streamed from disk. NOTE: the header size is fixed from statSync but the
* body is read lazily via createReadStream, so a file that GROWS or SHRINKS between stat and
* read makes modern-tar's packer throw ("exceeds given size" / "Size mismatch") and errors
* the whole stream - the run FAILS loudly (it is NOT silently tolerated). This is acceptable
* because the backup helper's stack dir is quiescent (the stack is stopped or the files are
* config); a mid-capture size change is a real problem the caller should see, not hide.
*/
export function buildTarStream(sources: TarFileSource[], inlineEntries: TarEntry[] = [], mtimeSecs = Math.floor(Date.now() / 1000)): ReadableStream<Uint8Array> {
const packer = createTarPacker();
const mtime = new Date(mtimeSecs * 1000);
(async () => {
try {
for (const e of inlineEntries) {
const w = packer.controller.add({ name: e.path.replace(/^\/+/, ''), size: e.content.length, mode: 0o644, mtime, uid: 0, gid: 0, uname: 'root', gname: 'root', type: 'file' });
await Readable.toWeb(Readable.from(Buffer.from(e.content))).pipeTo(w);
}
for (const s of sources) {
const size = statSync(s.diskPath).size;
const w = packer.controller.add({ name: s.archivePath.replace(/^\/+/, ''), size, mode: 0o644, mtime, uid: 0, gid: 0, uname: 'root', gname: 'root', type: 'file' });
await Readable.toWeb(createReadStream(s.diskPath)).pipeTo(w);
}
packer.controller.finalize();
} catch (err) {
// Surface the failure to the consumer by erroring the stream.
packer.controller.error(err);
}
})();
return packer.readable;
}
/** Build a tar (any path length via PAX). `mtimeSecs` injectable for deterministic tests. */
export function buildTar(entries: TarEntry[], mtimeSecs = Math.floor(Date.now() / 1000)): Promise<Uint8Array> {
return packTar(
+40
View File
@@ -0,0 +1,40 @@
/**
* Pure decision logic for staging a stack's files onto a `direct` remote host (no server
* deps - safe to unit test). The IO (helper container + put-archive) lives in
* stage-remote-stackfiles.ts; this module only decides IF and WHERE.
*/
export interface RemoteStagingPlan {
stage: boolean;
/** Absolute project dir on the remote host (<remoteBase>/<stack>), when stage is true. */
projectDir?: string;
reason: string;
}
/**
* Decide whether a `direct` deploy should stage files onto the remote host, and where.
*
* Staging happens when ALL hold:
* - operation is 'up' (down/stop/start/restart don't re-stage),
* - the env has a non-empty remote_stacks_dir configured,
* - there are stack files to stage.
* Setting remote_stacks_dir is the user's explicit "keep my stacks here on the host", so we
* stage the WHOLE stack dir (compose + .env + sibling config) - not only when the compose has
* relative binds. Relative binds still need it to resolve, but a bind-less stack needs its
* compose on the host just as much: to be manageable and BACKUPABLE (the helper bind-mounts the
* host dir; without the compose there, a backup can't capture it).
* Any miss => stage:false => the deploy takes the exact current code path (no --project-directory).
*/
export function planRemoteStaging(input: {
operation: string;
remoteStacksDir: string | null | undefined;
stackName: string;
composeContent: string;
hasStackFiles: boolean;
}): RemoteStagingPlan {
if (input.operation !== 'up') return { stage: false, reason: `operation ${input.operation} does not stage` };
if (!input.hasStackFiles) return { stage: false, reason: 'no stack files to stage' };
const base = typeof input.remoteStacksDir === 'string' ? input.remoteStacksDir.trim().replace(/\/+$/, '') : '';
if (!base) return { stage: false, reason: 'no remote_stacks_dir configured on the env' };
return { stage: true, projectDir: `${base}/${input.stackName}`, reason: 'direct env with remote_stacks_dir' };
}
+132 -17
View File
@@ -41,6 +41,7 @@ import {
import { unregisterSchedule } from './scheduler';
import { sendEventNotification } from './notifications';
import { deleteGitStackFiles, parseEnvFileContent } from './git';
import { isDeletableStackDir } from './stack-delete-guard';
import { cleanPem } from '$lib/utils/pem';
import { rewriteComposeVolumePaths, getHostDataDir } from './host-path';
import { getOrderValue } from './container-labels';
@@ -988,7 +989,8 @@ async function executeLocalCompose(
serviceName?: string,
build?: boolean,
noBuildCache?: boolean,
pullPolicy?: string
pullPolicy?: string,
remoteProjectDir?: string
): Promise<StackOperationResult> {
const logPrefix = `[Stack:${stackName}]`;
@@ -1142,6 +1144,15 @@ async function executeLocalCompose(
const useStdin = finalComposeContent !== composeContent;
const args = ['docker', 'compose', '-p', stackName];
// For a `direct` remote daemon we can't share Dockhand's filesystem, so relative binds
// (./config, ./data) are staged onto the remote host under `remoteProjectDir` and the
// compose is pointed there with --project-directory: compose resolves `./x` against that
// remote path (which need not exist locally) instead of the client cwd, so the daemon
// binds the staged files. undefined = current behavior (no staging, no flag).
if (remoteProjectDir) {
args.push('--project-directory', remoteProjectDir);
}
// Temp file for path-translated override content (cleaned up in finally block)
let tempOverridePath: string | undefined;
@@ -1169,6 +1180,12 @@ async function executeLocalCompose(
args.push('-f', overrideFile);
console.log(`${logPrefix} Including override file: ${basename(overrideFile)}`);
}
} else if (remoteProjectDir) {
// Internal stack + remote staging: --project-directory repoints compose's base to the
// remote path, which breaks the usual cwd auto-discovery of compose.yaml (that file is
// on Dockhand's host, not under the remote project dir). Point -f at the local file
// explicitly so compose reads it locally while resolving ./binds against the remote dir.
args.push('-f', composeFile);
}
// else: internal stack without path translation - no -f needed.
// Docker Compose auto-discovers compose.yaml + compose.override.yaml from cwd.
@@ -1683,6 +1700,34 @@ async function executeComposeCommand(
skipVerify: env.tlsSkipVerify ?? false
} : undefined;
// A direct daemon shares no filesystem with Dockhand, so relative binds
// (./config, ./data) would resolve to a path the remote daemon can't see and
// Docker would auto-create empty dirs. When the env has a `remote_stacks_dir`
// set AND the compose has relative binds, stage the stack files onto the remote
// host under <remoteDir>/<stack> and point compose there with --project-directory.
// The plan() decides IF/WHERE (pure, unit-tested); anything it declines takes the
// exact current path (remoteProjectDir stays undefined = no flag, no staging).
let remoteProjectDir: string | undefined;
{
const { getEnvSetting } = await import('./db');
const { planRemoteStaging } = await import('./remote-staging-plan');
const remoteStacksDir = await getEnvSetting('remote_stacks_dir', envId ?? undefined);
// Stage the WHOLE local stack dir (compose + relative-bind files like nginx.conf,
// ./data). The local stack dir is config-only - runtime data written by the remote
// container never comes back here - so this can't clobber live data. The tar is
// STREAMED from disk (O(1) RAM), so a large ./data doesn't buffer in memory.
const hasLocalDir = !!(operation === 'up' && workingDir && existsSync(workingDir));
const plan = planRemoteStaging({
operation, remoteStacksDir, stackName, composeContent, hasStackFiles: hasLocalDir,
});
if (plan.stage && plan.projectDir && workingDir) {
const { stageStackDirOnRemote } = await import('./stage-remote-stackfiles');
const { staged } = await stageStackDirOnRemote(envId!, plan.projectDir, workingDir);
console.log(`[Stack:${stackName}] direct env: staged ${staged} file(s) to ${plan.projectDir} on the remote host (${plan.reason})`);
remoteProjectDir = plan.projectDir;
}
}
return executeLocalCompose(
operation,
stackName,
@@ -1701,7 +1746,8 @@ async function executeComposeCommand(
serviceName,
build,
noBuildCache,
pullPolicy
pullPolicy,
remoteProjectDir
);
}
@@ -2331,11 +2377,66 @@ export async function downStack(
* Remove a stack completely (compose down + delete files + cleanup database)
* Uses stack locking to prevent concurrent operations.
*/
/**
* Compute exactly which on-disk directories a `removeStack(..., deleteFiles)` would delete,
* WITHOUT deleting anything. The delete-preview endpoint uses this so the confirm modal
* shows the user the same paths the backend will actually remove — one source of truth, no
* frontend/backend drift (the class of bug behind #675). Adopted stacks whose files live
* outside DATA_DIR are reported as `null` (Dockhand never deletes those).
*/
export async function computeStackDeletionPaths(
stackName: string,
envId?: number | null
): Promise<{ stackDir: string | null; gitDir: string | null; sourceType: string | null; namedVolumes: string[] }> {
const stackSource = await getStackSource(stackName, envId);
const stacksDir = getStacksDir();
// Named volumes compose created for this stack — exactly what `down --volumes` would
// remove (compose-managed, labeled with the project). Best-effort: a docker/API hiccup
// just yields an empty list (the delete still works; the modal just won't preview them).
let namedVolumes: string[] = [];
try {
const { listVolumes } = await import('./docker.js');
const vols = await listVolumes(envId);
namedVolumes = vols
.filter((v) => v.labels?.['com.docker.compose.project'] === stackName)
.map((v) => v.name)
.sort();
} catch { /* best-effort */ }
let stackDir: string | null = null;
if (stackSource?.composePath) {
const customDir = dirname(stackSource.composePath);
// SAME strict guard as removeStack (#675): strict subdir + basename match.
if (isDeletableStackDir(customDir, stacksDir, stackName) && existsSync(customDir)) {
stackDir = customDir;
}
}
if (!stackDir && !stackSource?.composePath) {
const defaultDir = await findStackDir(stackName, envId) || await getStackDir(stackName, envId);
if (existsSync(defaultDir)) stackDir = defaultDir;
}
// Git stacks additionally have a cloned repo dir that removeStack deletes.
let gitDir: string | null = null;
const gitStack = await getGitStackByName(stackName, envId);
if (gitStack) {
try {
const { getStackRepoPath } = await import('./git');
const repoPath = await getStackRepoPath(gitStack.id, gitStack.stackName, gitStack.environmentId);
if (repoPath && existsSync(repoPath)) gitDir = repoPath;
} catch { /* best-effort: no git dir shown if we can't resolve it */ }
}
return { stackDir, gitDir, sourceType: stackSource?.sourceType ?? null, namedVolumes };
}
export async function removeStack(
stackName: string,
envId?: number | null,
force = false,
removeVolumes = false
removeVolumes = false,
deleteFiles = true
): Promise<StackOperationResult> {
return withStackLock(stackName, async () => {
// Get compose file (may not exist for external stacks)
@@ -2395,6 +2496,23 @@ export async function removeStack(
// Remove any dynamically-spawned child containers not handled by compose
await cleanupOrphanStackContainers(stackName, envId, 'remove');
// Local stack files ARE deleted below, but only under the DATA_DIR strict guard
// (#675) - Dockhand owns that dir. A direct env's REMOTE staged dir has no such
// guard: remote_stacks_dir is a user path that can hold co-located user data, and
// nothing distinguishes a staged file from the user's own there. So we never delete
// it - a stale compose is safe residue; an rm -rf could wipe user data.
if (deleteFiles && envId != null) {
try {
const { getEnvironment, getEnvSetting } = await import('./db');
const env = await getEnvironment(envId);
if (env?.connectionType === 'direct') {
const remoteStacksDir = await getEnvSetting('remote_stacks_dir', envId);
const base = typeof remoteStacksDir === 'string' ? remoteStacksDir.trim().replace(/\/+$/, '') : '';
if (base) console.log(`[Stack:${stackName}] leaving staged files at ${base}/${stackName} on the remote host (not deleting - may hold user data)`);
}
} catch { /* log-only, never blocks removal */ }
}
} else {
// External stack - remove containers directly in parallel
const { removeContainer } = await import('./docker.js');
@@ -2461,16 +2579,10 @@ export async function removeStack(
let stackDir: string | null = null;
if (stackSource?.composePath) {
// Check if the compose path is within Dockhand's stacks directory
// Only delete if the compose dir is a STRICT SUBDIR of DATA_DIR/stacks/ whose
// basename matches the stack name (#675 guard; see stack-delete-guard.ts).
const customDir = dirname(stackSource.composePath);
const resolvedCustomDir = resolve(customDir);
const resolvedStacksDir = resolve(stacksDir);
// Only delete if the directory is within DATA_DIR/stacks/ (files we created)
// AND the directory basename matches the stack name exactly (for safety)
if (resolvedCustomDir.startsWith(resolvedStacksDir) &&
basename(resolvedCustomDir) === stackName &&
existsSync(customDir)) {
if (isDeletableStackDir(customDir, stacksDir, stackName) && existsSync(customDir)) {
stackDir = customDir;
}
}
@@ -2484,8 +2596,10 @@ export async function removeStack(
}
}
// Delete the directory if found
if (stackDir) {
// Delete the directory if found — but ONLY when the caller asked to remove files.
// "Remove stack" (deleteFiles=false) leaves the compose/.env/data on disk; "Remove
// stack + files" (default) deletes them.
if (stackDir && deleteFiles) {
try {
rmSync(stackDir, { recursive: true, force: true });
} catch (err: any) {
@@ -2512,19 +2626,20 @@ export async function removeStack(
cleanupErrors.push(`env vars: ${err.message}`);
}
// If git stack, clean up git stack record
// If git stack, clean up git stack record. The DB record always goes (the stack is
// gone); the cloned repo FILES on disk go only when deleteFiles is set.
try {
const gitStack = await getGitStackByName(stackName, envId);
if (gitStack) {
await deleteGitStack(gitStack.id);
await deleteGitStackFiles(gitStack.id, gitStack.stackName, gitStack.environmentId);
if (deleteFiles) await deleteGitStackFiles(gitStack.id, gitStack.stackName, gitStack.environmentId);
}
// Also cleanup any orphaned git stacks with NULL environment_id for this stack name
if (envId !== undefined && envId !== null) {
const orphanedGitStack = await getGitStackByName(stackName, null);
if (orphanedGitStack) {
await deleteGitStack(orphanedGitStack.id);
await deleteGitStackFiles(orphanedGitStack.id, orphanedGitStack.stackName, orphanedGitStack.environmentId);
if (deleteFiles) await deleteGitStackFiles(orphanedGitStack.id, orphanedGitStack.stackName, orphanedGitStack.environmentId);
}
}
} catch (err: any) {
+91
View File
@@ -0,0 +1,91 @@
/**
* Stage a stack's files onto a REMOTE `direct` daemon's host filesystem.
*
* A `direct` env has no agent and no shared FS with Dockhand, so a `docker compose up`
* over TCP resolves relative binds (`./config.yaml`, `./data`) against a path the remote
* daemon can't see - it auto-creates empty dirs. This function puts the stack dir onto the
* remote host under `remoteDir` (compose points there via --project-directory), using a
* throwaway helper container + the Docker put-archive API - the same mechanism the backup
* helper uses to write to a remote host. After this runs, the relative binds resolve.
*
* The tar is STREAMED from disk (O(1) RAM) via the same primitive the backup helper uses:
* a stack dir with a large relative bind (a few-GB ./data) must not buffer in memory.
* DIRECT streams (transportCanStream is true for everything but hawser-edge, which never
* reaches this code - staging is direct-only).
*
* DIRECT ONLY. Socket shares the FS; Hawser has an agent that writes files itself.
*/
import { runContainerWithStreaming } from './docker';
import { buildTarStream, type TarFileSource } from './backups/tar';
import { putContainerArchiveStreaming } from './backups/put-archive-stream';
import { readdirSync } from 'node:fs';
import { join, relative, sep } from 'node:path';
const STAGER_IMAGE = 'alpine:latest';
export interface StageResult {
staged: number;
remoteDir: string;
}
/** Walk a stack dir into TarFileSource[] (disk paths, no in-RAM contents), so the tar
* streams lazily. `.git` is skipped. There is deliberately NO per-file size cap: the tar
* streams from disk with O(1) RAM, so a large relative-bind file (a big ./data config)
* must stage in full - a cap would defeat the whole point of streaming. */
function collectSources(rootDir: string): TarFileSource[] {
const sources: TarFileSource[] = [];
const walk = (dir: string) => {
for (const entry of readdirSync(dir, { withFileTypes: true })) {
if (entry.name === '.git') continue;
const full = join(dir, entry.name);
if (entry.isDirectory()) { walk(full); continue; }
if (!entry.isFile()) continue;
// archivePath is POSIX-relative to rootDir (tar entries under remoteDir).
sources.push({ diskPath: full, archivePath: relative(rootDir, full).split(sep).join('/') });
}
};
walk(rootDir);
return sources;
}
/**
* Stream the stack dir at `localDir` onto the remote host at `remoteDir` via a helper
* container. Throws on failure so the caller fails loudly instead of deploying against
* empty dirs (a silent no-op deploy).
*
* @param envId the direct env whose daemon hosts the files
* @param remoteDir absolute path on the remote host (validated by the API layer)
* @param localDir the local stack dir to stage (its files land under remoteDir 1:1)
*/
export async function stageStackDirOnRemote(
envId: number,
remoteDir: string,
localDir: string
): Promise<StageResult> {
const sources = collectSources(localDir);
if (sources.length === 0) return { staged: 0, remoteDir };
const name = `dockhand-stage-${envId}-${Math.floor(Date.now() / 1000)}`;
await runContainerWithStreaming({
image: STAGER_IMAGE,
// The container only needs to exist for the put-archive in beforeStart; the bind mount
// itself creates remoteDir on the remote host, then it exits immediately.
cmd: ['sh', '-c', `mkdir -p '${remoteDir.replace(/'/g, "'\\''")}'`],
binds: [`${remoteDir}:${remoteDir}:rw`],
name,
envId,
timeout: 300000,
labels: { 'io.dockhand.role': 'stackfile-stager' },
// After create, before start: the bind mount already exists, so a streaming put-archive
// lands the files on the remote host FS under remoteDir with O(1) RAM.
beforeStart: async (containerId: string) => {
const tar = buildTarStream(sources);
const { streamed } = await putContainerArchiveStreaming(containerId, remoteDir, tar, envId);
// direct always streams; a false here would mean a transport we don't stage on.
if (!streamed) throw new Error(`remote staging: transport for env ${envId} cannot stream the stack dir`);
},
});
return { staged: sources.length, remoteDir };
}
+3
View File
@@ -95,6 +95,9 @@ export interface BackupOptions {
excludeCaches?: boolean;
webhookSuccess?: string;
webhookFailure?: string;
/** Stack-dir entries the user DEselected in the "Stack files on the host" picker; excluded
* from the stack-dir capture (never load-bearing compose/.env - the server re-guards). */
excludedStackFiles?: string[];
}
export function parseRetention(retention: string | RetentionPolicy | null | undefined): RetentionPolicy {
+7 -3
View File
@@ -1,6 +1,7 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { authorize } from '$lib/server/authorize';
import { requireBackups } from '$lib/server/backups/route-guards';
import { runRestore } from '$lib/server/backups';
import { validateRestoreRequest, type RestoreMode } from '$lib/server/backups/validate';
import { createJobResponse } from '$lib/server/sse';
@@ -10,9 +11,8 @@ import { guardSnapshotEnvAccess } from '$lib/server/backups/route-guards';
export const POST: RequestHandler = async (event) => {
const { request, cookies } = event;
const auth = await authorize(cookies);
if (auth.authEnabled && !await auth.can('backups', 'manage')) {
return json({ error: 'Permission denied' }, { status: 403 });
}
const rbacDenied = await requireBackups(auth, 'manage');
if (rbacDenied) return rbacDenied;
const body = await request.json();
const mode: RestoreMode = body.mode === 'in-place' ? 'in-place' : 'new-location';
@@ -25,6 +25,9 @@ export const POST: RequestHandler = async (event) => {
// The client sends false to bring the stack up without them. Only a stack redeploy
// consumes this; container restores ignore it.
const restoreSecrets = body.restoreSecrets !== false;
// New-location STACK restore: skip the captured compose/.env (data-only restore),
// and/or ADOPT the restored stack into Dockhand afterwards (opt-in). Both default off.
const skipStackFiles = body.skipStackFiles === true;
// Authorization gates run BEFORE request validation, so an unauthorized caller
// never learns anything about the request shape.
@@ -76,6 +79,7 @@ export const POST: RequestHandler = async (event) => {
postRestore,
volumeDestinations,
restoreSecrets,
skipStackFiles,
},
access,
// Stream progress to the client (the restore modal's log). Without this the
@@ -0,0 +1,32 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { authorize } from '$lib/server/authorize';
import { requireBackups } from '$lib/server/backups/route-guards';
import { probeStackDir } from '$lib/server/backups';
// GET /api/backup/stack-dir-listing?target=<stack>&env=<id>
// Probe the TARGET host and list the actual contents of the stack directory, so the
// create/edit backup dialog can show the resolved host path + let the user pick which
// stack-dir entries to back up. Returns { kind:'listed', hostPath, entries } or
// { kind:'unknown', reason } (never throws for an operational failure).
export const GET: RequestHandler = async ({ url, cookies }) => {
const auth = await authorize(cookies);
const denied = await requireBackups(auth, 'view');
if (denied) return denied;
const targetName = url.searchParams.get('target');
if (!targetName) return json({ error: 'target is required' }, { status: 400 });
const envParam = url.searchParams.get('env');
const environmentId = envParam ? parseInt(envParam) : null;
if (environmentId != null && auth.isEnterprise && !await auth.canAccessEnvironment(environmentId)) {
return json({ error: 'Environment access denied' }, { status: 403 });
}
try {
const result = await probeStackDir(targetName, environmentId);
return json(result);
} catch (e) {
return json({ kind: 'unknown', reason: e instanceof Error ? e.message : 'failed to probe stack directory' });
}
};
@@ -0,0 +1,31 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { authorize } from '$lib/server/authorize';
import { requireBackups } from '$lib/server/backups/route-guards';
import { previewStackBackupPath } from '$lib/server/backups';
// GET /api/backup/stack-path?target=<stack>&env=<id>
// Preview WHERE a stack's directory will be captured from on the host (the resolved host
// bind path), or `unknown` with a reason - so the create-schedule dialog can show it up front
// and warn before scheduling a backup that would hard-fail the stack-dir probe.
export const GET: RequestHandler = async ({ url, cookies }) => {
const auth = await authorize(cookies);
const denied = await requireBackups(auth, 'view');
if (denied) return denied;
const targetName = url.searchParams.get('target');
if (!targetName) return json({ error: 'target is required' }, { status: 400 });
const envParam = url.searchParams.get('env');
const environmentId = envParam ? parseInt(envParam) : null;
if (environmentId != null && auth.isEnterprise && !await auth.canAccessEnvironment(environmentId)) {
return json({ error: 'Environment access denied' }, { status: 403 });
}
try {
const result = await previewStackBackupPath(targetName, environmentId);
return json(result);
} catch (e) {
return json({ kind: 'unknown', reason: e instanceof Error ? e.message : 'failed to resolve stack path' });
}
};
@@ -0,0 +1,53 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { authorize } from '$lib/server/authorize';
import { getEnvironment, getEnvSetting, setEnvSetting } from '$lib/server/db';
// Per-env "remote stacks directory" for `direct` environments. A direct daemon has no
// agent and no shared filesystem with Dockhand, so relative bind files (./config.yaml,
// ./data) never reach it. When this is set, Dockhand stages the stack files onto the
// remote host under this dir before `docker compose up`. Empty/unset = current behavior.
// Stored via the generic per-env settings store (no schema column).
export const GET: RequestHandler = async ({ params, cookies }) => {
const auth = await authorize(cookies);
if (auth.authEnabled && !(await auth.can('environments', 'view'))) {
return json({ error: 'Permission denied' }, { status: 403 });
}
const id = parseInt(params.id);
const env = await getEnvironment(id);
if (!env) return json({ error: 'Environment not found' }, { status: 404 });
const stored = await getEnvSetting('remote_stacks_dir', id);
const remoteStacksDir = stored && typeof stored === 'string' && stored.trim() !== '' ? stored : null;
return json({ remoteStacksDir });
};
export const POST: RequestHandler = async ({ params, request, cookies }) => {
const auth = await authorize(cookies);
if (auth.authEnabled && !(await auth.can('environments', 'edit'))) {
return json({ error: 'Permission denied' }, { status: 403 });
}
const id = parseInt(params.id);
const env = await getEnvironment(id);
if (!env) return json({ error: 'Environment not found' }, { status: 404 });
const data = await request.json().catch(() => ({}));
const raw = data?.remoteStacksDir;
// null / '' clears the setting (reverts to current behavior).
if (raw === null || raw === undefined || raw === '') {
await setEnvSetting('remote_stacks_dir', '', id);
return json({ success: true, remoteStacksDir: null });
}
if (typeof raw !== 'string') {
return json({ error: 'remoteStacksDir must be a string or null' }, { status: 400 });
}
// Must be an absolute path on the remote host; reject relative or traversal-y input.
const value = raw.trim();
if (!value.startsWith('/') || value.includes('..')) {
return json({ error: 'remoteStacksDir must be an absolute path with no ".."' }, { status: 400 });
}
await setEnvSetting('remote_stacks_dir', value, id);
return json({ success: true, remoteStacksDir: value });
};
+59 -4
View File
@@ -13,6 +13,7 @@
import { getRepoTypeIcon, formatCron, runBackupAction, type BackupFormState } from '$lib/utils/backup';
import { toast } from 'svelte-sonner';
import VolumePicker from '$lib/components/backup/VolumePicker.svelte';
import StackFilesPicker from '$lib/components/backup/StackFilesPicker.svelte';
import DestinationPicker from '$lib/components/backup/DestinationPicker.svelte';
import { normalizeMounts, volumesForStack, type VolumeInfo } from '$lib/utils/mounts';
@@ -47,6 +48,45 @@
let allVolumes = $state(true);
let selectedVolumes = $state<string[]>([]);
// Stack files on the host (probe listing) — stack targets only.
type StackListing =
| { kind: 'listed'; hostPath: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
| { kind: 'tar'; localStackDir: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
| { kind: 'helper-failed'; reason: string }
| { kind: 'unknown'; reason: string }
| null;
let stackListing = $state<StackListing>(null);
// The probe helper couldn't run on the target - a backup can't run either, so block Save.
const stackHelperFailed = $derived(stackListing?.kind === 'helper-failed');
let loadingStackListing = $state(false);
let excludedStackFiles = $state<string[]>([]);
// The user-set "Remote stack path (for backup)" for direct/hawser envs, shown in the picker
// tooltip so the user sees THEIR configured path (empty when not set / socket env).
let stackRemoteDir = $state<string>('');
async function fetchStackListing(item: ContainerItem) {
if (item.type !== 'stack') { stackListing = null; return; }
loadingStackListing = true;
stackListing = null;
excludedStackFiles = [];
stackRemoteDir = '';
if (item.envId != null) {
fetch(`/api/environments/${item.envId}/remote-stacks-dir`)
.then((r) => (r.ok ? r.json() : null))
.then((d) => { stackRemoteDir = d?.remoteStacksDir ?? ''; })
.catch(() => { /* non-fatal - tooltip just omits the configured path */ });
}
try {
const envQ = item.envId != null ? `&env=${item.envId}` : '';
const res = await fetch(`/api/backup/stack-dir-listing?target=${encodeURIComponent(item.name)}${envQ}`);
stackListing = await res.json();
} catch (e) {
stackListing = { kind: 'unknown', reason: e instanceof Error ? e.message : 'probe failed' };
} finally {
loadingStackListing = false;
}
}
// Step 3: Schedule & Run
let saveSchedule = $state(false);
let schedule = $state('0 2 * * *');
@@ -126,6 +166,7 @@
function selectSource(item: ContainerItem) {
selectedItem = item;
step = 2;
fetchStackListing(item);
}
// Save the config, then (when run=true) hand it off to the parent's shared
@@ -141,7 +182,8 @@
destinationId: selectedDestId,
stopBeforeBackup,
allVolumes,
selectedVolumes
selectedVolumes,
options: excludedStackFiles.length > 0 ? { excludedStackFiles } : undefined
};
try {
const result = await runBackupAction({
@@ -182,7 +224,7 @@
</script>
<Dialog.Root bind:open onOpenChange={(isOpen) => { if (!isOpen && saving) return; open = isOpen; }}>
<Dialog.Content class="max-w-3xl h-[70vh] flex flex-col">
<Dialog.Content class="max-w-5xl h-[85vh] flex flex-col">
<Dialog.Header class="pb-0">
<Dialog.Title class="flex items-center gap-2 text-base">
<Package class="w-4 h-4" />Create backup
@@ -316,6 +358,19 @@
</div>
</div>
{#if selectedItem?.type === 'stack'}
<StackFilesPicker
listing={stackListing}
loading={loadingStackListing}
connectionType={selectedEnv?.connectionType}
envName={selectedEnv?.name}
envIcon={selectedEnv?.icon}
envId={selectedEnv?.id}
configuredStackPath={stackRemoteDir}
bind:excludedStackFiles
/>
{/if}
{#if selectedItem}
<VolumePicker
volumes={selectedItem.volumes}
@@ -367,12 +422,12 @@
<Button variant="outline" size="sm" onclick={() => step = 2} disabled={saving}>Back</Button>
<div class="flex-1"></div>
{#if saveSchedule}
<Button variant="outline" size="sm" onclick={() => saveAndMaybeRun(false)} disabled={saving || scheduleInvalid}>
<Button variant="outline" size="sm" onclick={() => saveAndMaybeRun(false)} disabled={saving || scheduleInvalid || stackHelperFailed}>
{#if saving}<Loader2 class="w-3.5 h-3.5 mr-1.5 animate-spin" />{:else}<CheckCircle2 class="w-3.5 h-3.5 mr-1.5" />{/if}
Save schedule only
</Button>
{/if}
<Button size="sm" onclick={() => saveAndMaybeRun(true)} disabled={saving || (saveSchedule && scheduleInvalid)}>
<Button size="sm" onclick={() => saveAndMaybeRun(true)} disabled={saving || (saveSchedule && scheduleInvalid) || stackHelperFailed}>
{#if saving}<Loader2 class="w-3.5 h-3.5 mr-1.5 animate-spin" />{:else}<Play class="w-3.5 h-3.5 mr-1.5" />{/if}
{saveSchedule ? 'Run & save schedule' : 'Run backup now'}
</Button>
+73 -5
View File
@@ -7,9 +7,10 @@
import { Badge } from '$lib/components/ui/badge';
import { TogglePill } from '$lib/components/ui/toggle-pill';
import CronEditor from '$lib/components/cron-editor.svelte';
import { Save, Play, Pause, RefreshCw, CheckCircle, XCircle, ChevronDown, Loader2, Plus, Trash2, Pencil, X, Check, HardDrive, Box, Layers, FileText } from 'lucide-svelte';
import { Save, Play, Pause, RefreshCw, CheckCircle, XCircle, ChevronDown, Loader2, Plus, Trash2, Pencil, X, Check, HardDrive, Box, Layers, FileText, FolderOpen, AlertTriangle } from 'lucide-svelte';
import RotateCwFadingClock from '$lib/components/icons/RotateCwFadingClock.svelte';
import VolumePicker from '$lib/components/backup/VolumePicker.svelte';
import StackFilesPicker from '$lib/components/backup/StackFilesPicker.svelte';
import DestinationPicker from '$lib/components/backup/DestinationPicker.svelte';
import LogConsole from '$lib/components/LogConsole.svelte';
import SnapshotsPanel from './SnapshotsPanel.svelte';
@@ -23,10 +24,11 @@
import { watchJob } from '$lib/utils/sse-fetch';
import ConfirmPopover from '$lib/components/ConfirmPopover.svelte';
import { getRepoTypeIcon, parseRetention, parseOptions, retentionSummary as getRetentionSummary, formatCron, runBackupAction, classifyJobResult, tagLogLine, fetchBackupExecutions, type BackupAction, type BackupFormState } from '$lib/utils/backup';
import { reconcileSelectedVolumeKeys } from '$lib/utils/mounts';
interface Props {
containerName: string;
volumes: { name: string; mountPoint: string; mountType?: 'volume' | 'bind' }[];
volumes: { key: string; name: string; mountPoint: string; mountType?: 'volume' | 'bind' }[];
type?: 'container' | 'stack';
environmentId?: number;
onConfigSaved?: () => void;
@@ -238,9 +240,12 @@
editLimitDownload = opts.limitDownload || '';
editWebhookSuccess = opts.webhookSuccess || '';
editWebhookFailure = opts.webhookFailure || '';
excludedStackFiles = Array.isArray(opts.excludedStackFiles) ? opts.excludedStackFiles : [];
editStopBefore = cfg.stopBeforeBackup;
editAllVolumes = cfg.selectedVolumes === null;
editSelectedVolumes = cfg.selectedVolumes ? (typeof cfg.selectedVolumes === 'string' ? JSON.parse(cfg.selectedVolumes) : cfg.selectedVolumes) : [];
const storedSel = cfg.selectedVolumes ? (typeof cfg.selectedVolumes === 'string' ? JSON.parse(cfg.selectedVolumes) : cfg.selectedVolumes) : [];
// Reconcile a stored selection (may hold legacy names) to the current volume keys.
editSelectedVolumes = reconcileSelectedVolumeKeys(volumes, storedSel);
showAdvanced = false;
}
@@ -283,9 +288,11 @@
if (editWebhookFailure !== (opts.webhookFailure || '')) return true;
if (editStopBefore !== cfg.stopBeforeBackup) return true;
if (editAllVolumes !== (cfg.selectedVolumes === null)) return true;
const origVolumes = cfg.selectedVolumes
const storedSel = cfg.selectedVolumes
? (typeof cfg.selectedVolumes === 'string' ? JSON.parse(cfg.selectedVolumes) : cfg.selectedVolumes)
: [];
// Reconcile the baseline to keys too, so a name->key format shift alone isn't seen as dirty.
const origVolumes = reconcileSelectedVolumeKeys(volumes, storedSel);
if (JSON.stringify([...editSelectedVolumes].sort()) !== JSON.stringify([...origVolumes].sort())) return true;
return false;
}
@@ -314,7 +321,8 @@
limitUpload: editLimitUpload || undefined,
limitDownload: editLimitDownload || undefined,
webhookSuccess: editWebhookSuccess || undefined,
webhookFailure: editWebhookFailure || undefined
webhookFailure: editWebhookFailure || undefined,
excludedStackFiles: excludedStackFiles.length > 0 ? excludedStackFiles : undefined
}
};
}
@@ -468,7 +476,53 @@
return `${sel.length} of ${volumes.length} volumes`;
}
// Resolved host stack-dir path (stacks only): WHERE the helper will read the stack folder
// from on the target host. Shown in the create form so the user sees what gets captured -
// and a clear warning up front when it can't be located (would hard-fail the backup).
let stackPath = $state<{ kind: 'candidate'; hostPath: string; composeFile: string } | { kind: 'unknown'; reason: string } | null>(null);
async function fetchStackPath() {
if (type !== 'stack') return;
try {
const params = new URLSearchParams({ target: containerName });
// Use the derived envId (prop OR the currently selected environment), the SAME
// value the rest of the panel uses for save/run - the bare `environmentId` prop is
// often undefined (the modal doesn't pass it), which made the server report
// "No environment specified".
if (envId != null) params.set('env', String(envId));
const res = await fetch(`/api/backup/stack-path?${params}`);
if (res.ok) stackPath = await res.json();
} catch { /* non-fatal: the path preview is informational */ }
}
// Stack files on the host (probe listing) — supersedes the plain stackPath callout for the
// picker, but stackPath is still fetched for the compose-file name shown elsewhere.
type StackListing =
| { kind: 'listed'; hostPath: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
| { kind: 'tar'; localStackDir: string; entries: { name: string; type: 'dir' | 'file'; size: number; capturedAs?: 'bind' | 'volume' }[] }
| { kind: 'unknown'; reason: string }
| null;
let stackListing = $state<StackListing>(null);
let loadingStackListing = $state(false);
let excludedStackFiles = $state<string[]>([]);
async function fetchStackListing() {
if (type !== 'stack') { stackListing = null; return; }
loadingStackListing = true;
try {
const params = new URLSearchParams({ target: containerName });
if (envId != null) params.set('env', String(envId));
const res = await fetch(`/api/backup/stack-dir-listing?${params}`);
stackListing = await res.json();
} catch (e) {
stackListing = { kind: 'unknown', reason: e instanceof Error ? e.message : 'probe failed' };
} finally {
loadingStackListing = false;
}
}
onMount(() => { fetchDestinations(); fetchConfigs(); });
// Resolve the stack path once envId is known (it's derived from $currentEnvironment, which
// may not be ready at mount). Re-runs if the environment changes.
$effect(() => { if (type === 'stack' && envId != null) { void fetchStackPath(); void fetchStackListing(); } });
</script>
<!-- Sub-tabs: Schedules (cheap) vs Snapshots. SnapshotsPanel is mounted as soon as
@@ -610,6 +664,20 @@
<TogglePill bind:checked={editStopBefore} />
<Label class="text-xs">Stop {type} during backup</Label>
</div>
<!-- Stack files on the host: probe the host, show the resolved dir + let the user
pick which entries to back up. Shown BEFORE the volume list. -->
{#if type === 'stack'}
<StackFilesPicker
listing={stackListing}
loading={loadingStackListing}
connectionType={fullEnv?.connectionType}
envName={fullEnv?.name}
envIcon={fullEnv?.icon}
envId={fullEnv?.id}
bind:excludedStackFiles
/>
{/if}
<!-- Volume picker (all-or-subset toggle + per-volume checkboxes) -->
<VolumePicker
volumes={volumes}
@@ -275,6 +275,9 @@
let formHost = $state('');
let formPort = $state(2375); // Default for direct Docker connection
let formProtocol = $state('http');
// direct-only: where Dockhand stages compose + relative-bind files on the remote host,
// so `./config`/`./data` binds resolve. Empty = current behavior (relative binds unsupported).
let formRemoteStacksDir = $state('');
let formTlsCa = $state('');
let formTlsCert = $state('');
let formTlsKey = $state('');
@@ -294,6 +297,11 @@
let formDiskWarningThreshold = $state(80);
let formDiskWarningThresholdGb = $state(50);
let formConnectionType = $state<ConnectionType>('socket');
// Envs that keep stack files on a REMOTE host, so backup needs a declared stack path:
// direct (Dockhand stages there, drives deploy + backup) and hawser (the agent keeps files
// at its STACKS_DIR; the path is backup-only, does not steer deploy).
const usesStackPath = (ct: ConnectionType) => ct === 'direct' || ct === 'hawser-standard' || ct === 'hawser-edge';
const isHawserConn = (ct: ConnectionType) => ct === 'hawser-standard' || ct === 'hawser-edge';
let formHawserToken = $state('');
let formLabels = $state<string[]>([]);
let newLabelInput = $state('');
@@ -598,6 +606,7 @@
loadImagePruneSettings(environment.id);
loadTimezone(environment.id);
loadDiskWarningSettings(environment.id);
if (usesStackPath(formConnectionType)) loadRemoteStacksDir(environment.id);
// Load Hawser token if edge mode
if (formConnectionType === 'hawser-edge') {
loadHawserToken(environment.id);
@@ -607,6 +616,7 @@
formHost = '';
formPort = 2375;
formProtocol = 'http';
formRemoteStacksDir = '';
formTlsCa = '';
formTlsCert = '';
formTlsKey = '';
@@ -883,6 +893,7 @@
if (newEnv?.id) {
await saveTimezone(newEnv.id);
await saveDiskWarningSettings(newEnv.id);
if (usesStackPath(formConnectionType)) await saveRemoteStacksDir(newEnv.id);
}
onSaved();
onClose();
@@ -1019,6 +1030,7 @@
await saveImagePruneSettings(environment.id);
await saveTimezone(environment.id);
await saveDiskWarningSettings(environment.id);
if (usesStackPath(formConnectionType)) await saveRemoteStacksDir(environment.id);
toast.success(`Updated environment: ${formName}`);
onSaved();
onClose();
@@ -1049,6 +1061,30 @@
}
}
async function loadRemoteStacksDir(envId: number) {
try {
const response = await fetch(`/api/environments/${envId}/remote-stacks-dir`);
if (response.ok) {
const data = await response.json();
formRemoteStacksDir = data.remoteStacksDir ?? '';
}
} catch (error) {
console.error('Failed to load remote stacks dir:', error);
}
}
async function saveRemoteStacksDir(envId: number) {
try {
await fetch(`/api/environments/${envId}/remote-stacks-dir`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ remoteStacksDir: formRemoteStacksDir.trim() || null })
});
} catch (error) {
console.error('Failed to save remote stacks dir:', error);
}
}
async function saveDiskWarningSettings(envId: number) {
try {
await fetch(`/api/environments/${envId}/disk-warning`, {
@@ -1607,7 +1643,7 @@
</Tabs.Trigger>
</Tabs.List>
<div class="overflow-y-auto py-4 h-[520px] [scrollbar-gutter:stable] pr-3">
<div class="overflow-y-auto py-4 h-[520px] [scrollbar-gutter:stable] pr-5">
<!-- General Tab (Connection Settings) -->
<Tabs.Content value="general" class="space-y-4 mt-0 h-full">
<!-- Name field -->
@@ -1911,6 +1947,75 @@
</div>
{/if}
<!-- Stack path for backup (direct = also drives deploy; hawser = backup-only) -->
{#if usesStackPath(formConnectionType)}
<div class="space-y-2">
<div class="flex items-center gap-1.5">
<Label for="edit-env-remote-stacks-dir">
{isHawserConn(formConnectionType) ? 'Remote stack path (for backup)' : 'Remote stacks directory'}
<span class="text-muted-foreground font-normal">(optional)</span>
</Label>
<Tooltip.Root>
<Tooltip.Trigger type="button" class="text-muted-foreground hover:text-foreground">
<HelpCircle class="w-3.5 h-3.5" />
</Tooltip.Trigger>
<Tooltip.Content class="w-80 z-[200]" side="right">
{#if isHawserConn(formConnectionType)}
<div class="space-y-2">
<p class="font-medium">Where the agent keeps stack files</p>
<p class="text-muted-foreground">
The Hawser agent stores each stack's folder at
<code class="bg-muted px-1 rounded">&lt;this path&gt;/&lt;stack&gt;</code>
on <span class="font-medium text-foreground">its own host</span>. Backup reads
the compose and config from there. This does <span class="font-medium text-foreground">not</span>
change where deploy or restore write - the agent always uses its own
<code class="bg-muted px-1 rounded">STACKS_DIR</code>; this only tells backup where to look.
</p>
<p class="text-muted-foreground">
So set it to MATCH the agent's <code class="bg-muted px-1 rounded">STACKS_DIR</code>
(default <code class="bg-muted px-1 rounded">/data/stacks</code>). Leave empty for
the default; set it only if the agent runs with a custom one.
</p>
</div>
{:else}
<div class="space-y-2">
<p class="font-medium">How this path is used</p>
<p class="text-muted-foreground">
A direct daemon shares no filesystem with Dockhand. When set, Dockhand
materialises each stack's whole folder - compose, <code class="bg-muted px-1 rounded">.env</code>,
and relative-bind files (<code class="bg-muted px-1 rounded">./config</code>,
<code class="bg-muted px-1 rounded">./data</code>) - into
<code class="bg-muted px-1 rounded">&lt;this path&gt;/&lt;stack&gt;</code>
<span class="font-medium text-foreground">on the remote host</span>, and runs compose there
via <code class="bg-muted px-1 rounded">--project-directory</code>.
</p>
<p class="text-muted-foreground">
This applies to normal stacks and git sync alike, so relative binds resolve
and every stack is backupable. Leave empty to use only named volumes or
absolute paths.
</p>
</div>
{/if}
</Tooltip.Content>
</Tooltip.Root>
</div>
<Input
id="edit-env-remote-stacks-dir"
bind:value={formRemoteStacksDir}
placeholder={isHawserConn(formConnectionType) ? '/data/stacks' : '/opt/dockhand/stacks'}
/>
<p class="text-xs text-muted-foreground">
{#if isHawserConn(formConnectionType)}
Absolute path on the agent's host where it keeps stack folders. Used only to back up
each stack's compose and config. Leave empty for the default <code class="bg-muted px-1 rounded">/data/stacks</code>.
{:else}
Absolute path on the remote host where Dockhand stages each stack's files before deploying.
Set it for stacks with relative binds, or to keep stacks backupable on a remote direct daemon.
{/if}
</p>
</div>
{/if}
<!-- Direct connection settings -->
{#if formConnectionType === 'direct'}
<div class="grid grid-cols-2 gap-4">