container updates respect the new image's command/entrypoint

src/lib/server/container-env-merge.ts

@@ -33,6 +33,39 @@
 export interface ImageEnvLabels {
 	Env?: string[];
 	Labels?: Record<string, string>;
+	/** The image's baked CMD / ENTRYPOINT, for the same value-aware rebase (#1371). */
+	Cmd?: string[] | null;
+	Entrypoint?: string[] | null;
+}
+
+/**
+ * Rebase a container's Cmd/Entrypoint onto the new image (#1371). Same flattening problem
+ * as Env/Labels: a container's Config.Cmd is set at create time whether it came from the
+ * image's CMD or a user `command:` override - spreading it verbatim onto a new image freezes
+ * the OLD image's command, so a container can start with a command the NEW image no longer
+ * provides (missing entrypoint script -> exit 127 -> restart loop).
+ *
+ * The recoverable signal is the same VALUE comparison:
+ *   container == oldImage  -> came from the old image, user never overrode it -> adopt newImage
+ *   container != oldImage  -> a real user/Compose override                    -> keep container
+ * Cmd/Entrypoint are ordered arrays, so equality is order-sensitive (JSON compare). When the
+ * old image value is unknown (null/undefined - caller couldn't inspect it) we KEEP the
+ * container's value (verbatim fallback), exactly as the env/label rebase does.
+ */
+export function rebaseCommand(
+	containerValue: string[] | null | undefined,
+	oldImageValue: string[] | null | undefined,
+	newImageValue: string[] | null | undefined
+): { value: string[] | null | undefined; adopted: boolean } {
+	// Old image not inspectable -> can't tell image-default from user override -> keep verbatim.
+	if (oldImageValue == null) return { value: containerValue, adopted: false };
+	const eq = (a: string[] | null | undefined, b: string[] | null | undefined) =>
+		JSON.stringify(a ?? null) === JSON.stringify(b ?? null);
+	// User overrode it (differs from the old image's baked value) -> keep the override.
+	if (!eq(containerValue, oldImageValue)) return { value: containerValue, adopted: false };
+	// Image-baked and untouched -> follow the new image. (Same value -> no-op, adopted:false.)
+	if (eq(newImageValue, oldImageValue)) return { value: containerValue, adopted: false };
+	return { value: newImageValue, adopted: true };
 }
 
 /**

src/lib/server/docker.ts

@@ -12,10 +12,12 @@ import * as http from 'node:http';
 import * as https from 'node:https';
 import * as tls from 'node:tls';
 import { createHash } from 'node:crypto';
+import { pumpWebStreamToWritable } from './stream-pump';
+import { computeRequestTimeoutMs } from './backups/request-timeout';
 import type { Environment } from './db';
 import { getSetting } from './db';
-import { getAdditionalVolumeBinds } from './mount-dedupe';
-import { rebaseEnvOntoImage, rebaseLabelsOntoImage, describeEnvRebase, describeLabelRebase, type ImageEnvLabels } from './container-env-merge';
+import { getAdditionalVolumeBinds, dedupeVolumesForRecreate } from './mount-dedupe';
+import { rebaseEnvOntoImage, rebaseLabelsOntoImage, rebaseCommand, describeEnvRebase, describeLabelRebase, type ImageEnvLabels } from './container-env-merge';
 import { encodeRegistryAuth } from './registry-auth';
 import { isSystemContainer, classifyEmptyDigestImage } from './scheduler/tasks/update-utils';
 import { deepDiff } from '../utils/diff.js';
@@ -417,12 +419,16 @@ export function httpsAgentRequest(
 			headers: reqHeaders,
 		};
 
-		if (!streaming) {
-			const isComposeOperation = path === '/_hawser/compose';
-			const composeTimeoutMs = parseInt(process.env.COMPOSE_TIMEOUT || '900') * 1000;
-			const isPrune = path.endsWith('/prune');
-			reqOptions.timeout = isComposeOperation ? composeTimeoutMs : isPrune ? 300000 : 30000;
-		}
+		// A stream request BODY (a large tar PUT) must not get the small-request idle timeout
+		// even though the RESPONSE isn't streamed - computeRequestTimeoutMs decides on both.
+		const streamingBody = typeof (options.body as ReadableStream | undefined)?.getReader === 'function';
+		const timeoutMs = computeRequestTimeoutMs({
+			path,
+			streamingBody,
+			streamingResponse: streaming,
+			composeTimeoutSecs: parseInt(process.env.COMPOSE_TIMEOUT || '900'),
+		});
+		if (timeoutMs !== null) reqOptions.timeout = timeoutMs;
 
 		// Honor AbortSignal from caller (e.g., AbortSignal.timeout(5000) for ping)
 		const signal = options.signal as AbortSignal | undefined;
@@ -493,19 +499,10 @@ export function httpsAgentRequest(
 			} else if (body instanceof Blob) {
 				body.arrayBuffer().then(ab => req.end(Buffer.from(ab)), reject);
 			} else if (typeof (body as ReadableStream).getReader === 'function') {
+				// Stream the body with backpressure (O(1) RAM). Without awaiting 'drain' this
+				// piled the whole tar into RAM on a slow mTLS peer - see stream-pump.ts.
 				const reader = (body as ReadableStream<Uint8Array>).getReader();
-				(async () => {
-					try {
-						while (true) {
-							const { done, value } = await reader.read();
-							if (done) break;
-							req.write(value);
-						}
-						req.end();
-					} catch (err) {
-						req.destroy(err as Error);
-					}
-				})();
+				pumpWebStreamToWritable(reader, req).catch(() => { /* req.destroy already called */ });
 			} else {
 				req.end();
 			}
@@ -574,7 +571,7 @@ function buildConfigFromEnv(env: Environment): DockerClientConfig {
 /**
  * Get Docker client configuration for an environment
  */
-async function getDockerConfig(envId?: number | null): Promise<DockerClientConfig> {
+export async function getDockerConfig(envId?: number | null): Promise<DockerClientConfig> {
 	if (!envId) {
 		throw new Error('No environment specified');
 	}
@@ -2016,7 +2013,12 @@ export async function recreateContainerFromInspect(
 		let oldImg = oldImageConfig ?? null;
 		if (!oldImg) {
 			const oldInspect = await inspectImage(inspectData.Image || config.Image, envId);
-			oldImg = { Env: (oldInspect as any)?.Config?.Env, Labels: (oldInspect as any)?.Config?.Labels };
+			oldImg = {
+				Env: (oldInspect as any)?.Config?.Env,
+				Labels: (oldInspect as any)?.Config?.Labels,
+				Cmd: (oldInspect as any)?.Config?.Cmd ?? null,
+				Entrypoint: (oldInspect as any)?.Config?.Entrypoint ?? null,
+			};
 		}
 		const newImg = await inspectImage(newImage, envId);
 		const oldEnv = oldImg?.Env || [];
@@ -2026,6 +2028,18 @@ export async function recreateContainerFromInspect(
 		createConfig.Env = rebaseEnvOntoImage(config.Env || [], oldEnv, newEnv);
 		createConfig.Labels = rebaseLabelsOntoImage(config.Labels || {}, oldLabels, newLabels);
 
+		// 4b. Rebase Cmd/Entrypoint the same way (#1371): a container's Config.Cmd is flattened at
+		// create time, so the OLD image's baked CMD is indistinguishable from a user `command:`
+		// override and would be frozen onto the new image - a container can then start with a
+		// command the new image no longer provides (missing script -> exit 127 -> restart loop).
+		// oldImg?.Cmd == null (caller didn't capture it / inspect failed) -> keep verbatim.
+		const cmdR = rebaseCommand(config.Cmd, oldImg?.Cmd, (newImg as any)?.Config?.Cmd);
+		const entR = rebaseCommand(config.Entrypoint, oldImg?.Entrypoint, (newImg as any)?.Config?.Entrypoint);
+		createConfig.Cmd = cmdR.value;
+		createConfig.Entrypoint = entR.value;
+		if (cmdR.adopted) log?.(`Cmd — adopted from new image: ${JSON.stringify(cmdR.value)} (was ${JSON.stringify(config.Cmd)})`);
+		if (entR.adopted) log?.(`Entrypoint — adopted from new image: ${JSON.stringify(entR.value)} (was ${JSON.stringify(config.Entrypoint)})`);
+
 		const envSummary = describeEnvRebase(config.Env || [], oldEnv, newEnv);
 		const labelSummary = describeLabelRebase(config.Labels || {}, oldLabels, newLabels);
 		const fmt = (label: string, keys: string[]) => keys.length ? `${label}: ${keys.join(', ')}` : '';
@@ -2094,39 +2108,20 @@ export async function recreateContainerFromInspect(
 		}
 	}
 
-	// Deduplicate: remove Config.Volumes entries that conflict with HostConfig.Tmpfs or Binds.
-	// Read-only containers get tmpfs at paths like /tmp that may also be declared as image volumes.
-	// Docker rejects duplicate mount points, so the tmpfs/bind mount wins over the volume declaration.
-	if (createConfig.Volumes && hostConfig) {
-		const mountedPaths = new Set<string>();
-		if (hostConfig.Tmpfs) {
-			for (const p of Object.keys(hostConfig.Tmpfs)) {
-				mountedPaths.add(p);
-			}
-		}
-		if (hostConfig.Binds) {
-			for (const b of hostConfig.Binds) {
-				const parts = b.split(':');
-				if (parts.length >= 2) mountedPaths.add(parts[1].split(':')[0]);
-			}
-		}
-		if (mountedPaths.size > 0) {
-			for (const volPath of Object.keys(createConfig.Volumes)) {
-				if (mountedPaths.has(volPath)) {
-					delete createConfig.Volumes[volPath];
-				}
-			}
-			if (Object.keys(createConfig.Volumes).length === 0) {
-				delete createConfig.Volumes;
-			}
-		}
+	// Deduplicate Config.Volumes against every mount point that will exist on the new container
+	// (Tmpfs, existing binds, the binds we re-add below, and inspect.Mounts). Docker rejects
+	// duplicate mount points; the tmpfs/bind/volume mount wins over the bare image-VOLUME
+	// declaration. Compute additionalBinds first so the dedup sees them (#1088 / #1363).
+	const additionalBinds = getAdditionalVolumeBinds(hostConfig || {}, inspectData.Mounts || []);
+	if (hostConfig) {
+		const kept = dedupeVolumesForRecreate(createConfig.Volumes, hostConfig, inspectData.Mounts || [], additionalBinds);
+		if (kept) createConfig.Volumes = kept;
+		else delete createConfig.Volumes;
 	}
-
-	const additionalBinds = getAdditionalVolumeBinds(hostConfig, inspectData.Mounts || []);
 	if (additionalBinds.length > 0) {
 		createConfig.HostConfig = {
 			...hostConfig,
-			Binds: [...(hostConfig.Binds || []), ...additionalBinds]
+			Binds: [...(hostConfig?.Binds || []), ...additionalBinds]
 		};
 	}
 
@@ -2298,32 +2293,18 @@ export async function createContainerFromMetadata(
 		}
 	}
 
-	// Volume/Tmpfs deduplication
-	if (createConfig.Volumes && hostConfig) {
-		const mountedPaths = new Set<string>();
-		if (hostConfig.Tmpfs) {
-			for (const p of Object.keys(hostConfig.Tmpfs)) mountedPaths.add(p);
-		}
-		if (hostConfig.Binds) {
-			for (const b of hostConfig.Binds) {
-				const parts = b.split(':');
-				if (parts.length >= 2) mountedPaths.add(parts[1].split(':')[0]);
-			}
-		}
-		if (mountedPaths.size > 0) {
-			for (const volPath of Object.keys(createConfig.Volumes)) {
-				if (mountedPaths.has(volPath)) delete createConfig.Volumes[volPath];
-			}
-			if (Object.keys(createConfig.Volumes).length === 0) delete createConfig.Volumes;
-		}
+	// Preserve anonymous volumes, then dedup Config.Volumes against every resulting mount point
+	// (incl. these added binds + metadata.mounts) so recreate never sends a duplicate (#1088 / #1363).
+	const additionalBinds = getAdditionalVolumeBinds(hostConfig || {}, metadata.mounts || []);
+	if (hostConfig) {
+		const kept = dedupeVolumesForRecreate(createConfig.Volumes, hostConfig, metadata.mounts || [], additionalBinds);
+		if (kept) createConfig.Volumes = kept;
+		else delete createConfig.Volumes;
 	}
-
-	// Preserve anonymous volumes
-	const additionalBinds = getAdditionalVolumeBinds(hostConfig, metadata.mounts || []);
 	if (additionalBinds.length > 0) {
 		createConfig.HostConfig = {
 			...hostConfig,
-			Binds: [...(hostConfig.Binds || []), ...additionalBinds]
+			Binds: [...(hostConfig?.Binds || []), ...additionalBinds]
 		};
 	}
 
@@ -3399,6 +3380,10 @@ export async function getRegistryAuth(
 	return { baseUrl, orgPath: parsed.path, authHeader };
 }
 
+// Catalog-failure classification lives in an import-light module so it's unit-testable
+// without pulling this file's better-sqlite3 imports into the test process (#873).
+export { classifyCatalogFailure, CATALOG_NOT_SUPPORTED_MSG } from './registry-catalog.js';
+
 // --- Harbor fallback for catalog and image search ---
 // Harbor denies access to the V2 _catalog endpoint for robot accounts.
 // We detect Harbor and use the native project API as a fallback.
@@ -4538,11 +4523,16 @@ export async function pruneImages(dangling = true, envId?: number | null) {
 	}
 
 	const { DEFAULT_GRYPE_IMAGE, DEFAULT_TRIVY_IMAGE } = await import('./scanner');
+	const { DEFAULT_HELPER_IMAGE } = await import('./backups/restic');
 	const grypeImg = (await getSetting('default_grype_image')) ?? DEFAULT_GRYPE_IMAGE;
 	const trivyImg = (await getSetting('default_trivy_image')) ?? DEFAULT_TRIVY_IMAGE;
+	// The backup/restore helper image is pulled once and reused for every backup; shield it from
+	// prune too (same holder-container trick), so an unused-image prune between backups doesn't
+	// force a slow re-pull (and a fail on an air-tight/offline host).
+	const helperImg = (await getSetting('default_backup_image')) || DEFAULT_HELPER_IMAGE;
 
 	const holderIds: string[] = [];
-	for (const image of [grypeImg, trivyImg]) {
+	for (const image of [grypeImg, trivyImg, helperImg]) {
 		try {
 			const safeName = `dockhand-prune-keep-${image.replace(/[^a-z0-9]/gi, '-')}-${Date.now()}`;
 			const created = await dockerJsonRequest<{ Id: string }>(
@@ -4861,6 +4851,40 @@ export async function runContainer(options: {
 }
 
 // Run a container with attached streams (for scanners that need real-time output)
+/**
+ * DIAGNOSTIC (issue #1344): when a helper's exit code can't be determined, pull the
+ * recent `docker events` for that container so the log shows WHAT acted on it (a
+ * `die` with exitCode, a `kill` with the signal, a `destroy`/`remove` and its actor).
+ * `docker events` carries `Actor.Attributes` incl. the signal and, on some daemons, the
+ * execID/name of who issued it. Bounded, non-streaming (since..until), best-effort.
+ */
+async function logRecentContainerEvents(containerId: string, name: string, sinceISO: string | undefined, envId: number | null | undefined): Promise<void> {
+	try {
+		const sinceSec = sinceISO ? Math.floor(new Date(sinceISO).getTime() / 1000) - 2 : Math.floor(Date.now() / 1000) - 120;
+		const untilSec = Math.floor(Date.now() / 1000) + 1;
+		const filters = encodeURIComponent(JSON.stringify({ container: [containerId] }));
+		const resp = await dockerFetch(`/events?since=${sinceSec}&until=${untilSec}&filters=${filters}`, {}, envId);
+		if (!resp.ok) { console.warn(`[runContainerWithStreaming] events fetch for ${name}: HTTP ${resp.status}`); return; }
+		const body = await resp.text();
+		// The events endpoint returns newline-delimited JSON objects.
+		const lines = body.split('\n').map((l) => l.trim()).filter(Boolean);
+		if (lines.length === 0) { console.warn(`[runContainerWithStreaming] no docker events for ${name} in the window (nothing recorded who touched it)`); return; }
+		for (const line of lines) {
+			try {
+				const ev = JSON.parse(line) as { status?: string; Action?: string; Actor?: { Attributes?: Record<string, string> }; time?: number };
+				const action = ev.Action ?? ev.status ?? '(unknown)';
+				const attrs = ev.Actor?.Attributes ?? {};
+				const sig = attrs.signal ? ` signal=${attrs.signal}` : '';
+				const code = attrs.exitCode ? ` exitCode=${attrs.exitCode}` : '';
+				const exec = attrs.execID ? ` execID=${attrs.execID}` : '';
+				console.warn(`[runContainerWithStreaming] EVENT ${name}: ${action}${sig}${code}${exec}`);
+			} catch { /* skip a non-JSON line */ }
+		}
+	} catch (err) {
+		console.warn(`[runContainerWithStreaming] events fetch failed for ${name}: ${(err as Error).message}`);
+	}
+}
+
 export async function runContainerWithStreaming(options: {
 	image: string;
 	cmd: string[];
@@ -4890,7 +4914,11 @@ export async function runContainerWithStreaming(options: {
 		Image: options.image,
 		Cmd: options.cmd,
 		Env: options.env || [],
-		Labels: options.labels || {},
+		// Every helper spawned here (backup/restore/probe, image scanner) is ephemeral and exits
+		// on purpose, so it must NOT fire a "Container died/exited" notification like a user's
+		// container would. dockhand.notify=false makes the event handler skip it
+		// (isNotifyDisabledByLabel). Caller labels win if they set notify explicitly.
+		Labels: { 'dockhand.notify': 'false', ...(options.labels || {}) },
 		Tty: false,
 		HostConfig: {
 			Binds: options.binds || [],
@@ -4938,6 +4966,12 @@ export async function runContainerWithStreaming(options: {
 			// Start container
 			await drainResponse(await dockerFetch(`/containers/${containerId}/start`, { method: 'POST' }, options.envId));
 
+			// DIAGNOSTIC (issue #1344): one line confirming the helper actually started,
+			// with the config that could explain an early death (bind count, network,
+			// user). Pairs with the "exited with code" / "EXIT-CODE INDETERMINATE" lines
+			// so a user's log shows the full lifecycle of a helper that dies mid-run.
+			console.log(`[runContainerWithStreaming] Started ${containerName} (binds=${(options.binds || []).length}, netMode=${options.networkMode ?? 'default'}, user=${options.user ?? 'default'})`);
+
 			// Create abort controller to cancel stderr stream when container exits
 			// On some Docker hosts (e.g. Synology NAS with older kernels), follow=true
 			// streams don't close when the container exits, causing indefinite hangs.
@@ -4960,34 +4994,100 @@ export async function runContainerWithStreaming(options: {
 			// Bounded-retry the wait; if it still can't be determined, fall back to
 			// inspecting the container's final State, and if that's also unavailable
 			// leave exitCode undefined — the guard below fails closed on undefined.
+			// HELPERS_WAIT_MODE (default 'wait'): the exit signal below is a single
+			// streaming `POST /wait`, held open with no bytes for the whole run. A
+			// socket-proxy / reverse-proxy / tcp DOCKER_HOST with a short idle timeout can
+			// sever it mid-run, leaving the exit code unreadable even though the helper is
+			// fine (#1344). Set HELPERS_WAIT_MODE=poll to POLL short inspect calls instead
+			// (nothing stays open for a proxy to cut). Applies to every helper this function
+			// runs — backup/restore AND the image scanner.
 			let exitCode: number | undefined;
-			for (let attempt = 0; attempt < 3 && exitCode === undefined; attempt++) {
-				try {
-					const waitResult = await dockerFetch(`/containers/${containerId}/wait`, { method: 'POST', streaming: true }, options.envId);
-					if (!waitResult.ok) await throwDockerError(waitResult);
-					const waitData = await waitResult.json() as { StatusCode?: number };
-					exitCode = waitData.StatusCode;
-					console.log(`[runContainerWithStreaming] Container exited with code: ${exitCode}`);
-				} catch (err) {
-					console.warn(`[runContainerWithStreaming] Wait attempt ${attempt + 1} failed: ${(err as Error).message}`);
+			if (process.env.HELPERS_WAIT_MODE === 'poll') {
+				// Bound by the caller's timeout (backup passes a long 'data' tier, the
+				// scanner passes 600_000). Fall back to 1h so poll mode never hangs.
+				// One line so the log confirms which exit-signal mode is active (helps
+				// diagnose whether HELPERS_WAIT_MODE=poll actually took effect - #1344).
+				console.log(`[runContainerWithStreaming] Awaiting exit of ${options.name ?? containerId.slice(0, 12)} via POLL mode`);
+				const deadline = Date.now() + (options.timeout && options.timeout > 0 ? options.timeout : 3_600_000);
+				while (exitCode === undefined && Date.now() < deadline) {
+					try {
+						const insp = await dockerFetch(`/containers/${containerId}/json`, {}, options.envId);
+						if (insp.ok) {
+							const data = await insp.json() as { State?: { Status?: string; Running?: boolean; Dead?: boolean; ExitCode?: number } };
+							const st = data.State;
+							if (st && st.Running === false && st.Status === 'exited' && typeof st.ExitCode === 'number') {
+								exitCode = st.ExitCode;
+								console.log(`[runContainerWithStreaming] Container exited with code: ${exitCode} (poll mode)`);
+								break;
+							}
+							if (st?.Dead || st?.Status === 'removing') break; // gone -> fail-closed diagnostics below
+						} else if (insp.status === 404) {
+							console.warn(`[runContainerWithStreaming] Poll: ${options.name ?? containerId.slice(0, 12)} not found (404) — container removed`);
+							break;
+						}
+					} catch (err) {
+						// A dropped short inspect is cheap to retry — keep polling to the deadline
+						// rather than bailing (unlike the streaming /wait path).
+						console.warn(`[runContainerWithStreaming] Poll inspect failed for ${options.name ?? containerId.slice(0, 12)}: ${(err as Error).message}`);
+					}
+					if (exitCode === undefined) await new Promise((r) => setTimeout(r, 1500));
+				}
+			} else {
+				for (let attempt = 0; attempt < 3 && exitCode === undefined; attempt++) {
+					try {
+						const waitResult = await dockerFetch(`/containers/${containerId}/wait`, { method: 'POST', streaming: true }, options.envId);
+						if (!waitResult.ok) await throwDockerError(waitResult);
+						const waitData = await waitResult.json() as { StatusCode?: number };
+						exitCode = waitData.StatusCode;
+						console.log(`[runContainerWithStreaming] Container exited with code: ${exitCode}`);
+					} catch (err) {
+						// DIAGNOSTIC (issue #1344): include the attempt count so a user whose
+						// helper's exit can't be read can tell us whether /wait failed once
+						// (transient) or all 3 times (the container vanished / daemon churn).
+						console.warn(`[runContainerWithStreaming] Wait attempt ${attempt + 1}/3 failed for ${options.name ?? containerId.slice(0, 12)}: ${(err as Error).message}`);
+					}
 				}
 			}
 			if (exitCode === undefined) {
 				// Fallback: inspect the container. Only trust a definitively-exited
 				// state; a still-running/restarting/unknown state stays undefined.
+				let containerStartedAt: string | undefined;
 				try {
 					const insp = await dockerFetch(`/containers/${containerId}/json`, {}, options.envId);
 					if (insp.ok) {
-						const data = await insp.json() as { State?: { Status?: string; Running?: boolean; ExitCode?: number } };
+						const data = await insp.json() as { State?: { Status?: string; Running?: boolean; Restarting?: boolean; Dead?: boolean; Paused?: boolean; ExitCode?: number; OOMKilled?: boolean; Error?: string; StartedAt?: string; FinishedAt?: string; Pid?: number } };
 						const st = data.State;
+						containerStartedAt = st?.StartedAt;
+						// DIAGNOSTIC (issue #1344): /wait gave no code and the fallback is
+						// about to leave exitCode undefined -> the user sees "could not
+						// determine container exit status". Log the FULL final State so a
+						// user hitting this can tell us WHY (OOMKilled, Dead, removed by an
+						// external reaper, still Running = /wait raced, or a daemon Error).
+						console.warn(
+							`[runContainerWithStreaming] EXIT-CODE INDETERMINATE for ${options.name ?? containerId.slice(0, 12)} ` +
+							`State=${JSON.stringify({
+								Status: st?.Status, Running: st?.Running, Restarting: st?.Restarting,
+								Dead: st?.Dead, Paused: st?.Paused, ExitCode: st?.ExitCode,
+								OOMKilled: st?.OOMKilled, Error: st?.Error,
+								StartedAt: st?.StartedAt, FinishedAt: st?.FinishedAt, Pid: st?.Pid,
+							})}`
+						);
 						if (st && st.Running === false && st.Status === 'exited' && typeof st.ExitCode === 'number') {
 							exitCode = st.ExitCode;
 							console.log(`[runContainerWithStreaming] Resolved exit code via inspect: ${exitCode}`);
 						}
+					} else {
+						console.warn(`[runContainerWithStreaming] EXIT-CODE INDETERMINATE for ${options.name ?? containerId.slice(0, 12)}: inspect returned HTTP ${insp.status} (container may already be gone/removed)`);
 					}
 				} catch (err) {
 					console.warn(`[runContainerWithStreaming] Inspect fallback failed: ${(err as Error).message}`);
 				}
+				// Still indeterminate -> pull docker events so the log shows WHO/WHAT
+				// acted on the container (die/kill/destroy + actor). Missing piece when
+				// something external removes the helper mid-run (#1344).
+				if (exitCode === undefined) {
+					await logRecentContainerEvents(containerId, options.name ?? containerId.slice(0, 12), containerStartedAt, options.envId);
+				}
 			}
 
 			// Container exited - abort stderr stream (it may be hanging on some Docker hosts)

src/lib/server/scheduler/tasks/container-update.ts

@@ -386,12 +386,12 @@ export async function runContainerUpdate(
 		// Get the actual image ID from inspect data
 		const currentImageId = inspectData.Image;
 
-		// Capture the OLD image's Env/Labels BEFORE the pull — the old digest may
-		// be untagged/GC'd afterward. Forwarded to the env/label rebase (#1226, #1256).
+		// Capture the OLD image's Env/Labels/Cmd/Entrypoint BEFORE the pull — the old digest may
+		// be untagged/GC'd afterward. Forwarded to the env/label/command rebase (#1226, #1256, #1371).
 		let oldImageConfig: ImageEnvLabels | null = null;
 		try {
 			const oldImg = await inspectImage(currentImageId, envId) as any;
-			oldImageConfig = { Env: oldImg?.Config?.Env, Labels: oldImg?.Config?.Labels };
+			oldImageConfig = { Env: oldImg?.Config?.Env, Labels: oldImg?.Config?.Labels, Cmd: oldImg?.Config?.Cmd ?? null, Entrypoint: oldImg?.Config?.Entrypoint ?? null };
 		} catch {
 			// Best-effort; rebase falls back if unavailable.
 		}
@@ -669,10 +669,30 @@ export async function recreateContainer(
 
 		const inspectData = await inspectContainer(container.id, envId) as any;
 		const imageName = imageNameOverride || inspectData.Config?.Image;
+		// Capture the parent's id BEFORE recreate. A recreate gives the parent a NEW id,
+		// and any child using `network_mode: service:parent` / `container:parent` stores
+		// that old id in its own HostConfig.NetworkMode (`container:<oldId>`). After the
+		// parent update those children still point at the dead id and fail to (re)start
+		// with "joining network namespace of container: No such container" (#570).
+		const oldParentId: string = inspectData.Id;
 
 		log?.(`Recreating container: ${containerName} (image: ${imageName})`);
 
 		await recreateContainerFromInspect(inspectData, imageName, envId, log, oldImageConfig);
+
+		// Parent recreate SUCCEEDED (a failure would have thrown and rolled the parent
+		// back to its original id, leaving children valid). Repoint any dependent
+		// children at the new parent. Best-effort and fully isolated: a child failure
+		// NEVER changes the parent's already-successful update. No-op when there are no
+		// such children — the common case pays nothing beyond one already-in-hand id.
+		try {
+			const reconnected = await reconnectDependentChildren(oldParentId, containerName, envId, log);
+			if (reconnected > 0) {
+				log?.(`Reconnected ${reconnected} dependent container${reconnected === 1 ? '' : 's'} to the new parent`);
+			}
+		} catch (e: any) {
+			log?.(`WARNING: dependent-child reconnect step errored (parent update stands): ${e?.message}`);
+		}
 		return { success: true };
 	} catch (error: any) {
 		log?.(`Failed to recreate container: ${error.message}`);
@@ -680,3 +700,52 @@ export async function recreateContainer(
 	}
 }
 
+/**
+ * After a parent container is recreated (new id), find containers whose network is
+ * shared into the parent by id (`network_mode: service:X`/`container:X`, which Docker
+ * lowers to `HostConfig.NetworkMode = "container:<parentId>"`) and repoint them at the
+ * new parent. A plain restart is NOT enough — the stale id is baked into the child's
+ * stored NetworkMode, so the child must be RECREATED with the reference rewritten. We
+ * rewrite to `container:<parentName>` (not the new id) so the child also survives future
+ * parent recreations. Best-effort per child; the caller isolates this from the parent
+ * update result. Returns the number of children repointed. (#570)
+ */
+async function reconnectDependentChildren(
+	oldParentId: string,
+	newParentName: string,
+	envId: number | undefined,
+	log?: (msg: string) => void,
+	visited: Set<string> = new Set()
+): Promise<number> {
+	if (visited.has(oldParentId)) return 0; // cycle guard for A<-B<-C chains
+	visited.add(oldParentId);
+
+	const all = await listContainers(true, envId); // include stopped
+	let count = 0;
+	for (const c of all) {
+		if (c.id === oldParentId) continue;
+		let ci: any;
+		try { ci = await inspectContainer(c.id, envId); } catch { continue; }
+		const mode: string = ci.HostConfig?.NetworkMode || '';
+		if (!mode.startsWith('container:')) continue;
+		const ref = mode.slice('container:'.length);
+		// Match the parent by full id, or defensively by short (12-char) id.
+		if (ref !== oldParentId && ref !== oldParentId.slice(0, 12) && !oldParentId.startsWith(ref)) continue;
+
+		log?.(`Reconnecting dependent container "${c.name}" -> container:${newParentName}`);
+		const oldChildId: string = ci.Id;
+		ci.HostConfig.NetworkMode = `container:${newParentName}`;
+		try {
+			const { Id: newChildId } = await recreateContainerFromInspect(ci, ci.Config.Image, envId, log);
+			count++;
+			// Chain: if this child was itself a parent to others, repoint them too.
+			if (newChildId && newChildId !== oldChildId) {
+				count += await reconnectDependentChildren(oldChildId, c.name, envId, log, visited);
+			}
+		} catch (e: any) {
+			log?.(`WARNING: failed to reconnect dependent "${c.name}": ${e?.message}`);
+		}
+	}
+	return count;
+}
+

src/routes/api/containers/batch-update-stream/+server.ts

@@ -161,10 +161,10 @@ export const POST: RequestHandler = async (event) => {
 				// Capture the OLD image's Env/Labels BEFORE pulling — once the tag is
 				// repointed the old digest may be untagged/GC'd and un-inspectable.
 				// Used by the env/label rebase in recreateContainer (#1226, #1256).
-				let oldImageConfig: { Env?: string[]; Labels?: Record<string, string> } | null = null;
+				let oldImageConfig: { Env?: string[]; Labels?: Record<string, string>; Cmd?: string[] | null; Entrypoint?: string[] | null } | null = null;
 				try {
 					const oldImg = await inspectImage(currentImageId, envIdNum) as any;
-					oldImageConfig = { Env: oldImg?.Config?.Env, Labels: oldImg?.Config?.Labels };
+					oldImageConfig = { Env: oldImg?.Config?.Env, Labels: oldImg?.Config?.Labels, Cmd: oldImg?.Config?.Cmd ?? null, Entrypoint: oldImg?.Config?.Entrypoint ?? null };
 				} catch {
 					// Best-effort; rebase will fall back if unavailable.
 				}