From d33185d1fab69debe2f91cd5cb13efddac1b31b1 Mon Sep 17 00:00:00 2001 From: jarek Date: Sun, 9 Aug 2026 10:31:28 +0200 Subject: [PATCH] container updates respect the new image's command/entrypoint --- src/lib/server/container-env-merge.ts | 33 +++ src/lib/server/docker.ts | 270 ++++++++++++------ .../scheduler/tasks/container-update.ts | 75 ++++- .../containers/batch-update-stream/+server.ts | 4 +- 4 files changed, 292 insertions(+), 90 deletions(-) diff --git a/src/lib/server/container-env-merge.ts b/src/lib/server/container-env-merge.ts index 012413a..b035f96 100644 --- a/src/lib/server/container-env-merge.ts +++ b/src/lib/server/container-env-merge.ts @@ -33,6 +33,39 @@ export interface ImageEnvLabels { Env?: string[]; Labels?: Record; + /** The image's baked CMD / ENTRYPOINT, for the same value-aware rebase (#1371). */ + Cmd?: string[] | null; + Entrypoint?: string[] | null; +} + +/** + * Rebase a container's Cmd/Entrypoint onto the new image (#1371). Same flattening problem + * as Env/Labels: a container's Config.Cmd is set at create time whether it came from the + * image's CMD or a user `command:` override - spreading it verbatim onto a new image freezes + * the OLD image's command, so a container can start with a command the NEW image no longer + * provides (missing entrypoint script -> exit 127 -> restart loop). + * + * The recoverable signal is the same VALUE comparison: + * container == oldImage -> came from the old image, user never overrode it -> adopt newImage + * container != oldImage -> a real user/Compose override -> keep container + * Cmd/Entrypoint are ordered arrays, so equality is order-sensitive (JSON compare). When the + * old image value is unknown (null/undefined - caller couldn't inspect it) we KEEP the + * container's value (verbatim fallback), exactly as the env/label rebase does. + */ +export function rebaseCommand( + containerValue: string[] | null | undefined, + oldImageValue: string[] | null | undefined, + newImageValue: string[] | null | undefined +): { value: string[] | null | undefined; adopted: boolean } { + // Old image not inspectable -> can't tell image-default from user override -> keep verbatim. + if (oldImageValue == null) return { value: containerValue, adopted: false }; + const eq = (a: string[] | null | undefined, b: string[] | null | undefined) => + JSON.stringify(a ?? null) === JSON.stringify(b ?? null); + // User overrode it (differs from the old image's baked value) -> keep the override. + if (!eq(containerValue, oldImageValue)) return { value: containerValue, adopted: false }; + // Image-baked and untouched -> follow the new image. (Same value -> no-op, adopted:false.) + if (eq(newImageValue, oldImageValue)) return { value: containerValue, adopted: false }; + return { value: newImageValue, adopted: true }; } /** diff --git a/src/lib/server/docker.ts b/src/lib/server/docker.ts index e52b22c..b49482a 100644 --- a/src/lib/server/docker.ts +++ b/src/lib/server/docker.ts @@ -12,10 +12,12 @@ import * as http from 'node:http'; import * as https from 'node:https'; import * as tls from 'node:tls'; import { createHash } from 'node:crypto'; +import { pumpWebStreamToWritable } from './stream-pump'; +import { computeRequestTimeoutMs } from './backups/request-timeout'; import type { Environment } from './db'; import { getSetting } from './db'; -import { getAdditionalVolumeBinds } from './mount-dedupe'; -import { rebaseEnvOntoImage, rebaseLabelsOntoImage, describeEnvRebase, describeLabelRebase, type ImageEnvLabels } from './container-env-merge'; +import { getAdditionalVolumeBinds, dedupeVolumesForRecreate } from './mount-dedupe'; +import { rebaseEnvOntoImage, rebaseLabelsOntoImage, rebaseCommand, describeEnvRebase, describeLabelRebase, type ImageEnvLabels } from './container-env-merge'; import { encodeRegistryAuth } from './registry-auth'; import { isSystemContainer, classifyEmptyDigestImage } from './scheduler/tasks/update-utils'; import { deepDiff } from '../utils/diff.js'; @@ -417,12 +419,16 @@ export function httpsAgentRequest( headers: reqHeaders, }; - if (!streaming) { - const isComposeOperation = path === '/_hawser/compose'; - const composeTimeoutMs = parseInt(process.env.COMPOSE_TIMEOUT || '900') * 1000; - const isPrune = path.endsWith('/prune'); - reqOptions.timeout = isComposeOperation ? composeTimeoutMs : isPrune ? 300000 : 30000; - } + // A stream request BODY (a large tar PUT) must not get the small-request idle timeout + // even though the RESPONSE isn't streamed - computeRequestTimeoutMs decides on both. + const streamingBody = typeof (options.body as ReadableStream | undefined)?.getReader === 'function'; + const timeoutMs = computeRequestTimeoutMs({ + path, + streamingBody, + streamingResponse: streaming, + composeTimeoutSecs: parseInt(process.env.COMPOSE_TIMEOUT || '900'), + }); + if (timeoutMs !== null) reqOptions.timeout = timeoutMs; // Honor AbortSignal from caller (e.g., AbortSignal.timeout(5000) for ping) const signal = options.signal as AbortSignal | undefined; @@ -493,19 +499,10 @@ export function httpsAgentRequest( } else if (body instanceof Blob) { body.arrayBuffer().then(ab => req.end(Buffer.from(ab)), reject); } else if (typeof (body as ReadableStream).getReader === 'function') { + // Stream the body with backpressure (O(1) RAM). Without awaiting 'drain' this + // piled the whole tar into RAM on a slow mTLS peer - see stream-pump.ts. const reader = (body as ReadableStream).getReader(); - (async () => { - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - req.write(value); - } - req.end(); - } catch (err) { - req.destroy(err as Error); - } - })(); + pumpWebStreamToWritable(reader, req).catch(() => { /* req.destroy already called */ }); } else { req.end(); } @@ -574,7 +571,7 @@ function buildConfigFromEnv(env: Environment): DockerClientConfig { /** * Get Docker client configuration for an environment */ -async function getDockerConfig(envId?: number | null): Promise { +export async function getDockerConfig(envId?: number | null): Promise { if (!envId) { throw new Error('No environment specified'); } @@ -2016,7 +2013,12 @@ export async function recreateContainerFromInspect( let oldImg = oldImageConfig ?? null; if (!oldImg) { const oldInspect = await inspectImage(inspectData.Image || config.Image, envId); - oldImg = { Env: (oldInspect as any)?.Config?.Env, Labels: (oldInspect as any)?.Config?.Labels }; + oldImg = { + Env: (oldInspect as any)?.Config?.Env, + Labels: (oldInspect as any)?.Config?.Labels, + Cmd: (oldInspect as any)?.Config?.Cmd ?? null, + Entrypoint: (oldInspect as any)?.Config?.Entrypoint ?? null, + }; } const newImg = await inspectImage(newImage, envId); const oldEnv = oldImg?.Env || []; @@ -2026,6 +2028,18 @@ export async function recreateContainerFromInspect( createConfig.Env = rebaseEnvOntoImage(config.Env || [], oldEnv, newEnv); createConfig.Labels = rebaseLabelsOntoImage(config.Labels || {}, oldLabels, newLabels); + // 4b. Rebase Cmd/Entrypoint the same way (#1371): a container's Config.Cmd is flattened at + // create time, so the OLD image's baked CMD is indistinguishable from a user `command:` + // override and would be frozen onto the new image - a container can then start with a + // command the new image no longer provides (missing script -> exit 127 -> restart loop). + // oldImg?.Cmd == null (caller didn't capture it / inspect failed) -> keep verbatim. + const cmdR = rebaseCommand(config.Cmd, oldImg?.Cmd, (newImg as any)?.Config?.Cmd); + const entR = rebaseCommand(config.Entrypoint, oldImg?.Entrypoint, (newImg as any)?.Config?.Entrypoint); + createConfig.Cmd = cmdR.value; + createConfig.Entrypoint = entR.value; + if (cmdR.adopted) log?.(`Cmd — adopted from new image: ${JSON.stringify(cmdR.value)} (was ${JSON.stringify(config.Cmd)})`); + if (entR.adopted) log?.(`Entrypoint — adopted from new image: ${JSON.stringify(entR.value)} (was ${JSON.stringify(config.Entrypoint)})`); + const envSummary = describeEnvRebase(config.Env || [], oldEnv, newEnv); const labelSummary = describeLabelRebase(config.Labels || {}, oldLabels, newLabels); const fmt = (label: string, keys: string[]) => keys.length ? `${label}: ${keys.join(', ')}` : ''; @@ -2094,39 +2108,20 @@ export async function recreateContainerFromInspect( } } - // Deduplicate: remove Config.Volumes entries that conflict with HostConfig.Tmpfs or Binds. - // Read-only containers get tmpfs at paths like /tmp that may also be declared as image volumes. - // Docker rejects duplicate mount points, so the tmpfs/bind mount wins over the volume declaration. - if (createConfig.Volumes && hostConfig) { - const mountedPaths = new Set(); - if (hostConfig.Tmpfs) { - for (const p of Object.keys(hostConfig.Tmpfs)) { - mountedPaths.add(p); - } - } - if (hostConfig.Binds) { - for (const b of hostConfig.Binds) { - const parts = b.split(':'); - if (parts.length >= 2) mountedPaths.add(parts[1].split(':')[0]); - } - } - if (mountedPaths.size > 0) { - for (const volPath of Object.keys(createConfig.Volumes)) { - if (mountedPaths.has(volPath)) { - delete createConfig.Volumes[volPath]; - } - } - if (Object.keys(createConfig.Volumes).length === 0) { - delete createConfig.Volumes; - } - } + // Deduplicate Config.Volumes against every mount point that will exist on the new container + // (Tmpfs, existing binds, the binds we re-add below, and inspect.Mounts). Docker rejects + // duplicate mount points; the tmpfs/bind/volume mount wins over the bare image-VOLUME + // declaration. Compute additionalBinds first so the dedup sees them (#1088 / #1363). + const additionalBinds = getAdditionalVolumeBinds(hostConfig || {}, inspectData.Mounts || []); + if (hostConfig) { + const kept = dedupeVolumesForRecreate(createConfig.Volumes, hostConfig, inspectData.Mounts || [], additionalBinds); + if (kept) createConfig.Volumes = kept; + else delete createConfig.Volumes; } - - const additionalBinds = getAdditionalVolumeBinds(hostConfig, inspectData.Mounts || []); if (additionalBinds.length > 0) { createConfig.HostConfig = { ...hostConfig, - Binds: [...(hostConfig.Binds || []), ...additionalBinds] + Binds: [...(hostConfig?.Binds || []), ...additionalBinds] }; } @@ -2298,32 +2293,18 @@ export async function createContainerFromMetadata( } } - // Volume/Tmpfs deduplication - if (createConfig.Volumes && hostConfig) { - const mountedPaths = new Set(); - if (hostConfig.Tmpfs) { - for (const p of Object.keys(hostConfig.Tmpfs)) mountedPaths.add(p); - } - if (hostConfig.Binds) { - for (const b of hostConfig.Binds) { - const parts = b.split(':'); - if (parts.length >= 2) mountedPaths.add(parts[1].split(':')[0]); - } - } - if (mountedPaths.size > 0) { - for (const volPath of Object.keys(createConfig.Volumes)) { - if (mountedPaths.has(volPath)) delete createConfig.Volumes[volPath]; - } - if (Object.keys(createConfig.Volumes).length === 0) delete createConfig.Volumes; - } + // Preserve anonymous volumes, then dedup Config.Volumes against every resulting mount point + // (incl. these added binds + metadata.mounts) so recreate never sends a duplicate (#1088 / #1363). + const additionalBinds = getAdditionalVolumeBinds(hostConfig || {}, metadata.mounts || []); + if (hostConfig) { + const kept = dedupeVolumesForRecreate(createConfig.Volumes, hostConfig, metadata.mounts || [], additionalBinds); + if (kept) createConfig.Volumes = kept; + else delete createConfig.Volumes; } - - // Preserve anonymous volumes - const additionalBinds = getAdditionalVolumeBinds(hostConfig, metadata.mounts || []); if (additionalBinds.length > 0) { createConfig.HostConfig = { ...hostConfig, - Binds: [...(hostConfig.Binds || []), ...additionalBinds] + Binds: [...(hostConfig?.Binds || []), ...additionalBinds] }; } @@ -3399,6 +3380,10 @@ export async function getRegistryAuth( return { baseUrl, orgPath: parsed.path, authHeader }; } +// Catalog-failure classification lives in an import-light module so it's unit-testable +// without pulling this file's better-sqlite3 imports into the test process (#873). +export { classifyCatalogFailure, CATALOG_NOT_SUPPORTED_MSG } from './registry-catalog.js'; + // --- Harbor fallback for catalog and image search --- // Harbor denies access to the V2 _catalog endpoint for robot accounts. // We detect Harbor and use the native project API as a fallback. @@ -4538,11 +4523,16 @@ export async function pruneImages(dangling = true, envId?: number | null) { } const { DEFAULT_GRYPE_IMAGE, DEFAULT_TRIVY_IMAGE } = await import('./scanner'); + const { DEFAULT_HELPER_IMAGE } = await import('./backups/restic'); const grypeImg = (await getSetting('default_grype_image')) ?? DEFAULT_GRYPE_IMAGE; const trivyImg = (await getSetting('default_trivy_image')) ?? DEFAULT_TRIVY_IMAGE; + // The backup/restore helper image is pulled once and reused for every backup; shield it from + // prune too (same holder-container trick), so an unused-image prune between backups doesn't + // force a slow re-pull (and a fail on an air-tight/offline host). + const helperImg = (await getSetting('default_backup_image')) || DEFAULT_HELPER_IMAGE; const holderIds: string[] = []; - for (const image of [grypeImg, trivyImg]) { + for (const image of [grypeImg, trivyImg, helperImg]) { try { const safeName = `dockhand-prune-keep-${image.replace(/[^a-z0-9]/gi, '-')}-${Date.now()}`; const created = await dockerJsonRequest<{ Id: string }>( @@ -4861,6 +4851,40 @@ export async function runContainer(options: { } // Run a container with attached streams (for scanners that need real-time output) +/** + * DIAGNOSTIC (issue #1344): when a helper's exit code can't be determined, pull the + * recent `docker events` for that container so the log shows WHAT acted on it (a + * `die` with exitCode, a `kill` with the signal, a `destroy`/`remove` and its actor). + * `docker events` carries `Actor.Attributes` incl. the signal and, on some daemons, the + * execID/name of who issued it. Bounded, non-streaming (since..until), best-effort. + */ +async function logRecentContainerEvents(containerId: string, name: string, sinceISO: string | undefined, envId: number | null | undefined): Promise { + try { + const sinceSec = sinceISO ? Math.floor(new Date(sinceISO).getTime() / 1000) - 2 : Math.floor(Date.now() / 1000) - 120; + const untilSec = Math.floor(Date.now() / 1000) + 1; + const filters = encodeURIComponent(JSON.stringify({ container: [containerId] })); + const resp = await dockerFetch(`/events?since=${sinceSec}&until=${untilSec}&filters=${filters}`, {}, envId); + if (!resp.ok) { console.warn(`[runContainerWithStreaming] events fetch for ${name}: HTTP ${resp.status}`); return; } + const body = await resp.text(); + // The events endpoint returns newline-delimited JSON objects. + const lines = body.split('\n').map((l) => l.trim()).filter(Boolean); + if (lines.length === 0) { console.warn(`[runContainerWithStreaming] no docker events for ${name} in the window (nothing recorded who touched it)`); return; } + for (const line of lines) { + try { + const ev = JSON.parse(line) as { status?: string; Action?: string; Actor?: { Attributes?: Record }; time?: number }; + const action = ev.Action ?? ev.status ?? '(unknown)'; + const attrs = ev.Actor?.Attributes ?? {}; + const sig = attrs.signal ? ` signal=${attrs.signal}` : ''; + const code = attrs.exitCode ? ` exitCode=${attrs.exitCode}` : ''; + const exec = attrs.execID ? ` execID=${attrs.execID}` : ''; + console.warn(`[runContainerWithStreaming] EVENT ${name}: ${action}${sig}${code}${exec}`); + } catch { /* skip a non-JSON line */ } + } + } catch (err) { + console.warn(`[runContainerWithStreaming] events fetch failed for ${name}: ${(err as Error).message}`); + } +} + export async function runContainerWithStreaming(options: { image: string; cmd: string[]; @@ -4890,7 +4914,11 @@ export async function runContainerWithStreaming(options: { Image: options.image, Cmd: options.cmd, Env: options.env || [], - Labels: options.labels || {}, + // Every helper spawned here (backup/restore/probe, image scanner) is ephemeral and exits + // on purpose, so it must NOT fire a "Container died/exited" notification like a user's + // container would. dockhand.notify=false makes the event handler skip it + // (isNotifyDisabledByLabel). Caller labels win if they set notify explicitly. + Labels: { 'dockhand.notify': 'false', ...(options.labels || {}) }, Tty: false, HostConfig: { Binds: options.binds || [], @@ -4938,6 +4966,12 @@ export async function runContainerWithStreaming(options: { // Start container await drainResponse(await dockerFetch(`/containers/${containerId}/start`, { method: 'POST' }, options.envId)); + // DIAGNOSTIC (issue #1344): one line confirming the helper actually started, + // with the config that could explain an early death (bind count, network, + // user). Pairs with the "exited with code" / "EXIT-CODE INDETERMINATE" lines + // so a user's log shows the full lifecycle of a helper that dies mid-run. + console.log(`[runContainerWithStreaming] Started ${containerName} (binds=${(options.binds || []).length}, netMode=${options.networkMode ?? 'default'}, user=${options.user ?? 'default'})`); + // Create abort controller to cancel stderr stream when container exits // On some Docker hosts (e.g. Synology NAS with older kernels), follow=true // streams don't close when the container exits, causing indefinite hangs. @@ -4960,34 +4994,100 @@ export async function runContainerWithStreaming(options: { // Bounded-retry the wait; if it still can't be determined, fall back to // inspecting the container's final State, and if that's also unavailable // leave exitCode undefined — the guard below fails closed on undefined. + // HELPERS_WAIT_MODE (default 'wait'): the exit signal below is a single + // streaming `POST /wait`, held open with no bytes for the whole run. A + // socket-proxy / reverse-proxy / tcp DOCKER_HOST with a short idle timeout can + // sever it mid-run, leaving the exit code unreadable even though the helper is + // fine (#1344). Set HELPERS_WAIT_MODE=poll to POLL short inspect calls instead + // (nothing stays open for a proxy to cut). Applies to every helper this function + // runs — backup/restore AND the image scanner. let exitCode: number | undefined; - for (let attempt = 0; attempt < 3 && exitCode === undefined; attempt++) { - try { - const waitResult = await dockerFetch(`/containers/${containerId}/wait`, { method: 'POST', streaming: true }, options.envId); - if (!waitResult.ok) await throwDockerError(waitResult); - const waitData = await waitResult.json() as { StatusCode?: number }; - exitCode = waitData.StatusCode; - console.log(`[runContainerWithStreaming] Container exited with code: ${exitCode}`); - } catch (err) { - console.warn(`[runContainerWithStreaming] Wait attempt ${attempt + 1} failed: ${(err as Error).message}`); + if (process.env.HELPERS_WAIT_MODE === 'poll') { + // Bound by the caller's timeout (backup passes a long 'data' tier, the + // scanner passes 600_000). Fall back to 1h so poll mode never hangs. + // One line so the log confirms which exit-signal mode is active (helps + // diagnose whether HELPERS_WAIT_MODE=poll actually took effect - #1344). + console.log(`[runContainerWithStreaming] Awaiting exit of ${options.name ?? containerId.slice(0, 12)} via POLL mode`); + const deadline = Date.now() + (options.timeout && options.timeout > 0 ? options.timeout : 3_600_000); + while (exitCode === undefined && Date.now() < deadline) { + try { + const insp = await dockerFetch(`/containers/${containerId}/json`, {}, options.envId); + if (insp.ok) { + const data = await insp.json() as { State?: { Status?: string; Running?: boolean; Dead?: boolean; ExitCode?: number } }; + const st = data.State; + if (st && st.Running === false && st.Status === 'exited' && typeof st.ExitCode === 'number') { + exitCode = st.ExitCode; + console.log(`[runContainerWithStreaming] Container exited with code: ${exitCode} (poll mode)`); + break; + } + if (st?.Dead || st?.Status === 'removing') break; // gone -> fail-closed diagnostics below + } else if (insp.status === 404) { + console.warn(`[runContainerWithStreaming] Poll: ${options.name ?? containerId.slice(0, 12)} not found (404) — container removed`); + break; + } + } catch (err) { + // A dropped short inspect is cheap to retry — keep polling to the deadline + // rather than bailing (unlike the streaming /wait path). + console.warn(`[runContainerWithStreaming] Poll inspect failed for ${options.name ?? containerId.slice(0, 12)}: ${(err as Error).message}`); + } + if (exitCode === undefined) await new Promise((r) => setTimeout(r, 1500)); + } + } else { + for (let attempt = 0; attempt < 3 && exitCode === undefined; attempt++) { + try { + const waitResult = await dockerFetch(`/containers/${containerId}/wait`, { method: 'POST', streaming: true }, options.envId); + if (!waitResult.ok) await throwDockerError(waitResult); + const waitData = await waitResult.json() as { StatusCode?: number }; + exitCode = waitData.StatusCode; + console.log(`[runContainerWithStreaming] Container exited with code: ${exitCode}`); + } catch (err) { + // DIAGNOSTIC (issue #1344): include the attempt count so a user whose + // helper's exit can't be read can tell us whether /wait failed once + // (transient) or all 3 times (the container vanished / daemon churn). + console.warn(`[runContainerWithStreaming] Wait attempt ${attempt + 1}/3 failed for ${options.name ?? containerId.slice(0, 12)}: ${(err as Error).message}`); + } } } if (exitCode === undefined) { // Fallback: inspect the container. Only trust a definitively-exited // state; a still-running/restarting/unknown state stays undefined. + let containerStartedAt: string | undefined; try { const insp = await dockerFetch(`/containers/${containerId}/json`, {}, options.envId); if (insp.ok) { - const data = await insp.json() as { State?: { Status?: string; Running?: boolean; ExitCode?: number } }; + const data = await insp.json() as { State?: { Status?: string; Running?: boolean; Restarting?: boolean; Dead?: boolean; Paused?: boolean; ExitCode?: number; OOMKilled?: boolean; Error?: string; StartedAt?: string; FinishedAt?: string; Pid?: number } }; const st = data.State; + containerStartedAt = st?.StartedAt; + // DIAGNOSTIC (issue #1344): /wait gave no code and the fallback is + // about to leave exitCode undefined -> the user sees "could not + // determine container exit status". Log the FULL final State so a + // user hitting this can tell us WHY (OOMKilled, Dead, removed by an + // external reaper, still Running = /wait raced, or a daemon Error). + console.warn( + `[runContainerWithStreaming] EXIT-CODE INDETERMINATE for ${options.name ?? containerId.slice(0, 12)} ` + + `State=${JSON.stringify({ + Status: st?.Status, Running: st?.Running, Restarting: st?.Restarting, + Dead: st?.Dead, Paused: st?.Paused, ExitCode: st?.ExitCode, + OOMKilled: st?.OOMKilled, Error: st?.Error, + StartedAt: st?.StartedAt, FinishedAt: st?.FinishedAt, Pid: st?.Pid, + })}` + ); if (st && st.Running === false && st.Status === 'exited' && typeof st.ExitCode === 'number') { exitCode = st.ExitCode; console.log(`[runContainerWithStreaming] Resolved exit code via inspect: ${exitCode}`); } + } else { + console.warn(`[runContainerWithStreaming] EXIT-CODE INDETERMINATE for ${options.name ?? containerId.slice(0, 12)}: inspect returned HTTP ${insp.status} (container may already be gone/removed)`); } } catch (err) { console.warn(`[runContainerWithStreaming] Inspect fallback failed: ${(err as Error).message}`); } + // Still indeterminate -> pull docker events so the log shows WHO/WHAT + // acted on the container (die/kill/destroy + actor). Missing piece when + // something external removes the helper mid-run (#1344). + if (exitCode === undefined) { + await logRecentContainerEvents(containerId, options.name ?? containerId.slice(0, 12), containerStartedAt, options.envId); + } } // Container exited - abort stderr stream (it may be hanging on some Docker hosts) diff --git a/src/lib/server/scheduler/tasks/container-update.ts b/src/lib/server/scheduler/tasks/container-update.ts index 54df3d5..e36b690 100644 --- a/src/lib/server/scheduler/tasks/container-update.ts +++ b/src/lib/server/scheduler/tasks/container-update.ts @@ -386,12 +386,12 @@ export async function runContainerUpdate( // Get the actual image ID from inspect data const currentImageId = inspectData.Image; - // Capture the OLD image's Env/Labels BEFORE the pull — the old digest may - // be untagged/GC'd afterward. Forwarded to the env/label rebase (#1226, #1256). + // Capture the OLD image's Env/Labels/Cmd/Entrypoint BEFORE the pull — the old digest may + // be untagged/GC'd afterward. Forwarded to the env/label/command rebase (#1226, #1256, #1371). let oldImageConfig: ImageEnvLabels | null = null; try { const oldImg = await inspectImage(currentImageId, envId) as any; - oldImageConfig = { Env: oldImg?.Config?.Env, Labels: oldImg?.Config?.Labels }; + oldImageConfig = { Env: oldImg?.Config?.Env, Labels: oldImg?.Config?.Labels, Cmd: oldImg?.Config?.Cmd ?? null, Entrypoint: oldImg?.Config?.Entrypoint ?? null }; } catch { // Best-effort; rebase falls back if unavailable. } @@ -669,10 +669,30 @@ export async function recreateContainer( const inspectData = await inspectContainer(container.id, envId) as any; const imageName = imageNameOverride || inspectData.Config?.Image; + // Capture the parent's id BEFORE recreate. A recreate gives the parent a NEW id, + // and any child using `network_mode: service:parent` / `container:parent` stores + // that old id in its own HostConfig.NetworkMode (`container:`). After the + // parent update those children still point at the dead id and fail to (re)start + // with "joining network namespace of container: No such container" (#570). + const oldParentId: string = inspectData.Id; log?.(`Recreating container: ${containerName} (image: ${imageName})`); await recreateContainerFromInspect(inspectData, imageName, envId, log, oldImageConfig); + + // Parent recreate SUCCEEDED (a failure would have thrown and rolled the parent + // back to its original id, leaving children valid). Repoint any dependent + // children at the new parent. Best-effort and fully isolated: a child failure + // NEVER changes the parent's already-successful update. No-op when there are no + // such children — the common case pays nothing beyond one already-in-hand id. + try { + const reconnected = await reconnectDependentChildren(oldParentId, containerName, envId, log); + if (reconnected > 0) { + log?.(`Reconnected ${reconnected} dependent container${reconnected === 1 ? '' : 's'} to the new parent`); + } + } catch (e: any) { + log?.(`WARNING: dependent-child reconnect step errored (parent update stands): ${e?.message}`); + } return { success: true }; } catch (error: any) { log?.(`Failed to recreate container: ${error.message}`); @@ -680,3 +700,52 @@ export async function recreateContainer( } } +/** + * After a parent container is recreated (new id), find containers whose network is + * shared into the parent by id (`network_mode: service:X`/`container:X`, which Docker + * lowers to `HostConfig.NetworkMode = "container:"`) and repoint them at the + * new parent. A plain restart is NOT enough — the stale id is baked into the child's + * stored NetworkMode, so the child must be RECREATED with the reference rewritten. We + * rewrite to `container:` (not the new id) so the child also survives future + * parent recreations. Best-effort per child; the caller isolates this from the parent + * update result. Returns the number of children repointed. (#570) + */ +async function reconnectDependentChildren( + oldParentId: string, + newParentName: string, + envId: number | undefined, + log?: (msg: string) => void, + visited: Set = new Set() +): Promise { + if (visited.has(oldParentId)) return 0; // cycle guard for A<-B<-C chains + visited.add(oldParentId); + + const all = await listContainers(true, envId); // include stopped + let count = 0; + for (const c of all) { + if (c.id === oldParentId) continue; + let ci: any; + try { ci = await inspectContainer(c.id, envId); } catch { continue; } + const mode: string = ci.HostConfig?.NetworkMode || ''; + if (!mode.startsWith('container:')) continue; + const ref = mode.slice('container:'.length); + // Match the parent by full id, or defensively by short (12-char) id. + if (ref !== oldParentId && ref !== oldParentId.slice(0, 12) && !oldParentId.startsWith(ref)) continue; + + log?.(`Reconnecting dependent container "${c.name}" -> container:${newParentName}`); + const oldChildId: string = ci.Id; + ci.HostConfig.NetworkMode = `container:${newParentName}`; + try { + const { Id: newChildId } = await recreateContainerFromInspect(ci, ci.Config.Image, envId, log); + count++; + // Chain: if this child was itself a parent to others, repoint them too. + if (newChildId && newChildId !== oldChildId) { + count += await reconnectDependentChildren(oldChildId, c.name, envId, log, visited); + } + } catch (e: any) { + log?.(`WARNING: failed to reconnect dependent "${c.name}": ${e?.message}`); + } + } + return count; +} + diff --git a/src/routes/api/containers/batch-update-stream/+server.ts b/src/routes/api/containers/batch-update-stream/+server.ts index 8b52d36..03ab088 100644 --- a/src/routes/api/containers/batch-update-stream/+server.ts +++ b/src/routes/api/containers/batch-update-stream/+server.ts @@ -161,10 +161,10 @@ export const POST: RequestHandler = async (event) => { // Capture the OLD image's Env/Labels BEFORE pulling — once the tag is // repointed the old digest may be untagged/GC'd and un-inspectable. // Used by the env/label rebase in recreateContainer (#1226, #1256). - let oldImageConfig: { Env?: string[]; Labels?: Record } | null = null; + let oldImageConfig: { Env?: string[]; Labels?: Record; Cmd?: string[] | null; Entrypoint?: string[] | null } | null = null; try { const oldImg = await inspectImage(currentImageId, envIdNum) as any; - oldImageConfig = { Env: oldImg?.Config?.Env, Labels: oldImg?.Config?.Labels }; + oldImageConfig = { Env: oldImg?.Config?.Env, Labels: oldImg?.Config?.Labels, Cmd: oldImg?.Config?.Cmd ?? null, Entrypoint: oldImg?.Config?.Entrypoint ?? null }; } catch { // Best-effort; rebase will fall back if unavailable. }