From 369a9b17cf0116b22ebebbf95f2dafbe92628f16 Mon Sep 17 00:00:00 2001 From: Jarek Krochmalski Date: Sun, 6 Sep 2026 17:03:47 +0530 Subject: [PATCH] resolve helper container exit code from a terminal created or dead state --- src/lib/server/docker.ts | 27 ++++++++++++------ src/lib/server/helper-wait-core.ts | 32 +++++++++++++++++++++ tests/helper-wait-core.test.ts | 45 +++++++++++++++++++++++++++++- 3 files changed, 94 insertions(+), 10 deletions(-) diff --git a/src/lib/server/docker.ts b/src/lib/server/docker.ts index 822b0d1..a4e9401 100644 --- a/src/lib/server/docker.ts +++ b/src/lib/server/docker.ts @@ -17,7 +17,7 @@ import { pumpWebStreamToWritable } from './stream-pump'; import { toWebReadableStream } from './node-readable-stream'; import { buildImagePruneFilters } from './image-prune-core'; import { computeRequestTimeoutMs } from './backups/request-timeout'; -import { helperWaitDeadline } from './helper-wait-core'; +import { helperWaitDeadline, helperExitFromState } from './helper-wait-core'; import type { Environment } from './db'; import { getSetting } from './db'; import { getAdditionalVolumeBinds, dedupeVolumesForRecreate } from './mount-dedupe'; @@ -5129,14 +5129,22 @@ export async function runContainerWithStreaming(options: { try { const insp = await dockerFetch(`/containers/${containerId}/json`, {}, options.envId); if (insp.ok) { - const data = await insp.json() as { State?: { Status?: string; Running?: boolean; Dead?: boolean; ExitCode?: number } }; + const data = await insp.json() as { State?: { Status?: string; Running?: boolean; Dead?: boolean; ExitCode?: number; Error?: string } }; const st = data.State; - if (st && st.Running === false && st.Status === 'exited' && typeof st.ExitCode === 'number') { - exitCode = st.ExitCode; - console.log(`[runContainerWithStreaming] Container exited with code: ${exitCode} (poll mode)`); + // Resolves for a normal `exited` AND for a container the daemon left + // terminally `created`/`dead` after a start failure (e.g. Docker 29.x + // exit 128 "failed to mount overlay: device or resource busy", #1487) - + // otherwise the unbounded poll waits forever for an `exited` that never comes. + const resolved = helperExitFromState(st); + if (resolved !== undefined) { + exitCode = resolved; + console.log(`[runContainerWithStreaming] Container exited with code: ${exitCode} (poll mode, status=${st?.Status})`); break; } - if (st?.Dead || st?.Status === 'removing') break; // gone -> fail-closed diagnostics below + // A `dead`/`removing` container that helperExitFromState didn't resolve + // (e.g. dead with a 0 exit and no Error) is still terminal - break to the + // fail-closed diagnostics rather than polling it to the deadline. + if (st?.Dead || st?.Status === 'removing') break; } else if (insp.status === 404) { console.warn(`[runContainerWithStreaming] Poll: ${options.name ?? containerId.slice(0, 12)} not found (404) — container removed`); break; @@ -5188,9 +5196,10 @@ export async function runContainerWithStreaming(options: { StartedAt: st?.StartedAt, FinishedAt: st?.FinishedAt, Pid: st?.Pid, })}` ); - if (st && st.Running === false && st.Status === 'exited' && typeof st.ExitCode === 'number') { - exitCode = st.ExitCode; - console.log(`[runContainerWithStreaming] Resolved exit code via inspect: ${exitCode}`); + const resolved = helperExitFromState(st); + if (resolved !== undefined) { + exitCode = resolved; + console.log(`[runContainerWithStreaming] Resolved exit code via inspect: ${exitCode} (status=${st?.Status})`); } } else { console.warn(`[runContainerWithStreaming] EXIT-CODE INDETERMINATE for ${options.name ?? containerId.slice(0, 12)}: inspect returned HTTP ${insp.status} (container may already be gone/removed)`); diff --git a/src/lib/server/helper-wait-core.ts b/src/lib/server/helper-wait-core.ts index 98e4526..8004ab4 100644 --- a/src/lib/server/helper-wait-core.ts +++ b/src/lib/server/helper-wait-core.ts @@ -26,3 +26,35 @@ export function helperWaitDeadline(timeout: number | undefined, now: number): nu const cap = helperWaitCapMs(timeout); return cap > 0 ? now + cap : Infinity; } + +/** The subset of Docker's container State the exit classifier reads. */ +export interface HelperContainerState { + Status?: string; + Running?: boolean; + ExitCode?: number; + Error?: string; +} + +/** + * The definitive exit code for a helper container's State, or undefined when the + * container is not yet in a terminal state (still creating/running -> keep waiting). + * + * A normal helper goes created -> running -> `exited`. But the daemon can also leave a + * container that FAILED TO START in a terminal `created` (or `dead`) state with a + * non-zero ExitCode and a State.Error - e.g. Docker 29.x / containerd leaving exit 128 + * "failed to mount overlay: device or resource busy" (#1487). That is terminal too: the + * container will never run, so its exit code must be resolved (fail-closed) instead of + * the poll loop waiting on an `exited` status that never arrives. + * + * A brand-new `created` container that is about to start (ExitCode 0, no Error) is NOT + * terminal - runContainerWithStreaming calls /start before polling, so a `created` seen + * with a non-zero ExitCode or an Error is a genuine start failure, not a pre-start race. + */ +export function helperExitFromState(state: HelperContainerState | undefined | null): number | undefined { + if (!state || state.Running !== false || typeof state.ExitCode !== 'number') return undefined; + if (state.Status === 'exited') return state.ExitCode; + if ((state.Status === 'created' || state.Status === 'dead') && (state.ExitCode !== 0 || !!state.Error)) { + return state.ExitCode; + } + return undefined; +} diff --git a/tests/helper-wait-core.test.ts b/tests/helper-wait-core.test.ts index d41e617..65576d9 100644 --- a/tests/helper-wait-core.test.ts +++ b/tests/helper-wait-core.test.ts @@ -1,6 +1,6 @@ // @ts-expect-error -- bun:test is a runtime built-in with no types installed import { describe, expect, test } from 'bun:test'; -import { helperWaitCapMs, helperWaitDeadline } from '../src/lib/server/helper-wait-core'; +import { helperWaitCapMs, helperWaitDeadline, helperExitFromState } from '../src/lib/server/helper-wait-core'; describe('helperWaitCapMs', () => { test('a positive timeout is the cap', () => { @@ -36,3 +36,46 @@ describe('helperWaitDeadline', () => { expect(Number.MAX_SAFE_INTEGER < helperWaitDeadline(undefined, 0)).toBe(true); }); }); + +describe('helperExitFromState', () => { + test('a normal exited container returns its exit code', () => { + expect(helperExitFromState({ Status: 'exited', Running: false, ExitCode: 0 })).toBe(0); + expect(helperExitFromState({ Status: 'exited', Running: false, ExitCode: 1 })).toBe(1); + }); + + test('a still-running container is not terminal (keep waiting)', () => { + expect(helperExitFromState({ Status: 'running', Running: true, ExitCode: 0 })).toBeUndefined(); + }); + + test('a brand-new created container about to start is NOT terminal (#1487 race guard)', () => { + // created + ExitCode 0 + no Error = pre-start, must keep waiting, not resolve to 0. + expect(helperExitFromState({ Status: 'created', Running: false, ExitCode: 0 })).toBeUndefined(); + }); + + test('the #1487 case: created + non-zero ExitCode + mount Error IS terminal', () => { + // Docker 29.x / containerd leaving the helper unstarted: exit 128, State.Error set. + expect(helperExitFromState({ + Status: 'created', Running: false, ExitCode: 128, + Error: 'failed to mount /var/lib/docker/rootfs/overlayfs/...: device or resource busy' + })).toBe(128); + }); + + test('created with a non-zero ExitCode (no Error) is terminal', () => { + expect(helperExitFromState({ Status: 'created', Running: false, ExitCode: 125 })).toBe(125); + }); + + test('created with ExitCode 0 but an Error present is terminal', () => { + expect(helperExitFromState({ Status: 'created', Running: false, ExitCode: 0, Error: 'oci runtime error' })).toBe(0); + }); + + test('a dead container with a non-zero exit code is terminal', () => { + expect(helperExitFromState({ Status: 'dead', Running: false, ExitCode: 137 })).toBe(137); + }); + + test('missing / partial state is not terminal', () => { + expect(helperExitFromState(undefined)).toBeUndefined(); + expect(helperExitFromState(null)).toBeUndefined(); + expect(helperExitFromState({ Status: 'exited', Running: false })).toBeUndefined(); // no ExitCode + expect(helperExitFromState({ Running: true })).toBeUndefined(); + }); +});