From c91f38eddffe4535c5dd06b086f1b3131c37a8af Mon Sep 17 00:00:00 2001 From: jarek Date: Thu, 13 Aug 2026 10:20:29 +0200 Subject: [PATCH] upload a Docker image .tar through the UI (docker load) for air-gapped hosts --- src/lib/server/docker.ts | 74 +++++++++++----- src/lib/server/images.ts | 88 +++++++++++++++++++ src/routes/api/images/load/+server.ts | 61 +++++++++++++ src/routes/images/+page.svelte | 46 ++++++++++ .../settings/auth/roles/RoleModal.svelte | 1 + 5 files changed, 246 insertions(+), 24 deletions(-) create mode 100644 src/lib/server/images.ts create mode 100644 src/routes/api/images/load/+server.ts diff --git a/src/lib/server/docker.ts b/src/lib/server/docker.ts index b49482a..28572a4 100644 --- a/src/lib/server/docker.ts +++ b/src/lib/server/docker.ts @@ -10,6 +10,7 @@ import { existsSync } from 'node:fs'; import { join } from 'node:path'; import * as http from 'node:http'; import * as https from 'node:https'; +import { Readable } from 'node:stream'; import * as tls from 'node:tls'; import { createHash } from 'node:crypto'; import { pumpWebStreamToWritable } from './stream-pump'; @@ -17,6 +18,7 @@ import { computeRequestTimeoutMs } from './backups/request-timeout'; import type { Environment } from './db'; import { getSetting } from './db'; import { getAdditionalVolumeBinds, dedupeVolumesForRecreate } from './mount-dedupe'; +import { resolveNanoCpusConflict } from './hostconfig-recreate'; import { rebaseEnvOntoImage, rebaseLabelsOntoImage, rebaseCommand, describeEnvRebase, describeLabelRebase, type ImageEnvLabels } from './container-env-merge'; import { encodeRegistryAuth } from './registry-auth'; import { isSystemContainer, classifyEmptyDigestImage } from './scheduler/tasks/update-utils'; @@ -726,15 +728,21 @@ export function unixSocketRequest( req.on('error', reject); - if (options.body) { - if (typeof options.body === 'string') { - req.write(options.body); - } else if (options.body instanceof Uint8Array || Buffer.isBuffer(options.body)) { - req.write(options.body); + if (options.body instanceof ReadableStream) { + // Stream a web ReadableStream (e.g. a large tar for /images/load) straight + // to the socket without buffering it in memory. Readable.fromWeb bridges + // web -> Node stream; pipe() ends the request when the source finishes. + Readable.fromWeb(options.body as any).on('error', reject).pipe(req); + } else { + if (options.body) { + if (typeof options.body === 'string') { + req.write(options.body); + } else if (options.body instanceof Uint8Array || Buffer.isBuffer(options.body)) { + req.write(options.body); + } } + req.end(); } - - req.end(); }); } @@ -805,15 +813,21 @@ export function unixSocketStreamRequest( req.on('error', reject); - if (options.body) { - if (typeof options.body === 'string') { - req.write(options.body); - } else if (options.body instanceof Uint8Array || Buffer.isBuffer(options.body)) { - req.write(options.body); + if (options.body instanceof ReadableStream) { + // Stream a web ReadableStream (e.g. a large tar for /images/load) straight + // to the socket without buffering it in memory. Readable.fromWeb bridges + // web -> Node stream; pipe() ends the request when the source finishes. + Readable.fromWeb(options.body as any).on('error', reject).pipe(req); + } else { + if (options.body) { + if (typeof options.body === 'string') { + req.write(options.body); + } else if (options.body instanceof Uint8Array || Buffer.isBuffer(options.body)) { + req.write(options.body); + } } + req.end(); } - - req.end(); }); } @@ -967,6 +981,12 @@ export async function dockerFetch( finalOptions.signal = AbortSignal.timeout(isComposeOperation ? composeTimeoutMs : isPrune ? 300000 : 30000); } + // A ReadableStream request body (e.g. a large tar for /images/load) requires + // duplex:'half' under Node's fetch, else it throws before sending. + if (typeof (finalOptions.body as ReadableStream | undefined)?.getReader === 'function') { + (finalOptions as RequestInit & { duplex?: string }).duplex = 'half'; + } + try { const response = await fetch(url, finalOptions); const elapsed = Date.now() - startTime; @@ -2066,6 +2086,13 @@ export async function recreateContainerFromInspect( delete createConfig.HostConfig.MemorySwappiness; } + // NanoCpus and CpuPeriod/CpuQuota are two ways to express the same CPU limit and are + // mutually exclusive at create time. Podman's inspect reports BOTH, so passing the whole + // HostConfig back trips its "NanoCpus conflicts with CpuPeriod and CpuQuota" (#1381). + if (resolveNanoCpusConflict(createConfig.HostConfig)) { + log?.('Dropped CpuPeriod/CpuQuota — they conflict with NanoCpus at create time (kept NanoCpus)'); + } + // container: mode shares the network namespace — Docker rejects // networking-related fields on the dependent container since they're // owned by the network provider container @@ -4994,19 +5021,18 @@ export async function runContainerWithStreaming(options: { // Bounded-retry the wait; if it still can't be determined, fall back to // inspecting the container's final State, and if that's also unavailable // leave exitCode undefined — the guard below fails closed on undefined. - // HELPERS_WAIT_MODE (default 'wait'): the exit signal below is a single - // streaming `POST /wait`, held open with no bytes for the whole run. A - // socket-proxy / reverse-proxy / tcp DOCKER_HOST with a short idle timeout can - // sever it mid-run, leaving the exit code unreadable even though the helper is - // fine (#1344). Set HELPERS_WAIT_MODE=poll to POLL short inspect calls instead - // (nothing stays open for a proxy to cut). Applies to every helper this function - // runs — backup/restore AND the image scanner. + // HELPERS_WAIT_MODE (default 'poll'): the exit signal is read by POLLING short + // inspect calls - nothing stays open for a proxy to cut. The alternative, a single + // streaming `POST /wait` held open with no bytes for the whole run, gets severed + // mid-run by a socket-proxy / reverse-proxy / tcp DOCKER_HOST with a short idle + // timeout, leaving the exit code unreadable even though the helper is fine (#1344). + // Poll is the safe default; set HELPERS_WAIT_MODE=wait to force the streaming /wait. + // Applies to every helper this function runs — backup/restore AND the image scanner. let exitCode: number | undefined; - if (process.env.HELPERS_WAIT_MODE === 'poll') { + if (process.env.HELPERS_WAIT_MODE !== 'wait') { // Bound by the caller's timeout (backup passes a long 'data' tier, the // scanner passes 600_000). Fall back to 1h so poll mode never hangs. - // One line so the log confirms which exit-signal mode is active (helps - // diagnose whether HELPERS_WAIT_MODE=poll actually took effect - #1344). + // One line so the log confirms which exit-signal mode is active (#1344). console.log(`[runContainerWithStreaming] Awaiting exit of ${options.name ?? containerId.slice(0, 12)} via POLL mode`); const deadline = Date.now() + (options.timeout && options.timeout > 0 ? options.timeout : 3_600_000); while (exitCode === undefined && Date.now() < deadline) { diff --git a/src/lib/server/images.ts b/src/lib/server/images.ts new file mode 100644 index 0000000..dffab5a --- /dev/null +++ b/src/lib/server/images.ts @@ -0,0 +1,88 @@ +/** + * Image operations that don't belong in the transport layer (docker.ts). Starts + * with `loadImage` (docker load from an uploaded tar); the rest of the image + * functions still live in docker.ts and can migrate here incrementally. + */ + +import { dockerFetch, getDockerConfig } from './docker'; + +/** A line from the /images/load progress stream. Docker sends `{stream: "..."}` for + * progress and `{errorDetail: {...}}` / `{error: "..."}` on failure. */ +export interface LoadProgress { + stream?: string; + error?: string; + errorDetail?: { message?: string }; +} + +/** + * Loads a Docker image from a tar stream (equivalent to `docker load`), for + * air-gapped hosts with no registry access. The tar is streamed straight to the + * daemon's `POST /images/load` without buffering it in memory, so a multi-GB image + * does not OOM Dockhand. `onProgress` receives each parsed progress line. + * + * Only supported on connection types where Dockhand streams the request body to the + * daemon: socket (local), direct HTTP, and direct HTTPS/mTLS. Hawser (standard/edge) + * is rejected up front - the WS transport can't stream a large upload - with an + * actionable error rather than a silent failure mid-transfer. + */ +export async function loadImage( + tar: ReadableStream, + onProgress?: (data: LoadProgress) => void, + envId?: number | null +): Promise { + const config = await getDockerConfig(envId); + if (config.connectionType === 'hawser-edge' || config.connectionType === 'hawser-standard') { + throw new Error( + 'Uploading an image tar is not supported on Hawser connections yet; use a local or direct-TCP environment.' + ); + } + + // quiet=false so the daemon streams progress (loaded image name) back. + const response = await dockerFetch( + '/images/load?quiet=false', + { + method: 'POST', + streaming: true, + headers: { 'Content-Type': 'application/x-tar' }, + body: tar + }, + envId + ); + + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`Failed to load image (HTTP ${response.status})${body ? `: ${body}` : ''}`); + } + + // Docker streams NDJSON progress inside a 200 response, and reports a failed load + // as an {errorDetail}/{error} line - capture it and throw after the stream ends. + const reader = response.body?.getReader(); + if (!reader) return; + const decoder = new TextDecoder(); + let buffer = ''; + let streamError: string | null = null; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + for (const line of lines) { + if (!line.trim()) continue; + try { + const data = JSON.parse(line) as LoadProgress; + if (data.error || data.errorDetail) { + streamError = data.errorDetail?.message || data.error || 'Image load failed'; + } + onProgress?.(data); + } catch { + // ignore non-JSON keepalive lines + } + } + } + + if (streamError) { + throw new Error(`Failed to load image: ${streamError}`); + } +} diff --git a/src/routes/api/images/load/+server.ts b/src/routes/api/images/load/+server.ts new file mode 100644 index 0000000..5ef249d --- /dev/null +++ b/src/routes/api/images/load/+server.ts @@ -0,0 +1,61 @@ +import { json } from '@sveltejs/kit'; +import { loadImage } from '$lib/server/images'; +import { authorize } from '$lib/server/authorize'; +import { auditImage } from '$lib/server/audit'; +import type { RequestHandler } from './$types'; + +/** + * POST /api/images/load - load a Docker image from an uploaded tar (docker load), + * for air-gapped hosts. The request BODY is the raw tar (Content-Type + * application/x-tar); it is streamed straight to the daemon without buffering, so a + * large image does not OOM. + * + * This is SYNCHRONOUS on purpose - it must NOT use the job-polling pattern. That + * pattern returns immediately and runs the operation in the background, but the tar + * lives in the request body, which is torn down once the response is sent; a + * backgrounded read would hang on a dead stream. So we consume the body and stream it + * to the daemon while the request is still open, then return the result. + * + * Local/socket and direct TCP only (loadImage rejects Hawser). + */ +export const POST: RequestHandler = async (event) => { + const { request, url, cookies } = event; + const auth = await authorize(cookies); + + const envIdParam = url.searchParams.get('env'); + const envId = envIdParam ? parseInt(envIdParam) : undefined; + + // Loading a foreign tar is a distinct, higher-trust operation than pulling from a + // registry (no scan-on-pull, arbitrary user-supplied image), so it has its own + // permission rather than reusing images:pull. + if (auth.authEnabled && !(await auth.can('images', 'load', envId))) { + return json({ error: 'Permission denied' }, { status: 403 }); + } + if (envId && auth.isEnterprise && !(await auth.canAccessEnvironment(envId))) { + return json({ error: 'Access denied to this environment' }, { status: 403 }); + } + + if (!request.body) { + return json({ error: 'Request body (an image tar) is required' }, { status: 400 }); + } + const tar = request.body as ReadableStream; + + await auditImage(event, 'load', 'tar-upload', 'tar-upload', envId); + + try { + let lastStream = ''; + await loadImage( + tar, + (data) => { + // The daemon's final `stream` line is e.g. "Loaded image: alpine:3.20". + if (data.stream) lastStream = data.stream.trim(); + }, + envId + ); + return json({ success: true, loaded: lastStream }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error('[images/load] failed:', message); + return json({ success: false, error: message }, { status: 500 }); + } +}; diff --git a/src/routes/images/+page.svelte b/src/routes/images/+page.svelte index 57934d0..ea87655 100644 --- a/src/routes/images/+page.svelte +++ b/src/routes/images/+page.svelte @@ -99,6 +99,39 @@ // Pull modal state let showPullModal = $state(false); + // Load-from-tar state + let loadFileInput = $state(null); + let loadingImage = $state(false); + + async function handleLoadTar(e: Event) { + const input = e.currentTarget as HTMLInputElement; + const file = input.files?.[0]; + input.value = ''; // reset so re-selecting the same file fires change again + if (!file) return; + + loadingImage = true; + const t = toast.loading(`Loading ${file.name}...`); + try { + // Stream the file body straight to the endpoint (no in-memory copy). + const response = await fetch(appendEnvParam('/api/images/load', envId), { + method: 'POST', + headers: { 'Content-Type': 'application/x-tar' }, + body: file + }); + const data = await response.json(); + if (data?.success) { + toast.success(data.loaded || `Loaded image from ${file.name}`, { id: t }); + await fetchImages(); + } else { + toast.error(data?.error || 'Failed to load image', { id: t }); + } + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Failed to load image', { id: t }); + } finally { + loadingImage = false; + } + } + // Run modal state let showRunModal = $state(false); let prefilledImage = $state(''); @@ -1032,6 +1065,19 @@ Pull {/if} + {#if $canAccess('images', 'load')} + + + {/if} {/if} diff --git a/src/routes/settings/auth/roles/RoleModal.svelte b/src/routes/settings/auth/roles/RoleModal.svelte index dc20b75..400283c 100644 --- a/src/routes/settings/auth/roles/RoleModal.svelte +++ b/src/routes/settings/auth/roles/RoleModal.svelte @@ -176,6 +176,7 @@ images: [ { key: 'view', label: 'View images' }, { key: 'pull', label: 'Pull images' }, + { key: 'load', label: 'Load images from tar' }, { key: 'push', label: 'Push images' }, { key: 'remove', label: 'Remove images' }, { key: 'build', label: 'Build images' },