Files
dockhand-upstream/src/lib/server/scheduler/tasks/container-update.ts
T
2026-06-15 14:56:51 +02:00

679 lines
21 KiB
TypeScript

/**
* Container Auto-Update Task
*
* Handles automatic container updates with vulnerability scanning.
* Uses direct Docker API recreation with full Config/HostConfig passthrough
* from inspect data. No compose commands for
* individual container updates, no manual field mapping, zero settings loss.
*/
import type { ScheduleTrigger, VulnerabilityCriteria } from '../../db';
import {
getAutoUpdateSettingById,
updateAutoUpdateLastChecked,
updateAutoUpdateLastUpdated,
createScheduleExecution,
updateScheduleExecution,
appendScheduleExecutionLog,
saveVulnerabilityScan,
getCombinedScanForImage
} from '../../db';
import {
pullImage,
listContainers,
inspectContainer,
stopContainer,
startContainer,
removeContainer,
checkImageUpdateAvailable,
getTempImageTag,
isDigestBasedImage,
getImageIdByTag,
removeTempImage,
tagImage,
connectContainerToNetwork,
disconnectContainerFromNetwork,
recreateContainerFromInspect
} from '../../docker';
import { getScannerSettings, scanImage, type ScanResult, type VulnerabilitySeverity } from '../../scanner';
import { sendEventNotification } from '../../notifications';
import { parseImageNameAndTag, shouldBlockUpdate, combineScanSummaries, isSystemContainer } from './update-utils';
import { isUpdateDisabledByLabel, isHiddenByLabel } from '../../container-labels';
// =============================================================================
// TYPES
// =============================================================================
interface ScanContext {
newImageId: string;
currentImageId: string;
envId: number | undefined;
vulnerabilityCriteria: VulnerabilityCriteria;
log: (msg: string) => void;
}
interface ScanOutcome {
blocked: boolean;
reason?: string;
scanResults?: ScanResult[];
scanSummary?: VulnerabilitySeverity;
}
interface ExecutionDetails {
mode: string;
newDigest?: string;
vulnerabilityCriteria: VulnerabilityCriteria;
reason?: string;
blockReason?: string;
summary: { checked: number; updated: number; blocked: number; failed: number };
containers: Array<{
name: string;
status: string;
blockReason?: string;
scannerResults?: Array<{
scanner: string;
critical: number;
high: number;
medium: number;
low: number;
negligible: number;
unknown: number;
}>;
}>;
scanResult?: {
summary: VulnerabilitySeverity;
scanners: string[];
scannedAt?: string;
scannerResults: Array<{
scanner: string;
critical: number;
high: number;
medium: number;
low: number;
negligible: number;
unknown: number;
}>;
};
}
// =============================================================================
// HELPER FUNCTIONS
// =============================================================================
/**
* Scan an image and check if the update should be blocked based on vulnerability criteria.
* Handles scanning, saving results, and comparing with current image for 'more_than_current'.
*/
async function scanAndCheckBlock(ctx: ScanContext): Promise<ScanOutcome> {
const { newImageId, currentImageId, envId, vulnerabilityCriteria, log } = ctx;
log(`Scanning new image for vulnerabilities...`);
const scanResults = await scanImage(newImageId, envId, (progress) => {
const scannerTag = progress.scanner ? `[${progress.scanner}]` : '[scan]';
if (progress.message) {
log(`${scannerTag} ${progress.message}`);
}
if (progress.output) {
log(`${scannerTag} ${progress.output}`);
}
});
if (scanResults.length === 0) {
return { blocked: false, scanResults };
}
const scanSummary = combineScanSummaries(scanResults);
log(`Scan result: ${scanSummary.critical} critical, ${scanSummary.high} high, ${scanSummary.medium} medium, ${scanSummary.low} low`);
// Save scan results
for (const result of scanResults) {
try {
await saveVulnerabilityScan({
environmentId: envId ?? null,
imageId: newImageId,
imageName: result.imageName,
scanner: result.scanner,
scannedAt: result.scannedAt,
scanDuration: result.scanDuration,
criticalCount: result.summary.critical,
highCount: result.summary.high,
mediumCount: result.summary.medium,
lowCount: result.summary.low,
negligibleCount: result.summary.negligible,
unknownCount: result.summary.unknown,
vulnerabilities: result.vulnerabilities,
error: result.error ?? null
});
} catch (saveError: any) {
log(`Warning: Could not save scan results: ${saveError.message}`);
}
}
// Handle 'more_than_current' criteria - need to get/scan current image
let currentScanSummary: VulnerabilitySeverity | undefined;
if (vulnerabilityCriteria === 'more_than_current') {
log(`Looking up cached scan for current image...`);
try {
const cachedScan = await getCombinedScanForImage(currentImageId, envId ?? null);
if (cachedScan) {
currentScanSummary = cachedScan;
log(`Cached scan: ${currentScanSummary.critical} critical, ${currentScanSummary.high} high`);
} else {
log(`No cached scan found, scanning current image...`);
const currentScanResults = await scanImage(currentImageId, envId, (progress) => {
const tag = progress.scanner ? `[${progress.scanner}]` : '[scan]';
if (progress.message) log(`${tag} ${progress.message}`);
});
if (currentScanResults.length > 0) {
currentScanSummary = combineScanSummaries(currentScanResults);
log(`Current image: ${currentScanSummary.critical} critical, ${currentScanSummary.high} high`);
// Save for future use
for (const result of currentScanResults) {
try {
await saveVulnerabilityScan({
environmentId: envId ?? null,
imageId: currentImageId,
imageName: result.imageName,
scanner: result.scanner,
scannedAt: result.scannedAt,
scanDuration: result.scanDuration,
criticalCount: result.summary.critical,
highCount: result.summary.high,
mediumCount: result.summary.medium,
lowCount: result.summary.low,
negligibleCount: result.summary.negligible,
unknownCount: result.summary.unknown,
vulnerabilities: result.vulnerabilities,
error: result.error ?? null
});
} catch { /* ignore */ }
}
}
}
} catch (cacheError: any) {
log(`Warning: Could not get current scan: ${cacheError.message}`);
}
}
// Check if update should be blocked
const { blocked, reason } = shouldBlockUpdate(vulnerabilityCriteria, scanSummary, currentScanSummary);
if (blocked) {
log(`UPDATE BLOCKED: ${reason}`);
return { blocked: true, reason, scanResults, scanSummary };
}
log(`Scan passed vulnerability criteria`);
return { blocked: false, scanResults, scanSummary };
}
/**
* Build scanner results array from scan results for execution details.
*/
function buildScannerResults(scanResults: ScanResult[]) {
return scanResults.map(r => ({
scanner: r.scanner,
critical: r.summary.critical,
high: r.summary.high,
medium: r.summary.medium,
low: r.summary.low,
negligible: r.summary.negligible,
unknown: r.summary.unknown
}));
}
/**
* Build execution details for a blocked update.
*/
function buildBlockedDetails(
containerName: string,
vulnerabilityCriteria: VulnerabilityCriteria,
reason: string,
scanResults: ScanResult[],
scanSummary: VulnerabilitySeverity
): ExecutionDetails {
const scannerResults = buildScannerResults(scanResults);
return {
mode: 'auto_update',
reason: 'vulnerabilities_found',
blockReason: reason,
vulnerabilityCriteria,
summary: { checked: 1, updated: 0, blocked: 1, failed: 0 },
containers: [{
name: containerName,
status: 'blocked',
blockReason: reason,
scannerResults
}],
scanResult: {
summary: scanSummary,
scanners: scanResults.map(r => r.scanner),
scannedAt: scanResults[0]?.scannedAt,
scannerResults
}
};
}
/**
* Build execution details for a successful update.
*/
function buildSuccessDetails(
containerName: string,
newDigest: string | undefined,
vulnerabilityCriteria: VulnerabilityCriteria,
scanResults?: ScanResult[],
scanSummary?: VulnerabilitySeverity
): ExecutionDetails {
const scannerResults = scanResults ? buildScannerResults(scanResults) : undefined;
return {
mode: 'auto_update',
newDigest,
vulnerabilityCriteria,
summary: { checked: 1, updated: 1, blocked: 0, failed: 0 },
containers: [{
name: containerName,
status: 'updated',
scannerResults
}],
scanResult: scanSummary ? {
summary: scanSummary,
scanners: scanResults?.map(r => r.scanner) || [],
scannedAt: scanResults?.[0]?.scannedAt,
scannerResults: scannerResults || []
} : undefined
};
}
// =============================================================================
// MAIN UPDATE FUNCTION
// =============================================================================
/**
* Execute a container auto-update.
*/
export async function runContainerUpdate(
settingId: number,
containerName: string,
environmentId: number | null | undefined,
triggeredBy: ScheduleTrigger
): Promise<void> {
const envId = environmentId ?? undefined;
const startTime = Date.now();
// Create execution record
const execution = await createScheduleExecution({
scheduleType: 'container_update',
scheduleId: settingId,
environmentId: environmentId ?? null,
entityName: containerName,
triggeredBy,
status: 'running'
});
await updateScheduleExecution(execution.id, {
startedAt: new Date().toISOString()
});
const log = (message: string) => {
console.log(`[Auto-update] ${message}`);
appendScheduleExecutionLog(execution.id, `[${new Date().toISOString()}] ${message}`);
};
try {
log(`Checking container: ${containerName}`);
await updateAutoUpdateLastChecked(containerName, envId);
// Find the container
const containers = await listContainers(true, envId);
const container = containers.find(c => c.name === containerName);
if (!container) {
log(`Container not found: ${containerName}`);
await updateScheduleExecution(execution.id, {
status: 'failed',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
errorMessage: 'Container not found'
});
return;
}
// Get the full container config to extract the image name (tag)
const inspectData = await inspectContainer(container.id, envId) as any;
const imageNameFromConfig = inspectData.Config?.Image;
if (!imageNameFromConfig) {
log(`Could not determine image name from container config`);
await updateScheduleExecution(execution.id, {
status: 'failed',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
errorMessage: 'Could not determine image name'
});
return;
}
// Prevent system containers (Dockhand/Hawser) from being updated
const systemContainerType = isSystemContainer(imageNameFromConfig);
if (systemContainerType) {
const reason = systemContainerType === 'dockhand'
? 'Cannot auto-update Dockhand itself'
: 'Cannot auto-update Hawser agent';
log(`Skipping ${systemContainerType} container - ${reason}`);
await updateScheduleExecution(execution.id, {
status: 'skipped',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
details: { reason }
});
return;
}
// Check dockhand.update label (label wins over DB settings)
if (isUpdateDisabledByLabel(inspectData.Config?.Labels)) {
log(`Skipping - dockhand.update=false label set on container`);
await updateScheduleExecution(execution.id, {
status: 'skipped',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
details: { reason: 'Skipped by dockhand.update=false label' }
});
return;
}
// Hidden containers are excluded from update polling and auto-updates (#1083)
if (isHiddenByLabel(inspectData.Config?.Labels)) {
log(`Skipping - dockhand.hidden=true label set on container`);
await updateScheduleExecution(execution.id, {
status: 'skipped',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
details: { reason: 'Skipped by dockhand.hidden=true label' }
});
return;
}
// Skip digest-pinned images - they are explicitly locked to a specific version
if (isDigestBasedImage(imageNameFromConfig)) {
log(`Skipping ${containerName} - image pinned to specific digest`);
await updateScheduleExecution(execution.id, {
status: 'skipped',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
details: { reason: 'Image pinned to specific digest' }
});
return;
}
// Get the actual image ID from inspect data
const currentImageId = inspectData.Image;
log(`Container is using image: ${imageNameFromConfig}`);
log(`Current image ID: ${currentImageId?.substring(0, 19)}`);
// Get scanner and schedule settings early to determine scan strategy
const [scannerSettings, updateSetting] = await Promise.all([
getScannerSettings(envId),
getAutoUpdateSettingById(settingId)
]);
const vulnerabilityCriteria = (updateSetting?.vulnerabilityCriteria || 'never') as VulnerabilityCriteria;
const shouldScan = scannerSettings.scanner !== 'none';
// =============================================================================
// CHECK FOR UPDATES
// =============================================================================
log(`Checking registry for updates: ${imageNameFromConfig}`);
const registryCheck = await checkImageUpdateAvailable(imageNameFromConfig, currentImageId, envId);
if (registryCheck.isLocalImage) {
log(`Local image detected - skipping (auto-update requires registry)`);
await updateScheduleExecution(execution.id, {
status: 'skipped',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
details: { reason: 'Local image - no registry available' }
});
return;
}
if (registryCheck.error) {
log(`Registry check error: ${registryCheck.error}`);
await updateScheduleExecution(execution.id, {
status: 'skipped',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
details: { reason: `Registry check failed: ${registryCheck.error}` }
});
return;
}
if (!registryCheck.hasUpdate) {
log(`Already up-to-date: ${containerName} is running the latest version`);
await updateScheduleExecution(execution.id, {
status: 'skipped',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
details: { reason: 'Already up-to-date' }
});
return;
}
log(`Update available! Registry digest: ${registryCheck.registryDigest?.substring(0, 19) || 'unknown'}`);
const newDigest = registryCheck.registryDigest;
// =============================================================================
// PULL & SCAN: Temp-tag protection flow
// =============================================================================
// 1. Pull new image (overwrites tag)
// 2. Restore original tag to OLD image (safety)
// 3. Tag new image with temp suffix
// 4. Scan temp image, block if needed
// 5. Re-tag to original, recreate container
// =============================================================================
let newImageId: string | null = null;
let scanOutcome: ScanOutcome = { blocked: false };
if (shouldScan && !isDigestBasedImage(imageNameFromConfig)) {
const tempTag = getTempImageTag(imageNameFromConfig);
log(`Using temp tag for safe pull: ${tempTag}`);
try {
// Pull new image
log(`Pulling new image: ${imageNameFromConfig}`);
await pullImage(imageNameFromConfig, undefined, envId);
// Get new image ID
newImageId = await getImageIdByTag(imageNameFromConfig, envId);
if (!newImageId) {
throw new Error('Failed to get new image ID after pull');
}
log(`New image pulled: ${newImageId.substring(0, 19)}`);
// Restore original tag to OLD image for safety
log(`Restoring original tag to safe image...`);
const [oldRepo, oldTag] = parseImageNameAndTag(imageNameFromConfig);
await tagImage(currentImageId, oldRepo, oldTag, envId);
// Tag new image with temp suffix
const [tempRepo, tempTagName] = parseImageNameAndTag(tempTag);
await tagImage(newImageId, tempRepo, tempTagName, envId);
log(`New image tagged as: ${tempTag}`);
// Scan new image (by ID, not temp tag - for proper cache storage)
try {
scanOutcome = await scanAndCheckBlock({
newImageId,
currentImageId,
envId,
vulnerabilityCriteria,
log
});
if (scanOutcome.blocked) {
log(`Removing blocked image: ${tempTag}`);
await removeTempImage(newImageId, envId);
await updateScheduleExecution(execution.id, {
status: 'skipped',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
details: buildBlockedDetails(
containerName,
vulnerabilityCriteria,
scanOutcome.reason!,
scanOutcome.scanResults!,
scanOutcome.scanSummary!
)
});
await sendEventNotification('auto_update_blocked', {
title: 'Auto-update blocked',
message: `Container "${containerName}" update blocked: ${scanOutcome.reason}`,
type: 'warning'
}, envId);
return;
}
} catch (scanError: any) {
log(`Scan failed: ${scanError.message}`);
log(`Removing temp image...`);
await removeTempImage(newImageId, envId);
await updateScheduleExecution(execution.id, {
status: 'failed',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
errorMessage: `Vulnerability scan failed: ${scanError.message}`
});
return;
}
// Re-tag approved image to original
log(`Re-tagging approved image to: ${imageNameFromConfig}`);
await tagImage(newImageId, oldRepo, oldTag, envId);
// Clean up temp tag
try {
await removeTempImage(tempTag, envId);
} catch { /* ignore */ }
} catch (pullError: any) {
log(`Pull failed: ${pullError.message}`);
await updateScheduleExecution(execution.id, {
status: 'failed',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
errorMessage: `Failed to pull image: ${pullError.message}`
});
return;
}
} else {
// No scanning - simple pull
log(`Pulling update (no vulnerability scan)...`);
try {
await pullImage(imageNameFromConfig, undefined, envId);
log(`Image pulled successfully`);
} catch (pullError: any) {
log(`Pull failed: ${pullError.message}`);
await updateScheduleExecution(execution.id, {
status: 'failed',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
errorMessage: `Failed to pull image: ${pullError.message}`
});
return;
}
}
// =============================================================================
// RECREATE CONTAINER (full config passthrough from inspect data)
// =============================================================================
log(`Recreating container with full config passthrough...`);
const result = await recreateContainer(containerName, envId, log, imageNameFromConfig);
if (result.success) {
await updateAutoUpdateLastUpdated(containerName, envId);
log(`Successfully updated container: ${containerName}`);
await updateScheduleExecution(execution.id, {
status: 'success',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
details: buildSuccessDetails(
containerName,
newDigest,
vulnerabilityCriteria,
scanOutcome.scanResults,
scanOutcome.scanSummary
)
});
await sendEventNotification('auto_update_success', {
title: 'Container auto-updated',
message: `Container "${containerName}" was updated to a new image version`,
type: 'success'
}, envId);
} else {
throw new Error(result.error || 'Failed to recreate container');
}
} catch (error: any) {
log(`Error: ${error.message}`);
await updateScheduleExecution(execution.id, {
status: 'failed',
completedAt: new Date().toISOString(),
duration: Date.now() - startTime,
errorMessage: error.message
});
await sendEventNotification('auto_update_failed', {
title: 'Auto-update failed',
message: `Container "${containerName}" auto-update failed: ${error.message}`,
type: 'error'
}, envId);
}
}
// =============================================================================
// EXPORTED HELPER FUNCTIONS
// =============================================================================
/**
* Recreate a container using full Config/HostConfig passthrough from inspect data.
* Passes inspect data directly to Docker API create, only changing the image.
* No manual field mapping — zero settings loss.
*/
export async function recreateContainer(
containerName: string,
envId?: number,
log?: (msg: string) => void,
imageNameOverride?: string
): Promise<{ success: boolean; error?: string }> {
try {
const containers = await listContainers(true, envId);
const container = containers.find(c => c.name === containerName);
if (!container) {
log?.(`Container not found: ${containerName}`);
return { success: false, error: `Container not found: ${containerName}` };
}
const inspectData = await inspectContainer(container.id, envId) as any;
const imageName = imageNameOverride || inspectData.Config?.Image;
log?.(`Recreating container: ${containerName} (image: ${imageName})`);
await recreateContainerFromInspect(inspectData, imageName, envId, log);
return { success: true };
} catch (error: any) {
log?.(`Failed to recreate container: ${error.message}`);
return { success: false, error: error.message };
}
}