1.0.38

src/lib/server/scheduler/tasks/backup.ts

@@ -0,0 +1,35 @@
+/**
+ * Backup Scheduler Task
+ *
+ * Handles scheduled backup execution.
+ */
+
+import type { ScheduleTrigger } from '../../db';
+import { runBackup } from '../../backups';
+
+/**
+ * Execute a scheduled backup.
+ */
+export async function runScheduledBackup(
+	configId: number,
+	entityName: string,
+	environmentId: number | null | undefined,
+	triggeredBy: ScheduleTrigger = 'cron'
+): Promise<void> {
+	console.log(`[Backup] Scheduled backup triggered for config ${configId} (${entityName}) by ${triggeredBy}`);
+
+	const validTrigger = triggeredBy === 'cron' || triggeredBy === 'manual' || triggeredBy === 'webhook'
+		? triggeredBy
+		: 'manual';
+
+	const result = await runBackup(configId, validTrigger);
+
+	if (result.status === 'success' || result.status === 'warning') {
+		console.log(`[Backup] Scheduled backup completed for config ${configId} (${entityName}), snapshot: ${result.snapshotId}`);
+	} else if (result.status === 'skipped') {
+		// A benign overlap (one already running) — not a failure.
+		console.log(`[Backup] Scheduled backup skipped for config ${configId} (${entityName}): ${result.reason}`);
+	} else if (result.status === 'error') {
+		console.error(`[Backup] Scheduled backup failed for config ${configId} (${entityName}): ${result.error}`);
+	}
+}

src/lib/server/scheduler/tasks/container-update.ts

@@ -32,8 +32,10 @@ import {
 	tagImage,
 	connectContainerToNetwork,
 	disconnectContainerFromNetwork,
-	recreateContainerFromInspect
+	recreateContainerFromInspect,
+	inspectImage
 } from '../../docker';
+import type { ImageEnvLabels } from '../../container-env-merge';
 import { getScannerSettings, scanImage, type ScanResult, type VulnerabilitySeverity } from '../../scanner';
 import { sendEventNotification } from '../../notifications';
 import { parseImageNameAndTag, combineScanSummaries, isSystemContainer, isPodmanInfraContainer } from './update-utils';
@@ -384,6 +386,16 @@ export async function runContainerUpdate(
 		// Get the actual image ID from inspect data
 		const currentImageId = inspectData.Image;
 
+		// Capture the OLD image's Env/Labels BEFORE the pull — the old digest may
+		// be untagged/GC'd afterward. Forwarded to the env/label rebase (#1226, #1256).
+		let oldImageConfig: ImageEnvLabels | null = null;
+		try {
+			const oldImg = await inspectImage(currentImageId, envId) as any;
+			oldImageConfig = { Env: oldImg?.Config?.Env, Labels: oldImg?.Config?.Labels };
+		} catch {
+			// Best-effort; rebase falls back if unavailable.
+		}
+
 		log(`Container is using image: ${imageNameFromConfig}`);
 		log(`Current image ID: ${currentImageId?.substring(0, 19)}`);
 
@@ -569,7 +581,11 @@ export async function runContainerUpdate(
 		// =============================================================================
 
 		log(`Recreating container with full config passthrough...`);
-		const result = await recreateContainer(containerName, envId, log, imageNameFromConfig);
+		const result = await recreateContainer(containerName, envId, {
+			log,
+			imageNameOverride: imageNameFromConfig,
+			oldImageConfig
+		});
 
 		if (result.success) {
 			await updateAutoUpdateLastUpdated(containerName, envId);
@@ -623,12 +639,25 @@ export async function runContainerUpdate(
  * Passes inspect data directly to Docker API create, only changing the image.
  * No manual field mapping — zero settings loss.
  */
+export interface RecreateContainerOptions {
+	/** Progress logger. */
+	log?: (msg: string) => void;
+	/** New image to recreate with (defaults to the container's current image). */
+	imageNameOverride?: string;
+	/**
+	 * OLD image Config (Env/Labels) captured BEFORE the new image was pulled —
+	 * forwarded to the env/label rebase (#1226, #1256), since the old image may
+	 * no longer be inspectable after the pull repoints the tag.
+	 */
+	oldImageConfig?: ImageEnvLabels | null;
+}
+
 export async function recreateContainer(
 	containerName: string,
 	envId?: number,
-	log?: (msg: string) => void,
-	imageNameOverride?: string
+	options: RecreateContainerOptions = {}
 ): Promise<{ success: boolean; error?: string }> {
+	const { log, imageNameOverride, oldImageConfig } = options;
 	try {
 		const containers = await listContainers(true, envId);
 		const container = containers.find(c => c.name === containerName);
@@ -643,7 +672,7 @@ export async function recreateContainer(
 
 		log?.(`Recreating container: ${containerName} (image: ${imageName})`);
 
-		await recreateContainerFromInspect(inspectData, imageName, envId, log);
+		await recreateContainerFromInspect(inspectData, imageName, envId, log, oldImageConfig);
 		return { success: true };
 	} catch (error: any) {
 		log?.(`Failed to recreate container: ${error.message}`);

src/lib/server/scheduler/tasks/env-update-check.ts

@@ -27,7 +27,9 @@ import {
 	getImageIdByTag,
 	removeTempImage,
 	tagImage,
+	inspectImage,
 } from '../../docker';
+import type { ImageEnvLabels } from '../../container-env-merge';
 import { sendEventNotification } from '../../notifications';
 import { getScannerSettings, scanImage, type VulnerabilitySeverity } from '../../scanner';
 import { parseImageNameAndTag, combineScanSummaries, isSystemContainer, isPodmanInfraContainer } from './update-utils';
@@ -42,6 +44,8 @@ interface UpdateInfo {
 	currentImageId: string;
 	currentDigest?: string;
 	newDigest?: string;
+	// OLD image Env/Labels captured before any pull — for the rebase (#1226, #1256).
+	oldImageConfig?: ImageEnvLabels | null;
 }
 
 // Track running update checks to prevent concurrent execution
@@ -160,13 +164,23 @@ export async function runEnvUpdateCheckJob(
 				}
 
 				if (result.hasUpdate) {
+					// Capture the OLD image's Env/Labels now, before any pull, for the
+					// env/label rebase (#1226, #1256).
+					let oldImageConfig: ImageEnvLabels | null = null;
+					try {
+						const oldImg = await inspectImage(currentImageId, environmentId) as any;
+						oldImageConfig = { Env: oldImg?.Config?.Env, Labels: oldImg?.Config?.Labels };
+					} catch {
+						// Best-effort; rebase falls back if unavailable.
+					}
 					updatesAvailable.push({
 						containerId: container.id,
 						containerName: container.name,
 						imageName,
 						currentImageId,
 						currentDigest: result.currentDigest,
-						newDigest: result.registryDigest
+						newDigest: result.registryDigest,
+						oldImageConfig
 					});
 					// Add to pending table immediately - will be removed on successful update
 					await addPendingContainerUpdate(environmentId, container.id, container.name, imageName);
@@ -374,8 +388,10 @@ export async function runEnvUpdateCheckJob(
 
 					// Recreate container with full config passthrough
 					await log(`  Recreating container...`);
-					const result = await recreateContainer(update.containerName, environmentId,
-						(msg) => { log(`  ${msg}`); });
+					const result = await recreateContainer(update.containerName, environmentId, {
+						log: (msg) => { log(`  ${msg}`); },
+						oldImageConfig: update.oldImageConfig
+					});
 					if (!result.success) throw new Error(result.error || 'Container recreation failed');
 
 					await log(`  Updated successfully`);

src/lib/server/scheduler/tasks/git-stack-sync.ts

@@ -11,7 +11,6 @@ import {
 	appendScheduleExecutionLog
 } from '../../db';
 import { deployGitStack } from '../../git';
-import { sendEventNotification } from '../../notifications';
 
 /**
  * Execute a git stack sync.
@@ -46,30 +45,16 @@ export async function runGitStackSync(
 	try {
 		log(`Starting sync for stack: ${stackName}`);
 
-		// Deploy the git stack (only if there are changes)
+		// Deploy the git stack (only if there are changes). deployGitStack now emits the
+		// git_sync_success/failed/skipped notification itself, so EVERY caller (webhook,
+		// manual, this scheduler) notifies uniformly — we no longer dispatch here (#1295).
 		const result = await deployGitStack(stackId, { force: false });
 
-		const envId = environmentId ?? undefined;
-
 		if (result.success) {
 			if (result.skipped) {
 				log(`No changes detected for stack: ${stackName}, skipping redeploy`);
-
-				// Send notification for skipped sync
-				await sendEventNotification('git_sync_skipped', {
-					title: 'Git sync skipped',
-					message: `Stack "${stackName}" sync skipped: no changes detected`,
-					type: 'info'
-				}, envId);
 			} else {
 				log(`Successfully deployed stack: ${stackName}`);
-
-				// Send notification for successful sync
-				await sendEventNotification('git_sync_success', {
-					title: 'Git stack deployed',
-					message: `Stack "${stackName}" was synced and deployed successfully`,
-					type: 'success'
-				}, envId);
 			}
 			if (result.output) log(result.output);
 
@@ -90,13 +75,6 @@ export async function runGitStackSync(
 			duration: Date.now() - startTime,
 			errorMessage: error.message
 		});
-
-		// Send notification for failed sync
-		const envId = environmentId ?? undefined;
-		await sendEventNotification('git_sync_failed', {
-			title: 'Git sync failed',
-			message: `Stack "${stackName}" sync failed: ${error.message}`,
-			type: 'error'
-		}, envId);
+		// Notification is emitted by deployGitStack (git_sync_failed); not re-sent here.
 	}
 }

src/lib/server/scheduler/tasks/repo-maintenance.ts

@@ -0,0 +1,166 @@
+/**
+ * Repository maintenance tasks — scheduled prune, check, auto-unlock.
+ * Run per-destination based on destination policies.
+ */
+
+import { runRepoTask, verifyBackup } from '$lib/server/backups';
+import { parsePoliciesJson } from '$lib/server/backups/helpers';
+import { getBackupDestination } from '$lib/server/db';
+import { createScheduleExecution, updateScheduleExecution, appendScheduleExecutionLog } from '$lib/server/db';
+import { sendEventNotification } from '$lib/server/notifications';
+
+async function runMaintenanceTask(
+	destinationId: number,
+	destinationName: string,
+	task: 'prune' | 'check',
+	scheduleType: 'repo_prune' | 'repo_check',
+	triggeredBy: 'cron' | 'manual' = 'cron'
+): Promise<void> {
+	const label = task === 'prune' ? 'Prune' : 'Check';
+	const execution = await createScheduleExecution({
+		scheduleType,
+		scheduleId: destinationId,
+		environmentId: null,
+		entityName: `${label}: ${destinationName}`,
+		triggeredBy,
+		status: 'running'
+	});
+	await updateScheduleExecution(execution.id, { startedAt: new Date().toISOString() });
+	const startTime = Date.now();
+	const log = (msg: string) => appendScheduleExecutionLog(execution.id, `[${new Date().toISOString()}] ${msg}`);
+
+	try {
+		const dest = await getBackupDestination(destinationId);
+		if (!dest) throw new Error('Destination not found');
+
+		console.log(`[Backup] Repo ${task} starting for "${destinationName}" (${triggeredBy})`);
+
+		// Auto-unlock before task if enabled in policies
+		const policies = parsePoliciesJson(dest.policies); // (audit #26/#37) safe-degrade + log on malformed
+		if (policies.autoUnlock) {
+			log('Auto-unlocking repository...');
+			await runRepoTask(destinationId, 'unlock');
+		}
+
+		log(`Running ${task}...`);
+		const result = await runRepoTask(destinationId, task);
+
+		if (result.success) {
+			log(`${label} completed: ${result.output}`);
+			console.log(`[Backup] Repo ${task} completed for "${destinationName}"`);
+			await updateScheduleExecution(execution.id, {
+				status: 'success',
+				completedAt: new Date().toISOString(),
+				duration: Date.now() - startTime
+			});
+			await sendEventNotification(`${scheduleType}_success` as 'repo_prune_success' | 'repo_check_success', {
+				title: `${label} completed — ${destinationName}`,
+				message: result.output || `${label} completed successfully`,
+				type: 'success'
+			});
+		} else {
+			throw new Error(result.error || `${label} failed`);
+		}
+	} catch (error) {
+		const msg = error instanceof Error ? error.message : String(error);
+		log(`${label} failed: ${msg}`);
+		console.log(`[Backup] Repo ${task} failed for "${destinationName}": ${msg}`);
+		await updateScheduleExecution(execution.id, {
+			status: 'failed',
+			completedAt: new Date().toISOString(),
+			duration: Date.now() - startTime,
+			errorMessage: msg
+		});
+		await sendEventNotification(`${scheduleType}_failed` as 'repo_prune_failed' | 'repo_check_failed', {
+			title: `${label} failed — ${destinationName}`,
+			message: msg,
+			type: 'error'
+		});
+	}
+}
+
+export async function runRepoPrune(
+	destinationId: number,
+	destinationName: string,
+	triggeredBy: 'cron' | 'manual' = 'cron'
+): Promise<void> {
+	return runMaintenanceTask(destinationId, destinationName, 'prune', 'repo_prune', triggeredBy);
+}
+
+export async function runRepoCheck(
+	destinationId: number,
+	destinationName: string,
+	triggeredBy: 'cron' | 'manual' = 'cron'
+): Promise<void> {
+	return runMaintenanceTask(destinationId, destinationName, 'check', 'repo_check', triggeredBy);
+}
+
+export async function runRepoVerify(
+	destinationId: number,
+	destinationName: string,
+	dataSubset: string,
+	triggeredBy: 'cron' | 'manual' = 'cron'
+): Promise<void> {
+	const execution = await createScheduleExecution({
+		scheduleType: 'repo_verify',
+		scheduleId: destinationId,
+		environmentId: null,
+		entityName: `Verify: ${destinationName}`,
+		triggeredBy,
+		status: 'running'
+	});
+	await updateScheduleExecution(execution.id, { startedAt: new Date().toISOString() });
+	const startTime = Date.now();
+	const log = (msg: string) => appendScheduleExecutionLog(execution.id, `[${new Date().toISOString()}] ${msg}`);
+
+	try {
+		const dest = await getBackupDestination(destinationId);
+		if (!dest) throw new Error('Destination not found');
+
+		console.log(`[Backup] Repo verify starting for "${destinationName}" (${triggeredBy}, ${dataSubset})`);
+
+		const policies = parsePoliciesJson(dest.policies); // (audit #26/#37) safe-degrade + log on malformed
+		if (policies.autoUnlock) {
+			log('Auto-unlocking repository...');
+			await runRepoTask(destinationId, 'unlock');
+		}
+
+		log(`Verifying ${dataSubset} of data...`);
+		const result = await verifyBackup(destinationId, {
+			dataSubset,
+			onProgress: (m) => log(m)
+		});
+
+		if (result.success) {
+			log(`Verify completed: ${result.output}`);
+			console.log(`[Backup] Repo verify completed for "${destinationName}"`);
+			await updateScheduleExecution(execution.id, {
+				status: 'success',
+				completedAt: new Date().toISOString(),
+				duration: Date.now() - startTime
+			});
+			await sendEventNotification('repo_verify_success', {
+				title: `Data verify completed — ${destinationName}`,
+				message: `Verified ${dataSubset} of data successfully`,
+				type: 'success'
+			});
+		} else {
+			throw new Error(result.error || 'Verify failed');
+		}
+	} catch (error) {
+		const msg = error instanceof Error ? error.message : String(error);
+		log(`Verify failed: ${msg}`);
+		console.log(`[Backup] Repo verify failed for "${destinationName}": ${msg}`);
+		await updateScheduleExecution(execution.id, {
+			status: 'failed',
+			completedAt: new Date().toISOString(),
+			duration: Date.now() - startTime,
+			errorMessage: msg
+		});
+		await sendEventNotification('repo_verify_failed', {
+			title: `Data verify failed — ${destinationName}`,
+			message: msg,
+			type: 'error'
+		});
+	}
+}

src/lib/server/scheduler/tasks/update-utils.ts

@@ -37,6 +37,34 @@ export function parseImageNameAndTag(imageName: string): [string, string] {
 	return [imageName.substring(0, lastColon), afterColon];
 }
 
+/**
+ * Classify a container whose RUNNING image carries NO local repo digests (#1288).
+ *
+ * An empty RepoDigests set is ambiguous: it is what you see BOTH for a genuinely
+ * local/built image (nothing to check against a registry) AND for a registry image
+ * whose tag has since moved to a newer pull — leaving the container on an old,
+ * now-untagged, digest-less image. The old code assumed "empty digests ⇒ local" and
+ * skipped the update check forever, stranding exactly the containers that DO need an
+ * update.
+ *
+ * The tie-breaker is whether the registry could resolve the container's Config.Image
+ * tag (looked up separately, since that's async I/O):
+ *  - registry answered  ⇒ it's a registry image, and since the running image has no
+ *    digest to match, an update is available.
+ *  - registry didn't    ⇒ truly local (or unreachable) ⇒ leave it classified local.
+ *
+ * @param registryDigest the digest returned for Config.Image's tag, or null/undefined
+ *                        if the registry couldn't resolve it.
+ */
+export function classifyEmptyDigestImage(
+	registryDigest: string | null | undefined
+): { hasUpdate: boolean; isLocalImage: boolean; registryDigest?: string } {
+	if (!registryDigest) {
+		return { hasUpdate: false, isLocalImage: true };
+	}
+	return { hasUpdate: true, isLocalImage: false, registryDigest };
+}
+
 /**
  * Determine if an update should be blocked based on vulnerability criteria.
  */