From 0a16bd9b17d5d2a54b8f1c5eae851766164d1391 Mon Sep 17 00:00:00 2001 From: KodekaTeam Date: Sat, 22 Aug 2026 16:12:58 +0700 Subject: [PATCH] feat(deployment): implement recovery for interrupted rebuilds --- src/server/routes/containers.ts | 13 +- .../services/deployment-lock.service.ts | 35 ++++- .../services/deployment-recovery.service.ts | 138 ++++++++++++++++++ tests/backend/deployment-recovery.test.ts | 47 ++++++ 4 files changed, 229 insertions(+), 4 deletions(-) create mode 100644 src/server/services/deployment-recovery.service.ts create mode 100644 tests/backend/deployment-recovery.test.ts diff --git a/src/server/routes/containers.ts b/src/server/routes/containers.ts index b72993d..2fee22e 100644 --- a/src/server/routes/containers.ts +++ b/src/server/routes/containers.ts @@ -42,6 +42,7 @@ import { startDeploymentLockHeartbeat, } from "../services/deployment-lock.service"; import { sanitizeDeploymentError } from "../services/deployment-error.service"; +import { reconcileInterruptedRebuild } from "../services/deployment-recovery.service"; import { formatDockerInspectMountBindings, type DockerInspectMount as DockerInspectRuntimeMount, @@ -1766,7 +1767,7 @@ export async function containerRoutes(app: FastifyInstance) { ); const container = await prisma.container.findFirst({ where: { id, server: { organizationId: req.organizationId! } }, - select: { id: true }, + include: { server: true }, }); if (!container) { return reply @@ -1774,6 +1775,11 @@ export async function containerRoutes(app: FastifyInstance) { .send({ success: false, error: "Container not found" }); } + await reconcileInterruptedRebuild({ + container, + inspectRuntime: ssh.dockerInspect, + }); + const data = await listDeployments({ containerId: id, organizationId: req.organizationId!, @@ -3805,6 +3811,11 @@ export async function containerRoutes(app: FastifyInstance) { let gitAccessToken: string | undefined; try { + await reconcileInterruptedRebuild({ + container, + inspectRuntime: ssh.dockerInspect, + }); + if (container.sourceType === "APP_INSTALLER") { const install = await prisma.appInstall.findFirst({ where: { diff --git a/src/server/services/deployment-lock.service.ts b/src/server/services/deployment-lock.service.ts index 3bf0a02..253e7e0 100644 --- a/src/server/services/deployment-lock.service.ts +++ b/src/server/services/deployment-lock.service.ts @@ -4,6 +4,7 @@ import prisma from "../lib/prisma"; export const DEFAULT_DEPLOYMENT_LOCK_TTL_MS = 15 * 60 * 1000; export const DEFAULT_DEPLOYMENT_LOCK_HEARTBEAT_MS = 60 * 1000; +export const DEFAULT_DEPLOYMENT_LOCK_STALE_MS = 2 * 60 * 1000; export class DeploymentLockConflictError extends Error { constructor() { @@ -15,17 +16,29 @@ export class DeploymentLockConflictError extends Error { export async function acquireDeploymentLock(input: { containerId: string; ttlMs?: number; + staleMs?: number; }) { + const now = new Date(); const token = randomUUID(); const expiresAt = new Date( Date.now() + Math.max(30_000, input.ttlMs ?? DEFAULT_DEPLOYMENT_LOCK_TTL_MS), ); + const staleBefore = new Date( + now.getTime() - + Math.max(30_000, input.staleMs ?? DEFAULT_DEPLOYMENT_LOCK_STALE_MS), + ); try { return await prisma.$transaction(async (tx) => { const reclaimed = await tx.deploymentLock.updateMany({ - where: { containerId: input.containerId, expiresAt: { lte: new Date() } }, + where: { + containerId: input.containerId, + OR: [ + { expiresAt: { lte: now } }, + { updatedAt: { lte: staleBefore } }, + ], + }, data: { token, expiresAt }, }); @@ -44,6 +57,24 @@ export async function acquireDeploymentLock(input: { } } +export function isDeploymentLockActive( + lock: { expiresAt: Date; updatedAt: Date } | null, + options: { now?: Date; staleMs?: number } = {}, +) { + if (!lock) return false; + + const now = options.now ?? new Date(); + const staleMs = Math.max( + 30_000, + options.staleMs ?? DEFAULT_DEPLOYMENT_LOCK_STALE_MS, + ); + + return ( + lock.expiresAt.getTime() > now.getTime() && + lock.updatedAt.getTime() > now.getTime() - staleMs + ); +} + export async function releaseDeploymentLock(input: { containerId: string; token: string; @@ -126,5 +157,3 @@ export function startDeploymentLockHeartbeat( }, }; } - - diff --git a/src/server/services/deployment-recovery.service.ts b/src/server/services/deployment-recovery.service.ts new file mode 100644 index 0000000..3405e1d --- /dev/null +++ b/src/server/services/deployment-recovery.service.ts @@ -0,0 +1,138 @@ +import type { Server } from "@prisma/client"; +import prisma from "../lib/prisma"; +import { isDeploymentLockActive } from "./deployment-lock.service"; + +const RUNTIME_CREATION_CLOCK_SKEW_MS = 5_000; +const INTERRUPTED_REBUILD_ERROR = + "Rebuild process was interrupted before completion could be confirmed"; + +export type DeploymentRecoveryInspect = { + Id?: unknown; + Image?: unknown; + Created?: unknown; + Config?: { Image?: unknown } | null; + State?: { Running?: unknown; Status?: unknown } | null; +}; + +export function classifyInterruptedRebuild(input: { + startedAt: Date; + inspect: DeploymentRecoveryInspect | null; +}) { + const createdAt = + typeof input.inspect?.Created === "string" + ? new Date(input.inspect.Created) + : null; + const createdDuringRebuild = + createdAt !== null && + Number.isFinite(createdAt.getTime()) && + createdAt.getTime() >= + input.startedAt.getTime() - RUNTIME_CREATION_CLOCK_SKEW_MS; + const stateStatus = String(input.inspect?.State?.Status ?? "") + .trim() + .toLowerCase(); + const running = + input.inspect?.State?.Running === true || stateStatus === "running"; + + return running && createdDuringRebuild ? "SUCCESS" : "FAILED"; +} + +export async function reconcileInterruptedRebuild(input: { + container: { + id: string; + dockerId: string | null; + name: string; + image: string; + server: Server; + }; + inspectRuntime: ( + server: Server, + runtimeRef: string, + ) => Promise; + now?: Date; +}) { + const now = input.now ?? new Date(); + const deployment = await prisma.deployment.findFirst({ + where: { + containerId: input.container.id, + trigger: "REBUILD", + status: "RUNNING", + }, + orderBy: { createdAt: "desc" }, + select: { id: true, startedAt: true, createdAt: true }, + }); + + if (!deployment) return null; + + const lock = await prisma.deploymentLock.findUnique({ + where: { containerId: input.container.id }, + select: { token: true, expiresAt: true, updatedAt: true }, + }); + if (isDeploymentLockActive(lock, { now })) return null; + + let inspect: DeploymentRecoveryInspect | null = null; + const runtimeRefs = Array.from( + new Set([input.container.dockerId, input.container.name].filter(Boolean)), + ) as string[]; + + for (const runtimeRef of runtimeRefs) { + try { + inspect = await input.inspectRuntime(input.container.server, runtimeRef); + break; + } catch { + // A rebuild commonly replaces the old Docker ID. Fall back to the + // stable container name before deciding the interrupted outcome. + } + } + + const startedAt = deployment.startedAt ?? deployment.createdAt; + const status = classifyInterruptedRebuild({ startedAt, inspect }); + const dockerId = + typeof inspect?.Id === "string" && inspect.Id.trim() + ? inspect.Id.trim().slice(0, 12) + : null; + const runtimeImage = + typeof inspect?.Config?.Image === "string" && inspect.Config.Image.trim() + ? inspect.Config.Image.trim() + : input.container.image; + const imageDigest = + typeof inspect?.Image === "string" && inspect.Image.trim() + ? inspect.Image.trim() + : null; + + const recovered = await prisma.$transaction(async (tx) => { + const updated = await tx.deployment.updateMany({ + where: { id: deployment.id, status: "RUNNING" }, + data: { + status, + completedAt: now, + error: status === "SUCCESS" ? null : INTERRUPTED_REBUILD_ERROR, + ...(status === "SUCCESS" + ? { image: runtimeImage, imageDigest } + : {}), + }, + }); + + if (updated.count !== 1) return false; + + if (lock) { + await tx.deploymentLock.deleteMany({ + where: { containerId: input.container.id, token: lock.token }, + }); + } + + if (status === "SUCCESS") { + await tx.container.update({ + where: { id: input.container.id }, + data: { + status: "RUNNING", + image: runtimeImage, + ...(dockerId ? { dockerId } : {}), + }, + }); + } + + return true; + }); + + return recovered ? { deploymentId: deployment.id, status } : null; +} diff --git a/tests/backend/deployment-recovery.test.ts b/tests/backend/deployment-recovery.test.ts new file mode 100644 index 0000000..cdbe44e --- /dev/null +++ b/tests/backend/deployment-recovery.test.ts @@ -0,0 +1,47 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { classifyInterruptedRebuild } from "../../src/server/services/deployment-recovery.service"; +import { isDeploymentLockActive } from "../../src/server/services/deployment-lock.service"; + +const startedAt = new Date("2026-08-22T10:00:00.000Z"); + +test("interrupted rebuild is recovered as successful for a new running container", () => { + assert.equal( + classifyInterruptedRebuild({ + startedAt, + inspect: { + Created: "2026-08-22T10:00:05.000Z", + State: { Running: true, Status: "running" }, + }, + }), + "SUCCESS", + ); +}); + +test("interrupted rebuild is not reported as successful for the old runtime", () => { + assert.equal( + classifyInterruptedRebuild({ + startedAt, + inspect: { + Created: "2026-08-20T10:00:00.000Z", + State: { Running: true, Status: "running" }, + }, + }), + "FAILED", + ); +}); + +test("deployment lock becomes inactive when its heartbeat is stale", () => { + const now = new Date("2026-08-22T10:05:00.000Z"); + + assert.equal( + isDeploymentLockActive( + { + expiresAt: new Date("2026-08-22T10:15:00.000Z"), + updatedAt: new Date("2026-08-22T10:00:00.000Z"), + }, + { now, staleMs: 120_000 }, + ), + false, + ); +});