feat(deployment): implement recovery for interrupted rebuilds

src/server/routes/containers.ts

@@ -42,6 +42,7 @@ import {
   startDeploymentLockHeartbeat,
 } from "../services/deployment-lock.service";
 import { sanitizeDeploymentError } from "../services/deployment-error.service";
+import { reconcileInterruptedRebuild } from "../services/deployment-recovery.service";
 import {
   formatDockerInspectMountBindings,
   type DockerInspectMount as DockerInspectRuntimeMount,
@@ -1766,7 +1767,7 @@ export async function containerRoutes(app: FastifyInstance) {
       );
       const container = await prisma.container.findFirst({
         where: { id, server: { organizationId: req.organizationId! } },
-        select: { id: true },
+        include: { server: true },
       });
       if (!container) {
         return reply
@@ -1774,6 +1775,11 @@ export async function containerRoutes(app: FastifyInstance) {
           .send({ success: false, error: "Container not found" });
       }
 
+      await reconcileInterruptedRebuild({
+        container,
+        inspectRuntime: ssh.dockerInspect,
+      });
+
       const data = await listDeployments({
         containerId: id,
         organizationId: req.organizationId!,
@@ -3805,6 +3811,11 @@ export async function containerRoutes(app: FastifyInstance) {
       let gitAccessToken: string | undefined;
 
       try {
+        await reconcileInterruptedRebuild({
+          container,
+          inspectRuntime: ssh.dockerInspect,
+        });
+
         if (container.sourceType === "APP_INSTALLER") {
           const install = await prisma.appInstall.findFirst({
             where: {

src/server/services/deployment-lock.service.ts

@@ -4,6 +4,7 @@ import prisma from "../lib/prisma";
 
 export const DEFAULT_DEPLOYMENT_LOCK_TTL_MS = 15 * 60 * 1000;
 export const DEFAULT_DEPLOYMENT_LOCK_HEARTBEAT_MS = 60 * 1000;
+export const DEFAULT_DEPLOYMENT_LOCK_STALE_MS = 2 * 60 * 1000;
 
 export class DeploymentLockConflictError extends Error {
   constructor() {
@@ -15,17 +16,29 @@ export class DeploymentLockConflictError extends Error {
 export async function acquireDeploymentLock(input: {
   containerId: string;
   ttlMs?: number;
+  staleMs?: number;
 }) {
+  const now = new Date();
   const token = randomUUID();
   const expiresAt = new Date(
     Date.now() +
       Math.max(30_000, input.ttlMs ?? DEFAULT_DEPLOYMENT_LOCK_TTL_MS),
   );
+  const staleBefore = new Date(
+    now.getTime() -
+      Math.max(30_000, input.staleMs ?? DEFAULT_DEPLOYMENT_LOCK_STALE_MS),
+  );
 
   try {
     return await prisma.$transaction(async (tx) => {
       const reclaimed = await tx.deploymentLock.updateMany({
-        where: { containerId: input.containerId, expiresAt: { lte: new Date() } },
+        where: {
+          containerId: input.containerId,
+          OR: [
+            { expiresAt: { lte: now } },
+            { updatedAt: { lte: staleBefore } },
+          ],
+        },
         data: { token, expiresAt },
       });
 
@@ -44,6 +57,24 @@ export async function acquireDeploymentLock(input: {
   }
 }
 
+export function isDeploymentLockActive(
+  lock: { expiresAt: Date; updatedAt: Date } | null,
+  options: { now?: Date; staleMs?: number } = {},
+) {
+  if (!lock) return false;
+
+  const now = options.now ?? new Date();
+  const staleMs = Math.max(
+    30_000,
+    options.staleMs ?? DEFAULT_DEPLOYMENT_LOCK_STALE_MS,
+  );
+
+  return (
+    lock.expiresAt.getTime() > now.getTime() &&
+    lock.updatedAt.getTime() > now.getTime() - staleMs
+  );
+}
+
 export async function releaseDeploymentLock(input: {
   containerId: string;
   token: string;
@@ -126,5 +157,3 @@ export function startDeploymentLockHeartbeat(
     },
   };
 }
-
-

src/server/services/deployment-recovery.service.ts

@@ -0,0 +1,138 @@
+import type { Server } from "@prisma/client";
+import prisma from "../lib/prisma";
+import { isDeploymentLockActive } from "./deployment-lock.service";
+
+const RUNTIME_CREATION_CLOCK_SKEW_MS = 5_000;
+const INTERRUPTED_REBUILD_ERROR =
+  "Rebuild process was interrupted before completion could be confirmed";
+
+export type DeploymentRecoveryInspect = {
+  Id?: unknown;
+  Image?: unknown;
+  Created?: unknown;
+  Config?: { Image?: unknown } | null;
+  State?: { Running?: unknown; Status?: unknown } | null;
+};
+
+export function classifyInterruptedRebuild(input: {
+  startedAt: Date;
+  inspect: DeploymentRecoveryInspect | null;
+}) {
+  const createdAt =
+    typeof input.inspect?.Created === "string"
+      ? new Date(input.inspect.Created)
+      : null;
+  const createdDuringRebuild =
+    createdAt !== null &&
+    Number.isFinite(createdAt.getTime()) &&
+    createdAt.getTime() >=
+      input.startedAt.getTime() - RUNTIME_CREATION_CLOCK_SKEW_MS;
+  const stateStatus = String(input.inspect?.State?.Status ?? "")
+    .trim()
+    .toLowerCase();
+  const running =
+    input.inspect?.State?.Running === true || stateStatus === "running";
+
+  return running && createdDuringRebuild ? "SUCCESS" : "FAILED";
+}
+
+export async function reconcileInterruptedRebuild(input: {
+  container: {
+    id: string;
+    dockerId: string | null;
+    name: string;
+    image: string;
+    server: Server;
+  };
+  inspectRuntime: (
+    server: Server,
+    runtimeRef: string,
+  ) => Promise<DeploymentRecoveryInspect>;
+  now?: Date;
+}) {
+  const now = input.now ?? new Date();
+  const deployment = await prisma.deployment.findFirst({
+    where: {
+      containerId: input.container.id,
+      trigger: "REBUILD",
+      status: "RUNNING",
+    },
+    orderBy: { createdAt: "desc" },
+    select: { id: true, startedAt: true, createdAt: true },
+  });
+
+  if (!deployment) return null;
+
+  const lock = await prisma.deploymentLock.findUnique({
+    where: { containerId: input.container.id },
+    select: { token: true, expiresAt: true, updatedAt: true },
+  });
+  if (isDeploymentLockActive(lock, { now })) return null;
+
+  let inspect: DeploymentRecoveryInspect | null = null;
+  const runtimeRefs = Array.from(
+    new Set([input.container.dockerId, input.container.name].filter(Boolean)),
+  ) as string[];
+
+  for (const runtimeRef of runtimeRefs) {
+    try {
+      inspect = await input.inspectRuntime(input.container.server, runtimeRef);
+      break;
+    } catch {
+      // A rebuild commonly replaces the old Docker ID. Fall back to the
+      // stable container name before deciding the interrupted outcome.
+    }
+  }
+
+  const startedAt = deployment.startedAt ?? deployment.createdAt;
+  const status = classifyInterruptedRebuild({ startedAt, inspect });
+  const dockerId =
+    typeof inspect?.Id === "string" && inspect.Id.trim()
+      ? inspect.Id.trim().slice(0, 12)
+      : null;
+  const runtimeImage =
+    typeof inspect?.Config?.Image === "string" && inspect.Config.Image.trim()
+      ? inspect.Config.Image.trim()
+      : input.container.image;
+  const imageDigest =
+    typeof inspect?.Image === "string" && inspect.Image.trim()
+      ? inspect.Image.trim()
+      : null;
+
+  const recovered = await prisma.$transaction(async (tx) => {
+    const updated = await tx.deployment.updateMany({
+      where: { id: deployment.id, status: "RUNNING" },
+      data: {
+        status,
+        completedAt: now,
+        error: status === "SUCCESS" ? null : INTERRUPTED_REBUILD_ERROR,
+        ...(status === "SUCCESS"
+          ? { image: runtimeImage, imageDigest }
+          : {}),
+      },
+    });
+
+    if (updated.count !== 1) return false;
+
+    if (lock) {
+      await tx.deploymentLock.deleteMany({
+        where: { containerId: input.container.id, token: lock.token },
+      });
+    }
+
+    if (status === "SUCCESS") {
+      await tx.container.update({
+        where: { id: input.container.id },
+        data: {
+          status: "RUNNING",
+          image: runtimeImage,
+          ...(dockerId ? { dockerId } : {}),
+        },
+      });
+    }
+
+    return true;
+  });
+
+  return recovered ? { deploymentId: deployment.id, status } : null;
+}

tests/backend/deployment-recovery.test.ts

@@ -0,0 +1,47 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { classifyInterruptedRebuild } from "../../src/server/services/deployment-recovery.service";
+import { isDeploymentLockActive } from "../../src/server/services/deployment-lock.service";
+
+const startedAt = new Date("2026-08-22T10:00:00.000Z");
+
+test("interrupted rebuild is recovered as successful for a new running container", () => {
+  assert.equal(
+    classifyInterruptedRebuild({
+      startedAt,
+      inspect: {
+        Created: "2026-08-22T10:00:05.000Z",
+        State: { Running: true, Status: "running" },
+      },
+    }),
+    "SUCCESS",
+  );
+});
+
+test("interrupted rebuild is not reported as successful for the old runtime", () => {
+  assert.equal(
+    classifyInterruptedRebuild({
+      startedAt,
+      inspect: {
+        Created: "2026-08-20T10:00:00.000Z",
+        State: { Running: true, Status: "running" },
+      },
+    }),
+    "FAILED",
+  );
+});
+
+test("deployment lock becomes inactive when its heartbeat is stale", () => {
+  const now = new Date("2026-08-22T10:05:00.000Z");
+
+  assert.equal(
+    isDeploymentLockActive(
+      {
+        expiresAt: new Date("2026-08-22T10:15:00.000Z"),
+        updatedAt: new Date("2026-08-22T10:00:00.000Z"),
+      },
+      { now, staleMs: 120_000 },
+    ),
+    false,
+  );
+});