From fea99407745680e31fbf6987dafc079e32a16a58 Mon Sep 17 00:00:00 2001 From: KodekaTeam Date: Thu, 3 Sep 2026 23:49:14 +0700 Subject: [PATCH] feat: Implement `.env` persistence during Git rebuilds - Added functionality to persist `.env` files to a private sidecar directory before Git checkout replacement. - Enhanced recovery of `.env` files from the sidecar after cloning, with fallback to templates if necessary. - Introduced validation for `envFilePath` to ensure it is an absolute path and integrated it into Docker run commands. - Updated deployment routes and rollback services to support the new environment file handling. - Added regression tests to verify the persistence and recovery of environment variables across rebuilds. --- src/server/routes/containers.ts | 41 ++++++++- .../services/deployment-rollback.service.ts | 8 ++ .../ssh-services/docker-containers.ts | 92 +++++++++++++++++++ .../deployment-lifecycle-regressions.test.ts | 82 +++++++++++++++++ tests/backend/deployment-strategy.test.ts | 25 +++++ 5 files changed, 246 insertions(+), 2 deletions(-) diff --git a/src/server/routes/containers.ts b/src/server/routes/containers.ts index 96bd5bf..a62467c 100644 --- a/src/server/routes/containers.ts +++ b/src/server/routes/containers.ts @@ -3116,6 +3116,38 @@ export async function containerRoutes(app: FastifyInstance) { body.data.content, ); + const deploymentPath = + container.deploymentSource?.deploymentPath?.trim(); + if ( + deploymentPath && + isGitRedeploySourceType(container.sourceType) + ) { + const persistentEnvPath = normalizeDeploymentChildPath( + deploymentPath, + container.deploymentSource?.buildPath, + ); + + if (!persistentEnvPath) { + throw new Error( + "The active container .env was updated, but its Git deployment path could not be validated for rebuild persistence", + ); + } + + try { + await ssh.writeDeploymentEnvFile( + container.server, + persistentEnvPath, + body.data.content, + ); + } catch (error) { + const cause = + error instanceof Error ? error.message : "unknown error"; + throw new Error( + `The active container .env was updated, but Doktainer could not persist it for the next rebuild: ${cause}`, + ); + } + } + await auditLog({ userId: req.userId, serverId: container.serverId, @@ -4983,8 +5015,13 @@ export async function containerRoutes(app: FastifyInstance) { ); } const prepared = gitDeploymentResult.preparedRuntime; - const targetRuntime: RuntimeReplacementSpec = { + const recoverablePreviousRuntime: RuntimeReplacementSpec = { ...previousRuntime, + envFilePath: + prepared.envFilePath ?? previousRuntime.envFilePath, + }; + const targetRuntime: RuntimeReplacementSpec = { + ...recoverablePreviousRuntime, ...prepared, networks: Array.from( new Set([ @@ -5016,7 +5053,7 @@ export async function containerRoutes(app: FastifyInstance) { deploymentId: runningDeployment.id, operationLabel: "rebuild", targetRuntime, - previousRuntime, + previousRuntime: recoverablePreviousRuntime, plan, finalize: finalizeGitRuntime, }); diff --git a/src/server/services/deployment-rollback.service.ts b/src/server/services/deployment-rollback.service.ts index baac8fb..7bf5b9b 100644 --- a/src/server/services/deployment-rollback.service.ts +++ b/src/server/services/deployment-rollback.service.ts @@ -37,6 +37,7 @@ export type RuntimeReplacementSpec = { image: string; ports: string; env: string; + envFilePath?: string; volumes: string; network: string; networks: string[]; @@ -209,6 +210,7 @@ function snapshotRuntime( image, ports: normalizeRollbackPortMappings(snapshot.ports), env: snapshotString(snapshot, "env"), + envFilePath: snapshotString(snapshot, "envFilePath") || undefined, volumes: snapshotString(snapshot, "volumes"), network, networks: normalizeRuntimeNetworks(network, snapshot.networks), @@ -422,6 +424,7 @@ function validateRuntimeBeforeMutation(input: { image: input.runtime.image, ports: input.ports, env: input.runtime.env, + envFilePath: input.runtime.envFilePath, volumes: input.runtime.volumes, network: input.runtime.network, cpuLimit: input.runtime.cpuLimit, @@ -554,6 +557,7 @@ export async function replaceRuntimeForRollback( image: input.targetRuntime.image, ports: targetPorts, env: input.targetRuntime.env, + envFilePath: input.targetRuntime.envFilePath, volumes: input.targetRuntime.volumes, network: input.targetRuntime.network, cpuLimit: input.targetRuntime.cpuLimit, @@ -664,6 +668,7 @@ export async function replaceRuntimeForRollback( image: input.previousRuntime.image, ports: input.previousRuntime.ports, env: input.previousRuntime.env, + envFilePath: input.previousRuntime.envFilePath, volumes: input.previousRuntime.volumes, network: input.previousRuntime.network, cpuLimit: input.previousRuntime.cpuLimit, @@ -839,6 +844,7 @@ export async function replaceRuntimeWithProxyCandidate(input: { image: input.targetRuntime.image, ports: candidatePorts, env: input.targetRuntime.env, + envFilePath: input.targetRuntime.envFilePath, volumes: input.targetRuntime.volumes, network: input.targetRuntime.network, cpuLimit: input.targetRuntime.cpuLimit, @@ -917,6 +923,7 @@ export async function replaceRuntimeWithProxyCandidate(input: { image: input.targetRuntime.image, ports: input.targetRuntime.ports, env: input.targetRuntime.env, + envFilePath: input.targetRuntime.envFilePath, volumes: input.targetRuntime.volumes, network: input.targetRuntime.network, cpuLimit: input.targetRuntime.cpuLimit, @@ -1071,6 +1078,7 @@ export async function replaceRuntimeWithProxyCandidate(input: { image: input.previousRuntime.image, ports: input.previousRuntime.ports, env: input.previousRuntime.env, + envFilePath: input.previousRuntime.envFilePath, volumes: input.previousRuntime.volumes, network: input.previousRuntime.network, cpuLimit: input.previousRuntime.cpuLimit, diff --git a/src/server/services/ssh-services/docker-containers.ts b/src/server/services/ssh-services/docker-containers.ts index be367c4..48311d5 100644 --- a/src/server/services/ssh-services/docker-containers.ts +++ b/src/server/services/ssh-services/docker-containers.ts @@ -199,6 +199,16 @@ function validateNetworkName(value?: string): string | null { return normalized; } +function validateRuntimeEnvFilePath(value?: string): string | null { + const candidate = value?.trim(); + if (!candidate) return null; + if (/\r|\n|\0/.test(candidate) || !pathPosix.isAbsolute(candidate)) { + throw new Error("Runtime env file path must be an absolute path"); + } + + return pathPosix.normalize(candidate); +} + function parsePortNumber(value: string, label: string): number { if (!/^\d+$/.test(value)) { throw new Error(`${label} must be numeric`); @@ -443,6 +453,7 @@ export function buildDockerRunCommand(opts: { image: string; ports?: string; env?: string; + envFilePath?: string; restartPolicy: string; volumes?: string; network?: string; @@ -503,6 +514,11 @@ export function buildDockerRunCommand(opts: { args.push("-e", envAssignment); } + const envFilePath = validateRuntimeEnvFilePath(opts.envFilePath); + if (envFilePath) { + args.push("--env-file", envFilePath); + } + for (const volumeMount of parseVolumeMounts( opts.volumes, opts.mountValidation, @@ -965,6 +981,7 @@ export type PreparedGitRuntime = { image: string; ports: string; env: string; + envFilePath?: string; volumes: string; network: string; restartPolicy: string; @@ -1699,6 +1716,7 @@ export async function runContainer( image: string; ports?: string; // "80:80,443:443" env?: string; // "KEY=val\nKEY2=val2" + envFilePath?: string; restartPolicy: string; volumes?: string; network?: string; @@ -2452,6 +2470,7 @@ async function bootstrapSourceProject( deploymentPath: string; buildPath?: string; sourceProject: Awaited>; + managedEnvFilePath?: string; }, ) { const buildPath = normalizeBuildSubdirectory(opts.buildPath); @@ -2472,6 +2491,7 @@ async function bootstrapSourceProject( `DEPLOY_PATH=${escapeShellArg(opts.deploymentPath)}`, `BUILD_PATH=${escapeShellArg(buildPath)}`, `ENV_TEMPLATE=${escapeShellArg(templateToCopy)}`, + `MANAGED_ENV_FILE=${escapeShellArg(opts.managedEnvFilePath ?? "")}`, 'if [ "$BUILD_PATH" = "." ]; then TARGET_PATH="$DEPLOY_PATH"; else TARGET_PATH="$DEPLOY_PATH/$BUILD_PATH"; fi', 'if [ ! -d "$TARGET_PATH" ]; then echo "Build path not found: $BUILD_PATH"; exit 1; fi', 'if [ -n "$ENV_TEMPLATE" ] && [ ! -f "$TARGET_PATH/.env" ] && [ -f "$TARGET_PATH/$ENV_TEMPLATE" ]; then cp "$TARGET_PATH/$ENV_TEMPLATE" "$TARGET_PATH/.env"; fi', @@ -2505,6 +2525,15 @@ async function bootstrapSourceProject( ...(sourceProject.hasBootstrapCacheDir ? ['chmod -R ugo+rwX "$TARGET_PATH/bootstrap/cache" || true'] : []), + 'if [ -n "$MANAGED_ENV_FILE" ] && [ -f "$TARGET_PATH/.env" ]; then', + ' MANAGED_ENV_DIR=$(dirname "$MANAGED_ENV_FILE")', + ' mkdir -p "$MANAGED_ENV_DIR"', + ' chmod 700 "$MANAGED_ENV_DIR" || true', + ' TEMP_ENV="${MANAGED_ENV_FILE}.tmp.$$"', + ' cp "$TARGET_PATH/.env" "$TEMP_ENV"', + ' chmod 600 "$TEMP_ENV" || true', + ' mv -f "$TEMP_ENV" "$MANAGED_ENV_FILE"', + "fi", ].join("\n"); await execStrict( @@ -2747,6 +2776,11 @@ export async function deployContainerFromGitSource( `doktainer/${sanitizeProjectName(projectName)}:latest`; const runOverride = resolveRunOverride(opts.startCommand); const shouldApplyRuntimeOverride = buildType === "DOCKERFILE"; + const managedEnvDirectory = `${deploymentPath}.doktainer`; + const managedRuntimeEnvFilePath = pathPosix.join( + managedEnvDirectory, + buildPath === "." ? "root.env" : "build.env", + ); const cloneCommand = [ "git", @@ -2764,9 +2798,47 @@ export async function deployContainerFromGitSource( "set -euo pipefail", "export GIT_TERMINAL_PROMPT=0", `DEPLOY_PATH=${escapeShellArg(deploymentPath)}`, + `BUILD_PATH=${escapeShellArg(buildPath)}`, + `MANAGED_ENV_DIR=${escapeShellArg(managedEnvDirectory)}`, + 'ROOT_ENV="$DEPLOY_PATH/.env"', + 'if [ "$BUILD_PATH" = "." ]; then TARGET_ENV="$ROOT_ENV"; else TARGET_ENV="$DEPLOY_PATH/$BUILD_PATH/.env"; fi', + 'ROOT_MANAGED_ENV="$MANAGED_ENV_DIR/root.env"', + 'if [ "$BUILD_PATH" = "." ]; then TARGET_MANAGED_ENV="$ROOT_MANAGED_ENV"; else TARGET_MANAGED_ENV="$MANAGED_ENV_DIR/build.env"; fi', + "persist_env() {", + ' SOURCE_ENV="$1"; MANAGED_ENV="$2"', + ' if [ ! -f "$SOURCE_ENV" ]; then return; fi', + ' mkdir -p "$MANAGED_ENV_DIR"', + ' chmod 700 "$MANAGED_ENV_DIR" || true', + ' TEMP_ENV="${MANAGED_ENV}.tmp.$$"', + ' cp "$SOURCE_ENV" "$TEMP_ENV"', + ' chmod 600 "$TEMP_ENV" || true', + ' mv -f "$TEMP_ENV" "$MANAGED_ENV"', + "}", + "restore_env() {", + ' MANAGED_ENV="$1"; DESTINATION_ENV="$2"', + ' if [ ! -f "$MANAGED_ENV" ]; then return; fi', + ' mkdir -p "$(dirname "$DESTINATION_ENV")"', + ' cp "$MANAGED_ENV" "$DESTINATION_ENV"', + ' chmod 600 "$DESTINATION_ENV" || true', + "}", 'mkdir -p "$(dirname "$DEPLOY_PATH")"', + 'persist_env "$ROOT_ENV" "$ROOT_MANAGED_ENV"', + 'if [ "$TARGET_ENV" != "$ROOT_ENV" ]; then persist_env "$TARGET_ENV" "$TARGET_MANAGED_ENV"; fi', 'rm -rf "$DEPLOY_PATH"', cloneCommand, + 'restore_env "$ROOT_MANAGED_ENV" "$ROOT_ENV"', + 'if [ "$TARGET_ENV" != "$ROOT_ENV" ]; then restore_env "$TARGET_MANAGED_ENV" "$TARGET_ENV"; fi', + 'if [ ! -f "$TARGET_ENV" ]; then', + ' TARGET_DIR=$(dirname "$TARGET_ENV")', + ' if [ -f "$TARGET_DIR/.env.example" ]; then ENV_TEMPLATE="$TARGET_DIR/.env.example"', + ' elif [ -f "$TARGET_DIR/.env.dist" ]; then ENV_TEMPLATE="$TARGET_DIR/.env.dist"', + ' elif [ -f "$TARGET_DIR/env" ]; then ENV_TEMPLATE="$TARGET_DIR/env"', + ' else ENV_TEMPLATE=""', + " fi", + ' if [ -n "$ENV_TEMPLATE" ]; then cp "$ENV_TEMPLATE" "$TARGET_ENV"; chmod 600 "$TARGET_ENV" || true; fi', + "fi", + 'persist_env "$ROOT_ENV" "$ROOT_MANAGED_ENV"', + 'if [ "$TARGET_ENV" != "$ROOT_ENV" ]; then persist_env "$TARGET_ENV" "$TARGET_MANAGED_ENV"; fi', ].join("\n"); try { @@ -2841,6 +2913,20 @@ export async function deployContainerFromGitSource( }; } + const hasManagedRuntimeEnvFile = ( + await execStrict( + server, + privilegedCommand( + server, + `bash -lc ${escapeShellArg(`if [ -f ${escapeShellArg(managedRuntimeEnvFilePath)} ]; then printf 1; else printf 0; fi`)}`, + ), + shortDockerCommandTimeout(DOCKER_INSPECT_TIMEOUT_MS), + ) + ).trim() === "1"; + const runtimeEnvFilePath = hasManagedRuntimeEnvFile + ? managedRuntimeEnvFilePath + : undefined; + if (buildType === "NIXPACKS") { const sourceProject = await inspectSourceProject(server, { deploymentPath, @@ -2850,6 +2936,7 @@ export async function deployContainerFromGitSource( deploymentPath, buildPath, sourceProject, + managedEnvFilePath: runtimeEnvFilePath, }); if ( sourceProject.likelyLaravel && @@ -2922,6 +3009,7 @@ export async function deployContainerFromGitSource( image: imageTag, ports: autoPorts ?? "", env: runtimeEnv ?? "", + envFilePath: runtimeEnvFilePath, restartPolicy: opts.restartPolicy?.trim() || "unless-stopped", volumes: opts.volumes ?? "", network: opts.network?.trim() || "bridge", @@ -2958,6 +3046,7 @@ export async function deployContainerFromGitSource( deploymentPath, buildPath, sourceProject, + managedEnvFilePath: runtimeEnvFilePath, }); if ( @@ -3041,6 +3130,7 @@ export async function deployContainerFromGitSource( image: imageTag, ports: autoPorts ?? "", env: runtimeEnv ?? "", + envFilePath: runtimeEnvFilePath, restartPolicy: opts.restartPolicy?.trim() || "unless-stopped", volumes: opts.volumes ?? "", network: opts.network?.trim() || "bridge", @@ -3081,6 +3171,7 @@ export async function deployContainerFromGitSource( image: imageTag, ports: autoPorts ?? "", env: opts.env ?? "", + envFilePath: runtimeEnvFilePath, restartPolicy: opts.restartPolicy?.trim() || "unless-stopped", volumes: opts.volumes ?? "", network: opts.network?.trim() || "bridge", @@ -3154,6 +3245,7 @@ export async function deployContainerFromGitSource( image: imageTag, ports: autoPorts ?? "", env: opts.env ?? "", + envFilePath: runtimeEnvFilePath, restartPolicy: opts.restartPolicy?.trim() || "unless-stopped", volumes: opts.volumes ?? "", network: opts.network?.trim() || "bridge", diff --git a/tests/backend/deployment-lifecycle-regressions.test.ts b/tests/backend/deployment-lifecycle-regressions.test.ts index 29a62d7..2652d6b 100644 --- a/tests/backend/deployment-lifecycle-regressions.test.ts +++ b/tests/backend/deployment-lifecycle-regressions.test.ts @@ -103,6 +103,88 @@ test("Git deployment resolves and returns the cloned commit SHA", () => { assert.match(gitDeployFunction, /preparedRuntime/); }); +test("Git rebuild preserves the project env outside the replaceable checkout", () => { + const source = readSource(DOCKER_CONTAINERS); + const start = source.indexOf( + "export async function deployContainerFromGitSource", + ); + assert.notEqual(start, -1); + const gitDeployFunction = source.slice(start); + + const persistIndex = gitDeployFunction.indexOf( + 'persist_env "$ROOT_ENV" "$ROOT_MANAGED_ENV"', + ); + const removeIndex = gitDeployFunction.indexOf('rm -rf "$DEPLOY_PATH"'); + const cloneIndex = gitDeployFunction.indexOf("cloneCommand,", removeIndex); + const restoreIndex = gitDeployFunction.indexOf( + 'restore_env "$ROOT_MANAGED_ENV" "$ROOT_ENV"', + ); + const templateIndex = gitDeployFunction.indexOf( + 'if [ -f "$TARGET_DIR/.env.example" ]', + ); + + assert.ok(persistIndex >= 0); + assert.ok(removeIndex > persistIndex); + assert.ok(cloneIndex > removeIndex); + assert.ok(restoreIndex > cloneIndex); + assert.ok(templateIndex > restoreIndex); + assert.match(gitDeployFunction, /managedEnvDirectory = `\$\{deploymentPath\}\.doktainer`/); + assert.match(gitDeployFunction, /chmod 700 .*MANAGED_ENV_DIR/); + assert.match(gitDeployFunction, /chmod 600 .*TEMP_ENV/); + assert.match(gitDeployFunction, /BUILD_PATH/); + assert.match(gitDeployFunction, /envFilePath: runtimeEnvFilePath/); +}); + +test("Git container env edits are mirrored to the deployment checkout", () => { + const source = readSource(CONTAINER_ROUTES); + const readRouteIndex = source.indexOf('"/:id/project-env"'); + const writeRouteIndex = source.indexOf( + '"/:id/project-env"', + readRouteIndex + 1, + ); + const nextRouteIndex = source.indexOf('"/:id/exec"', writeRouteIndex); + assert.ok(readRouteIndex >= 0); + assert.ok(writeRouteIndex > readRouteIndex); + assert.ok(nextRouteIndex > writeRouteIndex); + const envRoute = source.slice(writeRouteIndex, nextRouteIndex); + const containerWriteIndex = envRoute.indexOf("ssh.writeContainerFile"); + const gitSourceCheckIndex = envRoute.indexOf( + "isGitRedeploySourceType(container.sourceType)", + ); + const deploymentWriteIndex = envRoute.indexOf( + "ssh.writeDeploymentEnvFile", + containerWriteIndex, + ); + + assert.ok(containerWriteIndex >= 0); + assert.ok(gitSourceCheckIndex > containerWriteIndex); + assert.ok(deploymentWriteIndex > gitSourceCheckIndex); + assert.match(envRoute, /normalizeDeploymentChildPath/); + assert.match( + envRoute, + /active container \.env was updated, but Doktainer could not persist it for the next rebuild/, + ); +}); + +test("Git rebuild applies the managed env file to target and recovery runtimes", () => { + const source = readSource(CONTAINER_ROUTES); + const rebuildRoute = sourceBlock( + source, + '"/:id/rebuild"', + "GET /containers/:id/logs", + ); + + assert.match(rebuildRoute, /recoverablePreviousRuntime/); + assert.match( + rebuildRoute, + /prepared\.envFilePath \?\? previousRuntime\.envFilePath/, + ); + assert.match( + rebuildRoute, + /previousRuntime: recoverablePreviousRuntime/, + ); +}); + test("Git rebuild image tags are immutable and preserve registry ports", () => { assert.equal( toImmutableBuildImageTag( diff --git a/tests/backend/deployment-strategy.test.ts b/tests/backend/deployment-strategy.test.ts index fa3af40..f7db659 100644 --- a/tests/backend/deployment-strategy.test.ts +++ b/tests/backend/deployment-strategy.test.ts @@ -20,6 +20,31 @@ describe("deployment strategy", () => { assert.match(command, /'--memory' '512m'/); }); + it("applies a managed env file without embedding its contents in the command", () => { + const command = buildDockerRunCommand({ + name: "app", + image: "example/app:stable", + restartPolicy: "unless-stopped", + envFilePath: "/opt/doktainer/deployments/app.doktainer/root.env", + }); + + assert.match( + command, + /'--env-file' '\/opt\/doktainer\/deployments\/app\.doktainer\/root\.env'/, + ); + assert.doesNotMatch(command, /SECRET_VALUE/); + assert.throws( + () => + buildDockerRunCommand({ + name: "app", + image: "example/app:stable", + restartPolicy: "unless-stopped", + envFilePath: "relative/.env", + }), + /absolute path/, + ); + }); + it("uses atomic rename when no published ports are configured", () => { assert.equal(resolveDeploymentStrategy(""), "ATOMIC_RENAME"); assert.equal(resolveDeploymentStrategy(" "), "ATOMIC_RENAME");