persist restic cache and backup restore fixes
This commit is contained in:
3 files changed
+29
-4
No files matched your search
@@ -5,6 +5,7 @@
|
||||
* no globals, no DB. Safe to call in any context and to unit-test in isolation.
|
||||
*/
|
||||
|
||||
import { join } from 'node:path';
|
||||
import { isValidCron } from '../scheduler/cron-utils';
|
||||
import { privateIpReason, dangerousHostReason, isSafeWebhookUrl, isSafeNotificationUrl } from '../url-safety';
|
||||
import { BackupError, isLocalRepo } from './models';
|
||||
@@ -297,7 +298,12 @@ export function isRepoNotInitializedError(error: unknown): boolean {
|
||||
export function classifyBackupError(error: unknown): string {
|
||||
if (isRepoNotInitializedError(error)) return 'REPO_NOT_INIT';
|
||||
const raw = (error instanceof Error ? error.message : String(error ?? '')).toLowerCase();
|
||||
if (raw.includes('sigterm') || raw.includes('sigkill') || raw.includes('timed out') || raw.includes('timeout')) return 'RESTIC_TIMEOUT';
|
||||
// A killed/cancelled restic (slow remote backend, watchdog timeout) reports its
|
||||
// OWN exit as `signal terminated` / `context canceled`, and restic then prints a
|
||||
// MISLEADING `code:12 wrong password or no key found` because it was interrupted
|
||||
// mid key-load. Catch the termination markers FIRST so a timeout is never
|
||||
// mis-surfaced to the user as a bad password.
|
||||
if (raw.includes('sigterm') || raw.includes('sigkill') || raw.includes('signal terminated') || raw.includes('signal killed') || raw.includes('context canceled') || raw.includes('context cancelled') || raw.includes('timed out') || raw.includes('timeout')) return 'RESTIC_TIMEOUT';
|
||||
if (raw.includes('wrong password') || raw.includes('invalid password') || raw.includes('no key found')) return 'WRONG_PASSWORD';
|
||||
if (raw.includes('already locked') || raw.includes('unable to create lock') || raw.includes('repository is already locked')) return 'REPO_LOCKED';
|
||||
if (raw.includes('no volumes') || raw.includes('nothing to backup')) return 'NO_VOLUMES';
|
||||
@@ -399,6 +405,16 @@ export function buildResticEnv(
|
||||
for (const [k, v] of Object.entries(procEnv)) {
|
||||
if (v !== undefined && RESTIC_BASE_ENV_ALLOW.has(k)) env[k] = v;
|
||||
}
|
||||
// Persist restic's cache under DATA_DIR (a durable volume) instead of the default
|
||||
// $HOME/.cache/restic on the container's ephemeral RW layer, which is wiped on every
|
||||
// redeploy. A cold cache re-fetches the repo index from the remote backend (backblaze/
|
||||
// S3) on the first ls/snapshots/dump after a redeploy - ~3x slower and enough to trip a
|
||||
// reverse proxy's idle timeout. Only for the host-side restic (this fn feeds runLocal +
|
||||
// destination test); the helper container builds its env separately (buildHelperEnv). An
|
||||
// operator-set RESTIC_CACHE_DIR (copied in above) wins.
|
||||
if (!env.RESTIC_CACHE_DIR) {
|
||||
env.RESTIC_CACHE_DIR = join(procEnv.DATA_DIR || '/app/data', '.restic-cache');
|
||||
}
|
||||
env.RESTIC_REPOSITORY = opts.repository;
|
||||
env.RESTIC_PASSWORD = opts.password;
|
||||
for (const [k, v] of Object.entries(opts.envVars || {})) {
|
||||
|
||||
@@ -1046,7 +1046,12 @@ function reader() {
|
||||
async function listSnapshotVolumes(destination: any, snapshotId: string): Promise<string[]> {
|
||||
const { isReservedVolumeKey } = await import('./stackdir-plan');
|
||||
const run = await restic.runLocal(destination, ['ls', '--json', '--no-lock', snapshotId, '/volumes/']);
|
||||
if (run.exitCode !== 0) return [];
|
||||
// A FAILED `ls` (timeout / killed / slow remote backend) is NOT "the snapshot has no
|
||||
// volumes" - returning [] there makes restore report a MISLEADING "not found in snapshot,
|
||||
// Available: none" for a snapshot that actually holds data. Fail loud with the real reason.
|
||||
if (run.exitCode !== 0) {
|
||||
throw new BackupError('RESTIC', `could not list the snapshot's volumes: ${run.stderr.trim() || 'restic ls failed'}`, { exitCode: run.exitCode });
|
||||
}
|
||||
const names = new Set<string>();
|
||||
for (const line of run.stdout.split('\n')) {
|
||||
try {
|
||||
|
||||
@@ -1047,8 +1047,12 @@
|
||||
disabled={!canRun || restoring}
|
||||
variant={mode === 'in-place' ? 'destructive' : 'default'}
|
||||
>
|
||||
{#if restoring}<Loader2 class="mr-1.5 h-4 w-4 animate-spin" />{:else}<Play class="mr-1.5 h-4 w-4" />{/if}
|
||||
{mode === 'in-place' ? 'Overwrite & restore' : (postRestore !== 'none' ? 'Restore & start' : 'Restore')}
|
||||
{#if restoring}<Loader2 class="mr-1.5 h-4 w-4 animate-spin" />{:else if targetPreviewLoading}<Loader2 class="mr-1.5 h-4 w-4 animate-spin" />{:else}<Play class="mr-1.5 h-4 w-4" />{/if}
|
||||
{#if targetPreviewLoading && !restoring}
|
||||
Checking target…
|
||||
{:else}
|
||||
{mode === 'in-place' ? 'Overwrite & restore' : (postRestore !== 'none' ? 'Restore & start' : 'Restore')}
|
||||
{/if}
|
||||
</Button>
|
||||
{/if}
|
||||
{/if}
|
||||
|
||||
Reference in new issue
Block a user