v1.0.21
This commit is contained in:
1 parent
790ce092ee
commit
55f3101a19
51 files changed
+763
-599
No files matched your search
@@ -0,0 +1,91 @@
|
||||
import { setGlobalDispatcher, Agent } from 'undici';
|
||||
import dns from 'node:dns';
|
||||
import net from 'node:net';
|
||||
|
||||
const origLookup = dns.lookup.bind(dns);
|
||||
|
||||
// DNS cache: hostname → { address, family, expiresAt } (positive)
|
||||
// DNS negative cache: hostname → { error, expiresAt } (failed lookups)
|
||||
const dnsCache = new Map<string, { address: string; family: number; expiresAt: number }>();
|
||||
const dnsNegCache = new Map<string, { error: Error; expiresAt: number }>();
|
||||
const DNS_TTL_MS = 30_000;
|
||||
const DNS_NEG_TTL_MS = 10_000; // Cache failures for 10s to prevent DNS server storms
|
||||
|
||||
// In-flight deduplication: hostname → pending Promise<{address, family}>
|
||||
const inFlight = new Map<string, Promise<{ address: string; family: number }>>();
|
||||
|
||||
function lookupWithCache(hostname: string): Promise<{ address: string; family: number }> {
|
||||
// Positive cache hit
|
||||
const cached = dnsCache.get(hostname);
|
||||
if (cached) {
|
||||
if (cached.expiresAt > Date.now()) {
|
||||
return Promise.resolve({ address: cached.address, family: cached.family });
|
||||
}
|
||||
dnsCache.delete(hostname); // evict stale entry
|
||||
}
|
||||
|
||||
// Negative cache hit — don't hammer DNS for recently-failed hostnames
|
||||
const negCached = dnsNegCache.get(hostname);
|
||||
if (negCached) {
|
||||
if (negCached.expiresAt > Date.now()) {
|
||||
return Promise.reject(negCached.error);
|
||||
}
|
||||
dnsNegCache.delete(hostname);
|
||||
}
|
||||
|
||||
// In-flight deduplication
|
||||
const pending = inFlight.get(hostname);
|
||||
if (pending) return pending;
|
||||
|
||||
// Use getaddrinfo (libc) as primary — works through Docker's embedded DNS (127.0.0.11)
|
||||
// and respects --dns-result-order=ipv4first from entrypoint. This matches Bun's native
|
||||
// behavior which worked reliably on NAS environments where c-ares failed (#676).
|
||||
const promise = new Promise<{ address: string; family: number }>((resolve, reject) => {
|
||||
origLookup(hostname, { all: false }, (err, address, family) => {
|
||||
if (err) {
|
||||
// Cache the failure so parallel/subsequent requests don't all hammer DNS
|
||||
dnsNegCache.set(hostname, { error: err, expiresAt: Date.now() + DNS_NEG_TTL_MS });
|
||||
reject(err);
|
||||
} else {
|
||||
const result = { address: address as string, family: family as number };
|
||||
dnsCache.set(hostname, { ...result, expiresAt: Date.now() + DNS_TTL_MS });
|
||||
resolve(result);
|
||||
}
|
||||
});
|
||||
}).finally(() => {
|
||||
inFlight.delete(hostname);
|
||||
});
|
||||
|
||||
inFlight.set(hostname, promise);
|
||||
return promise;
|
||||
}
|
||||
|
||||
setGlobalDispatcher(
|
||||
new Agent({
|
||||
connect: {
|
||||
// Undici default is 10s. Increase to 30s for NAS environments with slow NAT/firewalls (#676).
|
||||
timeout: 30_000,
|
||||
lookup(hostname: string, opts: any, cb: any) {
|
||||
if (typeof opts === 'function') {
|
||||
cb = opts;
|
||||
opts = {};
|
||||
}
|
||||
|
||||
// IP addresses / localhost → no DNS needed
|
||||
if (net.isIP(hostname) || hostname === 'localhost') {
|
||||
return origLookup(hostname, opts, cb);
|
||||
}
|
||||
|
||||
lookupWithCache(hostname)
|
||||
.then(({ address, family }) => {
|
||||
if (opts.all) {
|
||||
cb(null, [{ address, family }]);
|
||||
} else {
|
||||
cb(null, address, family);
|
||||
}
|
||||
})
|
||||
.catch((err) => cb(err));
|
||||
}
|
||||
}
|
||||
})
|
||||
);
|
||||
+48
-32
@@ -10,6 +10,7 @@ import { existsSync, mkdirSync, rmSync, readdirSync } from 'node:fs';
|
||||
import { join, resolve } from 'node:path';
|
||||
import * as http from 'node:http';
|
||||
import * as https from 'node:https';
|
||||
import * as tls from 'node:tls';
|
||||
import { createHash } from 'node:crypto';
|
||||
import type { Environment } from './db';
|
||||
import { getStackEnvVarsAsRecord } from './db';
|
||||
@@ -345,7 +346,12 @@ function getHttpsAgent(config: DockerClientConfig): https.Agent {
|
||||
timeout: 30000,
|
||||
};
|
||||
|
||||
if (config.ca) agentOptions.ca = config.ca;
|
||||
if (config.ca) {
|
||||
// Include both the custom CA and Node.js built-in root certificates.
|
||||
// Node.js replaces the entire CA store when `ca` is set, unlike Bun which appends.
|
||||
// Without this, certs signed by intermediate CAs fail with "unable to get local issuer certificate".
|
||||
agentOptions.ca = [config.ca, ...tls.rootCertificates];
|
||||
}
|
||||
if (config.cert) agentOptions.cert = config.cert;
|
||||
if (config.key) agentOptions.key = config.key;
|
||||
if (config.skipVerify) agentOptions.rejectUnauthorized = false;
|
||||
@@ -874,7 +880,8 @@ export async function dockerFetch(
|
||||
headers,
|
||||
streaming || false,
|
||||
(streaming || path === '/_hawser/compose') ? 300000 : 30000, // 5 min for streaming/compose, 30s for normal
|
||||
isBinary
|
||||
isBinary,
|
||||
fetchOptions.signal ?? undefined
|
||||
);
|
||||
const elapsed = Date.now() - startTime;
|
||||
// Only warn for slow requests, but skip /stats which is expected to be slow (5-10s)
|
||||
@@ -1233,7 +1240,7 @@ export interface CreateContainerOptions {
|
||||
networkIpv6Address?: string;
|
||||
/** Gateway priority for the primary network (Docker Engine 28+) */
|
||||
networkGwPriority?: number;
|
||||
user?: string;
|
||||
user?: string | null;
|
||||
privileged?: boolean;
|
||||
healthcheck?: HealthcheckConfig;
|
||||
memory?: number;
|
||||
@@ -1286,7 +1293,7 @@ export interface CreateContainerOptions {
|
||||
// Device requests (GPU access, etc.)
|
||||
deviceRequests?: DeviceRequest[];
|
||||
// Container runtime (e.g., 'runc', 'nvidia' for GPU containers)
|
||||
runtime?: string;
|
||||
runtime?: string | null;
|
||||
// Read-only root filesystem
|
||||
readonlyRootfs?: boolean;
|
||||
// CPU pinning (e.g., "0-3", "0,1")
|
||||
@@ -1322,8 +1329,8 @@ export async function createContainer(options: CreateContainerOptions, envId?: n
|
||||
containerConfig.Cmd = options.cmd;
|
||||
}
|
||||
|
||||
if (options.user) {
|
||||
containerConfig.User = options.user;
|
||||
if (options.user !== undefined) {
|
||||
containerConfig.User = options.user ?? '';
|
||||
}
|
||||
|
||||
if (options.healthcheck) {
|
||||
@@ -1436,7 +1443,7 @@ export async function createContainer(options: CreateContainerOptions, envId?: n
|
||||
};
|
||||
}
|
||||
|
||||
if (options.privileged) {
|
||||
if (options.privileged !== undefined) {
|
||||
containerConfig.HostConfig.Privileged = options.privileged;
|
||||
}
|
||||
|
||||
@@ -1614,8 +1621,8 @@ export async function createContainer(options: CreateContainerOptions, envId?: n
|
||||
}
|
||||
|
||||
// Container runtime (e.g., 'nvidia' for GPU containers)
|
||||
if (options.runtime) {
|
||||
containerConfig.HostConfig.Runtime = options.runtime;
|
||||
if (options.runtime !== undefined) {
|
||||
containerConfig.HostConfig.Runtime = options.runtime ?? '';
|
||||
}
|
||||
|
||||
// Read-only root filesystem
|
||||
@@ -1782,6 +1789,13 @@ export async function recreateContainerFromInspect(
|
||||
HostConfig: hostConfig
|
||||
};
|
||||
|
||||
// Strip default MemorySwappiness — Podman + cgroupv2 rejects it.
|
||||
// Docker returns -1, Podman returns 0 when unset.
|
||||
const swappiness = createConfig.HostConfig?.MemorySwappiness;
|
||||
if (swappiness == null || swappiness === -1 || swappiness === 0) {
|
||||
delete createConfig.HostConfig.MemorySwappiness;
|
||||
}
|
||||
|
||||
// container:<name> mode shares the network namespace — Docker rejects
|
||||
// networking-related fields on the dependent container since they're
|
||||
// owned by the network provider container
|
||||
@@ -1883,6 +1897,11 @@ export async function recreateContainerFromInspect(
|
||||
[initialNetworkName]: endpointConfig
|
||||
}
|
||||
};
|
||||
// Container-level MacAddress conflicts with endpoint-level MacAddress.
|
||||
// Docker requires them to match or the top-level one to be empty.
|
||||
// The MAC is preserved in the endpoint config (correct location per API v1.44+),
|
||||
// so clear the top-level one to avoid the conflict error.
|
||||
delete createConfig.MacAddress;
|
||||
}
|
||||
|
||||
// 5. Create new container
|
||||
@@ -2236,7 +2255,7 @@ export function extractContainerOptions(inspectData: any): CreateContainerOption
|
||||
groupAdd: hostConfig.GroupAdd?.length > 0 ? hostConfig.GroupAdd : undefined,
|
||||
|
||||
// Memory swappiness
|
||||
memorySwappiness: hostConfig.MemorySwappiness !== null ? hostConfig.MemorySwappiness : undefined,
|
||||
memorySwappiness: hostConfig.MemorySwappiness != null && hostConfig.MemorySwappiness !== -1 && hostConfig.MemorySwappiness !== 0 ? hostConfig.MemorySwappiness : undefined,
|
||||
|
||||
// User namespace mode
|
||||
usernsMode: hostConfig.UsernsMode || undefined
|
||||
@@ -2665,6 +2684,10 @@ async function getRegistryBearerToken(registry: string, repo: string): Promise<s
|
||||
const cause = (e as any)?.cause;
|
||||
const causeMsg = cause ? ` (cause: ${cause})` : '';
|
||||
console.error('[Registry] Failed to get bearer token:', errorMsg + causeMsg);
|
||||
const causeStr = String(cause ?? errorMsg);
|
||||
if (causeStr.includes('EAI_AGAIN') || causeStr.includes('ENOTFOUND')) {
|
||||
console.error('[Registry] DNS resolution failed. If you are on a NAS (Synology, uGreen, QNAP), try adding --dns=8.8.8.8 to your docker run command or set {"dns": ["8.8.8.8"]} in /etc/docker/daemon.json');
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -2845,9 +2868,16 @@ export async function getRegistryManifestDigest(imageName: string): Promise<stri
|
||||
return null;
|
||||
}
|
||||
|
||||
return response.headers.get('Docker-Content-Digest');
|
||||
const digest = response.headers.get('Docker-Content-Digest');
|
||||
await drainResponse(response);
|
||||
return digest;
|
||||
} catch (e) {
|
||||
console.error(`[Registry] ${imageName}: ${e}`);
|
||||
const causeStr = String((e as any)?.cause ?? e);
|
||||
if (causeStr.includes('EAI_AGAIN') || causeStr.includes('ENOTFOUND')) {
|
||||
console.error(`[Registry] ${imageName}: DNS resolution failed. If you are on a NAS (Synology, uGreen, QNAP), add --dns=8.8.8.8 to your docker run command.`);
|
||||
} else {
|
||||
console.error(`[Registry] ${imageName}: ${e}`);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -3111,15 +3141,18 @@ export async function getDockerVersion(envId?: number | null) {
|
||||
*/
|
||||
export async function dockerPing(envId: number): Promise<boolean> {
|
||||
try {
|
||||
// Edge connections go WebSocket → agent → Docker daemon, which adds latency on slow hosts.
|
||||
// Use a longer timeout for edge to avoid false negatives on overloaded NAS/VPS devices.
|
||||
const config = await getDockerConfig(envId).catch(() => null);
|
||||
const timeoutMs = config?.connectionType === 'hawser-edge' ? 20000 : 5000;
|
||||
const response = await dockerFetch('/_ping', {
|
||||
signal: AbortSignal.timeout(5000)
|
||||
signal: AbortSignal.timeout(timeoutMs)
|
||||
}, envId);
|
||||
await drainResponse(response);
|
||||
return response.ok;
|
||||
} catch (error: any) {
|
||||
const msg = error?.message || String(error);
|
||||
if (msg.includes('unreachable')) {
|
||||
const config = await getDockerConfig(envId).catch(() => null);
|
||||
console.warn(`[Docker] ${config?.connectionType || 'direct'} ${config?.host || envId}: /_ping failed - host unreachable`);
|
||||
}
|
||||
return false;
|
||||
@@ -3286,24 +3319,7 @@ export interface NetworkInfo {
|
||||
export async function listNetworks(envId?: number | null): Promise<NetworkInfo[]> {
|
||||
const networks = await dockerJsonRequest<any[]>('/networks', {}, envId);
|
||||
|
||||
// Docker's /networks endpoint returns empty Containers - we need to inspect each network
|
||||
// to get the actual connected containers. Run inspections in parallel for performance.
|
||||
const networkDetails = await Promise.all(
|
||||
networks.map(async (network: any) => {
|
||||
try {
|
||||
const details = await dockerJsonRequest<any>(`/networks/${network.Id}`, {}, envId);
|
||||
return {
|
||||
...network,
|
||||
Containers: details.Containers || {}
|
||||
};
|
||||
} catch {
|
||||
// If inspection fails, return network with empty containers
|
||||
return network;
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
return networkDetails.map((network: any) => ({
|
||||
return networks.map((network: any) => ({
|
||||
id: network.Id,
|
||||
name: network.Name,
|
||||
driver: network.Driver,
|
||||
|
||||
+81
-18
@@ -44,6 +44,7 @@ export interface EdgeConnection {
|
||||
lastHeartbeat: number;
|
||||
pendingRequests: Map<string, PendingRequest>;
|
||||
pendingStreamRequests: Map<string, PendingStreamRequest>;
|
||||
pingInterval?: ReturnType<typeof setInterval>;
|
||||
lastMetrics?: {
|
||||
uptime?: number;
|
||||
cpuUsage?: number;
|
||||
@@ -77,7 +78,7 @@ declare global {
|
||||
var __hawserSendMessage: ((envId: number, message: string) => boolean) | undefined;
|
||||
var __hawserHandleContainerEvent: ((envId: number, event: ContainerEventMessage['event']) => Promise<void>) | undefined;
|
||||
var __hawserHandleMetrics: ((envId: number, metrics: MetricsMessage['metrics']) => Promise<void>) | undefined;
|
||||
var __hawserHandleMessage: ((ws: any, msg: any, connId: string) => Promise<void>) | undefined;
|
||||
var __hawserHandleMessage: ((ws: any, msg: any, connId: string, remoteIp?: string) => Promise<void>) | undefined;
|
||||
var __hawserHandleDisconnect: ((ws: any, connId: string) => void) | undefined;
|
||||
var __terminalHandleExecMessage: ((msg: any) => void) | undefined;
|
||||
}
|
||||
@@ -119,6 +120,11 @@ export function initializeEdgeManager(): void {
|
||||
conn.pendingRequests.clear();
|
||||
conn.pendingStreamRequests.clear();
|
||||
|
||||
if (conn.pingInterval) {
|
||||
clearInterval(conn.pingInterval);
|
||||
conn.pingInterval = undefined;
|
||||
}
|
||||
|
||||
conn.ws.close(1001, 'Connection timeout');
|
||||
edgeConnections.delete(envId);
|
||||
updateEnvironmentStatus(envId, null);
|
||||
@@ -255,11 +261,15 @@ globalThis.__hawserHandleMetrics = handleEdgeMetrics;
|
||||
export async function validateHawserToken(
|
||||
token: string
|
||||
): Promise<{ valid: boolean; environmentId?: number; tokenId?: number }> {
|
||||
// Get all active tokens
|
||||
const tokens = await db.select().from(hawserTokens).where(eq(hawserTokens.isActive, true));
|
||||
// Fast path: lookup by token prefix (first 8 chars) instead of iterating all tokens.
|
||||
// This reduces O(N) Argon2id verifications to O(1) DB lookup + 1 verify.
|
||||
const prefix = token.substring(0, 8);
|
||||
const candidates = await db
|
||||
.select()
|
||||
.from(hawserTokens)
|
||||
.where(and(eq(hawserTokens.tokenPrefix, prefix), eq(hawserTokens.isActive, true)));
|
||||
|
||||
// Check each token (tokens are hashed)
|
||||
for (const t of tokens) {
|
||||
for (const t of candidates) {
|
||||
try {
|
||||
const isValid = await verifyPassword(token, t.token);
|
||||
if (isValid) {
|
||||
@@ -276,7 +286,7 @@ export async function validateHawserToken(
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Invalid hash, continue checking
|
||||
// Invalid hash format, skip
|
||||
}
|
||||
}
|
||||
|
||||
@@ -371,6 +381,12 @@ export function closeEdgeConnection(environmentId: number): void {
|
||||
`Rejecting ${pendingCount} pending requests and ${streamCount} stream requests.`
|
||||
);
|
||||
|
||||
// Clear ping interval
|
||||
if (connection.pingInterval) {
|
||||
clearInterval(connection.pingInterval);
|
||||
connection.pingInterval = undefined;
|
||||
}
|
||||
|
||||
// Reject all pending requests
|
||||
for (const [requestId, pending] of connection.pendingRequests) {
|
||||
console.log(`[Hawser] Rejecting pending request ${requestId} due to environment deletion`);
|
||||
@@ -427,6 +443,12 @@ export function handleEdgeConnection(
|
||||
existing.pendingRequests.clear();
|
||||
existing.pendingStreamRequests.clear();
|
||||
|
||||
// Clear ping interval before closing
|
||||
if (existing.pingInterval) {
|
||||
clearInterval(existing.pingInterval);
|
||||
existing.pingInterval = undefined;
|
||||
}
|
||||
|
||||
// Immediately destroy TCP socket — no graceful close needed for replaced connections
|
||||
if (typeof existing.ws.terminate === 'function') {
|
||||
existing.ws.terminate();
|
||||
@@ -452,6 +474,17 @@ export function handleEdgeConnection(
|
||||
|
||||
edgeConnections.set(environmentId, connection);
|
||||
|
||||
// Start server-side ping interval to keep connection alive.
|
||||
// 5s is conservative against reverse proxies with aggressive idle timeouts.
|
||||
connection.pingInterval = setInterval(() => {
|
||||
try {
|
||||
connection.ws.send(JSON.stringify({ type: 'ping', timestamp: Date.now() }));
|
||||
} catch {
|
||||
clearInterval(connection.pingInterval!);
|
||||
connection.pingInterval = undefined;
|
||||
}
|
||||
}, 5000);
|
||||
|
||||
// Update environment record
|
||||
updateEnvironmentStatus(environmentId, connection);
|
||||
|
||||
@@ -499,7 +532,8 @@ export async function sendEdgeRequest(
|
||||
headers?: Record<string, string>,
|
||||
streaming = false,
|
||||
timeout = 30000,
|
||||
isBinary = false
|
||||
isBinary = false,
|
||||
signal?: AbortSignal
|
||||
): Promise<EdgeResponse> {
|
||||
const connection = edgeConnections.get(environmentId);
|
||||
if (!connection) {
|
||||
@@ -517,6 +551,27 @@ export async function sendEdgeRequest(
|
||||
reject(new Error('Request timeout'));
|
||||
}, timeout);
|
||||
|
||||
// Honor AbortSignal from caller (e.g., AbortSignal.timeout(5000) for dockerPing)
|
||||
if (signal) {
|
||||
if (signal.aborted) {
|
||||
clearTimeout(timeoutHandle);
|
||||
reject(new Error('Request aborted'));
|
||||
return;
|
||||
}
|
||||
signal.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
connection.pendingRequests.delete(requestId);
|
||||
if (streaming) {
|
||||
connection.pendingStreamRequests.delete(requestId);
|
||||
}
|
||||
clearTimeout(timeoutHandle);
|
||||
reject(new Error('Request aborted'));
|
||||
},
|
||||
{ once: true }
|
||||
);
|
||||
}
|
||||
|
||||
// For streaming requests, the Go agent sends 'stream' messages instead of a single 'response'.
|
||||
// We need to register a stream handler that collects all data and resolves when complete.
|
||||
if (streaming) {
|
||||
@@ -792,6 +847,12 @@ export function handleHeartbeat(environmentId: number): void {
|
||||
export function handleDisconnect(environmentId: number): void {
|
||||
const connection = edgeConnections.get(environmentId);
|
||||
if (connection) {
|
||||
// Clear ping interval
|
||||
if (connection.pingInterval) {
|
||||
clearInterval(connection.pingInterval);
|
||||
connection.pingInterval = undefined;
|
||||
}
|
||||
|
||||
// Reject all pending requests
|
||||
for (const [, pending] of connection.pendingRequests) {
|
||||
clearTimeout(pending.timeout);
|
||||
@@ -983,11 +1044,12 @@ export type HawserMessage =
|
||||
const wsToEnvId = new Map<any, number>();
|
||||
|
||||
// Auth fail cache to prevent brute-force token validation.
|
||||
// Entries are periodically cleaned up to prevent unbounded growth.
|
||||
// 5 min cooldown — hawser agents use exponential backoff (30-60s),
|
||||
// so a short cooldown lets every retry through.
|
||||
const hawserAuthFailCache = new Map<string, number>();
|
||||
const HAWSER_AUTH_FAIL_COOLDOWN_MS = 30_000;
|
||||
const HAWSER_AUTH_FAIL_COOLDOWN_MS = 5 * 60 * 1000; // 5 minutes
|
||||
|
||||
// Periodic cleanup of expired auth fail entries (every 60s)
|
||||
// Periodic cleanup of expired auth fail entries
|
||||
setInterval(() => {
|
||||
const now = Date.now();
|
||||
for (const [key, timestamp] of hawserAuthFailCache) {
|
||||
@@ -995,7 +1057,7 @@ setInterval(() => {
|
||||
hawserAuthFailCache.delete(key);
|
||||
}
|
||||
}
|
||||
}, 60_000);
|
||||
}, HAWSER_AUTH_FAIL_COOLDOWN_MS);
|
||||
|
||||
// ─── Reconnection storm throttle ───
|
||||
// Tracks per-environment reconnection frequency to detect storms
|
||||
@@ -1007,7 +1069,7 @@ interface ReconnectTrackerEntry {
|
||||
}
|
||||
const reconnectTracker = new Map<number, ReconnectTrackerEntry>();
|
||||
const RECONNECT_WINDOW_MS = 2 * 60 * 1000; // 2-minute sliding window
|
||||
const RECONNECT_BURST = 3; // allow 3 reconnections per window
|
||||
const RECONNECT_BURST = 10; // allow 10 reconnections per window
|
||||
const COOLDOWN_LEVELS_SECS = [30, 60, 120, 300]; // escalating cooldown in seconds
|
||||
const STABLE_THRESHOLD_MS = 5 * 60 * 1000; // stable connection resets tracker
|
||||
const STALE_TRACKER_MS = 10 * 60 * 1000; // clean up stale tracker entries
|
||||
@@ -1062,13 +1124,14 @@ function recordReconnection(envId: number): { allowed: true } | { allowed: false
|
||||
*
|
||||
* Registered as globalThis.__hawserHandleMessage for server.js to call.
|
||||
*/
|
||||
async function handleHawserWsMessage(ws: any, msg: any, connId: string): Promise<void> {
|
||||
async function handleHawserWsMessage(ws: any, msg: any, connId: string, remoteIp?: string): Promise<void> {
|
||||
if (msg.type === 'hello') {
|
||||
const remoteAddr = connId;
|
||||
const rateLimitKey = remoteIp || connId;
|
||||
|
||||
// Rate limit auth failures
|
||||
const lastFail = hawserAuthFailCache.get(remoteAddr);
|
||||
// Rate limit auth failures by remote IP (not connId which is unique per connection)
|
||||
const lastFail = hawserAuthFailCache.get(rateLimitKey);
|
||||
if (lastFail && Date.now() - lastFail < HAWSER_AUTH_FAIL_COOLDOWN_MS) {
|
||||
console.log(`[Hawser WS] Rate limited ${connId} (IP: ${rateLimitKey}) — ${Math.round((Date.now() - lastFail) / 1000)}s since last fail`);
|
||||
ws.send(JSON.stringify({ type: 'error', message: 'Too many failed attempts' }));
|
||||
ws.close(1008, 'Rate limited');
|
||||
return;
|
||||
@@ -1083,8 +1146,8 @@ async function handleHawserWsMessage(ws: any, msg: any, connId: string): Promise
|
||||
try {
|
||||
const result = await validateHawserToken(msg.token);
|
||||
if (!result.valid || !result.environmentId) {
|
||||
console.log(`[Hawser WS] Authentication failed for connection ${connId}`);
|
||||
hawserAuthFailCache.set(remoteAddr, Date.now());
|
||||
console.log(`[Hawser WS] Authentication failed for connection ${connId} (IP: ${rateLimitKey})`);
|
||||
hawserAuthFailCache.set(rateLimitKey, Date.now());
|
||||
ws.send(JSON.stringify({ type: 'error', message: 'Invalid token' }));
|
||||
ws.close(1008, 'Invalid token');
|
||||
return;
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { Cron } from 'croner';
|
||||
|
||||
/**
|
||||
* Get the next run time for a cron expression.
|
||||
* Uses legacyMode: false so day-of-month + day-of-week use AND logic.
|
||||
* @param cronExpression - The cron expression
|
||||
* @param timezone - Optional IANA timezone (e.g., 'Europe/Warsaw'). Defaults to local timezone.
|
||||
*/
|
||||
export function getNextRun(cronExpression: string, timezone?: string): Date | null {
|
||||
try {
|
||||
const options = timezone ? { timezone, legacyMode: false } : { legacyMode: false };
|
||||
const job = new Cron(cronExpression, options);
|
||||
const next = job.nextRun();
|
||||
job.stop();
|
||||
return next;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a cron expression is valid.
|
||||
*/
|
||||
export function isValidCron(cronExpression: string): boolean {
|
||||
try {
|
||||
const job = new Cron(cronExpression, { legacyMode: false });
|
||||
job.stop();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -107,11 +107,11 @@ export async function startScheduler(): Promise<void> {
|
||||
const defaultTimezone = await getDefaultTimezone();
|
||||
|
||||
// Start system cleanup jobs (static schedules with default timezone)
|
||||
cleanupJob = new Cron(scheduleCleanupCron, { timezone: defaultTimezone }, async () => {
|
||||
cleanupJob = new Cron(scheduleCleanupCron, { timezone: defaultTimezone, legacyMode: false }, async () => {
|
||||
await runScheduleCleanupJob();
|
||||
});
|
||||
|
||||
eventCleanupJob = new Cron(eventCleanupCron, { timezone: defaultTimezone }, async () => {
|
||||
eventCleanupJob = new Cron(eventCleanupCron, { timezone: defaultTimezone, legacyMode: false }, async () => {
|
||||
await runEventCleanupJob();
|
||||
});
|
||||
|
||||
@@ -127,15 +127,10 @@ export async function startScheduler(): Promise<void> {
|
||||
};
|
||||
|
||||
// Volume helper cleanup runs every 30 minutes to clean up expired browse containers
|
||||
volumeHelperCleanupJob = new Cron('*/30 * * * *', { timezone: defaultTimezone }, async () => {
|
||||
volumeHelperCleanupJob = new Cron('*/30 * * * *', { timezone: defaultTimezone, legacyMode: false }, async () => {
|
||||
await runVolumeHelperCleanupJob('cron', volumeCleanupFns);
|
||||
});
|
||||
|
||||
// Run volume helper cleanup immediately on startup to clean up stale containers
|
||||
runVolumeHelperCleanupJob('startup', volumeCleanupFns).catch(err => {
|
||||
const errorMsg = err instanceof Error ? err.message : String(err);
|
||||
console.error('[Scheduler] Error during startup volume helper cleanup:', errorMsg);
|
||||
});
|
||||
|
||||
console.log(`[Scheduler] System schedule cleanup: ${scheduleCleanupCron} [${defaultTimezone}]`);
|
||||
console.log(`[Scheduler] System event cleanup: ${eventCleanupCron} [${defaultTimezone}]`);
|
||||
@@ -331,7 +326,7 @@ export async function registerSchedule(
|
||||
const timezone = environmentId ? await getEnvironmentTimezone(environmentId) : 'UTC';
|
||||
|
||||
// Create new Cron instance with timezone
|
||||
const job = new Cron(cronExpression, { timezone }, async () => {
|
||||
const job = new Cron(cronExpression, { timezone, legacyMode: false }, async () => {
|
||||
// Defensive check: verify schedule still exists and is enabled
|
||||
if (type === 'container_update') {
|
||||
const setting = await getAutoUpdateSettingById(scheduleId);
|
||||
@@ -494,15 +489,15 @@ export async function refreshSystemJobs(): Promise<void> {
|
||||
}
|
||||
|
||||
// Re-create with new timezone
|
||||
cleanupJob = new Cron(scheduleCleanupCron, { timezone: defaultTimezone }, async () => {
|
||||
cleanupJob = new Cron(scheduleCleanupCron, { timezone: defaultTimezone, legacyMode: false }, async () => {
|
||||
await runScheduleCleanupJob();
|
||||
});
|
||||
|
||||
eventCleanupJob = new Cron(eventCleanupCron, { timezone: defaultTimezone }, async () => {
|
||||
eventCleanupJob = new Cron(eventCleanupCron, { timezone: defaultTimezone, legacyMode: false }, async () => {
|
||||
await runEventCleanupJob();
|
||||
});
|
||||
|
||||
volumeHelperCleanupJob = new Cron('*/30 * * * *', { timezone: defaultTimezone }, async () => {
|
||||
volumeHelperCleanupJob = new Cron('*/30 * * * *', { timezone: defaultTimezone, legacyMode: false }, async () => {
|
||||
await runVolumeHelperCleanupJob('cron', volumeCleanupFns);
|
||||
});
|
||||
|
||||
@@ -654,35 +649,9 @@ export async function triggerSystemJob(jobId: string): Promise<{ success: boolea
|
||||
// UTILITY FUNCTIONS
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Get the next run time for a cron expression.
|
||||
* @param cronExpression - The cron expression
|
||||
* @param timezone - Optional IANA timezone (e.g., 'Europe/Warsaw'). Defaults to local timezone.
|
||||
*/
|
||||
export function getNextRun(cronExpression: string, timezone?: string): Date | null {
|
||||
try {
|
||||
const options = timezone ? { timezone } : undefined;
|
||||
const job = new Cron(cronExpression, options);
|
||||
const next = job.nextRun();
|
||||
job.stop();
|
||||
return next;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a cron expression is valid.
|
||||
*/
|
||||
export function isValidCron(cronExpression: string): boolean {
|
||||
try {
|
||||
const job = new Cron(cronExpression);
|
||||
job.stop();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Imported from cron-utils.ts (isolated from DB deps for unit test compatibility)
|
||||
import { getNextRun, isValidCron } from './cron-utils';
|
||||
export { getNextRun, isValidCron };
|
||||
|
||||
/**
|
||||
* Get system schedules info for the API.
|
||||
|
||||
@@ -74,12 +74,17 @@ export async function runImagePrune(
|
||||
|
||||
// Extract space reclaimed and images removed from result
|
||||
const spaceReclaimed = result?.SpaceReclaimed || 0;
|
||||
// Count unique images by filtering Untagged entries that are not digest references
|
||||
// Docker returns multiple entries per image: Untagged (tag), Untagged (digest @sha256:), Deleted (layers)
|
||||
// We only count tag-based Untagged entries to get actual image count
|
||||
const imagesRemoved = result?.ImagesDeleted
|
||||
?.filter((img: any) => img.Untagged && !img.Untagged.includes('@sha256:'))
|
||||
.length || 0;
|
||||
// Count unique images removed.
|
||||
// Docker returns: Untagged (tag), Untagged (digest @sha256:), Deleted (layer sha256:)
|
||||
// For tagged images: count Untagged entries that are NOT digest references (tag-based)
|
||||
// For dangling images: there are no tag-based entries, only digest-based Untagged entries
|
||||
// So count tag-based Untagged first, fall back to digest-based Untagged for dangling prune
|
||||
const deleted = result?.ImagesDeleted || [];
|
||||
const tagEntries = deleted.filter((img: any) => img.Untagged && !img.Untagged.includes('@sha256:'));
|
||||
const digestEntries = deleted.filter((img: any) => img.Untagged && img.Untagged.includes('@sha256:'));
|
||||
const imagesRemoved = tagEntries.length > 0
|
||||
? tagEntries.length
|
||||
: digestEntries.length;
|
||||
|
||||
// Format space for human-readable output
|
||||
const formatBytes = (bytes: number): string => {
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
*
|
||||
* Provides compose-first stack operations for internal, git, and external stacks.
|
||||
* All lifecycle operations use docker compose commands.
|
||||
* v1.0.20
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, rmSync, readdirSync, cpSync, statSync, unlinkSync, renameSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
@@ -139,10 +138,6 @@ const stackLocks = new Map<string, Promise<void>>();
|
||||
// Track active TLS temp directories for cleanup on unexpected process exit
|
||||
const activeTlsDirs = new Set<string>();
|
||||
|
||||
// Cache of envId → daemon max API version (e.g. "1.43")
|
||||
// Populated lazily to avoid CLI/daemon version mismatch on older Docker hosts (e.g. Synology)
|
||||
const dockerApiVersionCache = new Map<string, string>();
|
||||
|
||||
// Register cleanup handlers once at module load
|
||||
if (typeof process !== 'undefined') {
|
||||
const cleanupTlsDirs = () => {
|
||||
@@ -158,85 +153,6 @@ if (typeof process !== 'undefined') {
|
||||
process.on('SIGTERM', () => { cleanupTlsDirs(); process.exit(143); });
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch and cache the Docker daemon's maximum supported API version for a given environment.
|
||||
* Used to set DOCKER_API_VERSION when spawning docker compose, preventing version mismatch
|
||||
* errors on older Docker hosts (e.g. Synology DSM).
|
||||
*
|
||||
* Strategy:
|
||||
* 1. Try Dockhand's HTTP API call to the daemon (works for all environment types)
|
||||
* 2. Fall back to `docker version` CLI command (works for local socket connections)
|
||||
*/
|
||||
async function getDockerApiVersionForCli(envId: number | null | undefined): Promise<string | undefined> {
|
||||
const key = String(envId ?? 'local');
|
||||
if (dockerApiVersionCache.has(key)) return dockerApiVersionCache.get(key);
|
||||
|
||||
// Strategy 1: Use Dockhand's HTTP API to query the daemon
|
||||
if (envId) {
|
||||
try {
|
||||
const { getDockerVersion } = await import('./docker.js');
|
||||
const version = await getDockerVersion(envId) as { ApiVersion?: string };
|
||||
const apiVersion: string | undefined = version?.ApiVersion;
|
||||
if (apiVersion) {
|
||||
console.log(`[Docker API Version] Detected daemon API version ${apiVersion} for env ${key} (via HTTP API)`);
|
||||
dockerApiVersionCache.set(key, apiVersion);
|
||||
return apiVersion;
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.warn(`[Docker API Version] HTTP API query failed for env ${key}: ${err?.message || err}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 2: Fall back to `docker version` CLI command
|
||||
// This handles local socket connections where envId is null and also
|
||||
// cases where the HTTP API query fails (e.g. daemon quirks on Synology)
|
||||
try {
|
||||
const apiVersion = await getDockerApiVersionViaCli();
|
||||
if (apiVersion) {
|
||||
console.log(`[Docker API Version] Detected daemon API version ${apiVersion} for env ${key} (via CLI)`);
|
||||
dockerApiVersionCache.set(key, apiVersion);
|
||||
return apiVersion;
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.warn(`[Docker API Version] CLI query failed for env ${key}: ${err?.message || err}`);
|
||||
}
|
||||
|
||||
console.warn(`[Docker API Version] Could not detect daemon API version for env ${key}`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the Docker daemon's API version using the `docker version` CLI command.
|
||||
* This is a fallback for when the HTTP API query fails or envId is null.
|
||||
*/
|
||||
function getDockerApiVersionViaCli(): Promise<string | undefined> {
|
||||
return new Promise((resolve) => {
|
||||
const proc = nodeSpawn('docker', ['version', '--format', '{{.Server.APIVersion}}'], {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
timeout: 5000,
|
||||
// Use the minimum Docker API version (1.25) for this probe command.
|
||||
// This ensures the probe itself doesn't fail due to the version mismatch
|
||||
// we're trying to detect.
|
||||
env: {
|
||||
PATH: process.env.PATH || '/usr/local/bin:/usr/bin:/bin',
|
||||
DOCKER_API_VERSION: '1.25'
|
||||
}
|
||||
});
|
||||
let stdout = '';
|
||||
proc.stdout.on('data', (data: Buffer) => { stdout += data.toString(); });
|
||||
proc.stderr?.on('data', () => {}); // drain stderr to prevent pipe buffer blocking
|
||||
proc.on('close', (code) => {
|
||||
const version = stdout.trim();
|
||||
if (code === 0 && /^\d+\.\d+$/.test(version)) {
|
||||
resolve(version);
|
||||
} else {
|
||||
resolve(undefined);
|
||||
}
|
||||
});
|
||||
proc.on('error', () => resolve(undefined));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a function with exclusive lock on a stack.
|
||||
* Prevents race conditions when multiple operations target the same stack.
|
||||
@@ -821,7 +737,7 @@ async function loginToRegistries(dockerHost?: string, logPrefix = '[Stack]', api
|
||||
if (dockerHost) {
|
||||
spawnEnv.DOCKER_HOST = dockerHost;
|
||||
}
|
||||
// Cap Docker CLI API version to prevent version mismatch errors
|
||||
// Pass through explicit DOCKER_API_VERSION if provided by caller
|
||||
if (apiVersion) {
|
||||
spawnEnv.DOCKER_API_VERSION = apiVersion;
|
||||
}
|
||||
@@ -997,13 +913,10 @@ async function executeLocalCompose(
|
||||
spawnEnv.DOCKER_HOST = process.env.DOCKER_HOST;
|
||||
}
|
||||
|
||||
// Auto-cap Docker CLI API version to the daemon's max supported version.
|
||||
// This fixes compatibility with older Docker daemons (e.g. Synology DSM) that
|
||||
// reject newer client versions. DOCKER_API_VERSION env var overrides this if set.
|
||||
const daemonApiVersion = process.env.DOCKER_API_VERSION
|
||||
?? await getDockerApiVersionForCli(envId);
|
||||
if (daemonApiVersion) {
|
||||
spawnEnv.DOCKER_API_VERSION = daemonApiVersion;
|
||||
// Honor explicit DOCKER_API_VERSION override from environment (user-controlled).
|
||||
// Otherwise let compose negotiate natively — 5.0.2 handles old daemons correctly.
|
||||
if (process.env.DOCKER_API_VERSION) {
|
||||
spawnEnv.DOCKER_API_VERSION = process.env.DOCKER_API_VERSION;
|
||||
}
|
||||
|
||||
// Check if .env file exists on disk (for legacy support decision)
|
||||
@@ -1162,7 +1075,7 @@ async function executeLocalCompose(
|
||||
console.log(`${logPrefix} Working directory:`, stackDir);
|
||||
console.log(`${logPrefix} Compose file:`, composeFile);
|
||||
console.log(`${logPrefix} DOCKER_HOST:`, dockerHost || '(local socket)');
|
||||
console.log(`${logPrefix} DOCKER_API_VERSION:`, daemonApiVersion || '(not set - using CLI default)');
|
||||
console.log(`${logPrefix} DOCKER_API_VERSION:`, spawnEnv.DOCKER_API_VERSION || '(not set - native negotiation)');
|
||||
console.log(`${logPrefix} Force recreate:`, forceRecreate ?? false);
|
||||
console.log(`${logPrefix} Remove volumes:`, removeVolumes ?? false);
|
||||
console.log(`${logPrefix} Service name:`, serviceName ?? '(all services)');
|
||||
@@ -1173,7 +1086,7 @@ async function executeLocalCompose(
|
||||
|
||||
// Login to registries before pulling images
|
||||
if (operation === 'up' || operation === 'pull') {
|
||||
await loginToRegistries(dockerHost, logPrefix, daemonApiVersion);
|
||||
await loginToRegistries(dockerHost, logPrefix, spawnEnv.DOCKER_API_VERSION);
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
Reference in new issue
Block a user