From f085d78fc19ba9477765e2ee8640a89262691b99 Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Sun, 15 Feb 2026 13:51:58 -0800 Subject: [PATCH] fix starting and restarts --- src/cli/commands/manage.ts | 5 +-- src/client/components/AppDetail.tsx | 2 +- src/client/styles/misc.ts | 2 ++ src/server/apps.ts | 19 ++++++++--- src/server/tunnels.ts | 51 ++++++++++++++++++++++++++--- src/shared/types.ts | 2 +- 6 files changed, 69 insertions(+), 12 deletions(-) diff --git a/src/cli/commands/manage.ts b/src/cli/commands/manage.ts index 87042ac..7f0a600 100644 --- a/src/cli/commands/manage.ts +++ b/src/cli/commands/manage.ts @@ -10,6 +10,7 @@ import { resolveAppName } from '../name' import { pushApp } from './sync' export const STATE_ICONS: Record = { + error: color.red('●'), running: color.green('●'), starting: color.yellow('◎'), stopped: color.gray('◯'), @@ -26,8 +27,8 @@ async function waitForState(name: string, target: string, timeout: number): Prom if (!app) return undefined if (app.state === target) return target // Terminal failure states — stop polling - if (target === 'running' && (app.state === 'stopped' || app.state === 'invalid')) return app.state - if (target === 'stopped' && app.state === 'invalid') return app.state + if (target === 'running' && (app.state === 'stopped' || app.state === 'invalid' || app.state === 'error')) return app.state + if (target === 'stopped' && (app.state === 'invalid' || app.state === 'error')) return app.state } // Timed out — return last known state const app: App | undefined = await get(`/api/apps/${name}`) diff --git a/src/client/components/AppDetail.tsx b/src/client/components/AppDetail.tsx index e2f3868..ba59c03 100644 --- a/src/client/components/AppDetail.tsx +++ b/src/client/components/AppDetail.tsx @@ -137,7 +137,7 @@ export function AppDetail({ app, render }: { app: App, render: () => void }) { - {app.state === 'stopped' && ( + {(app.state === 'stopped' || app.state === 'error') && ( diff --git a/src/client/styles/misc.ts b/src/client/styles/misc.ts index 0088ef2..180c11f 100644 --- a/src/client/styles/misc.ts +++ b/src/client/styles/misc.ts @@ -47,6 +47,7 @@ export const StatusDot = define('StatusDot', { flexShrink: 0, variants: { state: { + error: { background: theme('colors-statusInvalid') }, invalid: { background: theme('colors-statusInvalid') }, stopped: { background: theme('colors-statusStopped') }, starting: { background: theme('colors-statusStarting') }, @@ -151,6 +152,7 @@ export const TabContent = define('TabContent', { }) export const stateLabels: Record = { + error: 'Error', invalid: 'Invalid', stopped: 'Stopped', starting: 'Starting', diff --git a/src/server/apps.ts b/src/server/apps.ts index 2b5479e..37c79f6 100644 --- a/src/server/apps.ts +++ b/src/server/apps.ts @@ -23,6 +23,7 @@ const HEALTH_CHECK_TIMEOUT = 5000 const LOG_RETENTION_DAYS = 7 const MAX_LOGS = 100 const MAX_PORT = 3100 +const MAX_RESTART_ATTEMPTS = 5 const MIN_PORT = 3001 const RESTART_DELAYS = [1000, 2000, 4000, 8000, 16000, 32000] const SHUTDOWN_TIMEOUT = 10000 @@ -219,11 +220,12 @@ export async function renameApp(oldName: string, newName: string): Promise<{ ok: export function startApp(dir: string) { const app = _apps.get(dir) - if (!app || (app.state !== 'stopped' && app.state !== 'invalid')) return + if (!app || (app.state !== 'stopped' && app.state !== 'invalid' && app.state !== 'error')) return if (!isApp(dir)) return // Clear flags when explicitly starting app.manuallyStopped = false + app.restartAttempts = 0 app.error = undefined runApp(dir, getPort(dir)) } @@ -761,13 +763,22 @@ function saveApp(dir: string, pkg: any) { function scheduleRestart(app: App, dir: string) { const attempts = app.restartAttempts ?? 0 + + if (attempts >= MAX_RESTART_ATTEMPTS) { + info(app, `Too many restart failures (${attempts}/${MAX_RESTART_ATTEMPTS}), giving up.`) + app.state = 'error' + app.error = `Crashed ${attempts} times, restart disabled` + update() + return + } + const delayIndex = Math.min(attempts, RESTART_DELAYS.length - 1) const delay = RESTART_DELAYS[delayIndex]! app.restartAttempts = attempts + 1 app.lastRestartTime = Date.now() - info(app, `Scheduling restart in ${delay / 1000}s (attempt ${app.restartAttempts})...`) + info(app, `Scheduling restart in ${delay / 1000}s (attempt ${app.restartAttempts}/${MAX_RESTART_ATTEMPTS})...`) setTimeout(() => { // Double-check conditions before restarting @@ -793,8 +804,8 @@ function shouldAutoRestart(app: App, exitCode: number | null): boolean { // Don't restart if manually stopped if (app.manuallyStopped) return false - // Don't restart if app became invalid - if (app.state === 'invalid') return false + // Don't restart if app became invalid or hit error limit + if (app.state === 'invalid' || app.state === 'error') return false // Only restart on non-zero exit codes (crashes) if (exitCode === 0) return false diff --git a/src/server/tunnels.ts b/src/server/tunnels.ts index 37a16a2..e5fff6a 100644 --- a/src/server/tunnels.ts +++ b/src/server/tunnels.ts @@ -9,7 +9,12 @@ const SNEAKER_URL = process.env.SNEAKER_URL ?? 'https://toes.space' type TunnelConfig = Record +const RECONNECT_DELAYS = [5000, 10000, 20000, 30000, 60000] +const MAX_RECONNECT_ATTEMPTS = 10 + const _closing = new Set() +const _reconnectAttempts = new Map() +const _reconnectTimers = new Map() const _tunnels = new Map() const _tunnelPorts = new Map() @@ -47,6 +52,12 @@ export const isTunnelsAvailable = (): boolean => !!SNEAKER_URL export function closeAllTunnels() { + for (const [, timer] of _reconnectTimers) { + clearTimeout(timer) + } + _reconnectTimers.clear() + _reconnectAttempts.clear() + for (const [name, tunnel] of _tunnels) { _closing.add(name) tunnel.close() @@ -56,6 +67,8 @@ export function closeAllTunnels() { } export function closeTunnel(appName: string) { + cancelReconnect(appName) + const tunnel = _tunnels.get(appName) if (!tunnel) return @@ -133,7 +146,19 @@ export function renameTunnelConfig(oldName: string, newName: string) { saveConfig(config) } +function cancelReconnect(appName: string) { + const timer = _reconnectTimers.get(appName) + if (timer) { + clearTimeout(timer) + _reconnectTimers.delete(appName) + } + _reconnectAttempts.delete(appName) +} + function openTunnel(appName: string, port: number, subdomain?: string) { + // Cancel any pending reconnect timer to prevent duplicate loops + cancelReconnect(appName) + // Close existing tunnel if any const existing = _tunnels.get(appName) if (existing) { @@ -154,6 +179,9 @@ function openTunnel(appName: string, port: number, subdomain?: string) { onOpen(assignedSubdomain) { hostLog(`Tunnel open: ${appName} -> ${assignedSubdomain}`) + // Reset reconnect attempts on successful connection + _reconnectAttempts.delete(appName) + // Save subdomain for reconnection const config = loadConfig() if (config[appName]) { @@ -178,21 +206,36 @@ function openTunnel(appName: string, port: number, subdomain?: string) { return } - // Unexpected close — try to reconnect - hostLog(`Tunnel dropped: ${appName}, reconnecting in 5s...`) + // Check retry limit + const attempts = _reconnectAttempts.get(appName) ?? 0 + if (attempts >= MAX_RECONNECT_ATTEMPTS) { + hostLog(`Tunnel gave up: ${appName} (${attempts} failed attempts)`) + _reconnectAttempts.delete(appName) + return + } + + const delayIndex = Math.min(attempts, RECONNECT_DELAYS.length - 1) + const delay = RECONNECT_DELAYS[delayIndex]! + _reconnectAttempts.set(appName, attempts + 1) + + hostLog(`Tunnel dropped: ${appName}, reconnecting in ${delay / 1000}s (attempt ${attempts + 1}/${MAX_RECONNECT_ATTEMPTS})...`) const app = getApp(appName) if (app) { app.tunnelUrl = undefined update() } - setTimeout(() => { + const timer = setTimeout(() => { + _reconnectTimers.delete(appName) + // Only reconnect if still enabled in config const config = loadConfig() if (!config[appName]) return hostLog(`Tunnel reconnecting: ${appName}`) openTunnel(appName, port, config[appName]?.subdomain) - }, 5000) + }, delay) + + _reconnectTimers.set(appName, timer) }, onError(error) { diff --git a/src/shared/types.ts b/src/shared/types.ts index d0bdf71..c0fdd42 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -11,7 +11,7 @@ export interface Manifest { name: string } -export type AppState = 'invalid' | 'stopped' | 'starting' | 'running' | 'stopping' +export type AppState = 'error' | 'invalid' | 'stopped' | 'starting' | 'running' | 'stopping' export type LogLine = { time: number