From 769b564d207e95775b44bff708ac9d6ccfaae9e2 Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Fri, 30 Jan 2026 00:05:02 -0800 Subject: [PATCH] monster features --- src/server/apps.ts | 342 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 325 insertions(+), 17 deletions(-) diff --git a/src/server/apps.ts b/src/server/apps.ts index fdfddb0..8df3ee8 100644 --- a/src/server/apps.ts +++ b/src/server/apps.ts @@ -1,4 +1,4 @@ -import type { App as SharedApp, AppState, LogLine } from '@types' +import type { App as SharedApp, AppState } from '@types' import type { Subprocess } from 'bun' import { DEFAULT_EMOJI } from '@types' import { existsSync, readdirSync, readFileSync, renameSync, statSync, watch, writeFileSync } from 'fs' @@ -7,17 +7,37 @@ import { join } from 'path' export type { AppState } from '@types' export const APPS_DIR = join(process.env.DATA_DIR ?? '.', 'apps') + +const HEALTH_CHECK_FAILURES_BEFORE_RESTART = 3 +const HEALTH_CHECK_INTERVAL = 30000 +const HEALTH_CHECK_TIMEOUT = 5000 const MAX_LOGS = 100 +const MAX_PORT = 3100 +const MIN_PORT = 3001 +const RESTART_DELAYS = [1000, 2000, 4000, 8000, 16000, 32000] +const SHUTDOWN_TIMEOUT = 10000 +const STABLE_RUN_TIME = 60000 +const STARTUP_TIMEOUT = 30000 + +const _appPorts = new Map() const _apps = new Map() +const _availablePorts: number[] = [] const _listeners = new Set<() => void>() -let NEXT_PORT = 3001 +let _shuttingDown = false export type App = SharedApp & { + consecutiveHealthFailures?: number + healthCheckTimer?: Timer + lastRestartTime?: number + manuallyStopped?: boolean proc?: Subprocess + restartAttempts?: number + shutdownTimer?: Timer + startupTimer?: Timer } -type LoadResult = { pkg: any; error?: string } +type LoadResult = { pkg: any, error?: string } export const allApps = (): App[] => Array.from(_apps.values()) @@ -33,6 +53,8 @@ export const runningApps = (): App[] => allApps().filter(a => a.state === 'running') export function initApps() { + initPortPool() + setupShutdownHandlers() discoverApps() runApps() watchAppsDir() @@ -43,25 +65,26 @@ export function onChange(cb: () => void) { return () => _listeners.delete(cb) } -export function startApp(dir: string) { - const app = _apps.get(dir) - if (!app || app.state !== 'stopped') return - if (!isApp(dir)) return - runApp(dir, getPort()) -} - export function removeApp(dir: string) { const app = _apps.get(dir) if (!app) return + // Clear all timers + clearTimers(app) + if (app.state === 'running') app.proc?.kill() + // Release port if assigned + if (app.port) { + releasePort(app.port) + } + _apps.delete(dir) update() } -export function renameApp(oldName: string, newName: string): { ok: boolean; error?: string } { +export function renameApp(oldName: string, newName: string): { ok: boolean, error?: string } { const app = _apps.get(oldName) if (!app) return { ok: false, error: 'App not found' } @@ -77,8 +100,10 @@ export function renameApp(oldName: string, newName: string): { ok: boolean; erro // Stop the app if running const wasRunning = app.state === 'running' if (wasRunning) { + clearTimers(app) app.proc?.kill() app.proc = undefined + if (app.port) releasePort(app.port) app.port = undefined app.started = undefined } @@ -89,10 +114,19 @@ export function renameApp(oldName: string, newName: string): { ok: boolean; erro return { ok: false, error: `Failed to rename directory: ${e instanceof Error ? e.message : String(e)}` } } + // Transfer port mapping to new name + const oldPort = _appPorts.get(oldName) + if (oldPort !== undefined) { + _appPorts.delete(oldName) + _appPorts.set(newName, oldPort) + } + // Update the internal registry _apps.delete(oldName) app.name = newName app.state = 'stopped' + app.manuallyStopped = false + app.restartAttempts = 0 _apps.set(newName, app) update() @@ -105,13 +139,34 @@ export function renameApp(oldName: string, newName: string): { ok: boolean; erro return { ok: true } } +export function startApp(dir: string) { + const app = _apps.get(dir) + if (!app || app.state !== 'stopped') return + if (!isApp(dir)) return + + // Clear manually stopped flag when explicitly starting + app.manuallyStopped = false + runApp(dir, getPort(dir)) +} + export function stopApp(dir: string) { const app = _apps.get(dir) if (!app || app.state !== 'running') return info(app, 'Stopping...') app.state = 'stopping' + app.manuallyStopped = true update() + + // Clear health check timer + if (app.healthCheckTimer) { + clearInterval(app.healthCheckTimer) + app.healthCheckTimer = undefined + } + + // Start shutdown timeout - escalate to SIGKILL if needed + startShutdownTimeout(app) + app.proc?.kill() } @@ -124,7 +179,20 @@ export function updateAppIcon(dir: string, icon: string) { saveApp(dir, pkg) } -const getPort = () => NEXT_PORT++ +const clearTimers = (app: App) => { + if (app.startupTimer) { + clearTimeout(app.startupTimer) + app.startupTimer = undefined + } + if (app.shutdownTimer) { + clearTimeout(app.shutdownTimer) + app.shutdownTimer = undefined + } + if (app.healthCheckTimer) { + clearInterval(app.healthCheckTimer) + app.healthCheckTimer = undefined + } +} const info = (app: App, ...msg: string[]) => { console.log('šŸ¾', `[${app.name}]`, ...msg) @@ -152,6 +220,111 @@ function discoverApps() { } } +function getPort(appName?: string): number { + // Try to return the same port this app used before + if (appName) { + const previousPort = _appPorts.get(appName) + if (previousPort !== undefined) { + // Check if it's still in the available pool + const idx = _availablePorts.indexOf(previousPort) + if (idx !== -1) { + _availablePorts.splice(idx, 1) + return previousPort + } + // Port is in use by another app, fall through to get new one + } + } + + // Get next available port + const port = _availablePorts.shift() + if (port === undefined) { + // Pool exhausted - this shouldn't happen with 100 ports + throw new Error('No available ports') + } + + // Remember this port for the app + if (appName) { + _appPorts.set(appName, port) + } + + return port +} + +async function gracefulShutdown(signal: string) { + if (_shuttingDown) return + _shuttingDown = true + + console.log(`\nšŸ¾ Received ${signal}, shutting down gracefully...`) + + const running = runningApps() + if (running.length === 0) { + console.log('šŸ¾ No apps running, exiting.') + process.exit(0) + } + + console.log(`šŸ¾ Stopping ${running.length} app(s)...`) + + // Stop all running apps + for (const app of running) { + app.manuallyStopped = true + if (app.healthCheckTimer) { + clearInterval(app.healthCheckTimer) + app.healthCheckTimer = undefined + } + app.proc?.kill() + } + + // Wait for all apps to exit with timeout + const shutdownStart = Date.now() + const checkInterval = setInterval(() => { + const stillRunning = runningApps() + if (stillRunning.length === 0) { + clearInterval(checkInterval) + console.log('šŸ¾ All apps stopped, exiting.') + process.exit(0) + } + + // Check for timeout + if (Date.now() - shutdownStart > SHUTDOWN_TIMEOUT) { + clearInterval(checkInterval) + console.log(`šŸ¾ Shutdown timeout, forcing ${stillRunning.length} app(s) to stop...`) + for (const app of stillRunning) { + if (app.proc) { + app.proc.kill(9) // SIGKILL + } + } + // Give a moment for SIGKILL to take effect + setTimeout(() => { + console.log('šŸ¾ Forced shutdown complete, exiting.') + process.exit(1) + }, 500) + } + }, 100) +} + +function handleHealthCheckFailure(app: App) { + app.consecutiveHealthFailures = (app.consecutiveHealthFailures ?? 0) + 1 + info(app, `Health check failed (${app.consecutiveHealthFailures}/${HEALTH_CHECK_FAILURES_BEFORE_RESTART})`) + + if (app.consecutiveHealthFailures >= HEALTH_CHECK_FAILURES_BEFORE_RESTART) { + info(app, 'Too many health check failures, restarting...') + // Clear health check timer before killing + if (app.healthCheckTimer) { + clearInterval(app.healthCheckTimer) + app.healthCheckTimer = undefined + } + // Don't set manuallyStopped - we want auto-restart to kick in + app.proc?.kill() + } +} + +function initPortPool() { + _availablePorts.length = 0 + for (let port = MIN_PORT; port <= MAX_PORT; port++) { + _availablePorts.push(port) + } +} + function isDir(path: string): boolean { try { return statSync(path).isDirectory() @@ -181,8 +354,23 @@ function loadApp(dir: string): LoadResult { } } +function maybeResetBackoff(app: App) { + if (app.started && Date.now() - app.started >= STABLE_RUN_TIME) { + app.restartAttempts = 0 + } +} + +function releasePort(port: number) { + // Return port to pool if not already there + if (!_availablePorts.includes(port)) { + _availablePorts.push(port) + // Keep sorted for predictable allocation + _availablePorts.sort((a, b) => a - b) + } +} + async function runApp(dir: string, port: number) { - const { pkg, error } = loadApp(dir) + const { error } = loadApp(dir) if (error) return const app = _apps.get(dir) @@ -192,8 +380,17 @@ async function runApp(dir: string, port: number) { app.state = 'starting' app.port = port app.logs = [] + app.consecutiveHealthFailures = 0 update() + // Start startup timeout + app.startupTimer = setTimeout(() => { + if (app.state === 'starting') { + info(app, 'Startup timeout, killing process...') + app.proc?.kill() + } + }, STARTUP_TIMEOUT) + const cwd = join(APPS_DIR, dir) const needsInstall = !existsSync(join(cwd, 'node_modules')) @@ -210,12 +407,20 @@ async function runApp(dir: string, port: number) { stderr: 'pipe', }) - // Set state to running + // Clear startup timer and set state to running + if (app.startupTimer) { + clearTimeout(app.startupTimer) + app.startupTimer = undefined + } + app.state = 'running' app.proc = proc app.started = Date.now() update() + // Start health checks + startHealthChecks(app, port) + const streamOutput = async (stream: ReadableStream | null) => { if (!stream) return const reader = stream.getReader() @@ -238,17 +443,33 @@ async function runApp(dir: string, port: number) { // Handle process exit proc.exited.then(code => { + // Clear all timers + clearTimers(app) + + // Check if app was stable before crashing (for backoff reset) + maybeResetBackoff(app) + if (code !== 0) app.logs?.push({ time: Date.now(), text: `Exited with code ${code}` }) else app.logs?.push({ time: Date.now(), text: 'Stopped' }) + // Release port back to pool + if (app.port) { + releasePort(app.port) + } + // Reset to stopped state (or invalid if no longer valid) app.state = isApp(dir) ? 'stopped' : 'invalid' app.proc = undefined app.port = undefined app.started = undefined update() + + // Schedule restart if appropriate + if (shouldAutoRestart(app, code)) { + scheduleRestart(app, dir) + } }) } @@ -257,8 +478,92 @@ function saveApp(dir: string, pkg: any) { writeFileSync(path, JSON.stringify(pkg, null, 2) + '\n') } +function scheduleRestart(app: App, dir: string) { + const attempts = app.restartAttempts ?? 0 + const delayIndex = Math.min(attempts, RESTART_DELAYS.length - 1) + const delay = RESTART_DELAYS[delayIndex]! + + app.restartAttempts = attempts + 1 + app.lastRestartTime = Date.now() + + info(app, `Scheduling restart in ${delay / 1000}s (attempt ${app.restartAttempts})...`) + + setTimeout(() => { + // Double-check conditions before restarting + if (_shuttingDown) return + if (app.manuallyStopped) return + if (app.state !== 'stopped') return + if (!isApp(dir)) return + + info(app, 'Restarting...') + runApp(dir, getPort(dir)) + }, delay) +} + +function setupShutdownHandlers() { + process.on('SIGTERM', () => gracefulShutdown('SIGTERM')) + process.on('SIGINT', () => gracefulShutdown('SIGINT')) +} + +function shouldAutoRestart(app: App, exitCode: number | null): boolean { + // Don't restart during host shutdown + if (_shuttingDown) return false + + // Don't restart if manually stopped + if (app.manuallyStopped) return false + + // Don't restart if app became invalid + if (app.state === 'invalid') return false + + // Only restart on non-zero exit codes (crashes) + if (exitCode === 0) return false + + return true +} + +function startHealthChecks(app: App, port: number) { + app.healthCheckTimer = setInterval(async () => { + if (app.state !== 'running') { + if (app.healthCheckTimer) { + clearInterval(app.healthCheckTimer) + app.healthCheckTimer = undefined + } + return + } + + try { + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), HEALTH_CHECK_TIMEOUT) + + const response = await fetch(`http://localhost:${port}/`, { + signal: controller.signal, + }) + + clearTimeout(timeout) + + if (response.ok) { + // Reset consecutive failures on success + app.consecutiveHealthFailures = 0 + } else { + handleHealthCheckFailure(app) + } + } catch (e) { + handleHealthCheckFailure(app) + } + }, HEALTH_CHECK_INTERVAL) +} + +function startShutdownTimeout(app: App) { + app.shutdownTimer = setTimeout(() => { + if (app.proc && (app.state === 'stopping' || app.state === 'running')) { + info(app, 'Shutdown timeout, sending SIGKILL...') + app.proc.kill(9) // SIGKILL + } + }, SHUTDOWN_TIMEOUT) +} + function watchAppsDir() { - watch(APPS_DIR, { recursive: true }, (event, filename) => { + watch(APPS_DIR, { recursive: true }, (_event, filename) => { if (!filename) return // Extract the app directory name from the path (e.g., "myapp/package.json" -> "myapp") @@ -275,7 +580,7 @@ function watchAppsDir() { _apps.set(dir, { name: dir, state, icon, error }) update() if (!error) { - runApp(dir, getPort()) + runApp(dir, getPort(dir)) } return } @@ -284,6 +589,8 @@ function watchAppsDir() { // check if app was deleted if (!isDir(join(APPS_DIR, dir))) { + clearTimers(app) + if (app.port) releasePort(app.port) _apps.delete(dir) update() return @@ -301,12 +608,13 @@ function watchAppsDir() { // App became valid - start it if stopped if (!error && app.state === 'invalid') { app.state = 'stopped' - runApp(dir, getPort()) + runApp(dir, getPort(dir)) } // App became invalid - stop it if running if (error && app.state === 'running') { app.state = 'invalid' + clearTimers(app) app.proc?.kill() }