monster features

src/server/apps.ts

@@ -1,4 +1,4 @@
-import type { App as SharedApp, AppState, LogLine } from '@types'
+import type { App as SharedApp, AppState } from '@types'
 import type { Subprocess } from 'bun'
 import { DEFAULT_EMOJI } from '@types'
 import { existsSync, readdirSync, readFileSync, renameSync, statSync, watch, writeFileSync } from 'fs'
@@ -7,17 +7,37 @@ import { join } from 'path'
 export type { AppState } from '@types'
 
 export const APPS_DIR = join(process.env.DATA_DIR ?? '.', 'apps')
+
+const HEALTH_CHECK_FAILURES_BEFORE_RESTART = 3
+const HEALTH_CHECK_INTERVAL = 30000
+const HEALTH_CHECK_TIMEOUT = 5000
 const MAX_LOGS = 100
+const MAX_PORT = 3100
+const MIN_PORT = 3001
+const RESTART_DELAYS = [1000, 2000, 4000, 8000, 16000, 32000]
+const SHUTDOWN_TIMEOUT = 10000
+const STABLE_RUN_TIME = 60000
+const STARTUP_TIMEOUT = 30000
+
+const _appPorts = new Map<string, number>()
 const _apps = new Map<string, App>()
+const _availablePorts: number[] = []
 const _listeners = new Set<() => void>()
 
-let NEXT_PORT = 3001
+let _shuttingDown = false
 
 export type App = SharedApp & {
+  consecutiveHealthFailures?: number
+  healthCheckTimer?: Timer
+  lastRestartTime?: number
+  manuallyStopped?: boolean
   proc?: Subprocess
+  restartAttempts?: number
+  shutdownTimer?: Timer
+  startupTimer?: Timer
 }
 
-type LoadResult = { pkg: any; error?: string }
+type LoadResult = { pkg: any, error?: string }
 
 export const allApps = (): App[] =>
   Array.from(_apps.values())
@@ -33,6 +53,8 @@ export const runningApps = (): App[] =>
   allApps().filter(a => a.state === 'running')
 
 export function initApps() {
+  initPortPool()
+  setupShutdownHandlers()
   discoverApps()
   runApps()
   watchAppsDir()
@@ -43,25 +65,26 @@ export function onChange(cb: () => void) {
   return () => _listeners.delete(cb)
 }
 
-export function startApp(dir: string) {
-  const app = _apps.get(dir)
-  if (!app || app.state !== 'stopped') return
-  if (!isApp(dir)) return
-  runApp(dir, getPort())
-}
-
 export function removeApp(dir: string) {
   const app = _apps.get(dir)
   if (!app) return
 
+  // Clear all timers
+  clearTimers(app)
+
   if (app.state === 'running')
     app.proc?.kill()
 
+  // Release port if assigned
+  if (app.port) {
+    releasePort(app.port)
+  }
+
   _apps.delete(dir)
   update()
 }
 
-export function renameApp(oldName: string, newName: string): { ok: boolean; error?: string } {
+export function renameApp(oldName: string, newName: string): { ok: boolean, error?: string } {
   const app = _apps.get(oldName)
   if (!app) return { ok: false, error: 'App not found' }
 
@@ -77,8 +100,10 @@ export function renameApp(oldName: string, newName: string): { ok: boolean; erro
   // Stop the app if running
   const wasRunning = app.state === 'running'
   if (wasRunning) {
+    clearTimers(app)
     app.proc?.kill()
     app.proc = undefined
+    if (app.port) releasePort(app.port)
     app.port = undefined
     app.started = undefined
   }
@@ -89,10 +114,19 @@ export function renameApp(oldName: string, newName: string): { ok: boolean; erro
     return { ok: false, error: `Failed to rename directory: ${e instanceof Error ? e.message : String(e)}` }
   }
 
+  // Transfer port mapping to new name
+  const oldPort = _appPorts.get(oldName)
+  if (oldPort !== undefined) {
+    _appPorts.delete(oldName)
+    _appPorts.set(newName, oldPort)
+  }
+
   // Update the internal registry
   _apps.delete(oldName)
   app.name = newName
   app.state = 'stopped'
+  app.manuallyStopped = false
+  app.restartAttempts = 0
   _apps.set(newName, app)
 
   update()
@@ -105,13 +139,34 @@ export function renameApp(oldName: string, newName: string): { ok: boolean; erro
   return { ok: true }
 }
 
+export function startApp(dir: string) {
+  const app = _apps.get(dir)
+  if (!app || app.state !== 'stopped') return
+  if (!isApp(dir)) return
+
+  // Clear manually stopped flag when explicitly starting
+  app.manuallyStopped = false
+  runApp(dir, getPort(dir))
+}
+
 export function stopApp(dir: string) {
   const app = _apps.get(dir)
   if (!app || app.state !== 'running') return
 
   info(app, 'Stopping...')
   app.state = 'stopping'
+  app.manuallyStopped = true
   update()
+
+  // Clear health check timer
+  if (app.healthCheckTimer) {
+    clearInterval(app.healthCheckTimer)
+    app.healthCheckTimer = undefined
+  }
+
+  // Start shutdown timeout - escalate to SIGKILL if needed
+  startShutdownTimeout(app)
+
   app.proc?.kill()
 }
 
@@ -124,7 +179,20 @@ export function updateAppIcon(dir: string, icon: string) {
   saveApp(dir, pkg)
 }
 
-const getPort = () => NEXT_PORT++
+const clearTimers = (app: App) => {
+  if (app.startupTimer) {
+    clearTimeout(app.startupTimer)
+    app.startupTimer = undefined
+  }
+  if (app.shutdownTimer) {
+    clearTimeout(app.shutdownTimer)
+    app.shutdownTimer = undefined
+  }
+  if (app.healthCheckTimer) {
+    clearInterval(app.healthCheckTimer)
+    app.healthCheckTimer = undefined
+  }
+}
 
 const info = (app: App, ...msg: string[]) => {
   console.log('🐾', `[${app.name}]`, ...msg)
@@ -152,6 +220,111 @@ function discoverApps() {
   }
 }
 
+function getPort(appName?: string): number {
+  // Try to return the same port this app used before
+  if (appName) {
+    const previousPort = _appPorts.get(appName)
+    if (previousPort !== undefined) {
+      // Check if it's still in the available pool
+      const idx = _availablePorts.indexOf(previousPort)
+      if (idx !== -1) {
+        _availablePorts.splice(idx, 1)
+        return previousPort
+      }
+      // Port is in use by another app, fall through to get new one
+    }
+  }
+
+  // Get next available port
+  const port = _availablePorts.shift()
+  if (port === undefined) {
+    // Pool exhausted - this shouldn't happen with 100 ports
+    throw new Error('No available ports')
+  }
+
+  // Remember this port for the app
+  if (appName) {
+    _appPorts.set(appName, port)
+  }
+
+  return port
+}
+
+async function gracefulShutdown(signal: string) {
+  if (_shuttingDown) return
+  _shuttingDown = true
+
+  console.log(`\n🐾 Received ${signal}, shutting down gracefully...`)
+
+  const running = runningApps()
+  if (running.length === 0) {
+    console.log('🐾 No apps running, exiting.')
+    process.exit(0)
+  }
+
+  console.log(`🐾 Stopping ${running.length} app(s)...`)
+
+  // Stop all running apps
+  for (const app of running) {
+    app.manuallyStopped = true
+    if (app.healthCheckTimer) {
+      clearInterval(app.healthCheckTimer)
+      app.healthCheckTimer = undefined
+    }
+    app.proc?.kill()
+  }
+
+  // Wait for all apps to exit with timeout
+  const shutdownStart = Date.now()
+  const checkInterval = setInterval(() => {
+    const stillRunning = runningApps()
+    if (stillRunning.length === 0) {
+      clearInterval(checkInterval)
+      console.log('🐾 All apps stopped, exiting.')
+      process.exit(0)
+    }
+
+    // Check for timeout
+    if (Date.now() - shutdownStart > SHUTDOWN_TIMEOUT) {
+      clearInterval(checkInterval)
+      console.log(`🐾 Shutdown timeout, forcing ${stillRunning.length} app(s) to stop...`)
+      for (const app of stillRunning) {
+        if (app.proc) {
+          app.proc.kill(9) // SIGKILL
+        }
+      }
+      // Give a moment for SIGKILL to take effect
+      setTimeout(() => {
+        console.log('🐾 Forced shutdown complete, exiting.')
+        process.exit(1)
+      }, 500)
+    }
+  }, 100)
+}
+
+function handleHealthCheckFailure(app: App) {
+  app.consecutiveHealthFailures = (app.consecutiveHealthFailures ?? 0) + 1
+  info(app, `Health check failed (${app.consecutiveHealthFailures}/${HEALTH_CHECK_FAILURES_BEFORE_RESTART})`)
+
+  if (app.consecutiveHealthFailures >= HEALTH_CHECK_FAILURES_BEFORE_RESTART) {
+    info(app, 'Too many health check failures, restarting...')
+    // Clear health check timer before killing
+    if (app.healthCheckTimer) {
+      clearInterval(app.healthCheckTimer)
+      app.healthCheckTimer = undefined
+    }
+    // Don't set manuallyStopped - we want auto-restart to kick in
+    app.proc?.kill()
+  }
+}
+
+function initPortPool() {
+  _availablePorts.length = 0
+  for (let port = MIN_PORT; port <= MAX_PORT; port++) {
+    _availablePorts.push(port)
+  }
+}
+
 function isDir(path: string): boolean {
   try {
     return statSync(path).isDirectory()
@@ -181,8 +354,23 @@ function loadApp(dir: string): LoadResult {
   }
 }
 
+function maybeResetBackoff(app: App) {
+  if (app.started && Date.now() - app.started >= STABLE_RUN_TIME) {
+    app.restartAttempts = 0
+  }
+}
+
+function releasePort(port: number) {
+  // Return port to pool if not already there
+  if (!_availablePorts.includes(port)) {
+    _availablePorts.push(port)
+    // Keep sorted for predictable allocation
+    _availablePorts.sort((a, b) => a - b)
+  }
+}
+
 async function runApp(dir: string, port: number) {
-  const { pkg, error } = loadApp(dir)
+  const { error } = loadApp(dir)
   if (error) return
 
   const app = _apps.get(dir)
@@ -192,8 +380,17 @@ async function runApp(dir: string, port: number) {
   app.state = 'starting'
   app.port = port
   app.logs = []
+  app.consecutiveHealthFailures = 0
   update()
 
+  // Start startup timeout
+  app.startupTimer = setTimeout(() => {
+    if (app.state === 'starting') {
+      info(app, 'Startup timeout, killing process...')
+      app.proc?.kill()
+    }
+  }, STARTUP_TIMEOUT)
+
   const cwd = join(APPS_DIR, dir)
 
   const needsInstall = !existsSync(join(cwd, 'node_modules'))
@@ -210,12 +407,20 @@ async function runApp(dir: string, port: number) {
     stderr: 'pipe',
   })
 
-  // Set state to running
+  // Clear startup timer and set state to running
+  if (app.startupTimer) {
+    clearTimeout(app.startupTimer)
+    app.startupTimer = undefined
+  }
+
   app.state = 'running'
   app.proc = proc
   app.started = Date.now()
   update()
 
+  // Start health checks
+  startHealthChecks(app, port)
+
   const streamOutput = async (stream: ReadableStream<Uint8Array> | null) => {
     if (!stream) return
     const reader = stream.getReader()
@@ -238,17 +443,33 @@ async function runApp(dir: string, port: number) {
 
   // Handle process exit
   proc.exited.then(code => {
+    // Clear all timers
+    clearTimers(app)
+
+    // Check if app was stable before crashing (for backoff reset)
+    maybeResetBackoff(app)
+
     if (code !== 0)
       app.logs?.push({ time: Date.now(), text: `Exited with code ${code}` })
     else
       app.logs?.push({ time: Date.now(), text: 'Stopped' })
 
+    // Release port back to pool
+    if (app.port) {
+      releasePort(app.port)
+    }
+
     // Reset to stopped state (or invalid if no longer valid)
     app.state = isApp(dir) ? 'stopped' : 'invalid'
     app.proc = undefined
     app.port = undefined
     app.started = undefined
     update()
+
+    // Schedule restart if appropriate
+    if (shouldAutoRestart(app, code)) {
+      scheduleRestart(app, dir)
+    }
   })
 }
 
@@ -257,8 +478,92 @@ function saveApp(dir: string, pkg: any) {
   writeFileSync(path, JSON.stringify(pkg, null, 2) + '\n')
 }
 
+function scheduleRestart(app: App, dir: string) {
+  const attempts = app.restartAttempts ?? 0
+  const delayIndex = Math.min(attempts, RESTART_DELAYS.length - 1)
+  const delay = RESTART_DELAYS[delayIndex]!
+
+  app.restartAttempts = attempts + 1
+  app.lastRestartTime = Date.now()
+
+  info(app, `Scheduling restart in ${delay / 1000}s (attempt ${app.restartAttempts})...`)
+
+  setTimeout(() => {
+    // Double-check conditions before restarting
+    if (_shuttingDown) return
+    if (app.manuallyStopped) return
+    if (app.state !== 'stopped') return
+    if (!isApp(dir)) return
+
+    info(app, 'Restarting...')
+    runApp(dir, getPort(dir))
+  }, delay)
+}
+
+function setupShutdownHandlers() {
+  process.on('SIGTERM', () => gracefulShutdown('SIGTERM'))
+  process.on('SIGINT', () => gracefulShutdown('SIGINT'))
+}
+
+function shouldAutoRestart(app: App, exitCode: number | null): boolean {
+  // Don't restart during host shutdown
+  if (_shuttingDown) return false
+
+  // Don't restart if manually stopped
+  if (app.manuallyStopped) return false
+
+  // Don't restart if app became invalid
+  if (app.state === 'invalid') return false
+
+  // Only restart on non-zero exit codes (crashes)
+  if (exitCode === 0) return false
+
+  return true
+}
+
+function startHealthChecks(app: App, port: number) {
+  app.healthCheckTimer = setInterval(async () => {
+    if (app.state !== 'running') {
+      if (app.healthCheckTimer) {
+        clearInterval(app.healthCheckTimer)
+        app.healthCheckTimer = undefined
+      }
+      return
+    }
+
+    try {
+      const controller = new AbortController()
+      const timeout = setTimeout(() => controller.abort(), HEALTH_CHECK_TIMEOUT)
+
+      const response = await fetch(`http://localhost:${port}/`, {
+        signal: controller.signal,
+      })
+
+      clearTimeout(timeout)
+
+      if (response.ok) {
+        // Reset consecutive failures on success
+        app.consecutiveHealthFailures = 0
+      } else {
+        handleHealthCheckFailure(app)
+      }
+    } catch (e) {
+      handleHealthCheckFailure(app)
+    }
+  }, HEALTH_CHECK_INTERVAL)
+}
+
+function startShutdownTimeout(app: App) {
+  app.shutdownTimer = setTimeout(() => {
+    if (app.proc && (app.state === 'stopping' || app.state === 'running')) {
+      info(app, 'Shutdown timeout, sending SIGKILL...')
+      app.proc.kill(9) // SIGKILL
+    }
+  }, SHUTDOWN_TIMEOUT)
+}
+
 function watchAppsDir() {
-  watch(APPS_DIR, { recursive: true }, (event, filename) => {
+  watch(APPS_DIR, { recursive: true }, (_event, filename) => {
     if (!filename) return
 
     // Extract the app directory name from the path (e.g., "myapp/package.json" -> "myapp")
@@ -275,7 +580,7 @@ function watchAppsDir() {
       _apps.set(dir, { name: dir, state, icon, error })
       update()
       if (!error) {
-        runApp(dir, getPort())
+        runApp(dir, getPort(dir))
       }
       return
     }
@@ -284,6 +589,8 @@ function watchAppsDir() {
 
     // check if app was deleted
     if (!isDir(join(APPS_DIR, dir))) {
+      clearTimers(app)
+      if (app.port) releasePort(app.port)
       _apps.delete(dir)
       update()
       return
@@ -301,12 +608,13 @@ function watchAppsDir() {
     // App became valid - start it if stopped
     if (!error && app.state === 'invalid') {
       app.state = 'stopped'
-      runApp(dir, getPort())
+      runApp(dir, getPort(dir))
     }
 
     // App became invalid - stop it if running
     if (error && app.state === 'running') {
       app.state = 'invalid'
+      clearTimers(app)
       app.proc?.kill()
     }