toes/src/server/apps.ts
2026-01-30 20:39:01 -08:00

677 lines
18 KiB
TypeScript

import type { App as SharedApp, AppState } from '@types'
import type { Subprocess } from 'bun'
import { DEFAULT_EMOJI } from '@types'
import { existsSync, readdirSync, readFileSync, realpathSync, renameSync, statSync, watch, writeFileSync } from 'fs'
import { join, resolve } from 'path'
export type { AppState } from '@types'
export const APPS_DIR = process.env.APPS_DIR ?? resolve(join(process.env.DATA_DIR ?? '.', 'apps'))
const HEALTH_CHECK_FAILURES_BEFORE_RESTART = 3
const HEALTH_CHECK_INTERVAL = 30000
const HEALTH_CHECK_TIMEOUT = 5000
const MAX_LOGS = 100
const MAX_PORT = 3100
const MIN_PORT = 3001
const RESTART_DELAYS = [1000, 2000, 4000, 8000, 16000, 32000]
const SHUTDOWN_TIMEOUT = 10000
const STABLE_RUN_TIME = 60000
const STARTUP_TIMEOUT = 30000
const _appPorts = new Map<string, number>()
const _apps = new Map<string, App>()
const _availablePorts: number[] = []
const _listeners = new Set<() => void>()
let _shuttingDown = false
export type App = SharedApp & {
consecutiveHealthFailures?: number
healthCheckTimer?: Timer
lastRestartTime?: number
manuallyStopped?: boolean
proc?: Subprocess
restartAttempts?: number
shutdownTimer?: Timer
startupTimer?: Timer
}
type LoadResult = { pkg: any, error?: string }
export const allApps = (): App[] =>
Array.from(_apps.values())
.sort((a, b) => a.name.localeCompare(b.name))
export const getApp = (dir: string): App | undefined =>
_apps.get(dir)
export const runApps = () =>
allAppDirs().filter(isApp).forEach(startApp)
export const runningApps = (): App[] =>
allApps().filter(a => a.state === 'running')
export function initApps() {
initPortPool()
setupShutdownHandlers()
discoverApps()
runApps()
watchAppsDir()
}
export function onChange(cb: () => void) {
_listeners.add(cb)
return () => _listeners.delete(cb)
}
export function removeApp(dir: string) {
const app = _apps.get(dir)
if (!app) return
// Clear all timers
clearTimers(app)
if (app.state === 'running')
app.proc?.kill()
// Release port if assigned
if (app.port) {
releasePort(app.port)
}
_apps.delete(dir)
update()
}
export function renameApp(oldName: string, newName: string): { ok: boolean, error?: string } {
const app = _apps.get(oldName)
if (!app) return { ok: false, error: 'App not found' }
if (_apps.has(newName)) return { ok: false, error: 'An app with that name already exists' }
if (!/^[a-z][a-z0-9-]*$/.test(newName)) {
return { ok: false, error: 'Name must start with a letter and contain only lowercase letters, numbers, and hyphens' }
}
const oldPath = join(APPS_DIR, oldName)
const newPath = join(APPS_DIR, newName)
// Stop the app if running
const wasRunning = app.state === 'running'
if (wasRunning) {
clearTimers(app)
app.proc?.kill()
app.proc = undefined
if (app.port) releasePort(app.port)
app.port = undefined
app.started = undefined
}
try {
renameSync(oldPath, newPath)
} catch (e) {
return { ok: false, error: `Failed to rename directory: ${e instanceof Error ? e.message : String(e)}` }
}
// Transfer port mapping to new name
const oldPort = _appPorts.get(oldName)
if (oldPort !== undefined) {
_appPorts.delete(oldName)
_appPorts.set(newName, oldPort)
}
// Update the internal registry
_apps.delete(oldName)
app.name = newName
app.state = 'stopped'
app.manuallyStopped = false
app.restartAttempts = 0
_apps.set(newName, app)
update()
// Restart if it was running
if (wasRunning) {
startApp(newName)
}
return { ok: true }
}
export function startApp(dir: string) {
const app = _apps.get(dir)
if (!app || app.state !== 'stopped') return
if (!isApp(dir)) return
// Clear manually stopped flag when explicitly starting
app.manuallyStopped = false
runApp(dir, getPort(dir))
}
export async function restartApp(dir: string): Promise<void> {
const app = _apps.get(dir)
if (!app) return
// Stop if running
if (app.state === 'running' || app.state === 'starting') {
stopApp(dir)
// Poll until stopped (with timeout)
const maxWait = 10000 // 10 seconds
const pollInterval = 100
let waited = 0
while (app.state !== 'stopped' && waited < maxWait) {
await new Promise(resolve => setTimeout(resolve, pollInterval))
waited += pollInterval
}
if (app.state !== 'stopped') {
throw new Error(`App ${dir} failed to stop after ${maxWait}ms`)
}
}
// Start the app
startApp(dir)
}
export function stopApp(dir: string) {
const app = _apps.get(dir)
if (!app || app.state !== 'running') return
info(app, 'Stopping...')
app.state = 'stopping'
app.manuallyStopped = true
update()
// Clear health check timer
if (app.healthCheckTimer) {
clearInterval(app.healthCheckTimer)
app.healthCheckTimer = undefined
}
// Start shutdown timeout - escalate to SIGKILL if needed
startShutdownTimeout(app)
app.proc?.kill()
}
export function updateAppIcon(dir: string, icon: string) {
const { pkg, error } = loadApp(dir)
if (error) throw new Error(error)
pkg.toes ??= {}
pkg.toes.icon = icon
saveApp(dir, pkg)
}
const clearTimers = (app: App) => {
if (app.startupTimer) {
clearTimeout(app.startupTimer)
app.startupTimer = undefined
}
if (app.shutdownTimer) {
clearTimeout(app.shutdownTimer)
app.shutdownTimer = undefined
}
if (app.healthCheckTimer) {
clearInterval(app.healthCheckTimer)
app.healthCheckTimer = undefined
}
}
const info = (app: App, ...msg: string[]) => {
console.log('🐾', `[${app.name}]`, ...msg)
app.logs?.push({ time: Date.now(), text: msg.join(' ') })
}
const isApp = (dir: string): boolean =>
!loadApp(dir).error
const update = () => _listeners.forEach(cb => cb())
function allAppDirs() {
return readdirSync(APPS_DIR, { withFileTypes: true })
.filter(e => e.isDirectory() && existsSync(join(APPS_DIR, e.name, 'current')))
.map(e => e.name)
.sort()
}
function discoverApps() {
for (const dir of allAppDirs()) {
const { pkg, error } = loadApp(dir)
const state: AppState = error ? 'invalid' : 'stopped'
const icon = pkg.toes?.icon ?? DEFAULT_EMOJI
const tool = pkg.toes?.tool
_apps.set(dir, { name: dir, state, icon, error, tool })
}
}
function getPort(appName?: string): number {
// Try to return the same port this app used before
if (appName) {
const previousPort = _appPorts.get(appName)
if (previousPort !== undefined) {
// Check if it's still in the available pool
const idx = _availablePorts.indexOf(previousPort)
if (idx !== -1) {
_availablePorts.splice(idx, 1)
return previousPort
}
// Port is in use by another app, fall through to get new one
}
}
// Get next available port
const port = _availablePorts.shift()
if (port === undefined) {
// Pool exhausted - this shouldn't happen with 100 ports
throw new Error('No available ports')
}
// Remember this port for the app
if (appName) {
_appPorts.set(appName, port)
}
return port
}
async function gracefulShutdown(signal: string) {
if (_shuttingDown) return
_shuttingDown = true
console.log(`\n🐾 Received ${signal}, shutting down gracefully...`)
const running = runningApps()
if (running.length === 0) {
console.log('🐾 No apps running, exiting.')
process.exit(0)
}
console.log(`🐾 Stopping ${running.length} app(s)...`)
// Stop all running apps
for (const app of running) {
app.manuallyStopped = true
if (app.healthCheckTimer) {
clearInterval(app.healthCheckTimer)
app.healthCheckTimer = undefined
}
app.proc?.kill()
}
// Wait for all apps to exit with timeout
const shutdownStart = Date.now()
const checkInterval = setInterval(() => {
const stillRunning = runningApps()
if (stillRunning.length === 0) {
clearInterval(checkInterval)
console.log('🐾 All apps stopped, exiting.')
process.exit(0)
}
// Check for timeout
if (Date.now() - shutdownStart > SHUTDOWN_TIMEOUT) {
clearInterval(checkInterval)
console.log(`🐾 Shutdown timeout, forcing ${stillRunning.length} app(s) to stop...`)
for (const app of stillRunning) {
if (app.proc) {
app.proc.kill(9) // SIGKILL
}
}
// Give a moment for SIGKILL to take effect
setTimeout(() => {
console.log('🐾 Forced shutdown complete, exiting.')
process.exit(1)
}, 500)
}
}, 100)
}
function handleHealthCheckFailure(app: App) {
app.consecutiveHealthFailures = (app.consecutiveHealthFailures ?? 0) + 1
info(app, `Health check failed (${app.consecutiveHealthFailures}/${HEALTH_CHECK_FAILURES_BEFORE_RESTART})`)
if (app.consecutiveHealthFailures >= HEALTH_CHECK_FAILURES_BEFORE_RESTART) {
info(app, 'Too many health check failures, restarting...')
// Clear health check timer before killing
if (app.healthCheckTimer) {
clearInterval(app.healthCheckTimer)
app.healthCheckTimer = undefined
}
// Don't set manuallyStopped - we want auto-restart to kick in
app.proc?.kill()
}
}
function initPortPool() {
_availablePorts.length = 0
for (let port = MIN_PORT; port <= MAX_PORT; port++) {
_availablePorts.push(port)
}
}
function isDir(path: string): boolean {
try {
return statSync(path).isDirectory()
} catch {
return false
}
}
function loadApp(dir: string): LoadResult {
try {
const pkgPath = join(APPS_DIR, dir, 'current', 'package.json')
const file = readFileSync(pkgPath, 'utf-8')
try {
const json = JSON.parse(file)
if (json.scripts?.toes) {
return { pkg: json }
} else {
return { pkg: json, error: 'Missing scripts.toes in package.json' }
}
} catch (e) {
const error = `Invalid JSON in package.json: ${e instanceof Error ? e.message : String(e)}`
return { pkg: {}, error }
}
} catch (e) {
return { pkg: {}, error: 'Missing package.json' }
}
}
function maybeResetBackoff(app: App) {
if (app.started && Date.now() - app.started >= STABLE_RUN_TIME) {
app.restartAttempts = 0
}
}
function releasePort(port: number) {
// Return port to pool if not already there
if (!_availablePorts.includes(port)) {
_availablePorts.push(port)
// Keep sorted for predictable allocation
_availablePorts.sort((a, b) => a - b)
}
}
async function runApp(dir: string, port: number) {
const { error } = loadApp(dir)
if (error) return
const app = _apps.get(dir)
if (!app) return
// Set state to starting
app.state = 'starting'
app.port = port
app.logs = []
app.consecutiveHealthFailures = 0
update()
// Start startup timeout
app.startupTimer = setTimeout(() => {
if (app.state === 'starting') {
info(app, 'Startup timeout, killing process...')
app.proc?.kill()
}
}, STARTUP_TIMEOUT)
// Resolve symlink to actual timestamp directory
const currentLink = join(APPS_DIR, dir, 'current')
const cwd = realpathSync(currentLink)
const needsInstall = !existsSync(join(cwd, 'node_modules'))
if (needsInstall) info(app, 'Installing dependencies...')
const install = Bun.spawn(['bun', 'install'], { cwd, stdout: 'pipe', stderr: 'pipe' })
await install.exited
info(app, `Starting on port ${port}...`)
const proc = Bun.spawn(['bun', 'run', 'toes'], {
cwd,
env: { ...process.env, PORT: String(port), NO_AUTOPORT: 'true', APPS_DIR, TOES_URL: `http://localhost:${process.env.PORT || 3000}` },
stdout: 'pipe',
stderr: 'pipe',
})
// Clear startup timer and set state to running
if (app.startupTimer) {
clearTimeout(app.startupTimer)
app.startupTimer = undefined
}
app.state = 'running'
app.proc = proc
app.started = Date.now()
update()
// Start health checks
startHealthChecks(app, port)
const streamOutput = async (stream: ReadableStream<Uint8Array> | null) => {
if (!stream) return
const reader = stream.getReader()
const decoder = new TextDecoder()
while (true) {
const { done, value } = await reader.read()
if (done) break
const chunk = decoder.decode(value)
const lines = chunk.split('\n').map(l => l.trimEnd()).filter(Boolean)
for (const text of lines) {
info(app, text)
app.logs = (app.logs ?? []).slice(-MAX_LOGS)
}
if (lines.length) update()
}
}
streamOutput(proc.stdout)
streamOutput(proc.stderr)
// Handle process exit
proc.exited.then(code => {
// Clear all timers
clearTimers(app)
// Check if app was stable before crashing (for backoff reset)
maybeResetBackoff(app)
if (code !== 0)
app.logs?.push({ time: Date.now(), text: `Exited with code ${code}` })
else
app.logs?.push({ time: Date.now(), text: 'Stopped' })
// Release port back to pool
if (app.port) {
releasePort(app.port)
}
// Reset to stopped state (or invalid if no longer valid)
app.state = isApp(dir) ? 'stopped' : 'invalid'
app.proc = undefined
app.port = undefined
app.started = undefined
update()
// Schedule restart if appropriate
if (shouldAutoRestart(app, code)) {
scheduleRestart(app, dir)
}
})
}
function saveApp(dir: string, pkg: any) {
const path = join(APPS_DIR, dir, 'current', 'package.json')
writeFileSync(path, JSON.stringify(pkg, null, 2) + '\n')
}
function scheduleRestart(app: App, dir: string) {
const attempts = app.restartAttempts ?? 0
const delayIndex = Math.min(attempts, RESTART_DELAYS.length - 1)
const delay = RESTART_DELAYS[delayIndex]!
app.restartAttempts = attempts + 1
app.lastRestartTime = Date.now()
info(app, `Scheduling restart in ${delay / 1000}s (attempt ${app.restartAttempts})...`)
setTimeout(() => {
// Double-check conditions before restarting
if (_shuttingDown) return
if (app.manuallyStopped) return
if (app.state !== 'stopped') return
if (!isApp(dir)) return
info(app, 'Restarting...')
runApp(dir, getPort(dir))
}, delay)
}
function setupShutdownHandlers() {
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'))
process.on('SIGINT', () => gracefulShutdown('SIGINT'))
}
function shouldAutoRestart(app: App, exitCode: number | null): boolean {
// Don't restart during host shutdown
if (_shuttingDown) return false
// Don't restart if manually stopped
if (app.manuallyStopped) return false
// Don't restart if app became invalid
if (app.state === 'invalid') return false
// Only restart on non-zero exit codes (crashes)
if (exitCode === 0) return false
return true
}
function startHealthChecks(app: App, port: number) {
app.healthCheckTimer = setInterval(async () => {
if (app.state !== 'running') {
if (app.healthCheckTimer) {
clearInterval(app.healthCheckTimer)
app.healthCheckTimer = undefined
}
return
}
try {
const controller = new AbortController()
const timeout = setTimeout(() => controller.abort(), HEALTH_CHECK_TIMEOUT)
const response = await fetch(`http://localhost:${port}/`, {
signal: controller.signal,
})
clearTimeout(timeout)
if (response.ok) {
// Reset consecutive failures on success
app.consecutiveHealthFailures = 0
} else {
handleHealthCheckFailure(app)
}
} catch (e) {
handleHealthCheckFailure(app)
}
}, HEALTH_CHECK_INTERVAL)
}
function startShutdownTimeout(app: App) {
app.shutdownTimer = setTimeout(() => {
if (app.proc && (app.state === 'stopping' || app.state === 'running')) {
info(app, 'Shutdown timeout, sending SIGKILL...')
app.proc.kill(9) // SIGKILL
}
}, SHUTDOWN_TIMEOUT)
}
function watchAppsDir() {
watch(APPS_DIR, { recursive: true }, (_event, filename) => {
if (!filename) return
const parts = filename.split('/')
const dir = parts[0]!
// Ignore changes inside old timestamp dirs (but allow current/)
if (parts.length > 2 && parts[1] !== 'current') return
// For versioned apps, only care about changes to "current" directory
if (parts.length === 2 && parts[1] !== 'current' && parts[1] !== 'package.json') return
// Handle new directory appearing
if (!_apps.has(dir)) {
// Make sure the directory actually exists (avoids race with rename)
if (!isDir(join(APPS_DIR, dir))) return
const { pkg, error } = loadApp(dir)
const state: AppState = error ? 'invalid' : 'stopped'
const icon = pkg.toes?.icon
const tool = pkg.toes?.tool
_apps.set(dir, { name: dir, state, icon, error, tool })
update()
if (!error) {
runApp(dir, getPort(dir))
}
return
}
const app = _apps.get(dir)!
// check if app was deleted
if (!isDir(join(APPS_DIR, dir))) {
clearTimers(app)
if (app.port) releasePort(app.port)
_apps.delete(dir)
update()
return
}
// Only care about package.json changes for existing apps
if (!filename.endsWith('package.json')) return
const { pkg, error } = loadApp(dir)
// Update icon, tool, and error from package.json
const iconChanged = app.icon !== pkg.toes?.icon
const toolChanged = app.tool !== pkg.toes?.tool
app.icon = pkg.toes?.icon
app.tool = pkg.toes?.tool
app.error = error
// Broadcast if icon or tool changed
if (iconChanged || toolChanged) update()
// App became valid - start it if stopped
if (!error && app.state === 'invalid') {
app.state = 'stopped'
runApp(dir, getPort(dir))
}
// App became invalid - stop it if running
if (error && app.state === 'running') {
app.state = 'invalid'
clearTimers(app)
app.proc?.kill()
}
// Update state if already stopped/invalid
if (error && app.state === 'stopped') {
app.state = 'invalid'
update()
}
if (!error && app.state === 'invalid') {
app.state = 'stopped'
update()
}
})
}