677 lines
18 KiB
TypeScript
677 lines
18 KiB
TypeScript
import type { App as SharedApp, AppState } from '@types'
|
|
import type { Subprocess } from 'bun'
|
|
import { DEFAULT_EMOJI } from '@types'
|
|
import { existsSync, readdirSync, readFileSync, realpathSync, renameSync, statSync, watch, writeFileSync } from 'fs'
|
|
import { join, resolve } from 'path'
|
|
|
|
export type { AppState } from '@types'
|
|
|
|
export const APPS_DIR = process.env.APPS_DIR ?? resolve(join(process.env.DATA_DIR ?? '.', 'apps'))
|
|
|
|
const HEALTH_CHECK_FAILURES_BEFORE_RESTART = 3
|
|
const HEALTH_CHECK_INTERVAL = 30000
|
|
const HEALTH_CHECK_TIMEOUT = 5000
|
|
const MAX_LOGS = 100
|
|
const MAX_PORT = 3100
|
|
const MIN_PORT = 3001
|
|
const RESTART_DELAYS = [1000, 2000, 4000, 8000, 16000, 32000]
|
|
const SHUTDOWN_TIMEOUT = 10000
|
|
const STABLE_RUN_TIME = 60000
|
|
const STARTUP_TIMEOUT = 30000
|
|
|
|
const _appPorts = new Map<string, number>()
|
|
const _apps = new Map<string, App>()
|
|
const _availablePorts: number[] = []
|
|
const _listeners = new Set<() => void>()
|
|
|
|
let _shuttingDown = false
|
|
|
|
export type App = SharedApp & {
|
|
consecutiveHealthFailures?: number
|
|
healthCheckTimer?: Timer
|
|
lastRestartTime?: number
|
|
manuallyStopped?: boolean
|
|
proc?: Subprocess
|
|
restartAttempts?: number
|
|
shutdownTimer?: Timer
|
|
startupTimer?: Timer
|
|
}
|
|
|
|
type LoadResult = { pkg: any, error?: string }
|
|
|
|
export const allApps = (): App[] =>
|
|
Array.from(_apps.values())
|
|
.sort((a, b) => a.name.localeCompare(b.name))
|
|
|
|
export const getApp = (dir: string): App | undefined =>
|
|
_apps.get(dir)
|
|
|
|
export const runApps = () =>
|
|
allAppDirs().filter(isApp).forEach(startApp)
|
|
|
|
export const runningApps = (): App[] =>
|
|
allApps().filter(a => a.state === 'running')
|
|
|
|
export function initApps() {
|
|
initPortPool()
|
|
setupShutdownHandlers()
|
|
discoverApps()
|
|
runApps()
|
|
watchAppsDir()
|
|
}
|
|
|
|
export function onChange(cb: () => void) {
|
|
_listeners.add(cb)
|
|
return () => _listeners.delete(cb)
|
|
}
|
|
|
|
export function removeApp(dir: string) {
|
|
const app = _apps.get(dir)
|
|
if (!app) return
|
|
|
|
// Clear all timers
|
|
clearTimers(app)
|
|
|
|
if (app.state === 'running')
|
|
app.proc?.kill()
|
|
|
|
// Release port if assigned
|
|
if (app.port) {
|
|
releasePort(app.port)
|
|
}
|
|
|
|
_apps.delete(dir)
|
|
update()
|
|
}
|
|
|
|
export function renameApp(oldName: string, newName: string): { ok: boolean, error?: string } {
|
|
const app = _apps.get(oldName)
|
|
if (!app) return { ok: false, error: 'App not found' }
|
|
|
|
if (_apps.has(newName)) return { ok: false, error: 'An app with that name already exists' }
|
|
|
|
if (!/^[a-z][a-z0-9-]*$/.test(newName)) {
|
|
return { ok: false, error: 'Name must start with a letter and contain only lowercase letters, numbers, and hyphens' }
|
|
}
|
|
|
|
const oldPath = join(APPS_DIR, oldName)
|
|
const newPath = join(APPS_DIR, newName)
|
|
|
|
// Stop the app if running
|
|
const wasRunning = app.state === 'running'
|
|
if (wasRunning) {
|
|
clearTimers(app)
|
|
app.proc?.kill()
|
|
app.proc = undefined
|
|
if (app.port) releasePort(app.port)
|
|
app.port = undefined
|
|
app.started = undefined
|
|
}
|
|
|
|
try {
|
|
renameSync(oldPath, newPath)
|
|
} catch (e) {
|
|
return { ok: false, error: `Failed to rename directory: ${e instanceof Error ? e.message : String(e)}` }
|
|
}
|
|
|
|
// Transfer port mapping to new name
|
|
const oldPort = _appPorts.get(oldName)
|
|
if (oldPort !== undefined) {
|
|
_appPorts.delete(oldName)
|
|
_appPorts.set(newName, oldPort)
|
|
}
|
|
|
|
// Update the internal registry
|
|
_apps.delete(oldName)
|
|
app.name = newName
|
|
app.state = 'stopped'
|
|
app.manuallyStopped = false
|
|
app.restartAttempts = 0
|
|
_apps.set(newName, app)
|
|
|
|
update()
|
|
|
|
// Restart if it was running
|
|
if (wasRunning) {
|
|
startApp(newName)
|
|
}
|
|
|
|
return { ok: true }
|
|
}
|
|
|
|
export function startApp(dir: string) {
|
|
const app = _apps.get(dir)
|
|
if (!app || app.state !== 'stopped') return
|
|
if (!isApp(dir)) return
|
|
|
|
// Clear manually stopped flag when explicitly starting
|
|
app.manuallyStopped = false
|
|
runApp(dir, getPort(dir))
|
|
}
|
|
|
|
export async function restartApp(dir: string): Promise<void> {
|
|
const app = _apps.get(dir)
|
|
if (!app) return
|
|
|
|
// Stop if running
|
|
if (app.state === 'running' || app.state === 'starting') {
|
|
stopApp(dir)
|
|
|
|
// Poll until stopped (with timeout)
|
|
const maxWait = 10000 // 10 seconds
|
|
const pollInterval = 100
|
|
let waited = 0
|
|
|
|
while (app.state !== 'stopped' && waited < maxWait) {
|
|
await new Promise(resolve => setTimeout(resolve, pollInterval))
|
|
waited += pollInterval
|
|
}
|
|
|
|
if (app.state !== 'stopped') {
|
|
throw new Error(`App ${dir} failed to stop after ${maxWait}ms`)
|
|
}
|
|
}
|
|
|
|
// Start the app
|
|
startApp(dir)
|
|
}
|
|
|
|
export function stopApp(dir: string) {
|
|
const app = _apps.get(dir)
|
|
if (!app || app.state !== 'running') return
|
|
|
|
info(app, 'Stopping...')
|
|
app.state = 'stopping'
|
|
app.manuallyStopped = true
|
|
update()
|
|
|
|
// Clear health check timer
|
|
if (app.healthCheckTimer) {
|
|
clearInterval(app.healthCheckTimer)
|
|
app.healthCheckTimer = undefined
|
|
}
|
|
|
|
// Start shutdown timeout - escalate to SIGKILL if needed
|
|
startShutdownTimeout(app)
|
|
|
|
app.proc?.kill()
|
|
}
|
|
|
|
export function updateAppIcon(dir: string, icon: string) {
|
|
const { pkg, error } = loadApp(dir)
|
|
if (error) throw new Error(error)
|
|
|
|
pkg.toes ??= {}
|
|
pkg.toes.icon = icon
|
|
saveApp(dir, pkg)
|
|
}
|
|
|
|
const clearTimers = (app: App) => {
|
|
if (app.startupTimer) {
|
|
clearTimeout(app.startupTimer)
|
|
app.startupTimer = undefined
|
|
}
|
|
if (app.shutdownTimer) {
|
|
clearTimeout(app.shutdownTimer)
|
|
app.shutdownTimer = undefined
|
|
}
|
|
if (app.healthCheckTimer) {
|
|
clearInterval(app.healthCheckTimer)
|
|
app.healthCheckTimer = undefined
|
|
}
|
|
}
|
|
|
|
const info = (app: App, ...msg: string[]) => {
|
|
console.log('🐾', `[${app.name}]`, ...msg)
|
|
app.logs?.push({ time: Date.now(), text: msg.join(' ') })
|
|
}
|
|
|
|
const isApp = (dir: string): boolean =>
|
|
!loadApp(dir).error
|
|
|
|
const update = () => _listeners.forEach(cb => cb())
|
|
|
|
function allAppDirs() {
|
|
return readdirSync(APPS_DIR, { withFileTypes: true })
|
|
.filter(e => e.isDirectory() && existsSync(join(APPS_DIR, e.name, 'current')))
|
|
.map(e => e.name)
|
|
.sort()
|
|
}
|
|
|
|
function discoverApps() {
|
|
for (const dir of allAppDirs()) {
|
|
const { pkg, error } = loadApp(dir)
|
|
const state: AppState = error ? 'invalid' : 'stopped'
|
|
const icon = pkg.toes?.icon ?? DEFAULT_EMOJI
|
|
const tool = pkg.toes?.tool
|
|
_apps.set(dir, { name: dir, state, icon, error, tool })
|
|
}
|
|
}
|
|
|
|
function getPort(appName?: string): number {
|
|
// Try to return the same port this app used before
|
|
if (appName) {
|
|
const previousPort = _appPorts.get(appName)
|
|
if (previousPort !== undefined) {
|
|
// Check if it's still in the available pool
|
|
const idx = _availablePorts.indexOf(previousPort)
|
|
if (idx !== -1) {
|
|
_availablePorts.splice(idx, 1)
|
|
return previousPort
|
|
}
|
|
// Port is in use by another app, fall through to get new one
|
|
}
|
|
}
|
|
|
|
// Get next available port
|
|
const port = _availablePorts.shift()
|
|
if (port === undefined) {
|
|
// Pool exhausted - this shouldn't happen with 100 ports
|
|
throw new Error('No available ports')
|
|
}
|
|
|
|
// Remember this port for the app
|
|
if (appName) {
|
|
_appPorts.set(appName, port)
|
|
}
|
|
|
|
return port
|
|
}
|
|
|
|
async function gracefulShutdown(signal: string) {
|
|
if (_shuttingDown) return
|
|
_shuttingDown = true
|
|
|
|
console.log(`\n🐾 Received ${signal}, shutting down gracefully...`)
|
|
|
|
const running = runningApps()
|
|
if (running.length === 0) {
|
|
console.log('🐾 No apps running, exiting.')
|
|
process.exit(0)
|
|
}
|
|
|
|
console.log(`🐾 Stopping ${running.length} app(s)...`)
|
|
|
|
// Stop all running apps
|
|
for (const app of running) {
|
|
app.manuallyStopped = true
|
|
if (app.healthCheckTimer) {
|
|
clearInterval(app.healthCheckTimer)
|
|
app.healthCheckTimer = undefined
|
|
}
|
|
app.proc?.kill()
|
|
}
|
|
|
|
// Wait for all apps to exit with timeout
|
|
const shutdownStart = Date.now()
|
|
const checkInterval = setInterval(() => {
|
|
const stillRunning = runningApps()
|
|
if (stillRunning.length === 0) {
|
|
clearInterval(checkInterval)
|
|
console.log('🐾 All apps stopped, exiting.')
|
|
process.exit(0)
|
|
}
|
|
|
|
// Check for timeout
|
|
if (Date.now() - shutdownStart > SHUTDOWN_TIMEOUT) {
|
|
clearInterval(checkInterval)
|
|
console.log(`🐾 Shutdown timeout, forcing ${stillRunning.length} app(s) to stop...`)
|
|
for (const app of stillRunning) {
|
|
if (app.proc) {
|
|
app.proc.kill(9) // SIGKILL
|
|
}
|
|
}
|
|
// Give a moment for SIGKILL to take effect
|
|
setTimeout(() => {
|
|
console.log('🐾 Forced shutdown complete, exiting.')
|
|
process.exit(1)
|
|
}, 500)
|
|
}
|
|
}, 100)
|
|
}
|
|
|
|
function handleHealthCheckFailure(app: App) {
|
|
app.consecutiveHealthFailures = (app.consecutiveHealthFailures ?? 0) + 1
|
|
info(app, `Health check failed (${app.consecutiveHealthFailures}/${HEALTH_CHECK_FAILURES_BEFORE_RESTART})`)
|
|
|
|
if (app.consecutiveHealthFailures >= HEALTH_CHECK_FAILURES_BEFORE_RESTART) {
|
|
info(app, 'Too many health check failures, restarting...')
|
|
// Clear health check timer before killing
|
|
if (app.healthCheckTimer) {
|
|
clearInterval(app.healthCheckTimer)
|
|
app.healthCheckTimer = undefined
|
|
}
|
|
// Don't set manuallyStopped - we want auto-restart to kick in
|
|
app.proc?.kill()
|
|
}
|
|
}
|
|
|
|
function initPortPool() {
|
|
_availablePorts.length = 0
|
|
for (let port = MIN_PORT; port <= MAX_PORT; port++) {
|
|
_availablePorts.push(port)
|
|
}
|
|
}
|
|
|
|
function isDir(path: string): boolean {
|
|
try {
|
|
return statSync(path).isDirectory()
|
|
} catch {
|
|
return false
|
|
}
|
|
}
|
|
|
|
function loadApp(dir: string): LoadResult {
|
|
try {
|
|
const pkgPath = join(APPS_DIR, dir, 'current', 'package.json')
|
|
const file = readFileSync(pkgPath, 'utf-8')
|
|
|
|
try {
|
|
const json = JSON.parse(file)
|
|
|
|
if (json.scripts?.toes) {
|
|
return { pkg: json }
|
|
} else {
|
|
return { pkg: json, error: 'Missing scripts.toes in package.json' }
|
|
}
|
|
} catch (e) {
|
|
const error = `Invalid JSON in package.json: ${e instanceof Error ? e.message : String(e)}`
|
|
return { pkg: {}, error }
|
|
}
|
|
} catch (e) {
|
|
return { pkg: {}, error: 'Missing package.json' }
|
|
}
|
|
}
|
|
|
|
function maybeResetBackoff(app: App) {
|
|
if (app.started && Date.now() - app.started >= STABLE_RUN_TIME) {
|
|
app.restartAttempts = 0
|
|
}
|
|
}
|
|
|
|
function releasePort(port: number) {
|
|
// Return port to pool if not already there
|
|
if (!_availablePorts.includes(port)) {
|
|
_availablePorts.push(port)
|
|
// Keep sorted for predictable allocation
|
|
_availablePorts.sort((a, b) => a - b)
|
|
}
|
|
}
|
|
|
|
async function runApp(dir: string, port: number) {
|
|
const { error } = loadApp(dir)
|
|
if (error) return
|
|
|
|
const app = _apps.get(dir)
|
|
if (!app) return
|
|
|
|
// Set state to starting
|
|
app.state = 'starting'
|
|
app.port = port
|
|
app.logs = []
|
|
app.consecutiveHealthFailures = 0
|
|
update()
|
|
|
|
// Start startup timeout
|
|
app.startupTimer = setTimeout(() => {
|
|
if (app.state === 'starting') {
|
|
info(app, 'Startup timeout, killing process...')
|
|
app.proc?.kill()
|
|
}
|
|
}, STARTUP_TIMEOUT)
|
|
|
|
// Resolve symlink to actual timestamp directory
|
|
const currentLink = join(APPS_DIR, dir, 'current')
|
|
const cwd = realpathSync(currentLink)
|
|
|
|
const needsInstall = !existsSync(join(cwd, 'node_modules'))
|
|
if (needsInstall) info(app, 'Installing dependencies...')
|
|
const install = Bun.spawn(['bun', 'install'], { cwd, stdout: 'pipe', stderr: 'pipe' })
|
|
await install.exited
|
|
|
|
info(app, `Starting on port ${port}...`)
|
|
|
|
const proc = Bun.spawn(['bun', 'run', 'toes'], {
|
|
cwd,
|
|
env: { ...process.env, PORT: String(port), NO_AUTOPORT: 'true', APPS_DIR, TOES_URL: `http://localhost:${process.env.PORT || 3000}` },
|
|
stdout: 'pipe',
|
|
stderr: 'pipe',
|
|
})
|
|
|
|
// Clear startup timer and set state to running
|
|
if (app.startupTimer) {
|
|
clearTimeout(app.startupTimer)
|
|
app.startupTimer = undefined
|
|
}
|
|
|
|
app.state = 'running'
|
|
app.proc = proc
|
|
app.started = Date.now()
|
|
update()
|
|
|
|
// Start health checks
|
|
startHealthChecks(app, port)
|
|
|
|
const streamOutput = async (stream: ReadableStream<Uint8Array> | null) => {
|
|
if (!stream) return
|
|
const reader = stream.getReader()
|
|
const decoder = new TextDecoder()
|
|
while (true) {
|
|
const { done, value } = await reader.read()
|
|
if (done) break
|
|
const chunk = decoder.decode(value)
|
|
const lines = chunk.split('\n').map(l => l.trimEnd()).filter(Boolean)
|
|
for (const text of lines) {
|
|
info(app, text)
|
|
app.logs = (app.logs ?? []).slice(-MAX_LOGS)
|
|
}
|
|
if (lines.length) update()
|
|
}
|
|
}
|
|
|
|
streamOutput(proc.stdout)
|
|
streamOutput(proc.stderr)
|
|
|
|
// Handle process exit
|
|
proc.exited.then(code => {
|
|
// Clear all timers
|
|
clearTimers(app)
|
|
|
|
// Check if app was stable before crashing (for backoff reset)
|
|
maybeResetBackoff(app)
|
|
|
|
if (code !== 0)
|
|
app.logs?.push({ time: Date.now(), text: `Exited with code ${code}` })
|
|
else
|
|
app.logs?.push({ time: Date.now(), text: 'Stopped' })
|
|
|
|
// Release port back to pool
|
|
if (app.port) {
|
|
releasePort(app.port)
|
|
}
|
|
|
|
// Reset to stopped state (or invalid if no longer valid)
|
|
app.state = isApp(dir) ? 'stopped' : 'invalid'
|
|
app.proc = undefined
|
|
app.port = undefined
|
|
app.started = undefined
|
|
update()
|
|
|
|
// Schedule restart if appropriate
|
|
if (shouldAutoRestart(app, code)) {
|
|
scheduleRestart(app, dir)
|
|
}
|
|
})
|
|
}
|
|
|
|
function saveApp(dir: string, pkg: any) {
|
|
const path = join(APPS_DIR, dir, 'current', 'package.json')
|
|
writeFileSync(path, JSON.stringify(pkg, null, 2) + '\n')
|
|
}
|
|
|
|
function scheduleRestart(app: App, dir: string) {
|
|
const attempts = app.restartAttempts ?? 0
|
|
const delayIndex = Math.min(attempts, RESTART_DELAYS.length - 1)
|
|
const delay = RESTART_DELAYS[delayIndex]!
|
|
|
|
app.restartAttempts = attempts + 1
|
|
app.lastRestartTime = Date.now()
|
|
|
|
info(app, `Scheduling restart in ${delay / 1000}s (attempt ${app.restartAttempts})...`)
|
|
|
|
setTimeout(() => {
|
|
// Double-check conditions before restarting
|
|
if (_shuttingDown) return
|
|
if (app.manuallyStopped) return
|
|
if (app.state !== 'stopped') return
|
|
if (!isApp(dir)) return
|
|
|
|
info(app, 'Restarting...')
|
|
runApp(dir, getPort(dir))
|
|
}, delay)
|
|
}
|
|
|
|
function setupShutdownHandlers() {
|
|
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'))
|
|
process.on('SIGINT', () => gracefulShutdown('SIGINT'))
|
|
}
|
|
|
|
function shouldAutoRestart(app: App, exitCode: number | null): boolean {
|
|
// Don't restart during host shutdown
|
|
if (_shuttingDown) return false
|
|
|
|
// Don't restart if manually stopped
|
|
if (app.manuallyStopped) return false
|
|
|
|
// Don't restart if app became invalid
|
|
if (app.state === 'invalid') return false
|
|
|
|
// Only restart on non-zero exit codes (crashes)
|
|
if (exitCode === 0) return false
|
|
|
|
return true
|
|
}
|
|
|
|
function startHealthChecks(app: App, port: number) {
|
|
app.healthCheckTimer = setInterval(async () => {
|
|
if (app.state !== 'running') {
|
|
if (app.healthCheckTimer) {
|
|
clearInterval(app.healthCheckTimer)
|
|
app.healthCheckTimer = undefined
|
|
}
|
|
return
|
|
}
|
|
|
|
try {
|
|
const controller = new AbortController()
|
|
const timeout = setTimeout(() => controller.abort(), HEALTH_CHECK_TIMEOUT)
|
|
|
|
const response = await fetch(`http://localhost:${port}/`, {
|
|
signal: controller.signal,
|
|
})
|
|
|
|
clearTimeout(timeout)
|
|
|
|
if (response.ok) {
|
|
// Reset consecutive failures on success
|
|
app.consecutiveHealthFailures = 0
|
|
} else {
|
|
handleHealthCheckFailure(app)
|
|
}
|
|
} catch (e) {
|
|
handleHealthCheckFailure(app)
|
|
}
|
|
}, HEALTH_CHECK_INTERVAL)
|
|
}
|
|
|
|
function startShutdownTimeout(app: App) {
|
|
app.shutdownTimer = setTimeout(() => {
|
|
if (app.proc && (app.state === 'stopping' || app.state === 'running')) {
|
|
info(app, 'Shutdown timeout, sending SIGKILL...')
|
|
app.proc.kill(9) // SIGKILL
|
|
}
|
|
}, SHUTDOWN_TIMEOUT)
|
|
}
|
|
|
|
function watchAppsDir() {
|
|
watch(APPS_DIR, { recursive: true }, (_event, filename) => {
|
|
if (!filename) return
|
|
|
|
const parts = filename.split('/')
|
|
const dir = parts[0]!
|
|
|
|
// Ignore changes inside old timestamp dirs (but allow current/)
|
|
if (parts.length > 2 && parts[1] !== 'current') return
|
|
|
|
// For versioned apps, only care about changes to "current" directory
|
|
if (parts.length === 2 && parts[1] !== 'current' && parts[1] !== 'package.json') return
|
|
|
|
// Handle new directory appearing
|
|
if (!_apps.has(dir)) {
|
|
// Make sure the directory actually exists (avoids race with rename)
|
|
if (!isDir(join(APPS_DIR, dir))) return
|
|
|
|
const { pkg, error } = loadApp(dir)
|
|
const state: AppState = error ? 'invalid' : 'stopped'
|
|
const icon = pkg.toes?.icon
|
|
const tool = pkg.toes?.tool
|
|
_apps.set(dir, { name: dir, state, icon, error, tool })
|
|
update()
|
|
if (!error) {
|
|
runApp(dir, getPort(dir))
|
|
}
|
|
return
|
|
}
|
|
|
|
const app = _apps.get(dir)!
|
|
|
|
// check if app was deleted
|
|
if (!isDir(join(APPS_DIR, dir))) {
|
|
clearTimers(app)
|
|
if (app.port) releasePort(app.port)
|
|
_apps.delete(dir)
|
|
update()
|
|
return
|
|
}
|
|
|
|
// Only care about package.json changes for existing apps
|
|
if (!filename.endsWith('package.json')) return
|
|
|
|
const { pkg, error } = loadApp(dir)
|
|
|
|
// Update icon, tool, and error from package.json
|
|
const iconChanged = app.icon !== pkg.toes?.icon
|
|
const toolChanged = app.tool !== pkg.toes?.tool
|
|
app.icon = pkg.toes?.icon
|
|
app.tool = pkg.toes?.tool
|
|
app.error = error
|
|
|
|
// Broadcast if icon or tool changed
|
|
if (iconChanged || toolChanged) update()
|
|
|
|
// App became valid - start it if stopped
|
|
if (!error && app.state === 'invalid') {
|
|
app.state = 'stopped'
|
|
runApp(dir, getPort(dir))
|
|
}
|
|
|
|
// App became invalid - stop it if running
|
|
if (error && app.state === 'running') {
|
|
app.state = 'invalid'
|
|
clearTimers(app)
|
|
app.proc?.kill()
|
|
}
|
|
|
|
// Update state if already stopped/invalid
|
|
if (error && app.state === 'stopped') {
|
|
app.state = 'invalid'
|
|
update()
|
|
}
|
|
|
|
if (!error && app.state === 'invalid') {
|
|
app.state = 'stopped'
|
|
update()
|
|
}
|
|
})
|
|
}
|