toes/src/server/api/system.ts
Chris Wanstrath 195be426f1 Convert getAppMetrics to async and replace spawnSync with async Bun.spawn
spawnSync blocks the event loop while waiting for ps and du, which
stalls the SSE metrics stream and other requests. Running these
concurrently with async spawn (and Promise.all for du) keeps the
server responsive under load.
2026-03-16 16:32:17 -07:00

335 lines
9.3 KiB
TypeScript

import { allApps, APPS_DIR, onChange } from '$apps'
import { onHostLog } from '../tui'
import { Hype } from '@because/hype'
import { cpus, freemem, platform, totalmem } from 'os'
import { join } from 'path'
import { readFileSync, statfsSync } from 'fs'
export interface AppMetrics {
cpu: number
mem: number
disk: number
}
export interface SystemMetrics {
cpu: number // 0-100
ram: { used: number, total: number, percent: number }
disk: { used: number, total: number, percent: number }
apps: Record<string, AppMetrics>
}
export interface UnifiedLogLine {
time: number
app: string
text: string
}
const router = Hype.router()
// Unified log buffer
const unifiedLogs: UnifiedLogLine[] = []
const MAX_UNIFIED_LOGS = 200
const unifiedLogListeners = new Set<(line: UnifiedLogLine) => void>()
// Track last log counts per app for delta detection
const lastLogCounts = new Map<string, number>()
// CPU tracking
let lastCpuTimes: { idle: number, total: number } | null = null
function getCpuUsage(): number {
const cpuList = cpus()
let idle = 0
let total = 0
for (const cpu of cpuList) {
idle += cpu.times.idle
total += cpu.times.user + cpu.times.nice + cpu.times.sys + cpu.times.idle + cpu.times.irq
}
if (!lastCpuTimes) {
lastCpuTimes = { idle, total }
return 0
}
const idleDiff = idle - lastCpuTimes.idle
const totalDiff = total - lastCpuTimes.total
lastCpuTimes = { idle, total }
if (totalDiff === 0) return 0
return Math.round((1 - idleDiff / totalDiff) * 100)
}
function getMemoryUsage(): { used: number, total: number, percent: number } {
const total = totalmem()
if (platform() === 'linux') {
try {
const meminfo = readFileSync('/proc/meminfo', 'utf-8')
const available = meminfo.match(/MemAvailable:\s+(\d+)/)
if (available) {
const availableBytes = parseInt(available[1]!, 10) * 1024
const used = total - availableBytes
return { used, total, percent: Math.round((used / total) * 100) }
}
} catch {}
}
// macOS fallback
const free = freemem()
const used = total - free
return { used, total, percent: Math.round((used / total) * 100) }
}
function getDiskUsage(): { used: number, total: number, percent: number } {
try {
const stats = statfsSync('/')
const total = stats.blocks * stats.bsize
const free = stats.bfree * stats.bsize
const used = total - free
return {
used,
total,
percent: Math.round((used / total) * 100),
}
} catch {
return { used: 0, total: 0, percent: 0 }
}
}
// Per-app disk cache (updated every 30s)
let _appDiskCache: Record<string, number> = {}
let _appDiskLastUpdate = 0
const DISK_CACHE_TTL = 30000
async function getAppMetrics(): Promise<Record<string, AppMetrics>> {
const apps = allApps()
const running = apps.filter(a => a.proc?.pid)
const result: Record<string, AppMetrics> = {}
// CPU + MEM via ps (works on both macOS and Linux)
const pidToName = new Map<number, string>()
for (const app of running) {
pidToName.set(app.proc!.pid, app.name)
}
if (pidToName.size > 0) {
try {
const pids = [...pidToName.keys()].join(',')
const proc = Bun.spawn(['ps', '-o', 'pid=,%cpu=,rss=', '-p', pids], { stdout: 'pipe', stderr: 'ignore' })
const output = await new Response(proc.stdout).text()
await proc.exited
for (const line of output.split('\n')) {
const parts = line.trim().split(/\s+/)
if (parts.length < 3) continue
const pid = parseInt(parts[0]!, 10)
const cpu = parseFloat(parts[1]!) || 0
const mem = (parseInt(parts[2]!, 10) || 0) * 1024
const name = pidToName.get(pid)
if (name) result[name] = { cpu: Math.round(cpu), mem, disk: 0 }
}
} catch {}
}
// Disk usage per app (cached)
const now = Date.now()
if (now - _appDiskLastUpdate > DISK_CACHE_TTL) {
_appDiskLastUpdate = now
_appDiskCache = {}
const duResults = await Promise.all(
apps.map(async app => {
try {
const proc = Bun.spawn(['du', '-sk', join(APPS_DIR, app.name)], { stdout: 'pipe', stderr: 'ignore' })
const output = await new Response(proc.stdout).text()
await proc.exited
const kb = parseInt(output.trim().split('\t')[0]!, 10)
return { name: app.name, bytes: kb ? kb * 1024 : 0 }
} catch {
return { name: app.name, bytes: 0 }
}
})
)
for (const { name, bytes } of duResults) {
if (bytes) _appDiskCache[name] = bytes
}
}
// Merge disk into results, fill in stopped apps
for (const app of apps) {
if (!result[app.name]) result[app.name] = { cpu: 0, mem: 0, disk: 0 }
result[app.name]!.disk = _appDiskCache[app.name] ?? 0
}
return result
}
// Get current system metrics
router.get('/metrics', async c => {
const metrics: SystemMetrics = {
cpu: getCpuUsage(),
ram: getMemoryUsage(),
disk: getDiskUsage(),
apps: await getAppMetrics(),
}
return c.json(metrics)
})
// SSE stream for real-time metrics (updates every 2s)
router.sse('/metrics/stream', (send) => {
let queue = Promise.resolve()
const sendMetrics = () => {
queue = queue.then(async () => {
const metrics: SystemMetrics = {
cpu: getCpuUsage(),
ram: getMemoryUsage(),
disk: getDiskUsage(),
apps: await getAppMetrics(),
}
await send(metrics)
})
}
// Initial send
sendMetrics()
// Update every 2 seconds
const interval = setInterval(sendMetrics, 2000)
return () => clearInterval(interval)
})
// System info
const projectRoot = join(import.meta.dir, '../../..')
const pkg = JSON.parse(readFileSync(join(projectRoot, 'package.json'), 'utf-8'))
const sha = Bun.spawnSync(['git', 'rev-parse', '--short', 'HEAD'], { cwd: projectRoot }).stdout.toString().trim() || 'unknown'
let isUpdating = false
router.get('/info', c => {
return c.json({ version: pkg.version, sha })
})
// Get recent unified logs
router.get('/logs', c => {
const tail = c.req.query('tail')
const count = tail ? parseInt(tail, 10) : MAX_UNIFIED_LOGS
return c.json(unifiedLogs.slice(-count))
})
// Clear unified logs
router.post('/logs/clear', c => {
unifiedLogs.length = 0
return c.json({ ok: true })
})
// SSE stream for unified logs
router.sse('/logs/stream', (send) => {
// Send existing logs first
for (const line of unifiedLogs.slice(-50)) {
send(line)
}
// Subscribe to new logs
const listener = (line: UnifiedLogLine) => send(line)
unifiedLogListeners.add(listener)
return () => unifiedLogListeners.delete(listener)
})
// Collect logs from all apps
function collectLogs() {
const apps = allApps()
for (const app of apps) {
const logs = app.logs ?? []
const lastCount = lastLogCounts.get(app.name) ?? 0
// Get new logs since last check
const newLogs = logs.slice(lastCount)
lastLogCounts.set(app.name, logs.length)
for (const log of newLogs) {
const line: UnifiedLogLine = {
time: log.time,
app: app.name,
text: log.text,
}
// Add to buffer
unifiedLogs.push(line)
if (unifiedLogs.length > MAX_UNIFIED_LOGS) {
unifiedLogs.shift()
}
// Notify listeners
for (const listener of unifiedLogListeners) {
listener(line)
}
}
}
}
function pushHostLog(text: string) {
const line: UnifiedLogLine = { time: Date.now(), app: 'toes', text }
unifiedLogs.push(line)
if (unifiedLogs.length > MAX_UNIFIED_LOGS) unifiedLogs.shift()
for (const listener of unifiedLogListeners) listener(line)
}
// Subscribe to app changes to collect logs
onChange(collectLogs)
// Subscribe to host-level log messages
onHostLog(pushHostLog)
// Restart server (systemd brings it back)
router.post('/restart', c => {
setTimeout(() => process.exit(0), 100)
return c.json({ ok: true })
})
// Check for updates
router.get('/update', async c => {
try {
const gitFetch = await Bun.spawn(['git', 'fetch', 'origin', 'main'], { cwd: projectRoot }).exited
if (gitFetch !== 0) return c.json({ available: false })
const log = Bun.spawnSync(['git', 'log', 'HEAD..origin/main', '--oneline'], { cwd: projectRoot })
const output = log.stdout.toString().trim()
const commits = output ? output.split('\n') : []
const latest = Bun.spawnSync(['git', 'rev-parse', '--short', 'origin/main'], { cwd: projectRoot })
.stdout.toString().trim()
return c.json({
available: commits.length > 0,
current: sha,
latest: commits.length > 0 ? latest : sha,
commits,
})
} catch {
return c.json({ available: false })
}
})
// Apply update and restart (delegates to install/install.sh)
// The script ends with `systemctl restart toes`, killing this process,
// so we respond immediately and run it detached.
router.post('/update', c => {
if (isUpdating) return c.json({ ok: false, error: 'update already in progress' }, 409)
isUpdating = true
try {
const script = join(projectRoot, 'install/install.sh')
const proc = Bun.spawn(['bash', script], { cwd: projectRoot, stdout: 'ignore', stderr: 'ignore' })
proc.exited.then(code => { if (code !== 0) isUpdating = false })
.catch(() => { isUpdating = false })
return c.json({ ok: true })
} catch {
isUpdating = false
return c.json({ ok: false, error: 'failed to start update' }, 500)
}
})
export default router