From c4af302d9d5a379e7a6a9e2c0c621ea73a07b411 Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Thu, 29 Jan 2026 23:21:44 -0800 Subject: [PATCH] toes sync and friends --- TODO.txt | 4 +- apps/clock2/bun.lock | 38 ------- src/cli/index.ts | 234 +++++++++++++++++++++++++++++------------ src/lib/sync.ts | 47 +++++++++ src/server/api/sync.ts | 89 ++++++++++++++-- src/server/apps.ts | 11 ++ src/server/sync.ts | 57 +--------- src/shared/types.ts | 11 ++ tsconfig.json | 3 + 9 files changed, 322 insertions(+), 172 deletions(-) delete mode 100644 apps/clock2/bun.lock create mode 100644 src/lib/sync.ts diff --git a/TODO.txt b/TODO.txt index 7e32dc9..9bbf9c8 100644 --- a/TODO.txt +++ b/TODO.txt @@ -31,12 +31,12 @@ [x] `toes new` [x] `toes pull` [x] `toes push` -[ ] `toes sync` +[x] `toes sync` ## webui [x] list projects [x] start/stop/restart project -[ ] create project +[x] create project [ ] todo.txt [ ] ... diff --git a/apps/clock2/bun.lock b/apps/clock2/bun.lock deleted file mode 100644 index ee899ce..0000000 --- a/apps/clock2/bun.lock +++ /dev/null @@ -1,38 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "clock2", - "dependencies": { - "forge": "git+https://git.nose.space/defunkt/forge", - "hype": "git+https://git.nose.space/defunkt/hype", - }, - "devDependencies": { - "@types/bun": "latest", - }, - "peerDependencies": { - "typescript": "^5.9.2", - }, - }, - }, - "packages": { - "@types/bun": ["@types/bun@1.3.8", "", { "dependencies": { "bun-types": "1.3.8" } }, "sha512-3LvWJ2q5GerAXYxO2mffLTqOzEu5qnhEAlh48Vnu8WQfnmSwbgagjGZV6BoHKJztENYEDn6QmVd949W4uESRJA=="], - - "@types/node": ["@types/node@25.1.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-t7frlewr6+cbx+9Ohpl0NOTKXZNV9xHRmNOvql47BFJKcEG1CxtxlPEEe+gR9uhVWM4DwhnvTF110mIL4yP9RA=="], - - "bun-types": ["bun-types@1.3.8", "", { "dependencies": { "@types/node": "*" } }, "sha512-fL99nxdOWvV4LqjmC+8Q9kW3M4QTtTR1eePs94v5ctGqU8OeceWrSUaRw3JYb7tU3FkMIAjkueehrHPPPGKi5Q=="], - - "forge": ["@because/forge@git+https://git.nose.space/defunkt/forge#67180bb4f3f13b6eacba020e60592d4a76e2deda", { "peerDependencies": { "typescript": "^5" } }, "67180bb4f3f13b6eacba020e60592d4a76e2deda"], - - "hono": ["hono@4.11.7", "", {}, "sha512-l7qMiNee7t82bH3SeyUCt9UF15EVmaBvsppY2zQtrbIhl/yzBTny+YUxsVjSjQ6gaqaeVtZmGocom8TzBlA4Yw=="], - - "hype": ["@because/hype@git+https://git.nose.space/defunkt/hype#5ed8673274187958e3bee569aba0d33baf605bb1", { "dependencies": { "hono": "^4.10.4", "kleur": "^4.1.5" }, "peerDependencies": { "typescript": "^5" } }, "5ed8673274187958e3bee569aba0d33baf605bb1"], - - "kleur": ["kleur@4.1.5", "", {}, "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ=="], - - "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], - - "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], - } -} diff --git a/src/cli/index.ts b/src/cli/index.ts index eaf442b..3f55ce8 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,27 +1,16 @@ #!/usr/bin/env bun -import type { App, LogLine } from '@types' +import type { App, LogLine, Manifest } from '@types' import { loadGitignore } from '@gitignore' +import { computeHash, generateManifest } from '%sync' import { generateTemplates } from '@templates' import { program } from 'commander' -import { createHash } from 'crypto' -import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, unlinkSync, writeFileSync } from 'fs' +import { existsSync, mkdirSync, readFileSync, statSync, unlinkSync, watch, writeFileSync } from 'fs' import color from 'kleur' -import { basename, dirname, join, relative } from 'path' +import { basename, dirname, join } from 'path' import * as readline from 'readline' const HOST = `http://localhost:${process.env.PORT ?? 3000}` -interface FileInfo { - hash: string - mtime: string - size: number -} - -interface Manifest { - files: Record - name: string -} - const STATE_ICONS: Record = { running: color.green('●'), starting: color.yellow('◎'), @@ -115,48 +104,6 @@ async function del(url: string): Promise { } } -function computeHash(content: Buffer | string): string { - return createHash('sha256').update(content).digest('hex') -} - -function generateLocalManifest(appPath: string, appName: string): Manifest { - const files: Record = {} - const gitignore = loadGitignore(appPath) - - function walkDir(dir: string) { - if (!existsSync(dir)) return - - const entries = readdirSync(dir, { withFileTypes: true }) - - for (const entry of entries) { - const fullPath = join(dir, entry.name) - const relativePath = relative(appPath, fullPath) - - if (gitignore.shouldExclude(relativePath)) continue - - if (entry.isDirectory()) { - walkDir(fullPath) - } else if (entry.isFile()) { - const content = readFileSync(fullPath) - const stats = statSync(fullPath) - - files[relativePath] = { - hash: computeHash(content), - mtime: stats.mtime.toISOString(), - size: stats.size, - } - } - } - } - - walkDir(appPath) - - return { - name: appName, - files, - } -} - async function infoApp(arg?: string) { const name = resolveAppName(arg) if (!name) return @@ -351,13 +298,9 @@ async function pushApp() { return } - const appName = process.cwd().split('/').pop() - if (!appName) { - console.error('Could not determine app name from current directory') - return - } + const appName = basename(process.cwd()) - const localManifest = generateLocalManifest(process.cwd(), appName) + const localManifest = generateManifest(process.cwd(), appName) const result = await getManifest(appName) if (result === null) { @@ -483,11 +426,7 @@ async function pullApp() { return } - const appName = process.cwd().split('/').pop() - if (!appName) { - console.error('Could not determine app name from current directory') - return - } + const appName = basename(process.cwd()) const remoteManifest: Manifest | undefined = await get(`/api/sync/apps/${appName}/manifest`) if (!remoteManifest) { @@ -495,7 +434,7 @@ async function pullApp() { return } - const localManifest = generateLocalManifest(process.cwd(), appName) + const localManifest = generateManifest(process.cwd(), appName) const localFiles = new Set(Object.keys(localManifest.files)) const remoteFiles = new Set(Object.keys(remoteManifest.files)) @@ -558,6 +497,152 @@ async function pullApp() { console.log(color.green('✓ Pull complete')) } +async function syncApp() { + if (!isApp()) { + console.error('Not a toes app. Use `toes get ` to grab one.') + return + } + + const appName = basename(process.cwd()) + const gitignore = loadGitignore(process.cwd()) + const localHashes = new Map() + + // Initialize local hashes + const manifest = generateManifest(process.cwd(), appName) + for (const [path, info] of Object.entries(manifest.files)) { + localHashes.set(path, info.hash) + } + + console.log(`Syncing ${color.bold(appName)}...`) + + // Watch local files + const watcher = watch(process.cwd(), { recursive: true }, async (_event, filename) => { + if (!filename || gitignore.shouldExclude(filename)) return + + const fullPath = join(process.cwd(), filename) + + if (existsSync(fullPath) && statSync(fullPath).isFile()) { + const content = readFileSync(fullPath) + const hash = computeHash(content) + if (localHashes.get(filename) !== hash) { + localHashes.set(filename, hash) + await put(`/api/sync/apps/${appName}/files/${filename}`, content) + console.log(` ${color.green('↑')} ${filename}`) + } + } else if (!existsSync(fullPath)) { + localHashes.delete(filename) + await del(`/api/sync/apps/${appName}/files/${filename}`) + console.log(` ${color.red('✗')} ${filename}`) + } + }) + + // Connect to SSE for remote changes + const url = makeUrl(`/api/sync/apps/${appName}/watch`) + let res: Response + try { + res = await fetch(url) + if (!res.ok) { + console.error(`Failed to connect to server: ${res.status} ${res.statusText}`) + watcher.close() + return + } + } catch (error) { + handleError(error) + watcher.close() + return + } + + if (!res.body) { + console.error('No response body from server') + watcher.close() + return + } + + console.log(` Connected to server, watching for changes...`) + + const reader = res.body.getReader() + const decoder = new TextDecoder() + let buffer = '' + + try { + while (true) { + const { done, value } = await reader.read() + if (done) break + + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split('\n\n') + buffer = lines.pop() ?? '' + + for (const line of lines) { + if (!line.startsWith('data: ')) continue + const event = JSON.parse(line.slice(6)) as { type: 'change' | 'delete'; path: string; hash?: string } + + if (event.type === 'change') { + // Skip if we already have this version (handles echo from our own changes) + if (localHashes.get(event.path) === event.hash) continue + const content = await download(`/api/sync/apps/${appName}/files/${event.path}`) + if (content) { + const fullPath = join(process.cwd(), event.path) + mkdirSync(dirname(fullPath), { recursive: true }) + writeFileSync(fullPath, content) + localHashes.set(event.path, event.hash!) + console.log(` ${color.green('↓')} ${event.path}`) + } + } else if (event.type === 'delete') { + const fullPath = join(process.cwd(), event.path) + if (existsSync(fullPath)) { + unlinkSync(fullPath) + localHashes.delete(event.path) + console.log(` ${color.red('✗')} ${event.path} (remote)`) + } + } + } + } + } finally { + watcher.close() + } +} + +async function prompt(message: string): Promise { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }) + + return new Promise(resolve => { + rl.question(message, (answer) => { + rl.close() + resolve(answer) + }) + }) +} + +async function rmApp(arg?: string) { + const name = resolveAppName(arg) + if (!name) return + + const result = await getManifest(name) + if (result === null) return + if (!result.exists) { + console.error(`App not found on server: ${name}`) + return + } + + const expected = `sudo rm ${name}` + console.log(`This will ${color.red('permanently delete')} ${color.bold(name)} from the server.`) + const answer = await prompt(`Type "${expected}" to confirm: `) + + if (answer !== expected) { + console.log('Aborted.') + return + } + + const success = await del(`/api/sync/apps/${name}`) + if (success) { + console.log(color.green(`✓ Removed ${name}`)) + } +} + program .name('toes') .version('0.0.1', '-v, --version') @@ -632,4 +717,15 @@ program .description('Pull changes from server') .action(pullApp) +program + .command('sync') + .description('Watch and sync changes bidirectionally') + .action(syncApp) + +program + .command('rm') + .description('Remove an app from the server') + .argument('[name]', 'app name (uses current directory if omitted)') + .action(rmApp) + program.parse() \ No newline at end of file diff --git a/src/lib/sync.ts b/src/lib/sync.ts new file mode 100644 index 0000000..50e634a --- /dev/null +++ b/src/lib/sync.ts @@ -0,0 +1,47 @@ +export type { FileInfo, Manifest } from '@types' + +import type { FileInfo, Manifest } from '@types' +import { loadGitignore } from '@gitignore' +import { createHash } from 'crypto' +import { readdirSync, readFileSync, statSync } from 'fs' +import { join, relative } from 'path' + +export function computeHash(content: Buffer | string): string { + return createHash('sha256').update(content).digest('hex') +} + +export function generateManifest(appPath: string, appName: string): Manifest { + const files: Record = {} + const gitignore = loadGitignore(appPath) + + function walkDir(dir: string) { + const entries = readdirSync(dir, { withFileTypes: true }) + + for (const entry of entries) { + const fullPath = join(dir, entry.name) + const relativePath = relative(appPath, fullPath) + + if (gitignore.shouldExclude(relativePath)) continue + + if (entry.isDirectory()) { + walkDir(fullPath) + } else if (entry.isFile()) { + const content = readFileSync(fullPath) + const stats = statSync(fullPath) + + files[relativePath] = { + hash: computeHash(content), + mtime: stats.mtime.toISOString(), + size: stats.size, + } + } + } + } + + walkDir(appPath) + + return { + name: appName, + files, + } +} diff --git a/src/server/api/sync.ts b/src/server/api/sync.ts index 33f40ed..7eccfe7 100644 --- a/src/server/api/sync.ts +++ b/src/server/api/sync.ts @@ -1,9 +1,22 @@ -import { APPS_DIR, allApps } from '$apps' -import { generateManifest } from '../sync' -import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'fs' +import { APPS_DIR, allApps, removeApp } from '$apps' +import { computeHash, generateManifest } from '../sync' +import { loadGitignore } from '@gitignore' +import { existsSync, mkdirSync, readFileSync, rmSync, unlinkSync, watch, writeFileSync } from 'fs' import { dirname, join } from 'path' import { Hype } from '@because/hype' +interface FileChangeEvent { + hash?: string + path: string + type: 'change' | 'delete' +} + +function safePath(base: string, ...parts: string[]): string | null { + const resolved = join(base, ...parts) + if (!resolved.startsWith(base + '/') && resolved !== base) return null + return resolved +} + const router = Hype.router() router.get('/apps', c => c.json(allApps().map(a => a.name))) @@ -12,7 +25,8 @@ router.get('/apps/:app/manifest', c => { const appName = c.req.param('app') if (!appName) return c.json({ error: 'App not found' }, 404) - const appPath = join(APPS_DIR, appName) + const appPath = safePath(APPS_DIR, appName) + if (!appPath) return c.json({ error: 'Invalid path' }, 400) if (!existsSync(appPath)) return c.json({ error: 'App not found' }, 404) const manifest = generateManifest(appPath, appName) @@ -25,11 +39,14 @@ router.get('/apps/:app/files/:path{.+}', c => { if (!appName || !filePath) return c.json({ error: 'Invalid path' }, 400) - const fullPath = join(APPS_DIR, appName, filePath) + const fullPath = safePath(APPS_DIR, appName, filePath) + if (!fullPath) return c.json({ error: 'Invalid path' }, 400) if (!existsSync(fullPath)) return c.json({ error: 'File not found' }, 404) const content = readFileSync(fullPath) - return new Response(content) + return new Response(content, { + headers: { 'Content-Type': 'application/octet-stream' }, + }) }) router.put('/apps/:app/files/:path{.+}', async c => { @@ -38,7 +55,9 @@ router.put('/apps/:app/files/:path{.+}', async c => { if (!appName || !filePath) return c.json({ error: 'Invalid path' }, 400) - const fullPath = join(APPS_DIR, appName, filePath) + const fullPath = safePath(APPS_DIR, appName, filePath) + if (!fullPath) return c.json({ error: 'Invalid path' }, 400) + const dir = dirname(fullPath) // Ensure directory exists @@ -52,17 +71,71 @@ router.put('/apps/:app/files/:path{.+}', async c => { return c.json({ ok: true }) }) +router.delete('/apps/:app', c => { + const appName = c.req.param('app') + if (!appName) return c.json({ error: 'App not found' }, 404) + + const appPath = safePath(APPS_DIR, appName) + if (!appPath) return c.json({ error: 'Invalid path' }, 400) + if (!existsSync(appPath)) return c.json({ error: 'App not found' }, 404) + + removeApp(appName) + rmSync(appPath, { recursive: true }) + return c.json({ ok: true }) +}) + router.delete('/apps/:app/files/:path{.+}', c => { const appName = c.req.param('app') const filePath = c.req.param('path') if (!appName || !filePath) return c.json({ error: 'Invalid path' }, 400) - const fullPath = join(APPS_DIR, appName, filePath) + const fullPath = safePath(APPS_DIR, appName, filePath) + if (!fullPath) return c.json({ error: 'Invalid path' }, 400) if (!existsSync(fullPath)) return c.json({ error: 'File not found' }, 404) unlinkSync(fullPath) return c.json({ ok: true }) }) +router.sse('/apps/:app/watch', (send, c) => { + const appName = c.req.param('app') + const appPath = safePath(APPS_DIR, appName) + if (!appPath || !existsSync(appPath)) return + + const gitignore = loadGitignore(appPath) + let debounceTimer: Timer | null = null + const pendingChanges = new Map() + + const watcher = watch(appPath, { recursive: true }, (_event, filename) => { + if (!filename || gitignore.shouldExclude(filename)) return + + const fullPath = join(appPath, filename) + const type = existsSync(fullPath) ? 'change' : 'delete' + pendingChanges.set(filename, type) + + if (debounceTimer) clearTimeout(debounceTimer) + debounceTimer = setTimeout(() => { + for (const [path, changeType] of pendingChanges) { + const evt: FileChangeEvent = { type: changeType, path } + if (changeType === 'change') { + try { + const content = readFileSync(join(appPath, path)) + evt.hash = computeHash(content) + } catch { + continue // File was deleted between check and read + } + } + send(evt) + } + pendingChanges.clear() + }, 100) + }) + + return () => { + if (debounceTimer) clearTimeout(debounceTimer) + watcher.close() + } +}) + export default router diff --git a/src/server/apps.ts b/src/server/apps.ts index 5539834..2c18efa 100644 --- a/src/server/apps.ts +++ b/src/server/apps.ts @@ -50,6 +50,17 @@ export function startApp(dir: string) { runApp(dir, getPort()) } +export function removeApp(dir: string) { + const app = _apps.get(dir) + if (!app) return + + if (app.state === 'running') + app.proc?.kill() + + _apps.delete(dir) + update() +} + export function stopApp(dir: string) { const app = _apps.get(dir) if (!app || app.state !== 'running') return diff --git a/src/server/sync.ts b/src/server/sync.ts index f0f5e83..c5cfe60 100644 --- a/src/server/sync.ts +++ b/src/server/sync.ts @@ -1,55 +1,2 @@ -import { createHash } from 'crypto' -import { readdirSync, readFileSync, statSync } from 'fs' -import { join, relative } from 'path' -import { loadGitignore } from '@gitignore' - -export interface FileInfo { - hash: string - mtime: string - size: number -} - -export interface Manifest { - files: Record - name: string -} - -export function computeHash(content: Buffer | string): string { - return createHash('sha256').update(content).digest('hex') -} - -export function generateManifest(appPath: string, appName: string): Manifest { - const files: Record = {} - const gitignore = loadGitignore(appPath) - - function walkDir(dir: string) { - const entries = readdirSync(dir, { withFileTypes: true }) - - for (const entry of entries) { - const fullPath = join(dir, entry.name) - const relativePath = relative(appPath, fullPath) - - if (gitignore.shouldExclude(relativePath)) continue - - if (entry.isDirectory()) { - walkDir(fullPath) - } else if (entry.isFile()) { - const content = readFileSync(fullPath) - const stats = statSync(fullPath) - - files[relativePath] = { - hash: computeHash(content), - mtime: stats.mtime.toISOString(), - size: stats.size, - } - } - } - } - - walkDir(appPath) - - return { - name: appName, - files, - } -} +export type { FileInfo, Manifest } from '%sync' +export { computeHash, generateManifest } from '%sync' diff --git a/src/shared/types.ts b/src/shared/types.ts index 8e47fa5..cea3e9e 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -1,5 +1,16 @@ export const DEFAULT_EMOJI = '🖥️' +export interface FileInfo { + hash: string + mtime: string + size: number +} + +export interface Manifest { + files: Record + name: string +} + export type AppState = 'invalid' | 'stopped' | 'starting' | 'running' | 'stopping' export type LogLine = { diff --git a/tsconfig.json b/tsconfig.json index aa94996..a7afcc2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -33,6 +33,9 @@ ], "@*": [ "./src/shared/*" + ], + "%*": [ + "./src/lib/*" ] } }