From 8e3179f7d3f0ba16b69a07581810d8f1c5a8c0df Mon Sep 17 00:00:00 2001 From: Corey Johnson Date: Wed, 11 Mar 2026 19:08:51 -0700 Subject: [PATCH] Add ggwave audio calculator POC and documentation Includes a working Bun server and iOS Safari calculator UI that uses data-over-sound encoding/decoding via the ggwave library. Phone encodes expressions as audible chirps, Mac server decodes via microphone, evaluates, and sends result back via SSE. Tested reliably with ambient noise using Studio Display microphone. Includes gotchas documentation covering iOS audio, macOS mic permissions, WASM heap, and sample rate requirements. Co-Authored-By: Claude Haiku 4.5 --- docs/ggwave-gotchas.md | 29 ++++++++ tmp/bun.lock | 15 ++++ tmp/index.html | 146 ++++++++++++++++++++++++++++++++++++++ tmp/package.json | 7 ++ tmp/server.ts | 155 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 352 insertions(+) create mode 100644 docs/ggwave-gotchas.md create mode 100644 tmp/bun.lock create mode 100644 tmp/index.html create mode 100644 tmp/package.json create mode 100644 tmp/server.ts diff --git a/docs/ggwave-gotchas.md b/docs/ggwave-gotchas.md new file mode 100644 index 0000000..201f075 --- /dev/null +++ b/docs/ggwave-gotchas.md @@ -0,0 +1,29 @@ +# ggwave Gotchas + +## WASM + +- **WASM heap copies**: `encode()` returns Int8Array backed by WASM heap memory. Must copy data out (`new Uint8Array(rawBytes.length); copy.set(...)`) before using, or it gets corrupted. +- **Same instance required**: Using separate ggwave instances for encode/decode causes WASM heap pointer corruption. Use one instance for both. Max 4 instances allowed. +- **Encode output format**: `encode()` returns Int8Array of raw F32 *bytes*, not Float32Array. Reinterpret with `new Float32Array(bytesCopy.buffer)` for AudioBuffer. +- **API naming**: README says `TxProtocolId` but actual API uses `ProtocolId`. Check `Object.keys(ggwave)` when in doubt. + +## iOS Safari Audio + +- **AudioContext must be created synchronously** inside a user gesture handler (click/tap). Any `await` before `new AudioContext()` breaks the gesture chain and Safari blocks audio permanently for that context. +- **Silent mode bypass**: `navigator.audioSession.type = 'playback'` (iOS 17+) switches WebAudio from ringer channel to media channel, bypassing the hardware mute switch. Without this, the silent switch kills all WebAudio output. +- **Unlock pattern**: Create AudioContext → play a silent buffer → then await async init. Never reverse this order. + +## macOS Microphone (sox + CoreAudio) + +- **Explicit device names produce all-zero data**: `sox -t coreaudio "MacBook Air Microphone"` gets zeros due to macOS TCC permission scoping. Only `sox -d` (default device) gets actual mic access granted to the terminal app. +- **Workaround**: Change the default input device in System Settings > Sound > Input, then use `sox -d`. Or install `switchaudio-osx` (`brew install switchaudio-osx`) to change it programmatically. +- **MacBook Air mic disabled when lid is closed**: If using a monitor, the laptop mic won't work. Use the monitor's mic (e.g., Studio Display) instead. +- **Sample rate must match**: ggwave needs matching sample rates for encode/decode. Browser uses 48000Hz. Server must also use 48000Hz — sox will resample from the device's native rate automatically. + +## SSE with Bun + +- Bun's default idle timeout is 10 seconds, which kills SSE connections. Set `idleTimeout: 255` (max value) on `Bun.serve()`. + +## Half-Duplex Audio + +- Both sides use the same audible protocol (GGWAVE_PROTOCOL_AUDIBLE_FAST). A `playing` flag stops mic processing during playback to prevent self-hearing/feedback. 300ms gap after playback before resuming listening. diff --git a/tmp/bun.lock b/tmp/bun.lock new file mode 100644 index 0000000..c106d99 --- /dev/null +++ b/tmp/bun.lock @@ -0,0 +1,15 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "ggwave-poc", + "dependencies": { + "ggwave": "0.4.0", + }, + }, + }, + "packages": { + "ggwave": ["ggwave@0.4.0", "", {}, "sha512-+sKq0aIEVJ7zHj4Vw+Sj/RPa91xp76ihaG5gsOKZ8ojM5+uUu3NFzAspozwBx/zeaThxP5VeIkA2bbsfWpUd2g=="], + } +} diff --git a/tmp/index.html b/tmp/index.html new file mode 100644 index 0000000..e3864cb --- /dev/null +++ b/tmp/index.html @@ -0,0 +1,146 @@ + + + + + + Audio Calculator + + + + +
0
+
Tap any button to start
+
+ + + + + + + + + + + + + + + + + + + + + + + +
+ + + + diff --git a/tmp/package.json b/tmp/package.json new file mode 100644 index 0000000..7a263f5 --- /dev/null +++ b/tmp/package.json @@ -0,0 +1,7 @@ +{ + "name": "ggwave-poc", + "type": "module", + "dependencies": { + "ggwave": "0.4.0" + } +} diff --git a/tmp/server.ts b/tmp/server.ts new file mode 100644 index 0000000..1f56243 --- /dev/null +++ b/tmp/server.ts @@ -0,0 +1,155 @@ +import factory from 'ggwave' + +const PORT = 8888 +const SAMPLE_RATE = 48000 + +const ggwave = await factory() +const params = ggwave.getDefaultParameters() +params.sampleRateInp = SAMPLE_RATE +params.sampleRateOut = SAMPLE_RATE +params.sampleRate = SAMPLE_RATE +const instance = ggwave.init(params) + +// SSE clients waiting for results +const clients = new Set() + +// Half-duplex: don't process mic while playing +let playing = false + +function broadcast(data: object) { + const msg = `data: ${JSON.stringify(data)}\n\n` + for (const controller of clients) { + try { controller.enqueue(new TextEncoder().encode(msg)) } + catch { clients.delete(controller) } + } +} + +function evaluate(expr: string): string { + if (!/^[\d+\-*/.() ]+$/.test(expr)) return 'ERR' + try { return String(new Function(`return (${expr})`)()) } + catch { return 'ERR' } +} + +function decodeBytes(data: Int8Array): string { + return Array.from(data).map(b => String.fromCharCode(b & 0xff)).join('') +} + +async function playResult(text: string) { + playing = true + broadcast({ type: 'playing', result: text }) + + const waveform = ggwave.encode( + instance, text, + ggwave.ProtocolId.GGWAVE_PROTOCOL_AUDIBLE_FAST, + 50 + ) + const rawBytes = new Uint8Array(waveform.length) + rawBytes.set(new Uint8Array(waveform.buffer, waveform.byteOffset, waveform.length)) + + const play = Bun.spawn( + ['sox', '-t', 'raw', '-r', String(SAMPLE_RATE), '-c', '1', '-b', '32', '-e', 'floating-point', '-', '-d'], + { stdin: 'pipe', stdout: 'ignore', stderr: 'ignore' } + ) + play.stdin.write(rawBytes) + play.stdin.end() + await play.exited + + await new Promise(r => setTimeout(r, 300)) + playing = false + broadcast({ type: 'ready' }) + console.log('Played result:', text) +} + +function startMicListener() { + // Uses default device (-d). Explicit CoreAudio device names produce all-zero + // data due to macOS TCC permission scoping. + const sox = Bun.spawn( + ['sox', '-d', '-t', 'raw', '-r', String(SAMPLE_RATE), '-c', '1', '-b', '32', '-e', 'floating-point', '-'], + { stdout: 'pipe', stderr: 'pipe' } + ) + + new Response(sox.stderr).text().then(err => { + const deviceLine = err.split('\n').find(l => l.includes('Input File')) + if (deviceLine) console.log('Mic:', deviceLine.trim()) + }) + + const reader = sox.stdout.getReader() + const bytesPerFrame = params.samplesPerFrame * 4 + + let buffer = new Uint8Array(0) + + async function processAudio() { + while (true) { + const { done, value } = await reader.read() + if (done) break + if (playing) continue + + const newBuf = new Uint8Array(buffer.length + value.length) + newBuf.set(buffer) + newBuf.set(value, buffer.length) + buffer = newBuf + + while (buffer.length >= bytesPerFrame) { + const frame = buffer.slice(0, bytesPerFrame) + buffer = buffer.slice(bytesPerFrame) + + const decoded = ggwave.decode(instance, frame) + if (decoded && decoded.length > 0) { + const text = decodeBytes(decoded) + const result = evaluate(text) + console.log(`${text} = ${result}`) + broadcast({ type: 'received', expression: text, result }) + await playResult(result) + } + } + } + } + + processAudio().catch(err => console.error('Mic error:', err)) +} + +Bun.serve({ + port: PORT, + idleTimeout: 255, + async fetch(req) { + const url = new URL(req.url) + + if (url.pathname === '/ok') return new Response('ok') + + if (url.pathname === '/') { + return new Response(Bun.file(import.meta.dir + '/index.html'), { + headers: { 'Content-Type': 'text/html' }, + }) + } + + if (url.pathname === '/ggwave.js') { + return new Response(Bun.file(import.meta.dir + '/node_modules/ggwave/ggwave.js'), { + headers: { 'Content-Type': 'application/javascript' }, + }) + } + + if (url.pathname === '/ggwave.wasm') { + return new Response(Bun.file(import.meta.dir + '/node_modules/ggwave/ggwave.wasm'), { + headers: { 'Content-Type': 'application/wasm' }, + }) + } + + if (url.pathname === '/events') { + const stream = new ReadableStream({ + start(controller) { + clients.add(controller) + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify({ type: 'ready' })}\n\n`)) + }, + cancel(controller) { clients.delete(controller) }, + }) + return new Response(stream, { + headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive' }, + }) + } + + return new Response('not found', { status: 404 }) + }, +}) + +console.log(`Listening on http://localhost:${PORT}`) +startMicListener()