OMG, it works

scripts/deploy.ts

@@ -20,7 +20,7 @@ await $`ssh ${PI_HOST} "mkdir -p ${PI_DIR}"`
 
 // Sync files from . directory to Pi (only transfers changed files)
 console.log("Syncing files from . directory...")
-await $`rsync -avz --delete   --exclude-from='.gitignore' . ${PI_HOST}:${PI_DIR}/`
+await $`rsync -avz --delete  --exclude-from='../.gitignore' --exclude='.git' . ${PI_HOST}:${PI_DIR}/`
 
 // Make all TypeScript files executable
 console.log("Making scripts executable...")

src/agent/tools.ts

@@ -26,5 +26,7 @@ export const searchWeb = async (query: string) => {
     input: `Search the web for: ${query}`,
   })
 
+  await Bun.sleep(10000)
+
   return response.output_text
 }

src/buzz/player.ts

@@ -151,13 +151,26 @@ export class Player {
       stderr: "pipe",
     })
 
+    let bufferFinishTime = performance.now()
+    const format = this.#format
+    const bytesPerSecond = format.sampleRate * format.channels * 2
+
     const handle: StreamingPlayback = {
       isPlaying: true,
       write: (chunk: Uint8Array) => {
         if (handle.isPlaying) {
           proc.stdin.write(chunk)
+          const chunkDurationMs = (chunk.byteLength / bytesPerSecond) * 1000
+          bufferFinishTime = Math.max(performance.now(), bufferFinishTime) + chunkDurationMs
         }
       },
+      get bufferEmptyFor() {
+        const now = performance.now()
+        if (now > bufferFinishTime) {
+          return now - bufferFinishTime
+        }
+        return 0
+      },
       stop: async () => {
         if (!handle.isPlaying) return
         handle.isPlaying = false

src/buzz/utils.ts

@@ -33,6 +33,7 @@ export type StreamingPlayback = {
   isPlaying: boolean;
   write: (chunk: Uint8Array) => void;
   stop: () => Promise<void>;
+  bufferEmptyFor: number; // milliseconds since buffer became empty, 0 if not empty
 };
 
 // Streaming recording control handle

src/operator.ts

@@ -21,8 +21,8 @@ const runPhoneSystem = async (agentId: string, apiKey: string) => {
 
   let currentDialtone: Playback | undefined
   let currentBackgroundNoise: Playback | undefined
-  let currentPlayback = player.playStream()
+  let streamPlayback = player.playStream()
-  const waitingIndicator = new WaitingSounds(player)
+  const waitingIndicator = new WaitingSounds(player, streamPlayback)
 
   // Set up agent event listeners
   agent.events.connect((event) => {
@@ -42,13 +42,13 @@ const runPhoneSystem = async (agentId: string, apiKey: string) => {
       case "audio":
         waitingIndicator.stop()
         const audioBuffer = Buffer.from(event.audioBase64, "base64")
-        currentPlayback.write(audioBuffer)
+        streamPlayback.write(audioBuffer)
         break
 
       case "interruption":
         console.log("🛑 User interrupted")
-        currentPlayback?.stop()
+        streamPlayback?.stop()
-        currentPlayback = player.playStream() // Reset playback stream
+        streamPlayback = player.playStream() // Reset playback stream
         break
 
       case "tool_call":
@@ -66,7 +66,7 @@ const runPhoneSystem = async (agentId: string, apiKey: string) => {
 
       case "disconnected":
         console.log("\n👋 Conversation ended, returning to dialtone\n")
-        currentPlayback?.stop()
+        streamPlayback?.stop()
         state = "WAITING_FOR_VOICE"
         startDialtone()
         break
@@ -94,9 +94,7 @@ const runPhoneSystem = async (agentId: string, apiKey: string) => {
   const stopDialtone = async () => {
     await currentDialtone?.stop()
     currentDialtone = undefined
-    currentBackgroundNoise = await player.play(getSound("background"), {
+    currentBackgroundNoise = await player.play(getSound("background"), { repeat: true })
-      repeat: true,
-    })
   }
 
   const startConversation = async () => {
@@ -139,7 +137,7 @@ const runPhoneSystem = async (agentId: string, apiKey: string) => {
     console.log("\n\n🛑 Shutting down phone system...")
     await currentDialtone?.stop()
     await currentBackgroundNoise?.stop()
-    await currentPlayback?.stop()
+    await streamPlayback?.stop()
     await agent.stop()
     process.exit(0)
   }

src/utils/waiting-sounds.ts

@@ -1,57 +1,80 @@
-import Buzz, { type Player } from "../buzz/index.ts"
+import { type Player } from "../buzz/index.ts"
 import { join } from "path"
-import type { Playback } from "../buzz/utils.ts"
+import type { Playback, StreamingPlayback } from "../buzz/utils.ts"
 import { random } from "./index.ts"
 
 export class WaitingSounds {
-  currentPlayback?: Playback
+  typingPlayback?: Playback
+  speakingPlayback?: Playback
 
-  constructor(private player: Player) {}
+  constructor(private player: Player, private streamPlayback: StreamingPlayback) {}
 
   async start() {
-    if (this.currentPlayback) return // Already playing
+    if (this.typingPlayback) return // Already playing
 
-    // Now play randomly play things
+    this.#startTypingSounds()
+    this.#startSpeakingSounds()
+  }
+
+  async #startTypingSounds() {
+    return new Promise<void>(async (resolve) => {
+      do {
+        const value = Math.random() * 100
+        let dir: SoundDir
+        if (value > 33) {
+          dir = "typing"
+        } else {
+          dir = "clicking"
+        }
+        const typingSound = getSound(dir)
+        this.typingPlayback = await this.player.play(typingSound)
+        await this.typingPlayback.finished()
+      } while (this.typingPlayback)
+
+      resolve()
+    })
+  }
+
+  async #startSpeakingSounds() {
     const playedSounds = new Set<string>()
-    let lastSoundDir: SoundDir | undefined
+    let dir: SoundDir | undefined
-    do {
+    return new Promise<void>(async (resolve) => {
-      const dir = this.getRandomSoundDir(lastSoundDir)
+      // Don't start speaking until the stream playback buffer is empty!
-      const soundPath = getSound(dir, Array.from(playedSounds))
+      while (this.streamPlayback.bufferEmptyFor < 1000) {
-      playedSounds.add(soundPath)
+        await Bun.sleep(100)
-      lastSoundDir = dir
+      }
-      console.log(`🌭 playing ${soundPath}`)
 
-      const playback = await this.player.play(soundPath)
+      do {
-      this.currentPlayback = playback
+        this.streamPlayback.bufferEmptyFor
-      await playback.finished()
+        const lastSoundDir = dir
-    } while (this.currentPlayback)
+        const value = Math.random() * 100
+        if (lastSoundDir === "body-noises") {
+          dir = "apology"
+        } else if (value > 99 && !lastSoundDir) {
+          dir = "body-noises"
+        } else if (value > 75 && !lastSoundDir) {
+          dir = "stalling"
+        } else {
+          dir = undefined
+          await Bun.sleep(1000)
+        }
+
+        if (dir) {
+          const speakingSound = getSound(dir, Array.from(playedSounds))
+          this.speakingPlayback = await this.player.play(speakingSound)
+          playedSounds.add(speakingSound)
+          await this.speakingPlayback.finished()
+        }
+      } while (this.typingPlayback)
+      resolve()
+    })
   }
 
   async stop() {
-    if (!this.currentPlayback) return
+    if (!this.typingPlayback) return
-    await this.currentPlayback.finished()
-    this.currentPlayback = undefined
-  }
 
-  getRandomSoundDir(lastSoundDir?: SoundDir): SoundDir {
+    await Promise.all([this.typingPlayback.stop(), this.speakingPlayback?.finished()])
-    if (lastSoundDir === "body-noises") {
+    this.typingPlayback = undefined
-      return "apology"
-    }
-
-    const skipSpecialSounds =
-      (lastSoundDir !== "typing" && lastSoundDir !== "clicking") || !lastSoundDir
-
-    const value = Math.random() * 100
-    console.log(`🎲 got ${value}`)
-    if (value > 95 && !skipSpecialSounds) {
-      return "body-noises"
-    } else if (value > 66 && !skipSpecialSounds) {
-      return "stalling"
-    } else if (value > 33) {
-      return "clicking"
-    } else {
-      return "typing"
-    }
   }
 }
 
@@ -78,9 +101,3 @@ export const getSound = (dir: SoundDir, exclude: string[] = []): string => {
 
   return random(filteredSoundPaths)
 }
-
-const player = await Buzz.defaultPlayer()
-Buzz.setVolume(0.2)
-player.play(getSound("background"), { repeat: true })
-const x = new WaitingSounds(player)
-x.start()