diff --git a/.gitignore b/.gitignore index a14702c..ba79468 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,4 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json .idea # Finder (MacOS) folder config -.DS_Store +.DS_Store \ No newline at end of file diff --git a/scripts/deploy.ts b/scripts/deploy.ts index 59f9ce1..feb253e 100755 --- a/scripts/deploy.ts +++ b/scripts/deploy.ts @@ -20,7 +20,7 @@ await $`ssh ${PI_HOST} "mkdir -p ${PI_DIR}"` // Sync files from . directory to Pi (only transfers changed files) console.log("Syncing files from . directory...") -await $`rsync -avz --delete --exclude-from='.gitignore' . ${PI_HOST}:${PI_DIR}/` +await $`rsync -avz --delete --exclude-from='../.gitignore' --exclude='.git' . ${PI_HOST}:${PI_DIR}/` // Make all TypeScript files executable console.log("Making scripts executable...") diff --git a/sounds/apology/excuse-me1.wav b/sounds/apology/excuse-me1.wav index 43693fc..504afb2 100644 Binary files a/sounds/apology/excuse-me1.wav and b/sounds/apology/excuse-me1.wav differ diff --git a/sounds/apology/excuse-me2.wav b/sounds/apology/excuse-me2.wav index e6a86a1..95f726e 100644 Binary files a/sounds/apology/excuse-me2.wav and b/sounds/apology/excuse-me2.wav differ diff --git a/sounds/apology/excuse-me3.wav b/sounds/apology/excuse-me3.wav index d3ab00b..b1026c6 100644 Binary files a/sounds/apology/excuse-me3.wav and b/sounds/apology/excuse-me3.wav differ diff --git a/sounds/stalling/cough1.wav b/sounds/stalling/cough1.wav index 21e5148..0897991 100644 Binary files a/sounds/stalling/cough1.wav and b/sounds/stalling/cough1.wav differ diff --git a/sounds/stalling/cough2.wav b/sounds/stalling/cough2.wav index 1ae8acf..bc3f72c 100644 Binary files a/sounds/stalling/cough2.wav and b/sounds/stalling/cough2.wav differ diff --git a/sounds/stalling/hmm1.wav b/sounds/stalling/hmm1.wav index 155a129..460f5d1 100644 Binary files a/sounds/stalling/hmm1.wav and b/sounds/stalling/hmm1.wav differ diff --git a/sounds/stalling/hmm2.wav b/sounds/stalling/hmm2.wav index 169da0e..56eac19 100644 Binary files a/sounds/stalling/hmm2.wav and b/sounds/stalling/hmm2.wav differ diff --git a/sounds/stalling/one-sec.wav b/sounds/stalling/one-sec.wav index 7d1b497..daba071 100644 Binary files a/sounds/stalling/one-sec.wav and b/sounds/stalling/one-sec.wav differ diff --git a/sounds/stalling/sigh1.wav b/sounds/stalling/sigh1.wav index 66bdc8c..5aa7c43 100644 Binary files a/sounds/stalling/sigh1.wav and b/sounds/stalling/sigh1.wav differ diff --git a/sounds/stalling/sneeze.wav b/sounds/stalling/sneeze.wav index 99076de..83784c3 100644 Binary files a/sounds/stalling/sneeze.wav and b/sounds/stalling/sneeze.wav differ diff --git a/sounds/stalling/uh-huh.wav b/sounds/stalling/uh-huh.wav index 89d3cb6..e09da43 100644 Binary files a/sounds/stalling/uh-huh.wav and b/sounds/stalling/uh-huh.wav differ diff --git a/sounds/stalling/yeah.wav b/sounds/stalling/yeah.wav index e4adad0..5895f1f 100644 Binary files a/sounds/stalling/yeah.wav and b/sounds/stalling/yeah.wav differ diff --git a/src/agent/tools.ts b/src/agent/tools.ts index d508336..1e318d6 100644 --- a/src/agent/tools.ts +++ b/src/agent/tools.ts @@ -26,5 +26,7 @@ export const searchWeb = async (query: string) => { input: `Search the web for: ${query}`, }) + await Bun.sleep(10000) + return response.output_text } diff --git a/src/buzz/player.ts b/src/buzz/player.ts index df0dcea..04e251a 100644 --- a/src/buzz/player.ts +++ b/src/buzz/player.ts @@ -151,13 +151,26 @@ export class Player { stderr: "pipe", }) + let bufferFinishTime = performance.now() + const format = this.#format + const bytesPerSecond = format.sampleRate * format.channels * 2 + const handle: StreamingPlayback = { isPlaying: true, write: (chunk: Uint8Array) => { if (handle.isPlaying) { proc.stdin.write(chunk) + const chunkDurationMs = (chunk.byteLength / bytesPerSecond) * 1000 + bufferFinishTime = Math.max(performance.now(), bufferFinishTime) + chunkDurationMs } }, + get bufferEmptyFor() { + const now = performance.now() + if (now > bufferFinishTime) { + return now - bufferFinishTime + } + return 0 + }, stop: async () => { if (!handle.isPlaying) return handle.isPlaying = false diff --git a/src/buzz/utils.ts b/src/buzz/utils.ts index 20263ae..9c06042 100644 --- a/src/buzz/utils.ts +++ b/src/buzz/utils.ts @@ -33,6 +33,7 @@ export type StreamingPlayback = { isPlaying: boolean; write: (chunk: Uint8Array) => void; stop: () => Promise; + bufferEmptyFor: number; // milliseconds since buffer became empty, 0 if not empty }; // Streaming recording control handle diff --git a/src/operator.ts b/src/operator.ts index 2f19d94..fc02e02 100755 --- a/src/operator.ts +++ b/src/operator.ts @@ -21,8 +21,8 @@ const runPhoneSystem = async (agentId: string, apiKey: string) => { let currentDialtone: Playback | undefined let currentBackgroundNoise: Playback | undefined - let currentPlayback = player.playStream() - const waitingIndicator = new WaitingSounds(player) + let streamPlayback = player.playStream() + const waitingIndicator = new WaitingSounds(player, streamPlayback) // Set up agent event listeners agent.events.connect((event) => { @@ -42,13 +42,13 @@ const runPhoneSystem = async (agentId: string, apiKey: string) => { case "audio": waitingIndicator.stop() const audioBuffer = Buffer.from(event.audioBase64, "base64") - currentPlayback.write(audioBuffer) + streamPlayback.write(audioBuffer) break case "interruption": console.log("šŸ›‘ User interrupted") - currentPlayback?.stop() - currentPlayback = player.playStream() // Reset playback stream + streamPlayback?.stop() + streamPlayback = player.playStream() // Reset playback stream break case "tool_call": @@ -66,7 +66,7 @@ const runPhoneSystem = async (agentId: string, apiKey: string) => { case "disconnected": console.log("\nšŸ‘‹ Conversation ended, returning to dialtone\n") - currentPlayback?.stop() + streamPlayback?.stop() state = "WAITING_FOR_VOICE" startDialtone() break @@ -94,9 +94,7 @@ const runPhoneSystem = async (agentId: string, apiKey: string) => { const stopDialtone = async () => { await currentDialtone?.stop() currentDialtone = undefined - currentBackgroundNoise = await player.play(getSound("background"), { - repeat: true, - }) + currentBackgroundNoise = await player.play(getSound("background"), { repeat: true }) } const startConversation = async () => { @@ -139,7 +137,7 @@ const runPhoneSystem = async (agentId: string, apiKey: string) => { console.log("\n\nšŸ›‘ Shutting down phone system...") await currentDialtone?.stop() await currentBackgroundNoise?.stop() - await currentPlayback?.stop() + await streamPlayback?.stop() await agent.stop() process.exit(0) } diff --git a/src/utils/waiting-sounds.ts b/src/utils/waiting-sounds.ts index 9efd459..a46925b 100644 --- a/src/utils/waiting-sounds.ts +++ b/src/utils/waiting-sounds.ts @@ -1,57 +1,80 @@ -import Buzz, { type Player } from "../buzz/index.ts" +import { type Player } from "../buzz/index.ts" import { join } from "path" -import type { Playback } from "../buzz/utils.ts" +import type { Playback, StreamingPlayback } from "../buzz/utils.ts" import { random } from "./index.ts" export class WaitingSounds { - currentPlayback?: Playback + typingPlayback?: Playback + speakingPlayback?: Playback - constructor(private player: Player) {} + constructor(private player: Player, private streamPlayback: StreamingPlayback) {} async start() { - if (this.currentPlayback) return // Already playing + if (this.typingPlayback) return // Already playing - // Now play randomly play things + this.#startTypingSounds() + this.#startSpeakingSounds() + } + + async #startTypingSounds() { + return new Promise(async (resolve) => { + do { + const value = Math.random() * 100 + let dir: SoundDir + if (value > 33) { + dir = "typing" + } else { + dir = "clicking" + } + const typingSound = getSound(dir) + this.typingPlayback = await this.player.play(typingSound) + await this.typingPlayback.finished() + } while (this.typingPlayback) + + resolve() + }) + } + + async #startSpeakingSounds() { const playedSounds = new Set() - let lastSoundDir: SoundDir | undefined - do { - const dir = this.getRandomSoundDir(lastSoundDir) - const soundPath = getSound(dir, Array.from(playedSounds)) - playedSounds.add(soundPath) - lastSoundDir = dir - console.log(`šŸŒ­ playing ${soundPath}`) + let dir: SoundDir | undefined + return new Promise(async (resolve) => { + // Don't start speaking until the stream playback buffer is empty! + while (this.streamPlayback.bufferEmptyFor < 1000) { + await Bun.sleep(100) + } - const playback = await this.player.play(soundPath) - this.currentPlayback = playback - await playback.finished() - } while (this.currentPlayback) + do { + this.streamPlayback.bufferEmptyFor + const lastSoundDir = dir + const value = Math.random() * 100 + if (lastSoundDir === "body-noises") { + dir = "apology" + } else if (value > 99 && !lastSoundDir) { + dir = "body-noises" + } else if (value > 75 && !lastSoundDir) { + dir = "stalling" + } else { + dir = undefined + await Bun.sleep(1000) + } + + if (dir) { + const speakingSound = getSound(dir, Array.from(playedSounds)) + this.speakingPlayback = await this.player.play(speakingSound) + playedSounds.add(speakingSound) + await this.speakingPlayback.finished() + } + } while (this.typingPlayback) + resolve() + }) } async stop() { - if (!this.currentPlayback) return - await this.currentPlayback.finished() - this.currentPlayback = undefined - } + if (!this.typingPlayback) return - getRandomSoundDir(lastSoundDir?: SoundDir): SoundDir { - if (lastSoundDir === "body-noises") { - return "apology" - } - - const skipSpecialSounds = - (lastSoundDir !== "typing" && lastSoundDir !== "clicking") || !lastSoundDir - - const value = Math.random() * 100 - console.log(`šŸŽ² got ${value}`) - if (value > 95 && !skipSpecialSounds) { - return "body-noises" - } else if (value > 66 && !skipSpecialSounds) { - return "stalling" - } else if (value > 33) { - return "clicking" - } else { - return "typing" - } + await Promise.all([this.typingPlayback.stop(), this.speakingPlayback?.finished()]) + this.typingPlayback = undefined } } @@ -78,9 +101,3 @@ export const getSound = (dir: SoundDir, exclude: string[] = []): string => { return random(filteredSoundPaths) } - -const player = await Buzz.defaultPlayer() -Buzz.setVolume(0.2) -player.play(getSound("background"), { repeat: true }) -const x = new WaitingSounds(player) -x.start()