diff --git a/packages/whiteboard/public/whiteboard.png b/packages/whiteboard/public/whiteboard.png index 0bdf311..21b112d 100644 Binary files a/packages/whiteboard/public/whiteboard.png and b/packages/whiteboard/public/whiteboard.png differ diff --git a/packages/whiteboard/src/routes/voice.tsx b/packages/whiteboard/src/routes/voice.tsx index 152ad8f..399c207 100644 --- a/packages/whiteboard/src/routes/voice.tsx +++ b/packages/whiteboard/src/routes/voice.tsx @@ -1,23 +1,46 @@ +import { useRef, useState, useEffect } from "hono/jsx" import { useStreamingAI } from "../useStreamingAI" +import { useVideo } from "../useVideo" +import { VideoOverlay, type OverlayItem } from "../videoOverlay" import "../index.css" export default function Voice() { - const { audioError, transcript, isRecording, waitingForResponse } = useStreamingAI() + const { audioError, transcript, isRecording: audioRecording, waitingForResponse } = useStreamingAI() + const videoRef = useRef(null) + const video = useVideo(videoRef) + + const [overlays, setOverlays] = useState([]) let recordingStateClass = "" - if (isRecording) recordingStateClass = "border-red-500 border-4" + if (audioRecording) recordingStateClass = "border-red-500 border-4" else if (waitingForResponse) recordingStateClass = "border-yellow-500 border-4" return ( -
+
{audioError &&

Audio Error: {audioError}

} + {video.error &&

Video Error: {video.error}

} -
-

Voice Control

-
Hold Space key to record, release to transcribe
-
+ {transcript &&
{transcript}
} - {transcript &&
{transcript}
} + {!video.isRecording && ( + + )} + + + + {video.isRecording &&
Hold Space to ask a question
}
) } diff --git a/packages/whiteboard/src/server.ts b/packages/whiteboard/src/server.ts index 749214b..7444b90 100644 --- a/packages/whiteboard/src/server.ts +++ b/packages/whiteboard/src/server.ts @@ -57,6 +57,7 @@ const streamResponse = async (req: Request) => { const result = await run(agent, input, { stream: true }) const readableStream = result.toTextStream() as any // This DOES work, but typescript is a little confused so I cast it to any + console.log(`🌭`, readableStream) return new Response(readableStream, { headers: { "Content-Type": "text/plain", diff --git a/packages/whiteboard/src/tools.ts b/packages/whiteboard/src/tools.ts index d6ba564..d699c74 100644 --- a/packages/whiteboard/src/tools.ts +++ b/packages/whiteboard/src/tools.ts @@ -5,7 +5,7 @@ export const tools = [ tool({ name: "embed video", description: "Embed a video into the whiteboard", - parameters: z.object({ video: z.string().url() }), + parameters: z.object({ video: z.string() }), execute(input, context) { const { video } = input return `Video embedded: ${video}` diff --git a/packages/whiteboard/src/useStreamingAI.ts b/packages/whiteboard/src/useStreamingAI.ts index 6fa0d60..4dd2930 100644 --- a/packages/whiteboard/src/useStreamingAI.ts +++ b/packages/whiteboard/src/useStreamingAI.ts @@ -69,6 +69,6 @@ export function useStreamingAI() { isRecording, waitingForResponse, startRecording, - endRecording + endRecording, } -} \ No newline at end of file +} diff --git a/packages/whiteboard/src/useVideo.ts b/packages/whiteboard/src/useVideo.ts index c5fd6dc..88360dd 100644 --- a/packages/whiteboard/src/useVideo.ts +++ b/packages/whiteboard/src/useVideo.ts @@ -3,11 +3,10 @@ import { ensure } from "@workshop/shared/utils" interface UseVideoOptions { captureInterval?: number - onCapture?: (dataUrl: string) => void } export function useVideo(videoRef: RefObject, options: UseVideoOptions = {}) { - const { captureInterval = 1000, onCapture } = options + const { captureInterval = 10000 } = options const [isRecording, setIsRecording] = useState(false) const [error, setError] = useState(null) @@ -40,13 +39,27 @@ export function useVideo(videoRef: RefObject, options: UseVide ctx.drawImage(video, 0, 0, newWidth, newHeight) const dataURL = canvas.toDataURL("image/png") - if (onCapture) { - onCapture(dataURL) - } + uploadImage(dataURL) return dataURL } + const uploadImage = async (dataURL: string) => { + const formData = new FormData() + formData.append("imageData", dataURL) + + try { + const response = await fetch("/upload", { + method: "POST", + body: formData, + }) + const result = await response.json() + console.log("Image uploaded:", result) + } catch (error) { + console.error("Upload failed:", error) + } + } + const startCamera = async () => { try { const mediaStream = await navigator.mediaDevices.getUserMedia({