better

packages/whiteboard/src/routes/voice.tsx

@@ -1,23 +1,46 @@
+import { useRef, useState, useEffect } from "hono/jsx"
 import { useStreamingAI } from "../useStreamingAI"
+import { useVideo } from "../useVideo"
+import { VideoOverlay, type OverlayItem } from "../videoOverlay"
 import "../index.css"
 
 export default function Voice() {
-  const { audioError, transcript, isRecording, waitingForResponse } = useStreamingAI()
+  const { audioError, transcript, isRecording: audioRecording, waitingForResponse } = useStreamingAI()
+  const videoRef = useRef<HTMLVideoElement>(null)
+  const video = useVideo(videoRef)
+
+  const [overlays, setOverlays] = useState<OverlayItem[]>([])
 
   let recordingStateClass = ""
-  if (isRecording) recordingStateClass = "border-red-500 border-4"
+  if (audioRecording) recordingStateClass = "border-red-500 border-4"
   else if (waitingForResponse) recordingStateClass = "border-yellow-500 border-4"
 
   return (
-    <div class={`fixed inset-0 p-5 transition-all duration-300 ${recordingStateClass}`}>
+    <div class="p-8 h-dvh w-full flex flex-col justify-center align-middle">
       {audioError && <p class="text-red-500">Audio Error: {audioError}</p>}
+      {video.error && <p class="text-red-500">Video Error: {video.error}</p>}
 
-      <div>
-        <h3 class="text-xl font-bold">Voice Control</h3>
-        <div class="text-gray-600">Hold Space key to record, release to transcribe</div>
-      </div>
+      {transcript && <div class="absolute top-5 left-5 right-5 bg-white/90 p-4 rounded-lg">{transcript}</div>}
 
-      {transcript && <div class="mt-5 bg-white/90 p-4 rounded-lg">{transcript}</div>}
+      {!video.isRecording && (
+        <button
+          onClick={video.toggleRecording}
+          class="px-4 py-2 text-8xl rounded-2xl text-white bg-green-500 hover:bg-green-600"
+        >
+          Start Camera
+        </button>
+      )}
+
+      <VideoOverlay overlays={overlays} isRecording={video.isRecording}>
+        <video
+          ref={videoRef}
+          autoPlay
+          muted
+          playsInline
+          class={`w-full h-full object-cover transition-all duration-300 ${recordingStateClass}`}
+        />
+      </VideoOverlay>
+      {video.isRecording && <div class="text-sm italic text-center">Hold Space to ask a question</div>}
     </div>
   )
 }

packages/whiteboard/src/server.ts

@@ -57,6 +57,7 @@ const streamResponse = async (req: Request) => {
   const result = await run(agent, input, { stream: true })
   const readableStream = result.toTextStream() as any // This DOES work, but typescript is a little confused so I cast it to any
 
+  console.log(`🌭`, readableStream)
   return new Response(readableStream, {
     headers: {
       "Content-Type": "text/plain",

packages/whiteboard/src/tools.ts

@@ -5,7 +5,7 @@ export const tools = [
   tool({
     name: "embed video",
     description: "Embed a video into the whiteboard",
-    parameters: z.object({ video: z.string().url() }),
+    parameters: z.object({ video: z.string() }),
     execute(input, context) {
       const { video } = input
       return `Video embedded: ${video}`

packages/whiteboard/src/useStreamingAI.ts

@@ -69,6 +69,6 @@ export function useStreamingAI() {
     isRecording,
     waitingForResponse,
     startRecording,
-    endRecording
+    endRecording,
   }
-}
+}

packages/whiteboard/src/useVideo.ts

@@ -3,11 +3,10 @@ import { ensure } from "@workshop/shared/utils"
 
 interface UseVideoOptions {
   captureInterval?: number
-  onCapture?: (dataUrl: string) => void
 }
 
 export function useVideo(videoRef: RefObject<HTMLVideoElement>, options: UseVideoOptions = {}) {
-  const { captureInterval = 1000, onCapture } = options
+  const { captureInterval = 10000 } = options
   const [isRecording, setIsRecording] = useState(false)
   const [error, setError] = useState<string | null>(null)
 
@@ -40,13 +39,27 @@ export function useVideo(videoRef: RefObject<HTMLVideoElement>, options: UseVide
     ctx.drawImage(video, 0, 0, newWidth, newHeight)
     const dataURL = canvas.toDataURL("image/png")
 
-    if (onCapture) {
-      onCapture(dataURL)
-    }
+    uploadImage(dataURL)
 
     return dataURL
   }
 
+  const uploadImage = async (dataURL: string) => {
+    const formData = new FormData()
+    formData.append("imageData", dataURL)
+
+    try {
+      const response = await fetch("/upload", {
+        method: "POST",
+        body: formData,
+      })
+      const result = await response.json()
+      console.log("Image uploaded:", result)
+    } catch (error) {
+      console.error("Upload failed:", error)
+    }
+  }
+
   const startCamera = async () => {
     try {
       const mediaStream = await navigator.mediaDevices.getUserMedia({