diff --git a/packages/iago/README.md b/packages/iago/README.md new file mode 100644 index 0000000..3fb4cd5 --- /dev/null +++ b/packages/iago/README.md @@ -0,0 +1,15 @@ +# iago + +To install dependencies: + +```bash +bun install +``` + +To run: + +```bash +bun run index.ts +``` + +This project was created using `bun init` in bun v1.2.13. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime. diff --git a/packages/iago/bun.lock b/packages/iago/bun.lock new file mode 100644 index 0000000..c2b58fd --- /dev/null +++ b/packages/iago/bun.lock @@ -0,0 +1,25 @@ +{ + "lockfileVersion": 1, + "workspaces": { + "": { + "name": "iago", + "devDependencies": { + "@types/bun": "latest", + }, + "peerDependencies": { + "typescript": "^5", + }, + }, + }, + "packages": { + "@types/bun": ["@types/bun@1.2.16", "", { "dependencies": { "bun-types": "1.2.16" } }, "sha512-1aCZJ/6nSiViw339RsaNhkNoEloLaPzZhxMOYEa7OzRzO41IGg5n/7I43/ZIAW/c+Q6cT12Vf7fOZOoVIzb5BQ=="], + + "@types/node": ["@types/node@24.0.3", "", { "dependencies": { "undici-types": "~7.8.0" } }, "sha512-R4I/kzCYAdRLzfiCabn9hxWfbuHS573x+r0dJMkkzThEa7pbrcDWK+9zu3e7aBOouf+rQAciqPFMnxwr0aWgKg=="], + + "bun-types": ["bun-types@1.2.16", "", { "dependencies": { "@types/node": "*" } }, "sha512-ciXLrHV4PXax9vHvUrkvun9VPVGOVwbbbBF/Ev1cXz12lyEZMoJpIJABOfPcN9gDJRaiKF9MVbSygLg4NXu3/A=="], + + "typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="], + + "undici-types": ["undici-types@7.8.0", "", {}, "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw=="], + } +} diff --git a/packages/iago/package.json b/packages/iago/package.json new file mode 100644 index 0000000..dcc030e --- /dev/null +++ b/packages/iago/package.json @@ -0,0 +1,16 @@ +{ + "name": "@workshop/iago", + "module": "index.ts", + "type": "module", + "private": true, + "scripts": { + "dev": "bun run --hot server.ts", + "start": "bun run server.ts" + }, + "devDependencies": { + "@types/bun": "latest" + }, + "peerDependencies": { + "typescript": "^5" + } +} diff --git a/packages/iago/photo.jpg b/packages/iago/photo.jpg new file mode 100644 index 0000000..6041a65 Binary files /dev/null and b/packages/iago/photo.jpg differ diff --git a/packages/iago/public/talk.html b/packages/iago/public/talk.html new file mode 100644 index 0000000..8f97f00 --- /dev/null +++ b/packages/iago/public/talk.html @@ -0,0 +1,263 @@ + + + + + + Voice Recorder + + + +
+

Voice Recorder

+ + + +
+ Click the mic to start recording +
+ + + + + + +
+ + + + \ No newline at end of file diff --git a/packages/iago/server.ts b/packages/iago/server.ts new file mode 100644 index 0000000..1cf2fd9 --- /dev/null +++ b/packages/iago/server.ts @@ -0,0 +1,146 @@ +import { $ } from "bun" +import path from "path" +import { Hono } from "hono" +import { openai, createFile } from "./src/openai" +import { serveStatic } from "hono/bun" +import fs from "fs" + +const CAMERA = process.env.IAGO_CAMERA || "Set IAGO_CAMERA environment variable" +const PROMPT = "What text do you see in this image?" + +const IMAGE_PATH = "./photo.jpg" +const app = new Hono() + +// Serve static files from public directory +app.use("/*", serveStatic({ root: "./public" })) + +app.get("/", (c) => { + return c.json({ message: "Hello World" }) +}) + +// you know +app.get("/status", async (c) => { + if (!(await checkImagesnap())) + return c.json({ status: "imagesnap not found", devices: [], camera: CAMERA }) + + const devices = await getDevices() + return c.json({ devices, camera: CAMERA, status: devices.includes(CAMERA) ? "camera found" : "camera not found" }) +}) + +// take a picture with the camera +app.get("/capture", async (c) => { + try { + await runImagesnap() + const image = await Bun.file(IMAGE_PATH).arrayBuffer() + return new Response(image, { + headers: { "Content-Type": "image/jpeg" }, + }) + } catch (err: any) { + return c.json({ error: err.message }, 500) + } +}) + +// capture and analyze image +app.get("/analyze", async (c) => { + try { + // await runImagesnap() + + const fileId = await createFile(IMAGE_PATH) + + const result = await openai.responses.create({ + model: "gpt-4o", + input: [ + { + role: "user", + content: [ + { type: "input_text", text: PROMPT }, + { type: "input_image", file_id: fileId, detail: "high" } + ] + } + ] + }) + + return c.json({ result: result.output_text }) + } catch (err: any) { + return c.json({ error: err.message }, 500) + } +}) + +app.get("/speak", async (c) => { + const mp3 = await openai.audio.speech.create({ + model: "gpt-4o-mini-tts", + voice: "ash", + instructions: "Speak in an upbeat and goofy tone!", + input: "I put the MAN in MAN-UH!" + }) + + const buffer = Buffer.from(await mp3.arrayBuffer()) + + return new Response(buffer, { + headers: { + "Content-Type": "audio/mpeg", + "Content-Disposition": "inline; filename=speak.mp3" + }, + }) +}) + +app.get("/talk", async (c) => { + return c.redirect("/talk.html") +}) + +app.post("/transcribe", async (c) => { + try { + const formData = await c.req.formData() + const audioFile = formData.get("audio") as File + + if (!audioFile) { + return c.json({ error: "No audio file provided" }, 400) + } + + // Convert File to Buffer + const arrayBuffer = await audioFile.arrayBuffer() + const buffer = Buffer.from(arrayBuffer) + + // Create a temporary file + const tempPath = `./temp_audio_${Date.now()}.webm` + await Bun.write(tempPath, buffer) + + const transcription = await openai.audio.transcriptions.create({ + file: Bun.file(tempPath), + model: "gpt-4o-transcribe", + response_format: "text", + }) + + // Clean up temporary file + await fs.promises.unlink(tempPath) + + console.log(transcription) + return c.json({ transcription }) + } catch (err: any) { + console.error("Transcription error:", err) + return c.json({ error: err.message }, 500) + } +}) + +// Capture image using Bun.spawn +async function runImagesnap(): Promise { + const proc = await $`imagesnap -d ${CAMERA} ${IMAGE_PATH}` + if (proc.exitCode !== 0) throw new Error("imagesnap failed") +} + +// Check if imagesnap exists using Bun.spawnSync +async function checkImagesnap(): Promise { + const result = await $`which imagesnap` + return result.exitCode === 0 +} + +// Get devices using imagesnap +async function getDevices(): Promise { + const result = await $`imagesnap -l` + return result.stdout.toString().split("\n").filter(line => line.startsWith("=> ")).map(line => line.replace("=> ", "")) +} + +export default { + port: 3000, + fetch: app.fetch, +} \ No newline at end of file diff --git a/packages/iago/src/openai.ts b/packages/iago/src/openai.ts new file mode 100644 index 0000000..cf07a2c --- /dev/null +++ b/packages/iago/src/openai.ts @@ -0,0 +1,16 @@ +import OpenAI from "openai" + +export const openai = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY || "", +}) + + +// Function to create a file with the Files API +export async function createFile(filePath: string) { + const fileContent = Bun.file(filePath) + const result = await openai.files.create({ + file: fileContent, + purpose: "vision", + }) + return result.id +} \ No newline at end of file diff --git a/packages/iago/test.jpg b/packages/iago/test.jpg new file mode 100644 index 0000000..6041a65 Binary files /dev/null and b/packages/iago/test.jpg differ diff --git a/packages/iago/tsconfig.json b/packages/iago/tsconfig.json new file mode 100644 index 0000000..9c62f74 --- /dev/null +++ b/packages/iago/tsconfig.json @@ -0,0 +1,28 @@ +{ + "compilerOptions": { + // Environment setup & latest features + "lib": ["ESNext"], + "target": "ESNext", + "module": "ESNext", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +}