Move KV to separate files!

packages/http/src/routes/todos/[id].tsx

@@ -10,7 +10,7 @@ export const head: Head = {
 
 export const loader = async (req: Request, params: { id: string }) => {
   const todos = await KV.get("todos", {})
-  return { todos: todos[params.id] }
+  return { todos: todos![params.id] }
 }
 
 export const action = async (req: Request, params: { id: string }) => {

packages/shared/src/kv.ts

@@ -1,21 +1,7 @@
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"
+import { mkdir } from "node:fs/promises"
 import { dirname, join } from "node:path"
 import type { Reminder } from "@/reminders"
-
-export type Conversation = { message: string; role: "user" | "assistant" }
-
-export type EvalMessage = {
-  content: string
-  role: "user" | "assistant"
-  author?: string
-}
-
-export type EvalData = {
-  channel: string
-  rating: "good" | "bad"
-  context: EvalMessage[] // Last 5 messages leading up to the rated response
-  ratedBy: string
-}
+import { timeBomb } from "@/utils"
 
 export interface Keys {
   threads: Record<string, string> // threadId: previousResponseId
@@ -23,14 +9,18 @@ export interface Keys {
   todos: Record<string, string> // todoId: todoText
   evaluations: EvalData[] // evaluation data for dashboard import
 }
-const version = 10
+
+// Individual versions for each key type
+const keyVersions = {
+  threads: 1,
+  reminders: 1,
+  todos: 1,
+  evaluations: 1,
+} as const
 
 const set = async <T extends keyof Keys>(key: T, value: Keys[T]) => {
   try {
-    const store = readStore()
-    store[`${key}.${version}`] = value
-    writeStore(store)
-
+    await writeStore(key, value)
     return value
   } catch (error) {
     console.error(`Error storing key "${key}":`, error)
@@ -41,7 +31,7 @@ const set = async <T extends keyof Keys>(key: T, value: Keys[T]) => {
 const update = async <T extends keyof Keys>(
   key: T,
   defaultValue: Keys[T],
-  updateFn: (prev: Keys[T]) => Promise<Keys[T]> | Keys[T]
+  updateFn: (prev: Keys[T]) => Keys[T] | Promise<Keys[T]>
 ) => {
   try {
     const currentValue = await get(key)
@@ -52,56 +42,115 @@ const update = async <T extends keyof Keys>(
     throw error
   }
 }
-
-const get = async <T extends keyof Keys>(key: T, defaultValue?: Keys[T]): Promise<Keys[T]> => {
+const get = async <T extends keyof Keys>(key: T, defaultValue?: Keys[T]): Promise<Keys[T] | undefined> => {
   try {
-    const store = readStore()
-    return store[`${key}.${version}`] ?? defaultValue
+    return (await readStore(key)) ?? defaultValue
   } catch (error) {
     console.error(`Error retrieving key "${key}":`, error)
-    throw error
+    return defaultValue
   }
 }
 
-const remove = <T extends keyof Keys>(key: T): void => {
+const remove = async <T extends keyof Keys>(key: T): Promise<void> => {
   try {
-    const store = readStore()
-    delete store[`${key}.${version}`]
-    writeStore(store)
+    const keyPath = getStorePath(key)
+    const file = Bun.file(keyPath)
+    if (await file.exists()) {
+      await file.delete()
+    }
   } catch (error) {
     console.error(`Error removing key "${key}":`, error)
     throw error
   }
 }
 
-const readStore = (): Record<string, any> => {
+const readStore = async <T extends keyof Keys>(key: T): Promise<Keys[T] | undefined> => {
   try {
-    if (!existsSync(getStorePath())) {
-      return {}
+    const keyPath = getStorePath(key)
+    const file = Bun.file(keyPath)
+    if (!(await file.exists())) {
+      return undefined
     }
-    const data = readFileSync(getStorePath(), "utf-8")
+    const data = await file.text()
     return JSON.parse(data)
   } catch (error) {
-    console.error("Error reading KV store:", error)
-    return {}
+    console.error(`Error reading store for key "${key}":`, error)
+    return undefined
   }
 }
 
-const writeStore = (store: Record<string, any>): void => {
+const writeStore = async <T extends keyof Keys>(key: T, value: Keys[T]): Promise<void> => {
   try {
-    mkdirSync(dirname(getStorePath()), { recursive: true })
-    writeFileSync(getStorePath(), JSON.stringify(store, null, 2), "utf-8")
+    const keyPath = getStorePath(key)
+    await mkdir(dirname(keyPath), { recursive: true })
+    await Bun.write(keyPath, JSON.stringify(value, null, 2))
   } catch (error) {
-    console.error("Error writing KV store:", error)
+    console.error(`Error writing store for key "${key}":`, error)
     throw error
   }
 }
 
-const getStorePath = (): string => {
-  console.log(`🌭`, process.env.DATA_DIR)
+const getStorePath = <T extends keyof Keys>(key: T): string => {
   const rootDir = process.env.DATA_DIR ?? join(import.meta.dir, "..")
-  const store = join(rootDir, "data/kv.json")
-  return store
+  const kvDir = join(rootDir, "data/kv")
+  const version = keyVersions[key]
+  return join(kvDir, `${key}.v${version}.json`)
 }
 
+// Migration function to convert from old kv.json to new individual files
+const migrateFromOldFormat = async (): Promise<void> => {
+  timeBomb("2025-07-01", "This migration isn't needed anymore. Delete it.")
+
+  try {
+    const rootDir = process.env.DATA_DIR ?? join(import.meta.dir, "..")
+    const oldKvPath = join(rootDir, "data/kv.json")
+    const oldFile = Bun.file(oldKvPath)
+
+    if (!(await oldFile.exists())) {
+      return // No old file to migrate
+    }
+
+    console.log("Migrating from old kv.json format to new individual files...")
+
+    const oldData = await oldFile.json()
+    const migrations: Promise<void>[] = []
+
+    // Migrate each key from the old format
+    for (const [oldKey, value] of Object.entries(oldData)) {
+      const keyParts = oldKey.split(".")
+      const keyName = keyParts[0] as keyof Keys
+
+      // Only migrate if it's a valid key we recognize and value exists
+      if (keyName in keyVersions && value !== undefined && value !== null) {
+        console.log(`Migrating key: ${keyName}`)
+        migrations.push(writeStore(keyName, value as Keys[typeof keyName]))
+      }
+    }
+
+    await Promise.all(migrations)
+
+    const backupPath = join(rootDir, "data/kv.json.backup")
+    await Bun.write(backupPath, await oldFile.text())
+    await oldFile.delete()
+
+    console.log("Migration completed successfully. Old file backed up as kv.json.backup")
+  } catch (error) {
+    console.error("Error during migration:", error)
+  }
+}
+await migrateFromOldFormat()
+
 export default { set, get, remove, update }
+
+export type Conversation = { message: string; role: "user" | "assistant" }
+
+export type EvalData = {
+  channel: string
+  rating: "good" | "bad"
+  context: {
+    content: string
+    role: "user" | "assistant"
+    author?: string
+  }[] // Last 5 messages leading up to the rated response
+  ratedBy: string
+}

packages/shared/src/utils.ts

@@ -22,4 +22,15 @@ export const random = <T>(array: T[]): T => {
   return array[randomIndex]!
 }
 
+// timeBomb is an assert that only goes off in development and after a certain date. It logs a warning if the condition is not met.
+export const timeBomb = (date: string, message: string) => {
+  if (process.env.NODE_ENV === "production") return
+  const now = DateTime.now()
+  const targetDate = DateTime.fromISO(date)
+
+  if (now > targetDate) {
+    console.warn(`💣 Time bomb triggered: ${message}`)
+  }
+}
+
 export const zone = "America/Los_Angeles"

packages/spike/src/eval.ts

@@ -1,5 +1,5 @@
 import kv from "@workshop/shared/kv"
-import type { EvalMessage, EvalData } from "@workshop/shared/kv"
+import type { EvalData } from "@workshop/shared/kv"
 import type { Message, PartialMessage } from "discord.js"
 
 export const storeEvaluation = async (message: Message | PartialMessage, emoji: string, ratedBy: string) => {
@@ -49,7 +49,7 @@ export const storeEvaluation = async (message: Message | PartialMessage, emoji:
 }
 
 // Fetch conversation context around a message using Discord API
-const fetchConversationContext = async (message: Message, contextSize = 5): Promise<EvalMessage[]> => {
+const fetchConversationContext = async (message: Message, contextSize = 5): Promise<EvalData["context"]> => {
   try {
     // Fetch messages before the target message (the conversation leading up to this response)
     const messagesBefore = await message.channel.messages.fetch({
@@ -62,7 +62,6 @@ const fetchConversationContext = async (message: Message, contextSize = 5): Prom
       (a, b) => a.createdTimestamp - b.createdTimestamp
     )
 
-    // Convert to EvalMessage format
     return allMessages.map((msg) => ({
       content: msg.content,
       role: msg.author.bot ? "assistant" : "user",