From 0c8c5c73878a44b588af1ffa645563d9f94a9905 Mon Sep 17 00:00:00 2001 From: Corey Johnson Date: Mon, 23 Jun 2025 15:17:39 -0700 Subject: [PATCH] Move KV to separate files! --- packages/http/src/routes/todos/[id].tsx | 2 +- packages/shared/src/kv.ts | 139 ++++++++++++++++-------- packages/shared/src/utils.ts | 11 ++ packages/spike/src/eval.ts | 5 +- 4 files changed, 108 insertions(+), 49 deletions(-) diff --git a/packages/http/src/routes/todos/[id].tsx b/packages/http/src/routes/todos/[id].tsx index 2027284..8cd38be 100644 --- a/packages/http/src/routes/todos/[id].tsx +++ b/packages/http/src/routes/todos/[id].tsx @@ -10,7 +10,7 @@ export const head: Head = { export const loader = async (req: Request, params: { id: string }) => { const todos = await KV.get("todos", {}) - return { todos: todos[params.id] } + return { todos: todos![params.id] } } export const action = async (req: Request, params: { id: string }) => { diff --git a/packages/shared/src/kv.ts b/packages/shared/src/kv.ts index d7b2875..4821c9e 100644 --- a/packages/shared/src/kv.ts +++ b/packages/shared/src/kv.ts @@ -1,21 +1,7 @@ -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs" +import { mkdir } from "node:fs/promises" import { dirname, join } from "node:path" import type { Reminder } from "@/reminders" - -export type Conversation = { message: string; role: "user" | "assistant" } - -export type EvalMessage = { - content: string - role: "user" | "assistant" - author?: string -} - -export type EvalData = { - channel: string - rating: "good" | "bad" - context: EvalMessage[] // Last 5 messages leading up to the rated response - ratedBy: string -} +import { timeBomb } from "@/utils" export interface Keys { threads: Record // threadId: previousResponseId @@ -23,14 +9,18 @@ export interface Keys { todos: Record // todoId: todoText evaluations: EvalData[] // evaluation data for dashboard import } -const version = 10 + +// Individual versions for each key type +const keyVersions = { + threads: 1, + reminders: 1, + todos: 1, + evaluations: 1, +} as const const set = async (key: T, value: Keys[T]) => { try { - const store = readStore() - store[`${key}.${version}`] = value - writeStore(store) - + await writeStore(key, value) return value } catch (error) { console.error(`Error storing key "${key}":`, error) @@ -41,7 +31,7 @@ const set = async (key: T, value: Keys[T]) => { const update = async ( key: T, defaultValue: Keys[T], - updateFn: (prev: Keys[T]) => Promise | Keys[T] + updateFn: (prev: Keys[T]) => Keys[T] | Promise ) => { try { const currentValue = await get(key) @@ -52,56 +42,115 @@ const update = async ( throw error } } - -const get = async (key: T, defaultValue?: Keys[T]): Promise => { +const get = async (key: T, defaultValue?: Keys[T]): Promise => { try { - const store = readStore() - return store[`${key}.${version}`] ?? defaultValue + return (await readStore(key)) ?? defaultValue } catch (error) { console.error(`Error retrieving key "${key}":`, error) - throw error + return defaultValue } } -const remove = (key: T): void => { +const remove = async (key: T): Promise => { try { - const store = readStore() - delete store[`${key}.${version}`] - writeStore(store) + const keyPath = getStorePath(key) + const file = Bun.file(keyPath) + if (await file.exists()) { + await file.delete() + } } catch (error) { console.error(`Error removing key "${key}":`, error) throw error } } -const readStore = (): Record => { +const readStore = async (key: T): Promise => { try { - if (!existsSync(getStorePath())) { - return {} + const keyPath = getStorePath(key) + const file = Bun.file(keyPath) + if (!(await file.exists())) { + return undefined } - const data = readFileSync(getStorePath(), "utf-8") + const data = await file.text() return JSON.parse(data) } catch (error) { - console.error("Error reading KV store:", error) - return {} + console.error(`Error reading store for key "${key}":`, error) + return undefined } } -const writeStore = (store: Record): void => { +const writeStore = async (key: T, value: Keys[T]): Promise => { try { - mkdirSync(dirname(getStorePath()), { recursive: true }) - writeFileSync(getStorePath(), JSON.stringify(store, null, 2), "utf-8") + const keyPath = getStorePath(key) + await mkdir(dirname(keyPath), { recursive: true }) + await Bun.write(keyPath, JSON.stringify(value, null, 2)) } catch (error) { - console.error("Error writing KV store:", error) + console.error(`Error writing store for key "${key}":`, error) throw error } } -const getStorePath = (): string => { - console.log(`🌭`, process.env.DATA_DIR) +const getStorePath = (key: T): string => { const rootDir = process.env.DATA_DIR ?? join(import.meta.dir, "..") - const store = join(rootDir, "data/kv.json") - return store + const kvDir = join(rootDir, "data/kv") + const version = keyVersions[key] + return join(kvDir, `${key}.v${version}.json`) } +// Migration function to convert from old kv.json to new individual files +const migrateFromOldFormat = async (): Promise => { + timeBomb("2025-07-01", "This migration isn't needed anymore. Delete it.") + + try { + const rootDir = process.env.DATA_DIR ?? join(import.meta.dir, "..") + const oldKvPath = join(rootDir, "data/kv.json") + const oldFile = Bun.file(oldKvPath) + + if (!(await oldFile.exists())) { + return // No old file to migrate + } + + console.log("Migrating from old kv.json format to new individual files...") + + const oldData = await oldFile.json() + const migrations: Promise[] = [] + + // Migrate each key from the old format + for (const [oldKey, value] of Object.entries(oldData)) { + const keyParts = oldKey.split(".") + const keyName = keyParts[0] as keyof Keys + + // Only migrate if it's a valid key we recognize and value exists + if (keyName in keyVersions && value !== undefined && value !== null) { + console.log(`Migrating key: ${keyName}`) + migrations.push(writeStore(keyName, value as Keys[typeof keyName])) + } + } + + await Promise.all(migrations) + + const backupPath = join(rootDir, "data/kv.json.backup") + await Bun.write(backupPath, await oldFile.text()) + await oldFile.delete() + + console.log("Migration completed successfully. Old file backed up as kv.json.backup") + } catch (error) { + console.error("Error during migration:", error) + } +} +await migrateFromOldFormat() + export default { set, get, remove, update } + +export type Conversation = { message: string; role: "user" | "assistant" } + +export type EvalData = { + channel: string + rating: "good" | "bad" + context: { + content: string + role: "user" | "assistant" + author?: string + }[] // Last 5 messages leading up to the rated response + ratedBy: string +} diff --git a/packages/shared/src/utils.ts b/packages/shared/src/utils.ts index b7d2f6c..a2e9b48 100644 --- a/packages/shared/src/utils.ts +++ b/packages/shared/src/utils.ts @@ -22,4 +22,15 @@ export const random = (array: T[]): T => { return array[randomIndex]! } +// timeBomb is an assert that only goes off in development and after a certain date. It logs a warning if the condition is not met. +export const timeBomb = (date: string, message: string) => { + if (process.env.NODE_ENV === "production") return + const now = DateTime.now() + const targetDate = DateTime.fromISO(date) + + if (now > targetDate) { + console.warn(`💣 Time bomb triggered: ${message}`) + } +} + export const zone = "America/Los_Angeles" diff --git a/packages/spike/src/eval.ts b/packages/spike/src/eval.ts index 68f8266..6c46ce7 100644 --- a/packages/spike/src/eval.ts +++ b/packages/spike/src/eval.ts @@ -1,5 +1,5 @@ import kv from "@workshop/shared/kv" -import type { EvalMessage, EvalData } from "@workshop/shared/kv" +import type { EvalData } from "@workshop/shared/kv" import type { Message, PartialMessage } from "discord.js" export const storeEvaluation = async (message: Message | PartialMessage, emoji: string, ratedBy: string) => { @@ -49,7 +49,7 @@ export const storeEvaluation = async (message: Message | PartialMessage, emoji: } // Fetch conversation context around a message using Discord API -const fetchConversationContext = async (message: Message, contextSize = 5): Promise => { +const fetchConversationContext = async (message: Message, contextSize = 5): Promise => { try { // Fetch messages before the target message (the conversation leading up to this response) const messagesBefore = await message.channel.messages.fetch({ @@ -62,7 +62,6 @@ const fetchConversationContext = async (message: Message, contextSize = 5): Prom (a, b) => a.createdTimestamp - b.createdTimestamp ) - // Convert to EvalMessage format return allMessages.map((msg) => ({ content: msg.content, role: msg.author.bot ? "assistant" : "user",