From 7585f0e8a2a2715b4bb684364aea2ce27b2f5139 Mon Sep 17 00:00:00 2001 From: Corey Johnson Date: Mon, 29 Sep 2025 10:00:26 -0700 Subject: [PATCH] wip --- src/evaluator/evaluator.ts | 13 +++-- src/parser/highlight.js | 7 +++ src/parser/shrimp.grammar | 20 +++---- src/parser/shrimp.terms.ts | 16 +++-- src/parser/shrimp.test.ts | 110 +++++++++++++++++++++-------------- src/parser/shrimp.ts | 12 ++-- src/parser/test-helper.ts | 37 +++++++++--- src/server/debugPlugin.ts | 71 ++++++++++++++++++++++ src/server/editor.tsx | 12 +++- src/server/editorTheme.tsx | 34 +++++++++-- src/server/shrimpLanguage.ts | 11 ++++ 11 files changed, 256 insertions(+), 87 deletions(-) create mode 100644 src/server/debugPlugin.ts create mode 100644 src/server/shrimpLanguage.ts diff --git a/src/evaluator/evaluator.ts b/src/evaluator/evaluator.ts index f6b7797..a2269a8 100644 --- a/src/evaluator/evaluator.ts +++ b/src/evaluator/evaluator.ts @@ -1,5 +1,6 @@ import { nodeToString } from '@/evaluator/treeHelper' import { Tree, type SyntaxNode } from '@lezer/common' +import * as terms from '../parser/shrimp.terms.ts' type Context = Map @@ -29,19 +30,19 @@ const evaluateNode = (node: SyntaxNode, input: string, context: Context): any => const value = input.slice(node.from, node.to) try { - switch (node.name) { - case 'Number': { + switch (node.type.id) { + case terms.Number: { return parseFloat(value) } - case 'Identifier': { + case terms.Identifier: { if (!context.has(value)) { throw new Error(`Undefined identifier: ${value}`) } return context.get(value) } - case 'BinOp': { + case terms.BinOp: { let [left, op, right] = getChildren(node) left = assertNode(left, 'LeftOperand') @@ -66,7 +67,7 @@ const evaluateNode = (node: SyntaxNode, input: string, context: Context): any => } } - case 'Assignment': { + case terms.Assignment: { const [identifier, expr] = getChildren(node) const identifierNode = assertNode(identifier, 'Identifier') @@ -78,7 +79,7 @@ const evaluateNode = (node: SyntaxNode, input: string, context: Context): any => return value } - case 'Function': { + case terms.Function: { const [params, body] = getChildren(node) const paramNodes = getChildren(assertNode(params, 'Parameters')) diff --git a/src/parser/highlight.js b/src/parser/highlight.js index 5adeb55..374b6ff 100644 --- a/src/parser/highlight.js +++ b/src/parser/highlight.js @@ -1,7 +1,14 @@ +import { Identifier, Params } from '@/parser/shrimp.terms' import { styleTags, tags } from '@lezer/highlight' export const highlighting = styleTags({ Identifier: tags.name, Number: tags.number, String: tags.string, + Boolean: tags.bool, + Keyword: tags.keyword, + Operator: tags.operator, + // Params: tags.definition(tags.variableName), + 'Params/Identifier': tags.definition(tags.variableName), + Paren: tags.paren, }) diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index 1c25312..790ebbe 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -12,15 +12,15 @@ Boolean { "true" | "false" } String { '"' !["]* '"' } Identifier { $[A-Za-z_]$[A-Za-z_0-9-]* } - fn { "fn" } - arrow { "->" } - equals { "=" } - "+" - "-" - "*" - "/" - leftParen { "(" } - rightParen { ")" } + fn[@name=Keyword] { "fn" } + equals[@name=Operator] { "=" } + ":"[@name=Colon] + "+"[@name=Operator] + "-"[@name=Operator] + "*"[@name=Operator] + "/"[@name=Operator] + leftParen[@name=Paren] { "(" } + rightParen[@name=Paren] { ")" } } @precedence { @@ -45,7 +45,7 @@ BinOp { } Params { Identifier* } -Function { !function fn Params arrow expr } +Function { !function fn Params ":" expr } atom { Identifier | Number | String | Boolean | leftParen expr rightParen } diff --git a/src/parser/shrimp.terms.ts b/src/parser/shrimp.terms.ts index 0d856a9..07cd39e 100644 --- a/src/parser/shrimp.terms.ts +++ b/src/parser/shrimp.terms.ts @@ -3,9 +3,13 @@ export const Program = 1, Assignment = 2, Identifier = 3, - Function = 4, - Params = 5, - BinOp = 6, - Number = 11, - String = 12, - Boolean = 13 + equals = 4, + Function = 5, + fn = 6, + Params = 7, + BinOp = 9, + Number = 14, + String = 15, + Boolean = 16, + leftParen = 17, + rightParen = 18 diff --git a/src/parser/shrimp.test.ts b/src/parser/shrimp.test.ts index f64ecbe..7529434 100644 --- a/src/parser/shrimp.test.ts +++ b/src/parser/shrimp.test.ts @@ -1,59 +1,59 @@ -import { expectTree, regenerateParser } from '@/parser/test-helper' -import { beforeAll, describe, test } from 'bun:test' +import { regenerateParser } from '@/parser/test-helper' +import { expect, beforeAll, describe, test } from 'bun:test' describe('BinOp', () => { beforeAll(() => regenerateParser()) test('addition tests', () => { - expectTree('2 + 3').toMatch(` + expect('2 + 3').toMatchTree(` BinOp Number 2 - + + Operator + Number 3 `) }) test('subtraction tests', () => { - expectTree('5 - 2').toMatch(` + expect('5 - 2').toMatchTree(` BinOp Number 5 - - + Operator - Number 2 `) }) test('multiplication tests', () => { - expectTree('4 * 3').toMatch(` + expect('4 * 3').toMatchTree(` BinOp Number 4 - * + Operator * Number 3 `) }) test('division tests', () => { - expectTree('8 / 2').toMatch(` + expect('8 / 2').toMatchTree(` BinOp Number 8 - / + Operator / Number 2 `) }) test('mixed operations with precedence', () => { - expectTree('2 + 3 * 4 - 5 / 1').toMatch(` + expect('2 + 3 * 4 - 5 / 1').toMatchTree(` BinOp BinOp Number 2 - + + Operator + BinOp Number 3 - * + Operator * Number 4 - - + Operator - BinOp Number 5 - / + Operator / Number 1 `) }) @@ -63,39 +63,47 @@ describe('Fn', () => { beforeAll(() => regenerateParser()) test('parses function with single parameter', () => { - expectTree('fn x -> x + 1').toMatch(` + expect('fn x: x + 1').toMatchTree(` Function + Keyword fn Params Identifier x + Colon : BinOp Identifier x - + + Operator + Number 1`) }) test('parses function with multiple parameters', () => { - expectTree('fn x y -> x * y').toMatch(` + expect('fn x y: x * y').toMatchTree(` Function + Keyword fn Params Identifier x Identifier y + Colon : BinOp Identifier x - * + Operator * Identifier y`) }) test('parses nested functions', () => { - expectTree('fn x -> fn y -> x + y').toMatch(` + expect('fn x: fn y: x + y').toMatchTree(` Function + Keyword fn Params Identifier x + Colon : Function + Keyword fn Params Identifier y + Colon : BinOp Identifier x - + + Operator + Identifier y`) }) }) @@ -104,22 +112,22 @@ describe('Identifier', () => { beforeAll(() => regenerateParser()) test('parses hyphenated identifiers correctly', () => { - expectTree('my-var - another-var').toMatch(` + expect('my-var - another-var').toMatchTree(` BinOp Identifier my-var - - + Operator - Identifier another-var`) - expectTree('double--trouble - another-var').toMatch(` + expect('double--trouble - another-var').toMatchTree(` BinOp Identifier double--trouble - - + Operator - Identifier another-var`) - expectTree('tail-- - another-var').toMatch(` + expect('tail-- - another-var').toMatchTree(` BinOp Identifier tail-- - - + Operator - Identifier another-var`) }) }) @@ -128,26 +136,30 @@ describe('Assignment', () => { beforeAll(() => regenerateParser()) test('parses assignment with addition', () => { - expectTree('x = 5 + 3').toMatch(` + expect('x = 5 + 3').toMatchTree(` Assignment Identifier x + Operator = BinOp Number 5 - + + Operator + Number 3`) }) test('parses assignment with functions', () => { - expectTree('add = fn a b -> a + b').toMatch(` + expect('add = fn a b: a + b').toMatchTree(` Assignment Identifier add + Operator = Function + Keyword fn Params Identifier a Identifier b + Colon : BinOp Identifier a - + + Operator + Identifier b`) }) }) @@ -156,30 +168,38 @@ describe('Parentheses', () => { beforeAll(() => regenerateParser()) test('parses expressions with parentheses correctly', () => { - expectTree('(2 + 3) * 4').toMatch(` + expect('(2 + 3) * 4').toMatchTree(` BinOp + Paren ( BinOp Number 2 - + + Operator + Number 3 - * + Paren ) + Operator * Number 4`) }) test('parses nested parentheses correctly', () => { - expectTree('((1 + 2) * (3 - 4)) / 5').toMatch(` + expect('((1 + 2) * (3 - 4)) / 5').toMatchTree(` BinOp + Paren ( BinOp + Paren ( BinOp Number 1 - + + Operator + Number 2 - * + Paren ) + Operator * + Paren ( BinOp Number 3 - - + Operator - Number 4 - / + Paren ) + Paren ) + Operator / Number 5`) }) }) @@ -188,20 +208,22 @@ describe('multiline', () => { beforeAll(() => regenerateParser()) test('parses multiline expressions', () => { - expectTree(` + expect(` 5 + 4 - fn x -> x - 1 - `).toMatch(` + fn x: x - 1 + `).toMatchTree(` BinOp Number 5 - + + Operator + Number 4 Function + Keyword fn Params Identifier x + Colon : BinOp Identifier x - - + Operator - Number 1 `) }) diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index f798f00..f25632f 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -3,16 +3,16 @@ import {LRParser} from "@lezer/lr" import {highlighting} from "./highlight.js" export const parser = LRParser.deserialize({ version: 14, - states: "$OQVQPOOOkQPO'#CsO!fQQO'#C`O!nQPO'#CjOOQO'#Cs'#CsOVQPO'#CsOOQO'#Co'#CoQVQPOOOVQPO,58xOOQO'#Ck'#CkO#cQQO'#CaO#kQQO,58zOVQPO,58|OVQPO,58|O#pQPO,59_OOQO-E6h-E6hO$RQPO1G.dOOQO-E6i-E6iOVQPO1G.fOOQO1G.h1G.hO$yQPO1G.hOOQO1G.y1G.yO%qQPO7+$Q", - stateData: "&n~ObOS~ORPOZSO[SO]SOeQOhTO~OdWORgXVgXWgXXgXYgXZgX[gX]gX`gXegXhgXigX~ORXOfTP~OV[OW[OX]OY]OR^XZ^X[^X]^X`^Xe^Xh^X~ORXOfTX~OfbO~OV[OW[OX]OY]OieO~OV[OW[OX]OY]ORQiZQi[Qi]Qi`QieQihQiiQi~OV[OW[ORUiXUiYUiZUi[Ui]Ui`UieUihUiiUi~OV[OW[OX]OY]ORSqZSq[Sq]Sq`SqeSqhSqiSq~Oe]R]~", - goto: "!fhPPiPiriPPPPPPPu{PPP!RPPPi_UOTVW[]bRZQQVOR_VQYQRaYSROVQ^TQ`WQc[Qd]Rfb", - nodeNames: "⚠ Program Assignment Identifier Function Params BinOp * / + - Number String Boolean", + states: "$OQVQPOOOkQPO'#CuO!fQPO'#CaO!nQPO'#CoOOQO'#Cu'#CuOVQPO'#CuOOQO'#Ct'#CtQVQPOOOVQPO,58xOOQO'#Cp'#CpO#cQPO'#CcO#kQPO,58{OVQPO,59POVQPO,59PO#pQPO,59aOOQO-E6m-E6mO$RQPO1G.dOOQO-E6n-E6nOVQPO1G.gOOQO1G.k1G.kO$yQPO1G.kOOQO1G.{1G.{O%qQPO7+$R", + stateData: "&n~OgOS~ORPOUQO^SO_SO`SOaTO~OSWORiXUiXYiXZiX[iX]iX^iX_iX`iXaiXeiXbiX~ORXOWVP~OY[OZ[O[]O]]ORcXUcX^cX_cX`cXacXecX~ORXOWVX~OWbO~OY[OZ[O[]O]]ObeO~OY[OZ[O[]O]]ORQiUQi^Qi_Qi`QiaQieQibQi~OY[OZ[ORXiUXi[Xi]Xi^Xi_Xi`XiaXieXibXi~OY[OZ[O[]O]]ORTqUTq^Tq_Tq`TqaTqeTqbTq~OU`R`~", + goto: "!hjPPkPPkPtPkPPPPPPPPPw}PPP!Tk_UOTVW[]bRZQQVOR_VQYQRaYSROVQ^TQ`WQc[Qd]Rfb", + nodeNames: "⚠ Program Assignment Identifier Operator Function Keyword Params Colon BinOp Operator Operator Operator Operator Number String Boolean Paren Paren", maxTerm: 25, propSources: [highlighting], skippedNodes: [0], repeatNodeCount: 2, - tokenData: "*f~RjX^!spq!srs#hxy$Vyz$[z{$a{|$f}!O$k!P!Q$x!Q![$}!_!`%h!c!}%m#R#S%m#T#Y%m#Y#Z&R#Z#h%m#h#i)`#i#o%m#y#z!s$f$g!s#BY#BZ!s$IS$I_!s$I|$JO!s$JT$JU!s$KV$KW!s&FU&FV!s~!xYb~X^!spq!s#y#z!s$f$g!s#BY#BZ!s$IS$I_!s$I|$JO!s$JT$JU!s$KV$KW!s&FU&FV!s~#kTOr#hrs#zs;'S#h;'S;=`$P<%lO#h~$PO[~~$SP;=`<%l#h~$[Oh~~$aOi~~$fOV~~$kOX~R$pPYP!`!a$sQ$xOfQ~$}OW~~%SQZ~!O!P%Y!Q![$}~%]P!Q![%`~%ePZ~!Q![%`~%mOd~~%rTR~}!O%m!Q![%m!c!}%m#R#S%m#T#o%m~&WWR~}!O%m!Q![%m!c!}%m#R#S%m#T#U&p#U#b%m#b#c(x#c#o%m~&uVR~}!O%m!Q![%m!c!}%m#R#S%m#T#`%m#`#a'[#a#o%m~'aVR~}!O%m!Q![%m!c!}%m#R#S%m#T#g%m#g#h'v#h#o%m~'{VR~}!O%m!Q![%m!c!}%m#R#S%m#T#X%m#X#Y(b#Y#o%m~(iT]~R~}!O%m!Q![%m!c!}%m#R#S%m#T#o%m~)PTe~R~}!O%m!Q![%m!c!}%m#R#S%m#T#o%m~)eVR~}!O%m!Q![%m!c!}%m#R#S%m#T#f%m#f#g)z#g#o%m~*PVR~}!O%m!Q![%m!c!}%m#R#S%m#T#i%m#i#j'v#j#o%m", - tokenizers: [0, 1], + tokenData: "*f~RkX^!vpq!vrs#kxy$Yyz$_z{$d{|$i}!O$n!P!Q$s!Q![$x![!]%c!_!`%h!c!}%m#R#S%m#T#Y%m#Y#Z&R#Z#h%m#h#i)`#i#o%m#y#z!v$f$g!v#BY#BZ!v$IS$I_!v$I|$JO!v$JT$JU!v$KV$KW!v&FU&FV!v~!{Yg~X^!vpq!v#y#z!v$f$g!v#BY#BZ!v$IS$I_!v$I|$JO!v$JT$JU!v$KV$KW!v&FU&FV!v~#nTOr#krs#}s;'S#k;'S;=`$S<%lO#k~$SO_~~$VP;=`<%l#k~$_Oa~~$dOb~~$iOY~~$nO[~~$sO]~~$xOZ~~$}Q^~!O!P%T!Q![$x~%WP!Q![%Z~%`P^~!Q![%Z~%hOW~~%mOS~~%rTR~}!O%m!Q![%m!c!}%m#R#S%m#T#o%m~&WWR~}!O%m!Q![%m!c!}%m#R#S%m#T#U&p#U#b%m#b#c(x#c#o%m~&uVR~}!O%m!Q![%m!c!}%m#R#S%m#T#`%m#`#a'[#a#o%m~'aVR~}!O%m!Q![%m!c!}%m#R#S%m#T#g%m#g#h'v#h#o%m~'{VR~}!O%m!Q![%m!c!}%m#R#S%m#T#X%m#X#Y(b#Y#o%m~(iT`~R~}!O%m!Q![%m!c!}%m#R#S%m#T#o%m~)PTU~R~}!O%m!Q![%m!c!}%m#R#S%m#T#o%m~)eVR~}!O%m!Q![%m!c!}%m#R#S%m#T#f%m#f#g)z#g#o%m~*PVR~}!O%m!Q![%m!c!}%m#R#S%m#T#i%m#i#j'v#j#o%m", + tokenizers: [0], topRules: {"Program":[0,1]}, tokenPrec: 255 }) diff --git a/src/parser/test-helper.ts b/src/parser/test-helper.ts index 11e0d51..d1bc9ae 100644 --- a/src/parser/test-helper.ts +++ b/src/parser/test-helper.ts @@ -16,15 +16,38 @@ export const regenerateParser = async () => { await $`bun generate-parser ` } -export const expectTree = (input: string) => { - const tree = parser.parse(input) - return { - toMatch: (expected: string) => { - expect(treeToString(tree, input)).toEqual(trimWhitespace(expected)) - }, +// Type declaration for TypeScript +declare module 'bun:test' { + interface Matchers { + toMatchTree(expected: string): T } } +expect.extend({ + toMatchTree(received: unknown, expected: string) { + if (typeof received !== 'string') { + return { + message: () => 'toMatchTree can only be used with string values', + pass: false, + } + } + + const tree = parser.parse(received) + const actual = treeToString(tree, received) + const normalizedExpected = trimWhitespace(expected) + try { + // A hacky way to show the colorized diff in the test output + expect(actual).toEqual(normalizedExpected) + return { pass: true, message: () => '' } + } catch (error) { + return { + message: () => (error as Error).message, + pass: false, + } + } + }, +}) + const treeToString = (tree: Tree, input: string): string => { const lines: string[] = [] @@ -44,7 +67,7 @@ const treeToString = (tree: Tree, input: string): string => { } else { const cleanText = nodeName === 'String' ? text.slice(1, -1) : text // Node names that should be displayed as single tokens (operators, keywords) - const singleTokens = ['+', '-', '*', '/', '->'] + const singleTokens = ['+', '-', '*', '/', '->', 'fn', '=', 'equals'] if (singleTokens.includes(nodeName)) { lines.push(`${indent}${nodeName}`) } else { diff --git a/src/server/debugPlugin.ts b/src/server/debugPlugin.ts new file mode 100644 index 0000000..019cf72 --- /dev/null +++ b/src/server/debugPlugin.ts @@ -0,0 +1,71 @@ +import { + EditorView, + Decoration, + ViewPlugin, + ViewUpdate, + WidgetType, + type DecorationSet, +} from '@codemirror/view' +import { syntaxTree } from '@codemirror/language' + +const createDebugWidget = (tags: string) => + Decoration.widget({ + widget: new (class extends WidgetType { + toDOM() { + const div = document.createElement('div') + div.style.cssText = ` + position: fixed; + top: 10px; + right: 10px; + background: #000; + color: #00ff00; + padding: 8px; + font-family: monospace; + font-size: 12px; + border: 1px solid #333; + z-index: 1000; + max-width: 300px; + word-wrap: break-word; + white-space: pre-wrap; + ` + div.textContent = tags + return div + } + })(), + }) + +export const debugTags = ViewPlugin.fromClass( + class { + decorations: DecorationSet = Decoration.none + + constructor(view: EditorView) { + this.updateDecorations(view) + } + + update(update: ViewUpdate) { + if (update.docChanged || update.selectionSet) { + this.updateDecorations(update.view) + } + } + + updateDecorations(view: EditorView) { + const pos = view.state.selection.main.head + const tree = syntaxTree(view.state) + + let tags: string[] = [] + let node = tree.resolveInner(pos, -1) + + while (node) { + tags.push(node.type.name) + node = node.parent! + if (!node) break + } + + const debugText = tags.length ? tags.join(' > ') : 'No nodes' + this.decorations = Decoration.set([createDebugWidget(debugText).range(pos)]) + } + }, + { + decorations: (v) => v.decorations, + } +) diff --git a/src/server/editor.tsx b/src/server/editor.tsx index 0dcff77..5c16c33 100644 --- a/src/server/editor.tsx +++ b/src/server/editor.tsx @@ -1,6 +1,9 @@ import { basicSetup } from 'codemirror' import { EditorView } from '@codemirror/view' import { editorTheme } from './editorTheme' +import { shrimpLanguage } from './shrimpLanguage' +import { shrimpHighlighting } from './editorTheme' +import { debugTags } from '@/server/debugPlugin' export const Editor = () => { return ( @@ -10,9 +13,14 @@ export const Editor = () => { console.log('init editor') new EditorView({ - doc: '', + doc: `a = 3 +fn x y: x + y +aa = fn radius: 3.14 * radius * radius +b = true +c = "cyan" +`, parent: ref, - extensions: [basicSetup, editorTheme], + extensions: [basicSetup, editorTheme, shrimpLanguage(), shrimpHighlighting], }) }} /> diff --git a/src/server/editorTheme.tsx b/src/server/editorTheme.tsx index 2fd53f9..a18795d 100644 --- a/src/server/editorTheme.tsx +++ b/src/server/editorTheme.tsx @@ -1,28 +1,50 @@ import { EditorView } from '@codemirror/view' +import { HighlightStyle, syntaxHighlighting } from '@codemirror/language' +import { tags } from '@lezer/highlight' + +const highlightStyle = HighlightStyle.define([ + { tag: tags.keyword, color: '#C792EA' }, // fn - soft purple (Night Owl inspired) + { tag: tags.name, color: '#82AAFF' }, // identifiers - bright blue (Night Owl style) + { tag: tags.string, color: '#C3E88D' }, // strings - soft green + { tag: tags.number, color: '#F78C6C' }, // numbers - warm orange + { tag: tags.bool, color: '#FF5370' }, // booleans - coral red + { tag: tags.operator, color: '#89DDFF' }, // operators - cyan blue + { tag: tags.paren, color: '#676E95' }, // parens - muted blue-gray + { + tag: tags.definition(tags.variableName), + color: '#FFCB6B', // warm yellow + backgroundColor: '#1E2A4A', // dark blue background + padding: '1px 2px', + borderRadius: '2px', + fontWeight: '500', + }, +]) + +export const shrimpHighlighting = syntaxHighlighting(highlightStyle) export const editorTheme = EditorView.theme( { '&': { - color: '#7C70DA', - backgroundColor: '#40318D', + color: '#D6DEEB', // Night Owl text color + backgroundColor: '#011627', // Night Owl dark blue fontFamily: '"Pixeloid Mono", "Courier New", monospace', fontSize: '18px', height: '100%', }, '.cm-content': { - caretColor: '#7C70DA', + caretColor: '#80A4C2', // soft blue caret padding: '0px', minHeight: '100px', - borderBottom: '3px solid #7C70DA', + borderBottom: '3px solid #1E2A4A', }, '.cm-activeLine': { backgroundColor: 'transparent', }, '&.cm-focused .cm-cursor': { - borderLeftColor: '#7C70DA', + borderLeftColor: '#80A4C2', }, '&.cm-focused .cm-selectionBackground, ::selection': { - backgroundColor: '#5A4FCF', + backgroundColor: '#1D3B53', // darker blue selection }, '.cm-gutters': { display: 'none', diff --git a/src/server/shrimpLanguage.ts b/src/server/shrimpLanguage.ts new file mode 100644 index 0000000..65b8034 --- /dev/null +++ b/src/server/shrimpLanguage.ts @@ -0,0 +1,11 @@ +import { parser } from '../parser/shrimp' +import { LRLanguage, LanguageSupport } from '@codemirror/language' +import { highlighting } from '../parser/highlight.js' + +export const shrimpLanguage = () => { + const language = LRLanguage.define({ + parser: parser.configure({ props: [highlighting] }), + }) + + return new LanguageSupport(language) +}