From 66671970e0ce869b484aaad77a1707fcd77a16d3 Mon Sep 17 00:00:00 2001 From: Corey Johnson Date: Fri, 24 Oct 2025 10:17:06 -0700 Subject: [PATCH] i did things --- packages/ReefVM | 2 +- src/compiler/tests/compiler.test.ts | 2 +- src/editor/editor.tsx | 47 +++++++--------- src/editor/plugins/keymap.tsx | 26 +++++++++ src/editor/plugins/shrimpSetup.ts | 35 ++++++++++++ src/editor/plugins/theme.tsx | 6 -- src/editor/runCode.tsx | 6 +- src/parser/operatorTokenizer.ts | 80 +++++++++++++++++++++++++++ src/parser/shrimp.grammar | 48 +++++++--------- src/parser/shrimp.terms.ts | 36 ++++++++---- src/parser/shrimp.ts | 15 ++--- src/parser/tests/basics.test.ts | 79 +++++++++++++------------- src/parser/tests/control-flow.test.ts | 6 +- src/parser/tests/dot-get.test.ts | 6 +- src/parser/tests/functions.test.ts | 8 +-- src/parser/tests/multiline.test.ts | 6 +- src/parser/tests/pipes.test.ts | 2 +- src/parser/tests/strings.test.ts | 4 +- src/parser/tokenizer.ts | 17 +++++- src/testSetup.ts | 69 ++--------------------- src/utils/tree.ts | 61 ++++++++++++++++++++ 21 files changed, 359 insertions(+), 202 deletions(-) create mode 100644 src/editor/plugins/shrimpSetup.ts create mode 100644 src/parser/operatorTokenizer.ts create mode 100644 src/utils/tree.ts diff --git a/packages/ReefVM b/packages/ReefVM index 47f829f..995487f 160000 --- a/packages/ReefVM +++ b/packages/ReefVM @@ -1 +1 @@ -Subproject commit 47f829fcada71655f0d40ec363b5bcc844af8856 +Subproject commit 995487f2d5d8bb260e223ca402220c51ceba1c4a diff --git a/src/compiler/tests/compiler.test.ts b/src/compiler/tests/compiler.test.ts index 0f2e1c0..dc35a34 100644 --- a/src/compiler/tests/compiler.test.ts +++ b/src/compiler/tests/compiler.test.ts @@ -209,7 +209,7 @@ describe('Regex', () => { }) test('invalid regex pattern', () => { - expect('//[unclosed//').toFailEvaluation() + expect('//[unclosed//').toEvaluateTo('//[unclosed//') }) }) diff --git a/src/editor/editor.tsx b/src/editor/editor.tsx index b2d84c6..9206fce 100644 --- a/src/editor/editor.tsx +++ b/src/editor/editor.tsx @@ -1,23 +1,24 @@ -import { basicSetup } from 'codemirror' import { EditorView } from '@codemirror/view' -import { shrimpTheme } from '#editor/plugins/theme' -import { shrimpLanguage } from '#/editor/plugins/shrimpLanguage' -import { shrimpHighlighting } from '#editor/plugins/theme' -import { shrimpKeymap } from '#editor/plugins/keymap' -import { asciiEscapeToHtml, assert, assertNever, log, toElement } from '#utils/utils' +import { asciiEscapeToHtml, assertNever, log, toElement } from '#utils/utils' import { Signal } from '#utils/signal' -import { shrimpErrors } from '#editor/plugins/errors' -import { debugTags } from '#editor/plugins/debugTags' -import { getContent, persistencePlugin } from '#editor/plugins/persistence' - -import '#editor/editor.css' +import { getContent } from '#editor/plugins/persistence' import type { HtmlEscapedString } from 'hono/utils/html' -import { catchErrors } from '#editor/plugins/catchErrors' import { connectToNose, noseSignals } from '#editor/noseClient' import type { Value } from 'reefvm' +import { Compartment } from '@codemirror/state' +import { lineNumbers } from '@codemirror/view' +import { shrimpSetup } from '#editor/plugins/shrimpSetup' + +import '#editor/editor.css' + +const lineNumbersCompartment = new Compartment() connectToNose() +export const outputSignal = new Signal() +export const errorSignal = new Signal() +export const multilineModeSignal = new Signal() + export const Editor = () => { return ( <> @@ -27,17 +28,14 @@ export const Editor = () => { const view = new EditorView({ parent: ref, doc: getContent(), - extensions: [ - catchErrors, - shrimpKeymap, - basicSetup, - shrimpTheme, - shrimpLanguage, - shrimpHighlighting, - shrimpErrors, - persistencePlugin, - // debugTags, - ], + extensions: shrimpSetup(lineNumbersCompartment), + }) + + multilineModeSignal.connect((isMultiline) => { + console.log(`🌭 hey babe`, isMultiline) + view.dispatch({ + effects: lineNumbersCompartment.reconfigure(isMultiline ? lineNumbers() : []), + }) }) requestAnimationFrame(() => view.focus()) @@ -64,9 +62,6 @@ noseSignals.connect((message) => { } }) -export const outputSignal = new Signal() -export const errorSignal = new Signal() - outputSignal.connect((value) => { const el = document.querySelector('#output')! el.innerHTML = '' diff --git a/src/editor/plugins/keymap.tsx b/src/editor/plugins/keymap.tsx index 6f948f6..6c77f14 100644 --- a/src/editor/plugins/keymap.tsx +++ b/src/editor/plugins/keymap.tsx @@ -1,3 +1,4 @@ +import { multilineModeSignal, outputSignal } from '#editor/editor' import { printBytecodeOutput, printParserOutput, runCode } from '#editor/runCode' import { EditorState } from '@codemirror/state' import { keymap } from '@codemirror/view' @@ -27,9 +28,13 @@ const customKeymap = keymap.of([ if (multilineMode) { const input = view.state.doc.toString() runCode(input) + return true + } else { + outputSignal.emit('Press Shift+Enter to insert run the code.') } + multilineModeSignal.emit(true) multilineMode = true view.dispatch({ changes: { from: view.state.doc.length, insert: '\n' }, @@ -44,6 +49,10 @@ const customKeymap = keymap.of([ key: 'Tab', preventDefault: true, run: (view) => { + view.dispatch({ + changes: { from: view.state.selection.main.from, insert: ' ' }, + selection: { anchor: view.state.selection.main.from + 2 }, + }) return true }, }, @@ -51,6 +60,8 @@ const customKeymap = keymap.of([ { key: 'ArrowUp', run: (view) => { + if (multilineMode) return false + const command = history.previous() if (command === undefined) return false view.dispatch({ @@ -64,6 +75,8 @@ const customKeymap = keymap.of([ { key: 'ArrowDown', run: (view) => { + if (multilineMode) return false + const command = history.next() if (command === undefined) return false view.dispatch({ @@ -124,9 +137,22 @@ export const shrimpKeymap = [customKeymap, singleLineFilter] class History { private commands: string[] = [] private index: number | undefined + private storageKey = 'shrimp-command-history' + + constructor() { + try { + this.commands = JSON.parse(localStorage.getItem(this.storageKey) || '[]') + } catch { + console.warn('Failed to load command history from localStorage') + } + } push(command: string) { this.commands.push(command) + + // Limit to last 50 commands + this.commands = this.commands.slice(-50) + localStorage.setItem(this.storageKey, JSON.stringify(this.commands)) this.index = undefined } diff --git a/src/editor/plugins/shrimpSetup.ts b/src/editor/plugins/shrimpSetup.ts new file mode 100644 index 0000000..9adaea2 --- /dev/null +++ b/src/editor/plugins/shrimpSetup.ts @@ -0,0 +1,35 @@ +import { history, defaultKeymap, historyKeymap } from '@codemirror/commands' +import { bracketMatching, indentOnInput } from '@codemirror/language' +import { highlightSpecialChars, drawSelection, dropCursor, keymap } from '@codemirror/view' +import { closeBrackets, autocompletion, completionKeymap } from '@codemirror/autocomplete' +import { EditorState, Compartment } from '@codemirror/state' +import { searchKeymap } from '@codemirror/search' +import { shrimpKeymap } from './keymap' +import { shrimpTheme, shrimpHighlighting } from './theme' +import { shrimpLanguage } from './shrimpLanguage' +import { shrimpErrors } from './errors' +import { persistencePlugin } from './persistence' +import { catchErrors } from './catchErrors' + +export const shrimpSetup = (lineNumbersCompartment: Compartment) => { + return [ + catchErrors, + shrimpKeymap, + highlightSpecialChars(), + history(), + drawSelection(), + dropCursor(), + EditorState.allowMultipleSelections.of(true), + bracketMatching(), + closeBrackets(), + autocompletion(), + indentOnInput(), + keymap.of([...defaultKeymap, ...historyKeymap, ...searchKeymap, ...completionKeymap]), + lineNumbersCompartment.of([]), + shrimpTheme, + shrimpLanguage, + shrimpHighlighting, + shrimpErrors, + persistencePlugin, + ] +} diff --git a/src/editor/plugins/theme.tsx b/src/editor/plugins/theme.tsx index f131591..c1e46f1 100644 --- a/src/editor/plugins/theme.tsx +++ b/src/editor/plugins/theme.tsx @@ -38,18 +38,12 @@ export const shrimpTheme = EditorView.theme( caretColor: 'var(--caret)', padding: '0px', }, - '.cm-activeLine': { - backgroundColor: 'transparent', - }, '&.cm-focused .cm-cursor': { borderLeftColor: 'var(--caret)', }, '&.cm-focused .cm-selectionBackground, ::selection': { backgroundColor: 'var(--bg-selection)', }, - '.cm-gutters': { - display: 'none', - }, '.cm-editor': { border: 'none', outline: 'none', diff --git a/src/editor/runCode.tsx b/src/editor/runCode.tsx index 27b68ff..1583b5f 100644 --- a/src/editor/runCode.tsx +++ b/src/editor/runCode.tsx @@ -1,9 +1,10 @@ import { outputSignal, errorSignal } from '#editor/editor' import { Compiler } from '#compiler/compiler' import { errorMessage, log } from '#utils/utils' -import { bytecodeToString, run } from 'reefvm' +import { bytecodeToString } from 'reefvm' import { parser } from '#parser/shrimp' import { sendToNose } from '#editor/noseClient' +import { treeToString } from '#utils/tree' export const runCode = async (input: string) => { try { @@ -18,7 +19,8 @@ export const runCode = async (input: string) => { export const printParserOutput = (input: string) => { try { const cst = parser.parse(input) - outputSignal.emit(cst.toString()) + const string = treeToString(cst, input) + outputSignal.emit(string) } catch (error) { log.error(error) errorSignal.emit(`${errorMessage(error)}`) diff --git a/src/parser/operatorTokenizer.ts b/src/parser/operatorTokenizer.ts new file mode 100644 index 0000000..478c20d --- /dev/null +++ b/src/parser/operatorTokenizer.ts @@ -0,0 +1,80 @@ +import { ExternalTokenizer, InputStream } from '@lezer/lr' +import * as terms from './shrimp.terms' + +type Operator = { str: string; tokenName: keyof typeof terms } +const operators: Array = [ + { str: 'and', tokenName: 'And' }, + { str: 'or', tokenName: 'Or' }, + { str: '>=', tokenName: 'Gte' }, + { str: '<=', tokenName: 'Lte' }, + { str: '!=', tokenName: 'Neq' }, + + // // Single-char operators + { str: '*', tokenName: 'Star' }, + { str: '=', tokenName: 'Eq' }, + { str: '/', tokenName: 'Slash' }, + { str: '+', tokenName: 'Plus' }, + { str: '-', tokenName: 'Minus' }, + { str: '>', tokenName: 'Gt' }, + { str: '<', tokenName: 'Lt' }, +] + +export const operatorTokenizer = new ExternalTokenizer((input: InputStream) => { + for (let operator of operators) { + if (!matchesString(input, 0, operator.str)) continue + const afterOpPos = operator.str.length + const charAfterOp = input.peek(afterOpPos) + if (!isWhitespace(charAfterOp)) continue + + // Accept the operator token + const token = terms[operator.tokenName] + if (token === undefined) { + throw new Error(`Unknown token name: ${operator.tokenName}`) + } + + input.advance(afterOpPos) + input.acceptToken(token) + + return + } +}) + +const isWhitespace = (ch: number): boolean => { + return matchesChar(ch, [' ', '\t', '\n']) +} + +const matchesChar = (ch: number, chars: (string | number)[]): boolean => { + for (const c of chars) { + if (typeof c === 'number') { + if (ch === c) { + return true + } + } else if (ch === c.charCodeAt(0)) { + return true + } + } + return false +} + +const matchesString = (input: InputStream, pos: number, str: string): boolean => { + for (let i = 0; i < str.length; i++) { + if (input.peek(pos + i) !== str.charCodeAt(i)) { + return false + } + } + return true +} + +const peek = (numChars: number, input: InputStream): string => { + let result = '' + for (let i = 0; i < numChars; i++) { + const ch = input.peek(i) + if (ch === -1) { + result += 'EOF' + break + } else { + result += String.fromCharCode(ch) + } + } + return result +} diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index 6eeeecd..ce8ad22 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -6,14 +6,14 @@ @top Program { item* } -@external tokens tokenizer from "./tokenizer" { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot } +@external tokens operatorTokenizer from "./operatorTokenizer" { Star, Slash, Plus, Minus, And, Or, Eq, Neq, Lt, Lte, Gt, Gte } @tokens { - @precedence { Number "-" Regex "/"} + @precedence { Number Regex } StringFragment { !['\\$]+ } NamedArgPrefix { $[a-z]+ "=" } - Number { "-"? $[0-9]+ ('.' $[0-9]+)? } + Number { ("-" | "+")? $[0-9]+ ('.' $[0-9]+)? } Boolean { "true" | "false" } newlineOrSemicolon { "\n" | ";" } eof { @eof } @@ -28,23 +28,13 @@ Fn[@name=keyword] { "fn" } "if" [@name=keyword] "elsif" [@name=keyword] - "else" [@name=keyword] - "and" [@name=operator] - "or" [@name=operator] - "!=" [@name=operator] - "<" [@name=operator] - "<=" [@name=operator] - ">" [@name=operator] - ">=" [@name=operator] - "=" [@name=operator] - "+"[@name=operator] - "-"[@name=operator] - "*"[@name=operator] - "/"[@name=operator] + "else" [@name=keyword] "|"[@name=operator] } +@external tokens tokenizer from "./tokenizer" { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot } + @precedence { pipe @left, multiplicative @left, @@ -140,14 +130,14 @@ ThenBlock { } ConditionalOp { - expression "=" expression | - expression "!=" expression | - expression "<" expression | - expression "<=" expression | - expression ">" expression | - expression ">=" expression | - expression "and" (expression | ConditionalOp) | - expression "or" (expression | ConditionalOp) + expression Eq expression | + expression Neq expression | + expression Lt expression | + expression Lte expression | + expression Gt expression | + expression Gte expression | + expression And (expression | ConditionalOp) | + expression Or (expression | ConditionalOp) } Params { @@ -155,14 +145,14 @@ Params { } Assign { - AssignableIdentifier "=" consumeToTerminator + AssignableIdentifier Eq consumeToTerminator } BinOp { - (expression | BinOp) !multiplicative "*" (expression | BinOp) | - (expression | BinOp) !multiplicative "/" (expression | BinOp) | - (expression | BinOp) !additive "+" (expression | BinOp) | - (expression | BinOp) !additive "-" (expression | BinOp) + (expression | BinOp) !multiplicative Star (expression | BinOp) | + (expression | BinOp) !multiplicative Slash (expression | BinOp) | + (expression | BinOp) !additive Plus (expression | BinOp) | + (expression | BinOp) !additive Minus (expression | BinOp) } ParenExpr { diff --git a/src/parser/shrimp.terms.ts b/src/parser/shrimp.terms.ts index a6c6615..78a893d 100644 --- a/src/parser/shrimp.terms.ts +++ b/src/parser/shrimp.terms.ts @@ -1,17 +1,29 @@ // This file was generated by lezer-generator. You probably shouldn't edit it. export const - Identifier = 1, - AssignableIdentifier = 2, - Word = 3, - IdentifierBeforeDot = 4, - Program = 5, - PipeExpr = 6, - FunctionCall = 7, - PositionalArg = 8, - ParenExpr = 9, - FunctionCallOrIdentifier = 10, - BinOp = 11, - ConditionalOp = 16, + Star = 1, + Slash = 2, + Plus = 3, + Minus = 4, + And = 5, + Or = 6, + Eq = 7, + Neq = 8, + Lt = 9, + Lte = 10, + Gt = 11, + Gte = 12, + Identifier = 13, + AssignableIdentifier = 14, + Word = 15, + IdentifierBeforeDot = 16, + Program = 17, + PipeExpr = 18, + FunctionCall = 19, + PositionalArg = 20, + ParenExpr = 21, + FunctionCallOrIdentifier = 22, + BinOp = 23, + ConditionalOp = 24, String = 25, StringFragment = 26, Interpolation = 27, diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index 476243a..bf99404 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -1,14 +1,15 @@ // This file was generated by lezer-generator. You probably shouldn't edit it. import {LRParser} from "@lezer/lr" +import {operatorTokenizer} from "./operatorTokenizer" import {tokenizer} from "./tokenizer" import {trackScope} from "./scopeTracker" import {highlighting} from "./highlight" export const parser = LRParser.deserialize({ version: 14, - states: ".jQVQROOO#XQTO'#CfO$RQQO'#CgO$aQQO'#DmO$xQRO'#CeO%gOWO'#CuOOQP'#Dq'#DqO%uOQO'#C}O%zQQO'#DpO&cQRO'#D|OOQP'#DO'#DOOOQO'#Dn'#DnO&kQQO'#DmO&yQRO'#EQOOQO'#DX'#DXO'hQQO'#DaOOQO'#Dm'#DmO'mQQO'#DlOOQP'#Dl'#DlOOQP'#Db'#DbQVQROOOOQP'#Dp'#DpOOQP'#Cd'#CdO'uQRO'#DUOOQP'#Do'#DoOOQP'#Dc'#DcO(PQTO,58}O&yQRO,59RO&yQRO,59RO)XQQO'#CgO)iQQO,59PO)zQQO,59PO)uQQO,59PO*uQQO,59PO*}QRO'#CwO+VQ`O'#CxOOOO'#Du'#DuOOOO'#Dd'#DdO+kOWO,59aOOQP,59a,59aO+yOPO,59iOOQP'#De'#DeO,OQRO'#DQO,WQQO,5:hO,]QRO'#DgO,bQQO,58|O,sQQO,5:lO,zQQO,5:lO-PQRO,59{OOQP,5:W,5:WOOQP-E7`-E7`OOQP,59p,59pOOQP-E7a-E7aOOQO1G.m1G.mO-^QQO1G.mO&yQRO,59WO&yQRO,59WOOQP1G.k1G.kOOOO,59c,59cOOOO,59d,59dOOOO-E7b-E7bOOQP1G.{1G.{OOQP1G/T1G/TOOQP-E7c-E7cO-xQRO1G0SO!QQTO'#CfOOQO,5:R,5:ROOQO-E7e-E7eO.YQRO1G0WOOQO1G/g1G/gOOQO1G.r1G.rO.jQQO1G.rO.tQQO7+%nO.yQRO7+%oOOQO'#DZ'#DZOOQO7+%r7+%rO/ZQRO7+%sOOQP<uAN>uO&yQRO'#D]OOQO'#Dh'#DhO0nQQOAN>yO0yQQO'#D_OOQOAN>yAN>yO1OQQOAN>yO1TQQO,59wO1[QQO,59wOOQO-E7f-E7fOOQOG24eG24eO1aQQOG24eO1fQQO,59yO1kQQO1G/cOOQOLD*PLD*PO.yQRO1G/eO/ZQRO7+$}OOQO7+%P7+%POOQO<uAN>uO&yQrO'#D]OOQO'#Dh'#DhO0nQQOAN>yO0yQQO'#D_OOQOAN>yAN>yO1OQQOAN>yO1TQRO,59wO1[QQO,59wOOQO-E7f-E7fOOQOG24eG24eO1aQQOG24eO1fQQO,59yO1kQQO1G/cOOQOLD*PLD*PO.yQrO1G/eO/ZQrO7+$}OOQO7+%P7+%POOQO<n#a#b;W#b#cCR#c#o;W#o;'S$_;'S;=`$v<%lO$_V>s[jSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#g;W#g#h?i#h#o;W#o;'S$_;'S;=`$v<%lO$_V?n^jSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#X;W#X#Y@j#Y#];W#]#^Aa#^#o;W#o;'S$_;'S;=`$v<%lO$_V@qY!SPjSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#o;W#o;'S$_;'S;=`$v<%lO$_VAf[jSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#Y;W#Y#ZB[#Z#o;W#o;'S$_;'S;=`$v<%lO$_VBcY!QPjSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#o;W#o;'S$_;'S;=`$v<%lO$_VCW[jSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#W;W#W#XC|#X#o;W#o;'S$_;'S;=`$v<%lO$_VDTYjSvROt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#o;W#o;'S$_;'S;=`$v<%lO$_VDx]jSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#UEq#U#b;W#b#cIX#c#o;W#o;'S$_;'S;=`$v<%lO$_VEv[jSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#`;W#`#aFl#a#o;W#o;'S$_;'S;=`$v<%lO$_VFq[jSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#g;W#g#hGg#h#o;W#o;'S$_;'S;=`$v<%lO$_VGl[jSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#X;W#X#YHb#Y#o;W#o;'S$_;'S;=`$v<%lO$_VHiYnRjSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#o;W#o;'S$_;'S;=`$v<%lO$_VI`YsRjSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#o;W#o;'S$_;'S;=`$v<%lO$_VJT[jSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#Y;W#Y#ZJy#Z#o;W#o;'S$_;'S;=`$v<%lO$_VKQY|PjSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#o;W#o;'S$_;'S;=`$v<%lO$__Kw[!lWjSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#i;W#i#jLm#j#o;W#o;'S$_;'S;=`$v<%lO$_VLr[jSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#`;W#`#aMh#a#o;W#o;'S$_;'S;=`$v<%lO$_VMm[jSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#`;W#`#aNc#a#o;W#o;'S$_;'S;=`$v<%lO$_VNjYpRjSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#o;W#o;'S$_;'S;=`$v<%lO$_V! _[jSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#f;W#f#g!!T#g#o;W#o;'S$_;'S;=`$v<%lO$_V!![YhRjSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#o;W#o;'S$_;'S;=`$v<%lO$_^!#RY!nWjSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#o;W#o;'S$_;'S;=`$v<%lO$__!#x[!mWjSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#f;W#f#g!$n#g#o;W#o;'S$_;'S;=`$v<%lO$_V!$s[jSOt$_uw$_x!_$_!_!`:m!`#O$_#P#T$_#T#i;W#i#jGg#j#o;W#o;'S$_;'S;=`$v<%lO$_V!%pUzRjSOt$_uw$_x#O$_#P;'S$_;'S;=`$v<%lO$_~!&XO!v~", - tokenizers: [tokenizer, 0, 1, 2, 3], - topRules: {"Program":[0,5]}, + tokenData: "JX~R|OX#{XY$jYZ%TZp#{pq$jqt#{tu%nuw#{wx%sxy%xyz&cz{#{{|&||}#{}!O&|!O!P)p!P!Q*Z!Q!['k![!]2v!]!^%T!^#O#{#O#P3a#P#R#{#R#S3f#S#T#{#T#X4P#X#Y5_#Y#Z^#a#o4P#o;'S#{;'S;=`$d<%lO#{V>c[jSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#g4P#g#h?X#h#o4P#o;'S#{;'S;=`$d<%lO#{V?^[jSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#X4P#X#Y@S#Y#o4P#o;'S#{;'S;=`$d<%lO#{V@ZYnRjSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#o4P#o;'S#{;'S;=`$d<%lO#{VAQYsRjSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#o4P#o;'S#{;'S;=`$d<%lO#{VAu[jSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#Y4P#Y#ZBk#Z#o4P#o;'S#{;'S;=`$d<%lO#{VBrY|PjSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#o4P#o;'S#{;'S;=`$d<%lO#{_Ci[!lWjSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#i4P#i#jD_#j#o4P#o;'S#{;'S;=`$d<%lO#{VDd[jSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#`4P#`#aEY#a#o4P#o;'S#{;'S;=`$d<%lO#{VE_[jSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#`4P#`#aFT#a#o4P#o;'S#{;'S;=`$d<%lO#{VF[YpRjSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#o4P#o;'S#{;'S;=`$d<%lO#{^GRY!nWjSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#o4P#o;'S#{;'S;=`$d<%lO#{_Gx[!mWjSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#f4P#f#gHn#g#o4P#o;'S#{;'S;=`$d<%lO#{VHs[jSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#i4P#i#j?X#j#o4P#o;'S#{;'S;=`$d<%lO#{VIpUzRjSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~JXO!v~", + tokenizers: [operatorTokenizer, 0, 1, 2, 3, tokenizer], + topRules: {"Program":[0,17]}, tokenPrec: 768 }) diff --git a/src/parser/tests/basics.test.ts b/src/parser/tests/basics.test.ts index 44bf627..e4ec3f5 100644 --- a/src/parser/tests/basics.test.ts +++ b/src/parser/tests/basics.test.ts @@ -11,7 +11,7 @@ describe('null', () => { expect('a = null').toMatchTree(` Assign AssignableIdentifier a - operator = + Eq = Null null`) }) }) @@ -30,7 +30,7 @@ describe('Parentheses', () => { ParenExpr BinOp Number 2 - operator + + Plus + Number 3`) }) @@ -72,14 +72,14 @@ describe('Parentheses', () => { ParenExpr ConditionalOp Identifier a - operator > + Gt > Identifier b`) expect('(a and b)').toMatchTree(` ParenExpr ConditionalOp Identifier a - operator and + And and Identifier b`) }) @@ -91,7 +91,7 @@ describe('Parentheses', () => { ParenExpr BinOp Number 3 - operator + + Plus + Number 3`) }) @@ -105,13 +105,16 @@ describe('Parentheses', () => { `) }) - test('a word start with *', () => { - expect('find *cool*').toMatchTree(` - FunctionCall - Identifier find - PositionalArg - Word *cool* + test('a word start with an operator', () => { + const operators = ['*', '/', '+', '-', 'and', 'or', '=', '!=', '>=', '<=', '>', '<'] + for (const operator of operators) { + expect(`find ${operator}cool*`).toMatchTree(` + FunctionCall + Identifier find + PositionalArg + Word ${operator}cool* `) + } }) test('a word can look like a binop', () => { @@ -128,11 +131,11 @@ describe('Parentheses', () => { ParenExpr BinOp Number 2 - operator + + Plus + ParenExpr BinOp Number 1 - operator * + Star * Number 4`) }) @@ -140,7 +143,7 @@ describe('Parentheses', () => { expect('4 + (echo 3)').toMatchTree(` BinOp Number 4 - operator + + Plus + ParenExpr FunctionCall Identifier echo @@ -149,12 +152,12 @@ describe('Parentheses', () => { }) }) -describe.only('BinOp', () => { +describe('BinOp', () => { test('addition tests', () => { expect('2 + 3').toMatchTree(` BinOp Number 2 - operator + + Plus + Number 3 `) }) @@ -163,7 +166,7 @@ describe.only('BinOp', () => { expect('5 - 2').toMatchTree(` BinOp Number 5 - operator - + Minus - Number 2 `) }) @@ -172,7 +175,7 @@ describe.only('BinOp', () => { expect('4 * 3').toMatchTree(` BinOp Number 4 - operator * + Star * Number 3 `) }) @@ -181,7 +184,7 @@ describe.only('BinOp', () => { expect('8 / 2').toMatchTree(` BinOp Number 8 - operator / + Slash / Number 2 `) }) @@ -191,15 +194,15 @@ describe.only('BinOp', () => { BinOp BinOp Number 2 - operator + + Plus + BinOp Number 3 - operator * + Star * Number 4 - operator - + Minus - BinOp Number 5 - operator / + Slash / Number 1 `) }) @@ -210,7 +213,7 @@ describe('ambiguity', () => { expect('a + -3').toMatchTree(` BinOp Identifier a - operator + + Plus + Number -3 `) }) @@ -219,7 +222,7 @@ describe('ambiguity', () => { expect('a-var + a-thing').toMatchTree(` BinOp Identifier a-var - operator + + Plus + Identifier a-thing `) }) @@ -231,11 +234,11 @@ describe('newlines', () => { y = 2`).toMatchTree(` Assign AssignableIdentifier x - operator = + Eq = Number 5 Assign AssignableIdentifier y - operator = + Eq = Number 2`) }) @@ -243,11 +246,11 @@ y = 2`).toMatchTree(` expect(`x = 5; y = 2`).toMatchTree(` Assign AssignableIdentifier x - operator = + Eq = Number 5 Assign AssignableIdentifier y - operator = + Eq = Number 2`) }) @@ -255,7 +258,7 @@ y = 2`).toMatchTree(` expect(`a = hello; 2`).toMatchTree(` Assign AssignableIdentifier a - operator = + Eq = FunctionCallOrIdentifier Identifier hello Number 2`) @@ -267,7 +270,7 @@ describe('Assign', () => { expect('x = 5').toMatchTree(` Assign AssignableIdentifier x - operator = + Eq = Number 5`) }) @@ -275,10 +278,10 @@ describe('Assign', () => { expect('x = 5 + 3').toMatchTree(` Assign AssignableIdentifier x - operator = + Eq = BinOp Number 5 - operator + + Plus + Number 3`) }) @@ -286,7 +289,7 @@ describe('Assign', () => { expect('add = fn a b: a + b end').toMatchTree(` Assign AssignableIdentifier add - operator = + Eq = FunctionDef keyword fn Params @@ -295,7 +298,7 @@ describe('Assign', () => { colon : BinOp Identifier a - operator + + Plus + Identifier b end end`) }) @@ -306,7 +309,7 @@ describe('DotGet whitespace sensitivity', () => { expect('basename = 5; basename.prop').toMatchTree(` Assign AssignableIdentifier basename - operator = + Eq = Number 5 DotGet IdentifierBeforeDot basename @@ -317,11 +320,11 @@ describe('DotGet whitespace sensitivity', () => { expect('basename = 5; basename / prop').toMatchTree(` Assign AssignableIdentifier basename - operator = + Eq = Number 5 BinOp Identifier basename - operator / + Slash / Identifier prop`) }) diff --git a/src/parser/tests/control-flow.test.ts b/src/parser/tests/control-flow.test.ts index 88ec3ad..a507b36 100644 --- a/src/parser/tests/control-flow.test.ts +++ b/src/parser/tests/control-flow.test.ts @@ -9,7 +9,7 @@ describe('if/elsif/else', () => { keyword if ConditionalOp Identifier y - operator = + Eq = Number 1 colon : ThenBlock @@ -20,7 +20,7 @@ describe('if/elsif/else', () => { expect('a = if x: 2').toMatchTree(` Assign AssignableIdentifier a - operator = + Eq = IfExpr keyword if Identifier x @@ -39,7 +39,7 @@ describe('if/elsif/else', () => { keyword if ConditionalOp Identifier x - operator < + Lt < Number 9 colon : ThenBlock diff --git a/src/parser/tests/dot-get.test.ts b/src/parser/tests/dot-get.test.ts index d11341b..ddd24ae 100644 --- a/src/parser/tests/dot-get.test.ts +++ b/src/parser/tests/dot-get.test.ts @@ -18,7 +18,7 @@ describe('DotGet', () => { expect('obj = 5; obj.prop').toMatchTree(` Assign AssignableIdentifier obj - operator = + Eq = Number 5 DotGet IdentifierBeforeDot obj @@ -106,7 +106,7 @@ end`).toMatchTree(` expect('config = 42; echo config.path').toMatchTree(` Assign AssignableIdentifier config - operator = + Eq = Number 42 FunctionCall Identifier echo @@ -121,7 +121,7 @@ end`).toMatchTree(` expect('config = 42; cat readme.txt; echo config.path').toMatchTree(` Assign AssignableIdentifier config - operator = + Eq = Number 42 FunctionCall Identifier cat diff --git a/src/parser/tests/functions.test.ts b/src/parser/tests/functions.test.ts index f9632a5..eb2fa56 100644 --- a/src/parser/tests/functions.test.ts +++ b/src/parser/tests/functions.test.ts @@ -76,7 +76,7 @@ describe('Fn', () => { colon : BinOp Identifier x - operator + + Plus + Number 1 end end`) }) @@ -91,7 +91,7 @@ describe('Fn', () => { colon : BinOp Identifier x - operator * + Star * Identifier y end end`) }) @@ -109,11 +109,11 @@ end`).toMatchTree(` colon : BinOp Identifier x - operator * + Star * Identifier y BinOp Identifier x - operator + + Plus + Number 9 end end`) }) diff --git a/src/parser/tests/multiline.test.ts b/src/parser/tests/multiline.test.ts index f71faab..daf6d84 100644 --- a/src/parser/tests/multiline.test.ts +++ b/src/parser/tests/multiline.test.ts @@ -22,7 +22,7 @@ describe('multiline', () => { `).toMatchTree(` Assign AssignableIdentifier add - operator = + Eq = FunctionDef keyword fn Params @@ -31,10 +31,10 @@ describe('multiline', () => { colon : Assign AssignableIdentifier result - operator = + Eq = BinOp Identifier a - operator + + Plus + Identifier b FunctionCallOrIdentifier Identifier result diff --git a/src/parser/tests/pipes.test.ts b/src/parser/tests/pipes.test.ts index 61d6f73..bfd000e 100644 --- a/src/parser/tests/pipes.test.ts +++ b/src/parser/tests/pipes.test.ts @@ -51,7 +51,7 @@ describe('pipe expressions', () => { expect('result = echo hello | grep h').toMatchTree(` Assign AssignableIdentifier result - operator = + Eq = PipeExpr FunctionCall Identifier echo diff --git a/src/parser/tests/strings.test.ts b/src/parser/tests/strings.test.ts index 0fd6231..3f78f56 100644 --- a/src/parser/tests/strings.test.ts +++ b/src/parser/tests/strings.test.ts @@ -20,7 +20,7 @@ describe('string interpolation', () => { ParenExpr BinOp Identifier a - operator + + Plus + Identifier b StringFragment ! `) @@ -34,7 +34,7 @@ describe('string interpolation', () => { ParenExpr BinOp Identifier a - operator + + Plus + Identifier b `) }) diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts index 767c2b6..8963ffb 100644 --- a/src/parser/tokenizer.ts +++ b/src/parser/tokenizer.ts @@ -8,6 +8,12 @@ export const tokenizer = new ExternalTokenizer( const ch = getFullCodePoint(input, 0) if (!isWordChar(ch)) return + // Don't consume things that start with digits - let Number token handle it + if (isDigit(ch)) return + + // Don't consume things that start with - or + followed by a digit (negative/positive numbers) + if ((ch === 45 /* - */ || ch === 43) /* + */ && isDigit(input.peek(1))) return + const isValidStart = isLowercaseLetter(ch) || isEmoji(ch) const canBeWord = stack.canShift(Word) @@ -166,7 +172,16 @@ const chooseIdentifierToken = (input: InputStream, stack: Stack): number => { } const nextCh = getFullCodePoint(input, peekPos) - return nextCh === 61 /* = */ ? AssignableIdentifier : Identifier + if (nextCh === 61 /* = */) { + // Found '=', but check if it's followed by whitespace + // If '=' is followed by non-whitespace (like '=cool*'), it won't be tokenized as Eq + // In that case, this should be Identifier (for function call), not AssignableIdentifier + const charAfterEquals = getFullCodePoint(input, peekPos + 1) + if (isWhiteSpace(charAfterEquals) || charAfterEquals === -1 /* EOF */) { + return AssignableIdentifier + } + } + return Identifier } // Character classification helpers diff --git a/src/testSetup.ts b/src/testSetup.ts index 56b851c..799dbcb 100644 --- a/src/testSetup.ts +++ b/src/testSetup.ts @@ -1,10 +1,10 @@ import { expect } from 'bun:test' -import { Tree, TreeCursor } from '@lezer/common' import { parser } from '#parser/shrimp' import { $ } from 'bun' -import { assert, assertNever, errorMessage } from '#utils/utils' +import { assert, errorMessage } from '#utils/utils' import { Compiler } from '#compiler/compiler' -import { run, VM, type Value } from 'reefvm' +import { run, VM } from 'reefvm' +import { treeToString, VMResultToValue } from '#utils/tree' const regenerateParser = async () => { let generate = true @@ -131,10 +131,11 @@ expect.extend({ try { const compiler = new Compiler(received) const vm = new VM(compiler.bytecode) - await vm.run() + const value = await vm.run() return { - message: () => `Expected evaluation to fail, but it succeeded.`, + message: () => + `Expected evaluation to fail, but it succeeded with ${JSON.stringify(value)}`, pass: false, } } catch (error) { @@ -146,38 +147,6 @@ expect.extend({ }, }) -const treeToString = (tree: Tree, input: string): string => { - const lines: string[] = [] - - const addNode = (cursor: TreeCursor, depth: number) => { - if (!cursor.name) return - - const indent = ' '.repeat(depth) - const text = input.slice(cursor.from, cursor.to) - const nodeName = cursor.name // Save the node name before moving cursor - - if (cursor.firstChild()) { - lines.push(`${indent}${nodeName}`) - do { - addNode(cursor, depth + 1) - } while (cursor.nextSibling()) - cursor.parent() - } else { - const cleanText = nodeName === 'String' ? text.slice(1, -1) : text - lines.push(`${indent}${nodeName} ${cleanText}`) - } - } - - const cursor = tree.cursor() - if (cursor.firstChild()) { - do { - addNode(cursor, 0) - } while (cursor.nextSibling()) - } - - return lines.join('\n') -} - const trimWhitespace = (str: string): string => { const lines = str.split('\n').filter((line) => line.trim().length > 0) const firstLine = lines[0] @@ -196,29 +165,3 @@ const trimWhitespace = (str: string): string => { }) .join('\n') } - -const VMResultToValue = (result: Value): unknown => { - if ( - result.type === 'number' || - result.type === 'boolean' || - result.type === 'string' || - result.type === 'regex' - ) { - return result.value - } else if (result.type === 'null') { - return null - } else if (result.type === 'array') { - return result.value.map(VMResultToValue) - } else if (result.type === 'dict') { - const obj: Record = {} - for (const [key, val] of Object.entries(result.value)) { - obj[key] = VMResultToValue(val) - } - - return obj - } else if (result.type === 'function') { - return Function - } else { - assertNever(result) - } -} diff --git a/src/utils/tree.ts b/src/utils/tree.ts new file mode 100644 index 0000000..1682d21 --- /dev/null +++ b/src/utils/tree.ts @@ -0,0 +1,61 @@ +import { Tree, TreeCursor } from '@lezer/common' +import { assertNever } from '#utils/utils' +import { type Value } from 'reefvm' + +export const treeToString = (tree: Tree, input: string): string => { + const lines: string[] = [] + + const addNode = (cursor: TreeCursor, depth: number) => { + if (!cursor.name) return + + const indent = ' '.repeat(depth) + const text = input.slice(cursor.from, cursor.to) + const nodeName = cursor.name // Save the node name before moving cursor + + if (cursor.firstChild()) { + lines.push(`${indent}${nodeName}`) + do { + addNode(cursor, depth + 1) + } while (cursor.nextSibling()) + cursor.parent() + } else { + const cleanText = nodeName === 'String' ? text.slice(1, -1) : text + lines.push(`${indent}${nodeName} ${cleanText}`) + } + } + + const cursor = tree.cursor() + if (cursor.firstChild()) { + do { + addNode(cursor, 0) + } while (cursor.nextSibling()) + } + + return lines.join('\n') +} + +export const VMResultToValue = (result: Value): unknown => { + if ( + result.type === 'number' || + result.type === 'boolean' || + result.type === 'string' || + result.type === 'regex' + ) { + return result.value + } else if (result.type === 'null') { + return null + } else if (result.type === 'array') { + return result.value.map(VMResultToValue) + } else if (result.type === 'dict') { + const obj: Record = {} + for (const [key, val] of Object.entries(result.value)) { + obj[key] = VMResultToValue(val) + } + + return obj + } else if (result.type === 'function') { + return Function + } else { + assertNever(result) + } +}