From a28bdf74c928aba08b138c954b16ad436319a56e Mon Sep 17 00:00:00 2001 From: Corey Johnson Date: Mon, 6 Oct 2025 16:37:07 -0700 Subject: [PATCH] ok, cool --- README.md | 2 +- src/bin/shrimp | 2 +- src/editor/keymap.ts | 2 +- .../evaluator.test.ts | 0 src/{evaluator => interpreter}/evaluator.ts | 2 +- .../runtimeError.ts | 0 src/{evaluator => interpreter}/treeHelper.ts | 0 src/parser/shrimp.grammar | 76 ++-- src/parser/shrimp.terms.ts | 24 +- src/parser/shrimp.test.ts | 330 +++++++++--------- src/parser/shrimp.ts | 16 +- src/parser/tokenizers.ts | 9 +- src/testSetup.ts | 2 +- 13 files changed, 242 insertions(+), 223 deletions(-) rename src/{evaluator => interpreter}/evaluator.test.ts (100%) rename src/{evaluator => interpreter}/evaluator.ts (99%) rename src/{evaluator => interpreter}/runtimeError.ts (100%) rename src/{evaluator => interpreter}/treeHelper.ts (100%) diff --git a/README.md b/README.md index 2360f9b..437b913 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Shrimp is a shell-like scripting language that combines the simplicity of comman ## Current Status & Goals ### Today's Implementation Goals -1. **Interpreter Setup** - Rename evaluator to interpreter for clarity +1. ✅ **Interpreter Setup** - Renamed evaluator to interpreter for clarity 2. **Command Execution** - Support calling external commands and built-in functions 3. **Variable Assignment** - Implement assignment with validation using Lezer context tracking diff --git a/src/bin/shrimp b/src/bin/shrimp index e4a8397..d62415a 100755 --- a/src/bin/shrimp +++ b/src/bin/shrimp @@ -1,7 +1,7 @@ #! /usr/bin/env bun import { parser } from '../parser/shrimp.js' -import { evaluate } from '../evaluator/evaluator.js' +import { evaluate } from '../interpreter/evaluator.js' const log = (...args: any[]) => console.log(...args) log.error = (...args: any[]) => console.error(...args) diff --git a/src/editor/keymap.ts b/src/editor/keymap.ts index 7c22e04..6a74e82 100644 --- a/src/editor/keymap.ts +++ b/src/editor/keymap.ts @@ -1,5 +1,5 @@ import { outputSignal } from '#editor/editor' -import { evaluate } from '#evaluator/evaluator' +import { evaluate } from '#interpreter/evaluator' import { parser } from '#parser/shrimp' import { errorMessage, log } from '#utils/utils' import { keymap } from '@codemirror/view' diff --git a/src/evaluator/evaluator.test.ts b/src/interpreter/evaluator.test.ts similarity index 100% rename from src/evaluator/evaluator.test.ts rename to src/interpreter/evaluator.test.ts diff --git a/src/evaluator/evaluator.ts b/src/interpreter/evaluator.ts similarity index 99% rename from src/evaluator/evaluator.ts rename to src/interpreter/evaluator.ts index 5f0700e..3d9231c 100644 --- a/src/evaluator/evaluator.ts +++ b/src/interpreter/evaluator.ts @@ -1,6 +1,6 @@ import { Tree, type SyntaxNode } from '@lezer/common' import * as terms from '../parser/shrimp.terms.ts' -import { RuntimeError } from '#evaluator/runtimeError.ts' +import { RuntimeError } from '#interpreter/runtimeError.ts' import { assert } from 'console' import { assertNever } from '#utils/utils.tsx' import { matchingCommands, type CommandShape } from '#editor/commands.ts' diff --git a/src/evaluator/runtimeError.ts b/src/interpreter/runtimeError.ts similarity index 100% rename from src/evaluator/runtimeError.ts rename to src/interpreter/runtimeError.ts diff --git a/src/evaluator/treeHelper.ts b/src/interpreter/treeHelper.ts similarity index 100% rename from src/evaluator/treeHelper.ts rename to src/interpreter/treeHelper.ts diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index 2cc57ec..7661996 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -1,7 +1,13 @@ @external propSource highlighting from "./highlight.js" -@top Program { (Expression newline)* } -@tokens { +@skip { space } + +@top Program { (line newline)* } + +@tokens { + @precedence { Number "-" } + + NamedArgPrefix { $[a-z]+ "=" } Number { "-"? $[0-9]+ ('.' $[0-9]+)? } Boolean { "true" | "false" } String { '\'' !["]* '\'' } @@ -9,6 +15,11 @@ space { " " } leftParen { "(" } rightParen { ")" } + ":" + "fn" + "do" + "end" + "=" "+"[@name=operator] "-"[@name=operator] "*"[@name=operator] @@ -16,20 +27,28 @@ } @external tokens tokenizer from "./tokenizers" { Identifier, Word } + @precedence { multiplicative @left, additive @left + call } -Expression { - FunctionCall | +line { + FunctionCall | FunctionCallOrIdentifier | + FunctionDef | + Assignment | + expressionWithoutIdentifier +} + +expression { + expressionWithoutIdentifier | Identifier +} + +expressionWithoutIdentifier { BinOp | - ParenExpr | - Word | - String | - Number | - Boolean + valueWithoutIdentifier } @@ -38,11 +57,11 @@ FunctionCallOrIdentifier { } FunctionCall { - Identifier (~ambig space arg)+ + Identifier arg+ } arg { - PositionalArg | NamedArg | IncompleteNamedArg + PositionalArg | NamedArg } PositionalArg { @@ -50,29 +69,36 @@ PositionalArg { } NamedArg { - Identifier "=" value + NamedArgPrefix value } -IncompleteNamedArg { - Identifier "=" +FunctionDef { + "fn" Params ":" "do" expression "end" +} + +Params { + Identifier* +} + +Assignment { + Identifier "=" line } BinOp { - operand ~ambig space !multiplicative "*" space operand | - operand ~ambig space !multiplicative "/" space operand | - operand ~ambig space !additive "+" space operand | - operand ~ambig space !additive "-" space operand + expression !multiplicative "*" expression | + expression !multiplicative "/" expression | + expression !additive "+" expression | + expression !additive "-" expression } -operand { - value | BinOp -} - - ParenExpr { - leftParen Expression rightParen + leftParen (expressionWithoutIdentifier | FunctionCall | FunctionCallOrIdentifier) rightParen } value { - ParenExpr | Identifier | Word | String | Number | Boolean + valueWithoutIdentifier | Identifier +} + +valueWithoutIdentifier { + ParenExpr | Word | String | Number | Boolean } diff --git a/src/parser/shrimp.terms.ts b/src/parser/shrimp.terms.ts index 376b5c8..109eda1 100644 --- a/src/parser/shrimp.terms.ts +++ b/src/parser/shrimp.terms.ts @@ -3,14 +3,16 @@ export const Identifier = 1, Word = 2, Program = 3, - Expression = 4, - FunctionCall = 5, - PositionalArg = 6, - ParenExpr = 7, - String = 8, - Number = 9, - Boolean = 10, - NamedArg = 11, - IncompleteNamedArg = 12, - FunctionCallOrIdentifier = 13, - BinOp = 14 + FunctionCall = 4, + PositionalArg = 5, + ParenExpr = 6, + BinOp = 7, + FunctionCallOrIdentifier = 12, + String = 13, + Number = 14, + Boolean = 15, + NamedArg = 16, + NamedArgPrefix = 17, + FunctionDef = 18, + Params = 20, + Assignment = 24 diff --git a/src/parser/shrimp.test.ts b/src/parser/shrimp.test.ts index 8c9c6d7..4027d61 100644 --- a/src/parser/shrimp.test.ts +++ b/src/parser/shrimp.test.ts @@ -21,48 +21,43 @@ describe('calling functions', () => { test('call with no args', () => { expect('tail').toMatchTree(` - Expression - FunctionCallOrIdentifier - Identifier tail + FunctionCallOrIdentifier + Identifier tail `) }) test('call with arg', () => { expect('tail path').toMatchTree(` - Expression - FunctionCall - Identifier tail - PositionalArg - Identifier path + FunctionCall + Identifier tail + PositionalArg + Identifier path `) }) test('call with arg and named arg', () => { expect('tail path lines=30').toMatchTree(` - Expression - FunctionCall - Identifier tail - PositionalArg - Identifier path - NamedArg - Identifier lines - Number 30 + FunctionCall + Identifier tail + PositionalArg + Identifier path + NamedArg + NamedArgPrefix lines= + Number 30 `) }) test('command with arg that is also a command', () => { expect('tail tail').toMatchTree(` - Expression - FunctionCall + FunctionCall + Identifier tail + PositionalArg Identifier tail - PositionalArg - Identifier tail `) expect('tai').toMatchTree(` - Expression - FunctionCallOrIdentifier - Identifier tai + FunctionCallOrIdentifier + Identifier tai `) }) @@ -74,200 +69,199 @@ describe('calling functions', () => { test('Incomplete namedArg', () => { expect('tail lines=').toMatchTree(` - Expression - FunctionCall - Identifier tail - IncompleteNamedArg - Identifier lines - `) + FunctionCall + Identifier tail + NamedArg + NamedArgPrefix lines= + ⚠ + ⚠ `) }) }) describe('Identifier', () => { - test('fails on underscores and capital letters', () => { - expect('myVar').toFailParse() - expect('underscore_var').toFailParse() - expect('_leadingUnderscore').toFailParse() - expect('trailingUnderscore_').toFailParse() - expect('mixed-123_var').toFailParse() - }) - test('parses identifiers with emojis and dashes', () => { expect('moo-😊-34').toMatchTree(` - Expression - FunctionCallOrIdentifier - Identifier moo-😊-34`) + FunctionCallOrIdentifier + Identifier moo-😊-34`) }) }) describe('Parentheses', () => { test('parses expressions with parentheses correctly', () => { expect('(2 + 3)').toMatchTree(` - Expression - ParenExpr - Expression - BinOp - Number 2 - operator + - Number 3`) + ParenExpr + BinOp + Number 2 + operator + + Number 3`) }) test('allows parens in function calls', () => { expect('echo (3 + 3)').toMatchTree(` - Expression - FunctionCall - Identifier echo - PositionalArg - ParenExpr - Expression - BinOp - Number 3 - operator + - Number 3`) + FunctionCall + Identifier echo + PositionalArg + ParenExpr + BinOp + Number 3 + operator + + Number 3`) + }) + + test('nested parentheses', () => { + expect('(2 + (1 * 4))').toMatchTree(` + ParenExpr + BinOp + Number 2 + operator + + ParenExpr + BinOp + Number 1 + operator * + Number 4`) + }) + + test('Function in parentheses', () => { + expect('4 + (echo 3)').toMatchTree(` + BinOp + Number 4 + operator + + ParenExpr + FunctionCall + Identifier echo + PositionalArg + Number 3`) }) }) describe('BinOp', () => { test('addition tests', () => { expect('2 + 3').toMatchTree(` - Expression - BinOp - Number 2 - operator + - Number 3 + BinOp + Number 2 + operator + + Number 3 `) }) test('subtraction tests', () => { expect('5 - 2').toMatchTree(` - Expression - BinOp - Number 5 - operator - - Number 2 + BinOp + Number 5 + operator - + Number 2 `) }) test('multiplication tests', () => { expect('4 * 3').toMatchTree(` - Expression - BinOp - Number 4 - operator * - Number 3 + BinOp + Number 4 + operator * + Number 3 `) }) test('division tests', () => { expect('8 / 2').toMatchTree(` - Expression - BinOp - Number 8 - operator / - Number 2 + BinOp + Number 8 + operator / + Number 2 `) }) test('mixed operations with precedence', () => { expect('2 + 3 * 4 - 5 / 1').toMatchTree(` - Expression + BinOp BinOp + Number 2 + operator + BinOp - Number 2 - operator + - BinOp - Number 3 - operator * - Number 4 - operator - - BinOp - Number 5 - operator / - Number 1 + Number 3 + operator * + Number 4 + operator - + BinOp + Number 5 + operator / + Number 1 `) }) }) -// describe('Fn', () => { -// test('parses function with single parameter', () => { -// expect('fn x: x + 1').toMatchTree(` -// Function -// keyword fn -// Params -// Identifier x -// colon : -// BinOp -// Identifier x -// operator + -// Number 1`) -// }) +describe('Fn', () => { + test('parses function no parameters', () => { + expect('fn: do 1 end').toMatchTree(` + FunctionDef + fn fn + Params + : : + do do + Number 1 + end end`) + }) -// test('parses function with multiple parameters', () => { -// expect('fn x y: x * y').toMatchTree(` -// Function -// keyword fn -// Params -// Identifier x -// Identifier y -// colon : -// BinOp -// Identifier x -// operator * -// Identifier y`) -// }) + test('parses function with single parameter', () => { + expect('fn x: do x + 1 end').toMatchTree(` + FunctionDef + fn fn + Params + Identifier x + : : + do do + BinOp + Identifier x + operator + + Number 1 + end end`) + }) -// test('parses nested functions', () => { -// expect('fn x: fn y: x + y').toMatchTree(` -// Function -// keyword fn -// Params -// Identifier x -// colon : -// Function -// keyword fn -// Params -// Identifier y -// colon : -// BinOp -// Identifier x -// operator + -// Identifier y`) -// }) -// }) + test('parses function with multiple parameters', () => { + expect('fn x y: do x * y end').toMatchTree(` + FunctionDef + fn fn + Params + Identifier x + Identifier y + : : + do do + BinOp + Identifier x + operator * + Identifier y + end end`) + }) +}) -// describe('Identifier', () => { -// test('parses hyphenated identifiers correctly', () => { -// expect('my-var').toMatchTree(`Identifier my-var`) -// expect('double--trouble').toMatchTree(`Identifier double--trouble`) -// }) -// }) +describe('Assignment', () => { + test('parses assignment with addition', () => { + expect('x = 5 + 3').toMatchTree(` + Assignment + Identifier x + = = + BinOp + Number 5 + operator + + Number 3`) + }) -// describe('Assignment', () => { -// test('parses assignment with addition', () => { -// expect('x = 5 + 3').toMatchTree(` -// Assignment -// Identifier x -// operator = -// BinOp -// Number 5 -// operator + -// Number 3`) -// }) - -// test('parses assignment with functions', () => { -// expect('add = fn a b: a + b').toMatchTree(` -// Assignment -// Identifier add -// operator = -// Function -// keyword fn -// Params -// Identifier a -// Identifier b -// colon : -// BinOp -// Identifier a -// operator + -// Identifier b`) -// }) -// }) + test('parses assignment with functions', () => { + expect('add = fn a b: do a + b end').toMatchTree(` + Assignment + Identifier add + = = + FunctionDef + fn fn + Params + Identifier a + Identifier b + : : + do do + BinOp + Identifier a + operator + + Identifier b + end end`) + }) +}) diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index 8a9f492..36f6a81 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -4,16 +4,16 @@ import {tokenizer} from "./tokenizers" import {highlighting} from "./highlight.js" export const parser = LRParser.deserialize({ version: 14, - states: "$nQQOTOOOQOTO'#CcOfOPO'#CtOqOPO'#CtOOOO'#Cx'#CxO!POPO'#CxO![OPOOOOOO'#C`'#C`O!aOPO'#CoQQOTOOO!fOPO,58}O!|OTO'#CpO#TOPO,58{O#`OQO,59UOOOS,59Z,59ZOOOS-E6m-E6mOOOO1G.i1G.iOOOO'#Ct'#CtO#nOPO'#CtOOOO'#Cb'#CbOOOO'#Cs'#CsOOOO,59[,59[OOOO-E6n-E6nO#|OPO1G.pO$ROTO,59SO$cOTO7+$[OOOO1G.m1G.mOOOO< { let ch = getFullCodePoint(input, 0) @@ -7,18 +7,15 @@ export const tokenizer = new ExternalTokenizer((input: InputStream, stack: Stack let pos = getCharSize(ch) let isValidIdentifier = isLowercaseLetter(ch) || isEmoji(ch) + const canBeWord = stack.canShift(Word) while (true) { ch = getFullCodePoint(input, pos) if (isWhitespace(ch) || ch === -1) break - // Only stop at = if we could parse a NamedArg here - if (ch === 61 /* = */ && isValidIdentifier) { - break // Stop, let grammar handle identifier = value - } - // Track identifier validity if (!isLowercaseLetter(ch) && !isDigit(ch) && ch !== 45 && !isEmoji(ch)) { + if (!canBeWord) break isValidIdentifier = false } diff --git a/src/testSetup.ts b/src/testSetup.ts index 6d990ab..7f9c94b 100644 --- a/src/testSetup.ts +++ b/src/testSetup.ts @@ -3,7 +3,7 @@ import { Tree, TreeCursor } from '@lezer/common' import { parser } from '#parser/shrimp' import { $ } from 'bun' import { assert } from '#utils/utils' -import { evaluate } from '#evaluator/evaluator' +import { evaluate } from '#interpreter/evaluator' const regenerateParser = async () => { let generate = true