From f608c9e4c51573585322774079ea2cb7df5380b8 Mon Sep 17 00:00:00 2001 From: Corey Johnson Date: Fri, 3 Oct 2025 08:15:02 -0700 Subject: [PATCH] wip --- README.md | 96 ++++- src/editor/commands.ts | 21 ++ src/editor/editor.tsx | 2 +- src/evaluator/evaluator.test.ts | 85 ++++- src/evaluator/evaluator.ts | 276 +++++++-------- src/evaluator/runtimeError.ts | 16 + src/parser/highlight.js | 1 - src/parser/shrimp.grammar | 3 +- src/parser/shrimp.terms.ts | 23 +- src/parser/shrimp.test.ts | 604 ++++++++++++++++---------------- src/parser/shrimp.ts | 12 +- src/testSetup.ts | 2 +- 12 files changed, 653 insertions(+), 488 deletions(-) create mode 100644 src/evaluator/runtimeError.ts diff --git a/README.md b/README.md index 8afc0f2..d647a09 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,89 @@ -# bun-react-tailwind-template +# Shrimp Parser - Development Context -To install dependencies: +## Overview -```bash -bun install +Building a command-line language parser using Lezer (CodeMirror's parser system) with TypeScript. The goal is to create a prototype that can parse commands with arguments, similar to shell syntax, with inline hints for autocompletion. + +## Current Architecture + +### Grammar Structure (`shrimp.grammar`) + +- **Commands**: Can be complete (`Command`) or partial (`CommandPartial`) for autocomplete +- **Arguments**: Positional or named (with `name=value` syntax) +- **Key Challenge**: Handling arbitrary text (like file paths) as arguments without conflicting with operators/keywords + +### Tokenizer Setup (`tokenizers.ts`) + +- **Main tokenizer**: Returns `Command`, `CommandPartial`, or `Identifier` based on context +- **Command matching**: Uses `matchCommand()` to check against available commands +- **Context-aware**: Uses `stack.canShift()` to return appropriate token based on parse position +- **Issue**: Second occurrence of command name (e.g., `tail tail`) should be `Identifier` not `Command` + +### Key Design Decisions + +1. **External tokenizers over regular tokens** for commands to enable: + + - Dynamic command list (can change at runtime) + - Partial matching for autocomplete + - Context-aware tokenization + +2. **Virtual semicolons** for statement boundaries: + + - Using `insertSemicolon` external tokenizer + - Inserts at newlines/EOF to keep parser "inside" CommandCall + - Prevents `tail t` from parsing as two separate commands + +3. **UnquotedArg token** for paths/arbitrary text: + - Accepts anything except whitespace/parens/equals + - Only valid in command argument context + - Avoids conflicts with operators elsewhere + +### Current Problems + +1. **Parser completes CommandCall too early** + + - After `tail `, cursor shows position in `Program` not `CommandCall` + - Makes hint system harder to implement + +2. **Command token in wrong context** + + - `tail tail` - second "tail" returns `Command` token but should be `Identifier` + - Need better context checking in tokenizer + +3. **Inline hints need to be smarter** + - Must look backward to find command context + - Handle cases where parser has "completed" the command + +### Test Infrastructure + +- Custom test matchers: `toMatchTree`, `toEvaluateTo` +- Command source injection for testing: `setCommandSource()` +- Tests in `shrimp.test.ts` + +### File Structure + +``` +src/parser/ + shrimp.grammar - Lezer grammar definition + tokenizers.ts - External tokenizers + shrimp.ts - Generated parser + +src/editor/ + commands.ts - Command definitions + plugins/ + inlineHints.tsx - Autocomplete hint UI ``` -To start a development server: +## Next Steps -```bash -bun dev -``` +1. Fix tokenizer context checking with `stack.canShift()` +2. Improve hint detection for "after command with space" case +3. Consider if grammar structure changes would help -To run for production: +## Key Concepts to Remember -```bash -bun start -``` - -This project was created using `bun init` in bun v1.2.22. [Bun](https://bun.com) is a fast all-in-one JavaScript runtime. +- Lezer is LR parser - builds tree bottom-up +- External tokenizers run at each position +- `@skip { space }` makes whitespace invisible to parser +- Token precedence matters for overlap resolution +- `stack.canShift(tokenId)` checks if token is valid at current position diff --git a/src/editor/commands.ts b/src/editor/commands.ts index b3407f7..499c230 100644 --- a/src/editor/commands.ts +++ b/src/editor/commands.ts @@ -2,6 +2,7 @@ export type CommandShape = { command: string description?: string args: ArgShape[] + execute: string | ((...args: any[]) => any) } type ArgShape = @@ -29,6 +30,7 @@ const commandShapes: CommandShape[] = [ { command: 'ls', description: 'List the contents of a directory', + execute: './commands/ls.ts', args: [ { name: 'path', type: 'string', description: 'The path to list' }, { name: 'all', type: 'boolean', description: 'Show hidden files', default: false }, @@ -46,12 +48,14 @@ const commandShapes: CommandShape[] = [ { command: 'cd', description: 'Change the current working directory', + execute: './commands/cd.ts', args: [{ name: 'path', type: 'string', description: 'The path to change to' }], }, { command: 'cp', description: 'Copy files or directories', + execute: './commands/cp.ts', args: [ { name: 'source', type: 'string', description: 'Source file or directory' }, { name: 'destination', type: 'string', description: 'Destination path' }, @@ -63,6 +67,7 @@ const commandShapes: CommandShape[] = [ { command: 'mv', description: 'Move files or directories', + execute: './commands/mv.ts', args: [ { name: 'source', type: 'string', description: 'Source file or directory' }, { name: 'destination', type: 'string', description: 'Destination path' }, @@ -73,6 +78,7 @@ const commandShapes: CommandShape[] = [ { command: 'rm', description: 'Remove files or directories', + execute: './commands/rm.ts', args: [ { name: 'path', type: 'string', description: 'Path to remove' }, { name: 'recursive', type: 'boolean', description: 'Remove recursively', default: false }, @@ -84,6 +90,7 @@ const commandShapes: CommandShape[] = [ { command: 'mkdir', description: 'Create directories', + execute: './commands/mkdir.ts', args: [ { name: 'path', type: 'string', description: 'Directory path to create' }, { name: 'verbose', type: 'boolean', description: 'Verbose output', default: false }, @@ -93,6 +100,7 @@ const commandShapes: CommandShape[] = [ { command: 'touch', description: 'Create empty files or update timestamps', + execute: './commands/touch.ts', args: [ { name: 'path', type: 'string', description: 'File path to touch' }, { name: 'access', type: 'boolean', description: 'Update access time only', default: false }, @@ -108,6 +116,7 @@ const commandShapes: CommandShape[] = [ { command: 'echo', description: 'Display a string', + execute: './commands/echo.ts', args: [ { name: 'text', type: 'string', description: 'Text to display' }, { name: 'no-newline', type: 'boolean', description: "Don't append newline", default: false }, @@ -117,6 +126,7 @@ const commandShapes: CommandShape[] = [ { command: 'cat', description: 'Display file contents', + execute: './commands/cat.ts', args: [ { name: 'path', type: 'string', description: 'File to display' }, { name: 'numbered', type: 'boolean', description: 'Show line numbers', default: false }, @@ -126,6 +136,7 @@ const commandShapes: CommandShape[] = [ { command: 'head', description: 'Show first lines of input', + execute: './commands/head.ts', args: [ { name: 'path', type: 'string', description: 'File to read from' }, { name: 'lines', type: 'number', description: 'Number of lines', default: 10 }, @@ -135,6 +146,7 @@ const commandShapes: CommandShape[] = [ { command: 'tail', description: 'Show last lines of input', + execute: './commands/tail.ts', args: [ { name: 'path', type: 'string', description: 'File to read from' }, { name: 'lines', type: 'number', description: 'Number of lines', default: 10 }, @@ -145,6 +157,7 @@ const commandShapes: CommandShape[] = [ { command: 'grep', description: 'Search for patterns in text', + execute: './commands/grep.ts', args: [ { name: 'pattern', type: 'string', description: 'Pattern to search for' }, { @@ -161,6 +174,7 @@ const commandShapes: CommandShape[] = [ { command: 'sort', description: 'Sort input', + execute: './commands/sort.ts', args: [ { name: 'reverse', type: 'boolean', description: 'Sort in reverse order', default: false }, { @@ -176,6 +190,7 @@ const commandShapes: CommandShape[] = [ { command: 'uniq', description: 'Filter out repeated lines', + execute: './commands/uniq.ts', args: [ { name: 'count', type: 'boolean', description: 'Show count of occurrences', default: false }, { @@ -191,24 +206,28 @@ const commandShapes: CommandShape[] = [ { command: 'select', description: 'Select specific columns from data', + execute: './commands/select.ts', args: [{ name: 'columns', type: 'string', description: 'Columns to select' }], }, { command: 'where', description: 'Filter data based on conditions', + execute: './commands/where.ts', args: [{ name: 'condition', type: 'string', description: 'Filter condition' }], }, { command: 'group-by', description: 'Group data by column values', + execute: './commands/group-by.ts', args: [{ name: 'column', type: 'string', description: 'Column to group by' }], }, { command: 'ps', description: 'List running processes', + execute: './commands/ps.ts', args: [ { name: 'long', type: 'boolean', description: 'Show detailed information', default: false }, ], @@ -217,12 +236,14 @@ const commandShapes: CommandShape[] = [ { command: 'sys', description: 'Show system information', + execute: './commands/sys.ts', args: [], }, { command: 'which', description: 'Find the location of a command', + execute: './commands/which.ts', args: [ { name: 'command', type: 'string', description: 'Command to locate' }, { name: 'all', type: 'boolean', description: 'Show all matches', default: false }, diff --git a/src/editor/editor.tsx b/src/editor/editor.tsx index a1bf1b8..e173000 100644 --- a/src/editor/editor.tsx +++ b/src/editor/editor.tsx @@ -41,7 +41,7 @@ export const Editor = () => { shrimpLanguage(), shrimpHighlighting, shrimpErrors, - inlineHints, + // inlineHints, debugTags, ], }) diff --git a/src/evaluator/evaluator.test.ts b/src/evaluator/evaluator.test.ts index 88ac688..36e0e60 100644 --- a/src/evaluator/evaluator.test.ts +++ b/src/evaluator/evaluator.test.ts @@ -1,8 +1,83 @@ +import { resetCommandSource, setCommandSource, type CommandShape } from '#editor/commands' import { expect, test } from 'bun:test' -test('parses simple assignments', () => { - expect('number = 5').toEvaluateTo(5) - expect('number = -5.3').toEvaluateTo(-5.3) - expect(`string = 'abc'`).toEvaluateTo('abc') - expect('boolean = true').toEvaluateTo(true) +test('number literal', () => { + expect('42').toEvaluateTo(42) }) + +test('negative number', () => { + expect('-5').toEvaluateTo(-5) +}) + +test('string literal', () => { + expect(`'hello'`).toEvaluateTo('hello') +}) + +test('boolean true', () => { + expect('true').toEvaluateTo(true) +}) + +test('boolean false', () => { + expect('false').toEvaluateTo(false) +}) + +test('addition', () => { + expect('2 + 3').toEvaluateTo(5) +}) + +test('subtraction', () => { + expect('10 - 4').toEvaluateTo(6) +}) + +test('multiplication', () => { + expect('3 * 4').toEvaluateTo(12) +}) + +test('division', () => { + expect('15 / 3').toEvaluateTo(5) +}) + +test('assign number', () => { + expect('x = 5').toEvaluateTo(5) +}) + +test('emoji assignment to number', () => { + expect('πŸ’Ž = 5').toEvaluateTo(5) +}) + +test('assign string', () => { + expect(`name = 'Alice'`).toEvaluateTo('Alice') +}) + +test('assign expression', () => { + expect('sum = 2 + 3').toEvaluateTo(5) +}) + +test('parentheses', () => { + expect('(2 + 3) * 4').toEvaluateTo(20) +}) + +test('simple command', () => { + const commands: CommandShape[] = [ + { + command: 'echo', + args: [{ name: 'text', type: 'string' }], + execute: (text: string) => text, + }, + ] + + withCommands(commands, () => { + expect(`echo hello`).toEvaluateTo('hello') + }) +}) + +const withCommands = (commands: CommandShape[], fn: () => void) => { + try { + setCommandSource(() => commands) + fn() + } catch (e) { + throw e + } finally { + resetCommandSource() + } +} diff --git a/src/evaluator/evaluator.ts b/src/evaluator/evaluator.ts index d79a7a3..ac62ad5 100644 --- a/src/evaluator/evaluator.ts +++ b/src/evaluator/evaluator.ts @@ -1,36 +1,8 @@ -import { nodeToString } from '#/evaluator/treeHelper' import { Tree, type SyntaxNode } from '@lezer/common' import * as terms from '../parser/shrimp.terms.ts' -import { errorMessage } from '#utils/utils.ts' - -type Context = Map - -function getChildren(node: SyntaxNode): SyntaxNode[] { - const children = [] - let child = node.firstChild - while (child) { - children.push(child) - child = child.nextSibling - } - return children -} - -class RuntimeError extends Error { - constructor(message: string, private input: string, private from: number, private to: number) { - super(message) - this.name = 'RuntimeError' - this.message = `${message} at "${input.slice(from, to)}" (${from}:${to})` - } - - toReadableString(code: string) { - const pointer = ' '.repeat(this.from) + '^'.repeat(this.to - this.from) - const context = code.split('\n').slice(-2).join('\n') - return `${context}\n${pointer}\n${this.message}` - } -} +import { RuntimeError } from '#evaluator/runtimeError.ts' export const evaluate = (input: string, tree: Tree, context: Context) => { - // Just evaluate the top-level children, don't use iterate() let result = undefined let child = tree.topNode.firstChild try { @@ -42,7 +14,7 @@ export const evaluate = (input: string, tree: Tree, context: Context) => { if (error instanceof RuntimeError) { throw new Error(error.toReadableString(input)) } else { - throw new RuntimeError('Unknown error during evaluation', input, 0, input.length) + throw new Error('Unknown error during evaluation') } } @@ -50,138 +22,150 @@ export const evaluate = (input: string, tree: Tree, context: Context) => { } const evaluateNode = (node: SyntaxNode, input: string, context: Context): any => { - const value = input.slice(node.from, node.to) + const evalNode = syntaxNodeToEvalNode(node, input, context) - switch (node.type.id) { - case terms.Number: { - return parseFloat(value) - } + switch (evalNode.kind) { + case 'number': + case 'string': + case 'boolean': + return evalNode.value - case terms.String: { - return value.slice(1, -1) // Remove quotes - } - case terms.Boolean: { - return value === 'true' - } - - case terms.Identifier: { - if (!context.has(value)) { - throw new RuntimeError(`Undefined identifier: ${value}`, input, node.from, node.to) - } - return context.get(value) - } - - case terms.BinOp: { - let [left, op, right] = getChildren(node) - - left = assertNode(left, 'LeftOperand') - op = assertNode(op, 'Operator') - right = assertNode(right, 'RightOperand') - - const leftValue = evaluateNode(left, input, context) - const opValue = input.slice(op.from, op.to) - const rightValue = evaluateNode(right, input, context) - - switch (opValue) { - case '+': - return leftValue + rightValue - case '-': - return leftValue - rightValue - case '*': - return leftValue * rightValue - case '/': - return leftValue / rightValue - default: - throw new RuntimeError(`Unknown operator: ${opValue}`, input, op.from, op.to) + case 'identifier': { + const name = evalNode.name + if (context.has(name)) { + return context.get(name) + } else { + throw new RuntimeError(`Undefined variable "${name}"`, node.from, node.to) } } - case terms.Assignment: { - const [identifier, _operator, expr] = getChildren(node) - - const identifierNode = assertNode(identifier, 'Identifier') - const exprNode = assertNode(expr, 'Expression') - - const name = input.slice(identifierNode.from, identifierNode.to) - const value = evaluateNode(exprNode, input, context) + case 'assignment': { + const name = evalNode.name + const value = evaluateNode(evalNode.value.node, input, context) context.set(name, value) return value } - case terms.Function: { - const [params, body] = getChildren(node) + case 'binop': { + const left = evaluateNode(evalNode.left, input, context) + const right = evaluateNode(evalNode.right, input, context) - const paramNodes = getChildren(assertNode(params, 'Parameters')) - const bodyNode = assertNode(body, 'Body') - - const paramNames = paramNodes.map((param) => { - const paramNode = assertNode(param, 'Identifier') - return input.slice(paramNode.from, paramNode.to) - }) - - return (...args: any[]) => { - if (args.length !== paramNames.length) { - throw new RuntimeError( - `Expected ${paramNames.length} arguments, but got ${args.length}`, - input, - node.from, - node.to - ) - } - - const localContext = new Map(context) - paramNames.forEach((param, index) => { - localContext.set(param, args[index]) - }) - - return evaluateNode(bodyNode, input, localContext) + if (evalNode.op === '+') { + return left + right + } else if (evalNode.op === '-') { + return left - right + } else if (evalNode.op === '*') { + return left * right + } else if (evalNode.op === '/') { + return left / right + } else { + throw new RuntimeError(`Unsupported operator "${evalNode.op}"`, node.from, node.to) } } + } +} + +type Operators = '+' | '-' | '*' | '/' +type Context = Map +type EvalNode = + | { kind: 'number'; value: number; node: SyntaxNode } + | { kind: 'string'; value: string; node: SyntaxNode } + | { kind: 'boolean'; value: boolean; node: SyntaxNode } + | { kind: 'identifier'; name: string; node: SyntaxNode } + | { kind: 'binop'; op: Operators; left: SyntaxNode; right: SyntaxNode; node: SyntaxNode } + | { kind: 'assignment'; name: string; value: EvalNode; node: SyntaxNode } + | { kind: 'command'; name: string; args: EvalNode[]; node: SyntaxNode } + +const syntaxNodeToEvalNode = (node: SyntaxNode, input: string, context: Context): EvalNode => { + const value = input.slice(node.from, node.to) + + switch (node.type.id) { + case terms.Number: + return { kind: 'number', value: parseFloat(value), node } + + case terms.String: + return { kind: 'string', value: value.slice(1, -1), node } // Remove quotes + + case terms.Boolean: + return { kind: 'boolean', value: value === 'true', node } + + case terms.Identifier: + return { kind: 'identifier', name: value, node } + + case terms.BinOp: { + const [left, op, right] = destructure(node, ['*', '*', '*']) + const opString = input.slice(op.from, op.to) as Operators + return { kind: 'binop', op: opString, left, right, node } + } + + case terms.Assignment: { + const [identifier, _equals, expr] = destructure(node, [terms.Identifier, '*', '*']) + + const name = input.slice(identifier.from, identifier.to) + const value = syntaxNodeToEvalNode(expr, input, context) + + return { kind: 'assignment', name, value, node } + } + + case terms.ParenExpr: { + const [_leftParen, expr, _rightParen] = destructure(node, ['*', '*', '*']) + return syntaxNodeToEvalNode(expr, input, context) + } case terms.CommandCall: { - const commandNode = assertNode(node.firstChild, 'Command') - const commandIdentifier = assertNode(commandNode.firstChild, 'Identifier') - const command = input.slice(commandIdentifier.from, commandIdentifier.to) + const [_at, identifier, _leftParen, ...rest] = destructure(node, [ + '*', + terms.Identifier, + '*', + '*', + ]) + } - const args = getChildren(node) - .slice(1) - .map((argNode) => { - if (argNode.type.id === terms.Arg) { - return evaluateNode(argNode, input, context) - } else if (argNode.type.id === terms.NamedArg) { - return evaluateNode(argNode, input, context) - } else { - throw new RuntimeError( - `Unexpected argument type: ${argNode.type.name}`, - input, - argNode.from, - argNode.to - ) - } - }) - const commandName = input.slice(commandIdentifier.from, commandIdentifier.to) + throw new RuntimeError(`Unsupported node type "${node.type.name}"`, node.from, node.to) +} + +/* +The code below is a... +SIN AGAINST GOD! +...but it makes it easier to use above +*/ +type ExpectedType = '*' | number +function destructure(node: SyntaxNode, expected: [ExpectedType]): [SyntaxNode] +function destructure( + node: SyntaxNode, + expected: [ExpectedType, ExpectedType] +): [SyntaxNode, SyntaxNode] +function destructure( + node: SyntaxNode, + expected: [ExpectedType, ExpectedType, ExpectedType] +): [SyntaxNode, SyntaxNode, SyntaxNode] +function destructure(node: SyntaxNode, expected: ExpectedType[]): SyntaxNode[] { + const children: SyntaxNode[] = [] + let child = node.firstChild + while (child) { + children.push(child) + child = child.nextSibling + } + + if (children.length !== expected.length) { + throw new RuntimeError( + `${node.type.name} expected ${expected.length} children, got ${children.length}`, + node.from, + node.to + ) + } + + children.forEach((child, i) => { + const expectedType = expected[i] + if (expectedType !== '*' && child.type.id !== expectedType) { + throw new RuntimeError( + `Child ${i} of ${node.type.name} expected ${expectedType}, got ${child.type.id} (${child.type.name})`, + child.from, + child.to + ) } + }) - default: - const isLowerCase = node.type.name[0] == node.type.name[0]?.toLowerCase() - - // Ignore nodes with lowercase names, those are for syntax only - if (!isLowerCase) { - throw new RuntimeError( - `Unsupported node type "${node.type.name}"`, - input, - node.from, - node.to - ) - } - } -} - -const assertNode = (node: any, expectedName: string): SyntaxNode => { - if (!node) { - throw new Error(`Expected "${expectedName}", but got undefined`) - } - - return node + return children } diff --git a/src/evaluator/runtimeError.ts b/src/evaluator/runtimeError.ts new file mode 100644 index 0000000..95f24ba --- /dev/null +++ b/src/evaluator/runtimeError.ts @@ -0,0 +1,16 @@ +export class RuntimeError extends Error { + constructor(message: string, private from: number, private to: number) { + super(message) + this.name = 'RuntimeError' + this.message = message + } + + toReadableString(input: string) { + const pointer = ' '.repeat(this.from) + '^'.repeat(this.to - this.from) + const message = `${this.message} at "${input.slice(this.from, this.to)}" (${this.from}:${ + this.to + })` + + return `${input}\n${pointer}\n${message}` + } +} diff --git a/src/parser/highlight.js b/src/parser/highlight.js index 5a66e1d..6076527 100644 --- a/src/parser/highlight.js +++ b/src/parser/highlight.js @@ -1,5 +1,4 @@ import { styleTags, tags } from '@lezer/highlight' -import { Command, Identifier, Params } from '#/parser/shrimp.terms' export const highlighting = styleTags({ Identifier: tags.name, diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index d88ae3c..fda2857 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -75,4 +75,5 @@ BinOp { expr !additive "-" expr } -atom { Identifier ~command | Number | String | Boolean | leftParen expr rightParen } +ParenExpr { leftParen expr rightParen } +atom { Identifier ~command | Number | String | Boolean | ParenExpr } diff --git a/src/parser/shrimp.terms.ts b/src/parser/shrimp.terms.ts index e87e5b9..0ce3b13 100644 --- a/src/parser/shrimp.terms.ts +++ b/src/parser/shrimp.terms.ts @@ -4,7 +4,7 @@ export const Command = 2, CommandPartial = 3, UnquotedArg = 4, - insertedSemi = 30, + insertedSemi = 31, Program = 5, CommandCall = 6, NamedArg = 7, @@ -12,13 +12,14 @@ export const Number = 9, String = 10, Boolean = 11, - leftParen = 12, - Assignment = 13, - equals = 14, - Function = 15, - fn = 16, - Params = 17, - BinOp = 19, - rightParen = 24, - PartialNamedArg = 25, - Arg = 26 + ParenExpr = 12, + leftParen = 13, + Assignment = 14, + equals = 15, + Function = 16, + fn = 17, + Params = 18, + BinOp = 20, + rightParen = 25, + PartialNamedArg = 26, + Arg = 27 diff --git a/src/parser/shrimp.test.ts b/src/parser/shrimp.test.ts index 7df1c79..a5d188c 100644 --- a/src/parser/shrimp.test.ts +++ b/src/parser/shrimp.test.ts @@ -1,331 +1,331 @@ -import { expect, describe, test } from 'bun:test' -import { afterEach } from 'bun:test' -import { resetCommandSource, setCommandSource } from '#editor/commands' -import { beforeEach } from 'bun:test' -import './shrimp.grammar' // Importing this so changes cause it to retest! +// import { expect, describe, test } from 'bun:test' +// import { afterEach } from 'bun:test' +// import { resetCommandSource, setCommandSource } from '#editor/commands' +// import { beforeEach } from 'bun:test' +// import './shrimp.grammar' // Importing this so changes cause it to retest! -describe('calling commands', () => { - beforeEach(() => { - setCommandSource(() => [ - { command: 'tail', args: [{ name: 'path', type: 'string' }] }, - { command: 'head', args: [{ name: 'path', type: 'string' }] }, - { command: 'echo', args: [{ name: 'path', type: 'string' }] }, - ]) - }) +// describe('calling commands', () => { +// beforeEach(() => { +// setCommandSource(() => [ +// { command: 'tail', args: [{ name: 'path', type: 'string' }] }, +// { command: 'head', args: [{ name: 'path', type: 'string' }] }, +// { command: 'echo', args: [{ name: 'path', type: 'string' }] }, +// ]) +// }) - afterEach(() => { - resetCommandSource() - }) +// afterEach(() => { +// resetCommandSource() +// }) - test('basic', () => { - expect('tail path').toMatchTree(` - CommandCall - Command tail - Arg - Identifier path - `) +// test('basic', () => { +// expect('tail path').toMatchTree(` +// CommandCall +// Command tail +// Arg +// Identifier path +// `) - expect('tai').toMatchTree(` - CommandCall - CommandPartial tai - `) - }) +// expect('tai').toMatchTree(` +// CommandCall +// CommandPartial tai +// `) +// }) - test('command with arg that is also a command', () => { - expect('tail tail').toMatchTree(` - CommandCall - Command tail - Arg - Identifier tail - `) +// test('command with arg that is also a command', () => { +// expect('tail tail').toMatchTree(` +// CommandCall +// Command tail +// Arg +// Identifier tail +// `) - expect('tai').toMatchTree(` - CommandCall - CommandPartial tai - `) - }) +// expect('tai').toMatchTree(` +// CommandCall +// CommandPartial tai +// `) +// }) - test('when no commands match, falls back to Identifier', () => { - expect('omgwtf').toMatchTree(` - Identifier omgwtf - `) - }) +// test('when no commands match, falls back to Identifier', () => { +// expect('omgwtf').toMatchTree(` +// Identifier omgwtf +// `) +// }) - // In shrimp.test.ts, add to the 'calling commands' section - test('arg', () => { - expect('tail l').toMatchTree(` - CommandCall - Command tail - Arg - Identifier l - `) - }) +// // In shrimp.test.ts, add to the 'calling commands' section +// test('arg', () => { +// expect('tail l').toMatchTree(` +// CommandCall +// Command tail +// Arg +// Identifier l +// `) +// }) - test('partial namedArg', () => { - expect('tail lines=').toMatchTree(` - CommandCall - Command tail - PartialNamedArg - NamedArgPrefix lines= - `) - }) +// test('partial namedArg', () => { +// expect('tail lines=').toMatchTree(` +// CommandCall +// Command tail +// PartialNamedArg +// NamedArgPrefix lines= +// `) +// }) - test('complete namedArg', () => { - expect('tail lines=10').toMatchTree(` - CommandCall - Command tail - NamedArg - NamedArgPrefix lines= - Number 10 - `) - }) +// test('complete namedArg', () => { +// expect('tail lines=10').toMatchTree(` +// CommandCall +// Command tail +// NamedArg +// NamedArgPrefix lines= +// Number 10 +// `) +// }) - test('mixed positional and named args', () => { - expect('tail ../file.txt lines=5').toMatchTree(` - CommandCall - Command tail - Arg - UnquotedArg ../file.txt - NamedArg - NamedArgPrefix lines= - Number 5 - `) - }) +// test('mixed positional and named args', () => { +// expect('tail ../file.txt lines=5').toMatchTree(` +// CommandCall +// Command tail +// Arg +// UnquotedArg ../file.txt +// NamedArg +// NamedArgPrefix lines= +// Number 5 +// `) +// }) - test('named args', () => { - expect(`tail lines='5' path`).toMatchTree(` - CommandCall - Command tail - NamedArg - NamedArgPrefix lines= - String 5 - Arg - Identifier path - `) - }) +// test('named args', () => { +// expect(`tail lines='5' path`).toMatchTree(` +// CommandCall +// Command tail +// NamedArg +// NamedArgPrefix lines= +// String 5 +// Arg +// Identifier path +// `) +// }) - test('complex args', () => { - expect(`tail lines=(2 + 3) filter='error' (a + b)`).toMatchTree(` - CommandCall - Command tail - NamedArg - NamedArgPrefix lines= - paren ( - BinOp - Number 2 - operator + - Number 3 - paren ) - NamedArg - NamedArgPrefix filter= - String error - - Arg - paren ( - BinOp - Identifier a - operator + - Identifier b - paren ) - `) - }) -}) +// test('complex args', () => { +// expect(`tail lines=(2 + 3) filter='error' (a + b)`).toMatchTree(` +// CommandCall +// Command tail +// NamedArg +// NamedArgPrefix lines= +// paren ( +// BinOp +// Number 2 +// operator + +// Number 3 +// paren ) +// NamedArg +// NamedArgPrefix filter= +// String error -describe('Identifier', () => { - test('parses simple identifiers', () => { - expect('hyphenated-var').toMatchTree(`Identifier hyphenated-var`) - expect('var').toMatchTree(`Identifier var`) - expect('var123').toMatchTree(`Identifier var123`) - }) +// Arg +// paren ( +// BinOp +// Identifier a +// operator + +// Identifier b +// paren ) +// `) +// }) +// }) - test('fails on underscores and capital letters', () => { - expect('myVar').toFailParse() - expect('underscore_var').toFailParse() - expect('_leadingUnderscore').toFailParse() - expect('trailingUnderscore_').toFailParse() - expect('mixed-123_var').toFailParse() - }) +// describe('Identifier', () => { +// test('parses simple identifiers', () => { +// expect('hyphenated-var').toMatchTree(`Identifier hyphenated-var`) +// expect('var').toMatchTree(`Identifier var`) +// expect('var123').toMatchTree(`Identifier var123`) +// }) - test('parses identifiers with emojis', () => { - expect('var😊').toMatchTree(`Identifier var😊`) - expect('😊').toMatchTree(`Identifier 😊`) - }) -}) +// test('fails on underscores and capital letters', () => { +// expect('myVar').toFailParse() +// expect('underscore_var').toFailParse() +// expect('_leadingUnderscore').toFailParse() +// expect('trailingUnderscore_').toFailParse() +// expect('mixed-123_var').toFailParse() +// }) -describe('BinOp', () => { - test('addition tests', () => { - expect('2 + 3').toMatchTree(` - BinOp - Number 2 - operator + - Number 3 - `) - }) +// test('parses identifiers with emojis', () => { +// expect('var😊').toMatchTree(`Identifier var😊`) +// expect('😊').toMatchTree(`Identifier 😊`) +// }) +// }) - test('subtraction tests', () => { - expect('5 - 2').toMatchTree(` - BinOp - Number 5 - operator - - Number 2 - `) - }) +// describe('BinOp', () => { +// test('addition tests', () => { +// expect('2 + 3').toMatchTree(` +// BinOp +// Number 2 +// operator + +// Number 3 +// `) +// }) - test('multiplication tests', () => { - expect('4 * 3').toMatchTree(` - BinOp - Number 4 - operator * - Number 3 - `) - }) +// test('subtraction tests', () => { +// expect('5 - 2').toMatchTree(` +// BinOp +// Number 5 +// operator - +// Number 2 +// `) +// }) - test('division tests', () => { - expect('8 / 2').toMatchTree(` - BinOp - Number 8 - operator / - Number 2 - `) - }) +// test('multiplication tests', () => { +// expect('4 * 3').toMatchTree(` +// BinOp +// Number 4 +// operator * +// Number 3 +// `) +// }) - test('mixed operations with precedence', () => { - expect('2 + 3 * 4 - 5 / 1').toMatchTree(` - BinOp - BinOp - Number 2 - operator + - BinOp - Number 3 - operator * - Number 4 - operator - - BinOp - Number 5 - operator / - Number 1 - `) - }) -}) +// test('division tests', () => { +// expect('8 / 2').toMatchTree(` +// BinOp +// Number 8 +// operator / +// Number 2 +// `) +// }) -describe('Fn', () => { - test('parses function with single parameter', () => { - expect('fn x: x + 1').toMatchTree(` - Function - keyword fn - Params - Identifier x - colon : - BinOp - Identifier x - operator + - Number 1`) - }) +// test('mixed operations with precedence', () => { +// expect('2 + 3 * 4 - 5 / 1').toMatchTree(` +// BinOp +// BinOp +// Number 2 +// operator + +// BinOp +// Number 3 +// operator * +// Number 4 +// operator - +// BinOp +// Number 5 +// operator / +// Number 1 +// `) +// }) +// }) - test('parses function with multiple parameters', () => { - expect('fn x y: x * y').toMatchTree(` - Function - keyword fn - Params - Identifier x - Identifier y - colon : - BinOp - Identifier x - operator * - Identifier y`) - }) +// describe('Fn', () => { +// test('parses function with single parameter', () => { +// expect('fn x: x + 1').toMatchTree(` +// Function +// keyword fn +// Params +// Identifier x +// colon : +// BinOp +// Identifier x +// operator + +// Number 1`) +// }) - test('parses nested functions', () => { - expect('fn x: fn y: x + y').toMatchTree(` - Function - keyword fn - Params - Identifier x - colon : - Function - keyword fn - Params - Identifier y - colon : - BinOp - Identifier x - operator + - Identifier y`) - }) -}) +// test('parses function with multiple parameters', () => { +// expect('fn x y: x * y').toMatchTree(` +// Function +// keyword fn +// Params +// Identifier x +// Identifier y +// colon : +// BinOp +// Identifier x +// operator * +// Identifier y`) +// }) -describe('Identifier', () => { - test('parses hyphenated identifiers correctly', () => { - expect('my-var').toMatchTree(`Identifier my-var`) - expect('double--trouble').toMatchTree(`Identifier double--trouble`) - }) -}) +// test('parses nested functions', () => { +// expect('fn x: fn y: x + y').toMatchTree(` +// Function +// keyword fn +// Params +// Identifier x +// colon : +// Function +// keyword fn +// Params +// Identifier y +// colon : +// BinOp +// Identifier x +// operator + +// Identifier y`) +// }) +// }) -describe('Assignment', () => { - test('parses assignment with addition', () => { - expect('x = 5 + 3').toMatchTree(` - Assignment - Identifier x - operator = - BinOp - Number 5 - operator + - Number 3`) - }) +// describe('Identifier', () => { +// test('parses hyphenated identifiers correctly', () => { +// expect('my-var').toMatchTree(`Identifier my-var`) +// expect('double--trouble').toMatchTree(`Identifier double--trouble`) +// }) +// }) - test('parses assignment with functions', () => { - expect('add = fn a b: a + b').toMatchTree(` - Assignment - Identifier add - operator = - Function - keyword fn - Params - Identifier a - Identifier b - colon : - BinOp - Identifier a - operator + - Identifier b`) - }) -}) +// describe('Assignment', () => { +// test('parses assignment with addition', () => { +// expect('x = 5 + 3').toMatchTree(` +// Assignment +// Identifier x +// operator = +// BinOp +// Number 5 +// operator + +// Number 3`) +// }) -describe('Parentheses', () => { - test('parses expressions with parentheses correctly', () => { - expect('(2 + 3) * 4').toMatchTree(` - BinOp - paren ( - BinOp - Number 2 - operator + - Number 3 - paren ) - operator * - Number 4`) - }) +// test('parses assignment with functions', () => { +// expect('add = fn a b: a + b').toMatchTree(` +// Assignment +// Identifier add +// operator = +// Function +// keyword fn +// Params +// Identifier a +// Identifier b +// colon : +// BinOp +// Identifier a +// operator + +// Identifier b`) +// }) +// }) - test('parses nested parentheses correctly', () => { - expect('((1 + 2) * (3 - 4)) / 5').toMatchTree(` - BinOp - paren ( - BinOp - paren ( - BinOp - Number 1 - operator + - Number 2 - paren ) - operator * - paren ( - BinOp - Number 3 - operator - - Number 4 - paren ) - paren ) - operator / - Number 5`) - }) -}) +// describe('Parentheses', () => { +// test('parses expressions with parentheses correctly', () => { +// expect('(2 + 3) * 4').toMatchTree(` +// BinOp +// paren ( +// BinOp +// Number 2 +// operator + +// Number 3 +// paren ) +// operator * +// Number 4`) +// }) + +// test('parses nested parentheses correctly', () => { +// expect('((1 + 2) * (3 - 4)) / 5').toMatchTree(` +// BinOp +// paren ( +// BinOp +// paren ( +// BinOp +// Number 1 +// operator + +// Number 2 +// paren ) +// operator * +// paren ( +// BinOp +// Number 3 +// operator - +// Number 4 +// paren ) +// paren ) +// operator / +// Number 5`) +// }) +// }) diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index 55602b6..45b62f9 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -4,15 +4,15 @@ import {tokenizer, argTokenizer, insertSemicolon} from "./tokenizers" import {highlighting} from "./highlight.js" export const parser = LRParser.deserialize({ version: 14, - states: "%^OkQTOOOuQaO'#DPO!aQTO'#CkO!iQaO'#C}OOQ`'#DQ'#DQOOQl'#DP'#DPOVQTO'#DPO#fQnO'#CbO!uQaO'#C}QOQPOOOVQTO,59TOOQS'#Cx'#CxO#pQTO'#CmO#xQPO,59VOVQTO,59ZOVQTO,59ZOOQO'#DR'#DROOQO,59i,59iO#}QPO,59kOOQl'#DO'#DOO$tQnO'#CuOOQl'#Cv'#CvOOQl'#Cw'#CwO%RQnO,58|O%]QaO1G.oOOQS-E6v-E6vOVQTO1G.qOOQ`1G.u1G.uO%tQaO1G.uOOQl1G/V1G/VOOQl,58},58}OOQl-E6u-E6uO&]QaO7+$]", - stateData: "&w~OpOS~OPPOXTOYTOZTO[UO`QO~OQVORVO~PVO^YOdsXesXfsXgsXnsXvsXhsX~OPZObaP~Od^Oe^Of_Og_On`Ov`O~OPTOScOWdOXTOYTOZTO[UO~OnUXvUX~P!}OPZObaX~ObjO~Od^Oe^Of_Og_OhmO~OPTOScOXTOYTOZTO[UO~OWiXniXviX~P$`OnUavUa~P!}Od^Oe^Of_Og_On]iv]ih]i~Od^Oe^Ofcigcincivcihci~Od^Oe^Of_Og_On_qv_qh_q~OXg~", - goto: "#fvPPPPPPwzPPPPP!OP!OP!WP!OPPPPPzz!Z!aPPPP!g!j!q#O#bRWOTfVg]SOUY^_jR]QQgVRogQ[QRi[RXOSeVgRnd[SOUY^_jVcVdgQROQbUQhYQk^Ql_RpjTaRW", - nodeNames: "⚠ Identifier Command CommandPartial UnquotedArg Program CommandCall NamedArg NamedArgPrefix Number String Boolean paren Assignment operator Function keyword Params colon BinOp operator operator operator operator paren PartialNamedArg Arg", - maxTerm: 38, + states: "%^OkQTOOOuQaO'#DQO!aQTO'#ClO!iQaO'#DOOOQ`'#DR'#DROVQTO'#ChOOQl'#DQ'#DQO#fQnO'#CbO!uQaO'#DOQOQPOOOVQTO,59UOOQS'#Cy'#CyO#pQTO'#CnO#xQPO,59WOVQTO,59[OVQTO,59[OOQO'#DS'#DSOOQO,59j,59jO#}QPO,59SOOQl'#DP'#DPO$tQnO'#CvOOQl'#Cw'#CwOOQl'#Cx'#CxO%RQnO,58|O%]QaO1G.pOOQS-E6w-E6wOVQTO1G.rOOQ`1G.v1G.vO%tQaO1G.vOOQl1G.n1G.nOOQl,58},58}OOQl-E6v-E6vO&]QaO7+$^", + stateData: "&w~OqOS~OPPOXUOYUOZUO]TOaQO~OQVORVO~PVO_YOetXftXgtXhtXotXwtXitX~OPZOcbP~Oe^Of^Og_Oh_Oo`Ow`O~OPUOScOWdOXUOYUOZUO]TO~OoUXwUX~P!}OPZOcbX~OcjO~Oe^Of^Og_Oh_OimO~OPUOScOXUOYUOZUO]TO~OWjXojXwjX~P$`OoUawUa~P!}Oe^Of^Og_Oh_Oo^iw^ii^i~Oe^Of^Ogdihdiodiwdiidi~Oe^Of^Og_Oh_Oo`qw`qi`q~OXh~", + goto: "#rwPPPPPPx{PPPP!PP![P![P!dP![PPPPP{{!g!mPPPP!s!v!}#[#nRWOTfVgcUOTVY^_dgj]SOTY^_jR]QQgVRogQ[QRi[RXOSeVgRnd[SOTY^_jVcVdgQROQbTQhYQk^Ql_RpjTaRW", + nodeNames: "⚠ Identifier Command CommandPartial UnquotedArg Program CommandCall NamedArg NamedArgPrefix Number String Boolean ParenExpr paren Assignment operator Function keyword Params colon BinOp operator operator operator operator paren PartialNamedArg Arg", + maxTerm: 39, propSources: [highlighting], skippedNodes: [0], repeatNodeCount: 2, - tokenData: "*W~RjX^!spq!swx#hxy$lyz$qz{$v{|${}!O%Q!P!Q%s!Q![%Y![!]%x!]!^%}!_!`&S#T#Y&X#Y#Z&m#Z#h&X#h#i)[#i#o&X#y#z!s$f$g!s#BY#BZ!s$IS$I_!s$I|$JO!s$JT$JU!s$KV$KW!s&FU&FV!s~!xYp~X^!spq!s#y#z!s$f$g!s#BY#BZ!s$IS$I_!s$I|$JO!s$JT$JU!s$KV$KW!s&FU&FV!s~#kUOr#hsw#hwx#}x;'S#h;'S;=`$f<%lO#h~$SUY~Or#hsw#hwx#}x;'S#h;'S;=`$f<%lO#h~$iP;=`<%l#h~$qO[~~$vOh~~${Od~~%QOf~~%VPg~!Q![%Y~%_QX~!O!P%e!Q![%Y~%hP!Q![%k~%pPX~!Q![%k~%xOe~~%}Ob~~&SOv~~&XO^~Q&[S}!O&X!Q![&X!_!`&h#T#o&XQ&mOWQ~&pV}!O&X!Q![&X!_!`&h#T#U'V#U#b&X#b#c(y#c#o&X~'YU}!O&X!Q![&X!_!`&h#T#`&X#`#a'l#a#o&X~'oU}!O&X!Q![&X!_!`&h#T#g&X#g#h(R#h#o&X~(UU}!O&X!Q![&X!_!`&h#T#X&X#X#Y(h#Y#o&X~(mSZ~}!O&X!Q![&X!_!`&h#T#o&XR)OS`P}!O&X!Q![&X!_!`&h#T#o&X~)_U}!O&X!Q![&X!_!`&h#T#f&X#f#g)q#g#o&X~)tU}!O&X!Q![&X!_!`&h#T#i&X#i#j(R#j#o&X", + tokenData: "*W~RjX^!spq!swx#hxy$lyz$qz{$v{|${}!O%Q!P!Q%s!Q![%Y![!]%x!]!^%}!_!`&S#T#Y&X#Y#Z&m#Z#h&X#h#i)[#i#o&X#y#z!s$f$g!s#BY#BZ!s$IS$I_!s$I|$JO!s$JT$JU!s$KV$KW!s&FU&FV!s~!xYq~X^!spq!s#y#z!s$f$g!s#BY#BZ!s$IS$I_!s$I|$JO!s$JT$JU!s$KV$KW!s&FU&FV!s~#kUOr#hsw#hwx#}x;'S#h;'S;=`$f<%lO#h~$SUY~Or#hsw#hwx#}x;'S#h;'S;=`$f<%lO#h~$iP;=`<%l#h~$qO]~~$vOi~~${Oe~~%QOg~~%VPh~!Q![%Y~%_QX~!O!P%e!Q![%Y~%hP!Q![%k~%pPX~!Q![%k~%xOf~~%}Oc~~&SOw~~&XO_~Q&[S}!O&X!Q![&X!_!`&h#T#o&XQ&mOWQ~&pV}!O&X!Q![&X!_!`&h#T#U'V#U#b&X#b#c(y#c#o&X~'YU}!O&X!Q![&X!_!`&h#T#`&X#`#a'l#a#o&X~'oU}!O&X!Q![&X!_!`&h#T#g&X#g#h(R#h#o&X~(UU}!O&X!Q![&X!_!`&h#T#X&X#X#Y(h#Y#o&X~(mSZ~}!O&X!Q![&X!_!`&h#T#o&XR)OSaP}!O&X!Q![&X!_!`&h#T#o&X~)_U}!O&X!Q![&X!_!`&h#T#f&X#f#g)q#g#o&X~)tU}!O&X!Q![&X!_!`&h#T#i&X#i#j(R#j#o&X", tokenizers: [0, 1, tokenizer, argTokenizer, insertSemicolon], topRules: {"Program":[0,5]}, tokenPrec: 266 diff --git a/src/testSetup.ts b/src/testSetup.ts index d674692..d9bedd0 100644 --- a/src/testSetup.ts +++ b/src/testSetup.ts @@ -128,7 +128,7 @@ expect.extend({ } } catch (error) { return { - message: () => `Evaluation threw an error: ${(error as Error).message}`, + message: () => `Evaluation threw an error:\n${(error as Error).message}`, pass: false, } }