diff --git a/CLAUDE.md b/CLAUDE.md index 356bd09..e0e372d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -207,6 +207,19 @@ Implementation files: **Why this matters**: This enables shell-like file paths (`readme.txt`) while supporting dictionary/array access (`config.path`) without quotes, determined entirely at parse time based on lexical scope. +**Array and dict literals**: Square brackets `[]` create both arrays and dicts, distinguished by content: +- **Arrays**: Space/newline/semicolon-separated args that work like calling a function → `[1 2 3]` (call functions using parens eg `[1 (double 4) 200]`) +- **Dicts**: NamedArg syntax (key=value pairs) → `[a=1 b=2]` +- **Empty array**: `[]` (standard empty brackets) +- **Empty dict**: `[=]` (exactly this, no spaces) + +Implementation details: +- Grammar rules (shrimp.grammar:194-201): Dict uses `NamedArg` nodes, Array uses `expression` nodes +- Parser distinguishes at parse time based on whether first element contains `=` +- Both support multiline, comments, and nesting +- Separators: spaces, newlines (`\n`), or semicolons (`;`) work interchangeably +- Test files: `src/parser/tests/literals.test.ts` and `src/compiler/tests/literals.test.ts` + **EOF handling**: The grammar uses `(statement | newlineOrSemicolon)+ eof?` to handle empty lines and end-of-file without infinite loops. ## Compiler Architecture diff --git a/bin/repl b/bin/repl index b614e26..052f481 100755 --- a/bin/repl +++ b/bin/repl @@ -1,24 +1,12 @@ #!/usr/bin/env bun import { Compiler } from '../src/compiler/compiler' -import { VM, type Value, Scope, bytecodeToString } from 'reefvm' +import { colors, formatValue, globals } from '../src/prelude' +import { VM, Scope, bytecodeToString } from 'reefvm' import * as readline from 'readline' import { readFileSync, writeFileSync } from 'fs' import { basename } from 'path' -const colors = { - reset: '\x1b[0m', - bright: '\x1b[1m', - dim: '\x1b[2m', - cyan: '\x1b[36m', - yellow: '\x1b[33m', - green: '\x1b[32m', - red: '\x1b[31m', - blue: '\x1b[34m', - magenta: '\x1b[35m', - pink: '\x1b[38;2;255;105;180m' -} - async function repl() { const commands = ['/clear', '/reset', '/vars', '/funcs', '/history', '/bytecode', '/exit', '/save', '/quit'] @@ -60,7 +48,7 @@ async function repl() { return } - vm ||= new VM({ instructions: [], constants: [] }, nativeFunctions) + vm ||= new VM({ instructions: [], constants: [] }, globals) if (['/exit', 'exit', '/quit', 'quit'].includes(trimmed)) { console.log(`\n${colors.yellow}Goodbye!${colors.reset}`) @@ -159,7 +147,7 @@ async function repl() { codeHistory.push(trimmed) try { - const compiler = new Compiler(trimmed) + const compiler = new Compiler(trimmed, Object.keys(globals)) vm.appendBytecode(compiler.bytecode) @@ -186,40 +174,6 @@ async function repl() { }) } - -function formatValue(value: Value, inner = false): string { - switch (value.type) { - case 'string': - return `${colors.green}'${value.value}'${colors.reset}` - case 'number': - return `${colors.cyan}${value.value}${colors.reset}` - case 'boolean': - return `${colors.yellow}${value.value}${colors.reset}` - case 'null': - return `${colors.dim}null${colors.reset}` - case 'array': { - const items = value.value.map(x => formatValue(x, true)).join(' ') - return `${inner ? '(' : ''}${colors.blue}list${colors.reset} ${items}${inner ? ')' : ''}` - } - case 'dict': { - const entries = Array.from(value.value.entries()) - .map(([k, v]) => `${k}=${formatValue(v, true)}`) - .join(' ') - return `${inner ? '(' : ''}${colors.magenta}dict${colors.reset} ${entries}${inner ? ')' : ''}` - } - case 'function': { - const params = value.params.join(', ') - return `${colors.dim}${colors.reset}` - } - case 'native': - return `${colors.dim}${colors.reset}` - case 'regex': - return `${colors.magenta}${value.value}${colors.reset}` - default: - return String(value) - } -} - function formatVariables(scope: Scope, onlyFunctions = false): string { const vars: string[] = [] @@ -257,7 +211,7 @@ async function loadFile(filePath: string): Promise<{ vm: VM; codeHistory: string console.log(`${colors.dim}Loading ${basename(filePath)}...${colors.reset}`) - const vm = new VM({ instructions: [], constants: [] }, nativeFunctions) + const vm = new VM({ instructions: [], constants: [] }, globals) await vm.run() const codeHistory: string[] = [] @@ -313,43 +267,4 @@ function showWelcome() { console.log() } -const nativeFunctions = { - echo: (...args: any[]) => { - console.log(...args) - }, - len: (value: any) => { - if (typeof value === 'string') return value.length - if (Array.isArray(value)) return value.length - if (value && typeof value === 'object') return Object.keys(value).length - return 0 - }, - type: (value: any) => { - if (value === null) return 'null' - if (Array.isArray(value)) return 'array' - return typeof value - }, - range: (start: number, end: number | null) => { - if (end === null) { - end = start - start = 0 - } - const result: number[] = [] - for (let i = start; i <= end; i++) { - result.push(i) - } - return result - }, - join: (arr: any[], sep: string = ',') => { - return arr.join(sep) - }, - split: (str: string, sep: string = ',') => { - return str.split(sep) - }, - upper: (str: string) => str.toUpperCase(), - lower: (str: string) => str.toLowerCase(), - trim: (str: string) => str.trim(), - list: (...args: any[]) => args, - dict: (atNamed = {}) => atNamed -} - await repl() diff --git a/bin/shrimp b/bin/shrimp index eed1d48..49cd7f3 100755 --- a/bin/shrimp +++ b/bin/shrimp @@ -1,57 +1,18 @@ #!/usr/bin/env bun import { Compiler } from '../src/compiler/compiler' -import { VM, toValue, fromValue, bytecodeToString } from 'reefvm' +import { colors, globals } from '../src/prelude' +import { VM, fromValue, bytecodeToString } from 'reefvm' import { readFileSync, writeFileSync, mkdirSync } from 'fs' import { randomUUID } from "crypto" import { spawn } from 'child_process' import { join } from 'path' -const colors = { - reset: '\x1b[0m', - bright: '\x1b[1m', - dim: '\x1b[2m', - red: '\x1b[31m', - yellow: '\x1b[33m', - cyan: '\x1b[36m', - magenta: '\x1b[35m', - pink: '\x1b[38;2;255;105;180m' -} - -const nativeFunctions = { - echo: (...args: any[]) => console.log(...args), - len: (value: any) => { - if (typeof value === 'string') return value.length - if (Array.isArray(value)) return value.length - if (value && typeof value === 'object') return Object.keys(value).length - return 0 - }, - type: (value: any) => toValue(value).type, - range: (start: number, end: number | null) => { - if (end === null) { - end = start - start = 0 - } - const result: number[] = [] - for (let i = start; i <= end; i++) { - result.push(i) - } - return result - }, - join: (arr: any[], sep: string = ',') => arr.join(sep), - split: (str: string, sep: string = ',') => str.split(sep), - upper: (str: string) => str.toUpperCase(), - lower: (str: string) => str.toLowerCase(), - trim: (str: string) => str.trim(), - list: (...args: any[]) => args, - dict: (atNamed = {}) => atNamed -} - async function runFile(filePath: string) { try { const code = readFileSync(filePath, 'utf-8') - const compiler = new Compiler(code) - const vm = new VM(compiler.bytecode, nativeFunctions) + const compiler = new Compiler(code, Object.keys(globals)) + const vm = new VM(compiler.bytecode, globals) await vm.run() return vm.stack.length ? fromValue(vm.stack[vm.stack.length - 1]) : null } catch (error: any) { diff --git a/bun.lock b/bun.lock index 7561a48..005ca60 100644 --- a/bun.lock +++ b/bun.lock @@ -62,7 +62,7 @@ "hono": ["hono@4.9.8", "", {}, "sha512-JW8Bb4RFWD9iOKxg5PbUarBYGM99IcxFl2FPBo2gSJO11jjUDqlP1Bmfyqt8Z/dGhIQ63PMA9LdcLefXyIasyg=="], - "reefvm": ["reefvm@git+https://git.nose.space/defunkt/reefvm#e54207067734d2186cd788c3654b675b493c2585", { "peerDependencies": { "typescript": "^5" } }, "e54207067734d2186cd788c3654b675b493c2585"], + "reefvm": ["reefvm@git+https://git.nose.space/defunkt/reefvm#030eb7487165b3ba502965a8b7fa09c4b5fdb0da", { "peerDependencies": { "typescript": "^5" } }, "030eb7487165b3ba502965a8b7fa09c4b5fdb0da"], "style-mod": ["style-mod@4.1.2", "", {}, "sha512-wnD1HyVqpJUI2+eKZ+eo1UwghftP6yuFheBqqe+bWCotBjC2K1YnteJILRMs3SM4V/0dLEW1SC27MWP5y+mwmw=="], diff --git a/example.shrimp b/example.shrimp index 53c563b..f1a9a05 100644 --- a/example.shrimp +++ b/example.shrimp @@ -42,13 +42,13 @@ a-file = file.txt 3 # symbols can be assigned to functions. The body of the function comes after a colon `:` -add = fn x y: x + y +add = do x y: x + y add 1 2 --- 3 # Functions can have multiple lines, they are terminated with `end` -sub = fn x y: +sub = do x y: x - y end @@ -82,9 +82,25 @@ add 1 (sub 5 2) 4 +# Arrays use square brackets with space-separated elements +numbers = [1 2 3] +shopping-list = [apples bananas carrots] +empty-array = [] + +# Dicts use square brackets with key=value pairs +config = [name=Shrimp version=1.0 debug=true] +empty-dict = [=] + +# Nested structures work naturally +nested = [ + users=[ + [name=Alice age=30] + [name=Bob age=25] + ] + settings=[debug=true timeout=5000] +] + # HOLD UP -- how do we handle arrays? -- how do we handle hashes? - conditionals - loops \ No newline at end of file diff --git a/package.json b/package.json index 4adf26d..2ac8e0c 100644 --- a/package.json +++ b/package.json @@ -3,13 +3,11 @@ "version": "0.1.0", "private": true, "type": "module", - "workspaces": [ - "packages/*" - ], "scripts": { "dev": "bun generate-parser && bun --hot src/server/server.tsx", "generate-parser": "lezer-generator src/parser/shrimp.grammar --typeScript -o src/parser/shrimp.ts", - "repl": "bun generate-parser && bun bin/repl" + "repl": "bun generate-parser && bun bin/repl", + "update-reef": "cd packages/ReefVM && git pull origin main" }, "dependencies": { "@codemirror/view": "^6.38.3", diff --git a/src/compiler/compiler.ts b/src/compiler/compiler.ts index b7e6274..e6b006b 100644 --- a/src/compiler/compiler.ts +++ b/src/compiler/compiler.ts @@ -1,6 +1,7 @@ import { CompilerError } from '#compiler/compilerError.ts' import { parser } from '#parser/shrimp.ts' import * as terms from '#parser/shrimp.terms' +import { setGlobals } from '#parser/tokenizer' import type { SyntaxNode, Tree } from '@lezer/common' import { assert, errorMessage } from '#utils/utils' import { toBytecode, type Bytecode, type ProgramItem, bytecodeToString } from 'reefvm' @@ -53,8 +54,9 @@ export class Compiler { bytecode: Bytecode pipeCounter = 0 - constructor(public input: string) { + constructor(public input: string, globals?: string[]) { try { + if (globals) setGlobals(globals) const cst = parser.parse(input) const errors = checkTreeForErrors(cst) @@ -218,6 +220,9 @@ export class Compiler { case '/': instructions.push(['DIV']) break + case '%': + instructions.push(['MOD']) + break default: throw new CompilerError(`Unsupported binary operator: ${opValue}`, op.from, op.to) } @@ -265,6 +270,9 @@ export class Compiler { } case terms.FunctionCallOrIdentifier: { + if (node.firstChild?.name === 'DotGet') + return this.#compileNode(node.firstChild, input) + return [['TRY_CALL', value]] } @@ -358,7 +366,7 @@ export class Compiler { const opValue = input.slice(op.from, op.to) switch (opValue) { - case '=': + case '==': instructions.push(...leftInstructions, ...rightInstructions, ['EQ']) break @@ -468,6 +476,44 @@ export class Compiler { return instructions } + case terms.Array: { + const children = getAllChildren(node) + + // We can easily parse [=] as an empty dict, but `[ = ]` is tougher. + // = can be a valid word, and is also valid inside words, so for now we cheat + // and check for arrays that look like `[ = ]` to interpret them as + // empty dicts + if (children.length === 1 && children[0]!.name === 'Word') { + const child = children[0]! + if (input.slice(child.from, child.to) === '=') { + return [['MAKE_DICT', 0]] + } + } + + const instructions: ProgramItem[] = children.map((x) => this.#compileNode(x, input)).flat() + instructions.push(['MAKE_ARRAY', children.length]) + return instructions + } + + case terms.Dict: { + const children = getAllChildren(node) + const instructions: ProgramItem[] = [] + + children.forEach((node) => { + const keyNode = node.firstChild + const valueNode = node.firstChild!.nextSibling + + // name= -> name + const key = input.slice(keyNode!.from, keyNode!.to).slice(0, -1) + instructions.push(['PUSH', key]) + + instructions.push(...this.#compileNode(valueNode!, input)) + }) + + instructions.push(['MAKE_DICT', children.length]) + return instructions + } + default: throw new CompilerError( `Compiler doesn't know how to handle a "${node.type.name}" node.`, diff --git a/src/compiler/tests/compiler.test.ts b/src/compiler/tests/compiler.test.ts index 46ee0b7..ef02035 100644 --- a/src/compiler/tests/compiler.test.ts +++ b/src/compiler/tests/compiler.test.ts @@ -38,6 +38,12 @@ describe('compiler', () => { expect('15 / 3').toEvaluateTo(5) }) + test('modulo', () => { + expect('44 % 2').toEvaluateTo(0) + expect('44 % 3').toEvaluateTo(2) + expect('3 % 4').toEvaluateTo(3) + }) + test('assign number', () => { expect('x = 5').toEvaluateTo(5) }) @@ -105,7 +111,7 @@ describe('compiler', () => { expect(`(10 > 20)`).toEvaluateTo(false) expect(`(4 <= 9)`).toEvaluateTo(true) expect(`(15 >= 20)`).toEvaluateTo(false) - expect(`(7 = 7)`).toEvaluateTo(true) + expect(`(7 == 7)`).toEvaluateTo(true) expect(`(5 != 5)`).toEvaluateTo(false) expect(`('shave' and 'haircut')`).toEvaluateTo('haircut') expect(`(false and witness)`).toEvaluateTo(false) diff --git a/src/compiler/tests/literals.test.ts b/src/compiler/tests/literals.test.ts new file mode 100644 index 0000000..666a4b5 --- /dev/null +++ b/src/compiler/tests/literals.test.ts @@ -0,0 +1,157 @@ +import { describe } from 'bun:test' +import { expect, test } from 'bun:test' + +describe('array literals', () => { + test('work with numbers', () => { + expect('[1 2 3]').toEvaluateTo([1, 2, 3]) + }) + + test('work with strings', () => { + expect("['one' 'two' 'three']").toEvaluateTo(['one', 'two', 'three']) + }) + + test('work with identifiers', () => { + expect('[one two three]').toEvaluateTo(['one', 'two', 'three']) + }) + + test('can be nested', () => { + expect('[one [two [three]]]').toEvaluateTo(['one', ['two', ['three']]]) + }) + + test('can span multiple lines', () => { + expect(`[ + 1 + 2 + 3 + ]`).toEvaluateTo([1, 2, 3]) + }) + + test('can span multiple w/o calling functions', () => { + expect(`[ + one + two + three + ]`).toEvaluateTo(['one', 'two', 'three']) + }) + + test('empty arrays', () => { + expect('[]').toEvaluateTo([]) + }) + + test('mixed types', () => { + expect("[1 'two' three true null]").toEvaluateTo([1, 'two', 'three', true, null]) + }) + + test('semicolons as separators', () => { + expect('[1; 2; 3]').toEvaluateTo([1, 2, 3]) + }) + + test('expressions in arrays', () => { + expect('[(1 + 2) (3 * 4)]').toEvaluateTo([3, 12]) + }) + + test('mixed separators - spaces and newlines', () => { + expect(`[1 2 +3 4]`).toEvaluateTo([1, 2, 3, 4]) + }) + + test('mixed separators - spaces and semicolons', () => { + expect('[1 2; 3 4]').toEvaluateTo([1, 2, 3, 4]) + }) + + test('empty lines within arrays', () => { + expect(`[1 + +2]`).toEvaluateTo([1, 2]) + }) + + test('comments within arrays', () => { + expect(`[1 # first + 2 # second + ]`).toEvaluateTo([1, 2]) + }) + + test('complex nested multiline', () => { + expect(`[ + [1 2] + [3 4] + [5 6] +]`).toEvaluateTo([ + [1, 2], + [3, 4], + [5, 6], + ]) + }) + + test('boolean and null literals', () => { + expect('[true false null]').toEvaluateTo([true, false, null]) + }) + + test('regex literals', () => { + expect('[//[0-9]+//]').toEvaluateTo([/[0-9]+/]) + }) + + test('trailing newlines', () => { + expect(`[ +1 +2 +]`).toEvaluateTo([1, 2]) + }) +}) + +describe('dict literals', () => { + test('work with numbers', () => { + expect('[a=1 b=2 c=3]').toEvaluateTo({ a: 1, b: 2, c: 3 }) + }) + + test('work with strings', () => { + expect("[a='one' b='two' c='three']").toEvaluateTo({ a: 'one', b: 'two', c: 'three' }) + }) + + test('work with identifiers', () => { + expect('[a=one b=two c=three]').toEvaluateTo({ a: 'one', b: 'two', c: 'three' }) + }) + + test('can be nested', () => { + expect('[a=one b=[two [c=three]]]').toEvaluateTo({ a: 'one', b: ['two', { c: 'three' }] }) + }) + + test('can span multiple lines', () => { + expect(`[ + a=1 + b=2 + c=3 + ]`).toEvaluateTo({ a: 1, b: 2, c: 3 }) + }) + + test('empty dict', () => { + expect('[=]').toEvaluateTo({}) + expect('[ = ]').toEvaluateTo({}) + }) + + test('mixed types', () => { + expect("[a=1 b='two' c=three d=true e=null]").toEvaluateTo({ + a: 1, + b: 'two', + c: 'three', + d: true, + e: null, + }) + }) + + test('semicolons as separators', () => { + expect('[a=1; b=2; c=3]').toEvaluateTo({ a: 1, b: 2, c: 3 }) + }) + + test('expressions in dicts', () => { + expect('[a=(1 + 2) b=(3 * 4)]').toEvaluateTo({ a: 3, b: 12 }) + }) + + test('empty lines within dicts', () => { + expect(`[a=1 + + b=2 + + c=3]`).toEvaluateTo({ a: 1, b: 2, c: 3 }) + }) +}) diff --git a/src/parser/operatorTokenizer.ts b/src/parser/operatorTokenizer.ts index 478c20d..ee1dc44 100644 --- a/src/parser/operatorTokenizer.ts +++ b/src/parser/operatorTokenizer.ts @@ -8,6 +8,7 @@ const operators: Array = [ { str: '>=', tokenName: 'Gte' }, { str: '<=', tokenName: 'Lte' }, { str: '!=', tokenName: 'Neq' }, + { str: '==', tokenName: 'EqEq' }, // // Single-char operators { str: '*', tokenName: 'Star' }, @@ -17,6 +18,7 @@ const operators: Array = [ { str: '-', tokenName: 'Minus' }, { str: '>', tokenName: 'Gt' }, { str: '<', tokenName: 'Lt' }, + { str: '%', tokenName: 'Modulo' }, ] export const operatorTokenizer = new ExternalTokenizer((input: InputStream) => { diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index 0968765..f74f5ea 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -6,7 +6,7 @@ @top Program { item* } -@external tokens operatorTokenizer from "./operatorTokenizer" { Star, Slash, Plus, Minus, And, Or, Eq, Neq, Lt, Lte, Gt, Gte } +@external tokens operatorTokenizer from "./operatorTokenizer" { Star, Slash, Plus, Minus, And, Or, Eq, EqEq, Neq, Lt, Lte, Gt, Gte, Modulo } @tokens { @precedence { Number Regex } @@ -33,6 +33,9 @@ @precedence { pipe @left, + or @left, + and @left, + comparison @left, multiplicative @left, additive @left, call @@ -52,6 +55,7 @@ consumeToTerminator { FunctionDef | Assign | BinOp | + ConditionalOp | expressionWithoutIdentifier } @@ -129,14 +133,14 @@ SingleLineThenBlock { } ConditionalOp { - expression Eq expression | - expression Neq expression | - expression Lt expression | - expression Lte expression | - expression Gt expression | - expression Gte expression | - expression And (expression | ConditionalOp) | - expression Or (expression | ConditionalOp) + expression !comparison EqEq expression | + expression !comparison Neq expression | + expression !comparison Lt expression | + expression !comparison Lte expression | + expression !comparison Gt expression | + expression !comparison Gte expression | + (expression | ConditionalOp) !and And (expression | ConditionalOp) | + (expression | ConditionalOp) !or Or (expression | ConditionalOp) } Params { @@ -148,6 +152,7 @@ Assign { } BinOp { + expression !multiplicative Modulo expression | (expression | BinOp) !multiplicative Star (expression | BinOp) | (expression | BinOp) !multiplicative Slash (expression | BinOp) | (expression | BinOp) !additive Plus (expression | BinOp) | @@ -191,6 +196,15 @@ EscapeSeq { "\\" ("$" | "n" | "t" | "r" | "\\" | "'") } +Dict { + "[=]" | + "[" newlineOrSemicolon* NamedArg (newlineOrSemicolon | NamedArg)* "]" +} + +Array { + "[" newlineOrSemicolon* (expression (newlineOrSemicolon | expression)*)? "]" +} + // We need expressionWithoutIdentifier to avoid conflicts in consumeToTerminator. // Without this, when parsing "my-var" at statement level, the parser can't decide: // - ambiguousFunctionCall → FunctionCallOrIdentifier → Identifier @@ -200,7 +214,7 @@ EscapeSeq { // to go through ambiguousFunctionCall (which is what we want semantically). // Yes, it is annoying and I gave up trying to use GLR to fix it. expressionWithoutIdentifier { - ParenExpr | Word | String | Number | Boolean | Regex | @specialize[@name=Null] + ParenExpr | Word | String | Number | Boolean | Regex | Dict | Array | @specialize[@name=Null] } block { diff --git a/src/parser/shrimp.terms.ts b/src/parser/shrimp.terms.ts index 6ea2f2a..144d69b 100644 --- a/src/parser/shrimp.terms.ts +++ b/src/parser/shrimp.terms.ts @@ -7,43 +7,47 @@ export const And = 5, Or = 6, Eq = 7, - Neq = 8, - Lt = 9, - Lte = 10, - Gt = 11, - Gte = 12, - Identifier = 13, - AssignableIdentifier = 14, - Word = 15, - IdentifierBeforeDot = 16, - Do = 17, - Program = 18, - PipeExpr = 19, - FunctionCall = 20, - DotGet = 21, - Number = 22, - ParenExpr = 23, - FunctionCallOrIdentifier = 24, - BinOp = 25, - String = 26, - StringFragment = 27, - Interpolation = 28, - EscapeSeq = 29, - Boolean = 30, - Regex = 31, - Null = 32, - ConditionalOp = 33, - FunctionDef = 34, - Params = 35, - colon = 36, - keyword = 50, - PositionalArg = 38, - Underscore = 39, - NamedArg = 40, - NamedArgPrefix = 41, - IfExpr = 43, - SingleLineThenBlock = 45, - ThenBlock = 46, - ElseIfExpr = 47, - ElseExpr = 49, - Assign = 51 + EqEq = 8, + Neq = 9, + Lt = 10, + Lte = 11, + Gt = 12, + Gte = 13, + Modulo = 14, + Identifier = 15, + AssignableIdentifier = 16, + Word = 17, + IdentifierBeforeDot = 18, + Do = 19, + Program = 20, + PipeExpr = 21, + FunctionCall = 22, + DotGet = 23, + Number = 24, + ParenExpr = 25, + FunctionCallOrIdentifier = 26, + BinOp = 27, + String = 28, + StringFragment = 29, + Interpolation = 30, + EscapeSeq = 31, + Boolean = 32, + Regex = 33, + Dict = 34, + NamedArg = 35, + NamedArgPrefix = 36, + FunctionDef = 37, + Params = 38, + colon = 39, + keyword = 54, + Underscore = 41, + Array = 42, + Null = 43, + ConditionalOp = 44, + PositionalArg = 45, + IfExpr = 47, + SingleLineThenBlock = 49, + ThenBlock = 50, + ElseIfExpr = 51, + ElseExpr = 53, + Assign = 55 diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index 40ba69f..da34621 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -4,24 +4,24 @@ import {operatorTokenizer} from "./operatorTokenizer" import {tokenizer, specializeKeyword} from "./tokenizer" import {trackScope} from "./scopeTracker" import {highlighting} from "./highlight" -const spec_Identifier = {__proto__:null,null:64, end:74, if:88, elseif:96, else:100} +const spec_Identifier = {__proto__:null,end:80, null:86, if:96, elseif:104, else:108} export const parser = LRParser.deserialize({ version: 14, - states: "/SQYQbOOO!TOpO'#CqO#aQcO'#CtO$ZOSO'#CvO%aQcO'#DsOOQa'#Ds'#DsO&gQcO'#DrO'OQRO'#CuO'^QcO'#DnO'uQbO'#D{OOQ`'#DO'#DOO'}QbO'#CsOOQ`'#Do'#DoO(oQbO'#DnO(}QbO'#EROOQ`'#DX'#DXO)lQRO'#DaOOQ`'#Dn'#DnO)qQQO'#DmOOQ`'#Dm'#DmOOQ`'#Db'#DbQYQbOOO)yObO,59]OOQa'#Dr'#DrOOQ`'#DS'#DSO*RQbO'#DUOOQ`'#EQ'#EQOOQ`'#Df'#DfO*]QbO,59[O*pQbO'#CxO*xQWO'#CyOOOO'#Du'#DuOOOO'#Dc'#DcO+^OSO,59bOOQa,59b,59bO(}QbO,59aO(}QbO,59aOOQ`'#Dd'#DdO+lQbO'#DPO+tQQO,5:gO+yQRO,59_O-`QRO'#CuO-pQRO,59_O-|QQO,59_O.RQQO,59_O.ZQbO'#DgO.fQbO,59ZO.wQRO,5:mO/OQQO,5:mO/TQbO,59{OOQ`,5:X,5:XOOQ`-E7`-E7`OOQa1G.w1G.wOOQ`,59p,59pOOQ`-E7d-E7dOOOO,59d,59dOOOO,59e,59eOOOO-E7a-E7aOOQa1G.|1G.|OOQa1G.{1G.{O/_QcO1G.{OOQ`-E7b-E7bO/yQbO1G0ROOQa1G.y1G.yO(}QbO,59iO(}QbO,59iO!YQbO'#CtO$iQbO'#CpOOQ`,5:R,5:ROOQ`-E7e-E7eO0WQbO1G0XOOQ`1G/g1G/gO0eQbO7+%mO0jQbO7+%nOOQO1G/T1G/TO0zQRO1G/TOOQ`'#DZ'#DZO1UQbO7+%sO1ZQbO7+%tOOQ`<tAN>tO(}QbO'#D]OOQ`'#Dh'#DhO2nQbOAN>zO2yQQO'#D_OOQ`AN>zAN>zO3OQbOAN>zO3TQRO,59wO3[QQO,59wOOQ`-E7f-E7fOOQ`G24fG24fO3aQbOG24fO3fQQO,59yO3kQQO1G/cOOQ`LD*QLD*QO0jQbO1G/eO1ZQbO7+$}OOQ`7+%P7+%POOQ`<i~RzOX#uXY$dYZ$}Zp#upq$dqs#ust%htu'Puw#uwx'Uxy'Zyz'tz{#u{|(_|}#u}!O(_!O!P#u!P!Q+R!Q![(|![!]3n!]!^$}!^#O#u#O#P4X#P#R#u#R#S4^#S#T#u#T#Y4w#Y#Z6V#Z#b4w#b#c:e#c#f4w#f#g;[#g#h4w#h#idS#zUkSOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uS$aP;=`<%l#u^$kUkS!_YOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU%UUkS!qQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#u^%oZkS!`YOY%hYZ#uZt%htu&buw%hwx&bx#O%h#O#P&b#P;'S%h;'S;=`&y<%lO%hY&gS!`YOY&bZ;'S&b;'S;=`&s<%lO&bY&vP;=`<%l&b^&|P;=`<%l%h~'UO!j~~'ZO!h~U'bUkS!eQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU'{UkS!sQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU(dWkSOt#uuw#ux!Q#u!Q![(|![#O#u#P;'S#u;'S;=`$^<%lO#uU)TYkSfQOt#uuw#ux!O#u!O!P)s!P!Q#u!Q![(|![#O#u#P;'S#u;'S;=`$^<%lO#uU)xWkSOt#uuw#ux!Q#u!Q![*b![#O#u#P;'S#u;'S;=`$^<%lO#uU*iWkSfQOt#uuw#ux!Q#u!Q![*b![#O#u#P;'S#u;'S;=`$^<%lO#uU+WWkSOt#uuw#ux!P#u!P!Q+p!Q#O#u#P;'S#u;'S;=`$^<%lO#uU+u^kSOY,qYZ#uZt,qtu-tuw,qwx-tx!P,q!P!Q#u!Q!},q!}#O2g#O#P0S#P;'S,q;'S;=`3h<%lO,qU,x^kSoQOY,qYZ#uZt,qtu-tuw,qwx-tx!P,q!P!Q0i!Q!},q!}#O2g#O#P0S#P;'S,q;'S;=`3h<%lO,qQ-yXoQOY-tZ!P-t!P!Q.f!Q!}-t!}#O/T#O#P0S#P;'S-t;'S;=`0c<%lO-tQ.iP!P!Q.lQ.qUoQ#Z#[.l#]#^.l#a#b.l#g#h.l#i#j.l#m#n.lQ/WVOY/TZ#O/T#O#P/m#P#Q-t#Q;'S/T;'S;=`/|<%lO/TQ/pSOY/TZ;'S/T;'S;=`/|<%lO/TQ0PP;=`<%l/TQ0VSOY-tZ;'S-t;'S;=`0c<%lO-tQ0fP;=`<%l-tU0nWkSOt#uuw#ux!P#u!P!Q1W!Q#O#u#P;'S#u;'S;=`$^<%lO#uU1_bkSoQOt#uuw#ux#O#u#P#Z#u#Z#[1W#[#]#u#]#^1W#^#a#u#a#b1W#b#g#u#g#h1W#h#i#u#i#j1W#j#m#u#m#n1W#n;'S#u;'S;=`$^<%lO#uU2l[kSOY2gYZ#uZt2gtu/Tuw2gwx/Tx#O2g#O#P/m#P#Q,q#Q;'S2g;'S;=`3b<%lO2gU3eP;=`<%l2gU3kP;=`<%l,qU3uUkStQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#u~4^O!k~U4eUkSwQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU4|YkSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#o4w#o;'S#u;'S;=`$^<%lO#uU5sUyQkSOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU6[ZkSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#U6}#U#o4w#o;'S#u;'S;=`$^<%lO#uU7S[kSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#`4w#`#a7x#a#o4w#o;'S#u;'S;=`$^<%lO#uU7}[kSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#g4w#g#h8s#h#o4w#o;'S#u;'S;=`$^<%lO#uU8x[kSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#X4w#X#Y9n#Y#o4w#o;'S#u;'S;=`$^<%lO#uU9uYnQkSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#o4w#o;'S#u;'S;=`$^<%lO#u^:lY!lWkSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#o4w#o;'S#u;'S;=`$^<%lO#u^;cY!nWkSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#o4w#o;'S#u;'S;=`$^<%lO#u^QUzQkSOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#u~>iO!w~", - tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, new LocalTokenGroup("[~RP!O!PU~ZO!d~~", 11)], - topRules: {"Program":[0,18]}, - specialized: [{term: 13, get: (value: any, stack: any) => (specializeKeyword(value, stack) << 1), external: specializeKeyword},{term: 13, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}], - tokenPrec: 860 + repeatNodeCount: 10, + tokenData: "AO~R|OX#{XY$jYZ%TZp#{pq$jqs#{st%ntu'Vuw#{wx'[xy'ayz'zz{#{{|(e|}#{}!O(e!O!P#{!P!Q+X!Q![)S![!]3t!]!^%T!^!}#{!}#O4_#O#P6T#P#Q6Y#Q#R#{#R#S6s#S#T#{#T#Y7^#Y#Z8l#Z#b7^#b#ch#i#o7^#o#p#{#p#q@`#q;'S#{;'S;=`$d<%l~#{~O#{~~@yS$QUmSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{S$gP;=`<%l#{^$qUmS!fYOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U%[UmS!xQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{^%uZmS!gYOY%nYZ#{Zt%ntu&huw%nwx&hx#O%n#O#P&h#P;'S%n;'S;=`'P<%lO%nY&mS!gYOY&hZ;'S&h;'S;=`&y<%lO&hY&|P;=`<%l&h^'SP;=`<%l%n~'[O!q~~'aO!o~U'hUmS!lQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U(RUmS!}QOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U(jWmSOt#{uw#{x!Q#{!Q![)S![#O#{#P;'S#{;'S;=`$d<%lO#{U)ZYmShQOt#{uw#{x!O#{!O!P)y!P!Q#{!Q![)S![#O#{#P;'S#{;'S;=`$d<%lO#{U*OWmSOt#{uw#{x!Q#{!Q![*h![#O#{#P;'S#{;'S;=`$d<%lO#{U*oWmShQOt#{uw#{x!Q#{!Q![*h![#O#{#P;'S#{;'S;=`$d<%lO#{U+^WmSOt#{uw#{x!P#{!P!Q+v!Q#O#{#P;'S#{;'S;=`$d<%lO#{U+{^mSOY,wYZ#{Zt,wtu-zuw,wwx-zx!P,w!P!Q#{!Q!},w!}#O2m#O#P0Y#P;'S,w;'S;=`3n<%lO,wU-O^mSqQOY,wYZ#{Zt,wtu-zuw,wwx-zx!P,w!P!Q0o!Q!},w!}#O2m#O#P0Y#P;'S,w;'S;=`3n<%lO,wQ.PXqQOY-zZ!P-z!P!Q.l!Q!}-z!}#O/Z#O#P0Y#P;'S-z;'S;=`0i<%lO-zQ.oP!P!Q.rQ.wUqQ#Z#[.r#]#^.r#a#b.r#g#h.r#i#j.r#m#n.rQ/^VOY/ZZ#O/Z#O#P/s#P#Q-z#Q;'S/Z;'S;=`0S<%lO/ZQ/vSOY/ZZ;'S/Z;'S;=`0S<%lO/ZQ0VP;=`<%l/ZQ0]SOY-zZ;'S-z;'S;=`0i<%lO-zQ0lP;=`<%l-zU0tWmSOt#{uw#{x!P#{!P!Q1^!Q#O#{#P;'S#{;'S;=`$d<%lO#{U1ebmSqQOt#{uw#{x#O#{#P#Z#{#Z#[1^#[#]#{#]#^1^#^#a#{#a#b1^#b#g#{#g#h1^#h#i#{#i#j1^#j#m#{#m#n1^#n;'S#{;'S;=`$d<%lO#{U2r[mSOY2mYZ#{Zt2mtu/Zuw2mwx/Zx#O2m#O#P/s#P#Q,w#Q;'S2m;'S;=`3h<%lO2mU3kP;=`<%l2mU3qP;=`<%l,wU3{UmSwQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U4fW!wQmSOt#{uw#{x!_#{!_!`5O!`#O#{#P;'S#{;'S;=`$d<%lO#{U5TVmSOt#{uw#{x#O#{#P#Q5j#Q;'S#{;'S;=`$d<%lO#{U5qU!vQmSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~6YO!r~U6aU!|QmSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U6zUmSyQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U7cYmSOt#{uw#{x!_#{!_!`8R!`#O#{#P#T#{#T#o7^#o;'S#{;'S;=`$d<%lO#{U8YUtQmSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U8qZmSOt#{uw#{x!_#{!_!`8R!`#O#{#P#T#{#T#U9d#U#o7^#o;'S#{;'S;=`$d<%lO#{U9i[mSOt#{uw#{x!_#{!_!`8R!`#O#{#P#T#{#T#`7^#`#a:_#a#o7^#o;'S#{;'S;=`$d<%lO#{U:d[mSOt#{uw#{x!_#{!_!`8R!`#O#{#P#T#{#T#g7^#g#h;Y#h#o7^#o;'S#{;'S;=`$d<%lO#{U;_[mSOt#{uw#{x!_#{!_!`8R!`#O#{#P#T#{#T#X7^#X#Yo[!tWmSOt#{uw#{x!_#{!_!`8R!`#O#{#P#T#{#T#f7^#f#g?e#g#o7^#o;'S#{;'S;=`$d<%lO#{U?j[mSOt#{uw#{x!_#{!_!`8R!`#O#{#P#T#{#T#i7^#i#j;Y#j#o7^#o;'S#{;'S;=`$d<%lO#{U@gU!OQmSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~AOO#R~", + tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, new LocalTokenGroup("[~RP!O!PU~ZO!k~~", 11)], + topRules: {"Program":[0,20]}, + specialized: [{term: 15, get: (value: any, stack: any) => (specializeKeyword(value, stack) << 1), external: specializeKeyword},{term: 15, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}], + tokenPrec: 1132 }) diff --git a/src/parser/tests/basics.test.ts b/src/parser/tests/basics.test.ts index f92e034..da6d4bb 100644 --- a/src/parser/tests/basics.test.ts +++ b/src/parser/tests/basics.test.ts @@ -36,6 +36,19 @@ describe('Identifier', () => { FunctionCallOrIdentifier Identifier 𝜋`) }) + + test('parses identifiers with queries', () => { + expect('even? 20').toMatchTree(` + FunctionCall + Identifier even? + PositionalArg + Number 20`) + + expect('even?').toMatchTree(` + FunctionCallOrIdentifier + Identifier even?`) + }) + }) describe('Unicode Symbol Support', () => { @@ -395,6 +408,15 @@ describe('BinOp', () => { `) }) + test('modulo tests', () => { + expect('4 % 3').toMatchTree(` + BinOp + Number 4 + Modulo % + Number 3 + `) + }) + test('mixed operations with precedence', () => { expect('2 + 3 * 4 - 5 / 1').toMatchTree(` BinOp @@ -570,4 +592,90 @@ describe('Comments', () => { Slash / Identifier prop`) }) +}) + +describe('Conditional ops', () => { + test('or can be chained', () => { + expect(` + is-positive = do x: + if x == 3 or x == 4 or x == 5: + true + end + end + `).toMatchTree(` +Assign + AssignableIdentifier is-positive + Eq = + FunctionDef + Do do + Params + Identifier x + colon : + IfExpr + keyword if + ConditionalOp + ConditionalOp + ConditionalOp + Identifier x + EqEq == + Number 3 + Or or + ConditionalOp + Identifier x + EqEq == + Number 4 + Or or + ConditionalOp + Identifier x + EqEq == + Number 5 + colon : + ThenBlock + Boolean true + keyword end + keyword end + `) + }) + + test('and can be chained', () => { + expect(` + is-positive = do x: + if x == 3 and x == 4 and x == 5: + true + end + end + `).toMatchTree(` +Assign + AssignableIdentifier is-positive + Eq = + FunctionDef + Do do + Params + Identifier x + colon : + IfExpr + keyword if + ConditionalOp + ConditionalOp + ConditionalOp + Identifier x + EqEq == + Number 3 + And and + ConditionalOp + Identifier x + EqEq == + Number 4 + And and + ConditionalOp + Identifier x + EqEq == + Number 5 + colon : + ThenBlock + Boolean true + keyword end + keyword end + `) + }) }) \ No newline at end of file diff --git a/src/parser/tests/control-flow.test.ts b/src/parser/tests/control-flow.test.ts index 11c81d0..70efddb 100644 --- a/src/parser/tests/control-flow.test.ts +++ b/src/parser/tests/control-flow.test.ts @@ -4,12 +4,12 @@ import '../shrimp.grammar' // Importing this so changes cause it to retest! describe('if/elseif/else', () => { test('parses single line if', () => { - expect(`if y = 1: 'cool' end`).toMatchTree(` + expect(`if y == 1: 'cool' end`).toMatchTree(` IfExpr keyword if ConditionalOp Identifier y - Eq = + EqEq == Number 1 colon : SingleLineThenBlock diff --git a/src/parser/tests/literals.test.ts b/src/parser/tests/literals.test.ts new file mode 100644 index 0000000..693da17 --- /dev/null +++ b/src/parser/tests/literals.test.ts @@ -0,0 +1,492 @@ +import { expect, describe, test } from 'bun:test' + +import '../shrimp.grammar' // Importing this so changes cause it to retest! + +describe('array literals', () => { + test('work with numbers', () => { + expect('[1 2 3]').toMatchTree(` + Array + Number 1 + Number 2 + Number 3 + `) + }) + + test('work with strings', () => { + expect("['one' 'two' 'three']").toMatchTree(` + Array + String + StringFragment one + String + StringFragment two + String + StringFragment three + `) + }) + + test('work with identifiers', () => { + expect('[one two three]').toMatchTree(` + Array + Identifier one + Identifier two + Identifier three + `) + }) + + test('can be nested', () => { + expect('[one [two [three]]]').toMatchTree(` + Array + Identifier one + Array + Identifier two + Array + Identifier three + `) + }) + + test('can span multiple lines', () => { + expect(`[ + 1 + 2 + 3 + ]`).toMatchTree(` + Array + Number 1 + Number 2 + Number 3 + `) + }) + + test('can span multiple w/o calling functions', () => { + expect(`[ + one + two + three + ]`).toMatchTree(` + Array + Identifier one + Identifier two + Identifier three + `) + }) + + test('empty arrays', () => { + expect('[]').toMatchTree(` + Array [] + `) + }) + + test('mixed types', () => { + expect("[1 'two' three true null]").toMatchTree(` + Array + Number 1 + String + StringFragment two + Identifier three + Boolean true + Null null + `) + }) + + test('semicolons as separators', () => { + expect('[1; 2; 3]').toMatchTree(` + Array + Number 1 + Number 2 + Number 3 + `) + }) + + test('expressions in arrays', () => { + expect('[(1 + 2) (3 * 4)]').toMatchTree(` + Array + ParenExpr + BinOp + Number 1 + Plus + + Number 2 + ParenExpr + BinOp + Number 3 + Star * + Number 4 + `) + }) + + test('mixed separators - spaces and newlines', () => { + expect(`[1 2 +3 4]`).toMatchTree(` + Array + Number 1 + Number 2 + Number 3 + Number 4 + `) + }) + + test('mixed separators - spaces and semicolons', () => { + expect('[1 2; 3 4]').toMatchTree(` + Array + Number 1 + Number 2 + Number 3 + Number 4 + `) + }) + + test('empty lines within arrays', () => { + expect(`[1 + +2]`).toMatchTree(` + Array + Number 1 + Number 2 + `) + }) + + test('comments within arrays', () => { + expect(`[ # something... + 1 # first + 2 # second + ]`).toMatchTree(` + Array + Number 1 + Number 2 + `) + }) + + test('complex nested multiline', () => { + expect(`[ + [1 2] + [3 4] + [5 6] +]`).toMatchTree(` + Array + Array + Number 1 + Number 2 + Array + Number 3 + Number 4 + Array + Number 5 + Number 6 + `) + }) + + test('boolean and null literals', () => { + expect('[true false null]').toMatchTree(` + Array + Boolean true + Boolean false + Null null + `) + }) + + test('regex literals', () => { + expect('[//[0-9]+//]').toMatchTree(` + Array + Regex //[0-9]+// + `) + }) + + test('trailing newlines', () => { + expect(`[ +1 +2 +]`).toMatchTree(` + Array + Number 1 + Number 2 + `) + }) +}) + +describe('dict literals', () => { + test('work with numbers', () => { + expect('[a=1 b=2 c=3]').toMatchTree(` + Dict + NamedArg + NamedArgPrefix a= + Number 1 + NamedArg + NamedArgPrefix b= + Number 2 + NamedArg + NamedArgPrefix c= + Number 3 + `) + }) + + test('work with strings', () => { + expect("[a='one' b='two' c='three']").toMatchTree(` + Dict + NamedArg + NamedArgPrefix a= + String + StringFragment one + NamedArg + NamedArgPrefix b= + String + StringFragment two + NamedArg + NamedArgPrefix c= + String + StringFragment three + `) + }) + + test('work with identifiers', () => { + expect('[a=one b=two c=three]').toMatchTree(` + Dict + NamedArg + NamedArgPrefix a= + Identifier one + NamedArg + NamedArgPrefix b= + Identifier two + NamedArg + NamedArgPrefix c= + Identifier three + `) + }) + + test('can be nested', () => { + expect('[a=one b=[two [c=three]]]').toMatchTree(` + Dict + NamedArg + NamedArgPrefix a= + Identifier one + NamedArg + NamedArgPrefix b= + Array + Identifier two + Dict + NamedArg + NamedArgPrefix c= + Identifier three + `) + }) + + test('can span multiple lines', () => { + expect(`[ + a=1 + b=2 + c=3 + ]`).toMatchTree(` + Dict + NamedArg + NamedArgPrefix a= + Number 1 + NamedArg + NamedArgPrefix b= + Number 2 + NamedArg + NamedArgPrefix c= + Number 3 + `) + }) + test('empty dict', () => { + expect('[=]').toMatchTree(` + Dict [=] + `) + + expect('[ = ]').toMatchTree(` + Array + Word = + `) + }) + + test('mixed types', () => { + expect("[a=1 b='two' c=three d=true e=null]").toMatchTree(` + Dict + NamedArg + NamedArgPrefix a= + Number 1 + NamedArg + NamedArgPrefix b= + String + StringFragment two + NamedArg + NamedArgPrefix c= + Identifier three + NamedArg + NamedArgPrefix d= + Boolean true + NamedArg + NamedArgPrefix e= + Null null + `) + }) + + test('semicolons as separators', () => { + expect('[a=1; b=2; c=3]').toMatchTree(` + Dict + NamedArg + NamedArgPrefix a= + Number 1 + NamedArg + NamedArgPrefix b= + Number 2 + NamedArg + NamedArgPrefix c= + Number 3 + `) + }) + + test('expressions in dicts', () => { + expect('[a=(1 + 2) b=(3 * 4)]').toMatchTree(` + Dict + NamedArg + NamedArgPrefix a= + ParenExpr + BinOp + Number 1 + Plus + + Number 2 + NamedArg + NamedArgPrefix b= + ParenExpr + BinOp + Number 3 + Star * + Number 4 + `) + }) + + test('mixed separators - spaces and newlines', () => { + expect(`[a=1 b=2 +c=3]`).toMatchTree(` + Dict + NamedArg + NamedArgPrefix a= + Number 1 + NamedArg + NamedArgPrefix b= + Number 2 + NamedArg + NamedArgPrefix c= + Number 3 + `) + }) + + test('empty lines within dicts', () => { + expect(`[a=1 + +b=2 + +c=3]`).toMatchTree(` + Dict + NamedArg + NamedArgPrefix a= + Number 1 + NamedArg + NamedArgPrefix b= + Number 2 + NamedArg + NamedArgPrefix c= + Number 3 + `) + }) + + test('comments within dicts', () => { + expect(`[ # something... + a=1 # first + b=2 # second + + c=3 + ]`).toMatchTree(` + Dict + NamedArg + NamedArgPrefix a= + Number 1 + NamedArg + NamedArgPrefix b= + Number 2 + NamedArg + NamedArgPrefix c= + Number 3 + `) + }) + + test('complex nested multiline', () => { + expect(`[ + a=[a=1 b=2] + b=[b=3 c=4] + c=[c=5 d=6] +]`).toMatchTree(` + Dict + NamedArg + NamedArgPrefix a= + Dict + NamedArg + NamedArgPrefix a= + Number 1 + NamedArg + NamedArgPrefix b= + Number 2 + NamedArg + NamedArgPrefix b= + Dict + NamedArg + NamedArgPrefix b= + Number 3 + NamedArg + NamedArgPrefix c= + Number 4 + NamedArg + NamedArgPrefix c= + Dict + NamedArg + NamedArgPrefix c= + Number 5 + NamedArg + NamedArgPrefix d= + Number 6 + `) + }) + + test('boolean and null literals', () => { + expect('[a=true b=false c=null]').toMatchTree(` + Dict + NamedArg + NamedArgPrefix a= + Boolean true + NamedArg + NamedArgPrefix b= + Boolean false + NamedArg + NamedArgPrefix c= + Null null + `) + }) + + test('regex literals', () => { + expect('[pattern=//[0-9]+//]').toMatchTree(` + Dict + NamedArg + NamedArgPrefix pattern= + Regex //[0-9]+// + `) + }) + + test('trailing newlines', () => { + expect(`[ +a=1 +b=2 +c=3 + +]`).toMatchTree(` + Dict + NamedArg + NamedArgPrefix a= + Number 1 + NamedArg + NamedArgPrefix b= + Number 2 + NamedArg + NamedArgPrefix c= + Number 3 + `) + }) +}) diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts index 0db5545..e4fc895 100644 --- a/src/parser/tokenizer.ts +++ b/src/parser/tokenizer.ts @@ -6,6 +6,13 @@ export function specializeKeyword(ident: string) { return ident === 'do' ? Do : -1 } +// tell the dotGet searcher about builtin globals +export const globals: string[] = [] +export const setGlobals = (newGlobals: string[]) => { + globals.length = 0 + globals.push(...newGlobals) +} + // The only chars that can't be words are whitespace, apostrophes, closing parens, and EOF. export const tokenizer = new ExternalTokenizer( @@ -112,7 +119,7 @@ const consumeWordToken = ( } // Track identifier validity: must be lowercase, digit, dash, or emoji/unicode - if (!isLowercaseLetter(ch) && !isDigit(ch) && ch !== 45 /* - */ && !isEmojiOrUnicode(ch)) { + if (!isLowercaseLetter(ch) && !isDigit(ch) && ch !== 45 /* - */ && ch !== 63 /* ? */ && !isEmojiOrUnicode(ch)) { if (!canBeWord) break isValidIdentifier = false } @@ -152,7 +159,7 @@ const checkForDotGet = (input: InputStream, stack: Stack, pos: number): number | // If identifier is in scope, this is property access (e.g., obj.prop) // If not in scope, it should be consumed as a Word (e.g., file.txt) - return context?.scope.has(identifierText) ? IdentifierBeforeDot : null + return context?.scope.has(identifierText) || globals.includes(identifierText) ? IdentifierBeforeDot : null } // Decide between AssignableIdentifier and Identifier using grammar state + peek-ahead @@ -195,7 +202,13 @@ const isWhiteSpace = (ch: number): boolean => { } const isWordChar = (ch: number): boolean => { - return !isWhiteSpace(ch) && ch !== 10 /* \n */ && ch !== 41 /* ) */ && ch !== -1 /* EOF */ + return ( + !isWhiteSpace(ch) && + ch !== 10 /* \n */ && + ch !== 41 /* ) */ && + ch !== 93 /* ] */ && + ch !== -1 /* EOF */ + ) } const isLowercaseLetter = (ch: number): boolean => { diff --git a/src/prelude/dict.ts b/src/prelude/dict.ts new file mode 100644 index 0000000..9642a15 --- /dev/null +++ b/src/prelude/dict.ts @@ -0,0 +1,35 @@ +import { type Value, toString, toValue } from 'reefvm' + +export const dict = { + keys: (dict: Record) => Object.keys(dict), + values: (dict: Record) => Object.values(dict), + entries: (dict: Record) => Object.entries(dict).map(([k, v]) => ({ key: k, value: v })), + 'has?': (dict: Record, key: string) => key in dict, + get: (dict: Record, key: string, defaultValue: any = null) => dict[key] ?? defaultValue, + set: (dict: Value, key: Value, value: Value) => { + const map = dict.value as Map + map.set(toString(key), value) + return dict + }, + merge: (...dicts: Record[]) => Object.assign({}, ...dicts), + 'empty?': (dict: Record) => Object.keys(dict).length === 0, + map: async (dict: Record, cb: Function) => { + const result: Record = {} + for (const [key, value] of Object.entries(dict)) { + result[key] = await cb(value, key) + } + return result + }, + filter: async (dict: Record, cb: Function) => { + const result: Record = {} + for (const [key, value] of Object.entries(dict)) { + if (await cb(value, key)) result[key] = value + } + return result + }, + 'from-entries': (entries: [string, any][]) => Object.fromEntries(entries), +} + + // raw functions deal directly in Value types, meaning we can modify collection + // careful - the MUST return a Value! + ; (dict.set as any).raw = true diff --git a/src/prelude/index.ts b/src/prelude/index.ts new file mode 100644 index 0000000..facf4b8 --- /dev/null +++ b/src/prelude/index.ts @@ -0,0 +1,149 @@ +// The prelude creates all the builtin Shrimp functions. + +import { + type Value, toValue, + extractParamInfo, isWrapped, getOriginalFunction, +} from 'reefvm' + +import { dict } from './dict' +import { load } from './load' +import { list } from './list' +import { math } from './math' +import { str } from './str' + +export const globals = { + dict, + load, + list, + math, + str, + + // hello + echo: (...args: any[]) => { + console.log(...args.map(a => { + const v = toValue(a) + return ['array', 'dict'].includes(v.type) ? formatValue(v, true) : v.value + })) + return toValue(null) + }, + + // info + type: (v: any) => toValue(v).type, + inspect: (v: any) => formatValue(toValue(v)), + describe: (v: any) => { + const val = toValue(v) + return `#<${val.type}: ${formatValue(val)}>` + }, + length: (v: any) => { + const value = toValue(v) + switch (value.type) { + case 'string': case 'array': return value.value.length + case 'dict': return value.value.size + default: return 0 + } + }, + + // type predicates + 'string?': (v: any) => toValue(v).type === 'string', + 'number?': (v: any) => toValue(v).type === 'number', + 'boolean?': (v: any) => toValue(v).type === 'boolean', + 'array?': (v: any) => toValue(v).type === 'array', + 'dict?': (v: any) => toValue(v).type === 'dict', + 'function?': (v: any) => { + const t = toValue(v).type + return t === 'function' || t === 'native' + }, + 'null?': (v: any) => toValue(v).type === 'null', + 'some?': (v: any) => toValue(v).type !== 'null', + + // boolean/logic + not: (v: any) => !v, + + // utilities + inc: (n: number) => n + 1, + dec: (n: number) => n - 1, + identity: (v: any) => v, + + // collections + at: (collection: any, index: number | string) => collection[index], + range: (start: number, end: number | null) => { + if (end === null) { + end = start + start = 0 + } + const result: number[] = [] + for (let i = start; i <= end; i++) { + result.push(i) + } + return result + }, + 'empty?': (v: any) => { + const value = toValue(v) + switch (value.type) { + case 'string': case 'array': + return value.value.length === 0 + case 'dict': + return value.value.size === 0 + default: + return false + } + }, + + // enumerables + each: async (list: any[], cb: Function) => { + for (const value of list) await cb(value) + return list + }, + +} + +export const colors = { + reset: '\x1b[0m', + bright: '\x1b[1m', + dim: '\x1b[2m', + cyan: '\x1b[36m', + yellow: '\x1b[33m', + green: '\x1b[32m', + red: '\x1b[31m', + blue: '\x1b[34m', + magenta: '\x1b[35m', + pink: '\x1b[38;2;255;105;180m' +} + +export function formatValue(value: Value, inner = false): string { + switch (value.type) { + case 'string': + return `${colors.green}'${value.value.replaceAll("'", "\\'")}${colors.green}'${colors.reset}` + case 'number': + return `${colors.cyan}${value.value}${colors.reset}` + case 'boolean': + return `${colors.yellow}${value.value}${colors.reset}` + case 'null': + return `${colors.dim}null${colors.reset}` + case 'array': { + const items = value.value.map(x => formatValue(x, true)).join(' ') + return `${colors.blue}[${colors.reset}${items}${colors.blue}]${colors.reset}` + } + case 'dict': { + const entries = Array.from(value.value.entries()) + .map(([k, v]) => `${k}${colors.blue}=${colors.reset}${formatValue(v, true)}`) + .join(' ') + if (entries.length === 0) + return `${colors.blue}[=]${colors.reset}` + return `${colors.blue}[${colors.reset}${entries}${colors.blue}]${colors.reset}` + } + case 'function': { + const params = value.params.length ? '(' + value.params.join(' ') + ')' : '' + return `${colors.dim}${colors.reset}` + } + case 'native': + const fn = isWrapped(value.fn) ? getOriginalFunction(value.fn) : value.fn + const info = extractParamInfo(fn) + const params = info.params.length ? '(' + info.params.join(' ') + ')' : '' + return `${colors.dim}${colors.reset}` + case 'regex': + return `${colors.magenta}${value.value}${colors.reset}` + default: + return String(value) + } +} \ No newline at end of file diff --git a/src/prelude/list.ts b/src/prelude/list.ts new file mode 100644 index 0000000..eb013ef --- /dev/null +++ b/src/prelude/list.ts @@ -0,0 +1,89 @@ +export const list = { + slice: (list: any[], start: number, end?: number) => list.slice(start, end), + map: async (list: any[], cb: Function) => { + let acc: any[] = [] + for (const value of list) acc.push(await cb(value)) + return acc + }, + filter: async (list: any[], cb: Function) => { + let acc: any[] = [] + for (const value of list) { + if (await cb(value)) acc.push(value) + } + return acc + }, + reduce: async (list: any[], cb: Function, initial: any) => { + let acc = initial + for (const value of list) acc = await cb(acc, value) + return acc + }, + find: async (list: any[], cb: Function) => { + for (const value of list) { + if (await cb(value)) return value + } + return null + }, + + // predicates + 'empty?': (list: any[]) => list.length === 0, + 'contains?': (list: any[], item: any) => list.includes(item), + 'any?': async (list: any[], cb: Function) => { + for (const value of list) { + if (await cb(value)) return true + } + return false + }, + 'all?': async (list: any[], cb: Function) => { + for (const value of list) { + if (!await cb(value)) return false + } + return true + }, + + // sequence operations + reverse: (list: any[]) => list.slice().reverse(), + sort: (list: any[], cb?: (a: any, b: any) => number) => list.slice().sort(cb), + concat: (...lists: any[][]) => lists.flat(1), + flatten: (list: any[], depth: number = 1) => list.flat(depth), + unique: (list: any[]) => Array.from(new Set(list)), + zip: (list1: any[], list2: any[]) => list1.map((item, i) => [item, list2[i]]), + + // access + first: (list: any[]) => list[0] ?? null, + last: (list: any[]) => list[list.length - 1] ?? null, + rest: (list: any[]) => list.slice(1), + take: (list: any[], n: number) => list.slice(0, n), + drop: (list: any[], n: number) => list.slice(n), + append: (list: any[], item: any) => [...list, item], + prepend: (list: any[], item: any) => [item, ...list], + 'index-of': (list: any[], item: any) => list.indexOf(item), + + // utilities + sum: (list: any[]) => list.reduce((acc, x) => acc + x, 0), + count: async (list: any[], cb: Function) => { + let count = 0 + for (const value of list) { + if (await cb(value)) count++ + } + return count + }, + partition: async (list: any[], cb: Function) => { + const truthy: any[] = [] + const falsy: any[] = [] + for (const value of list) { + if (await cb(value)) truthy.push(value) + else falsy.push(value) + } + return [truthy, falsy] + }, + compact: (list: any[]) => list.filter(x => x != null), + 'group-by': async (list: any[], cb: Function) => { + const groups: Record = {} + for (const value of list) { + const key = String(await cb(value)) + if (!groups[key]) groups[key] = [] + groups[key].push(value) + } + return groups + }, +} \ No newline at end of file diff --git a/src/prelude/load.ts b/src/prelude/load.ts new file mode 100644 index 0000000..3f317c1 --- /dev/null +++ b/src/prelude/load.ts @@ -0,0 +1,29 @@ +import { resolve } from 'path' +import { readFileSync } from 'fs' +import { Compiler } from '#compiler/compiler' +import { type Value, VM, Scope } from 'reefvm' + +export const load = async function (this: VM, path: string): Promise> { + const scope = this.scope + const pc = this.pc + + const fullPath = resolve(path) + '.sh' + const code = readFileSync(fullPath, 'utf-8') + + this.pc = this.instructions.length + this.scope = new Scope(scope) + const compiled = new Compiler(code) + this.appendBytecode(compiled.bytecode) + + await this.continue() + + const module: Record = {} + for (const [name, value] of this.scope.locals.entries()) + module[name] = value + + this.scope = scope + this.pc = pc + this.stopped = false + + return module +} \ No newline at end of file diff --git a/src/prelude/math.ts b/src/prelude/math.ts new file mode 100644 index 0000000..21f2f57 --- /dev/null +++ b/src/prelude/math.ts @@ -0,0 +1,21 @@ +export const math = { + abs: (n: number) => Math.abs(n), + floor: (n: number) => Math.floor(n), + ceil: (n: number) => Math.ceil(n), + round: (n: number) => Math.round(n), + min: (...nums: number[]) => Math.min(...nums), + max: (...nums: number[]) => Math.max(...nums), + pow: (base: number, exp: number) => Math.pow(base, exp), + sqrt: (n: number) => Math.sqrt(n), + random: () => Math.random(), + clamp: (n: number, min: number, max: number) => Math.min(Math.max(n, min), max), + sign: (n: number) => Math.sign(n), + trunc: (n: number) => Math.trunc(n), + + // predicates + 'even?': (n: number) => n % 2 === 0, + 'odd?': (n: number) => n % 2 !== 0, + 'positive?': (n: number) => n > 0, + 'negative?': (n: number) => n < 0, + 'zero?': (n: number) => n === 0, +} \ No newline at end of file diff --git a/src/prelude/str.ts b/src/prelude/str.ts new file mode 100644 index 0000000..fa0d657 --- /dev/null +++ b/src/prelude/str.ts @@ -0,0 +1,33 @@ +// strings +export const str = { + join: (arr: string[], sep: string = ',') => arr.join(sep), + split: (str: string, sep: string = ',') => str.split(sep), + 'to-upper': (str: string) => str.toUpperCase(), + 'to-lower': (str: string) => str.toLowerCase(), + trim: (str: string) => str.trim(), + + // predicates + 'starts-with?': (str: string, prefix: string) => str.startsWith(prefix), + 'ends-with?': (str: string, suffix: string) => str.endsWith(suffix), + 'contains?': (str: string, substr: string) => str.includes(substr), + 'empty?': (str: string) => str.length === 0, + + // inspection + 'index-of': (str: string, search: string) => str.indexOf(search), + 'last-index-of': (str: string, search: string) => str.lastIndexOf(search), + + // transformations + replace: (str: string, search: string, replacement: string) => str.replace(search, replacement), + 'replace-all': (str: string, search: string, replacement: string) => str.replaceAll(search, replacement), + slice: (str: string, start: number, end?: number | null) => str.slice(start, end ?? undefined), + substring: (str: string, start: number, end?: number | null) => str.substring(start, end ?? undefined), + repeat: (str: string, count: number) => str.repeat(count), + 'pad-start': (str: string, length: number, pad: string = ' ') => str.padStart(length, pad), + 'pad-end': (str: string, length: number, pad: string = ' ') => str.padEnd(length, pad), + lines: (str: string) => str.split('\n'), + chars: (str: string) => str.split(''), + + // regex + match: (str: string, regex: RegExp) => str.match(regex), + 'test?': (str: string, regex: RegExp) => regex.test(str), +} \ No newline at end of file diff --git a/src/prelude/tests/load.ts b/src/prelude/tests/load.ts new file mode 100644 index 0000000..7ce8172 --- /dev/null +++ b/src/prelude/tests/load.ts @@ -0,0 +1,42 @@ +import { expect, describe, test } from 'bun:test' +import { globals } from '#prelude' + +describe('use', () => { + test(`imports all a file's functions`, async () => { + expect(` + math = load ./src/prelude/tests/math + math.double 4 + `).toEvaluateTo(8, globals) + + expect(` + math = load ./src/prelude/tests/math + math.double (math.double 4) + `).toEvaluateTo(16, globals) + + expect(` + math = load ./src/prelude/tests/math + dbl = math.double + dbl (dbl 2) + `).toEvaluateTo(8, globals) + + expect(` + math = load ./src/prelude/tests/math + math.pi + `).toEvaluateTo(3.14, globals) + + expect(` + math = load ./src/prelude/tests/math + math | at 🥧 + `).toEvaluateTo(3.14159265359, globals) + + expect(` + math = load ./src/prelude/tests/math + math.🥧 + `).toEvaluateTo(3.14159265359, globals) + + expect(` + math = load ./src/prelude/tests/math + math.add1 5 + `).toEvaluateTo(6, globals) + }) +}) diff --git a/src/prelude/tests/math.sh b/src/prelude/tests/math.sh new file mode 100644 index 0000000..14dc504 --- /dev/null +++ b/src/prelude/tests/math.sh @@ -0,0 +1,4 @@ +🥧 = 3.14159265359 +pi = 3.14 +add1 = do x: x + 1 end +double = do x: x * 2 end \ No newline at end of file diff --git a/src/prelude/tests/prelude.test.ts b/src/prelude/tests/prelude.test.ts new file mode 100644 index 0000000..ef7d8d6 --- /dev/null +++ b/src/prelude/tests/prelude.test.ts @@ -0,0 +1,598 @@ +import { expect, describe, test } from 'bun:test' +import { globals } from '#prelude' + +describe('string operations', () => { + test('to-upper converts to uppercase', async () => { + await expect(`str.to-upper 'hello'`).toEvaluateTo('HELLO', globals) + await expect(`str.to-upper 'Hello World!'`).toEvaluateTo('HELLO WORLD!', globals) + }) + + test('to-lower converts to lowercase', async () => { + await expect(`str.to-lower 'HELLO'`).toEvaluateTo('hello', globals) + await expect(`str.to-lower 'Hello World!'`).toEvaluateTo('hello world!', globals) + }) + + test('trim removes whitespace', async () => { + await expect(`str.trim ' hello '`).toEvaluateTo('hello', globals) + await expect(`str.trim '\\n\\thello\\t\\n'`).toEvaluateTo('hello', globals) + }) + + test('split divides string by separator', async () => { + await expect(`str.split 'a,b,c' ','`).toEvaluateTo(['a', 'b', 'c'], globals) + await expect(`str.split 'hello' ''`).toEvaluateTo(['h', 'e', 'l', 'l', 'o'], globals) + }) + + test('split with comma separator', async () => { + await expect(`str.split 'a,b,c' ','`).toEvaluateTo(['a', 'b', 'c'], globals) + }) + + test('join combines array elements', async () => { + await expect(`str.join ['a' 'b' 'c'] '-'`).toEvaluateTo('a-b-c', globals) + await expect(`str.join ['hello' 'world'] ' '`).toEvaluateTo('hello world', globals) + }) + + test('join with comma separator', async () => { + await expect(`str.join ['a' 'b' 'c'] ','`).toEvaluateTo('a,b,c', globals) + }) + + test('starts-with? checks string prefix', async () => { + await expect(`str.starts-with? 'hello' 'hel'`).toEvaluateTo(true, globals) + await expect(`str.starts-with? 'hello' 'bye'`).toEvaluateTo(false, globals) + }) + + test('ends-with? checks string suffix', async () => { + await expect(`str.ends-with? 'hello' 'lo'`).toEvaluateTo(true, globals) + await expect(`str.ends-with? 'hello' 'he'`).toEvaluateTo(false, globals) + }) + + test('contains? checks for substring', async () => { + await expect(`str.contains? 'hello world' 'o w'`).toEvaluateTo(true, globals) + await expect(`str.contains? 'hello' 'bye'`).toEvaluateTo(false, globals) + }) + + test('empty? checks if string is empty', async () => { + await expect(`str.empty? ''`).toEvaluateTo(true, globals) + await expect(`str.empty? 'hello'`).toEvaluateTo(false, globals) + }) + + test('replace replaces first occurrence', async () => { + await expect(`str.replace 'hello hello' 'hello' 'hi'`).toEvaluateTo('hi hello', globals) + }) + + test('replace-all replaces all occurrences', async () => { + await expect(`str.replace-all 'hello hello' 'hello' 'hi'`).toEvaluateTo('hi hi', globals) + }) + + test('slice extracts substring', async () => { + await expect(`str.slice 'hello' 1 3`).toEvaluateTo('el', globals) + await expect(`str.slice 'hello' 2 null`).toEvaluateTo('llo', globals) + }) + + test('repeat repeats string', async () => { + await expect(`str.repeat 'ha' 3`).toEvaluateTo('hahaha', globals) + }) + + test('pad-start pads beginning', async () => { + await expect(`str.pad-start '5' 3 '0'`).toEvaluateTo('005', globals) + }) + + test('pad-end pads end', async () => { + await expect(`str.pad-end '5' 3 '0'`).toEvaluateTo('500', globals) + }) + + test('lines splits by newlines', async () => { + await expect(`str.lines 'a\\nb\\nc'`).toEvaluateTo(['a', 'b', 'c'], globals) + }) + + test('chars splits into characters', async () => { + await expect(`str.chars 'abc'`).toEvaluateTo(['a', 'b', 'c'], globals) + }) + + test('index-of finds substring position', async () => { + await expect(`str.index-of 'hello world' 'world'`).toEvaluateTo(6, globals) + await expect(`str.index-of 'hello' 'bye'`).toEvaluateTo(-1, globals) + }) + + test('last-index-of finds last occurrence', async () => { + await expect(`str.last-index-of 'hello hello' 'hello'`).toEvaluateTo(6, globals) + }) +}) + +describe('type predicates', () => { + test('string? checks for string type', async () => { + await expect(`string? 'hello'`).toEvaluateTo(true, globals) + await expect(`string? 42`).toEvaluateTo(false, globals) + }) + + test('number? checks for number type', async () => { + await expect(`number? 42`).toEvaluateTo(true, globals) + await expect(`number? 'hello'`).toEvaluateTo(false, globals) + }) + + test('boolean? checks for boolean type', async () => { + await expect(`boolean? true`).toEvaluateTo(true, globals) + await expect(`boolean? 42`).toEvaluateTo(false, globals) + }) + + test('array? checks for array type', async () => { + await expect(`array? [1 2 3]`).toEvaluateTo(true, globals) + await expect(`array? 42`).toEvaluateTo(false, globals) + }) + + test('dict? checks for dict type', async () => { + await expect(`dict? [a=1]`).toEvaluateTo(true, globals) + await expect(`dict? []`).toEvaluateTo(false, globals) + }) + + test('null? checks for null type', async () => { + await expect(`null? null`).toEvaluateTo(true, globals) + await expect(`null? 42`).toEvaluateTo(false, globals) + }) + + test('some? checks for non-null', async () => { + await expect(`some? 42`).toEvaluateTo(true, globals) + await expect(`some? null`).toEvaluateTo(false, globals) + }) +}) + +describe('boolean logic', () => { + test('not negates value', async () => { + await expect(`not true`).toEvaluateTo(false, globals) + await expect(`not false`).toEvaluateTo(true, globals) + await expect(`not 42`).toEvaluateTo(false, globals) + await expect(`not null`).toEvaluateTo(true, globals) + }) +}) + +describe('utilities', () => { + test('inc increments by 1', async () => { + await expect(`inc 5`).toEvaluateTo(6, globals) + await expect(`inc -1`).toEvaluateTo(0, globals) + }) + + test('dec decrements by 1', async () => { + await expect(`dec 5`).toEvaluateTo(4, globals) + await expect(`dec 0`).toEvaluateTo(-1, globals) + }) + + test('identity returns value as-is', async () => { + await expect(`identity 42`).toEvaluateTo(42, globals) + await expect(`identity 'hello'`).toEvaluateTo('hello', globals) + }) +}) + +describe('introspection', () => { + test('type returns proper types', async () => { + await expect(`type 'hello'`).toEvaluateTo('string', globals) + await expect(`type 42`).toEvaluateTo('number', globals) + await expect(`type true`).toEvaluateTo('boolean', globals) + await expect(`type false`).toEvaluateTo('boolean', globals) + await expect(`type null`).toEvaluateTo('null', globals) + await expect(`type [1 2 3]`).toEvaluateTo('array', globals) + await expect(`type [a=1 b=2]`).toEvaluateTo('dict', globals) + }) + + test('length', async () => { + await expect(`length 'hello'`).toEvaluateTo(5, globals) + await expect(`length [1 2 3]`).toEvaluateTo(3, globals) + await expect(`length [a=1 b=2]`).toEvaluateTo(2, globals) + await expect(`length 42`).toEvaluateTo(0, globals) + await expect(`length true`).toEvaluateTo(0, globals) + await expect(`length null`).toEvaluateTo(0, globals) + }) + + test('inspect formats values', async () => { + // Just test that inspect returns something for now + // (we'd need more complex assertion to check the actual format) + await expect(`type (inspect 'hello')`).toEvaluateTo('string', globals) + }) + + test('describe describes values', async () => { + // Just test that inspect returns something for now + // (we'd need more complex assertion to check the actual format) + await expect(`describe 'hello'`).toEvaluateTo("#", globals) + }) +}) + +describe('collections', () => { + test('literal array creates array from arguments', async () => { + await expect(`[ 1 2 3 ]`).toEvaluateTo([1, 2, 3], globals) + await expect(`['a' 'b']`).toEvaluateTo(['a', 'b'], globals) + await expect(`[]`).toEvaluateTo([], globals) + }) + + test('literal dict creates object from named arguments', async () => { + await expect(`[ a=1 b=2 ]`).toEvaluateTo({ a: 1, b: 2 }, globals) + await expect(`[=]`).toEvaluateTo({}, globals) + }) + + test('at retrieves element at index', async () => { + await expect(`at [10 20 30] 0`).toEvaluateTo(10, globals) + await expect(`at [10 20 30] 2`).toEvaluateTo(30, globals) + }) + + test('at retrieves property from object', async () => { + await expect(`at [name='test'] 'name'`).toEvaluateTo('test', globals) + }) + + test('slice extracts array subset', async () => { + await expect(`list.slice [1 2 3 4 5] 1 3`).toEvaluateTo([2, 3], globals) + await expect(`list.slice [1 2 3 4 5] 2 5`).toEvaluateTo([3, 4, 5], globals) + }) + + test('range creates number sequence', async () => { + await expect(`range 0 5`).toEvaluateTo([0, 1, 2, 3, 4, 5], globals) + await expect(`range 3 6`).toEvaluateTo([3, 4, 5, 6], globals) + }) + + test('range with single argument starts from 0', async () => { + await expect(`range 3 null`).toEvaluateTo([0, 1, 2, 3], globals) + await expect(`range 0 null`).toEvaluateTo([0], globals) + }) + + test('empty? checks if list, dict, string is empty', async () => { + await expect(`empty? []`).toEvaluateTo(true, globals) + await expect(`empty? [1]`).toEvaluateTo(false, globals) + + await expect(`empty? [=]`).toEvaluateTo(true, globals) + await expect(`empty? [a=true]`).toEvaluateTo(false, globals) + + await expect(`empty? ''`).toEvaluateTo(true, globals) + await expect(`empty? 'cat'`).toEvaluateTo(false, globals) + await expect(`empty? meow`).toEvaluateTo(false, globals) + }) + + test('list.filter keeps matching elements', async () => { + await expect(` + is-positive = do x: + x == 3 or x == 4 or x == 5 + end + list.filter [1 2 3 4 5] is-positive + `).toEvaluateTo([3, 4, 5], globals) + }) + + test('list.reduce accumulates values', async () => { + await expect(` + add = do acc x: + acc + x + end + list.reduce [1 2 3 4] add 0 + `).toEvaluateTo(10, globals) + }) + + test('list.find returns first match', async () => { + await expect(` + is-four = do x: + x == 4 + end + list.find [1 2 4 5] is-four + `).toEvaluateTo(4, globals) + }) + + test('list.find returns null if no match', async () => { + await expect(` + is-ten = do x: x == 10 end + list.find [1 2 3] is-ten + `).toEvaluateTo(null, globals) + }) + + test('list.empty? checks if list is empty', async () => { + await expect(`list.empty? []`).toEvaluateTo(true, globals) + await expect(`list.empty? [1]`).toEvaluateTo(false, globals) + }) + + test('list.contains? checks for element', async () => { + await expect(`list.contains? [1 2 3] 2`).toEvaluateTo(true, globals) + await expect(`list.contains? [1 2 3] 5`).toEvaluateTo(false, globals) + }) + + test('list.reverse reverses array', async () => { + await expect(`list.reverse [1 2 3]`).toEvaluateTo([3, 2, 1], globals) + }) + + test('list.concat combines arrays', async () => { + await expect(`list.concat [1 2] [3 4]`).toEvaluateTo([1, 2, 3, 4], globals) + }) + + test('list.flatten flattens nested arrays', async () => { + await expect(`list.flatten [[1 2] [3 4]] 1`).toEvaluateTo([1, 2, 3, 4], globals) + }) + + test('list.unique removes duplicates', async () => { + await expect(`list.unique [1 2 2 3 1]`).toEvaluateTo([1, 2, 3], globals) + }) + + test('list.zip combines two arrays', async () => { + await expect(`list.zip [1 2] [3 4]`).toEvaluateTo([[1, 3], [2, 4]], globals) + }) + + test('list.first returns first element', async () => { + await expect(`list.first [1 2 3]`).toEvaluateTo(1, globals) + await expect(`list.first []`).toEvaluateTo(null, globals) + }) + + test('list.last returns last element', async () => { + await expect(`list.last [1 2 3]`).toEvaluateTo(3, globals) + await expect(`list.last []`).toEvaluateTo(null, globals) + }) + + test('list.rest returns all but first', async () => { + await expect(`list.rest [1 2 3]`).toEvaluateTo([2, 3], globals) + }) + + test('list.take returns first n elements', async () => { + await expect(`list.take [1 2 3 4 5] 3`).toEvaluateTo([1, 2, 3], globals) + }) + + test('list.drop skips first n elements', async () => { + await expect(`list.drop [1 2 3 4 5] 2`).toEvaluateTo([3, 4, 5], globals) + }) + + test('list.append adds to end', async () => { + await expect(`list.append [1 2] 3`).toEvaluateTo([1, 2, 3], globals) + }) + + test('list.prepend adds to start', async () => { + await expect(`list.prepend [2 3] 1`).toEvaluateTo([1, 2, 3], globals) + }) + + test('list.index-of finds element index', async () => { + await expect(`list.index-of [1 2 3] 2`).toEvaluateTo(1, globals) + await expect(`list.index-of [1 2 3] 5`).toEvaluateTo(-1, globals) + }) + + test('list.any? checks if any element matches', async () => { + await expect(` + gt-three = do x: x > 3 end + list.any? [1 2 4 5] gt-three + `).toEvaluateTo(true, globals) + await expect(` + gt-ten = do x: x > 10 end + list.any? [1 2 3] gt-ten + `).toEvaluateTo(false, globals) + }) + + test('list.all? checks if all elements match', async () => { + await expect(` + positive = do x: x > 0 end + list.all? [1 2 3] positive + `).toEvaluateTo(true, globals) + await expect(` + positive = do x: x > 0 end + list.all? [1 -2 3] positive + `).toEvaluateTo(false, globals) + }) + + test('list.sum adds all numbers', async () => { + await expect(`list.sum [1 2 3 4]`).toEvaluateTo(10, globals) + await expect(`list.sum []`).toEvaluateTo(0, globals) + }) + + test('list.count counts matching elements', async () => { + await expect(` + gt-two = do x: x > 2 end + list.count [1 2 3 4 5] gt-two + `).toEvaluateTo(3, globals) + }) + + test('list.partition splits array by predicate', async () => { + await expect(` + gt-two = do x: x > 2 end + list.partition [1 2 3 4 5] gt-two + `).toEvaluateTo([[3, 4, 5], [1, 2]], globals) + }) + + test('list.compact removes null values', async () => { + await expect(`list.compact [1 null 2 null 3]`).toEvaluateTo([1, 2, 3], globals) + }) + + test('list.group-by groups by key function', async () => { + await expect(` + get-type = do x: + if (string? x): + 'str' + else: + 'num' + end + end + list.group-by ['a' 1 'b' 2] get-type + `).toEvaluateTo({ str: ['a', 'b'], num: [1, 2] }, globals) + }) +}) + +describe('enumerables', () => { + test('map transforms array elements', async () => { + await expect(` + double = do x: x * 2 end + list.map [1 2 3] double + `).toEvaluateTo([2, 4, 6], globals) + }) + + test('map handles empty array', async () => { + await expect(` + double = do x: x * 2 end + list.map [] double + `).toEvaluateTo([], globals) + }) + + test('each iterates over array', async () => { + // Note: each doesn't return the results, it returns null + // We can test it runs by checking the return value + await expect(` + double = do x: x * 2 end + each [1 2 3] double + `).toEvaluateTo([1, 2, 3], globals) + }) + + test('each handles empty array', async () => { + await expect(` + fn = do x: x end + each [] fn + `).toEvaluateTo([], globals) + }) +}) + +describe('dict operations', () => { + test('dict.keys returns all keys', async () => { + const result = await (async () => { + const { Compiler } = await import('#compiler/compiler') + const { run, fromValue } = await import('reefvm') + const { setGlobals } = await import('#parser/tokenizer') + setGlobals(Object.keys(globals)) + const c = new Compiler('dict.keys [a=1 b=2 c=3]') + const r = await run(c.bytecode, globals) + return fromValue(r) + })() + // Check that all expected keys are present (order may vary) + expect(result.sort()).toEqual(['a', 'b', 'c']) + }) + + test('dict.values returns all values', async () => { + const result = await (async () => { + const { Compiler } = await import('#compiler/compiler') + const { run, fromValue } = await import('reefvm') + const { setGlobals } = await import('#parser/tokenizer') + setGlobals(Object.keys(globals)) + const c = new Compiler('dict.values [a=1 b=2]') + const r = await run(c.bytecode, globals) + return fromValue(r) + })() + // Check that all expected values are present (order may vary) + expect(result.sort()).toEqual([1, 2]) + }) + + test('dict.has? checks for key', async () => { + await expect(`dict.has? [a=1 b=2] 'a'`).toEvaluateTo(true, globals) + await expect(`dict.has? [a=1 b=2] 'c'`).toEvaluateTo(false, globals) + }) + + test('dict.get retrieves value with default', async () => { + await expect(`dict.get [a=1] 'a' 0`).toEvaluateTo(1, globals) + await expect(`dict.get [a=1] 'b' 99`).toEvaluateTo(99, globals) + }) + + test('dict.set sets value', async () => { + await expect(`map = [a=1]; dict.set map 'b' 99; map.b`).toEvaluateTo(99, globals) + await expect(`map = [a=1]; dict.set map 'a' 100; map.a`).toEvaluateTo(100, globals) + }) + + test('dict.empty? checks if dict is empty', async () => { + await expect(`dict.empty? [=]`).toEvaluateTo(true, globals) + await expect(`dict.empty? [a=1]`).toEvaluateTo(false, globals) + }) + + test('dict.merge combines dicts', async () => { + await expect(`dict.merge [a=1] [b=2]`).toEvaluateTo({ a: 1, b: 2 }, globals) + }) + + test('dict.map transforms values', async () => { + await expect(` + double = do v k: v * 2 end + dict.map [a=1 b=2] double + `).toEvaluateTo({ a: 2, b: 4 }, globals) + }) + + test('dict.filter keeps matching entries', async () => { + await expect(` + gt-one = do v k: v > 1 end + dict.filter [a=1 b=2 c=3] gt-one + `).toEvaluateTo({ b: 2, c: 3 }, globals) + }) + + test('dict.from-entries creates dict from array', async () => { + await expect(`dict.from-entries [['a' 1] ['b' 2]]`).toEvaluateTo({ a: 1, b: 2 }, globals) + }) +}) + +describe('math operations', () => { + test('math.abs returns absolute value', async () => { + await expect(`math.abs -5`).toEvaluateTo(5, globals) + await expect(`math.abs 5`).toEvaluateTo(5, globals) + }) + + test('math.floor rounds down', async () => { + await expect(`math.floor 3.7`).toEvaluateTo(3, globals) + }) + + test('math.ceil rounds up', async () => { + await expect(`math.ceil 3.2`).toEvaluateTo(4, globals) + }) + + test('math.round rounds to nearest', async () => { + await expect(`math.round 3.4`).toEvaluateTo(3, globals) + await expect(`math.round 3.6`).toEvaluateTo(4, globals) + }) + + test('math.min returns minimum', async () => { + await expect(`math.min 5 2 8 1`).toEvaluateTo(1, globals) + }) + + test('math.max returns maximum', async () => { + await expect(`math.max 5 2 8 1`).toEvaluateTo(8, globals) + }) + + test('math.pow computes power', async () => { + await expect(`math.pow 2 3`).toEvaluateTo(8, globals) + }) + + test('math.sqrt computes square root', async () => { + await expect(`math.sqrt 16`).toEvaluateTo(4, globals) + }) + + test('math.even? checks if even', async () => { + await expect(`math.even? 4`).toEvaluateTo(true, globals) + await expect(`math.even? 5`).toEvaluateTo(false, globals) + }) + + test('math.odd? checks if odd', async () => { + await expect(`math.odd? 5`).toEvaluateTo(true, globals) + await expect(`math.odd? 4`).toEvaluateTo(false, globals) + }) + + test('math.positive? checks if positive', async () => { + await expect(`math.positive? 5`).toEvaluateTo(true, globals) + await expect(`math.positive? -5`).toEvaluateTo(false, globals) + await expect(`math.positive? 0`).toEvaluateTo(false, globals) + }) + + test('math.negative? checks if negative', async () => { + await expect(`math.negative? -5`).toEvaluateTo(true, globals) + await expect(`math.negative? 5`).toEvaluateTo(false, globals) + }) + + test('math.zero? checks if zero', async () => { + await expect(`math.zero? 0`).toEvaluateTo(true, globals) + await expect(`math.zero? 5`).toEvaluateTo(false, globals) + }) + + test('math.clamp restricts value to range', async () => { + await expect(`math.clamp 5 0 10`).toEvaluateTo(5, globals) + await expect(`math.clamp -5 0 10`).toEvaluateTo(0, globals) + await expect(`math.clamp 15 0 10`).toEvaluateTo(10, globals) + }) + + test('math.sign returns sign of number', async () => { + await expect(`math.sign 5`).toEvaluateTo(1, globals) + await expect(`math.sign -5`).toEvaluateTo(-1, globals) + await expect(`math.sign 0`).toEvaluateTo(0, globals) + }) + + test('math.trunc truncates decimal', async () => { + await expect(`math.trunc 3.7`).toEvaluateTo(3, globals) + await expect(`math.trunc -3.7`).toEvaluateTo(-3, globals) + }) +}) + +// describe('echo', () => { +// test('echo returns null value', async () => { +// await expect(`echo 'hello' 'world'`).toEvaluateTo(null, globalFunctions) +// }) + +// test('echo with array', async () => { +// await expect(`echo [1 2 3]`).toEvaluateTo(null, globalFunctions) +// }) + +// test('echo with multiple arguments', async () => { +// await expect(`echo 'test' 42 true`).toEvaluateTo(null, globalFunctions) +// }) +// }) diff --git a/src/testSetup.ts b/src/testSetup.ts index 8e1f4b8..f47218a 100644 --- a/src/testSetup.ts +++ b/src/testSetup.ts @@ -1,5 +1,6 @@ import { expect } from 'bun:test' import { parser } from '#parser/shrimp' +import { setGlobals } from '#parser/tokenizer' import { $ } from 'bun' import { assert, errorMessage } from '#utils/utils' import { Compiler } from '#compiler/compiler' @@ -30,7 +31,7 @@ await regenerateParser() // Type declaration for TypeScript declare module 'bun:test' { interface Matchers { - toMatchTree(expected: string): T + toMatchTree(expected: string, globals?: Record): T toMatchExpression(expected: string): T toFailParse(): T toEvaluateTo(expected: unknown, globals?: Record): Promise @@ -39,9 +40,10 @@ declare module 'bun:test' { } expect.extend({ - toMatchTree(received: unknown, expected: string) { + toMatchTree(received: unknown, expected: string, globals?: Record) { assert(typeof received === 'string', 'toMatchTree can only be used with string values') + if (globals) setGlobals(Object.keys(globals)) const tree = parser.parse(received) const actual = treeToString(tree, received) const normalizedExpected = trimWhitespace(expected) @@ -93,14 +95,11 @@ expect.extend({ } }, - async toEvaluateTo( - received: unknown, - expected: unknown, - globals: Record = {} - ) { + async toEvaluateTo(received: unknown, expected: unknown, globals: Record = {}) { assert(typeof received === 'string', 'toEvaluateTo can only be used with string values') try { + if (globals) setGlobals(Object.keys(globals)) const compiler = new Compiler(received) const result = await run(compiler.bytecode, globals) let value = VMResultToValue(result) @@ -109,13 +108,10 @@ expect.extend({ if (expected instanceof RegExp) expected = String(expected) if (value instanceof RegExp) value = String(value) - if (value === expected) { - return { pass: true } - } else { - return { - message: () => `Expected evaluation to be ${expected}, but got ${value}`, - pass: false, - } + expect(value).toEqual(expected) + return { + message: () => `Expected evaluation to be ${expected}, but got ${value}`, + pass: true, } } catch (error) { return { diff --git a/src/utils/tree.ts b/src/utils/tree.ts index 1682d21..45a9318 100644 --- a/src/utils/tree.ts +++ b/src/utils/tree.ts @@ -1,6 +1,5 @@ import { Tree, TreeCursor } from '@lezer/common' -import { assertNever } from '#utils/utils' -import { type Value } from 'reefvm' +import { type Value, fromValue } from 'reefvm' export const treeToString = (tree: Tree, input: string): string => { const lines: string[] = [] @@ -35,27 +34,5 @@ export const treeToString = (tree: Tree, input: string): string => { } export const VMResultToValue = (result: Value): unknown => { - if ( - result.type === 'number' || - result.type === 'boolean' || - result.type === 'string' || - result.type === 'regex' - ) { - return result.value - } else if (result.type === 'null') { - return null - } else if (result.type === 'array') { - return result.value.map(VMResultToValue) - } else if (result.type === 'dict') { - const obj: Record = {} - for (const [key, val] of Object.entries(result.value)) { - obj[key] = VMResultToValue(val) - } - - return obj - } else if (result.type === 'function') { - return Function - } else { - assertNever(result) - } + return result.type === 'function' ? Function : fromValue(result) }