$ can be in an identifier now

Reviewed-on: #52
Reviewed-by: probablycorey <probablycorey@gmail.com>

.gitignore

@@ -34,6 +34,7 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
 .DS_Store
 
 /tmp
+vscode-extension/tmp
 /docs
 
 *.vsix

bin/parser-tree.ts

@@ -0,0 +1,192 @@
+#!/usr/bin/env bun
+
+// WARNING: [[ No human has been anywhere near this file. It's pure Claude slop. 
+//             Enter at your own risk. ]]
+
+import { readFileSync } from 'fs'
+
+type CallInfo = {
+  method: string
+  line: number
+  calls: Set<string>
+  isRecursive?: boolean
+}
+
+// Parse the parser file and extract method calls
+function analyzeParser(filePath: string): Map<string, CallInfo> {
+  const content = readFileSync(filePath, 'utf-8')
+  const lines = content.split('\n')
+  const methods = new Map<string, CallInfo>()
+
+  // Find all method definitions
+  const methodRegex = /^\s*(\w+)\s*\([^)]*\):\s*/
+
+  let currentMethod: string | null = null
+  let braceDepth = 0
+  let classDepth = 0
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i] || ''
+
+    // Track if we're inside the Parser class
+    if (line.includes('class Parser')) {
+      classDepth = braceDepth + 1 // Will be the depth after we process this line's brace
+    }
+
+    // Check for method definition (only inside class, at class level)
+    // Check BEFORE incrementing braceDepth
+    if (classDepth > 0 && braceDepth === classDepth) {
+      const methodMatch = line.match(methodRegex)
+      if (methodMatch && !line.includes('class ')) {
+        currentMethod = methodMatch[1]!
+        methods.set(currentMethod, {
+          method: currentMethod,
+          line: i + 1,
+          calls: new Set()
+        })
+      }
+    }
+
+    // Track brace depth
+    braceDepth += (line.match(/{/g) || []).length
+    braceDepth -= (line.match(/}/g) || []).length
+
+    // Find method calls within current method
+    if (currentMethod && braceDepth > 0) {
+      // Match this.methodName() calls
+      const callRegex = /this\.(\w+)\s*\(/g
+      let match
+      while ((match = callRegex.exec(line)) !== null) {
+        const calledMethod = match[1]!
+        const info = methods.get(currentMethod)!
+        info.calls.add(calledMethod)
+
+        // Mark recursive calls
+        if (calledMethod === currentMethod) {
+          info.isRecursive = true
+        }
+      }
+    }
+
+    // Reset when method ends
+    if (braceDepth === 0) {
+      currentMethod = null
+    }
+  }
+
+  return methods
+}
+
+// Build tree structure starting from a root method
+function buildTree(
+  method: string,
+  callGraph: Map<string, CallInfo>,
+  visited: Set<string>,
+  indent = '',
+  isLast = true,
+  depth = 0,
+  maxDepth = 3
+): string[] {
+  const lines: string[] = []
+  const info = callGraph.get(method)
+
+  if (!info) return lines
+
+  // Add current method
+  const prefix = depth === 0 ? '' : (isLast ? '└─> ' : '├─> ')
+  const suffix = info.isRecursive ? ' (recursive)' : ''
+  const lineNum = `[line ${info.line}]`
+  lines.push(`${indent}${prefix}${method}() ${lineNum}${suffix}`)
+
+  // Stop if we've reached max depth
+  if (depth >= maxDepth) {
+    return lines
+  }
+
+  // Prevent infinite recursion in tree display
+  if (visited.has(method)) {
+    return lines
+  }
+
+  const newVisited = new Set(visited)
+  newVisited.add(method)
+
+  // Helper methods to filter out (low-level utilities)
+  const helperPatterns = /^(is|next|peek|expect|current|op)/i
+
+  // Get sorted unique calls (filter out recursive self-calls for display)
+  const calls = Array.from(info.calls)
+    .filter(c => callGraph.has(c)) // Only show parser methods
+    .filter(c => c !== method) // Don't show immediate self-recursion
+    .filter(c => !helperPatterns.test(c)) // Filter out helpers
+    .sort()
+
+  // Add children
+  const newIndent = indent + (isLast ? '     ' : '│    ')
+  calls.forEach((call, idx) => {
+    const childLines = buildTree(
+      call,
+      callGraph,
+      newVisited,
+      newIndent,
+      idx === calls.length - 1,
+      depth + 1,
+      maxDepth
+    )
+    lines.push(...childLines)
+  })
+
+  return lines
+}
+
+// Main
+const parserPath = './src/parser/parser2.ts'
+const maxDepth = parseInt(process.argv[2] || '5')
+
+console.log('Parser Call Tree for', parserPath)
+console.log(`Max depth: ${maxDepth}`)
+console.log('═'.repeat(60))
+console.log()
+
+const callGraph = analyzeParser(parserPath)
+
+// Start from parse() method
+const tree = buildTree('parse', callGraph, new Set(), '', true, 0, maxDepth)
+console.log(tree.join('\n'))
+
+// Show some stats
+console.log('\n' + '═'.repeat(60))
+console.log('Stats:')
+console.log(`  Total methods: ${callGraph.size}`)
+console.log(`  Entry point: parse()`)
+
+// Find methods that are never called (potential dead code or entry points)
+const allCalled = new Set<string>()
+for (const info of callGraph.values()) {
+  info.calls.forEach(c => allCalled.add(c))
+}
+
+const uncalled = Array.from(callGraph.keys())
+  .filter(m => !allCalled.has(m) && m !== 'parse')
+  .sort()
+
+if (uncalled.length > 0) {
+  console.log(`\n  Uncalled methods: ${uncalled.join(', ')}`)
+}
+
+// Find most-called methods
+const callCount = new Map<string, number>()
+for (const info of callGraph.values()) {
+  for (const called of info.calls) {
+    callCount.set(called, (callCount.get(called) || 0) + 1)
+  }
+}
+
+const topCalled = Array.from(callCount.entries())
+  .sort((a, b) => b[1] - a[1])
+  .slice(0, 5)
+
+console.log(`\n  Most-called methods:`)
+for (const [method, count] of topCalled) {
+  console.log(`    ${method}() - called ${count} times`)
+}

bin/shrimp

@@ -1,7 +1,7 @@
 #!/usr/bin/env bun
 
 import { colors, globals as prelude } from '../src/prelude'
-import { treeToString } from '../src/utils/tree'
+import { treeToString2 } from '../src/utils/tree'
 import { runCode, runFile, compileFile, parseCode } from '../src'
 import { resolve } from 'path'
 import { bytecodeToString } from 'reefvm'
@@ -31,7 +31,7 @@ ${colors.bright}Options:${colors.reset}
 }
 
 function showVersion() {
-  console.log('🦐 v0.0.1')
+  console.log('🦐 v0.0.1 (non-lezer parser)')
 }
 
 async function evalCode(code: string, imports: string[]) {
@@ -143,7 +143,7 @@ async function main() {
       process.exit(1)
     }
     const input = readFileSync(file, 'utf-8')
-    console.log(treeToString(parseCode(input), input))
+    console.log(treeToString2(parseCode(input).topNode, input))
     return
   }
 

bun.lock

@@ -16,6 +16,8 @@
         "@lezer/highlight": "^1.2.1",
         "@lezer/lr": "^1.4.2",
         "@types/bun": "latest",
+        "diff": "^8.0.2",
+        "kleur": "^4.1.5",
       },
     },
   },
@@ -60,8 +62,12 @@
 
     "csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="],
 
+    "diff": ["diff@8.0.2", "", {}, "sha512-sSuxWU5j5SR9QQji/o2qMvqRNYRDOcBTgsJ/DeCf4iSN4gW+gNMXM7wFIP+fdXZxoNiAnHUTGjCr+TSWXdRDKg=="],
+
     "hono": ["hono@4.10.4", "", {}, "sha512-YG/fo7zlU3KwrBL5vDpWKisLYiM+nVstBQqfr7gCPbSYURnNEP9BDxEMz8KfsDR9JX0lJWDRNc6nXX31v7ZEyg=="],
 
+    "kleur": ["kleur@4.1.5", "", {}, "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ=="],
+
     "reefvm": ["reefvm@git+https://git.nose.space/defunkt/reefvm#3e2e68b31f504347225a4d705c7568a0957d629e", { "peerDependencies": { "typescript": "^5" } }, "3e2e68b31f504347225a4d705c7568a0957d629e"],
 
     "style-mod": ["style-mod@4.1.3", "", {}, "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ=="],

examples/d20.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env shrimp
+# usage: dice <sides>
+
+import math only=random
+import list only=first
+import str only=[replace starts-with?]
+
+sides = $.args | first 
+sides ??= 20
+
+if sides | starts-with? d: 
+  sides = replace sides //\D// ''
+end
+
+sides = number sides 
+
+echo 'Rolling d$sides...'
+random 1 sides | echo

examples/license.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env shrimp
+
+year = date.now | date.year
+project = fs.pwd | fs.basename | str.titlecase
+
+{
+
+Copyright $year $project Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the “Software”), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE.
+
+}
+
+| str.trim
+| echo

examples/password.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env shrimp
+# usage: password <length> [!spaced] [!symbols]
+
+if ($.args | list.contains? -h):
+  echo 'usage: password <length> [!spaced] [!symbols]'
+  exit
+end
+
+password = do n=22 symbols=true spaced=true:
+  chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
+  if symbols: chars += '!@#%^&*-=()[]<>' end
+
+  out = []
+  i = 0
+  max = length chars
+
+  while i < n:
+    idx = math.floor ((math.random) * max)
+    ch = chars | at idx
+    list.push out ch
+    i += 1
+  end
+
+  if spaced:
+    pos1 = math.floor((n - 2) / 3)
+    pos2 = math.floor((n - 2) * 2 / 3)
+
+    list.insert out pos2 ' '
+    list.insert out pos1 ' '
+  end
+
+  str.join out ''
+end
+
+missing-arg? = do x: $.args | list.contains? x | not end
+
+num = $.args | list.reject (do x: x | str.starts-with? ! end) | list.first
+
+password num symbols=(missing-arg? !symbols) spaced=(missing-arg? !spaced) | echo

examples/scripts.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env shrimp
+
+if not fs.exists? 'package.json':
+  echo '🦐 package.json not found'
+  exit 1
+end
+
+package = fs.read 'package.json' | json.decode
+package.scripts | dict.keys | list.sort | each do x: echo x end

package.json

@@ -5,12 +5,12 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "dev": "bun generate-parser && bun --hot src/server/server.tsx",
+    "dev": "bun --hot src/server/server.tsx",
-    "generate-parser": "lezer-generator src/parser/shrimp.grammar --typeScript -o src/parser/shrimp.ts",
+    "repl": "bun bin/repl",
-    "repl": "bun generate-parser && bun bin/repl",
     "update-reef": "rm -rf ~/.bun/install/cache/ && rm bun.lock && bun update reefvm",
     "cli:install": "ln -s \"$(pwd)/bin/shrimp\" ~/.bun/bin/shrimp",
-    "cli:remove": "rm ~/.bun/bin/shrimp"
+    "cli:remove": "rm ~/.bun/bin/shrimp",
+    "check": "bunx tsc --noEmit"
   },
   "dependencies": {
     "@codemirror/view": "^6.38.3",
@@ -24,7 +24,9 @@
   "devDependencies": {
     "@lezer/highlight": "^1.2.1",
     "@lezer/lr": "^1.4.2",
-    "@types/bun": "latest"
+    "@types/bun": "latest",
+    "diff": "^8.0.2",
+    "kleur": "^4.1.5"
   },
   "prettier": {
     "semi": false,

src/compiler/compiler.ts

@@ -1,9 +1,7 @@
 import { CompilerError } from '#compiler/compilerError.ts'
-import { parser } from '#parser/shrimp.ts'
+import { parse, setGlobals } from '#parser/parser2'
-import * as terms from '#parser/shrimp.terms'
+import { SyntaxNode, Tree } from '#parser/node'
-import { setGlobals } from '#parser/tokenizer'
 import { tokenizeCurlyString } from '#parser/curlyTokenizer'
-import type { SyntaxNode, Tree } from '@lezer/common'
 import { assert, errorMessage } from '#utils/utils'
 import { toBytecode, type Bytecode, type ProgramItem, bytecodeToString } from 'reefvm'
 import {
@@ -63,7 +61,8 @@ export class Compiler {
   constructor(public input: string, globals?: string[] | Record<string, any>) {
     try {
       if (globals) setGlobals(Array.isArray(globals) ? globals : Object.keys(globals))
-      const cst = parser.parse(input)
+      const ast = parse(input)
+      const cst = new Tree(ast)
       const errors = checkTreeForErrors(cst)
 
       const firstError = errors[0]
@@ -89,7 +88,7 @@ export class Compiler {
   }
 
   #compileCst(cst: Tree, input: string) {
-    const isProgram = cst.topNode.type.id === terms.Program
+    const isProgram = cst.topNode.type.is('Program')
     assert(isProgram, `Expected Program node, got ${cst.topNode.type.name}`)
 
     let child = cst.topNode.firstChild
@@ -105,8 +104,8 @@ export class Compiler {
     const value = input.slice(node.from, node.to)
     if (DEBUG) console.log(`🫦  ${node.name}: ${value}`)
 
-    switch (node.type.id) {
+    switch (node.type.name) {
-      case terms.Number:
+      case 'Number':
         // Handle sign prefix for hex, binary, and octal literals
         // Number() doesn't parse '-0xFF', '+0xFF', '-0o77', etc. correctly
         let numberValue: number
@@ -123,8 +122,8 @@ export class Compiler {
 
         return [[`PUSH`, numberValue]]
 
-      case terms.String: {
+      case 'String': {
-        if (node.firstChild?.type.id === terms.CurlyString)
+        if (node.firstChild?.type.is('CurlyString'))
           return this.#compileCurlyString(value, input)
 
         const { parts, hasInterpolation } = getStringParts(node, input)
@@ -141,19 +140,19 @@ export class Compiler {
         parts.forEach((part) => {
           const partValue = input.slice(part.from, part.to)
 
-          switch (part.type.id) {
+          switch (part.type.name) {
-            case terms.StringFragment:
+            case 'StringFragment':
               // Plain text fragment - just push as-is
               instructions.push(['PUSH', partValue])
               break
 
-            case terms.EscapeSeq:
+            case 'EscapeSeq':
               // Process escape sequence and push the result
               const processed = processEscapeSeq(partValue)
               instructions.push(['PUSH', processed])
               break
 
-            case terms.Interpolation:
+            case 'Interpolation':
               // Interpolation contains either Identifier or ParenExpr (the $ is anonymous)
               const child = part.firstChild
               if (!child) {
@@ -177,15 +176,15 @@ export class Compiler {
         return instructions
       }
 
-      case terms.Boolean: {
+      case 'Boolean': {
         return [[`PUSH`, value === 'true']]
       }
 
-      case terms.Null: {
+      case 'Null': {
         return [[`PUSH`, null]]
       }
 
-      case terms.Regex: {
+      case 'Regex': {
         // remove the surrounding slashes and any flags
         const [_, pattern, flags] = value.match(/^\/\/(.*)\/\/([gimsuy]*)$/) || []
         if (!pattern) {
@@ -202,15 +201,15 @@ export class Compiler {
         return [['PUSH', regex]]
       }
 
-      case terms.Identifier: {
+      case 'Identifier': {
         return [[`TRY_LOAD`, value]]
       }
 
-      case terms.Word: {
+      case 'Word': {
         return [['PUSH', value]]
       }
 
-      case terms.DotGet: {
+      case 'DotGet': {
         // DotGet is parsed into a nested tree because it's hard to parse it into a flat one.
         // However, we want a flat tree - so we're going to pretend like we are getting one from the parser.
         //
@@ -222,7 +221,7 @@ export class Compiler {
         instructions.push(['TRY_LOAD', objectName])
 
         const flattenProperty = (prop: SyntaxNode): void => {
-          if (prop.type.id === terms.DotGet) {
+          if (prop.type.is('DotGet')) {
             const nestedParts = getDotGetParts(prop, input)
 
             const nestedObjectValue = input.slice(nestedParts.object.from, nestedParts.object.to)
@@ -231,7 +230,7 @@ export class Compiler {
 
             flattenProperty(nestedParts.property)
           } else {
-            if (prop.type.id === terms.ParenExpr) {
+            if (prop.type.is('ParenExpr')) {
               instructions.push(...this.#compileNode(prop, input))
             } else {
               const propertyValue = input.slice(prop.from, prop.to)
@@ -245,7 +244,7 @@ export class Compiler {
         return instructions
       }
 
-      case terms.BinOp: {
+      case 'BinOp': {
         const { left, op, right } = getBinaryParts(node)
         const instructions: ProgramItem[] = []
         instructions.push(...this.#compileNode(left, input))
@@ -293,7 +292,7 @@ export class Compiler {
         return instructions
       }
 
-      case terms.Assign: {
+      case 'Assign': {
         const assignParts = getAssignmentParts(node)
         const instructions: ProgramItem[] = []
 
@@ -324,7 +323,7 @@ export class Compiler {
         return instructions
       }
 
-      case terms.CompoundAssign: {
+      case 'CompoundAssign': {
         const { identifier, operator, right } = getCompoundAssignmentParts(node)
         const identifierName = input.slice(identifier.from, identifier.to)
         const instructions: ProgramItem[] = []
@@ -386,14 +385,14 @@ export class Compiler {
         return instructions
       }
 
-      case terms.ParenExpr: {
+      case 'ParenExpr': {
         const child = node.firstChild
         if (!child) return [] // I guess it is empty parentheses?
 
         return this.#compileNode(child, input)
       }
 
-      case terms.FunctionDef: {
+      case 'FunctionDef': {
         const { paramNames, bodyNodes, catchVariable, catchBody, finallyBody } =
           getFunctionDefParts(node, input)
         const instructions: ProgramItem[] = []
@@ -439,8 +438,8 @@ export class Compiler {
         return instructions
       }
 
-      case terms.FunctionCallOrIdentifier: {
+      case 'FunctionCallOrIdentifier': {
-        if (node.firstChild?.type.id === terms.DotGet) {
+        if (node.firstChild?.type.is('DotGet')) {
           const instructions: ProgramItem[] = []
           const callLabel: Label = `.call_dotget_${++this.labelCount}`
           const afterLabel: Label = `.after_dotget_${++this.labelCount}`
@@ -482,8 +481,8 @@ export class Compiler {
       PUSH 1              ; Named count
       CALL
       */
-      case terms.FunctionCallWithNewlines:
+
-      case terms.FunctionCall: {
+      case 'FunctionCall': {
         const { identifierNode, namedArgs, positionalArgs } = getFunctionCallParts(node, input)
         const instructions: ProgramItem[] = []
         instructions.push(...this.#compileNode(identifierNode, input))
@@ -505,7 +504,7 @@ export class Compiler {
         return instructions
       }
 
-      case terms.Block: {
+      case 'Block': {
         const children = getAllChildren(node)
         const instructions: ProgramItem[] = []
 
@@ -520,7 +519,7 @@ export class Compiler {
         return instructions
       }
 
-      case terms.FunctionCallWithBlock: {
+      case 'FunctionCallWithBlock': {
         const [fn, _colon, ...block] = getAllChildren(node)
         let instructions: ProgramItem[] = []
 
@@ -538,13 +537,13 @@ export class Compiler {
         instructions.push(['RETURN'])
         instructions.push([`${afterLabel}:`])
 
-        if (fn?.type.id === terms.FunctionCallOrIdentifier) {
+        if (fn?.type.is('FunctionCallOrIdentifier')) {
           instructions.push(['LOAD', input.slice(fn!.from, fn!.to)])
           instructions.push(['MAKE_FUNCTION', [], fnLabel])
           instructions.push(['PUSH', 1])
           instructions.push(['PUSH', 0])
           instructions.push(['CALL'])
-        } else if (fn?.type.id === terms.FunctionCall) {
+        } else if (fn?.type.is('FunctionCall')) {
           let body = this.#compileNode(fn!, input)
           const namedArgCount = (body[body.length - 2]![1] as number) * 2
           const startSlice = body.length - namedArgCount - 3
@@ -567,7 +566,7 @@ export class Compiler {
         return instructions
       }
 
-      case terms.TryExpr: {
+      case 'TryExpr': {
         const { tryBlock, catchVariable, catchBody, finallyBody } = getTryExprParts(node, input)
 
         return this.#compileTryCatchFinally(
@@ -579,12 +578,14 @@ export class Compiler {
         )
       }
 
-      case terms.Throw: {
+      case 'Throw':
+      case 'Not': {
+        const keyword = node.type.is('Throw') ? 'Throw' : 'Not'
         const children = getAllChildren(node)
         const [_throwKeyword, expression] = children
         if (!expression) {
           throw new CompilerError(
-            `Throw expected expression, got ${children.length} children`,
+            `${keyword} expected expression, got ${children.length} children`,
             node.from,
             node.to
           )
@@ -592,12 +593,12 @@ export class Compiler {
 
         const instructions: ProgramItem[] = []
         instructions.push(...this.#compileNode(expression, input))
-        instructions.push(['THROW'])
+        instructions.push([keyword.toUpperCase()]) // THROW or NOT
 
         return instructions
       }
 
-      case terms.IfExpr: {
+      case 'IfExpr': {
         const { conditionNode, thenBlock, elseIfBlocks, elseThenBlock } = getIfExprParts(
           node,
           input
@@ -640,7 +641,7 @@ export class Compiler {
       }
 
       // - `EQ`, `NEQ`, `LT`, `GT`, `LTE`, `GTE` - Pop 2, push boolean
-      case terms.ConditionalOp: {
+      case 'ConditionalOp': {
         const instructions: ProgramItem[] = []
         const { left, op, right } = getBinaryParts(node)
         const leftInstructions: ProgramItem[] = this.#compileNode(left, input)
@@ -715,7 +716,7 @@ export class Compiler {
         return instructions
       }
 
-      case terms.PipeExpr: {
+      case 'PipeExpr': {
         const { pipedFunctionCall, pipeReceivers } = getPipeExprParts(node)
         if (!pipedFunctionCall || pipeReceivers.length === 0) {
           throw new CompilerError('PipeExpr must have at least two operands', node.from, node.to)
@@ -737,11 +738,11 @@ export class Compiler {
           instructions.push(...this.#compileNode(identifierNode, input))
 
           const isUnderscoreInPositionalArgs = positionalArgs.some(
-            (arg) => arg.type.id === terms.Underscore
+            (arg) => arg.type.is('Underscore')
           )
           const isUnderscoreInNamedArgs = namedArgs.some((arg) => {
             const { valueNode } = getNamedArgParts(arg, input)
-            return valueNode.type.id === terms.Underscore
+            return valueNode.type.is('Underscore')
           })
 
           const shouldPushPositionalArg = !isUnderscoreInPositionalArgs && !isUnderscoreInNamedArgs
@@ -752,7 +753,7 @@ export class Compiler {
           }
 
           positionalArgs.forEach((arg) => {
-            if (arg.type.id === terms.Underscore) {
+            if (arg.type.is('Underscore')) {
               instructions.push(['LOAD', pipeValName])
             } else {
               instructions.push(...this.#compileNode(arg, input))
@@ -762,7 +763,7 @@ export class Compiler {
           namedArgs.forEach((arg) => {
             const { name, valueNode } = getNamedArgParts(arg, input)
             instructions.push(['PUSH', name])
-            if (valueNode.type.id === terms.Underscore) {
+            if (valueNode.type.is('Underscore')) {
               instructions.push(['LOAD', pipeValName])
             } else {
               instructions.push(...this.#compileNode(valueNode, input))
@@ -777,14 +778,14 @@ export class Compiler {
         return instructions
       }
 
-      case terms.Array: {
+      case 'Array': {
         const children = getAllChildren(node)
 
         // We can easily parse [=] as an empty dict, but `[ = ]` is tougher.
         // = can be a valid word, and is also valid inside words, so for now we cheat
         // and check for arrays that look like `[ = ]` to interpret them as
         // empty dicts
-        if (children.length === 1 && children[0]!.type.id === terms.Word) {
+        if (children.length === 1 && children[0]!.type.is('Word')) {
           const child = children[0]!
           if (input.slice(child.from, child.to) === '=') {
             return [['MAKE_DICT', 0]]
@@ -796,7 +797,7 @@ export class Compiler {
         return instructions
       }
 
-      case terms.Dict: {
+      case 'Dict': {
         const children = getAllChildren(node)
         const instructions: ProgramItem[] = []
 
@@ -805,7 +806,7 @@ export class Compiler {
           const valueNode = node.firstChild!.nextSibling
 
           // name= -> name
-          const key = input.slice(keyNode!.from, keyNode!.to).slice(0, -1)
+          const key = input.slice(keyNode!.from, keyNode!.to).replace(/\s*=$/, '')
           instructions.push(['PUSH', key])
 
           instructions.push(...this.#compileNode(valueNode!, input))
@@ -815,7 +816,7 @@ export class Compiler {
         return instructions
       }
 
-      case terms.WhileExpr: {
+      case 'WhileExpr': {
         const [_while, test, _colon, block] = getAllChildren(node)
         const instructions: ProgramItem[] = []
 
@@ -833,11 +834,11 @@ export class Compiler {
         return instructions
       }
 
-      case terms.Import: {
+      case 'Import': {
         const instructions: ProgramItem[] = []
         const [_import, ...nodes] = getAllChildren(node)
-        const args = nodes.filter(node => node.type.id === terms.Identifier)
+        const args = nodes.filter(node => node.type.is('Identifier'))
-        const namedArgs = nodes.filter(node => node.type.id === terms.NamedArg)
+        const namedArgs = nodes.filter(node => node.type.is('NamedArg'))
 
         instructions.push(['LOAD', 'import'])
 
@@ -858,13 +859,13 @@ export class Compiler {
         return instructions
       }
 
-      case terms.Comment: {
+      case 'Comment': {
         return [] // ignore comments
       }
 
       default:
         throw new CompilerError(
-          `Compiler doesn't know how to handle a "${node.type.name}" (${node.type.id}) node.`,
+          `Compiler doesn't know how to handle a "${node.type.name}" node.`,
           node.from,
           node.to
         )

src/compiler/tests/literals.test.ts

@@ -151,18 +151,22 @@ describe('array literals', () => {
 describe('dict literals', () => {
   test('work with numbers', () => {
     expect('[a=1 b=2 c=3]').toEvaluateTo({ a: 1, b: 2, c: 3 })
+    expect('[a = 1 b = 2 c = 3]').toEvaluateTo({ a: 1, b: 2, c: 3 })
   })
 
   test('work with strings', () => {
     expect("[a='one' b='two' c='three']").toEvaluateTo({ a: 'one', b: 'two', c: 'three' })
+    expect("[a = 'one' b = 'two' c = 'three']").toEvaluateTo({ a: 'one', b: 'two', c: 'three' })
   })
 
   test('work with identifiers', () => {
     expect('[a=one b=two c=three]').toEvaluateTo({ a: 'one', b: 'two', c: 'three' })
+    expect('[a = one b = two c = three]').toEvaluateTo({ a: 'one', b: 'two', c: 'three' })
   })
 
   test('can be nested', () => {
     expect('[a=one b=[two [c=three]]]').toEvaluateTo({ a: 'one', b: ['two', { c: 'three' }] })
+    expect('[a = one b = [two [c = three]]]').toEvaluateTo({ a: 'one', b: ['two', { c: 'three' }] })
   })
 
   test('can span multiple lines', () => {
@@ -171,6 +175,12 @@ describe('dict literals', () => {
         b=2
         c=3
       ]`).toEvaluateTo({ a: 1, b: 2, c: 3 })
+
+    expect(`[
+        a = 1
+        b = 2
+        c = 3
+      ]`).toEvaluateTo({ a: 1, b: 2, c: 3 })
   })
 
   test('empty dict', () => {
@@ -190,10 +200,12 @@ describe('dict literals', () => {
 
   test('semicolons as separators', () => {
     expect('[a=1; b=2; c=3]').toEvaluateTo({ a: 1, b: 2, c: 3 })
+    expect('[a = 1; b = 2; c = 3]').toEvaluateTo({ a: 1, b: 2, c: 3 })
   })
 
   test('expressions in dicts', () => {
     expect('[a=(1 + 2) b=(3 * 4)]').toEvaluateTo({ a: 3, b: 12 })
+    expect('[a = (1 + 2) b = (3 * 4)]').toEvaluateTo({ a: 3, b: 12 })
   })
 
   test('empty lines within dicts', () => {
@@ -246,7 +258,7 @@ describe('curly strings', () => {
   test('interpolation edge cases', () => {
     expect(`{[a=1 b=2 c={wild}]}`).toEvaluateTo(`[a=1 b=2 c={wild}]`)
     expect(`a = 1;b = 2;c = 3;{$a $b $c}`).toEvaluateTo(`1 2 3`)
-    expect(`a = 1;b = 2;c = 3;{$a$b$c}`).toEvaluateTo(`123`)
+    expect(`a = 1;b = 2;c = 3;{$(a)$(b)$(c)}`).toEvaluateTo(`123`)
   })
 })
 

src/compiler/utils.ts

@@ -1,9 +1,9 @@
 import { CompilerError } from '#compiler/compilerError.ts'
-import * as terms from '#parser/shrimp.terms'
+import type { SyntaxNode, Tree } from '#parser/node'
-import type { SyntaxNode, Tree } from '@lezer/common'
 
 export const checkTreeForErrors = (tree: Tree): CompilerError[] => {
   const errors: CompilerError[] = []
+
   tree.iterate({
     enter: (node) => {
       if (node.type.isError) {
@@ -23,7 +23,7 @@ export const getAllChildren = (node: SyntaxNode): SyntaxNode[] => {
     child = child.nextSibling
   }
 
-  return children.filter((n) => n.type.id !== terms.Comment)
+  return children.filter((n) => !n.type.is('Comment'))
 }
 
 export const getBinaryParts = (node: SyntaxNode) => {
@@ -50,15 +50,14 @@ export const getAssignmentParts = (node: SyntaxNode) => {
   }
 
   // array destructuring
-  if (left && left.type.id === terms.Array) {
+  if (left && left.type.is('Array')) {
-    const identifiers = getAllChildren(left).filter((child) => child.type.id === terms.Identifier)
+    const identifiers = getAllChildren(left).filter((child) => child.type.is('Identifier'))
     return { arrayPattern: identifiers, right }
   }
 
-  if (!left || left.type.id !== terms.AssignableIdentifier) {
+  if (!left || !left.type.is('AssignableIdentifier')) {
     throw new CompilerError(
-      `Assign left child must be an AssignableIdentifier or Array, got ${
+      `Assign left child must be an AssignableIdentifier or Array, got ${left ? left.type.name : 'none'
-        left ? left.type.name : 'none'
       }`,
       node.from,
       node.to
@@ -72,10 +71,9 @@ export const getCompoundAssignmentParts = (node: SyntaxNode) => {
   const children = getAllChildren(node)
   const [left, operator, right] = children
 
-  if (!left || left.type.id !== terms.AssignableIdentifier) {
+  if (!left || !left.type.is('AssignableIdentifier')) {
     throw new CompilerError(
-      `CompoundAssign left child must be an AssignableIdentifier, got ${
+      `CompoundAssign left child must be an AssignableIdentifier, got ${left ? left.type.name : 'none'
-        left ? left.type.name : 'none'
       }`,
       node.from,
       node.to
@@ -104,7 +102,7 @@ export const getFunctionDefParts = (node: SyntaxNode, input: string) => {
   }
 
   const paramNames = getAllChildren(paramsNode).map((param) => {
-    if (param.type.id !== terms.Identifier && param.type.id !== terms.NamedParam) {
+    if (!param.type.is('Identifier') && !param.type.is('NamedParam')) {
       throw new CompilerError(
         `FunctionDef params must be Identifier or NamedParam, got ${param.type.name}`,
         param.from,
@@ -123,7 +121,7 @@ export const getFunctionDefParts = (node: SyntaxNode, input: string) => {
   let finallyBody: SyntaxNode | undefined
 
   for (const child of rest) {
-    if (child.type.id === terms.CatchExpr) {
+    if (child.type.is('CatchExpr')) {
       catchExpr = child
       const catchChildren = getAllChildren(child)
       const [_catchKeyword, identifierNode, _colon, body] = catchChildren
@@ -136,7 +134,7 @@ export const getFunctionDefParts = (node: SyntaxNode, input: string) => {
       }
       catchVariable = input.slice(identifierNode.from, identifierNode.to)
       catchBody = body
-    } else if (child.type.id === terms.FinallyExpr) {
+    } else if (child.type.is('FinallyExpr')) {
       finallyExpr = child
       const finallyChildren = getAllChildren(child)
       const [_finallyKeyword, _colon, body] = finallyChildren
@@ -165,9 +163,9 @@ export const getFunctionCallParts = (node: SyntaxNode, input: string) => {
     throw new CompilerError(`FunctionCall expected at least 1 child, got 0`, node.from, node.to)
   }
 
-  const namedArgs = args.filter((arg) => arg.type.id === terms.NamedArg)
+  const namedArgs = args.filter((arg) => arg.type.is('NamedArg'))
   const positionalArgs = args
-    .filter((arg) => arg.type.id === terms.PositionalArg)
+    .filter((arg) => arg.type.is('PositionalArg'))
     .map((arg) => {
       const child = arg.firstChild
       if (!child) throw new CompilerError(`PositionalArg has no child`, arg.from, arg.to)
@@ -208,13 +206,13 @@ export const getIfExprParts = (node: SyntaxNode, input: string) => {
   rest.forEach((child) => {
     const parts = getAllChildren(child)
 
-    if (child.type.id === terms.ElseExpr) {
+    if (child.type.is('ElseExpr')) {
       if (parts.length !== 3) {
         const message = `ElseExpr expected 1 child, got ${parts.length}`
         throw new CompilerError(message, child.from, child.to)
       }
       elseThenBlock = parts.at(-1)
-    } else if (child.type.id === terms.ElseIfExpr) {
+    } else if (child.type.is('ElseIfExpr')) {
       const [_else, _if, conditional, _colon, thenBlock] = parts
       if (!conditional || !thenBlock) {
         const names = parts.map((p) => p.type.name).join(', ')
@@ -249,10 +247,10 @@ export const getStringParts = (node: SyntaxNode, input: string) => {
   // The text is just between the quotes
   const parts = children.filter((child) => {
     return (
-      child.type.id === terms.StringFragment ||
+      child.type.is('StringFragment') ||
-      child.type.id === terms.Interpolation ||
+      child.type.is('Interpolation') ||
-      child.type.id === terms.EscapeSeq ||
+      child.type.is('EscapeSeq') ||
-      child.type.id === terms.CurlyString
+      child.type.is('CurlyString')
 
     )
   })
@@ -260,10 +258,10 @@ export const getStringParts = (node: SyntaxNode, input: string) => {
   // Validate each part is the expected type
   parts.forEach((part) => {
     if (
-      part.type.id !== terms.StringFragment &&
+      part.type.is('StringFragment') &&
-      part.type.id !== terms.Interpolation &&
+      part.type.is('Interpolation') &&
-      part.type.id !== terms.EscapeSeq &&
+      part.type.is('EscapeSeq') &&
-      part.type.id !== terms.CurlyString
+      part.type.is('CurlyString')
     ) {
       throw new CompilerError(
         `String child must be StringFragment, Interpolation, or EscapeSeq, got ${part.type.name}`,
@@ -276,7 +274,7 @@ export const getStringParts = (node: SyntaxNode, input: string) => {
   // hasInterpolation means the string has interpolation ($var) or escape sequences (\n)
   // A simple string like 'hello' has one StringFragment but no interpolation
   const hasInterpolation = parts.some(
-    (p) => p.type.id === terms.Interpolation || p.type.id === terms.EscapeSeq
+    (p) => p.type.is('Interpolation') || p.type.is('EscapeSeq')
   )
   return { parts, hasInterpolation }
 }
@@ -293,7 +291,7 @@ export const getDotGetParts = (node: SyntaxNode, input: string) => {
     )
   }
 
-  if (object.type.id !== terms.IdentifierBeforeDot && object.type.id !== terms.Dollar) {
+  if (!object.type.is('IdentifierBeforeDot')) {
     throw new CompilerError(
       `DotGet object must be an IdentifierBeforeDot, got ${object.type.name}`,
       object.from,
@@ -301,7 +299,7 @@ export const getDotGetParts = (node: SyntaxNode, input: string) => {
     )
   }
 
-  if (![terms.Identifier, terms.Number, terms.ParenExpr, terms.DotGet].includes(property.type.id)) {
+  if (!['Identifier', 'Number', 'ParenExpr', 'DotGet'].includes(property.type.name)) {
     throw new CompilerError(
       `DotGet property must be an Identifier, Number, ParenExpr, or DotGet, got ${property.type.name}`,
       property.from,
@@ -335,7 +333,7 @@ export const getTryExprParts = (node: SyntaxNode, input: string) => {
   let finallyBody: SyntaxNode | undefined
 
   rest.forEach((child) => {
-    if (child.type.id === terms.CatchExpr) {
+    if (child.type.is('CatchExpr')) {
       catchExpr = child
       const catchChildren = getAllChildren(child)
       const [_catchKeyword, identifierNode, _colon, body] = catchChildren
@@ -348,7 +346,7 @@ export const getTryExprParts = (node: SyntaxNode, input: string) => {
       }
       catchVariable = input.slice(identifierNode.from, identifierNode.to)
       catchBody = body
-    } else if (child.type.id === terms.FinallyExpr) {
+    } else if (child.type.is('FinallyExpr')) {
       finallyExpr = child
       const finallyChildren = getAllChildren(child)
       const [_finallyKeyword, _colon, body] = finallyChildren

src/editor/commands.ts

@@ -245,7 +245,7 @@ const commandShapes: CommandShape[] = [
 ] as const
 
 let commandSource = () => commandShapes
-export const setCommandSource = (do: () => CommandShape[]) => {
+export const setCommandSource = (fn: () => CommandShape[]) => {
   commandSource = fn
 }
 

src/index.ts

@@ -1,15 +1,15 @@
 import { readFileSync } from 'fs'
 import { VM, fromValue, toValue, isValue, type Bytecode } from 'reefvm'
-import { type Tree } from '@lezer/common'
 import { Compiler } from '#compiler/compiler'
-import { parser } from '#parser/shrimp'
+import { parse } from '#parser/parser2'
-import { globals as parserGlobals, setGlobals as setParserGlobals } from '#parser/tokenizer'
+import { Tree } from '#parser/node'
+import { globals as parserGlobals, setGlobals as setParserGlobals } from '#parser/parser2'
 import { globals as prelude } from '#prelude'
 
 export { Compiler } from '#compiler/compiler'
-export { parser } from '#parser/shrimp'
+export { parse } from '#parser/parser2'
+export { type SyntaxNode, Tree } from '#parser/node'
 export { globals as prelude } from '#prelude'
-export type { Tree } from '@lezer/common'
 export { type Value, type Bytecode } from 'reefvm'
 export { toValue, fromValue, isValue, Scope, VM, bytecodeToString } from 'reefvm'
 
@@ -105,8 +105,8 @@ export function parseCode(code: string, globals?: Record<string, any>): Tree {
     const globalNames = [...Object.keys(prelude), ...(globals ? Object.keys(globals) : [])]
 
     setParserGlobals(globalNames)
-    const result = parser.parse(code)
+    const result = parse(code)
     setParserGlobals(oldGlobals)
 
-    return result
+    return new Tree(result)
 }

src/parser/curlyTokenizer.ts

@@ -1,6 +1,6 @@
-import { parser } from '#parser/shrimp.ts'
+import { parse } from '#parser/parser2'
-import type { SyntaxNode } from '@lezer/common'
+import type { SyntaxNode } from '#parser/node'
-import { isIdentStart, isIdentChar } from './tokenizer'
+import { isIdentStart, isIdentChar } from './tokenizer2'
 
 // Turns a { curly string } into strings and nodes for interpolation
 export const tokenizeCurlyString = (value: string): (string | [string, SyntaxNode])[] => {
@@ -37,8 +37,8 @@ export const tokenizeCurlyString = (value: string): (string | [string, SyntaxNod
         }
 
         const input = value.slice(start + 2, pos) // skip '$('
-        tokens.push([input, parser.parse(input).topNode])
+        tokens.push([input, parse(input)])
-        start = ++pos // skip ')'
+        start = pos + 1 // start after ')'
       } else {
         char = value[++pos]
         if (!char) break
@@ -48,7 +48,7 @@ export const tokenizeCurlyString = (value: string): (string | [string, SyntaxNod
           char = value[++pos]
 
         const input = value.slice(start + 1, pos) // skip '$'
-        tokens.push([input, parser.parse(input).topNode])
+        tokens.push([input, parse(input)])
         start = pos-- // backtrack and start over
       }
     }

src/parser/node.ts

@@ -0,0 +1,270 @@
+import { type Token, TokenType } from './tokenizer2'
+
+export type NodeType =
+  | 'Program'
+  | 'Block'
+
+  | 'FunctionCall'
+  | 'FunctionCallOrIdentifier'
+  | 'FunctionCallWithBlock'
+  | 'PositionalArg'
+  | 'NamedArg'
+  | 'NamedArgPrefix'
+
+  | 'FunctionDef'
+  | 'Params'
+  | 'NamedParam'
+
+  | 'Null'
+  | 'Boolean'
+  | 'Number'
+  | 'String'
+  | 'StringFragment'
+  | 'CurlyString'
+  | 'DoubleQuote'
+  | 'EscapeSeq'
+  | 'Interpolation'
+  | 'Regex'
+  | 'Identifier'
+  | 'AssignableIdentifier'
+  | 'IdentifierBeforeDot'
+  | 'Word'
+  | 'Array'
+  | 'Dict'
+  | 'Comment'
+
+  | 'BinOp'
+  | 'ConditionalOp'
+  | 'ParenExpr'
+  | 'Assign'
+  | 'CompoundAssign'
+  | 'DotGet'
+  | 'PipeExpr'
+
+  | 'IfExpr'
+  | 'ElseIfExpr'
+  | 'ElseExpr'
+  | 'WhileExpr'
+  | 'TryExpr'
+  | 'CatchExpr'
+  | 'FinallyExpr'
+  | 'Throw'
+
+  | 'Not'
+  | 'Eq'
+  | 'Modulo'
+  | 'Plus'
+  | 'Star'
+  | 'Slash'
+
+  | 'Import'
+  | 'Do'
+  | 'Underscore'
+  | 'colon'
+  | 'keyword'
+  | 'operator'
+
+// TODO: remove this when we switch from lezer
+export const operators: Record<string, any> = {
+  // Logic
+  'and': 'And',
+  'or': 'Or',
+
+  // Bitwise
+  'band': 'Band',
+  'bor': 'Bor',
+  'bxor': 'Bxor',
+  '>>>': 'Ushr',
+  '>>': 'Shr',
+  '<<': 'Shl',
+
+  // Comparison
+  '>=': 'Gte',
+  '<=': 'Lte',
+  '>': 'Gt',
+  '<': 'Lt',
+  '!=': 'Neq',
+  '==': 'EqEq',
+
+  // Compound assignment operators
+  '??=': 'NullishEq',
+  '+=': 'PlusEq',
+  '-=': 'MinusEq',
+  '*=': 'StarEq',
+  '/=': 'SlashEq',
+  '%=': 'ModuloEq',
+
+  // Nullish coalescing
+  '??': 'NullishCoalesce',
+
+  // Math
+  '*': 'Star',
+  '**': 'StarStar',
+  '=': 'Eq',
+  '/': 'Slash',
+  '+': 'Plus',
+  '-': 'Minus',
+  '%': 'Modulo',
+
+  // Dotget
+  '.': 'Dot',
+
+  // Pipe
+  '|': 'operator',
+}
+
+export class Tree {
+  constructor(public topNode: SyntaxNode) { }
+
+  get length(): number {
+    return this.topNode.to
+  }
+
+  cursor() {
+    return {
+      type: this.topNode.type,
+      from: this.topNode.from,
+      to: this.topNode.to,
+      node: this.topNode,
+    }
+  }
+
+  iterate(options: { enter: (node: SyntaxNode) => void }) {
+    const iter = (node: SyntaxNode) => {
+      for (const n of node.children) iter(n)
+      options.enter(node)
+    }
+
+    iter(this.topNode)
+  }
+}
+
+export class SyntaxNode {
+  #type: NodeType
+  #isError = false
+  from: number
+  to: number
+  parent: SyntaxNode | null
+  children: SyntaxNode[] = []
+
+  constructor(type: NodeType, from: number, to: number, parent: SyntaxNode | null = null) {
+    this.#type = type
+    this.from = from
+    this.to = to
+    this.parent = parent
+  }
+
+  static from(token: Token, parent?: SyntaxNode): SyntaxNode {
+    return new SyntaxNode(TokenType[token.type] as NodeType, token.from, token.to, parent ?? null)
+  }
+
+  get type(): { type: NodeType, name: NodeType, isError: boolean, is: (other: NodeType) => boolean } {
+    return {
+      type: this.#type,
+      name: this.#type,
+      isError: this.#isError,
+      is: (other: NodeType) => other === this.#type
+    }
+  }
+
+  set type(name: NodeType) {
+    this.#type = name
+  }
+
+  get name(): string {
+    return this.type.name
+  }
+
+  get isError(): boolean {
+    return this.#isError
+  }
+
+  set isError(err: boolean) {
+    this.#isError = err
+  }
+
+  get firstChild(): SyntaxNode | null {
+    return this.children[0] ?? null
+  }
+
+  get lastChild(): SyntaxNode | null {
+    return this.children.at(-1) ?? null
+  }
+
+  get nextSibling(): SyntaxNode | null {
+    if (!this.parent) return null
+    const siblings = this.parent.children
+    const index = siblings.indexOf(this)
+    return index >= 0 && index < siblings.length - 1 ? siblings[index + 1]! : null
+  }
+
+  get prevSibling(): SyntaxNode | null {
+    if (!this.parent) return null
+    const siblings = this.parent.children
+    const index = siblings.indexOf(this)
+    return index > 0 ? siblings[index - 1]! : null
+  }
+
+  add(node: SyntaxNode) {
+    node.parent = this
+    this.children.push(node)
+  }
+
+  push(...nodes: SyntaxNode[]): SyntaxNode {
+    nodes.forEach(child => child.parent = this)
+    this.children.push(...nodes)
+    return this
+  }
+
+  toString(): string {
+    return this.type.name
+  }
+}
+
+// Operator precedence (binding power) - higher = tighter binding
+export const precedence: Record<string, number> = {
+  // Logical
+  'or': 10,
+  'and': 20,
+
+  // Comparison
+  '==': 30,
+  '!=': 30,
+  '<': 30,
+  '>': 30,
+  '<=': 30,
+  '>=': 30,
+
+  // Nullish coalescing
+  '??': 35,
+
+  // Bitwise shifts (lower precedence than addition)
+  '<<': 37,
+  '>>': 37,
+  '>>>': 37,
+
+  // Addition/Subtraction
+  '+': 40,
+  '-': 40,
+
+  // Bitwise AND/OR/XOR (higher precedence than addition)
+  'band': 45,
+  'bor': 45,
+  'bxor': 45,
+
+  // Multiplication/Division/Modulo
+  '*': 50,
+  '/': 50,
+  '%': 50,
+
+  // Exponentiation (right-associative)
+  '**': 60,
+}
+
+export const conditionals = new Set([
+  '==', '!=', '<', '>', '<=', '>=', '??', 'and', 'or'
+])
+
+export const compounds = [
+  '??=', '+=', '-=', '*=', '/=', '%='
+]

src/parser/operatorTokenizer.ts

@@ -1,99 +0,0 @@
-import { ExternalTokenizer, InputStream } from '@lezer/lr'
-import * as terms from './shrimp.terms'
-
-type Operator = { str: string; tokenName: keyof typeof terms }
-const operators: Array<Operator> = [
-  { str: 'and', tokenName: 'And' },
-  { str: 'or', tokenName: 'Or' },
-  { str: 'band', tokenName: 'Band' },
-  { str: 'bor', tokenName: 'Bor' },
-  { str: 'bxor', tokenName: 'Bxor' },
-  { str: '>>>', tokenName: 'Ushr' },  // Must come before >>
-  { str: '>>', tokenName: 'Shr' },
-  { str: '<<', tokenName: 'Shl' },
-  { str: '>=', tokenName: 'Gte' },
-  { str: '<=', tokenName: 'Lte' },
-  { str: '!=', tokenName: 'Neq' },
-  { str: '==', tokenName: 'EqEq' },
-
-  // Compound assignment operators (must come before single-char operators)
-  { str: '??=', tokenName: 'NullishEq' },
-  { str: '+=', tokenName: 'PlusEq' },
-  { str: '-=', tokenName: 'MinusEq' },
-  { str: '*=', tokenName: 'StarEq' },
-  { str: '/=', tokenName: 'SlashEq' },
-  { str: '%=', tokenName: 'ModuloEq' },
-
-  // Nullish coalescing (must come before it could be mistaken for other tokens)
-  { str: '??', tokenName: 'NullishCoalesce' },
-
-  // Single-char operators
-  { str: '*', tokenName: 'Star' },
-  { str: '=', tokenName: 'Eq' },
-  { str: '/', tokenName: 'Slash' },
-  { str: '+', tokenName: 'Plus' },
-  { str: '-', tokenName: 'Minus' },
-  { str: '>', tokenName: 'Gt' },
-  { str: '<', tokenName: 'Lt' },
-  { str: '%', tokenName: 'Modulo' },
-]
-
-export const operatorTokenizer = new ExternalTokenizer((input: InputStream) => {
-  for (let operator of operators) {
-    if (!matchesString(input, 0, operator.str)) continue
-    const afterOpPos = operator.str.length
-    const charAfterOp = input.peek(afterOpPos)
-    if (!isWhitespace(charAfterOp)) continue
-
-    // Accept the operator token
-    const token = terms[operator.tokenName]
-    if (token === undefined) {
-      throw new Error(`Unknown token name: ${operator.tokenName}`)
-    }
-
-    input.advance(afterOpPos)
-    input.acceptToken(token)
-
-    return
-  }
-})
-
-const isWhitespace = (ch: number): boolean => {
-  return matchesChar(ch, [' ', '\t', '\n'])
-}
-
-const matchesChar = (ch: number, chars: (string | number)[]): boolean => {
-  for (const c of chars) {
-    if (typeof c === 'number') {
-      if (ch === c) {
-        return true
-      }
-    } else if (ch === c.charCodeAt(0)) {
-      return true
-    }
-  }
-  return false
-}
-
-const matchesString = (input: InputStream, pos: number, str: string): boolean => {
-  for (let i = 0; i < str.length; i++) {
-    if (input.peek(pos + i) !== str.charCodeAt(i)) {
-      return false
-    }
-  }
-  return true
-}
-
-const peek = (numChars: number, input: InputStream): string => {
-  let result = ''
-  for (let i = 0; i < numChars; i++) {
-    const ch = input.peek(i)
-    if (ch === -1) {
-      result += 'EOF'
-      break
-    } else {
-      result += String.fromCharCode(ch)
-    }
-  }
-  return result
-}

src/parser/parser2.ts

@@ -0,0 +1,991 @@
+import { CompilerError } from '#compiler/compilerError'
+import { Scanner, type Token, TokenType } from './tokenizer2'
+import { SyntaxNode, operators, precedence, conditionals, compounds } from './node'
+import { parseString } from './stringParser'
+
+const $T = TokenType
+
+// tell the dotGet searcher about builtin globals
+export const globals: string[] = []
+export const setGlobals = (newGlobals: string[] | Record<string, any>) => {
+  globals.length = 0
+  globals.push(...(Array.isArray(newGlobals) ? newGlobals : Object.keys(newGlobals)))
+}
+
+export const parse = (input: string): SyntaxNode => {
+  const parser = new Parser()
+  return parser.parse(input)
+}
+
+class Scope {
+  parent?: Scope
+  set = new Set<string>()
+
+  constructor(parent?: Scope) {
+    this.parent = parent
+
+    // no parent means this is global scope
+    if (!parent) for (const name of globals) this.add(name)
+  }
+
+  add(key: string) {
+    this.set.add(key)
+  }
+
+  has(key: string): boolean {
+    return this.set.has(key) || this.parent?.has(key) || false
+  }
+}
+
+export class Parser {
+  tokens: Token[] = []
+  pos = 0
+  inParens = 0
+  input = ''
+  scope = new Scope
+  inTestExpr = false
+
+  parse(input: string): SyntaxNode {
+    const scanner = new Scanner()
+    this.tokens = scanner.tokenize(input)
+    this.pos = 0
+    this.input = input
+    this.scope = new Scope()
+    this.inTestExpr = false
+
+    const node = new SyntaxNode('Program', 0, input.length)
+
+    while (!this.isEOF()) {
+      if (this.is($T.Newline) || this.is($T.Semicolon)) {
+        this.next()
+        continue
+      }
+
+      const prevPos = this.pos
+      const stmt = this.statement()
+      if (stmt) node.add(stmt)
+
+      if (this.pos === prevPos && !this.isEOF())
+        throw `parser didn't advance - you need to call next()\n\n       ${this.input}\n`
+    }
+
+    return node
+  }
+
+  //
+  // parse foundation nodes - statements, expressions
+  //
+
+  // statement is a line of code
+  statement(): SyntaxNode | null {
+    if (this.is($T.Comment))
+      return this.comment()
+
+    while (this.is($T.Newline) || this.is($T.Semicolon))
+      this.next()
+
+    if (this.isEOF() || this.isExprEndKeyword())
+      return null
+
+    return this.expression()
+  }
+
+  // expressions can be found in four places:
+  // 1. line of code
+  // 2. right side of assignment
+  // 3. if/while conditions
+  // 4. inside (parens)
+  expression(allowPipe = true): SyntaxNode {
+    let expr
+
+    // x = value
+    if (this.is($T.Identifier) && (
+      this.nextIs($T.Operator, '=') || compounds.some(x => this.nextIs($T.Operator, x))
+    ))
+      expr = this.assign()
+
+    // if, while, do, etc
+    else if (this.is($T.Keyword))
+      expr = this.keywords()
+
+    // dotget
+    else if (this.nextIs($T.Operator, '.'))
+      expr = this.dotGetFunctionCall()
+
+    // echo hello world
+    else if (this.is($T.Identifier) && !this.nextIs($T.Operator) && !this.nextIsExprEnd())
+      expr = this.functionCall()
+
+    // bare-function-call
+    else if (this.is($T.Identifier) && this.nextIsExprEnd())
+      expr = this.functionCallOrIdentifier()
+
+    // everything else
+    else
+      expr = this.exprWithPrecedence()
+
+    // check for destructuring
+    if (expr.type.is('Array') && this.is($T.Operator, '='))
+      return this.destructure(expr)
+
+    // check for parens function call
+    // ex: (ref my-func) my-arg
+    if (expr.type.is('ParenExpr') && !this.isExprEnd())
+      expr = this.functionCall(expr)
+
+    // if dotget is followed by binary operator, continue parsing as binary expression
+    if (expr.type.is('DotGet') && this.is($T.Operator) && !this.is($T.Operator, '|'))
+      expr = this.dotGetBinOp(expr)
+
+    // one | echo
+    if (allowPipe && this.isPipe())
+      return this.pipe(expr)
+
+    // regular
+    else
+      return expr
+  }
+
+  // piping | stuff | is | cool
+  pipe(left: SyntaxNode): SyntaxNode {
+    const canLookPastNewlines = this.inParens === 0
+    const parts: SyntaxNode[] = [left]
+
+    while (this.isPipe()) {
+      // consume newlines before pipe (only if not in parens)
+      if (canLookPastNewlines) {
+        while (this.is($T.Newline)) this.next()
+      }
+
+      const pipeOp = this.op('|')
+      pipeOp.type = 'operator'
+      parts.push(pipeOp)
+
+      // consume newlines after pipe (only if not in parens)
+      if (canLookPastNewlines) {
+        while (this.is($T.Newline)) this.next()
+      }
+
+      // parse right side - don't allow nested pipes
+      parts.push(this.expression(false))
+    }
+
+    const node = new SyntaxNode('PipeExpr', parts[0]!.from, parts.at(-1)!.to)
+    return node.push(...parts)
+  }
+
+  // Pratt parser - parses expressions with precedence climbing
+  // bp = binding precedence
+  exprWithPrecedence(minBp = 0): SyntaxNode {
+    let left = this.value()
+
+    // infix operators with precedence
+    while (this.is($T.Operator)) {
+      const op = this.current().value!
+      const bp = precedence[op]
+
+      // operator has lower precedence than required, stop
+      if (bp === undefined || bp < minBp) break
+
+      const opNode = this.op()
+
+      // right-associative operators (like **) use same bp, others use bp + 1
+      const nextMinBp = op === '**' ? bp : bp + 1
+
+      // parse right-hand side with higher precedence
+      const right = this.exprWithPrecedence(nextMinBp)
+
+      const nodeType = conditionals.has(op) ? 'ConditionalOp' : 'BinOp'
+      const node = new SyntaxNode(nodeType, left.from, right.to)
+
+      node.push(left, opNode, right)
+      left = node
+    }
+
+    return left
+  }
+
+  // if, while, do, etc
+  keywords(): SyntaxNode {
+    if (this.is($T.Keyword, 'if'))
+      return this.if()
+
+    if (this.is($T.Keyword, 'while'))
+      return this.while()
+
+    if (this.is($T.Keyword, 'do'))
+      return this.do()
+
+    if (this.is($T.Keyword, 'try'))
+      return this.try()
+
+    if (this.is($T.Keyword, 'throw'))
+      return this.throw()
+
+    if (this.is($T.Keyword, 'not'))
+      return this.not()
+
+    if (this.is($T.Keyword, 'import'))
+      return this.import()
+
+    return this.expect($T.Keyword, 'if/while/do/import') as never
+  }
+
+  // value can be an atom or a (parens that gets turned into an atom)
+  // values are used in a few places:
+  // 1. function arguments
+  // 2. array/dict members
+  // 3. binary operations
+  // 4. anywhere an expression can be used
+  value(): SyntaxNode {
+    if (this.is($T.OpenParen))
+      return this.parens()
+
+    if (this.is($T.OpenBracket))
+      return this.arrayOrDict()
+
+    // dotget
+    if (this.nextIs($T.Operator, '.'))
+      return this.dotGet()
+
+    return this.atom()
+  }
+
+  //
+  // parse specific nodes
+  //
+
+  // raw determines whether we just want the SyntaxNodes or we want to
+  // wrap them in a PositionalArg
+  arg(raw = false): SyntaxNode {
+    // 'do' is a special function arg - it doesn't need to be wrapped
+    // in parens. otherwise, args are regular value()s
+    const val = this.is($T.Keyword, 'do') ? this.do() : this.value()
+
+    if (raw) {
+      return val
+    } else {
+      const arg = new SyntaxNode('PositionalArg', val.from, val.to)
+      if (val.isError) arg.isError = true
+      arg.add(val)
+      return arg
+    }
+  }
+
+  // [ 1 2 3 ]
+  array(): SyntaxNode {
+    const open = this.expect($T.OpenBracket)
+
+    const values = []
+    while (!this.is($T.CloseBracket) && !this.isEOF()) {
+      if (this.is($T.Semicolon) || this.is($T.Newline)) {
+        this.next()
+        continue
+      }
+
+      if (this.is($T.Comment)) {
+        values.push(this.comment())
+        continue
+      }
+
+      values.push(this.value())
+    }
+
+    const close = this.expect($T.CloseBracket)
+
+    const node = new SyntaxNode('Array', open.from, close.to)
+    return node.push(...values)
+  }
+
+  // which are we dealing with? ignores leading newlines and comments
+  arrayOrDict(): SyntaxNode {
+    let peek = 1
+    let curr = this.peek(peek++)
+    let isDict = false
+
+    while (curr && curr.type !== $T.CloseBracket) {
+      // definitely a dict
+      if (curr.type === $T.NamedArgPrefix) {
+        isDict = true
+        break
+      }
+
+      // empty dict
+      if (curr.type === $T.Operator && curr.value === '=') {
+        isDict = true
+        break
+      }
+
+      // [ a = true ]
+      const next = this.peek(peek)
+      if (next?.type === $T.Operator && next.value === '=') {
+        isDict = true
+        break
+      }
+
+      // probably an array
+      if (curr.type !== $T.Comment && curr.type !== $T.Semicolon && curr.type !== $T.Newline)
+        break
+
+      curr = this.peek(peek++)
+    }
+
+    return isDict ? this.dict() : this.array()
+  }
+
+  // x = true
+  assign(): SyntaxNode {
+    const ident = this.assignableIdentifier()
+    const opToken = this.current()!
+    const op = this.op()
+    const expr = this.expression()
+
+    const node = new SyntaxNode(
+      opToken.value === '=' ? 'Assign' : 'CompoundAssign',
+      ident.from,
+      expr.to
+    )
+
+    return node.push(ident, op, expr)
+  }
+
+  // identifier used in assignment (TODO: legacy lezer quirk)
+  assignableIdentifier(): SyntaxNode {
+    const token = this.expect($T.Identifier)
+    this.scope.add(token.value!)
+    const node = SyntaxNode.from(token)
+    node.type = 'AssignableIdentifier'
+    return node
+  }
+
+  // atoms are the basic building blocks: literals, identifiers, words
+  atom(): SyntaxNode {
+    if (this.is($T.String))
+      return this.string()
+
+    if (this.isAny($T.Null, $T.Boolean, $T.Number, $T.Identifier, $T.Word, $T.Regex, $T.Underscore))
+      return SyntaxNode.from(this.next())
+
+    const next = this.next()
+    throw new CompilerError(`Unexpected token: ${TokenType[next.type]}`, next.from, next.to)
+  }
+
+  // blocks in if, do, special calls, etc
+  // `: something end`
+  //
+  // `blockNode` determines whether we return [colon, BlockNode, end] or
+  // just a list of statements like [colon, stmt1, stmt2, end]
+  block(blockNode = true): SyntaxNode[] {
+    const stmts: SyntaxNode[] = []
+    const colon = this.colon()
+
+    while (!this.isExprEndKeyword() && !this.isEOF()) {
+      const stmt = this.statement()
+      if (stmt) stmts.push(stmt)
+    }
+
+    const out = [colon]
+
+    if (blockNode) {
+      const block = new SyntaxNode('Block', stmts[0]!.from, stmts.at(-1)!.to)
+      block.push(...stmts)
+      out.push(block)
+    } else {
+      out.push(...stmts)
+    }
+
+    return out
+  }
+
+  // catch err: block
+  catch(): SyntaxNode {
+    const keyword = this.keyword('catch')
+
+    let catchVar
+    if (this.is($T.Identifier))
+      catchVar = this.identifier()
+
+    const block = this.block()
+
+    const node = new SyntaxNode('CatchExpr', keyword.from, block.at(-1)!.to)
+
+    node.push(keyword)
+    if (catchVar) node.push(catchVar)
+    return node.push(...block)
+  }
+
+  // colon
+  colon(): SyntaxNode {
+    const colon = SyntaxNode.from(this.expect($T.Colon))
+    colon.type = 'colon' // TODO lezer legacy
+    return colon
+  }
+
+  // # comment
+  comment(): SyntaxNode {
+    return SyntaxNode.from(this.expect($T.Comment))
+  }
+
+  // [ a b c ] = [ 1 2 3 ]
+  destructure(array: SyntaxNode): SyntaxNode {
+    const eq = this.op('=')
+    const val = this.expression()
+
+    for (const ident of array.children) {
+      const varName = this.input.slice(ident.from, ident.to)
+      this.scope.add(varName)
+    }
+
+    const node = new SyntaxNode('Assign', array.from, val.to)
+    return node.push(array, eq, val)
+  }
+
+  // [ a=1 b=true c='three' ]
+  dict(): SyntaxNode {
+    const open = this.expect($T.OpenBracket)
+    let isError = false
+
+    // empty dict [=] or [ = ]
+    if (this.is($T.Operator, '=') && this.nextIs($T.CloseBracket)) {
+      const _op = this.next()
+      const close = this.next()
+      return new SyntaxNode('Dict', open.from, close.to)
+    }
+
+    const values = []
+    while (!this.is($T.CloseBracket) && !this.isEOF()) {
+      if (this.is($T.Semicolon) || this.is($T.Newline)) {
+        this.next()
+        continue
+      }
+
+      if (this.is($T.Comment)) {
+        values.push(this.comment())
+        continue
+      }
+
+      // check for named arg with space after it (vs connected)
+      if (this.nextIs($T.Operator, '=')) {
+        const ident = this.identifier()
+        const op = this.op('=')
+        const prefix = new SyntaxNode('NamedArgPrefix', ident.from, op.to)
+
+        if (this.is($T.CloseBracket) || this.is($T.Semicolon) || this.is($T.Newline)) {
+          const node = new SyntaxNode('NamedArg', ident.from, op.to)
+          node.isError = true
+          isError = true
+          values.push(node.push(prefix))
+        } else {
+          const val = this.arg(true)
+          const node = new SyntaxNode('NamedArg', ident.from, val.to)
+          values.push(node.push(prefix, val))
+        }
+      } else {
+        const arg = this.is($T.NamedArgPrefix) ? this.namedArg() : this.arg()
+        if (arg.isError) isError = true
+        values.push(arg)
+      }
+    }
+
+    const close = this.expect($T.CloseBracket)
+
+    const node = new SyntaxNode('Dict', open.from, close.to)
+    node.isError = isError
+    return node.push(...values)
+  }
+
+  // FunctionDef `do x y: something end`
+  do(): SyntaxNode {
+    const doNode = this.keyword('do')
+    doNode.type = 'Do'
+    this.scope = new Scope(this.scope)
+
+    const params = []
+    while (!this.is($T.Colon) && !this.isExprEnd()) {
+      let varName = this.current().value!
+      if (varName.endsWith('=')) varName = varName.slice(0, varName.length - 1)
+      this.scope.add(varName)
+
+      let arg
+      if (this.is($T.Identifier))
+        arg = this.identifier()
+      else if (this.is($T.NamedArgPrefix))
+        arg = this.namedParam()
+      else
+        throw new CompilerError(`Expected Identifier or NamedArgPrefix, got ${TokenType[this.current().type]}`, this.current().from, this.current().to)
+
+      params.push(arg)
+    }
+
+    const block = this.block(false)
+    let catchNode, finalNode
+
+    if (this.is($T.Keyword, 'catch'))
+      catchNode = this.catch()
+
+    if (this.is($T.Keyword, 'finally'))
+      finalNode = this.finally()
+
+    const end = this.keyword('end')
+
+    let last = block.at(-1)
+    if (finalNode) last = finalNode.children.at(-1)!
+    else if (catchNode) last = catchNode.children.at(-1)!
+
+    const node = new SyntaxNode('FunctionDef', doNode.from, last!.to)
+
+    node.add(doNode)
+
+    const paramsNode = new SyntaxNode(
+      'Params',
+      params[0]?.from ?? 0,
+      params.at(-1)?.to ?? 0
+    )
+
+    if (params.length) paramsNode.push(...params)
+    node.add(paramsNode)
+
+    this.scope = this.scope.parent!
+
+    node.push(...block)
+
+    if (catchNode) node.push(catchNode)
+    if (finalNode) node.push(finalNode)
+
+    return node.push(end)
+  }
+
+  // config.path
+  dotGet(): SyntaxNode {
+    const left = this.identifier()
+    const ident = this.input.slice(left.from, left.to)
+
+    // not in scope, just return Word
+    if (!this.scope.has(ident))
+      return this.word(left)
+
+    if (left.type.is('Identifier')) left.type = 'IdentifierBeforeDot'
+
+    let parts = []
+    while (this.is($T.Operator, '.')) {
+      this.next()
+      parts.push(this.is($T.OpenParen) ? this.parens() : this.atom())
+    }
+
+    // TODO lezer legacy - we can do a flat DotGet if we remove this
+    const nodes = parts.length > 1 ? collapseDotGets(parts) : undefined
+
+    const node = new SyntaxNode('DotGet', left.from, parts.at(-1)!.to)
+    return nodes ? node.push(left, nodes!) : node.push(left, ...parts)
+  }
+
+  // continue parsing dotget/word binary operation
+  dotGetBinOp(left: SyntaxNode): SyntaxNode {
+    while (this.is($T.Operator) && !this.is($T.Operator, '|')) {
+      const op = this.current().value!
+      const bp = precedence[op]
+      if (bp === undefined) break
+
+      const opNode = this.op()
+      const right = this.exprWithPrecedence(bp + 1)
+
+      const nodeType = conditionals.has(op) ? 'ConditionalOp' : 'BinOp'
+      const node = new SyntaxNode(nodeType, left.from, right.to)
+      node.push(left, opNode, right)
+      left = node
+    }
+    return left
+  }
+
+  // dotget in a statement/expression (something.blah) or (something.blah arg1)
+  dotGetFunctionCall(): SyntaxNode {
+    const dotGet = this.dotGet()
+
+    // if followed by a binary operator (not pipe), return dotGet/Word as-is for expression parser
+    if (this.is($T.Operator) && !this.is($T.Operator, '|'))
+      return dotGet
+
+    // dotget not in scope, regular Word
+    if (dotGet.type.is('Word')) return dotGet
+
+    if (this.isExprEnd())
+      return this.functionCallOrIdentifier(dotGet)
+    else
+      return this.functionCall(dotGet)
+  }
+
+  // can be used in functions or try block
+  finally(): SyntaxNode {
+    const keyword = this.keyword('finally')
+    const block = this.block()
+    const node = new SyntaxNode('FinallyExpr', keyword.from, block.at(-1)!.to)
+
+    return node.push(keyword, ...block)
+  }
+
+  // you're lookin at it
+  functionCall(fn?: SyntaxNode): SyntaxNode {
+    const ident = fn ?? this.identifier()
+    let isError = false
+
+    const args: SyntaxNode[] = []
+    while (!this.isExprEnd()) {
+      const arg = this.is($T.NamedArgPrefix) ? this.namedArg() : this.arg()
+      if (arg.isError) isError = true
+      args.push(arg)
+    }
+
+    const node = new SyntaxNode('FunctionCall', ident.from, (args.at(-1) || ident).to)
+    node.push(ident, ...args)
+
+    if (isError) node.isError = true
+
+    if (!this.inTestExpr && this.is($T.Colon)) {
+      const block = this.block()
+      const end = this.keyword('end')
+      const blockNode = new SyntaxNode('FunctionCallWithBlock', node.from, end.to)
+      return blockNode.push(node, ...block, end)
+    }
+
+    return node
+  }
+
+  // bare identifier in an expression
+  functionCallOrIdentifier(inner?: SyntaxNode) {
+    if (!inner && this.nextIs($T.Operator, '.')) {
+      inner = this.dotGet()
+
+      // if the dotGet was just a Word, bail
+      if (inner.type.is('Word')) return inner
+    }
+
+    inner ??= this.identifier()
+
+    const wrapper = new SyntaxNode('FunctionCallOrIdentifier', inner.from, inner.to)
+    wrapper.push(inner)
+
+    if (!this.inTestExpr && this.is($T.Colon)) {
+      const block = this.block()
+      const end = this.keyword('end')
+      const node = new SyntaxNode('FunctionCallWithBlock', wrapper.from, end.to)
+      return node.push(wrapper, ...block, end)
+    }
+
+    return wrapper
+  }
+
+  // function and variable names
+  identifier(): SyntaxNode {
+    return SyntaxNode.from(this.expect($T.Identifier))
+  }
+
+  // if something: blah end
+  // if something: blah else: blah end
+  // if something: blah else if something: blah else: blah end
+  if(): SyntaxNode {
+    const ifNode = this.keyword('if')
+    const test = this.testExpr()
+    const ifBlock = this.block()
+
+    const node = new SyntaxNode('IfExpr', ifNode.from, ifBlock.at(-1)!.to)
+    node.push(ifNode, test)
+    node.push(...ifBlock)
+
+    while (this.is($T.Keyword, 'else') && this.nextIs($T.Keyword, 'if')) {
+      const elseWord = this.keyword('else')
+      const ifWord = this.keyword('if')
+      const elseIfTest = this.testExpr()
+      const elseIfBlock = this.block()
+      const elseIfNode = new SyntaxNode('ElseIfExpr', elseWord.from, elseIfBlock.at(-1)!.to)
+      elseIfNode.push(elseWord, ifWord, elseIfTest)
+      elseIfNode.push(...elseIfBlock)
+      node.push(elseIfNode)
+    }
+
+    if (this.is($T.Keyword, 'else') && this.nextIs($T.Colon)) {
+      const elseWord = this.keyword('else')
+      const elseBlock = this.block()
+      const elseNode = new SyntaxNode('ElseExpr', elseWord.from, elseBlock.at(-1)!.to)
+      elseNode.push(elseWord)
+      elseNode.push(...elseBlock)
+      node.push(elseNode)
+    }
+
+    return node.push(this.keyword('end'))
+  }
+
+  import(): SyntaxNode {
+    const keyword = this.keyword('import')
+
+    const args: SyntaxNode[] = []
+    while (!this.isExprEnd()) {
+      if (this.is($T.NamedArgPrefix)) {
+        const prefix = SyntaxNode.from(this.next())
+        const val = this.value()
+        const arg = new SyntaxNode('NamedArg', prefix.from, val.to)
+        arg.push(prefix, val)
+        args.push(arg)
+      } else {
+        args.push(this.identifier())
+      }
+    }
+
+    const node = new SyntaxNode('Import', keyword.from, args.at(-1)!.to)
+    node.add(keyword)
+    return node.push(...args)
+  }
+
+  // if, while, do, etc
+  keyword(name: string): SyntaxNode {
+    const node = SyntaxNode.from(this.expect($T.Keyword, name))
+    node.type = 'keyword' // TODO lezer legacy
+    return node
+  }
+
+  // abc= true
+  namedArg(): SyntaxNode {
+    const prefix = SyntaxNode.from(this.expect($T.NamedArgPrefix))
+
+    if (this.isExprEnd()) {
+      const node = new SyntaxNode('NamedArg', prefix.from, prefix.to)
+      node.isError = true
+      return node.push(prefix)
+    }
+
+    const val = this.arg(true)
+    const node = new SyntaxNode('NamedArg', prefix.from, val.to)
+    return node.push(prefix, val)
+  }
+
+  // abc= null|true|123|'hi'
+  namedParam(): SyntaxNode {
+    const prefix = SyntaxNode.from(this.expect($T.NamedArgPrefix))
+    const val = this.value()
+
+    if (!['Null', 'Boolean', 'Number', 'String'].includes(val.type.name))
+      throw new CompilerError(`Default value must be null, boolean, number, or string, got ${val.type.name}`, val.from, val.to)
+
+    const node = new SyntaxNode('NamedParam', prefix.from, val.to)
+    return node.push(prefix, val)
+  }
+
+  // not blah
+  not(): SyntaxNode {
+    const keyword = this.keyword('not')
+    const val = this.expression()
+    const node = new SyntaxNode('Not', keyword.from, val.to)
+    return node.push(keyword, val)
+  }
+
+  // operators like + - =
+  op(op?: string): SyntaxNode {
+    const token = op ? this.expect($T.Operator, op) : this.expect($T.Operator)
+    const name = operators[token.value!]
+    if (!name) throw new CompilerError(`Operator not registered: ${token.value!}`, token.from, token.to)
+    return new SyntaxNode(name, token.from, token.to)
+  }
+
+  // ( expressions in parens )
+  parens(): SyntaxNode {
+    this.inParens++
+    const open = this.expect($T.OpenParen)
+    const child = this.expression()
+    const close = this.expect($T.CloseParen)
+    this.inParens--
+
+    const node = new SyntaxNode('ParenExpr', open.from, close.to)
+    node.add(child)
+
+    return node
+  }
+
+  // 'hell yes' "hell no" { hell if i know }
+  string(): SyntaxNode {
+    const token = this.expect($T.String)
+    return parseString(this.input, token.from, token.to, this)
+  }
+
+  // if TEST: blah end
+  testExpr(): SyntaxNode {
+    this.inTestExpr = true
+    const expr = this.expression()
+    this.inTestExpr = false
+    return expr
+  }
+
+  // throw blah
+  throw(): SyntaxNode {
+    const keyword = this.keyword('throw')
+    const val = this.expression()
+    const node = new SyntaxNode('Throw', keyword.from, val.to)
+    return node.push(keyword, val)
+  }
+
+  // try: blah catch e: blah end
+  try(): SyntaxNode {
+    const tryNode = this.keyword('try')
+    const tryBlock = this.block()
+    let last = tryBlock.at(-1)
+    let catchNode, finalNode
+
+    if (this.is($T.Keyword, 'catch'))
+      catchNode = this.catch()
+
+    if (this.is($T.Keyword, 'finally'))
+      finalNode = this.finally()
+
+    const end = this.keyword('end')
+
+    if (finalNode) last = finalNode.children.at(-1)
+    else if (catchNode) last = catchNode.children.at(-1)
+
+    const node = new SyntaxNode('TryExpr', tryNode.from, last!.to)
+    node.push(tryNode, ...tryBlock)
+
+    if (catchNode)
+      node.push(catchNode)
+
+    if (finalNode)
+      node.push(finalNode)
+
+    return node.push(end)
+  }
+
+  // while test: blah end
+  while(): SyntaxNode {
+    const keyword = this.keyword('while')
+    const test = this.testExpr()
+    const block = this.block()
+    const end = this.keyword('end')
+
+    const node = new SyntaxNode('WhileExpr', keyword.from, end.to)
+    return node.push(keyword, test, ...block, end)
+  }
+
+  // readme.txt (when `readme` isn't in scope)
+  word(start?: SyntaxNode): SyntaxNode {
+    const parts = [start ?? this.expect($T.Word)]
+
+    while (this.is($T.Operator, '.')) {
+      this.next()
+      if (this.isAny($T.Word, $T.Identifier, $T.Number))
+        parts.push(this.next())
+    }
+
+    return new SyntaxNode('Word', parts[0]!.from, parts.at(-1)!.to)
+  }
+
+  //
+  // helpers
+  //
+
+  current(): Token {
+    return this.tokens[this.pos] || { type: TokenType.Newline, from: 0, to: 0 }
+  }
+
+  peek(offset = 1): Token | undefined {
+    return this.tokens[this.pos + offset]
+  }
+
+  // look past newlines to check for a specific token
+  peekPastNewlines(type: TokenType, value?: string): boolean {
+    let offset = 1
+    let peek = this.peek(offset)
+
+    while (peek && peek.type === $T.Newline)
+      peek = this.peek(++offset)
+
+    if (!peek || peek.type !== type) return false
+    if (value !== undefined && peek.value !== value) return false
+    return true
+  }
+
+  next(): Token {
+    const token = this.current()
+    this.pos++
+    return token
+  }
+
+  is(type: TokenType, value?: string): boolean {
+    const token = this.current()
+    if (!token || token.type !== type) return false
+    if (value !== undefined && token.value !== value) return false
+    return true
+  }
+
+  isAny(...type: TokenType[]): boolean {
+    return type.some(x => this.is(x))
+  }
+
+  nextIs(type: TokenType, value?: string): boolean {
+    const token = this.peek()
+    if (!token || token.type !== type) return false
+    if (value !== undefined && token.value !== value) return false
+    return true
+  }
+
+  nextIsAny(...type: TokenType[]): boolean {
+    return type.some(x => this.nextIs(x))
+  }
+
+  isExprEnd(): boolean {
+    return this.isAny($T.Colon, $T.Semicolon, $T.Newline, $T.CloseParen, $T.CloseBracket) ||
+      this.is($T.Operator, '|') ||
+      this.isExprEndKeyword() || !this.current()
+  }
+
+  nextIsExprEnd(): boolean {
+    // pipes act like expression end for function arg parsing
+    if (this.nextIs($T.Operator, '|'))
+      return true
+
+    return this.nextIsAny($T.Colon, $T.Semicolon, $T.Newline, $T.CloseBracket, $T.CloseParen) ||
+      this.nextIs($T.Keyword, 'end') || this.nextIs($T.Keyword, 'else') ||
+      this.nextIs($T.Keyword, 'catch') || this.nextIs($T.Keyword, 'finally') ||
+      !this.peek()
+  }
+
+  isExprEndKeyword(): boolean {
+    return this.is($T.Keyword, 'end') || this.is($T.Keyword, 'else') ||
+      this.is($T.Keyword, 'catch') || this.is($T.Keyword, 'finally')
+  }
+
+  isPipe(): boolean {
+    // inside parens, only look for pipes on same line (don't look past newlines)
+    const canLookPastNewlines = this.inParens === 0
+
+    return this.is($T.Operator, '|') ||
+      (canLookPastNewlines && this.peekPastNewlines($T.Operator, '|'))
+  }
+
+  expect(type: TokenType, value?: string): Token | never {
+    if (!this.is(type, value)) {
+      const token = this.current()
+      throw new CompilerError(`Expected ${TokenType[type]}${value ? ` "${value}"` : ''}, got ${TokenType[token?.type || 0]}${token?.value ? ` "${token.value}"` : ''} at position ${this.pos}`, token.from, token.to)
+    }
+    return this.next()
+  }
+
+  isEOF(): boolean {
+    return this.pos >= this.tokens.length
+  }
+}
+
+// TODO lezer legacy
+function collapseDotGets(origNodes: SyntaxNode[]): SyntaxNode {
+  const nodes = [...origNodes]
+  let right = nodes.pop()!
+
+  while (nodes.length > 0) {
+    const left = nodes.pop()!
+
+    if (left.type.is('Identifier')) left.type = 'IdentifierBeforeDot'
+
+    const dot = new SyntaxNode("DotGet", left.from, right.to)
+    dot.push(left, right)
+
+    right = dot
+  }
+
+  return right
+}

src/parser/parserScopeContext.ts

@@ -1,129 +0,0 @@
-import { ContextTracker, InputStream } from '@lezer/lr'
-import * as terms from './shrimp.terms'
-
-export class Scope {
-  constructor(public parent: Scope | null, public vars = new Set<string>()) { }
-
-  has(name: string): boolean {
-    return this.vars.has(name) || (this.parent?.has(name) ?? false)
-  }
-
-  hash(): number {
-    let h = 0
-    for (const name of this.vars) {
-      for (let i = 0; i < name.length; i++) {
-        h = (h << 5) - h + name.charCodeAt(i)
-        h |= 0
-      }
-    }
-    if (this.parent) {
-      h = (h << 5) - h + this.parent.hash()
-      h |= 0
-    }
-    return h
-  }
-
-  // Static methods that return new Scopes (immutable operations)
-
-  static add(scope: Scope, ...names: string[]): Scope {
-    const newVars = new Set(scope.vars)
-    names.forEach((name) => newVars.add(name))
-    return new Scope(scope.parent, newVars)
-  }
-
-  push(): Scope {
-    return new Scope(this, new Set())
-  }
-
-  pop(): Scope {
-    return this.parent ?? this
-  }
-}
-
-// Tracker context that combines Scope with temporary pending identifiers
-class TrackerContext {
-  constructor(public scope: Scope, public pendingIds: string[] = []) { }
-}
-
-// Extract identifier text from input stream
-const readIdentifierText = (input: InputStream, start: number, end: number): string => {
-  let text = ''
-  for (let i = start; i < end; i++) {
-    const offset = i - input.pos
-    const ch = input.peek(offset)
-    if (ch === -1) break
-    text += String.fromCharCode(ch)
-  }
-  return text
-}
-
-let inParams = false
-
-export const trackScope = new ContextTracker<TrackerContext>({
-  start: new TrackerContext(new Scope(null, new Set())),
-
-  shift(context, term, stack, input) {
-    if (term == terms.Do) inParams = true
-
-    if (term === terms.AssignableIdentifier) {
-      const text = readIdentifierText(input, input.pos, stack.pos)
-      return new TrackerContext(Scope.add(context.scope, text), context.pendingIds)
-    }
-
-    if (inParams && term === terms.Identifier) {
-      const text = readIdentifierText(input, input.pos, stack.pos)
-      return new TrackerContext(context.scope, [...context.pendingIds, text])
-    }
-
-    // Track identifiers in array destructuring: [ a b ] = ...
-    if (!inParams && term === terms.Identifier && isArrayDestructuring(input)) {
-      const text = readIdentifierText(input, input.pos, stack.pos)
-      return new TrackerContext(Scope.add(context.scope, text), context.pendingIds)
-    }
-
-    return context
-  },
-
-  reduce(context, term) {
-    if (term === terms.Params) {
-      inParams = false
-      let newScope = context.scope.push()
-      if (context.pendingIds.length > 0) {
-        newScope = Scope.add(newScope, ...context.pendingIds)
-      }
-      return new TrackerContext(newScope, [])
-    }
-
-    // Pop scope when exiting function
-    if (term === terms.FunctionDef) {
-      return new TrackerContext(context.scope.pop(), [])
-    }
-
-    return context
-  },
-
-  hash: (context) => context.scope.hash(),
-})
-
-// Check if we're parsing array destructuring: [ a b ] = ...
-const isArrayDestructuring = (input: InputStream): boolean => {
-  let pos = 0
-
-  // Find closing bracket
-  while (pos < 200 && input.peek(pos) !== 93 /* ] */) {
-    if (input.peek(pos) === -1) return false // EOF
-    pos++
-  }
-
-  if (input.peek(pos) !== 93 /* ] */) return false
-  pos++
-
-  // Skip whitespace
-  while (input.peek(pos) === 32 /* space */ ||
-    input.peek(pos) === 9 /* tab */ ||
-    input.peek(pos) === 10 /* \n */) {
-    pos++
-  }
-
-  return input.peek(pos) === 61 /* = */
-}

src/parser/shrimp.grammar

@@ -1,293 +0,0 @@
-@external propSource highlighting from "./highlight"
-
-@context trackScope from "./parserScopeContext"
-
-@skip { space | Comment }
-
-@top Program { item* }
-
-@external tokens operatorTokenizer from "./operatorTokenizer" { Star, Slash, Plus, Minus, And, Or, Eq, EqEq, Neq, Lt, Lte, Gt, Gte, Modulo, PlusEq, MinusEq, StarEq, SlashEq, ModuloEq, Band, Bor, Bxor, Shl, Shr, Ushr, NullishCoalesce, NullishEq }
-
-@tokens {
-  @precedence { Number Regex }
-
-  StringFragment { !['\\$]+ }
-  DoubleQuote { '"' !["]* '"' }
-  NamedArgPrefix { $[a-z] $[a-z0-9-]* "=" }
-  Number {
-    ("-" | "+")? "0x" $[0-9a-fA-F]+ |
-    ("-" | "+")? "0b" $[01]+ |
-    ("-" | "+")? "0o" $[0-7]+ |
-    ("-" | "+")? $[0-9]+ ("_"? $[0-9]+)* ('.' $[0-9]+ ("_"? $[0-9]+)*)?
-  }
-  Boolean { "true" | "false" }
-  semicolon { ";" }
-  eof { @eof }
-  space { " " | "\t" }
-  Comment { "#" ![\n]* }
-  leftParen { "(" }
-  rightParen { ")" }
-  colon[closedBy="end", @name="colon"] { ":" }
-  Underscore { "_" }
-  Dollar { "$" }
-  Regex { "//" (![/\\\n[] | "\\" ![\n] | "[" (![\n\\\]] | "\\" ![\n])* "]")+ ("//" $[gimsuy]*)? } // Stolen from the lezer JavaScript grammar
-  "|"[@name=operator]
-}
-
-newlineOrSemicolon { newline | semicolon }
-
-end { @specialize[@name=keyword]<Identifier, "end"> }
-while { @specialize[@name=keyword]<Identifier, "while"> }
-if { @specialize[@name=keyword]<Identifier, "if"> }
-else { @specialize[@name=keyword]<Identifier, "else"> }
-try { @specialize[@name=keyword]<Identifier, "try"> }
-catch { @specialize[@name=keyword]<Identifier, "catch"> }
-finally { @specialize[@name=keyword]<Identifier, "finally"> }
-throw { @specialize[@name=keyword]<Identifier, "throw"> }
-import { @specialize[@name=keyword]<Identifier, "import"> }
-null { @specialize[@name=Null]<Identifier, "null"> }
-
-@external tokens tokenizer from "./tokenizer" { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, CurlyString }
-@external tokens pipeStartsLineTokenizer from "./tokenizer" { newline, pipeStartsLine }
-@external specialize {Identifier} specializeKeyword from "./tokenizer" { Do }
-
-@precedence {
-  pipe @left,
-  or @left,
-  and @left,
-  nullish @left,
-  comparison @left,
-  multiplicative @left,
-  additive @left,
-  bitwise @left,
-  call,
-  functionWithNewlines
-}
-
-item {
-  consumeToTerminator newlineOrSemicolon |
-  consumeToTerminator eof |
-  newlineOrSemicolon // allow blank lines
-}
-
-consumeToTerminator {
-  PipeExpr |
-  WhileExpr |
-  FunctionCallWithBlock |
-  ambiguousFunctionCall |
-  TryExpr |
-  Throw |
-  Import |
-  IfExpr |
-  FunctionDef |
-  CompoundAssign |
-  Assign |
-  BinOp |
-  ConditionalOp |
-  expressionWithoutIdentifier
-}
-
-PipeExpr {
-  pipeOperand (!pipe (pipeStartsLine? "|") newlineOrSemicolon* pipeOperand)+
-}
-
-pipeOperand {
-  consumeToTerminator
-}
-
-WhileExpr {
-  while (ConditionalOp | expression) colon Block end
-}
-
-Block {
-  consumeToTerminator | newlineOrSemicolon block
-}
-
-FunctionCallWithBlock {
-  ambiguousFunctionCall colon Block CatchExpr? FinallyExpr? end
-}
-
-FunctionCallOrIdentifier {
-  DotGet | Identifier
-}
-
-ambiguousFunctionCall {
-  FunctionCall | FunctionCallOrIdentifier
-}
-
-FunctionCall {
-  (DotGet | Identifier | ParenExpr) arg+
-}
-
-arg {
-  PositionalArg | NamedArg
-}
-
-PositionalArg {
-  expression | FunctionDef | Underscore
-}
-
-NamedArg {
-  NamedArgPrefix (expression | FunctionDef | Underscore)
-}
-
-FunctionDef {
-  Do Params colon (consumeToTerminator | newlineOrSemicolon block) CatchExpr? FinallyExpr? end
-}
-
-ifTest {
-  ConditionalOp | expression | FunctionCall
-}
-
-IfExpr {
-  if ifTest colon Block ElseIfExpr* ElseExpr? end
-}
-
-ElseIfExpr {
-  else if ifTest colon Block
-}
-
-ElseExpr {
-  else colon Block
-}
-
-TryExpr {
-  try colon Block CatchExpr? FinallyExpr? end
-}
-
-CatchExpr {
-  catch Identifier colon Block
-}
-
-FinallyExpr {
-  finally colon Block
-}
-
-Throw {
-  throw (BinOp | ConditionalOp | expression)
-}
-
-// this has to be in the parse tree so the scope tracker can use it
-Import { 
-  import NamedArg* Identifier+ NamedArg*
-}
-
-ConditionalOp {
-  expression !comparison EqEq expression |
-  expression !comparison Neq expression |
-  expression !comparison Lt expression |
-  expression !comparison Lte expression |
-  expression !comparison Gt expression |
-  expression !comparison Gte expression |
-  (expression | ConditionalOp) !and And (expression | ConditionalOp) |
-  (expression | ConditionalOp) !or Or (expression | ConditionalOp) |
-  (expression | ConditionalOp) !nullish NullishCoalesce (expression | ConditionalOp)
-}
-
-Params {
-  Identifier* NamedParam*
-}
-
-NamedParam {
-  NamedArgPrefix (String | Number | Boolean | null)
-}
-
-Assign {
-  (AssignableIdentifier | Array) Eq consumeToTerminator
-}
-
-CompoundAssign {
-  AssignableIdentifier (PlusEq | MinusEq | StarEq | SlashEq | ModuloEq | NullishEq) consumeToTerminator
-}
-
-BinOp {
-  expression !multiplicative Modulo expression |
-  (expression | BinOp) !multiplicative Star (expression | BinOp) |
-  (expression | BinOp) !multiplicative Slash (expression | BinOp) |
-  (expression | BinOp) !additive Plus (expression | BinOp) |
-  (expression | BinOp) !additive Minus (expression | BinOp) |
-  (expression | BinOp) !bitwise Band (expression | BinOp) |
-  (expression | BinOp) !bitwise Bor (expression | BinOp) |
-  (expression | BinOp) !bitwise Bxor (expression | BinOp) |
-  (expression | BinOp) !bitwise Shl (expression | BinOp) |
-  (expression | BinOp) !bitwise Shr (expression | BinOp) |
-  (expression | BinOp) !bitwise Ushr (expression | BinOp)
-}
-
-ParenExpr {
-  leftParen newlineOrSemicolon* (
-    FunctionCallWithNewlines |
-    IfExpr |
-    ambiguousFunctionCall |
-    BinOp newlineOrSemicolon* |
-    expressionWithoutIdentifier |
-    ConditionalOp newlineOrSemicolon* |
-    PipeExpr |
-    FunctionDef
-  )
-  rightParen
-}
-
-FunctionCallWithNewlines[@name=FunctionCall] {
-  (DotGet | Identifier | ParenExpr) newlineOrSemicolon+ arg !functionWithNewlines (newlineOrSemicolon+ arg)* newlineOrSemicolon*
-}
-
-expression {
-  expressionWithoutIdentifier | DotGet | Identifier
-}
-
-
-@local tokens {
-  dot { "." }
-}
-
-@skip {} {
-  DotGet {
-    IdentifierBeforeDot dot (DotGet | Number | Identifier | ParenExpr) |
-    Dollar dot (DotGet | Number | Identifier | ParenExpr)
-  }
-
-  String { 
-    "'" stringContent* "'" | CurlyString | DoubleQuote
-  }
-}
-
-stringContent {
-  StringFragment |
-  Interpolation |
-  EscapeSeq
-}
-
-Interpolation {
-  "$" FunctionCallOrIdentifier |
-  "$" ParenExpr
-}
-
-EscapeSeq {
-  "\\" ("$" | "n" | "t" | "r" | "\\" | "'")
-}
-
-Dict {
-  "[=]" |
-  "[" newlineOrSemicolon* NamedArg (newlineOrSemicolon | NamedArg)* "]"
-}
-
-Array {
-  "[" newlineOrSemicolon* (expression (newlineOrSemicolon | expression)*)? "]"
-}
-
-// We need expressionWithoutIdentifier to avoid conflicts in consumeToTerminator.
-// Without this, when parsing "my-var" at statement level, the parser can't decide:
-//   - ambiguousFunctionCall → FunctionCallOrIdentifier → Identifier
-//   - expression → Identifier
-// Both want the same Identifier token! So we use expressionWithoutIdentifier
-// to remove Identifier from the second path, forcing standalone identifiers
-// to go through ambiguousFunctionCall (which is what we want semantically).
-// Yes, it is annoying and I gave up trying to use GLR to fix it.
-expressionWithoutIdentifier {
-  ParenExpr | Word | String | Number | Boolean | Regex | Dict | Array | null
-}
-
-block {
-  (consumeToTerminator? newlineOrSemicolon)*
-}

src/parser/shrimp.grammar.d.ts

@@ -1,4 +0,0 @@
-declare module '*.grammar' {
-  const content: string
-  export default content
-}

src/parser/shrimp.terms.ts

@@ -1,81 +0,0 @@
-// This file was generated by lezer-generator. You probably shouldn't edit it.
-export const
-  Star = 1,
-  Slash = 2,
-  Plus = 3,
-  Minus = 4,
-  And = 5,
-  Or = 6,
-  Eq = 7,
-  EqEq = 8,
-  Neq = 9,
-  Lt = 10,
-  Lte = 11,
-  Gt = 12,
-  Gte = 13,
-  Modulo = 14,
-  PlusEq = 15,
-  MinusEq = 16,
-  StarEq = 17,
-  SlashEq = 18,
-  ModuloEq = 19,
-  Band = 20,
-  Bor = 21,
-  Bxor = 22,
-  Shl = 23,
-  Shr = 24,
-  Ushr = 25,
-  NullishCoalesce = 26,
-  NullishEq = 27,
-  Identifier = 28,
-  AssignableIdentifier = 29,
-  Word = 30,
-  IdentifierBeforeDot = 31,
-  CurlyString = 32,
-  newline = 101,
-  pipeStartsLine = 102,
-  Do = 33,
-  Comment = 34,
-  Program = 35,
-  PipeExpr = 36,
-  WhileExpr = 38,
-  keyword = 84,
-  ConditionalOp = 40,
-  ParenExpr = 41,
-  FunctionCallWithNewlines = 42,
-  DotGet = 43,
-  Number = 44,
-  Dollar = 45,
-  PositionalArg = 46,
-  FunctionDef = 47,
-  Params = 48,
-  NamedParam = 49,
-  NamedArgPrefix = 50,
-  String = 51,
-  StringFragment = 52,
-  Interpolation = 53,
-  FunctionCallOrIdentifier = 54,
-  EscapeSeq = 55,
-  DoubleQuote = 56,
-  Boolean = 57,
-  Null = 58,
-  colon = 59,
-  CatchExpr = 60,
-  Block = 62,
-  FinallyExpr = 63,
-  Underscore = 66,
-  NamedArg = 67,
-  IfExpr = 68,
-  FunctionCall = 70,
-  ElseIfExpr = 71,
-  ElseExpr = 73,
-  BinOp = 74,
-  Regex = 75,
-  Dict = 76,
-  Array = 77,
-  FunctionCallWithBlock = 78,
-  TryExpr = 79,
-  Throw = 81,
-  Import = 83,
-  CompoundAssign = 85,
-  Assign = 86

src/parser/shrimp.ts

@@ -1,27 +0,0 @@
-// This file was generated by lezer-generator. You probably shouldn't edit it.
-import {LRParser, LocalTokenGroup} from "@lezer/lr"
-import {operatorTokenizer} from "./operatorTokenizer"
-import {tokenizer, pipeStartsLineTokenizer, specializeKeyword} from "./tokenizer"
-import {trackScope} from "./parserScopeContext"
-import {highlighting} from "./highlight"
-const spec_Identifier = {__proto__:null,while:78, null:116, catch:122, finally:128, end:130, if:138, else:144, try:160, throw:164, import:168}
-export const parser = LRParser.deserialize({
-  version: 14,
-  states: "?[QYQ!SOOOOQ!Q'#Ek'#EkO!sO!bO'#DXO%kQ!TO'#DdO&UOSO'#DaOOQ!R'#Da'#DaO)SQ!TO'#EnOOQ!Q'#E{'#E{O)pQRO'#DxO+xQ!TO'#EjO,fQ!SO'#DVOOQ!R'#Dz'#DzO/WQ!SO'#D{OOQ!R'#En'#EnO/_Q!TO'#EnO1cQ!TO'#EmO2qQ!TO'#EjO3OQRO'#ETOOQ!Q'#Ej'#EjO3gQ!SO'#EjO3nQrO'#EiOOQ!Q'#Ei'#EiOOQ!Q'#EV'#EVQYQ!SOOO4PQbO'#D]O4[QbO'#DrO5YQbO'#DSO6WQQO'#D}O5YQbO'#EPO6]QbO'#ERO6eObO,59sOOQ!Q'#D['#D[O6vQbO'#DqOOQ!Q'#Eq'#EqOOQ!Q'#E_'#E_O7QQ!SO,5:`OOQ!R'#Em'#EmO8QQbO'#DcO8`QWO'#DeOOOO'#Es'#EsOOOO'#E['#E[O8tOSO,59{OOQ!R,59{,59{O5YQbO,5:dO5YQbO,5:dO5YQbO,5:dO5YQbO,5:dO5YQbO,59pO5YQbO,59pO5YQbO,59pO5YQbO,59pOOQ!Q'#EX'#EXO,fQ!SO,59qO9SQ!TO'#DdO9^Q!TO'#EnO9hQsO,59qO9uQQO,59qO9zQrO,59qO:VQrO,59qO:eQsO,59qO;TQsO,59qO;[QrO'#DQO;dQ!SO,5:gO;kQrO,5:fOOQ!R,5:g,5:gO;yQ!SO,5:gO<WQbO,5:pO<WQbO,5:oOYQ!SO,5:hO=kQ!SO,59lOOQ!Q,5;T,5;TOYQ!SO'#EWO>]QQO'#EWOOQ!Q-E8T-E8TOOQ!Q'#EY'#EYO>bQbO'#D^O>mQbO'#D_OOQO'#EZ'#EZO>eQQO'#D^O?RQQO,59wO?WQcO'#EmO@TQRO'#EzOAQQRO'#EzOOQO'#Ez'#EzOAXQQO,5:^OA^QRO,59nOAeQRO,59nOYQ!SO,5:iOAsQ!TO,5:kOCXQ!TO,5:kOC{Q!TO,5:kODYQ!SO,5:mOOQ!Q'#Ec'#EcO6]QbO,5:mOOQ!R1G/_1G/_OOQ!Q,5:],5:]OOQ!Q-E8]-E8]OOOO'#Dd'#DdOOOO,59},59}OOOO,5:P,5:POOOO-E8Y-E8YOOQ!R1G/g1G/gOOQ!R1G0O1G0OOF_Q!TO1G0OOFiQ!TO1G0OOG}Q!TO1G0OOHXQ!TO1G0OOHfQ!TO1G0OOOQ!R1G/[1G/[OI}Q!TO1G/[OJUQ!TO1G/[OJ]Q!TO1G/[OKbQ!TO1G/[OJdQ!TO1G/[OOQ!Q-E8V-E8VOKxQsO1G/]OLVQQO1G/]OL[QrO1G/]OLgQrO1G/]OLuQsO1G/]OL|QsO1G/]OMTQ!SO,59rOM_QrO1G/]OOQ!R1G/]1G/]OMjQrO1G0QOOQ!R1G0R1G0ROMxQ!SO1G0ROOQp'#Ea'#EaOMjQrO1G0QOOQ!R1G0Q1G0QOOQ!Q'#Eb'#EbOMxQ!SO1G0RONVQ!SO1G0[ONwQ!SO1G0ZO! iQ!SO'#DlO! }Q!SO'#DlO!!_QbO1G0SOOQ!Q-E8U-E8UOYQ!SO,5:rOOQ!Q,5:r,5:rOYQ!SO,5:rOOQ!Q-E8W-E8WO!!jQQO,59xOOQO,59y,59yOOQO-E8X-E8XOYQ!SO1G/cOYQ!SO1G/xOYQ!SO1G/YO!!rQbO1G0TO!!}Q!SO1G0XO!#rQ!SO1G0XOOQ!Q-E8a-E8aO!#yQrO7+$wOOQ!R7+$w7+$wO!$UQrO1G/^O!$aQrO7+%lOOQ!R7+%l7+%lO!$oQ!SO7+%mOOQ!R7+%m7+%mOOQp-E8_-E8_OOQ!Q-E8`-E8`OOQ!Q'#E]'#E]O!$|QrO'#E]O!%[Q!SO'#EyOOQ`,5:W,5:WO!%lQbO'#DjO!%qQQO'#DmOOQ!Q7+%n7+%nO!%vQbO7+%nO!%{QbO7+%nOOQ!Q1G0^1G0^OYQ!SO1G0^O!&TQ!SO7+$}O!&fQ!SO7+$}O!&sQbO7+%dO!&{QbO7+$tOOQ!Q7+%o7+%oO!'QQbO7+%oO!'VQbO7+%oO!'_Q!SO7+%sOOQ!R<<Hc<<HcO!(SQ!SO7+$xO!(aQrO7+$xOOQ!R<<IW<<IWOOQ!R<<IX<<IXOOQ!Q,5:w,5:wOOQ!Q-E8Z-E8ZO!(lQQO,5:UOYQ!SO,5:XOOQ!Q<<IY<<IYO!(qQbO<<IYOOQ!Q7+%x7+%xOOQ!Q<<Hi<<HiO!(vQbO<<HiO!({QbO<<HiO!)TQbO<<HiOOQ`'#E`'#E`O!)`QbO<<IOO!)hQbO'#DwOOQ!Q<<IO<<IOO!)pQbO<<IOOOQ!Q<<H`<<H`OOQ!Q<<IZ<<IZO!)uQbO<<IZOOQp,5:x,5:xO!)zQ!SO<<HdOOQp-E8[-E8[OYQ!SO1G/pOOQ`1G/s1G/sOOQ!QAN>tAN>tOOQ!QAN>TAN>TO!*XQbOAN>TO!*^QbOAN>TOOQ`-E8^-E8^OOQ!QAN>jAN>jO!*fQbOAN>jO4[QbO,5:aOYQ!SO,5:cOOQ!QAN>uAN>uPMTQ!SO'#EXOOQ`7+%[7+%[OOQ!QG23oG23oO!*kQbOG23oP!)kQbO'#DuOOQ!QG24UG24UO!*pQQO1G/{OOQ`1G/}1G/}OOQ!QLD)ZLD)ZOYQ!SO7+%gOOQ`<<IR<<IRO!*uObO,59sO!+WO!bO'#DX",
-  stateData: "!+`~O#[OSrOS~OlROmaOn]OoQOpTOqhOwjO|]O}QO!YTO!Z]O![]O!giO!m]O!rkO!tlO!vmO#XPO#`PO#cYO#fSO#qZO#r[O~O#dnO~OltOn]OoQOpTOqhO|]O}QO!SpO!YTO!Z]O![]O!doO!m]O#cYO#fSO#qZO#r[OP#aXQ#aXR#aXS#aXT#aXU#aXW#aXX#aXY#aXZ#aX[#aX]#aX^#aXd#aXe#aXf#aXg#aXh#aXi#aXj#aXu!WX!]!WX#Y!WX#p!WX~O#X!WX#`!WX#t!WX!_!WX!b!WX!c!WX!j!WX~P!xO!UwO#fzO#huO#ivO~OltOn]OoQOpTOqhO|]O}QO!SpO!YTO!Z]O![]O!doO!m]O#cYO#fSO#qZO#r[OP#bXQ#bXR#bXS#bXT#bXU#bXW#bXX#bXY#bXZ#bX[#bX]#bX^#bXd#bXe#bXf#bXg#bXh#bXi#bXj#bXu#bX#Y#bX#p#bX~O#X#bX#`#bX#t#bX!]#bX!_#bX!b#bX!c#bX!j#bX~P&dOP|OQ|OR}OS}OT!QOU!ROW!POX!POY!POZ!PO[!PO]!PO^{Od!OOe!OOf!OOg!OOh!OOi!OOj!SO~OP|OQ|OR}OS}Od!OOe!OOf!OOg!OOh!OOi!OOu#^X#Y#^X~O#X#^X#`#^X#t#^X!_#^X!b#^X!c#^X#p#^X!j#^X~P+QOl!VOmaOn]OoQOpTOqhOwjO|]O}QO!YTO!Z]O![]O!giO!m]O!rkO!tlO!vmO#XPO#`PO#cYO#fSO#qZO#r[O~OltOn]OoQOpTO|]O}QO!SpO!YTO!Z]O![]O!m]O#XPO#`PO#cYO#fSO#qZO#r[O~O#s!bO~P.POV!dO#X#bX#`#bX#t#bX!_#bX!b#bX!c#bX!j#bX~P'iOP#aXQ#aXR#aXS#aXT#aXU#aXW#aXX#aXY#aXZ#aX[#aX]#aX^#aXd#aXe#aXf#aXg#aXh#aXi#aXj#aXu#^X#Y#^X~O#X#^X#`#^X#t#^X!_#^X!b#^X!c#^X#p#^X!j#^X~P/{Ou#^X#X#^X#Y#^X#`#^X#t#^X!_#^X!b#^X!c#^X#p#^X!j#^X~OT!QOU!ROj!SO~P2POV!dO_!eO`!eOa!eOb!eOc!eOk!eO~O!]!fO~P2POu!iO#XPO#Y!jO#`PO#t!hO~Ol!lO!S!nO!]!QP~Ol!rOn]OoQOpTO|]O}QO!YTO!Z]O![]O!m]O#cYO#fSO#qZO#r[O~OltOn]OoQOpTO|]O}QO!YTO!Z]O![]O!m]O#cYO#fSO#qZO#r[O~O!]!yO~Ol!lO!SpO~Ol#QOoQO|#QO}QO#cYO~OqhO!d#RO~P5YOqhO!SpO!doOu!ha!]!ha#X!ha#Y!ha#`!ha#t!ha#p!ha!_!ha!b!ha!c!ha!j!ha~P5YOl#TOo&PO}&PO#cYO~O#f#VO#h#VO#i#VO#j#VO#k#VO#l#VO~O!UwO#f#XO#huO#ivO~O#XPO#`PO~P!xO#XPO#`PO~P&dO#XPO#`PO#p#oO~P+QO#p#oO~O#p#oOu#^X#Y#^X~O!]!fO#p#oOu#^X#Y#^X~O#p#oO~P/{OT!QOU!ROj!SO#XPO#`POu#^X#Y#^X~O#p#oO~P:lOu!iO#Y!jO~O#s#qO~P.PO!SpO#XPO#`PO#s#uO~O#XPO#`PO#s#qO~P5YOlROmaOn]OoQOpTOqhOwjO|]O}QO!YTO!Z]O![]O!giO!m]O!rkO!tlO!vmO#cYO#fSO#qZO#r[O~Ou!iO#Y!jO#Xta#`ta#tta#pta!_ta!bta!cta!jta~Ou$QO~Ol!lO!S!nO!]!QX~OpTO|$TO!YTO!Z$TO![$TO#fSO~O!]$VO~OqhO!SpO!doOT#aXU#aXW#aXX#aXY#aXZ#aX[#aX]#aXj#aX!]#aX~P5YOT!QOU!ROj!SO!]#nX~OT!QOU!ROW!POX!POY!POZ!PO[!PO]!POj!SO~O!]#nX~P@cO!]$WO~O!]$XO~P@cOT!QOU!ROj!SO!]$XO~Ou!sa#X!sa#Y!sa#`!sa#t!sa!_!sa!b!sa!c!sa#p!sa!j!sa~P)pOu!sa#X!sa#Y!sa#`!sa#t!sa!_!sa!b!sa!c!sa#p!sa!j!sa~OP|OQ|OR}OS}Od!OOe!OOf!OOg!OOh!OOi!OO~PBgOT!QOU!ROj!SO~PBgOl!lO!SpOu!ua#X!ua#Y!ua#`!ua#t!ua!_!ua!b!ua!c!ua#p!ua!j!ua~O^{OR!liS!lid!lie!lif!lig!lih!lii!liu!li#X!li#Y!li#`!li#t!li#p!li!_!li!b!li!c!li!j!li~OP!liQ!li~PEQOP|OQ|O~PEQOP|OQ|Od!lie!lif!lig!lih!lii!liu!li#X!li#Y!li#`!li#t!li#p!li!_!li!b!li!c!li!j!li~OR!liS!li~PFsOR}OS}O^{O~PFsOR}OS}O~PFsOW!POX!POY!POZ!PO[!PO]!POTxijxiuxi#Xxi#Yxi#`xi#txi#pxi!]xi!_xi!bxi!cxi!jxi~OU!RO~PHpOU!RO~PISOUxi~PHpOT!QOU!ROjxiuxi#Xxi#Yxi#`xi#txi#pxi!]xi!_xi!bxi!cxi!jxi~OW!POX!POY!POZ!PO[!PO]!PO~PJdO#XPO#`PO#p$_O~P+QO#p$_O~O#p$_Ou#^X#Y#^X~O!]!fO#p$_Ou#^X#Y#^X~O#p$_O~P/{O#p$_O~P:lOqhO!doO~P.PO#XPO#`PO#p$_O~O!SpO#XPO#`PO#s$bO~O#XPO#`PO#s$dO~P5YOu!iO#Y!jO#X!xi#`!xi#t!xi!_!xi!b!xi!c!xi#p!xi!j!xi~Ou!iO#Y!jO#X!wi#`!wi#t!wi!_!wi!b!wi!c!wi#p!wi!j!wi~Ou!iO#Y!jO!_!`X!b!`X!c!`X!j!`X~O!_#mP!b#mP!c#mP!j#mP~PYO!_$kO!b$lO!c$mO~O!S!nO!]!Qa~O!_$kO!b$lO!c$vO~O!SpOu!ui#X!ui#Y!ui#`!ui#t!ui!_!ui!b!ui!c!ui#p!ui!j!ui~Ol!lO~P!!}O#XPO#`PO#p$zO~O#XPO#`PO#pzi~O!SpO#XPO#`PO#s$}O~O#XPO#`PO#s%OO~P5YOu!iO#XPO#Y!jO#`PO~O!_#mX!b#mX!c#mX!j#mX~PYOl%RO~O!]%SO~O!c%TO~O!b$lO!c%TO~Ou!iO!_$kO!b$lO!c%WO#Y!jO~O!_#mP!b#mP!c#mP~PYO!c%_O!j%^O~O!c%aO~O!c%bO~O!b$lO!c%bO~O!SpOu!uq#X!uq#Y!uq#`!uq#t!uq!_!uq!b!uq!c!uq#p!uq!j!uq~OqhO!doO#pzq~P.PO#XPO#`PO#pzq~O!]%gO~O!c%iO~O!c%jO~O!b$lO!c%jO~O!_$kO!b$lO!c%jO~O!c%nO!j%^O~O!]%qO!g%pO~O!c%nO~O!c%rO~OqhO!doO#pzy~P.PO!c%uO~O!b$lO!c%uO~O!c%xO~O!c%{O~O!]%|O~Ol#QOo&PO|#QO}&PO#cYO~O#d&OO~O|!m~",
-  goto: "<[#pPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP#qP$_P$w%x'['bPP(v)S*P*SP*YP+d+h+dPPPP,TP,a,yPPP-a#qP.R.oP.s.yP/s0z$_$_P$_P$_P$_$_2T2Z2g3c3q3{4R4Y4`4j4p4z5UPPPPP5d5h6dP7v9oPP:|P;^PPPPP;b;h;nxbOg!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|Q!ZYR#i!U}bOYg!U!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|x`Og!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|Q!^YS!si%pQ!xjQ!|lQ#`!RQ#b!QQ#e!SR#l!U|UOgi!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%p%q%|!W]RU[jlps{|}!O!P!Q!R!S!V!W!`!c!r#m#r#w$c${%e%sS!WY!US#Qn&OR#UuQ!YYR#h!UxROg!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|!WtRU[jlps{|}!O!P!Q!R!S!V!W!`!c!r#m#r#w$c${%e%sS!VY!US!ri%pS#Qn&OR#TueqRUs!V!W!r#m${%e%sxbOg!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|doRUs!V!W!r#m${%e%sQ!ZYQ#RpR#i!UR!qhX!oh!m!p$S#Y]ORUY[gijlps{|}!O!P!Q!R!S!U!V!W!`!c!d!e!f!i!r!y#m#r#w#{$O$Q$V$W$X$c$i$q$s${%S%e%g%p%q%s%|R$T!nTwSy|VOYg!U!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|R#UuQ$o#|Q$x$YQ%Y$rR%l%ZQ#|!fQ$Y!yQ$t$WQ$u$XQ%h%SQ%t%gQ%z%qR%}%|Q$n#|Q$w$YQ%U$oQ%X$rQ%c$xS%k%Y%ZR%v%ldqRUs!V!W!r#m${%e%sQ!a[[#Om!}#P$Z$[$yQ#p!`X#s!a#p#t$a|VOYg!U!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|T!ui%pT%[$t%]Q%`$tR%o%]xXOg!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|Q!XYQ!{lQ#Y|Q#]}Q#_!OR#g!U#Z]ORUY[gijlps{|}!O!P!Q!R!S!U!V!W!`!c!d!e!f!i!r!y#m#r#w#{$O$Q$V$W$X$c$i$q$s${%S%e%g%p%q%s%|![]RU[ijlps{|}!O!P!Q!R!S!V!W!`!c!r#m#r#w$c${%e%p%s}^OYg!U!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|QgOR!kg^!gd!_#x#y#z$h$rR#}!gQ!UYQ!`[d#f!U!`#m#n$O$^$q${%e%sS#m!V!WS#n!X!^Q$O!iS$^#g#lQ$q$QQ${$`R%e$|Q!mhQ!}mU$R!m!}$[R$[#PQ!phQ$S!mT$U!p$SQySR#WyS$i#{$sR%Q$iQ$|$`R%f$|YsRU!V!W!rR#SsQ%]$tR%m%]Q#t!aQ$a#pT$e#t$aQ#w!cQ$c#rT$f#w$cQ#PmQ$Z!}U$]#P$Z$yR$y$[TfOgSdOgS!_Y!UQ#x!dQ#y!e`#z!f!y$W$X%S%g%q%|Q$P!iU$h#{$i$sS$p$O$QQ$r$VR%V$qSeOg|!TY[!U!V!W!X!^!`!i#g#l#m#n$O$Q$^$`$q${$|%e%sQ!hdW#s!a#p#t$aW#v!c#r#w$c`#{!f!y$W$X%S%g%q%|U$g#{$i$sQ$s$VR%P$h|WOYg!U!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|doRUs!V!W!r#m${%e%sQ!c[S!ti%pQ!wjQ!zlQ#RpQ#Y{Q#Z|Q#[}Q#^!OQ#`!PQ#a!QQ#c!RQ#d!SQ#r!`X#v!c#r#w$cx_Og!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|![tRU[ijlps{|}!O!P!Q!R!S!V!W!`!c!r#m#r#w$c${%e%p%sQ!]YR#k!U[rRUs!V!W!rQ$`#mV%d${%e%sTxSyQ$j#{R%Z$sQ!viR%y%pxcOg!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|Q![YR#j!U",
-  nodeNames: "⚠ Star Slash Plus Minus And Or Eq EqEq Neq Lt Lte Gt Gte Modulo PlusEq MinusEq StarEq SlashEq ModuloEq Band Bor Bxor Shl Shr Ushr NullishCoalesce NullishEq Identifier AssignableIdentifier Word IdentifierBeforeDot CurlyString Do Comment Program PipeExpr operator WhileExpr keyword ConditionalOp ParenExpr FunctionCall DotGet Number Dollar PositionalArg FunctionDef Params NamedParam NamedArgPrefix String StringFragment Interpolation FunctionCallOrIdentifier EscapeSeq DoubleQuote Boolean Null colon CatchExpr keyword Block FinallyExpr keyword keyword Underscore NamedArg IfExpr keyword FunctionCall ElseIfExpr keyword ElseExpr BinOp Regex Dict Array FunctionCallWithBlock TryExpr keyword Throw keyword Import keyword CompoundAssign Assign",
-  maxTerm: 128,
-  context: trackScope,
-  nodeProps: [
-    ["closedBy", 59,"end"]
-  ],
-  propSources: [highlighting],
-  skippedNodes: [0,34],
-  repeatNodeCount: 13,
-  tokenData: "Lp~R}OX$OXY$mYp$Opq$mqr$Ors%Wst'^tu(uuw$Owx(|xy)Ryz)lz{$O{|*V|}$O}!O*V!O!P$O!P!Q3r!Q!R*w!R![-l![!]<_!]!^<x!^!}$O!}#O=c#O#P?X#P#Q?^#Q#R$O#R#S?w#S#T$O#T#Y@b#Y#ZA|#Z#b@b#b#cGj#c#f@b#f#gHm#g#h@b#h#iIp#i#o@b#o#p$O#p#qLQ#q;'S$O;'S;=`$g<%l~$O~O$O~~LkS$TU!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OS$jP;=`<%l$O^$tU!US#[YOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU%]Z!USOr%Wrs&Ost%Wtu&iuw%Wwx&ix#O%W#O#P&i#P;'S%W;'S;=`'W<%lO%WU&VU!YQ!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OQ&lTOr&irs&{s;'S&i;'S;=`'Q<%lO&iQ'QO!YQQ'TP;=`<%l&iU'ZP;=`<%l%W^'eZrY!USOY'^YZ$OZt'^tu(Wuw'^wx(Wx#O'^#O#P(W#P;'S'^;'S;=`(o<%lO'^Y(]SrYOY(WZ;'S(W;'S;=`(i<%lO(WY(lP;=`<%l(W^(rP;=`<%l'^^(|O#h[}Q~)RO#f~U)YU!US#cQOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU)sU!US#pQOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU*[X!USOt$Ouw$Ox!Q$O!Q!R*w!R![-l![#O$O#P;'S$O;'S;=`$g<%lO$OU+Ob!US|QOt$Ouw$Ox!O$O!O!P,W!P!Q$O!Q![-l![#O$O#P#R$O#R#S.i#S#U$O#U#V/W#V#c$O#c#d0l#d#l$O#l#m1z#m;'S$O;'S;=`$g<%lO$OU,]W!USOt$Ouw$Ox!Q$O!Q![,u![#O$O#P;'S$O;'S;=`$g<%lO$OU,|Y!US|QOt$Ouw$Ox!Q$O!Q![,u![#O$O#P#R$O#R#S,W#S;'S$O;'S;=`$g<%lO$OU-s[!US|QOt$Ouw$Ox!O$O!O!P,W!P!Q$O!Q![-l![#O$O#P#R$O#R#S.i#S;'S$O;'S;=`$g<%lO$OU.nW!USOt$Ouw$Ox!Q$O!Q![-l![#O$O#P;'S$O;'S;=`$g<%lO$OU/]X!USOt$Ouw$Ox!Q$O!Q!R/x!R!S/x!S#O$O#P;'S$O;'S;=`$g<%lO$OU0PX!US|QOt$Ouw$Ox!Q$O!Q!R/x!R!S/x!S#O$O#P;'S$O;'S;=`$g<%lO$OU0qW!USOt$Ouw$Ox!Q$O!Q!Y1Z!Y#O$O#P;'S$O;'S;=`$g<%lO$OU1bW!US|QOt$Ouw$Ox!Q$O!Q!Y1Z!Y#O$O#P;'S$O;'S;=`$g<%lO$OU2P[!USOt$Ouw$Ox!Q$O!Q![2u![!c$O!c!i2u!i#O$O#P#T$O#T#Z2u#Z;'S$O;'S;=`$g<%lO$OU2|[!US|QOt$Ouw$Ox!Q$O!Q![2u![!c$O!c!i2u!i#O$O#P#T$O#T#Z2u#Z;'S$O;'S;=`$g<%lO$OU3wW!USOt$Ouw$Ox!P$O!P!Q4a!Q#O$O#P;'S$O;'S;=`$g<%lO$OU4f^!USOY5bYZ$OZt5btu6euw5bwx6ex!P5b!P!Q$O!Q!}5b!}#O;W#O#P8s#P;'S5b;'S;=`<X<%lO5bU5i^!US!mQOY5bYZ$OZt5btu6euw5bwx6ex!P5b!P!Q9Y!Q!}5b!}#O;W#O#P8s#P;'S5b;'S;=`<X<%lO5bQ6jX!mQOY6eZ!P6e!P!Q7V!Q!}6e!}#O7t#O#P8s#P;'S6e;'S;=`9S<%lO6eQ7YP!P!Q7]Q7bU!mQ#Z#[7]#]#^7]#a#b7]#g#h7]#i#j7]#m#n7]Q7wVOY7tZ#O7t#O#P8^#P#Q6e#Q;'S7t;'S;=`8m<%lO7tQ8aSOY7tZ;'S7t;'S;=`8m<%lO7tQ8pP;=`<%l7tQ8vSOY6eZ;'S6e;'S;=`9S<%lO6eQ9VP;=`<%l6eU9_W!USOt$Ouw$Ox!P$O!P!Q9w!Q#O$O#P;'S$O;'S;=`$g<%lO$OU:Ob!US!mQOt$Ouw$Ox#O$O#P#Z$O#Z#[9w#[#]$O#]#^9w#^#a$O#a#b9w#b#g$O#g#h9w#h#i$O#i#j9w#j#m$O#m#n9w#n;'S$O;'S;=`$g<%lO$OU;][!USOY;WYZ$OZt;Wtu7tuw;Wwx7tx#O;W#O#P8^#P#Q5b#Q;'S;W;'S;=`<R<%lO;WU<UP;=`<%l;WU<[P;=`<%l5bU<fU!US!]QOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU=PU!US#`QOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU=jW#rQ!USOt$Ouw$Ox!_$O!_!`>S!`#O$O#P;'S$O;'S;=`$g<%lO$OU>XV!USOt$Ouw$Ox#O$O#P#Q>n#Q;'S$O;'S;=`$g<%lO$OU>uU#qQ!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$O~?^O#i~U?eU#sQ!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU@OU!US!dQOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU@g^!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#o@b#o;'S$O;'S;=`$g<%lO$OUAjU!SQ!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OUBR_!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#UCQ#U#o@b#o;'S$O;'S;=`$g<%lO$OUCV`!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#`@b#`#aDX#a#o@b#o;'S$O;'S;=`$g<%lO$OUD^`!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#g@b#g#hE`#h#o@b#o;'S$O;'S;=`$g<%lO$OUEe`!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#X@b#X#YFg#Y#o@b#o;'S$O;'S;=`$g<%lO$OUFn^!ZQ!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#o@b#o;'S$O;'S;=`$g<%lO$O^Gq^#jW!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#o@b#o;'S$O;'S;=`$g<%lO$O^Ht^#lW!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#o@b#o;'S$O;'S;=`$g<%lO$O^Iw`#kW!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#f@b#f#gJy#g#o@b#o;'S$O;'S;=`$g<%lO$OUKO`!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#i@b#i#jE`#j#o@b#o;'S$O;'S;=`$g<%lO$OULXUuQ!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$O~LpO#t~",
-  tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, pipeStartsLineTokenizer, new LocalTokenGroup("[~RP!O!PU~ZO#d~~", 11)],
-  topRules: {"Program":[0,35]},
-  specialized: [{term: 28, get: (value: any, stack: any) => (specializeKeyword(value, stack) << 1), external: specializeKeyword},{term: 28, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}],
-  tokenPrec: 2589
-})

src/parser/stringParser.ts

@@ -0,0 +1,258 @@
+import { SyntaxNode } from './node'
+
+/**
+ * Parse string contents into fragments, interpolations, and escape sequences.
+ *
+ * Input: full string including quotes, e.g. "'hello $name'"
+ * Output: SyntaxNode tree with StringFragment, Interpolation, EscapeSeq children
+ */
+export const parseString = (input: string, from: number, to: number, parser: any): SyntaxNode => {
+  const stringNode = new SyntaxNode('String', from, to)
+  const content = input.slice(from, to)
+
+  // Determine string type
+  const firstChar = content[0]
+
+  // Double-quoted strings: no interpolation or escapes
+  if (firstChar === '"') {
+    const fragment = new SyntaxNode('DoubleQuote', from, to)
+    stringNode.add(fragment)
+    return stringNode
+  }
+
+  // Curly strings: interpolation but no escapes
+  if (firstChar === '{') {
+    parseCurlyString(stringNode, input, from, to, parser)
+    return stringNode
+  }
+
+  // Single-quoted strings: interpolation and escapes
+  if (firstChar === "'") {
+    parseSingleQuoteString(stringNode, input, from, to, parser)
+    return stringNode
+  }
+
+  throw `Unknown string type starting with: ${firstChar}`
+}
+
+/**
+ * Parse single-quoted string: 'hello $name\n'
+ * Supports: interpolation ($var, $(expr)), escape sequences (\n, \$, etc)
+ */
+const parseSingleQuoteString = (stringNode: SyntaxNode, input: string, from: number, to: number, parser: any) => {
+  let pos = from + 1 // Skip opening '
+  let fragmentStart = pos
+
+  while (pos < to - 1) { // -1 to skip closing '
+    const char = input[pos]
+
+    // Escape sequence
+    if (char === '\\' && pos + 1 < to - 1) {
+      // Push accumulated fragment
+      if (pos > fragmentStart) {
+        const frag = new SyntaxNode('StringFragment', fragmentStart, pos)
+        stringNode.add(frag)
+      }
+
+      // Add escape sequence node
+      const escNode = new SyntaxNode('EscapeSeq', pos, pos + 2)
+      stringNode.add(escNode)
+
+      pos += 2
+      fragmentStart = pos
+      continue
+    }
+
+    // Interpolation
+    if (char === '$') {
+      // Push accumulated fragment
+      if (pos > fragmentStart) {
+        const frag = new SyntaxNode('StringFragment', fragmentStart, pos)
+        stringNode.add(frag)
+      }
+
+      pos++ // Skip $
+
+      // Parse interpolation content
+      if (input[pos] === '(') {
+        // Expression interpolation: $(expr)
+        const interpStart = pos - 1 // Include the $
+        const exprResult = parseInterpolationExpr(input, pos, parser)
+        const interpNode = new SyntaxNode('Interpolation', interpStart, exprResult.endPos)
+        interpNode.add(exprResult.node)
+        stringNode.add(interpNode)
+        pos = exprResult.endPos
+      } else {
+        // Variable interpolation: $name
+        const interpStart = pos - 1
+        const identEnd = findIdentifierEnd(input, pos, to - 1)
+        const identNode = new SyntaxNode('FunctionCallOrIdentifier', pos, identEnd)
+        const innerIdent = new SyntaxNode('Identifier', pos, identEnd)
+        identNode.add(innerIdent)
+
+        const interpNode = new SyntaxNode('Interpolation', interpStart, identEnd)
+        interpNode.add(identNode)
+        stringNode.add(interpNode)
+        pos = identEnd
+      }
+
+      fragmentStart = pos
+      continue
+    }
+
+    pos++
+  }
+
+  // Push final fragment
+  if (pos > fragmentStart && fragmentStart < to - 1) {
+    const frag = new SyntaxNode('StringFragment', fragmentStart, pos)
+    stringNode.add(frag)
+  }
+}
+
+/**
+ * Parse curly string: { hello $name }
+ * Supports: interpolation ($var, $(expr)), nested braces
+ * Does NOT support: escape sequences (raw content)
+ */
+const parseCurlyString = (stringNode: SyntaxNode, input: string, from: number, to: number, parser: any) => {
+  let pos = from + 1 // Skip opening {
+  let fragmentStart = from // Include the opening { in the fragment
+  let depth = 1
+
+  while (pos < to && depth > 0) {
+    const char = input[pos]
+
+    // Track brace nesting
+    if (char === '{') {
+      depth++
+      pos++
+      continue
+    }
+
+    if (char === '}') {
+      depth--
+      if (depth === 0) {
+        // Push final fragment including closing }
+        const frag = new SyntaxNode('CurlyString', fragmentStart, pos + 1)
+        stringNode.add(frag)
+        break
+      }
+      pos++
+      continue
+    }
+
+    // Interpolation
+    if (char === '$') {
+      // Push accumulated fragment
+      if (pos > fragmentStart) {
+        const frag = new SyntaxNode('CurlyString', fragmentStart, pos)
+        stringNode.add(frag)
+      }
+
+      pos++ // Skip $
+
+      // Parse interpolation content
+      if (input[pos] === '(') {
+        // Expression interpolation: $(expr)
+        const interpStart = pos - 1
+        const exprResult = parseInterpolationExpr(input, pos, parser)
+        const interpNode = new SyntaxNode('Interpolation', interpStart, exprResult.endPos)
+        interpNode.add(exprResult.node)
+        stringNode.add(interpNode)
+        pos = exprResult.endPos
+      } else {
+        // Variable interpolation: $name
+        const interpStart = pos - 1
+        const identEnd = findIdentifierEnd(input, pos, to)
+        const identNode = new SyntaxNode('FunctionCallOrIdentifier', pos, identEnd)
+        const innerIdent = new SyntaxNode('Identifier', pos, identEnd)
+        identNode.add(innerIdent)
+
+        const interpNode = new SyntaxNode('Interpolation', interpStart, identEnd)
+        interpNode.add(identNode)
+        stringNode.add(interpNode)
+        pos = identEnd
+      }
+
+      fragmentStart = pos
+      continue
+    }
+
+    pos++
+  }
+}
+
+/**
+ * Parse a parenthesized expression interpolation: $(a + b)
+ * Returns the parsed expression node and the position after the closing )
+ * pos is position of the opening ( in the full input string
+ */
+const parseInterpolationExpr = (input: string, pos: number, parser: any): { node: SyntaxNode, endPos: number } => {
+  // Find matching closing paren
+  let depth = 1
+  let start = pos
+  let end = pos + 1 // Start after opening (
+
+  while (end < input.length && depth > 0) {
+    if (input[end] === '(') depth++
+    if (input[end] === ')') {
+      depth--
+      if (depth === 0) break
+    }
+    end++
+  }
+
+  const exprContent = input.slice(start + 1, end) // Content between ( and )
+  const closeParen = end
+  end++ // Move past closing )
+
+  // Use the main parser to parse the expression
+  const exprNode = parser.parse(exprContent)
+
+  // Get the first real node (skip Program wrapper)
+  const innerNode = exprNode.firstChild || exprNode
+
+  // Adjust node positions: they're relative to exprContent, need to offset to full input
+  const offset = start + 1 // Position where exprContent starts in full input
+  adjustNodePositions(innerNode, offset)
+
+  // Wrap in ParenExpr - use positions in the full string
+  const parenNode = new SyntaxNode('ParenExpr', start, closeParen + 1)
+  parenNode.add(innerNode)
+
+  return { node: parenNode, endPos: end }
+}
+
+/**
+ * Recursively adjust all node positions by adding an offset
+ */
+const adjustNodePositions = (node: SyntaxNode, offset: number) => {
+  node.from += offset
+  node.to += offset
+
+  for (const child of node.children) {
+    adjustNodePositions(child, offset)
+  }
+}
+
+/**
+ * Find the end position of an identifier starting at pos
+ * Identifiers: lowercase letter or emoji, followed by letters/digits/dashes/emoji
+ */
+const findIdentifierEnd = (input: string, pos: number, maxPos: number): number => {
+  let end = pos
+
+  while (end < maxPos) {
+    const char = input[end]!
+
+    // Stop at non-identifier characters
+    if (!/[a-z0-9\-?]/.test(char)) {
+      break
+    }
+
+    end++
+  }
+
+  return end
+}

src/parser/tests/basics.test.ts

@@ -1,7 +1,5 @@
 import { expect, describe, test } from 'bun:test'
 
-import '../shrimp.grammar' // Importing this so changes cause it to retest!
-
 describe('null', () => {
   test('parses null', () => {
     expect('null').toMatchTree(`Null null`)
@@ -810,44 +808,6 @@ describe('Nullish coalescing operator', () => {
   })
 })
 
-describe('DotGet whitespace sensitivity', () => {
-  test('no whitespace - DotGet works when identifier in scope', () => {
-    expect('basename = 5; basename.prop').toMatchTree(`
-      Assign
-        AssignableIdentifier basename
-        Eq =
-        Number 5
-      FunctionCallOrIdentifier
-        DotGet
-          IdentifierBeforeDot basename
-          Identifier prop`)
-  })
-
-  test('space before dot - NOT DotGet, parses as division', () => {
-    expect('basename = 5; basename / prop').toMatchTree(`
-      Assign
-        AssignableIdentifier basename
-        Eq =
-        Number 5
-      BinOp
-        Identifier basename
-        Slash /
-        Identifier prop`)
-  })
-
-  test('dot followed by slash is Word, not DotGet', () => {
-    expect('basename ./cool').toMatchTree(`
-      FunctionCall
-        Identifier basename
-        PositionalArg
-          Word ./cool`)
-  })
-
-  test('identifier not in scope with dot becomes Word', () => {
-    expect('readme.txt').toMatchTree(`Word readme.txt`)
-  })
-})
-
 describe('Comments', () => {
   test('are greedy', () => {
     expect(`
@@ -897,61 +857,6 @@ basename = 5 # very astute
   })
 })
 
-describe('Array destructuring', () => {
-  test('parses array pattern with two variables', () => {
-    expect('[ a b ] = [ 1 2 3 4]').toMatchTree(`
-      Assign
-        Array
-          Identifier a
-          Identifier b
-        Eq =
-        Array
-          Number 1
-          Number 2
-          Number 3
-          Number 4`)
-  })
-
-  test('parses array pattern with one variable', () => {
-    expect('[ x ] = [ 42 ]').toMatchTree(`
-      Assign
-        Array
-          Identifier x
-        Eq =
-        Array
-          Number 42`)
-  })
-
-  test('parses array pattern with emoji identifiers', () => {
-    expect('[ 🚀 💎 ] = [ 1 2 ]').toMatchTree(`
-      Assign
-        Array
-          Identifier 🚀
-          Identifier 💎
-        Eq =
-        Array
-          Number 1
-          Number 2`)
-  })
-
-  test('works with dotget', () => {
-    expect('[ a ] = [ [1 2 3] ]; a.1').toMatchTree(`
-      Assign
-        Array
-          Identifier a
-        Eq =
-        Array
-          Array
-            Number 1
-            Number 2
-            Number 3
-      FunctionCallOrIdentifier
-        DotGet
-          IdentifierBeforeDot a
-          Number 1`)
-  })
-})
-
 describe('Conditional ops', () => {
   test('or can be chained', () => {
     expect(`
@@ -1037,34 +942,3 @@ Assign
         `)
   })
 })
-
-describe('import', () => {
-  test('parses single import', () => {
-    expect(`import str`).toMatchTree(`
-      Import
-        keyword import
-        Identifier str
-      `)
-  })
-
-  test('parses multiple imports', () => {
-    expect(`import str math list`).toMatchTree(`
-      Import
-        keyword import
-        Identifier str
-        Identifier math
-        Identifier list
-      `)
-  })
-
-  test('parses named args', () => {
-    expect(`import str only=ends-with?`).toMatchTree(`
-      Import
-        keyword import
-        Identifier str
-        NamedArg
-          NamedArgPrefix only=
-          Identifier ends-with?
-      `)
-  })
-})

src/parser/tests/bitwise.test.ts

@@ -1,7 +1,5 @@
 import { expect, describe, test } from 'bun:test'
 
-import '../shrimp.grammar' // Importing this so changes cause it to retest!
-
 describe('bitwise operators - grammar', () => {
   test('parses band (bitwise AND)', () => {
     expect('5 band 3').toMatchTree(`

src/parser/tests/control-flow.test.ts

@@ -1,7 +1,5 @@
 import { expect, describe, test } from 'bun:test'
 
-import '../shrimp.grammar' // Importing this so changes cause it to retest!
-
 describe('if/else if/else', () => {
   test('parses single line if', () => {
     expect(`if y == 1: 'cool' end`).toMatchTree(`
@@ -24,6 +22,7 @@ describe('if/else if/else', () => {
         Eq =
         IfExpr
           keyword if
+          FunctionCallOrIdentifier
             Identifier x
           colon :
           Block
@@ -59,6 +58,7 @@ describe('if/else if/else', () => {
     end`).toMatchTree(`
       IfExpr
         keyword if
+        FunctionCallOrIdentifier
           Identifier with-else
         colon :
         Block
@@ -82,6 +82,7 @@ describe('if/else if/else', () => {
     end`).toMatchTree(`
       IfExpr
         keyword if
+        FunctionCallOrIdentifier
           Identifier with-else-if
         colon :
         Block
@@ -90,6 +91,7 @@ describe('if/else if/else', () => {
         ElseIfExpr
           keyword else
           keyword if
+          FunctionCallOrIdentifier
             Identifier another-condition
           colon :
           Block
@@ -111,6 +113,7 @@ describe('if/else if/else', () => {
     end`).toMatchTree(`
       IfExpr
         keyword if
+        FunctionCallOrIdentifier
           Identifier with-else-if-else
         colon :
         Block
@@ -119,6 +122,7 @@ describe('if/else if/else', () => {
         ElseIfExpr
           keyword else
           keyword if
+          FunctionCallOrIdentifier
             Identifier another-condition
           colon :
           Block
@@ -127,6 +131,7 @@ describe('if/else if/else', () => {
         ElseIfExpr
           keyword else
           keyword if
+          FunctionCallOrIdentifier
             Identifier yet-another-condition
           colon :
           Block
@@ -173,7 +178,7 @@ describe('if/else if/else', () => {
     `)
   })
 
-  test('parses function calls in if tests', () => {
+  test("parses paren'd function calls in if tests", () => {
     expect(`if (var? 'abc'): true end`).toMatchTree(`
       IfExpr
         keyword if
@@ -214,7 +219,7 @@ describe('if/else if/else', () => {
     `)
   })
 
-  test('parses function calls in else-if tests', () => {
+  test("parses paren'd function calls in else-if tests", () => {
     expect(`if false: true else if (var? 'abc'): true end`).toMatchTree(`
       IfExpr
         keyword if

src/parser/tests/destructuring.test.ts

@@ -0,0 +1,56 @@
+import { expect, describe, test } from 'bun:test'
+
+describe('Array destructuring', () => {
+  test('parses array pattern with two variables', () => {
+    expect('[ a b ] = [ 1 2 3 4]').toMatchTree(`
+      Assign
+        Array
+          Identifier a
+          Identifier b
+        Eq =
+        Array
+          Number 1
+          Number 2
+          Number 3
+          Number 4`)
+  })
+
+  test('parses array pattern with one variable', () => {
+    expect('[ x ] = [ 42 ]').toMatchTree(`
+      Assign
+        Array
+          Identifier x
+        Eq =
+        Array
+          Number 42`)
+  })
+
+  test('parses array pattern with emoji identifiers', () => {
+    expect('[ 🚀 💎 ] = [ 1 2 ]').toMatchTree(`
+      Assign
+        Array
+          Identifier 🚀
+          Identifier 💎
+        Eq =
+        Array
+          Number 1
+          Number 2`)
+  })
+
+  test('works with dotget', () => {
+    expect('[ a ] = [ [1 2 3] ]; a.1').toMatchTree(`
+      Assign
+        Array
+          Identifier a
+        Eq =
+        Array
+          Array
+            Number 1
+            Number 2
+            Number 3
+      FunctionCallOrIdentifier
+        DotGet
+          IdentifierBeforeDot a
+          Number 1`)
+  })
+})

src/parser/tests/dot-get.test.ts

@@ -1,6 +1,44 @@
 import { describe, test, expect } from 'bun:test'
 import '../../testSetup'
 
+describe('DotGet whitespace sensitivity', () => {
+  test('no whitespace - DotGet works when identifier in scope', () => {
+    expect('basename = 5; basename.prop').toMatchTree(`
+      Assign
+        AssignableIdentifier basename
+        Eq =
+        Number 5
+      FunctionCallOrIdentifier
+        DotGet
+          IdentifierBeforeDot basename
+          Identifier prop`)
+  })
+
+  test('space before dot - NOT DotGet, parses as division', () => {
+    expect('basename = 5; basename / prop').toMatchTree(`
+      Assign
+        AssignableIdentifier basename
+        Eq =
+        Number 5
+      BinOp
+        Identifier basename
+        Slash /
+        Identifier prop`)
+  })
+
+  test('dot followed by slash is Word, not DotGet', () => {
+    expect('basename ./cool').toMatchTree(`
+      FunctionCall
+        Identifier basename
+        PositionalArg
+          Word ./cool`)
+  })
+
+  test('identifier not in scope with dot becomes Word', () => {
+    expect('readme.txt').toMatchTree(`Word readme.txt`)
+  })
+})
+
 describe('DotGet', () => {
   test('readme.txt is Word when readme not in scope', () => {
     expect('readme.txt').toMatchTree(`Word readme.txt`)
@@ -199,7 +237,7 @@ end`).toMatchTree(`
     `)
   })
 
-  test("dot get doesn't work with spaces", () => {
+  test.skip("dot get doesn't work with spaces", () => {
     expect('obj . prop').toMatchTree(`
       FunctionCall
         Identifier obj

src/parser/tests/exceptions.test.ts

@@ -1,7 +1,5 @@
 import { expect, describe, test } from 'bun:test'
 
-import '../shrimp.grammar' // Importing this so changes cause it to retest!
-
 describe('try/catch/finally/throw', () => {
   test('parses try with catch', () => {
     expect(`try:
@@ -139,10 +137,23 @@ describe('try/catch/finally/throw', () => {
     `)
   })
 
+  test('parses throw statement with BinOp', () => {
+    expect("throw 'error message:' + msg").toMatchTree(`
+      Throw
+        keyword throw
+        BinOp
+          String
+            StringFragment error message:
+          Plus +
+          Identifier msg
+    `)
+  })
+
   test('parses throw statement with identifier', () => {
     expect('throw error-object').toMatchTree(`
       Throw
         keyword throw
+        FunctionCallOrIdentifier
           Identifier error-object
     `)
   })

src/parser/tests/function-blocks.test.ts

@@ -1,7 +1,5 @@
 import { expect, describe, test } from 'bun:test'
 
-import '../shrimp.grammar' // Importing this so changes cause it to retest!
-
 describe('single line function blocks', () => {
   test('work with no args', () => {
     expect(`trap: echo bye bye end`).toMatchTree(`

src/parser/tests/functions.test.ts

@@ -1,7 +1,5 @@
 import { expect, describe, test } from 'bun:test'
 
-import '../shrimp.grammar' // Importing this so changes cause it to retest!
-
 describe('calling functions', () => {
   test('call with no args', () => {
     expect('tail').toMatchTree(`
@@ -43,6 +41,58 @@ describe('calling functions', () => {
     `)
   })
 
+  test('call with function', () => {
+    expect(`tail do x: x end`).toMatchTree(`
+      FunctionCall
+        Identifier tail
+        PositionalArg
+          FunctionDef
+            Do do
+            Params
+              Identifier x
+            colon :
+            FunctionCallOrIdentifier
+              Identifier x
+            keyword end
+    `)
+  })
+
+  test('call with arg and function', () => {
+    expect(`tail true do x: x end`).toMatchTree(`
+      FunctionCall
+        Identifier tail
+        PositionalArg
+          Boolean true
+        PositionalArg
+          FunctionDef
+            Do do
+            Params
+              Identifier x
+            colon :
+            FunctionCallOrIdentifier
+              Identifier x
+            keyword end
+    `)
+  })
+
+  test('call with function in named arg', () => {
+    expect(`tail callback=do x: x end`).toMatchTree(`
+      FunctionCall
+        Identifier tail
+        NamedArg
+          NamedArgPrefix callback=
+          FunctionDef
+            Do do
+            Params
+              Identifier x
+            colon :
+            FunctionCallOrIdentifier
+              Identifier x
+            keyword end
+      `)
+  })
+
+
   test('command with arg that is also a command', () => {
     expect('tail tail').toMatchTree(`
       FunctionCall
@@ -64,7 +114,7 @@ describe('calling functions', () => {
         NamedArg
           NamedArgPrefix lines=
           ⚠
-      ⚠ `)
+      ⚠`)
   })
 })
 

src/parser/tests/import.test.ts

@@ -0,0 +1,32 @@
+import { expect, describe, test } from 'bun:test'
+
+describe('import', () => {
+  test('parses single import', () => {
+    expect(`import str`).toMatchTree(`
+      Import
+        keyword import
+        Identifier str
+      `)
+  })
+
+  test('parses multiple imports', () => {
+    expect(`import str math list`).toMatchTree(`
+      Import
+        keyword import
+        Identifier str
+        Identifier math
+        Identifier list
+      `)
+  })
+
+  test('parses named args', () => {
+    expect(`import str only=ends-with?`).toMatchTree(`
+      Import
+        keyword import
+        Identifier str
+        NamedArg
+          NamedArgPrefix only=
+          Identifier ends-with?
+      `)
+  })
+})

src/parser/tests/literals.test.ts

@@ -1,7 +1,5 @@
 import { expect, describe, test } from 'bun:test'
 
-import '../shrimp.grammar' // Importing this so changes cause it to retest!
-
 describe('number literals', () => {
   test('binary numbers', () => {
     expect('0b110').toMatchTree(`
@@ -336,6 +334,22 @@ describe('dict literals', () => {
     `)
   })
 
+  test('work with functions', () => {
+    expect(`[trap=do x: x end]`).toMatchTree(`
+      Dict
+        NamedArg
+          NamedArgPrefix trap=
+          FunctionDef
+            Do do
+            Params
+              Identifier x
+            colon :
+            FunctionCallOrIdentifier
+              Identifier x
+            keyword end
+    `)
+  })
+
   test('can be nested', () => {
     expect('[a=one b=[two [c=three]]]').toMatchTree(`
       Dict
@@ -371,14 +385,35 @@ describe('dict literals', () => {
           Number 3
     `)
   })
+
+  test('can have spaces between equals', () => {
+    expect(`[
+      a = 1
+      b = 2
+      c = 3
+    ]`).toMatchTree(`
+      Dict
+        NamedArg
+          NamedArgPrefix a =
+          Number 1
+        NamedArg
+          NamedArgPrefix b =
+          Number 2
+        NamedArg
+          NamedArgPrefix c =
+          Number 3
+    `)
+  })
+
   test('empty dict', () => {
     expect('[=]').toMatchTree(`
       Dict [=]
     `)
+  })
 
+  test('empty dict w whitespace', () => {
     expect('[ = ]').toMatchTree(`
-      Array
+      Dict [ = ]
-        Word =
     `)
   })
 

src/parser/tests/multiline.test.ts

@@ -1,7 +1,5 @@
 import { expect, describe, test } from 'bun:test'
 
-import '../shrimp.grammar' // Importing this so changes cause it to retest!
-
 describe('multiline', () => {
   test('parses multiline strings', () => {
     expect(`'first'\n'second'`).toMatchTree(`

src/parser/tests/pipes.test.ts

@@ -1,7 +1,4 @@
 import { expect, describe, test } from 'bun:test'
-import { parser } from '../shrimp'
-
-import '../shrimp.grammar' // Importing this so changes cause it to retest!
 
 describe('pipe expressions', () => {
   test('simple pipe expression', () => {
@@ -176,6 +173,43 @@ describe('pipe expressions', () => {
           Identifier echo
       `)
   })
+
+  test('parenthesized expressions can be piped', () => {
+    expect(`(1 + 2) | echo`).toMatchTree(`
+      PipeExpr
+        ParenExpr
+          BinOp
+            Number 1
+            Plus +
+            Number 2
+        operator |
+        FunctionCallOrIdentifier
+          Identifier echo
+      `)
+  })
+
+  test('complex parenthesized expressions with pipes', () => {
+    expect(`((math.random) * 10 + 1) | math.floor`).toMatchTree(`
+      PipeExpr
+        ParenExpr
+          BinOp
+            BinOp
+              ParenExpr
+                FunctionCallOrIdentifier
+                  DotGet
+                    IdentifierBeforeDot math
+                    Identifier random
+              Star *
+              Number 10
+            Plus +
+            Number 1
+        operator |
+        FunctionCallOrIdentifier
+          DotGet
+            IdentifierBeforeDot math
+            Identifier floor
+      `)
+  })
 })
 
 describe('pipe continuation', () => {
@@ -309,7 +343,7 @@ grep h`).toMatchTree(`
       Identifier split
       PositionalArg
         String
-          StringFragment  
+          StringFragment (space)
     operator |
     FunctionCall
       Identifier map
@@ -333,3 +367,41 @@ grep h`).toMatchTree(`
   `)
   })
 })
+
+describe('Underscore', () => {
+  test('works in pipes', () => {
+    expect(`sub 3 1 | div (sub 110 9 | sub 1) _ | div 5`).toMatchTree(`
+      PipeExpr
+        FunctionCall
+          Identifier sub
+          PositionalArg
+            Number 3
+          PositionalArg
+            Number 1
+        operator |
+        FunctionCall
+          Identifier div
+          PositionalArg
+            ParenExpr
+              PipeExpr
+                FunctionCall
+                  Identifier sub
+                  PositionalArg
+                    Number 110
+                  PositionalArg
+                    Number 9
+                operator |
+                FunctionCall
+                  Identifier sub
+                  PositionalArg
+                    Number 1
+          PositionalArg
+            Underscore _
+        operator |
+        FunctionCall
+          Identifier div
+          PositionalArg
+            Number 5
+    `)
+  })
+})

src/parser/tests/strings.test.ts

@@ -1,7 +1,5 @@
 import { expect, describe, test } from 'bun:test'
 
-import '../shrimp.grammar' // Importing this so changes cause it to retest!
-
 describe('string interpolation', () => {
   test('string with variable interpolation', () => {
     expect("'hello $name'").toMatchTree(`

src/parser/tests/tokens.test.ts

@@ -0,0 +1,750 @@
+import { expect, describe, test } from 'bun:test'
+
+describe('constant types', () => {
+  test('null', () => {
+    expect(`null`).toBeToken('Null')
+  })
+
+  test('boolean', () => {
+    expect(`true`).toMatchToken('Boolean', 'true')
+    expect(`false`).toMatchToken('Boolean', 'false')
+  })
+})
+
+describe('numbers', () => {
+  test('non-numbers', () => {
+    expect(`1st`).toMatchToken('Word', '1st')
+    expect(`1_`).toMatchToken('Word', '1_')
+    expect(`100.`).toMatchTokens(
+      { type: 'Number', value: '100' },
+      { type: 'Operator', value: '.' },
+    )
+  })
+
+  test('simple numbers', () => {
+    expect(`1`).toMatchToken('Number', '1')
+    expect(`200`).toMatchToken('Number', '200')
+    expect(`5.20`).toMatchToken('Number', '5.20')
+    expect(`0.20`).toMatchToken('Number', '0.20')
+    expect(`-20`).toMatchToken('Number', '-20')
+    expect(`+20`).toMatchToken('Number', '+20')
+    expect(`-2134.34`).toMatchToken('Number', '-2134.34')
+    expect(`+20.5325`).toMatchToken('Number', '+20.5325')
+    expect(`1_000`).toMatchToken('Number', '1_000')
+    expect(`53_232_220`).toMatchToken('Number', '53_232_220')
+  })
+
+  test('binary numbers', () => {
+    expect('0b110').toMatchToken('Number', '0b110')
+  })
+
+  test('hex numbers', () => {
+    expect('0xdeadbeef').toMatchToken('Number', '0xdeadbeef')
+    expect('0x02d3f4').toMatchToken('Number', '0x02d3f4')
+  })
+
+  test('hex numbers uppercase', () => {
+    expect('0xFF').toMatchToken('Number', '0xFF')
+  })
+
+  test('octal numbers', () => {
+    expect('0o644').toMatchToken('Number', '0o644')
+    expect('0o055').toMatchToken('Number', '0o055')
+  })
+
+  test('negative binary', () => {
+    expect('-0b110').toMatchToken('Number', '-0b110')
+  })
+
+  test('negative hex', () => {
+    expect('-0xFF').toMatchToken('Number', '-0xFF')
+  })
+
+  test('negative octal', () => {
+    expect('-0o755').toMatchToken('Number', '-0o755')
+  })
+
+  test('positive prefix binary', () => {
+    expect('+0b110').toMatchToken('Number', '+0b110')
+  })
+
+  test('positive prefix hex', () => {
+    expect('+0xFF').toMatchToken('Number', '+0xFF')
+  })
+
+  test('positive prefix octal', () => {
+    expect('+0o644').toMatchToken('Number', '+0o644')
+  })
+
+  test('underscores in number', () => {
+    expect(`1_000`).toMatchToken('Number', '1_000')
+    expect(`1_0`).toMatchToken('Number', '1_0')
+    expect('0b11_0').toMatchToken('Number', '0b11_0')
+    expect('0xdead_beef').toMatchToken('Number', '0xdead_beef')
+    expect('0o64_4').toMatchToken('Number', '0o64_4')
+  })
+})
+
+describe('identifiers', () => {
+  test('regular', () => {
+    expect('name').toBeToken('Identifier')
+    expect('bobby-mcgee').toBeToken('Identifier')
+    expect('starts-with?').toBeToken('Identifier')
+    expect('📢').toMatchToken('Identifier', '📢')
+    expect(' 📢 ').toMatchToken('Identifier', '📢')
+    expect(' oink-🐷-oink').toMatchToken('Identifier', 'oink-🐷-oink')
+    expect('$').toMatchToken('Identifier', '$')
+    expect('$cool').toMatchToken('Identifier', '$cool')
+  })
+
+  test('one character identifiers', () => {
+    expect('a').toMatchToken('Identifier', 'a')
+    expect('z').toMatchToken('Identifier', 'z')
+    expect('$').toMatchToken('Identifier', '$')
+    expect('📢').toMatchToken('Identifier', '📢')
+    expect('?').toBeToken('Word') // ? alone is not valid identifier start
+  })
+
+  test('two character identifiers', () => {
+    expect('ab').toMatchToken('Identifier', 'ab')
+    expect('a1').toMatchToken('Identifier', 'a1')
+    expect('a-').toMatchToken('Identifier', 'a-')
+    expect('a?').toMatchToken('Identifier', 'a?') // ? valid at end
+    expect('ab?').toMatchToken('Identifier', 'ab?')
+  })
+
+  test('three+ character identifiers', () => {
+    expect('abc').toMatchToken('Identifier', 'abc')
+    expect('a-b').toMatchToken('Identifier', 'a-b')
+    expect('a1b').toMatchToken('Identifier', 'a1b')
+    expect('abc?').toMatchToken('Identifier', 'abc?') // ? valid at end
+    expect('a-b-c?').toMatchToken('Identifier', 'a-b-c?')
+  })
+
+  test('edge cases', () => {
+    expect('-bobby-mcgee').toBeToken('Word')
+    expect('starts-with??').toMatchToken('Identifier', 'starts-with??')
+    expect('starts?with?').toMatchToken('Identifier', 'starts?with?')
+    expect('a??b').toMatchToken('Identifier', 'a??b')
+    expect('oink-oink!').toBeToken('Word')
+    expect('dog#pound').toMatchToken('Word', 'dog#pound')
+    expect('http://website.com').toMatchToken('Word', 'http://website.com')
+    expect('school$cool').toMatchToken('Identifier', 'school$cool')
+    expect('EXIT:').toMatchTokens(
+      { type: 'Word', value: 'EXIT' },
+      { type: 'Colon' },
+    )
+    expect(`if y == 1: 'cool' end`).toMatchTokens(
+      { type: 'Keyword', value: 'if' },
+      { type: 'Identifier', value: 'y' },
+      { type: 'Operator', value: '==' },
+      { type: 'Number', value: '1' },
+      { type: 'Colon' },
+      { type: 'String', value: `'cool'` },
+      { type: 'Keyword', value: 'end' },
+    )
+  })
+})
+
+describe('paths', () => {
+  test('starting with ./', () => {
+    expect('./tmp').toMatchToken('Word', './tmp')
+  })
+
+  test('starting with /', () => {
+    expect('/home/chris/dev').toMatchToken('Word', '/home/chris/dev')
+  })
+
+  test('identifiers with dots tokenize separately', () => {
+    expect('readme.txt').toMatchTokens(
+      { type: 'Identifier', value: 'readme' },
+      { type: 'Operator', value: '.' },
+      { type: 'Identifier', value: 'txt' },
+    )
+  })
+
+  test('words (non-identifiers) consume dots', () => {
+    expect('README.md').toMatchToken('Word', 'README.md')
+  })
+
+  test('all sorts of weird stuff', () => {
+    expect('dog#pound').toMatchToken('Word', 'dog#pound')
+    expect('my/kinda/place').toMatchToken('my/kinda/place')
+    expect('file://%/$##/@40!/index.php').toMatchToken('Word', 'file://%/$##/@40!/index.php')
+  })
+})
+
+describe('strings', () => {
+  test('single quoted', () => {
+    expect(`'hello world'`).toMatchToken('String', `'hello world'`)
+    expect(`'it\\'s a beautiful world'`).toMatchToken("'it\\'s a beautiful world'")
+  })
+
+  test('double quoted', () => {
+    expect(`"hello world"`).toMatchToken('String', `"hello world"`)
+    expect(`"it's a beautiful world"`).toMatchToken('String', `"it's a beautiful world"`)
+  })
+
+  test('empty strings', () => {
+    expect(`''`).toMatchToken('String', `''`)
+    expect(`""`).toMatchToken('String', `""`)
+  })
+
+  test('escape sequences', () => {
+    expect(`'hello\\nworld'`).toMatchToken('String', `'hello\\nworld'`)
+    expect(`'tab\\there'`).toMatchToken('String', `'tab\\there'`)
+    expect(`'quote\\''`).toMatchToken('String', `'quote\\''`)
+    expect(`'backslash\\\\'`).toMatchToken('String', `'backslash\\\\'`)
+    expect(`'dollar\\$sign'`).toMatchToken('String', `'dollar\\$sign'`)
+  })
+
+  test('unclosed strings - error case', () => {
+    // These should either fail or produce unexpected results
+    expect(`'hello`).toMatchToken('String', `'hello`)
+    expect(`"world`).toMatchToken('String', `"world`)
+  })
+})
+
+describe('curly strings', () => {
+  test('curly quoted', () => {
+    expect('{ one two three }').toMatchToken('String', `{ one two three }`)
+  })
+
+  test('work on multiple lines', () => {
+    expect(`{
+      one
+      two
+      three }`).toMatchToken('String', `{
+      one
+      two
+      three }`)
+  })
+
+  test('can contain other curlies', () => {
+    expect(`{ { one }
+      two
+      { three } }`).toMatchToken('String', `{ { one }
+      two
+      { three } }`)
+  })
+
+  test('empty curly string', () => {
+    expect('{}').toMatchToken('String', '{}')
+  })
+
+  test('unclosed curly string - error case', () => {
+    // Should either fail or produce unexpected results
+    expect('{ hello').toMatchToken('String', '{ hello')
+    expect('{ nested { unclosed }').toMatchToken('String', '{ nested { unclosed }')
+  })
+})
+
+describe('operators', () => {
+  test('math operators', () => {
+    // assignment
+    expect('=').toMatchToken('Operator', '=')
+
+    // logic
+    expect('or').toMatchToken('Operator', 'or')
+    expect('and').toMatchToken('Operator', 'and')
+
+    // bitwise
+    expect('band').toMatchToken('Operator', 'band')
+    expect('bor').toMatchToken('Operator', 'bor')
+    expect('bxor').toMatchToken('Operator', 'bxor')
+    expect('>>>').toMatchToken('Operator', '>>>')
+    expect('>>').toMatchToken('Operator', '>>')
+    expect('<<').toMatchToken('Operator', '<<')
+
+    // compound assignment
+    expect('??=').toMatchToken('Operator', '??=')
+    expect('+=').toMatchToken('Operator', '+=')
+    expect('-=').toMatchToken('Operator', '-=')
+    expect('*=').toMatchToken('Operator', '*=')
+    expect('/=').toMatchToken('Operator', '/=')
+    expect('%=').toMatchToken('Operator', '%=')
+
+    // nullish
+    expect('??').toMatchToken('Operator', '??')
+
+    // math
+    expect('**').toMatchToken('Operator', '**')
+    expect('*').toMatchToken('Operator', '*')
+    expect('/').toMatchToken('Operator', '/')
+    expect('+').toMatchToken('Operator', '+')
+    expect('-').toMatchToken('Operator', '-')
+    expect('%').toMatchToken('Operator', '%')
+
+    // comparison
+    expect('>=').toMatchToken('Operator', '>=')
+    expect('<=').toMatchToken('Operator', '<=')
+    expect('!=').toMatchToken('Operator', '!=')
+    expect('==').toMatchToken('Operator', '==')
+    expect('>').toMatchToken('Operator', '>')
+    expect('<').toMatchToken('Operator', '<')
+
+    // property access
+    expect('.').toMatchToken('Operator', '.')
+  })
+})
+
+describe('keywords', () => {
+  test('keywords', () => {
+    expect(`import`).toMatchToken('Keyword', 'import')
+
+    expect(`end`).toMatchToken('Keyword', 'end')
+    expect(`do`).toMatchToken('Keyword', 'do')
+
+    expect(`while`).toMatchToken('Keyword', 'while')
+
+    expect(`if`).toMatchToken('Keyword', 'if')
+    expect(`else`).toMatchToken('Keyword', 'else')
+
+    expect(`try`).toMatchToken('Keyword', 'try')
+    expect(`catch`).toMatchToken('Keyword', 'catch')
+    expect(`finally`).toMatchToken('Keyword', 'finally')
+    expect(`throw`).toMatchToken('Keyword', 'throw')
+    expect(`not`).toMatchToken('Keyword', 'not')
+  })
+})
+
+describe('regex', () => {
+  test('use double slash', () => {
+    expect(`//[0-9]+//`).toMatchToken('Regex', '//[0-9]+//')
+  })
+})
+
+describe('punctuation', () => {
+  test('underscore', () => {
+    expect(`_`).toBeToken('Underscore')
+    expect(`__`).toMatchToken('Word', '__')
+  })
+
+  test('semicolon', () => {
+    expect(`;`).toBeToken('Semicolon')
+  })
+
+  test('newline', () => {
+    expect('\n').toBeToken('Newline')
+  })
+
+  test('colon', () => {
+    expect(':').toBeToken('Colon')
+  })
+})
+
+describe('comments', () => {
+  test('comments', () => {
+    expect(`# hey friends`).toMatchToken('Comment', '# hey friends')
+    expect(`#hey-friends`).toMatchToken('Comment', '#hey-friends')
+  })
+})
+
+describe('brackets', () => {
+  test('parens', () => {
+    expect(`(`).toBeToken('OpenParen')
+    expect(`)`).toBeToken('CloseParen')
+  })
+
+  test('staples', () => {
+    expect(`[`).toBeToken('OpenBracket')
+    expect(`]`).toBeToken('CloseBracket')
+  })
+})
+
+describe('multiple tokens', () => {
+  test('constants work fine', () => {
+    expect(`null true false`).toMatchTokens(
+      { type: 'Null' },
+      { type: 'Boolean', value: 'true' },
+      { type: 'Boolean', value: 'false' },
+    )
+  })
+
+  test('numbers', () => {
+    expect(`100 -400.42 null`).toMatchTokens(
+      { type: 'Number', value: '100' },
+      { type: 'Number', value: '-400.42' },
+      { type: 'Null' },
+    )
+  })
+
+  test('whitespace', () => {
+    expect(`
+      'hello world'
+
+      'goodbye world'
+    `).toMatchTokens(
+      { type: 'Newline' },
+      { type: 'String', value: "'hello world'" },
+      { type: 'Newline' },
+      { type: 'Newline' },
+      { type: 'String', value: "'goodbye world'" },
+      { type: 'Newline' },
+    )
+  })
+
+  test('newline in parens is ignored', () => {
+    expect(`(
+      'hello world'
+
+      'goodbye world'
+     )`).toMatchTokens(
+      { type: 'OpenParen' },
+      { type: 'String', value: "'hello world'" },
+      { type: 'String', value: "'goodbye world'" },
+      { type: 'CloseParen' },
+    )
+  })
+
+  test('newline in brackets is ignored', () => {
+    expect(`[
+      a b
+c d
+
+e
+
+f
+
+    ]`).toMatchTokens(
+      { type: 'OpenBracket' },
+      { type: 'Identifier', value: "a" },
+      { type: 'Identifier', value: "b" },
+      { type: 'Identifier', value: "c" },
+      { type: 'Identifier', value: "d" },
+      { type: 'Identifier', value: "e" },
+      { type: 'Identifier', value: "f" },
+      { type: 'CloseBracket' },
+    )
+  })
+
+  test('function call', () => {
+    expect('echo hello world').toMatchTokens(
+      { type: 'Identifier', value: 'echo' },
+      { type: 'Identifier', value: 'hello' },
+      { type: 'Identifier', value: 'world' },
+    )
+  })
+
+  test('function call w/ parens', () => {
+    expect('echo(bold hello world)').toMatchTokens(
+      { type: 'Identifier', value: 'echo' },
+      { type: 'OpenParen' },
+      { type: 'Identifier', value: 'bold' },
+      { type: 'Identifier', value: 'hello' },
+      { type: 'Identifier', value: 'world' },
+      { type: 'CloseParen' },
+    )
+
+    expect('echo (bold hello world)').toMatchTokens(
+      { type: 'Identifier', value: 'echo' },
+      { type: 'OpenParen' },
+      { type: 'Identifier', value: 'bold' },
+      { type: 'Identifier', value: 'hello' },
+      { type: 'Identifier', value: 'world' },
+      { type: 'CloseParen' },
+    )
+  })
+
+  test('assignment', () => {
+    expect('x = 5').toMatchTokens(
+      { type: 'Identifier', value: 'x' },
+      { type: 'Operator', value: '=' },
+      { type: 'Number', value: '5' },
+    )
+  })
+
+  test('math expression', () => {
+    expect('1 + 2 * 3').toMatchTokens(
+      { type: 'Number', value: '1' },
+      { type: 'Operator', value: '+' },
+      { type: 'Number', value: '2' },
+      { type: 'Operator', value: '*' },
+      { type: 'Number', value: '3' },
+    )
+  })
+
+  test('inline comment', () => {
+    expect('x = 5 # set x').toMatchTokens(
+      { type: 'Identifier', value: 'x' },
+      { type: 'Operator', value: '=' },
+      { type: 'Number', value: '5' },
+      { type: 'Comment', value: '# set x' },
+    )
+  })
+
+  test('line comment', () => {
+    expect('x = 5 \n# hello\n set x').toMatchTokens(
+      { type: 'Identifier', value: 'x' },
+      { type: 'Operator', value: '=' },
+      { type: 'Number', value: '5' },
+      { type: 'Newline' },
+      { type: 'Comment', value: '# hello' },
+      { type: 'Newline' },
+      { type: 'Identifier', value: 'set' },
+      { type: 'Identifier', value: 'x' },
+    )
+  })
+
+  test('colons separate tokens', () => {
+    expect('x do: y').toMatchTokens(
+      { type: 'Identifier', value: 'x' },
+      { type: 'Keyword', value: 'do' },
+      { type: 'Colon' },
+      { type: 'Identifier', value: 'y' },
+    )
+
+    expect('x: y').toMatchTokens(
+      { type: 'Identifier', value: 'x' },
+      { type: 'Colon' },
+      { type: 'Identifier', value: 'y' },
+    )
+
+    expect('5: y').toMatchTokens(
+      { type: 'Number', value: '5' },
+      { type: 'Colon' },
+      { type: 'Identifier', value: 'y' },
+    )
+
+
+    expect(`if (var? 'abc'): y`).toMatchTokens(
+      { type: 'Keyword', value: 'if' },
+      { type: 'OpenParen' },
+      { type: 'Identifier', value: 'var?' },
+      { type: 'String', value: `'abc'` },
+      { type: 'CloseParen' },
+      { type: 'Colon' },
+      { type: 'Identifier', value: 'y' },
+    )
+
+    expect(`
+do x:
+  y
+end`).toMatchTokens(
+      { type: 'Newline' },
+      { type: 'Keyword', value: 'do' },
+      { type: 'Identifier', value: 'x' },
+      { type: 'Colon' },
+      { type: 'Newline' },
+      { type: 'Identifier', value: 'y' },
+      { type: 'Newline' },
+      { type: 'Keyword', value: 'end' },
+    )
+  })
+
+  test('semicolons separate statements', () => {
+    expect('x; y').toMatchTokens(
+      { type: 'Identifier', value: 'x' },
+      { type: 'Semicolon' },
+      { type: 'Identifier', value: 'y' },
+    )
+  })
+
+  test('semicolons in parens', () => {
+    expect('(x; y)').toMatchTokens(
+      { type: 'OpenParen' },
+      { type: 'Identifier', value: 'x' },
+      { type: 'Semicolon' },
+      { type: 'Identifier', value: 'y' },
+      { type: 'CloseParen' },
+    )
+  })
+
+  test('dot operator beginning word with slash', () => {
+    expect(`(basename ./cool)`).toMatchTokens(
+      { 'type': 'OpenParen' },
+      { 'type': 'Identifier', 'value': 'basename' },
+      { 'type': 'Word', 'value': './cool' },
+      { 'type': 'CloseParen' }
+    )
+  })
+
+  test('dot word after identifier with space', () => {
+    expect(`expand-path .git`).toMatchTokens(
+      { 'type': 'Identifier', 'value': 'expand-path' },
+      { 'type': 'Word', 'value': '.git' },
+    )
+  })
+
+  test('dot operator after identifier without space', () => {
+    expect(`config.path`).toMatchTokens(
+      { 'type': 'Identifier', 'value': 'config' },
+      { 'type': 'Operator', 'value': '.' },
+      { 'type': 'Identifier', 'value': 'path' },
+    )
+  })
+})
+
+describe('nesting edge cases', () => {
+  test('deeply nested parens', () => {
+    expect('((nested))').toMatchTokens(
+      { type: 'OpenParen' },
+      { type: 'OpenParen' },
+      { type: 'Identifier', value: 'nested' },
+      { type: 'CloseParen' },
+      { type: 'CloseParen' },
+    )
+  })
+
+  test('mixed nesting', () => {
+    expect('([combo])').toMatchTokens(
+      { type: 'OpenParen' },
+      { type: 'OpenBracket' },
+      { type: 'Identifier', value: 'combo' },
+      { type: 'CloseBracket' },
+      { type: 'CloseParen' },
+    )
+  })
+})
+
+describe('invalid numbers that should be words', () => {
+  test('invalid binary', () => {
+    expect('0b2').toMatchToken('Word', '0b2')
+    expect('0b123').toMatchToken('Word', '0b123')
+  })
+
+  test('invalid octal', () => {
+    expect('0o8').toMatchToken('Word', '0o8')
+    expect('0o999').toMatchToken('Word', '0o999')
+  })
+
+  test('invalid hex', () => {
+    expect('0xGGG').toMatchToken('Word', '0xGGG')
+    expect('0xZZZ').toMatchToken('Word', '0xZZZ')
+  })
+
+  test('multiple decimal points', () => {
+    expect('1.2.3').toMatchToken('Word', '1.2.3')
+  })
+})
+
+describe('unicode and emoji', () => {
+  test('greek letters', () => {
+    expect('αβγ').toMatchToken('Identifier', 'αβγ')
+    expect('delta-δ').toMatchToken('Identifier', 'delta-δ')
+  })
+
+  test('math symbols', () => {
+    expect('∑').toMatchToken('Identifier', '∑')
+    expect('∏').toMatchToken('Identifier', '∏')
+  })
+
+  test('CJK characters', () => {
+    expect('你好').toMatchToken('Identifier', '你好')
+    expect('こんにちは').toMatchToken('Identifier', 'こんにちは')
+  })
+})
+
+describe('empty and whitespace input', () => {
+  test('empty string', () => {
+    expect('').toMatchTokens()
+  })
+
+  test('only whitespace', () => {
+    expect('   ').toMatchTokens()
+  })
+
+  test('only tabs', () => {
+    expect('\t\t\t').toMatchTokens()
+  })
+
+  test('only newlines', () => {
+    expect('\n\n\n').toMatchTokens(
+      { type: 'Newline' },
+      { type: 'Newline' },
+      { type: 'Newline' },
+    )
+  })
+})
+
+describe('named args', () => {
+  test("don't need spaces", () => {
+    expect(`named=arg`).toMatchTokens(
+      { type: 'NamedArgPrefix', value: 'named=' },
+      { type: 'Identifier', value: 'arg' },
+    )
+  })
+
+  test("can have spaces", () => {
+    expect(`named= arg`).toMatchTokens(
+      { type: 'NamedArgPrefix', value: 'named=' },
+      { type: 'Identifier', value: 'arg' },
+    )
+  })
+
+  test("can include numbers", () => {
+    expect(`named123= arg`).toMatchTokens(
+      { type: 'NamedArgPrefix', value: 'named123=' },
+      { type: 'Identifier', value: 'arg' },
+    )
+  })
+})
+
+describe('dot operator', () => {
+  test('standalone dot', () => {
+    expect('.').toMatchToken('Operator', '.')
+  })
+
+  test('dot between identifiers tokenizes as separate tokens', () => {
+    expect('config.path').toMatchTokens(
+      { type: 'Identifier', value: 'config' },
+      { type: 'Operator', value: '.' },
+      { type: 'Identifier', value: 'path' },
+    )
+  })
+
+  test('dot with number', () => {
+    expect('array.0').toMatchTokens(
+      { type: 'Identifier', value: 'array' },
+      { type: 'Operator', value: '.' },
+      { type: 'Number', value: '0' },
+    )
+  })
+
+  test('chained dots', () => {
+    expect('a.b.c').toMatchTokens(
+      { type: 'Identifier', value: 'a' },
+      { type: 'Operator', value: '.' },
+      { type: 'Identifier', value: 'b' },
+      { type: 'Operator', value: '.' },
+      { type: 'Identifier', value: 'c' },
+    )
+  })
+
+  test('identifier-like paths tokenize separately', () => {
+    expect('readme.txt').toMatchTokens(
+      { type: 'Identifier', value: 'readme' },
+      { type: 'Operator', value: '.' },
+      { type: 'Identifier', value: 'txt' },
+    )
+  })
+
+  test('word-like paths remain as single token', () => {
+    expect('./file.txt').toMatchToken('Word', './file.txt')
+    expect('README.TXT').toMatchToken('Word', 'README.TXT')
+  })
+
+  test('dot with paren expression', () => {
+    expect('obj.(1 + 2)').toMatchTokens(
+      { type: 'Identifier', value: 'obj' },
+      { type: 'Operator', value: '.' },
+      { type: 'OpenParen' },
+      { type: 'Number', value: '1' },
+      { type: 'Operator', value: '+' },
+      { type: 'Number', value: '2' },
+      { type: 'CloseParen' },
+    )
+  })
+
+  test('chained dot with paren expression', () => {
+    expect('obj.items.(i)').toMatchTokens(
+      { type: 'Identifier', value: 'obj' },
+      { type: 'Operator', value: '.' },
+      { type: 'Identifier', value: 'items' },
+      { type: 'Operator', value: '.' },
+      { type: 'OpenParen' },
+      { type: 'Identifier', value: 'i' },
+      { type: 'CloseParen' },
+    )
+  })
+})

src/parser/tokenizer.ts

@@ -1,389 +0,0 @@
-import { ExternalTokenizer, InputStream, Stack } from '@lezer/lr'
-import { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, Do, CurlyString, DotGet, newline, pipeStartsLine } from './shrimp.terms'
-
-// doobie doobie do (we need the `do` keyword to know when we're defining params)
-export function specializeKeyword(ident: string) {
-  return ident === 'do' ? Do : -1
-}
-
-// tell the dotGet searcher about builtin globals
-export const globals: string[] = []
-export const setGlobals = (newGlobals: string[] | Record<string, any>) => {
-  globals.length = 0
-  globals.push(...(Array.isArray(newGlobals) ? newGlobals : Object.keys(newGlobals)))
-}
-
-// The only chars that can't be words are whitespace, apostrophes, closing parens, and EOF.
-
-export const tokenizer = new ExternalTokenizer(
-  (input: InputStream, stack: Stack) => {
-    const ch = getFullCodePoint(input, 0)
-
-    // Handle curly strings
-    if (ch === 123 /* { */) return consumeCurlyString(input, stack)
-
-    if (!isWordChar(ch)) return
-
-    // Don't consume things that start with digits - let Number token handle it
-    if (isDigit(ch)) return
-
-    // Don't consume things that start with - or + followed by a digit (negative/positive numbers)
-    if ((ch === 45 /* - */ || ch === 43) /* + */ && isDigit(input.peek(1))) return
-
-    const isValidStart = isIdentStart(ch)
-    const canBeWord = stack.canShift(Word)
-
-    // Consume all word characters, tracking if it remains a valid identifier
-    const { pos, isValidIdentifier, stoppedAtDot } = consumeWordToken(
-      input,
-      isValidStart,
-      canBeWord
-    )
-
-    // Check if we should emit IdentifierBeforeDot for property access
-    if (stoppedAtDot) {
-      const dotGetToken = checkForDotGet(input, stack, pos)
-
-      if (dotGetToken) {
-        input.advance(pos)
-        input.acceptToken(dotGetToken)
-      } else {
-        // Not in scope - continue consuming the dot as part of the word
-        const afterDot = consumeRestOfWord(input, pos + 1, canBeWord)
-        input.advance(afterDot)
-        input.acceptToken(Word)
-      }
-
-      return
-    }
-
-    // Advance past the token we consumed
-    input.advance(pos)
-
-    // Choose which token to emit
-    if (isValidIdentifier) {
-      const token = chooseIdentifierToken(input, stack)
-      input.acceptToken(token)
-    } else {
-      input.acceptToken(Word)
-    }
-  },
-  { contextual: true }
-)
-
-// Build identifier text from input stream, handling surrogate pairs for emoji
-const buildIdentifierText = (input: InputStream, length: number): string => {
-  let text = ''
-  for (let i = 0; i < length; i++) {
-    const charCode = input.peek(i)
-    if (charCode === -1) break
-
-    // Handle surrogate pairs for emoji (UTF-16 encoding)
-    if (charCode >= 0xd800 && charCode <= 0xdbff && i + 1 < length) {
-      const low = input.peek(i + 1)
-      if (low >= 0xdc00 && low <= 0xdfff) {
-        text += String.fromCharCode(charCode, low)
-        i++ // Skip the low surrogate
-        continue
-      }
-    }
-    text += String.fromCharCode(charCode)
-  }
-  return text
-}
-
-// Consume word characters, tracking if it remains a valid identifier
-// Returns the position after consuming, whether it's a valid identifier, and if we stopped at a dot
-const consumeWordToken = (
-  input: InputStream,
-  isValidStart: boolean,
-  canBeWord: boolean
-): { pos: number; isValidIdentifier: boolean; stoppedAtDot: boolean } => {
-  let pos = getCharSize(getFullCodePoint(input, 0))
-  let isValidIdentifier = isValidStart
-  let stoppedAtDot = false
-
-  while (true) {
-    const ch = getFullCodePoint(input, pos)
-
-    // Stop at dot if we have a valid identifier (might be property access)
-    if (ch === 46 /* . */ && isValidIdentifier) {
-      stoppedAtDot = true
-      break
-    }
-
-    // Stop if we hit a non-word character
-    if (!isWordChar(ch)) break
-
-    // Context-aware termination: semicolon/colon can end a word if followed by whitespace
-    // This allows `hello; 2` to parse correctly while `hello;world` stays as one word
-    if (canBeWord && (ch === 59 /* ; */ || ch === 58) /* : */) {
-      const nextCh = getFullCodePoint(input, pos + 1)
-      if (!isWordChar(nextCh)) break
-    }
-
-    // Track identifier validity: must be lowercase, digit, dash, or emoji/unicode
-    if (!isIdentChar(ch)) {
-      if (!canBeWord) break
-      isValidIdentifier = false
-    }
-
-    pos += getCharSize(ch)
-  }
-
-  return { pos, isValidIdentifier, stoppedAtDot }
-}
-
-// Consume the rest of a word after we've decided not to treat a dot as DotGet
-// Used when we have "file.txt" - we already consumed "file", now consume ".txt"
-const consumeRestOfWord = (input: InputStream, startPos: number, canBeWord: boolean): number => {
-  let pos = startPos
-  while (true) {
-    const ch = getFullCodePoint(input, pos)
-
-    // Stop if we hit a non-word character
-    if (!isWordChar(ch)) break
-
-    // Context-aware termination for semicolon/colon
-    if (canBeWord && (ch === 59 /* ; */ || ch === 58) /* : */) {
-      const nextCh = getFullCodePoint(input, pos + 1)
-      if (!isWordChar(nextCh)) break
-    }
-
-    pos += getCharSize(ch)
-  }
-  return pos
-}
-
-// Consumes { curly strings } and tracks braces so you can { have { braces { inside { braces } } }
-const consumeCurlyString = (input: InputStream, stack: Stack) => {
-  if (!stack.canShift(CurlyString)) return
-
-  let depth = 0
-  let pos = 0
-
-  while (true) {
-    const ch = input.peek(pos)
-    if (ch < 0) return // EOF - invalid
-
-    if (ch === 123) depth++ // {
-    else if (ch === 125) { // }
-      depth--
-      if (depth === 0) {
-        pos++ // consume final }
-        break
-      }
-    }
-
-    pos++
-  }
-
-  input.acceptToken(CurlyString, pos)
-}
-
-// Check if this identifier is in scope (for property access detection)
-// Returns IdentifierBeforeDot token if in scope, null otherwise
-const checkForDotGet = (input: InputStream, stack: Stack, pos: number): number | null => {
-  const identifierText = buildIdentifierText(input, pos)
-  const context = stack.context as { scope: { has(name: string): boolean } } | undefined
-
-  // Check if identifier is in scope (lexical scope or globals)
-  const inScope = context?.scope.has(identifierText) || globals.includes(identifierText)
-
-  // property access
-  if (inScope) return IdentifierBeforeDot
-
-  // Not in scope - check if we're inside a DotGet chain
-  // Inside the @skip {} block where DotGet is defined, Word cannot be shifted
-  // but Identifier can be. This tells us we're at the RHS of a DotGet.
-  const canShiftIdentifier = stack.canShift(Identifier)
-  const canShiftWord = stack.canShift(Word)
-  const inDotGetChain = canShiftIdentifier && !canShiftWord
-
-  // continue if we're inside a DotGet
-  return inDotGetChain ? IdentifierBeforeDot : null
-}
-
-// Decide between AssignableIdentifier and Identifier using grammar state + peek-ahead
-const chooseIdentifierToken = (input: InputStream, stack: Stack): number => {
-  const canAssignable = stack.canShift(AssignableIdentifier)
-  const canRegular = stack.canShift(Identifier)
-
-  // Only one option is valid - use it
-  if (canAssignable && !canRegular) return AssignableIdentifier
-  if (canRegular && !canAssignable) return Identifier
-
-  // Both possible (ambiguous context) - peek ahead for '=' to disambiguate
-  // This happens at statement start where both `x = 5` (assign) and `echo x` (call) are valid
-  let peekPos = 0
-  while (true) {
-    const ch = getFullCodePoint(input, peekPos)
-    if (isWhiteSpace(ch)) {
-      peekPos += getCharSize(ch)
-    } else {
-      break
-    }
-  }
-
-  const nextCh = getFullCodePoint(input, peekPos)
-  const nextCh2 = getFullCodePoint(input, peekPos + 1)
-  const nextCh3 = getFullCodePoint(input, peekPos + 2)
-
-  // Check for ??= (three-character compound operator)
-  if (nextCh === 63 /* ? */ && nextCh2 === 63 /* ? */ && nextCh3 === 61 /* = */) {
-    const charAfterOp = getFullCodePoint(input, peekPos + 3)
-    if (isWhiteSpace(charAfterOp) || charAfterOp === -1 /* EOF */) {
-      return AssignableIdentifier
-    }
-  }
-
-  // Check for compound assignment operators: +=, -=, *=, /=, %=
-  if (
-    [43 /* + */, 45 /* - */, 42 /* * */, 47 /* / */, 37 /* % */].includes(nextCh) &&
-    nextCh2 === 61 /* = */
-  ) {
-    // Found compound operator, check if it's followed by whitespace
-    const charAfterOp = getFullCodePoint(input, peekPos + 2)
-    if (isWhiteSpace(charAfterOp) || charAfterOp === -1 /* EOF */) {
-      return AssignableIdentifier
-    }
-  }
-
-  if (nextCh === 61 /* = */) {
-    // Found '=', but check if it's followed by whitespace
-    // If '=' is followed by non-whitespace (like '=cool*'), it won't be tokenized as Eq
-    // In that case, this should be Identifier (for function call), not AssignableIdentifier
-    const charAfterEquals = getFullCodePoint(input, peekPos + 1)
-    if (isWhiteSpace(charAfterEquals) || charAfterEquals === -1 /* EOF */) {
-      return AssignableIdentifier
-    }
-  }
-  return Identifier
-}
-
-// Character classification helpers
-export const isIdentStart = (ch: number): boolean => {
-  return isLowercaseLetter(ch) || isEmojiOrUnicode(ch)
-}
-
-export const isIdentChar = (ch: number): boolean => {
-  return isLowercaseLetter(ch) || isDigit(ch) || ch === 45 /* - */ || ch === 63 /* ? */ || isEmojiOrUnicode(ch)
-}
-
-const isWhiteSpace = (ch: number): boolean => {
-  return ch === 32 /* space */ || ch === 9 /* tab */ || ch === 13 /* \r */
-}
-
-const isWordChar = (ch: number): boolean => {
-  return (
-    !isWhiteSpace(ch) &&
-    ch !== 10 /* \n */ &&
-    ch !== 41 /* ) */ &&
-    ch !== 93 /* ] */ &&
-    ch !== -1 /* EOF */
-  )
-}
-
-const isLowercaseLetter = (ch: number): boolean => {
-  return ch >= 97 && ch <= 122 // a-z
-}
-
-const isDigit = (ch: number): boolean => {
-  return ch >= 48 && ch <= 57 // 0-9
-}
-
-const getFullCodePoint = (input: InputStream, pos: number): number => {
-  const ch = input.peek(pos)
-
-  // Check if this is a high surrogate (0xD800-0xDBFF)
-  if (ch >= 0xd800 && ch <= 0xdbff) {
-    const low = input.peek(pos + 1)
-    // Check if next is low surrogate (0xDC00-0xDFFF)
-    if (low >= 0xdc00 && low <= 0xdfff) {
-      // Combine surrogate pair into full code point
-      return 0x10000 + ((ch & 0x3ff) << 10) + (low & 0x3ff)
-    }
-  }
-
-  return ch
-}
-
-const isEmojiOrUnicode = (ch: number): boolean => {
-  return (
-    // Basic Emoticons
-    (ch >= 0x1f600 && ch <= 0x1f64f) ||
-    // Miscellaneous Symbols and Pictographs
-    (ch >= 0x1f300 && ch <= 0x1f5ff) ||
-    // Transport and Map Symbols
-    (ch >= 0x1f680 && ch <= 0x1f6ff) ||
-    // Regional Indicator Symbols (flags)
-    (ch >= 0x1f1e6 && ch <= 0x1f1ff) ||
-    // Miscellaneous Symbols (hearts, stars, weather)
-    (ch >= 0x2600 && ch <= 0x26ff) ||
-    // Dingbats (scissors, pencils, etc)
-    (ch >= 0x2700 && ch <= 0x27bf) ||
-    // Supplemental Symbols and Pictographs (newer emojis)
-    (ch >= 0x1f900 && ch <= 0x1f9ff) ||
-    // Symbols and Pictographs Extended-A (newest emojis)
-    (ch >= 0x1fa70 && ch <= 0x1faff) ||
-    // Various Asian Characters with emoji presentation
-    (ch >= 0x1f018 && ch <= 0x1f270) ||
-    // Variation Selectors (for emoji presentation)
-    (ch >= 0xfe00 && ch <= 0xfe0f) ||
-    // Additional miscellaneous items
-    (ch >= 0x238c && ch <= 0x2454) ||
-    // Combining Diacritical Marks for Symbols
-    (ch >= 0x20d0 && ch <= 0x20ff) ||
-    // Latin-1 Supplement (includes ², ³, ¹ and other special chars)
-    (ch >= 0x00a0 && ch <= 0x00ff) ||
-    // Greek and Coptic (U+0370-U+03FF)
-    (ch >= 0x0370 && ch <= 0x03ff) ||
-    // Mathematical Alphanumeric Symbols (U+1D400-U+1D7FF)
-    (ch >= 0x1d400 && ch <= 0x1d7ff) ||
-    // Mathematical Operators (U+2200-U+22FF)
-    (ch >= 0x2200 && ch <= 0x22ff) ||
-    // Superscripts and Subscripts (U+2070-U+209F)
-    (ch >= 0x2070 && ch <= 0x209f) ||
-    // Arrows (U+2190-U+21FF)
-    (ch >= 0x2190 && ch <= 0x21ff) ||
-    // Hiragana (U+3040-U+309F)
-    (ch >= 0x3040 && ch <= 0x309f) ||
-    // Katakana (U+30A0-U+30FF)
-    (ch >= 0x30a0 && ch <= 0x30ff) ||
-    // CJK Unified Ideographs (U+4E00-U+9FFF)
-    (ch >= 0x4e00 && ch <= 0x9fff)
-  )
-}
-
-const getCharSize = (ch: number) => (ch > 0xffff ? 2 : 1) // emoji takes 2 UTF-16 code units
-
-export const pipeStartsLineTokenizer = new ExternalTokenizer((input: InputStream, stack: Stack) => {
-  const ch = input.peek(0)
-
-  if (ch !== 10 /* \n */) return
-
-  // ignore whitespace
-  let offset = 1
-  let lastNewlineOffset = 0
-
-  while (true) {
-    const ch = input.peek(offset)
-    if (ch === 10 /* \n */) {
-      lastNewlineOffset = offset
-      offset++
-    } else if (isWhiteSpace(ch)) {
-      offset++
-    } else {
-      break
-    }
-  }
-
-  // look for pipe after skipping empty lines
-  if (input.peek(offset) === 124 /* | */) {
-    input.advance(lastNewlineOffset + 1)
-    input.acceptToken(pipeStartsLine)
-  } else {
-    input.advance(1)
-    input.acceptToken(newline)
-  }
-})

src/parser/tokenizer2.ts

@@ -0,0 +1,594 @@
+const DEBUG = process.env.DEBUG || false
+
+export type Token = {
+  type: TokenType
+  value?: string,
+  from: number,
+  to: number,
+}
+
+export enum TokenType {
+  Comment,
+
+  Keyword,
+  Operator,
+
+  Newline,
+  Semicolon,
+  Colon,
+  Underscore,
+
+  OpenParen,
+  CloseParen,
+  OpenBracket,
+  CloseBracket,
+
+  Identifier,
+  Word,
+  NamedArgPrefix,
+
+  Null,
+  Boolean,
+  Number,
+  String,
+  Regex,
+}
+
+const valueTokens = new Set([
+  TokenType.Comment,
+  TokenType.Keyword, TokenType.Operator,
+  TokenType.Identifier, TokenType.Word, TokenType.NamedArgPrefix,
+  TokenType.Boolean, TokenType.Number, TokenType.String, TokenType.Regex,
+  TokenType.Underscore
+])
+
+const operators = new Set([
+  // assignment
+  '=',
+
+  // logic
+  'or',
+  'and',
+
+  // bitwise
+  'band',
+  'bor',
+  'bxor',
+  '>>>',
+  '>>',
+  '<<',
+
+  // compound assignment
+  '??=',
+  '+=',
+  '-=',
+  '*=',
+  '/=',
+  '%=',
+
+  // nullish
+  '??',
+
+  // math
+  '**',
+  '*',
+  '/',
+  '+',
+  '-',
+  '%',
+
+  // comparison
+  '>=',
+  '<=',
+  '!=',
+  '==',
+  '>',
+  '<',
+
+  // property access
+  '.',
+
+  // pipe
+  '|',
+])
+
+const keywords = new Set([
+  'import',
+  'end',
+  'do',
+  'if',
+  'while',
+  'if',
+  'else',
+  'try',
+  'catch',
+  'finally',
+  'throw',
+  'not',
+])
+
+// helper
+function c(strings: TemplateStringsArray, ...values: any[]) {
+  return strings.reduce((result, str, i) => result + str + (values[i] ?? ""), "").charCodeAt(0)
+}
+
+function s(c: number): string {
+  return String.fromCharCode(c)
+}
+
+export class Scanner {
+  input = ''
+  pos = 0
+  start = 0
+  char = 0
+  prev = 0
+  inParen = 0
+  inBracket = 0
+  tokens: Token[] = []
+  prevIsWhitespace = true
+
+  reset() {
+    this.input = ''
+    this.pos = 0
+    this.start = 0
+    this.char = 0
+    this.prev = 0
+    this.tokens.length = 0
+    this.prevIsWhitespace = true
+  }
+
+  peek(count = 0): number {
+    return getFullCodePoint(this.input, this.pos + count)
+  }
+
+  next(): number {
+    this.prevIsWhitespace = isWhitespace(this.char)
+    this.prev = this.char
+    this.char = this.peek()
+    this.pos += getCharSize(this.char)
+
+    return this.char
+  }
+
+  push(type: TokenType, from?: number, to?: number) {
+    from ??= this.start
+    to ??= this.pos - getCharSize(this.char)
+    if (to < from) to = from
+
+    this.tokens.push(Object.assign({}, {
+      type,
+      from,
+      to,
+    }, valueTokens.has(type) ? { value: this.input.slice(from, to) } : {}))
+
+    if (DEBUG) {
+      const tok = this.tokens.at(-1)
+      console.log(`≫ PUSH(${from},${to})`, TokenType[tok?.type || 0], '—', tok?.value)
+    }
+
+    this.start = this.pos
+  }
+
+  pushChar(type: TokenType) {
+    this.push(type, this.pos - 1, this.pos)
+  }
+
+  // turn shrimp code into shrimp tokens that get fed into the parser
+  tokenize(input: string): Token[] {
+    this.reset()
+    this.input = input
+    this.next()
+
+    while (this.char > 0) {
+      const char = this.char
+
+      if (char === c`#`) {
+        this.readComment()
+        continue
+      }
+
+      if (isBracket(char)) {
+        this.readBracket()
+        continue
+      }
+
+      if (isStringDelim(char)) {
+        this.readString(char)
+        continue
+      }
+
+      if (char === c`{`) {
+        this.readCurlyString()
+        continue
+      }
+
+      if (isIdentStart(char)) {
+        this.readWordOrIdent(true) // true = started with identifier char
+        continue
+      }
+
+      if (isDigit(char) || ((char === c`-` || char === c`+`) && isDigit(this.peek()))) {
+        this.readNumber()
+        continue
+      }
+
+      if (char === c`:`) {
+        this.pushChar(TokenType.Colon)
+        this.next()
+        continue
+      }
+
+      // whitespace-sensitive dot as operator (property access) only after identifier/number
+      if (char === c`.`) {
+        if (this.canBeDotGet(this.tokens.at(-1))) {
+          this.pushChar(TokenType.Operator)
+          this.next()
+          continue
+        }
+      }
+
+      if (char === c`/` && this.peek() === c`/`) {
+        this.readRegex()
+        continue
+      }
+
+      if (isWordChar(char)) {
+        this.readWordOrIdent(false) // false = didn't start with identifier char
+        continue
+      }
+
+      if (char === c`\n`) {
+        if (this.inParen === 0 && this.inBracket === 0)
+          this.pushChar(TokenType.Newline)
+        this.next()
+        continue
+      }
+
+      if (char === c`;`) {
+        this.pushChar(TokenType.Semicolon)
+        this.next()
+        continue
+      }
+
+      this.next()
+    }
+
+    return this.tokens
+  }
+
+  readComment() {
+    this.start = this.pos - 1
+    while (this.char !== c`\n` && this.char > 0) this.next()
+    this.push(TokenType.Comment)
+  }
+
+  readBracket() {
+    switch (this.char) {
+      case c`(`:
+        this.inParen++
+        this.pushChar(TokenType.OpenParen); break
+      case c`)`:
+        this.inParen--
+        this.pushChar(TokenType.CloseParen); break
+      case c`[`:
+        this.inBracket++
+        this.pushChar(TokenType.OpenBracket); break
+      case c`]`:
+        this.inBracket--
+        this.pushChar(TokenType.CloseBracket); break
+    }
+    this.next()
+  }
+
+  readString(delim: number) {
+    this.start = this.pos - 1
+    this.next() // skip opening delim
+    while (this.char > 0 && (this.char !== delim || (this.char === delim && this.prev === c`\\`)))
+      this.next()
+    this.next() // skip closing delim
+
+    this.push(TokenType.String)
+  }
+
+  readCurlyString() {
+    this.start = this.pos - 1
+    let depth = 1
+    this.next()
+
+    while (depth > 0 && this.char > 0) {
+      if (this.char === c`{`) depth++
+      if (this.char === c`}`) depth--
+      this.next()
+    }
+
+    this.push(TokenType.String)
+  }
+
+  readWordOrIdent(startedWithIdentChar: boolean) {
+    this.start = this.pos - getCharSize(this.char)
+
+    while (isWordChar(this.char)) {
+      // stop at colon if followed by whitespace (e.g., 'do x: echo x end')
+      if (this.char === c`:`) {
+        const nextCh = this.peek()
+        if (isWhitespace(nextCh) || nextCh === 0) break
+      }
+
+      // stop at equal sign (named arg) - but only if what we've read so far is an identifier
+      if (this.char === c`=`) {
+        const soFar = this.input.slice(this.start, this.pos - getCharSize(this.char))
+        if (isIdentifer(soFar)) {
+          this.next()
+          break
+        }
+      }
+
+      // stop at dot only if it would create a valid property access
+      // AND only if we started with an identifier character (not for Words like README.txt)
+      if (startedWithIdentChar && this.char === c`.`) {
+        const nextCh = this.peek()
+        if (isIdentStart(nextCh) || isDigit(nextCh) || nextCh === c`(`) {
+          const soFar = this.input.slice(this.start, this.pos - getCharSize(this.char))
+          if (isIdentifer(soFar)) break
+        }
+      }
+
+      this.next()
+    }
+
+    const word = this.input.slice(this.start, this.pos - getCharSize(this.char))
+
+    // classify the token based on what we read
+    if (word === '_')
+      this.push(TokenType.Underscore)
+
+    else if (word === 'null')
+      this.push(TokenType.Null)
+
+    else if (word === 'true' || word === 'false')
+      this.push(TokenType.Boolean)
+
+    else if (isKeyword(word))
+      this.push(TokenType.Keyword)
+
+    else if (isOperator(word))
+      this.push(TokenType.Operator)
+
+    else if (isIdentifer(word))
+      this.push(TokenType.Identifier)
+
+    else if (word.endsWith('='))
+      this.push(TokenType.NamedArgPrefix)
+
+    else
+      this.push(TokenType.Word)
+  }
+
+  readNumber() {
+    this.start = this.pos - 1
+    while (isWordChar(this.char)) {
+      // stop at dot unless it's part of the number
+      if (this.char === c`.`) {
+        const nextCh = this.peek()
+        if (!isDigit(nextCh)) break
+      }
+
+      // stop at colon
+      if (this.char === c`:`) {
+        const nextCh = this.peek()
+        if (isWhitespace(nextCh) || nextCh === 0) break
+      }
+      this.next()
+    }
+    const ident = this.input.slice(this.start, this.pos - 1)
+    this.push(isNumber(ident) ? TokenType.Number : TokenType.Word)
+  }
+
+  readRegex() {
+    this.start = this.pos - 1
+    this.next() // skip 2nd /
+
+    while (this.char > 0) {
+      if (this.char === c`/` && this.peek() === c`/`) {
+        this.next() // skip /
+        this.next() // skip /
+
+        // read regex flags
+        while (this.char > 0 && isIdentStart(this.char))
+          this.next()
+
+        // validate regex
+        const to = this.pos - getCharSize(this.char)
+        const regexText = this.input.slice(this.start, to)
+        const [_, pattern, flags] = regexText.match(/^\/\/(.*)\/\/([gimsuy]*)$/) || []
+
+        if (pattern) {
+          try {
+            new RegExp(pattern, flags)
+            this.push(TokenType.Regex)
+            break
+          } catch (e) {
+            // invalid regex - fall through to Word
+          }
+        }
+
+        // invalid regex is treated as Word
+        this.push(TokenType.Word)
+        break
+      }
+
+      this.next()
+    }
+  }
+
+  canBeDotGet(lastToken?: Token): boolean {
+    return !this.prevIsWhitespace && !!lastToken &&
+      (lastToken.type === TokenType.Identifier ||
+        lastToken.type === TokenType.Number ||
+        lastToken.type === TokenType.CloseParen ||
+        lastToken.type === TokenType.CloseBracket)
+  }
+}
+
+const isNumber = (word: string): boolean => {
+  // regular number
+  if (/^[+-]?\d+(_?\d+)*(\.(\d+(_?\d+)*))?$/.test(word))
+    return true
+
+  // binary
+  if (/^[+-]?0b[01]+(_?[01]+)*(\.[01](_?[01]*))?$/.test(word))
+    return true
+
+  // octal
+  if (/^[+-]?0o[0-7]+(_?[0-7]+)*(\.[0-7](_?[0-7]*))?$/.test(word))
+    return true
+
+  // hex
+  if (/^[+-]?0x[0-9a-f]+([0-9a-f]_?[0-9a-f]+)*(\.([0-9a-f]_?[0-9a-f]*))?$/i.test(word))
+    return true
+
+  return false
+}
+
+const isIdentifer = (s: string): boolean => {
+  if (s.length === 0) return false
+
+  let pos = 0
+  const chars = []
+  while (pos < s.length) {
+    const out = getFullCodePoint(s, pos)
+    pos += getCharSize(out)
+    chars.push(out)
+  }
+
+  if (chars.length === 1)
+    return isIdentStart(chars[0]!)
+  else if (chars.length === 2)
+    return isIdentStart(chars[0]!) && isIdentEnd(chars[1]!)
+  else
+    return isIdentStart(chars[0]!) &&
+      chars.slice(1, chars.length - 1).every(isIdentChar) &&
+      isIdentEnd(chars.at(-1)!)
+}
+
+const isStringDelim = (ch: number): boolean => {
+  return ch === c`'` || ch === c`"`
+}
+
+export const isIdentStart = (char: number | string): boolean => {
+  let ch = typeof char === 'string' ? char.charCodeAt(0) : char
+  return isLowercaseLetter(ch) || isEmojiOrUnicode(ch) || ch === 36 /* $ */
+}
+
+export const isIdentChar = (char: number | string): boolean => {
+  let ch = typeof char === 'string' ? char.charCodeAt(0) : char
+  return isIdentStart(ch) || isDigit(ch) || ch === 45 /* - */ || ch === 63 /* ? */
+}
+
+const isIdentEnd = (char: number | string): boolean => {
+  return isIdentChar(char)
+}
+
+const isLowercaseLetter = (ch: number): boolean => {
+  return ch >= 97 && ch <= 122 // a-z
+}
+
+const isDigit = (ch: number): boolean => {
+  return ch >= 48 && ch <= 57 // 0-9
+}
+
+const isWhitespace = (ch: number): boolean => {
+  return ch === 32 /* space */ || ch === 9 /* tab */ ||
+    ch === 13 /* \r */ || ch === 10 /* \n */ ||
+    ch === -1 || ch === 0 /* EOF */
+}
+
+const isWordChar = (ch: number): boolean => {
+  return (
+    !isWhitespace(ch) &&
+    ch !== 10 /* \n */ &&
+    ch !== 59 /* ; */ &&
+    ch !== 40 /* ( */ &&
+    ch !== 41 /* ) */ &&
+    ch !== 93 /* ] */ &&
+    ch !== -1 /* EOF */
+  )
+}
+
+const isOperator = (word: string): boolean => {
+  return operators.has(word)
+}
+
+const isKeyword = (word: string): boolean => {
+  return keywords.has(word)
+}
+
+const isBracket = (char: number): boolean => {
+  return char === c`(` || char === c`)` || char === c`[` || char === c`]`
+}
+
+const getCharSize = (ch: number) =>
+  (ch > 0xffff ? 2 : 1) // emoji takes 2 UTF-16 code units
+
+const getFullCodePoint = (input: string, pos: number): number => {
+  const ch = input[pos]?.charCodeAt(0) || 0
+
+  // Check if this is a high surrogate (0xD800-0xDBFF)
+  if (ch >= 0xd800 && ch <= 0xdbff) {
+    const low = input[pos + 1]?.charCodeAt(0) || 0
+    // Check if next is low surrogate (0xDC00-0xDFFF)
+    if (low >= 0xdc00 && low <= 0xdfff) {
+      // Combine surrogate pair into full code point
+      return 0x10000 + ((ch & 0x3ff) << 10) + (low & 0x3ff)
+    }
+  }
+
+  return ch
+}
+
+const isEmojiOrUnicode = (ch: number): boolean => {
+  return (
+    // Basic Emoticons
+    (ch >= 0x1f600 && ch <= 0x1f64f) ||
+    // Miscellaneous Symbols and Pictographs
+    (ch >= 0x1f300 && ch <= 0x1f5ff) ||
+    // Transport and Map Symbols
+    (ch >= 0x1f680 && ch <= 0x1f6ff) ||
+    // Regional Indicator Symbols (flags)
+    (ch >= 0x1f1e6 && ch <= 0x1f1ff) ||
+    // Miscellaneous Symbols (hearts, stars, weather)
+    (ch >= 0x2600 && ch <= 0x26ff) ||
+    // Dingbats (scissors, pencils, etc)
+    (ch >= 0x2700 && ch <= 0x27bf) ||
+    // Supplemental Symbols and Pictographs (newer emojis)
+    (ch >= 0x1f900 && ch <= 0x1f9ff) ||
+    // Symbols and Pictographs Extended-A (newest emojis)
+    (ch >= 0x1fa70 && ch <= 0x1faff) ||
+    // Various Asian Characters with emoji presentation
+    (ch >= 0x1f018 && ch <= 0x1f270) ||
+    // Variation Selectors (for emoji presentation)
+    (ch >= 0xfe00 && ch <= 0xfe0f) ||
+    // Additional miscellaneous items
+    (ch >= 0x238c && ch <= 0x2454) ||
+    // Combining Diacritical Marks for Symbols
+    (ch >= 0x20d0 && ch <= 0x20ff) ||
+    // Latin-1 Supplement (includes ², ³, ¹ and other special chars)
+    (ch >= 0x00a0 && ch <= 0x00ff) ||
+    // Greek and Coptic (U+0370-U+03FF)
+    (ch >= 0x0370 && ch <= 0x03ff) ||
+    // Mathematical Alphanumeric Symbols (U+1D400-U+1D7FF)
+    (ch >= 0x1d400 && ch <= 0x1d7ff) ||
+    // Mathematical Operators (U+2200-U+22FF)
+    (ch >= 0x2200 && ch <= 0x22ff) ||
+    // Superscripts and Subscripts (U+2070-U+209F)
+    (ch >= 0x2070 && ch <= 0x209f) ||
+    // Arrows (U+2190-U+21FF)
+    (ch >= 0x2190 && ch <= 0x21ff) ||
+    // Hiragana (U+3040-U+309F)
+    (ch >= 0x3040 && ch <= 0x309f) ||
+    // Katakana (U+30A0-U+30FF)
+    (ch >= 0x30a0 && ch <= 0x30ff) ||
+    // CJK Unified Ideographs (U+4E00-U+9FFF)
+    (ch >= 0x4e00 && ch <= 0x9fff)
+  )
+}

src/prelude/date.ts

@@ -0,0 +1,12 @@
+export const date = {
+  now: () => Date.now(),
+  year: (time: number) => (new Date(time)).getFullYear(),
+  month: (time: number) => (new Date(time)).getMonth(),
+  date: (time: number) => (new Date(time)).getDate(),
+  hour: (time: number) => (new Date(time)).getHours(),
+  minute: (time: number) => (new Date(time)).getMinutes(),
+  second: (time: number) => (new Date(time)).getSeconds(),
+  ms: (time: number) => (new Date(time)).getMilliseconds(),
+  new: (year: number, month: number, day: number, hour = 0, minute = 0, second = 0, ms = 0) =>
+    new Date(year, month, day, hour, minute, second, ms).getTime()
+}

src/prelude/index.ts

@@ -6,6 +6,7 @@ import {
   extractParamInfo, isWrapped, getOriginalFunction,
 } from 'reefvm'
 
+import { date } from './date'
 import { dict } from './dict'
 import { fs } from './fs'
 import { json } from './json'
@@ -13,8 +14,10 @@ import { load } from './load'
 import { list } from './list'
 import { math } from './math'
 import { str } from './str'
+import { types } from './types'
 
-export const globals = {
+export const globals: Record<string, any> = {
+  date,
   dict,
   fs,
   json,
@@ -34,7 +37,6 @@ export const globals = {
       name: Bun.argv[2] || '(shrimp)',
       path: resolve(join('.', Bun.argv[2] ?? ''))
     },
-
   },
 
   // hello
@@ -84,20 +86,9 @@ export const globals = {
   exit: (num: number) => process.exit(num ?? 0),
 
   // type predicates
-  'string?': (v: any) => toValue(v).type === 'string',
-  'number?': (v: any) => toValue(v).type === 'number',
-  'boolean?': (v: any) => toValue(v).type === 'boolean',
-  'array?': (v: any) => toValue(v).type === 'array',
-  'dict?': (v: any) => toValue(v).type === 'dict',
-  'function?': (v: any) => {
-    const t = toValue(v).type
-    return t === 'function' || t === 'native'
-  },
-  'null?': (v: any) => toValue(v).type === 'null',
   'some?': (v: any) => toValue(v).type !== 'null',
 
   // boolean/logic
-  not: (v: any) => !v,
   bnot: (n: number) => ~(n | 0),
 
   // utilities
@@ -191,8 +182,8 @@ export function formatValue(value: Value, inner = false): string {
       return `${colors.blue}[${colors.reset}${items}${colors.blue}]${colors.reset}`
     }
     case 'dict': {
-      const entries = Array.from(value.value.entries())
+      const entries = Array.from(value.value.entries()).reverse()
-        .map(([k, v]) => `${k}${colors.blue}=${colors.reset}${formatValue(v, true)}`)
+        .map(([k, v]) => `${k.trim()}${colors.blue}=${colors.reset}${formatValue(v, true)}`)
         .join(' ')
       if (entries.length === 0)
         return `${colors.blue}[=]${colors.reset}`
@@ -213,3 +204,7 @@ export function formatValue(value: Value, inner = false): string {
       return String(value)
   }
 }
+
+// add types functions to top-level namespace
+for (const [key, value] of Object.entries(types))
+  globals[key] = value

src/prelude/math.ts

@@ -16,7 +16,10 @@ export const math = {
     if (n < 0) throw new Error(`sqrt: cannot take square root of negative number ${n}`)
     return Math.sqrt(n)
   },
-  random: () => Math.random(),
+  random: (min = 0, max = 1) => {
+    if (min === 0 && max === 1) return Math.random()
+    return Math.floor(Math.random() * (max - min + 1)) + min
+  },
   clamp: (n: number, min: number, max: number) => {
     if (min > max) throw new Error(`clamp: min (${min}) must be less than or equal to max (${max})`)
     return Math.min(Math.max(n, min), max)

src/prelude/str.ts

@@ -28,6 +28,16 @@ export const str = {
   },
   'pad-start': (str: string, length: number, pad: string = ' ') => String(str ?? '').padStart(length, pad),
   'pad-end': (str: string, length: number, pad: string = ' ') => String(str ?? '').padEnd(length, pad),
+  capitalize: (str: string) => {
+    const s = String(str ?? '')
+    return s.charAt(0).toUpperCase() + s.slice(1).toLowerCase()
+  },
+  titlecase: (s: string) => {
+    return String(s ?? '')
+      .split(' ')
+      .map(str.capitalize)
+      .join(' ')
+  },
   lines: (str: string) => String(str ?? '').split('\n'),
   chars: (str: string) => String(str ?? '').split(''),
 

src/prelude/tests/date.test.ts

@@ -0,0 +1,170 @@
+import { expect, describe, test } from 'bun:test'
+
+describe('date', () => {
+  test('date.now returns current timestamp', () => {
+    expect(`date.now | number?`).toEvaluateTo(true)
+
+    expect(`(date.now) > 1577836800000`).toEvaluateTo(true)
+  })
+
+  test('date.new creates timestamp from components', () => {
+    expect(`
+      t = date.new 2024 0 1 12 0 0 500
+      [
+        (date.year t)
+        (date.month t)
+        (date.date t)
+        (date.hour t)
+        (date.minute t)
+        (date.second t)
+        (date.ms t)
+      ]
+    `).toEvaluateTo([2024, 0, 1, 12, 0, 0, 500])
+  })
+
+  test('date.new with minimal arguments', () => {
+    expect(`
+      t = date.new 2024 5 15
+      [
+        (date.year t)
+        (date.month t)
+        (date.date t)
+        (date.hour t)
+        (date.minute t)
+        (date.second t)
+        (date.ms t)
+      ]
+    `).toEvaluateTo([2024, 5, 15, 0, 0, 0, 0])
+  })
+
+  test('date.year extracts year', () => {
+    expect(`
+      t = date.new 2024 0 1
+      date.year t
+    `).toEvaluateTo(2024)
+
+    expect(`
+      t = date.new 1999 11 31
+      date.year t
+    `).toEvaluateTo(1999)
+  })
+
+  test('date.month extracts month (0-indexed)', () => {
+    // January = 0, December = 11
+    expect(`
+      jan = date.new 2024 0 1
+      dec = date.new 2024 11 31
+      [(date.month jan) (date.month dec)]
+    `).toEvaluateTo([0, 11])
+  })
+
+  test('date.date extracts day of month', () => {
+    expect(`
+      t = date.new 2024 5 15
+      date.date t
+    `).toEvaluateTo(15)
+
+    expect(`
+      date.new 2024 0 1 | date.date
+    `).toEvaluateTo(1)
+  })
+
+  test('date.hour extracts hour', () => {
+    expect(`
+      t = date.new 2024 0 1 14 30 45
+      date.hour t
+    `).toEvaluateTo(14)
+
+    expect(`
+      t = date.new 2024 0 1 0 0 0
+      date.hour t
+    `).toEvaluateTo(0)
+  })
+
+  test('date.minute extracts minute', () => {
+    expect(`
+      t = date.new 2024 0 1 14 30 45
+      date.minute t
+    `).toEvaluateTo(30)
+  })
+
+  test('date.second extracts second', () => {
+    expect(`
+      t = date.new 2024 0 1 14 30 45
+      date.second t
+    `).toEvaluateTo(45)
+  })
+
+  test('date.ms extracts milliseconds', () => {
+    expect(`
+      t = date.new 2024 0 1 14 30 45 250
+      date.ms t
+    `).toEvaluateTo(250)
+  })
+
+  test('round-trip: create and extract components', () => {
+    expect(`
+      t = date.new 2024 6 4 15 30 45 123
+      year = date.year t
+      month = date.month t
+      day = date.date t
+      hour = date.hour t
+      min = date.minute t
+      sec = date.second t
+      ms = date.ms t
+      [year month day hour min sec ms]
+    `).toEvaluateTo([2024, 6, 4, 15, 30, 45, 123])
+  })
+
+  test('edge cases - midnight', () => {
+    expect(`
+      t = date.new 2024 0 1 0 0 0 0
+      [
+        (date.hour t)
+        (date.minute t)
+        (date.second t)
+        (date.ms t)
+      ]
+    `).toEvaluateTo([0, 0, 0, 0])
+  })
+
+  test('edge cases - end of day', () => {
+    expect(`
+      t = date.new 2024 0 1 23 59 59 999
+      [
+        (date.hour t)
+        (date.minute t)
+        (date.second t)
+        (date.ms t)
+      ]
+    `).toEvaluateTo([23, 59, 59, 999])
+  })
+
+  test('edge cases - leap year', () => {
+    expect(`
+      t = date.new 2024 1 29
+      [
+        (date.year t)
+        (date.month t)
+        (date.date t)
+      ]
+    `).toEvaluateTo([2024, 1, 29])
+  })
+
+  test('combining date functions with arithmetic', () => {
+    expect(`
+      t = date.new 2024 5 15 10 30 0
+      next-hour = date.new 2024 5 15 11 30 0
+      (date.hour next-hour) - (date.hour t)
+    `).toEvaluateTo(1)
+  })
+
+  test('using date.now in calculations', () => {
+    // Check that date.now is in the past compared to a future timestamp
+    expect(`
+      now = (date.now)
+      future = date.new 2030 0 1
+      future > now
+    `).toEvaluateTo(true)
+  })
+})

src/prelude/tests/info.test.ts

@@ -1,90 +1,89 @@
 import { expect, describe, test } from 'bun:test'
-import { globals } from '#prelude'
 
 describe('var and var?', () => {
   test('var? checks if a variable exists', async () => {
-    await expect(`var? 'nada'`).toEvaluateTo(false, globals)
+    await expect(`var? 'nada'`).toEvaluateTo(false)
-    await expect(`var? 'info'`).toEvaluateTo(false, globals)
+    await expect(`var? 'info'`).toEvaluateTo(false)
-    await expect(`abc = abc; var? 'abc'`).toEvaluateTo(true, globals)
+    await expect(`abc = abc; var? 'abc'`).toEvaluateTo(true)
-    await expect(`var? 'var?'`).toEvaluateTo(true, globals)
+    await expect(`var? 'var?'`).toEvaluateTo(true)
 
-    await expect(`var? 'dict'`).toEvaluateTo(true, globals)
+    await expect(`var? 'dict'`).toEvaluateTo(true)
-    await expect(`var? dict`).toEvaluateTo(true, globals)
+    await expect(`var? dict`).toEvaluateTo(true)
   })
 
   test('var returns a value or null', async () => {
-    await expect(`var 'nada'`).toEvaluateTo(null, globals)
+    await expect(`var 'nada'`).toEvaluateTo(null)
-    await expect(`var nada`).toEvaluateTo(null, globals)
+    await expect(`var nada`).toEvaluateTo(null)
-    await expect(`var 'info'`).toEvaluateTo(null, globals)
+    await expect(`var 'info'`).toEvaluateTo(null)
-    await expect(`abc = my-string; var 'abc'`).toEvaluateTo('my-string', globals)
+    await expect(`abc = my-string; var 'abc'`).toEvaluateTo('my-string')
-    await expect(`abc = my-string; var abc`).toEvaluateTo(null, globals)
+    await expect(`abc = my-string; var abc`).toEvaluateTo(null)
   })
 })
 
 describe('type predicates', () => {
   test('string? checks for string type', async () => {
-    await expect(`string? 'hello'`).toEvaluateTo(true, globals)
+    await expect(`string? 'hello'`).toEvaluateTo(true)
-    await expect(`string? 42`).toEvaluateTo(false, globals)
+    await expect(`string? 42`).toEvaluateTo(false)
   })
 
   test('number? checks for number type', async () => {
-    await expect(`number? 42`).toEvaluateTo(true, globals)
+    await expect(`number? 42`).toEvaluateTo(true)
-    await expect(`number? 'hello'`).toEvaluateTo(false, globals)
+    await expect(`number? 'hello'`).toEvaluateTo(false)
   })
 
   test('boolean? checks for boolean type', async () => {
-    await expect(`boolean? true`).toEvaluateTo(true, globals)
+    await expect(`boolean? true`).toEvaluateTo(true)
-    await expect(`boolean? 42`).toEvaluateTo(false, globals)
+    await expect(`boolean? 42`).toEvaluateTo(false)
   })
 
   test('array? checks for array type', async () => {
-    await expect(`array? [1 2 3]`).toEvaluateTo(true, globals)
+    await expect(`array? [1 2 3]`).toEvaluateTo(true)
-    await expect(`array? 42`).toEvaluateTo(false, globals)
+    await expect(`array? 42`).toEvaluateTo(false)
   })
 
   test('dict? checks for dict type', async () => {
-    await expect(`dict? [a=1]`).toEvaluateTo(true, globals)
+    await expect(`dict? [a=1]`).toEvaluateTo(true)
-    await expect(`dict? []`).toEvaluateTo(false, globals)
+    await expect(`dict? []`).toEvaluateTo(false)
   })
 
   test('null? checks for null type', async () => {
-    await expect(`null? null`).toEvaluateTo(true, globals)
+    await expect(`null? null`).toEvaluateTo(true)
-    await expect(`null? 42`).toEvaluateTo(false, globals)
+    await expect(`null? 42`).toEvaluateTo(false)
   })
 
   test('some? checks for non-null', async () => {
-    await expect(`some? 42`).toEvaluateTo(true, globals)
+    await expect(`some? 42`).toEvaluateTo(true)
-    await expect(`some? null`).toEvaluateTo(false, globals)
+    await expect(`some? null`).toEvaluateTo(false)
   })
 })
 
 describe('introspection', () => {
   test('type returns proper types', async () => {
-    await expect(`type 'hello'`).toEvaluateTo('string', globals)
+    await expect(`type 'hello'`).toEvaluateTo('string')
-    await expect(`type 42`).toEvaluateTo('number', globals)
+    await expect(`type 42`).toEvaluateTo('number')
-    await expect(`type true`).toEvaluateTo('boolean', globals)
+    await expect(`type true`).toEvaluateTo('boolean')
-    await expect(`type false`).toEvaluateTo('boolean', globals)
+    await expect(`type false`).toEvaluateTo('boolean')
-    await expect(`type null`).toEvaluateTo('null', globals)
+    await expect(`type null`).toEvaluateTo('null')
-    await expect(`type [1 2 3]`).toEvaluateTo('array', globals)
+    await expect(`type [1 2 3]`).toEvaluateTo('array')
-    await expect(`type [a=1 b=2]`).toEvaluateTo('dict', globals)
+    await expect(`type [a=1 b=2]`).toEvaluateTo('dict')
   })
 
   test('inspect formats values', async () => {
-    await expect(`inspect 'hello'`).toEvaluateTo("\u001b[32m'hello\u001b[32m'\u001b[0m", globals)
+    await expect(`inspect 'hello'`).toEvaluateTo("\u001b[32m'hello\u001b[32m'\u001b[0m")
   })
 
   test('describe describes values', async () => {
-    await expect(`describe 'hello'`).toEvaluateTo("#<string: \u001b[32m'hello\u001b[32m'\u001b[0m>", globals)
+    await expect(`describe 'hello'`).toEvaluateTo("#<string: \u001b[32m'hello\u001b[32m'\u001b[0m>")
   })
 })
 
 describe('environment', () => {
   test('args is an array', async () => {
-    await expect(`array? $.args`).toEvaluateTo(true, globals)
+    await expect(`array? $.args`).toEvaluateTo(true)
   })
 
   test('args can be accessed', async () => {
-    await expect(`type $.args`).toEvaluateTo('array', globals)
+    await expect(`type $.args`).toEvaluateTo('array')
   })
 
   test('argv includes more than just the args', async () => {
@@ -106,35 +105,35 @@ describe('ref', () => {
 
 describe('$ global dictionary', () => {
   test('$.args is an array', async () => {
-    await expect(`$.args | array?`).toEvaluateTo(true, globals)
+    await expect(`$.args | array?`).toEvaluateTo(true)
   })
 
   test('$.args can be accessed', async () => {
-    await expect(`$.args | type`).toEvaluateTo('array', globals)
+    await expect(`$.args | type`).toEvaluateTo('array')
   })
 
   test('$.script.name is a string', async () => {
-    await expect(`$.script.name | string?`).toEvaluateTo(true, globals)
+    await expect(`$.script.name | string?`).toEvaluateTo(true)
   })
 
   test('$.script.path is a string', async () => {
-    await expect(`$.script.path | string?`).toEvaluateTo(true, globals)
+    await expect(`$.script.path | string?`).toEvaluateTo(true)
   })
 
   test('$.env is a dict', async () => {
-    await expect(`$.env | dict?`).toEvaluateTo(true, globals)
+    await expect(`$.env | dict?`).toEvaluateTo(true)
   })
 
   test('$.pid is a number', async () => {
-    await expect(`$.pid | number?`).toEvaluateTo(true, globals)
+    await expect(`$.pid | number?`).toEvaluateTo(true)
-    await expect(`$.pid > 0`).toEvaluateTo(true, globals)
+    await expect(`$.pid > 0`).toEvaluateTo(true)
   })
 
   test('$.cwd is a string', async () => {
-    await expect(`$.cwd | string?`).toEvaluateTo(true, globals)
+    await expect(`$.cwd | string?`).toEvaluateTo(true)
   })
 
   test('$.cwd returns current working directory', async () => {
-    await expect(`$.cwd`).toEvaluateTo(process.cwd(), globals)
+    await expect(`$.cwd`).toEvaluateTo(process.cwd())
   })
 })

src/prelude/tests/load.test.ts

@@ -1,42 +1,41 @@
 import { expect, describe, test } from 'bun:test'
-import { globals } from '#prelude'
 
 describe('loading a file', () => {
   test(`imports all a file's functions`, async () => {
     expect(`
       math = load ./src/prelude/tests/math.sh
       math.double 4
-    `).toEvaluateTo(8, globals)
+    `).toEvaluateTo(8)
 
     expect(`
       math = load ./src/prelude/tests/math.sh
       math.double (math.double 4)
-    `).toEvaluateTo(16, globals)
+    `).toEvaluateTo(16)
 
     expect(`
       math = load ./src/prelude/tests/math.sh
       dbl = ref math.double
       dbl (dbl 2)
-    `).toEvaluateTo(8, globals)
+    `).toEvaluateTo(8)
 
     expect(`
       math = load ./src/prelude/tests/math.sh
       math.pi
-    `).toEvaluateTo(3.14, globals)
+    `).toEvaluateTo(3.14)
 
     expect(`
       math = load ./src/prelude/tests/math.sh
       math | at 🥧
-    `).toEvaluateTo(3.14159265359, globals)
+    `).toEvaluateTo(3.14159265359)
 
     expect(`
       math = load ./src/prelude/tests/math.sh
       math.🥧
-    `).toEvaluateTo(3.14159265359, globals)
+    `).toEvaluateTo(3.14159265359)
 
     expect(`
       math = load ./src/prelude/tests/math.sh
       math.add1 5
-    `).toEvaluateTo(6, globals)
+    `).toEvaluateTo(6)
   })
 })

src/prelude/tests/prelude.test.ts

@@ -1,5 +1,4 @@
 import { expect, describe, test } from 'bun:test'
-import { globals } from '#prelude'
 
 describe('string operations', () => {
   test('to-upper converts to uppercase', async () => {
@@ -17,6 +16,18 @@ describe('string operations', () => {
     await expect(`str.trim '\\n\\thello\\t\\n'`).toEvaluateTo('hello')
   })
 
+  test('capitalize makes first char uppercase', async () => {
+    await expect(`str.capitalize 'hello'`).toEvaluateTo('Hello')
+    await expect(`str.capitalize 'HELLO'`).toEvaluateTo('Hello')
+    await expect(`str.capitalize 'hello world'`).toEvaluateTo('Hello world')
+  })
+
+  test('titlecase capitalizes each word', async () => {
+    await expect(`str.titlecase 'hello world'`).toEvaluateTo('Hello World')
+    await expect(`str.titlecase 'HELLO WORLD'`).toEvaluateTo('Hello World')
+    await expect(`str.titlecase 'the quick brown fox'`).toEvaluateTo('The Quick Brown Fox')
+  })
+
   test('split divides string by separator', async () => {
     await expect(`str.split 'a,b,c' ','`).toEvaluateTo(['a', 'b', 'c'])
     await expect(`str.split 'hello' ''`).toEvaluateTo(['h', 'e', 'l', 'l', 'o'])
@@ -106,6 +117,17 @@ describe('boolean logic', () => {
     await expect(`not 42`).toEvaluateTo(false)
     await expect(`not null`).toEvaluateTo(true)
   })
+
+  test('not works with function calls', async () => {
+    await expect(`equals = do x y: x == y end; not equals 5 5`).toEvaluateTo(false)
+    await expect(`equals = do x y: x == y end; not equals 5 10`).toEvaluateTo(true)
+  })
+
+  test('not works with binary operations and comparisons', async () => {
+    await expect(`not 5 > 10`).toEvaluateTo(true)
+    await expect(`not 10 > 5`).toEvaluateTo(false)
+    await expect(`not true and false`).toEvaluateTo(true)
+  })
 })
 
 describe('utilities', () => {

src/prelude/tests/types.test.ts

@@ -0,0 +1,143 @@
+import { expect, describe, test } from 'bun:test'
+
+describe('type predicates', () => {
+  test('boolean? checks if value is boolean', async () => {
+    await expect(`boolean? true`).toEvaluateTo(true)
+    await expect(`boolean? false`).toEvaluateTo(true)
+    await expect(`boolean? 42`).toEvaluateTo(false)
+    await expect(`boolean? 'hello'`).toEvaluateTo(false)
+    await expect(`boolean? null`).toEvaluateTo(false)
+    await expect(`boolean? [1 2 3]`).toEvaluateTo(false)
+  })
+
+  test('number? checks if value is number', async () => {
+    await expect(`number? 42`).toEvaluateTo(true)
+    await expect(`number? 3.14`).toEvaluateTo(true)
+    await expect(`number? 0`).toEvaluateTo(true)
+    await expect(`number? -5`).toEvaluateTo(true)
+    await expect(`number? 'hello'`).toEvaluateTo(false)
+    await expect(`number? true`).toEvaluateTo(false)
+    await expect(`number? null`).toEvaluateTo(false)
+  })
+
+  test('string? checks if value is string', async () => {
+    await expect(`string? 'hello'`).toEvaluateTo(true)
+    await expect(`string? ''`).toEvaluateTo(true)
+    await expect(`string? world`).toEvaluateTo(true)
+    await expect(`string? 42`).toEvaluateTo(false)
+    await expect(`string? true`).toEvaluateTo(false)
+    await expect(`string? null`).toEvaluateTo(false)
+    await expect(`string? [1 2 3]`).toEvaluateTo(false)
+  })
+
+  test('array? checks if value is array', async () => {
+    await expect(`array? [1 2 3]`).toEvaluateTo(true)
+    await expect(`array? []`).toEvaluateTo(true)
+    await expect(`array? ['a' 'b']`).toEvaluateTo(true)
+    await expect(`array? [a=1 b=2]`).toEvaluateTo(false)
+    await expect(`array? 42`).toEvaluateTo(false)
+    await expect(`array? 'hello'`).toEvaluateTo(false)
+    await expect(`array? null`).toEvaluateTo(false)
+  })
+
+  test('list? is alias for array?', async () => {
+    await expect(`list? [1 2 3]`).toEvaluateTo(true)
+    await expect(`list? []`).toEvaluateTo(true)
+    await expect(`list? [a=1 b=2]`).toEvaluateTo(false)
+  })
+
+  test('dict? checks if value is dict', async () => {
+    await expect(`dict? [a=1 b=2]`).toEvaluateTo(true)
+    await expect(`dict? [=]`).toEvaluateTo(true)
+    await expect(`dict? [1 2 3]`).toEvaluateTo(false)
+    await expect(`dict? []`).toEvaluateTo(false)
+    await expect(`dict? 42`).toEvaluateTo(false)
+    await expect(`dict? 'hello'`).toEvaluateTo(false)
+  })
+
+  test('function? checks if value is function', async () => {
+    await expect(`
+      my-fn = do x: x * 2 end
+      function? my-fn
+    `).toEvaluateTo(true)
+    await expect(`function? inc`).toEvaluateTo(true)
+    await expect(`function? list.map`).toEvaluateTo(true)
+    await expect(`function? 42`).toEvaluateTo(false)
+    await expect(`function? 'hello'`).toEvaluateTo(false)
+    await expect(`function? [1 2 3]`).toEvaluateTo(false)
+  })
+
+  test('null? checks if value is null', async () => {
+    await expect(`null? null`).toEvaluateTo(true)
+    await expect(`null? 0`).toEvaluateTo(false)
+    await expect(`null? false`).toEvaluateTo(false)
+    await expect(`null? ''`).toEvaluateTo(false)
+    await expect(`null? []`).toEvaluateTo(false)
+  })
+})
+
+describe('type coercion', () => {
+  test('boolean coerces to boolean', async () => {
+    await expect(`boolean true`).toEvaluateTo(true)
+    await expect(`boolean false`).toEvaluateTo(false)
+    await expect(`boolean 1`).toEvaluateTo(true)
+    await expect(`boolean 0`).toEvaluateTo(false)
+    await expect(`boolean 'hello'`).toEvaluateTo(true)
+    await expect(`boolean ''`).toEvaluateTo(false)
+    await expect(`boolean null`).toEvaluateTo(false)
+    await expect(`boolean [1 2 3]`).toEvaluateTo(true)
+  })
+
+  test('number coerces to number', async () => {
+    await expect(`number 42`).toEvaluateTo(42)
+    await expect(`number '42'`).toEvaluateTo(42)
+    await expect(`number '3.14'`).toEvaluateTo(3.14)
+    await expect(`number true`).toEvaluateTo(1)
+    await expect(`number false`).toEvaluateTo(0)
+  })
+
+  test('string coerces to string', async () => {
+    await expect(`string 'hello'`).toEvaluateTo('hello')
+    await expect(`string 42`).toEvaluateTo('42')
+    await expect(`string true`).toEvaluateTo('true')
+    await expect(`string false`).toEvaluateTo('false')
+    await expect(`string null`).toEvaluateTo('null')
+  })
+})
+
+describe('type predicates in conditionals', () => {
+  test('using type predicates in if statements', async () => {
+    await expect(`
+      x = 42
+      if (number? x):
+        'is-num'
+      else:
+        'not-num'
+      end
+    `).toEvaluateTo('is-num')
+  })
+
+  test('filtering by type', async () => {
+    await expect(`
+      items = [1 'hello' 2 'world' 3]
+      list.filter items number?
+    `).toEvaluateTo([1, 2, 3])
+  })
+
+  test('filtering strings', async () => {
+    await expect(`
+      items = [1 'hello' 2 'world' 3]
+      list.filter items string?
+    `).toEvaluateTo(['hello', 'world'])
+  })
+
+  test('checking for functions', async () => {
+    await expect(`
+      double = do x: x * 2 end
+      not-fn = 42
+      is-fn = function? double
+      is-not-fn = function? not-fn
+      is-fn and (not is-not-fn)
+    `).toEvaluateTo(true)
+  })
+})

src/prelude/types.ts

@@ -0,0 +1,22 @@
+import { toValue } from 'reefvm'
+
+export const types = {
+  'boolean?': (v: any) => toValue(v).type === 'boolean',
+  boolean: (v: any) => Boolean(v),
+
+  'number?': (v: any) => toValue(v).type === 'number',
+  number: (v: any) => Number(v),
+
+  'string?': (v: any) => toValue(v).type === 'string',
+  string: (v: any) => String(v),
+
+
+  'array?': (v: any) => toValue(v).type === 'array',
+  'list?': (v: any) => toValue(v).type === 'array',
+
+  'dict?': (v: any) => toValue(v).type === 'dict',
+
+  'function?': (v: any) => ['function', 'native'].includes(toValue(v).type),
+
+  'null?': (v: any) => toValue(v).type === 'null',
+}

src/testSetup.ts

@@ -1,33 +1,14 @@
 import { expect } from 'bun:test'
-import { parser } from '#parser/shrimp'
+import { diffLines } from 'diff'
-import { setGlobals } from '#parser/tokenizer'
+import color from 'kleur'
+import { Scanner, TokenType, type Token } from '#parser/tokenizer2'
+import { parse, setGlobals } from '#parser/parser2'
+import { Tree } from '#parser/node'
 import { globals as prelude } from '#prelude'
-import { $ } from 'bun'
 import { assert, errorMessage } from '#utils/utils'
 import { Compiler } from '#compiler/compiler'
 import { run, VM } from 'reefvm'
-import { treeToString, VMResultToValue } from '#utils/tree'
+import { treeToString2, VMResultToValue } from '#utils/tree'
-
-const regenerateParser = async () => {
-  let generate = true
-  try {
-    const grammarStat = await Bun.file('./src/parser/shrimp.grammar').stat()
-    const tokenizerStat = await Bun.file('./src/parser/tokenizer.ts').stat()
-    const parserStat = await Bun.file('./src/parser/shrimp.ts').stat()
-
-    if (grammarStat.mtime <= parserStat.mtime && tokenizerStat.mtime <= parserStat.mtime) {
-      generate = false
-    }
-  } catch (e) {
-    console.error('Error checking or regenerating parser:', e)
-  } finally {
-    if (generate) {
-      await $`bun generate-parser`
-    }
-  }
-}
-
-await regenerateParser()
 
 // Type declaration for TypeScript
 declare module 'bun:test' {
@@ -37,6 +18,9 @@ declare module 'bun:test' {
     toFailParse(): T
     toEvaluateTo(expected: unknown, globals?: Record<string, any>): Promise<T>
     toFailEvaluation(): Promise<T>
+    toBeToken(expected: string): T
+    toMatchToken(typeOrValue: string, value?: string): T
+    toMatchTokens(...tokens: { type: string, value?: string }[]): T
   }
 }
 
@@ -46,8 +30,8 @@ expect.extend({
 
     const allGlobals = { ...prelude, ...(globals || {}) }
     setGlobals(Object.keys(allGlobals))
-    const tree = parser.parse(received)
+    const tree = parse(received)
-    const actual = treeToString(tree, received)
+    const actual = treeToString2(tree, received)
     const normalizedExpected = trimWhitespace(expected)
 
     try {
@@ -66,7 +50,8 @@ expect.extend({
     assert(typeof received === 'string', 'toFailParse can only be used with string values')
 
     try {
-      const tree = parser.parse(received)
+      const node = parse(received)
+      const tree = new Tree(node)
       let hasErrors = false
       tree.iterate({
         enter(n) {
@@ -83,7 +68,7 @@ expect.extend({
           pass: true,
         }
       } else {
-        const actual = treeToString(tree, received)
+        const actual = treeToString2(node, received)
         return {
           message: () => `Expected input to fail parsing, but it parsed successfully:\n${actual}`,
           pass: false,
@@ -144,8 +129,107 @@ expect.extend({
       }
     }
   },
+  toBeToken(received: unknown, expected: string) {
+    assert(typeof received === 'string', 'toBeToken can only be used with string values')
+
+    try {
+      const tokens = tokenize(received)
+      const value = tokens[0] as Token
+      const target = TokenType[expected as keyof typeof TokenType]
+
+      if (!value) {
+        return {
+          message: () => `Expected token type to be ${expected}, but got ${value}`,
+          pass: false,
+        }
+      }
+
+      return {
+        message: () => `Expected token type to be ${expected}, but got ${TokenType[value.type]}`,
+        pass: value.type === target
+      }
+    } catch (error) {
+      return {
+        message: () => `Tokenization failed: ${errorMessage(error)}`,
+        pass: false,
+      }
+    }
+  },
+  toMatchToken(received: unknown, typeOrValue: string, value?: string) {
+    assert(typeof received === 'string', 'toMatchToken can only be used with string values')
+    const expectedValue = value ? value : typeOrValue
+    const expectedType = value ? typeOrValue : undefined
+
+    try {
+      const tokens = tokenize(received)
+      const token = tokens[0] as Token
+
+      if (!token) {
+        return {
+          message: () => `Expected token to be ${expectedValue.replaceAll('\n', '\\n')}, got ${token}`,
+          pass: false,
+        }
+      }
+
+      if (expectedType && TokenType[expectedType as keyof typeof TokenType] !== token.type) {
+        return {
+          message: () => `Expected token to be ${expectedType}, but got ${TokenType[token.type]}`,
+          pass: false
+        }
+      }
+
+      return {
+        message: () => `Expected token to be ${expectedValue.replaceAll('\n', '\\n')}, but got ${token.value}`,
+        pass: token.value === expectedValue
+      }
+    } catch (error) {
+      return {
+        message: () => `Tokenization failed: ${errorMessage(error)} `,
+        pass: false,
+      }
+    }
+  },
+  toMatchTokens(received: unknown, ...tokens: { type: string, value?: string }[]) {
+    assert(typeof received === 'string', 'toMatchTokens can only be used with string values')
+
+    try {
+      const result = tokenize(received).map(t => toHumanToken(t))
+
+      if (result.length === 0 && tokens.length > 0) {
+        return {
+          message: () => `Expected tokens ${JSON.stringify(tokens)}, got nothing`,
+          pass: false,
+        }
+      }
+
+      const expected = JSON.stringify(tokens, null, 2)
+      const actual = JSON.stringify(result, null, 2)
+
+      return {
+        message: () => `Tokens don't match: \n\n${diff(actual, expected)}`,
+        pass: expected == actual
+      }
+    } catch (error) {
+      return {
+        message: () => `Tokenization failed: ${errorMessage(error)} `,
+        pass: false,
+      }
+    }
+  }
 })
 
+const tokenize = (code: string): Token[] => {
+  const scanner = new Scanner
+  return scanner.tokenize(code)
+}
+
+const toHumanToken = (tok: Token): { type: string, value?: string } => {
+  return {
+    type: TokenType[tok.type],
+    value: tok.value
+  }
+}
+
 const trimWhitespace = (str: string): string => {
   const lines = str.split('\n').filter((line) => line.trim().length > 0)
   const firstLine = lines[0]
@@ -157,10 +241,33 @@ const trimWhitespace = (str: string): string => {
       if (!line.startsWith(leadingWhitespace)) {
         let foundWhitespace = line.match(/^(\s*)/)?.[1] || ''
         throw new Error(
-          `Line has inconsistent leading whitespace: "${line}" (found "${foundWhitespace}", expected "${leadingWhitespace}")`
+          `Line has inconsistent leading whitespace: "${line}"(found "${foundWhitespace}", expected "${leadingWhitespace}")`
         )
       }
       return line.slice(leadingWhitespace.length)
     })
     .join('\n')
 }
+
+const diff = (a: string, b: string): string => {
+  const expected = a.trim()
+  const actual = b.trim()
+  const lines = []
+
+  if (expected !== actual) {
+    const changes = diffLines(actual, expected)
+    for (const part of changes) {
+      const sign = part.added ? "+" : part.removed ? "-" : " "
+      let line = sign + part.value
+      if (part.added) {
+        line = color.green(line)
+      } else if (part.removed) {
+        line = color.red(line)
+      }
+
+      lines.push(line.endsWith("\n") || line.endsWith("\n\u001b[39m") ? line : line + "\n")
+    }
+  }
+
+  return lines.join('\n')
+}

src/utils/tree.ts

@@ -1,5 +1,50 @@
 import { Tree, TreeCursor } from '@lezer/common'
 import { type Value, fromValue } from 'reefvm'
+import { SyntaxNode } from '#parser/node'
+
+const nodeToString = (node: SyntaxNode, input: string, depth = 0): string => {
+  const indent = '  '.repeat(depth)
+  const text = input.slice(node.from, node.to)
+  const nodeName = node.name
+
+  if (node.firstChild) {
+    return `${indent}${nodeName}`
+  } else {
+    // Only strip quotes from whole String nodes (legacy DoubleQuote), not StringFragment/EscapeSeq/CurlyString
+    let cleanText = nodeName === 'String' ? text.slice(1, -1) : text
+    if (cleanText === ' ') cleanText = '(space)'
+    return cleanText ? `${indent}${nodeName} ${cleanText}` : `${indent}${nodeName}`
+  }
+}
+
+export const treeToString2 = (tree: SyntaxNode, input: string, depth = 0): string => {
+  let lines = []
+  let node: SyntaxNode | null = tree
+
+  if (node.name === 'Program') node = node.firstChild
+
+  while (node) {
+    // If this node is an error, print ⚠ instead of its content
+    if (node.isError && !node.firstChild) {
+      lines.push('  '.repeat(depth) + '⚠')
+    } else {
+      lines.push(nodeToString(node, input, depth))
+
+      if (node.firstChild) {
+        lines.push(treeToString2(node.firstChild, input, depth + 1))
+      }
+
+      // If this node has an error, add ⚠ after its children
+      if (node.isError && node.firstChild) {
+        lines.push('  '.repeat(depth === 0 ? 0 : depth + 1) + '⚠')
+      }
+    }
+
+    node = node.nextSibling
+  }
+
+  return lines.join('\n')
+}
 
 export const treeToString = (tree: Tree, input: string): string => {
   const lines: string[] = []

vscode-extension/server/src/metadata/prelude-completions.ts

@@ -8,6 +8,57 @@ export type CompletionMetadata = {
 
 export const completions = {
   modules: {
+  "date": {
+    "now": {
+      "params": []
+    },
+    "year": {
+      "params": [
+        "time"
+      ]
+    },
+    "month": {
+      "params": [
+        "time"
+      ]
+    },
+    "date": {
+      "params": [
+        "time"
+      ]
+    },
+    "hour": {
+      "params": [
+        "time"
+      ]
+    },
+    "minute": {
+      "params": [
+        "time"
+      ]
+    },
+    "second": {
+      "params": [
+        "time"
+      ]
+    },
+    "ms": {
+      "params": [
+        "time"
+      ]
+    },
+    "new": {
+      "params": [
+        "year",
+        "month",
+        "day",
+        "hour",
+        "minute",
+        "second",
+        "ms"
+      ]
+    }
+  },
   "dict": {
     "keys": {
       "params": [
@@ -529,7 +580,10 @@ export const completions = {
       ]
     },
     "random": {
-      "params": []
+      "params": [
+        "min",
+        "max"
+      ]
     },
     "clamp": {
       "params": [
@@ -685,6 +739,16 @@ export const completions = {
         "pad"
       ]
     },
+    "capitalize": {
+      "params": [
+        "str"
+      ]
+    },
+    "titlecase": {
+      "params": [
+        "s"
+      ]
+    },
     "lines": {
       "params": [
         "str"

vscode-extension/server/src/metadata/prelude-names.ts

@@ -6,7 +6,9 @@ export const PRELUDE_NAMES = [
   "array?",
   "at",
   "bnot",
+  "boolean",
   "boolean?",
+  "date",
   "dec",
   "describe",
   "dict",
@@ -24,15 +26,18 @@ export const PRELUDE_NAMES = [
   "json",
   "length",
   "list",
+  "list?",
   "load",
   "math",
   "not",
   "null?",
+  "number",
   "number?",
   "range",
   "ref",
   "some?",
   "str",
+  "string",
   "string?",
   "type",
   "var",

vscode-extension/server/src/server.ts

@@ -17,7 +17,6 @@ import {
   CompletionItemKind,
   TextDocumentChangeEvent,
 } from 'vscode-languageserver/node'
-import { setGlobals } from '../../../src/parser/tokenizer'
 import { globals } from '../../../src/prelude'
 
 // Initialize parser with prelude globals so it knows dict/list/str are in scope

vscode-extension/tmp/test-dotget-parse.ts

@@ -1,41 +0,0 @@
-import { parser } from '../../src/parser/shrimp'
-import { setGlobals } from '../../src/parser/tokenizer'
-import { PRELUDE_NAMES } from '../server/src/prelude-names'
-
-// Set globals for DotGet detection
-setGlobals(PRELUDE_NAMES as unknown as string[])
-
-// Test cases - does incomplete DotGet parse correctly?
-const testCases = [
-  'dict.',
-  'dict.g',
-  'dict.get',
-  '$.',
-  '$.e',
-  '$.env',
-]
-
-for (const code of testCases) {
-  console.log(`\nTesting: "${code}"`)
-  const tree = parser.parse(code)
-  const cursor = tree.cursor()
-
-  // Print the parse tree
-  const printTree = (depth = 0) => {
-    const indent = '  '.repeat(depth)
-    console.log(`${indent}${cursor.name} [${cursor.from}-${cursor.to}]`)
-
-    if (cursor.firstChild()) {
-      do {
-        printTree(depth + 1)
-      } while (cursor.nextSibling())
-      cursor.parent()
-    }
-  }
-
-  printTree()
-
-  // Check at cursor position (end of string)
-  const node = tree.resolveInner(code.length, -1)
-  console.log(`Node at end: ${node.name} (type: ${node.type.id})`)
-}