new parser(-ish)

bin/shrimp

@@ -1,7 +1,7 @@
 #!/usr/bin/env bun
 
 import { colors, globals as prelude } from '../src/prelude'
-import { treeToString2 } from '../src/utils/tree'
+import { treeToString } from '../src/utils/tree'
 import { runCode, runFile, compileFile, parseCode } from '../src'
 import { resolve } from 'path'
 import { bytecodeToString } from 'reefvm'
@@ -143,7 +143,7 @@ async function main() {
       process.exit(1)
     }
     const input = readFileSync(file, 'utf-8')
-    console.log(treeToString2(parseCode(input), input))
+    console.log(treeToString(parseCode(input), input))
     return
   }
 

src/compiler/compiler.ts

@@ -1,10 +1,8 @@
 import { CompilerError } from '#compiler/compilerError.ts'
-import { parse } from '#parser/parser2'
+import { parseToTree as parse } from '#parser/parser2'
-import { SyntaxNode, Tree } from '#parser/node'
+import { Tree, SyntaxNode } from '#parser/node'
-import { parser } from '#parser/shrimp.ts'
 import * as terms from '#parser/shrimp.terms'
 import { setGlobals } from '#parser/tokenizer'
-import { tokenizeCurlyString } from '#parser/curlyTokenizer'
 import { assert, errorMessage } from '#utils/utils'
 import { toBytecode, type Bytecode, type ProgramItem, bytecodeToString } from 'reefvm'
 import {
@@ -64,8 +62,7 @@ export class Compiler {
   constructor(public input: string, globals?: string[] | Record<string, any>) {
     try {
       if (globals) setGlobals(Array.isArray(globals) ? globals : Object.keys(globals))
-      const ast = parse(input)
+      const cst = parse(input)
-      const cst = new Tree(ast)
       // const errors = checkTreeForErrors(cst)
 
       // const firstError = errors[0]
@@ -91,8 +88,8 @@ export class Compiler {
   }
 
   #compileCst(cst: Tree, input: string) {
-    const isProgram = cst.topNode.type.id === terms.Program
+    const isProgram = cst.topNode.typeId === terms.Program
-    assert(isProgram, `Expected Program node, got ${cst.topNode.type.name}`)
+    assert(isProgram, `Expected Program node, got ${cst.topNode.type}`)
 
     let child = cst.topNode.firstChild
     while (child) {
@@ -107,7 +104,7 @@ export class Compiler {
     const value = input.slice(node.from, node.to)
     if (DEBUG) console.log(`🫦  ${node.name}: ${value}`)
 
-    switch (node.type.id) {
+    switch (node.typeId) {
       case terms.Number:
         // Handle sign prefix for hex, binary, and octal literals
         // Number() doesn't parse '-0xFF', '+0xFF', '-0o77', etc. correctly
@@ -126,9 +123,6 @@ export class Compiler {
         return [[`PUSH`, numberValue]]
 
       case terms.String: {
-        if (node.firstChild?.type.id === terms.CurlyString)
-          return this.#compileCurlyString(value, input)
-
         const { parts, hasInterpolation } = getStringParts(node, input)
 
         // Simple string without interpolation or escapes - extract text directly
@@ -143,7 +137,7 @@ export class Compiler {
         parts.forEach((part) => {
           const partValue = input.slice(part.from, part.to)
 
-          switch (part.type.id) {
+          switch (part.typeId) {
             case terms.StringFragment:
               // Plain text fragment - just push as-is
               instructions.push(['PUSH', partValue])
@@ -167,7 +161,7 @@ export class Compiler {
 
             default:
               throw new CompilerError(
-                `Unexpected string part: ${part.type.name}`,
+                `Unexpected string part: ${part.type}`,
                 part.from,
                 part.to
               )
@@ -224,7 +218,7 @@ export class Compiler {
         instructions.push(['TRY_LOAD', objectName])
 
         const flattenProperty = (prop: SyntaxNode): void => {
-          if (prop.type.id === terms.DotGet) {
+          if (prop.typeId === terms.DotGet) {
             const nestedParts = getDotGetParts(prop, input)
 
             const nestedObjectValue = input.slice(nestedParts.object.from, nestedParts.object.to)
@@ -233,7 +227,7 @@ export class Compiler {
 
             flattenProperty(nestedParts.property)
           } else {
-            if (prop.type.id === terms.ParenExpr) {
+            if (prop.typeId === terms.ParenExpr) {
               instructions.push(...this.#compileNode(prop, input))
             } else {
               const propertyValue = input.slice(prop.from, prop.to)
@@ -442,7 +436,7 @@ export class Compiler {
       }
 
       case terms.FunctionCallOrIdentifier: {
-        if (node.firstChild?.type.id === terms.DotGet) {
+        if (node.firstChild?.typeId === terms.DotGet) {
           const instructions: ProgramItem[] = []
           const callLabel: Label = `.call_dotget_${++this.labelCount}`
           const afterLabel: Label = `.after_dotget_${++this.labelCount}`
@@ -533,20 +527,20 @@ export class Compiler {
         instructions.push([`${fnLabel}:`])
         instructions.push(
           ...block
-            .filter((x) => x.type.name !== 'keyword')
+            .filter((x) => x.type !== 'keyword')
             .map((x) => this.#compileNode(x!, input))
             .flat()
         )
         instructions.push(['RETURN'])
         instructions.push([`${afterLabel}:`])
 
-        if (fn?.type.id === terms.FunctionCallOrIdentifier) {
+        if (fn?.typeId === terms.FunctionCallOrIdentifier) {
           instructions.push(['LOAD', input.slice(fn!.from, fn!.to)])
           instructions.push(['MAKE_FUNCTION', [], fnLabel])
           instructions.push(['PUSH', 1])
           instructions.push(['PUSH', 0])
           instructions.push(['CALL'])
-        } else if (fn?.type.id === terms.FunctionCall) {
+        } else if (fn?.typeId === terms.FunctionCall) {
           let body = this.#compileNode(fn!, input)
           const namedArgCount = (body[body.length - 2]![1] as number) * 2
           const startSlice = body.length - namedArgCount - 3
@@ -739,11 +733,11 @@ export class Compiler {
           instructions.push(...this.#compileNode(identifierNode, input))
 
           const isUnderscoreInPositionalArgs = positionalArgs.some(
-            (arg) => arg.type.id === terms.Underscore
+            (arg) => arg.typeId === terms.Underscore
           )
           const isUnderscoreInNamedArgs = namedArgs.some((arg) => {
             const { valueNode } = getNamedArgParts(arg, input)
-            return valueNode.type.id === terms.Underscore
+            return valueNode.typeId === terms.Underscore
           })
 
           const shouldPushPositionalArg = !isUnderscoreInPositionalArgs && !isUnderscoreInNamedArgs
@@ -754,7 +748,7 @@ export class Compiler {
           }
 
           positionalArgs.forEach((arg) => {
-            if (arg.type.id === terms.Underscore) {
+            if (arg.typeId === terms.Underscore) {
               instructions.push(['LOAD', pipeValName])
             } else {
               instructions.push(...this.#compileNode(arg, input))
@@ -764,7 +758,7 @@ export class Compiler {
           namedArgs.forEach((arg) => {
             const { name, valueNode } = getNamedArgParts(arg, input)
             instructions.push(['PUSH', name])
-            if (valueNode.type.id === terms.Underscore) {
+            if (valueNode.typeId === terms.Underscore) {
               instructions.push(['LOAD', pipeValName])
             } else {
               instructions.push(...this.#compileNode(valueNode, input))
@@ -786,7 +780,7 @@ export class Compiler {
         // = can be a valid word, and is also valid inside words, so for now we cheat
         // and check for arrays that look like `[ = ]` to interpret them as
         // empty dicts
-        if (children.length === 1 && children[0]!.type.id === terms.Word) {
+        if (children.length === 1 && children[0]!.typeId === terms.Word) {
           const child = children[0]!
           if (input.slice(child.from, child.to) === '=') {
             return [['MAKE_DICT', 0]]
@@ -838,8 +832,8 @@ export class Compiler {
       case terms.Import: {
         const instructions: ProgramItem[] = []
         const [_import, ...nodes] = getAllChildren(node)
-        const args = nodes.filter(node => node.type.id === terms.Identifier)
+        const args = nodes.filter(node => node.typeId === terms.Identifier)
-        const namedArgs = nodes.filter(node => node.type.id === terms.NamedArg)
+        const namedArgs = nodes.filter(node => node.typeId === terms.NamedArg)
 
         instructions.push(['LOAD', 'import'])
 
@@ -866,7 +860,7 @@ export class Compiler {
 
       default:
         throw new CompilerError(
-          `Compiler doesn't know how to handle a "${node.type.name}" (${node.type.id}) node.`,
+          `Compiler doesn't know how to handle a "${node.type}" (${node.typeId}) node.`,
           node.from,
           node.to
         )
@@ -920,26 +914,4 @@ export class Compiler {
 
     return instructions
   }
-
-  #compileCurlyString(value: string, input: string): ProgramItem[] {
-    const instructions: ProgramItem[] = []
-    const nodes = tokenizeCurlyString(value)
-
-    nodes.forEach((node) => {
-      if (typeof node === 'string') {
-        instructions.push(['PUSH', node])
-      } else {
-        const [input, topNode] = node
-        let child = topNode.firstChild
-        while (child) {
-          instructions.push(...this.#compileNode(child, input))
-          child = child.nextSibling
-        }
-      }
-    })
-
-    instructions.push(['STR_CONCAT', nodes.length])
-
-    return instructions
-  }
 }

src/compiler/utils.ts

@@ -1,6 +1,6 @@
 import { CompilerError } from '#compiler/compilerError.ts'
-import * as terms from '#parser/shrimp.terms'
 import type { SyntaxNode, Tree } from '#parser/node'
+import * as terms from '#parser/shrimp.terms'
 
 export const checkTreeForErrors = (tree: Tree): CompilerError[] => {
   const errors: CompilerError[] = []
@@ -24,7 +24,7 @@ export const getAllChildren = (node: SyntaxNode): SyntaxNode[] => {
     child = child.nextSibling
   }
 
-  return children.filter((n) => n.type.id !== terms.Comment)
+  return children.filter((n) => n.typeId !== terms.Comment)
 }
 
 export const getBinaryParts = (node: SyntaxNode) => {
@@ -51,15 +51,14 @@ export const getAssignmentParts = (node: SyntaxNode) => {
   }
 
   // array destructuring
-  if (left && left.type.id === terms.Array) {
+  if (left && left.typeId === terms.Array) {
-    const identifiers = getAllChildren(left).filter((child) => child.type.id === terms.Identifier)
+    const identifiers = getAllChildren(left).filter((child) => child.typeId === terms.Identifier)
     return { arrayPattern: identifiers, right }
   }
 
-  if (!left || left.type.id !== terms.AssignableIdentifier) {
+  if (!left || left.typeId !== terms.AssignableIdentifier) {
     throw new CompilerError(
-      `Assign left child must be an AssignableIdentifier or Array, got ${
+      `Assign left child must be an AssignableIdentifier or Array, got ${left ? left.type : 'none'
-        left ? left.type.name : 'none'
       }`,
       node.from,
       node.to
@@ -73,10 +72,9 @@ export const getCompoundAssignmentParts = (node: SyntaxNode) => {
   const children = getAllChildren(node)
   const [left, operator, right] = children
 
-  if (!left || left.type.id !== terms.AssignableIdentifier) {
+  if (!left || left.typeId !== terms.AssignableIdentifier) {
     throw new CompilerError(
-      `CompoundAssign left child must be an AssignableIdentifier, got ${
+      `CompoundAssign left child must be an AssignableIdentifier, got ${left ? left.type : 'none'
-        left ? left.type.name : 'none'
       }`,
       node.from,
       node.to
@@ -105,9 +103,9 @@ export const getFunctionDefParts = (node: SyntaxNode, input: string) => {
   }
 
   const paramNames = getAllChildren(paramsNode).map((param) => {
-    if (param.type.id !== terms.Identifier && param.type.id !== terms.NamedParam) {
+    if (param.typeId !== terms.Identifier && param.typeId !== terms.NamedParam) {
       throw new CompilerError(
-        `FunctionDef params must be Identifier or NamedParam, got ${param.type.name}`,
+        `FunctionDef params must be Identifier or NamedParam, got ${param.type}`,
         param.from,
         param.to
       )
@@ -124,7 +122,7 @@ export const getFunctionDefParts = (node: SyntaxNode, input: string) => {
   let finallyBody: SyntaxNode | undefined
 
   for (const child of rest) {
-    if (child.type.id === terms.CatchExpr) {
+    if (child.typeId === terms.CatchExpr) {
       catchExpr = child
       const catchChildren = getAllChildren(child)
       const [_catchKeyword, identifierNode, _colon, body] = catchChildren
@@ -137,7 +135,7 @@ export const getFunctionDefParts = (node: SyntaxNode, input: string) => {
       }
       catchVariable = input.slice(identifierNode.from, identifierNode.to)
       catchBody = body
-    } else if (child.type.id === terms.FinallyExpr) {
+    } else if (child.typeId === terms.FinallyExpr) {
       finallyExpr = child
       const finallyChildren = getAllChildren(child)
       const [_finallyKeyword, _colon, body] = finallyChildren
@@ -149,7 +147,7 @@ export const getFunctionDefParts = (node: SyntaxNode, input: string) => {
         )
       }
       finallyBody = body
-    } else if (child.type.name === 'keyword' && input.slice(child.from, child.to) === 'end') {
+    } else if (child.type === 'keyword' && input.slice(child.from, child.to) === 'end') {
       // Skip the end keyword
     } else {
       bodyNodes.push(child)
@@ -166,9 +164,9 @@ export const getFunctionCallParts = (node: SyntaxNode, input: string) => {
     throw new CompilerError(`FunctionCall expected at least 1 child, got 0`, node.from, node.to)
   }
 
-  const namedArgs = args.filter((arg) => arg.type.id === terms.NamedArg)
+  const namedArgs = args.filter((arg) => arg.typeId === terms.NamedArg)
   const positionalArgs = args
-    .filter((arg) => arg.type.id === terms.PositionalArg)
+    .filter((arg) => arg.typeId === terms.PositionalArg)
     .map((arg) => {
       const child = arg.firstChild
       if (!child) throw new CompilerError(`PositionalArg has no child`, arg.from, arg.to)
@@ -209,16 +207,16 @@ export const getIfExprParts = (node: SyntaxNode, input: string) => {
   rest.forEach((child) => {
     const parts = getAllChildren(child)
 
-    if (child.type.id === terms.ElseExpr) {
+    if (child.typeId === terms.ElseExpr) {
       if (parts.length !== 3) {
         const message = `ElseExpr expected 1 child, got ${parts.length}`
         throw new CompilerError(message, child.from, child.to)
       }
       elseThenBlock = parts.at(-1)
-    } else if (child.type.id === terms.ElseIfExpr) {
+    } else if (child.typeId === terms.ElseIfExpr) {
       const [_else, _if, conditional, _colon, thenBlock] = parts
       if (!conditional || !thenBlock) {
-        const names = parts.map((p) => p.type.name).join(', ')
+        const names = parts.map((p) => p.type).join(', ')
         const message = `ElseIfExpr expected conditional and thenBlock, got ${names}`
         throw new CompilerError(message, child.from, child.to)
       }
@@ -250,10 +248,10 @@ export const getStringParts = (node: SyntaxNode, input: string) => {
   // The text is just between the quotes
   const parts = children.filter((child) => {
     return (
-      child.type.id === terms.StringFragment ||
+      child.typeId === terms.StringFragment ||
-      child.type.id === terms.Interpolation ||
+      child.typeId === terms.Interpolation ||
-      child.type.id === terms.EscapeSeq ||
+      child.typeId === terms.EscapeSeq ||
-      child.type.id === terms.CurlyString
+      child.typeId === terms.CurlyString
 
     )
   })
@@ -261,13 +259,13 @@ export const getStringParts = (node: SyntaxNode, input: string) => {
   // Validate each part is the expected type
   parts.forEach((part) => {
     if (
-      part.type.id !== terms.StringFragment &&
+      part.typeId !== terms.StringFragment &&
-      part.type.id !== terms.Interpolation &&
+      part.typeId !== terms.Interpolation &&
-      part.type.id !== terms.EscapeSeq &&
+      part.typeId !== terms.EscapeSeq &&
-      part.type.id !== terms.CurlyString
+      part.typeId !== terms.CurlyString
     ) {
       throw new CompilerError(
-        `String child must be StringFragment, Interpolation, or EscapeSeq, got ${part.type.name}`,
+        `String child must be StringFragment, Interpolation, or EscapeSeq, got ${part.type}`,
         part.from,
         part.to
       )
@@ -277,7 +275,7 @@ export const getStringParts = (node: SyntaxNode, input: string) => {
   // hasInterpolation means the string has interpolation ($var) or escape sequences (\n)
   // A simple string like 'hello' has one StringFragment but no interpolation
   const hasInterpolation = parts.some(
-    (p) => p.type.id === terms.Interpolation || p.type.id === terms.EscapeSeq
+    (p) => p.typeId === terms.Interpolation || p.typeId === terms.EscapeSeq
   )
   return { parts, hasInterpolation }
 }
@@ -294,17 +292,17 @@ export const getDotGetParts = (node: SyntaxNode, input: string) => {
     )
   }
 
-  if (object.type.id !== terms.IdentifierBeforeDot && object.type.id !== terms.Dollar) {
+  if (object.typeId !== terms.IdentifierBeforeDot && object.typeId !== terms.Dollar) {
     throw new CompilerError(
-      `DotGet object must be an IdentifierBeforeDot, got ${object.type.name}`,
+      `DotGet object must be an IdentifierBeforeDot, got ${object.type}`,
       object.from,
       object.to
     )
   }
 
-  if (![terms.Identifier, terms.Number, terms.ParenExpr, terms.DotGet].includes(property.type.id)) {
+  if (![terms.Identifier, terms.Number, terms.ParenExpr, terms.DotGet].includes(property.typeId)) {
     throw new CompilerError(
-      `DotGet property must be an Identifier, Number, ParenExpr, or DotGet, got ${property.type.name}`,
+      `DotGet property must be an Identifier, Number, ParenExpr, or DotGet, got ${property.type}`,
       property.from,
       property.to
     )
@@ -336,7 +334,7 @@ export const getTryExprParts = (node: SyntaxNode, input: string) => {
   let finallyBody: SyntaxNode | undefined
 
   rest.forEach((child) => {
-    if (child.type.id === terms.CatchExpr) {
+    if (child.typeId === terms.CatchExpr) {
       catchExpr = child
       const catchChildren = getAllChildren(child)
       const [_catchKeyword, identifierNode, _colon, body] = catchChildren
@@ -349,7 +347,7 @@ export const getTryExprParts = (node: SyntaxNode, input: string) => {
       }
       catchVariable = input.slice(identifierNode.from, identifierNode.to)
       catchBody = body
-    } else if (child.type.id === terms.FinallyExpr) {
+    } else if (child.typeId === terms.FinallyExpr) {
       finallyExpr = child
       const finallyChildren = getAllChildren(child)
       const [_finallyKeyword, _colon, body] = finallyChildren

src/index.ts

@@ -1,15 +1,15 @@
 import { readFileSync } from 'fs'
 import { VM, fromValue, toValue, isValue, type Bytecode } from 'reefvm'
+import { type Tree } from '@lezer/common'
 import { Compiler } from '#compiler/compiler'
-import { parse } from '#parser/parser2'
+import { parser } from '#parser/shrimp'
-import { type SyntaxNode, Tree } from '#parser/node'
 import { globals as parserGlobals, setGlobals as setParserGlobals } from '#parser/tokenizer'
 import { globals as prelude } from '#prelude'
 
 export { Compiler } from '#compiler/compiler'
-export { parse } from '#parser/parser2'
+export { parser } from '#parser/shrimp'
-export { type SyntaxNode, Tree } from '#parser/node'
 export { globals as prelude } from '#prelude'
+export type { Tree } from '@lezer/common'
 export { type Value, type Bytecode } from 'reefvm'
 export { toValue, fromValue, isValue, Scope, VM, bytecodeToString } from 'reefvm'
 
@@ -41,7 +41,7 @@ export class Shrimp {
         return isValue(result) ? fromValue(result, this.vm) : result
     }
 
-    parse(code: string): SyntaxNode {
+    parse(code: string): Tree {
         return parseCode(code, this.globals)
     }
 
@@ -105,8 +105,8 @@ export function parseCode(code: string, globals?: Record<string, any>): Tree {
     const globalNames = [...Object.keys(prelude), ...(globals ? Object.keys(globals) : [])]
 
     setParserGlobals(globalNames)
-    const result = parse(code)
+    const result = parser.parse(code)
     setParserGlobals(oldGlobals)
 
-    return new Tree(result)
+    return result
 }

src/parser/curlyTokenizer.ts

@@ -1,6 +1,5 @@
 import { parser } from '#parser/shrimp.ts'
-import { parse } from '#parser/parser2'
+import type { SyntaxNode } from '@lezer/common'
-import type { SyntaxNode } from '#parser/node'
 import { isIdentStart, isIdentChar } from './tokenizer'
 
 // Turns a { curly string } into strings and nodes for interpolation
@@ -38,7 +37,7 @@ export const tokenizeCurlyString = (value: string): (string | [string, SyntaxNod
         }
 
         const input = value.slice(start + 2, pos) // skip '$('
-        tokens.push([input, parse(input)])
+        tokens.push([input, parser.parse(input).topNode])
         start = ++pos // skip ')'
       } else {
         char = value[++pos]
@@ -49,7 +48,7 @@ export const tokenizeCurlyString = (value: string): (string | [string, SyntaxNod
           char = value[++pos]
 
         const input = value.slice(start + 1, pos) // skip '$'
-        tokens.push([input, parse(input)])
+        tokens.push([input, parser.parse(input).topNode])
         start = pos-- // backtrack and start over
       }
     }

src/parser/node.ts

@@ -1,5 +1,5 @@
 import { type Token, TokenType } from './tokenizer2'
-import * as term from './shrimp.terms'
+import { nameToId } from './terms'
 
 export type NodeType =
   | 'Program'
@@ -58,7 +58,6 @@ export type NodeType =
 
   | 'Import'
   | 'Do'
-  | 'Underscore'
   | 'colon'
   | 'keyword'
   | 'operator'
@@ -114,224 +113,32 @@ export const operators: Record<string, any> = {
 
 export class Tree {
   constructor(public topNode: SyntaxNode) { }
-
-  get length(): number {
-    return this.topNode.to
-  }
-
-  cursor() {
-    return {
-      type: this.topNode.type,
-      from: this.topNode.from,
-      to: this.topNode.to,
-      node: this.topNode,
-    }
-  }
-}
-
-// TODO: TEMPORARY SHIM
-class SyntaxNodeType {
-  constructor(public nodeType: NodeType) { }
-
-  is(other: string) {
-    return this.nodeType === other
-  }
-
-  get id(): number {
-    switch (this.nodeType) {
-      case 'Program':
-        return term.Program
-
-      case 'Block':
-        return term.Block
-
-      case 'FunctionCall':
-        return term.FunctionCall
-
-      case 'FunctionCallOrIdentifier':
-        return term.FunctionCallOrIdentifier
-
-      case 'FunctionCallWithBlock':
-        return term.FunctionCallWithBlock
-
-      case 'PositionalArg':
-        return term.PositionalArg
-
-      case 'NamedArg':
-        return term.NamedArg
-
-      case 'FunctionDef':
-        return term.FunctionDef
-
-      case 'Params':
-        return term.Params
-
-      case 'NamedParam':
-        return term.NamedParam
-
-      case 'Null':
-        return term.Null
-
-      case 'Boolean':
-        return term.Boolean
-
-      case 'Number':
-        return term.Number
-
-      case 'String':
-        return term.String
-
-      case 'StringFragment':
-        return term.StringFragment
-
-      case 'CurlyString':
-        return term.CurlyString
-
-      case 'DoubleQuote':
-        return term.DoubleQuote
-
-      case 'EscapeSeq':
-        return term.EscapeSeq
-
-      case 'Interpolation':
-        return term.Interpolation
-
-      case 'Regex':
-        return term.Regex
-
-      case 'Identifier':
-        return term.Identifier
-
-      case 'AssignableIdentifier':
-        return term.AssignableIdentifier
-
-      case 'IdentifierBeforeDot':
-        return term.IdentifierBeforeDot
-
-      case 'Word':
-        return term.Word
-
-      case 'Array':
-        return term.Array
-
-      case 'Dict':
-        return term.Dict
-
-      case 'Comment':
-        return term.Comment
-
-      case 'BinOp':
-        return term.BinOp
-
-      case 'ConditionalOp':
-        return term.ConditionalOp
-
-      case 'ParenExpr':
-        return term.ParenExpr
-
-      case 'Assign':
-        return term.Assign
-
-      case 'CompoundAssign':
-        return term.CompoundAssign
-
-      case 'DotGet':
-        return term.DotGet
-
-      case 'PipeExpr':
-        return term.PipeExpr
-
-      case 'IfExpr':
-        return term.IfExpr
-
-      case 'ElseIfExpr':
-        return term.ElseIfExpr
-
-      case 'ElseExpr':
-        return term.ElseExpr
-
-      case 'WhileExpr':
-        return term.WhileExpr
-
-      case 'TryExpr':
-        return term.TryExpr
-
-      case 'CatchExpr':
-        return term.CatchExpr
-
-      case 'FinallyExpr':
-        return term.FinallyExpr
-
-      case 'Throw':
-        return term.Throw
-
-      case 'Eq':
-        return term.Eq
-
-      case 'Modulo':
-        return term.Modulo
-
-      case 'Plus':
-        return term.Plus
-
-      case 'Star':
-        return term.Star
-
-      case 'Slash':
-        return term.Slash
-
-      case 'Import':
-        return term.Import
-
-      case 'Do':
-        return term.Do
-
-      case 'Underscore':
-        return term.Underscore
-
-      case 'colon':
-        return term.colon
-
-      case 'keyword':
-        return term.keyword
-
-    }
-    return 0
-  }
-
-  get name(): string {
-    return this.nodeType
-  }
 }
 
 export class SyntaxNode {
-  #type: NodeType
+  type: NodeType
   from: number
   to: number
   parent: SyntaxNode | null
   children: SyntaxNode[] = []
 
   constructor(type: NodeType, from: number, to: number, parent: SyntaxNode | null = null) {
-    this.#type = type
+    this.type = type
     this.from = from
     this.to = to
     this.parent = parent
   }
 
+  get typeId(): number {
+    return nameToId(this.type)
+  }
+
   static from(token: Token, parent?: SyntaxNode): SyntaxNode {
     return new SyntaxNode(TokenType[token.type] as NodeType, token.from, token.to, parent ?? null)
   }
 
-  get type(): SyntaxNodeType {
-    return new SyntaxNodeType(this.#type)
-  }
-
-  set type(name: NodeType) {
-    this.#type = name
-  }
-
   get name(): string {
-    return this.type.name
+    return this.type
   }
 
   get isError(): boolean {
@@ -372,7 +179,7 @@ export class SyntaxNode {
   }
 
   toString(): string {
-    return this.type.name
+    return this.type
   }
 }
 
@@ -393,7 +200,7 @@ export const precedence: Record<string, number> = {
   // Nullish coalescing
   '??': 35,
 
-  // Bitwise shifts (lower precedence than addition)
+  // Bitwise shift (lower precedence than addition)
   '<<': 37,
   '>>': 37,
   '>>>': 37,
@@ -402,7 +209,7 @@ export const precedence: Record<string, number> = {
   '+': 40,
   '-': 40,
 
-  // Bitwise AND/OR/XOR (higher precedence than addition)
+  // Bitwise AND/OR/XOR (between addition and multiplication)
   'band': 45,
   'bor': 45,
   'bxor': 45,

src/parser/parser2.ts

@@ -1,5 +1,5 @@
 import { Scanner, type Token, TokenType } from './tokenizer2'
-import { SyntaxNode, operators, precedence, conditionals, compounds } from './node'
+import { Tree, SyntaxNode, operators, precedence, conditionals, compounds } from './node'
 import { globals } from './tokenizer'
 import { parseString } from './stringParser'
 
@@ -10,6 +10,10 @@ export const parse = (input: string): SyntaxNode => {
   return parser.parse(input)
 }
 
+export const parseToTree = (input: string): Tree => {
+  return new Tree(parse(input))
+}
+
 class Scope {
   parent?: Scope
   set = new Set<string>()
@@ -118,17 +122,19 @@ export class Parser {
       expr = this.exprWithPrecedence()
 
     // check for destructuring
-    if (expr.type.is('Array') && this.is($T.Operator, '='))
+    if (expr.type === 'Array' && this.is($T.Operator, '='))
       return this.destructure(expr)
 
     // check for parens function call
     // ex: (ref my-func) my-arg
-    if (expr.type.is('ParenExpr') && !this.isExprEnd())
+    // but not if followed by operator: (x) + 1
+    if (expr.type === 'ParenExpr' && !this.isExprEnd() && !this.is($T.Operator))
       expr = this.functionCall(expr)
 
-    // if dotget is followed by binary operator, continue parsing as binary expression
+    // if there's an operator (not pipe), continue with precedence parsing
-    if (expr.type.is('DotGet') && this.is($T.Operator) && !this.is($T.Operator, '|'))
+    if (this.is($T.Operator) && !this.isPipe()) {
-      expr = this.dotGetBinOp(expr)
+      expr = this.continueWithPrecedence(expr)
+    }
 
     // one | echo
     if (allowPipe && this.isPipe())
@@ -139,6 +145,33 @@ export class Parser {
       return expr
   }
 
+  // Continue parsing with precedence after we already have a left side
+  continueWithPrecedence(left: SyntaxNode, minBp = 0): SyntaxNode {
+    while (this.is($T.Operator)) {
+      const op = this.current().value!
+      const bp = precedence[op]
+
+      // operator has lower precedence than required, stop
+      if (bp === undefined || bp < minBp) break
+
+      const opNode = this.op()
+
+      // right-associative operators (like **) use same bp, others use bp + 1
+      const nextMinBp = op === '**' ? bp : bp + 1
+
+      // parse right-hand side with higher precedence
+      const right = this.exprWithPrecedence(nextMinBp)
+
+      const nodeType = conditionals.has(op) ? 'ConditionalOp' : 'BinOp'
+      const node = new SyntaxNode(nodeType, left.from, right.to)
+
+      node.push(left, opNode, right)
+      left = node
+    }
+
+    return left
+  }
+
   // piping | stuff | is | cool
   pipe(left: SyntaxNode): SyntaxNode {
     const canLookPastNewlines = this.inParens === 0
@@ -245,22 +278,6 @@ export class Parser {
   // parse specific nodes
   //
 
-  // raw determines whether we just want the SyntaxNodes or we want to
-  // wrap them in a PositionalArg
-  arg(raw = false): SyntaxNode {
-    // 'do' is a special function arg - it doesn't need to be wrapped
-    // in parens. otherwise, args are regular value()s
-    const val = this.is($T.Keyword, 'do') ? this.do() : this.value()
-
-    if (raw) {
-      return val
-    } else {
-      const arg = new SyntaxNode('PositionalArg', val.from, val.to)
-      arg.add(val)
-      return arg
-    }
-  }
-
   // [ 1 2 3 ]
   array(): SyntaxNode {
     const open = this.expect($T.OpenBracket)
@@ -341,7 +358,7 @@ export class Parser {
   }
 
   // atoms are the basic building blocks: literals, identifiers, words
-  atom(): SyntaxNode {
+  atom() {
     if (this.is($T.String))
       return this.string()
 
@@ -445,7 +462,15 @@ export class Parser {
         continue
       }
 
-      values.push(this.is($T.NamedArgPrefix) ? this.namedArg() : this.arg())
+      if (this.is($T.NamedArgPrefix)) {
+        const prefix = SyntaxNode.from(this.next())
+        const val = this.is($T.Keyword, 'do') ? this.do() : this.value()
+        const arg = new SyntaxNode('NamedArg', prefix.from, val.to)
+        arg.push(prefix, val)
+        values.push(arg)
+      } else {
+        values.push(this.value())
+      }
     }
 
     const close = this.expect($T.CloseBracket)
@@ -524,7 +549,7 @@ export class Parser {
     if (!this.scope.has(ident))
       return this.word(left)
 
-    if (left.type.is('Identifier')) left.type = 'IdentifierBeforeDot'
+    if (left.type === 'Identifier') left.type = 'IdentifierBeforeDot'
 
     let parts = []
     while (this.is($T.Operator, '.')) {
@@ -539,37 +564,19 @@ export class Parser {
     return nodes ? node.push(left, nodes!) : node.push(left, ...parts)
   }
 
-  // continue parsing dotget/word binary operation
-  dotGetBinOp(left: SyntaxNode): SyntaxNode {
-    while (this.is($T.Operator) && !this.is($T.Operator, '|')) {
-      const op = this.current().value!
-      const bp = precedence[op]
-      if (bp === undefined) break
-
-      const opNode = this.op()
-      const right = this.exprWithPrecedence(bp + 1)
-
-      const nodeType = conditionals.has(op) ? 'ConditionalOp' : 'BinOp'
-      const node = new SyntaxNode(nodeType, left.from, right.to)
-      node.push(left, opNode, right)
-      left = node
-    }
-    return left
-  }
-
   // dotget in a statement/expression (something.blah) or (something.blah arg1)
   dotGetFunctionCall(): SyntaxNode {
     const dotGet = this.dotGet()
 
-    // if followed by a binary operator (not pipe), return dotGet/Word as-is for expression parser
+    // dotget not in scope, regular Word
-    if (this.is($T.Operator) && !this.is($T.Operator, '|'))
+    if (dotGet.type === 'Word') return dotGet
+
+    if (this.is($T.Operator) && !this.isPipe())
       return dotGet
 
-    // dotget not in scope, regular Word
+    else if (this.isPipe() || this.isExprEnd())
-    if (dotGet.type.is('Word')) return dotGet
-
-    if (this.isExprEnd())
       return this.functionCallOrIdentifier(dotGet)
+
     else
       return this.functionCall(dotGet)
   }
@@ -588,8 +595,17 @@ export class Parser {
     const ident = fn ?? this.identifier()
 
     const args: SyntaxNode[] = []
-    while (!this.isExprEnd())
+    while (!this.isExprEnd() && !this.is($T.Operator, '|')) {
-      args.push(this.is($T.NamedArgPrefix) ? this.namedArg() : this.arg())
+      if (this.is($T.NamedArgPrefix)) {
+        args.push(this.namedArg())
+      } else {
+        // 'do' is the only keyword allowed as a function argument
+        const val = this.is($T.Keyword, 'do') ? this.do() : this.value()
+        const arg = new SyntaxNode('PositionalArg', val.from, val.to)
+        arg.add(val)
+        args.push(arg)
+      }
+    }
 
     const node = new SyntaxNode('FunctionCall', ident.from, (args.at(-1) || ident).to)
     node.push(ident, ...args)
@@ -610,7 +626,7 @@ export class Parser {
       inner = this.dotGet()
 
       // if the dotGet was just a Word, bail
-      if (inner.type.is('Word')) return inner
+      if (inner.type === 'Word') return inner
     }
 
     inner ??= this.identifier()
@@ -699,7 +715,7 @@ export class Parser {
   // abc= true
   namedArg(): SyntaxNode {
     const prefix = SyntaxNode.from(this.expect($T.NamedArgPrefix))
-    const val = this.arg(true)
+    const val = this.value()
     const node = new SyntaxNode('NamedArg', prefix.from, val.to)
     return node.push(prefix, val)
   }
@@ -709,7 +725,7 @@ export class Parser {
     const prefix = SyntaxNode.from(this.expect($T.NamedArgPrefix))
     const val = this.value()
 
-    if (!['Null', 'Boolean', 'Number', 'String'].includes(val.type.name))
+    if (!['Null', 'Boolean', 'Number', 'String'].includes(val.type))
       throw `[namedParam] default value must be Null|Bool|Num|Str, got ${val.type}\n\n       ${this.input}\n`
 
     const node = new SyntaxNode('NamedParam', prefix.from, val.to)
@@ -755,7 +771,7 @@ export class Parser {
   // throw blah
   throw(): SyntaxNode {
     const keyword = this.keyword('throw')
-    const val = this.expression()
+    const val = this.value()
     const node = new SyntaxNode('Throw', keyword.from, val.to)
     return node.push(keyword, val)
   }
@@ -869,7 +885,6 @@ export class Parser {
 
   isExprEnd(): boolean {
     return this.isAny($T.Colon, $T.Semicolon, $T.Newline, $T.CloseParen, $T.CloseBracket) ||
-      this.is($T.Operator, '|') ||
       this.isExprEndKeyword() || !this.current()
   }
 
@@ -918,9 +933,9 @@ function collapseDotGets(origNodes: SyntaxNode[]): SyntaxNode {
   while (nodes.length > 0) {
     const left = nodes.pop()!
 
-    if (left.type.is('Identifier')) left.type = 'IdentifierBeforeDot'
+    if (left.type === 'Identifier') left.type = 'IdentifierBeforeDot'
 
-    const dot = new SyntaxNode("DotGet", left.from, right.to)
+    const dot = new SyntaxNode("DotGet", left.from, right.to);
     dot.push(left, right)
 
     right = dot

src/parser/stringParser.ts

@@ -1,32 +1,27 @@
 import { SyntaxNode } from './node'
 
-/**
+
- * Parse string contents into fragments, interpolations, and escape sequences.
+// Parse string contents into fragments, interpolations, and escape sequences.
- *
- * Input: full string including quotes, e.g. "'hello $name'"
- * Output: SyntaxNode tree with StringFragment, Interpolation, EscapeSeq children
- */
 export const parseString = (input: string, from: number, to: number, parser: any): SyntaxNode => {
   const stringNode = new SyntaxNode('String', from, to)
   const content = input.slice(from, to)
 
-  // Determine string type
   const firstChar = content[0]
 
-  // Double-quoted strings: no interpolation or escapes
+  // double quotes: no interpolation or escapes
   if (firstChar === '"') {
     const fragment = new SyntaxNode('DoubleQuote', from, to)
     stringNode.add(fragment)
     return stringNode
   }
 
-  // Curly strings: interpolation but no escapes
+  // curlies: interpolation but no escapes
   if (firstChar === '{') {
     parseCurlyString(stringNode, input, from, to, parser)
     return stringNode
   }
 
-  // Single-quoted strings: interpolation and escapes
+  // single-quotes: interpolation and escapes
   if (firstChar === "'") {
     parseSingleQuoteString(stringNode, input, from, to, parser)
     return stringNode
@@ -35,26 +30,19 @@ export const parseString = (input: string, from: number, to: number, parser: any
   throw `Unknown string type starting with: ${firstChar}`
 }
 
-/**
- * Parse single-quoted string: 'hello $name\n'
- * Supports: interpolation ($var, $(expr)), escape sequences (\n, \$, etc)
- */
 const parseSingleQuoteString = (stringNode: SyntaxNode, input: string, from: number, to: number, parser: any) => {
-  let pos = from + 1 // Skip opening '
+  let pos = from + 1 // skip opening '
   let fragmentStart = pos
 
   while (pos < to - 1) { // -1 to skip closing '
     const char = input[pos]
 
-    // Escape sequence
     if (char === '\\' && pos + 1 < to - 1) {
-      // Push accumulated fragment
       if (pos > fragmentStart) {
         const frag = new SyntaxNode('StringFragment', fragmentStart, pos)
         stringNode.add(frag)
       }
 
-      // Add escape sequence node
       const escNode = new SyntaxNode('EscapeSeq', pos, pos + 2)
       stringNode.add(escNode)
 
@@ -63,19 +51,15 @@ const parseSingleQuoteString = (stringNode: SyntaxNode, input: string, from: num
       continue
     }
 
-    // Interpolation
     if (char === '$') {
-      // Push accumulated fragment
       if (pos > fragmentStart) {
         const frag = new SyntaxNode('StringFragment', fragmentStart, pos)
         stringNode.add(frag)
       }
 
-      pos++ // Skip $
+      pos++ // skip $
 
-      // Parse interpolation content
       if (input[pos] === '(') {
-        // Expression interpolation: $(expr)
         const interpStart = pos - 1 // Include the $
         const exprResult = parseInterpolationExpr(input, pos, parser)
         const interpNode = new SyntaxNode('Interpolation', interpStart, exprResult.endPos)
@@ -83,7 +67,6 @@ const parseSingleQuoteString = (stringNode: SyntaxNode, input: string, from: num
         stringNode.add(interpNode)
         pos = exprResult.endPos
       } else {
-        // Variable interpolation: $name
         const interpStart = pos - 1
         const identEnd = findIdentifierEnd(input, pos, to - 1)
         const identNode = new SyntaxNode('FunctionCallOrIdentifier', pos, identEnd)
@@ -103,27 +86,21 @@ const parseSingleQuoteString = (stringNode: SyntaxNode, input: string, from: num
     pos++
   }
 
-  // Push final fragment
   if (pos > fragmentStart && fragmentStart < to - 1) {
     const frag = new SyntaxNode('StringFragment', fragmentStart, pos)
     stringNode.add(frag)
   }
 }
 
-/**
- * Parse curly string: { hello $name }
- * Supports: interpolation ($var, $(expr)), nested braces
- * Does NOT support: escape sequences (raw content)
- */
 const parseCurlyString = (stringNode: SyntaxNode, input: string, from: number, to: number, parser: any) => {
-  let pos = from + 1 // Skip opening {
+  let pos = from + 1 // skip opening {
-  let fragmentStart = from // Include the opening { in the fragment
+  let fragmentStart = from // include the opening { in the fragment
   let depth = 1
 
   while (pos < to && depth > 0) {
     const char = input[pos]
 
-    // Track brace nesting
+    // track nesting
     if (char === '{') {
       depth++
       pos++
@@ -133,7 +110,6 @@ const parseCurlyString = (stringNode: SyntaxNode, input: string, from: number, t
     if (char === '}') {
       depth--
       if (depth === 0) {
-        // Push final fragment including closing }
         const frag = new SyntaxNode('CurlyString', fragmentStart, pos + 1)
         stringNode.add(frag)
         break
@@ -142,19 +118,29 @@ const parseCurlyString = (stringNode: SyntaxNode, input: string, from: number, t
       continue
     }
 
-    // Interpolation
+    if (char === '\\' && pos + 1 < to && input[pos + 1] === '$') {
-    if (char === '$') {
-      // Push accumulated fragment
       if (pos > fragmentStart) {
         const frag = new SyntaxNode('CurlyString', fragmentStart, pos)
         stringNode.add(frag)
       }
 
-      pos++ // Skip $
+      const escapedFrag = new SyntaxNode('CurlyString', pos + 1, pos + 2)
+      stringNode.add(escapedFrag)
+
+      pos += 2 // skip \ and $
+      fragmentStart = pos
+      continue
+    }
+
+    if (char === '$') {
+      if (pos > fragmentStart) {
+        const frag = new SyntaxNode('CurlyString', fragmentStart, pos)
+        stringNode.add(frag)
+      }
+
+      pos++ // skip $
 
-      // Parse interpolation content
       if (input[pos] === '(') {
-        // Expression interpolation: $(expr)
         const interpStart = pos - 1
         const exprResult = parseInterpolationExpr(input, pos, parser)
         const interpNode = new SyntaxNode('Interpolation', interpStart, exprResult.endPos)
@@ -162,7 +148,6 @@ const parseCurlyString = (stringNode: SyntaxNode, input: string, from: number, t
         stringNode.add(interpNode)
         pos = exprResult.endPos
       } else {
-        // Variable interpolation: $name
         const interpStart = pos - 1
         const identEnd = findIdentifierEnd(input, pos, to)
         const identNode = new SyntaxNode('FunctionCallOrIdentifier', pos, identEnd)
@@ -183,16 +168,10 @@ const parseCurlyString = (stringNode: SyntaxNode, input: string, from: number, t
   }
 }
 
-/**
- * Parse a parenthesized expression interpolation: $(a + b)
- * Returns the parsed expression node and the position after the closing )
- * pos is position of the opening ( in the full input string
- */
 const parseInterpolationExpr = (input: string, pos: number, parser: any): { node: SyntaxNode, endPos: number } => {
-  // Find matching closing paren
   let depth = 1
   let start = pos
-  let end = pos + 1 // Start after opening (
+  let end = pos + 1 // start after opening (
 
   while (end < input.length && depth > 0) {
     if (input[end] === '(') depth++
@@ -205,28 +184,21 @@ const parseInterpolationExpr = (input: string, pos: number, parser: any): { node
 
   const exprContent = input.slice(start + 1, end) // Content between ( and )
   const closeParen = end
-  end++ // Move past closing )
+  end++ // move past closing )
 
-  // Use the main parser to parse the expression
   const exprNode = parser.parse(exprContent)
 
-  // Get the first real node (skip Program wrapper)
   const innerNode = exprNode.firstChild || exprNode
 
-  // Adjust node positions: they're relative to exprContent, need to offset to full input
+  const offset = start + 1 // position where exprContent starts in input
-  const offset = start + 1 // Position where exprContent starts in full input
   adjustNodePositions(innerNode, offset)
 
-  // Wrap in ParenExpr - use positions in the full string
   const parenNode = new SyntaxNode('ParenExpr', start, closeParen + 1)
   parenNode.add(innerNode)
 
   return { node: parenNode, endPos: end }
 }
 
-/**
- * Recursively adjust all node positions by adding an offset
- */
 const adjustNodePositions = (node: SyntaxNode, offset: number) => {
   node.from += offset
   node.to += offset
@@ -236,15 +208,11 @@ const adjustNodePositions = (node: SyntaxNode, offset: number) => {
   }
 }
 
-/**
- * Find the end position of an identifier starting at pos
- * Identifiers: lowercase letter or emoji, followed by letters/digits/dashes/emoji
- */
 const findIdentifierEnd = (input: string, pos: number, maxPos: number): number => {
   let end = pos
 
   while (end < maxPos) {
-    const char = input[end]
+    const char = input[end]!
 
     // Stop at non-identifier characters
     if (!/[a-z0-9\-?]/.test(char)) {

src/parser/terms.ts

@@ -0,0 +1,86 @@
+import * as terms from '#parser/shrimp.terms'
+
+export function nameToId(name: string): number {
+  switch (name) {
+    case 'Star': return terms.Star
+    case 'Slash': return terms.Slash
+    case 'Plus': return terms.Plus
+    case 'Minus': return terms.Minus
+    case 'And': return terms.And
+    case 'Or': return terms.Or
+    case 'Eq': return terms.Eq
+    case 'EqEq': return terms.EqEq
+    case 'Neq': return terms.Neq
+    case 'Lt': return terms.Lt
+    case 'Lte': return terms.Lte
+    case 'Gt': return terms.Gt
+    case 'Gte': return terms.Gte
+    case 'Modulo': return terms.Modulo
+    case 'PlusEq': return terms.PlusEq
+    case 'MinusEq': return terms.MinusEq
+    case 'StarEq': return terms.StarEq
+    case 'SlashEq': return terms.SlashEq
+    case 'ModuloEq': return terms.ModuloEq
+    case 'Band': return terms.Band
+    case 'Bor': return terms.Bor
+    case 'Bxor': return terms.Bxor
+    case 'Shl': return terms.Shl
+    case 'Shr': return terms.Shr
+    case 'Ushr': return terms.Ushr
+    case 'NullishCoalesce': return terms.NullishCoalesce
+    case 'NullishEq': return terms.NullishEq
+    case 'Identifier': return terms.Identifier
+    case 'AssignableIdentifier': return terms.AssignableIdentifier
+    case 'Word': return terms.Word
+    case 'IdentifierBeforeDot': return terms.IdentifierBeforeDot
+    case 'CurlyString': return terms.CurlyString
+    case 'newline': return terms.newline
+    case 'pipeStartsLine': return terms.pipeStartsLine
+    case 'Do': return terms.Do
+    case 'Comment': return terms.Comment
+    case 'Program': return terms.Program
+    case 'PipeExpr': return terms.PipeExpr
+    case 'WhileExpr': return terms.WhileExpr
+    case 'keyword': return terms.keyword
+    case 'ConditionalOp': return terms.ConditionalOp
+    case 'ParenExpr': return terms.ParenExpr
+    case 'FunctionCallWithNewlines': return terms.FunctionCallWithNewlines
+    case 'DotGet': return terms.DotGet
+    case 'Number': return terms.Number
+    case 'Dollar': return terms.Dollar
+    case 'PositionalArg': return terms.PositionalArg
+    case 'FunctionDef': return terms.FunctionDef
+    case 'Params': return terms.Params
+    case 'NamedParam': return terms.NamedParam
+    case 'NamedArgPrefix': return terms.NamedArgPrefix
+    case 'String': return terms.String
+    case 'StringFragment': return terms.StringFragment
+    case 'Interpolation': return terms.Interpolation
+    case 'FunctionCallOrIdentifier': return terms.FunctionCallOrIdentifier
+    case 'EscapeSeq': return terms.EscapeSeq
+    case 'DoubleQuote': return terms.DoubleQuote
+    case 'Boolean': return terms.Boolean
+    case 'Null': return terms.Null
+    case 'colon': return terms.colon
+    case 'CatchExpr': return terms.CatchExpr
+    case 'Block': return terms.Block
+    case 'FinallyExpr': return terms.FinallyExpr
+    case 'Underscore': return terms.Underscore
+    case 'NamedArg': return terms.NamedArg
+    case 'IfExpr': return terms.IfExpr
+    case 'FunctionCall': return terms.FunctionCall
+    case 'ElseIfExpr': return terms.ElseIfExpr
+    case 'ElseExpr': return terms.ElseExpr
+    case 'BinOp': return terms.BinOp
+    case 'Regex': return terms.Regex
+    case 'Dict': return terms.Dict
+    case 'Array': return terms.Array
+    case 'FunctionCallWithBlock': return terms.FunctionCallWithBlock
+    case 'TryExpr': return terms.TryExpr
+    case 'Throw': return terms.Throw
+    case 'Import': return terms.Import
+    case 'CompoundAssign': return terms.CompoundAssign
+    case 'Assign': return terms.Assign
+    default: throw `unknown term: ${name}`
+  }
+}

src/parser/tests/exceptions.test.ts

@@ -139,24 +139,11 @@ describe('try/catch/finally/throw', () => {
     `)
   })
 
-  test('parses throw statement with BinOp', () => {
-    expect("throw 'error message:' + msg").toMatchTree(`
-      Throw
-        keyword throw
-        BinOp
-          String
-            StringFragment error message:
-          Plus +
-          Identifier msg
-    `)
-  })
-
   test('parses throw statement with identifier', () => {
     expect('throw error-object').toMatchTree(`
       Throw
         keyword throw
-        FunctionCallOrIdentifier
+        Identifier error-object
-          Identifier error-object
     `)
   })
 

src/parser/tests/functions.test.ts

@@ -43,58 +43,6 @@ describe('calling functions', () => {
     `)
   })
 
-  test('call with function', () => {
-    expect(`tail do x: x end`).toMatchTree(`
-      FunctionCall
-        Identifier tail
-        PositionalArg
-          FunctionDef
-            Do do
-            Params
-              Identifier x
-            colon :
-            FunctionCallOrIdentifier
-              Identifier x
-            keyword end
-    `)
-  })
-
-  test('call with arg and function', () => {
-    expect(`tail true do x: x end`).toMatchTree(`
-      FunctionCall
-        Identifier tail
-        PositionalArg
-          Boolean true
-        PositionalArg
-          FunctionDef
-            Do do
-            Params
-              Identifier x
-            colon :
-            FunctionCallOrIdentifier
-              Identifier x
-            keyword end
-    `)
-  })
-
-  test('call with function in named arg', () => {
-    expect(`tail callback=do x: x end`).toMatchTree(`
-      FunctionCall
-        Identifier tail
-        NamedArg
-          NamedArgPrefix callback=
-          FunctionDef
-            Do do
-            Params
-              Identifier x
-            colon :
-            FunctionCallOrIdentifier
-              Identifier x
-            keyword end
-      `)
-  })
-
-
   test('command with arg that is also a command', () => {
     expect('tail tail').toMatchTree(`
       FunctionCall

src/parser/tests/literals.test.ts

@@ -336,22 +336,6 @@ describe('dict literals', () => {
     `)
   })
 
-  test('work with functions', () => {
-    expect(`[trap=do x: x end]`).toMatchTree(`
-      Dict
-        NamedArg
-          NamedArgPrefix trap=
-          FunctionDef
-            Do do
-            Params
-              Identifier x
-            colon :
-            FunctionCallOrIdentifier
-              Identifier x
-            keyword end
-    `)
-  })
-
   test('can be nested', () => {
     expect('[a=one b=[two [c=three]]]').toMatchTree(`
       Dict

src/parser/tests/pipes.test.ts

@@ -176,43 +176,6 @@ describe('pipe expressions', () => {
           Identifier echo
       `)
   })
-
-  test('parenthesized expressions can be piped', () => {
-    expect(`(1 + 2) | echo`).toMatchTree(`
-      PipeExpr
-        ParenExpr
-          BinOp
-            Number 1
-            Plus +
-            Number 2
-        operator |
-        FunctionCallOrIdentifier
-          Identifier echo
-      `)
-  })
-
-  test('complex parenthesized expressions with pipes', () => {
-    expect(`((math.random) * 10 + 1) | math.floor`).toMatchTree(`
-      PipeExpr
-        ParenExpr
-          BinOp
-            BinOp
-              ParenExpr
-                FunctionCallOrIdentifier
-                  DotGet
-                    IdentifierBeforeDot math
-                    Identifier random
-              Star *
-              Number 10
-            Plus +
-            Number 1
-        operator |
-        FunctionCallOrIdentifier
-          DotGet
-            IdentifierBeforeDot math
-            Identifier floor
-      `)
-  })
 })
 
 describe('pipe continuation', () => {
@@ -346,7 +309,7 @@ grep h`).toMatchTree(`
       Identifier split
       PositionalArg
         String
-          StringFragment (space)
+          StringFragment  
     operator |
     FunctionCall
       Identifier map
@@ -370,41 +333,3 @@ grep h`).toMatchTree(`
   `)
   })
 })
-
-describe('Underscore', () => {
-  test('works in pipes', () => {
-    expect(`sub 3 1 | div (sub 110 9 | sub 1) _ | div 5`).toMatchTree(`
-      PipeExpr
-        FunctionCall
-          Identifier sub
-          PositionalArg
-            Number 3
-          PositionalArg
-            Number 1
-        operator |
-        FunctionCall
-          Identifier div
-          PositionalArg
-            ParenExpr
-              PipeExpr
-                FunctionCall
-                  Identifier sub
-                  PositionalArg
-                    Number 110
-                  PositionalArg
-                    Number 9
-                operator |
-                FunctionCall
-                  Identifier sub
-                  PositionalArg
-                    Number 1
-          PositionalArg
-            Underscore _
-        operator |
-        FunctionCall
-          Identifier div
-          PositionalArg
-            Number 5
-    `)
-  })
-})

src/parser/tokenizer2.ts

@@ -38,8 +38,7 @@ const valueTokens = new Set([
   TokenType.Comment,
   TokenType.Keyword, TokenType.Operator,
   TokenType.Identifier, TokenType.Word, TokenType.NamedArgPrefix,
-  TokenType.Boolean, TokenType.Number, TokenType.String, TokenType.Regex,
+  TokenType.Boolean, TokenType.Number, TokenType.String, TokenType.Regex
-  TokenType.Underscore
 ])
 
 const operators = new Set([
@@ -107,8 +106,8 @@ const keywords = new Set([
 ])
 
 // helper
-function c(strings: TemplateStringsArray, ...values: any[]) {
+function c(strings: TemplateStringsArray) {
-  return strings.reduce((result, str, i) => result + str + (values[i] ?? ""), "").charCodeAt(0)
+  return strings[0]!.charCodeAt(0)
 }
 
 function s(c: number): string {
@@ -290,7 +289,7 @@ export class Scanner {
   }
 
   readCurlyString() {
-    this.start = this.pos - 1
+    this.start = this.pos - 1  // include opening {
     let depth = 1
     this.next()
 
@@ -339,7 +338,7 @@ export class Scanner {
 
     // classify the token based on what we read
     if (word === '_')
-      this.push(TokenType.Underscore)
+      this.pushChar(TokenType.Underscore)
 
     else if (word === 'null')
       this.push(TokenType.Null)
@@ -387,37 +386,25 @@ export class Scanner {
     this.start = this.pos - 1
     this.next() // skip 2nd /
 
+    let foundClosing = false
     while (this.char > 0) {
       if (this.char === c`/` && this.peek() === c`/`) {
         this.next() // skip /
         this.next() // skip /
-
+        foundClosing = true
-        // read regex flags
-        while (this.char > 0 && isIdentStart(this.char))
-          this.next()
-
-        // validate regex
-        const to = this.pos - getCharSize(this.char)
-        const regexText = this.input.slice(this.start, to)
-        const [_, pattern, flags] = regexText.match(/^\/\/(.*)\/\/([gimsuy]*)$/) || []
-
-        if (pattern) {
-          try {
-            new RegExp(pattern, flags)
-            this.push(TokenType.Regex)
-            break
-          } catch (e) {
-            // invalid regex - fall through to Word
-          }
-        }
-
-        // invalid regex is treated as Word
-        this.push(TokenType.Word)
         break
       }
 
       this.next()
     }
+
+    const closing = new Set([c`g`, c`i`, c`m`, c`s`, c`u`, c`y`])
+
+    // read flags (e.g., 'gi', 'gim', etc.)
+    if (foundClosing)
+      while (closing.has(this.char)) this.next()
+
+    this.push(TokenType.Regex)
   }
 
   canBeDotGet(lastToken?: Token): boolean {

src/prelude/tests/info.test.ts

@@ -1,89 +1,90 @@
 import { expect, describe, test } from 'bun:test'
+import { globals } from '#prelude'
 
 describe('var and var?', () => {
   test('var? checks if a variable exists', async () => {
-    await expect(`var? 'nada'`).toEvaluateTo(false)
+    await expect(`var? 'nada'`).toEvaluateTo(false, globals)
-    await expect(`var? 'info'`).toEvaluateTo(false)
+    await expect(`var? 'info'`).toEvaluateTo(false, globals)
-    await expect(`abc = abc; var? 'abc'`).toEvaluateTo(true)
+    await expect(`abc = abc; var? 'abc'`).toEvaluateTo(true, globals)
-    await expect(`var? 'var?'`).toEvaluateTo(true)
+    await expect(`var? 'var?'`).toEvaluateTo(true, globals)
 
-    await expect(`var? 'dict'`).toEvaluateTo(true)
+    await expect(`var? 'dict'`).toEvaluateTo(true, globals)
-    await expect(`var? dict`).toEvaluateTo(true)
+    await expect(`var? dict`).toEvaluateTo(true, globals)
   })
 
   test('var returns a value or null', async () => {
-    await expect(`var 'nada'`).toEvaluateTo(null)
+    await expect(`var 'nada'`).toEvaluateTo(null, globals)
-    await expect(`var nada`).toEvaluateTo(null)
+    await expect(`var nada`).toEvaluateTo(null, globals)
-    await expect(`var 'info'`).toEvaluateTo(null)
+    await expect(`var 'info'`).toEvaluateTo(null, globals)
-    await expect(`abc = my-string; var 'abc'`).toEvaluateTo('my-string')
+    await expect(`abc = my-string; var 'abc'`).toEvaluateTo('my-string', globals)
-    await expect(`abc = my-string; var abc`).toEvaluateTo(null)
+    await expect(`abc = my-string; var abc`).toEvaluateTo(null, globals)
   })
 })
 
 describe('type predicates', () => {
   test('string? checks for string type', async () => {
-    await expect(`string? 'hello'`).toEvaluateTo(true)
+    await expect(`string? 'hello'`).toEvaluateTo(true, globals)
-    await expect(`string? 42`).toEvaluateTo(false)
+    await expect(`string? 42`).toEvaluateTo(false, globals)
   })
 
   test('number? checks for number type', async () => {
-    await expect(`number? 42`).toEvaluateTo(true)
+    await expect(`number? 42`).toEvaluateTo(true, globals)
-    await expect(`number? 'hello'`).toEvaluateTo(false)
+    await expect(`number? 'hello'`).toEvaluateTo(false, globals)
   })
 
   test('boolean? checks for boolean type', async () => {
-    await expect(`boolean? true`).toEvaluateTo(true)
+    await expect(`boolean? true`).toEvaluateTo(true, globals)
-    await expect(`boolean? 42`).toEvaluateTo(false)
+    await expect(`boolean? 42`).toEvaluateTo(false, globals)
   })
 
   test('array? checks for array type', async () => {
-    await expect(`array? [1 2 3]`).toEvaluateTo(true)
+    await expect(`array? [1 2 3]`).toEvaluateTo(true, globals)
-    await expect(`array? 42`).toEvaluateTo(false)
+    await expect(`array? 42`).toEvaluateTo(false, globals)
   })
 
   test('dict? checks for dict type', async () => {
-    await expect(`dict? [a=1]`).toEvaluateTo(true)
+    await expect(`dict? [a=1]`).toEvaluateTo(true, globals)
-    await expect(`dict? []`).toEvaluateTo(false)
+    await expect(`dict? []`).toEvaluateTo(false, globals)
   })
 
   test('null? checks for null type', async () => {
-    await expect(`null? null`).toEvaluateTo(true)
+    await expect(`null? null`).toEvaluateTo(true, globals)
-    await expect(`null? 42`).toEvaluateTo(false)
+    await expect(`null? 42`).toEvaluateTo(false, globals)
   })
 
   test('some? checks for non-null', async () => {
-    await expect(`some? 42`).toEvaluateTo(true)
+    await expect(`some? 42`).toEvaluateTo(true, globals)
-    await expect(`some? null`).toEvaluateTo(false)
+    await expect(`some? null`).toEvaluateTo(false, globals)
   })
 })
 
 describe('introspection', () => {
   test('type returns proper types', async () => {
-    await expect(`type 'hello'`).toEvaluateTo('string')
+    await expect(`type 'hello'`).toEvaluateTo('string', globals)
-    await expect(`type 42`).toEvaluateTo('number')
+    await expect(`type 42`).toEvaluateTo('number', globals)
-    await expect(`type true`).toEvaluateTo('boolean')
+    await expect(`type true`).toEvaluateTo('boolean', globals)
-    await expect(`type false`).toEvaluateTo('boolean')
+    await expect(`type false`).toEvaluateTo('boolean', globals)
-    await expect(`type null`).toEvaluateTo('null')
+    await expect(`type null`).toEvaluateTo('null', globals)
-    await expect(`type [1 2 3]`).toEvaluateTo('array')
+    await expect(`type [1 2 3]`).toEvaluateTo('array', globals)
-    await expect(`type [a=1 b=2]`).toEvaluateTo('dict')
+    await expect(`type [a=1 b=2]`).toEvaluateTo('dict', globals)
   })
 
   test('inspect formats values', async () => {
-    await expect(`inspect 'hello'`).toEvaluateTo("\u001b[32m'hello\u001b[32m'\u001b[0m")
+    await expect(`inspect 'hello'`).toEvaluateTo("\u001b[32m'hello\u001b[32m'\u001b[0m", globals)
   })
 
   test('describe describes values', async () => {
-    await expect(`describe 'hello'`).toEvaluateTo("#<string: \u001b[32m'hello\u001b[32m'\u001b[0m>")
+    await expect(`describe 'hello'`).toEvaluateTo("#<string: \u001b[32m'hello\u001b[32m'\u001b[0m>", globals)
   })
 })
 
 describe('environment', () => {
   test('args is an array', async () => {
-    await expect(`array? $.args`).toEvaluateTo(true)
+    await expect(`array? $.args`).toEvaluateTo(true, globals)
   })
 
   test('args can be accessed', async () => {
-    await expect(`type $.args`).toEvaluateTo('array')
+    await expect(`type $.args`).toEvaluateTo('array', globals)
   })
 
   test('argv includes more than just the args', async () => {
@@ -105,35 +106,35 @@ describe('ref', () => {
 
 describe('$ global dictionary', () => {
   test('$.args is an array', async () => {
-    await expect(`$.args | array?`).toEvaluateTo(true)
+    await expect(`$.args | array?`).toEvaluateTo(true, globals)
   })
 
   test('$.args can be accessed', async () => {
-    await expect(`$.args | type`).toEvaluateTo('array')
+    await expect(`$.args | type`).toEvaluateTo('array', globals)
   })
 
   test('$.script.name is a string', async () => {
-    await expect(`$.script.name | string?`).toEvaluateTo(true)
+    await expect(`$.script.name | string?`).toEvaluateTo(true, globals)
   })
 
   test('$.script.path is a string', async () => {
-    await expect(`$.script.path | string?`).toEvaluateTo(true)
+    await expect(`$.script.path | string?`).toEvaluateTo(true, globals)
   })
 
   test('$.env is a dict', async () => {
-    await expect(`$.env | dict?`).toEvaluateTo(true)
+    await expect(`$.env | dict?`).toEvaluateTo(true, globals)
   })
 
   test('$.pid is a number', async () => {
-    await expect(`$.pid | number?`).toEvaluateTo(true)
+    await expect(`$.pid | number?`).toEvaluateTo(true, globals)
-    await expect(`$.pid > 0`).toEvaluateTo(true)
+    await expect(`$.pid > 0`).toEvaluateTo(true, globals)
   })
 
   test('$.cwd is a string', async () => {
-    await expect(`$.cwd | string?`).toEvaluateTo(true)
+    await expect(`$.cwd | string?`).toEvaluateTo(true, globals)
   })
 
   test('$.cwd returns current working directory', async () => {
-    await expect(`$.cwd`).toEvaluateTo(process.cwd())
+    await expect(`$.cwd`).toEvaluateTo(process.cwd(), globals)
   })
 })

src/prelude/tests/load.test.ts

@@ -1,41 +1,42 @@
 import { expect, describe, test } from 'bun:test'
+import { globals } from '#prelude'
 
 describe('loading a file', () => {
   test(`imports all a file's functions`, async () => {
     expect(`
       math = load ./src/prelude/tests/math.sh
       math.double 4
-    `).toEvaluateTo(8)
+    `).toEvaluateTo(8, globals)
 
     expect(`
       math = load ./src/prelude/tests/math.sh
       math.double (math.double 4) 
-    `).toEvaluateTo(16)
+    `).toEvaluateTo(16, globals)
 
     expect(`
       math = load ./src/prelude/tests/math.sh
       dbl = ref math.double 
       dbl (dbl 2)
-    `).toEvaluateTo(8)
+    `).toEvaluateTo(8, globals)
 
     expect(`
       math = load ./src/prelude/tests/math.sh
       math.pi
-    `).toEvaluateTo(3.14)
+    `).toEvaluateTo(3.14, globals)
 
     expect(`
       math = load ./src/prelude/tests/math.sh
       math | at 🥧
-    `).toEvaluateTo(3.14159265359)
+    `).toEvaluateTo(3.14159265359, globals)
 
     expect(`
       math = load ./src/prelude/tests/math.sh
       math.🥧
-    `).toEvaluateTo(3.14159265359)
+    `).toEvaluateTo(3.14159265359, globals)
 
     expect(`
       math = load ./src/prelude/tests/math.sh
       math.add1 5
-    `).toEvaluateTo(6)
+    `).toEvaluateTo(6, globals)
   })
 })

src/prelude/tests/prelude.test.ts

@@ -1,4 +1,5 @@
 import { expect, describe, test } from 'bun:test'
+import { globals } from '#prelude'
 
 describe('string operations', () => {
   test('to-upper converts to uppercase', async () => {

src/utils/tree.ts

@@ -11,9 +11,8 @@ const nodeToString = (node: SyntaxNode, input: string, depth = 0): string => {
     return `${indent}${nodeName}`
   } else {
     // Only strip quotes from whole String nodes (legacy DoubleQuote), not StringFragment/EscapeSeq/CurlyString
-    let cleanText = nodeName === 'String' ? text.slice(1, -1) : text
+    const cleanText = nodeName === 'String' ? text.slice(1, -1) : text
-    if (cleanText === ' ') cleanText = '(space)'
+    return `${indent}${nodeName} ${cleanText}`
-    return cleanText ? `${indent}${nodeName} ${cleanText}` : `${indent}${nodeName}`
   }
 }