Compare commits

..

No commits in common. "main" and "broken-shrimp" have entirely different histories.

60 changed files with 1498 additions and 4394 deletions

3
.gitignore vendored
View File

@ -34,7 +34,4 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
.DS_Store .DS_Store
/tmp /tmp
vscode-extension/tmp
/docs /docs
*.vsix

View File

@ -1,192 +0,0 @@
#!/usr/bin/env bun
// WARNING: [[ No human has been anywhere near this file. It's pure Claude slop.
// Enter at your own risk. ]]
import { readFileSync } from 'fs'
type CallInfo = {
method: string
line: number
calls: Set<string>
isRecursive?: boolean
}
// Parse the parser file and extract method calls
function analyzeParser(filePath: string): Map<string, CallInfo> {
const content = readFileSync(filePath, 'utf-8')
const lines = content.split('\n')
const methods = new Map<string, CallInfo>()
// Find all method definitions
const methodRegex = /^\s*(\w+)\s*\([^)]*\):\s*/
let currentMethod: string | null = null
let braceDepth = 0
let classDepth = 0
for (let i = 0; i < lines.length; i++) {
const line = lines[i] || ''
// Track if we're inside the Parser class
if (line.includes('class Parser')) {
classDepth = braceDepth + 1 // Will be the depth after we process this line's brace
}
// Check for method definition (only inside class, at class level)
// Check BEFORE incrementing braceDepth
if (classDepth > 0 && braceDepth === classDepth) {
const methodMatch = line.match(methodRegex)
if (methodMatch && !line.includes('class ')) {
currentMethod = methodMatch[1]!
methods.set(currentMethod, {
method: currentMethod,
line: i + 1,
calls: new Set()
})
}
}
// Track brace depth
braceDepth += (line.match(/{/g) || []).length
braceDepth -= (line.match(/}/g) || []).length
// Find method calls within current method
if (currentMethod && braceDepth > 0) {
// Match this.methodName() calls
const callRegex = /this\.(\w+)\s*\(/g
let match
while ((match = callRegex.exec(line)) !== null) {
const calledMethod = match[1]!
const info = methods.get(currentMethod)!
info.calls.add(calledMethod)
// Mark recursive calls
if (calledMethod === currentMethod) {
info.isRecursive = true
}
}
}
// Reset when method ends
if (braceDepth === 0) {
currentMethod = null
}
}
return methods
}
// Build tree structure starting from a root method
function buildTree(
method: string,
callGraph: Map<string, CallInfo>,
visited: Set<string>,
indent = '',
isLast = true,
depth = 0,
maxDepth = 3
): string[] {
const lines: string[] = []
const info = callGraph.get(method)
if (!info) return lines
// Add current method
const prefix = depth === 0 ? '' : (isLast ? '└─> ' : '├─> ')
const suffix = info.isRecursive ? ' (recursive)' : ''
const lineNum = `[line ${info.line}]`
lines.push(`${indent}${prefix}${method}() ${lineNum}${suffix}`)
// Stop if we've reached max depth
if (depth >= maxDepth) {
return lines
}
// Prevent infinite recursion in tree display
if (visited.has(method)) {
return lines
}
const newVisited = new Set(visited)
newVisited.add(method)
// Helper methods to filter out (low-level utilities)
const helperPatterns = /^(is|next|peek|expect|current|op)/i
// Get sorted unique calls (filter out recursive self-calls for display)
const calls = Array.from(info.calls)
.filter(c => callGraph.has(c)) // Only show parser methods
.filter(c => c !== method) // Don't show immediate self-recursion
.filter(c => !helperPatterns.test(c)) // Filter out helpers
.sort()
// Add children
const newIndent = indent + (isLast ? ' ' : '│ ')
calls.forEach((call, idx) => {
const childLines = buildTree(
call,
callGraph,
newVisited,
newIndent,
idx === calls.length - 1,
depth + 1,
maxDepth
)
lines.push(...childLines)
})
return lines
}
// Main
const parserPath = './src/parser/parser2.ts'
const maxDepth = parseInt(process.argv[2] || '5')
console.log('Parser Call Tree for', parserPath)
console.log(`Max depth: ${maxDepth}`)
console.log('═'.repeat(60))
console.log()
const callGraph = analyzeParser(parserPath)
// Start from parse() method
const tree = buildTree('parse', callGraph, new Set(), '', true, 0, maxDepth)
console.log(tree.join('\n'))
// Show some stats
console.log('\n' + '═'.repeat(60))
console.log('Stats:')
console.log(` Total methods: ${callGraph.size}`)
console.log(` Entry point: parse()`)
// Find methods that are never called (potential dead code or entry points)
const allCalled = new Set<string>()
for (const info of callGraph.values()) {
info.calls.forEach(c => allCalled.add(c))
}
const uncalled = Array.from(callGraph.keys())
.filter(m => !allCalled.has(m) && m !== 'parse')
.sort()
if (uncalled.length > 0) {
console.log(`\n Uncalled methods: ${uncalled.join(', ')}`)
}
// Find most-called methods
const callCount = new Map<string, number>()
for (const info of callGraph.values()) {
for (const called of info.calls) {
callCount.set(called, (callCount.get(called) || 0) + 1)
}
}
const topCalled = Array.from(callCount.entries())
.sort((a, b) => b[1] - a[1])
.slice(0, 5)
console.log(`\n Most-called methods:`)
for (const [method, count] of topCalled) {
console.log(` ${method}() - called ${count} times`)
}

View File

@ -1,7 +1,7 @@
#!/usr/bin/env bun #!/usr/bin/env bun
import { colors, globals as prelude } from '../src/prelude' import { colors, globals as prelude } from '../src/prelude'
import { treeToString2 } from '../src/utils/tree' import { treeToString } from '../src/utils/tree'
import { runCode, runFile, compileFile, parseCode } from '../src' import { runCode, runFile, compileFile, parseCode } from '../src'
import { resolve } from 'path' import { resolve } from 'path'
import { bytecodeToString } from 'reefvm' import { bytecodeToString } from 'reefvm'
@ -31,7 +31,7 @@ ${colors.bright}Options:${colors.reset}
} }
function showVersion() { function showVersion() {
console.log('🦐 v0.0.1 (non-lezer parser)') console.log('🦐 v0.0.1')
} }
async function evalCode(code: string, imports: string[]) { async function evalCode(code: string, imports: string[]) {
@ -143,7 +143,7 @@ async function main() {
process.exit(1) process.exit(1)
} }
const input = readFileSync(file, 'utf-8') const input = readFileSync(file, 'utf-8')
console.log(treeToString2(parseCode(input).topNode, input)) console.log(treeToString(parseCode(input), input))
return return
} }

View File

@ -16,8 +16,6 @@
"@lezer/highlight": "^1.2.1", "@lezer/highlight": "^1.2.1",
"@lezer/lr": "^1.4.2", "@lezer/lr": "^1.4.2",
"@types/bun": "latest", "@types/bun": "latest",
"diff": "^8.0.2",
"kleur": "^4.1.5",
}, },
}, },
}, },
@ -62,12 +60,8 @@
"csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="], "csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="],
"diff": ["diff@8.0.2", "", {}, "sha512-sSuxWU5j5SR9QQji/o2qMvqRNYRDOcBTgsJ/DeCf4iSN4gW+gNMXM7wFIP+fdXZxoNiAnHUTGjCr+TSWXdRDKg=="],
"hono": ["hono@4.10.4", "", {}, "sha512-YG/fo7zlU3KwrBL5vDpWKisLYiM+nVstBQqfr7gCPbSYURnNEP9BDxEMz8KfsDR9JX0lJWDRNc6nXX31v7ZEyg=="], "hono": ["hono@4.10.4", "", {}, "sha512-YG/fo7zlU3KwrBL5vDpWKisLYiM+nVstBQqfr7gCPbSYURnNEP9BDxEMz8KfsDR9JX0lJWDRNc6nXX31v7ZEyg=="],
"kleur": ["kleur@4.1.5", "", {}, "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ=="],
"reefvm": ["reefvm@git+https://git.nose.space/defunkt/reefvm#3e2e68b31f504347225a4d705c7568a0957d629e", { "peerDependencies": { "typescript": "^5" } }, "3e2e68b31f504347225a4d705c7568a0957d629e"], "reefvm": ["reefvm@git+https://git.nose.space/defunkt/reefvm#3e2e68b31f504347225a4d705c7568a0957d629e", { "peerDependencies": { "typescript": "^5" } }, "3e2e68b31f504347225a4d705c7568a0957d629e"],
"style-mod": ["style-mod@4.1.3", "", {}, "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ=="], "style-mod": ["style-mod@4.1.3", "", {}, "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ=="],

View File

@ -1,18 +0,0 @@
#!/usr/bin/env shrimp
# usage: dice <sides>
import math only=random
import list only=first
import str only=[replace starts-with?]
sides = $.args | first
sides ??= 20
if sides | starts-with? d:
sides = replace sides //\D// ''
end
sides = number sides
echo 'Rolling d$sides...'
random 1 sides | echo

View File

@ -1,31 +0,0 @@
#!/usr/bin/env shrimp
year = date.now | date.year
project = fs.pwd | fs.basename | str.titlecase
{
Copyright $year $project Authors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the “Software”), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.
}
| str.trim
| echo

View File

@ -1,39 +0,0 @@
#!/usr/bin/env shrimp
# usage: password <length> [!spaced] [!symbols]
if ($.args | list.contains? -h):
echo 'usage: password <length> [!spaced] [!symbols]'
exit
end
password = do n=22 symbols=true spaced=true:
chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
if symbols: chars += '!@#%^&*-=()[]<>' end
out = []
i = 0
max = length chars
while i < n:
idx = math.floor ((math.random) * max)
ch = chars | at idx
list.push out ch
i += 1
end
if spaced:
pos1 = math.floor((n - 2) / 3)
pos2 = math.floor((n - 2) * 2 / 3)
list.insert out pos2 ' '
list.insert out pos1 ' '
end
str.join out ''
end
missing-arg? = do x: $.args | list.contains? x | not end
num = $.args | list.reject (do x: x | str.starts-with? ! end) | list.first
password num symbols=(missing-arg? !symbols) spaced=(missing-arg? !spaced) | echo

View File

@ -1,9 +0,0 @@
#!/usr/bin/env shrimp
if not fs.exists? 'package.json':
echo '🦐 package.json not found'
exit 1
end
package = fs.read 'package.json' | json.decode
package.scripts | dict.keys | list.sort | each do x: echo x end

View File

@ -5,12 +5,12 @@
"private": true, "private": true,
"type": "module", "type": "module",
"scripts": { "scripts": {
"dev": "bun --hot src/server/server.tsx", "dev": "bun generate-parser && bun --hot src/server/server.tsx",
"repl": "bun bin/repl", "generate-parser": "lezer-generator src/parser/shrimp.grammar --typeScript -o src/parser/shrimp.ts",
"repl": "bun generate-parser && bun bin/repl",
"update-reef": "rm -rf ~/.bun/install/cache/ && rm bun.lock && bun update reefvm", "update-reef": "rm -rf ~/.bun/install/cache/ && rm bun.lock && bun update reefvm",
"cli:install": "ln -s \"$(pwd)/bin/shrimp\" ~/.bun/bin/shrimp", "cli:install": "ln -s \"$(pwd)/bin/shrimp\" ~/.bun/bin/shrimp",
"cli:remove": "rm ~/.bun/bin/shrimp", "cli:remove": "rm ~/.bun/bin/shrimp"
"check": "bunx tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@codemirror/view": "^6.38.3", "@codemirror/view": "^6.38.3",
@ -24,9 +24,7 @@
"devDependencies": { "devDependencies": {
"@lezer/highlight": "^1.2.1", "@lezer/highlight": "^1.2.1",
"@lezer/lr": "^1.4.2", "@lezer/lr": "^1.4.2",
"@types/bun": "latest", "@types/bun": "latest"
"diff": "^8.0.2",
"kleur": "^4.1.5"
}, },
"prettier": { "prettier": {
"semi": false, "semi": false,

View File

@ -1,7 +1,9 @@
import { CompilerError } from '#compiler/compilerError.ts' import { CompilerError } from '#compiler/compilerError.ts'
import { parse, setGlobals } from '#parser/parser2' import { parser } from '#parser/shrimp.ts'
import { SyntaxNode, Tree } from '#parser/node' import * as terms from '#parser/shrimp.terms'
import { setGlobals } from '#parser/tokenizer'
import { tokenizeCurlyString } from '#parser/curlyTokenizer' import { tokenizeCurlyString } from '#parser/curlyTokenizer'
import type { SyntaxNode, Tree } from '@lezer/common'
import { assert, errorMessage } from '#utils/utils' import { assert, errorMessage } from '#utils/utils'
import { toBytecode, type Bytecode, type ProgramItem, bytecodeToString } from 'reefvm' import { toBytecode, type Bytecode, type ProgramItem, bytecodeToString } from 'reefvm'
import { import {
@ -61,8 +63,7 @@ export class Compiler {
constructor(public input: string, globals?: string[] | Record<string, any>) { constructor(public input: string, globals?: string[] | Record<string, any>) {
try { try {
if (globals) setGlobals(Array.isArray(globals) ? globals : Object.keys(globals)) if (globals) setGlobals(Array.isArray(globals) ? globals : Object.keys(globals))
const ast = parse(input) const cst = parser.parse(input)
const cst = new Tree(ast)
const errors = checkTreeForErrors(cst) const errors = checkTreeForErrors(cst)
const firstError = errors[0] const firstError = errors[0]
@ -88,7 +89,7 @@ export class Compiler {
} }
#compileCst(cst: Tree, input: string) { #compileCst(cst: Tree, input: string) {
const isProgram = cst.topNode.type.is('Program') const isProgram = cst.topNode.type.id === terms.Program
assert(isProgram, `Expected Program node, got ${cst.topNode.type.name}`) assert(isProgram, `Expected Program node, got ${cst.topNode.type.name}`)
let child = cst.topNode.firstChild let child = cst.topNode.firstChild
@ -104,8 +105,8 @@ export class Compiler {
const value = input.slice(node.from, node.to) const value = input.slice(node.from, node.to)
if (DEBUG) console.log(`🫦 ${node.name}: ${value}`) if (DEBUG) console.log(`🫦 ${node.name}: ${value}`)
switch (node.type.name) { switch (node.type.id) {
case 'Number': case terms.Number:
// Handle sign prefix for hex, binary, and octal literals // Handle sign prefix for hex, binary, and octal literals
// Number() doesn't parse '-0xFF', '+0xFF', '-0o77', etc. correctly // Number() doesn't parse '-0xFF', '+0xFF', '-0o77', etc. correctly
let numberValue: number let numberValue: number
@ -122,8 +123,8 @@ export class Compiler {
return [[`PUSH`, numberValue]] return [[`PUSH`, numberValue]]
case 'String': { case terms.String: {
if (node.firstChild?.type.is('CurlyString')) if (node.firstChild?.type.id === terms.CurlyString)
return this.#compileCurlyString(value, input) return this.#compileCurlyString(value, input)
const { parts, hasInterpolation } = getStringParts(node, input) const { parts, hasInterpolation } = getStringParts(node, input)
@ -140,19 +141,19 @@ export class Compiler {
parts.forEach((part) => { parts.forEach((part) => {
const partValue = input.slice(part.from, part.to) const partValue = input.slice(part.from, part.to)
switch (part.type.name) { switch (part.type.id) {
case 'StringFragment': case terms.StringFragment:
// Plain text fragment - just push as-is // Plain text fragment - just push as-is
instructions.push(['PUSH', partValue]) instructions.push(['PUSH', partValue])
break break
case 'EscapeSeq': case terms.EscapeSeq:
// Process escape sequence and push the result // Process escape sequence and push the result
const processed = processEscapeSeq(partValue) const processed = processEscapeSeq(partValue)
instructions.push(['PUSH', processed]) instructions.push(['PUSH', processed])
break break
case 'Interpolation': case terms.Interpolation:
// Interpolation contains either Identifier or ParenExpr (the $ is anonymous) // Interpolation contains either Identifier or ParenExpr (the $ is anonymous)
const child = part.firstChild const child = part.firstChild
if (!child) { if (!child) {
@ -176,15 +177,15 @@ export class Compiler {
return instructions return instructions
} }
case 'Boolean': { case terms.Boolean: {
return [[`PUSH`, value === 'true']] return [[`PUSH`, value === 'true']]
} }
case 'Null': { case terms.Null: {
return [[`PUSH`, null]] return [[`PUSH`, null]]
} }
case 'Regex': { case terms.Regex: {
// remove the surrounding slashes and any flags // remove the surrounding slashes and any flags
const [_, pattern, flags] = value.match(/^\/\/(.*)\/\/([gimsuy]*)$/) || [] const [_, pattern, flags] = value.match(/^\/\/(.*)\/\/([gimsuy]*)$/) || []
if (!pattern) { if (!pattern) {
@ -201,15 +202,15 @@ export class Compiler {
return [['PUSH', regex]] return [['PUSH', regex]]
} }
case 'Identifier': { case terms.Identifier: {
return [[`TRY_LOAD`, value]] return [[`TRY_LOAD`, value]]
} }
case 'Word': { case terms.Word: {
return [['PUSH', value]] return [['PUSH', value]]
} }
case 'DotGet': { case terms.DotGet: {
// DotGet is parsed into a nested tree because it's hard to parse it into a flat one. // DotGet is parsed into a nested tree because it's hard to parse it into a flat one.
// However, we want a flat tree - so we're going to pretend like we are getting one from the parser. // However, we want a flat tree - so we're going to pretend like we are getting one from the parser.
// //
@ -221,7 +222,7 @@ export class Compiler {
instructions.push(['TRY_LOAD', objectName]) instructions.push(['TRY_LOAD', objectName])
const flattenProperty = (prop: SyntaxNode): void => { const flattenProperty = (prop: SyntaxNode): void => {
if (prop.type.is('DotGet')) { if (prop.type.id === terms.DotGet) {
const nestedParts = getDotGetParts(prop, input) const nestedParts = getDotGetParts(prop, input)
const nestedObjectValue = input.slice(nestedParts.object.from, nestedParts.object.to) const nestedObjectValue = input.slice(nestedParts.object.from, nestedParts.object.to)
@ -230,7 +231,7 @@ export class Compiler {
flattenProperty(nestedParts.property) flattenProperty(nestedParts.property)
} else { } else {
if (prop.type.is('ParenExpr')) { if (prop.type.id === terms.ParenExpr) {
instructions.push(...this.#compileNode(prop, input)) instructions.push(...this.#compileNode(prop, input))
} else { } else {
const propertyValue = input.slice(prop.from, prop.to) const propertyValue = input.slice(prop.from, prop.to)
@ -244,7 +245,7 @@ export class Compiler {
return instructions return instructions
} }
case 'BinOp': { case terms.BinOp: {
const { left, op, right } = getBinaryParts(node) const { left, op, right } = getBinaryParts(node)
const instructions: ProgramItem[] = [] const instructions: ProgramItem[] = []
instructions.push(...this.#compileNode(left, input)) instructions.push(...this.#compileNode(left, input))
@ -292,7 +293,7 @@ export class Compiler {
return instructions return instructions
} }
case 'Assign': { case terms.Assign: {
const assignParts = getAssignmentParts(node) const assignParts = getAssignmentParts(node)
const instructions: ProgramItem[] = [] const instructions: ProgramItem[] = []
@ -323,7 +324,7 @@ export class Compiler {
return instructions return instructions
} }
case 'CompoundAssign': { case terms.CompoundAssign: {
const { identifier, operator, right } = getCompoundAssignmentParts(node) const { identifier, operator, right } = getCompoundAssignmentParts(node)
const identifierName = input.slice(identifier.from, identifier.to) const identifierName = input.slice(identifier.from, identifier.to)
const instructions: ProgramItem[] = [] const instructions: ProgramItem[] = []
@ -385,14 +386,14 @@ export class Compiler {
return instructions return instructions
} }
case 'ParenExpr': { case terms.ParenExpr: {
const child = node.firstChild const child = node.firstChild
if (!child) return [] // I guess it is empty parentheses? if (!child) return [] // I guess it is empty parentheses?
return this.#compileNode(child, input) return this.#compileNode(child, input)
} }
case 'FunctionDef': { case terms.FunctionDef: {
const { paramNames, bodyNodes, catchVariable, catchBody, finallyBody } = const { paramNames, bodyNodes, catchVariable, catchBody, finallyBody } =
getFunctionDefParts(node, input) getFunctionDefParts(node, input)
const instructions: ProgramItem[] = [] const instructions: ProgramItem[] = []
@ -438,8 +439,8 @@ export class Compiler {
return instructions return instructions
} }
case 'FunctionCallOrIdentifier': { case terms.FunctionCallOrIdentifier: {
if (node.firstChild?.type.is('DotGet')) { if (node.firstChild?.type.id === terms.DotGet) {
const instructions: ProgramItem[] = [] const instructions: ProgramItem[] = []
const callLabel: Label = `.call_dotget_${++this.labelCount}` const callLabel: Label = `.call_dotget_${++this.labelCount}`
const afterLabel: Label = `.after_dotget_${++this.labelCount}` const afterLabel: Label = `.after_dotget_${++this.labelCount}`
@ -481,8 +482,8 @@ export class Compiler {
PUSH 1 ; Named count PUSH 1 ; Named count
CALL CALL
*/ */
case terms.FunctionCallWithNewlines:
case 'FunctionCall': { case terms.FunctionCall: {
const { identifierNode, namedArgs, positionalArgs } = getFunctionCallParts(node, input) const { identifierNode, namedArgs, positionalArgs } = getFunctionCallParts(node, input)
const instructions: ProgramItem[] = [] const instructions: ProgramItem[] = []
instructions.push(...this.#compileNode(identifierNode, input)) instructions.push(...this.#compileNode(identifierNode, input))
@ -504,7 +505,7 @@ export class Compiler {
return instructions return instructions
} }
case 'Block': { case terms.Block: {
const children = getAllChildren(node) const children = getAllChildren(node)
const instructions: ProgramItem[] = [] const instructions: ProgramItem[] = []
@ -519,7 +520,7 @@ export class Compiler {
return instructions return instructions
} }
case 'FunctionCallWithBlock': { case terms.FunctionCallWithBlock: {
const [fn, _colon, ...block] = getAllChildren(node) const [fn, _colon, ...block] = getAllChildren(node)
let instructions: ProgramItem[] = [] let instructions: ProgramItem[] = []
@ -537,13 +538,13 @@ export class Compiler {
instructions.push(['RETURN']) instructions.push(['RETURN'])
instructions.push([`${afterLabel}:`]) instructions.push([`${afterLabel}:`])
if (fn?.type.is('FunctionCallOrIdentifier')) { if (fn?.type.id === terms.FunctionCallOrIdentifier) {
instructions.push(['LOAD', input.slice(fn!.from, fn!.to)]) instructions.push(['LOAD', input.slice(fn!.from, fn!.to)])
instructions.push(['MAKE_FUNCTION', [], fnLabel]) instructions.push(['MAKE_FUNCTION', [], fnLabel])
instructions.push(['PUSH', 1]) instructions.push(['PUSH', 1])
instructions.push(['PUSH', 0]) instructions.push(['PUSH', 0])
instructions.push(['CALL']) instructions.push(['CALL'])
} else if (fn?.type.is('FunctionCall')) { } else if (fn?.type.id === terms.FunctionCall) {
let body = this.#compileNode(fn!, input) let body = this.#compileNode(fn!, input)
const namedArgCount = (body[body.length - 2]![1] as number) * 2 const namedArgCount = (body[body.length - 2]![1] as number) * 2
const startSlice = body.length - namedArgCount - 3 const startSlice = body.length - namedArgCount - 3
@ -566,7 +567,7 @@ export class Compiler {
return instructions return instructions
} }
case 'TryExpr': { case terms.TryExpr: {
const { tryBlock, catchVariable, catchBody, finallyBody } = getTryExprParts(node, input) const { tryBlock, catchVariable, catchBody, finallyBody } = getTryExprParts(node, input)
return this.#compileTryCatchFinally( return this.#compileTryCatchFinally(
@ -578,14 +579,12 @@ export class Compiler {
) )
} }
case 'Throw': case terms.Throw: {
case 'Not': {
const keyword = node.type.is('Throw') ? 'Throw' : 'Not'
const children = getAllChildren(node) const children = getAllChildren(node)
const [_throwKeyword, expression] = children const [_throwKeyword, expression] = children
if (!expression) { if (!expression) {
throw new CompilerError( throw new CompilerError(
`${keyword} expected expression, got ${children.length} children`, `Throw expected expression, got ${children.length} children`,
node.from, node.from,
node.to node.to
) )
@ -593,12 +592,12 @@ export class Compiler {
const instructions: ProgramItem[] = [] const instructions: ProgramItem[] = []
instructions.push(...this.#compileNode(expression, input)) instructions.push(...this.#compileNode(expression, input))
instructions.push([keyword.toUpperCase()]) // THROW or NOT instructions.push(['THROW'])
return instructions return instructions
} }
case 'IfExpr': { case terms.IfExpr: {
const { conditionNode, thenBlock, elseIfBlocks, elseThenBlock } = getIfExprParts( const { conditionNode, thenBlock, elseIfBlocks, elseThenBlock } = getIfExprParts(
node, node,
input input
@ -641,7 +640,7 @@ export class Compiler {
} }
// - `EQ`, `NEQ`, `LT`, `GT`, `LTE`, `GTE` - Pop 2, push boolean // - `EQ`, `NEQ`, `LT`, `GT`, `LTE`, `GTE` - Pop 2, push boolean
case 'ConditionalOp': { case terms.ConditionalOp: {
const instructions: ProgramItem[] = [] const instructions: ProgramItem[] = []
const { left, op, right } = getBinaryParts(node) const { left, op, right } = getBinaryParts(node)
const leftInstructions: ProgramItem[] = this.#compileNode(left, input) const leftInstructions: ProgramItem[] = this.#compileNode(left, input)
@ -716,7 +715,7 @@ export class Compiler {
return instructions return instructions
} }
case 'PipeExpr': { case terms.PipeExpr: {
const { pipedFunctionCall, pipeReceivers } = getPipeExprParts(node) const { pipedFunctionCall, pipeReceivers } = getPipeExprParts(node)
if (!pipedFunctionCall || pipeReceivers.length === 0) { if (!pipedFunctionCall || pipeReceivers.length === 0) {
throw new CompilerError('PipeExpr must have at least two operands', node.from, node.to) throw new CompilerError('PipeExpr must have at least two operands', node.from, node.to)
@ -738,11 +737,11 @@ export class Compiler {
instructions.push(...this.#compileNode(identifierNode, input)) instructions.push(...this.#compileNode(identifierNode, input))
const isUnderscoreInPositionalArgs = positionalArgs.some( const isUnderscoreInPositionalArgs = positionalArgs.some(
(arg) => arg.type.is('Underscore') (arg) => arg.type.id === terms.Underscore
) )
const isUnderscoreInNamedArgs = namedArgs.some((arg) => { const isUnderscoreInNamedArgs = namedArgs.some((arg) => {
const { valueNode } = getNamedArgParts(arg, input) const { valueNode } = getNamedArgParts(arg, input)
return valueNode.type.is('Underscore') return valueNode.type.id === terms.Underscore
}) })
const shouldPushPositionalArg = !isUnderscoreInPositionalArgs && !isUnderscoreInNamedArgs const shouldPushPositionalArg = !isUnderscoreInPositionalArgs && !isUnderscoreInNamedArgs
@ -753,7 +752,7 @@ export class Compiler {
} }
positionalArgs.forEach((arg) => { positionalArgs.forEach((arg) => {
if (arg.type.is('Underscore')) { if (arg.type.id === terms.Underscore) {
instructions.push(['LOAD', pipeValName]) instructions.push(['LOAD', pipeValName])
} else { } else {
instructions.push(...this.#compileNode(arg, input)) instructions.push(...this.#compileNode(arg, input))
@ -763,7 +762,7 @@ export class Compiler {
namedArgs.forEach((arg) => { namedArgs.forEach((arg) => {
const { name, valueNode } = getNamedArgParts(arg, input) const { name, valueNode } = getNamedArgParts(arg, input)
instructions.push(['PUSH', name]) instructions.push(['PUSH', name])
if (valueNode.type.is('Underscore')) { if (valueNode.type.id === terms.Underscore) {
instructions.push(['LOAD', pipeValName]) instructions.push(['LOAD', pipeValName])
} else { } else {
instructions.push(...this.#compileNode(valueNode, input)) instructions.push(...this.#compileNode(valueNode, input))
@ -778,14 +777,14 @@ export class Compiler {
return instructions return instructions
} }
case 'Array': { case terms.Array: {
const children = getAllChildren(node) const children = getAllChildren(node)
// We can easily parse [=] as an empty dict, but `[ = ]` is tougher. // We can easily parse [=] as an empty dict, but `[ = ]` is tougher.
// = can be a valid word, and is also valid inside words, so for now we cheat // = can be a valid word, and is also valid inside words, so for now we cheat
// and check for arrays that look like `[ = ]` to interpret them as // and check for arrays that look like `[ = ]` to interpret them as
// empty dicts // empty dicts
if (children.length === 1 && children[0]!.type.is('Word')) { if (children.length === 1 && children[0]!.type.id === terms.Word) {
const child = children[0]! const child = children[0]!
if (input.slice(child.from, child.to) === '=') { if (input.slice(child.from, child.to) === '=') {
return [['MAKE_DICT', 0]] return [['MAKE_DICT', 0]]
@ -797,7 +796,7 @@ export class Compiler {
return instructions return instructions
} }
case 'Dict': { case terms.Dict: {
const children = getAllChildren(node) const children = getAllChildren(node)
const instructions: ProgramItem[] = [] const instructions: ProgramItem[] = []
@ -806,7 +805,7 @@ export class Compiler {
const valueNode = node.firstChild!.nextSibling const valueNode = node.firstChild!.nextSibling
// name= -> name // name= -> name
const key = input.slice(keyNode!.from, keyNode!.to).replace(/\s*=$/, '') const key = input.slice(keyNode!.from, keyNode!.to).slice(0, -1)
instructions.push(['PUSH', key]) instructions.push(['PUSH', key])
instructions.push(...this.#compileNode(valueNode!, input)) instructions.push(...this.#compileNode(valueNode!, input))
@ -816,7 +815,7 @@ export class Compiler {
return instructions return instructions
} }
case 'WhileExpr': { case terms.WhileExpr: {
const [_while, test, _colon, block] = getAllChildren(node) const [_while, test, _colon, block] = getAllChildren(node)
const instructions: ProgramItem[] = [] const instructions: ProgramItem[] = []
@ -834,11 +833,11 @@ export class Compiler {
return instructions return instructions
} }
case 'Import': { case terms.Import: {
const instructions: ProgramItem[] = [] const instructions: ProgramItem[] = []
const [_import, ...nodes] = getAllChildren(node) const [_import, ...nodes] = getAllChildren(node)
const args = nodes.filter(node => node.type.is('Identifier')) const args = nodes.filter(node => node.type.id === terms.Identifier)
const namedArgs = nodes.filter(node => node.type.is('NamedArg')) const namedArgs = nodes.filter(node => node.type.id === terms.NamedArg)
instructions.push(['LOAD', 'import']) instructions.push(['LOAD', 'import'])
@ -859,13 +858,13 @@ export class Compiler {
return instructions return instructions
} }
case 'Comment': { case terms.Comment: {
return [] // ignore comments return [] // ignore comments
} }
default: default:
throw new CompilerError( throw new CompilerError(
`Compiler doesn't know how to handle a "${node.type.name}" node.`, `Compiler doesn't know how to handle a "${node.type.name}" (${node.type.id}) node.`,
node.from, node.from,
node.to node.to
) )

View File

@ -151,22 +151,18 @@ describe('array literals', () => {
describe('dict literals', () => { describe('dict literals', () => {
test('work with numbers', () => { test('work with numbers', () => {
expect('[a=1 b=2 c=3]').toEvaluateTo({ a: 1, b: 2, c: 3 }) expect('[a=1 b=2 c=3]').toEvaluateTo({ a: 1, b: 2, c: 3 })
expect('[a = 1 b = 2 c = 3]').toEvaluateTo({ a: 1, b: 2, c: 3 })
}) })
test('work with strings', () => { test('work with strings', () => {
expect("[a='one' b='two' c='three']").toEvaluateTo({ a: 'one', b: 'two', c: 'three' }) expect("[a='one' b='two' c='three']").toEvaluateTo({ a: 'one', b: 'two', c: 'three' })
expect("[a = 'one' b = 'two' c = 'three']").toEvaluateTo({ a: 'one', b: 'two', c: 'three' })
}) })
test('work with identifiers', () => { test('work with identifiers', () => {
expect('[a=one b=two c=three]').toEvaluateTo({ a: 'one', b: 'two', c: 'three' }) expect('[a=one b=two c=three]').toEvaluateTo({ a: 'one', b: 'two', c: 'three' })
expect('[a = one b = two c = three]').toEvaluateTo({ a: 'one', b: 'two', c: 'three' })
}) })
test('can be nested', () => { test('can be nested', () => {
expect('[a=one b=[two [c=three]]]').toEvaluateTo({ a: 'one', b: ['two', { c: 'three' }] }) expect('[a=one b=[two [c=three]]]').toEvaluateTo({ a: 'one', b: ['two', { c: 'three' }] })
expect('[a = one b = [two [c = three]]]').toEvaluateTo({ a: 'one', b: ['two', { c: 'three' }] })
}) })
test('can span multiple lines', () => { test('can span multiple lines', () => {
@ -175,12 +171,6 @@ describe('dict literals', () => {
b=2 b=2
c=3 c=3
]`).toEvaluateTo({ a: 1, b: 2, c: 3 }) ]`).toEvaluateTo({ a: 1, b: 2, c: 3 })
expect(`[
a = 1
b = 2
c = 3
]`).toEvaluateTo({ a: 1, b: 2, c: 3 })
}) })
test('empty dict', () => { test('empty dict', () => {
@ -200,12 +190,10 @@ describe('dict literals', () => {
test('semicolons as separators', () => { test('semicolons as separators', () => {
expect('[a=1; b=2; c=3]').toEvaluateTo({ a: 1, b: 2, c: 3 }) expect('[a=1; b=2; c=3]').toEvaluateTo({ a: 1, b: 2, c: 3 })
expect('[a = 1; b = 2; c = 3]').toEvaluateTo({ a: 1, b: 2, c: 3 })
}) })
test('expressions in dicts', () => { test('expressions in dicts', () => {
expect('[a=(1 + 2) b=(3 * 4)]').toEvaluateTo({ a: 3, b: 12 }) expect('[a=(1 + 2) b=(3 * 4)]').toEvaluateTo({ a: 3, b: 12 })
expect('[a = (1 + 2) b = (3 * 4)]').toEvaluateTo({ a: 3, b: 12 })
}) })
test('empty lines within dicts', () => { test('empty lines within dicts', () => {
@ -258,7 +246,7 @@ describe('curly strings', () => {
test('interpolation edge cases', () => { test('interpolation edge cases', () => {
expect(`{[a=1 b=2 c={wild}]}`).toEvaluateTo(`[a=1 b=2 c={wild}]`) expect(`{[a=1 b=2 c={wild}]}`).toEvaluateTo(`[a=1 b=2 c={wild}]`)
expect(`a = 1;b = 2;c = 3;{$a $b $c}`).toEvaluateTo(`1 2 3`) expect(`a = 1;b = 2;c = 3;{$a $b $c}`).toEvaluateTo(`1 2 3`)
expect(`a = 1;b = 2;c = 3;{$(a)$(b)$(c)}`).toEvaluateTo(`123`) expect(`a = 1;b = 2;c = 3;{$a$b$c}`).toEvaluateTo(`123`)
}) })
}) })

View File

@ -1,9 +1,9 @@
import { CompilerError } from '#compiler/compilerError.ts' import { CompilerError } from '#compiler/compilerError.ts'
import type { SyntaxNode, Tree } from '#parser/node' import * as terms from '#parser/shrimp.terms'
import type { SyntaxNode, Tree } from '@lezer/common'
export const checkTreeForErrors = (tree: Tree): CompilerError[] => { export const checkTreeForErrors = (tree: Tree): CompilerError[] => {
const errors: CompilerError[] = [] const errors: CompilerError[] = []
tree.iterate({ tree.iterate({
enter: (node) => { enter: (node) => {
if (node.type.isError) { if (node.type.isError) {
@ -23,7 +23,7 @@ export const getAllChildren = (node: SyntaxNode): SyntaxNode[] => {
child = child.nextSibling child = child.nextSibling
} }
return children.filter((n) => !n.type.is('Comment')) return children.filter((n) => n.type.id !== terms.Comment)
} }
export const getBinaryParts = (node: SyntaxNode) => { export const getBinaryParts = (node: SyntaxNode) => {
@ -50,14 +50,15 @@ export const getAssignmentParts = (node: SyntaxNode) => {
} }
// array destructuring // array destructuring
if (left && left.type.is('Array')) { if (left && left.type.id === terms.Array) {
const identifiers = getAllChildren(left).filter((child) => child.type.is('Identifier')) const identifiers = getAllChildren(left).filter((child) => child.type.id === terms.Identifier)
return { arrayPattern: identifiers, right } return { arrayPattern: identifiers, right }
} }
if (!left || !left.type.is('AssignableIdentifier')) { if (!left || left.type.id !== terms.AssignableIdentifier) {
throw new CompilerError( throw new CompilerError(
`Assign left child must be an AssignableIdentifier or Array, got ${left ? left.type.name : 'none' `Assign left child must be an AssignableIdentifier or Array, got ${
left ? left.type.name : 'none'
}`, }`,
node.from, node.from,
node.to node.to
@ -71,9 +72,10 @@ export const getCompoundAssignmentParts = (node: SyntaxNode) => {
const children = getAllChildren(node) const children = getAllChildren(node)
const [left, operator, right] = children const [left, operator, right] = children
if (!left || !left.type.is('AssignableIdentifier')) { if (!left || left.type.id !== terms.AssignableIdentifier) {
throw new CompilerError( throw new CompilerError(
`CompoundAssign left child must be an AssignableIdentifier, got ${left ? left.type.name : 'none' `CompoundAssign left child must be an AssignableIdentifier, got ${
left ? left.type.name : 'none'
}`, }`,
node.from, node.from,
node.to node.to
@ -102,7 +104,7 @@ export const getFunctionDefParts = (node: SyntaxNode, input: string) => {
} }
const paramNames = getAllChildren(paramsNode).map((param) => { const paramNames = getAllChildren(paramsNode).map((param) => {
if (!param.type.is('Identifier') && !param.type.is('NamedParam')) { if (param.type.id !== terms.Identifier && param.type.id !== terms.NamedParam) {
throw new CompilerError( throw new CompilerError(
`FunctionDef params must be Identifier or NamedParam, got ${param.type.name}`, `FunctionDef params must be Identifier or NamedParam, got ${param.type.name}`,
param.from, param.from,
@ -121,7 +123,7 @@ export const getFunctionDefParts = (node: SyntaxNode, input: string) => {
let finallyBody: SyntaxNode | undefined let finallyBody: SyntaxNode | undefined
for (const child of rest) { for (const child of rest) {
if (child.type.is('CatchExpr')) { if (child.type.id === terms.CatchExpr) {
catchExpr = child catchExpr = child
const catchChildren = getAllChildren(child) const catchChildren = getAllChildren(child)
const [_catchKeyword, identifierNode, _colon, body] = catchChildren const [_catchKeyword, identifierNode, _colon, body] = catchChildren
@ -134,7 +136,7 @@ export const getFunctionDefParts = (node: SyntaxNode, input: string) => {
} }
catchVariable = input.slice(identifierNode.from, identifierNode.to) catchVariable = input.slice(identifierNode.from, identifierNode.to)
catchBody = body catchBody = body
} else if (child.type.is('FinallyExpr')) { } else if (child.type.id === terms.FinallyExpr) {
finallyExpr = child finallyExpr = child
const finallyChildren = getAllChildren(child) const finallyChildren = getAllChildren(child)
const [_finallyKeyword, _colon, body] = finallyChildren const [_finallyKeyword, _colon, body] = finallyChildren
@ -163,9 +165,9 @@ export const getFunctionCallParts = (node: SyntaxNode, input: string) => {
throw new CompilerError(`FunctionCall expected at least 1 child, got 0`, node.from, node.to) throw new CompilerError(`FunctionCall expected at least 1 child, got 0`, node.from, node.to)
} }
const namedArgs = args.filter((arg) => arg.type.is('NamedArg')) const namedArgs = args.filter((arg) => arg.type.id === terms.NamedArg)
const positionalArgs = args const positionalArgs = args
.filter((arg) => arg.type.is('PositionalArg')) .filter((arg) => arg.type.id === terms.PositionalArg)
.map((arg) => { .map((arg) => {
const child = arg.firstChild const child = arg.firstChild
if (!child) throw new CompilerError(`PositionalArg has no child`, arg.from, arg.to) if (!child) throw new CompilerError(`PositionalArg has no child`, arg.from, arg.to)
@ -206,13 +208,13 @@ export const getIfExprParts = (node: SyntaxNode, input: string) => {
rest.forEach((child) => { rest.forEach((child) => {
const parts = getAllChildren(child) const parts = getAllChildren(child)
if (child.type.is('ElseExpr')) { if (child.type.id === terms.ElseExpr) {
if (parts.length !== 3) { if (parts.length !== 3) {
const message = `ElseExpr expected 1 child, got ${parts.length}` const message = `ElseExpr expected 1 child, got ${parts.length}`
throw new CompilerError(message, child.from, child.to) throw new CompilerError(message, child.from, child.to)
} }
elseThenBlock = parts.at(-1) elseThenBlock = parts.at(-1)
} else if (child.type.is('ElseIfExpr')) { } else if (child.type.id === terms.ElseIfExpr) {
const [_else, _if, conditional, _colon, thenBlock] = parts const [_else, _if, conditional, _colon, thenBlock] = parts
if (!conditional || !thenBlock) { if (!conditional || !thenBlock) {
const names = parts.map((p) => p.type.name).join(', ') const names = parts.map((p) => p.type.name).join(', ')
@ -247,10 +249,10 @@ export const getStringParts = (node: SyntaxNode, input: string) => {
// The text is just between the quotes // The text is just between the quotes
const parts = children.filter((child) => { const parts = children.filter((child) => {
return ( return (
child.type.is('StringFragment') || child.type.id === terms.StringFragment ||
child.type.is('Interpolation') || child.type.id === terms.Interpolation ||
child.type.is('EscapeSeq') || child.type.id === terms.EscapeSeq ||
child.type.is('CurlyString') child.type.id === terms.CurlyString
) )
}) })
@ -258,10 +260,10 @@ export const getStringParts = (node: SyntaxNode, input: string) => {
// Validate each part is the expected type // Validate each part is the expected type
parts.forEach((part) => { parts.forEach((part) => {
if ( if (
part.type.is('StringFragment') && part.type.id !== terms.StringFragment &&
part.type.is('Interpolation') && part.type.id !== terms.Interpolation &&
part.type.is('EscapeSeq') && part.type.id !== terms.EscapeSeq &&
part.type.is('CurlyString') part.type.id !== terms.CurlyString
) { ) {
throw new CompilerError( throw new CompilerError(
`String child must be StringFragment, Interpolation, or EscapeSeq, got ${part.type.name}`, `String child must be StringFragment, Interpolation, or EscapeSeq, got ${part.type.name}`,
@ -274,7 +276,7 @@ export const getStringParts = (node: SyntaxNode, input: string) => {
// hasInterpolation means the string has interpolation ($var) or escape sequences (\n) // hasInterpolation means the string has interpolation ($var) or escape sequences (\n)
// A simple string like 'hello' has one StringFragment but no interpolation // A simple string like 'hello' has one StringFragment but no interpolation
const hasInterpolation = parts.some( const hasInterpolation = parts.some(
(p) => p.type.is('Interpolation') || p.type.is('EscapeSeq') (p) => p.type.id === terms.Interpolation || p.type.id === terms.EscapeSeq
) )
return { parts, hasInterpolation } return { parts, hasInterpolation }
} }
@ -291,7 +293,7 @@ export const getDotGetParts = (node: SyntaxNode, input: string) => {
) )
} }
if (!object.type.is('IdentifierBeforeDot')) { if (object.type.id !== terms.IdentifierBeforeDot && object.type.id !== terms.Dollar) {
throw new CompilerError( throw new CompilerError(
`DotGet object must be an IdentifierBeforeDot, got ${object.type.name}`, `DotGet object must be an IdentifierBeforeDot, got ${object.type.name}`,
object.from, object.from,
@ -299,7 +301,7 @@ export const getDotGetParts = (node: SyntaxNode, input: string) => {
) )
} }
if (!['Identifier', 'Number', 'ParenExpr', 'DotGet'].includes(property.type.name)) { if (![terms.Identifier, terms.Number, terms.ParenExpr, terms.DotGet].includes(property.type.id)) {
throw new CompilerError( throw new CompilerError(
`DotGet property must be an Identifier, Number, ParenExpr, or DotGet, got ${property.type.name}`, `DotGet property must be an Identifier, Number, ParenExpr, or DotGet, got ${property.type.name}`,
property.from, property.from,
@ -333,7 +335,7 @@ export const getTryExprParts = (node: SyntaxNode, input: string) => {
let finallyBody: SyntaxNode | undefined let finallyBody: SyntaxNode | undefined
rest.forEach((child) => { rest.forEach((child) => {
if (child.type.is('CatchExpr')) { if (child.type.id === terms.CatchExpr) {
catchExpr = child catchExpr = child
const catchChildren = getAllChildren(child) const catchChildren = getAllChildren(child)
const [_catchKeyword, identifierNode, _colon, body] = catchChildren const [_catchKeyword, identifierNode, _colon, body] = catchChildren
@ -346,7 +348,7 @@ export const getTryExprParts = (node: SyntaxNode, input: string) => {
} }
catchVariable = input.slice(identifierNode.from, identifierNode.to) catchVariable = input.slice(identifierNode.from, identifierNode.to)
catchBody = body catchBody = body
} else if (child.type.is('FinallyExpr')) { } else if (child.type.id === terms.FinallyExpr) {
finallyExpr = child finallyExpr = child
const finallyChildren = getAllChildren(child) const finallyChildren = getAllChildren(child)
const [_finallyKeyword, _colon, body] = finallyChildren const [_finallyKeyword, _colon, body] = finallyChildren

View File

@ -245,7 +245,7 @@ const commandShapes: CommandShape[] = [
] as const ] as const
let commandSource = () => commandShapes let commandSource = () => commandShapes
export const setCommandSource = (fn: () => CommandShape[]) => { export const setCommandSource = (do: () => CommandShape[]) => {
commandSource = fn commandSource = fn
} }

View File

@ -1,15 +1,15 @@
import { readFileSync } from 'fs' import { readFileSync } from 'fs'
import { VM, fromValue, toValue, isValue, type Bytecode } from 'reefvm' import { VM, fromValue, toValue, isValue, type Bytecode } from 'reefvm'
import { type Tree } from '@lezer/common'
import { Compiler } from '#compiler/compiler' import { Compiler } from '#compiler/compiler'
import { parse } from '#parser/parser2' import { parser } from '#parser/shrimp'
import { Tree } from '#parser/node' import { globals as parserGlobals, setGlobals as setParserGlobals } from '#parser/tokenizer'
import { globals as parserGlobals, setGlobals as setParserGlobals } from '#parser/parser2'
import { globals as prelude } from '#prelude' import { globals as prelude } from '#prelude'
export { Compiler } from '#compiler/compiler' export { Compiler } from '#compiler/compiler'
export { parse } from '#parser/parser2' export { parser } from '#parser/shrimp'
export { type SyntaxNode, Tree } from '#parser/node'
export { globals as prelude } from '#prelude' export { globals as prelude } from '#prelude'
export type { Tree } from '@lezer/common'
export { type Value, type Bytecode } from 'reefvm' export { type Value, type Bytecode } from 'reefvm'
export { toValue, fromValue, isValue, Scope, VM, bytecodeToString } from 'reefvm' export { toValue, fromValue, isValue, Scope, VM, bytecodeToString } from 'reefvm'
@ -105,8 +105,8 @@ export function parseCode(code: string, globals?: Record<string, any>): Tree {
const globalNames = [...Object.keys(prelude), ...(globals ? Object.keys(globals) : [])] const globalNames = [...Object.keys(prelude), ...(globals ? Object.keys(globals) : [])]
setParserGlobals(globalNames) setParserGlobals(globalNames)
const result = parse(code) const result = parser.parse(code)
setParserGlobals(oldGlobals) setParserGlobals(oldGlobals)
return new Tree(result) return result
} }

View File

@ -1,6 +1,6 @@
import { parse } from '#parser/parser2' import { parser } from '#parser/shrimp.ts'
import type { SyntaxNode } from '#parser/node' import type { SyntaxNode } from '@lezer/common'
import { isIdentStart, isIdentChar } from './tokenizer2' import { isIdentStart, isIdentChar } from './tokenizer'
// Turns a { curly string } into strings and nodes for interpolation // Turns a { curly string } into strings and nodes for interpolation
export const tokenizeCurlyString = (value: string): (string | [string, SyntaxNode])[] => { export const tokenizeCurlyString = (value: string): (string | [string, SyntaxNode])[] => {
@ -37,8 +37,8 @@ export const tokenizeCurlyString = (value: string): (string | [string, SyntaxNod
} }
const input = value.slice(start + 2, pos) // skip '$(' const input = value.slice(start + 2, pos) // skip '$('
tokens.push([input, parse(input)]) tokens.push([input, parser.parse(input).topNode])
start = pos + 1 // start after ')' start = ++pos // skip ')'
} else { } else {
char = value[++pos] char = value[++pos]
if (!char) break if (!char) break
@ -48,7 +48,7 @@ export const tokenizeCurlyString = (value: string): (string | [string, SyntaxNod
char = value[++pos] char = value[++pos]
const input = value.slice(start + 1, pos) // skip '$' const input = value.slice(start + 1, pos) // skip '$'
tokens.push([input, parse(input)]) tokens.push([input, parser.parse(input).topNode])
start = pos-- // backtrack and start over start = pos-- // backtrack and start over
} }
} }

View File

@ -1,270 +0,0 @@
import { type Token, TokenType } from './tokenizer2'
export type NodeType =
| 'Program'
| 'Block'
| 'FunctionCall'
| 'FunctionCallOrIdentifier'
| 'FunctionCallWithBlock'
| 'PositionalArg'
| 'NamedArg'
| 'NamedArgPrefix'
| 'FunctionDef'
| 'Params'
| 'NamedParam'
| 'Null'
| 'Boolean'
| 'Number'
| 'String'
| 'StringFragment'
| 'CurlyString'
| 'DoubleQuote'
| 'EscapeSeq'
| 'Interpolation'
| 'Regex'
| 'Identifier'
| 'AssignableIdentifier'
| 'IdentifierBeforeDot'
| 'Word'
| 'Array'
| 'Dict'
| 'Comment'
| 'BinOp'
| 'ConditionalOp'
| 'ParenExpr'
| 'Assign'
| 'CompoundAssign'
| 'DotGet'
| 'PipeExpr'
| 'IfExpr'
| 'ElseIfExpr'
| 'ElseExpr'
| 'WhileExpr'
| 'TryExpr'
| 'CatchExpr'
| 'FinallyExpr'
| 'Throw'
| 'Not'
| 'Eq'
| 'Modulo'
| 'Plus'
| 'Star'
| 'Slash'
| 'Import'
| 'Do'
| 'Underscore'
| 'colon'
| 'keyword'
| 'operator'
// TODO: remove this when we switch from lezer
export const operators: Record<string, any> = {
// Logic
'and': 'And',
'or': 'Or',
// Bitwise
'band': 'Band',
'bor': 'Bor',
'bxor': 'Bxor',
'>>>': 'Ushr',
'>>': 'Shr',
'<<': 'Shl',
// Comparison
'>=': 'Gte',
'<=': 'Lte',
'>': 'Gt',
'<': 'Lt',
'!=': 'Neq',
'==': 'EqEq',
// Compound assignment operators
'??=': 'NullishEq',
'+=': 'PlusEq',
'-=': 'MinusEq',
'*=': 'StarEq',
'/=': 'SlashEq',
'%=': 'ModuloEq',
// Nullish coalescing
'??': 'NullishCoalesce',
// Math
'*': 'Star',
'**': 'StarStar',
'=': 'Eq',
'/': 'Slash',
'+': 'Plus',
'-': 'Minus',
'%': 'Modulo',
// Dotget
'.': 'Dot',
// Pipe
'|': 'operator',
}
export class Tree {
constructor(public topNode: SyntaxNode) { }
get length(): number {
return this.topNode.to
}
cursor() {
return {
type: this.topNode.type,
from: this.topNode.from,
to: this.topNode.to,
node: this.topNode,
}
}
iterate(options: { enter: (node: SyntaxNode) => void }) {
const iter = (node: SyntaxNode) => {
for (const n of node.children) iter(n)
options.enter(node)
}
iter(this.topNode)
}
}
export class SyntaxNode {
#type: NodeType
#isError = false
from: number
to: number
parent: SyntaxNode | null
children: SyntaxNode[] = []
constructor(type: NodeType, from: number, to: number, parent: SyntaxNode | null = null) {
this.#type = type
this.from = from
this.to = to
this.parent = parent
}
static from(token: Token, parent?: SyntaxNode): SyntaxNode {
return new SyntaxNode(TokenType[token.type] as NodeType, token.from, token.to, parent ?? null)
}
get type(): { type: NodeType, name: NodeType, isError: boolean, is: (other: NodeType) => boolean } {
return {
type: this.#type,
name: this.#type,
isError: this.#isError,
is: (other: NodeType) => other === this.#type
}
}
set type(name: NodeType) {
this.#type = name
}
get name(): string {
return this.type.name
}
get isError(): boolean {
return this.#isError
}
set isError(err: boolean) {
this.#isError = err
}
get firstChild(): SyntaxNode | null {
return this.children[0] ?? null
}
get lastChild(): SyntaxNode | null {
return this.children.at(-1) ?? null
}
get nextSibling(): SyntaxNode | null {
if (!this.parent) return null
const siblings = this.parent.children
const index = siblings.indexOf(this)
return index >= 0 && index < siblings.length - 1 ? siblings[index + 1]! : null
}
get prevSibling(): SyntaxNode | null {
if (!this.parent) return null
const siblings = this.parent.children
const index = siblings.indexOf(this)
return index > 0 ? siblings[index - 1]! : null
}
add(node: SyntaxNode) {
node.parent = this
this.children.push(node)
}
push(...nodes: SyntaxNode[]): SyntaxNode {
nodes.forEach(child => child.parent = this)
this.children.push(...nodes)
return this
}
toString(): string {
return this.type.name
}
}
// Operator precedence (binding power) - higher = tighter binding
export const precedence: Record<string, number> = {
// Logical
'or': 10,
'and': 20,
// Comparison
'==': 30,
'!=': 30,
'<': 30,
'>': 30,
'<=': 30,
'>=': 30,
// Nullish coalescing
'??': 35,
// Bitwise shifts (lower precedence than addition)
'<<': 37,
'>>': 37,
'>>>': 37,
// Addition/Subtraction
'+': 40,
'-': 40,
// Bitwise AND/OR/XOR (higher precedence than addition)
'band': 45,
'bor': 45,
'bxor': 45,
// Multiplication/Division/Modulo
'*': 50,
'/': 50,
'%': 50,
// Exponentiation (right-associative)
'**': 60,
}
export const conditionals = new Set([
'==', '!=', '<', '>', '<=', '>=', '??', 'and', 'or'
])
export const compounds = [
'??=', '+=', '-=', '*=', '/=', '%='
]

View File

@ -0,0 +1,99 @@
import { ExternalTokenizer, InputStream } from '@lezer/lr'
import * as terms from './shrimp.terms'
type Operator = { str: string; tokenName: keyof typeof terms }
const operators: Array<Operator> = [
{ str: 'and', tokenName: 'And' },
{ str: 'or', tokenName: 'Or' },
{ str: 'band', tokenName: 'Band' },
{ str: 'bor', tokenName: 'Bor' },
{ str: 'bxor', tokenName: 'Bxor' },
{ str: '>>>', tokenName: 'Ushr' }, // Must come before >>
{ str: '>>', tokenName: 'Shr' },
{ str: '<<', tokenName: 'Shl' },
{ str: '>=', tokenName: 'Gte' },
{ str: '<=', tokenName: 'Lte' },
{ str: '!=', tokenName: 'Neq' },
{ str: '==', tokenName: 'EqEq' },
// Compound assignment operators (must come before single-char operators)
{ str: '??=', tokenName: 'NullishEq' },
{ str: '+=', tokenName: 'PlusEq' },
{ str: '-=', tokenName: 'MinusEq' },
{ str: '*=', tokenName: 'StarEq' },
{ str: '/=', tokenName: 'SlashEq' },
{ str: '%=', tokenName: 'ModuloEq' },
// Nullish coalescing (must come before it could be mistaken for other tokens)
{ str: '??', tokenName: 'NullishCoalesce' },
// Single-char operators
{ str: '*', tokenName: 'Star' },
{ str: '=', tokenName: 'Eq' },
{ str: '/', tokenName: 'Slash' },
{ str: '+', tokenName: 'Plus' },
{ str: '-', tokenName: 'Minus' },
{ str: '>', tokenName: 'Gt' },
{ str: '<', tokenName: 'Lt' },
{ str: '%', tokenName: 'Modulo' },
]
export const operatorTokenizer = new ExternalTokenizer((input: InputStream) => {
for (let operator of operators) {
if (!matchesString(input, 0, operator.str)) continue
const afterOpPos = operator.str.length
const charAfterOp = input.peek(afterOpPos)
if (!isWhitespace(charAfterOp)) continue
// Accept the operator token
const token = terms[operator.tokenName]
if (token === undefined) {
throw new Error(`Unknown token name: ${operator.tokenName}`)
}
input.advance(afterOpPos)
input.acceptToken(token)
return
}
})
const isWhitespace = (ch: number): boolean => {
return matchesChar(ch, [' ', '\t', '\n'])
}
const matchesChar = (ch: number, chars: (string | number)[]): boolean => {
for (const c of chars) {
if (typeof c === 'number') {
if (ch === c) {
return true
}
} else if (ch === c.charCodeAt(0)) {
return true
}
}
return false
}
const matchesString = (input: InputStream, pos: number, str: string): boolean => {
for (let i = 0; i < str.length; i++) {
if (input.peek(pos + i) !== str.charCodeAt(i)) {
return false
}
}
return true
}
const peek = (numChars: number, input: InputStream): string => {
let result = ''
for (let i = 0; i < numChars; i++) {
const ch = input.peek(i)
if (ch === -1) {
result += 'EOF'
break
} else {
result += String.fromCharCode(ch)
}
}
return result
}

View File

@ -1,991 +0,0 @@
import { CompilerError } from '#compiler/compilerError'
import { Scanner, type Token, TokenType } from './tokenizer2'
import { SyntaxNode, operators, precedence, conditionals, compounds } from './node'
import { parseString } from './stringParser'
const $T = TokenType
// tell the dotGet searcher about builtin globals
export const globals: string[] = []
export const setGlobals = (newGlobals: string[] | Record<string, any>) => {
globals.length = 0
globals.push(...(Array.isArray(newGlobals) ? newGlobals : Object.keys(newGlobals)))
}
export const parse = (input: string): SyntaxNode => {
const parser = new Parser()
return parser.parse(input)
}
class Scope {
parent?: Scope
set = new Set<string>()
constructor(parent?: Scope) {
this.parent = parent
// no parent means this is global scope
if (!parent) for (const name of globals) this.add(name)
}
add(key: string) {
this.set.add(key)
}
has(key: string): boolean {
return this.set.has(key) || this.parent?.has(key) || false
}
}
export class Parser {
tokens: Token[] = []
pos = 0
inParens = 0
input = ''
scope = new Scope
inTestExpr = false
parse(input: string): SyntaxNode {
const scanner = new Scanner()
this.tokens = scanner.tokenize(input)
this.pos = 0
this.input = input
this.scope = new Scope()
this.inTestExpr = false
const node = new SyntaxNode('Program', 0, input.length)
while (!this.isEOF()) {
if (this.is($T.Newline) || this.is($T.Semicolon)) {
this.next()
continue
}
const prevPos = this.pos
const stmt = this.statement()
if (stmt) node.add(stmt)
if (this.pos === prevPos && !this.isEOF())
throw `parser didn't advance - you need to call next()\n\n ${this.input}\n`
}
return node
}
//
// parse foundation nodes - statements, expressions
//
// statement is a line of code
statement(): SyntaxNode | null {
if (this.is($T.Comment))
return this.comment()
while (this.is($T.Newline) || this.is($T.Semicolon))
this.next()
if (this.isEOF() || this.isExprEndKeyword())
return null
return this.expression()
}
// expressions can be found in four places:
// 1. line of code
// 2. right side of assignment
// 3. if/while conditions
// 4. inside (parens)
expression(allowPipe = true): SyntaxNode {
let expr
// x = value
if (this.is($T.Identifier) && (
this.nextIs($T.Operator, '=') || compounds.some(x => this.nextIs($T.Operator, x))
))
expr = this.assign()
// if, while, do, etc
else if (this.is($T.Keyword))
expr = this.keywords()
// dotget
else if (this.nextIs($T.Operator, '.'))
expr = this.dotGetFunctionCall()
// echo hello world
else if (this.is($T.Identifier) && !this.nextIs($T.Operator) && !this.nextIsExprEnd())
expr = this.functionCall()
// bare-function-call
else if (this.is($T.Identifier) && this.nextIsExprEnd())
expr = this.functionCallOrIdentifier()
// everything else
else
expr = this.exprWithPrecedence()
// check for destructuring
if (expr.type.is('Array') && this.is($T.Operator, '='))
return this.destructure(expr)
// check for parens function call
// ex: (ref my-func) my-arg
if (expr.type.is('ParenExpr') && !this.isExprEnd())
expr = this.functionCall(expr)
// if dotget is followed by binary operator, continue parsing as binary expression
if (expr.type.is('DotGet') && this.is($T.Operator) && !this.is($T.Operator, '|'))
expr = this.dotGetBinOp(expr)
// one | echo
if (allowPipe && this.isPipe())
return this.pipe(expr)
// regular
else
return expr
}
// piping | stuff | is | cool
pipe(left: SyntaxNode): SyntaxNode {
const canLookPastNewlines = this.inParens === 0
const parts: SyntaxNode[] = [left]
while (this.isPipe()) {
// consume newlines before pipe (only if not in parens)
if (canLookPastNewlines) {
while (this.is($T.Newline)) this.next()
}
const pipeOp = this.op('|')
pipeOp.type = 'operator'
parts.push(pipeOp)
// consume newlines after pipe (only if not in parens)
if (canLookPastNewlines) {
while (this.is($T.Newline)) this.next()
}
// parse right side - don't allow nested pipes
parts.push(this.expression(false))
}
const node = new SyntaxNode('PipeExpr', parts[0]!.from, parts.at(-1)!.to)
return node.push(...parts)
}
// Pratt parser - parses expressions with precedence climbing
// bp = binding precedence
exprWithPrecedence(minBp = 0): SyntaxNode {
let left = this.value()
// infix operators with precedence
while (this.is($T.Operator)) {
const op = this.current().value!
const bp = precedence[op]
// operator has lower precedence than required, stop
if (bp === undefined || bp < minBp) break
const opNode = this.op()
// right-associative operators (like **) use same bp, others use bp + 1
const nextMinBp = op === '**' ? bp : bp + 1
// parse right-hand side with higher precedence
const right = this.exprWithPrecedence(nextMinBp)
const nodeType = conditionals.has(op) ? 'ConditionalOp' : 'BinOp'
const node = new SyntaxNode(nodeType, left.from, right.to)
node.push(left, opNode, right)
left = node
}
return left
}
// if, while, do, etc
keywords(): SyntaxNode {
if (this.is($T.Keyword, 'if'))
return this.if()
if (this.is($T.Keyword, 'while'))
return this.while()
if (this.is($T.Keyword, 'do'))
return this.do()
if (this.is($T.Keyword, 'try'))
return this.try()
if (this.is($T.Keyword, 'throw'))
return this.throw()
if (this.is($T.Keyword, 'not'))
return this.not()
if (this.is($T.Keyword, 'import'))
return this.import()
return this.expect($T.Keyword, 'if/while/do/import') as never
}
// value can be an atom or a (parens that gets turned into an atom)
// values are used in a few places:
// 1. function arguments
// 2. array/dict members
// 3. binary operations
// 4. anywhere an expression can be used
value(): SyntaxNode {
if (this.is($T.OpenParen))
return this.parens()
if (this.is($T.OpenBracket))
return this.arrayOrDict()
// dotget
if (this.nextIs($T.Operator, '.'))
return this.dotGet()
return this.atom()
}
//
// parse specific nodes
//
// raw determines whether we just want the SyntaxNodes or we want to
// wrap them in a PositionalArg
arg(raw = false): SyntaxNode {
// 'do' is a special function arg - it doesn't need to be wrapped
// in parens. otherwise, args are regular value()s
const val = this.is($T.Keyword, 'do') ? this.do() : this.value()
if (raw) {
return val
} else {
const arg = new SyntaxNode('PositionalArg', val.from, val.to)
if (val.isError) arg.isError = true
arg.add(val)
return arg
}
}
// [ 1 2 3 ]
array(): SyntaxNode {
const open = this.expect($T.OpenBracket)
const values = []
while (!this.is($T.CloseBracket) && !this.isEOF()) {
if (this.is($T.Semicolon) || this.is($T.Newline)) {
this.next()
continue
}
if (this.is($T.Comment)) {
values.push(this.comment())
continue
}
values.push(this.value())
}
const close = this.expect($T.CloseBracket)
const node = new SyntaxNode('Array', open.from, close.to)
return node.push(...values)
}
// which are we dealing with? ignores leading newlines and comments
arrayOrDict(): SyntaxNode {
let peek = 1
let curr = this.peek(peek++)
let isDict = false
while (curr && curr.type !== $T.CloseBracket) {
// definitely a dict
if (curr.type === $T.NamedArgPrefix) {
isDict = true
break
}
// empty dict
if (curr.type === $T.Operator && curr.value === '=') {
isDict = true
break
}
// [ a = true ]
const next = this.peek(peek)
if (next?.type === $T.Operator && next.value === '=') {
isDict = true
break
}
// probably an array
if (curr.type !== $T.Comment && curr.type !== $T.Semicolon && curr.type !== $T.Newline)
break
curr = this.peek(peek++)
}
return isDict ? this.dict() : this.array()
}
// x = true
assign(): SyntaxNode {
const ident = this.assignableIdentifier()
const opToken = this.current()!
const op = this.op()
const expr = this.expression()
const node = new SyntaxNode(
opToken.value === '=' ? 'Assign' : 'CompoundAssign',
ident.from,
expr.to
)
return node.push(ident, op, expr)
}
// identifier used in assignment (TODO: legacy lezer quirk)
assignableIdentifier(): SyntaxNode {
const token = this.expect($T.Identifier)
this.scope.add(token.value!)
const node = SyntaxNode.from(token)
node.type = 'AssignableIdentifier'
return node
}
// atoms are the basic building blocks: literals, identifiers, words
atom(): SyntaxNode {
if (this.is($T.String))
return this.string()
if (this.isAny($T.Null, $T.Boolean, $T.Number, $T.Identifier, $T.Word, $T.Regex, $T.Underscore))
return SyntaxNode.from(this.next())
const next = this.next()
throw new CompilerError(`Unexpected token: ${TokenType[next.type]}`, next.from, next.to)
}
// blocks in if, do, special calls, etc
// `: something end`
//
// `blockNode` determines whether we return [colon, BlockNode, end] or
// just a list of statements like [colon, stmt1, stmt2, end]
block(blockNode = true): SyntaxNode[] {
const stmts: SyntaxNode[] = []
const colon = this.colon()
while (!this.isExprEndKeyword() && !this.isEOF()) {
const stmt = this.statement()
if (stmt) stmts.push(stmt)
}
const out = [colon]
if (blockNode) {
const block = new SyntaxNode('Block', stmts[0]!.from, stmts.at(-1)!.to)
block.push(...stmts)
out.push(block)
} else {
out.push(...stmts)
}
return out
}
// catch err: block
catch(): SyntaxNode {
const keyword = this.keyword('catch')
let catchVar
if (this.is($T.Identifier))
catchVar = this.identifier()
const block = this.block()
const node = new SyntaxNode('CatchExpr', keyword.from, block.at(-1)!.to)
node.push(keyword)
if (catchVar) node.push(catchVar)
return node.push(...block)
}
// colon
colon(): SyntaxNode {
const colon = SyntaxNode.from(this.expect($T.Colon))
colon.type = 'colon' // TODO lezer legacy
return colon
}
// # comment
comment(): SyntaxNode {
return SyntaxNode.from(this.expect($T.Comment))
}
// [ a b c ] = [ 1 2 3 ]
destructure(array: SyntaxNode): SyntaxNode {
const eq = this.op('=')
const val = this.expression()
for (const ident of array.children) {
const varName = this.input.slice(ident.from, ident.to)
this.scope.add(varName)
}
const node = new SyntaxNode('Assign', array.from, val.to)
return node.push(array, eq, val)
}
// [ a=1 b=true c='three' ]
dict(): SyntaxNode {
const open = this.expect($T.OpenBracket)
let isError = false
// empty dict [=] or [ = ]
if (this.is($T.Operator, '=') && this.nextIs($T.CloseBracket)) {
const _op = this.next()
const close = this.next()
return new SyntaxNode('Dict', open.from, close.to)
}
const values = []
while (!this.is($T.CloseBracket) && !this.isEOF()) {
if (this.is($T.Semicolon) || this.is($T.Newline)) {
this.next()
continue
}
if (this.is($T.Comment)) {
values.push(this.comment())
continue
}
// check for named arg with space after it (vs connected)
if (this.nextIs($T.Operator, '=')) {
const ident = this.identifier()
const op = this.op('=')
const prefix = new SyntaxNode('NamedArgPrefix', ident.from, op.to)
if (this.is($T.CloseBracket) || this.is($T.Semicolon) || this.is($T.Newline)) {
const node = new SyntaxNode('NamedArg', ident.from, op.to)
node.isError = true
isError = true
values.push(node.push(prefix))
} else {
const val = this.arg(true)
const node = new SyntaxNode('NamedArg', ident.from, val.to)
values.push(node.push(prefix, val))
}
} else {
const arg = this.is($T.NamedArgPrefix) ? this.namedArg() : this.arg()
if (arg.isError) isError = true
values.push(arg)
}
}
const close = this.expect($T.CloseBracket)
const node = new SyntaxNode('Dict', open.from, close.to)
node.isError = isError
return node.push(...values)
}
// FunctionDef `do x y: something end`
do(): SyntaxNode {
const doNode = this.keyword('do')
doNode.type = 'Do'
this.scope = new Scope(this.scope)
const params = []
while (!this.is($T.Colon) && !this.isExprEnd()) {
let varName = this.current().value!
if (varName.endsWith('=')) varName = varName.slice(0, varName.length - 1)
this.scope.add(varName)
let arg
if (this.is($T.Identifier))
arg = this.identifier()
else if (this.is($T.NamedArgPrefix))
arg = this.namedParam()
else
throw new CompilerError(`Expected Identifier or NamedArgPrefix, got ${TokenType[this.current().type]}`, this.current().from, this.current().to)
params.push(arg)
}
const block = this.block(false)
let catchNode, finalNode
if (this.is($T.Keyword, 'catch'))
catchNode = this.catch()
if (this.is($T.Keyword, 'finally'))
finalNode = this.finally()
const end = this.keyword('end')
let last = block.at(-1)
if (finalNode) last = finalNode.children.at(-1)!
else if (catchNode) last = catchNode.children.at(-1)!
const node = new SyntaxNode('FunctionDef', doNode.from, last!.to)
node.add(doNode)
const paramsNode = new SyntaxNode(
'Params',
params[0]?.from ?? 0,
params.at(-1)?.to ?? 0
)
if (params.length) paramsNode.push(...params)
node.add(paramsNode)
this.scope = this.scope.parent!
node.push(...block)
if (catchNode) node.push(catchNode)
if (finalNode) node.push(finalNode)
return node.push(end)
}
// config.path
dotGet(): SyntaxNode {
const left = this.identifier()
const ident = this.input.slice(left.from, left.to)
// not in scope, just return Word
if (!this.scope.has(ident))
return this.word(left)
if (left.type.is('Identifier')) left.type = 'IdentifierBeforeDot'
let parts = []
while (this.is($T.Operator, '.')) {
this.next()
parts.push(this.is($T.OpenParen) ? this.parens() : this.atom())
}
// TODO lezer legacy - we can do a flat DotGet if we remove this
const nodes = parts.length > 1 ? collapseDotGets(parts) : undefined
const node = new SyntaxNode('DotGet', left.from, parts.at(-1)!.to)
return nodes ? node.push(left, nodes!) : node.push(left, ...parts)
}
// continue parsing dotget/word binary operation
dotGetBinOp(left: SyntaxNode): SyntaxNode {
while (this.is($T.Operator) && !this.is($T.Operator, '|')) {
const op = this.current().value!
const bp = precedence[op]
if (bp === undefined) break
const opNode = this.op()
const right = this.exprWithPrecedence(bp + 1)
const nodeType = conditionals.has(op) ? 'ConditionalOp' : 'BinOp'
const node = new SyntaxNode(nodeType, left.from, right.to)
node.push(left, opNode, right)
left = node
}
return left
}
// dotget in a statement/expression (something.blah) or (something.blah arg1)
dotGetFunctionCall(): SyntaxNode {
const dotGet = this.dotGet()
// if followed by a binary operator (not pipe), return dotGet/Word as-is for expression parser
if (this.is($T.Operator) && !this.is($T.Operator, '|'))
return dotGet
// dotget not in scope, regular Word
if (dotGet.type.is('Word')) return dotGet
if (this.isExprEnd())
return this.functionCallOrIdentifier(dotGet)
else
return this.functionCall(dotGet)
}
// can be used in functions or try block
finally(): SyntaxNode {
const keyword = this.keyword('finally')
const block = this.block()
const node = new SyntaxNode('FinallyExpr', keyword.from, block.at(-1)!.to)
return node.push(keyword, ...block)
}
// you're lookin at it
functionCall(fn?: SyntaxNode): SyntaxNode {
const ident = fn ?? this.identifier()
let isError = false
const args: SyntaxNode[] = []
while (!this.isExprEnd()) {
const arg = this.is($T.NamedArgPrefix) ? this.namedArg() : this.arg()
if (arg.isError) isError = true
args.push(arg)
}
const node = new SyntaxNode('FunctionCall', ident.from, (args.at(-1) || ident).to)
node.push(ident, ...args)
if (isError) node.isError = true
if (!this.inTestExpr && this.is($T.Colon)) {
const block = this.block()
const end = this.keyword('end')
const blockNode = new SyntaxNode('FunctionCallWithBlock', node.from, end.to)
return blockNode.push(node, ...block, end)
}
return node
}
// bare identifier in an expression
functionCallOrIdentifier(inner?: SyntaxNode) {
if (!inner && this.nextIs($T.Operator, '.')) {
inner = this.dotGet()
// if the dotGet was just a Word, bail
if (inner.type.is('Word')) return inner
}
inner ??= this.identifier()
const wrapper = new SyntaxNode('FunctionCallOrIdentifier', inner.from, inner.to)
wrapper.push(inner)
if (!this.inTestExpr && this.is($T.Colon)) {
const block = this.block()
const end = this.keyword('end')
const node = new SyntaxNode('FunctionCallWithBlock', wrapper.from, end.to)
return node.push(wrapper, ...block, end)
}
return wrapper
}
// function and variable names
identifier(): SyntaxNode {
return SyntaxNode.from(this.expect($T.Identifier))
}
// if something: blah end
// if something: blah else: blah end
// if something: blah else if something: blah else: blah end
if(): SyntaxNode {
const ifNode = this.keyword('if')
const test = this.testExpr()
const ifBlock = this.block()
const node = new SyntaxNode('IfExpr', ifNode.from, ifBlock.at(-1)!.to)
node.push(ifNode, test)
node.push(...ifBlock)
while (this.is($T.Keyword, 'else') && this.nextIs($T.Keyword, 'if')) {
const elseWord = this.keyword('else')
const ifWord = this.keyword('if')
const elseIfTest = this.testExpr()
const elseIfBlock = this.block()
const elseIfNode = new SyntaxNode('ElseIfExpr', elseWord.from, elseIfBlock.at(-1)!.to)
elseIfNode.push(elseWord, ifWord, elseIfTest)
elseIfNode.push(...elseIfBlock)
node.push(elseIfNode)
}
if (this.is($T.Keyword, 'else') && this.nextIs($T.Colon)) {
const elseWord = this.keyword('else')
const elseBlock = this.block()
const elseNode = new SyntaxNode('ElseExpr', elseWord.from, elseBlock.at(-1)!.to)
elseNode.push(elseWord)
elseNode.push(...elseBlock)
node.push(elseNode)
}
return node.push(this.keyword('end'))
}
import(): SyntaxNode {
const keyword = this.keyword('import')
const args: SyntaxNode[] = []
while (!this.isExprEnd()) {
if (this.is($T.NamedArgPrefix)) {
const prefix = SyntaxNode.from(this.next())
const val = this.value()
const arg = new SyntaxNode('NamedArg', prefix.from, val.to)
arg.push(prefix, val)
args.push(arg)
} else {
args.push(this.identifier())
}
}
const node = new SyntaxNode('Import', keyword.from, args.at(-1)!.to)
node.add(keyword)
return node.push(...args)
}
// if, while, do, etc
keyword(name: string): SyntaxNode {
const node = SyntaxNode.from(this.expect($T.Keyword, name))
node.type = 'keyword' // TODO lezer legacy
return node
}
// abc= true
namedArg(): SyntaxNode {
const prefix = SyntaxNode.from(this.expect($T.NamedArgPrefix))
if (this.isExprEnd()) {
const node = new SyntaxNode('NamedArg', prefix.from, prefix.to)
node.isError = true
return node.push(prefix)
}
const val = this.arg(true)
const node = new SyntaxNode('NamedArg', prefix.from, val.to)
return node.push(prefix, val)
}
// abc= null|true|123|'hi'
namedParam(): SyntaxNode {
const prefix = SyntaxNode.from(this.expect($T.NamedArgPrefix))
const val = this.value()
if (!['Null', 'Boolean', 'Number', 'String'].includes(val.type.name))
throw new CompilerError(`Default value must be null, boolean, number, or string, got ${val.type.name}`, val.from, val.to)
const node = new SyntaxNode('NamedParam', prefix.from, val.to)
return node.push(prefix, val)
}
// not blah
not(): SyntaxNode {
const keyword = this.keyword('not')
const val = this.expression()
const node = new SyntaxNode('Not', keyword.from, val.to)
return node.push(keyword, val)
}
// operators like + - =
op(op?: string): SyntaxNode {
const token = op ? this.expect($T.Operator, op) : this.expect($T.Operator)
const name = operators[token.value!]
if (!name) throw new CompilerError(`Operator not registered: ${token.value!}`, token.from, token.to)
return new SyntaxNode(name, token.from, token.to)
}
// ( expressions in parens )
parens(): SyntaxNode {
this.inParens++
const open = this.expect($T.OpenParen)
const child = this.expression()
const close = this.expect($T.CloseParen)
this.inParens--
const node = new SyntaxNode('ParenExpr', open.from, close.to)
node.add(child)
return node
}
// 'hell yes' "hell no" { hell if i know }
string(): SyntaxNode {
const token = this.expect($T.String)
return parseString(this.input, token.from, token.to, this)
}
// if TEST: blah end
testExpr(): SyntaxNode {
this.inTestExpr = true
const expr = this.expression()
this.inTestExpr = false
return expr
}
// throw blah
throw(): SyntaxNode {
const keyword = this.keyword('throw')
const val = this.expression()
const node = new SyntaxNode('Throw', keyword.from, val.to)
return node.push(keyword, val)
}
// try: blah catch e: blah end
try(): SyntaxNode {
const tryNode = this.keyword('try')
const tryBlock = this.block()
let last = tryBlock.at(-1)
let catchNode, finalNode
if (this.is($T.Keyword, 'catch'))
catchNode = this.catch()
if (this.is($T.Keyword, 'finally'))
finalNode = this.finally()
const end = this.keyword('end')
if (finalNode) last = finalNode.children.at(-1)
else if (catchNode) last = catchNode.children.at(-1)
const node = new SyntaxNode('TryExpr', tryNode.from, last!.to)
node.push(tryNode, ...tryBlock)
if (catchNode)
node.push(catchNode)
if (finalNode)
node.push(finalNode)
return node.push(end)
}
// while test: blah end
while(): SyntaxNode {
const keyword = this.keyword('while')
const test = this.testExpr()
const block = this.block()
const end = this.keyword('end')
const node = new SyntaxNode('WhileExpr', keyword.from, end.to)
return node.push(keyword, test, ...block, end)
}
// readme.txt (when `readme` isn't in scope)
word(start?: SyntaxNode): SyntaxNode {
const parts = [start ?? this.expect($T.Word)]
while (this.is($T.Operator, '.')) {
this.next()
if (this.isAny($T.Word, $T.Identifier, $T.Number))
parts.push(this.next())
}
return new SyntaxNode('Word', parts[0]!.from, parts.at(-1)!.to)
}
//
// helpers
//
current(): Token {
return this.tokens[this.pos] || { type: TokenType.Newline, from: 0, to: 0 }
}
peek(offset = 1): Token | undefined {
return this.tokens[this.pos + offset]
}
// look past newlines to check for a specific token
peekPastNewlines(type: TokenType, value?: string): boolean {
let offset = 1
let peek = this.peek(offset)
while (peek && peek.type === $T.Newline)
peek = this.peek(++offset)
if (!peek || peek.type !== type) return false
if (value !== undefined && peek.value !== value) return false
return true
}
next(): Token {
const token = this.current()
this.pos++
return token
}
is(type: TokenType, value?: string): boolean {
const token = this.current()
if (!token || token.type !== type) return false
if (value !== undefined && token.value !== value) return false
return true
}
isAny(...type: TokenType[]): boolean {
return type.some(x => this.is(x))
}
nextIs(type: TokenType, value?: string): boolean {
const token = this.peek()
if (!token || token.type !== type) return false
if (value !== undefined && token.value !== value) return false
return true
}
nextIsAny(...type: TokenType[]): boolean {
return type.some(x => this.nextIs(x))
}
isExprEnd(): boolean {
return this.isAny($T.Colon, $T.Semicolon, $T.Newline, $T.CloseParen, $T.CloseBracket) ||
this.is($T.Operator, '|') ||
this.isExprEndKeyword() || !this.current()
}
nextIsExprEnd(): boolean {
// pipes act like expression end for function arg parsing
if (this.nextIs($T.Operator, '|'))
return true
return this.nextIsAny($T.Colon, $T.Semicolon, $T.Newline, $T.CloseBracket, $T.CloseParen) ||
this.nextIs($T.Keyword, 'end') || this.nextIs($T.Keyword, 'else') ||
this.nextIs($T.Keyword, 'catch') || this.nextIs($T.Keyword, 'finally') ||
!this.peek()
}
isExprEndKeyword(): boolean {
return this.is($T.Keyword, 'end') || this.is($T.Keyword, 'else') ||
this.is($T.Keyword, 'catch') || this.is($T.Keyword, 'finally')
}
isPipe(): boolean {
// inside parens, only look for pipes on same line (don't look past newlines)
const canLookPastNewlines = this.inParens === 0
return this.is($T.Operator, '|') ||
(canLookPastNewlines && this.peekPastNewlines($T.Operator, '|'))
}
expect(type: TokenType, value?: string): Token | never {
if (!this.is(type, value)) {
const token = this.current()
throw new CompilerError(`Expected ${TokenType[type]}${value ? ` "${value}"` : ''}, got ${TokenType[token?.type || 0]}${token?.value ? ` "${token.value}"` : ''} at position ${this.pos}`, token.from, token.to)
}
return this.next()
}
isEOF(): boolean {
return this.pos >= this.tokens.length
}
}
// TODO lezer legacy
function collapseDotGets(origNodes: SyntaxNode[]): SyntaxNode {
const nodes = [...origNodes]
let right = nodes.pop()!
while (nodes.length > 0) {
const left = nodes.pop()!
if (left.type.is('Identifier')) left.type = 'IdentifierBeforeDot'
const dot = new SyntaxNode("DotGet", left.from, right.to)
dot.push(left, right)
right = dot
}
return right
}

View File

@ -0,0 +1,129 @@
import { ContextTracker, InputStream } from '@lezer/lr'
import * as terms from './shrimp.terms'
export class Scope {
constructor(public parent: Scope | null, public vars = new Set<string>()) { }
has(name: string): boolean {
return this.vars.has(name) || (this.parent?.has(name) ?? false)
}
hash(): number {
let h = 0
for (const name of this.vars) {
for (let i = 0; i < name.length; i++) {
h = (h << 5) - h + name.charCodeAt(i)
h |= 0
}
}
if (this.parent) {
h = (h << 5) - h + this.parent.hash()
h |= 0
}
return h
}
// Static methods that return new Scopes (immutable operations)
static add(scope: Scope, ...names: string[]): Scope {
const newVars = new Set(scope.vars)
names.forEach((name) => newVars.add(name))
return new Scope(scope.parent, newVars)
}
push(): Scope {
return new Scope(this, new Set())
}
pop(): Scope {
return this.parent ?? this
}
}
// Tracker context that combines Scope with temporary pending identifiers
class TrackerContext {
constructor(public scope: Scope, public pendingIds: string[] = []) { }
}
// Extract identifier text from input stream
const readIdentifierText = (input: InputStream, start: number, end: number): string => {
let text = ''
for (let i = start; i < end; i++) {
const offset = i - input.pos
const ch = input.peek(offset)
if (ch === -1) break
text += String.fromCharCode(ch)
}
return text
}
let inParams = false
export const trackScope = new ContextTracker<TrackerContext>({
start: new TrackerContext(new Scope(null, new Set())),
shift(context, term, stack, input) {
if (term == terms.Do) inParams = true
if (term === terms.AssignableIdentifier) {
const text = readIdentifierText(input, input.pos, stack.pos)
return new TrackerContext(Scope.add(context.scope, text), context.pendingIds)
}
if (inParams && term === terms.Identifier) {
const text = readIdentifierText(input, input.pos, stack.pos)
return new TrackerContext(context.scope, [...context.pendingIds, text])
}
// Track identifiers in array destructuring: [ a b ] = ...
if (!inParams && term === terms.Identifier && isArrayDestructuring(input)) {
const text = readIdentifierText(input, input.pos, stack.pos)
return new TrackerContext(Scope.add(context.scope, text), context.pendingIds)
}
return context
},
reduce(context, term) {
if (term === terms.Params) {
inParams = false
let newScope = context.scope.push()
if (context.pendingIds.length > 0) {
newScope = Scope.add(newScope, ...context.pendingIds)
}
return new TrackerContext(newScope, [])
}
// Pop scope when exiting function
if (term === terms.FunctionDef) {
return new TrackerContext(context.scope.pop(), [])
}
return context
},
hash: (context) => context.scope.hash(),
})
// Check if we're parsing array destructuring: [ a b ] = ...
const isArrayDestructuring = (input: InputStream): boolean => {
let pos = 0
// Find closing bracket
while (pos < 200 && input.peek(pos) !== 93 /* ] */) {
if (input.peek(pos) === -1) return false // EOF
pos++
}
if (input.peek(pos) !== 93 /* ] */) return false
pos++
// Skip whitespace
while (input.peek(pos) === 32 /* space */ ||
input.peek(pos) === 9 /* tab */ ||
input.peek(pos) === 10 /* \n */) {
pos++
}
return input.peek(pos) === 61 /* = */
}

293
src/parser/shrimp.grammar Normal file
View File

@ -0,0 +1,293 @@
@external propSource highlighting from "./highlight"
@context trackScope from "./parserScopeContext"
@skip { space | Comment }
@top Program { item* }
@external tokens operatorTokenizer from "./operatorTokenizer" { Star, Slash, Plus, Minus, And, Or, Eq, EqEq, Neq, Lt, Lte, Gt, Gte, Modulo, PlusEq, MinusEq, StarEq, SlashEq, ModuloEq, Band, Bor, Bxor, Shl, Shr, Ushr, NullishCoalesce, NullishEq }
@tokens {
@precedence { Number Regex }
StringFragment { !['\\$]+ }
DoubleQuote { '"' !["]* '"' }
NamedArgPrefix { $[a-z] $[a-z0-9-]* "=" }
Number {
("-" | "+")? "0x" $[0-9a-fA-F]+ |
("-" | "+")? "0b" $[01]+ |
("-" | "+")? "0o" $[0-7]+ |
("-" | "+")? $[0-9]+ ("_"? $[0-9]+)* ('.' $[0-9]+ ("_"? $[0-9]+)*)?
}
Boolean { "true" | "false" }
semicolon { ";" }
eof { @eof }
space { " " | "\t" }
Comment { "#" ![\n]* }
leftParen { "(" }
rightParen { ")" }
colon[closedBy="end", @name="colon"] { ":" }
Underscore { "_" }
Dollar { "$" }
Regex { "//" (![/\\\n[] | "\\" ![\n] | "[" (![\n\\\]] | "\\" ![\n])* "]")+ ("//" $[gimsuy]*)? } // Stolen from the lezer JavaScript grammar
"|"[@name=operator]
}
newlineOrSemicolon { newline | semicolon }
end { @specialize[@name=keyword]<Identifier, "end"> }
while { @specialize[@name=keyword]<Identifier, "while"> }
if { @specialize[@name=keyword]<Identifier, "if"> }
else { @specialize[@name=keyword]<Identifier, "else"> }
try { @specialize[@name=keyword]<Identifier, "try"> }
catch { @specialize[@name=keyword]<Identifier, "catch"> }
finally { @specialize[@name=keyword]<Identifier, "finally"> }
throw { @specialize[@name=keyword]<Identifier, "throw"> }
import { @specialize[@name=keyword]<Identifier, "import"> }
null { @specialize[@name=Null]<Identifier, "null"> }
@external tokens tokenizer from "./tokenizer" { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, CurlyString }
@external tokens pipeStartsLineTokenizer from "./tokenizer" { newline, pipeStartsLine }
@external specialize {Identifier} specializeKeyword from "./tokenizer" { Do }
@precedence {
pipe @left,
or @left,
and @left,
nullish @left,
comparison @left,
multiplicative @left,
additive @left,
bitwise @left,
call,
functionWithNewlines
}
item {
consumeToTerminator newlineOrSemicolon |
consumeToTerminator eof |
newlineOrSemicolon // allow blank lines
}
consumeToTerminator {
PipeExpr |
WhileExpr |
FunctionCallWithBlock |
ambiguousFunctionCall |
TryExpr |
Throw |
Import |
IfExpr |
FunctionDef |
CompoundAssign |
Assign |
BinOp |
ConditionalOp |
expressionWithoutIdentifier
}
PipeExpr {
pipeOperand (!pipe (pipeStartsLine? "|") newlineOrSemicolon* pipeOperand)+
}
pipeOperand {
consumeToTerminator
}
WhileExpr {
while (ConditionalOp | expression) colon Block end
}
Block {
consumeToTerminator | newlineOrSemicolon block
}
FunctionCallWithBlock {
ambiguousFunctionCall colon Block CatchExpr? FinallyExpr? end
}
FunctionCallOrIdentifier {
DotGet | Identifier
}
ambiguousFunctionCall {
FunctionCall | FunctionCallOrIdentifier
}
FunctionCall {
(DotGet | Identifier | ParenExpr) arg+
}
arg {
PositionalArg | NamedArg
}
PositionalArg {
expression | FunctionDef | Underscore
}
NamedArg {
NamedArgPrefix (expression | FunctionDef | Underscore)
}
FunctionDef {
Do Params colon (consumeToTerminator | newlineOrSemicolon block) CatchExpr? FinallyExpr? end
}
ifTest {
ConditionalOp | expression | FunctionCall
}
IfExpr {
if ifTest colon Block ElseIfExpr* ElseExpr? end
}
ElseIfExpr {
else if ifTest colon Block
}
ElseExpr {
else colon Block
}
TryExpr {
try colon Block CatchExpr? FinallyExpr? end
}
CatchExpr {
catch Identifier colon Block
}
FinallyExpr {
finally colon Block
}
Throw {
throw (BinOp | ConditionalOp | expression)
}
// this has to be in the parse tree so the scope tracker can use it
Import {
import NamedArg* Identifier+ NamedArg*
}
ConditionalOp {
expression !comparison EqEq expression |
expression !comparison Neq expression |
expression !comparison Lt expression |
expression !comparison Lte expression |
expression !comparison Gt expression |
expression !comparison Gte expression |
(expression | ConditionalOp) !and And (expression | ConditionalOp) |
(expression | ConditionalOp) !or Or (expression | ConditionalOp) |
(expression | ConditionalOp) !nullish NullishCoalesce (expression | ConditionalOp)
}
Params {
Identifier* NamedParam*
}
NamedParam {
NamedArgPrefix (String | Number | Boolean | null)
}
Assign {
(AssignableIdentifier | Array) Eq consumeToTerminator
}
CompoundAssign {
AssignableIdentifier (PlusEq | MinusEq | StarEq | SlashEq | ModuloEq | NullishEq) consumeToTerminator
}
BinOp {
expression !multiplicative Modulo expression |
(expression | BinOp) !multiplicative Star (expression | BinOp) |
(expression | BinOp) !multiplicative Slash (expression | BinOp) |
(expression | BinOp) !additive Plus (expression | BinOp) |
(expression | BinOp) !additive Minus (expression | BinOp) |
(expression | BinOp) !bitwise Band (expression | BinOp) |
(expression | BinOp) !bitwise Bor (expression | BinOp) |
(expression | BinOp) !bitwise Bxor (expression | BinOp) |
(expression | BinOp) !bitwise Shl (expression | BinOp) |
(expression | BinOp) !bitwise Shr (expression | BinOp) |
(expression | BinOp) !bitwise Ushr (expression | BinOp)
}
ParenExpr {
leftParen newlineOrSemicolon* (
FunctionCallWithNewlines |
IfExpr |
ambiguousFunctionCall |
BinOp newlineOrSemicolon* |
expressionWithoutIdentifier |
ConditionalOp newlineOrSemicolon* |
PipeExpr |
FunctionDef
)
rightParen
}
FunctionCallWithNewlines[@name=FunctionCall] {
(DotGet | Identifier | ParenExpr) newlineOrSemicolon+ arg !functionWithNewlines (newlineOrSemicolon+ arg)* newlineOrSemicolon*
}
expression {
expressionWithoutIdentifier | DotGet | Identifier
}
@local tokens {
dot { "." }
}
@skip {} {
DotGet {
IdentifierBeforeDot dot (DotGet | Number | Identifier | ParenExpr) |
Dollar dot (DotGet | Number | Identifier | ParenExpr)
}
String {
"'" stringContent* "'" | CurlyString | DoubleQuote
}
}
stringContent {
StringFragment |
Interpolation |
EscapeSeq
}
Interpolation {
"$" FunctionCallOrIdentifier |
"$" ParenExpr
}
EscapeSeq {
"\\" ("$" | "n" | "t" | "r" | "\\" | "'")
}
Dict {
"[=]" |
"[" newlineOrSemicolon* NamedArg (newlineOrSemicolon | NamedArg)* "]"
}
Array {
"[" newlineOrSemicolon* (expression (newlineOrSemicolon | expression)*)? "]"
}
// We need expressionWithoutIdentifier to avoid conflicts in consumeToTerminator.
// Without this, when parsing "my-var" at statement level, the parser can't decide:
// - ambiguousFunctionCall → FunctionCallOrIdentifier → Identifier
// - expression → Identifier
// Both want the same Identifier token! So we use expressionWithoutIdentifier
// to remove Identifier from the second path, forcing standalone identifiers
// to go through ambiguousFunctionCall (which is what we want semantically).
// Yes, it is annoying and I gave up trying to use GLR to fix it.
expressionWithoutIdentifier {
ParenExpr | Word | String | Number | Boolean | Regex | Dict | Array | null
}
block {
(consumeToTerminator? newlineOrSemicolon)*
}

4
src/parser/shrimp.grammar.d.ts vendored Normal file
View File

@ -0,0 +1,4 @@
declare module '*.grammar' {
const content: string
export default content
}

View File

@ -0,0 +1,81 @@
// This file was generated by lezer-generator. You probably shouldn't edit it.
export const
Star = 1,
Slash = 2,
Plus = 3,
Minus = 4,
And = 5,
Or = 6,
Eq = 7,
EqEq = 8,
Neq = 9,
Lt = 10,
Lte = 11,
Gt = 12,
Gte = 13,
Modulo = 14,
PlusEq = 15,
MinusEq = 16,
StarEq = 17,
SlashEq = 18,
ModuloEq = 19,
Band = 20,
Bor = 21,
Bxor = 22,
Shl = 23,
Shr = 24,
Ushr = 25,
NullishCoalesce = 26,
NullishEq = 27,
Identifier = 28,
AssignableIdentifier = 29,
Word = 30,
IdentifierBeforeDot = 31,
CurlyString = 32,
newline = 101,
pipeStartsLine = 102,
Do = 33,
Comment = 34,
Program = 35,
PipeExpr = 36,
WhileExpr = 38,
keyword = 84,
ConditionalOp = 40,
ParenExpr = 41,
FunctionCallWithNewlines = 42,
DotGet = 43,
Number = 44,
Dollar = 45,
PositionalArg = 46,
FunctionDef = 47,
Params = 48,
NamedParam = 49,
NamedArgPrefix = 50,
String = 51,
StringFragment = 52,
Interpolation = 53,
FunctionCallOrIdentifier = 54,
EscapeSeq = 55,
DoubleQuote = 56,
Boolean = 57,
Null = 58,
colon = 59,
CatchExpr = 60,
Block = 62,
FinallyExpr = 63,
Underscore = 66,
NamedArg = 67,
IfExpr = 68,
FunctionCall = 70,
ElseIfExpr = 71,
ElseExpr = 73,
BinOp = 74,
Regex = 75,
Dict = 76,
Array = 77,
FunctionCallWithBlock = 78,
TryExpr = 79,
Throw = 81,
Import = 83,
CompoundAssign = 85,
Assign = 86

27
src/parser/shrimp.ts Normal file
View File

@ -0,0 +1,27 @@
// This file was generated by lezer-generator. You probably shouldn't edit it.
import {LRParser, LocalTokenGroup} from "@lezer/lr"
import {operatorTokenizer} from "./operatorTokenizer"
import {tokenizer, pipeStartsLineTokenizer, specializeKeyword} from "./tokenizer"
import {trackScope} from "./parserScopeContext"
import {highlighting} from "./highlight"
const spec_Identifier = {__proto__:null,while:78, null:116, catch:122, finally:128, end:130, if:138, else:144, try:160, throw:164, import:168}
export const parser = LRParser.deserialize({
version: 14,
states: "?[QYQ!SOOOOQ!Q'#Ek'#EkO!sO!bO'#DXO%kQ!TO'#DdO&UOSO'#DaOOQ!R'#Da'#DaO)SQ!TO'#EnOOQ!Q'#E{'#E{O)pQRO'#DxO+xQ!TO'#EjO,fQ!SO'#DVOOQ!R'#Dz'#DzO/WQ!SO'#D{OOQ!R'#En'#EnO/_Q!TO'#EnO1cQ!TO'#EmO2qQ!TO'#EjO3OQRO'#ETOOQ!Q'#Ej'#EjO3gQ!SO'#EjO3nQrO'#EiOOQ!Q'#Ei'#EiOOQ!Q'#EV'#EVQYQ!SOOO4PQbO'#D]O4[QbO'#DrO5YQbO'#DSO6WQQO'#D}O5YQbO'#EPO6]QbO'#ERO6eObO,59sOOQ!Q'#D['#D[O6vQbO'#DqOOQ!Q'#Eq'#EqOOQ!Q'#E_'#E_O7QQ!SO,5:`OOQ!R'#Em'#EmO8QQbO'#DcO8`QWO'#DeOOOO'#Es'#EsOOOO'#E['#E[O8tOSO,59{OOQ!R,59{,59{O5YQbO,5:dO5YQbO,5:dO5YQbO,5:dO5YQbO,5:dO5YQbO,59pO5YQbO,59pO5YQbO,59pO5YQbO,59pOOQ!Q'#EX'#EXO,fQ!SO,59qO9SQ!TO'#DdO9^Q!TO'#EnO9hQsO,59qO9uQQO,59qO9zQrO,59qO:VQrO,59qO:eQsO,59qO;TQsO,59qO;[QrO'#DQO;dQ!SO,5:gO;kQrO,5:fOOQ!R,5:g,5:gO;yQ!SO,5:gO<WQbO,5:pO<WQbO,5:oOYQ!SO,5:hO=kQ!SO,59lOOQ!Q,5;T,5;TOYQ!SO'#EWO>]QQO'#EWOOQ!Q-E8T-E8TOOQ!Q'#EY'#EYO>bQbO'#D^O>mQbO'#D_OOQO'#EZ'#EZO>eQQO'#D^O?RQQO,59wO?WQcO'#EmO@TQRO'#EzOAQQRO'#EzOOQO'#Ez'#EzOAXQQO,5:^OA^QRO,59nOAeQRO,59nOYQ!SO,5:iOAsQ!TO,5:kOCXQ!TO,5:kOC{Q!TO,5:kODYQ!SO,5:mOOQ!Q'#Ec'#EcO6]QbO,5:mOOQ!R1G/_1G/_OOQ!Q,5:],5:]OOQ!Q-E8]-E8]OOOO'#Dd'#DdOOOO,59},59}OOOO,5:P,5:POOOO-E8Y-E8YOOQ!R1G/g1G/gOOQ!R1G0O1G0OOF_Q!TO1G0OOFiQ!TO1G0OOG}Q!TO1G0OOHXQ!TO1G0OOHfQ!TO1G0OOOQ!R1G/[1G/[OI}Q!TO1G/[OJUQ!TO1G/[OJ]Q!TO1G/[OKbQ!TO1G/[OJdQ!TO1G/[OOQ!Q-E8V-E8VOKxQsO1G/]OLVQQO1G/]OL[QrO1G/]OLgQrO1G/]OLuQsO1G/]OL|QsO1G/]OMTQ!SO,59rOM_QrO1G/]OOQ!R1G/]1G/]OMjQrO1G0QOOQ!R1G0R1G0ROMxQ!SO1G0ROOQp'#Ea'#EaOMjQrO1G0QOOQ!R1G0Q1G0QOOQ!Q'#Eb'#EbOMxQ!SO1G0RONVQ!SO1G0[ONwQ!SO1G0ZO! iQ!SO'#DlO! }Q!SO'#DlO!!_QbO1G0SOOQ!Q-E8U-E8UOYQ!SO,5:rOOQ!Q,5:r,5:rOYQ!SO,5:rOOQ!Q-E8W-E8WO!!jQQO,59xOOQO,59y,59yOOQO-E8X-E8XOYQ!SO1G/cOYQ!SO1G/xOYQ!SO1G/YO!!rQbO1G0TO!!}Q!SO1G0XO!#rQ!SO1G0XOOQ!Q-E8a-E8aO!#yQrO7+$wOOQ!R7+$w7+$wO!$UQrO1G/^O!$aQrO7+%lOOQ!R7+%l7+%lO!$oQ!SO7+%mOOQ!R7+%m7+%mOOQp-E8_-E8_OOQ!Q-E8`-E8`OOQ!Q'#E]'#E]O!$|QrO'#E]O!%[Q!SO'#EyOOQ`,5:W,5:WO!%lQbO'#DjO!%qQQO'#DmOOQ!Q7+%n7+%nO!%vQbO7+%nO!%{QbO7+%nOOQ!Q1G0^1G0^OYQ!SO1G0^O!&TQ!SO7+$}O!&fQ!SO7+$}O!&sQbO7+%dO!&{QbO7+$tOOQ!Q7+%o7+%oO!'QQbO7+%oO!'VQbO7+%oO!'_Q!SO7+%sOOQ!R<<Hc<<HcO!(SQ!SO7+$xO!(aQrO7+$xOOQ!R<<IW<<IWOOQ!R<<IX<<IXOOQ!Q,5:w,5:wOOQ!Q-E8Z-E8ZO!(lQQO,5:UOYQ!SO,5:XOOQ!Q<<IY<<IYO!(qQbO<<IYOOQ!Q7+%x7+%xOOQ!Q<<Hi<<HiO!(vQbO<<HiO!({QbO<<HiO!)TQbO<<HiOOQ`'#E`'#E`O!)`QbO<<IOO!)hQbO'#DwOOQ!Q<<IO<<IOO!)pQbO<<IOOOQ!Q<<H`<<H`OOQ!Q<<IZ<<IZO!)uQbO<<IZOOQp,5:x,5:xO!)zQ!SO<<HdOOQp-E8[-E8[OYQ!SO1G/pOOQ`1G/s1G/sOOQ!QAN>tAN>tOOQ!QAN>TAN>TO!*XQbOAN>TO!*^QbOAN>TOOQ`-E8^-E8^OOQ!QAN>jAN>jO!*fQbOAN>jO4[QbO,5:aOYQ!SO,5:cOOQ!QAN>uAN>uPMTQ!SO'#EXOOQ`7+%[7+%[OOQ!QG23oG23oO!*kQbOG23oP!)kQbO'#DuOOQ!QG24UG24UO!*pQQO1G/{OOQ`1G/}1G/}OOQ!QLD)ZLD)ZOYQ!SO7+%gOOQ`<<IR<<IRO!*uObO,59sO!+WO!bO'#DX",
stateData: "!+`~O#[OSrOS~OlROmaOn]OoQOpTOqhOwjO|]O}QO!YTO!Z]O![]O!giO!m]O!rkO!tlO!vmO#XPO#`PO#cYO#fSO#qZO#r[O~O#dnO~OltOn]OoQOpTOqhO|]O}QO!SpO!YTO!Z]O![]O!doO!m]O#cYO#fSO#qZO#r[OP#aXQ#aXR#aXS#aXT#aXU#aXW#aXX#aXY#aXZ#aX[#aX]#aX^#aXd#aXe#aXf#aXg#aXh#aXi#aXj#aXu!WX!]!WX#Y!WX#p!WX~O#X!WX#`!WX#t!WX!_!WX!b!WX!c!WX!j!WX~P!xO!UwO#fzO#huO#ivO~OltOn]OoQOpTOqhO|]O}QO!SpO!YTO!Z]O![]O!doO!m]O#cYO#fSO#qZO#r[OP#bXQ#bXR#bXS#bXT#bXU#bXW#bXX#bXY#bXZ#bX[#bX]#bX^#bXd#bXe#bXf#bXg#bXh#bXi#bXj#bXu#bX#Y#bX#p#bX~O#X#bX#`#bX#t#bX!]#bX!_#bX!b#bX!c#bX!j#bX~P&dOP|OQ|OR}OS}OT!QOU!ROW!POX!POY!POZ!PO[!PO]!PO^{Od!OOe!OOf!OOg!OOh!OOi!OOj!SO~OP|OQ|OR}OS}Od!OOe!OOf!OOg!OOh!OOi!OOu#^X#Y#^X~O#X#^X#`#^X#t#^X!_#^X!b#^X!c#^X#p#^X!j#^X~P+QOl!VOmaOn]OoQOpTOqhOwjO|]O}QO!YTO!Z]O![]O!giO!m]O!rkO!tlO!vmO#XPO#`PO#cYO#fSO#qZO#r[O~OltOn]OoQOpTO|]O}QO!SpO!YTO!Z]O![]O!m]O#XPO#`PO#cYO#fSO#qZO#r[O~O#s!bO~P.POV!dO#X#bX#`#bX#t#bX!_#bX!b#bX!c#bX!j#bX~P'iOP#aXQ#aXR#aXS#aXT#aXU#aXW#aXX#aXY#aXZ#aX[#aX]#aX^#aXd#aXe#aXf#aXg#aXh#aXi#aXj#aXu#^X#Y#^X~O#X#^X#`#^X#t#^X!_#^X!b#^X!c#^X#p#^X!j#^X~P/{Ou#^X#X#^X#Y#^X#`#^X#t#^X!_#^X!b#^X!c#^X#p#^X!j#^X~OT!QOU!ROj!SO~P2POV!dO_!eO`!eOa!eOb!eOc!eOk!eO~O!]!fO~P2POu!iO#XPO#Y!jO#`PO#t!hO~Ol!lO!S!nO!]!QP~Ol!rOn]OoQOpTO|]O}QO!YTO!Z]O![]O!m]O#cYO#fSO#qZO#r[O~OltOn]OoQOpTO|]O}QO!YTO!Z]O![]O!m]O#cYO#fSO#qZO#r[O~O!]!yO~Ol!lO!SpO~Ol#QOoQO|#QO}QO#cYO~OqhO!d#RO~P5YOqhO!SpO!doOu!ha!]!ha#X!ha#Y!ha#`!ha#t!ha#p!ha!_!ha!b!ha!c!ha!j!ha~P5YOl#TOo&PO}&PO#cYO~O#f#VO#h#VO#i#VO#j#VO#k#VO#l#VO~O!UwO#f#XO#huO#ivO~O#XPO#`PO~P!xO#XPO#`PO~P&dO#XPO#`PO#p#oO~P+QO#p#oO~O#p#oOu#^X#Y#^X~O!]!fO#p#oOu#^X#Y#^X~O#p#oO~P/{OT!QOU!ROj!SO#XPO#`POu#^X#Y#^X~O#p#oO~P:lOu!iO#Y!jO~O#s#qO~P.PO!SpO#XPO#`PO#s#uO~O#XPO#`PO#s#qO~P5YOlROmaOn]OoQOpTOqhOwjO|]O}QO!YTO!Z]O![]O!giO!m]O!rkO!tlO!vmO#cYO#fSO#qZO#r[O~Ou!iO#Y!jO#Xta#`ta#tta#pta!_ta!bta!cta!jta~Ou$QO~Ol!lO!S!nO!]!QX~OpTO|$TO!YTO!Z$TO![$TO#fSO~O!]$VO~OqhO!SpO!doOT#aXU#aXW#aXX#aXY#aXZ#aX[#aX]#aXj#aX!]#aX~P5YOT!QOU!ROj!SO!]#nX~OT!QOU!ROW!POX!POY!POZ!PO[!PO]!POj!SO~O!]#nX~P@cO!]$WO~O!]$XO~P@cOT!QOU!ROj!SO!]$XO~Ou!sa#X!sa#Y!sa#`!sa#t!sa!_!sa!b!sa!c!sa#p!sa!j!sa~P)pOu!sa#X!sa#Y!sa#`!sa#t!sa!_!sa!b!sa!c!sa#p!sa!j!sa~OP|OQ|OR}OS}Od!OOe!OOf!OOg!OOh!OOi!OO~PBgOT!QOU!ROj!SO~PBgOl!lO!SpOu!ua#X!ua#Y!ua#`!ua#t!ua!_!ua!b!ua!c!ua#p!ua!j!ua~O^{OR!liS!lid!lie!lif!lig!lih!lii!liu!li#X!li#Y!li#`!li#t!li#p!li!_!li!b!li!c!li!j!li~OP!liQ!li~PEQOP|OQ|O~PEQOP|OQ|Od!lie!lif!lig!lih!lii!liu!li#X!li#Y!li#`!li#t!li#p!li!_!li!b!li!c!li!j!li~OR!liS!li~PFsOR}OS}O^{O~PFsOR}OS}O~PFsOW!POX!POY!POZ!PO[!PO]!POTxijxiuxi#Xxi#Yxi#`xi#txi#pxi!]xi!_xi!bxi!cxi!jxi~OU!RO~PHpOU!RO~PISOUxi~PHpOT!QOU!ROjxiuxi#Xxi#Yxi#`xi#txi#pxi!]xi!_xi!bxi!cxi!jxi~OW!POX!POY!POZ!PO[!PO]!PO~PJdO#XPO#`PO#p$_O~P+QO#p$_O~O#p$_Ou#^X#Y#^X~O!]!fO#p$_Ou#^X#Y#^X~O#p$_O~P/{O#p$_O~P:lOqhO!doO~P.PO#XPO#`PO#p$_O~O!SpO#XPO#`PO#s$bO~O#XPO#`PO#s$dO~P5YOu!iO#Y!jO#X!xi#`!xi#t!xi!_!xi!b!xi!c!xi#p!xi!j!xi~Ou!iO#Y!jO#X!wi#`!wi#t!wi!_!wi!b!wi!c!wi#p!wi!j!wi~Ou!iO#Y!jO!_!`X!b!`X!c!`X!j!`X~O!_#mP!b#mP!c#mP!j#mP~PYO!_$kO!b$lO!c$mO~O!S!nO!]!Qa~O!_$kO!b$lO!c$vO~O!SpOu!ui#X!ui#Y!ui#`!ui#t!ui!_!ui!b!ui!c!ui#p!ui!j!ui~Ol!lO~P!!}O#XPO#`PO#p$zO~O#XPO#`PO#pzi~O!SpO#XPO#`PO#s$}O~O#XPO#`PO#s%OO~P5YOu!iO#XPO#Y!jO#`PO~O!_#mX!b#mX!c#mX!j#mX~PYOl%RO~O!]%SO~O!c%TO~O!b$lO!c%TO~Ou!iO!_$kO!b$lO!c%WO#Y!jO~O!_#mP!b#mP!c#mP~PYO!c%_O!j%^O~O!c%aO~O!c%bO~O!b$lO!c%bO~O!SpOu!uq#X!uq#Y!uq#`!uq#t!uq!_!uq!b!uq!c!uq#p!uq!j!uq~OqhO!doO#pzq~P.PO#XPO#`PO#pzq~O!]%gO~O!c%iO~O!c%jO~O!b$lO!c%jO~O!_$kO!b$lO!c%jO~O!c%nO!j%^O~O!]%qO!g%pO~O!c%nO~O!c%rO~OqhO!doO#pzy~P.PO!c%uO~O!b$lO!c%uO~O!c%xO~O!c%{O~O!]%|O~Ol#QOo&PO|#QO}&PO#cYO~O#d&OO~O|!m~",
goto: "<[#pPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP#qP$_P$w%x'['bPP(v)S*P*SP*YP+d+h+dPPPP,TP,a,yPPP-a#qP.R.oP.s.yP/s0z$_$_P$_P$_P$_$_2T2Z2g3c3q3{4R4Y4`4j4p4z5UPPPPP5d5h6dP7v9oPP:|P;^PPPPP;b;h;nxbOg!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|Q!ZYR#i!U}bOYg!U!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|x`Og!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|Q!^YS!si%pQ!xjQ!|lQ#`!RQ#b!QQ#e!SR#l!U|UOgi!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%p%q%|!W]RU[jlps{|}!O!P!Q!R!S!V!W!`!c!r#m#r#w$c${%e%sS!WY!US#Qn&OR#UuQ!YYR#h!UxROg!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|!WtRU[jlps{|}!O!P!Q!R!S!V!W!`!c!r#m#r#w$c${%e%sS!VY!US!ri%pS#Qn&OR#TueqRUs!V!W!r#m${%e%sxbOg!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|doRUs!V!W!r#m${%e%sQ!ZYQ#RpR#i!UR!qhX!oh!m!p$S#Y]ORUY[gijlps{|}!O!P!Q!R!S!U!V!W!`!c!d!e!f!i!r!y#m#r#w#{$O$Q$V$W$X$c$i$q$s${%S%e%g%p%q%s%|R$T!nTwSy|VOYg!U!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|R#UuQ$o#|Q$x$YQ%Y$rR%l%ZQ#|!fQ$Y!yQ$t$WQ$u$XQ%h%SQ%t%gQ%z%qR%}%|Q$n#|Q$w$YQ%U$oQ%X$rQ%c$xS%k%Y%ZR%v%ldqRUs!V!W!r#m${%e%sQ!a[[#Om!}#P$Z$[$yQ#p!`X#s!a#p#t$a|VOYg!U!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|T!ui%pT%[$t%]Q%`$tR%o%]xXOg!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|Q!XYQ!{lQ#Y|Q#]}Q#_!OR#g!U#Z]ORUY[gijlps{|}!O!P!Q!R!S!U!V!W!`!c!d!e!f!i!r!y#m#r#w#{$O$Q$V$W$X$c$i$q$s${%S%e%g%p%q%s%|![]RU[ijlps{|}!O!P!Q!R!S!V!W!`!c!r#m#r#w$c${%e%p%s}^OYg!U!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|QgOR!kg^!gd!_#x#y#z$h$rR#}!gQ!UYQ!`[d#f!U!`#m#n$O$^$q${%e%sS#m!V!WS#n!X!^Q$O!iS$^#g#lQ$q$QQ${$`R%e$|Q!mhQ!}mU$R!m!}$[R$[#PQ!phQ$S!mT$U!p$SQySR#WyS$i#{$sR%Q$iQ$|$`R%f$|YsRU!V!W!rR#SsQ%]$tR%m%]Q#t!aQ$a#pT$e#t$aQ#w!cQ$c#rT$f#w$cQ#PmQ$Z!}U$]#P$Z$yR$y$[TfOgSdOgS!_Y!UQ#x!dQ#y!e`#z!f!y$W$X%S%g%q%|Q$P!iU$h#{$i$sS$p$O$QQ$r$VR%V$qSeOg|!TY[!U!V!W!X!^!`!i#g#l#m#n$O$Q$^$`$q${$|%e%sQ!hdW#s!a#p#t$aW#v!c#r#w$c`#{!f!y$W$X%S%g%q%|U$g#{$i$sQ$s$VR%P$h|WOYg!U!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|doRUs!V!W!r#m${%e%sQ!c[S!ti%pQ!wjQ!zlQ#RpQ#Y{Q#Z|Q#[}Q#^!OQ#`!PQ#a!QQ#c!RQ#d!SQ#r!`X#v!c#r#w$cx_Og!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|![tRU[ijlps{|}!O!P!Q!R!S!V!W!`!c!r#m#r#w$c${%e%p%sQ!]YR#k!U[rRUs!V!W!rQ$`#mV%d${%e%sTxSyQ$j#{R%Z$sQ!viR%y%pxcOg!d!e!f!i!y#{$O$Q$V$W$X$i$q$s%S%g%q%|Q![YR#j!U",
nodeNames: "⚠ Star Slash Plus Minus And Or Eq EqEq Neq Lt Lte Gt Gte Modulo PlusEq MinusEq StarEq SlashEq ModuloEq Band Bor Bxor Shl Shr Ushr NullishCoalesce NullishEq Identifier AssignableIdentifier Word IdentifierBeforeDot CurlyString Do Comment Program PipeExpr operator WhileExpr keyword ConditionalOp ParenExpr FunctionCall DotGet Number Dollar PositionalArg FunctionDef Params NamedParam NamedArgPrefix String StringFragment Interpolation FunctionCallOrIdentifier EscapeSeq DoubleQuote Boolean Null colon CatchExpr keyword Block FinallyExpr keyword keyword Underscore NamedArg IfExpr keyword FunctionCall ElseIfExpr keyword ElseExpr BinOp Regex Dict Array FunctionCallWithBlock TryExpr keyword Throw keyword Import keyword CompoundAssign Assign",
maxTerm: 128,
context: trackScope,
nodeProps: [
["closedBy", 59,"end"]
],
propSources: [highlighting],
skippedNodes: [0,34],
repeatNodeCount: 13,
tokenData: "Lp~R}OX$OXY$mYp$Opq$mqr$Ors%Wst'^tu(uuw$Owx(|xy)Ryz)lz{$O{|*V|}$O}!O*V!O!P$O!P!Q3r!Q!R*w!R![-l![!]<_!]!^<x!^!}$O!}#O=c#O#P?X#P#Q?^#Q#R$O#R#S?w#S#T$O#T#Y@b#Y#ZA|#Z#b@b#b#cGj#c#f@b#f#gHm#g#h@b#h#iIp#i#o@b#o#p$O#p#qLQ#q;'S$O;'S;=`$g<%l~$O~O$O~~LkS$TU!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OS$jP;=`<%l$O^$tU!US#[YOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU%]Z!USOr%Wrs&Ost%Wtu&iuw%Wwx&ix#O%W#O#P&i#P;'S%W;'S;=`'W<%lO%WU&VU!YQ!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OQ&lTOr&irs&{s;'S&i;'S;=`'Q<%lO&iQ'QO!YQQ'TP;=`<%l&iU'ZP;=`<%l%W^'eZrY!USOY'^YZ$OZt'^tu(Wuw'^wx(Wx#O'^#O#P(W#P;'S'^;'S;=`(o<%lO'^Y(]SrYOY(WZ;'S(W;'S;=`(i<%lO(WY(lP;=`<%l(W^(rP;=`<%l'^^(|O#h[}Q~)RO#f~U)YU!US#cQOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU)sU!US#pQOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU*[X!USOt$Ouw$Ox!Q$O!Q!R*w!R![-l![#O$O#P;'S$O;'S;=`$g<%lO$OU+Ob!US|QOt$Ouw$Ox!O$O!O!P,W!P!Q$O!Q![-l![#O$O#P#R$O#R#S.i#S#U$O#U#V/W#V#c$O#c#d0l#d#l$O#l#m1z#m;'S$O;'S;=`$g<%lO$OU,]W!USOt$Ouw$Ox!Q$O!Q![,u![#O$O#P;'S$O;'S;=`$g<%lO$OU,|Y!US|QOt$Ouw$Ox!Q$O!Q![,u![#O$O#P#R$O#R#S,W#S;'S$O;'S;=`$g<%lO$OU-s[!US|QOt$Ouw$Ox!O$O!O!P,W!P!Q$O!Q![-l![#O$O#P#R$O#R#S.i#S;'S$O;'S;=`$g<%lO$OU.nW!USOt$Ouw$Ox!Q$O!Q![-l![#O$O#P;'S$O;'S;=`$g<%lO$OU/]X!USOt$Ouw$Ox!Q$O!Q!R/x!R!S/x!S#O$O#P;'S$O;'S;=`$g<%lO$OU0PX!US|QOt$Ouw$Ox!Q$O!Q!R/x!R!S/x!S#O$O#P;'S$O;'S;=`$g<%lO$OU0qW!USOt$Ouw$Ox!Q$O!Q!Y1Z!Y#O$O#P;'S$O;'S;=`$g<%lO$OU1bW!US|QOt$Ouw$Ox!Q$O!Q!Y1Z!Y#O$O#P;'S$O;'S;=`$g<%lO$OU2P[!USOt$Ouw$Ox!Q$O!Q![2u![!c$O!c!i2u!i#O$O#P#T$O#T#Z2u#Z;'S$O;'S;=`$g<%lO$OU2|[!US|QOt$Ouw$Ox!Q$O!Q![2u![!c$O!c!i2u!i#O$O#P#T$O#T#Z2u#Z;'S$O;'S;=`$g<%lO$OU3wW!USOt$Ouw$Ox!P$O!P!Q4a!Q#O$O#P;'S$O;'S;=`$g<%lO$OU4f^!USOY5bYZ$OZt5btu6euw5bwx6ex!P5b!P!Q$O!Q!}5b!}#O;W#O#P8s#P;'S5b;'S;=`<X<%lO5bU5i^!US!mQOY5bYZ$OZt5btu6euw5bwx6ex!P5b!P!Q9Y!Q!}5b!}#O;W#O#P8s#P;'S5b;'S;=`<X<%lO5bQ6jX!mQOY6eZ!P6e!P!Q7V!Q!}6e!}#O7t#O#P8s#P;'S6e;'S;=`9S<%lO6eQ7YP!P!Q7]Q7bU!mQ#Z#[7]#]#^7]#a#b7]#g#h7]#i#j7]#m#n7]Q7wVOY7tZ#O7t#O#P8^#P#Q6e#Q;'S7t;'S;=`8m<%lO7tQ8aSOY7tZ;'S7t;'S;=`8m<%lO7tQ8pP;=`<%l7tQ8vSOY6eZ;'S6e;'S;=`9S<%lO6eQ9VP;=`<%l6eU9_W!USOt$Ouw$Ox!P$O!P!Q9w!Q#O$O#P;'S$O;'S;=`$g<%lO$OU:Ob!US!mQOt$Ouw$Ox#O$O#P#Z$O#Z#[9w#[#]$O#]#^9w#^#a$O#a#b9w#b#g$O#g#h9w#h#i$O#i#j9w#j#m$O#m#n9w#n;'S$O;'S;=`$g<%lO$OU;][!USOY;WYZ$OZt;Wtu7tuw;Wwx7tx#O;W#O#P8^#P#Q5b#Q;'S;W;'S;=`<R<%lO;WU<UP;=`<%l;WU<[P;=`<%l5bU<fU!US!]QOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU=PU!US#`QOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU=jW#rQ!USOt$Ouw$Ox!_$O!_!`>S!`#O$O#P;'S$O;'S;=`$g<%lO$OU>XV!USOt$Ouw$Ox#O$O#P#Q>n#Q;'S$O;'S;=`$g<%lO$OU>uU#qQ!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$O~?^O#i~U?eU#sQ!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU@OU!US!dQOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU@g^!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#o@b#o;'S$O;'S;=`$g<%lO$OUAjU!SQ!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OUBR_!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#UCQ#U#o@b#o;'S$O;'S;=`$g<%lO$OUCV`!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#`@b#`#aDX#a#o@b#o;'S$O;'S;=`$g<%lO$OUD^`!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#g@b#g#hE`#h#o@b#o;'S$O;'S;=`$g<%lO$OUEe`!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#X@b#X#YFg#Y#o@b#o;'S$O;'S;=`$g<%lO$OUFn^!ZQ!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#o@b#o;'S$O;'S;=`$g<%lO$O^Gq^#jW!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#o@b#o;'S$O;'S;=`$g<%lO$O^Ht^#lW!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#o@b#o;'S$O;'S;=`$g<%lO$O^Iw`#kW!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#f@b#f#gJy#g#o@b#o;'S$O;'S;=`$g<%lO$OUKO`!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#i@b#i#jE`#j#o@b#o;'S$O;'S;=`$g<%lO$OULXUuQ!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$O~LpO#t~",
tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, pipeStartsLineTokenizer, new LocalTokenGroup("[~RP!O!PU~ZO#d~~", 11)],
topRules: {"Program":[0,35]},
specialized: [{term: 28, get: (value: any, stack: any) => (specializeKeyword(value, stack) << 1), external: specializeKeyword},{term: 28, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}],
tokenPrec: 2589
})

View File

@ -1,258 +0,0 @@
import { SyntaxNode } from './node'
/**
* Parse string contents into fragments, interpolations, and escape sequences.
*
* Input: full string including quotes, e.g. "'hello $name'"
* Output: SyntaxNode tree with StringFragment, Interpolation, EscapeSeq children
*/
export const parseString = (input: string, from: number, to: number, parser: any): SyntaxNode => {
const stringNode = new SyntaxNode('String', from, to)
const content = input.slice(from, to)
// Determine string type
const firstChar = content[0]
// Double-quoted strings: no interpolation or escapes
if (firstChar === '"') {
const fragment = new SyntaxNode('DoubleQuote', from, to)
stringNode.add(fragment)
return stringNode
}
// Curly strings: interpolation but no escapes
if (firstChar === '{') {
parseCurlyString(stringNode, input, from, to, parser)
return stringNode
}
// Single-quoted strings: interpolation and escapes
if (firstChar === "'") {
parseSingleQuoteString(stringNode, input, from, to, parser)
return stringNode
}
throw `Unknown string type starting with: ${firstChar}`
}
/**
* Parse single-quoted string: 'hello $name\n'
* Supports: interpolation ($var, $(expr)), escape sequences (\n, \$, etc)
*/
const parseSingleQuoteString = (stringNode: SyntaxNode, input: string, from: number, to: number, parser: any) => {
let pos = from + 1 // Skip opening '
let fragmentStart = pos
while (pos < to - 1) { // -1 to skip closing '
const char = input[pos]
// Escape sequence
if (char === '\\' && pos + 1 < to - 1) {
// Push accumulated fragment
if (pos > fragmentStart) {
const frag = new SyntaxNode('StringFragment', fragmentStart, pos)
stringNode.add(frag)
}
// Add escape sequence node
const escNode = new SyntaxNode('EscapeSeq', pos, pos + 2)
stringNode.add(escNode)
pos += 2
fragmentStart = pos
continue
}
// Interpolation
if (char === '$') {
// Push accumulated fragment
if (pos > fragmentStart) {
const frag = new SyntaxNode('StringFragment', fragmentStart, pos)
stringNode.add(frag)
}
pos++ // Skip $
// Parse interpolation content
if (input[pos] === '(') {
// Expression interpolation: $(expr)
const interpStart = pos - 1 // Include the $
const exprResult = parseInterpolationExpr(input, pos, parser)
const interpNode = new SyntaxNode('Interpolation', interpStart, exprResult.endPos)
interpNode.add(exprResult.node)
stringNode.add(interpNode)
pos = exprResult.endPos
} else {
// Variable interpolation: $name
const interpStart = pos - 1
const identEnd = findIdentifierEnd(input, pos, to - 1)
const identNode = new SyntaxNode('FunctionCallOrIdentifier', pos, identEnd)
const innerIdent = new SyntaxNode('Identifier', pos, identEnd)
identNode.add(innerIdent)
const interpNode = new SyntaxNode('Interpolation', interpStart, identEnd)
interpNode.add(identNode)
stringNode.add(interpNode)
pos = identEnd
}
fragmentStart = pos
continue
}
pos++
}
// Push final fragment
if (pos > fragmentStart && fragmentStart < to - 1) {
const frag = new SyntaxNode('StringFragment', fragmentStart, pos)
stringNode.add(frag)
}
}
/**
* Parse curly string: { hello $name }
* Supports: interpolation ($var, $(expr)), nested braces
* Does NOT support: escape sequences (raw content)
*/
const parseCurlyString = (stringNode: SyntaxNode, input: string, from: number, to: number, parser: any) => {
let pos = from + 1 // Skip opening {
let fragmentStart = from // Include the opening { in the fragment
let depth = 1
while (pos < to && depth > 0) {
const char = input[pos]
// Track brace nesting
if (char === '{') {
depth++
pos++
continue
}
if (char === '}') {
depth--
if (depth === 0) {
// Push final fragment including closing }
const frag = new SyntaxNode('CurlyString', fragmentStart, pos + 1)
stringNode.add(frag)
break
}
pos++
continue
}
// Interpolation
if (char === '$') {
// Push accumulated fragment
if (pos > fragmentStart) {
const frag = new SyntaxNode('CurlyString', fragmentStart, pos)
stringNode.add(frag)
}
pos++ // Skip $
// Parse interpolation content
if (input[pos] === '(') {
// Expression interpolation: $(expr)
const interpStart = pos - 1
const exprResult = parseInterpolationExpr(input, pos, parser)
const interpNode = new SyntaxNode('Interpolation', interpStart, exprResult.endPos)
interpNode.add(exprResult.node)
stringNode.add(interpNode)
pos = exprResult.endPos
} else {
// Variable interpolation: $name
const interpStart = pos - 1
const identEnd = findIdentifierEnd(input, pos, to)
const identNode = new SyntaxNode('FunctionCallOrIdentifier', pos, identEnd)
const innerIdent = new SyntaxNode('Identifier', pos, identEnd)
identNode.add(innerIdent)
const interpNode = new SyntaxNode('Interpolation', interpStart, identEnd)
interpNode.add(identNode)
stringNode.add(interpNode)
pos = identEnd
}
fragmentStart = pos
continue
}
pos++
}
}
/**
* Parse a parenthesized expression interpolation: $(a + b)
* Returns the parsed expression node and the position after the closing )
* pos is position of the opening ( in the full input string
*/
const parseInterpolationExpr = (input: string, pos: number, parser: any): { node: SyntaxNode, endPos: number } => {
// Find matching closing paren
let depth = 1
let start = pos
let end = pos + 1 // Start after opening (
while (end < input.length && depth > 0) {
if (input[end] === '(') depth++
if (input[end] === ')') {
depth--
if (depth === 0) break
}
end++
}
const exprContent = input.slice(start + 1, end) // Content between ( and )
const closeParen = end
end++ // Move past closing )
// Use the main parser to parse the expression
const exprNode = parser.parse(exprContent)
// Get the first real node (skip Program wrapper)
const innerNode = exprNode.firstChild || exprNode
// Adjust node positions: they're relative to exprContent, need to offset to full input
const offset = start + 1 // Position where exprContent starts in full input
adjustNodePositions(innerNode, offset)
// Wrap in ParenExpr - use positions in the full string
const parenNode = new SyntaxNode('ParenExpr', start, closeParen + 1)
parenNode.add(innerNode)
return { node: parenNode, endPos: end }
}
/**
* Recursively adjust all node positions by adding an offset
*/
const adjustNodePositions = (node: SyntaxNode, offset: number) => {
node.from += offset
node.to += offset
for (const child of node.children) {
adjustNodePositions(child, offset)
}
}
/**
* Find the end position of an identifier starting at pos
* Identifiers: lowercase letter or emoji, followed by letters/digits/dashes/emoji
*/
const findIdentifierEnd = (input: string, pos: number, maxPos: number): number => {
let end = pos
while (end < maxPos) {
const char = input[end]!
// Stop at non-identifier characters
if (!/[a-z0-9\-?]/.test(char)) {
break
}
end++
}
return end
}

View File

@ -1,5 +1,7 @@
import { expect, describe, test } from 'bun:test' import { expect, describe, test } from 'bun:test'
import '../shrimp.grammar' // Importing this so changes cause it to retest!
describe('null', () => { describe('null', () => {
test('parses null', () => { test('parses null', () => {
expect('null').toMatchTree(`Null null`) expect('null').toMatchTree(`Null null`)
@ -808,6 +810,44 @@ describe('Nullish coalescing operator', () => {
}) })
}) })
describe('DotGet whitespace sensitivity', () => {
test('no whitespace - DotGet works when identifier in scope', () => {
expect('basename = 5; basename.prop').toMatchTree(`
Assign
AssignableIdentifier basename
Eq =
Number 5
FunctionCallOrIdentifier
DotGet
IdentifierBeforeDot basename
Identifier prop`)
})
test('space before dot - NOT DotGet, parses as division', () => {
expect('basename = 5; basename / prop').toMatchTree(`
Assign
AssignableIdentifier basename
Eq =
Number 5
BinOp
Identifier basename
Slash /
Identifier prop`)
})
test('dot followed by slash is Word, not DotGet', () => {
expect('basename ./cool').toMatchTree(`
FunctionCall
Identifier basename
PositionalArg
Word ./cool`)
})
test('identifier not in scope with dot becomes Word', () => {
expect('readme.txt').toMatchTree(`Word readme.txt`)
})
})
describe('Comments', () => { describe('Comments', () => {
test('are greedy', () => { test('are greedy', () => {
expect(` expect(`
@ -857,6 +897,61 @@ basename = 5 # very astute
}) })
}) })
describe('Array destructuring', () => {
test('parses array pattern with two variables', () => {
expect('[ a b ] = [ 1 2 3 4]').toMatchTree(`
Assign
Array
Identifier a
Identifier b
Eq =
Array
Number 1
Number 2
Number 3
Number 4`)
})
test('parses array pattern with one variable', () => {
expect('[ x ] = [ 42 ]').toMatchTree(`
Assign
Array
Identifier x
Eq =
Array
Number 42`)
})
test('parses array pattern with emoji identifiers', () => {
expect('[ 🚀 💎 ] = [ 1 2 ]').toMatchTree(`
Assign
Array
Identifier 🚀
Identifier 💎
Eq =
Array
Number 1
Number 2`)
})
test('works with dotget', () => {
expect('[ a ] = [ [1 2 3] ]; a.1').toMatchTree(`
Assign
Array
Identifier a
Eq =
Array
Array
Number 1
Number 2
Number 3
FunctionCallOrIdentifier
DotGet
IdentifierBeforeDot a
Number 1`)
})
})
describe('Conditional ops', () => { describe('Conditional ops', () => {
test('or can be chained', () => { test('or can be chained', () => {
expect(` expect(`
@ -942,3 +1037,34 @@ Assign
`) `)
}) })
}) })
describe('import', () => {
test('parses single import', () => {
expect(`import str`).toMatchTree(`
Import
keyword import
Identifier str
`)
})
test('parses multiple imports', () => {
expect(`import str math list`).toMatchTree(`
Import
keyword import
Identifier str
Identifier math
Identifier list
`)
})
test('parses named args', () => {
expect(`import str only=ends-with?`).toMatchTree(`
Import
keyword import
Identifier str
NamedArg
NamedArgPrefix only=
Identifier ends-with?
`)
})
})

View File

@ -1,5 +1,7 @@
import { expect, describe, test } from 'bun:test' import { expect, describe, test } from 'bun:test'
import '../shrimp.grammar' // Importing this so changes cause it to retest!
describe('bitwise operators - grammar', () => { describe('bitwise operators - grammar', () => {
test('parses band (bitwise AND)', () => { test('parses band (bitwise AND)', () => {
expect('5 band 3').toMatchTree(` expect('5 band 3').toMatchTree(`

View File

@ -1,5 +1,7 @@
import { expect, describe, test } from 'bun:test' import { expect, describe, test } from 'bun:test'
import '../shrimp.grammar' // Importing this so changes cause it to retest!
describe('if/else if/else', () => { describe('if/else if/else', () => {
test('parses single line if', () => { test('parses single line if', () => {
expect(`if y == 1: 'cool' end`).toMatchTree(` expect(`if y == 1: 'cool' end`).toMatchTree(`
@ -22,7 +24,6 @@ describe('if/else if/else', () => {
Eq = Eq =
IfExpr IfExpr
keyword if keyword if
FunctionCallOrIdentifier
Identifier x Identifier x
colon : colon :
Block Block
@ -58,7 +59,6 @@ describe('if/else if/else', () => {
end`).toMatchTree(` end`).toMatchTree(`
IfExpr IfExpr
keyword if keyword if
FunctionCallOrIdentifier
Identifier with-else Identifier with-else
colon : colon :
Block Block
@ -82,7 +82,6 @@ describe('if/else if/else', () => {
end`).toMatchTree(` end`).toMatchTree(`
IfExpr IfExpr
keyword if keyword if
FunctionCallOrIdentifier
Identifier with-else-if Identifier with-else-if
colon : colon :
Block Block
@ -91,7 +90,6 @@ describe('if/else if/else', () => {
ElseIfExpr ElseIfExpr
keyword else keyword else
keyword if keyword if
FunctionCallOrIdentifier
Identifier another-condition Identifier another-condition
colon : colon :
Block Block
@ -113,7 +111,6 @@ describe('if/else if/else', () => {
end`).toMatchTree(` end`).toMatchTree(`
IfExpr IfExpr
keyword if keyword if
FunctionCallOrIdentifier
Identifier with-else-if-else Identifier with-else-if-else
colon : colon :
Block Block
@ -122,7 +119,6 @@ describe('if/else if/else', () => {
ElseIfExpr ElseIfExpr
keyword else keyword else
keyword if keyword if
FunctionCallOrIdentifier
Identifier another-condition Identifier another-condition
colon : colon :
Block Block
@ -131,7 +127,6 @@ describe('if/else if/else', () => {
ElseIfExpr ElseIfExpr
keyword else keyword else
keyword if keyword if
FunctionCallOrIdentifier
Identifier yet-another-condition Identifier yet-another-condition
colon : colon :
Block Block
@ -178,7 +173,7 @@ describe('if/else if/else', () => {
`) `)
}) })
test("parses paren'd function calls in if tests", () => { test('parses function calls in if tests', () => {
expect(`if (var? 'abc'): true end`).toMatchTree(` expect(`if (var? 'abc'): true end`).toMatchTree(`
IfExpr IfExpr
keyword if keyword if
@ -219,7 +214,7 @@ describe('if/else if/else', () => {
`) `)
}) })
test("parses paren'd function calls in else-if tests", () => { test('parses function calls in else-if tests', () => {
expect(`if false: true else if (var? 'abc'): true end`).toMatchTree(` expect(`if false: true else if (var? 'abc'): true end`).toMatchTree(`
IfExpr IfExpr
keyword if keyword if

View File

@ -1,56 +0,0 @@
import { expect, describe, test } from 'bun:test'
describe('Array destructuring', () => {
test('parses array pattern with two variables', () => {
expect('[ a b ] = [ 1 2 3 4]').toMatchTree(`
Assign
Array
Identifier a
Identifier b
Eq =
Array
Number 1
Number 2
Number 3
Number 4`)
})
test('parses array pattern with one variable', () => {
expect('[ x ] = [ 42 ]').toMatchTree(`
Assign
Array
Identifier x
Eq =
Array
Number 42`)
})
test('parses array pattern with emoji identifiers', () => {
expect('[ 🚀 💎 ] = [ 1 2 ]').toMatchTree(`
Assign
Array
Identifier 🚀
Identifier 💎
Eq =
Array
Number 1
Number 2`)
})
test('works with dotget', () => {
expect('[ a ] = [ [1 2 3] ]; a.1').toMatchTree(`
Assign
Array
Identifier a
Eq =
Array
Array
Number 1
Number 2
Number 3
FunctionCallOrIdentifier
DotGet
IdentifierBeforeDot a
Number 1`)
})
})

View File

@ -1,44 +1,6 @@
import { describe, test, expect } from 'bun:test' import { describe, test, expect } from 'bun:test'
import '../../testSetup' import '../../testSetup'
describe('DotGet whitespace sensitivity', () => {
test('no whitespace - DotGet works when identifier in scope', () => {
expect('basename = 5; basename.prop').toMatchTree(`
Assign
AssignableIdentifier basename
Eq =
Number 5
FunctionCallOrIdentifier
DotGet
IdentifierBeforeDot basename
Identifier prop`)
})
test('space before dot - NOT DotGet, parses as division', () => {
expect('basename = 5; basename / prop').toMatchTree(`
Assign
AssignableIdentifier basename
Eq =
Number 5
BinOp
Identifier basename
Slash /
Identifier prop`)
})
test('dot followed by slash is Word, not DotGet', () => {
expect('basename ./cool').toMatchTree(`
FunctionCall
Identifier basename
PositionalArg
Word ./cool`)
})
test('identifier not in scope with dot becomes Word', () => {
expect('readme.txt').toMatchTree(`Word readme.txt`)
})
})
describe('DotGet', () => { describe('DotGet', () => {
test('readme.txt is Word when readme not in scope', () => { test('readme.txt is Word when readme not in scope', () => {
expect('readme.txt').toMatchTree(`Word readme.txt`) expect('readme.txt').toMatchTree(`Word readme.txt`)
@ -237,7 +199,7 @@ end`).toMatchTree(`
`) `)
}) })
test.skip("dot get doesn't work with spaces", () => { test("dot get doesn't work with spaces", () => {
expect('obj . prop').toMatchTree(` expect('obj . prop').toMatchTree(`
FunctionCall FunctionCall
Identifier obj Identifier obj

View File

@ -1,5 +1,7 @@
import { expect, describe, test } from 'bun:test' import { expect, describe, test } from 'bun:test'
import '../shrimp.grammar' // Importing this so changes cause it to retest!
describe('try/catch/finally/throw', () => { describe('try/catch/finally/throw', () => {
test('parses try with catch', () => { test('parses try with catch', () => {
expect(`try: expect(`try:
@ -137,23 +139,10 @@ describe('try/catch/finally/throw', () => {
`) `)
}) })
test('parses throw statement with BinOp', () => {
expect("throw 'error message:' + msg").toMatchTree(`
Throw
keyword throw
BinOp
String
StringFragment error message:
Plus +
Identifier msg
`)
})
test('parses throw statement with identifier', () => { test('parses throw statement with identifier', () => {
expect('throw error-object').toMatchTree(` expect('throw error-object').toMatchTree(`
Throw Throw
keyword throw keyword throw
FunctionCallOrIdentifier
Identifier error-object Identifier error-object
`) `)
}) })

View File

@ -1,5 +1,7 @@
import { expect, describe, test } from 'bun:test' import { expect, describe, test } from 'bun:test'
import '../shrimp.grammar' // Importing this so changes cause it to retest!
describe('single line function blocks', () => { describe('single line function blocks', () => {
test('work with no args', () => { test('work with no args', () => {
expect(`trap: echo bye bye end`).toMatchTree(` expect(`trap: echo bye bye end`).toMatchTree(`

View File

@ -1,5 +1,7 @@
import { expect, describe, test } from 'bun:test' import { expect, describe, test } from 'bun:test'
import '../shrimp.grammar' // Importing this so changes cause it to retest!
describe('calling functions', () => { describe('calling functions', () => {
test('call with no args', () => { test('call with no args', () => {
expect('tail').toMatchTree(` expect('tail').toMatchTree(`
@ -41,58 +43,6 @@ describe('calling functions', () => {
`) `)
}) })
test('call with function', () => {
expect(`tail do x: x end`).toMatchTree(`
FunctionCall
Identifier tail
PositionalArg
FunctionDef
Do do
Params
Identifier x
colon :
FunctionCallOrIdentifier
Identifier x
keyword end
`)
})
test('call with arg and function', () => {
expect(`tail true do x: x end`).toMatchTree(`
FunctionCall
Identifier tail
PositionalArg
Boolean true
PositionalArg
FunctionDef
Do do
Params
Identifier x
colon :
FunctionCallOrIdentifier
Identifier x
keyword end
`)
})
test('call with function in named arg', () => {
expect(`tail callback=do x: x end`).toMatchTree(`
FunctionCall
Identifier tail
NamedArg
NamedArgPrefix callback=
FunctionDef
Do do
Params
Identifier x
colon :
FunctionCallOrIdentifier
Identifier x
keyword end
`)
})
test('command with arg that is also a command', () => { test('command with arg that is also a command', () => {
expect('tail tail').toMatchTree(` expect('tail tail').toMatchTree(`
FunctionCall FunctionCall

View File

@ -1,32 +0,0 @@
import { expect, describe, test } from 'bun:test'
describe('import', () => {
test('parses single import', () => {
expect(`import str`).toMatchTree(`
Import
keyword import
Identifier str
`)
})
test('parses multiple imports', () => {
expect(`import str math list`).toMatchTree(`
Import
keyword import
Identifier str
Identifier math
Identifier list
`)
})
test('parses named args', () => {
expect(`import str only=ends-with?`).toMatchTree(`
Import
keyword import
Identifier str
NamedArg
NamedArgPrefix only=
Identifier ends-with?
`)
})
})

View File

@ -1,5 +1,7 @@
import { expect, describe, test } from 'bun:test' import { expect, describe, test } from 'bun:test'
import '../shrimp.grammar' // Importing this so changes cause it to retest!
describe('number literals', () => { describe('number literals', () => {
test('binary numbers', () => { test('binary numbers', () => {
expect('0b110').toMatchTree(` expect('0b110').toMatchTree(`
@ -334,22 +336,6 @@ describe('dict literals', () => {
`) `)
}) })
test('work with functions', () => {
expect(`[trap=do x: x end]`).toMatchTree(`
Dict
NamedArg
NamedArgPrefix trap=
FunctionDef
Do do
Params
Identifier x
colon :
FunctionCallOrIdentifier
Identifier x
keyword end
`)
})
test('can be nested', () => { test('can be nested', () => {
expect('[a=one b=[two [c=three]]]').toMatchTree(` expect('[a=one b=[two [c=three]]]').toMatchTree(`
Dict Dict
@ -385,35 +371,14 @@ describe('dict literals', () => {
Number 3 Number 3
`) `)
}) })
test('can have spaces between equals', () => {
expect(`[
a = 1
b = 2
c = 3
]`).toMatchTree(`
Dict
NamedArg
NamedArgPrefix a =
Number 1
NamedArg
NamedArgPrefix b =
Number 2
NamedArg
NamedArgPrefix c =
Number 3
`)
})
test('empty dict', () => { test('empty dict', () => {
expect('[=]').toMatchTree(` expect('[=]').toMatchTree(`
Dict [=] Dict [=]
`) `)
})
test('empty dict w whitespace', () => {
expect('[ = ]').toMatchTree(` expect('[ = ]').toMatchTree(`
Dict [ = ] Array
Word =
`) `)
}) })

View File

@ -1,5 +1,7 @@
import { expect, describe, test } from 'bun:test' import { expect, describe, test } from 'bun:test'
import '../shrimp.grammar' // Importing this so changes cause it to retest!
describe('multiline', () => { describe('multiline', () => {
test('parses multiline strings', () => { test('parses multiline strings', () => {
expect(`'first'\n'second'`).toMatchTree(` expect(`'first'\n'second'`).toMatchTree(`

View File

@ -1,4 +1,7 @@
import { expect, describe, test } from 'bun:test' import { expect, describe, test } from 'bun:test'
import { parser } from '../shrimp'
import '../shrimp.grammar' // Importing this so changes cause it to retest!
describe('pipe expressions', () => { describe('pipe expressions', () => {
test('simple pipe expression', () => { test('simple pipe expression', () => {
@ -173,43 +176,6 @@ describe('pipe expressions', () => {
Identifier echo Identifier echo
`) `)
}) })
test('parenthesized expressions can be piped', () => {
expect(`(1 + 2) | echo`).toMatchTree(`
PipeExpr
ParenExpr
BinOp
Number 1
Plus +
Number 2
operator |
FunctionCallOrIdentifier
Identifier echo
`)
})
test('complex parenthesized expressions with pipes', () => {
expect(`((math.random) * 10 + 1) | math.floor`).toMatchTree(`
PipeExpr
ParenExpr
BinOp
BinOp
ParenExpr
FunctionCallOrIdentifier
DotGet
IdentifierBeforeDot math
Identifier random
Star *
Number 10
Plus +
Number 1
operator |
FunctionCallOrIdentifier
DotGet
IdentifierBeforeDot math
Identifier floor
`)
})
}) })
describe('pipe continuation', () => { describe('pipe continuation', () => {
@ -343,7 +309,7 @@ grep h`).toMatchTree(`
Identifier split Identifier split
PositionalArg PositionalArg
String String
StringFragment (space) StringFragment
operator | operator |
FunctionCall FunctionCall
Identifier map Identifier map
@ -367,41 +333,3 @@ grep h`).toMatchTree(`
`) `)
}) })
}) })
describe('Underscore', () => {
test('works in pipes', () => {
expect(`sub 3 1 | div (sub 110 9 | sub 1) _ | div 5`).toMatchTree(`
PipeExpr
FunctionCall
Identifier sub
PositionalArg
Number 3
PositionalArg
Number 1
operator |
FunctionCall
Identifier div
PositionalArg
ParenExpr
PipeExpr
FunctionCall
Identifier sub
PositionalArg
Number 110
PositionalArg
Number 9
operator |
FunctionCall
Identifier sub
PositionalArg
Number 1
PositionalArg
Underscore _
operator |
FunctionCall
Identifier div
PositionalArg
Number 5
`)
})
})

View File

@ -1,5 +1,7 @@
import { expect, describe, test } from 'bun:test' import { expect, describe, test } from 'bun:test'
import '../shrimp.grammar' // Importing this so changes cause it to retest!
describe('string interpolation', () => { describe('string interpolation', () => {
test('string with variable interpolation', () => { test('string with variable interpolation', () => {
expect("'hello $name'").toMatchTree(` expect("'hello $name'").toMatchTree(`

View File

@ -1,750 +0,0 @@
import { expect, describe, test } from 'bun:test'
describe('constant types', () => {
test('null', () => {
expect(`null`).toBeToken('Null')
})
test('boolean', () => {
expect(`true`).toMatchToken('Boolean', 'true')
expect(`false`).toMatchToken('Boolean', 'false')
})
})
describe('numbers', () => {
test('non-numbers', () => {
expect(`1st`).toMatchToken('Word', '1st')
expect(`1_`).toMatchToken('Word', '1_')
expect(`100.`).toMatchTokens(
{ type: 'Number', value: '100' },
{ type: 'Operator', value: '.' },
)
})
test('simple numbers', () => {
expect(`1`).toMatchToken('Number', '1')
expect(`200`).toMatchToken('Number', '200')
expect(`5.20`).toMatchToken('Number', '5.20')
expect(`0.20`).toMatchToken('Number', '0.20')
expect(`-20`).toMatchToken('Number', '-20')
expect(`+20`).toMatchToken('Number', '+20')
expect(`-2134.34`).toMatchToken('Number', '-2134.34')
expect(`+20.5325`).toMatchToken('Number', '+20.5325')
expect(`1_000`).toMatchToken('Number', '1_000')
expect(`53_232_220`).toMatchToken('Number', '53_232_220')
})
test('binary numbers', () => {
expect('0b110').toMatchToken('Number', '0b110')
})
test('hex numbers', () => {
expect('0xdeadbeef').toMatchToken('Number', '0xdeadbeef')
expect('0x02d3f4').toMatchToken('Number', '0x02d3f4')
})
test('hex numbers uppercase', () => {
expect('0xFF').toMatchToken('Number', '0xFF')
})
test('octal numbers', () => {
expect('0o644').toMatchToken('Number', '0o644')
expect('0o055').toMatchToken('Number', '0o055')
})
test('negative binary', () => {
expect('-0b110').toMatchToken('Number', '-0b110')
})
test('negative hex', () => {
expect('-0xFF').toMatchToken('Number', '-0xFF')
})
test('negative octal', () => {
expect('-0o755').toMatchToken('Number', '-0o755')
})
test('positive prefix binary', () => {
expect('+0b110').toMatchToken('Number', '+0b110')
})
test('positive prefix hex', () => {
expect('+0xFF').toMatchToken('Number', '+0xFF')
})
test('positive prefix octal', () => {
expect('+0o644').toMatchToken('Number', '+0o644')
})
test('underscores in number', () => {
expect(`1_000`).toMatchToken('Number', '1_000')
expect(`1_0`).toMatchToken('Number', '1_0')
expect('0b11_0').toMatchToken('Number', '0b11_0')
expect('0xdead_beef').toMatchToken('Number', '0xdead_beef')
expect('0o64_4').toMatchToken('Number', '0o64_4')
})
})
describe('identifiers', () => {
test('regular', () => {
expect('name').toBeToken('Identifier')
expect('bobby-mcgee').toBeToken('Identifier')
expect('starts-with?').toBeToken('Identifier')
expect('📢').toMatchToken('Identifier', '📢')
expect(' 📢 ').toMatchToken('Identifier', '📢')
expect(' oink-🐷-oink').toMatchToken('Identifier', 'oink-🐷-oink')
expect('$').toMatchToken('Identifier', '$')
expect('$cool').toMatchToken('Identifier', '$cool')
})
test('one character identifiers', () => {
expect('a').toMatchToken('Identifier', 'a')
expect('z').toMatchToken('Identifier', 'z')
expect('$').toMatchToken('Identifier', '$')
expect('📢').toMatchToken('Identifier', '📢')
expect('?').toBeToken('Word') // ? alone is not valid identifier start
})
test('two character identifiers', () => {
expect('ab').toMatchToken('Identifier', 'ab')
expect('a1').toMatchToken('Identifier', 'a1')
expect('a-').toMatchToken('Identifier', 'a-')
expect('a?').toMatchToken('Identifier', 'a?') // ? valid at end
expect('ab?').toMatchToken('Identifier', 'ab?')
})
test('three+ character identifiers', () => {
expect('abc').toMatchToken('Identifier', 'abc')
expect('a-b').toMatchToken('Identifier', 'a-b')
expect('a1b').toMatchToken('Identifier', 'a1b')
expect('abc?').toMatchToken('Identifier', 'abc?') // ? valid at end
expect('a-b-c?').toMatchToken('Identifier', 'a-b-c?')
})
test('edge cases', () => {
expect('-bobby-mcgee').toBeToken('Word')
expect('starts-with??').toMatchToken('Identifier', 'starts-with??')
expect('starts?with?').toMatchToken('Identifier', 'starts?with?')
expect('a??b').toMatchToken('Identifier', 'a??b')
expect('oink-oink!').toBeToken('Word')
expect('dog#pound').toMatchToken('Word', 'dog#pound')
expect('http://website.com').toMatchToken('Word', 'http://website.com')
expect('school$cool').toMatchToken('Identifier', 'school$cool')
expect('EXIT:').toMatchTokens(
{ type: 'Word', value: 'EXIT' },
{ type: 'Colon' },
)
expect(`if y == 1: 'cool' end`).toMatchTokens(
{ type: 'Keyword', value: 'if' },
{ type: 'Identifier', value: 'y' },
{ type: 'Operator', value: '==' },
{ type: 'Number', value: '1' },
{ type: 'Colon' },
{ type: 'String', value: `'cool'` },
{ type: 'Keyword', value: 'end' },
)
})
})
describe('paths', () => {
test('starting with ./', () => {
expect('./tmp').toMatchToken('Word', './tmp')
})
test('starting with /', () => {
expect('/home/chris/dev').toMatchToken('Word', '/home/chris/dev')
})
test('identifiers with dots tokenize separately', () => {
expect('readme.txt').toMatchTokens(
{ type: 'Identifier', value: 'readme' },
{ type: 'Operator', value: '.' },
{ type: 'Identifier', value: 'txt' },
)
})
test('words (non-identifiers) consume dots', () => {
expect('README.md').toMatchToken('Word', 'README.md')
})
test('all sorts of weird stuff', () => {
expect('dog#pound').toMatchToken('Word', 'dog#pound')
expect('my/kinda/place').toMatchToken('my/kinda/place')
expect('file://%/$##/@40!/index.php').toMatchToken('Word', 'file://%/$##/@40!/index.php')
})
})
describe('strings', () => {
test('single quoted', () => {
expect(`'hello world'`).toMatchToken('String', `'hello world'`)
expect(`'it\\'s a beautiful world'`).toMatchToken("'it\\'s a beautiful world'")
})
test('double quoted', () => {
expect(`"hello world"`).toMatchToken('String', `"hello world"`)
expect(`"it's a beautiful world"`).toMatchToken('String', `"it's a beautiful world"`)
})
test('empty strings', () => {
expect(`''`).toMatchToken('String', `''`)
expect(`""`).toMatchToken('String', `""`)
})
test('escape sequences', () => {
expect(`'hello\\nworld'`).toMatchToken('String', `'hello\\nworld'`)
expect(`'tab\\there'`).toMatchToken('String', `'tab\\there'`)
expect(`'quote\\''`).toMatchToken('String', `'quote\\''`)
expect(`'backslash\\\\'`).toMatchToken('String', `'backslash\\\\'`)
expect(`'dollar\\$sign'`).toMatchToken('String', `'dollar\\$sign'`)
})
test('unclosed strings - error case', () => {
// These should either fail or produce unexpected results
expect(`'hello`).toMatchToken('String', `'hello`)
expect(`"world`).toMatchToken('String', `"world`)
})
})
describe('curly strings', () => {
test('curly quoted', () => {
expect('{ one two three }').toMatchToken('String', `{ one two three }`)
})
test('work on multiple lines', () => {
expect(`{
one
two
three }`).toMatchToken('String', `{
one
two
three }`)
})
test('can contain other curlies', () => {
expect(`{ { one }
two
{ three } }`).toMatchToken('String', `{ { one }
two
{ three } }`)
})
test('empty curly string', () => {
expect('{}').toMatchToken('String', '{}')
})
test('unclosed curly string - error case', () => {
// Should either fail or produce unexpected results
expect('{ hello').toMatchToken('String', '{ hello')
expect('{ nested { unclosed }').toMatchToken('String', '{ nested { unclosed }')
})
})
describe('operators', () => {
test('math operators', () => {
// assignment
expect('=').toMatchToken('Operator', '=')
// logic
expect('or').toMatchToken('Operator', 'or')
expect('and').toMatchToken('Operator', 'and')
// bitwise
expect('band').toMatchToken('Operator', 'band')
expect('bor').toMatchToken('Operator', 'bor')
expect('bxor').toMatchToken('Operator', 'bxor')
expect('>>>').toMatchToken('Operator', '>>>')
expect('>>').toMatchToken('Operator', '>>')
expect('<<').toMatchToken('Operator', '<<')
// compound assignment
expect('??=').toMatchToken('Operator', '??=')
expect('+=').toMatchToken('Operator', '+=')
expect('-=').toMatchToken('Operator', '-=')
expect('*=').toMatchToken('Operator', '*=')
expect('/=').toMatchToken('Operator', '/=')
expect('%=').toMatchToken('Operator', '%=')
// nullish
expect('??').toMatchToken('Operator', '??')
// math
expect('**').toMatchToken('Operator', '**')
expect('*').toMatchToken('Operator', '*')
expect('/').toMatchToken('Operator', '/')
expect('+').toMatchToken('Operator', '+')
expect('-').toMatchToken('Operator', '-')
expect('%').toMatchToken('Operator', '%')
// comparison
expect('>=').toMatchToken('Operator', '>=')
expect('<=').toMatchToken('Operator', '<=')
expect('!=').toMatchToken('Operator', '!=')
expect('==').toMatchToken('Operator', '==')
expect('>').toMatchToken('Operator', '>')
expect('<').toMatchToken('Operator', '<')
// property access
expect('.').toMatchToken('Operator', '.')
})
})
describe('keywords', () => {
test('keywords', () => {
expect(`import`).toMatchToken('Keyword', 'import')
expect(`end`).toMatchToken('Keyword', 'end')
expect(`do`).toMatchToken('Keyword', 'do')
expect(`while`).toMatchToken('Keyword', 'while')
expect(`if`).toMatchToken('Keyword', 'if')
expect(`else`).toMatchToken('Keyword', 'else')
expect(`try`).toMatchToken('Keyword', 'try')
expect(`catch`).toMatchToken('Keyword', 'catch')
expect(`finally`).toMatchToken('Keyword', 'finally')
expect(`throw`).toMatchToken('Keyword', 'throw')
expect(`not`).toMatchToken('Keyword', 'not')
})
})
describe('regex', () => {
test('use double slash', () => {
expect(`//[0-9]+//`).toMatchToken('Regex', '//[0-9]+//')
})
})
describe('punctuation', () => {
test('underscore', () => {
expect(`_`).toBeToken('Underscore')
expect(`__`).toMatchToken('Word', '__')
})
test('semicolon', () => {
expect(`;`).toBeToken('Semicolon')
})
test('newline', () => {
expect('\n').toBeToken('Newline')
})
test('colon', () => {
expect(':').toBeToken('Colon')
})
})
describe('comments', () => {
test('comments', () => {
expect(`# hey friends`).toMatchToken('Comment', '# hey friends')
expect(`#hey-friends`).toMatchToken('Comment', '#hey-friends')
})
})
describe('brackets', () => {
test('parens', () => {
expect(`(`).toBeToken('OpenParen')
expect(`)`).toBeToken('CloseParen')
})
test('staples', () => {
expect(`[`).toBeToken('OpenBracket')
expect(`]`).toBeToken('CloseBracket')
})
})
describe('multiple tokens', () => {
test('constants work fine', () => {
expect(`null true false`).toMatchTokens(
{ type: 'Null' },
{ type: 'Boolean', value: 'true' },
{ type: 'Boolean', value: 'false' },
)
})
test('numbers', () => {
expect(`100 -400.42 null`).toMatchTokens(
{ type: 'Number', value: '100' },
{ type: 'Number', value: '-400.42' },
{ type: 'Null' },
)
})
test('whitespace', () => {
expect(`
'hello world'
'goodbye world'
`).toMatchTokens(
{ type: 'Newline' },
{ type: 'String', value: "'hello world'" },
{ type: 'Newline' },
{ type: 'Newline' },
{ type: 'String', value: "'goodbye world'" },
{ type: 'Newline' },
)
})
test('newline in parens is ignored', () => {
expect(`(
'hello world'
'goodbye world'
)`).toMatchTokens(
{ type: 'OpenParen' },
{ type: 'String', value: "'hello world'" },
{ type: 'String', value: "'goodbye world'" },
{ type: 'CloseParen' },
)
})
test('newline in brackets is ignored', () => {
expect(`[
a b
c d
e
f
]`).toMatchTokens(
{ type: 'OpenBracket' },
{ type: 'Identifier', value: "a" },
{ type: 'Identifier', value: "b" },
{ type: 'Identifier', value: "c" },
{ type: 'Identifier', value: "d" },
{ type: 'Identifier', value: "e" },
{ type: 'Identifier', value: "f" },
{ type: 'CloseBracket' },
)
})
test('function call', () => {
expect('echo hello world').toMatchTokens(
{ type: 'Identifier', value: 'echo' },
{ type: 'Identifier', value: 'hello' },
{ type: 'Identifier', value: 'world' },
)
})
test('function call w/ parens', () => {
expect('echo(bold hello world)').toMatchTokens(
{ type: 'Identifier', value: 'echo' },
{ type: 'OpenParen' },
{ type: 'Identifier', value: 'bold' },
{ type: 'Identifier', value: 'hello' },
{ type: 'Identifier', value: 'world' },
{ type: 'CloseParen' },
)
expect('echo (bold hello world)').toMatchTokens(
{ type: 'Identifier', value: 'echo' },
{ type: 'OpenParen' },
{ type: 'Identifier', value: 'bold' },
{ type: 'Identifier', value: 'hello' },
{ type: 'Identifier', value: 'world' },
{ type: 'CloseParen' },
)
})
test('assignment', () => {
expect('x = 5').toMatchTokens(
{ type: 'Identifier', value: 'x' },
{ type: 'Operator', value: '=' },
{ type: 'Number', value: '5' },
)
})
test('math expression', () => {
expect('1 + 2 * 3').toMatchTokens(
{ type: 'Number', value: '1' },
{ type: 'Operator', value: '+' },
{ type: 'Number', value: '2' },
{ type: 'Operator', value: '*' },
{ type: 'Number', value: '3' },
)
})
test('inline comment', () => {
expect('x = 5 # set x').toMatchTokens(
{ type: 'Identifier', value: 'x' },
{ type: 'Operator', value: '=' },
{ type: 'Number', value: '5' },
{ type: 'Comment', value: '# set x' },
)
})
test('line comment', () => {
expect('x = 5 \n# hello\n set x').toMatchTokens(
{ type: 'Identifier', value: 'x' },
{ type: 'Operator', value: '=' },
{ type: 'Number', value: '5' },
{ type: 'Newline' },
{ type: 'Comment', value: '# hello' },
{ type: 'Newline' },
{ type: 'Identifier', value: 'set' },
{ type: 'Identifier', value: 'x' },
)
})
test('colons separate tokens', () => {
expect('x do: y').toMatchTokens(
{ type: 'Identifier', value: 'x' },
{ type: 'Keyword', value: 'do' },
{ type: 'Colon' },
{ type: 'Identifier', value: 'y' },
)
expect('x: y').toMatchTokens(
{ type: 'Identifier', value: 'x' },
{ type: 'Colon' },
{ type: 'Identifier', value: 'y' },
)
expect('5: y').toMatchTokens(
{ type: 'Number', value: '5' },
{ type: 'Colon' },
{ type: 'Identifier', value: 'y' },
)
expect(`if (var? 'abc'): y`).toMatchTokens(
{ type: 'Keyword', value: 'if' },
{ type: 'OpenParen' },
{ type: 'Identifier', value: 'var?' },
{ type: 'String', value: `'abc'` },
{ type: 'CloseParen' },
{ type: 'Colon' },
{ type: 'Identifier', value: 'y' },
)
expect(`
do x:
y
end`).toMatchTokens(
{ type: 'Newline' },
{ type: 'Keyword', value: 'do' },
{ type: 'Identifier', value: 'x' },
{ type: 'Colon' },
{ type: 'Newline' },
{ type: 'Identifier', value: 'y' },
{ type: 'Newline' },
{ type: 'Keyword', value: 'end' },
)
})
test('semicolons separate statements', () => {
expect('x; y').toMatchTokens(
{ type: 'Identifier', value: 'x' },
{ type: 'Semicolon' },
{ type: 'Identifier', value: 'y' },
)
})
test('semicolons in parens', () => {
expect('(x; y)').toMatchTokens(
{ type: 'OpenParen' },
{ type: 'Identifier', value: 'x' },
{ type: 'Semicolon' },
{ type: 'Identifier', value: 'y' },
{ type: 'CloseParen' },
)
})
test('dot operator beginning word with slash', () => {
expect(`(basename ./cool)`).toMatchTokens(
{ 'type': 'OpenParen' },
{ 'type': 'Identifier', 'value': 'basename' },
{ 'type': 'Word', 'value': './cool' },
{ 'type': 'CloseParen' }
)
})
test('dot word after identifier with space', () => {
expect(`expand-path .git`).toMatchTokens(
{ 'type': 'Identifier', 'value': 'expand-path' },
{ 'type': 'Word', 'value': '.git' },
)
})
test('dot operator after identifier without space', () => {
expect(`config.path`).toMatchTokens(
{ 'type': 'Identifier', 'value': 'config' },
{ 'type': 'Operator', 'value': '.' },
{ 'type': 'Identifier', 'value': 'path' },
)
})
})
describe('nesting edge cases', () => {
test('deeply nested parens', () => {
expect('((nested))').toMatchTokens(
{ type: 'OpenParen' },
{ type: 'OpenParen' },
{ type: 'Identifier', value: 'nested' },
{ type: 'CloseParen' },
{ type: 'CloseParen' },
)
})
test('mixed nesting', () => {
expect('([combo])').toMatchTokens(
{ type: 'OpenParen' },
{ type: 'OpenBracket' },
{ type: 'Identifier', value: 'combo' },
{ type: 'CloseBracket' },
{ type: 'CloseParen' },
)
})
})
describe('invalid numbers that should be words', () => {
test('invalid binary', () => {
expect('0b2').toMatchToken('Word', '0b2')
expect('0b123').toMatchToken('Word', '0b123')
})
test('invalid octal', () => {
expect('0o8').toMatchToken('Word', '0o8')
expect('0o999').toMatchToken('Word', '0o999')
})
test('invalid hex', () => {
expect('0xGGG').toMatchToken('Word', '0xGGG')
expect('0xZZZ').toMatchToken('Word', '0xZZZ')
})
test('multiple decimal points', () => {
expect('1.2.3').toMatchToken('Word', '1.2.3')
})
})
describe('unicode and emoji', () => {
test('greek letters', () => {
expect('αβγ').toMatchToken('Identifier', 'αβγ')
expect('delta-δ').toMatchToken('Identifier', 'delta-δ')
})
test('math symbols', () => {
expect('∑').toMatchToken('Identifier', '∑')
expect('∏').toMatchToken('Identifier', '∏')
})
test('CJK characters', () => {
expect('你好').toMatchToken('Identifier', '你好')
expect('こんにちは').toMatchToken('Identifier', 'こんにちは')
})
})
describe('empty and whitespace input', () => {
test('empty string', () => {
expect('').toMatchTokens()
})
test('only whitespace', () => {
expect(' ').toMatchTokens()
})
test('only tabs', () => {
expect('\t\t\t').toMatchTokens()
})
test('only newlines', () => {
expect('\n\n\n').toMatchTokens(
{ type: 'Newline' },
{ type: 'Newline' },
{ type: 'Newline' },
)
})
})
describe('named args', () => {
test("don't need spaces", () => {
expect(`named=arg`).toMatchTokens(
{ type: 'NamedArgPrefix', value: 'named=' },
{ type: 'Identifier', value: 'arg' },
)
})
test("can have spaces", () => {
expect(`named= arg`).toMatchTokens(
{ type: 'NamedArgPrefix', value: 'named=' },
{ type: 'Identifier', value: 'arg' },
)
})
test("can include numbers", () => {
expect(`named123= arg`).toMatchTokens(
{ type: 'NamedArgPrefix', value: 'named123=' },
{ type: 'Identifier', value: 'arg' },
)
})
})
describe('dot operator', () => {
test('standalone dot', () => {
expect('.').toMatchToken('Operator', '.')
})
test('dot between identifiers tokenizes as separate tokens', () => {
expect('config.path').toMatchTokens(
{ type: 'Identifier', value: 'config' },
{ type: 'Operator', value: '.' },
{ type: 'Identifier', value: 'path' },
)
})
test('dot with number', () => {
expect('array.0').toMatchTokens(
{ type: 'Identifier', value: 'array' },
{ type: 'Operator', value: '.' },
{ type: 'Number', value: '0' },
)
})
test('chained dots', () => {
expect('a.b.c').toMatchTokens(
{ type: 'Identifier', value: 'a' },
{ type: 'Operator', value: '.' },
{ type: 'Identifier', value: 'b' },
{ type: 'Operator', value: '.' },
{ type: 'Identifier', value: 'c' },
)
})
test('identifier-like paths tokenize separately', () => {
expect('readme.txt').toMatchTokens(
{ type: 'Identifier', value: 'readme' },
{ type: 'Operator', value: '.' },
{ type: 'Identifier', value: 'txt' },
)
})
test('word-like paths remain as single token', () => {
expect('./file.txt').toMatchToken('Word', './file.txt')
expect('README.TXT').toMatchToken('Word', 'README.TXT')
})
test('dot with paren expression', () => {
expect('obj.(1 + 2)').toMatchTokens(
{ type: 'Identifier', value: 'obj' },
{ type: 'Operator', value: '.' },
{ type: 'OpenParen' },
{ type: 'Number', value: '1' },
{ type: 'Operator', value: '+' },
{ type: 'Number', value: '2' },
{ type: 'CloseParen' },
)
})
test('chained dot with paren expression', () => {
expect('obj.items.(i)').toMatchTokens(
{ type: 'Identifier', value: 'obj' },
{ type: 'Operator', value: '.' },
{ type: 'Identifier', value: 'items' },
{ type: 'Operator', value: '.' },
{ type: 'OpenParen' },
{ type: 'Identifier', value: 'i' },
{ type: 'CloseParen' },
)
})
})

389
src/parser/tokenizer.ts Normal file
View File

@ -0,0 +1,389 @@
import { ExternalTokenizer, InputStream, Stack } from '@lezer/lr'
import { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, Do, CurlyString, DotGet, newline, pipeStartsLine } from './shrimp.terms'
// doobie doobie do (we need the `do` keyword to know when we're defining params)
export function specializeKeyword(ident: string) {
return ident === 'do' ? Do : -1
}
// tell the dotGet searcher about builtin globals
export const globals: string[] = []
export const setGlobals = (newGlobals: string[] | Record<string, any>) => {
globals.length = 0
globals.push(...(Array.isArray(newGlobals) ? newGlobals : Object.keys(newGlobals)))
}
// The only chars that can't be words are whitespace, apostrophes, closing parens, and EOF.
export const tokenizer = new ExternalTokenizer(
(input: InputStream, stack: Stack) => {
const ch = getFullCodePoint(input, 0)
// Handle curly strings
if (ch === 123 /* { */) return consumeCurlyString(input, stack)
if (!isWordChar(ch)) return
// Don't consume things that start with digits - let Number token handle it
if (isDigit(ch)) return
// Don't consume things that start with - or + followed by a digit (negative/positive numbers)
if ((ch === 45 /* - */ || ch === 43) /* + */ && isDigit(input.peek(1))) return
const isValidStart = isIdentStart(ch)
const canBeWord = stack.canShift(Word)
// Consume all word characters, tracking if it remains a valid identifier
const { pos, isValidIdentifier, stoppedAtDot } = consumeWordToken(
input,
isValidStart,
canBeWord
)
// Check if we should emit IdentifierBeforeDot for property access
if (stoppedAtDot) {
const dotGetToken = checkForDotGet(input, stack, pos)
if (dotGetToken) {
input.advance(pos)
input.acceptToken(dotGetToken)
} else {
// Not in scope - continue consuming the dot as part of the word
const afterDot = consumeRestOfWord(input, pos + 1, canBeWord)
input.advance(afterDot)
input.acceptToken(Word)
}
return
}
// Advance past the token we consumed
input.advance(pos)
// Choose which token to emit
if (isValidIdentifier) {
const token = chooseIdentifierToken(input, stack)
input.acceptToken(token)
} else {
input.acceptToken(Word)
}
},
{ contextual: true }
)
// Build identifier text from input stream, handling surrogate pairs for emoji
const buildIdentifierText = (input: InputStream, length: number): string => {
let text = ''
for (let i = 0; i < length; i++) {
const charCode = input.peek(i)
if (charCode === -1) break
// Handle surrogate pairs for emoji (UTF-16 encoding)
if (charCode >= 0xd800 && charCode <= 0xdbff && i + 1 < length) {
const low = input.peek(i + 1)
if (low >= 0xdc00 && low <= 0xdfff) {
text += String.fromCharCode(charCode, low)
i++ // Skip the low surrogate
continue
}
}
text += String.fromCharCode(charCode)
}
return text
}
// Consume word characters, tracking if it remains a valid identifier
// Returns the position after consuming, whether it's a valid identifier, and if we stopped at a dot
const consumeWordToken = (
input: InputStream,
isValidStart: boolean,
canBeWord: boolean
): { pos: number; isValidIdentifier: boolean; stoppedAtDot: boolean } => {
let pos = getCharSize(getFullCodePoint(input, 0))
let isValidIdentifier = isValidStart
let stoppedAtDot = false
while (true) {
const ch = getFullCodePoint(input, pos)
// Stop at dot if we have a valid identifier (might be property access)
if (ch === 46 /* . */ && isValidIdentifier) {
stoppedAtDot = true
break
}
// Stop if we hit a non-word character
if (!isWordChar(ch)) break
// Context-aware termination: semicolon/colon can end a word if followed by whitespace
// This allows `hello; 2` to parse correctly while `hello;world` stays as one word
if (canBeWord && (ch === 59 /* ; */ || ch === 58) /* : */) {
const nextCh = getFullCodePoint(input, pos + 1)
if (!isWordChar(nextCh)) break
}
// Track identifier validity: must be lowercase, digit, dash, or emoji/unicode
if (!isIdentChar(ch)) {
if (!canBeWord) break
isValidIdentifier = false
}
pos += getCharSize(ch)
}
return { pos, isValidIdentifier, stoppedAtDot }
}
// Consume the rest of a word after we've decided not to treat a dot as DotGet
// Used when we have "file.txt" - we already consumed "file", now consume ".txt"
const consumeRestOfWord = (input: InputStream, startPos: number, canBeWord: boolean): number => {
let pos = startPos
while (true) {
const ch = getFullCodePoint(input, pos)
// Stop if we hit a non-word character
if (!isWordChar(ch)) break
// Context-aware termination for semicolon/colon
if (canBeWord && (ch === 59 /* ; */ || ch === 58) /* : */) {
const nextCh = getFullCodePoint(input, pos + 1)
if (!isWordChar(nextCh)) break
}
pos += getCharSize(ch)
}
return pos
}
// Consumes { curly strings } and tracks braces so you can { have { braces { inside { braces } } }
const consumeCurlyString = (input: InputStream, stack: Stack) => {
if (!stack.canShift(CurlyString)) return
let depth = 0
let pos = 0
while (true) {
const ch = input.peek(pos)
if (ch < 0) return // EOF - invalid
if (ch === 123) depth++ // {
else if (ch === 125) { // }
depth--
if (depth === 0) {
pos++ // consume final }
break
}
}
pos++
}
input.acceptToken(CurlyString, pos)
}
// Check if this identifier is in scope (for property access detection)
// Returns IdentifierBeforeDot token if in scope, null otherwise
const checkForDotGet = (input: InputStream, stack: Stack, pos: number): number | null => {
const identifierText = buildIdentifierText(input, pos)
const context = stack.context as { scope: { has(name: string): boolean } } | undefined
// Check if identifier is in scope (lexical scope or globals)
const inScope = context?.scope.has(identifierText) || globals.includes(identifierText)
// property access
if (inScope) return IdentifierBeforeDot
// Not in scope - check if we're inside a DotGet chain
// Inside the @skip {} block where DotGet is defined, Word cannot be shifted
// but Identifier can be. This tells us we're at the RHS of a DotGet.
const canShiftIdentifier = stack.canShift(Identifier)
const canShiftWord = stack.canShift(Word)
const inDotGetChain = canShiftIdentifier && !canShiftWord
// continue if we're inside a DotGet
return inDotGetChain ? IdentifierBeforeDot : null
}
// Decide between AssignableIdentifier and Identifier using grammar state + peek-ahead
const chooseIdentifierToken = (input: InputStream, stack: Stack): number => {
const canAssignable = stack.canShift(AssignableIdentifier)
const canRegular = stack.canShift(Identifier)
// Only one option is valid - use it
if (canAssignable && !canRegular) return AssignableIdentifier
if (canRegular && !canAssignable) return Identifier
// Both possible (ambiguous context) - peek ahead for '=' to disambiguate
// This happens at statement start where both `x = 5` (assign) and `echo x` (call) are valid
let peekPos = 0
while (true) {
const ch = getFullCodePoint(input, peekPos)
if (isWhiteSpace(ch)) {
peekPos += getCharSize(ch)
} else {
break
}
}
const nextCh = getFullCodePoint(input, peekPos)
const nextCh2 = getFullCodePoint(input, peekPos + 1)
const nextCh3 = getFullCodePoint(input, peekPos + 2)
// Check for ??= (three-character compound operator)
if (nextCh === 63 /* ? */ && nextCh2 === 63 /* ? */ && nextCh3 === 61 /* = */) {
const charAfterOp = getFullCodePoint(input, peekPos + 3)
if (isWhiteSpace(charAfterOp) || charAfterOp === -1 /* EOF */) {
return AssignableIdentifier
}
}
// Check for compound assignment operators: +=, -=, *=, /=, %=
if (
[43 /* + */, 45 /* - */, 42 /* * */, 47 /* / */, 37 /* % */].includes(nextCh) &&
nextCh2 === 61 /* = */
) {
// Found compound operator, check if it's followed by whitespace
const charAfterOp = getFullCodePoint(input, peekPos + 2)
if (isWhiteSpace(charAfterOp) || charAfterOp === -1 /* EOF */) {
return AssignableIdentifier
}
}
if (nextCh === 61 /* = */) {
// Found '=', but check if it's followed by whitespace
// If '=' is followed by non-whitespace (like '=cool*'), it won't be tokenized as Eq
// In that case, this should be Identifier (for function call), not AssignableIdentifier
const charAfterEquals = getFullCodePoint(input, peekPos + 1)
if (isWhiteSpace(charAfterEquals) || charAfterEquals === -1 /* EOF */) {
return AssignableIdentifier
}
}
return Identifier
}
// Character classification helpers
export const isIdentStart = (ch: number): boolean => {
return isLowercaseLetter(ch) || isEmojiOrUnicode(ch)
}
export const isIdentChar = (ch: number): boolean => {
return isLowercaseLetter(ch) || isDigit(ch) || ch === 45 /* - */ || ch === 63 /* ? */ || isEmojiOrUnicode(ch)
}
const isWhiteSpace = (ch: number): boolean => {
return ch === 32 /* space */ || ch === 9 /* tab */ || ch === 13 /* \r */
}
const isWordChar = (ch: number): boolean => {
return (
!isWhiteSpace(ch) &&
ch !== 10 /* \n */ &&
ch !== 41 /* ) */ &&
ch !== 93 /* ] */ &&
ch !== -1 /* EOF */
)
}
const isLowercaseLetter = (ch: number): boolean => {
return ch >= 97 && ch <= 122 // a-z
}
const isDigit = (ch: number): boolean => {
return ch >= 48 && ch <= 57 // 0-9
}
const getFullCodePoint = (input: InputStream, pos: number): number => {
const ch = input.peek(pos)
// Check if this is a high surrogate (0xD800-0xDBFF)
if (ch >= 0xd800 && ch <= 0xdbff) {
const low = input.peek(pos + 1)
// Check if next is low surrogate (0xDC00-0xDFFF)
if (low >= 0xdc00 && low <= 0xdfff) {
// Combine surrogate pair into full code point
return 0x10000 + ((ch & 0x3ff) << 10) + (low & 0x3ff)
}
}
return ch
}
const isEmojiOrUnicode = (ch: number): boolean => {
return (
// Basic Emoticons
(ch >= 0x1f600 && ch <= 0x1f64f) ||
// Miscellaneous Symbols and Pictographs
(ch >= 0x1f300 && ch <= 0x1f5ff) ||
// Transport and Map Symbols
(ch >= 0x1f680 && ch <= 0x1f6ff) ||
// Regional Indicator Symbols (flags)
(ch >= 0x1f1e6 && ch <= 0x1f1ff) ||
// Miscellaneous Symbols (hearts, stars, weather)
(ch >= 0x2600 && ch <= 0x26ff) ||
// Dingbats (scissors, pencils, etc)
(ch >= 0x2700 && ch <= 0x27bf) ||
// Supplemental Symbols and Pictographs (newer emojis)
(ch >= 0x1f900 && ch <= 0x1f9ff) ||
// Symbols and Pictographs Extended-A (newest emojis)
(ch >= 0x1fa70 && ch <= 0x1faff) ||
// Various Asian Characters with emoji presentation
(ch >= 0x1f018 && ch <= 0x1f270) ||
// Variation Selectors (for emoji presentation)
(ch >= 0xfe00 && ch <= 0xfe0f) ||
// Additional miscellaneous items
(ch >= 0x238c && ch <= 0x2454) ||
// Combining Diacritical Marks for Symbols
(ch >= 0x20d0 && ch <= 0x20ff) ||
// Latin-1 Supplement (includes ², ³, ¹ and other special chars)
(ch >= 0x00a0 && ch <= 0x00ff) ||
// Greek and Coptic (U+0370-U+03FF)
(ch >= 0x0370 && ch <= 0x03ff) ||
// Mathematical Alphanumeric Symbols (U+1D400-U+1D7FF)
(ch >= 0x1d400 && ch <= 0x1d7ff) ||
// Mathematical Operators (U+2200-U+22FF)
(ch >= 0x2200 && ch <= 0x22ff) ||
// Superscripts and Subscripts (U+2070-U+209F)
(ch >= 0x2070 && ch <= 0x209f) ||
// Arrows (U+2190-U+21FF)
(ch >= 0x2190 && ch <= 0x21ff) ||
// Hiragana (U+3040-U+309F)
(ch >= 0x3040 && ch <= 0x309f) ||
// Katakana (U+30A0-U+30FF)
(ch >= 0x30a0 && ch <= 0x30ff) ||
// CJK Unified Ideographs (U+4E00-U+9FFF)
(ch >= 0x4e00 && ch <= 0x9fff)
)
}
const getCharSize = (ch: number) => (ch > 0xffff ? 2 : 1) // emoji takes 2 UTF-16 code units
export const pipeStartsLineTokenizer = new ExternalTokenizer((input: InputStream, stack: Stack) => {
const ch = input.peek(0)
if (ch !== 10 /* \n */) return
// ignore whitespace
let offset = 1
let lastNewlineOffset = 0
while (true) {
const ch = input.peek(offset)
if (ch === 10 /* \n */) {
lastNewlineOffset = offset
offset++
} else if (isWhiteSpace(ch)) {
offset++
} else {
break
}
}
// look for pipe after skipping empty lines
if (input.peek(offset) === 124 /* | */) {
input.advance(lastNewlineOffset + 1)
input.acceptToken(pipeStartsLine)
} else {
input.advance(1)
input.acceptToken(newline)
}
})

View File

@ -1,594 +0,0 @@
const DEBUG = process.env.DEBUG || false
export type Token = {
type: TokenType
value?: string,
from: number,
to: number,
}
export enum TokenType {
Comment,
Keyword,
Operator,
Newline,
Semicolon,
Colon,
Underscore,
OpenParen,
CloseParen,
OpenBracket,
CloseBracket,
Identifier,
Word,
NamedArgPrefix,
Null,
Boolean,
Number,
String,
Regex,
}
const valueTokens = new Set([
TokenType.Comment,
TokenType.Keyword, TokenType.Operator,
TokenType.Identifier, TokenType.Word, TokenType.NamedArgPrefix,
TokenType.Boolean, TokenType.Number, TokenType.String, TokenType.Regex,
TokenType.Underscore
])
const operators = new Set([
// assignment
'=',
// logic
'or',
'and',
// bitwise
'band',
'bor',
'bxor',
'>>>',
'>>',
'<<',
// compound assignment
'??=',
'+=',
'-=',
'*=',
'/=',
'%=',
// nullish
'??',
// math
'**',
'*',
'/',
'+',
'-',
'%',
// comparison
'>=',
'<=',
'!=',
'==',
'>',
'<',
// property access
'.',
// pipe
'|',
])
const keywords = new Set([
'import',
'end',
'do',
'if',
'while',
'if',
'else',
'try',
'catch',
'finally',
'throw',
'not',
])
// helper
function c(strings: TemplateStringsArray, ...values: any[]) {
return strings.reduce((result, str, i) => result + str + (values[i] ?? ""), "").charCodeAt(0)
}
function s(c: number): string {
return String.fromCharCode(c)
}
export class Scanner {
input = ''
pos = 0
start = 0
char = 0
prev = 0
inParen = 0
inBracket = 0
tokens: Token[] = []
prevIsWhitespace = true
reset() {
this.input = ''
this.pos = 0
this.start = 0
this.char = 0
this.prev = 0
this.tokens.length = 0
this.prevIsWhitespace = true
}
peek(count = 0): number {
return getFullCodePoint(this.input, this.pos + count)
}
next(): number {
this.prevIsWhitespace = isWhitespace(this.char)
this.prev = this.char
this.char = this.peek()
this.pos += getCharSize(this.char)
return this.char
}
push(type: TokenType, from?: number, to?: number) {
from ??= this.start
to ??= this.pos - getCharSize(this.char)
if (to < from) to = from
this.tokens.push(Object.assign({}, {
type,
from,
to,
}, valueTokens.has(type) ? { value: this.input.slice(from, to) } : {}))
if (DEBUG) {
const tok = this.tokens.at(-1)
console.log(`≫ PUSH(${from},${to})`, TokenType[tok?.type || 0], '—', tok?.value)
}
this.start = this.pos
}
pushChar(type: TokenType) {
this.push(type, this.pos - 1, this.pos)
}
// turn shrimp code into shrimp tokens that get fed into the parser
tokenize(input: string): Token[] {
this.reset()
this.input = input
this.next()
while (this.char > 0) {
const char = this.char
if (char === c`#`) {
this.readComment()
continue
}
if (isBracket(char)) {
this.readBracket()
continue
}
if (isStringDelim(char)) {
this.readString(char)
continue
}
if (char === c`{`) {
this.readCurlyString()
continue
}
if (isIdentStart(char)) {
this.readWordOrIdent(true) // true = started with identifier char
continue
}
if (isDigit(char) || ((char === c`-` || char === c`+`) && isDigit(this.peek()))) {
this.readNumber()
continue
}
if (char === c`:`) {
this.pushChar(TokenType.Colon)
this.next()
continue
}
// whitespace-sensitive dot as operator (property access) only after identifier/number
if (char === c`.`) {
if (this.canBeDotGet(this.tokens.at(-1))) {
this.pushChar(TokenType.Operator)
this.next()
continue
}
}
if (char === c`/` && this.peek() === c`/`) {
this.readRegex()
continue
}
if (isWordChar(char)) {
this.readWordOrIdent(false) // false = didn't start with identifier char
continue
}
if (char === c`\n`) {
if (this.inParen === 0 && this.inBracket === 0)
this.pushChar(TokenType.Newline)
this.next()
continue
}
if (char === c`;`) {
this.pushChar(TokenType.Semicolon)
this.next()
continue
}
this.next()
}
return this.tokens
}
readComment() {
this.start = this.pos - 1
while (this.char !== c`\n` && this.char > 0) this.next()
this.push(TokenType.Comment)
}
readBracket() {
switch (this.char) {
case c`(`:
this.inParen++
this.pushChar(TokenType.OpenParen); break
case c`)`:
this.inParen--
this.pushChar(TokenType.CloseParen); break
case c`[`:
this.inBracket++
this.pushChar(TokenType.OpenBracket); break
case c`]`:
this.inBracket--
this.pushChar(TokenType.CloseBracket); break
}
this.next()
}
readString(delim: number) {
this.start = this.pos - 1
this.next() // skip opening delim
while (this.char > 0 && (this.char !== delim || (this.char === delim && this.prev === c`\\`)))
this.next()
this.next() // skip closing delim
this.push(TokenType.String)
}
readCurlyString() {
this.start = this.pos - 1
let depth = 1
this.next()
while (depth > 0 && this.char > 0) {
if (this.char === c`{`) depth++
if (this.char === c`}`) depth--
this.next()
}
this.push(TokenType.String)
}
readWordOrIdent(startedWithIdentChar: boolean) {
this.start = this.pos - getCharSize(this.char)
while (isWordChar(this.char)) {
// stop at colon if followed by whitespace (e.g., 'do x: echo x end')
if (this.char === c`:`) {
const nextCh = this.peek()
if (isWhitespace(nextCh) || nextCh === 0) break
}
// stop at equal sign (named arg) - but only if what we've read so far is an identifier
if (this.char === c`=`) {
const soFar = this.input.slice(this.start, this.pos - getCharSize(this.char))
if (isIdentifer(soFar)) {
this.next()
break
}
}
// stop at dot only if it would create a valid property access
// AND only if we started with an identifier character (not for Words like README.txt)
if (startedWithIdentChar && this.char === c`.`) {
const nextCh = this.peek()
if (isIdentStart(nextCh) || isDigit(nextCh) || nextCh === c`(`) {
const soFar = this.input.slice(this.start, this.pos - getCharSize(this.char))
if (isIdentifer(soFar)) break
}
}
this.next()
}
const word = this.input.slice(this.start, this.pos - getCharSize(this.char))
// classify the token based on what we read
if (word === '_')
this.push(TokenType.Underscore)
else if (word === 'null')
this.push(TokenType.Null)
else if (word === 'true' || word === 'false')
this.push(TokenType.Boolean)
else if (isKeyword(word))
this.push(TokenType.Keyword)
else if (isOperator(word))
this.push(TokenType.Operator)
else if (isIdentifer(word))
this.push(TokenType.Identifier)
else if (word.endsWith('='))
this.push(TokenType.NamedArgPrefix)
else
this.push(TokenType.Word)
}
readNumber() {
this.start = this.pos - 1
while (isWordChar(this.char)) {
// stop at dot unless it's part of the number
if (this.char === c`.`) {
const nextCh = this.peek()
if (!isDigit(nextCh)) break
}
// stop at colon
if (this.char === c`:`) {
const nextCh = this.peek()
if (isWhitespace(nextCh) || nextCh === 0) break
}
this.next()
}
const ident = this.input.slice(this.start, this.pos - 1)
this.push(isNumber(ident) ? TokenType.Number : TokenType.Word)
}
readRegex() {
this.start = this.pos - 1
this.next() // skip 2nd /
while (this.char > 0) {
if (this.char === c`/` && this.peek() === c`/`) {
this.next() // skip /
this.next() // skip /
// read regex flags
while (this.char > 0 && isIdentStart(this.char))
this.next()
// validate regex
const to = this.pos - getCharSize(this.char)
const regexText = this.input.slice(this.start, to)
const [_, pattern, flags] = regexText.match(/^\/\/(.*)\/\/([gimsuy]*)$/) || []
if (pattern) {
try {
new RegExp(pattern, flags)
this.push(TokenType.Regex)
break
} catch (e) {
// invalid regex - fall through to Word
}
}
// invalid regex is treated as Word
this.push(TokenType.Word)
break
}
this.next()
}
}
canBeDotGet(lastToken?: Token): boolean {
return !this.prevIsWhitespace && !!lastToken &&
(lastToken.type === TokenType.Identifier ||
lastToken.type === TokenType.Number ||
lastToken.type === TokenType.CloseParen ||
lastToken.type === TokenType.CloseBracket)
}
}
const isNumber = (word: string): boolean => {
// regular number
if (/^[+-]?\d+(_?\d+)*(\.(\d+(_?\d+)*))?$/.test(word))
return true
// binary
if (/^[+-]?0b[01]+(_?[01]+)*(\.[01](_?[01]*))?$/.test(word))
return true
// octal
if (/^[+-]?0o[0-7]+(_?[0-7]+)*(\.[0-7](_?[0-7]*))?$/.test(word))
return true
// hex
if (/^[+-]?0x[0-9a-f]+([0-9a-f]_?[0-9a-f]+)*(\.([0-9a-f]_?[0-9a-f]*))?$/i.test(word))
return true
return false
}
const isIdentifer = (s: string): boolean => {
if (s.length === 0) return false
let pos = 0
const chars = []
while (pos < s.length) {
const out = getFullCodePoint(s, pos)
pos += getCharSize(out)
chars.push(out)
}
if (chars.length === 1)
return isIdentStart(chars[0]!)
else if (chars.length === 2)
return isIdentStart(chars[0]!) && isIdentEnd(chars[1]!)
else
return isIdentStart(chars[0]!) &&
chars.slice(1, chars.length - 1).every(isIdentChar) &&
isIdentEnd(chars.at(-1)!)
}
const isStringDelim = (ch: number): boolean => {
return ch === c`'` || ch === c`"`
}
export const isIdentStart = (char: number | string): boolean => {
let ch = typeof char === 'string' ? char.charCodeAt(0) : char
return isLowercaseLetter(ch) || isEmojiOrUnicode(ch) || ch === 36 /* $ */
}
export const isIdentChar = (char: number | string): boolean => {
let ch = typeof char === 'string' ? char.charCodeAt(0) : char
return isIdentStart(ch) || isDigit(ch) || ch === 45 /* - */ || ch === 63 /* ? */
}
const isIdentEnd = (char: number | string): boolean => {
return isIdentChar(char)
}
const isLowercaseLetter = (ch: number): boolean => {
return ch >= 97 && ch <= 122 // a-z
}
const isDigit = (ch: number): boolean => {
return ch >= 48 && ch <= 57 // 0-9
}
const isWhitespace = (ch: number): boolean => {
return ch === 32 /* space */ || ch === 9 /* tab */ ||
ch === 13 /* \r */ || ch === 10 /* \n */ ||
ch === -1 || ch === 0 /* EOF */
}
const isWordChar = (ch: number): boolean => {
return (
!isWhitespace(ch) &&
ch !== 10 /* \n */ &&
ch !== 59 /* ; */ &&
ch !== 40 /* ( */ &&
ch !== 41 /* ) */ &&
ch !== 93 /* ] */ &&
ch !== -1 /* EOF */
)
}
const isOperator = (word: string): boolean => {
return operators.has(word)
}
const isKeyword = (word: string): boolean => {
return keywords.has(word)
}
const isBracket = (char: number): boolean => {
return char === c`(` || char === c`)` || char === c`[` || char === c`]`
}
const getCharSize = (ch: number) =>
(ch > 0xffff ? 2 : 1) // emoji takes 2 UTF-16 code units
const getFullCodePoint = (input: string, pos: number): number => {
const ch = input[pos]?.charCodeAt(0) || 0
// Check if this is a high surrogate (0xD800-0xDBFF)
if (ch >= 0xd800 && ch <= 0xdbff) {
const low = input[pos + 1]?.charCodeAt(0) || 0
// Check if next is low surrogate (0xDC00-0xDFFF)
if (low >= 0xdc00 && low <= 0xdfff) {
// Combine surrogate pair into full code point
return 0x10000 + ((ch & 0x3ff) << 10) + (low & 0x3ff)
}
}
return ch
}
const isEmojiOrUnicode = (ch: number): boolean => {
return (
// Basic Emoticons
(ch >= 0x1f600 && ch <= 0x1f64f) ||
// Miscellaneous Symbols and Pictographs
(ch >= 0x1f300 && ch <= 0x1f5ff) ||
// Transport and Map Symbols
(ch >= 0x1f680 && ch <= 0x1f6ff) ||
// Regional Indicator Symbols (flags)
(ch >= 0x1f1e6 && ch <= 0x1f1ff) ||
// Miscellaneous Symbols (hearts, stars, weather)
(ch >= 0x2600 && ch <= 0x26ff) ||
// Dingbats (scissors, pencils, etc)
(ch >= 0x2700 && ch <= 0x27bf) ||
// Supplemental Symbols and Pictographs (newer emojis)
(ch >= 0x1f900 && ch <= 0x1f9ff) ||
// Symbols and Pictographs Extended-A (newest emojis)
(ch >= 0x1fa70 && ch <= 0x1faff) ||
// Various Asian Characters with emoji presentation
(ch >= 0x1f018 && ch <= 0x1f270) ||
// Variation Selectors (for emoji presentation)
(ch >= 0xfe00 && ch <= 0xfe0f) ||
// Additional miscellaneous items
(ch >= 0x238c && ch <= 0x2454) ||
// Combining Diacritical Marks for Symbols
(ch >= 0x20d0 && ch <= 0x20ff) ||
// Latin-1 Supplement (includes ², ³, ¹ and other special chars)
(ch >= 0x00a0 && ch <= 0x00ff) ||
// Greek and Coptic (U+0370-U+03FF)
(ch >= 0x0370 && ch <= 0x03ff) ||
// Mathematical Alphanumeric Symbols (U+1D400-U+1D7FF)
(ch >= 0x1d400 && ch <= 0x1d7ff) ||
// Mathematical Operators (U+2200-U+22FF)
(ch >= 0x2200 && ch <= 0x22ff) ||
// Superscripts and Subscripts (U+2070-U+209F)
(ch >= 0x2070 && ch <= 0x209f) ||
// Arrows (U+2190-U+21FF)
(ch >= 0x2190 && ch <= 0x21ff) ||
// Hiragana (U+3040-U+309F)
(ch >= 0x3040 && ch <= 0x309f) ||
// Katakana (U+30A0-U+30FF)
(ch >= 0x30a0 && ch <= 0x30ff) ||
// CJK Unified Ideographs (U+4E00-U+9FFF)
(ch >= 0x4e00 && ch <= 0x9fff)
)
}

View File

@ -1,12 +0,0 @@
export const date = {
now: () => Date.now(),
year: (time: number) => (new Date(time)).getFullYear(),
month: (time: number) => (new Date(time)).getMonth(),
date: (time: number) => (new Date(time)).getDate(),
hour: (time: number) => (new Date(time)).getHours(),
minute: (time: number) => (new Date(time)).getMinutes(),
second: (time: number) => (new Date(time)).getSeconds(),
ms: (time: number) => (new Date(time)).getMilliseconds(),
new: (year: number, month: number, day: number, hour = 0, minute = 0, second = 0, ms = 0) =>
new Date(year, month, day, hour, minute, second, ms).getTime()
}

View File

@ -6,7 +6,6 @@ import {
extractParamInfo, isWrapped, getOriginalFunction, extractParamInfo, isWrapped, getOriginalFunction,
} from 'reefvm' } from 'reefvm'
import { date } from './date'
import { dict } from './dict' import { dict } from './dict'
import { fs } from './fs' import { fs } from './fs'
import { json } from './json' import { json } from './json'
@ -14,10 +13,8 @@ import { load } from './load'
import { list } from './list' import { list } from './list'
import { math } from './math' import { math } from './math'
import { str } from './str' import { str } from './str'
import { types } from './types'
export const globals: Record<string, any> = { export const globals = {
date,
dict, dict,
fs, fs,
json, json,
@ -37,6 +34,7 @@ export const globals: Record<string, any> = {
name: Bun.argv[2] || '(shrimp)', name: Bun.argv[2] || '(shrimp)',
path: resolve(join('.', Bun.argv[2] ?? '')) path: resolve(join('.', Bun.argv[2] ?? ''))
}, },
}, },
// hello // hello
@ -86,9 +84,20 @@ export const globals: Record<string, any> = {
exit: (num: number) => process.exit(num ?? 0), exit: (num: number) => process.exit(num ?? 0),
// type predicates // type predicates
'string?': (v: any) => toValue(v).type === 'string',
'number?': (v: any) => toValue(v).type === 'number',
'boolean?': (v: any) => toValue(v).type === 'boolean',
'array?': (v: any) => toValue(v).type === 'array',
'dict?': (v: any) => toValue(v).type === 'dict',
'function?': (v: any) => {
const t = toValue(v).type
return t === 'function' || t === 'native'
},
'null?': (v: any) => toValue(v).type === 'null',
'some?': (v: any) => toValue(v).type !== 'null', 'some?': (v: any) => toValue(v).type !== 'null',
// boolean/logic // boolean/logic
not: (v: any) => !v,
bnot: (n: number) => ~(n | 0), bnot: (n: number) => ~(n | 0),
// utilities // utilities
@ -182,8 +191,8 @@ export function formatValue(value: Value, inner = false): string {
return `${colors.blue}[${colors.reset}${items}${colors.blue}]${colors.reset}` return `${colors.blue}[${colors.reset}${items}${colors.blue}]${colors.reset}`
} }
case 'dict': { case 'dict': {
const entries = Array.from(value.value.entries()).reverse() const entries = Array.from(value.value.entries())
.map(([k, v]) => `${k.trim()}${colors.blue}=${colors.reset}${formatValue(v, true)}`) .map(([k, v]) => `${k}${colors.blue}=${colors.reset}${formatValue(v, true)}`)
.join(' ') .join(' ')
if (entries.length === 0) if (entries.length === 0)
return `${colors.blue}[=]${colors.reset}` return `${colors.blue}[=]${colors.reset}`
@ -204,7 +213,3 @@ export function formatValue(value: Value, inner = false): string {
return String(value) return String(value)
} }
} }
// add types functions to top-level namespace
for (const [key, value] of Object.entries(types))
globals[key] = value

View File

@ -16,10 +16,7 @@ export const math = {
if (n < 0) throw new Error(`sqrt: cannot take square root of negative number ${n}`) if (n < 0) throw new Error(`sqrt: cannot take square root of negative number ${n}`)
return Math.sqrt(n) return Math.sqrt(n)
}, },
random: (min = 0, max = 1) => { random: () => Math.random(),
if (min === 0 && max === 1) return Math.random()
return Math.floor(Math.random() * (max - min + 1)) + min
},
clamp: (n: number, min: number, max: number) => { clamp: (n: number, min: number, max: number) => {
if (min > max) throw new Error(`clamp: min (${min}) must be less than or equal to max (${max})`) if (min > max) throw new Error(`clamp: min (${min}) must be less than or equal to max (${max})`)
return Math.min(Math.max(n, min), max) return Math.min(Math.max(n, min), max)

View File

@ -28,16 +28,6 @@ export const str = {
}, },
'pad-start': (str: string, length: number, pad: string = ' ') => String(str ?? '').padStart(length, pad), 'pad-start': (str: string, length: number, pad: string = ' ') => String(str ?? '').padStart(length, pad),
'pad-end': (str: string, length: number, pad: string = ' ') => String(str ?? '').padEnd(length, pad), 'pad-end': (str: string, length: number, pad: string = ' ') => String(str ?? '').padEnd(length, pad),
capitalize: (str: string) => {
const s = String(str ?? '')
return s.charAt(0).toUpperCase() + s.slice(1).toLowerCase()
},
titlecase: (s: string) => {
return String(s ?? '')
.split(' ')
.map(str.capitalize)
.join(' ')
},
lines: (str: string) => String(str ?? '').split('\n'), lines: (str: string) => String(str ?? '').split('\n'),
chars: (str: string) => String(str ?? '').split(''), chars: (str: string) => String(str ?? '').split(''),

View File

@ -1,170 +0,0 @@
import { expect, describe, test } from 'bun:test'
describe('date', () => {
test('date.now returns current timestamp', () => {
expect(`date.now | number?`).toEvaluateTo(true)
expect(`(date.now) > 1577836800000`).toEvaluateTo(true)
})
test('date.new creates timestamp from components', () => {
expect(`
t = date.new 2024 0 1 12 0 0 500
[
(date.year t)
(date.month t)
(date.date t)
(date.hour t)
(date.minute t)
(date.second t)
(date.ms t)
]
`).toEvaluateTo([2024, 0, 1, 12, 0, 0, 500])
})
test('date.new with minimal arguments', () => {
expect(`
t = date.new 2024 5 15
[
(date.year t)
(date.month t)
(date.date t)
(date.hour t)
(date.minute t)
(date.second t)
(date.ms t)
]
`).toEvaluateTo([2024, 5, 15, 0, 0, 0, 0])
})
test('date.year extracts year', () => {
expect(`
t = date.new 2024 0 1
date.year t
`).toEvaluateTo(2024)
expect(`
t = date.new 1999 11 31
date.year t
`).toEvaluateTo(1999)
})
test('date.month extracts month (0-indexed)', () => {
// January = 0, December = 11
expect(`
jan = date.new 2024 0 1
dec = date.new 2024 11 31
[(date.month jan) (date.month dec)]
`).toEvaluateTo([0, 11])
})
test('date.date extracts day of month', () => {
expect(`
t = date.new 2024 5 15
date.date t
`).toEvaluateTo(15)
expect(`
date.new 2024 0 1 | date.date
`).toEvaluateTo(1)
})
test('date.hour extracts hour', () => {
expect(`
t = date.new 2024 0 1 14 30 45
date.hour t
`).toEvaluateTo(14)
expect(`
t = date.new 2024 0 1 0 0 0
date.hour t
`).toEvaluateTo(0)
})
test('date.minute extracts minute', () => {
expect(`
t = date.new 2024 0 1 14 30 45
date.minute t
`).toEvaluateTo(30)
})
test('date.second extracts second', () => {
expect(`
t = date.new 2024 0 1 14 30 45
date.second t
`).toEvaluateTo(45)
})
test('date.ms extracts milliseconds', () => {
expect(`
t = date.new 2024 0 1 14 30 45 250
date.ms t
`).toEvaluateTo(250)
})
test('round-trip: create and extract components', () => {
expect(`
t = date.new 2024 6 4 15 30 45 123
year = date.year t
month = date.month t
day = date.date t
hour = date.hour t
min = date.minute t
sec = date.second t
ms = date.ms t
[year month day hour min sec ms]
`).toEvaluateTo([2024, 6, 4, 15, 30, 45, 123])
})
test('edge cases - midnight', () => {
expect(`
t = date.new 2024 0 1 0 0 0 0
[
(date.hour t)
(date.minute t)
(date.second t)
(date.ms t)
]
`).toEvaluateTo([0, 0, 0, 0])
})
test('edge cases - end of day', () => {
expect(`
t = date.new 2024 0 1 23 59 59 999
[
(date.hour t)
(date.minute t)
(date.second t)
(date.ms t)
]
`).toEvaluateTo([23, 59, 59, 999])
})
test('edge cases - leap year', () => {
expect(`
t = date.new 2024 1 29
[
(date.year t)
(date.month t)
(date.date t)
]
`).toEvaluateTo([2024, 1, 29])
})
test('combining date functions with arithmetic', () => {
expect(`
t = date.new 2024 5 15 10 30 0
next-hour = date.new 2024 5 15 11 30 0
(date.hour next-hour) - (date.hour t)
`).toEvaluateTo(1)
})
test('using date.now in calculations', () => {
// Check that date.now is in the past compared to a future timestamp
expect(`
now = (date.now)
future = date.new 2030 0 1
future > now
`).toEvaluateTo(true)
})
})

View File

@ -1,89 +1,90 @@
import { expect, describe, test } from 'bun:test' import { expect, describe, test } from 'bun:test'
import { globals } from '#prelude'
describe('var and var?', () => { describe('var and var?', () => {
test('var? checks if a variable exists', async () => { test('var? checks if a variable exists', async () => {
await expect(`var? 'nada'`).toEvaluateTo(false) await expect(`var? 'nada'`).toEvaluateTo(false, globals)
await expect(`var? 'info'`).toEvaluateTo(false) await expect(`var? 'info'`).toEvaluateTo(false, globals)
await expect(`abc = abc; var? 'abc'`).toEvaluateTo(true) await expect(`abc = abc; var? 'abc'`).toEvaluateTo(true, globals)
await expect(`var? 'var?'`).toEvaluateTo(true) await expect(`var? 'var?'`).toEvaluateTo(true, globals)
await expect(`var? 'dict'`).toEvaluateTo(true) await expect(`var? 'dict'`).toEvaluateTo(true, globals)
await expect(`var? dict`).toEvaluateTo(true) await expect(`var? dict`).toEvaluateTo(true, globals)
}) })
test('var returns a value or null', async () => { test('var returns a value or null', async () => {
await expect(`var 'nada'`).toEvaluateTo(null) await expect(`var 'nada'`).toEvaluateTo(null, globals)
await expect(`var nada`).toEvaluateTo(null) await expect(`var nada`).toEvaluateTo(null, globals)
await expect(`var 'info'`).toEvaluateTo(null) await expect(`var 'info'`).toEvaluateTo(null, globals)
await expect(`abc = my-string; var 'abc'`).toEvaluateTo('my-string') await expect(`abc = my-string; var 'abc'`).toEvaluateTo('my-string', globals)
await expect(`abc = my-string; var abc`).toEvaluateTo(null) await expect(`abc = my-string; var abc`).toEvaluateTo(null, globals)
}) })
}) })
describe('type predicates', () => { describe('type predicates', () => {
test('string? checks for string type', async () => { test('string? checks for string type', async () => {
await expect(`string? 'hello'`).toEvaluateTo(true) await expect(`string? 'hello'`).toEvaluateTo(true, globals)
await expect(`string? 42`).toEvaluateTo(false) await expect(`string? 42`).toEvaluateTo(false, globals)
}) })
test('number? checks for number type', async () => { test('number? checks for number type', async () => {
await expect(`number? 42`).toEvaluateTo(true) await expect(`number? 42`).toEvaluateTo(true, globals)
await expect(`number? 'hello'`).toEvaluateTo(false) await expect(`number? 'hello'`).toEvaluateTo(false, globals)
}) })
test('boolean? checks for boolean type', async () => { test('boolean? checks for boolean type', async () => {
await expect(`boolean? true`).toEvaluateTo(true) await expect(`boolean? true`).toEvaluateTo(true, globals)
await expect(`boolean? 42`).toEvaluateTo(false) await expect(`boolean? 42`).toEvaluateTo(false, globals)
}) })
test('array? checks for array type', async () => { test('array? checks for array type', async () => {
await expect(`array? [1 2 3]`).toEvaluateTo(true) await expect(`array? [1 2 3]`).toEvaluateTo(true, globals)
await expect(`array? 42`).toEvaluateTo(false) await expect(`array? 42`).toEvaluateTo(false, globals)
}) })
test('dict? checks for dict type', async () => { test('dict? checks for dict type', async () => {
await expect(`dict? [a=1]`).toEvaluateTo(true) await expect(`dict? [a=1]`).toEvaluateTo(true, globals)
await expect(`dict? []`).toEvaluateTo(false) await expect(`dict? []`).toEvaluateTo(false, globals)
}) })
test('null? checks for null type', async () => { test('null? checks for null type', async () => {
await expect(`null? null`).toEvaluateTo(true) await expect(`null? null`).toEvaluateTo(true, globals)
await expect(`null? 42`).toEvaluateTo(false) await expect(`null? 42`).toEvaluateTo(false, globals)
}) })
test('some? checks for non-null', async () => { test('some? checks for non-null', async () => {
await expect(`some? 42`).toEvaluateTo(true) await expect(`some? 42`).toEvaluateTo(true, globals)
await expect(`some? null`).toEvaluateTo(false) await expect(`some? null`).toEvaluateTo(false, globals)
}) })
}) })
describe('introspection', () => { describe('introspection', () => {
test('type returns proper types', async () => { test('type returns proper types', async () => {
await expect(`type 'hello'`).toEvaluateTo('string') await expect(`type 'hello'`).toEvaluateTo('string', globals)
await expect(`type 42`).toEvaluateTo('number') await expect(`type 42`).toEvaluateTo('number', globals)
await expect(`type true`).toEvaluateTo('boolean') await expect(`type true`).toEvaluateTo('boolean', globals)
await expect(`type false`).toEvaluateTo('boolean') await expect(`type false`).toEvaluateTo('boolean', globals)
await expect(`type null`).toEvaluateTo('null') await expect(`type null`).toEvaluateTo('null', globals)
await expect(`type [1 2 3]`).toEvaluateTo('array') await expect(`type [1 2 3]`).toEvaluateTo('array', globals)
await expect(`type [a=1 b=2]`).toEvaluateTo('dict') await expect(`type [a=1 b=2]`).toEvaluateTo('dict', globals)
}) })
test('inspect formats values', async () => { test('inspect formats values', async () => {
await expect(`inspect 'hello'`).toEvaluateTo("\u001b[32m'hello\u001b[32m'\u001b[0m") await expect(`inspect 'hello'`).toEvaluateTo("\u001b[32m'hello\u001b[32m'\u001b[0m", globals)
}) })
test('describe describes values', async () => { test('describe describes values', async () => {
await expect(`describe 'hello'`).toEvaluateTo("#<string: \u001b[32m'hello\u001b[32m'\u001b[0m>") await expect(`describe 'hello'`).toEvaluateTo("#<string: \u001b[32m'hello\u001b[32m'\u001b[0m>", globals)
}) })
}) })
describe('environment', () => { describe('environment', () => {
test('args is an array', async () => { test('args is an array', async () => {
await expect(`array? $.args`).toEvaluateTo(true) await expect(`array? $.args`).toEvaluateTo(true, globals)
}) })
test('args can be accessed', async () => { test('args can be accessed', async () => {
await expect(`type $.args`).toEvaluateTo('array') await expect(`type $.args`).toEvaluateTo('array', globals)
}) })
test('argv includes more than just the args', async () => { test('argv includes more than just the args', async () => {
@ -105,35 +106,35 @@ describe('ref', () => {
describe('$ global dictionary', () => { describe('$ global dictionary', () => {
test('$.args is an array', async () => { test('$.args is an array', async () => {
await expect(`$.args | array?`).toEvaluateTo(true) await expect(`$.args | array?`).toEvaluateTo(true, globals)
}) })
test('$.args can be accessed', async () => { test('$.args can be accessed', async () => {
await expect(`$.args | type`).toEvaluateTo('array') await expect(`$.args | type`).toEvaluateTo('array', globals)
}) })
test('$.script.name is a string', async () => { test('$.script.name is a string', async () => {
await expect(`$.script.name | string?`).toEvaluateTo(true) await expect(`$.script.name | string?`).toEvaluateTo(true, globals)
}) })
test('$.script.path is a string', async () => { test('$.script.path is a string', async () => {
await expect(`$.script.path | string?`).toEvaluateTo(true) await expect(`$.script.path | string?`).toEvaluateTo(true, globals)
}) })
test('$.env is a dict', async () => { test('$.env is a dict', async () => {
await expect(`$.env | dict?`).toEvaluateTo(true) await expect(`$.env | dict?`).toEvaluateTo(true, globals)
}) })
test('$.pid is a number', async () => { test('$.pid is a number', async () => {
await expect(`$.pid | number?`).toEvaluateTo(true) await expect(`$.pid | number?`).toEvaluateTo(true, globals)
await expect(`$.pid > 0`).toEvaluateTo(true) await expect(`$.pid > 0`).toEvaluateTo(true, globals)
}) })
test('$.cwd is a string', async () => { test('$.cwd is a string', async () => {
await expect(`$.cwd | string?`).toEvaluateTo(true) await expect(`$.cwd | string?`).toEvaluateTo(true, globals)
}) })
test('$.cwd returns current working directory', async () => { test('$.cwd returns current working directory', async () => {
await expect(`$.cwd`).toEvaluateTo(process.cwd()) await expect(`$.cwd`).toEvaluateTo(process.cwd(), globals)
}) })
}) })

View File

@ -1,41 +1,42 @@
import { expect, describe, test } from 'bun:test' import { expect, describe, test } from 'bun:test'
import { globals } from '#prelude'
describe('loading a file', () => { describe('loading a file', () => {
test(`imports all a file's functions`, async () => { test(`imports all a file's functions`, async () => {
expect(` expect(`
math = load ./src/prelude/tests/math.sh math = load ./src/prelude/tests/math.sh
math.double 4 math.double 4
`).toEvaluateTo(8) `).toEvaluateTo(8, globals)
expect(` expect(`
math = load ./src/prelude/tests/math.sh math = load ./src/prelude/tests/math.sh
math.double (math.double 4) math.double (math.double 4)
`).toEvaluateTo(16) `).toEvaluateTo(16, globals)
expect(` expect(`
math = load ./src/prelude/tests/math.sh math = load ./src/prelude/tests/math.sh
dbl = ref math.double dbl = ref math.double
dbl (dbl 2) dbl (dbl 2)
`).toEvaluateTo(8) `).toEvaluateTo(8, globals)
expect(` expect(`
math = load ./src/prelude/tests/math.sh math = load ./src/prelude/tests/math.sh
math.pi math.pi
`).toEvaluateTo(3.14) `).toEvaluateTo(3.14, globals)
expect(` expect(`
math = load ./src/prelude/tests/math.sh math = load ./src/prelude/tests/math.sh
math | at 🥧 math | at 🥧
`).toEvaluateTo(3.14159265359) `).toEvaluateTo(3.14159265359, globals)
expect(` expect(`
math = load ./src/prelude/tests/math.sh math = load ./src/prelude/tests/math.sh
math.🥧 math.🥧
`).toEvaluateTo(3.14159265359) `).toEvaluateTo(3.14159265359, globals)
expect(` expect(`
math = load ./src/prelude/tests/math.sh math = load ./src/prelude/tests/math.sh
math.add1 5 math.add1 5
`).toEvaluateTo(6) `).toEvaluateTo(6, globals)
}) })
}) })

View File

@ -1,4 +1,5 @@
import { expect, describe, test } from 'bun:test' import { expect, describe, test } from 'bun:test'
import { globals } from '#prelude'
describe('string operations', () => { describe('string operations', () => {
test('to-upper converts to uppercase', async () => { test('to-upper converts to uppercase', async () => {
@ -16,18 +17,6 @@ describe('string operations', () => {
await expect(`str.trim '\\n\\thello\\t\\n'`).toEvaluateTo('hello') await expect(`str.trim '\\n\\thello\\t\\n'`).toEvaluateTo('hello')
}) })
test('capitalize makes first char uppercase', async () => {
await expect(`str.capitalize 'hello'`).toEvaluateTo('Hello')
await expect(`str.capitalize 'HELLO'`).toEvaluateTo('Hello')
await expect(`str.capitalize 'hello world'`).toEvaluateTo('Hello world')
})
test('titlecase capitalizes each word', async () => {
await expect(`str.titlecase 'hello world'`).toEvaluateTo('Hello World')
await expect(`str.titlecase 'HELLO WORLD'`).toEvaluateTo('Hello World')
await expect(`str.titlecase 'the quick brown fox'`).toEvaluateTo('The Quick Brown Fox')
})
test('split divides string by separator', async () => { test('split divides string by separator', async () => {
await expect(`str.split 'a,b,c' ','`).toEvaluateTo(['a', 'b', 'c']) await expect(`str.split 'a,b,c' ','`).toEvaluateTo(['a', 'b', 'c'])
await expect(`str.split 'hello' ''`).toEvaluateTo(['h', 'e', 'l', 'l', 'o']) await expect(`str.split 'hello' ''`).toEvaluateTo(['h', 'e', 'l', 'l', 'o'])
@ -117,17 +106,6 @@ describe('boolean logic', () => {
await expect(`not 42`).toEvaluateTo(false) await expect(`not 42`).toEvaluateTo(false)
await expect(`not null`).toEvaluateTo(true) await expect(`not null`).toEvaluateTo(true)
}) })
test('not works with function calls', async () => {
await expect(`equals = do x y: x == y end; not equals 5 5`).toEvaluateTo(false)
await expect(`equals = do x y: x == y end; not equals 5 10`).toEvaluateTo(true)
})
test('not works with binary operations and comparisons', async () => {
await expect(`not 5 > 10`).toEvaluateTo(true)
await expect(`not 10 > 5`).toEvaluateTo(false)
await expect(`not true and false`).toEvaluateTo(true)
})
}) })
describe('utilities', () => { describe('utilities', () => {

View File

@ -1,143 +0,0 @@
import { expect, describe, test } from 'bun:test'
describe('type predicates', () => {
test('boolean? checks if value is boolean', async () => {
await expect(`boolean? true`).toEvaluateTo(true)
await expect(`boolean? false`).toEvaluateTo(true)
await expect(`boolean? 42`).toEvaluateTo(false)
await expect(`boolean? 'hello'`).toEvaluateTo(false)
await expect(`boolean? null`).toEvaluateTo(false)
await expect(`boolean? [1 2 3]`).toEvaluateTo(false)
})
test('number? checks if value is number', async () => {
await expect(`number? 42`).toEvaluateTo(true)
await expect(`number? 3.14`).toEvaluateTo(true)
await expect(`number? 0`).toEvaluateTo(true)
await expect(`number? -5`).toEvaluateTo(true)
await expect(`number? 'hello'`).toEvaluateTo(false)
await expect(`number? true`).toEvaluateTo(false)
await expect(`number? null`).toEvaluateTo(false)
})
test('string? checks if value is string', async () => {
await expect(`string? 'hello'`).toEvaluateTo(true)
await expect(`string? ''`).toEvaluateTo(true)
await expect(`string? world`).toEvaluateTo(true)
await expect(`string? 42`).toEvaluateTo(false)
await expect(`string? true`).toEvaluateTo(false)
await expect(`string? null`).toEvaluateTo(false)
await expect(`string? [1 2 3]`).toEvaluateTo(false)
})
test('array? checks if value is array', async () => {
await expect(`array? [1 2 3]`).toEvaluateTo(true)
await expect(`array? []`).toEvaluateTo(true)
await expect(`array? ['a' 'b']`).toEvaluateTo(true)
await expect(`array? [a=1 b=2]`).toEvaluateTo(false)
await expect(`array? 42`).toEvaluateTo(false)
await expect(`array? 'hello'`).toEvaluateTo(false)
await expect(`array? null`).toEvaluateTo(false)
})
test('list? is alias for array?', async () => {
await expect(`list? [1 2 3]`).toEvaluateTo(true)
await expect(`list? []`).toEvaluateTo(true)
await expect(`list? [a=1 b=2]`).toEvaluateTo(false)
})
test('dict? checks if value is dict', async () => {
await expect(`dict? [a=1 b=2]`).toEvaluateTo(true)
await expect(`dict? [=]`).toEvaluateTo(true)
await expect(`dict? [1 2 3]`).toEvaluateTo(false)
await expect(`dict? []`).toEvaluateTo(false)
await expect(`dict? 42`).toEvaluateTo(false)
await expect(`dict? 'hello'`).toEvaluateTo(false)
})
test('function? checks if value is function', async () => {
await expect(`
my-fn = do x: x * 2 end
function? my-fn
`).toEvaluateTo(true)
await expect(`function? inc`).toEvaluateTo(true)
await expect(`function? list.map`).toEvaluateTo(true)
await expect(`function? 42`).toEvaluateTo(false)
await expect(`function? 'hello'`).toEvaluateTo(false)
await expect(`function? [1 2 3]`).toEvaluateTo(false)
})
test('null? checks if value is null', async () => {
await expect(`null? null`).toEvaluateTo(true)
await expect(`null? 0`).toEvaluateTo(false)
await expect(`null? false`).toEvaluateTo(false)
await expect(`null? ''`).toEvaluateTo(false)
await expect(`null? []`).toEvaluateTo(false)
})
})
describe('type coercion', () => {
test('boolean coerces to boolean', async () => {
await expect(`boolean true`).toEvaluateTo(true)
await expect(`boolean false`).toEvaluateTo(false)
await expect(`boolean 1`).toEvaluateTo(true)
await expect(`boolean 0`).toEvaluateTo(false)
await expect(`boolean 'hello'`).toEvaluateTo(true)
await expect(`boolean ''`).toEvaluateTo(false)
await expect(`boolean null`).toEvaluateTo(false)
await expect(`boolean [1 2 3]`).toEvaluateTo(true)
})
test('number coerces to number', async () => {
await expect(`number 42`).toEvaluateTo(42)
await expect(`number '42'`).toEvaluateTo(42)
await expect(`number '3.14'`).toEvaluateTo(3.14)
await expect(`number true`).toEvaluateTo(1)
await expect(`number false`).toEvaluateTo(0)
})
test('string coerces to string', async () => {
await expect(`string 'hello'`).toEvaluateTo('hello')
await expect(`string 42`).toEvaluateTo('42')
await expect(`string true`).toEvaluateTo('true')
await expect(`string false`).toEvaluateTo('false')
await expect(`string null`).toEvaluateTo('null')
})
})
describe('type predicates in conditionals', () => {
test('using type predicates in if statements', async () => {
await expect(`
x = 42
if (number? x):
'is-num'
else:
'not-num'
end
`).toEvaluateTo('is-num')
})
test('filtering by type', async () => {
await expect(`
items = [1 'hello' 2 'world' 3]
list.filter items number?
`).toEvaluateTo([1, 2, 3])
})
test('filtering strings', async () => {
await expect(`
items = [1 'hello' 2 'world' 3]
list.filter items string?
`).toEvaluateTo(['hello', 'world'])
})
test('checking for functions', async () => {
await expect(`
double = do x: x * 2 end
not-fn = 42
is-fn = function? double
is-not-fn = function? not-fn
is-fn and (not is-not-fn)
`).toEvaluateTo(true)
})
})

View File

@ -1,22 +0,0 @@
import { toValue } from 'reefvm'
export const types = {
'boolean?': (v: any) => toValue(v).type === 'boolean',
boolean: (v: any) => Boolean(v),
'number?': (v: any) => toValue(v).type === 'number',
number: (v: any) => Number(v),
'string?': (v: any) => toValue(v).type === 'string',
string: (v: any) => String(v),
'array?': (v: any) => toValue(v).type === 'array',
'list?': (v: any) => toValue(v).type === 'array',
'dict?': (v: any) => toValue(v).type === 'dict',
'function?': (v: any) => ['function', 'native'].includes(toValue(v).type),
'null?': (v: any) => toValue(v).type === 'null',
}

View File

@ -1,14 +1,33 @@
import { expect } from 'bun:test' import { expect } from 'bun:test'
import { diffLines } from 'diff' import { parser } from '#parser/shrimp'
import color from 'kleur' import { setGlobals } from '#parser/tokenizer'
import { Scanner, TokenType, type Token } from '#parser/tokenizer2'
import { parse, setGlobals } from '#parser/parser2'
import { Tree } from '#parser/node'
import { globals as prelude } from '#prelude' import { globals as prelude } from '#prelude'
import { $ } from 'bun'
import { assert, errorMessage } from '#utils/utils' import { assert, errorMessage } from '#utils/utils'
import { Compiler } from '#compiler/compiler' import { Compiler } from '#compiler/compiler'
import { run, VM } from 'reefvm' import { run, VM } from 'reefvm'
import { treeToString2, VMResultToValue } from '#utils/tree' import { treeToString, VMResultToValue } from '#utils/tree'
const regenerateParser = async () => {
let generate = true
try {
const grammarStat = await Bun.file('./src/parser/shrimp.grammar').stat()
const tokenizerStat = await Bun.file('./src/parser/tokenizer.ts').stat()
const parserStat = await Bun.file('./src/parser/shrimp.ts').stat()
if (grammarStat.mtime <= parserStat.mtime && tokenizerStat.mtime <= parserStat.mtime) {
generate = false
}
} catch (e) {
console.error('Error checking or regenerating parser:', e)
} finally {
if (generate) {
await $`bun generate-parser`
}
}
}
await regenerateParser()
// Type declaration for TypeScript // Type declaration for TypeScript
declare module 'bun:test' { declare module 'bun:test' {
@ -18,9 +37,6 @@ declare module 'bun:test' {
toFailParse(): T toFailParse(): T
toEvaluateTo(expected: unknown, globals?: Record<string, any>): Promise<T> toEvaluateTo(expected: unknown, globals?: Record<string, any>): Promise<T>
toFailEvaluation(): Promise<T> toFailEvaluation(): Promise<T>
toBeToken(expected: string): T
toMatchToken(typeOrValue: string, value?: string): T
toMatchTokens(...tokens: { type: string, value?: string }[]): T
} }
} }
@ -30,8 +46,8 @@ expect.extend({
const allGlobals = { ...prelude, ...(globals || {}) } const allGlobals = { ...prelude, ...(globals || {}) }
setGlobals(Object.keys(allGlobals)) setGlobals(Object.keys(allGlobals))
const tree = parse(received) const tree = parser.parse(received)
const actual = treeToString2(tree, received) const actual = treeToString(tree, received)
const normalizedExpected = trimWhitespace(expected) const normalizedExpected = trimWhitespace(expected)
try { try {
@ -50,8 +66,7 @@ expect.extend({
assert(typeof received === 'string', 'toFailParse can only be used with string values') assert(typeof received === 'string', 'toFailParse can only be used with string values')
try { try {
const node = parse(received) const tree = parser.parse(received)
const tree = new Tree(node)
let hasErrors = false let hasErrors = false
tree.iterate({ tree.iterate({
enter(n) { enter(n) {
@ -68,7 +83,7 @@ expect.extend({
pass: true, pass: true,
} }
} else { } else {
const actual = treeToString2(node, received) const actual = treeToString(tree, received)
return { return {
message: () => `Expected input to fail parsing, but it parsed successfully:\n${actual}`, message: () => `Expected input to fail parsing, but it parsed successfully:\n${actual}`,
pass: false, pass: false,
@ -129,107 +144,8 @@ expect.extend({
} }
} }
}, },
toBeToken(received: unknown, expected: string) {
assert(typeof received === 'string', 'toBeToken can only be used with string values')
try {
const tokens = tokenize(received)
const value = tokens[0] as Token
const target = TokenType[expected as keyof typeof TokenType]
if (!value) {
return {
message: () => `Expected token type to be ${expected}, but got ${value}`,
pass: false,
}
}
return {
message: () => `Expected token type to be ${expected}, but got ${TokenType[value.type]}`,
pass: value.type === target
}
} catch (error) {
return {
message: () => `Tokenization failed: ${errorMessage(error)}`,
pass: false,
}
}
},
toMatchToken(received: unknown, typeOrValue: string, value?: string) {
assert(typeof received === 'string', 'toMatchToken can only be used with string values')
const expectedValue = value ? value : typeOrValue
const expectedType = value ? typeOrValue : undefined
try {
const tokens = tokenize(received)
const token = tokens[0] as Token
if (!token) {
return {
message: () => `Expected token to be ${expectedValue.replaceAll('\n', '\\n')}, got ${token}`,
pass: false,
}
}
if (expectedType && TokenType[expectedType as keyof typeof TokenType] !== token.type) {
return {
message: () => `Expected token to be ${expectedType}, but got ${TokenType[token.type]}`,
pass: false
}
}
return {
message: () => `Expected token to be ${expectedValue.replaceAll('\n', '\\n')}, but got ${token.value}`,
pass: token.value === expectedValue
}
} catch (error) {
return {
message: () => `Tokenization failed: ${errorMessage(error)} `,
pass: false,
}
}
},
toMatchTokens(received: unknown, ...tokens: { type: string, value?: string }[]) {
assert(typeof received === 'string', 'toMatchTokens can only be used with string values')
try {
const result = tokenize(received).map(t => toHumanToken(t))
if (result.length === 0 && tokens.length > 0) {
return {
message: () => `Expected tokens ${JSON.stringify(tokens)}, got nothing`,
pass: false,
}
}
const expected = JSON.stringify(tokens, null, 2)
const actual = JSON.stringify(result, null, 2)
return {
message: () => `Tokens don't match: \n\n${diff(actual, expected)}`,
pass: expected == actual
}
} catch (error) {
return {
message: () => `Tokenization failed: ${errorMessage(error)} `,
pass: false,
}
}
}
}) })
const tokenize = (code: string): Token[] => {
const scanner = new Scanner
return scanner.tokenize(code)
}
const toHumanToken = (tok: Token): { type: string, value?: string } => {
return {
type: TokenType[tok.type],
value: tok.value
}
}
const trimWhitespace = (str: string): string => { const trimWhitespace = (str: string): string => {
const lines = str.split('\n').filter((line) => line.trim().length > 0) const lines = str.split('\n').filter((line) => line.trim().length > 0)
const firstLine = lines[0] const firstLine = lines[0]
@ -248,26 +164,3 @@ const trimWhitespace = (str: string): string => {
}) })
.join('\n') .join('\n')
} }
const diff = (a: string, b: string): string => {
const expected = a.trim()
const actual = b.trim()
const lines = []
if (expected !== actual) {
const changes = diffLines(actual, expected)
for (const part of changes) {
const sign = part.added ? "+" : part.removed ? "-" : " "
let line = sign + part.value
if (part.added) {
line = color.green(line)
} else if (part.removed) {
line = color.red(line)
}
lines.push(line.endsWith("\n") || line.endsWith("\n\u001b[39m") ? line : line + "\n")
}
}
return lines.join('\n')
}

View File

@ -1,50 +1,5 @@
import { Tree, TreeCursor } from '@lezer/common' import { Tree, TreeCursor } from '@lezer/common'
import { type Value, fromValue } from 'reefvm' import { type Value, fromValue } from 'reefvm'
import { SyntaxNode } from '#parser/node'
const nodeToString = (node: SyntaxNode, input: string, depth = 0): string => {
const indent = ' '.repeat(depth)
const text = input.slice(node.from, node.to)
const nodeName = node.name
if (node.firstChild) {
return `${indent}${nodeName}`
} else {
// Only strip quotes from whole String nodes (legacy DoubleQuote), not StringFragment/EscapeSeq/CurlyString
let cleanText = nodeName === 'String' ? text.slice(1, -1) : text
if (cleanText === ' ') cleanText = '(space)'
return cleanText ? `${indent}${nodeName} ${cleanText}` : `${indent}${nodeName}`
}
}
export const treeToString2 = (tree: SyntaxNode, input: string, depth = 0): string => {
let lines = []
let node: SyntaxNode | null = tree
if (node.name === 'Program') node = node.firstChild
while (node) {
// If this node is an error, print ⚠ instead of its content
if (node.isError && !node.firstChild) {
lines.push(' '.repeat(depth) + '⚠')
} else {
lines.push(nodeToString(node, input, depth))
if (node.firstChild) {
lines.push(treeToString2(node.firstChild, input, depth + 1))
}
// If this node has an error, add ⚠ after its children
if (node.isError && node.firstChild) {
lines.push(' '.repeat(depth === 0 ? 0 : depth + 1) + '⚠')
}
}
node = node.nextSibling
}
return lines.join('\n')
}
export const treeToString = (tree: Tree, input: string): string => { export const treeToString = (tree: Tree, input: string): string => {
const lines: string[] = [] const lines: string[] = []

4
vscode-extension/.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
node_modules
client/dist
server/dist
*.vsix

View File

@ -1,12 +1,12 @@
import { TextDocument, Position } from 'vscode-languageserver-textdocument' import { TextDocument, Position } from 'vscode-languageserver-textdocument'
import { Diagnostic, DiagnosticSeverity } from 'vscode-languageserver/node' import { Diagnostic, DiagnosticSeverity } from 'vscode-languageserver/node'
import { Tree } from '@lezer/common' import { parser } from '../../../src/parser/shrimp'
import { Compiler } from '../../../src/compiler/compiler' import { Compiler } from '../../../src/compiler/compiler'
import { CompilerError } from '../../../src/compiler/compilerError' import { CompilerError } from '../../../src/compiler/compilerError'
export const buildDiagnostics = (textDocument: TextDocument, tree: Tree): Diagnostic[] => { export const buildDiagnostics = (textDocument: TextDocument): Diagnostic[] => {
const text = textDocument.getText() const text = textDocument.getText()
const diagnostics = getParseErrors(textDocument, tree) const diagnostics = getParseErrors(textDocument)
if (diagnostics.length > 0) { if (diagnostics.length > 0) {
return diagnostics return diagnostics
@ -59,7 +59,9 @@ const unknownDiagnostic = (message: string): Diagnostic => {
return diagnostic return diagnostic
} }
const getParseErrors = (textDocument: TextDocument, tree: Tree): Diagnostic[] => { const getParseErrors = (textDocument: TextDocument): Diagnostic[] => {
const tree = parser.parse(textDocument.getText())
const ranges: { start: Position; end: Position }[] = [] const ranges: { start: Position; end: Position }[] = []
tree.iterate({ tree.iterate({
enter(n) { enter(n) {

View File

@ -8,57 +8,6 @@ export type CompletionMetadata = {
export const completions = { export const completions = {
modules: { modules: {
"date": {
"now": {
"params": []
},
"year": {
"params": [
"time"
]
},
"month": {
"params": [
"time"
]
},
"date": {
"params": [
"time"
]
},
"hour": {
"params": [
"time"
]
},
"minute": {
"params": [
"time"
]
},
"second": {
"params": [
"time"
]
},
"ms": {
"params": [
"time"
]
},
"new": {
"params": [
"year",
"month",
"day",
"hour",
"minute",
"second",
"ms"
]
}
},
"dict": { "dict": {
"keys": { "keys": {
"params": [ "params": [
@ -580,10 +529,7 @@ export const completions = {
] ]
}, },
"random": { "random": {
"params": [ "params": []
"min",
"max"
]
}, },
"clamp": { "clamp": {
"params": [ "params": [
@ -739,16 +685,6 @@ export const completions = {
"pad" "pad"
] ]
}, },
"capitalize": {
"params": [
"str"
]
},
"titlecase": {
"params": [
"s"
]
},
"lines": { "lines": {
"params": [ "params": [
"str" "str"

View File

@ -6,9 +6,7 @@ export const PRELUDE_NAMES = [
"array?", "array?",
"at", "at",
"bnot", "bnot",
"boolean",
"boolean?", "boolean?",
"date",
"dec", "dec",
"describe", "describe",
"dict", "dict",
@ -26,18 +24,15 @@ export const PRELUDE_NAMES = [
"json", "json",
"length", "length",
"list", "list",
"list?",
"load", "load",
"math", "math",
"not", "not",
"null?", "null?",
"number",
"number?", "number?",
"range", "range",
"ref", "ref",
"some?", "some?",
"str", "str",
"string",
"string?", "string?",
"type", "type",
"var", "var",

View File

@ -1,6 +1,6 @@
import { parser } from '../../../src/parser/shrimp' import { parser } from '../../../src/parser/shrimp'
import * as Terms from '../../../src/parser/shrimp.terms' import * as Terms from '../../../src/parser/shrimp.terms'
import { SyntaxNode, Tree } from '@lezer/common' import { SyntaxNode } from '@lezer/common'
import { TextDocument } from 'vscode-languageserver-textdocument' import { TextDocument } from 'vscode-languageserver-textdocument'
import { import {
SemanticTokensBuilder, SemanticTokensBuilder,
@ -28,7 +28,9 @@ export const TOKEN_MODIFIERS = [
SemanticTokenModifiers.readonly, SemanticTokenModifiers.readonly,
] ]
export function buildSemanticTokens(document: TextDocument, tree: Tree): number[] { export function buildSemanticTokens(document: TextDocument): number[] {
const text = document.getText()
const tree = parser.parse(text)
const builder = new SemanticTokensBuilder() const builder = new SemanticTokensBuilder()
const scopeTracker = new EditorScopeAnalyzer(document) const scopeTracker = new EditorScopeAnalyzer(document)

View File

@ -7,7 +7,6 @@ import { PRELUDE_NAMES } from './metadata/prelude-names'
import { parser } from '../../../src/parser/shrimp' import { parser } from '../../../src/parser/shrimp'
import { setGlobals } from '../../../src/parser/tokenizer' import { setGlobals } from '../../../src/parser/tokenizer'
import { Compiler } from '../../../src/compiler/compiler' import { Compiler } from '../../../src/compiler/compiler'
import { Tree } from '@lezer/common'
import { import {
InitializeResult, InitializeResult,
TextDocuments, TextDocuments,
@ -15,9 +14,7 @@ import {
createConnection, createConnection,
ProposedFeatures, ProposedFeatures,
CompletionItemKind, CompletionItemKind,
TextDocumentChangeEvent,
} from 'vscode-languageserver/node' } from 'vscode-languageserver/node'
import { globals } from '../../../src/prelude'
// Initialize parser with prelude globals so it knows dict/list/str are in scope // Initialize parser with prelude globals so it knows dict/list/str are in scope
setGlobals(PRELUDE_NAMES) setGlobals(PRELUDE_NAMES)
@ -26,16 +23,12 @@ const connection = createConnection(ProposedFeatures.all)
const documents = new TextDocuments(TextDocument) const documents = new TextDocuments(TextDocument)
documents.listen(connection) documents.listen(connection)
const documentTrees = new Map<string, Tree>()
// Server capabilities // Server capabilities
connection.onInitialize(handleInitialize) connection.onInitialize(handleInitialize)
// Language features // Language features
connection.languages.semanticTokens.on(handleSemanticTokens) connection.languages.semanticTokens.on(handleSemanticTokens)
documents.onDidOpen(handleDocumentOpen)
documents.onDidChangeContent(handleDocumentChange) documents.onDidChangeContent(handleDocumentChange)
documents.onDidClose(handleDocumentClose)
connection.onCompletion(handleCompletion) connection.onCompletion(handleCompletion)
connection.onSignatureHelp(handleSignatureHelp) connection.onSignatureHelp(handleSignatureHelp)
@ -46,7 +39,10 @@ connection.onRequest('shrimp/bytecode', handleBytecode)
// Start listening // Start listening
connection.listen() connection.listen()
// ============================================================================
// Handler implementations // Handler implementations
// ============================================================================
function handleInitialize(): InitializeResult { function handleInitialize(): InitializeResult {
connection.console.log('🦐 Server initialized with capabilities') connection.console.log('🦐 Server initialized with capabilities')
const result: InitializeResult = { const result: InitializeResult = {
@ -71,42 +67,22 @@ function handleInitialize(): InitializeResult {
return result return result
} }
function handleDocumentOpen(event: TextDocumentChangeEvent<TextDocument>) {
const document = event.document
setGlobals(Object.keys(globals))
const tree = parser.parse(document.getText())
documentTrees.set(document.uri, tree)
}
function handleSemanticTokens(params: any) { function handleSemanticTokens(params: any) {
const document = documents.get(params.textDocument.uri) const document = documents.get(params.textDocument.uri)
if (!document) return { data: [] } if (!document) return { data: [] }
const tree = documentTrees.get(params.textDocument.uri) const data = buildSemanticTokens(document)
if (!tree) return { data: [] }
const data = buildSemanticTokens(document, tree)
return { data } return { data }
} }
function handleDocumentChange(change: TextDocumentChangeEvent<TextDocument>) { function handleDocumentChange(change: any) {
const document = change.document const textDocument = change.document
const diagnostics = buildDiagnostics(textDocument)
// Parse and cache connection.sendDiagnostics({ uri: textDocument.uri, diagnostics })
setGlobals(Object.keys(globals))
const tree = parser.parse(document.getText())
documentTrees.set(document.uri, tree)
// Build diagnostics using cached tree
const diagnostics = buildDiagnostics(document, tree)
connection.sendDiagnostics({ uri: document.uri, diagnostics })
}
function handleDocumentClose(event: TextDocumentChangeEvent<TextDocument>) {
documentTrees.delete(event.document.uri)
} }
function handleCompletion(params: any) { function handleCompletion(params: any) {
console.log(`🌭 YOU ARE COMPLETING at position`, params.position)
const document = documents.get(params.textDocument.uri) const document = documents.get(params.textDocument.uri)
if (!document) { if (!document) {
console.log('❌ No document found') console.log('❌ No document found')
@ -122,10 +98,7 @@ function handleCompletion(params: any) {
const contextCompletions = provideCompletions(document, position) const contextCompletions = provideCompletions(document, position)
console.log(`🎯 Context completions count: ${contextCompletions.length}`) console.log(`🎯 Context completions count: ${contextCompletions.length}`)
if (contextCompletions.length > 0) { if (contextCompletions.length > 0) {
console.log( console.log(`✅ Returning ${contextCompletions.length} completions:`, contextCompletions.map(c => c.label).join(', '))
`✅ Returning ${contextCompletions.length} completions:`,
contextCompletions.map((c) => c.label).join(', ')
)
return contextCompletions return contextCompletions
} }
@ -156,13 +129,8 @@ function handleParseTree(params: { uri: string }) {
const document = documents.get(params.uri) const document = documents.get(params.uri)
if (!document) return 'Document not found' if (!document) return 'Document not found'
const tree = documentTrees.get(params.uri)
if (!tree) {
connection.console.error(`🦐 No cached tree for ${params.uri}`)
return 'No cached parse tree available'
}
const text = document.getText() const text = document.getText()
const tree = parser.parse(text)
const cursor = tree.cursor() const cursor = tree.cursor()
let formatted = '' let formatted = ''

View File

@ -0,0 +1,41 @@
import { parser } from '../../src/parser/shrimp'
import { setGlobals } from '../../src/parser/tokenizer'
import { PRELUDE_NAMES } from '../server/src/prelude-names'
// Set globals for DotGet detection
setGlobals(PRELUDE_NAMES as unknown as string[])
// Test cases - does incomplete DotGet parse correctly?
const testCases = [
'dict.',
'dict.g',
'dict.get',
'$.',
'$.e',
'$.env',
]
for (const code of testCases) {
console.log(`\nTesting: "${code}"`)
const tree = parser.parse(code)
const cursor = tree.cursor()
// Print the parse tree
const printTree = (depth = 0) => {
const indent = ' '.repeat(depth)
console.log(`${indent}${cursor.name} [${cursor.from}-${cursor.to}]`)
if (cursor.firstChild()) {
do {
printTree(depth + 1)
} while (cursor.nextSibling())
cursor.parent()
}
}
printTree()
// Check at cursor position (end of string)
const node = tree.resolveInner(code.length, -1)
console.log(`Node at end: ${node.name} (type: ${node.type.id})`)
}