probablycorey/shrimp
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Parser 2.0 (Major Delezer) #52

Merged
defunkt merged 35 commits from parser2 into main 2025-12-08 16:35:34 +00:00
Conversation 4 Commits 35 Files Changed 33 +84 -36
6 changed files with 84 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 21e7ed41af - Show all commits

10
src/compiler/compiler.ts
View File

@ -66,12 +66,12 @@ export class Compiler {
if (globals) setGlobals(Array.isArray(globals) ? globals : Object.keys(globals))
const ast = parse(input)
const cst = new Tree(ast)
// const errors = checkTreeForErrors(cst)
const errors = checkTreeForErrors(cst)
// const firstError = errors[0]
// if (firstError) {
// throw firstError
// }
const firstError = errors[0]
if (firstError) {
throw firstError
}
this.#compileCst(cst, input)
this.bytecode = toBytecode(this.instructions)

20
src/compiler/utils.ts
View File

@ -5,13 +5,13 @@ import type { SyntaxNode, Tree } from '#parser/node'
export const checkTreeForErrors = (tree: Tree): CompilerError[] => {
const errors: CompilerError[] = []
// tree.iterate({
// enter: (node) => {
// if (node.type.isError) {
// errors.push(new CompilerError(`Unexpected syntax.`, node.from, node.to))
// }
// },
// })
tree.iterate({
enter: (node) => {
if (node.type.isError) {
errors.push(new CompilerError(`Unexpected syntax.`, node.from, node.to))
}
},
})
return errors
}
@ -58,8 +58,7 @@ export const getAssignmentParts = (node: SyntaxNode) => {
if (!left || left.type.id !== terms.AssignableIdentifier) {
throw new CompilerError(
`Assign left child must be an AssignableIdentifier or Array, got ${
left ? left.type.name : 'none'
`Assign left child must be an AssignableIdentifier or Array, got ${left ? left.type.name : 'none'
}`,
node.from,
node.to
@ -75,8 +74,7 @@ export const getCompoundAssignmentParts = (node: SyntaxNode) => {
if (!left || left.type.id !== terms.AssignableIdentifier) {
throw new CompilerError(
`CompoundAssign left child must be an AssignableIdentifier, got ${
left ? left.type.name : 'none'
`CompoundAssign left child must be an AssignableIdentifier, got ${left ? left.type.name : 'none'
}`,
node.from,
node.to

17
src/parser/node.ts
View File

@ -128,6 +128,15 @@ export class Tree {
node: this.topNode,
}
}
iterate(options: { enter: (node: SyntaxNode) => void }) {
const iter = (node: SyntaxNode) => {
for (const n of node.children) iter(n)
options.enter(node)
}
iter(this.topNode)
}
}
// TODO: TEMPORARY SHIM
@ -295,7 +304,6 @@ class SyntaxNodeType {
case 'keyword':
return term.keyword
}
return 0
}
@ -307,6 +315,7 @@ class SyntaxNodeType {
export class SyntaxNode {
#type: NodeType
#isError = false
from: number
to: number
parent: SyntaxNode | null
@ -336,7 +345,11 @@ export class SyntaxNode {
}
get isError(): boolean {
return false
return this.#isError
}
set isError(err: boolean) {
this.#isError = err
}
get firstChild(): SyntaxNode | null {

52
src/parser/parser2.ts
View File

@ -1,7 +1,9 @@
import { CompilerError } from '#compiler/compilerError'
import { Scanner, type Token, TokenType } from './tokenizer2'
import { SyntaxNode, operators, precedence, conditionals, compounds } from './node'
import { globals } from './tokenizer'
import { parseString } from './stringParser'
import { Compiler } from '#compiler/compiler'
const $T = TokenType
@ -256,6 +258,7 @@ export class Parser {
return val
} else {
const arg = new SyntaxNode('PositionalArg', val.from, val.to)
if (val.isError) arg.isError = true
arg.add(val)
return arg
}
@ -356,7 +359,7 @@ export class Parser {
return SyntaxNode.from(this.next())
const next = this.next()
throw `[atom] unexpected token ${TokenType[next.type]}: ${JSON.stringify(next)}\n\n ${this.input}\n`
throw new CompilerError(`Unexpected token: ${TokenType[next.type]}`, next.from, next.to)
}
// blocks in if, do, special calls, etc
@ -432,6 +435,7 @@ export class Parser {
// [ a=1 b=true c='three' ]
dict(): SyntaxNode {
const open = this.expect($T.OpenBracket)
let isError = false
// empty dict [=] or [ = ]
if (this.is($T.Operator, '=') && this.nextIs($T.CloseBracket)) {
@ -456,20 +460,29 @@ export class Parser {
if (this.nextIs($T.Operator, '=')) {
const ident = this.identifier()
const op = this.op('=')
const val = this.arg(true)
const prefix = new SyntaxNode('NamedArgPrefix', ident.from, op.to)
const node = new SyntaxNode('NamedArg', ident.from, val.to)
node.add(prefix)
node.add(val)
values.push(node)
if (this.is($T.CloseBracket) || this.is($T.Semicolon) || this.is($T.Newline)) {
const node = new SyntaxNode('NamedArg', ident.from, op.to)
node.isError = true
isError = true
values.push(node.push(prefix))
} else {
const val = this.arg(true)
const node = new SyntaxNode('NamedArg', ident.from, val.to)
values.push(node.push(prefix, val))
}
} else {
values.push(this.is($T.NamedArgPrefix) ? this.namedArg() : this.arg())
const arg = this.is($T.NamedArgPrefix) ? this.namedArg() : this.arg()
if (arg.isError) isError = true
values.push(arg)
}
}
const close = this.expect($T.CloseBracket)
const node = new SyntaxNode('Dict', open.from, close.to)
node.isError = isError
return node.push(...values)
}
@ -491,7 +504,7 @@ export class Parser {
else if (this.is($T.NamedArgPrefix))
arg = this.namedParam()
else
throw `[do] expected Identifier or NamedArgPrefix, got ${JSON.stringify(this.current())}\n\n ${this.input}\n`
throw new CompilerError(`Expected Identifier or NamedArgPrefix, got ${TokenType[this.current().type]}`, this.current().from, this.current().to)
params.push(arg)
}
@ -605,14 +618,20 @@ export class Parser {
// you're lookin at it
functionCall(fn?: SyntaxNode): SyntaxNode {
const ident = fn ?? this.identifier()
let isError = false
const args: SyntaxNode[] = []
while (!this.isExprEnd())
args.push(this.is($T.NamedArgPrefix) ? this.namedArg() : this.arg())
while (!this.isExprEnd()) {
const arg = this.is($T.NamedArgPrefix) ? this.namedArg() : this.arg()
if (arg.isError) isError = true
args.push(arg)
}
const node = new SyntaxNode('FunctionCall', ident.from, (args.at(-1) || ident).to)
node.push(ident, ...args)
if (isError) node.isError = true
if (!this.inTestExpr && this.is($T.Colon)) {
const block = this.block()
const end = this.keyword('end')
@ -718,6 +737,13 @@ export class Parser {
// abc= true
namedArg(): SyntaxNode {
const prefix = SyntaxNode.from(this.expect($T.NamedArgPrefix))
if (this.isExprEnd()) {
const node = new SyntaxNode('NamedArg', prefix.from, prefix.to)
node.isError = true
return node.push(prefix)
}
const val = this.arg(true)
const node = new SyntaxNode('NamedArg', prefix.from, val.to)
return node.push(prefix, val)
@ -729,7 +755,7 @@ export class Parser {
const val = this.value()
if (!['Null', 'Boolean', 'Number', 'String'].includes(val.type.name))
throw `[namedParam] default value must be Null|Bool|Num|Str, got ${val.type}\n\n ${this.input}\n`
throw new CompilerError(`Default value must be null, boolean, number, or string, got ${val.type.name}`, val.from, val.to)
const node = new SyntaxNode('NamedParam', prefix.from, val.to)
return node.push(prefix, val)
@ -739,7 +765,7 @@ export class Parser {
op(op?: string): SyntaxNode {
const token = op ? this.expect($T.Operator, op) : this.expect($T.Operator)
const name = operators[token.value!]
if (!name) throw `[op] operator not registered: ${token.value!}\n\n ${this.input}\n`
if (!name) throw new CompilerError(`Operator not registered: ${token.value!}`, token.from, token.to)
return new SyntaxNode(name, token.from, token.to)
}
@ -919,7 +945,7 @@ export class Parser {
expect(type: TokenType, value?: string): Token | never {
if (!this.is(type, value)) {
const token = this.current()
throw `expected ${TokenType[type]}${value ? ` "${value}"` : ''}, got ${TokenType[token?.type || 0]}${token?.value ? ` "${token.value}"` : ''} at position ${this.pos}\n\n ${this.input}\n`
throw new CompilerError(`Expected ${TokenType[type]}${value ? ` "${value}"` : ''}, got ${TokenType[token?.type || 0]}${token?.value ? ` "${token.value}"` : ''} at position ${this.pos}`, token.from, token.to)
}
return this.next()
}

4
src/parser/tests/functions.test.ts
View File

@ -109,14 +109,14 @@ describe('calling functions', () => {
`)
})
test.skip('Incomplete namedArg', () => {
test('Incomplete namedArg', () => {
expect('tail lines=').toMatchTree(`
FunctionCall
Identifier tail
NamedArg
NamedArgPrefix lines=
⚠
⚠ `)
⚠`)
})
})

17
src/utils/tree.ts
View File

@ -24,10 +24,21 @@ export const treeToString2 = (tree: SyntaxNode, input: string, depth = 0): strin
if (node.name === 'Program') node = node.firstChild
while (node) {
lines.push(nodeToString(node, input, depth))
// If this node is an error, print ⚠ instead of its content
if (node.isError && !node.firstChild) {
lines.push(' '.repeat(depth) + '⚠')
} else {
lines.push(nodeToString(node, input, depth))
if (node.firstChild)
lines.push(treeToString2(node.firstChild, input, depth + 1))
if (node.firstChild) {
lines.push(treeToString2(node.firstChild, input, depth + 1))
}
// If this node has an error, add ⚠ after its children
if (node.isError && node.firstChild) {
lines.push(' '.repeat(depth === 0 ? 0 : depth + 1) + '⚠')
}
}
node = node.nextSibling
}