feat(parser): add scope-aware dot operator tokenization
🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
219397339c
commit
7e819f9c67
|
|
@ -1,5 +1,6 @@
|
||||||
import { ExternalTokenizer, InputStream, Stack } from '@lezer/lr'
|
import { ExternalTokenizer, InputStream, Stack } from '@lezer/lr'
|
||||||
import { Identifier, Word } from './shrimp.terms'
|
import { Identifier, Word } from './shrimp.terms'
|
||||||
|
import type { Scope } from './scopeTracker'
|
||||||
|
|
||||||
// The only chars that can't be words are whitespace, apostrophes, closing parens, and EOF.
|
// The only chars that can't be words are whitespace, apostrophes, closing parens, and EOF.
|
||||||
|
|
||||||
|
|
@ -14,6 +15,20 @@ export const tokenizer = new ExternalTokenizer((input: InputStream, stack: Stack
|
||||||
while (true) {
|
while (true) {
|
||||||
ch = getFullCodePoint(input, pos)
|
ch = getFullCodePoint(input, pos)
|
||||||
|
|
||||||
|
// Check for dot and scope - property access detection
|
||||||
|
if (ch === 46 /* . */ && isValidIdentifier) {
|
||||||
|
const identifierText = input.read(0, pos)
|
||||||
|
const scope = stack.context as Scope | undefined
|
||||||
|
|
||||||
|
if (scope?.has(identifierText)) {
|
||||||
|
// In scope - stop here, let grammar parse property access
|
||||||
|
input.advance(pos)
|
||||||
|
input.acceptToken(Identifier)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Not in scope - continue consuming as Word (fall through)
|
||||||
|
}
|
||||||
|
|
||||||
if (!isWordChar(ch)) break
|
if (!isWordChar(ch)) break
|
||||||
|
|
||||||
// Certain characters might end a word or identifier if they are followed by whitespace.
|
// Certain characters might end a word or identifier if they are followed by whitespace.
|
||||||
|
|
@ -34,7 +49,7 @@ export const tokenizer = new ExternalTokenizer((input: InputStream, stack: Stack
|
||||||
|
|
||||||
input.advance(pos)
|
input.advance(pos)
|
||||||
input.acceptToken(isValidIdentifier ? Identifier : Word)
|
input.acceptToken(isValidIdentifier ? Identifier : Word)
|
||||||
})
|
}, { contextual: true })
|
||||||
|
|
||||||
const isWhiteSpace = (ch: number): boolean => {
|
const isWhiteSpace = (ch: number): boolean => {
|
||||||
return ch === 32 /* space */ || ch === 10 /* \n */ || ch === 9 /* tab */ || ch === 13 /* \r */
|
return ch === 32 /* space */ || ch === 10 /* \n */ || ch === 9 /* tab */ || ch === 13 /* \r */
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user