probablycorey/shrimp
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(tokenizer): use canShift to emit AssignableIdentifier vs Identifier

Browse Source
This commit is contained in:
Corey Johnson 2025-10-17 18:34:57 -07:00
parent b2c5db77b2
commit 1e6fabf954
1 changed files with 11 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/parser/tokenizer.ts
View File

@ -1,5 +1,5 @@
import { ExternalTokenizer, InputStream, Stack } from '@lezer/lr' import { ExternalTokenizer, InputStream, Stack } from '@lezer/lr'
import { Identifier, Word, IdentifierBeforeDot } from './shrimp.terms' import { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot } from './shrimp.terms'
import type { Scope } from './scopeTracker' import type { Scope } from './scopeTracker'
// The only chars that can't be words are whitespace, apostrophes, closing parens, and EOF. // The only chars that can't be words are whitespace, apostrophes, closing parens, and EOF.
@ -66,7 +66,16 @@ export const tokenizer = new ExternalTokenizer(
} }
input.advance(pos) input.advance(pos)
input.acceptToken(isValidIdentifier ? Identifier : Word) if (isValidIdentifier) {
// Use canShift to decide which identifier type
if (stack.canShift(AssignableIdentifier)) {
input.acceptToken(AssignableIdentifier)
} else {
input.acceptToken(Identifier)
}
} else {
input.acceptToken(Word)
}
}, },
{ contextual: true } { contextual: true }
) )