100 lines
2.8 KiB
TypeScript
100 lines
2.8 KiB
TypeScript
import { ExternalTokenizer, InputStream } from '@lezer/lr'
|
|
import * as terms from './shrimp.terms'
|
|
|
|
type Operator = { str: string; tokenName: keyof typeof terms }
|
|
const operators: Array<Operator> = [
|
|
{ str: 'and', tokenName: 'And' },
|
|
{ str: 'or', tokenName: 'Or' },
|
|
{ str: 'band', tokenName: 'Band' },
|
|
{ str: 'bor', tokenName: 'Bor' },
|
|
{ str: 'bxor', tokenName: 'Bxor' },
|
|
{ str: '>>>', tokenName: 'Ushr' }, // Must come before >>
|
|
{ str: '>>', tokenName: 'Shr' },
|
|
{ str: '<<', tokenName: 'Shl' },
|
|
{ str: '>=', tokenName: 'Gte' },
|
|
{ str: '<=', tokenName: 'Lte' },
|
|
{ str: '!=', tokenName: 'Neq' },
|
|
{ str: '==', tokenName: 'EqEq' },
|
|
|
|
// Compound assignment operators (must come before single-char operators)
|
|
{ str: '??=', tokenName: 'NullishEq' },
|
|
{ str: '+=', tokenName: 'PlusEq' },
|
|
{ str: '-=', tokenName: 'MinusEq' },
|
|
{ str: '*=', tokenName: 'StarEq' },
|
|
{ str: '/=', tokenName: 'SlashEq' },
|
|
{ str: '%=', tokenName: 'ModuloEq' },
|
|
|
|
// Nullish coalescing (must come before it could be mistaken for other tokens)
|
|
{ str: '??', tokenName: 'NullishCoalesce' },
|
|
|
|
// Single-char operators
|
|
{ str: '*', tokenName: 'Star' },
|
|
{ str: '=', tokenName: 'Eq' },
|
|
{ str: '/', tokenName: 'Slash' },
|
|
{ str: '+', tokenName: 'Plus' },
|
|
{ str: '-', tokenName: 'Minus' },
|
|
{ str: '>', tokenName: 'Gt' },
|
|
{ str: '<', tokenName: 'Lt' },
|
|
{ str: '%', tokenName: 'Modulo' },
|
|
]
|
|
|
|
export const operatorTokenizer = new ExternalTokenizer((input: InputStream) => {
|
|
for (let operator of operators) {
|
|
if (!matchesString(input, 0, operator.str)) continue
|
|
const afterOpPos = operator.str.length
|
|
const charAfterOp = input.peek(afterOpPos)
|
|
if (!isWhitespace(charAfterOp)) continue
|
|
|
|
// Accept the operator token
|
|
const token = terms[operator.tokenName]
|
|
if (token === undefined) {
|
|
throw new Error(`Unknown token name: ${operator.tokenName}`)
|
|
}
|
|
|
|
input.advance(afterOpPos)
|
|
input.acceptToken(token)
|
|
|
|
return
|
|
}
|
|
})
|
|
|
|
const isWhitespace = (ch: number): boolean => {
|
|
return matchesChar(ch, [' ', '\t', '\n'])
|
|
}
|
|
|
|
const matchesChar = (ch: number, chars: (string | number)[]): boolean => {
|
|
for (const c of chars) {
|
|
if (typeof c === 'number') {
|
|
if (ch === c) {
|
|
return true
|
|
}
|
|
} else if (ch === c.charCodeAt(0)) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
const matchesString = (input: InputStream, pos: number, str: string): boolean => {
|
|
for (let i = 0; i < str.length; i++) {
|
|
if (input.peek(pos + i) !== str.charCodeAt(i)) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
const peek = (numChars: number, input: InputStream): string => {
|
|
let result = ''
|
|
for (let i = 0; i < numChars; i++) {
|
|
const ch = input.peek(i)
|
|
if (ch === -1) {
|
|
result += 'EOF'
|
|
break
|
|
} else {
|
|
result += String.fromCharCode(ch)
|
|
}
|
|
}
|
|
return result
|
|
}
|