import { ExternalTokenizer, InputStream } from '@lezer/lr' import * as terms from './shrimp.terms' type Operator = { str: string; tokenName: keyof typeof terms } const operators: Array = [ { str: 'and', tokenName: 'And' }, { str: 'or', tokenName: 'Or' }, { str: 'band', tokenName: 'Band' }, { str: 'bor', tokenName: 'Bor' }, { str: 'bxor', tokenName: 'Bxor' }, { str: '>>>', tokenName: 'Ushr' }, // Must come before >> { str: '>>', tokenName: 'Shr' }, { str: '<<', tokenName: 'Shl' }, { str: '>=', tokenName: 'Gte' }, { str: '<=', tokenName: 'Lte' }, { str: '!=', tokenName: 'Neq' }, { str: '==', tokenName: 'EqEq' }, // Compound assignment operators (must come before single-char operators) { str: '+=', tokenName: 'PlusEq' }, { str: '-=', tokenName: 'MinusEq' }, { str: '*=', tokenName: 'StarEq' }, { str: '/=', tokenName: 'SlashEq' }, { str: '%=', tokenName: 'ModuloEq' }, // Single-char operators { str: '*', tokenName: 'Star' }, { str: '=', tokenName: 'Eq' }, { str: '/', tokenName: 'Slash' }, { str: '+', tokenName: 'Plus' }, { str: '-', tokenName: 'Minus' }, { str: '>', tokenName: 'Gt' }, { str: '<', tokenName: 'Lt' }, { str: '%', tokenName: 'Modulo' }, ] export const operatorTokenizer = new ExternalTokenizer((input: InputStream) => { for (let operator of operators) { if (!matchesString(input, 0, operator.str)) continue const afterOpPos = operator.str.length const charAfterOp = input.peek(afterOpPos) if (!isWhitespace(charAfterOp)) continue // Accept the operator token const token = terms[operator.tokenName] if (token === undefined) { throw new Error(`Unknown token name: ${operator.tokenName}`) } input.advance(afterOpPos) input.acceptToken(token) return } }) const isWhitespace = (ch: number): boolean => { return matchesChar(ch, [' ', '\t', '\n']) } const matchesChar = (ch: number, chars: (string | number)[]): boolean => { for (const c of chars) { if (typeof c === 'number') { if (ch === c) { return true } } else if (ch === c.charCodeAt(0)) { return true } } return false } const matchesString = (input: InputStream, pos: number, str: string): boolean => { for (let i = 0; i < str.length; i++) { if (input.peek(pos + i) !== str.charCodeAt(i)) { return false } } return true } const peek = (numChars: number, input: InputStream): string => { let result = '' for (let i = 0; i < numChars; i++) { const ch = input.peek(i) if (ch === -1) { result += 'EOF' break } else { result += String.fromCharCode(ch) } } return result }