From ab12212df2afe7decd4277d192bf36d253ff1c9c Mon Sep 17 00:00:00 2001 From: Corey Johnson Date: Thu, 6 Nov 2025 09:23:18 -0800 Subject: [PATCH] Make the extension know about the prelude --- src/compiler/tests/pipe.test.ts | 14 ++++++++++++++ src/parser/tokenizer.ts | 17 ++++++++++++++--- src/testSetup.ts | 9 ++++++--- src/tests/shrimp.test.ts | 2 +- .../server/src/scopeTracker.test.ts | 6 ++++++ vscode-extension/server/src/scopeTracker.ts | 3 +++ 6 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/compiler/tests/pipe.test.ts b/src/compiler/tests/pipe.test.ts index 5c0aab8..1d08dec 100644 --- a/src/compiler/tests/pipe.test.ts +++ b/src/compiler/tests/pipe.test.ts @@ -78,4 +78,18 @@ describe('pipe expressions', () => { div = do a b: a / b end sub 3 1 | div (sub 110 9 | sub 1) _ | div 5`).toEvaluateTo(10) }) + + test('pipe with prelude functions (list.reverse and list.map)', () => { + expect(` + double = do x: x * 2 end + range 1 3 | list.reverse | list.map double + `).toEvaluateTo([6, 4, 2]) + }) + + test('pipe with prelude function (echo)', () => { + expect(` + get-msg = do: 'hello' end + get-msg | echo + `).toEvaluateTo(null) + }) }) diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts index cbaac67..ba8da48 100644 --- a/src/parser/tokenizer.ts +++ b/src/parser/tokenizer.ts @@ -119,7 +119,13 @@ const consumeWordToken = ( } // Track identifier validity: must be lowercase, digit, dash, or emoji/unicode - if (!isLowercaseLetter(ch) && !isDigit(ch) && ch !== 45 /* - */ && ch !== 63 /* ? */ && !isEmojiOrUnicode(ch)) { + if ( + !isLowercaseLetter(ch) && + !isDigit(ch) && + ch !== 45 /* - */ && + ch !== 63 /* ? */ && + !isEmojiOrUnicode(ch) + ) { if (!canBeWord) break isValidIdentifier = false } @@ -159,7 +165,9 @@ const checkForDotGet = (input: InputStream, stack: Stack, pos: number): number | // If identifier is in scope, this is property access (e.g., obj.prop) // If not in scope, it should be consumed as a Word (e.g., file.txt) - return context?.scope.has(identifierText) || globals.includes(identifierText) ? IdentifierBeforeDot : null + return context?.scope.has(identifierText) || globals.includes(identifierText) + ? IdentifierBeforeDot + : null } // Decide between AssignableIdentifier and Identifier using grammar state + peek-ahead @@ -187,7 +195,10 @@ const chooseIdentifierToken = (input: InputStream, stack: Stack): number => { const nextCh2 = getFullCodePoint(input, peekPos + 1) // Check for compound assignment operators: +=, -=, *=, /=, %= - if ([43/* + */, 45/* - */, 42/* * */, 47/* / */, 37/* % */].includes(nextCh) && nextCh2 === 61/* = */) { + if ( + [43 /* + */, 45 /* - */, 42 /* * */, 47 /* / */, 37 /* % */].includes(nextCh) && + nextCh2 === 61 /* = */ + ) { // Found compound operator, check if it's followed by whitespace const charAfterOp = getFullCodePoint(input, peekPos + 2) if (isWhiteSpace(charAfterOp) || charAfterOp === -1 /* EOF */) { diff --git a/src/testSetup.ts b/src/testSetup.ts index d2c1652..c476ba2 100644 --- a/src/testSetup.ts +++ b/src/testSetup.ts @@ -1,6 +1,7 @@ import { expect } from 'bun:test' import { parser } from '#parser/shrimp' import { setGlobals } from '#parser/tokenizer' +import { globals as prelude } from '#prelude' import { $ } from 'bun' import { assert, errorMessage } from '#utils/utils' import { Compiler } from '#compiler/compiler' @@ -43,7 +44,8 @@ expect.extend({ toMatchTree(received: unknown, expected: string, globals?: Record) { assert(typeof received === 'string', 'toMatchTree can only be used with string values') - if (globals) setGlobals(Object.keys(globals)) + const allGlobals = { ...prelude, ...(globals || {}) } + setGlobals(Object.keys(allGlobals)) const tree = parser.parse(received) const actual = treeToString(tree, received) const normalizedExpected = trimWhitespace(expected) @@ -99,9 +101,10 @@ expect.extend({ assert(typeof received === 'string', 'toEvaluateTo can only be used with string values') try { - if (globals) setGlobals(Object.keys(globals)) + const allGlobals = { ...prelude, ...(globals || {}) } + setGlobals(Object.keys(allGlobals)) const compiler = new Compiler(received) - const result = await run(compiler.bytecode, globals) + const result = await run(compiler.bytecode, allGlobals) let value = VMResultToValue(result) // Just treat regex as strings for comparison purposes diff --git a/src/tests/shrimp.test.ts b/src/tests/shrimp.test.ts index 3f7a6f8..19a0553 100644 --- a/src/tests/shrimp.test.ts +++ b/src/tests/shrimp.test.ts @@ -50,4 +50,4 @@ describe('Shrimp', () => { await shrimp.run('abc = nothing') expect(shrimp.get('abc')).toEqual('nothing') }) -}) \ No newline at end of file +}) diff --git a/vscode-extension/server/src/scopeTracker.test.ts b/vscode-extension/server/src/scopeTracker.test.ts index 9604e36..8f53d41 100644 --- a/vscode-extension/server/src/scopeTracker.test.ts +++ b/vscode-extension/server/src/scopeTracker.test.ts @@ -135,6 +135,12 @@ end` const xInEcho = identifiers[identifiers.length - 1] expect(tracker.isInScope('x', xInEcho)).toBe(true) }) + + test('the prelude functions are always in scope', () => { + const code = `echo "Hello, World!"` + const { tree, tracker } = parseAndGetScope(code) + expect(tracker.isInScope('echo', tree.topNode)).toBe(true) + }) }) const parseAndGetScope = (code: string) => { diff --git a/vscode-extension/server/src/scopeTracker.ts b/vscode-extension/server/src/scopeTracker.ts index 70ebf73..094d5c0 100644 --- a/vscode-extension/server/src/scopeTracker.ts +++ b/vscode-extension/server/src/scopeTracker.ts @@ -1,6 +1,7 @@ import { SyntaxNode } from '@lezer/common' import { TextDocument } from 'vscode-languageserver-textdocument' import * as Terms from '../../../src/parser/shrimp.terms' +import { globals } from '../../../src/prelude' /** * Tracks variables in scope at a given position in the parse tree. @@ -12,6 +13,8 @@ export class ScopeTracker { constructor(document: TextDocument) { this.document = document + const preludeKeys = Object.keys(globals) + this.scopeCache.set(0, new Set(preludeKeys)) } /** -- 2.50.1