From ab12212df2afe7decd4277d192bf36d253ff1c9c Mon Sep 17 00:00:00 2001 From: Corey Johnson Date: Thu, 6 Nov 2025 09:23:18 -0800 Subject: [PATCH 1/5] Make the extension know about the prelude --- src/compiler/tests/pipe.test.ts | 14 ++++++++++++++ src/parser/tokenizer.ts | 17 ++++++++++++++--- src/testSetup.ts | 9 ++++++--- src/tests/shrimp.test.ts | 2 +- .../server/src/scopeTracker.test.ts | 6 ++++++ vscode-extension/server/src/scopeTracker.ts | 3 +++ 6 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/compiler/tests/pipe.test.ts b/src/compiler/tests/pipe.test.ts index 5c0aab8..1d08dec 100644 --- a/src/compiler/tests/pipe.test.ts +++ b/src/compiler/tests/pipe.test.ts @@ -78,4 +78,18 @@ describe('pipe expressions', () => { div = do a b: a / b end sub 3 1 | div (sub 110 9 | sub 1) _ | div 5`).toEvaluateTo(10) }) + + test('pipe with prelude functions (list.reverse and list.map)', () => { + expect(` + double = do x: x * 2 end + range 1 3 | list.reverse | list.map double + `).toEvaluateTo([6, 4, 2]) + }) + + test('pipe with prelude function (echo)', () => { + expect(` + get-msg = do: 'hello' end + get-msg | echo + `).toEvaluateTo(null) + }) }) diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts index cbaac67..ba8da48 100644 --- a/src/parser/tokenizer.ts +++ b/src/parser/tokenizer.ts @@ -119,7 +119,13 @@ const consumeWordToken = ( } // Track identifier validity: must be lowercase, digit, dash, or emoji/unicode - if (!isLowercaseLetter(ch) && !isDigit(ch) && ch !== 45 /* - */ && ch !== 63 /* ? */ && !isEmojiOrUnicode(ch)) { + if ( + !isLowercaseLetter(ch) && + !isDigit(ch) && + ch !== 45 /* - */ && + ch !== 63 /* ? */ && + !isEmojiOrUnicode(ch) + ) { if (!canBeWord) break isValidIdentifier = false } @@ -159,7 +165,9 @@ const checkForDotGet = (input: InputStream, stack: Stack, pos: number): number | // If identifier is in scope, this is property access (e.g., obj.prop) // If not in scope, it should be consumed as a Word (e.g., file.txt) - return context?.scope.has(identifierText) || globals.includes(identifierText) ? IdentifierBeforeDot : null + return context?.scope.has(identifierText) || globals.includes(identifierText) + ? IdentifierBeforeDot + : null } // Decide between AssignableIdentifier and Identifier using grammar state + peek-ahead @@ -187,7 +195,10 @@ const chooseIdentifierToken = (input: InputStream, stack: Stack): number => { const nextCh2 = getFullCodePoint(input, peekPos + 1) // Check for compound assignment operators: +=, -=, *=, /=, %= - if ([43/* + */, 45/* - */, 42/* * */, 47/* / */, 37/* % */].includes(nextCh) && nextCh2 === 61/* = */) { + if ( + [43 /* + */, 45 /* - */, 42 /* * */, 47 /* / */, 37 /* % */].includes(nextCh) && + nextCh2 === 61 /* = */ + ) { // Found compound operator, check if it's followed by whitespace const charAfterOp = getFullCodePoint(input, peekPos + 2) if (isWhiteSpace(charAfterOp) || charAfterOp === -1 /* EOF */) { diff --git a/src/testSetup.ts b/src/testSetup.ts index d2c1652..c476ba2 100644 --- a/src/testSetup.ts +++ b/src/testSetup.ts @@ -1,6 +1,7 @@ import { expect } from 'bun:test' import { parser } from '#parser/shrimp' import { setGlobals } from '#parser/tokenizer' +import { globals as prelude } from '#prelude' import { $ } from 'bun' import { assert, errorMessage } from '#utils/utils' import { Compiler } from '#compiler/compiler' @@ -43,7 +44,8 @@ expect.extend({ toMatchTree(received: unknown, expected: string, globals?: Record) { assert(typeof received === 'string', 'toMatchTree can only be used with string values') - if (globals) setGlobals(Object.keys(globals)) + const allGlobals = { ...prelude, ...(globals || {}) } + setGlobals(Object.keys(allGlobals)) const tree = parser.parse(received) const actual = treeToString(tree, received) const normalizedExpected = trimWhitespace(expected) @@ -99,9 +101,10 @@ expect.extend({ assert(typeof received === 'string', 'toEvaluateTo can only be used with string values') try { - if (globals) setGlobals(Object.keys(globals)) + const allGlobals = { ...prelude, ...(globals || {}) } + setGlobals(Object.keys(allGlobals)) const compiler = new Compiler(received) - const result = await run(compiler.bytecode, globals) + const result = await run(compiler.bytecode, allGlobals) let value = VMResultToValue(result) // Just treat regex as strings for comparison purposes diff --git a/src/tests/shrimp.test.ts b/src/tests/shrimp.test.ts index 3f7a6f8..19a0553 100644 --- a/src/tests/shrimp.test.ts +++ b/src/tests/shrimp.test.ts @@ -50,4 +50,4 @@ describe('Shrimp', () => { await shrimp.run('abc = nothing') expect(shrimp.get('abc')).toEqual('nothing') }) -}) \ No newline at end of file +}) diff --git a/vscode-extension/server/src/scopeTracker.test.ts b/vscode-extension/server/src/scopeTracker.test.ts index 9604e36..8f53d41 100644 --- a/vscode-extension/server/src/scopeTracker.test.ts +++ b/vscode-extension/server/src/scopeTracker.test.ts @@ -135,6 +135,12 @@ end` const xInEcho = identifiers[identifiers.length - 1] expect(tracker.isInScope('x', xInEcho)).toBe(true) }) + + test('the prelude functions are always in scope', () => { + const code = `echo "Hello, World!"` + const { tree, tracker } = parseAndGetScope(code) + expect(tracker.isInScope('echo', tree.topNode)).toBe(true) + }) }) const parseAndGetScope = (code: string) => { diff --git a/vscode-extension/server/src/scopeTracker.ts b/vscode-extension/server/src/scopeTracker.ts index 70ebf73..094d5c0 100644 --- a/vscode-extension/server/src/scopeTracker.ts +++ b/vscode-extension/server/src/scopeTracker.ts @@ -1,6 +1,7 @@ import { SyntaxNode } from '@lezer/common' import { TextDocument } from 'vscode-languageserver-textdocument' import * as Terms from '../../../src/parser/shrimp.terms' +import { globals } from '../../../src/prelude' /** * Tracks variables in scope at a given position in the parse tree. @@ -12,6 +13,8 @@ export class ScopeTracker { constructor(document: TextDocument) { this.document = document + const preludeKeys = Object.keys(globals) + this.scopeCache.set(0, new Set(preludeKeys)) } /** From 82a97c0a5a83c3e8f9d73cc688af6d4c28f027a3 Mon Sep 17 00:00:00 2001 From: Corey Johnson Date: Thu, 6 Nov 2025 10:21:02 -0800 Subject: [PATCH 2/5] Better names --- CLAUDE.md | 2 +- src/parser/{scopeTracker.ts => parserScopeContext.ts} | 0 src/parser/shrimp.grammar | 2 +- src/parser/shrimp.ts | 2 +- .../{scopeTracker.test.ts => editorScopeAnalyzer.test.ts} | 6 +++--- .../src/{scopeTracker.ts => editorScopeAnalyzer.ts} | 2 +- vscode-extension/server/src/semanticTokens.ts | 8 ++++---- 7 files changed, 11 insertions(+), 11 deletions(-) rename src/parser/{scopeTracker.ts => parserScopeContext.ts} (100%) rename vscode-extension/server/src/{scopeTracker.test.ts => editorScopeAnalyzer.test.ts} (96%) rename vscode-extension/server/src/{scopeTracker.ts => editorScopeAnalyzer.ts} (99%) diff --git a/CLAUDE.md b/CLAUDE.md index e0e372d..ab367e6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -200,7 +200,7 @@ function parseExpression(input: string) { - **Not in scope** → Parses as `Word("obj.prop")` → compiles to `PUSH 'obj.prop'` (treated as file path/string) Implementation files: -- **src/parser/scopeTracker.ts**: ContextTracker that maintains immutable scope chain +- **src/parser/parserScopeContext.ts**: ContextTracker that maintains immutable scope chain - **src/parser/tokenizer.ts**: External tokenizer checks `stack.context` to decide if dot creates DotGet or Word - Scope tracking: Captures variables from assignments (`x = 5`) and function parameters (`fn x:`) - See `src/parser/tests/dot-get.test.ts` for comprehensive examples diff --git a/src/parser/scopeTracker.ts b/src/parser/parserScopeContext.ts similarity index 100% rename from src/parser/scopeTracker.ts rename to src/parser/parserScopeContext.ts diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index e9bc2ec..ac38814 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -1,6 +1,6 @@ @external propSource highlighting from "./highlight" -@context trackScope from "./scopeTracker" +@context trackScope from "./parserScopeContext" @skip { space | Comment } diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index afd774f..ffb958e 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -2,7 +2,7 @@ import {LRParser, LocalTokenGroup} from "@lezer/lr" import {operatorTokenizer} from "./operatorTokenizer" import {tokenizer, specializeKeyword} from "./tokenizer" -import {trackScope} from "./scopeTracker" +import {trackScope} from "./parserScopeContext" import {highlighting} from "./highlight" const spec_Identifier = {__proto__:null,if:66, null:94, catch:100, finally:106, end:108, else:116, while:130, try:136, throw:140} export const parser = LRParser.deserialize({ diff --git a/vscode-extension/server/src/scopeTracker.test.ts b/vscode-extension/server/src/editorScopeAnalyzer.test.ts similarity index 96% rename from vscode-extension/server/src/scopeTracker.test.ts rename to vscode-extension/server/src/editorScopeAnalyzer.test.ts index 9604e36..1a41d0a 100644 --- a/vscode-extension/server/src/scopeTracker.test.ts +++ b/vscode-extension/server/src/editorScopeAnalyzer.test.ts @@ -1,10 +1,10 @@ import { test, expect, describe } from 'bun:test' -import { ScopeTracker } from './scopeTracker' +import { EditorScopeAnalyzer } from './editorScopeAnalyzer' import { TextDocument } from 'vscode-languageserver-textdocument' import { parser } from '../../../src/parser/shrimp' import * as Terms from '../../../src/parser/shrimp.terms' -describe('ScopeTracker', () => { +describe('EditorScopeAnalyzer', () => { test('top-level assignment is in scope', () => { const code = 'x = 5\necho x' const { tree, tracker } = parseAndGetScope(code) @@ -140,6 +140,6 @@ end` const parseAndGetScope = (code: string) => { const document = TextDocument.create('test://test.sh', 'shrimp', 1, code) const tree = parser.parse(code) - const tracker = new ScopeTracker(document) + const tracker = new EditorScopeAnalyzer(document) return { document, tree, tracker } } diff --git a/vscode-extension/server/src/scopeTracker.ts b/vscode-extension/server/src/editorScopeAnalyzer.ts similarity index 99% rename from vscode-extension/server/src/scopeTracker.ts rename to vscode-extension/server/src/editorScopeAnalyzer.ts index 70ebf73..711704c 100644 --- a/vscode-extension/server/src/scopeTracker.ts +++ b/vscode-extension/server/src/editorScopeAnalyzer.ts @@ -6,7 +6,7 @@ import * as Terms from '../../../src/parser/shrimp.terms' * Tracks variables in scope at a given position in the parse tree. * Used to distinguish identifiers (in scope) from words (not in scope). */ -export class ScopeTracker { +export class EditorScopeAnalyzer { private document: TextDocument private scopeCache = new Map>() diff --git a/vscode-extension/server/src/semanticTokens.ts b/vscode-extension/server/src/semanticTokens.ts index d8fc0ea..3aaca34 100644 --- a/vscode-extension/server/src/semanticTokens.ts +++ b/vscode-extension/server/src/semanticTokens.ts @@ -7,7 +7,7 @@ import { SemanticTokenTypes, SemanticTokenModifiers, } from 'vscode-languageserver/node' -import { ScopeTracker } from './scopeTracker' +import { EditorScopeAnalyzer } from './editorScopeAnalyzer' export const TOKEN_TYPES = [ SemanticTokenTypes.function, @@ -32,7 +32,7 @@ export function buildSemanticTokens(document: TextDocument): number[] { const text = document.getText() const tree = parser.parse(text) const builder = new SemanticTokensBuilder() - const scopeTracker = new ScopeTracker(document) + const scopeTracker = new EditorScopeAnalyzer(document) walkTree(tree.topNode, document, builder, scopeTracker) @@ -77,7 +77,7 @@ function walkTree( node: SyntaxNode, document: TextDocument, builder: SemanticTokensBuilder, - scopeTracker: ScopeTracker + scopeTracker: EditorScopeAnalyzer ) { // Special handling for NamedArgPrefix to split "name=" into two tokens if (node.type.id === Terms.NamedArgPrefix) { @@ -104,7 +104,7 @@ type TokenInfo = { type: number; modifiers: number } | undefined function getTokenType( node: SyntaxNode, document: TextDocument, - scopeTracker: ScopeTracker + scopeTracker: EditorScopeAnalyzer ): TokenInfo { const nodeTypeId = node.type.id const parentTypeId = node.parent?.type.id From 47d1ea1a0b2944f88eb96436f2f1d6af1f00c0ec Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Thu, 6 Nov 2025 09:28:13 -0800 Subject: [PATCH 3/5] use string.quoted for strings (for now) (until defunkt gets a new vscode theme) --- vscode-extension/server/src/semanticTokens.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/vscode-extension/server/src/semanticTokens.ts b/vscode-extension/server/src/semanticTokens.ts index d8fc0ea..95f60fd 100644 --- a/vscode-extension/server/src/semanticTokens.ts +++ b/vscode-extension/server/src/semanticTokens.ts @@ -12,7 +12,7 @@ import { ScopeTracker } from './scopeTracker' export const TOKEN_TYPES = [ SemanticTokenTypes.function, SemanticTokenTypes.variable, - SemanticTokenTypes.string, + 'string.quoted', SemanticTokenTypes.number, SemanticTokenTypes.operator, SemanticTokenTypes.keyword, @@ -161,7 +161,7 @@ function getTokenType( // If not in scope, treat as string (like a Word) if (!scopeTracker.isInScope(identifierText, node)) { return { - type: TOKEN_TYPES.indexOf(SemanticTokenTypes.string), + type: TOKEN_TYPES.indexOf('string.quoted'), modifiers: 0, } } @@ -188,9 +188,14 @@ function getTokenType( case Terms.String: case Terms.StringFragment: + return { + type: TOKEN_TYPES.indexOf('string.quoted'), + modifiers: 0, + } + case Terms.Word: return { - type: TOKEN_TYPES.indexOf(SemanticTokenTypes.string), + type: TOKEN_TYPES.indexOf('string.quoted'), modifiers: 0, } From 4494cbce916bf7b319e86bb0d24c0f956b841a7b Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Thu, 6 Nov 2025 09:31:56 -0800 Subject: [PATCH 4/5] Revert "use string.quoted for strings (for now) (until defunkt gets a new vscode theme)" This reverts commit 47c3fda4c860b692c5c70a754937e3edea5f523a. --- vscode-extension/server/src/semanticTokens.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/vscode-extension/server/src/semanticTokens.ts b/vscode-extension/server/src/semanticTokens.ts index 95f60fd..d8fc0ea 100644 --- a/vscode-extension/server/src/semanticTokens.ts +++ b/vscode-extension/server/src/semanticTokens.ts @@ -12,7 +12,7 @@ import { ScopeTracker } from './scopeTracker' export const TOKEN_TYPES = [ SemanticTokenTypes.function, SemanticTokenTypes.variable, - 'string.quoted', + SemanticTokenTypes.string, SemanticTokenTypes.number, SemanticTokenTypes.operator, SemanticTokenTypes.keyword, @@ -161,7 +161,7 @@ function getTokenType( // If not in scope, treat as string (like a Word) if (!scopeTracker.isInScope(identifierText, node)) { return { - type: TOKEN_TYPES.indexOf('string.quoted'), + type: TOKEN_TYPES.indexOf(SemanticTokenTypes.string), modifiers: 0, } } @@ -188,14 +188,9 @@ function getTokenType( case Terms.String: case Terms.StringFragment: - return { - type: TOKEN_TYPES.indexOf('string.quoted'), - modifiers: 0, - } - case Terms.Word: return { - type: TOKEN_TYPES.indexOf('string.quoted'), + type: TOKEN_TYPES.indexOf(SemanticTokenTypes.string), modifiers: 0, } From 061452a334f3495a6798f4528f078e87550c030a Mon Sep 17 00:00:00 2001 From: Corey Johnson Date: Thu, 6 Nov 2025 10:21:02 -0800 Subject: [PATCH 5/5] Better names --- CLAUDE.md | 2 +- src/parser/{scopeTracker.ts => parserScopeContext.ts} | 0 src/parser/shrimp.grammar | 2 +- src/parser/shrimp.ts | 2 +- .../{scopeTracker.test.ts => editorScopeAnalyzer.test.ts} | 6 +++--- .../src/{scopeTracker.ts => editorScopeAnalyzer.ts} | 2 +- vscode-extension/server/src/semanticTokens.ts | 8 ++++---- 7 files changed, 11 insertions(+), 11 deletions(-) rename src/parser/{scopeTracker.ts => parserScopeContext.ts} (100%) rename vscode-extension/server/src/{scopeTracker.test.ts => editorScopeAnalyzer.test.ts} (96%) rename vscode-extension/server/src/{scopeTracker.ts => editorScopeAnalyzer.ts} (99%) diff --git a/CLAUDE.md b/CLAUDE.md index e0e372d..ab367e6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -200,7 +200,7 @@ function parseExpression(input: string) { - **Not in scope** → Parses as `Word("obj.prop")` → compiles to `PUSH 'obj.prop'` (treated as file path/string) Implementation files: -- **src/parser/scopeTracker.ts**: ContextTracker that maintains immutable scope chain +- **src/parser/parserScopeContext.ts**: ContextTracker that maintains immutable scope chain - **src/parser/tokenizer.ts**: External tokenizer checks `stack.context` to decide if dot creates DotGet or Word - Scope tracking: Captures variables from assignments (`x = 5`) and function parameters (`fn x:`) - See `src/parser/tests/dot-get.test.ts` for comprehensive examples diff --git a/src/parser/scopeTracker.ts b/src/parser/parserScopeContext.ts similarity index 100% rename from src/parser/scopeTracker.ts rename to src/parser/parserScopeContext.ts diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index e9bc2ec..ac38814 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -1,6 +1,6 @@ @external propSource highlighting from "./highlight" -@context trackScope from "./scopeTracker" +@context trackScope from "./parserScopeContext" @skip { space | Comment } diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index afd774f..ffb958e 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -2,7 +2,7 @@ import {LRParser, LocalTokenGroup} from "@lezer/lr" import {operatorTokenizer} from "./operatorTokenizer" import {tokenizer, specializeKeyword} from "./tokenizer" -import {trackScope} from "./scopeTracker" +import {trackScope} from "./parserScopeContext" import {highlighting} from "./highlight" const spec_Identifier = {__proto__:null,if:66, null:94, catch:100, finally:106, end:108, else:116, while:130, try:136, throw:140} export const parser = LRParser.deserialize({ diff --git a/vscode-extension/server/src/scopeTracker.test.ts b/vscode-extension/server/src/editorScopeAnalyzer.test.ts similarity index 96% rename from vscode-extension/server/src/scopeTracker.test.ts rename to vscode-extension/server/src/editorScopeAnalyzer.test.ts index 8f53d41..10043c2 100644 --- a/vscode-extension/server/src/scopeTracker.test.ts +++ b/vscode-extension/server/src/editorScopeAnalyzer.test.ts @@ -1,10 +1,10 @@ import { test, expect, describe } from 'bun:test' -import { ScopeTracker } from './scopeTracker' +import { EditorScopeAnalyzer } from './editorScopeAnalyzer' import { TextDocument } from 'vscode-languageserver-textdocument' import { parser } from '../../../src/parser/shrimp' import * as Terms from '../../../src/parser/shrimp.terms' -describe('ScopeTracker', () => { +describe('EditorScopeAnalyzer', () => { test('top-level assignment is in scope', () => { const code = 'x = 5\necho x' const { tree, tracker } = parseAndGetScope(code) @@ -146,6 +146,6 @@ end` const parseAndGetScope = (code: string) => { const document = TextDocument.create('test://test.sh', 'shrimp', 1, code) const tree = parser.parse(code) - const tracker = new ScopeTracker(document) + const tracker = new EditorScopeAnalyzer(document) return { document, tree, tracker } } diff --git a/vscode-extension/server/src/scopeTracker.ts b/vscode-extension/server/src/editorScopeAnalyzer.ts similarity index 99% rename from vscode-extension/server/src/scopeTracker.ts rename to vscode-extension/server/src/editorScopeAnalyzer.ts index 094d5c0..449c3df 100644 --- a/vscode-extension/server/src/scopeTracker.ts +++ b/vscode-extension/server/src/editorScopeAnalyzer.ts @@ -7,7 +7,7 @@ import { globals } from '../../../src/prelude' * Tracks variables in scope at a given position in the parse tree. * Used to distinguish identifiers (in scope) from words (not in scope). */ -export class ScopeTracker { +export class EditorScopeAnalyzer { private document: TextDocument private scopeCache = new Map>() diff --git a/vscode-extension/server/src/semanticTokens.ts b/vscode-extension/server/src/semanticTokens.ts index d8fc0ea..3aaca34 100644 --- a/vscode-extension/server/src/semanticTokens.ts +++ b/vscode-extension/server/src/semanticTokens.ts @@ -7,7 +7,7 @@ import { SemanticTokenTypes, SemanticTokenModifiers, } from 'vscode-languageserver/node' -import { ScopeTracker } from './scopeTracker' +import { EditorScopeAnalyzer } from './editorScopeAnalyzer' export const TOKEN_TYPES = [ SemanticTokenTypes.function, @@ -32,7 +32,7 @@ export function buildSemanticTokens(document: TextDocument): number[] { const text = document.getText() const tree = parser.parse(text) const builder = new SemanticTokensBuilder() - const scopeTracker = new ScopeTracker(document) + const scopeTracker = new EditorScopeAnalyzer(document) walkTree(tree.topNode, document, builder, scopeTracker) @@ -77,7 +77,7 @@ function walkTree( node: SyntaxNode, document: TextDocument, builder: SemanticTokensBuilder, - scopeTracker: ScopeTracker + scopeTracker: EditorScopeAnalyzer ) { // Special handling for NamedArgPrefix to split "name=" into two tokens if (node.type.id === Terms.NamedArgPrefix) { @@ -104,7 +104,7 @@ type TokenInfo = { type: number; modifiers: number } | undefined function getTokenType( node: SyntaxNode, document: TextDocument, - scopeTracker: ScopeTracker + scopeTracker: EditorScopeAnalyzer ): TokenInfo { const nodeTypeId = node.type.id const parentTypeId = node.parent?.type.id