From 47c3fda4c860b692c5c70a754937e3edea5f523a Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Thu, 6 Nov 2025 09:28:13 -0800 Subject: [PATCH 1/3] use string.quoted for strings (for now) (until defunkt gets a new vscode theme) --- vscode-extension/server/src/semanticTokens.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/vscode-extension/server/src/semanticTokens.ts b/vscode-extension/server/src/semanticTokens.ts index d8fc0ea..95f60fd 100644 --- a/vscode-extension/server/src/semanticTokens.ts +++ b/vscode-extension/server/src/semanticTokens.ts @@ -12,7 +12,7 @@ import { ScopeTracker } from './scopeTracker' export const TOKEN_TYPES = [ SemanticTokenTypes.function, SemanticTokenTypes.variable, - SemanticTokenTypes.string, + 'string.quoted', SemanticTokenTypes.number, SemanticTokenTypes.operator, SemanticTokenTypes.keyword, @@ -161,7 +161,7 @@ function getTokenType( // If not in scope, treat as string (like a Word) if (!scopeTracker.isInScope(identifierText, node)) { return { - type: TOKEN_TYPES.indexOf(SemanticTokenTypes.string), + type: TOKEN_TYPES.indexOf('string.quoted'), modifiers: 0, } } @@ -188,9 +188,14 @@ function getTokenType( case Terms.String: case Terms.StringFragment: + return { + type: TOKEN_TYPES.indexOf('string.quoted'), + modifiers: 0, + } + case Terms.Word: return { - type: TOKEN_TYPES.indexOf(SemanticTokenTypes.string), + type: TOKEN_TYPES.indexOf('string.quoted'), modifiers: 0, } From 7645efc4f952997425a7dc096630c139c836b2b4 Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Thu, 6 Nov 2025 09:31:56 -0800 Subject: [PATCH 2/3] Revert "use string.quoted for strings (for now) (until defunkt gets a new vscode theme)" This reverts commit 47c3fda4c860b692c5c70a754937e3edea5f523a. --- vscode-extension/server/src/semanticTokens.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/vscode-extension/server/src/semanticTokens.ts b/vscode-extension/server/src/semanticTokens.ts index 95f60fd..d8fc0ea 100644 --- a/vscode-extension/server/src/semanticTokens.ts +++ b/vscode-extension/server/src/semanticTokens.ts @@ -12,7 +12,7 @@ import { ScopeTracker } from './scopeTracker' export const TOKEN_TYPES = [ SemanticTokenTypes.function, SemanticTokenTypes.variable, - 'string.quoted', + SemanticTokenTypes.string, SemanticTokenTypes.number, SemanticTokenTypes.operator, SemanticTokenTypes.keyword, @@ -161,7 +161,7 @@ function getTokenType( // If not in scope, treat as string (like a Word) if (!scopeTracker.isInScope(identifierText, node)) { return { - type: TOKEN_TYPES.indexOf('string.quoted'), + type: TOKEN_TYPES.indexOf(SemanticTokenTypes.string), modifiers: 0, } } @@ -188,14 +188,9 @@ function getTokenType( case Terms.String: case Terms.StringFragment: - return { - type: TOKEN_TYPES.indexOf('string.quoted'), - modifiers: 0, - } - case Terms.Word: return { - type: TOKEN_TYPES.indexOf('string.quoted'), + type: TOKEN_TYPES.indexOf(SemanticTokenTypes.string), modifiers: 0, } From 82a97c0a5a83c3e8f9d73cc688af6d4c28f027a3 Mon Sep 17 00:00:00 2001 From: Corey Johnson Date: Thu, 6 Nov 2025 10:21:02 -0800 Subject: [PATCH 3/3] Better names --- CLAUDE.md | 2 +- src/parser/{scopeTracker.ts => parserScopeContext.ts} | 0 src/parser/shrimp.grammar | 2 +- src/parser/shrimp.ts | 2 +- .../{scopeTracker.test.ts => editorScopeAnalyzer.test.ts} | 6 +++--- .../src/{scopeTracker.ts => editorScopeAnalyzer.ts} | 2 +- vscode-extension/server/src/semanticTokens.ts | 8 ++++---- 7 files changed, 11 insertions(+), 11 deletions(-) rename src/parser/{scopeTracker.ts => parserScopeContext.ts} (100%) rename vscode-extension/server/src/{scopeTracker.test.ts => editorScopeAnalyzer.test.ts} (96%) rename vscode-extension/server/src/{scopeTracker.ts => editorScopeAnalyzer.ts} (99%) diff --git a/CLAUDE.md b/CLAUDE.md index e0e372d..ab367e6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -200,7 +200,7 @@ function parseExpression(input: string) { - **Not in scope** → Parses as `Word("obj.prop")` → compiles to `PUSH 'obj.prop'` (treated as file path/string) Implementation files: -- **src/parser/scopeTracker.ts**: ContextTracker that maintains immutable scope chain +- **src/parser/parserScopeContext.ts**: ContextTracker that maintains immutable scope chain - **src/parser/tokenizer.ts**: External tokenizer checks `stack.context` to decide if dot creates DotGet or Word - Scope tracking: Captures variables from assignments (`x = 5`) and function parameters (`fn x:`) - See `src/parser/tests/dot-get.test.ts` for comprehensive examples diff --git a/src/parser/scopeTracker.ts b/src/parser/parserScopeContext.ts similarity index 100% rename from src/parser/scopeTracker.ts rename to src/parser/parserScopeContext.ts diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index e9bc2ec..ac38814 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -1,6 +1,6 @@ @external propSource highlighting from "./highlight" -@context trackScope from "./scopeTracker" +@context trackScope from "./parserScopeContext" @skip { space | Comment } diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index afd774f..ffb958e 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -2,7 +2,7 @@ import {LRParser, LocalTokenGroup} from "@lezer/lr" import {operatorTokenizer} from "./operatorTokenizer" import {tokenizer, specializeKeyword} from "./tokenizer" -import {trackScope} from "./scopeTracker" +import {trackScope} from "./parserScopeContext" import {highlighting} from "./highlight" const spec_Identifier = {__proto__:null,if:66, null:94, catch:100, finally:106, end:108, else:116, while:130, try:136, throw:140} export const parser = LRParser.deserialize({ diff --git a/vscode-extension/server/src/scopeTracker.test.ts b/vscode-extension/server/src/editorScopeAnalyzer.test.ts similarity index 96% rename from vscode-extension/server/src/scopeTracker.test.ts rename to vscode-extension/server/src/editorScopeAnalyzer.test.ts index 9604e36..1a41d0a 100644 --- a/vscode-extension/server/src/scopeTracker.test.ts +++ b/vscode-extension/server/src/editorScopeAnalyzer.test.ts @@ -1,10 +1,10 @@ import { test, expect, describe } from 'bun:test' -import { ScopeTracker } from './scopeTracker' +import { EditorScopeAnalyzer } from './editorScopeAnalyzer' import { TextDocument } from 'vscode-languageserver-textdocument' import { parser } from '../../../src/parser/shrimp' import * as Terms from '../../../src/parser/shrimp.terms' -describe('ScopeTracker', () => { +describe('EditorScopeAnalyzer', () => { test('top-level assignment is in scope', () => { const code = 'x = 5\necho x' const { tree, tracker } = parseAndGetScope(code) @@ -140,6 +140,6 @@ end` const parseAndGetScope = (code: string) => { const document = TextDocument.create('test://test.sh', 'shrimp', 1, code) const tree = parser.parse(code) - const tracker = new ScopeTracker(document) + const tracker = new EditorScopeAnalyzer(document) return { document, tree, tracker } } diff --git a/vscode-extension/server/src/scopeTracker.ts b/vscode-extension/server/src/editorScopeAnalyzer.ts similarity index 99% rename from vscode-extension/server/src/scopeTracker.ts rename to vscode-extension/server/src/editorScopeAnalyzer.ts index 70ebf73..711704c 100644 --- a/vscode-extension/server/src/scopeTracker.ts +++ b/vscode-extension/server/src/editorScopeAnalyzer.ts @@ -6,7 +6,7 @@ import * as Terms from '../../../src/parser/shrimp.terms' * Tracks variables in scope at a given position in the parse tree. * Used to distinguish identifiers (in scope) from words (not in scope). */ -export class ScopeTracker { +export class EditorScopeAnalyzer { private document: TextDocument private scopeCache = new Map>() diff --git a/vscode-extension/server/src/semanticTokens.ts b/vscode-extension/server/src/semanticTokens.ts index d8fc0ea..3aaca34 100644 --- a/vscode-extension/server/src/semanticTokens.ts +++ b/vscode-extension/server/src/semanticTokens.ts @@ -7,7 +7,7 @@ import { SemanticTokenTypes, SemanticTokenModifiers, } from 'vscode-languageserver/node' -import { ScopeTracker } from './scopeTracker' +import { EditorScopeAnalyzer } from './editorScopeAnalyzer' export const TOKEN_TYPES = [ SemanticTokenTypes.function, @@ -32,7 +32,7 @@ export function buildSemanticTokens(document: TextDocument): number[] { const text = document.getText() const tree = parser.parse(text) const builder = new SemanticTokensBuilder() - const scopeTracker = new ScopeTracker(document) + const scopeTracker = new EditorScopeAnalyzer(document) walkTree(tree.topNode, document, builder, scopeTracker) @@ -77,7 +77,7 @@ function walkTree( node: SyntaxNode, document: TextDocument, builder: SemanticTokensBuilder, - scopeTracker: ScopeTracker + scopeTracker: EditorScopeAnalyzer ) { // Special handling for NamedArgPrefix to split "name=" into two tokens if (node.type.id === Terms.NamedArgPrefix) { @@ -104,7 +104,7 @@ type TokenInfo = { type: number; modifiers: number } | undefined function getTokenType( node: SyntaxNode, document: TextDocument, - scopeTracker: ScopeTracker + scopeTracker: EditorScopeAnalyzer ): TokenInfo { const nodeTypeId = node.type.id const parentTypeId = node.parent?.type.id