Compare commits

..

17 Commits

Author SHA1 Message Date
b7a65e07dc fix test issues 2025-11-06 21:26:37 -08:00
8299022b4f fix edge case 2025-11-06 21:24:05 -08:00
131c943fc6 interpolation in { curly strings } 2025-11-06 21:24:03 -08:00
866da86862 curly -> Curly 2025-11-06 21:23:31 -08:00
5ac0b02044 { curly strings } 2025-11-06 21:23:31 -08:00
e0095b110f Merge pull request 'Whoops, I had two things called scopeTracker' (#26) from name-things-better into main
Reviewed-on: #26
2025-11-07 00:58:36 +00:00
a38932a833 Merge remote-tracking branch 'origin/main' into name-things-better 2025-11-06 16:58:16 -08:00
03596aab5b Merge branch 'name-things-better' of 54.219.130.253:probablycorey/shrimp into name-things-better 2025-11-06 16:58:12 -08:00
bd1dbe75f3 Merge pull request 'Make the extension know about the prelude' (#25) from extension-update into main
Reviewed-on: #25
2025-11-07 00:57:47 +00:00
669e58b71e Merge branch 'extension-update' of 54.219.130.253:probablycorey/shrimp into extension-update 2025-11-06 16:57:21 -08:00
152aac269f Merge remote-tracking branch 'origin/main' into extension-update 2025-11-06 16:56:34 -08:00
a428e98d41 Merge branch 'main' into extension-update 2025-11-07 00:56:08 +00:00
061452a334 Better names 2025-11-06 10:22:37 -08:00
4494cbce91 Revert "use string.quoted for strings (for now) (until defunkt gets a new vscode theme)"
This reverts commit 47c3fda4c8.
2025-11-06 10:22:37 -08:00
47d1ea1a0b use string.quoted for strings (for now) (until defunkt gets a new vscode theme) 2025-11-06 10:22:37 -08:00
82a97c0a5a Better names 2025-11-06 10:21:02 -08:00
ab12212df2 Make the extension know about the prelude 2025-11-06 09:23:18 -08:00
12 changed files with 49 additions and 18 deletions

View File

@ -200,7 +200,7 @@ function parseExpression(input: string) {
- **Not in scope** → Parses as `Word("obj.prop")` → compiles to `PUSH 'obj.prop'` (treated as file path/string) - **Not in scope** → Parses as `Word("obj.prop")` → compiles to `PUSH 'obj.prop'` (treated as file path/string)
Implementation files: Implementation files:
- **src/parser/scopeTracker.ts**: ContextTracker that maintains immutable scope chain - **src/parser/parserScopeContext.ts**: ContextTracker that maintains immutable scope chain
- **src/parser/tokenizer.ts**: External tokenizer checks `stack.context` to decide if dot creates DotGet or Word - **src/parser/tokenizer.ts**: External tokenizer checks `stack.context` to decide if dot creates DotGet or Word
- Scope tracking: Captures variables from assignments (`x = 5`) and function parameters (`fn x:`) - Scope tracking: Captures variables from assignments (`x = 5`) and function parameters (`fn x:`)
- See `src/parser/tests/dot-get.test.ts` for comprehensive examples - See `src/parser/tests/dot-get.test.ts` for comprehensive examples

View File

@ -78,4 +78,18 @@ describe('pipe expressions', () => {
div = do a b: a / b end div = do a b: a / b end
sub 3 1 | div (sub 110 9 | sub 1) _ | div 5`).toEvaluateTo(10) sub 3 1 | div (sub 110 9 | sub 1) _ | div 5`).toEvaluateTo(10)
}) })
test('pipe with prelude functions (list.reverse and list.map)', () => {
expect(`
double = do x: x * 2 end
range 1 3 | list.reverse | list.map double
`).toEvaluateTo([6, 4, 2])
})
test('pipe with prelude function (echo)', () => {
expect(`
get-msg = do: 'hello' end
get-msg | length
`).toEvaluateTo(5)
})
}) })

View File

@ -83,7 +83,7 @@ end
test('custom tags', () => { test('custom tags', () => {
expect(` expect(`
list = tag ul class=list list = tag ul class='list'
ribbit: ribbit:
list: list:
li border-bottom='1px solid black' one li border-bottom='1px solid black' one

View File

@ -1,6 +1,6 @@
@external propSource highlighting from "./highlight" @external propSource highlighting from "./highlight"
@context trackScope from "./scopeTracker" @context trackScope from "./parserScopeContext"
@skip { space | Comment } @skip { space | Comment }

View File

@ -2,7 +2,7 @@
import {LRParser, LocalTokenGroup} from "@lezer/lr" import {LRParser, LocalTokenGroup} from "@lezer/lr"
import {operatorTokenizer} from "./operatorTokenizer" import {operatorTokenizer} from "./operatorTokenizer"
import {tokenizer, specializeKeyword} from "./tokenizer" import {tokenizer, specializeKeyword} from "./tokenizer"
import {trackScope} from "./scopeTracker" import {trackScope} from "./parserScopeContext"
import {highlighting} from "./highlight" import {highlighting} from "./highlight"
const spec_Identifier = {__proto__:null,if:68, null:96, catch:102, finally:108, end:110, else:118, while:132, try:138, throw:142} const spec_Identifier = {__proto__:null,if:68, null:96, catch:102, finally:108, end:110, else:118, while:132, try:138, throw:142}
export const parser = LRParser.deserialize({ export const parser = LRParser.deserialize({

View File

@ -189,7 +189,9 @@ const checkForDotGet = (input: InputStream, stack: Stack, pos: number): number |
// If identifier is in scope, this is property access (e.g., obj.prop) // If identifier is in scope, this is property access (e.g., obj.prop)
// If not in scope, it should be consumed as a Word (e.g., file.txt) // If not in scope, it should be consumed as a Word (e.g., file.txt)
return context?.scope.has(identifierText) || globals.includes(identifierText) ? IdentifierBeforeDot : null return context?.scope.has(identifierText) || globals.includes(identifierText)
? IdentifierBeforeDot
: null
} }
// Decide between AssignableIdentifier and Identifier using grammar state + peek-ahead // Decide between AssignableIdentifier and Identifier using grammar state + peek-ahead
@ -217,7 +219,10 @@ const chooseIdentifierToken = (input: InputStream, stack: Stack): number => {
const nextCh2 = getFullCodePoint(input, peekPos + 1) const nextCh2 = getFullCodePoint(input, peekPos + 1)
// Check for compound assignment operators: +=, -=, *=, /=, %= // Check for compound assignment operators: +=, -=, *=, /=, %=
if ([43/* + */, 45/* - */, 42/* * */, 47/* / */, 37/* % */].includes(nextCh) && nextCh2 === 61/* = */) { if (
[43 /* + */, 45 /* - */, 42 /* * */, 47 /* / */, 37 /* % */].includes(nextCh) &&
nextCh2 === 61 /* = */
) {
// Found compound operator, check if it's followed by whitespace // Found compound operator, check if it's followed by whitespace
const charAfterOp = getFullCodePoint(input, peekPos + 2) const charAfterOp = getFullCodePoint(input, peekPos + 2)
if (isWhiteSpace(charAfterOp) || charAfterOp === -1 /* EOF */) { if (isWhiteSpace(charAfterOp) || charAfterOp === -1 /* EOF */) {

View File

@ -1,6 +1,7 @@
import { expect } from 'bun:test' import { expect } from 'bun:test'
import { parser } from '#parser/shrimp' import { parser } from '#parser/shrimp'
import { setGlobals } from '#parser/tokenizer' import { setGlobals } from '#parser/tokenizer'
import { globals as prelude } from '#prelude'
import { $ } from 'bun' import { $ } from 'bun'
import { assert, errorMessage } from '#utils/utils' import { assert, errorMessage } from '#utils/utils'
import { Compiler } from '#compiler/compiler' import { Compiler } from '#compiler/compiler'
@ -43,7 +44,8 @@ expect.extend({
toMatchTree(received: unknown, expected: string, globals?: Record<string, any>) { toMatchTree(received: unknown, expected: string, globals?: Record<string, any>) {
assert(typeof received === 'string', 'toMatchTree can only be used with string values') assert(typeof received === 'string', 'toMatchTree can only be used with string values')
if (globals) setGlobals(Object.keys(globals)) const allGlobals = { ...prelude, ...(globals || {}) }
setGlobals(Object.keys(allGlobals))
const tree = parser.parse(received) const tree = parser.parse(received)
const actual = treeToString(tree, received) const actual = treeToString(tree, received)
const normalizedExpected = trimWhitespace(expected) const normalizedExpected = trimWhitespace(expected)
@ -99,9 +101,10 @@ expect.extend({
assert(typeof received === 'string', 'toEvaluateTo can only be used with string values') assert(typeof received === 'string', 'toEvaluateTo can only be used with string values')
try { try {
if (globals) setGlobals(Object.keys(globals)) const allGlobals = { ...prelude, ...(globals || {}) }
setGlobals(Object.keys(allGlobals))
const compiler = new Compiler(received) const compiler = new Compiler(received)
const result = await run(compiler.bytecode, globals) const result = await run(compiler.bytecode, allGlobals)
let value = VMResultToValue(result) let value = VMResultToValue(result)
// Just treat regex as strings for comparison purposes // Just treat regex as strings for comparison purposes

View File

@ -50,4 +50,4 @@ describe('Shrimp', () => {
await shrimp.run('abc = nothing') await shrimp.run('abc = nothing')
expect(shrimp.get('abc')).toEqual('nothing') expect(shrimp.get('abc')).toEqual('nothing')
}) })
}) })

View File

@ -1,10 +1,10 @@
import { test, expect, describe } from 'bun:test' import { test, expect, describe } from 'bun:test'
import { ScopeTracker } from './scopeTracker' import { EditorScopeAnalyzer } from './editorScopeAnalyzer'
import { TextDocument } from 'vscode-languageserver-textdocument' import { TextDocument } from 'vscode-languageserver-textdocument'
import { parser } from '../../../src/parser/shrimp' import { parser } from '../../../src/parser/shrimp'
import * as Terms from '../../../src/parser/shrimp.terms' import * as Terms from '../../../src/parser/shrimp.terms'
describe('ScopeTracker', () => { describe('EditorScopeAnalyzer', () => {
test('top-level assignment is in scope', () => { test('top-level assignment is in scope', () => {
const code = 'x = 5\necho x' const code = 'x = 5\necho x'
const { tree, tracker } = parseAndGetScope(code) const { tree, tracker } = parseAndGetScope(code)
@ -135,11 +135,17 @@ end`
const xInEcho = identifiers[identifiers.length - 1] const xInEcho = identifiers[identifiers.length - 1]
expect(tracker.isInScope('x', xInEcho)).toBe(true) expect(tracker.isInScope('x', xInEcho)).toBe(true)
}) })
test('the prelude functions are always in scope', () => {
const code = `echo "Hello, World!"`
const { tree, tracker } = parseAndGetScope(code)
expect(tracker.isInScope('echo', tree.topNode)).toBe(true)
})
}) })
const parseAndGetScope = (code: string) => { const parseAndGetScope = (code: string) => {
const document = TextDocument.create('test://test.sh', 'shrimp', 1, code) const document = TextDocument.create('test://test.sh', 'shrimp', 1, code)
const tree = parser.parse(code) const tree = parser.parse(code)
const tracker = new ScopeTracker(document) const tracker = new EditorScopeAnalyzer(document)
return { document, tree, tracker } return { document, tree, tracker }
} }

View File

@ -1,17 +1,20 @@
import { SyntaxNode } from '@lezer/common' import { SyntaxNode } from '@lezer/common'
import { TextDocument } from 'vscode-languageserver-textdocument' import { TextDocument } from 'vscode-languageserver-textdocument'
import * as Terms from '../../../src/parser/shrimp.terms' import * as Terms from '../../../src/parser/shrimp.terms'
import { globals } from '../../../src/prelude'
/** /**
* Tracks variables in scope at a given position in the parse tree. * Tracks variables in scope at a given position in the parse tree.
* Used to distinguish identifiers (in scope) from words (not in scope). * Used to distinguish identifiers (in scope) from words (not in scope).
*/ */
export class ScopeTracker { export class EditorScopeAnalyzer {
private document: TextDocument private document: TextDocument
private scopeCache = new Map<number, Set<string>>() private scopeCache = new Map<number, Set<string>>()
constructor(document: TextDocument) { constructor(document: TextDocument) {
this.document = document this.document = document
const preludeKeys = Object.keys(globals)
this.scopeCache.set(0, new Set(preludeKeys))
} }
/** /**

View File

@ -7,7 +7,7 @@ import {
SemanticTokenTypes, SemanticTokenTypes,
SemanticTokenModifiers, SemanticTokenModifiers,
} from 'vscode-languageserver/node' } from 'vscode-languageserver/node'
import { ScopeTracker } from './scopeTracker' import { EditorScopeAnalyzer } from './editorScopeAnalyzer'
export const TOKEN_TYPES = [ export const TOKEN_TYPES = [
SemanticTokenTypes.function, SemanticTokenTypes.function,
@ -32,7 +32,7 @@ export function buildSemanticTokens(document: TextDocument): number[] {
const text = document.getText() const text = document.getText()
const tree = parser.parse(text) const tree = parser.parse(text)
const builder = new SemanticTokensBuilder() const builder = new SemanticTokensBuilder()
const scopeTracker = new ScopeTracker(document) const scopeTracker = new EditorScopeAnalyzer(document)
walkTree(tree.topNode, document, builder, scopeTracker) walkTree(tree.topNode, document, builder, scopeTracker)
@ -77,7 +77,7 @@ function walkTree(
node: SyntaxNode, node: SyntaxNode,
document: TextDocument, document: TextDocument,
builder: SemanticTokensBuilder, builder: SemanticTokensBuilder,
scopeTracker: ScopeTracker scopeTracker: EditorScopeAnalyzer
) { ) {
// Special handling for NamedArgPrefix to split "name=" into two tokens // Special handling for NamedArgPrefix to split "name=" into two tokens
if (node.type.id === Terms.NamedArgPrefix) { if (node.type.id === Terms.NamedArgPrefix) {
@ -104,7 +104,7 @@ type TokenInfo = { type: number; modifiers: number } | undefined
function getTokenType( function getTokenType(
node: SyntaxNode, node: SyntaxNode,
document: TextDocument, document: TextDocument,
scopeTracker: ScopeTracker scopeTracker: EditorScopeAnalyzer
): TokenInfo { ): TokenInfo {
const nodeTypeId = node.type.id const nodeTypeId = node.type.id
const parentTypeId = node.parent?.type.id const parentTypeId = node.parent?.type.id