diff --git a/src/compiler/utils.ts b/src/compiler/utils.ts index 312962f..cbdd1bc 100644 --- a/src/compiler/utils.ts +++ b/src/compiler/utils.ts @@ -70,9 +70,9 @@ export const getFunctionDefParts = (node: SyntaxNode, input: string) => { } const paramNames = getAllChildren(paramsNode).map((param) => { - if (param.type.id !== terms.AssignableIdentifier) { + if (param.type.id !== terms.Identifier) { throw new CompilerError( - `FunctionDef params must be AssignableIdentifiers, got ${param.type.name}`, + `FunctionDef params must be Identifier, got ${param.type.name}`, param.from, param.to ) diff --git a/src/parser/scopeTracker.ts b/src/parser/scopeTracker.ts index 8854cad..af2a32c 100644 --- a/src/parser/scopeTracker.ts +++ b/src/parser/scopeTracker.ts @@ -57,26 +57,30 @@ const readIdentifierText = (input: InputStream, start: number, end: number): str return text } +let inParams = false + export const trackScope = new ContextTracker({ start: new TrackerContext(new Scope(null, new Set())), shift(context, term, stack, input) { - if (term !== terms.AssignableIdentifier) return context + if (term == terms.Do) inParams = true - const text = readIdentifierText(input, input.pos, stack.pos) - return new TrackerContext(context.scope, [...context.pendingIds, text]) + if (term === terms.AssignableIdentifier) { + const text = readIdentifierText(input, input.pos, stack.pos) + return new TrackerContext(Scope.add(context.scope, text), context.pendingIds) + } + + if (inParams && term === terms.Identifier) { + const text = readIdentifierText(input, input.pos, stack.pos) + return new TrackerContext(context.scope, [...context.pendingIds, text]) + } + + return context }, reduce(context, term) { - // Add assignment variable to scope - if (term === terms.Assign) { - const varName = context.pendingIds.at(-1) - if (!varName) return context - return new TrackerContext(Scope.add(context.scope, varName), context.pendingIds.slice(0, -1)) - } - - // Push new scope and add all parameters if (term === terms.Params) { + inParams = false let newScope = context.scope.push() if (context.pendingIds.length > 0) { newScope = Scope.add(newScope, ...context.pendingIds) diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index 8695018..3c517ce 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -28,6 +28,7 @@ } @external tokens tokenizer from "./tokenizer" { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot } +@external specialize {Identifier} specializeKeyword from "./tokenizer" { Do } @precedence { pipe @left, @@ -91,11 +92,11 @@ FunctionDef { } singleLineFunctionDef { - @specialize[@name=keyword] Params colon consumeToTerminator @specialize[@name=keyword] + Do Params colon consumeToTerminator @specialize[@name=keyword] } multilineFunctionDef { - @specialize[@name=keyword] Params colon newlineOrSemicolon block @specialize[@name=keyword] + Do Params colon newlineOrSemicolon block @specialize[@name=keyword] } IfExpr { @@ -134,7 +135,7 @@ ConditionalOp { } Params { - AssignableIdentifier* + Identifier* } Assign { @@ -163,7 +164,7 @@ expression { @skip {} { DotGet { - IdentifierBeforeDot dot Identifier + IdentifierBeforeDot dot (Number | Identifier) } String { "'" stringContent* "'" } diff --git a/src/parser/shrimp.terms.ts b/src/parser/shrimp.terms.ts index e7fce78..36e7098 100644 --- a/src/parser/shrimp.terms.ts +++ b/src/parser/shrimp.terms.ts @@ -16,24 +16,25 @@ export const AssignableIdentifier = 14, Word = 15, IdentifierBeforeDot = 16, - Program = 17, - PipeExpr = 18, - FunctionCall = 19, - DotGet = 20, - PositionalArg = 21, - ParenExpr = 22, - FunctionCallOrIdentifier = 23, - BinOp = 24, - ConditionalOp = 25, - FunctionDef = 26, + Do = 17, + Program = 18, + PipeExpr = 19, + FunctionCall = 20, + DotGet = 21, + Number = 22, + PositionalArg = 23, + ParenExpr = 24, + FunctionCallOrIdentifier = 25, + BinOp = 26, + ConditionalOp = 27, + FunctionDef = 28, + Params = 29, + colon = 30, keyword = 50, - Params = 28, - colon = 29, - String = 31, - StringFragment = 32, - Interpolation = 33, - EscapeSeq = 34, - Number = 35, + String = 32, + StringFragment = 33, + Interpolation = 34, + EscapeSeq = 35, Boolean = 36, Regex = 37, Null = 38, diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index 3f0a131..1dfa9d0 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -1,27 +1,27 @@ // This file was generated by lezer-generator. You probably shouldn't edit it. import {LRParser, LocalTokenGroup} from "@lezer/lr" import {operatorTokenizer} from "./operatorTokenizer" -import {tokenizer} from "./tokenizer" +import {tokenizer, specializeKeyword} from "./tokenizer" import {trackScope} from "./scopeTracker" import {highlighting} from "./highlight" -const spec_Identifier = {__proto__:null,do:54, end:60, null:76, if:88, elseif:96, else:100} +const spec_Identifier = {__proto__:null,end:62, null:76, if:88, elseif:96, else:100} export const parser = LRParser.deserialize({ version: 14, - states: ".jQVQbOOO!QOpO'#CpO#^QcO'#CsO$WQRO'#CtO$fQcO'#DmO$}QbO'#DtOOQ`'#Cv'#CvO%VQbO'#CrO%wOSO'#C{OOQa'#Dr'#DrO&VQcO'#DqOOQ`'#Dn'#DnO&nQbO'#DmO&|QbO'#EQOOQ`'#DX'#DXO'kQRO'#DaOOQ`'#Dm'#DmO'pQQO'#DlOOQ`'#Dl'#DlOOQ`'#Db'#DbQVQbOOO'xO`O,59[OOQa'#Dq'#DqOOQ`'#Cq'#CqO'}QbO'#DUOOQ`'#Dp'#DpOOQ`'#Dc'#DcO(XQbO,59ZO&|QbO,59`O&|QbO,59`OOQ`'#Dd'#DdO(uQbO'#CxO(}QQO,5:`O)nQRO'#CtO*OQRO,59^O*aQRO,59^O*[QQO,59^O+[QQO,59^O+dQbO'#C}O+lQWO'#DOOOOO'#Dz'#DzOOOO'#Df'#DfO,QOSO,59gOOQa,59g,59gO,`QbO'#DgO,hQbO,59YO,yQRO,5:lO-QQQO,5:lO-VQbO,59{OOQ`,5:W,5:WOOQ`-E7`-E7`OOQa1G.v1G.vOOQ`,59p,59pOOQ`-E7a-E7aOOQa1G.z1G.zO-aQcO1G.zOOQ`-E7b-E7bO-{QbO1G/zO&|QbO,59aO&|QbO,59aOOQa1G.x1G.xOOOO,59i,59iOOOO,59j,59jOOOO-E7d-E7dOOQa1G/R1G/RO!VQbO'#CsOOQ`,5:R,5:ROOQ`-E7e-E7eO.YQbO1G0WOOQ`1G/g1G/gO.gQbO7+%fO.lQbO7+%gOOQO1G.{1G.{O.yQRO1G.{OOQ`'#DZ'#DZOOQ`7+%r7+%rO/TQbO7+%sOOQ`<mAN>mO&|QbO'#D]OOQ`'#Dh'#DhO0bQbOAN>yO0mQQO'#D_OOQ`AN>yAN>yO0rQbOAN>yO0wQRO,59wO1OQQO,59wOOQ`-E7f-E7fOOQ`G24eG24eO1TQbOG24eO1YQQO,59yO1_QQO1G/cOOQ`LD*PLD*PO.lQbO1G/eO/TQbO7+$}OOQ`7+%P7+%POOQ`<mAN>mO&|QbO'#D]OOQ`'#Dh'#DhO0eQbOAN>yO0pQQO'#D_OOQ`AN>yAN>yO0uQbOAN>yO0zQRO,59wO1RQQO,59wOOQ`-E7f-E7fOOQ`G24eG24eO1WQbOG24eO1]QQO,59yO1bQQO1G/cOOQ`LD*PLD*PO.oQbO1G/eO/WQbO7+$}OOQ`7+%P7+%POOQ`< spec_Identifier[value] || -1}], - tokenPrec: 756 + topRules: {"Program":[0,18]}, + specialized: [{term: 13, get: (value: any, stack: any) => (specializeKeyword(value, stack) << 1), external: specializeKeyword},{term: 13, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}], + tokenPrec: 759 }) diff --git a/src/parser/tests/basics.test.ts b/src/parser/tests/basics.test.ts index 1158bb1..f9f5e28 100644 --- a/src/parser/tests/basics.test.ts +++ b/src/parser/tests/basics.test.ts @@ -299,10 +299,10 @@ describe('Assign', () => { AssignableIdentifier add Eq = FunctionDef - keyword do + Do do Params - AssignableIdentifier a - AssignableIdentifier b + Identifier a + Identifier b colon : BinOp Identifier a diff --git a/src/parser/tests/dot-get.test.ts b/src/parser/tests/dot-get.test.ts index f2461bc..bcec05f 100644 --- a/src/parser/tests/dot-get.test.ts +++ b/src/parser/tests/dot-get.test.ts @@ -30,9 +30,9 @@ describe('DotGet', () => { test('function parameters are in scope within function body', () => { expect('do config: config.path end').toMatchTree(` FunctionDef - keyword do + Do do Params - AssignableIdentifier config + Identifier config colon : FunctionCallOrIdentifier DotGet @@ -45,9 +45,9 @@ describe('DotGet', () => { test('parameters out of scope outside function', () => { expect('do x: x.prop end; x.prop').toMatchTree(` FunctionDef - keyword do + Do do Params - AssignableIdentifier x + Identifier x colon : FunctionCallOrIdentifier DotGet @@ -64,10 +64,10 @@ describe('DotGet', () => { y.bar end`).toMatchTree(` FunctionDef - keyword do + Do do Params - AssignableIdentifier x - AssignableIdentifier y + Identifier x + Identifier y colon : FunctionCallOrIdentifier DotGet @@ -87,18 +87,18 @@ end`).toMatchTree(` do y: y.inner end end`).toMatchTree(` FunctionDef - keyword do + Do do Params - AssignableIdentifier x + Identifier x colon : FunctionCallOrIdentifier DotGet IdentifierBeforeDot x Identifier outer FunctionDef - keyword do + Do do Params - AssignableIdentifier y + Identifier y colon : FunctionCallOrIdentifier DotGet @@ -208,4 +208,70 @@ end`).toMatchTree(` PositionalArg Identifier prop`) }) + + test('readme.1 is Word when readme not in scope', () => { + expect('readme.1').toMatchTree(`Word readme.1`) + }) + + test('readme.1 is Word when used in function', () => { + expect('echo readme.1').toMatchTree(` + FunctionCall + Identifier echo + PositionalArg + Word readme.1`) + }) + + test('obj.1 is DotGet when obj is assigned', () => { + expect('obj = 5; obj.1').toMatchTree(` + Assign + AssignableIdentifier obj + Eq = + Number 5 + FunctionCallOrIdentifier + DotGet + IdentifierBeforeDot obj + Number 1 + `) + }) + + test('obj.1 arg is DotGet when obj is assigned', () => { + expect('obj = 5; obj.1').toMatchTree(` + Assign + AssignableIdentifier obj + Eq = + Number 5 + FunctionCallOrIdentifier + DotGet + IdentifierBeforeDot obj + Number 1 + `) + }) + + test('dot get index works as function w/ args', () => { + expect(`io = list (do x: echo x end); io.0 heya`).toMatchTree(` + Assign + AssignableIdentifier io + Eq = + FunctionCall + Identifier list + PositionalArg + ParenExpr + FunctionDef + Do do + Params + Identifier x + colon : + FunctionCall + Identifier echo + PositionalArg + Identifier x + keyword end + FunctionCall + DotGet + IdentifierBeforeDot io + Number 0 + PositionalArg + Identifier heya + `) + }) }) diff --git a/src/parser/tests/functions.test.ts b/src/parser/tests/functions.test.ts index 8fb1419..9249101 100644 --- a/src/parser/tests/functions.test.ts +++ b/src/parser/tests/functions.test.ts @@ -60,7 +60,7 @@ describe('Do', () => { test('parses function no parameters', () => { expect('do: 1 end').toMatchTree(` FunctionDef - keyword do + Do do Params colon : Number 1 @@ -70,9 +70,9 @@ describe('Do', () => { test('parses function with single parameter', () => { expect('do x: x + 1 end').toMatchTree(` FunctionDef - keyword do + Do do Params - AssignableIdentifier x + Identifier x colon : BinOp Identifier x @@ -84,10 +84,10 @@ describe('Do', () => { test('parses function with multiple parameters', () => { expect('do x y: x * y end').toMatchTree(` FunctionDef - keyword do + Do do Params - AssignableIdentifier x - AssignableIdentifier y + Identifier x + Identifier y colon : BinOp Identifier x @@ -102,10 +102,10 @@ describe('Do', () => { x + 9 end`).toMatchTree(` FunctionDef - keyword do + Do do Params - AssignableIdentifier x - AssignableIdentifier y + Identifier x + Identifier y colon : BinOp Identifier x @@ -124,9 +124,9 @@ end`).toMatchTree(` AssignableIdentifier fnnn Eq = FunctionDef - keyword do + Do do Params - AssignableIdentifier x + Identifier x colon : FunctionCallOrIdentifier Identifier x @@ -139,9 +139,9 @@ end`).toMatchTree(` AssignableIdentifier enddd Eq = FunctionDef - keyword do + Do do Params - AssignableIdentifier x + Identifier x colon : FunctionCallOrIdentifier Identifier x diff --git a/src/parser/tests/multiline.test.ts b/src/parser/tests/multiline.test.ts index 27bddc7..0d0a40d 100644 --- a/src/parser/tests/multiline.test.ts +++ b/src/parser/tests/multiline.test.ts @@ -24,10 +24,10 @@ describe('multiline', () => { AssignableIdentifier add Eq = FunctionDef - keyword do + Do do Params - AssignableIdentifier a - AssignableIdentifier b + Identifier a + Identifier b colon : Assign AssignableIdentifier result @@ -61,10 +61,10 @@ end Number 3 FunctionDef - keyword do + Do do Params - AssignableIdentifier x - AssignableIdentifier y + Identifier x + Identifier y colon : FunctionCallOrIdentifier Identifier x diff --git a/src/parser/tests/pipes.test.ts b/src/parser/tests/pipes.test.ts index 9d105b3..9e87114 100644 --- a/src/parser/tests/pipes.test.ts +++ b/src/parser/tests/pipes.test.ts @@ -75,13 +75,27 @@ describe('pipe expressions', () => { Identifier each PositionalArg FunctionDef - keyword do + Do do Params - AssignableIdentifier x + Identifier x colon : FunctionCallOrIdentifier Identifier x keyword end `) }) + + test(`double trouble (do keyword isn't over eager)`, () => { + expect(` + double 2 | double`).toMatchTree(` + PipeExpr + FunctionCall + Identifier double + PositionalArg + Number 2 + operator | + FunctionCallOrIdentifier + Identifier double + `) + }) }) diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts index 8963ffb..beea8e2 100644 --- a/src/parser/tokenizer.ts +++ b/src/parser/tokenizer.ts @@ -1,5 +1,10 @@ import { ExternalTokenizer, InputStream, Stack } from '@lezer/lr' -import { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot } from './shrimp.terms' +import { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, Do } from './shrimp.terms' + +// doobie doobie do (we need the `do` keyword to know when we're defining params) +export function specializeKeyword(ident: string) { + return ident === 'do' ? Do : -1 +} // The only chars that can't be words are whitespace, apostrophes, closing parens, and EOF.