diff --git a/src/compiler/utils.ts b/src/compiler/utils.ts index 312962f..bcf0587 100644 --- a/src/compiler/utils.ts +++ b/src/compiler/utils.ts @@ -70,9 +70,9 @@ export const getFunctionDefParts = (node: SyntaxNode, input: string) => { } const paramNames = getAllChildren(paramsNode).map((param) => { - if (param.type.id !== terms.AssignableIdentifier) { + if (param.type.id !== terms.Identifier) { throw new CompilerError( - `FunctionDef params must be AssignableIdentifiers, got ${param.type.name}`, + `FunctionDef params must be Identifier, got ${param.type.name}`, param.from, param.to ) @@ -219,9 +219,9 @@ export const getDotGetParts = (node: SyntaxNode, input: string) => { ) } - if (property.type.id !== terms.Identifier) { + if (property.type.id !== terms.Identifier && property.type.id !== terms.Number) { throw new CompilerError( - `DotGet property must be an Identifier, got ${property.type.name}`, + `DotGet property must be an Identifier or Number, got ${property.type.name}`, property.from, property.to ) diff --git a/src/parser/scopeTracker.ts b/src/parser/scopeTracker.ts index 8854cad..af2a32c 100644 --- a/src/parser/scopeTracker.ts +++ b/src/parser/scopeTracker.ts @@ -57,26 +57,30 @@ const readIdentifierText = (input: InputStream, start: number, end: number): str return text } +let inParams = false + export const trackScope = new ContextTracker({ start: new TrackerContext(new Scope(null, new Set())), shift(context, term, stack, input) { - if (term !== terms.AssignableIdentifier) return context + if (term == terms.Do) inParams = true - const text = readIdentifierText(input, input.pos, stack.pos) - return new TrackerContext(context.scope, [...context.pendingIds, text]) + if (term === terms.AssignableIdentifier) { + const text = readIdentifierText(input, input.pos, stack.pos) + return new TrackerContext(Scope.add(context.scope, text), context.pendingIds) + } + + if (inParams && term === terms.Identifier) { + const text = readIdentifierText(input, input.pos, stack.pos) + return new TrackerContext(context.scope, [...context.pendingIds, text]) + } + + return context }, reduce(context, term) { - // Add assignment variable to scope - if (term === terms.Assign) { - const varName = context.pendingIds.at(-1) - if (!varName) return context - return new TrackerContext(Scope.add(context.scope, varName), context.pendingIds.slice(0, -1)) - } - - // Push new scope and add all parameters if (term === terms.Params) { + inParams = false let newScope = context.scope.push() if (context.pendingIds.length > 0) { newScope = Scope.add(newScope, ...context.pendingIds) diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index 35fdcbc..c9298c3 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -29,6 +29,7 @@ } @external tokens tokenizer from "./tokenizer" { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot } +@external specialize {Identifier} specializeKeyword from "./tokenizer" { Do } @precedence { pipe @left, @@ -47,7 +48,6 @@ item { consumeToTerminator { PipeExpr | ambiguousFunctionCall | - DotGet | IfExpr | FunctionDef | Assign | @@ -64,7 +64,7 @@ pipeOperand { } FunctionCallOrIdentifier { - Identifier + DotGet | Identifier } ambiguousFunctionCall { @@ -72,7 +72,7 @@ ambiguousFunctionCall { } FunctionCall { - Identifier arg+ + (DotGet | Identifier) arg+ } arg { @@ -93,11 +93,11 @@ FunctionDef { } singleLineFunctionDef { - @specialize[@name=keyword] Params colon consumeToTerminator @specialize[@name=keyword] + Do Params colon consumeToTerminator @specialize[@name=keyword] } multilineFunctionDef { - @specialize[@name=keyword] Params colon newlineOrSemicolon block @specialize[@name=keyword] + Do Params colon newlineOrSemicolon block @specialize[@name=keyword] } IfExpr { @@ -140,7 +140,7 @@ ConditionalOp { } Params { - AssignableIdentifier* + Identifier* } Assign { @@ -169,7 +169,7 @@ expression { @skip {} { DotGet { - IdentifierBeforeDot dot Identifier + IdentifierBeforeDot dot (Number | Identifier) } String { "'" stringContent* "'" } diff --git a/src/parser/shrimp.terms.ts b/src/parser/shrimp.terms.ts index 2b65bbc..e27b327 100644 --- a/src/parser/shrimp.terms.ts +++ b/src/parser/shrimp.terms.ts @@ -16,27 +16,28 @@ export const AssignableIdentifier = 14, Word = 15, IdentifierBeforeDot = 16, - Program = 17, - PipeExpr = 18, - FunctionCall = 19, - PositionalArg = 20, - ParenExpr = 21, - FunctionCallOrIdentifier = 22, - BinOp = 23, - ConditionalOp = 24, - FunctionDef = 25, + Do = 17, + Program = 18, + PipeExpr = 19, + FunctionCall = 20, + DotGet = 21, + Number = 22, + PositionalArg = 23, + ParenExpr = 24, + FunctionCallOrIdentifier = 25, + BinOp = 26, + ConditionalOp = 27, + FunctionDef = 28, + Params = 29, + colon = 30, keyword = 50, - Params = 27, - colon = 28, - String = 30, - StringFragment = 31, - Interpolation = 32, - EscapeSeq = 33, - Number = 34, - Boolean = 35, - Regex = 36, - Null = 37, - DotGet = 38, + String = 32, + StringFragment = 33, + Interpolation = 34, + EscapeSeq = 35, + Boolean = 36, + Regex = 37, + Null = 38, Underscore = 39, NamedArg = 40, NamedArgPrefix = 41, diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index ba5f038..1289469 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -1,27 +1,27 @@ // This file was generated by lezer-generator. You probably shouldn't edit it. import {LRParser, LocalTokenGroup} from "@lezer/lr" import {operatorTokenizer} from "./operatorTokenizer" -import {tokenizer} from "./tokenizer" +import {tokenizer, specializeKeyword} from "./tokenizer" import {trackScope} from "./scopeTracker" import {highlighting} from "./highlight" -const spec_Identifier = {__proto__:null,do:52, end:58, null:74, if:88, elseif:96, else:100} +const spec_Identifier = {__proto__:null,end:62, null:76, if:88, elseif:96, else:100} export const parser = LRParser.deserialize({ version: 14, - states: ".vQYQbOOO#[QcO'#CrO$UQRO'#CsO$dQcO'#DnO${QbO'#DtOOQ`'#Cu'#CuO%TQbO'#CqO%uOSO'#CzOOQa'#Dr'#DrO&TOpO'#DSO&YQcO'#DqOOQ`'#Do'#DoO&qQbO'#DnO'PQbO'#EROOQ`'#DX'#DXO'nQRO'#DaOOQ`'#Dn'#DnO'sQQO'#DmOOQ`'#Dm'#DmOOQ`'#Db'#DbQYQbOOOOQa'#Dq'#DqOOQ`'#Cp'#CpO'{QbO'#DUOOQ`'#Dp'#DpOOQ`'#Dc'#DcO(VQbO,59ZO'PQbO,59_O'PQbO,59_OOQ`'#Dd'#DdO(sQbO'#CwO({QQO,5:`O)lQRO'#CsO)|QRO,59]O*_QRO,59]O*YQQO,59]O+YQQO,59]O+bQbO'#C|O+jQWO'#C}OOOO'#Dz'#DzOOOO'#Df'#DfO,OOSO,59fOOQa,59f,59fO,^O`O,59nO,cQbO'#DgO,hQbO,59YO,yQRO,5:mO-QQQO,5:mO-VQbO,59{OOQ`,5:X,5:XOOQ`-E7`-E7`OOQ`,59p,59pOOQ`-E7a-E7aOOQa1G.y1G.yO-aQcO1G.yOOQ`-E7b-E7bO-{QbO1G/zO'PQbO,59`O'PQbO,59`OOQa1G.w1G.wOOOO,59h,59hOOOO,59i,59iOOOO-E7d-E7dOOQa1G/Q1G/QOOQa1G/Y1G/YO!TQbO'#CrOOQ`,5:R,5:ROOQ`-E7e-E7eO.YQbO1G0XOOQ`1G/g1G/gO.gQbO7+%fO.lQbO7+%gOOQO1G.z1G.zO.|QRO1G.zOOQ`'#DZ'#DZO/WQbO7+%sO/]QbO7+%tOOQ`<mAN>mO'PQbO'#D]OOQ`'#Dh'#DhO0pQbOAN>zO0{QQO'#D_OOQ`AN>zAN>zO1QQbOAN>zO1VQRO,59wO1^QQO,59wOOQ`-E7f-E7fOOQ`G24fG24fO1cQbOG24fO1hQQO,59yO1mQQO1G/cOOQ`LD*QLD*QO.lQbO1G/eO/]QbO7+$}OOQ`7+%P7+%POOQ`<nAN>nO'PQbO'#D]OOQ`'#Dh'#DhO0vQbOAN>zO1RQQO'#D_OOQ`AN>zAN>zO1WQbOAN>zO1]QRO,59wO1dQQO,59wOOQ`-E7f-E7fOOQ`G24fG24fO1iQbOG24fO1nQQO,59yO1sQQO1G/cOOQ`LD*QLD*QO.rQbO1G/eO/cQbO7+$}OOQ`7+%P7+%POOQ`<i~RzOX#uXY$dYZ$}Zp#upq$dqs#ust%htu'Puw#uwx'Uxy'Zyz'tz{#u{|(_|}#u}!O(_!O!P#u!P!Q+R!Q![(|![!]3n!]!^$}!^#O#u#O#P4X#P#R#u#R#S4^#S#T#u#T#Y4w#Y#Z6V#Z#b4w#b#c:e#c#f4w#f#g;[#g#h4w#h#idS#zUoSOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uS$aP;=`<%l#u^$kUoS!_YOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU%UUoS!jQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#u^%oZoS!`YOY%hYZ#uZt%htu&buw%hwx&bx#O%h#O#P&b#P;'S%h;'S;=`&y<%lO%hY&gS!`YOY&bZ;'S&b;'S;=`&s<%lO&bY&vP;=`<%l&b^&|P;=`<%l%h~'UO!o~~'ZO!m~U'bUoS!gQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU'{UoS!lQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU(dWoSOt#uuw#ux!Q#u!Q![(|![#O#u#P;'S#u;'S;=`$^<%lO#uU)TYoSrQOt#uuw#ux!O#u!O!P)s!P!Q#u!Q![(|![#O#u#P;'S#u;'S;=`$^<%lO#uU)xWoSOt#uuw#ux!Q#u!Q![*b![#O#u#P;'S#u;'S;=`$^<%lO#uU*iWoSrQOt#uuw#ux!Q#u!Q![*b![#O#u#P;'S#u;'S;=`$^<%lO#uU+WWoSOt#uuw#ux!P#u!P!Q+p!Q#O#u#P;'S#u;'S;=`$^<%lO#uU+u^oSOY,qYZ#uZt,qtu-tuw,qwx-tx!P,q!P!Q#u!Q!},q!}#O2g#O#P0S#P;'S,q;'S;=`3h<%lO,qU,x^oStQOY,qYZ#uZt,qtu-tuw,qwx-tx!P,q!P!Q0i!Q!},q!}#O2g#O#P0S#P;'S,q;'S;=`3h<%lO,qQ-yXtQOY-tZ!P-t!P!Q.f!Q!}-t!}#O/T#O#P0S#P;'S-t;'S;=`0c<%lO-tQ.iP!P!Q.lQ.qUtQ#Z#[.l#]#^.l#a#b.l#g#h.l#i#j.l#m#n.lQ/WVOY/TZ#O/T#O#P/m#P#Q-t#Q;'S/T;'S;=`/|<%lO/TQ/pSOY/TZ;'S/T;'S;=`/|<%lO/TQ0PP;=`<%l/TQ0VSOY-tZ;'S-t;'S;=`0c<%lO-tQ0fP;=`<%l-tU0nWoSOt#uuw#ux!P#u!P!Q1W!Q#O#u#P;'S#u;'S;=`$^<%lO#uU1_boStQOt#uuw#ux#O#u#P#Z#u#Z#[1W#[#]#u#]#^1W#^#a#u#a#b1W#b#g#u#g#h1W#h#i#u#i#j1W#j#m#u#m#n1W#n;'S#u;'S;=`$^<%lO#uU2l[oSOY2gYZ#uZt2gtu/Tuw2gwx/Tx#O2g#O#P/m#P#Q,q#Q;'S2g;'S;=`3b<%lO2gU3eP;=`<%l2gU3kP;=`<%l,qU3uUoSlQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#u~4^O!p~U4eUoSwQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU4|YoSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#o4w#o;'S#u;'S;=`$^<%lO#uU5sUyQoSOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU6[ZoSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#U6}#U#o4w#o;'S#u;'S;=`$^<%lO#uU7S[oSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#`4w#`#a7x#a#o4w#o;'S#u;'S;=`$^<%lO#uU7}[oSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#g4w#g#h8s#h#o4w#o;'S#u;'S;=`$^<%lO#uU8x[oSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#X4w#X#Y9n#Y#o4w#o;'S#u;'S;=`$^<%lO#uU9uYsQoSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#o4w#o;'S#u;'S;=`$^<%lO#u^:lY!qWoSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#o4w#o;'S#u;'S;=`$^<%lO#u^;cY!sWoSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#o4w#o;'S#u;'S;=`$^<%lO#u^QUzQoSOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#u~>iO!w~", - tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, new LocalTokenGroup("[~RP!O!PU~ZO!t~~", 11)], - topRules: {"Program":[0,17]}, - specialized: [{term: 13, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}], - tokenPrec: 770 + tokenData: ">i~RzOX#uXY$dYZ$}Zp#upq$dqs#ust%htu'Puw#uwx'Uxy'Zyz'tz{#u{|(_|}#u}!O(_!O!P#u!P!Q+R!Q![(|![!]3n!]!^$}!^#O#u#O#P4X#P#R#u#R#S4^#S#T#u#T#Y4w#Y#Z6V#Z#b4w#b#c:e#c#f4w#f#g;[#g#h4w#h#idS#zUqSOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uS$aP;=`<%l#u^$kUqS!_YOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU%UUqS!kQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#u^%oZqS!`YOY%hYZ#uZt%htu&buw%hwx&bx#O%h#O#P&b#P;'S%h;'S;=`&y<%lO%hY&gS!`YOY&bZ;'S&b;'S;=`&s<%lO&bY&vP;=`<%l&b^&|P;=`<%l%h~'UO!p~~'ZO!n~U'bUqS!hQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU'{UqS!mQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU(dWqSOt#uuw#ux!Q#u!Q![(|![#O#u#P;'S#u;'S;=`$^<%lO#uU)TYqSfQOt#uuw#ux!O#u!O!P)s!P!Q#u!Q![(|![#O#u#P;'S#u;'S;=`$^<%lO#uU)xWqSOt#uuw#ux!Q#u!Q![*b![#O#u#P;'S#u;'S;=`$^<%lO#uU*iWqSfQOt#uuw#ux!Q#u!Q![*b![#O#u#P;'S#u;'S;=`$^<%lO#uU+WWqSOt#uuw#ux!P#u!P!Q+p!Q#O#u#P;'S#u;'S;=`$^<%lO#uU+u^qSOY,qYZ#uZt,qtu-tuw,qwx-tx!P,q!P!Q#u!Q!},q!}#O2g#O#P0S#P;'S,q;'S;=`3h<%lO,qU,x^qSuQOY,qYZ#uZt,qtu-tuw,qwx-tx!P,q!P!Q0i!Q!},q!}#O2g#O#P0S#P;'S,q;'S;=`3h<%lO,qQ-yXuQOY-tZ!P-t!P!Q.f!Q!}-t!}#O/T#O#P0S#P;'S-t;'S;=`0c<%lO-tQ.iP!P!Q.lQ.qUuQ#Z#[.l#]#^.l#a#b.l#g#h.l#i#j.l#m#n.lQ/WVOY/TZ#O/T#O#P/m#P#Q-t#Q;'S/T;'S;=`/|<%lO/TQ/pSOY/TZ;'S/T;'S;=`/|<%lO/TQ0PP;=`<%l/TQ0VSOY-tZ;'S-t;'S;=`0c<%lO-tQ0fP;=`<%l-tU0nWqSOt#uuw#ux!P#u!P!Q1W!Q#O#u#P;'S#u;'S;=`$^<%lO#uU1_bqSuQOt#uuw#ux#O#u#P#Z#u#Z#[1W#[#]#u#]#^1W#^#a#u#a#b1W#b#g#u#g#h1W#h#i#u#i#j1W#j#m#u#m#n1W#n;'S#u;'S;=`$^<%lO#uU2l[qSOY2gYZ#uZt2gtu/Tuw2gwx/Tx#O2g#O#P/m#P#Q,q#Q;'S2g;'S;=`3b<%lO2gU3eP;=`<%l2gU3kP;=`<%l,qU3uUqSnQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#u~4^O!q~U4eUqSwQOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU4|YqSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#o4w#o;'S#u;'S;=`$^<%lO#uU5sUyQqSOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#uU6[ZqSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#U6}#U#o4w#o;'S#u;'S;=`$^<%lO#uU7S[qSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#`4w#`#a7x#a#o4w#o;'S#u;'S;=`$^<%lO#uU7}[qSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#g4w#g#h8s#h#o4w#o;'S#u;'S;=`$^<%lO#uU8x[qSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#X4w#X#Y9n#Y#o4w#o;'S#u;'S;=`$^<%lO#uU9uYtQqSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#o4w#o;'S#u;'S;=`$^<%lO#u^:lY!rWqSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#o4w#o;'S#u;'S;=`$^<%lO#u^;cY!tWqSOt#uuw#ux!_#u!_!`5l!`#O#u#P#T#u#T#o4w#o;'S#u;'S;=`$^<%lO#u^QUzQqSOt#uuw#ux#O#u#P;'S#u;'S;=`$^<%lO#u~>iO!w~", + tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, new LocalTokenGroup("[~RP!O!PU~ZO!d~~", 11)], + topRules: {"Program":[0,18]}, + specialized: [{term: 13, get: (value: any, stack: any) => (specializeKeyword(value, stack) << 1), external: specializeKeyword},{term: 13, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}], + tokenPrec: 776 }) diff --git a/src/parser/tests/basics.test.ts b/src/parser/tests/basics.test.ts index a135f50..f92e034 100644 --- a/src/parser/tests/basics.test.ts +++ b/src/parser/tests/basics.test.ts @@ -497,10 +497,10 @@ describe('Assign', () => { AssignableIdentifier add Eq = FunctionDef - keyword do + Do do Params - AssignableIdentifier a - AssignableIdentifier b + Identifier a + Identifier b colon : BinOp Identifier a @@ -517,9 +517,10 @@ describe('DotGet whitespace sensitivity', () => { AssignableIdentifier basename Eq = Number 5 - DotGet - IdentifierBeforeDot basename - Identifier prop`) + FunctionCallOrIdentifier + DotGet + IdentifierBeforeDot basename + Identifier prop`) }) test('space before dot - NOT DotGet, parses as division', () => { diff --git a/src/parser/tests/dot-get.test.ts b/src/parser/tests/dot-get.test.ts index fd1aacb..bcec05f 100644 --- a/src/parser/tests/dot-get.test.ts +++ b/src/parser/tests/dot-get.test.ts @@ -20,22 +20,24 @@ describe('DotGet', () => { AssignableIdentifier obj Eq = Number 5 - DotGet - IdentifierBeforeDot obj - Identifier prop + FunctionCallOrIdentifier + DotGet + IdentifierBeforeDot obj + Identifier prop `) }) test('function parameters are in scope within function body', () => { expect('do config: config.path end').toMatchTree(` FunctionDef - keyword do + Do do Params - AssignableIdentifier config + Identifier config colon : - DotGet - IdentifierBeforeDot config - Identifier path + FunctionCallOrIdentifier + DotGet + IdentifierBeforeDot config + Identifier path keyword end `) }) @@ -43,13 +45,14 @@ describe('DotGet', () => { test('parameters out of scope outside function', () => { expect('do x: x.prop end; x.prop').toMatchTree(` FunctionDef - keyword do + Do do Params - AssignableIdentifier x + Identifier x colon : - DotGet - IdentifierBeforeDot x - Identifier prop + FunctionCallOrIdentifier + DotGet + IdentifierBeforeDot x + Identifier prop keyword end Word x.prop `) @@ -61,17 +64,19 @@ describe('DotGet', () => { y.bar end`).toMatchTree(` FunctionDef - keyword do + Do do Params - AssignableIdentifier x - AssignableIdentifier y + Identifier x + Identifier y colon : - DotGet - IdentifierBeforeDot x - Identifier foo - DotGet - IdentifierBeforeDot y - Identifier bar + FunctionCallOrIdentifier + DotGet + IdentifierBeforeDot x + Identifier foo + FunctionCallOrIdentifier + DotGet + IdentifierBeforeDot y + Identifier bar keyword end `) }) @@ -82,21 +87,23 @@ end`).toMatchTree(` do y: y.inner end end`).toMatchTree(` FunctionDef - keyword do + Do do Params - AssignableIdentifier x + Identifier x colon : - DotGet - IdentifierBeforeDot x - Identifier outer - FunctionDef - keyword do - Params - AssignableIdentifier y - colon : + FunctionCallOrIdentifier DotGet - IdentifierBeforeDot y - Identifier inner + IdentifierBeforeDot x + Identifier outer + FunctionDef + Do do + Params + Identifier y + colon : + FunctionCallOrIdentifier + DotGet + IdentifierBeforeDot y + Identifier inner keyword end keyword end `) @@ -117,6 +124,62 @@ end`).toMatchTree(` `) }) + test('dot get works as bare function', () => { + expect('io = dict print=echo; io.print').toMatchTree(` + Assign + AssignableIdentifier io + Eq = + FunctionCall + Identifier dict + NamedArg + NamedArgPrefix print= + Identifier echo + FunctionCallOrIdentifier + DotGet + IdentifierBeforeDot io + Identifier print + `) + }) + + test('dot get works as function w/ args', () => { + expect('io = dict print=echo; io.print heya').toMatchTree(` + Assign + AssignableIdentifier io + Eq = + FunctionCall + Identifier dict + NamedArg + NamedArgPrefix print= + Identifier echo + FunctionCall + DotGet + IdentifierBeforeDot io + Identifier print + PositionalArg + Identifier heya + `) + }) + + test('dot get works as function in parens', () => { + expect('io = dict print=echo; (io.print heya)').toMatchTree(` + Assign + AssignableIdentifier io + Eq = + FunctionCall + Identifier dict + NamedArg + NamedArgPrefix print= + Identifier echo + ParenExpr + FunctionCall + DotGet + IdentifierBeforeDot io + Identifier print + PositionalArg + Identifier heya + `) + }) + test('mixed file paths and dot get', () => { expect('config = 42; cat readme.txt; echo config.path').toMatchTree(` Assign @@ -145,4 +208,70 @@ end`).toMatchTree(` PositionalArg Identifier prop`) }) + + test('readme.1 is Word when readme not in scope', () => { + expect('readme.1').toMatchTree(`Word readme.1`) + }) + + test('readme.1 is Word when used in function', () => { + expect('echo readme.1').toMatchTree(` + FunctionCall + Identifier echo + PositionalArg + Word readme.1`) + }) + + test('obj.1 is DotGet when obj is assigned', () => { + expect('obj = 5; obj.1').toMatchTree(` + Assign + AssignableIdentifier obj + Eq = + Number 5 + FunctionCallOrIdentifier + DotGet + IdentifierBeforeDot obj + Number 1 + `) + }) + + test('obj.1 arg is DotGet when obj is assigned', () => { + expect('obj = 5; obj.1').toMatchTree(` + Assign + AssignableIdentifier obj + Eq = + Number 5 + FunctionCallOrIdentifier + DotGet + IdentifierBeforeDot obj + Number 1 + `) + }) + + test('dot get index works as function w/ args', () => { + expect(`io = list (do x: echo x end); io.0 heya`).toMatchTree(` + Assign + AssignableIdentifier io + Eq = + FunctionCall + Identifier list + PositionalArg + ParenExpr + FunctionDef + Do do + Params + Identifier x + colon : + FunctionCall + Identifier echo + PositionalArg + Identifier x + keyword end + FunctionCall + DotGet + IdentifierBeforeDot io + Number 0 + PositionalArg + Identifier heya + `) + }) }) diff --git a/src/parser/tests/functions.test.ts b/src/parser/tests/functions.test.ts index 8fb1419..9249101 100644 --- a/src/parser/tests/functions.test.ts +++ b/src/parser/tests/functions.test.ts @@ -60,7 +60,7 @@ describe('Do', () => { test('parses function no parameters', () => { expect('do: 1 end').toMatchTree(` FunctionDef - keyword do + Do do Params colon : Number 1 @@ -70,9 +70,9 @@ describe('Do', () => { test('parses function with single parameter', () => { expect('do x: x + 1 end').toMatchTree(` FunctionDef - keyword do + Do do Params - AssignableIdentifier x + Identifier x colon : BinOp Identifier x @@ -84,10 +84,10 @@ describe('Do', () => { test('parses function with multiple parameters', () => { expect('do x y: x * y end').toMatchTree(` FunctionDef - keyword do + Do do Params - AssignableIdentifier x - AssignableIdentifier y + Identifier x + Identifier y colon : BinOp Identifier x @@ -102,10 +102,10 @@ describe('Do', () => { x + 9 end`).toMatchTree(` FunctionDef - keyword do + Do do Params - AssignableIdentifier x - AssignableIdentifier y + Identifier x + Identifier y colon : BinOp Identifier x @@ -124,9 +124,9 @@ end`).toMatchTree(` AssignableIdentifier fnnn Eq = FunctionDef - keyword do + Do do Params - AssignableIdentifier x + Identifier x colon : FunctionCallOrIdentifier Identifier x @@ -139,9 +139,9 @@ end`).toMatchTree(` AssignableIdentifier enddd Eq = FunctionDef - keyword do + Do do Params - AssignableIdentifier x + Identifier x colon : FunctionCallOrIdentifier Identifier x diff --git a/src/parser/tests/multiline.test.ts b/src/parser/tests/multiline.test.ts index 7ac995a..87e5c65 100644 --- a/src/parser/tests/multiline.test.ts +++ b/src/parser/tests/multiline.test.ts @@ -24,10 +24,10 @@ describe('multiline', () => { AssignableIdentifier add Eq = FunctionDef - keyword do + Do do Params - AssignableIdentifier a - AssignableIdentifier b + Identifier a + Identifier b colon : Assign AssignableIdentifier result @@ -61,10 +61,10 @@ end Number 3 FunctionDef - keyword do + Do do Params - AssignableIdentifier x - AssignableIdentifier y + Identifier x + Identifier y colon : FunctionCallOrIdentifier Identifier x diff --git a/src/parser/tests/pipes.test.ts b/src/parser/tests/pipes.test.ts index 9d105b3..9e87114 100644 --- a/src/parser/tests/pipes.test.ts +++ b/src/parser/tests/pipes.test.ts @@ -75,13 +75,27 @@ describe('pipe expressions', () => { Identifier each PositionalArg FunctionDef - keyword do + Do do Params - AssignableIdentifier x + Identifier x colon : FunctionCallOrIdentifier Identifier x keyword end `) }) + + test(`double trouble (do keyword isn't over eager)`, () => { + expect(` + double 2 | double`).toMatchTree(` + PipeExpr + FunctionCall + Identifier double + PositionalArg + Number 2 + operator | + FunctionCallOrIdentifier + Identifier double + `) + }) }) diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts index 8df852a..0db5545 100644 --- a/src/parser/tokenizer.ts +++ b/src/parser/tokenizer.ts @@ -1,5 +1,10 @@ import { ExternalTokenizer, InputStream, Stack } from '@lezer/lr' -import { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot } from './shrimp.terms' +import { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, Do } from './shrimp.terms' + +// doobie doobie do (we need the `do` keyword to know when we're defining params) +export function specializeKeyword(ident: string) { + return ident === 'do' ? Do : -1 +} // The only chars that can't be words are whitespace, apostrophes, closing parens, and EOF.