diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index ce8ad22..276a5e0 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -21,14 +21,9 @@ leftParen { "(" } rightParen { ")" } colon[closedBy="end", @name="colon"] { ":" } - end[openedBy="colon", @name="end"] { "end" } Underscore { "_" } Null { "null" } Regex { "//" (![/\\\n[] | "\\" ![\n] | "[" (![\n\\\]] | "\\" ![\n])* "]")+ ("//" $[gimsuy]*)? } // Stolen from the lezer JavaScript grammar - Fn[@name=keyword] { "fn" } - "if" [@name=keyword] - "elsif" [@name=keyword] - "else" [@name=keyword] "|"[@name=operator] } @@ -98,11 +93,11 @@ FunctionDef { } singleLineFunctionDef { - Fn Params colon consumeToTerminator end + @specialize[@name=keyword] Params colon consumeToTerminator @specialize[@name=keyword] } multilineFunctionDef { - Fn Params colon newlineOrSemicolon block end + @specialize[@name=keyword] Params colon newlineOrSemicolon block @specialize[@name=keyword] } IfExpr { @@ -110,19 +105,19 @@ IfExpr { } singleLineIf { - "if" (ConditionalOp | expression) colon ThenBlock { consumeToTerminator } + @specialize[@name=keyword] (ConditionalOp | expression) colon ThenBlock { consumeToTerminator } } multilineIf { - "if" (ConditionalOp | expression) colon newlineOrSemicolon ThenBlock ElsifExpr* ElseExpr? end + @specialize[@name=keyword] (ConditionalOp | expression) colon newlineOrSemicolon ThenBlock ElsifExpr* ElseExpr? @specialize[@name=keyword] } ElsifExpr { - "elsif" (ConditionalOp | expression) colon newlineOrSemicolon ThenBlock + @specialize[@name=keyword] (ConditionalOp | expression) colon newlineOrSemicolon ThenBlock } ElseExpr { - "else" colon newlineOrSemicolon ThenBlock + @specialize[@name=keyword] colon newlineOrSemicolon ThenBlock } ThenBlock { @@ -163,9 +158,14 @@ expression { expressionWithoutIdentifier | DotGet | Identifier } + +@local tokens { + dot { "." } +} + @skip {} { DotGet { - IdentifierBeforeDot "." Identifier + IdentifierBeforeDot dot Identifier } String { "'" stringContent* "'" } diff --git a/src/parser/shrimp.terms.ts b/src/parser/shrimp.terms.ts index 78a893d..cef00ea 100644 --- a/src/parser/shrimp.terms.ts +++ b/src/parser/shrimp.terms.ts @@ -34,10 +34,9 @@ export const Null = 32, DotGet = 33, FunctionDef = 34, - Fn = 35, + keyword = 50, Params = 36, colon = 37, - end = 38, Underscore = 39, NamedArg = 40, NamedArgPrefix = 41, diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index bf99404..fda1ab5 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -1,26 +1,27 @@ // This file was generated by lezer-generator. You probably shouldn't edit it. -import {LRParser} from "@lezer/lr" +import {LRParser, LocalTokenGroup} from "@lezer/lr" import {operatorTokenizer} from "./operatorTokenizer" import {tokenizer} from "./tokenizer" import {trackScope} from "./scopeTracker" import {highlighting} from "./highlight" +const spec_Identifier = {__proto__:null,fn:70, end:76, if:88, elsif:96, else:100} export const parser = LRParser.deserialize({ version: 14, - states: ".jQVQrOOO#XQuO'#CrO$RQRO'#CsO$aQRO'#DmO$xQrO'#CqO%gOWO'#CuOOQq'#Dq'#DqO%uOQO'#C}O%zQRO'#DpO&cQrO'#D|OOQp'#DO'#DOOOQO'#Dn'#DnO&kQQO'#DmO&yQrO'#EQOOQO'#DX'#DXO'hQRO'#DaOOQO'#Dm'#DmO'mQQO'#DlOOQp'#Dl'#DlOOQp'#Db'#DbQVQrOOOOQq'#Dp'#DpOOQp'#Cp'#CpO'uQrO'#DUOOQp'#Do'#DoOOQp'#Dc'#DcO(PQtO,59ZO&yQrO,59_O&yQrO,59_O)XQRO'#CsO)iQRO,59]O)zQRO,59]O)uQQO,59]O*uQQO,59]O*}QrO'#CwO+VQ`O'#CxOOOO'#Du'#DuOOOO'#Dd'#DdO+kOWO,59aOOQq,59a,59aO+yOpO,59iOOQp'#De'#DeO,OQrO'#DQO,WQQO,5:hO,]QrO'#DgO,bQQO,59YO,sQRO,5:lO,zQQO,5:lO-PQrO,59{OOQp,5:W,5:WOOQp-E7`-E7`OOQp,59p,59pOOQp-E7a-E7aOOQP1G.y1G.yO-^QRO1G.yO&yQrO,59`O&yQrO,59`OOQq1G.w1G.wOOOO,59c,59cOOOO,59d,59dOOOO-E7b-E7bOOQq1G.{1G.{OOQq1G/T1G/TOOQp-E7c-E7cO-xQrO1G0SO!QQtO'#CrOOQO,5:R,5:ROOQO-E7e-E7eO.YQrO1G0WOOQO1G/g1G/gOOQO1G.z1G.zO.jQRO1G.zO.tQQO7+%nO.yQrO7+%oOOQO'#DZ'#DZOOQO7+%r7+%rO/ZQrO7+%sOOQp<uAN>uO&yQrO'#D]OOQO'#Dh'#DhO0nQQOAN>yO0yQQO'#D_OOQOAN>yAN>yO1OQQOAN>yO1TQRO,59wO1[QQO,59wOOQO-E7f-E7fOOQOG24eG24eO1aQQOG24eO1fQQO,59yO1kQQO1G/cOOQOLD*PLD*PO.yQrO1G/eO/ZQrO7+$}OOQO7+%P7+%POOQO<uAN>uO&yQbO'#D]OOQ`'#Dh'#DhO0nQbOAN>yO0yQQO'#D_OOQ`AN>yAN>yO1OQbOAN>yO1TQRO,59wO1[QQO,59wOOQ`-E7f-E7fOOQ`G24eG24eO1aQbOG24eO1fQQO,59yO1kQQO1G/cOOQ`LD*PLD*PO.yQbO1G/eO/ZQbO7+$}OOQ`7+%P7+%POOQ`<^#a#o4P#o;'S#{;'S;=`$d<%lO#{V>c[jSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#g4P#g#h?X#h#o4P#o;'S#{;'S;=`$d<%lO#{V?^[jSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#X4P#X#Y@S#Y#o4P#o;'S#{;'S;=`$d<%lO#{V@ZYnRjSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#o4P#o;'S#{;'S;=`$d<%lO#{VAQYsRjSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#o4P#o;'S#{;'S;=`$d<%lO#{VAu[jSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#Y4P#Y#ZBk#Z#o4P#o;'S#{;'S;=`$d<%lO#{VBrY|PjSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#o4P#o;'S#{;'S;=`$d<%lO#{_Ci[!lWjSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#i4P#i#jD_#j#o4P#o;'S#{;'S;=`$d<%lO#{VDd[jSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#`4P#`#aEY#a#o4P#o;'S#{;'S;=`$d<%lO#{VE_[jSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#`4P#`#aFT#a#o4P#o;'S#{;'S;=`$d<%lO#{VF[YpRjSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#o4P#o;'S#{;'S;=`$d<%lO#{^GRY!nWjSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#o4P#o;'S#{;'S;=`$d<%lO#{_Gx[!mWjSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#f4P#f#gHn#g#o4P#o;'S#{;'S;=`$d<%lO#{VHs[jSOt#{uw#{x!_#{!_!`4t!`#O#{#P#T#{#T#i4P#i#j?X#j#o4P#o;'S#{;'S;=`$d<%lO#{VIpUzRjSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~JXO!v~", - tokenizers: [operatorTokenizer, 0, 1, 2, 3, tokenizer], + tokenData: "?p~RyOX#rXY$aYZ$zZp#rpq$aqt#rtu%euw#rwx%jxy%oyz&Yz{#r{|&s|}#r}!O&s!O!P#r!P!Q)g!Q!['b![!]2S!]!^$z!^#O#r#O#P2m#P#R#r#R#S2r#S#T#r#T#Y3]#Y#Z4k#Z#b3]#b#c8y#c#f3]#f#gV#g#o3]#o;'S#r;'S;=`$Z<%lO#rU>[[jSOt#ruw#rx!_#r!_!`4Q!`#O#r#P#T#r#T#i3]#i#j7X#j#o3]#o;'S#r;'S;=`$Z<%lO#rU?XUzQjSOt#ruw#rx#O#r#P;'S#r;'S;=`$Z<%lO#r~?pO!v~", + tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, new LocalTokenGroup("[~RP!O!PU~ZO!o~~", 11)], topRules: {"Program":[0,17]}, + specialized: [{term: 13, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}], tokenPrec: 768 }) diff --git a/src/parser/tests/basics.test.ts b/src/parser/tests/basics.test.ts index e4ec3f5..454dd03 100644 --- a/src/parser/tests/basics.test.ts +++ b/src/parser/tests/basics.test.ts @@ -300,7 +300,7 @@ describe('Assign', () => { Identifier a Plus + Identifier b - end end`) + keyword end`) }) }) diff --git a/src/parser/tests/control-flow.test.ts b/src/parser/tests/control-flow.test.ts index a507b36..33cf734 100644 --- a/src/parser/tests/control-flow.test.ts +++ b/src/parser/tests/control-flow.test.ts @@ -45,7 +45,7 @@ describe('if/elsif/else', () => { ThenBlock FunctionCallOrIdentifier Identifier yes - end end + keyword end `) }) @@ -68,7 +68,7 @@ describe('if/elsif/else', () => { ThenBlock FunctionCallOrIdentifier Identifier y - end end + keyword end `) }) @@ -92,7 +92,7 @@ describe('if/elsif/else', () => { ThenBlock FunctionCallOrIdentifier Identifier y - end end + keyword end `) }) @@ -133,7 +133,21 @@ describe('if/elsif/else', () => { ThenBlock FunctionCallOrIdentifier Identifier oh-no - end end + keyword end + `) + }) + + test('does not parse identifiers that start with if', () => { + expect('iffy = if true: 2').toMatchTree(` + Assign + AssignableIdentifier iffy + Eq = + IfExpr + keyword if + Boolean true + colon : + ThenBlock + Number 2 `) }) }) diff --git a/src/parser/tests/dot-get.test.ts b/src/parser/tests/dot-get.test.ts index ddd24ae..a97caeb 100644 --- a/src/parser/tests/dot-get.test.ts +++ b/src/parser/tests/dot-get.test.ts @@ -36,7 +36,7 @@ describe('DotGet', () => { DotGet IdentifierBeforeDot config Identifier path - end end + keyword end `) }) @@ -50,7 +50,7 @@ describe('DotGet', () => { DotGet IdentifierBeforeDot x Identifier prop - end end + keyword end Word x.prop `) }) @@ -72,7 +72,7 @@ end`).toMatchTree(` DotGet IdentifierBeforeDot y Identifier bar - end end + keyword end `) }) @@ -97,8 +97,8 @@ end`).toMatchTree(` DotGet IdentifierBeforeDot y Identifier inner - end end - end end + keyword end + keyword end `) }) diff --git a/src/parser/tests/functions.test.ts b/src/parser/tests/functions.test.ts index eb2fa56..a9f0ff0 100644 --- a/src/parser/tests/functions.test.ts +++ b/src/parser/tests/functions.test.ts @@ -64,7 +64,7 @@ describe('Fn', () => { Params colon : Number 1 - end end`) + keyword end`) }) test('parses function with single parameter', () => { @@ -78,7 +78,7 @@ describe('Fn', () => { Identifier x Plus + Number 1 - end end`) + keyword end`) }) test('parses function with multiple parameters', () => { @@ -93,7 +93,7 @@ describe('Fn', () => { Identifier x Star * Identifier y - end end`) + keyword end`) }) test('parses multiline function with multiple statements', () => { @@ -115,6 +115,36 @@ end`).toMatchTree(` Identifier x Plus + Number 9 - end end`) + keyword end`) + }) + + test('does not parse identifiers that start with fn', () => { + expect('fnnn = fn x: x end').toMatchTree(` + Assign + AssignableIdentifier fnnn + Eq = + FunctionDef + keyword fn + Params + AssignableIdentifier x + colon : + FunctionCallOrIdentifier + Identifier x + keyword end`) + }) + + test('does not parse identifiers that start with end', () => { + expect('enddd = fn x: x end').toMatchTree(` + Assign + AssignableIdentifier enddd + Eq = + FunctionDef + keyword fn + Params + AssignableIdentifier x + colon : + FunctionCallOrIdentifier + Identifier x + keyword end`) }) }) diff --git a/src/parser/tests/multiline.test.ts b/src/parser/tests/multiline.test.ts index daf6d84..4ee052a 100644 --- a/src/parser/tests/multiline.test.ts +++ b/src/parser/tests/multiline.test.ts @@ -39,7 +39,7 @@ describe('multiline', () => { FunctionCallOrIdentifier Identifier result - end end + keyword end FunctionCall Identifier add PositionalArg @@ -68,7 +68,7 @@ end colon : FunctionCallOrIdentifier Identifier x - end end + keyword end `) }) }) diff --git a/src/parser/tests/pipes.test.ts b/src/parser/tests/pipes.test.ts index bfd000e..ecddf87 100644 --- a/src/parser/tests/pipes.test.ts +++ b/src/parser/tests/pipes.test.ts @@ -81,7 +81,7 @@ describe('pipe expressions', () => { colon : FunctionCallOrIdentifier Identifier x - end end + keyword end `) }) })