diff --git a/src/compiler/tests/pipe.test.ts b/src/compiler/tests/pipe.test.ts index b8f92f1..4d1669f 100644 --- a/src/compiler/tests/pipe.test.ts +++ b/src/compiler/tests/pipe.test.ts @@ -93,7 +93,6 @@ describe('pipe expressions', () => { `).toEvaluateTo(5) }) - test('string literals can be piped', () => { expect(`'hey there' | str.to-upper`).toEvaluateTo('HEY THERE') }) diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index e8f2fa4..4f2e032 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -21,7 +21,7 @@ ("-" | "+")? $[0-9]+ ("_"? $[0-9]+)* ('.' $[0-9]+ ("_"? $[0-9]+)*)? } Boolean { "true" | "false" } - newlineOrSemicolon { "\n" | ";" } + semicolon { ";" } eof { @eof } space { " " | "\t" } Comment { "#" ![\n]* } @@ -34,6 +34,8 @@ "|"[@name=operator] } +newlineOrSemicolon { newline | semicolon } + end { @specialize[@name=keyword] } while { @specialize[@name=keyword] } if { @specialize[@name=keyword] } @@ -46,6 +48,7 @@ import { @specialize[@name=keyword] } null { @specialize[@name=Null] } @external tokens tokenizer from "./tokenizer" { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, CurlyString } +@external tokens pipeStartsLineTokenizer from "./tokenizer" { newline, pipeStartsLine } @external specialize {Identifier} specializeKeyword from "./tokenizer" { Do } @precedence { @@ -85,7 +88,7 @@ consumeToTerminator { } PipeExpr { - pipeOperand (!pipe "|" pipeOperand)+ + pipeOperand (!pipe (pipeStartsLine? "|") newlineOrSemicolon* pipeOperand)+ } pipeOperand { diff --git a/src/parser/shrimp.terms.ts b/src/parser/shrimp.terms.ts index 2085974..cc00537 100644 --- a/src/parser/shrimp.terms.ts +++ b/src/parser/shrimp.terms.ts @@ -32,6 +32,8 @@ export const Word = 30, IdentifierBeforeDot = 31, CurlyString = 32, + newline = 101, + pipeStartsLine = 102, Do = 33, Comment = 34, Program = 35, diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index 8b7359e..7064a86 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -1,17 +1,17 @@ // This file was generated by lezer-generator. You probably shouldn't edit it. import {LRParser, LocalTokenGroup} from "@lezer/lr" import {operatorTokenizer} from "./operatorTokenizer" -import {tokenizer, specializeKeyword} from "./tokenizer" +import {tokenizer, pipeStartsLineTokenizer, specializeKeyword} from "./tokenizer" import {trackScope} from "./parserScopeContext" import {highlighting} from "./highlight" const spec_Identifier = {__proto__:null,while:78, null:116, catch:122, finally:128, end:130, if:138, else:144, try:160, throw:164, import:168} export const parser = LRParser.deserialize({ version: 14, - states: ">`QYQbOOO!pOpO'#DXO%eQcO'#DdO%{OSO'#DaOOQa'#Da'#DaO(vQcO'#EjOOQ`'#Ex'#ExO)aQRO'#DxO+fQcO'#EhO,PQbO'#DVOOQa'#Dz'#DzO.kQbO'#D{OOQa'#Ej'#EjO.rQcO'#EjO0pQcO'#EiO1uQcO'#EhO2SQRO'#ETOOQ`'#Eh'#EhO2kQbO'#EhO2rQQO'#EgOOQ`'#Eg'#EgOOQ`'#EV'#EVQYQbOOO2}QbO'#D]O3YQbO'#DrO4WQbO'#DSO5UQQO'#D}O4WQbO'#EPO5ZQbO'#ERO5cObO,59sOOQ`'#D['#D[O5tQbO'#DqOOQ`'#En'#EnOOQ`'#E_'#E_O6OQbO,5:`OOQa'#Ei'#EiO6xQbO'#DcO7WQWO'#DeOOOO'#Ep'#EpOOOO'#E['#E[O7lOSO,59{OOQa,59{,59{O4WQbO,5:dO4WQbO,5:dO4WQbO,5:dO4WQbO,5:dO4WQbO,59pO4WQbO,59pO4WQbO,59pO4WQbO,59pOOQ`'#EX'#EXO,PQbO,59qO7zQcO'#DdO8RQcO'#EjO8YQRO,59qO8dQQO,59qO8iQQO,59qO8qQQO,59qO8|QRO,59qO9fQRO,59qO9mQQO'#DQO9rQbO,5:gO9yQQO,5:fOOQa,5:g,5:gO:UQbO,5:gO:`QbO,5:pO:`QbO,5:oO;sQbO,5:hO;zQbO,59lOOQ`,5;R,5;RO:`QbO'#EWOOQ`-E8T-E8TOOQ`'#EY'#EYOXQRO'#EwO?UQRO'#EwOOQO'#Ew'#EwO?]QQO,5:^O?bQRO,59nO?iQRO,59nO;sQbO,5:iO?wQcO,5:kOAVQcO,5:kOAsQcO,5:kOBhQbO,5:mOOQ`'#Ec'#EcO5ZQbO,5:mOOQa1G/_1G/_OOQ`,5:],5:]OOQ`-E8]-E8]OOOO'#Dd'#DdOOOO,59},59}OOOO,5:P,5:POOOO-E8Y-E8YOOQa1G/g1G/gOOQa1G0O1G0OODaQcO1G0OODkQcO1G0OOEyQcO1G0OOFTQcO1G0OOFbQcO1G0OOOQa1G/[1G/[OGsQcO1G/[OGzQcO1G/[OHRQcO1G/[OIQQcO1G/[OHYQcO1G/[OOQ`-E8V-E8VOIhQRO1G/]OIrQQO1G/]OIwQQO1G/]OJPQQO1G/]OJ[QRO1G/]OJcQRO1G/]OJjQbO,59rOJtQQO1G/]OOQa1G/]1G/]OJ|QQO1G0QOOQa1G0R1G0ROKXQbO1G0ROOQO'#Ea'#EaOJ|QQO1G0QOOQa1G0Q1G0QOOQ`'#Eb'#EbOKXQbO1G0ROKcQbO1G0[OK}QbO1G0ZOLiQbO'#DlOLzQbO'#DlOM_QbO1G0SOOQ`-E8U-E8UOOQ`,5:r,5:rOOQ`-E8W-E8WOMjQQO,59xOOQO,59y,59yOOQO-E8X-E8XOMrQbO1G/cO;sQbO1G/xO;sQbO1G/YOMyQbO1G0TONUQbO1G0XONsQbO1G0XOOQ`-E8a-E8aONzQQO7+$wOOQa7+$w7+$wO! SQQO1G/^O! [QQO7+%lOOQa7+%l7+%lO! gQbO7+%mOOQa7+%m7+%mOOQO-E8_-E8_OOQ`-E8`-E8`OOQ`'#E]'#E]O! qQQO'#E]O! yQbO'#EvOOQ`,5:W,5:WO!!^QbO'#DjO!!cQQO'#DmOOQ`7+%n7+%nO!!hQbO7+%nO!!mQbO7+%nO!!uQbO7+$}O!#TQbO7+$}O!#eQbO7+%dO!#mQbO7+$tOOQ`7+%o7+%oO!#rQbO7+%oO!#wQbO7+%oO!$PQbO7+%sOOQa<tAN>tOOQ`AN>TAN>TO!&pQbOAN>TO!&uQbOAN>TOOQ`-E8^-E8^OOQ`AN>jAN>jO!&}QbOAN>jO3YQbO,5:aO;sQbO,5:cOOQ`AN>uAN>uPJjQbO'#EXOOQ`7+%[7+%[OOQ`G23oG23oO!'SQbOG23oP!&SQbO'#DuOOQ`G24UG24UO!'XQQO1G/{OOQ`1G/}1G/}OOQ`LD)ZLD)ZO;sQbO7+%gOOQ`<gO!]$SO~O!]$TO~P>gOT!POU!QOj!RO!]$TO~Ou!sa#`!sa#q!sa!_!sa!b!sa!c!sa#m!sa!j!sa~P)aOP{OQ{OR|OS|Od}Oe}Of}Og}Oh}Oi}O~Ou!sa#`!sa#q!sa!_!sa!b!sa!c!sa#m!sa!j!sa~P@eOT!POU!QOj!ROu!sa#`!sa#q!sa!_!sa!b!sa!c!sa#m!sa!j!sa~Ol!jO!SoOu!ua#`!ua#q!ua!_!ua!b!ua!c!ua#m!ua!j!ua~O^zOR!liS!lid!lie!lif!lig!lih!lii!liu!li#`!li#q!li#m!li!_!li!b!li!c!li!j!li~OP!liQ!li~PCYOP{OQ{O~PCYOP{OQ{Od!lie!lif!lig!lih!lii!liu!li#`!li#q!li#m!li!_!li!b!li!c!li!j!li~OR!liS!li~PDuOR|OS|O^zO~PDuOR|OS|O~PDuOW!OOX!OOY!OOZ!OO[!OO]!OOTxijxiuxi#`xi#qxi#mxi!]xi!_xi!bxi!cxi!jxi~OU!QO~PFlOU!QO~PGOOUxi~PFlOT!POU!QOjxiuxi#`xi#qxi#mxi!]xi!_xi!bxi!cxi!jxi~OW!OOX!OOY!OOZ!OO[!OO]!OO~PHYO#`!SO#m$ZO~P*qO#m$ZO~O#m$ZOu#[X~O!]!eO#m$ZOu#[X~O#m$ZO~P/]O#m$ZO~P9TOqgO!dnO~P-gO#`!SO#m$ZO~O!SoO#`#qO#p$^O~O#`#tO#p$`O~P4WOu!hO#`!xi#q!xi!_!xi!b!xi!c!xi#m!xi!j!xi~Ou!hO#`!wi#q!wi!_!wi!b!wi!c!wi#m!wi!j!wi~Ou!hO!_!`X!b!`X!c!`X!j!`X~O#`$cO!_#jP!b#jP!c#jP!j#jP~P:`O!_$gO!b$hO!c$iO~O!S!lO!]!Qa~O#`$mO~P:`O!_$gO!b$hO!c$pO~O!SoOu!ui#`!ui#q!ui!_!ui!b!ui!c!ui#m!ui!j!ui~Ol!jO~PNUO#`!SO#m$tO~O#`!SO#mzi~O!SoO#`#qO#p$wO~O#`#tO#p$xO~P4WOu!hO#`$yO~O#`$cO!_#jX!b#jX!c#jX!j#jX~P:`Ol${O~O!]$|O~O!c$}O~O!b$hO!c$}O~Ou!hO!_$gO!b$hO!c%PO~O#`$cO!_#jP!b#jP!c#jP~P:`O!c%WO!j%VO~O!c%YO~O!c%ZO~O!b$hO!c%ZO~O!SoOu!uq#`!uq#q!uq!_!uq!b!uq!c!uq#m!uq!j!uq~OqgO!dnO#mzq~P-gO#`!SO#mzq~O!]%`O~O!c%bO~O!c%cO~O!b$hO!c%cO~O!_$gO!b$hO!c%cO~O!c%gO!j%VO~O!]%jO!g%iO~O!c%gO~O!c%kO~OqgO!dnO#mzy~P-gO!c%nO~O!b$hO!c%nO~O!c%qO~O!c%tO~O!]%uO~Ol#OOo%xO|#OO}%xO#_XO~O#a%wO~O|!m~", - goto: "9g#mPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP#nP$XP$n%l&{'RPP(d(p)j)mP)sP*z+O*zPPPP+hP+t,^PPP,t#nP-f.PP.T.ZP/Q0U$X$XP$XP$XP$X$X1[1b1n2b2p2z3Q3X3_3i3o3y4TPPP4c4g5[7QPPP8[P8lPPPPP8p8v8|raOf!c!d!e!h!w#y$R$S$T$e$m$|%`%j%uQ!YXR#g!TwaOXf!T!c!d!e!h!w#y$R$S$T$e$m$|%`%j%ur_Of!c!d!e!h!w#y$R$S$T$e$m$|%`%j%uQ!]XS!qh%iQ!viQ!zkQ#^!QQ#`!PQ#c!RR#j!TvTOfh!c!d!e!h!w#y$R$S$T$e$m$|%`%i%j%u!W[QTZikorz{|}!O!P!Q!R!U!V!_!b!p#k#p#u$_$u%^%lS!VX!TS#Om%wR#StQ!XXR#f!TrQOf!c!d!e!h!w#y$R$S$T$e$m$|%`%j%u!WsQTZikorz{|}!O!P!Q!R!U!V!_!b!p#k#p#u$_$u%^%lS!UX!TS!ph%iS#Om%wR#RtepQTr!U!V!p#k$u%^%lraOf!c!d!e!h!w#y$R$S$T$e$m$|%`%j%udnQTr!U!V!p#k$u%^%lQ!YXQ#PoR#g!TR!ogX!mg!k!n$O#S[OQTXZfhikorz{|}!O!P!Q!R!T!U!V!_!b!c!d!e!h!p!w#k#p#u#y$R$S$T$_$e$m$u$|%^%`%i%j%l%uR$P!lTvRxvUOXf!T!c!d!e!h!w#y$R$S$T$e$m$|%`%j%uR#StQ$k#zQ$r$UQ%R$lR%e%SQ#z!eQ$U!wQ$n$SQ$o$TQ%a$|Q%m%`Q%s%jR%v%uQ$j#zQ$q$UQ%O$kQ%Q$lQ%[$rS%d%R%SR%o%edpQTr!U!V!p#k$u%^%lQ!`Z[!|l!{!}$V$W$sQ#n!_X#q!`#n#r$]vUOXf!T!c!d!e!h!w#y$R$S$T$e$m$|%`%j%uT!sh%iT%T$n%UQ%X$nR%h%UrWOf!c!d!e!h!w#y$R$S$T$e$m$|%`%j%uQ!WXQ!ykQ#W{Q#Z|Q#]}R#e!T#T[OQTXZfhikorz{|}!O!P!Q!R!T!U!V!_!b!c!d!e!h!p!w#k#p#u#y$R$S$T$_$e$m$u$|%^%`%i%j%l%u![[QTZhikorz{|}!O!P!Q!R!U!V!_!b!p#k#p#u$_$u%^%i%lw]OXf!T!c!d!e!h!w#y$R$S$T$e$m$|%`%j%uQfOR!if^!fc!^#v#w#x$d$lR#{!fQ!TXQ!_Z`#d!T!_#k#l$Y$u%^%lS#k!U!VS#l!W!]S$Y#e#jQ$u$[R%^$vQ!kgQ!{lU#}!k!{$WR$W!}Q!ngQ$O!kT$Q!n$OQxRR#UxS$e#y$mR$z$eQ$v$[R%_$vYrQT!U!V!pR#QrQ%U$nR%f%UQ#r!`Q$]#nT$a#r$]Q#u!bQ$_#pT$b#u$_Q!}lQ$V!{U$X!}$V$sR$s$WTeOfScOfS!^X!TQ#v!cQ#w!d`#x!e!w$S$T$|%`%j%uQ#|!hU$d#y$e$mR$l$RvVOXf!T!c!d!e!h!w#y$R$S$T$e$m$|%`%j%udnQTr!U!V!p#k$u%^%lQ!bZS!rh%iQ!uiQ!xkQ#PoQ#WzQ#X{Q#Y|Q#[}Q#^!OQ#_!PQ#a!QQ#b!RQ#p!_X#t!b#p#u$_r^Of!c!d!e!h!w#y$R$S$T$e$m$|%`%j%u![sQTZhikorz{|}!O!P!Q!R!U!V!_!b!p#k#p#u$_$u%^%i%lQ![XR#i!T[qQTr!U!V!pQ$[#kV%]$u%^%lTwRxQ$f#yR%S$mQ!thR%r%irbOf!c!d!e!h!w#y$R$S$T$e$m$|%`%j%uQ!ZXR#h!T", + states: "?[QYQ!SOOOOQ!Q'#Ek'#EkO!sO!bO'#DXO%kQ!TO'#DdO&UOSO'#DaOOQ!R'#Da'#DaO)SQ!TO'#EnOOQ!Q'#E{'#E{O)pQRO'#DxO+xQ!TO'#EjO,fQ!SO'#DVOOQ!R'#Dz'#DzO/WQ!SO'#D{OOQ!R'#En'#EnO/_Q!TO'#EnO1cQ!TO'#EmO2qQ!TO'#EjO3OQRO'#ETOOQ!Q'#Ej'#EjO3gQ!SO'#EjO3nQrO'#EiOOQ!Q'#Ei'#EiOOQ!Q'#EV'#EVQYQ!SOOO4PQbO'#D]O4[QbO'#DrO5YQbO'#DSO6WQQO'#D}O5YQbO'#EPO6]QbO'#ERO6eObO,59sOOQ!Q'#D['#D[O6vQbO'#DqOOQ!Q'#Eq'#EqOOQ!Q'#E_'#E_O7QQ!SO,5:`OOQ!R'#Em'#EmO8QQbO'#DcO8`QWO'#DeOOOO'#Es'#EsOOOO'#E['#E[O8tOSO,59{OOQ!R,59{,59{O5YQbO,5:dO5YQbO,5:dO5YQbO,5:dO5YQbO,5:dO5YQbO,59pO5YQbO,59pO5YQbO,59pO5YQbO,59pOOQ!Q'#EX'#EXO,fQ!SO,59qO9SQ!TO'#DdO9^Q!TO'#EnO9hQsO,59qO9uQQO,59qO9zQrO,59qO:VQrO,59qO:eQsO,59qO;TQsO,59qO;[QrO'#DQO;dQ!SO,5:gO;kQrO,5:fOOQ!R,5:g,5:gO;yQ!SO,5:gO]QQO'#EWOOQ!Q-E8T-E8TOOQ!Q'#EY'#EYO>bQbO'#D^O>mQbO'#D_OOQO'#EZ'#EZO>eQQO'#D^O?RQQO,59wO?WQcO'#EmO@TQRO'#EzOAQQRO'#EzOOQO'#Ez'#EzOAXQQO,5:^OA^QRO,59nOAeQRO,59nOYQ!SO,5:iOAsQ!TO,5:kOCXQ!TO,5:kOC{Q!TO,5:kODYQ!SO,5:mOOQ!Q'#Ec'#EcO6]QbO,5:mOOQ!R1G/_1G/_OOQ!Q,5:],5:]OOQ!Q-E8]-E8]OOOO'#Dd'#DdOOOO,59},59}OOOO,5:P,5:POOOO-E8Y-E8YOOQ!R1G/g1G/gOOQ!R1G0O1G0OOF_Q!TO1G0OOFiQ!TO1G0OOG}Q!TO1G0OOHXQ!TO1G0OOHfQ!TO1G0OOOQ!R1G/[1G/[OI}Q!TO1G/[OJUQ!TO1G/[OJ]Q!TO1G/[OKbQ!TO1G/[OJdQ!TO1G/[OOQ!Q-E8V-E8VOKxQsO1G/]OLVQQO1G/]OL[QrO1G/]OLgQrO1G/]OLuQsO1G/]OL|QsO1G/]OMTQ!SO,59rOM_QrO1G/]OOQ!R1G/]1G/]OMjQrO1G0QOOQ!R1G0R1G0ROMxQ!SO1G0ROOQp'#Ea'#EaOMjQrO1G0QOOQ!R1G0Q1G0QOOQ!Q'#Eb'#EbOMxQ!SO1G0RONVQ!SO1G0[ONwQ!SO1G0ZO! iQ!SO'#DlO! }Q!SO'#DlO!!_QbO1G0SOOQ!Q-E8U-E8UOYQ!SO,5:rOOQ!Q,5:r,5:rOYQ!SO,5:rOOQ!Q-E8W-E8WO!!jQQO,59xOOQO,59y,59yOOQO-E8X-E8XOYQ!SO1G/cOYQ!SO1G/xOYQ!SO1G/YO!!rQbO1G0TO!!}Q!SO1G0XO!#rQ!SO1G0XOOQ!Q-E8a-E8aO!#yQrO7+$wOOQ!R7+$w7+$wO!$UQrO1G/^O!$aQrO7+%lOOQ!R7+%l7+%lO!$oQ!SO7+%mOOQ!R7+%m7+%mOOQp-E8_-E8_OOQ!Q-E8`-E8`OOQ!Q'#E]'#E]O!$|QrO'#E]O!%[Q!SO'#EyOOQ`,5:W,5:WO!%lQbO'#DjO!%qQQO'#DmOOQ!Q7+%n7+%nO!%vQbO7+%nO!%{QbO7+%nOOQ!Q1G0^1G0^OYQ!SO1G0^O!&TQ!SO7+$}O!&fQ!SO7+$}O!&sQbO7+%dO!&{QbO7+$tOOQ!Q7+%o7+%oO!'QQbO7+%oO!'VQbO7+%oO!'_Q!SO7+%sOOQ!R<tAN>tOOQ!QAN>TAN>TO!*XQbOAN>TO!*^QbOAN>TOOQ`-E8^-E8^OOQ!QAN>jAN>jO!*fQbOAN>jO4[QbO,5:aOYQ!SO,5:cOOQ!QAN>uAN>uPMTQ!SO'#EXOOQ`7+%[7+%[OOQ!QG23oG23oO!*kQbOG23oP!)kQbO'#DuOOQ!QG24UG24UO!*pQQO1G/{OOQ`1G/}1G/}OOQ!QLD)ZLD)ZOYQ!SO7+%gOOQ`<V!`#O$R#P;'S$R;'S;=`$j<%lO$RU>[V!USOt$Ruw$Rx#O$R#P#Q>q#Q;'S$R;'S;=`$j<%lO$RU>xU#nQ!USOt$Ruw$Rx#O$R#P;'S$R;'S;=`$j<%lO$R~?aO#f~U?hU#pQ!USOt$Ruw$Rx#O$R#P;'S$R;'S;=`$j<%lO$RU@RU!US!dQOt$Ruw$Rx#O$R#P;'S$R;'S;=`$j<%lO$RU@j^!USOt$Ruw$Rx}$R}!O@e!O!Q$R!Q![@e![!_$R!_!`Af!`#O$R#P#T$R#T#o@e#o;'S$R;'S;=`$j<%lO$RUAmU!SQ!USOt$Ruw$Rx#O$R#P;'S$R;'S;=`$j<%lO$RUBU_!USOt$Ruw$Rx}$R}!O@e!O!Q$R!Q![@e![!_$R!_!`Af!`#O$R#P#T$R#T#UCT#U#o@e#o;'S$R;'S;=`$j<%lO$RUCY`!USOt$Ruw$Rx}$R}!O@e!O!Q$R!Q![@e![!_$R!_!`Af!`#O$R#P#T$R#T#`@e#`#aD[#a#o@e#o;'S$R;'S;=`$j<%lO$RUDa`!USOt$Ruw$Rx}$R}!O@e!O!Q$R!Q![@e![!_$R!_!`Af!`#O$R#P#T$R#T#g@e#g#hEc#h#o@e#o;'S$R;'S;=`$j<%lO$RUEh`!USOt$Ruw$Rx}$R}!O@e!O!Q$R!Q![@e![!_$R!_!`Af!`#O$R#P#T$R#T#X@e#X#YFj#Y#o@e#o;'S$R;'S;=`$j<%lO$RUFq^!ZQ!USOt$Ruw$Rx}$R}!O@e!O!Q$R!Q![@e![!_$R!_!`Af!`#O$R#P#T$R#T#o@e#o;'S$R;'S;=`$j<%lO$R^Gt^#gW!USOt$Ruw$Rx}$R}!O@e!O!Q$R!Q![@e![!_$R!_!`Af!`#O$R#P#T$R#T#o@e#o;'S$R;'S;=`$j<%lO$R^Hw^#iW!USOt$Ruw$Rx}$R}!O@e!O!Q$R!Q![@e![!_$R!_!`Af!`#O$R#P#T$R#T#o@e#o;'S$R;'S;=`$j<%lO$R^Iz`#hW!USOt$Ruw$Rx}$R}!O@e!O!Q$R!Q![@e![!_$R!_!`Af!`#O$R#P#T$R#T#f@e#f#gJ|#g#o@e#o;'S$R;'S;=`$j<%lO$RUKR`!USOt$Ruw$Rx}$R}!O@e!O!Q$R!Q![@e![!_$R!_!`Af!`#O$R#P#T$R#T#i@e#i#jEc#j#o@e#o;'S$R;'S;=`$j<%lO$RUL[UuQ!USOt$Ruw$Rx#O$R#P;'S$R;'S;=`$j<%lO$R~LsO#q~", - tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, new LocalTokenGroup("[~RP!O!PU~ZO#a~~", 11)], + tokenData: "Lp~R}OX$OXY$mYp$Opq$mqr$Ors%Wst'^tu(uuw$Owx(|xy)Ryz)lz{$O{|*V|}$O}!O*V!O!P$O!P!Q3r!Q!R*w!R![-l![!]<_!]!^S!`#O$O#P;'S$O;'S;=`$g<%lO$OU>XV!USOt$Ouw$Ox#O$O#P#Q>n#Q;'S$O;'S;=`$g<%lO$OU>uU#qQ!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$O~?^O#i~U?eU#sQ!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU@OU!US!dQOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU@g^!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#o@b#o;'S$O;'S;=`$g<%lO$OUAjU!SQ!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OUBR_!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#UCQ#U#o@b#o;'S$O;'S;=`$g<%lO$OUCV`!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#`@b#`#aDX#a#o@b#o;'S$O;'S;=`$g<%lO$OUD^`!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#g@b#g#hE`#h#o@b#o;'S$O;'S;=`$g<%lO$OUEe`!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#X@b#X#YFg#Y#o@b#o;'S$O;'S;=`$g<%lO$OUFn^!ZQ!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#o@b#o;'S$O;'S;=`$g<%lO$O^Gq^#jW!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#o@b#o;'S$O;'S;=`$g<%lO$O^Ht^#lW!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#o@b#o;'S$O;'S;=`$g<%lO$O^Iw`#kW!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#f@b#f#gJy#g#o@b#o;'S$O;'S;=`$g<%lO$OUKO`!USOt$Ouw$Ox}$O}!O@b!O!Q$O!Q![@b![!_$O!_!`Ac!`#O$O#P#T$O#T#i@b#i#jE`#j#o@b#o;'S$O;'S;=`$g<%lO$OULXUuQ!USOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$O~LpO#t~", + tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, pipeStartsLineTokenizer, new LocalTokenGroup("[~RP!O!PU~ZO#d~~", 11)], topRules: {"Program":[0,35]}, specialized: [{term: 28, get: (value: any, stack: any) => (specializeKeyword(value, stack) << 1), external: specializeKeyword},{term: 28, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}], - tokenPrec: 2428 + tokenPrec: 2589 }) diff --git a/src/parser/tests/pipes.test.ts b/src/parser/tests/pipes.test.ts index 0d5da6a..e1ed0a9 100644 --- a/src/parser/tests/pipes.test.ts +++ b/src/parser/tests/pipes.test.ts @@ -1,4 +1,5 @@ import { expect, describe, test } from 'bun:test' +import { parser } from '../shrimp' import '../shrimp.grammar' // Importing this so changes cause it to retest! @@ -176,3 +177,159 @@ describe('pipe expressions', () => { `) }) }) + +describe('pipe continuation', () => { + test('pipe on next line', () => { + expect(`hello +| echo`).toMatchTree(` + PipeExpr + FunctionCallOrIdentifier + Identifier hello + operator | + FunctionCallOrIdentifier + Identifier echo + `) + + expect(`echo hello +| grep h`).toMatchTree(` + PipeExpr + FunctionCall + Identifier echo + PositionalArg + Identifier hello + operator | + FunctionCall + Identifier grep + PositionalArg + Identifier h + `) + }) + + test('pipe on next non-empty line', () => { + expect(`hello + + +| echo`).toMatchTree(` + PipeExpr + FunctionCallOrIdentifier + Identifier hello + operator | + FunctionCallOrIdentifier + Identifier echo + `) + }) + + test('multi-line pipe chain', () => { + expect(`echo hello +| grep h +| sort`).toMatchTree(` + PipeExpr + FunctionCall + Identifier echo + PositionalArg + Identifier hello + operator | + FunctionCall + Identifier grep + PositionalArg + Identifier h + operator | + FunctionCallOrIdentifier + Identifier sort + `) + }) + + test('pipe with indentation', () => { + expect(`echo hello + | grep h + | sort`).toMatchTree(` + PipeExpr + FunctionCall + Identifier echo + PositionalArg + Identifier hello + operator | + FunctionCall + Identifier grep + PositionalArg + Identifier h + operator | + FunctionCallOrIdentifier + Identifier sort + `) + }) + + test('pipe after operand on next line (trailing pipe style)', () => { + expect(`echo hello | +grep h`).toMatchTree(` + PipeExpr + FunctionCall + Identifier echo + PositionalArg + Identifier hello + operator | + FunctionCall + Identifier grep + PositionalArg + Identifier h + `) + }) + + test('same-line pipes still work', () => { + expect('echo hello | grep h | sort').toMatchTree(` + PipeExpr + FunctionCall + Identifier echo + PositionalArg + Identifier hello + operator | + FunctionCall + Identifier grep + PositionalArg + Identifier h + operator | + FunctionCallOrIdentifier + Identifier sort + `) + }) + + test('lots of pipes', () => { + expect(` +'this should help readability in long chains' + | split ' ' + | map (ref str.to-upper) + | join '-' + | echo +`).toMatchTree(` + PipeExpr + String + StringFragment this should help readability in long chains + operator | + FunctionCall + Identifier split + PositionalArg + String + StringFragment + operator | + FunctionCall + Identifier map + PositionalArg + ParenExpr + FunctionCall + Identifier ref + PositionalArg + DotGet + IdentifierBeforeDot str + Identifier to-upper + operator | + FunctionCall + Identifier join + PositionalArg + String + StringFragment - + operator | + FunctionCallOrIdentifier + Identifier echo + `) + }) +}) diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts index 75c027a..cbecdb7 100644 --- a/src/parser/tokenizer.ts +++ b/src/parser/tokenizer.ts @@ -1,5 +1,5 @@ import { ExternalTokenizer, InputStream, Stack } from '@lezer/lr' -import { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, Do, CurlyString, DotGet } from './shrimp.terms' +import { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, Do, CurlyString, DotGet, newline, pipeStartsLine } from './shrimp.terms' // doobie doobie do (we need the `do` keyword to know when we're defining params) export function specializeKeyword(ident: string) { @@ -356,3 +356,34 @@ const isEmojiOrUnicode = (ch: number): boolean => { } const getCharSize = (ch: number) => (ch > 0xffff ? 2 : 1) // emoji takes 2 UTF-16 code units + +export const pipeStartsLineTokenizer = new ExternalTokenizer((input: InputStream, stack: Stack) => { + const ch = input.peek(0) + + if (ch !== 10 /* \n */) return + + // ignore whitespace + let offset = 1 + let lastNewlineOffset = 0 + + while (true) { + const ch = input.peek(offset) + if (ch === 10 /* \n */) { + lastNewlineOffset = offset + offset++ + } else if (isWhiteSpace(ch)) { + offset++ + } else { + break + } + } + + // look for pipe after skipping empty lines + if (input.peek(offset) === 124 /* | */) { + input.advance(lastNewlineOffset + 1) + input.acceptToken(pipeStartsLine) + } else { + input.advance(1) + input.acceptToken(newline) + } +})