diff --git a/src/compiler/tests/pipe.test.ts b/src/compiler/tests/pipe.test.ts index b8f92f1..4d1669f 100644 --- a/src/compiler/tests/pipe.test.ts +++ b/src/compiler/tests/pipe.test.ts @@ -93,7 +93,6 @@ describe('pipe expressions', () => { `).toEvaluateTo(5) }) - test('string literals can be piped', () => { expect(`'hey there' | str.to-upper`).toEvaluateTo('HEY THERE') }) diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index 80adcdb..532a569 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -21,7 +21,7 @@ ("-" | "+")? $[0-9]+ ("_"? $[0-9]+)* ('.' $[0-9]+ ("_"? $[0-9]+)*)? } Boolean { "true" | "false" } - newlineOrSemicolon { "\n" | ";" } + semicolon { ";" } eof { @eof } space { " " | "\t" } Comment { "#" ![\n]* } @@ -33,6 +33,8 @@ "|"[@name=operator] } +newlineOrSemicolon { newline | semicolon } + end { @specialize[@name=keyword] } while { @specialize[@name=keyword] } if { @specialize[@name=keyword] } @@ -45,6 +47,7 @@ import { @specialize[@name=keyword] } null { @specialize[@name=Null] } @external tokens tokenizer from "./tokenizer" { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, CurlyString } +@external tokens pipeStartsLineTokenizer from "./tokenizer" { newline, pipeStartsLine } @external specialize {Identifier} specializeKeyword from "./tokenizer" { Do } @precedence { @@ -84,7 +87,7 @@ consumeToTerminator { } PipeExpr { - pipeOperand (!pipe "|" pipeOperand)+ + pipeOperand (!pipe (pipeStartsLine? "|") newlineOrSemicolon* pipeOperand)+ } pipeOperand { diff --git a/src/parser/shrimp.terms.ts b/src/parser/shrimp.terms.ts index 0f3ba5b..cf01b79 100644 --- a/src/parser/shrimp.terms.ts +++ b/src/parser/shrimp.terms.ts @@ -32,6 +32,8 @@ export const Word = 30, IdentifierBeforeDot = 31, CurlyString = 32, + newline = 100, + pipeStartsLine = 101, Do = 33, Comment = 34, Program = 35, diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index 051f00e..305a9bd 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -1,17 +1,17 @@ // This file was generated by lezer-generator. You probably shouldn't edit it. import {LRParser, LocalTokenGroup} from "@lezer/lr" import {operatorTokenizer} from "./operatorTokenizer" -import {tokenizer, specializeKeyword} from "./tokenizer" +import {tokenizer, pipeStartsLineTokenizer, specializeKeyword} from "./tokenizer" import {trackScope} from "./parserScopeContext" import {highlighting} from "./highlight" const spec_Identifier = {__proto__:null,while:78, null:112, catch:118, finally:124, end:126, if:134, else:140, try:158, throw:162, import:166} export const parser = LRParser.deserialize({ version: 14, - states: "=|QYQbOOO!mOpO'#DXO!rOSO'#D`OOQa'#D`'#D`O%mQcO'#DvO(mQcO'#EiOOQ`'#Ew'#EwO)WQRO'#DwO+]QcO'#EgO+vQbO'#DVOOQa'#Dy'#DyO.[QbO'#DzOOQa'#Ei'#EiO.cQcO'#EiO0aQcO'#EhO1fQcO'#EgO1sQRO'#ESOOQ`'#Eg'#EgO2[QbO'#EgO2cQQO'#EfOOQ`'#Ef'#EfOOQ`'#EU'#EUQYQbOOO2nQbO'#D[O2yQbO'#DpO3tQbO'#DSO4oQQO'#D|O3tQbO'#EOO4tQbO'#EQO4|ObO,59sO5XQbO'#DbO5aQWO'#DcOOOO'#Eo'#EoOOOO'#EZ'#EZO5uOSO,59zOOQa,59z,59zOOQ`'#DZ'#DZO6TQbO'#DoOOQ`'#Em'#EmOOQ`'#E^'#E^O6_QbO,5:^OOQa'#Eh'#EhO3tQbO,5:cO3tQbO,5:cO3tQbO,5:cO3tQbO,5:cO3tQbO,59pO3tQbO,59pO3tQbO,59pO3tQbO,59pOOQ`'#EW'#EWO+vQbO,59qO7XQcO'#DvO7`QcO'#EiO7gQRO,59qO7qQQO,59qO7vQQO,59qO8OQQO,59qO8ZQRO,59qO8sQRO,59qO8zQQO'#DQO9PQbO,5:fO9WQQO,5:eOOQa,5:f,5:fO9cQbO,5:fO9mQbO,5:oO9mQbO,5:nO:}QbO,5:gO;UQbO,59lOOQ`,5;Q,5;QO9mQbO'#EVOOQ`-E8S-E8SOOQ`'#EX'#EXO;pQbO'#D]O;{QbO'#D^OOQO'#EY'#EYO;sQQO'#D]O`QRO'#EvOOQO'#Ev'#EvO>gQQO,5:[O>lQRO,59nO>sQRO,59nO:}QbO,5:hO?RQcO,5:jO@aQcO,5:jO@}QcO,5:jOArQbO,5:lOOQ`'#Eb'#EbO4tQbO,5:lOOQa1G/_1G/_OOOO,59|,59|OOOO,59},59}OOOO-E8X-E8XOOQa1G/f1G/fOOQ`,5:Z,5:ZOOQ`-E8[-E8[OOQa1G/}1G/}OCkQcO1G/}OCuQcO1G/}OETQcO1G/}OE_QcO1G/}OElQcO1G/}OOQa1G/[1G/[OF}QcO1G/[OGUQcO1G/[OG]QcO1G/[OH[QcO1G/[OGdQcO1G/[OOQ`-E8U-E8UOHrQRO1G/]OH|QQO1G/]OIRQQO1G/]OIZQQO1G/]OIfQRO1G/]OImQRO1G/]OItQbO,59rOJOQQO1G/]OOQa1G/]1G/]OJWQQO1G0POOQa1G0Q1G0QOJcQbO1G0QOOQO'#E`'#E`OJWQQO1G0POOQa1G0P1G0POOQ`'#Ea'#EaOJcQbO1G0QOJmQbO1G0ZOKXQbO1G0YOKsQbO'#DjOLUQbO'#DjOLiQbO1G0ROOQ`-E8T-E8TOOQ`,5:q,5:qOOQ`-E8V-E8VOLtQQO,59wOOQO,59x,59xOOQO-E8W-E8WOL|QbO1G/bO:}QbO1G/vO:}QbO1G/YOMTQbO1G0SOM`QbO1G0WOM}QbO1G0WOOQ`-E8`-E8`ONUQQO7+$wOOQa7+$w7+$wON^QQO1G/^ONfQQO7+%kOOQa7+%k7+%kONqQbO7+%lOOQa7+%l7+%lOOQO-E8^-E8^OOQ`-E8_-E8_OOQ`'#E['#E[ON{QQO'#E[O! TQbO'#EuOOQ`,5:U,5:UO! hQbO'#DhO! mQQO'#DkOOQ`7+%m7+%mO! rQbO7+%mO! wQbO7+%mO!!PQbO7+$|O!!_QbO7+$|O!!oQbO7+%bO!!wQbO7+$tOOQ`7+%n7+%nO!!|QbO7+%nO!#RQbO7+%nO!#ZQbO7+%rOOQa<sAN>sOOQ`AN>SAN>SO!%zQbOAN>SO!&PQbOAN>SOOQ`-E8]-E8]OOQ`AN>hAN>hO!&XQbOAN>hO2yQbO,5:_O:}QbO,5:aOOQ`AN>tAN>tPItQbO'#EWOOQ`7+%Y7+%YOOQ`G23nG23nO!&^QbOG23nP!%^QbO'#DsOOQ`G24SG24SO!&cQQO1G/yOOQ`1G/{1G/{OOQ`LD)YLD)YO:}QbO7+%eOOQ`<xQYQ!SOOOOQ!Q'#Ej'#EjO!pO!bO'#DXO!uOSO'#D`OOQ!R'#D`'#D`O%sQ!TO'#DvO(yQ!TO'#EmOOQ!Q'#Ez'#EzO)gQRO'#DwO+oQ!TO'#EiO,]Q!SO'#DVOOQ!R'#Dy'#DyO.wQ!SO'#DzOOQ!R'#Em'#EmO/OQ!TO'#EmO1SQ!TO'#ElO2bQ!TO'#EiO2oQRO'#ESOOQ!Q'#Ei'#EiO3WQ!SO'#EiO3_QrO'#EhOOQ!Q'#Eh'#EhOOQ!Q'#EU'#EUQYQ!SOOO3pQbO'#D[O3{QbO'#DpO4vQbO'#DSO5qQQO'#D|O4vQbO'#EOO5vQbO'#EQO6OObO,59sO6ZQbO'#DbO6cQWO'#DcOOOO'#Er'#ErOOOO'#EZ'#EZO6wOSO,59zOOQ!R,59z,59zOOQ!Q'#DZ'#DZO7VQbO'#DoOOQ!Q'#Ep'#EpOOQ!Q'#E^'#E^O7aQ!SO,5:^OOQ!R'#El'#ElO4vQbO,5:cO4vQbO,5:cO4vQbO,5:cO4vQbO,5:cO4vQbO,59pO4vQbO,59pO4vQbO,59pO4vQbO,59pOOQ!Q'#EW'#EWO,]Q!SO,59qO8aQ!TO'#DvO8kQ!TO'#EmO8uQsO,59qO9SQQO,59qO9XQrO,59qO9dQrO,59qO9rQsO,59qO:bQsO,59qO:iQrO'#DQO:qQ!SO,5:fO:xQrO,5:eOOQ!R,5:f,5:fO;WQ!SO,5:fO;eQbO,5:oO;eQbO,5:nOYQ!SO,5:gO]QQO,59vO>bQcO'#ElO?_QRO'#EyO@[QRO'#EyOOQO'#Ey'#EyO@cQQO,5:[O@hQRO,59nO@oQRO,59nOYQ!SO,5:hO@}Q!TO,5:jOBcQ!TO,5:jOCVQ!TO,5:jOCdQ!SO,5:lOOQ!Q'#Eb'#EbO5vQbO,5:lOOQ!R1G/_1G/_OOOO,59|,59|OOOO,59},59}OOOO-E8X-E8XOOQ!R1G/f1G/fOOQ!Q,5:Z,5:ZOOQ!Q-E8[-E8[OOQ!R1G/}1G/}OEiQ!TO1G/}OEsQ!TO1G/}OGXQ!TO1G/}OGcQ!TO1G/}OGpQ!TO1G/}OOQ!R1G/[1G/[OIXQ!TO1G/[OI`Q!TO1G/[OIgQ!TO1G/[OJlQ!TO1G/[OInQ!TO1G/[OOQ!Q-E8U-E8UOKSQsO1G/]OKaQQO1G/]OKfQrO1G/]OKqQrO1G/]OLPQsO1G/]OLWQsO1G/]OL_Q!SO,59rOLiQrO1G/]OOQ!R1G/]1G/]OLtQrO1G0POOQ!R1G0Q1G0QOMSQ!SO1G0QOOQp'#E`'#E`OLtQrO1G0POOQ!R1G0P1G0POOQ!Q'#Ea'#EaOMSQ!SO1G0QOMaQ!SO1G0ZONRQ!SO1G0YONsQ!SO'#DjO! XQ!SO'#DjO! iQbO1G0ROOQ!Q-E8T-E8TOYQ!SO,5:qOOQ!Q,5:q,5:qOYQ!SO,5:qOOQ!Q-E8V-E8VO! tQQO,59wOOQO,59x,59xOOQO-E8W-E8WOYQ!SO1G/bOYQ!SO1G/vOYQ!SO1G/YO! |QbO1G0SO!!XQ!SO1G0WO!!|Q!SO1G0WOOQ!Q-E8`-E8`O!#TQrO7+$wOOQ!R7+$w7+$wO!#`QrO1G/^O!#kQrO7+%kOOQ!R7+%k7+%kO!#yQ!SO7+%lOOQ!R7+%l7+%lOOQp-E8^-E8^OOQ!Q-E8_-E8_OOQ!Q'#E['#E[O!$WQrO'#E[O!$fQ!SO'#ExOOQ`,5:U,5:UO!$vQbO'#DhO!${QQO'#DkOOQ!Q7+%m7+%mO!%QQbO7+%mO!%VQbO7+%mOOQ!Q1G0]1G0]OYQ!SO1G0]O!%_Q!SO7+$|O!%pQ!SO7+$|O!%}QbO7+%bO!&VQbO7+$tOOQ!Q7+%n7+%nO!&[QbO7+%nO!&aQbO7+%nO!&iQ!SO7+%rOOQ!R<sAN>sOOQ!QAN>SAN>SO!)cQbOAN>SO!)hQbOAN>SOOQ`-E8]-E8]OOQ!QAN>hAN>hO!)pQbOAN>hO3{QbO,5:_OYQ!SO,5:aOOQ!QAN>tAN>tPL_Q!SO'#EWOOQ`7+%Y7+%YOOQ!QG23nG23nO!)uQbOG23nP!(uQbO'#DsOOQ!QG24SG24SO!)zQQO1G/yOOQ`1G/{1G/{OOQ!QLD)YLD)YOYQ!SO7+%eOOQ`<T!`#O$R#P;'S$R;'S;=`$j<%lO$RU>YV!TSOt$Ruw$Rx#O$R#P#Q>o#Q;'S$R;'S;=`$j<%lO$RU>vU#mQ!TSOt$Ruw$Rx#O$R#P;'S$R;'S;=`$j<%lO$R~?_O#e~U?fU#oQ!TSOt$Ruw$Rx#O$R#P;'S$R;'S;=`$j<%lO$RU@PU!TS!bQOt$Ruw$Rx#O$R#P;'S$R;'S;=`$j<%lO$RU@h^!TSOt$Ruw$Rx}$R}!O@c!O!Q$R!Q![@c![!_$R!_!`Ad!`#O$R#P#T$R#T#o@c#o;'S$R;'S;=`$j<%lO$RUAkU!RQ!TSOt$Ruw$Rx#O$R#P;'S$R;'S;=`$j<%lO$RUBS_!TSOt$Ruw$Rx}$R}!O@c!O!Q$R!Q![@c![!_$R!_!`Ad!`#O$R#P#T$R#T#UCR#U#o@c#o;'S$R;'S;=`$j<%lO$RUCW`!TSOt$Ruw$Rx}$R}!O@c!O!Q$R!Q![@c![!_$R!_!`Ad!`#O$R#P#T$R#T#`@c#`#aDY#a#o@c#o;'S$R;'S;=`$j<%lO$RUD_`!TSOt$Ruw$Rx}$R}!O@c!O!Q$R!Q![@c![!_$R!_!`Ad!`#O$R#P#T$R#T#g@c#g#hEa#h#o@c#o;'S$R;'S;=`$j<%lO$RUEf`!TSOt$Ruw$Rx}$R}!O@c!O!Q$R!Q![@c![!_$R!_!`Ad!`#O$R#P#T$R#T#X@c#X#YFh#Y#o@c#o;'S$R;'S;=`$j<%lO$RUFo^!XQ!TSOt$Ruw$Rx}$R}!O@c!O!Q$R!Q![@c![!_$R!_!`Ad!`#O$R#P#T$R#T#o@c#o;'S$R;'S;=`$j<%lO$R^Gr^#fW!TSOt$Ruw$Rx}$R}!O@c!O!Q$R!Q![@c![!_$R!_!`Ad!`#O$R#P#T$R#T#o@c#o;'S$R;'S;=`$j<%lO$R^Hu^#hW!TSOt$Ruw$Rx}$R}!O@c!O!Q$R!Q![@c![!_$R!_!`Ad!`#O$R#P#T$R#T#o@c#o;'S$R;'S;=`$j<%lO$R^Ix`#gW!TSOt$Ruw$Rx}$R}!O@c!O!Q$R!Q![@c![!_$R!_!`Ad!`#O$R#P#T$R#T#f@c#f#gJz#g#o@c#o;'S$R;'S;=`$j<%lO$RUKP`!TSOt$Ruw$Rx}$R}!O@c!O!Q$R!Q![@c![!_$R!_!`Ad!`#O$R#P#T$R#T#i@c#i#jEa#j#o@c#o;'S$R;'S;=`$j<%lO$RULYUuQ!TSOt$Ruw$Rx#O$R#P;'S$R;'S;=`$j<%lO$R~LqO#p~", - tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, new LocalTokenGroup("[~RP!O!PU~ZO#`~~", 11)], + tokenData: "Ln~R}OX$OXY$mYp$Opq$mqr$Ors%Wst'^tu(uuw$Owx(zxy)Pyz)jz{$O{|*T|}$O}!O*T!O!P$O!P!Q3p!Q!R*u!R![-j![!]<]!]!^Q!`#O$O#P;'S$O;'S;=`$g<%lO$OU>VV!TSOt$Ouw$Ox#O$O#P#Q>l#Q;'S$O;'S;=`$g<%lO$OU>sU#pQ!TSOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$O~?[O#h~U?cU#rQ!TSOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU?|U!TS!bQOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OU@e^!TSOt$Ouw$Ox}$O}!O@`!O!Q$O!Q![@`![!_$O!_!`Aa!`#O$O#P#T$O#T#o@`#o;'S$O;'S;=`$g<%lO$OUAhU!RQ!TSOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$OUBP_!TSOt$Ouw$Ox}$O}!O@`!O!Q$O!Q![@`![!_$O!_!`Aa!`#O$O#P#T$O#T#UCO#U#o@`#o;'S$O;'S;=`$g<%lO$OUCT`!TSOt$Ouw$Ox}$O}!O@`!O!Q$O!Q![@`![!_$O!_!`Aa!`#O$O#P#T$O#T#`@`#`#aDV#a#o@`#o;'S$O;'S;=`$g<%lO$OUD[`!TSOt$Ouw$Ox}$O}!O@`!O!Q$O!Q![@`![!_$O!_!`Aa!`#O$O#P#T$O#T#g@`#g#hE^#h#o@`#o;'S$O;'S;=`$g<%lO$OUEc`!TSOt$Ouw$Ox}$O}!O@`!O!Q$O!Q![@`![!_$O!_!`Aa!`#O$O#P#T$O#T#X@`#X#YFe#Y#o@`#o;'S$O;'S;=`$g<%lO$OUFl^!XQ!TSOt$Ouw$Ox}$O}!O@`!O!Q$O!Q![@`![!_$O!_!`Aa!`#O$O#P#T$O#T#o@`#o;'S$O;'S;=`$g<%lO$O^Go^#iW!TSOt$Ouw$Ox}$O}!O@`!O!Q$O!Q![@`![!_$O!_!`Aa!`#O$O#P#T$O#T#o@`#o;'S$O;'S;=`$g<%lO$O^Hr^#kW!TSOt$Ouw$Ox}$O}!O@`!O!Q$O!Q![@`![!_$O!_!`Aa!`#O$O#P#T$O#T#o@`#o;'S$O;'S;=`$g<%lO$O^Iu`#jW!TSOt$Ouw$Ox}$O}!O@`!O!Q$O!Q![@`![!_$O!_!`Aa!`#O$O#P#T$O#T#f@`#f#gJw#g#o@`#o;'S$O;'S;=`$g<%lO$OUJ|`!TSOt$Ouw$Ox}$O}!O@`!O!Q$O!Q![@`![!_$O!_!`Aa!`#O$O#P#T$O#T#i@`#i#jE^#j#o@`#o;'S$O;'S;=`$g<%lO$OULVUuQ!TSOt$Ouw$Ox#O$O#P;'S$O;'S;=`$g<%lO$O~LnO#s~", + tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, pipeStartsLineTokenizer, new LocalTokenGroup("[~RP!O!PU~ZO#c~~", 11)], topRules: {"Program":[0,35]}, specialized: [{term: 28, get: (value: any, stack: any) => (specializeKeyword(value, stack) << 1), external: specializeKeyword},{term: 28, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}], - tokenPrec: 2370 + tokenPrec: 2531 }) diff --git a/src/parser/tests/pipes.test.ts b/src/parser/tests/pipes.test.ts index 0d5da6a..e1ed0a9 100644 --- a/src/parser/tests/pipes.test.ts +++ b/src/parser/tests/pipes.test.ts @@ -1,4 +1,5 @@ import { expect, describe, test } from 'bun:test' +import { parser } from '../shrimp' import '../shrimp.grammar' // Importing this so changes cause it to retest! @@ -176,3 +177,159 @@ describe('pipe expressions', () => { `) }) }) + +describe('pipe continuation', () => { + test('pipe on next line', () => { + expect(`hello +| echo`).toMatchTree(` + PipeExpr + FunctionCallOrIdentifier + Identifier hello + operator | + FunctionCallOrIdentifier + Identifier echo + `) + + expect(`echo hello +| grep h`).toMatchTree(` + PipeExpr + FunctionCall + Identifier echo + PositionalArg + Identifier hello + operator | + FunctionCall + Identifier grep + PositionalArg + Identifier h + `) + }) + + test('pipe on next non-empty line', () => { + expect(`hello + + +| echo`).toMatchTree(` + PipeExpr + FunctionCallOrIdentifier + Identifier hello + operator | + FunctionCallOrIdentifier + Identifier echo + `) + }) + + test('multi-line pipe chain', () => { + expect(`echo hello +| grep h +| sort`).toMatchTree(` + PipeExpr + FunctionCall + Identifier echo + PositionalArg + Identifier hello + operator | + FunctionCall + Identifier grep + PositionalArg + Identifier h + operator | + FunctionCallOrIdentifier + Identifier sort + `) + }) + + test('pipe with indentation', () => { + expect(`echo hello + | grep h + | sort`).toMatchTree(` + PipeExpr + FunctionCall + Identifier echo + PositionalArg + Identifier hello + operator | + FunctionCall + Identifier grep + PositionalArg + Identifier h + operator | + FunctionCallOrIdentifier + Identifier sort + `) + }) + + test('pipe after operand on next line (trailing pipe style)', () => { + expect(`echo hello | +grep h`).toMatchTree(` + PipeExpr + FunctionCall + Identifier echo + PositionalArg + Identifier hello + operator | + FunctionCall + Identifier grep + PositionalArg + Identifier h + `) + }) + + test('same-line pipes still work', () => { + expect('echo hello | grep h | sort').toMatchTree(` + PipeExpr + FunctionCall + Identifier echo + PositionalArg + Identifier hello + operator | + FunctionCall + Identifier grep + PositionalArg + Identifier h + operator | + FunctionCallOrIdentifier + Identifier sort + `) + }) + + test('lots of pipes', () => { + expect(` +'this should help readability in long chains' + | split ' ' + | map (ref str.to-upper) + | join '-' + | echo +`).toMatchTree(` + PipeExpr + String + StringFragment this should help readability in long chains + operator | + FunctionCall + Identifier split + PositionalArg + String + StringFragment + operator | + FunctionCall + Identifier map + PositionalArg + ParenExpr + FunctionCall + Identifier ref + PositionalArg + DotGet + IdentifierBeforeDot str + Identifier to-upper + operator | + FunctionCall + Identifier join + PositionalArg + String + StringFragment - + operator | + FunctionCallOrIdentifier + Identifier echo + `) + }) +}) diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts index 8ad55c2..2248336 100644 --- a/src/parser/tokenizer.ts +++ b/src/parser/tokenizer.ts @@ -1,5 +1,5 @@ import { ExternalTokenizer, InputStream, Stack } from '@lezer/lr' -import { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, Do, CurlyString } from './shrimp.terms' +import { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, Do, CurlyString, newline, pipeStartsLine } from './shrimp.terms' // doobie doobie do (we need the `do` keyword to know when we're defining params) export function specializeKeyword(ident: string) { @@ -346,3 +346,34 @@ const isEmojiOrUnicode = (ch: number): boolean => { } const getCharSize = (ch: number) => (ch > 0xffff ? 2 : 1) // emoji takes 2 UTF-16 code units + +export const pipeStartsLineTokenizer = new ExternalTokenizer((input: InputStream, stack: Stack) => { + const ch = input.peek(0) + + if (ch !== 10 /* \n */) return + + // ignore whitespace + let offset = 1 + let lastNewlineOffset = 0 + + while (true) { + const ch = input.peek(offset) + if (ch === 10 /* \n */) { + lastNewlineOffset = offset + offset++ + } else if (isWhiteSpace(ch)) { + offset++ + } else { + break + } + } + + // look for pipe after skipping empty lines + if (input.peek(offset) === 124 /* | */) { + input.advance(lastNewlineOffset + 1) + input.acceptToken(pipeStartsLine) + } else { + input.advance(1) + input.acceptToken(newline) + } +})