From 503ca411551ab9fc941b70a4fadd56d2ddab885d Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Thu, 6 Nov 2025 10:40:09 -0800 Subject: [PATCH] { curly strings } --- src/compiler/tests/literals.test.ts | 26 +++++++++++++++++++++++ src/parser/shrimp.grammar | 6 ++++-- src/parser/shrimp.terms.ts | 1 + src/parser/shrimp.ts | 14 ++++++------ src/parser/tests/strings.test.ts | 28 ++++++++++++++++++++++++ src/parser/tokenizer.ts | 33 ++++++++++++++++++++++++++++- 6 files changed, 98 insertions(+), 10 deletions(-) diff --git a/src/compiler/tests/literals.test.ts b/src/compiler/tests/literals.test.ts index 45f9fba..705b825 100644 --- a/src/compiler/tests/literals.test.ts +++ b/src/compiler/tests/literals.test.ts @@ -193,3 +193,29 @@ describe('dict literals', () => { c=3]`).toEvaluateTo({ a: 1, b: 2, c: 3 }) }) }) + +describe('curly strings', () => { + test('work on one line', () => { + expect('{ one two three }').toEvaluateTo(" one two three ") + }) + + test('work on multiple lines', () => { + expect(`{ + one + two + three + }`).toEvaluateTo("\n one\n two\n three\n ") + }) + + test('can contain other curlies', () => { + expect(`{ + { one } + two + { three } + }`).toEvaluateTo("\n { one }\n two\n { three }\n ") + }) + + test("don't interpolate", () => { + expect(`{ sum is $(a + b)! }`).toEvaluateTo(` sum is $(a + b)! `) + }) +}) \ No newline at end of file diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index f4ac3e7..e658096 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -41,7 +41,7 @@ finally { @specialize[@name=keyword] } throw { @specialize[@name=keyword] } null { @specialize[@name=Null] } -@external tokens tokenizer from "./tokenizer" { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot } +@external tokens tokenizer from "./tokenizer" { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, curlyString } @external specialize {Identifier} specializeKeyword from "./tokenizer" { Do } @precedence { @@ -233,7 +233,9 @@ expression { IdentifierBeforeDot dot (Number | Identifier | ParenExpr) } - String { "'" stringContent* "'" } + String { + "'" stringContent* "'" | curlyString + } } stringContent { diff --git a/src/parser/shrimp.terms.ts b/src/parser/shrimp.terms.ts index 3da47bb..beab2bb 100644 --- a/src/parser/shrimp.terms.ts +++ b/src/parser/shrimp.terms.ts @@ -31,6 +31,7 @@ export const AssignableIdentifier = 29, Word = 30, IdentifierBeforeDot = 31, + curlyString = 95, Do = 32, Comment = 33, Program = 34, diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index 901d667..47b13e6 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -7,11 +7,11 @@ import {highlighting} from "./highlight" const spec_Identifier = {__proto__:null,while:76, null:108, catch:114, finally:120, end:122, if:130, else:136, try:154, throw:158} export const parser = LRParser.deserialize({ version: 14, - states: "vQcO,5:hO?dQcO,5:hOOQa1G/^1G/^OOOO,59{,59{OOOO,59|,59|OOOO-E8T-E8TOOQa1G/e1G/eOOQ`,5:X,5:XOOQ`-E8W-E8WOOQa1G/{1G/{OA`QcO1G/{OAjQcO1G/{OBxQcO1G/{OCSQcO1G/{OCaQcO1G/{OOQa1G/Z1G/ZODrQcO1G/ZODyQcO1G/ZOEQQcO1G/ZOFPQcO1G/ZOEXQcO1G/ZOOQ`-E8Q-E8QOFgQRO1G/[OFqQQO1G/[OFvQQO1G/[OGOQQO1G/[OGZQRO1G/[OGbQRO1G/[OGiQbO,59qOGsQQO1G/[OOQa1G/[1G/[OG{QQO1G/}OOQa1G0O1G0OOHWQbO1G0OOOQO'#E['#E[OG{QQO1G/}OOQa1G/}1G/}OOQ`'#E]'#E]OHWQbO1G0OOHbQbO1G0VOH|QbO1G0UOIhQbO'#DhOIyQbO'#DhOJ^QbO1G0POOQ`-E8P-E8POOQ`,5:m,5:mOOQ`-E8R-E8ROJiQQO,59vOOQO,59w,59wOOQO-E8S-E8SOJqQbO1G/aO9jQbO1G/tO9jQbO1G/XOJxQbO1G0QOKTQQO7+$vOOQa7+$v7+$vOK]QQO1G/]OKeQQO7+%iOOQa7+%i7+%iOKpQbO7+%jOOQa7+%j7+%jOOQO-E8Y-E8YOOQ`-E8Z-E8ZOOQ`'#EW'#EWOKzQQO'#EWOLSQbO'#EpOOQ`,5:S,5:SOLgQbO'#DfOLlQQO'#DiOOQ`7+%k7+%kOLqQbO7+%kOLvQbO7+%kOMOQbO7+${OM^QbO7+${OMnQbO7+%`OMvQbO7+$sOOQ`7+%l7+%lOM{QbO7+%lONQQbO7+%lOOQa<qAN>qOOQ`AN>RAN>RO!![QbOAN>RO!!aQbOAN>ROOQ`-E8X-E8XOOQ`AN>fAN>fO!!iQbOAN>fO2TQbO,5:]O9jQbO,5:_OOQ`AN>rAN>rPGiQbO'#ESOOQ`7+%W7+%WOOQ`G23mG23mO!!nQbOG23mP! nQbO'#DqOOQ`G24QG24QO!!sQQO1G/wOOQ`1G/y1G/yOOQ`LD)XLD)XO9jQbO7+%cOOQ`<UOT}OU!OOj!POt!pa#Y!pa#k!pa!Z!pa!^!pa!_!pa#g!pa!f!pa~O^xOR!iiS!iid!iie!iif!iig!iih!iii!iit!ii#Y!ii#k!ii#g!ii!Z!ii!^!ii!_!ii!f!ii~OP!iiQ!ii~P@XOPyOQyO~P@XOPyOQyOd!iie!iif!iig!iih!iii!iit!ii#Y!ii#k!ii#g!ii!Z!ii!^!ii!_!ii!f!ii~OR!iiS!ii~PAtORzOSzO^xO~PAtORzOSzO~PAtOW|OX|OY|OZ|O[|O]|OTwijwitwi#Ywi#kwi#gwi!Xwi!Zwi!^wi!_wi!fwi~OU!OO~PCkOU!OO~PC}OUwi~PCkOT}OU!OOjwitwi#Ywi#kwi#gwi!Xwi!Zwi!^wi!_wi!fwi~OW|OX|OY|OZ|O[|O]|O~PEXO#Y!QO#g$QO~P*RO#g$QO~O#g$QOt#UX~O!X!cO#g$QOt#UX~O#g$QO~P.WO#g$QO~P7WOpfO!`rO~P,kO#Y!QO#g$QO~O!QsO#Y#kO#j$TO~O#Y#nO#j$VO~P2xOt!fO#Y!si#k!si!Z!si!^!si!_!si#g!si!f!si~Ot!fO#Y!ri#k!ri!Z!ri!^!ri!_!ri#g!ri!f!ri~Ot!fO!Z![X!^![X!_![X!f![X~O#Y$YO!Z#dP!^#dP!_#dP!f#dP~P8cO!Z$^O!^$_O!_$`O~O!Q!jO!X!Oa~O#Y$dO~P8cO!Z$^O!^$_O!_$gO~O#Y!QO#g$jO~O#Y!QO#gyi~O!QsO#Y#kO#j$mO~O#Y#nO#j$nO~P2xOt!fO#Y$oO~O#Y$YO!Z#dX!^#dX!_#dX!f#dX~P8cOl$qO~O!X$rO~O!_$sO~O!^$_O!_$sO~Ot!fO!Z$^O!^$_O!_$uO~O#Y$YO!Z#dP!^#dP!_#dP~P8cO!_$|O!f${O~O!_%OO~O!_%PO~O!^$_O!_%PO~OpfO!`rO#gyq~P,kO#Y!QO#gyq~O!X%UO~O!_%WO~O!_%XO~O!^$_O!_%XO~O!Z$^O!^$_O!_%XO~O!_%]O!f${O~O!X%`O!c%_O~O!_%]O~O!_%aO~OpfO!`rO#gyy~P,kO!_%dO~O!^$_O!_%dO~O!_%gO~O!_%jO~O!X%kO~O{!j~", - goto: "8f#gPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP#hP$RP$h%f&t&zP(U(b)[)_P)eP*l*lPPP*pP*|+fPPP+|#hP,f-PP-T-Z-pP.g/k$R$RP$RP$R$R0q0w1T1w1}2X2_2f2l2v2|3WPPP3b3f4Z6PPPP7ZP7kPPPPP7o7u7{r`Oe!a!b!c!f!u#s#{#|#}$[$d$r%U%`%kQ!WWR#a!Rw`OWe!R!a!b!c!f!u#s#{#|#}$[$d$r%U%`%kr^Oe!a!b!c!f!u#s#{#|#}$[$d$r%U%`%kQ!ZWS!og%_Q!thQ!xjQ#W!OQ#Y}Q#]!PR#d!RvSOeg!a!b!c!f!u#s#{#|#}$[$d$r%U%_%`%k!WZRSYhjsvxyz{|}!O!P!S!T!]!`!n#e#j#o$U$k%S%bS!TW!RQ!ykR!zlQ!VWR#`!RrROe!a!b!c!f!u#s#{#|#}$[$d$r%U%`%k!WwRSYhjsvxyz{|}!O!P!S!T!]!`!n#e#j#o$U$k%S%bS!SW!RT!ng%_etRSv!S!T!n#e$k%S%br`Oe!a!b!c!f!u#s#{#|#}$[$d$r%U%`%kdrRSv!S!T!n#e$k%S%bQ!WWQ#OsR#a!RR!mfX!kf!i!l#x#SZORSWYeghjsvxyz{|}!O!P!R!S!T!]!`!a!b!c!f!n!u#e#j#o#s#{#|#}$U$[$d$k$r%S%U%_%`%b%kR#y!jTnQpQ$b#tQ$i$OQ$w$cR%Z$xQ#t!cQ$O!uQ$e#|Q$f#}Q%V$rQ%c%UQ%i%`R%l%kQ$a#tQ$h$OQ$t$bQ$v$cQ%Q$iS%Y$w$xR%e%ZdtRSv!S!T!n#e$k%S%bQ!^YQ#h!]X#k!^#h#l$SvTOWe!R!a!b!c!f!u#s#{#|#}$[$d$r%U%`%kT!qg%_T$y$e$zQ$}$eR%^$zwTOWe!R!a!b!c!f!u#s#{#|#}$[$d$r%U%`%krVOe!a!b!c!f!u#s#{#|#}$[$d$r%U%`%kQ!UWQ!wjQ#QyQ#TzQ#V{R#_!R#TZORSWYeghjsvxyz{|}!O!P!R!S!T!]!`!a!b!c!f!n!u#e#j#o#s#{#|#}$U$[$d$k$r%S%U%_%`%b%k![ZRSYghjsvxyz{|}!O!P!S!T!]!`!n#e#j#o$U$k%S%_%bw[OWe!R!a!b!c!f!u#s#{#|#}$[$d$r%U%`%kQeOR!ge^!db![#p#q#r$Z$cR#u!dQ!RWQ!]Y`#^!R!]#e#f$P$k%S%bS#e!S!TS#f!U!ZS$P#_#dQ$k$RR%S$lQ!ifR#w!iQ!lfQ#x!iT#z!l#xQpQR!|pS$[#s$dR$p$[Q$l$RR%T$lYvRS!S!T!nR#PvQ$z$eR%[$zQ#l!^Q$S#hT$W#l$SQ#o!`Q$U#jT$X#o$UTdOeSbOeS![W!RQ#p!aQ#q!b`#r!c!u#|#}$r%U%`%kQ#v!fU$Z#s$[$dR$c#{vUOWe!R!a!b!c!f!u#s#{#|#}$[$d$r%U%`%kdrRSv!S!T!n#e$k%S%bQ!`YS!pg%_Q!shQ!vjQ#OsQ#QxQ#RyQ#SzQ#U{Q#W|Q#X}Q#Z!OQ#[!PQ#j!]X#n!`#j#o$Ur]Oe!a!b!c!f!u#s#{#|#}$[$d$r%U%`%k![wRSYghjsvxyz{|}!O!P!S!T!]!`!n#e#j#o$U$k%S%_%bQ!YWR#c!R[uRSv!S!T!nQ$R#eV%R$k%S%bToQpQ$]#sR$x$dQ!rgR%h%_raOe!a!b!c!f!u#s#{#|#}$[$d$r%U%`%kQ!XWR#b!R", + states: "TQcO,5:hO?cQcO,5:hO@PQcO,5:hOOQa1G/^1G/^OOOO,59{,59{OOOO,59|,59|OOOO-E8T-E8TOOQa1G/e1G/eOOQ`,5:X,5:XOOQ`-E8W-E8WOOQa1G/{1G/{OA{QcO1G/{OBVQcO1G/{OCeQcO1G/{OCoQcO1G/{OC|QcO1G/{OOQa1G/Z1G/ZOE_QcO1G/ZOEfQcO1G/ZOEmQcO1G/ZOFlQcO1G/ZOEtQcO1G/ZOOQ`-E8Q-E8QOGSQRO1G/[OG^QQO1G/[OGcQQO1G/[OGkQQO1G/[OGvQRO1G/[OG}QRO1G/[OHUQbO,59qOH`QQO1G/[OOQa1G/[1G/[OHhQQO1G/}OOQa1G0O1G0OOHsQbO1G0OOOQO'#E['#E[OHhQQO1G/}OOQa1G/}1G/}OOQ`'#E]'#E]OHsQbO1G0OOH}QbO1G0VOIiQbO1G0UOJTQbO'#DhOJfQbO'#DhOJyQbO1G0POOQ`-E8P-E8POOQ`,5:m,5:mOOQ`-E8R-E8ROKUQQO,59vOOQO,59w,59wOOQO-E8S-E8SOK^QbO1G/aO:SQbO1G/tO:SQbO1G/XOKeQbO1G0QOKpQQO7+$vOOQa7+$v7+$vOKxQQO1G/]OLQQQO7+%iOOQa7+%i7+%iOL]QbO7+%jOOQa7+%j7+%jOOQO-E8Y-E8YOOQ`-E8Z-E8ZOOQ`'#EW'#EWOLgQQO'#EWOLoQbO'#EqOOQ`,5:S,5:SOMSQbO'#DfOMXQQO'#DiOOQ`7+%k7+%kOM^QbO7+%kOMcQbO7+%kOMkQbO7+${OMyQbO7+${ONZQbO7+%`ONcQbO7+$sOOQ`7+%l7+%lONhQbO7+%lONmQbO7+%lOOQa<qAN>qOOQ`AN>RAN>RO!!wQbOAN>RO!!|QbOAN>ROOQ`-E8X-E8XOOQ`AN>fAN>fO!#UQbOAN>fO2dQbO,5:]O:SQbO,5:_OOQ`AN>rAN>rPHUQbO'#ESOOQ`7+%W7+%WOOQ`G23mG23mO!#ZQbOG23mP!!ZQbO'#DqOOQ`G24QG24QO!#`QQO1G/wOOQ`1G/y1G/yOOQ`LD)XLD)XO:SQbO7+%cOOQ`<qOT!OOU!POj!QOt!pa#Z!pa#l!pa!Z!pa!^!pa!_!pa#h!pa!f!pa~O^yOR!iiS!iid!iie!iif!iig!iih!iii!iit!ii#Z!ii#l!ii#h!ii!Z!ii!^!ii!_!ii!f!ii~OP!iiQ!ii~P@tOPzOQzO~P@tOPzOQzOd!iie!iif!iig!iih!iii!iit!ii#Z!ii#l!ii#h!ii!Z!ii!^!ii!_!ii!f!ii~OR!iiS!ii~PBaOR{OS{O^yO~PBaOR{OS{O~PBaOW}OX}OY}OZ}O[}O]}OTwijwitwi#Zwi#lwi#hwi!Xwi!Zwi!^wi!_wi!fwi~OU!PO~PDWOU!PO~PDjOUwi~PDWOT!OOU!POjwitwi#Zwi#lwi#hwi!Xwi!Zwi!^wi!_wi!fwi~OW}OX}OY}OZ}O[}O]}O~PEtO#Z!RO#h$RO~P*[O#h$RO~O#h$ROt#VX~O!X!dO#h$ROt#VX~O#h$RO~P.gO#h$RO~P7mOpgO!`sO~P,wO#Z!RO#h$RO~O!QtO#Z#lO#k$UO~O#Z#oO#k$WO~P3[Ot!gO#Z!si#l!si!Z!si!^!si!_!si#h!si!f!si~Ot!gO#Z!ri#l!ri!Z!ri!^!ri!_!ri#h!ri!f!ri~Ot!gO!Z![X!^![X!_![X!f![X~O#Z$ZO!Z#eP!^#eP!_#eP!f#eP~P8xO!Z$_O!^$`O!_$aO~O!Q!kO!X!Oa~O#Z$eO~P8xO!Z$_O!^$`O!_$hO~O#Z!RO#h$kO~O#Z!RO#hyi~O!QtO#Z#lO#k$nO~O#Z#oO#k$oO~P3[Ot!gO#Z$pO~O#Z$ZO!Z#eX!^#eX!_#eX!f#eX~P8xOl$rO~O!X$sO~O!_$tO~O!^$`O!_$tO~Ot!gO!Z$_O!^$`O!_$vO~O#Z$ZO!Z#eP!^#eP!_#eP~P8xO!_$}O!f$|O~O!_%PO~O!_%QO~O!^$`O!_%QO~OpgO!`sO#hyq~P,wO#Z!RO#hyq~O!X%VO~O!_%XO~O!_%YO~O!^$`O!_%YO~O!Z$_O!^$`O!_%YO~O!_%^O!f$|O~O!X%aO!c%`O~O!_%^O~O!_%bO~OpgO!`sO#hyy~P,wO!_%eO~O!^$`O!_%eO~O!_%hO~O!_%kO~O!X%lO~O{!j~", + goto: "8g#hPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP#iP$SP$i%g&u&{P(V(c)])`P)fP*m*mPPP*qP*}+gPPP+}#iP,g-QP-U-[-qP.h/l$S$SP$SP$S$S0r0x1U1x2O2Y2`2g2m2w2}3XPPPP3c3g4[6QPPP7[P7lPPPPP7p7v7|raOf!b!c!d!g!v#t#|#}$O$]$e$s%V%a%lQ!XXR#b!SwaOXf!S!b!c!d!g!v#t#|#}$O$]$e$s%V%a%lr_Of!b!c!d!g!v#t#|#}$O$]$e$s%V%a%lQ![XS!ph%`Q!uiQ!ykQ#X!PQ#Z!OQ#^!QR#e!SvTOfh!b!c!d!g!v#t#|#}$O$]$e$s%V%`%a%l!W[STZiktwyz{|}!O!P!Q!T!U!^!a!o#f#k#p$V$l%T%cS!UX!SQ!zlR!{mQ!WXR#a!SrSOf!b!c!d!g!v#t#|#}$O$]$e$s%V%a%l!WxSTZiktwyz{|}!O!P!Q!T!U!^!a!o#f#k#p$V$l%T%cS!TX!ST!oh%`euSTw!T!U!o#f$l%T%craOf!b!c!d!g!v#t#|#}$O$]$e$s%V%a%ldsSTw!T!U!o#f$l%T%cQ!XXQ#PtR#b!SR!ngX!lg!j!m#y#S[OSTXZfhiktwyz{|}!O!P!Q!S!T!U!^!a!b!c!d!g!o!v#f#k#p#t#|#}$O$V$]$e$l$s%T%V%`%a%c%lR#z!kToQqQ$c#uQ$j$PQ$x$dR%[$yQ#u!dQ$P!vQ$f#}Q$g$OQ%W$sQ%d%VQ%j%aR%m%lQ$b#uQ$i$PQ$u$cQ$w$dQ%R$jS%Z$x$yR%f%[duSTw!T!U!o#f$l%T%cQ!_ZQ#i!^X#l!_#i#m$TvUOXf!S!b!c!d!g!v#t#|#}$O$]$e$s%V%a%lT!rh%`T$z$f${Q%O$fR%_${wUOXf!S!b!c!d!g!v#t#|#}$O$]$e$s%V%a%lrWOf!b!c!d!g!v#t#|#}$O$]$e$s%V%a%lQ!VXQ!xkQ#RzQ#U{Q#W|R#`!S#T[OSTXZfhiktwyz{|}!O!P!Q!S!T!U!^!a!b!c!d!g!o!v#f#k#p#t#|#}$O$V$]$e$l$s%T%V%`%a%c%l![[STZhiktwyz{|}!O!P!Q!T!U!^!a!o#f#k#p$V$l%T%`%cw]OXf!S!b!c!d!g!v#t#|#}$O$]$e$s%V%a%lQfOR!hf^!ec!]#q#r#s$[$dR#v!eQ!SXQ!^Z`#_!S!^#f#g$Q$l%T%cS#f!T!US#g!V![S$Q#`#eQ$l$SR%T$mQ!jgR#x!jQ!mgQ#y!jT#{!m#yQqQR!}qS$]#t$eR$q$]Q$m$SR%U$mYwST!T!U!oR#QwQ${$fR%]${Q#m!_Q$T#iT$X#m$TQ#p!aQ$V#kT$Y#p$VTeOfScOfS!]X!SQ#q!bQ#r!c`#s!d!v#}$O$s%V%a%lQ#w!gU$[#t$]$eR$d#|vVOXf!S!b!c!d!g!v#t#|#}$O$]$e$s%V%a%ldsSTw!T!U!o#f$l%T%cQ!aZS!qh%`Q!tiQ!wkQ#PtQ#RyQ#SzQ#T{Q#V|Q#X}Q#Y!OQ#[!PQ#]!QQ#k!^X#o!a#k#p$Vr^Of!b!c!d!g!v#t#|#}$O$]$e$s%V%a%l![xSTZhiktwyz{|}!O!P!Q!T!U!^!a!o#f#k#p$V$l%T%`%cQ!ZXR#d!S[vSTw!T!U!oQ$S#fV%S$l%T%cTpQqQ$^#tR$y$eQ!shR%i%`rbOf!b!c!d!g!v#t#|#}$O$]$e$s%V%a%lQ!YXR#c!S", nodeNames: "⚠ Star Slash Plus Minus And Or Eq EqEq Neq Lt Lte Gt Gte Modulo PlusEq MinusEq StarEq SlashEq ModuloEq Band Bor Bxor Shl Shr Ushr NullishCoalesce NullishEq Identifier AssignableIdentifier Word IdentifierBeforeDot Do Comment Program PipeExpr operator WhileExpr keyword ConditionalOp ParenExpr FunctionCall DotGet Number PositionalArg FunctionDef Params NamedParam NamedArgPrefix String StringFragment Interpolation EscapeSeq Boolean Null colon CatchExpr keyword Block FinallyExpr keyword keyword Underscore NamedArg IfExpr keyword FunctionCall ElseIfExpr keyword ElseExpr FunctionCallOrIdentifier BinOp Regex Dict Array FunctionCallWithBlock TryExpr keyword Throw keyword CompoundAssign Assign", - maxTerm: 119, + maxTerm: 120, context: trackScope, nodeProps: [ ["closedBy", 55,"end"] @@ -19,9 +19,9 @@ export const parser = LRParser.deserialize({ propSources: [highlighting], skippedNodes: [0,33], repeatNodeCount: 12, - tokenData: "IS~R}OX$OXY$mYZ%WZp$Opq$mqs$Ost%qtu'Yuw$Owx'_xy'dyz'}z{$O{|(h|}$O}!O(h!O!P$O!P!Q0o!Q!R)Y!R![+w![!]9[!]!^%W!^!}$O!}#O9u#O#P;k#P#Q;p#Q#R$O#R#S`#Z#be_!SSOt$Ouw$Ox}$O}!O`#Z#be_!SSOt$Ouw$Ox}$O}!O (specializeKeyword(value, stack) << 1), external: specializeKeyword},{term: 28, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}], - tokenPrec: 2202 + tokenPrec: 2229 }) diff --git a/src/parser/tests/strings.test.ts b/src/parser/tests/strings.test.ts index 3f78f56..288315b 100644 --- a/src/parser/tests/strings.test.ts +++ b/src/parser/tests/strings.test.ts @@ -127,3 +127,31 @@ describe('string escape sequences', () => { `) }) }) + +describe('curly strings', () => { + test('work on one line', () => { + expect('{ one two three }').toMatchTree(` + String one two three + `) + }) + + test('work on multiple lines', () => { + expect(`{ + one + two + three }`).toMatchTree(` + String + one + two + three `) + }) + + test('can contain other curlies', () => { + expect(`{ { one } + two + { three } }`).toMatchTree(` + String { one } + two + { three } `) + }) +}) \ No newline at end of file diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts index d18a515..0189033 100644 --- a/src/parser/tokenizer.ts +++ b/src/parser/tokenizer.ts @@ -1,5 +1,5 @@ import { ExternalTokenizer, InputStream, Stack } from '@lezer/lr' -import { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, Do } from './shrimp.terms' +import { Identifier, AssignableIdentifier, Word, IdentifierBeforeDot, Do, curlyString } from './shrimp.terms' // doobie doobie do (we need the `do` keyword to know when we're defining params) export function specializeKeyword(ident: string) { @@ -18,6 +18,12 @@ export const setGlobals = (newGlobals: string[] | Record) => { export const tokenizer = new ExternalTokenizer( (input: InputStream, stack: Stack) => { const ch = getFullCodePoint(input, 0) + + // Handle curly strings + if (ch === 123) { // { + return consumeCurlyString(input, stack) + } + if (!isWordChar(ch)) return // Don't consume things that start with digits - let Number token handle it @@ -157,6 +163,31 @@ const consumeRestOfWord = (input: InputStream, startPos: number, canBeWord: bool return pos } +const consumeCurlyString = (input: InputStream, stack: Stack) => { + if (!stack.canShift(curlyString)) return + + let depth = 0 + let pos = 0 + + while (true) { + const ch = input.peek(pos) + if (ch < 0) return // EOF - invalid + + if (ch === 123) depth++ // { + else if (ch === 125) { // } + depth-- + if (depth === 0) { + pos++ // consume final } + break + } + } + + pos++ + } + + input.acceptToken(curlyString, pos) +} + // Check if this identifier is in scope (for property access detection) // Returns IdentifierBeforeDot token if in scope, null otherwise const checkForDotGet = (input: InputStream, stack: Stack, pos: number): number | null => {