Track that scope!

src/parser/shrimp.grammar

@@ -2,7 +2,7 @@
 
 @context trackScope from "./scopeTracker"
 
-@skip { space | comment }
+@skip { space | Comment }
 
 @top Program { item* }
 
@@ -18,7 +18,7 @@
   newlineOrSemicolon { "\n" | ";" }
   eof { @eof }
   space { " " | "\t" }
-  comment { "#" ![\n]* }
+  Comment { "#" " " ![\n]* }
   leftParen { "(" }
   rightParen { ")" }
   colon[closedBy="end", @name="colon"] { ":" }

src/parser/shrimp.terms.ts

@@ -24,42 +24,43 @@ export const
   Word = 22,
   IdentifierBeforeDot = 23,
   Do = 24,
-  Program = 25,
+  Comment = 25,
-  PipeExpr = 26,
+  Program = 26,
-  FunctionCall = 27,
+  PipeExpr = 27,
-  DotGet = 28,
+  FunctionCall = 28,
-  Number = 29,
+  DotGet = 29,
-  ParenExpr = 30,
+  Number = 30,
-  FunctionCallOrIdentifier = 31,
+  ParenExpr = 31,
-  BinOp = 32,
+  FunctionCallOrIdentifier = 32,
-  String = 33,
+  BinOp = 33,
-  StringFragment = 34,
+  String = 34,
-  Interpolation = 35,
+  StringFragment = 35,
-  EscapeSeq = 36,
+  Interpolation = 36,
-  Boolean = 37,
+  EscapeSeq = 37,
-  Regex = 38,
+  Boolean = 38,
-  Dict = 39,
+  Regex = 39,
-  NamedArg = 40,
+  Dict = 40,
-  NamedArgPrefix = 41,
+  NamedArg = 41,
-  FunctionDef = 42,
+  NamedArgPrefix = 42,
-  Params = 43,
+  FunctionDef = 43,
-  NamedParam = 44,
+  Params = 44,
-  Null = 45,
+  NamedParam = 45,
-  colon = 46,
+  Null = 46,
-  CatchExpr = 47,
+  colon = 47,
-  keyword = 68,
+  CatchExpr = 48,
-  Block = 49,
+  keyword = 69,
-  FinallyExpr = 50,
+  Block = 50,
-  Underscore = 53,
+  FinallyExpr = 51,
-  Array = 54,
+  Underscore = 54,
-  ConditionalOp = 55,
+  Array = 55,
-  PositionalArg = 56,
+  ConditionalOp = 56,
-  WhileExpr = 58,
+  PositionalArg = 57,
-  FunctionCallWithBlock = 60,
+  WhileExpr = 59,
-  TryExpr = 61,
+  FunctionCallWithBlock = 61,
-  Throw = 63,
+  TryExpr = 62,
-  IfExpr = 65,
+  Throw = 64,
-  ElseIfExpr = 67,
+  IfExpr = 66,
-  ElseExpr = 69,
+  ElseIfExpr = 68,
-  CompoundAssign = 70,
+  ElseExpr = 70,
-  Assign = 71
+  CompoundAssign = 71,
+  Assign = 72

src/parser/shrimp.ts

@@ -4,24 +4,24 @@ import {operatorTokenizer} from "./operatorTokenizer"
 import {tokenizer, specializeKeyword} from "./tokenizer"
 import {trackScope} from "./scopeTracker"
 import {highlighting} from "./highlight"
-const spec_Identifier = {__proto__:null,null:90, catch:96, finally:102, end:104, while:118, try:124, throw:128, if:132, else:136}
+const spec_Identifier = {__proto__:null,null:92, catch:98, finally:104, end:106, while:120, try:126, throw:130, if:134, else:138}
 export const parser = LRParser.deserialize({
   version: 14,
-  states: "9OQYQbOOO#zQcO'#C{O$zOSO'#C}OOQa'#DT'#DTO&TQbO'#DdO'iQcO'#E]OOQa'#E]'#E]O(lQcO'#E]O)nQcO'#E[O*UQRO'#C|O+eQcO'#EWO+uQcO'#EWO,PQbO'#CzO,wOpO'#CxOOQ`'#EX'#EXO,|QbO'#EWO-WQRO'#DtOOQ`'#EW'#EWO-lQQO'#EVOOQ`'#EV'#EVOOQ`'#Dv'#DvQYQbOOO-tQbO'#DWO.PQbO'#DhO.tQQO'#DkO.PQbO'#DmO.PQbO'#DoO.yQbO'#DUOOQa'#E['#E[OOQ`'#Df'#DfOOQ`'#Ek'#EkOOQ`'#EO'#EOO/TQbO,59cO/}QbO'#DPO0VQWO'#DQOOOO'#E_'#E_OOOO'#Dw'#DwO0kOSO,59iOOQa,59i,59iOOQ`'#Dx'#DxO0yQbO,5:OO1QQQO,59oOOQa,5:O,5:OO1]QbO,5:OO1gQbO,5:aO.PQbO,59hO.PQbO,59hO.PQbO,59hO.PQbO,5:PO.PQbO,5:PO.PQbO,5:PO1zQRO,59fO2RQRO,59fO2dQRO,59fO2_QQO,59fO2oQQO,59fO2wObO,59dO3SQbO'#EPO3_QbO,59bO3yQbO,5:UO1gQbO,5:`OOQ`,5:q,5:qOOQ`-E7t-E7tOOQ`'#Dy'#DyO4aQbO'#DXO4lQbO'#DYOOQO'#Dz'#DzO4dQQO'#DXO4zQQO,59rO5kQRO,5:SO5rQRO,5:SO3yQbO,5:VO5}QcO,5:XO6yQcO,5:XO7ZQcO,5:XO7eQRO,5:ZO7lQRO,5:ZOOQ`,59p,59pOOQ`-E7|-E7|OOOO,59k,59kOOOO,59l,59lOOOO-E7u-E7uOOQa1G/T1G/TOOQ`-E7v-E7vO7wQQO1G/ZOOQa1G/j1G/jO8SQbO1G/jOOQO'#D|'#D|O7wQQO1G/ZOOQa1G/Z1G/ZOOQ`'#D}'#D}O8SQbO1G/jOOQ`1G/{1G/{OOQa1G/S1G/SO9OQcO1G/SO9YQcO1G/SO9dQcO1G/SOOQa1G/k1G/kO;YQcO1G/kO;aQcO1G/kO;hQcO1G/kOOQa1G/Q1G/QOOQa1G/O1G/OO!dQbO'#C{O;oQbO'#CwOOQ`,5:k,5:kOOQ`-E7}-E7}OOQ`'#D_'#D_O;|QbO'#D_O<pQbO1G/pOOQ`1G/z1G/zOOQ`-E7w-E7wO<{QQO,59sOOQO,59t,59tOOQO-E7x-E7xO=TQbO1G/^O3yQbO1G/nO=kQbO1G/qO3yQbO1G/uO=vQQO7+$uOOQa7+$u7+$uO>RQbO7+%UOOQa7+%U7+%UOOQO-E7z-E7zOOQ`-E7{-E7{OOQ`'#D{'#D{O>]QQO'#D{O>bQbO'#EhOOQ`,59y,59yO?UQbO'#D]O?ZQQO'#D`OOQ`7+%[7+%[O?`QbO7+%[O?eQbO7+%[O?mQbO7+$xO?xQbO7+$xO@iQbO7+%YOOQ`7+%]7+%]O@nQbO7+%]O@sQbO7+%]O@{QbO7+%aOOQa<<Ha<<HaOOQa<<Hp<<HpOOQ`,5:g,5:gOOQ`-E7y-E7yOATQQO,59wO3yQbO,59zOOQ`<<Hv<<HvOAYQbO<<HvOOQ`<<Hd<<HdOA_QbO<<HdOAdQbO<<HdOAlQbO<<HdOOQ`<<Ht<<HtOOQ`<<Hw<<HwOAwQbO<<HwOOQ`'#EQ'#EQOA|QbO<<H{OBUQbO'#DsOOQ`<<H{<<H{OB^QbO<<H{O3yQbO1G/cOOQ`1G/f1G/fOOQ`AN>bAN>bOOQ`AN>OAN>OOBcQbOAN>OOBhQbOAN>OOOQ`AN>cAN>cOOQ`-E8O-E8OOOQ`AN>gAN>gOBpQbOAN>gO.PQbO,5:]O3yQbO,5:_OOQ`7+$}7+$}OOQ`G23jG23jOBuQbOG23jPBXQbO'#DqOOQ`G24RG24ROBzQRO1G/wOCRQRO1G/wOOQ`1G/y1G/yOOQ`LD)ULD)UO3yQbO7+%cOOQ`<<H}<<H}",
+  states: "9OQYQbOOO#zQcO'#C|O$zOSO'#DOOOQa'#DU'#DUO&TQbO'#DeO'iQcO'#E]OOQa'#E]'#E]O(lQcO'#E]O)nQcO'#E[O*UQRO'#C}O+eQcO'#EWO+uQcO'#EWO,PQbO'#C{O,wOpO'#CyOOQ`'#EX'#EXO,|QbO'#EWO-WQRO'#DuOOQ`'#EW'#EWO-lQQO'#EVOOQ`'#EV'#EVOOQ`'#Dw'#DwQYQbOOO-tQbO'#DXO.PQbO'#DiO.tQQO'#DlO.PQbO'#DnO.PQbO'#DpO.yQbO'#DVOOQa'#E['#E[OOQ`'#Dg'#DgOOQ`'#Ek'#EkOOQ`'#EP'#EPO/TQbO,59dO/}QbO'#DQO0VQWO'#DROOOO'#E_'#E_OOOO'#Dx'#DxO0kOSO,59jOOQa,59j,59jOOQ`'#Dy'#DyO0yQbO,5:PO1QQQO,59pOOQa,5:P,5:PO1]QbO,5:PO1gQbO,5:bO.PQbO,59iO.PQbO,59iO.PQbO,59iO.PQbO,5:QO.PQbO,5:QO.PQbO,5:QO1zQRO,59gO2RQRO,59gO2dQRO,59gO2_QQO,59gO2oQQO,59gO2wObO,59eO3SQbO'#EQO3_QbO,59cO3yQbO,5:VO1gQbO,5:aOOQ`,5:q,5:qOOQ`-E7u-E7uOOQ`'#Dz'#DzO4aQbO'#DYO4lQbO'#DZOOQO'#D{'#D{O4dQQO'#DYO4zQQO,59sO5kQRO,5:TO5rQRO,5:TO3yQbO,5:WO5}QcO,5:YO6yQcO,5:YO7ZQcO,5:YO7eQRO,5:[O7lQRO,5:[OOQ`,59q,59qOOQ`-E7}-E7}OOOO,59l,59lOOOO,59m,59mOOOO-E7v-E7vOOQa1G/U1G/UOOQ`-E7w-E7wO7wQQO1G/[OOQa1G/k1G/kO8SQbO1G/kOOQO'#D}'#D}O7wQQO1G/[OOQa1G/[1G/[OOQ`'#EO'#EOO8SQbO1G/kOOQ`1G/|1G/|OOQa1G/T1G/TO9OQcO1G/TO9YQcO1G/TO9dQcO1G/TOOQa1G/l1G/lO;YQcO1G/lO;aQcO1G/lO;hQcO1G/lOOQa1G/R1G/ROOQa1G/P1G/PO!dQbO'#C|O;oQbO'#CxOOQ`,5:l,5:lOOQ`-E8O-E8OOOQ`'#D`'#D`O;|QbO'#D`O<pQbO1G/qOOQ`1G/{1G/{OOQ`-E7x-E7xO<{QQO,59tOOQO,59u,59uOOQO-E7y-E7yO=TQbO1G/_O3yQbO1G/oO=kQbO1G/rO3yQbO1G/vO=vQQO7+$vOOQa7+$v7+$vO>RQbO7+%VOOQa7+%V7+%VOOQO-E7{-E7{OOQ`-E7|-E7|OOQ`'#D|'#D|O>]QQO'#D|O>bQbO'#EhOOQ`,59z,59zO?UQbO'#D^O?ZQQO'#DaOOQ`7+%]7+%]O?`QbO7+%]O?eQbO7+%]O?mQbO7+$yO?xQbO7+$yO@iQbO7+%ZOOQ`7+%^7+%^O@nQbO7+%^O@sQbO7+%^O@{QbO7+%bOOQa<<Hb<<HbOOQa<<Hq<<HqOOQ`,5:h,5:hOOQ`-E7z-E7zOATQQO,59xO3yQbO,59{OOQ`<<Hw<<HwOAYQbO<<HwOOQ`<<He<<HeOA_QbO<<HeOAdQbO<<HeOAlQbO<<HeOOQ`<<Hu<<HuOOQ`<<Hx<<HxOAwQbO<<HxOOQ`'#ER'#EROA|QbO<<H|OBUQbO'#DtOOQ`<<H|<<H|OB^QbO<<H|O3yQbO1G/dOOQ`1G/g1G/gOOQ`AN>cAN>cOOQ`AN>PAN>POBcQbOAN>POBhQbOAN>POOQ`AN>dAN>dOOQ`-E8P-E8POOQ`AN>hAN>hOBpQbOAN>hO.PQbO,5:^O3yQbO,5:`OOQ`7+%O7+%OOOQ`G23kG23kOBuQbOG23kPBXQbO'#DrOOQ`G24SG24SOBzQRO1G/xOCRQRO1G/xOOQ`1G/z1G/zOOQ`LD)VLD)VO3yQbO7+%dOOQ`<<IO<<IO",
-  stateData: "Ca~O!wOS!xOS~OdPOe`OfUOg]OhfOmUOuUOvUO}UO!]gO!`hO!biO!djO!}[O#QQO#XRO#YSO#ZcO~OdlOfUOg]OhfOmUOuUOvUOykO}UO!VmO!}[O#QQO#XRO#YSO!ZoX#ZoX#`oX#^oX!QoX!ToX!UoX!foX~OP#OXQ#OXR#OXS#OXT#OXU#OXW#OXX#OXY#OXZ#OX[#OX]#OX^#OX!OoX~P!dOrsO#QvO#SqO#TrO~OdlOfUOg]OmUOuUOvUOykO}UO!}[O#QQO#XRO#YSO#ZwO~O#]zO~P%YOP#PXQ#PXR#PXS#PXT#PXU#PXW#PXX#PXY#PXZ#PX[#PX]#PX^#PX#Z#PX#`#PX!Q#PX!T#PX!U#PX!f#PX~OdlOfUOg]OhfOmUOuUOvUOykO}UO!VmO!}[O#QQO#XRO#YSO#^#PX~P&[OV|O~P&[OP#OXQ#OXR#OXS#OXT#OXU#OXW#OXX#OXY#OXZ#OX[#OX]#OX^#OX~O#Z!zX#`!zX!Q!zX!T!zX!U!zX!f!zX~P(sOP!OOQ!OOR!POS!POT!ROU!SOW!QOX!QOY!QOZ!QO[!QO]!QO^}O~O#Z!zX#`!zX!Q!zX!T!zX!U!zX!f!zX~OP!OOQ!OOR!POS!PO~P+POT!ROU!SO~P+POdPOfUOg]OhfOmUOuUOvUO}UO!}[O#QQO#XRO#YSO~O!|!YO~O!O!]O!Z!ZO~P+POV|O_!^O`!^Oa!^Ob!^Oc!^O~O#Z!_O#`!_O~Od!aOy!cO!O{P~OdlOfUOg]OmUOuUOvUO}UO!}[O#QQO#XRO#YSO~O!O!iO~OhfO!V!oO~P.POhfOykO!VmO!Oka!Zka#Zka#`ka#^ka!Qka!Tka!Uka!fka~P.POd!qO!}[O~O#Q!rO#S!rO#T!rO#U!rO#V!rO#W!rO~OrsO#Q!tO#SqO#TrO~O#]!wO~P%YOykO#Z!yO#]!{O~O#Z!|O#]!wO~P.POe`O!]gO!`hO!biO!djO~P,PO#^#XO~P(sOP!OOQ!OOR!POS!PO#^#XO~OT!ROU!SO#^#XO~O!Z!ZO#^#XO~Od#YOm#YO!}[O~Od#ZOg]O!}[O~O!Z!ZO#Zja#`ja#^ja!Qja!Tja!Uja!fja~Oe`O!]gO!`hO!biO!djO#Z#`O~P,POd!aOy!cO!O{X~Om#eOu#eO}#eO#QQO~O!O#gO~OT!ROU!SOW!QOX!QOY!QOZ!QO[!QO]!QO~O!O#hO~P5POT!ROU!SO!O#hO~O#Z!aa#`!aa!Q!aa!T!aa!U!aa!f!aa~P*UO#Z!aa#`!aa!Q!aa!T!aa!U!aa!f!aa~OP!OOQ!OOR!POS!PO~P6eOT!ROU!SO~P6eO!O#jO~P5POT!ROU!SO!O#jO~OykO#Z!yO#]#lO~O#Z!|O#]#nO~P.PO^}ORpiSpi#Zpi#`pi#^pi!Qpi!Tpi!Upi!fpi~OPpiQpi~P8^OP!OOQ!OO~P8^OP!OOQ!OORpiSpi#Zpi#`pi#^pi!Qpi!Tpi!Upi!fpi~OW!QOX!QOY!QOZ!QO[!QO]!QOT!Xi#Z!Xi#`!Xi#^!Xi!O!Xi!Q!Xi!T!Xi!U!Xi!f!Xi~OU!SO~P:XOU!SO~P:kOU!Xi~P:XOhfOykO!VmO~P.POe`O!]gO!`hO!biO!djO#Z#qO!Q#[P!T#[P!U#[P!f#[P~P,PO!Q#uO!T#vO!U#wO~Oy!cO!O{a~Oe`O!]gO!`hO!biO!djO#Z#{O~P,PO!Q#uO!T#vO!U#}O~OykO#Z!yO#]$RO~O#Z!|O#]$SO~P.PO#Z$TO~Oe`O!]gO!`hO!biO!djO#Z#qO!Q#[X!T#[X!U#[X!f#[X~P,POd$VO~O!O$WO~O!U$XO~O!T#vO!U$XO~O!Q#uO!T#vO!U$ZO~Oe`O!]gO!`hO!biO!djO#Z#qO!Q#[P!T#[P!U#[P~P,PO!U$_O~O!U$`O~O!T#vO!U$`O~O!U$eO!f$dO~O!O$gO~O!U$iO~O!U$jO~O!T#vO!U$jO~O!Q#uO!T#vO!U$jO~O!U$mO~O!U$oO!f$dO~O!O$rO!d$qO~O!U$oO~O!U$tO~O!T#vO!U$tO~O!U$wO~O!U${O~O!O$|O~P5POT!ROU!SO!O$|O~Omv~",
+  stateData: "Ca~O!xOSiOS~OdPOe`OfUOg]OhfOnUOvUOwUO!OUO!^gO!ahO!ciO!ejO!}[O#QQO#XRO#YSO#ZcO~OdlOfUOg]OhfOnUOvUOwUOzkO!OUO!WmO!}[O#QQO#XRO#YSO![pX#ZpX#`pX#^pX!RpX!UpX!VpX!gpX~OP#OXQ#OXR#OXS#OXT#OXU#OXW#OXX#OXY#OXZ#OX[#OX]#OX^#OX!PpX~P!dOssO#QvO#SqO#TrO~OdlOfUOg]OnUOvUOwUOzkO!OUO!}[O#QQO#XRO#YSO#ZwO~O#]zO~P%YOP#PXQ#PXR#PXS#PXT#PXU#PXW#PXX#PXY#PXZ#PX[#PX]#PX^#PX#Z#PX#`#PX!R#PX!U#PX!V#PX!g#PX~OdlOfUOg]OhfOnUOvUOwUOzkO!OUO!WmO!}[O#QQO#XRO#YSO#^#PX~P&[OV|O~P&[OP#OXQ#OXR#OXS#OXT#OXU#OXW#OXX#OXY#OXZ#OX[#OX]#OX^#OX~O#Z!zX#`!zX!R!zX!U!zX!V!zX!g!zX~P(sOP!OOQ!OOR!POS!POT!ROU!SOW!QOX!QOY!QOZ!QO[!QO]!QO^}O~O#Z!zX#`!zX!R!zX!U!zX!V!zX!g!zX~OP!OOQ!OOR!POS!PO~P+POT!ROU!SO~P+POdPOfUOg]OhfOnUOvUOwUO!OUO!}[O#QQO#XRO#YSO~O!|!YO~O!P!]O![!ZO~P+POV|O_!^O`!^Oa!^Ob!^Oc!^O~O#Z!_O#`!_O~Od!aOz!cO!P|P~OdlOfUOg]OnUOvUOwUO!OUO!}[O#QQO#XRO#YSO~O!P!iO~OhfO!W!oO~P.POhfOzkO!WmO!Pla![la#Zla#`la#^la!Rla!Ula!Vla!gla~P.POd!qO!}[O~O#Q!rO#S!rO#T!rO#U!rO#V!rO#W!rO~OssO#Q!tO#SqO#TrO~O#]!wO~P%YOzkO#Z!yO#]!{O~O#Z!|O#]!wO~P.POe`O!^gO!ahO!ciO!ejO~P,PO#^#XO~P(sOP!OOQ!OOR!POS!PO#^#XO~OT!ROU!SO#^#XO~O![!ZO#^#XO~Od#YOn#YO!}[O~Od#ZOg]O!}[O~O![!ZO#Zka#`ka#^ka!Rka!Uka!Vka!gka~Oe`O!^gO!ahO!ciO!ejO#Z#`O~P,POd!aOz!cO!P|X~On#eOv#eO!O#eO#QQO~O!P#gO~OT!ROU!SOW!QOX!QOY!QOZ!QO[!QO]!QO~O!P#hO~P5POT!ROU!SO!P#hO~O#Z!ba#`!ba!R!ba!U!ba!V!ba!g!ba~P*UO#Z!ba#`!ba!R!ba!U!ba!V!ba!g!ba~OP!OOQ!OOR!POS!PO~P6eOT!ROU!SO~P6eO!P#jO~P5POT!ROU!SO!P#jO~OzkO#Z!yO#]#lO~O#Z!|O#]#nO~P.PO^}ORqiSqi#Zqi#`qi#^qi!Rqi!Uqi!Vqi!gqi~OPqiQqi~P8^OP!OOQ!OO~P8^OP!OOQ!OORqiSqi#Zqi#`qi#^qi!Rqi!Uqi!Vqi!gqi~OW!QOX!QOY!QOZ!QO[!QO]!QOT!Yi#Z!Yi#`!Yi#^!Yi!P!Yi!R!Yi!U!Yi!V!Yi!g!Yi~OU!SO~P:XOU!SO~P:kOU!Yi~P:XOhfOzkO!WmO~P.POe`O!^gO!ahO!ciO!ejO#Z#qO!R#[P!U#[P!V#[P!g#[P~P,PO!R#uO!U#vO!V#wO~Oz!cO!P|a~Oe`O!^gO!ahO!ciO!ejO#Z#{O~P,PO!R#uO!U#vO!V#}O~OzkO#Z!yO#]$RO~O#Z!|O#]$SO~P.PO#Z$TO~Oe`O!^gO!ahO!ciO!ejO#Z#qO!R#[X!U#[X!V#[X!g#[X~P,POd$VO~O!P$WO~O!V$XO~O!U#vO!V$XO~O!R#uO!U#vO!V$ZO~Oe`O!^gO!ahO!ciO!ejO#Z#qO!R#[P!U#[P!V#[P~P,PO!V$_O~O!V$`O~O!U#vO!V$`O~O!V$eO!g$dO~O!P$gO~O!V$iO~O!V$jO~O!U#vO!V$jO~O!R#uO!U#vO!V$jO~O!V$mO~O!V$oO!g$dO~O!P$rO!e$qO~O!V$oO~O!V$tO~O!U#vO!V$tO~O!V$wO~O!V${O~O!P$|O~P5POT!ROU!SO!P$|O~Onw~",
-  goto: "3}#`PPPPPPPPPPPPPPPPPPPPPPPPPP#a#v$[P%[#v&b'QP(O(OPP(S(}P)b*R*UPP*[P*h+QPPP+h,e-^P-eP-e-eP-eP-eP-wP-{-e-e.R.X._.e.k.u.|/W/b/k/rPPPP/x/|0jPP1S2mP3lPPPPPPPP3pPP3vpaOe|!]!^!i#`#g#h#j#s#{$W$g$r$|R!W[u^O[e|!Z!]!^!i#`#g#h#j#s#{$W$g$r$|rPO[e|!]!^!i#`#g#h#j#s#{$W$g$r$||lPSTgijkpx{}!O!P!Q!R!S!x!}#Z#[#m$qR#Z!ZrTO[e|!]!^!i#`#g#h#j#s#{$W$g$r$||UPSTgijkpx{}!O!P!Q!R!S!x!}#Z#[#m$qQ!qqQ#Y!YR#[!ZpYOe|!]!^!i#`#g#h#j#s#{$W$g$r$|Q!U[Q!kiQ#P!OR#S!P!pUOPST[egijkpx{|}!O!P!Q!R!S!]!^!i!x!}#Z#[#`#g#h#j#m#s#{$W$g$q$r$|R#e!cTsQu!qUOPST[egijkpx{|}!O!P!Q!R!S!]!^!i!x!}#Z#[#`#g#h#j#m#s#{$W$g$q$r$|YnPTp#Z#[QySQ!vxX!yy!v!z#kpaOe|!]!^!i#`#g#h#j#s#{$W$g$r$|YmPTp#Z#[Q!W[R!okR!ffX!df!b!e#dQ#y#aQ$P#iQ$]#zR$l$^Q#a!]Q#i!iQ#|#hQ$Q#jQ$h$WQ$s$gQ$z$rR$}$|Q#x#aQ$O#iQ$Y#yQ$[#zQ$a$PS$k$]$^R$u$l!OUPST[gijkpx{}!O!P!Q!R!S!x!}#Z#[#m$qqVOe|!]!^!i#`#g#h#j#s#{$W$g$r$|pZOe|!]!^!i#`#g#h#j#s#{$W$g$r$|Q!V[Q!hgQ!liQ!njQ#T!SQ#V!RR$y$qZnPTp#Z#[qaOe|!]!^!i#`#g#h#j#s#{$W$g$r$|T$b$Q$cQ$f$QR$p$cQeOR!`eQuQR!suQxSR!uxQ!bfR#c!bQ!efQ#d!bT#f!e#dS#s#`#{R$U#sQ!zyQ#k!vT#o!z#kQ!}{Q#m!xT#p!}#mWpPT#Z#[R!ppS![_!XR#^![Q$c$QR$n$cTdOeSbOeQ#O|`#_!]!i#h#j$W$g$r$|Q#b!^U#r#`#s#{R#z#gp_Oe|!]!^!i#`#g#h#j#s#{$W$g$r$|Q!X[R#]!ZrXO[e|!]!^!i#`#g#h#j#s#{$W$g$r$|YmPTp#Z#[Q{SQ!ggQ!jiQ!mjQ!okQ!xxW!|{!x!}#mQ#P}Q#Q!OQ#R!PQ#T!QQ#U!RQ#W!SR$x$qpWOe|!]!^!i#`#g#h#j#s#{$W$g$r$||lPSTgijkpx{}!O!P!Q!R!S!x!}#Z#[#m$qR!T[TtQuQ#t#`R$^#{ZoPTp#Z#[",
+  goto: "3}#`PPPPPPPPPPPPPPPPPPPPPPPPPPP#a#v$[P%[#v&b'QP(O(OPP(S(}P)b*R*UPP*[P*h+QPPP+h,e-^P-eP-e-eP-eP-eP-wP-{-e-e.R.X._.e.k.u.|/W/b/k/rPPP/x/|0jPP1S2mP3lPPPPPPPP3pPP3vpaOe|!]!^!i#`#g#h#j#s#{$W$g$r$|R!W[u^O[e|!Z!]!^!i#`#g#h#j#s#{$W$g$r$|rPO[e|!]!^!i#`#g#h#j#s#{$W$g$r$||lPSTgijkpx{}!O!P!Q!R!S!x!}#Z#[#m$qR#Z!ZrTO[e|!]!^!i#`#g#h#j#s#{$W$g$r$||UPSTgijkpx{}!O!P!Q!R!S!x!}#Z#[#m$qQ!qqQ#Y!YR#[!ZpYOe|!]!^!i#`#g#h#j#s#{$W$g$r$|Q!U[Q!kiQ#P!OR#S!P!pUOPST[egijkpx{|}!O!P!Q!R!S!]!^!i!x!}#Z#[#`#g#h#j#m#s#{$W$g$q$r$|R#e!cTsQu!qUOPST[egijkpx{|}!O!P!Q!R!S!]!^!i!x!}#Z#[#`#g#h#j#m#s#{$W$g$q$r$|YnPTp#Z#[QySQ!vxX!yy!v!z#kpaOe|!]!^!i#`#g#h#j#s#{$W$g$r$|YmPTp#Z#[Q!W[R!okR!ffX!df!b!e#dQ#y#aQ$P#iQ$]#zR$l$^Q#a!]Q#i!iQ#|#hQ$Q#jQ$h$WQ$s$gQ$z$rR$}$|Q#x#aQ$O#iQ$Y#yQ$[#zQ$a$PS$k$]$^R$u$l!OUPST[gijkpx{}!O!P!Q!R!S!x!}#Z#[#m$qqVOe|!]!^!i#`#g#h#j#s#{$W$g$r$|pZOe|!]!^!i#`#g#h#j#s#{$W$g$r$|Q!V[Q!hgQ!liQ!njQ#T!SQ#V!RR$y$qZnPTp#Z#[qaOe|!]!^!i#`#g#h#j#s#{$W$g$r$|T$b$Q$cQ$f$QR$p$cQeOR!`eQuQR!suQxSR!uxQ!bfR#c!bQ!efQ#d!bT#f!e#dS#s#`#{R$U#sQ!zyQ#k!vT#o!z#kQ!}{Q#m!xT#p!}#mWpPT#Z#[R!ppS![_!XR#^![Q$c$QR$n$cTdOeSbOeQ#O|`#_!]!i#h#j$W$g$r$|Q#b!^U#r#`#s#{R#z#gp_Oe|!]!^!i#`#g#h#j#s#{$W$g$r$|Q!X[R#]!ZrXO[e|!]!^!i#`#g#h#j#s#{$W$g$r$|YmPTp#Z#[Q{SQ!ggQ!jiQ!mjQ!okQ!xxW!|{!x!}#mQ#P}Q#Q!OQ#R!PQ#T!QQ#U!RQ#W!SR$x$qpWOe|!]!^!i#`#g#h#j#s#{$W$g$r$||lPSTgijkpx{}!O!P!Q!R!S!x!}#Z#[#m$qR!T[TtQuQ#t#`R$^#{ZoPTp#Z#[",
-  nodeNames: "⚠ Star Slash Plus Minus And Or Eq EqEq Neq Lt Lte Gt Gte Modulo PlusEq MinusEq StarEq SlashEq ModuloEq Identifier AssignableIdentifier Word IdentifierBeforeDot Do Program PipeExpr FunctionCall DotGet Number ParenExpr FunctionCallOrIdentifier BinOp String StringFragment Interpolation EscapeSeq Boolean Regex Dict NamedArg NamedArgPrefix FunctionDef Params NamedParam Null colon CatchExpr keyword Block FinallyExpr keyword keyword Underscore Array ConditionalOp PositionalArg operator WhileExpr keyword FunctionCallWithBlock TryExpr keyword Throw keyword IfExpr keyword ElseIfExpr keyword ElseExpr CompoundAssign Assign",
+  nodeNames: "⚠ Star Slash Plus Minus And Or Eq EqEq Neq Lt Lte Gt Gte Modulo PlusEq MinusEq StarEq SlashEq ModuloEq Identifier AssignableIdentifier Word IdentifierBeforeDot Do Comment Program PipeExpr FunctionCall DotGet Number ParenExpr FunctionCallOrIdentifier BinOp String StringFragment Interpolation EscapeSeq Boolean Regex Dict NamedArg NamedArgPrefix FunctionDef Params NamedParam Null colon CatchExpr keyword Block FinallyExpr keyword keyword Underscore Array ConditionalOp PositionalArg operator WhileExpr keyword FunctionCallWithBlock TryExpr keyword Throw keyword IfExpr keyword ElseIfExpr keyword ElseExpr CompoundAssign Assign",
   maxTerm: 108,
   context: trackScope,
   nodeProps: [
-    ["closedBy", 46,"end"]
+    ["closedBy", 47,"end"]
   ],
   propSources: [highlighting],
-  skippedNodes: [0],
+  skippedNodes: [0,25],
   repeatNodeCount: 11,
-  tokenData: "C_~R|OX#{XY$jYZ%TZp#{pq$jqs#{st%ntu'Vuw#{wx'[xy'ayz'zz{#{{|(e|}#{}!O+X!O!P#{!P!Q-n!Q![)S![!]6Z!]!^%T!^!}#{!}#O6t#O#P8j#P#Q8o#Q#R#{#R#S9Y#S#T#{#T#Y,Y#Y#Z9s#Z#b,Y#b#c>q#c#f,Y#f#g?n#g#h,Y#h#i@k#i#o,Y#o#p#{#p#qBo#q;'S#{;'S;=`$d<%l~#{~O#{~~CYS$QUrSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{S$gP;=`<%l#{^$qUrS!wYOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U%[UrS#ZQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{^%uZrS!xYOY%nYZ#{Zt%ntu&huw%nwx&hx#O%n#O#P&h#P;'S%n;'S;=`'P<%lO%nY&mS!xYOY&hZ;'S&h;'S;=`&y<%lO&hY&|P;=`<%l&h^'SP;=`<%l%n~'[O#S~~'aO#Q~U'hUrS!}QOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U(RUrS#^QOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U(jWrSOt#{uw#{x!Q#{!Q![)S![#O#{#P;'S#{;'S;=`$d<%lO#{U)ZYrSmQOt#{uw#{x!O#{!O!P)y!P!Q#{!Q![)S![#O#{#P;'S#{;'S;=`$d<%lO#{U*OWrSOt#{uw#{x!Q#{!Q![*h![#O#{#P;'S#{;'S;=`$d<%lO#{U*oWrSmQOt#{uw#{x!Q#{!Q![*h![#O#{#P;'S#{;'S;=`$d<%lO#{U+^^rSOt#{uw#{x}#{}!O,Y!O!Q#{!Q![)S![!_#{!_!`-T!`#O#{#P#T#{#T#o,Y#o;'S#{;'S;=`$d<%lO#{U,_[rSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#o,Y#o;'S#{;'S;=`$d<%lO#{U-[UyQrSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U-sWrSOt#{uw#{x!P#{!P!Q.]!Q#O#{#P;'S#{;'S;=`$d<%lO#{U.b^rSOY/^YZ#{Zt/^tu0auw/^wx0ax!P/^!P!Q#{!Q!}/^!}#O5S#O#P2o#P;'S/^;'S;=`6T<%lO/^U/e^rSvQOY/^YZ#{Zt/^tu0auw/^wx0ax!P/^!P!Q3U!Q!}/^!}#O5S#O#P2o#P;'S/^;'S;=`6T<%lO/^Q0fXvQOY0aZ!P0a!P!Q1R!Q!}0a!}#O1p#O#P2o#P;'S0a;'S;=`3O<%lO0aQ1UP!P!Q1XQ1^UvQ#Z#[1X#]#^1X#a#b1X#g#h1X#i#j1X#m#n1XQ1sVOY1pZ#O1p#O#P2Y#P#Q0a#Q;'S1p;'S;=`2i<%lO1pQ2]SOY1pZ;'S1p;'S;=`2i<%lO1pQ2lP;=`<%l1pQ2rSOY0aZ;'S0a;'S;=`3O<%lO0aQ3RP;=`<%l0aU3ZWrSOt#{uw#{x!P#{!P!Q3s!Q#O#{#P;'S#{;'S;=`$d<%lO#{U3zbrSvQOt#{uw#{x#O#{#P#Z#{#Z#[3s#[#]#{#]#^3s#^#a#{#a#b3s#b#g#{#g#h3s#h#i#{#i#j3s#j#m#{#m#n3s#n;'S#{;'S;=`$d<%lO#{U5X[rSOY5SYZ#{Zt5Stu1puw5Swx1px#O5S#O#P2Y#P#Q/^#Q;'S5S;'S;=`5}<%lO5SU6QP;=`<%l5SU6WP;=`<%l/^U6bUrS!OQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U6{W#YQrSOt#{uw#{x!_#{!_!`7e!`#O#{#P;'S#{;'S;=`$d<%lO#{U7jVrSOt#{uw#{x#O#{#P#Q8P#Q;'S#{;'S;=`$d<%lO#{U8WU#XQrSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~8oO#T~U8vU#]QrSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U9aUrS!VQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U9x]rSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#U:q#U#o,Y#o;'S#{;'S;=`$d<%lO#{U:v^rSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#`,Y#`#a;r#a#o,Y#o;'S#{;'S;=`$d<%lO#{U;w^rSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#g,Y#g#h<s#h#o,Y#o;'S#{;'S;=`$d<%lO#{U<x^rSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#X,Y#X#Y=t#Y#o,Y#o;'S#{;'S;=`$d<%lO#{U={[uQrSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#o,Y#o;'S#{;'S;=`$d<%lO#{^>x[#UWrSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#o,Y#o;'S#{;'S;=`$d<%lO#{^?u[#WWrSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#o,Y#o;'S#{;'S;=`$d<%lO#{^@r^#VWrSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#f,Y#f#gAn#g#o,Y#o;'S#{;'S;=`$d<%lO#{UAs^rSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#i,Y#i#j<s#j#o,Y#o;'S#{;'S;=`$d<%lO#{UBvU!ZQrSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~C_O#`~",
+  tokenData: "C|~R|OX#{XY$jYZ%TZp#{pq$jqs#{st%ntu'tuw#{wx'yxy(Oyz(iz{#{{|)S|}#{}!O+v!O!P#{!P!Q.]!Q![)q![!]6x!]!^%T!^!}#{!}#O7c#O#P9X#P#Q9^#Q#R#{#R#S9w#S#T#{#T#Y,w#Y#Z:b#Z#b,w#b#c?`#c#f,w#f#g@]#g#h,w#h#iAY#i#o,w#o#p#{#p#qC^#q;'S#{;'S;=`$d<%l~#{~O#{~~CwS$QUsSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{S$gP;=`<%l#{^$qUsS!xYOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U%[UsS#ZQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{^%sWsSOp#{pq&]qt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{^&dZiYsSOY&]YZ#{Zt&]tu'Vuw&]wx'Vx#O&]#O#P'V#P;'S&];'S;=`'n<%lO&]Y'[SiYOY'VZ;'S'V;'S;=`'h<%lO'VY'kP;=`<%l'V^'qP;=`<%l&]~'yO#S~~(OO#Q~U(VUsS!}QOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U(pUsS#^QOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U)XWsSOt#{uw#{x!Q#{!Q![)q![#O#{#P;'S#{;'S;=`$d<%lO#{U)xYsSnQOt#{uw#{x!O#{!O!P*h!P!Q#{!Q![)q![#O#{#P;'S#{;'S;=`$d<%lO#{U*mWsSOt#{uw#{x!Q#{!Q![+V![#O#{#P;'S#{;'S;=`$d<%lO#{U+^WsSnQOt#{uw#{x!Q#{!Q![+V![#O#{#P;'S#{;'S;=`$d<%lO#{U+{^sSOt#{uw#{x}#{}!O,w!O!Q#{!Q![)q![!_#{!_!`-r!`#O#{#P#T#{#T#o,w#o;'S#{;'S;=`$d<%lO#{U,|[sSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#o,w#o;'S#{;'S;=`$d<%lO#{U-yUzQsSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U.bWsSOt#{uw#{x!P#{!P!Q.z!Q#O#{#P;'S#{;'S;=`$d<%lO#{U/P^sSOY/{YZ#{Zt/{tu1Ouw/{wx1Ox!P/{!P!Q#{!Q!}/{!}#O5q#O#P3^#P;'S/{;'S;=`6r<%lO/{U0S^sSwQOY/{YZ#{Zt/{tu1Ouw/{wx1Ox!P/{!P!Q3s!Q!}/{!}#O5q#O#P3^#P;'S/{;'S;=`6r<%lO/{Q1TXwQOY1OZ!P1O!P!Q1p!Q!}1O!}#O2_#O#P3^#P;'S1O;'S;=`3m<%lO1OQ1sP!P!Q1vQ1{UwQ#Z#[1v#]#^1v#a#b1v#g#h1v#i#j1v#m#n1vQ2bVOY2_Z#O2_#O#P2w#P#Q1O#Q;'S2_;'S;=`3W<%lO2_Q2zSOY2_Z;'S2_;'S;=`3W<%lO2_Q3ZP;=`<%l2_Q3aSOY1OZ;'S1O;'S;=`3m<%lO1OQ3pP;=`<%l1OU3xWsSOt#{uw#{x!P#{!P!Q4b!Q#O#{#P;'S#{;'S;=`$d<%lO#{U4ibsSwQOt#{uw#{x#O#{#P#Z#{#Z#[4b#[#]#{#]#^4b#^#a#{#a#b4b#b#g#{#g#h4b#h#i#{#i#j4b#j#m#{#m#n4b#n;'S#{;'S;=`$d<%lO#{U5v[sSOY5qYZ#{Zt5qtu2_uw5qwx2_x#O5q#O#P2w#P#Q/{#Q;'S5q;'S;=`6l<%lO5qU6oP;=`<%l5qU6uP;=`<%l/{U7PUsS!PQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U7jW#YQsSOt#{uw#{x!_#{!_!`8S!`#O#{#P;'S#{;'S;=`$d<%lO#{U8XVsSOt#{uw#{x#O#{#P#Q8n#Q;'S#{;'S;=`$d<%lO#{U8uU#XQsSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~9^O#T~U9eU#]QsSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U:OUsS!WQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U:g]sSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#U;`#U#o,w#o;'S#{;'S;=`$d<%lO#{U;e^sSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#`,w#`#a<a#a#o,w#o;'S#{;'S;=`$d<%lO#{U<f^sSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#g,w#g#h=b#h#o,w#o;'S#{;'S;=`$d<%lO#{U=g^sSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#X,w#X#Y>c#Y#o,w#o;'S#{;'S;=`$d<%lO#{U>j[vQsSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#o,w#o;'S#{;'S;=`$d<%lO#{^?g[#UWsSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#o,w#o;'S#{;'S;=`$d<%lO#{^@d[#WWsSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#o,w#o;'S#{;'S;=`$d<%lO#{^Aa^#VWsSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#f,w#f#gB]#g#o,w#o;'S#{;'S;=`$d<%lO#{UBb^sSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#i,w#i#j=b#j#o,w#o;'S#{;'S;=`$d<%lO#{UCeU![QsSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~C|O#`~",
   tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, new LocalTokenGroup("[~RP!O!PU~ZO!|~~", 11)],
-  topRules: {"Program":[0,25]},
+  topRules: {"Program":[0,26]},
   specialized: [{term: 20, get: (value: any, stack: any) => (specializeKeyword(value, stack) << 1), external: specializeKeyword},{term: 20, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}],
   tokenPrec: 1578
 })

src/parser/tests/basics.test.ts

@@ -48,7 +48,6 @@ describe('Identifier', () => {
       FunctionCallOrIdentifier
         Identifier even?`)
   })
-
 })
 
 describe('Unicode Symbol Support', () => {
@@ -637,26 +636,51 @@ describe('DotGet whitespace sensitivity', () => {
 })
 
 describe('Comments', () => {
-  test('are barely there', () => {
+  test('are greedy', () => {
-    expect(`x = 5 # one banana\ny = 2 # two bananas`).toMatchTree(`
+    expect(`
+x = 5 # one banana
+y = 2 # two bananas`).toMatchTree(`
       Assign
         AssignableIdentifier x
         Eq =
         Number 5
+      Comment # one banana
       Assign
         AssignableIdentifier y
         Eq =
-        Number 2`)
+        Number 2
+      Comment # two bananas`)
 
-    expect('# some comment\nbasename = 5 # very astute\n basename / prop\n# good info').toMatchTree(`
+    expect(`
-      Assign
+# some comment
-        AssignableIdentifier basename
+basename = 5 # very astute
-        Eq =
+ basename / prop
-        Number 5
+# good info`).toMatchTree(`
-      BinOp
+  Comment # some comment
-        Identifier basename
+  Assign
-        Slash /
+    AssignableIdentifier basename
-        Identifier prop`)
+    Eq =
+    Number 5
+  Comment # very astute
+  BinOp
+    Identifier basename
+    Slash /
+    Identifier prop
+  Comment # good info`)
+  })
+
+  test('words with # are not considered comments', () => {
+    expect('find #hashtag-file.txt').toMatchTree(`
+      FunctionCall
+        Identifier find
+        PositionalArg
+          Word #hashtag-file.txt`)
+  })
+
+  test('hastags in strings are not comments', () => {
+    expect("'this is not a #comment'").toMatchTree(`
+      String
+        StringFragment this is not a #comment`)
   })
 })
 
@@ -697,7 +721,6 @@ describe('Array destructuring', () => {
           Number 2`)
   })
 
-
   test('works with dotget', () => {
     expect('[ a ] = [ [1 2 3] ]; a.1').toMatchTree(`
       Assign

vscode-extension/package.json

@@ -83,7 +83,7 @@
     "compile": "bun run compile:client && bun run compile:server",
     "compile:client": "bun build client/src/extension.ts --outdir client/dist --target node --format cjs --external vscode",
     "compile:server": "bun build server/src/server.ts --outdir server/dist --target node --format cjs",
-    "watch": "bun run compile:client --watch",
+    "watch": "bun run compile:client --watch & bun run compile:server --watch",
     "package": "bun run compile:client --minify && bun run compile:server --minify",
     "check-types": "tsc --noEmit",
     "build-and-install": "bun run package && bunx @vscode/vsce package --allow-missing-repository && code --install-extension shrimp-*.vsix"

vscode-extension/server/src/scopeTracker.test.ts

@@ -0,0 +1,145 @@
+import { test, expect, describe } from 'bun:test'
+import { ScopeTracker } from './scopeTracker'
+import { TextDocument } from 'vscode-languageserver-textdocument'
+import { parser } from '../../../src/parser/shrimp'
+import * as Terms from '../../../src/parser/shrimp.terms'
+
+describe('ScopeTracker', () => {
+  test('top-level assignment is in scope', () => {
+    const code = 'x = 5\necho x'
+    const { tree, tracker } = parseAndGetScope(code)
+
+    // Find the 'x' identifier in 'echo x'
+    const identifiers: any[] = []
+    tree.topNode.cursor().iterate((node: any) => {
+      if (node.type.id === Terms.Identifier) {
+        identifiers.push(node.node)
+      }
+    })
+
+    // Second identifier should be the 'x' in 'echo x'
+    const xInEcho = identifiers[1]
+    expect(xInEcho).toBeDefined()
+    expect(tracker.isInScope('x', xInEcho)).toBe(true)
+  })
+
+  test('undeclared variable is not in scope', () => {
+    const code = 'echo x'
+    const { tree, tracker } = parseAndGetScope(code)
+
+    // Find the 'x' identifier
+    let xNode: any = null
+    tree.topNode.cursor().iterate((node: any) => {
+      if (node.type.id === Terms.Identifier) {
+        xNode = node.node
+      }
+    })
+
+    expect(xNode).toBeDefined()
+    expect(tracker.isInScope('x', xNode)).toBe(false)
+  })
+
+  test('function parameter is in scope inside function', () => {
+    const code = `greet = do name:
+  echo name
+end`
+    const { tree, tracker } = parseAndGetScope(code)
+
+    // Find all identifiers
+    const identifiers: any[] = []
+    tree.topNode.cursor().iterate((node: any) => {
+      if (node.type.id === Terms.Identifier) {
+        identifiers.push(node.node)
+      }
+    })
+
+    // Find the 'name' in 'echo name' (should be last identifier)
+    const nameInEcho = identifiers[identifiers.length - 1]
+    expect(tracker.isInScope('name', nameInEcho)).toBe(true)
+  })
+
+  test('assignment before usage is in scope', () => {
+    const code = `x = 5
+y = 10
+echo x y`
+    const { tree, tracker } = parseAndGetScope(code)
+
+    // Find identifiers
+    const identifiers: any[] = []
+    tree.topNode.cursor().iterate((node: any) => {
+      if (node.type.id === Terms.Identifier) {
+        identifiers.push(node.node)
+      }
+    })
+
+    // Last two identifiers should be 'x' and 'y' in 'echo x y'
+    const xInEcho = identifiers[identifiers.length - 2]
+    const yInEcho = identifiers[identifiers.length - 1]
+
+    expect(tracker.isInScope('x', xInEcho)).toBe(true)
+    expect(tracker.isInScope('y', yInEcho)).toBe(true)
+  })
+
+  test('assignment after usage is not in scope', () => {
+    const code = `echo x
+x = 5`
+    const { tree, tracker } = parseAndGetScope(code)
+
+    // Find the first 'x' identifier (in echo)
+    let xNode: any = null
+    tree.topNode.cursor().iterate((node: any) => {
+      if (node.type.id === Terms.Identifier && !xNode) {
+        xNode = node.node
+      }
+    })
+
+    expect(tracker.isInScope('x', xNode)).toBe(false)
+  })
+
+  test('nested function has access to outer scope', () => {
+    const code = `x = 5
+greet = do:
+  echo x
+end`
+    const { tree, tracker } = parseAndGetScope(code)
+
+    // Find all identifiers
+    const identifiers: any[] = []
+    tree.topNode.cursor().iterate((node: any) => {
+      if (node.type.id === Terms.Identifier) {
+        identifiers.push(node.node)
+      }
+    })
+
+    // Find the 'x' in 'echo x' (should be last identifier)
+    const xInEcho = identifiers[identifiers.length - 1]
+    expect(tracker.isInScope('x', xInEcho)).toBe(true)
+  })
+
+  test('inner function parameter shadows outer variable', () => {
+    const code = `x = 5
+greet = do x:
+  echo x
+end`
+    const { tree, tracker } = parseAndGetScope(code)
+
+    // Find all identifiers
+    const identifiers: any[] = []
+    tree.topNode.cursor().iterate((node: any) => {
+      if (node.type.id === Terms.Identifier) {
+        identifiers.push(node.node)
+      }
+    })
+
+    // The 'x' in 'echo x' should have 'x' in scope (from parameter)
+    const xInEcho = identifiers[identifiers.length - 1]
+    expect(tracker.isInScope('x', xInEcho)).toBe(true)
+  })
+})
+
+const parseAndGetScope = (code: string) => {
+  const document = TextDocument.create('test://test.sh', 'shrimp', 1, code)
+  const tree = parser.parse(code)
+  const tracker = new ScopeTracker(document)
+  return { document, tree, tracker }
+}

vscode-extension/server/src/scopeTracker.ts

@@ -0,0 +1,135 @@
+import { SyntaxNode } from '@lezer/common'
+import { TextDocument } from 'vscode-languageserver-textdocument'
+import * as Terms from '../../../src/parser/shrimp.terms'
+
+/**
+ * Tracks variables in scope at a given position in the parse tree.
+ * Used to distinguish identifiers (in scope) from words (not in scope).
+ */
+export class ScopeTracker {
+  private document: TextDocument
+  private scopeCache = new Map<number, Set<string>>()
+
+  constructor(document: TextDocument) {
+    this.document = document
+  }
+
+  /**
+   * Check if a name is in scope at the given node's position.
+   */
+  isInScope(name: string, node: SyntaxNode): boolean {
+    const scope = this.getScopeAt(node)
+    return scope.has(name)
+  }
+
+  /**
+   * Get all variables in scope at the given node's position.
+   */
+  private getScopeAt(node: SyntaxNode): Set<string> {
+    const position = node.from
+
+    // Check cache first
+    if (this.scopeCache.has(position)) {
+      return this.scopeCache.get(position)!
+    }
+
+    const scope = new Set<string>()
+
+    // Find all containing function definitions
+    const containingFunctions = this.findContainingFunctions(node)
+
+    // Collect scope from each containing function (inner to outer)
+    for (const fnNode of containingFunctions) {
+      this.collectParams(fnNode, scope)
+      this.collectAssignments(fnNode, position, scope)
+    }
+
+    // Collect top-level assignments
+    const root = this.getRoot(node)
+    this.collectAssignments(root, position, scope)
+
+    this.scopeCache.set(position, scope)
+    return scope
+  }
+
+  /**
+   * Find all function definitions that contain the given node.
+   */
+  private findContainingFunctions(node: SyntaxNode): SyntaxNode[] {
+    const functions: SyntaxNode[] = []
+    let current = node.parent
+
+    while (current) {
+      if (current.type.id === Terms.FunctionDef) {
+        functions.unshift(current) // Add to beginning for outer-to-inner order
+      }
+      current = current.parent
+    }
+
+    return functions
+  }
+
+  /**
+   * Get the root node of the tree.
+   */
+  private getRoot(node: SyntaxNode): SyntaxNode {
+    let current = node
+    while (current.parent) {
+      current = current.parent
+    }
+    return current
+  }
+
+  /**
+   * Collect parameter names from a function definition.
+   */
+  private collectParams(fnNode: SyntaxNode, scope: Set<string>) {
+    let child = fnNode.firstChild
+    while (child) {
+      if (child.type.id === Terms.Params) {
+        let param = child.firstChild
+        while (param) {
+          if (param.type.id === Terms.Identifier) {
+            const text = this.document.getText({
+              start: this.document.positionAt(param.from),
+              end: this.document.positionAt(param.to),
+            })
+            scope.add(text)
+          }
+          param = param.nextSibling
+        }
+        break
+      }
+      child = child.nextSibling
+    }
+  }
+
+  /**
+   * Collect assignment names from a scope node that occur before the given position.
+   */
+  private collectAssignments(scopeNode: SyntaxNode, beforePosition: number, scope: Set<string>) {
+    const cursor = scopeNode.cursor()
+
+    cursor.iterate((node) => {
+      // Stop if we've passed the position we're checking
+      if (node.from >= beforePosition) return false
+
+      if (node.type.id === Terms.Assign) {
+        const assignNode = node.node
+        const child = assignNode.firstChild
+        if (child?.type.id === Terms.AssignableIdentifier) {
+          const text = this.document.getText({
+            start: this.document.positionAt(child.from),
+            end: this.document.positionAt(child.to),
+          })
+          scope.add(text)
+        }
+      }
+
+      // Don't descend into nested functions unless it's the current scope
+      if (node.type.id === Terms.FunctionDef && node.node !== scopeNode) {
+        return false
+      }
+    })
+  }
+}

vscode-extension/server/src/semanticTokens.ts

@@ -2,7 +2,12 @@ import { parser } from '../../../src/parser/shrimp'
 import * as Terms from '../../../src/parser/shrimp.terms'
 import { SyntaxNode } from '@lezer/common'
 import { TextDocument } from 'vscode-languageserver-textdocument'
-import { SemanticTokensBuilder, SemanticTokenTypes } from 'vscode-languageserver/node'
+import {
+  SemanticTokensBuilder,
+  SemanticTokenTypes,
+  SemanticTokenModifiers,
+} from 'vscode-languageserver/node'
+import { ScopeTracker } from './scopeTracker'
 
 export const TOKEN_TYPES = [
   SemanticTokenTypes.function,
@@ -14,56 +19,154 @@ export const TOKEN_TYPES = [
   SemanticTokenTypes.parameter,
   SemanticTokenTypes.property,
   SemanticTokenTypes.regexp,
+  SemanticTokenTypes.comment,
 ]
 
-export const TOKEN_MODIFIERS: string[] = []
+export const TOKEN_MODIFIERS = [
+  SemanticTokenModifiers.declaration,
+  SemanticTokenModifiers.modification,
+  SemanticTokenModifiers.readonly,
+]
 
 export function buildSemanticTokens(document: TextDocument): number[] {
   const text = document.getText()
   const tree = parser.parse(text)
   const builder = new SemanticTokensBuilder()
+  const scopeTracker = new ScopeTracker(document)
 
-  walkTree(tree.topNode, document, builder)
+  walkTree(tree.topNode, document, builder, scopeTracker)
 
   return builder.build().data
 }
 
 // Walk the tree and collect tokens
-function walkTree(node: SyntaxNode, document: TextDocument, builder: SemanticTokensBuilder) {
+function walkTree(
-  const tokenType = getTokenType(node.type.id)
+  node: SyntaxNode,
+  document: TextDocument,
+  builder: SemanticTokensBuilder,
+  scopeTracker: ScopeTracker
+) {
+  const tokenInfo = getTokenType(node, document, scopeTracker)
 
-  if (tokenType !== undefined) {
+  if (tokenInfo !== undefined) {
     const start = document.positionAt(node.from)
     const length = node.to - node.from
-    builder.push(start.line, start.character, length, tokenType, 0)
+    builder.push(start.line, start.character, length, tokenInfo.type, tokenInfo.modifiers)
   }
 
   let child = node.firstChild
   while (child) {
-    walkTree(child, document, builder)
+    walkTree(child, document, builder, scopeTracker)
     child = child.nextSibling
   }
 }
 
-// Map Lezer node IDs to semantic token type indices
+// Map Lezer node IDs to semantic token type indices and modifiers
-function getTokenType(nodeTypeId: number): number | undefined {
+type TokenInfo = { type: number; modifiers: number } | undefined
-  switch (nodeTypeId) {
+function getTokenType(
-    case Terms.FunctionCall:
+  node: SyntaxNode,
-    case Terms.FunctionDef:
+  document: TextDocument,
-      return TOKEN_TYPES.indexOf(SemanticTokenTypes.function)
+  scopeTracker: ScopeTracker
+): TokenInfo {
+  const nodeTypeId = node.type.id
+  const parentTypeId = node.parent?.type.id
 
+  // Special case for now, eventually keywords will go away
+  if (node.type.name === 'keyword') {
+    return {
+      type: TOKEN_TYPES.indexOf(SemanticTokenTypes.keyword),
+      modifiers: 0,
+    }
+  }
+
+  switch (nodeTypeId) {
     case Terms.Identifier:
+      // Check parent to determine context
+      if (parentTypeId === Terms.FunctionCall) {
+        return {
+          type: TOKEN_TYPES.indexOf(SemanticTokenTypes.function),
+          modifiers: 0,
+        }
+      }
+      if (parentTypeId === Terms.FunctionDef) {
+        return {
+          type: TOKEN_TYPES.indexOf(SemanticTokenTypes.function),
+          modifiers: getModifierBits(SemanticTokenModifiers.declaration),
+        }
+      }
+      if (parentTypeId === Terms.FunctionCallOrIdentifier) {
+        return {
+          type: TOKEN_TYPES.indexOf(SemanticTokenTypes.function),
+          modifiers: 0,
+        }
+      }
+      if (parentTypeId === Terms.Params) {
+        return {
+          type: TOKEN_TYPES.indexOf(SemanticTokenTypes.parameter),
+          modifiers: 0,
+        }
+      }
+      if (parentTypeId === Terms.DotGet) {
+        return {
+          type: TOKEN_TYPES.indexOf(SemanticTokenTypes.property),
+          modifiers: 0,
+        }
+      }
+
+      // Special case: Identifier in PositionalArg - check scope
+      if (parentTypeId === Terms.PositionalArg) {
+        const identifierText = document.getText({
+          start: document.positionAt(node.from),
+          end: document.positionAt(node.to),
+        })
+
+        // If not in scope, treat as string (like a Word)
+        if (!scopeTracker.isInScope(identifierText, node)) {
+          return {
+            type: TOKEN_TYPES.indexOf(SemanticTokenTypes.string),
+            modifiers: 0,
+          }
+        }
+        // If in scope, fall through to treat as variable
+      }
+
+      // Otherwise it's a regular variable
+      return {
+        type: TOKEN_TYPES.indexOf(SemanticTokenTypes.variable),
+        modifiers: 0,
+      }
+
+    case Terms.IdentifierBeforeDot:
+      return {
+        type: TOKEN_TYPES.indexOf(SemanticTokenTypes.variable),
+        modifiers: 0,
+      }
+
+    case Terms.NamedArgPrefix:
+      return {
+        type: TOKEN_TYPES.indexOf(SemanticTokenTypes.property),
+        modifiers: 0,
+      }
+
     case Terms.AssignableIdentifier:
-    case Terms.FunctionCallOrIdentifier:
+      return {
-      return TOKEN_TYPES.indexOf(SemanticTokenTypes.variable)
+        type: TOKEN_TYPES.indexOf(SemanticTokenTypes.variable),
+        modifiers: getModifierBits(SemanticTokenModifiers.modification),
+      }
 
     case Terms.String:
     case Terms.StringFragment:
     case Terms.Word:
-      return TOKEN_TYPES.indexOf(SemanticTokenTypes.string)
+      return {
+        type: TOKEN_TYPES.indexOf(SemanticTokenTypes.string),
+        modifiers: 0,
+      }
 
     case Terms.Number:
-      return TOKEN_TYPES.indexOf(SemanticTokenTypes.number)
+      return {
+        type: TOKEN_TYPES.indexOf(SemanticTokenTypes.number),
+        modifiers: 0,
+      }
 
     case Terms.Plus:
     case Terms.Minus:
@@ -79,23 +182,40 @@ function getTokenType(nodeTypeId: number): number | undefined {
     case Terms.Modulo:
     case Terms.And:
     case Terms.Or:
-      return TOKEN_TYPES.indexOf(SemanticTokenTypes.operator)
+      return {
+        type: TOKEN_TYPES.indexOf(SemanticTokenTypes.operator),
+        modifiers: 0,
+      }
 
-    case Terms.keyword:
     case Terms.Do:
-      return TOKEN_TYPES.indexOf(SemanticTokenTypes.keyword)
+    case Terms.colon:
-
+      return {
-    case Terms.Params:
+        type: TOKEN_TYPES.indexOf(SemanticTokenTypes.keyword),
-    case Terms.NamedParam:
+        modifiers: 0,
-      return TOKEN_TYPES.indexOf(SemanticTokenTypes.parameter)
+      }
-
-    case Terms.DotGet:
-      return TOKEN_TYPES.indexOf(SemanticTokenTypes.property)
 
     case Terms.Regex:
-      return TOKEN_TYPES.indexOf(SemanticTokenTypes.regexp)
+      return {
+        type: TOKEN_TYPES.indexOf(SemanticTokenTypes.regexp),
+        modifiers: 0,
+      }
+
+    case Terms.Comment:
+      return {
+        type: TOKEN_TYPES.indexOf(SemanticTokenTypes.comment),
+        modifiers: 0,
+      }
 
     default:
       return undefined
   }
 }
+
+const getModifierBits = (...modifiers: SemanticTokenModifiers[]): number => {
+  let bits = 0
+  for (const modifier of modifiers) {
+    const index = TOKEN_MODIFIERS.indexOf(modifier)
+    if (index !== -1) bits |= 1 << index
+  }
+  return bits
+}

vscode-extension/server/src/server.ts

@@ -86,27 +86,38 @@ function handleParseTree(params: { uri: string }) {
 
   const text = document.getText()
   const tree = parser.parse(text)
-  const treeString = tree.toString()
+  const cursor = tree.cursor()
 
-  // Format with indentation, without parentheses
   let formatted = ''
-  let indent = 0
+  let depth = 0
-  for (let i = 0; i < treeString.length; i++) {
+
-    const char = treeString[i]
+  const printNode = () => {
-    if (char === '(') {
+    const nodeName = cursor.name
-      formatted += '\n'
+    const nodeText = text.slice(cursor.from, cursor.to)
-      indent++
+    const indent = '  '.repeat(depth)
-      formatted += '  '.repeat(indent)
+
-    } else if (char === ')') {
+    formatted += `${indent}${nodeName}`
-      indent--
+    if (nodeText) {
-    } else if (char === ',') {
+      const escapedText = nodeText.replace(/\n/g, '\\n').replace(/\r/g, '\\r')
-      formatted += '\n'
+      formatted += ` "${escapedText}"`
-      formatted += '  '.repeat(indent)
+    }
-    } else {
+    formatted += '\n'
-      formatted += char
+  }
+
+  const traverse = (): void => {
+    printNode()
+
+    if (cursor.firstChild()) {
+      depth++
+      do {
+        traverse()
+      } while (cursor.nextSibling())
+      cursor.parent()
+      depth--
     }
   }
 
+  traverse()
   return formatted
 }