diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index 01f95b6..1cad017 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -2,7 +2,7 @@ @context trackScope from "./scopeTracker" -@skip { space | comment } +@skip { space | Comment } @top Program { item* } @@ -18,7 +18,7 @@ newlineOrSemicolon { "\n" | ";" } eof { @eof } space { " " | "\t" } - comment { "#" ![\n]* } + Comment { "#" " " ![\n]* } leftParen { "(" } rightParen { ")" } colon[closedBy="end", @name="colon"] { ":" } diff --git a/src/parser/shrimp.terms.ts b/src/parser/shrimp.terms.ts index 05f3d5a..bcdf7b8 100644 --- a/src/parser/shrimp.terms.ts +++ b/src/parser/shrimp.terms.ts @@ -24,42 +24,43 @@ export const Word = 22, IdentifierBeforeDot = 23, Do = 24, - Program = 25, - PipeExpr = 26, - FunctionCall = 27, - DotGet = 28, - Number = 29, - ParenExpr = 30, - FunctionCallOrIdentifier = 31, - BinOp = 32, - String = 33, - StringFragment = 34, - Interpolation = 35, - EscapeSeq = 36, - Boolean = 37, - Regex = 38, - Dict = 39, - NamedArg = 40, - NamedArgPrefix = 41, - FunctionDef = 42, - Params = 43, - NamedParam = 44, - Null = 45, - colon = 46, - CatchExpr = 47, - keyword = 68, - Block = 49, - FinallyExpr = 50, - Underscore = 53, - Array = 54, - ConditionalOp = 55, - PositionalArg = 56, - WhileExpr = 58, - FunctionCallWithBlock = 60, - TryExpr = 61, - Throw = 63, - IfExpr = 65, - ElseIfExpr = 67, - ElseExpr = 69, - CompoundAssign = 70, - Assign = 71 + Comment = 25, + Program = 26, + PipeExpr = 27, + FunctionCall = 28, + DotGet = 29, + Number = 30, + ParenExpr = 31, + FunctionCallOrIdentifier = 32, + BinOp = 33, + String = 34, + StringFragment = 35, + Interpolation = 36, + EscapeSeq = 37, + Boolean = 38, + Regex = 39, + Dict = 40, + NamedArg = 41, + NamedArgPrefix = 42, + FunctionDef = 43, + Params = 44, + NamedParam = 45, + Null = 46, + colon = 47, + CatchExpr = 48, + keyword = 69, + Block = 50, + FinallyExpr = 51, + Underscore = 54, + Array = 55, + ConditionalOp = 56, + PositionalArg = 57, + WhileExpr = 59, + FunctionCallWithBlock = 61, + TryExpr = 62, + Throw = 64, + IfExpr = 66, + ElseIfExpr = 68, + ElseExpr = 70, + CompoundAssign = 71, + Assign = 72 diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index 5afc5d8..c8c217c 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -4,24 +4,24 @@ import {operatorTokenizer} from "./operatorTokenizer" import {tokenizer, specializeKeyword} from "./tokenizer" import {trackScope} from "./scopeTracker" import {highlighting} from "./highlight" -const spec_Identifier = {__proto__:null,null:90, catch:96, finally:102, end:104, while:118, try:124, throw:128, if:132, else:136} +const spec_Identifier = {__proto__:null,null:92, catch:98, finally:104, end:106, while:120, try:126, throw:130, if:134, else:138} export const parser = LRParser.deserialize({ version: 14, - states: "9OQYQbOOO#zQcO'#C{O$zOSO'#C}OOQa'#DT'#DTO&TQbO'#DdO'iQcO'#E]OOQa'#E]'#E]O(lQcO'#E]O)nQcO'#E[O*UQRO'#C|O+eQcO'#EWO+uQcO'#EWO,PQbO'#CzO,wOpO'#CxOOQ`'#EX'#EXO,|QbO'#EWO-WQRO'#DtOOQ`'#EW'#EWO-lQQO'#EVOOQ`'#EV'#EVOOQ`'#Dv'#DvQYQbOOO-tQbO'#DWO.PQbO'#DhO.tQQO'#DkO.PQbO'#DmO.PQbO'#DoO.yQbO'#DUOOQa'#E['#E[OOQ`'#Df'#DfOOQ`'#Ek'#EkOOQ`'#EO'#EOO/TQbO,59cO/}QbO'#DPO0VQWO'#DQOOOO'#E_'#E_OOOO'#Dw'#DwO0kOSO,59iOOQa,59i,59iOOQ`'#Dx'#DxO0yQbO,5:OO1QQQO,59oOOQa,5:O,5:OO1]QbO,5:OO1gQbO,5:aO.PQbO,59hO.PQbO,59hO.PQbO,59hO.PQbO,5:PO.PQbO,5:PO.PQbO,5:PO1zQRO,59fO2RQRO,59fO2dQRO,59fO2_QQO,59fO2oQQO,59fO2wObO,59dO3SQbO'#EPO3_QbO,59bO3yQbO,5:UO1gQbO,5:`OOQ`,5:q,5:qOOQ`-E7t-E7tOOQ`'#Dy'#DyO4aQbO'#DXO4lQbO'#DYOOQO'#Dz'#DzO4dQQO'#DXO4zQQO,59rO5kQRO,5:SO5rQRO,5:SO3yQbO,5:VO5}QcO,5:XO6yQcO,5:XO7ZQcO,5:XO7eQRO,5:ZO7lQRO,5:ZOOQ`,59p,59pOOQ`-E7|-E7|OOOO,59k,59kOOOO,59l,59lOOOO-E7u-E7uOOQa1G/T1G/TOOQ`-E7v-E7vO7wQQO1G/ZOOQa1G/j1G/jO8SQbO1G/jOOQO'#D|'#D|O7wQQO1G/ZOOQa1G/Z1G/ZOOQ`'#D}'#D}O8SQbO1G/jOOQ`1G/{1G/{OOQa1G/S1G/SO9OQcO1G/SO9YQcO1G/SO9dQcO1G/SOOQa1G/k1G/kO;YQcO1G/kO;aQcO1G/kO;hQcO1G/kOOQa1G/Q1G/QOOQa1G/O1G/OO!dQbO'#C{O;oQbO'#CwOOQ`,5:k,5:kOOQ`-E7}-E7}OOQ`'#D_'#D_O;|QbO'#D_ORQbO7+%UOOQa7+%U7+%UOOQO-E7z-E7zOOQ`-E7{-E7{OOQ`'#D{'#D{O>]QQO'#D{O>bQbO'#EhOOQ`,59y,59yO?UQbO'#D]O?ZQQO'#D`OOQ`7+%[7+%[O?`QbO7+%[O?eQbO7+%[O?mQbO7+$xO?xQbO7+$xO@iQbO7+%YOOQ`7+%]7+%]O@nQbO7+%]O@sQbO7+%]O@{QbO7+%aOOQa<bAN>bOOQ`AN>OAN>OOBcQbOAN>OOBhQbOAN>OOOQ`AN>cAN>cOOQ`-E8O-E8OOOQ`AN>gAN>gOBpQbOAN>gO.PQbO,5:]O3yQbO,5:_OOQ`7+$}7+$}OOQ`G23jG23jOBuQbOG23jPBXQbO'#DqOOQ`G24RG24ROBzQRO1G/wOCRQRO1G/wOOQ`1G/y1G/yOOQ`LD)ULD)UO3yQbO7+%cOOQ`<RQbO7+%VOOQa7+%V7+%VOOQO-E7{-E7{OOQ`-E7|-E7|OOQ`'#D|'#D|O>]QQO'#D|O>bQbO'#EhOOQ`,59z,59zO?UQbO'#D^O?ZQQO'#DaOOQ`7+%]7+%]O?`QbO7+%]O?eQbO7+%]O?mQbO7+$yO?xQbO7+$yO@iQbO7+%ZOOQ`7+%^7+%^O@nQbO7+%^O@sQbO7+%^O@{QbO7+%bOOQa<cAN>cOOQ`AN>PAN>POBcQbOAN>POBhQbOAN>POOQ`AN>dAN>dOOQ`-E8P-E8POOQ`AN>hAN>hOBpQbOAN>hO.PQbO,5:^O3yQbO,5:`OOQ`7+%O7+%OOOQ`G23kG23kOBuQbOG23kPBXQbO'#DrOOQ`G24SG24SOBzQRO1G/xOCRQRO1G/xOOQ`1G/z1G/zOOQ`LD)VLD)VO3yQbO7+%dOOQ`<q#c#f,Y#f#g?n#g#h,Y#h#i@k#i#o,Y#o#p#{#p#qBo#q;'S#{;'S;=`$d<%l~#{~O#{~~CYS$QUrSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{S$gP;=`<%l#{^$qUrS!wYOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U%[UrS#ZQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{^%uZrS!xYOY%nYZ#{Zt%ntu&huw%nwx&hx#O%n#O#P&h#P;'S%n;'S;=`'P<%lO%nY&mS!xYOY&hZ;'S&h;'S;=`&y<%lO&hY&|P;=`<%l&h^'SP;=`<%l%n~'[O#S~~'aO#Q~U'hUrS!}QOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U(RUrS#^QOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U(jWrSOt#{uw#{x!Q#{!Q![)S![#O#{#P;'S#{;'S;=`$d<%lO#{U)ZYrSmQOt#{uw#{x!O#{!O!P)y!P!Q#{!Q![)S![#O#{#P;'S#{;'S;=`$d<%lO#{U*OWrSOt#{uw#{x!Q#{!Q![*h![#O#{#P;'S#{;'S;=`$d<%lO#{U*oWrSmQOt#{uw#{x!Q#{!Q![*h![#O#{#P;'S#{;'S;=`$d<%lO#{U+^^rSOt#{uw#{x}#{}!O,Y!O!Q#{!Q![)S![!_#{!_!`-T!`#O#{#P#T#{#T#o,Y#o;'S#{;'S;=`$d<%lO#{U,_[rSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#o,Y#o;'S#{;'S;=`$d<%lO#{U-[UyQrSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U-sWrSOt#{uw#{x!P#{!P!Q.]!Q#O#{#P;'S#{;'S;=`$d<%lO#{U.b^rSOY/^YZ#{Zt/^tu0auw/^wx0ax!P/^!P!Q#{!Q!}/^!}#O5S#O#P2o#P;'S/^;'S;=`6T<%lO/^U/e^rSvQOY/^YZ#{Zt/^tu0auw/^wx0ax!P/^!P!Q3U!Q!}/^!}#O5S#O#P2o#P;'S/^;'S;=`6T<%lO/^Q0fXvQOY0aZ!P0a!P!Q1R!Q!}0a!}#O1p#O#P2o#P;'S0a;'S;=`3O<%lO0aQ1UP!P!Q1XQ1^UvQ#Z#[1X#]#^1X#a#b1X#g#h1X#i#j1X#m#n1XQ1sVOY1pZ#O1p#O#P2Y#P#Q0a#Q;'S1p;'S;=`2i<%lO1pQ2]SOY1pZ;'S1p;'S;=`2i<%lO1pQ2lP;=`<%l1pQ2rSOY0aZ;'S0a;'S;=`3O<%lO0aQ3RP;=`<%l0aU3ZWrSOt#{uw#{x!P#{!P!Q3s!Q#O#{#P;'S#{;'S;=`$d<%lO#{U3zbrSvQOt#{uw#{x#O#{#P#Z#{#Z#[3s#[#]#{#]#^3s#^#a#{#a#b3s#b#g#{#g#h3s#h#i#{#i#j3s#j#m#{#m#n3s#n;'S#{;'S;=`$d<%lO#{U5X[rSOY5SYZ#{Zt5Stu1puw5Swx1px#O5S#O#P2Y#P#Q/^#Q;'S5S;'S;=`5}<%lO5SU6QP;=`<%l5SU6WP;=`<%l/^U6bUrS!OQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U6{W#YQrSOt#{uw#{x!_#{!_!`7e!`#O#{#P;'S#{;'S;=`$d<%lO#{U7jVrSOt#{uw#{x#O#{#P#Q8P#Q;'S#{;'S;=`$d<%lO#{U8WU#XQrSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~8oO#T~U8vU#]QrSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U9aUrS!VQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U9x]rSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#U:q#U#o,Y#o;'S#{;'S;=`$d<%lO#{U:v^rSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#`,Y#`#a;r#a#o,Y#o;'S#{;'S;=`$d<%lO#{U;w^rSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#g,Y#g#hx[#UWrSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#o,Y#o;'S#{;'S;=`$d<%lO#{^?u[#WWrSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#o,Y#o;'S#{;'S;=`$d<%lO#{^@r^#VWrSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#f,Y#f#gAn#g#o,Y#o;'S#{;'S;=`$d<%lO#{UAs^rSOt#{uw#{x}#{}!O,Y!O!_#{!_!`-T!`#O#{#P#T#{#T#i,Y#i#jc#Y#o,w#o;'S#{;'S;=`$d<%lO#{U>j[vQsSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#o,w#o;'S#{;'S;=`$d<%lO#{^?g[#UWsSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#o,w#o;'S#{;'S;=`$d<%lO#{^@d[#WWsSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#o,w#o;'S#{;'S;=`$d<%lO#{^Aa^#VWsSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#f,w#f#gB]#g#o,w#o;'S#{;'S;=`$d<%lO#{UBb^sSOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#i,w#i#j=b#j#o,w#o;'S#{;'S;=`$d<%lO#{UCeU![QsSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~C|O#`~", tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, new LocalTokenGroup("[~RP!O!PU~ZO!|~~", 11)], - topRules: {"Program":[0,25]}, + topRules: {"Program":[0,26]}, specialized: [{term: 20, get: (value: any, stack: any) => (specializeKeyword(value, stack) << 1), external: specializeKeyword},{term: 20, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}], tokenPrec: 1578 }) diff --git a/src/parser/tests/basics.test.ts b/src/parser/tests/basics.test.ts index b4092a9..dea9d7b 100644 --- a/src/parser/tests/basics.test.ts +++ b/src/parser/tests/basics.test.ts @@ -48,7 +48,6 @@ describe('Identifier', () => { FunctionCallOrIdentifier Identifier even?`) }) - }) describe('Unicode Symbol Support', () => { @@ -637,26 +636,51 @@ describe('DotGet whitespace sensitivity', () => { }) describe('Comments', () => { - test('are barely there', () => { - expect(`x = 5 # one banana\ny = 2 # two bananas`).toMatchTree(` + test('are greedy', () => { + expect(` +x = 5 # one banana +y = 2 # two bananas`).toMatchTree(` Assign AssignableIdentifier x Eq = Number 5 + Comment # one banana Assign AssignableIdentifier y Eq = - Number 2`) + Number 2 + Comment # two bananas`) - expect('# some comment\nbasename = 5 # very astute\n basename / prop\n# good info').toMatchTree(` - Assign - AssignableIdentifier basename - Eq = - Number 5 - BinOp - Identifier basename - Slash / - Identifier prop`) + expect(` +# some comment +basename = 5 # very astute + basename / prop +# good info`).toMatchTree(` + Comment # some comment + Assign + AssignableIdentifier basename + Eq = + Number 5 + Comment # very astute + BinOp + Identifier basename + Slash / + Identifier prop + Comment # good info`) + }) + + test('words with # are not considered comments', () => { + expect('find #hashtag-file.txt').toMatchTree(` + FunctionCall + Identifier find + PositionalArg + Word #hashtag-file.txt`) + }) + + test('hastags in strings are not comments', () => { + expect("'this is not a #comment'").toMatchTree(` + String + StringFragment this is not a #comment`) }) }) @@ -697,7 +721,6 @@ describe('Array destructuring', () => { Number 2`) }) - test('works with dotget', () => { expect('[ a ] = [ [1 2 3] ]; a.1').toMatchTree(` Assign @@ -800,4 +823,4 @@ Assign keyword end `) }) -}) \ No newline at end of file +})