From ef4184726d2bb20956692678f3cee93c60397417 Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Fri, 7 Nov 2025 19:48:33 -0800 Subject: [PATCH] allow _ in numbers (10_000_000) --- src/parser/shrimp.grammar | 2 +- src/parser/shrimp.ts | 2 +- src/parser/tests/basics.test.ts | 40 +++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/parser/shrimp.grammar b/src/parser/shrimp.grammar index 97908d9..0b7b0b7 100644 --- a/src/parser/shrimp.grammar +++ b/src/parser/shrimp.grammar @@ -13,7 +13,7 @@ StringFragment { !['\\$]+ } NamedArgPrefix { $[a-z-]+ "=" } - Number { ("-" | "+")? $[0-9]+ ('.' $[0-9]+)? } + Number { ("-" | "+")? $[0-9]+ ("_"? $[0-9]+)* ('.' $[0-9]+ ("_"? $[0-9]+)*)? } Boolean { "true" | "false" } newlineOrSemicolon { "\n" | ";" } eof { @eof } diff --git a/src/parser/shrimp.ts b/src/parser/shrimp.ts index fa41719..0bb0d38 100644 --- a/src/parser/shrimp.ts +++ b/src/parser/shrimp.ts @@ -19,7 +19,7 @@ export const parser = LRParser.deserialize({ propSources: [highlighting], skippedNodes: [0,25], repeatNodeCount: 11, - tokenData: "C|~R|OX#{XY$jYZ%TZp#{pq$jqs#{st%ntu'tuw#{wx'yxy(Oyz(iz{#{{|)S|}#{}!O+v!O!P#{!P!Q.]!Q![)q![!]6x!]!^%T!^!}#{!}#O7c#O#P9X#P#Q9^#Q#R#{#R#S9w#S#T#{#T#Y,w#Y#Z:b#Z#b,w#b#c?`#c#f,w#f#g@]#g#h,w#h#iAY#i#o,w#o#p#{#p#qC^#q;'S#{;'S;=`$d<%l~#{~O#{~~CwS$QU|SOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{S$gP;=`<%l#{^$qU|S!xYOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U%[U|S#YQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{^%sW|SOp#{pq&]qt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{^&dZiY|SOY&]YZ#{Zt&]tu'Vuw&]wx'Vx#O&]#O#P'V#P;'S&];'S;=`'n<%lO&]Y'[SiYOY'VZ;'S'V;'S;=`'h<%lO'VY'kP;=`<%l'V^'qP;=`<%l&]~'yO#T~~(OO#R~U(VU|S!}QOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U(pU|S#]QOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U)XW|SOt#{uw#{x!Q#{!Q![)q![#O#{#P;'S#{;'S;=`$d<%lO#{U)xY|SuQOt#{uw#{x!O#{!O!P*h!P!Q#{!Q![)q![#O#{#P;'S#{;'S;=`$d<%lO#{U*mW|SOt#{uw#{x!Q#{!Q![+V![#O#{#P;'S#{;'S;=`$d<%lO#{U+^W|SuQOt#{uw#{x!Q#{!Q![+V![#O#{#P;'S#{;'S;=`$d<%lO#{U+{^|SOt#{uw#{x}#{}!O,w!O!Q#{!Q![)q![!_#{!_!`-r!`#O#{#P#T#{#T#o,w#o;'S#{;'S;=`$d<%lO#{U,|[|SOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#o,w#o;'S#{;'S;=`$d<%lO#{U-yUzQ|SOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U.bW|SOt#{uw#{x!P#{!P!Q.z!Q#O#{#P;'S#{;'S;=`$d<%lO#{U/P^|SOY/{YZ#{Zt/{tu1Ouw/{wx1Ox!P/{!P!Q#{!Q!}/{!}#O5q#O#P3^#P;'S/{;'S;=`6r<%lO/{U0S^|S!aQOY/{YZ#{Zt/{tu1Ouw/{wx1Ox!P/{!P!Q3s!Q!}/{!}#O5q#O#P3^#P;'S/{;'S;=`6r<%lO/{Q1TX!aQOY1OZ!P1O!P!Q1p!Q!}1O!}#O2_#O#P3^#P;'S1O;'S;=`3m<%lO1OQ1sP!P!Q1vQ1{U!aQ#Z#[1v#]#^1v#a#b1v#g#h1v#i#j1v#m#n1vQ2bVOY2_Z#O2_#O#P2w#P#Q1O#Q;'S2_;'S;=`3W<%lO2_Q2zSOY2_Z;'S2_;'S;=`3W<%lO2_Q3ZP;=`<%l2_Q3aSOY1OZ;'S1O;'S;=`3m<%lO1OQ3pP;=`<%l1OU3xW|SOt#{uw#{x!P#{!P!Q4b!Q#O#{#P;'S#{;'S;=`$d<%lO#{U4ib|S!aQOt#{uw#{x#O#{#P#Z#{#Z#[4b#[#]#{#]#^4b#^#a#{#a#b4b#b#g#{#g#h4b#h#i#{#i#j4b#j#m#{#m#n4b#n;'S#{;'S;=`$d<%lO#{U5v[|SOY5qYZ#{Zt5qtu2_uw5qwx2_x#O5q#O#P2w#P#Q/{#Q;'S5q;'S;=`6l<%lO5qU6oP;=`<%l5qU6uP;=`<%l/{U7PU|S!RQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U7jW#_Q|SOt#{uw#{x!_#{!_!`8S!`#O#{#P;'S#{;'S;=`$d<%lO#{U8XV|SOt#{uw#{x#O#{#P#Q8n#Q;'S#{;'S;=`$d<%lO#{U8uU#^Q|SOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~9^O#U~U9eU#`Q|SOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U:OU|S!YQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U:g]|SOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#U;`#U#o,w#o;'S#{;'S;=`$d<%lO#{U;e^|SOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#`,w#`#ac#Y#o,w#o;'S#{;'S;=`$d<%lO#{U>j[!PQ|SOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#o,w#o;'S#{;'S;=`$d<%lO#{^?g[#VW|SOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#o,w#o;'S#{;'S;=`$d<%lO#{^@d[#XW|SOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#o,w#o;'S#{;'S;=`$d<%lO#{^Aa^#WW|SOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#f,w#f#gB]#g#o,w#o;'S#{;'S;=`$d<%lO#{UBb^|SOt#{uw#{x}#{}!O,w!O!_#{!_!`-r!`#O#{#P#T#{#T#i,w#i#j=b#j#o,w#o;'S#{;'S;=`$d<%lO#{UCeUlQ|SOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~C|O#a~", + tokenData: "DY~R|OX#{XY$jYZ%TZp#{pq$jqs#{st%ntu'tuw#{wx'yxy(Oyz(iz{#{{|)S|}#{}!O,S!O!P#{!P!Q.i!Q![)q![!]7U!]!^%T!^!}#{!}#O7o#O#P9e#P#Q9j#Q#R#{#R#S:T#S#T#{#T#Y-T#Y#Z:n#Z#b-T#b#c?l#c#f-T#f#g@i#g#h-T#h#iAf#i#o-T#o#p#{#p#qCj#q;'S#{;'S;=`$d<%l~#{~O#{~~DTS$QU|SOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{S$gP;=`<%l#{^$qU|S!xYOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U%[U|S#YQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{^%sW|SOp#{pq&]qt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{^&dZiY|SOY&]YZ#{Zt&]tu'Vuw&]wx'Vx#O&]#O#P'V#P;'S&];'S;=`'n<%lO&]Y'[SiYOY'VZ;'S'V;'S;=`'h<%lO'VY'kP;=`<%l'V^'qP;=`<%l&]~'yO#T~~(OO#R~U(VU|S!}QOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U(pU|S#]QOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U)XW|SOt#{uw#{x!Q#{!Q![)q![#O#{#P;'S#{;'S;=`$d<%lO#{U)x[|SuQOt#{uw#{x!O#{!O!P*n!P!Q#{!Q![)q![#O#{#P#R#{#R#S)S#S;'S#{;'S;=`$d<%lO#{U*sW|SOt#{uw#{x!Q#{!Q![+]![#O#{#P;'S#{;'S;=`$d<%lO#{U+dY|SuQOt#{uw#{x!Q#{!Q![+]![#O#{#P#R#{#R#S*n#S;'S#{;'S;=`$d<%lO#{U,X^|SOt#{uw#{x}#{}!O-T!O!Q#{!Q![)q![!_#{!_!`.O!`#O#{#P#T#{#T#o-T#o;'S#{;'S;=`$d<%lO#{U-Y[|SOt#{uw#{x}#{}!O-T!O!_#{!_!`.O!`#O#{#P#T#{#T#o-T#o;'S#{;'S;=`$d<%lO#{U.VUzQ|SOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U.nW|SOt#{uw#{x!P#{!P!Q/W!Q#O#{#P;'S#{;'S;=`$d<%lO#{U/]^|SOY0XYZ#{Zt0Xtu1[uw0Xwx1[x!P0X!P!Q#{!Q!}0X!}#O5}#O#P3j#P;'S0X;'S;=`7O<%lO0XU0`^|S!aQOY0XYZ#{Zt0Xtu1[uw0Xwx1[x!P0X!P!Q4P!Q!}0X!}#O5}#O#P3j#P;'S0X;'S;=`7O<%lO0XQ1aX!aQOY1[Z!P1[!P!Q1|!Q!}1[!}#O2k#O#P3j#P;'S1[;'S;=`3y<%lO1[Q2PP!P!Q2SQ2XU!aQ#Z#[2S#]#^2S#a#b2S#g#h2S#i#j2S#m#n2SQ2nVOY2kZ#O2k#O#P3T#P#Q1[#Q;'S2k;'S;=`3d<%lO2kQ3WSOY2kZ;'S2k;'S;=`3d<%lO2kQ3gP;=`<%l2kQ3mSOY1[Z;'S1[;'S;=`3y<%lO1[Q3|P;=`<%l1[U4UW|SOt#{uw#{x!P#{!P!Q4n!Q#O#{#P;'S#{;'S;=`$d<%lO#{U4ub|S!aQOt#{uw#{x#O#{#P#Z#{#Z#[4n#[#]#{#]#^4n#^#a#{#a#b4n#b#g#{#g#h4n#h#i#{#i#j4n#j#m#{#m#n4n#n;'S#{;'S;=`$d<%lO#{U6S[|SOY5}YZ#{Zt5}tu2kuw5}wx2kx#O5}#O#P3T#P#Q0X#Q;'S5};'S;=`6x<%lO5}U6{P;=`<%l5}U7RP;=`<%l0XU7]U|S!RQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U7vW#_Q|SOt#{uw#{x!_#{!_!`8`!`#O#{#P;'S#{;'S;=`$d<%lO#{U8eV|SOt#{uw#{x#O#{#P#Q8z#Q;'S#{;'S;=`$d<%lO#{U9RU#^Q|SOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~9jO#U~U9qU#`Q|SOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U:[U|S!YQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U:s]|SOt#{uw#{x}#{}!O-T!O!_#{!_!`.O!`#O#{#P#T#{#T#U;l#U#o-T#o;'S#{;'S;=`$d<%lO#{U;q^|SOt#{uw#{x}#{}!O-T!O!_#{!_!`.O!`#O#{#P#T#{#T#`-T#`#ao#Y#o-T#o;'S#{;'S;=`$d<%lO#{U>v[!PQ|SOt#{uw#{x}#{}!O-T!O!_#{!_!`.O!`#O#{#P#T#{#T#o-T#o;'S#{;'S;=`$d<%lO#{^?s[#VW|SOt#{uw#{x}#{}!O-T!O!_#{!_!`.O!`#O#{#P#T#{#T#o-T#o;'S#{;'S;=`$d<%lO#{^@p[#XW|SOt#{uw#{x}#{}!O-T!O!_#{!_!`.O!`#O#{#P#T#{#T#o-T#o;'S#{;'S;=`$d<%lO#{^Am^#WW|SOt#{uw#{x}#{}!O-T!O!_#{!_!`.O!`#O#{#P#T#{#T#f-T#f#gBi#g#o-T#o;'S#{;'S;=`$d<%lO#{UBn^|SOt#{uw#{x}#{}!O-T!O!_#{!_!`.O!`#O#{#P#T#{#T#i-T#i#j=n#j#o-T#o;'S#{;'S;=`$d<%lO#{UCqUlQ|SOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~DYO#a~", tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, new LocalTokenGroup("[~RP!O!PU~ZO#P~~", 11)], topRules: {"Program":[0,26]}, specialized: [{term: 20, get: (value: any, stack: any) => (specializeKeyword(value, stack) << 1), external: specializeKeyword},{term: 20, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}], diff --git a/src/parser/tests/basics.test.ts b/src/parser/tests/basics.test.ts index dea9d7b..0635cf8 100644 --- a/src/parser/tests/basics.test.ts +++ b/src/parser/tests/basics.test.ts @@ -370,6 +370,46 @@ describe('Parentheses', () => { }) }) +describe('Number literals', () => { + test('allows underscores in integer literals', () => { + expect('10_000').toMatchTree(`Number 10_000`) + expect('1_000_000').toMatchTree(`Number 1_000_000`) + expect('100_000').toMatchTree(`Number 100_000`) + }) + + test('allows underscores in decimal literals', () => { + expect('3.14_159').toMatchTree(`Number 3.14_159`) + expect('1_000.50').toMatchTree(`Number 1_000.50`) + expect('0.000_001').toMatchTree(`Number 0.000_001`) + }) + + test('allows underscores in negative numbers', () => { + expect('-10_000').toMatchTree(`Number -10_000`) + expect('-3.14_159').toMatchTree(`Number -3.14_159`) + }) + + test('allows underscores in positive numbers with explicit sign', () => { + expect('+10_000').toMatchTree(`Number +10_000`) + expect('+3.14_159').toMatchTree(`Number +3.14_159`) + }) + + test('works in expressions', () => { + expect('1_000 + 2_000').toMatchTree(` + BinOp + Number 1_000 + Plus + + Number 2_000`) + }) + + test('works in function calls', () => { + expect('echo 10_000').toMatchTree(` + FunctionCall + Identifier echo + PositionalArg + Number 10_000`) + }) +}) + describe('BinOp', () => { test('addition tests', () => { expect('2 + 3').toMatchTree(`