Merge pull request 'Allow _ in numbers (10_000_000)' (#34) from underscore-in-numbers into main

Reviewed-on: #34

src/parser/shrimp.grammar

@@ -13,7 +13,7 @@
 
   StringFragment { !['\\$]+ }
   NamedArgPrefix { $[a-z] $[a-z0-9-]* "=" }
-  Number { ("-" | "+")? $[0-9]+ ('.' $[0-9]+)? }
+  Number { ("-" | "+")? $[0-9]+ ("_"? $[0-9]+)* ('.' $[0-9]+ ("_"? $[0-9]+)*)? }
   Boolean { "true" | "false" }
   newlineOrSemicolon { "\n" | ";" }
   eof { @eof }

src/parser/shrimp.ts

@@ -19,7 +19,7 @@ export const parser = LRParser.deserialize({
   propSources: [highlighting],
   skippedNodes: [0,25],
   repeatNodeCount: 12,
-  tokenData: "Cl~R|OX#{XY$jYZ%TZp#{pq$jqs#{st%ntu'Vuw#{wx'[xy'ayz'zz{#{{|(e|}#{}!O(e!O!P#{!P!Q+X!Q![)S![!]3t!]!^%T!^!}#{!}#O4_#O#P6T#P#Q6Y#Q#R#{#R#S6s#S#T#{#T#Y7^#Y#Z8x#Z#b7^#b#c>f#c#f7^#f#g?i#g#h7^#h#i@l#i#o7^#o#p#{#p#qB|#q;'S#{;'S;=`$d<%l~#{~O#{~~CgS$QUzSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{S$gP;=`<%l#{^$qUzS!zYOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U%[UzS#QQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{^%uZiYzSOY%nYZ#{Zt%ntu&huw%nwx&hx#O%n#O#P&h#P;'S%n;'S;=`'P<%lO%nY&mSiYOY&hZ;'S&h;'S;=`&y<%lO&hY&|P;=`<%l&h^'SP;=`<%l%n~'[O#V~~'aO#T~U'hUzS#PQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U(RUzS#_QOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U(jWzSOt#{uw#{x!Q#{!Q![)S![#O#{#P;'S#{;'S;=`$d<%lO#{U)ZYzSsQOt#{uw#{x!O#{!O!P)y!P!Q#{!Q![)S![#O#{#P;'S#{;'S;=`$d<%lO#{U*OWzSOt#{uw#{x!Q#{!Q![*h![#O#{#P;'S#{;'S;=`$d<%lO#{U*oWzSsQOt#{uw#{x!Q#{!Q![*h![#O#{#P;'S#{;'S;=`$d<%lO#{U+^WzSOt#{uw#{x!P#{!P!Q+v!Q#O#{#P;'S#{;'S;=`$d<%lO#{U+{^zSOY,wYZ#{Zt,wtu-zuw,wwx-zx!P,w!P!Q#{!Q!},w!}#O2m#O#P0Y#P;'S,w;'S;=`3n<%lO,wU-O^zS!bQOY,wYZ#{Zt,wtu-zuw,wwx-zx!P,w!P!Q0o!Q!},w!}#O2m#O#P0Y#P;'S,w;'S;=`3n<%lO,wQ.PX!bQOY-zZ!P-z!P!Q.l!Q!}-z!}#O/Z#O#P0Y#P;'S-z;'S;=`0i<%lO-zQ.oP!P!Q.rQ.wU!bQ#Z#[.r#]#^.r#a#b.r#g#h.r#i#j.r#m#n.rQ/^VOY/ZZ#O/Z#O#P/s#P#Q-z#Q;'S/Z;'S;=`0S<%lO/ZQ/vSOY/ZZ;'S/Z;'S;=`0S<%lO/ZQ0VP;=`<%l/ZQ0]SOY-zZ;'S-z;'S;=`0i<%lO-zQ0lP;=`<%l-zU0tWzSOt#{uw#{x!P#{!P!Q1^!Q#O#{#P;'S#{;'S;=`$d<%lO#{U1ebzS!bQOt#{uw#{x#O#{#P#Z#{#Z#[1^#[#]#{#]#^1^#^#a#{#a#b1^#b#g#{#g#h1^#h#i#{#i#j1^#j#m#{#m#n1^#n;'S#{;'S;=`$d<%lO#{U2r[zSOY2mYZ#{Zt2mtu/Zuw2mwx/Zx#O2m#O#P/s#P#Q,w#Q;'S2m;'S;=`3h<%lO2mU3kP;=`<%l2mU3qP;=`<%l,wU3{UzS!PQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U4fW#aQzSOt#{uw#{x!_#{!_!`5O!`#O#{#P;'S#{;'S;=`$d<%lO#{U5TVzSOt#{uw#{x#O#{#P#Q5j#Q;'S#{;'S;=`$d<%lO#{U5qU#`QzSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~6YO#W~U6aU#bQzSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U6zUzS!WQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U7c^zSOt#{uw#{x}#{}!O7^!O!Q#{!Q![7^![!_#{!_!`8_!`#O#{#P#T#{#T#o7^#o;'S#{;'S;=`$d<%lO#{U8fUxQzSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U8}_zSOt#{uw#{x}#{}!O7^!O!Q#{!Q![7^![!_#{!_!`8_!`#O#{#P#T#{#T#U9|#U#o7^#o;'S#{;'S;=`$d<%lO#{U:R`zSOt#{uw#{x}#{}!O7^!O!Q#{!Q![7^![!_#{!_!`8_!`#O#{#P#T#{#T#`7^#`#a;T#a#o7^#o;'S#{;'S;=`$d<%lO#{U;Y`zSOt#{uw#{x}#{}!O7^!O!Q#{!Q![7^![!_#{!_!`8_!`#O#{#P#T#{#T#g7^#g#h<[#h#o7^#o;'S#{;'S;=`$d<%lO#{U<a`zSOt#{uw#{x}#{}!O7^!O!Q#{!Q![7^![!_#{!_!`8_!`#O#{#P#T#{#T#X7^#X#Y=c#Y#o7^#o;'S#{;'S;=`$d<%lO#{U=j^}QzSOt#{uw#{x}#{}!O7^!O!Q#{!Q![7^![!_#{!_!`8_!`#O#{#P#T#{#T#o7^#o;'S#{;'S;=`$d<%lO#{^>m^#XWzSOt#{uw#{x}#{}!O7^!O!Q#{!Q![7^![!_#{!_!`8_!`#O#{#P#T#{#T#o7^#o;'S#{;'S;=`$d<%lO#{^?p^#ZWzSOt#{uw#{x}#{}!O7^!O!Q#{!Q![7^![!_#{!_!`8_!`#O#{#P#T#{#T#o7^#o;'S#{;'S;=`$d<%lO#{^@s`#YWzSOt#{uw#{x}#{}!O7^!O!Q#{!Q![7^![!_#{!_!`8_!`#O#{#P#T#{#T#f7^#f#gAu#g#o7^#o;'S#{;'S;=`$d<%lO#{UAz`zSOt#{uw#{x}#{}!O7^!O!Q#{!Q![7^![!_#{!_!`8_!`#O#{#P#T#{#T#i7^#i#j<[#j#o7^#o;'S#{;'S;=`$d<%lO#{UCTUlQzSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~ClO#c~",
+  tokenData: "Cx~R|OX#{XY$jYZ%TZp#{pq$jqs#{st%ntu'Vuw#{wx'[xy'ayz'zz{#{{|(e|}#{}!O(e!O!P#{!P!Q+e!Q![)S![!]4Q!]!^%T!^!}#{!}#O4k#O#P6a#P#Q6f#Q#R#{#R#S7P#S#T#{#T#Y7j#Y#Z9U#Z#b7j#b#c>r#c#f7j#f#g?u#g#h7j#h#i@x#i#o7j#o#p#{#p#qCY#q;'S#{;'S;=`$d<%l~#{~O#{~~CsS$QUzSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{S$gP;=`<%l#{^$qUzS!zYOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U%[UzS#QQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{^%uZiYzSOY%nYZ#{Zt%ntu&huw%nwx&hx#O%n#O#P&h#P;'S%n;'S;=`'P<%lO%nY&mSiYOY&hZ;'S&h;'S;=`&y<%lO&hY&|P;=`<%l&h^'SP;=`<%l%n~'[O#V~~'aO#T~U'hUzS#PQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U(RUzS#_QOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U(jWzSOt#{uw#{x!Q#{!Q![)S![#O#{#P;'S#{;'S;=`$d<%lO#{U)Z[zSsQOt#{uw#{x!O#{!O!P*P!P!Q#{!Q![)S![#O#{#P#R#{#R#S(e#S;'S#{;'S;=`$d<%lO#{U*UWzSOt#{uw#{x!Q#{!Q![*n![#O#{#P;'S#{;'S;=`$d<%lO#{U*uYzSsQOt#{uw#{x!Q#{!Q![*n![#O#{#P#R#{#R#S*P#S;'S#{;'S;=`$d<%lO#{U+jWzSOt#{uw#{x!P#{!P!Q,S!Q#O#{#P;'S#{;'S;=`$d<%lO#{U,X^zSOY-TYZ#{Zt-Ttu.Wuw-Twx.Wx!P-T!P!Q#{!Q!}-T!}#O2y#O#P0f#P;'S-T;'S;=`3z<%lO-TU-[^zS!bQOY-TYZ#{Zt-Ttu.Wuw-Twx.Wx!P-T!P!Q0{!Q!}-T!}#O2y#O#P0f#P;'S-T;'S;=`3z<%lO-TQ.]X!bQOY.WZ!P.W!P!Q.x!Q!}.W!}#O/g#O#P0f#P;'S.W;'S;=`0u<%lO.WQ.{P!P!Q/OQ/TU!bQ#Z#[/O#]#^/O#a#b/O#g#h/O#i#j/O#m#n/OQ/jVOY/gZ#O/g#O#P0P#P#Q.W#Q;'S/g;'S;=`0`<%lO/gQ0SSOY/gZ;'S/g;'S;=`0`<%lO/gQ0cP;=`<%l/gQ0iSOY.WZ;'S.W;'S;=`0u<%lO.WQ0xP;=`<%l.WU1QWzSOt#{uw#{x!P#{!P!Q1j!Q#O#{#P;'S#{;'S;=`$d<%lO#{U1qbzS!bQOt#{uw#{x#O#{#P#Z#{#Z#[1j#[#]#{#]#^1j#^#a#{#a#b1j#b#g#{#g#h1j#h#i#{#i#j1j#j#m#{#m#n1j#n;'S#{;'S;=`$d<%lO#{U3O[zSOY2yYZ#{Zt2ytu/guw2ywx/gx#O2y#O#P0P#P#Q-T#Q;'S2y;'S;=`3t<%lO2yU3wP;=`<%l2yU3}P;=`<%l-TU4XUzS!PQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U4rW#aQzSOt#{uw#{x!_#{!_!`5[!`#O#{#P;'S#{;'S;=`$d<%lO#{U5aVzSOt#{uw#{x#O#{#P#Q5v#Q;'S#{;'S;=`$d<%lO#{U5}U#`QzSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~6fO#W~U6mU#bQzSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U7WUzS!WQOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U7o^zSOt#{uw#{x}#{}!O7j!O!Q#{!Q![7j![!_#{!_!`8k!`#O#{#P#T#{#T#o7j#o;'S#{;'S;=`$d<%lO#{U8rUxQzSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{U9Z_zSOt#{uw#{x}#{}!O7j!O!Q#{!Q![7j![!_#{!_!`8k!`#O#{#P#T#{#T#U:Y#U#o7j#o;'S#{;'S;=`$d<%lO#{U:_`zSOt#{uw#{x}#{}!O7j!O!Q#{!Q![7j![!_#{!_!`8k!`#O#{#P#T#{#T#`7j#`#a;a#a#o7j#o;'S#{;'S;=`$d<%lO#{U;f`zSOt#{uw#{x}#{}!O7j!O!Q#{!Q![7j![!_#{!_!`8k!`#O#{#P#T#{#T#g7j#g#h<h#h#o7j#o;'S#{;'S;=`$d<%lO#{U<m`zSOt#{uw#{x}#{}!O7j!O!Q#{!Q![7j![!_#{!_!`8k!`#O#{#P#T#{#T#X7j#X#Y=o#Y#o7j#o;'S#{;'S;=`$d<%lO#{U=v^}QzSOt#{uw#{x}#{}!O7j!O!Q#{!Q![7j![!_#{!_!`8k!`#O#{#P#T#{#T#o7j#o;'S#{;'S;=`$d<%lO#{^>y^#XWzSOt#{uw#{x}#{}!O7j!O!Q#{!Q![7j![!_#{!_!`8k!`#O#{#P#T#{#T#o7j#o;'S#{;'S;=`$d<%lO#{^?|^#ZWzSOt#{uw#{x}#{}!O7j!O!Q#{!Q![7j![!_#{!_!`8k!`#O#{#P#T#{#T#o7j#o;'S#{;'S;=`$d<%lO#{^AP`#YWzSOt#{uw#{x}#{}!O7j!O!Q#{!Q![7j![!_#{!_!`8k!`#O#{#P#T#{#T#f7j#f#gBR#g#o7j#o;'S#{;'S;=`$d<%lO#{UBW`zSOt#{uw#{x}#{}!O7j!O!Q#{!Q![7j![!_#{!_!`8k!`#O#{#P#T#{#T#i7j#i#j<h#j#o7j#o;'S#{;'S;=`$d<%lO#{UCaUlQzSOt#{uw#{x#O#{#P;'S#{;'S;=`$d<%lO#{~CxO#c~",
   tokenizers: [operatorTokenizer, 1, 2, 3, tokenizer, new LocalTokenGroup("[~RP!O!PU~ZO#R~~", 11)],
   topRules: {"Program":[0,26]},
   specialized: [{term: 20, get: (value: any, stack: any) => (specializeKeyword(value, stack) << 1), external: specializeKeyword},{term: 20, get: (value: keyof typeof spec_Identifier) => spec_Identifier[value] || -1}],

src/parser/tests/basics.test.ts

@@ -442,6 +442,46 @@ describe('Parentheses', () => {
   })
 })
 
+describe('Number literals', () => {
+  test('allows underscores in integer literals', () => {
+    expect('10_000').toMatchTree(`Number 10_000`)
+    expect('1_000_000').toMatchTree(`Number 1_000_000`)
+    expect('100_000').toMatchTree(`Number 100_000`)
+  })
+
+  test('allows underscores in decimal literals', () => {
+    expect('3.14_159').toMatchTree(`Number 3.14_159`)
+    expect('1_000.50').toMatchTree(`Number 1_000.50`)
+    expect('0.000_001').toMatchTree(`Number 0.000_001`)
+  })
+
+  test('allows underscores in negative numbers', () => {
+    expect('-10_000').toMatchTree(`Number -10_000`)
+    expect('-3.14_159').toMatchTree(`Number -3.14_159`)
+  })
+
+  test('allows underscores in positive numbers with explicit sign', () => {
+    expect('+10_000').toMatchTree(`Number +10_000`)
+    expect('+3.14_159').toMatchTree(`Number +3.14_159`)
+  })
+
+  test('works in expressions', () => {
+    expect('1_000 + 2_000').toMatchTree(`
+      BinOp
+        Number 1_000
+        Plus +
+        Number 2_000`)
+  })
+
+  test('works in function calls', () => {
+    expect('echo 10_000').toMatchTree(`
+      FunctionCall
+        Identifier echo
+        PositionalArg
+          Number 10_000`)
+  })
+})
+
 describe('BinOp', () => {
   test('addition tests', () => {
     expect('2 + 3').toMatchTree(`