make more compiler tests pass

src/parser/node.ts

@@ -58,6 +58,7 @@ export type NodeType =
 
   | 'Import'
   | 'Do'
+  | 'Underscore'
   | 'colon'
   | 'keyword'
   | 'operator'
@@ -272,6 +273,9 @@ class SyntaxNodeType {
       case 'Do':
         return term.Do
 
+      case 'Underscore':
+        return term.Underscore
+
       case 'colon':
         return term.colon
 
@@ -355,7 +359,7 @@ export class SyntaxNode {
   }
 
   toString(): string {
-    return this.type
+    return this.type.name
   }
 }
 

src/parser/parser2.ts

@@ -341,7 +341,7 @@ export class Parser {
     if (this.is($T.String))
       return this.string()
 
-    if (this.isAny($T.Null, $T.Boolean, $T.Number, $T.Identifier, $T.Word, $T.Regex))
+    if (this.isAny($T.Null, $T.Boolean, $T.Number, $T.Identifier, $T.Word, $T.Regex, $T.Underscore))
       return SyntaxNode.from(this.next())
 
     const next = this.next()

src/parser/tests/pipes.test.ts

@@ -295,10 +295,10 @@ grep h`).toMatchTree(`
 
   test('lots of pipes', () => {
     expect(`
-'this should help readability in long chains' 
-  | split ' ' 
-  | map (ref str.to-upper) 
-  | join '-' 
+'this should help readability in long chains'
+  | split ' '
+  | map (ref str.to-upper)
+  | join '-'
   | echo
 `).toMatchTree(`
   PipeExpr
@@ -309,7 +309,7 @@ grep h`).toMatchTree(`
       Identifier split
       PositionalArg
         String
-          StringFragment  
+          StringFragment (space)
     operator |
     FunctionCall
       Identifier map
@@ -333,3 +333,41 @@ grep h`).toMatchTree(`
   `)
   })
 })
+
+describe('Underscore', () => {
+  test('works in pipes', () => {
+    expect(`sub 3 1 | div (sub 110 9 | sub 1) _ | div 5`).toMatchTree(`
+      PipeExpr
+        FunctionCall
+          Identifier sub
+          PositionalArg
+            Number 3
+          PositionalArg
+            Number 1
+        operator |
+        FunctionCall
+          Identifier div
+          PositionalArg
+            ParenExpr
+              PipeExpr
+                FunctionCall
+                  Identifier sub
+                  PositionalArg
+                    Number 110
+                  PositionalArg
+                    Number 9
+                operator |
+                FunctionCall
+                  Identifier sub
+                  PositionalArg
+                    Number 1
+          PositionalArg
+            Underscore _
+        operator |
+        FunctionCall
+          Identifier div
+          PositionalArg
+            Number 5
+    `)
+  })
+})

src/parser/tokenizer2.ts

@@ -38,7 +38,8 @@ const valueTokens = new Set([
   TokenType.Comment,
   TokenType.Keyword, TokenType.Operator,
   TokenType.Identifier, TokenType.Word, TokenType.NamedArgPrefix,
-  TokenType.Boolean, TokenType.Number, TokenType.String, TokenType.Regex
+  TokenType.Boolean, TokenType.Number, TokenType.String, TokenType.Regex,
+  TokenType.Underscore
 ])
 
 const operators = new Set([
@@ -337,7 +338,7 @@ export class Scanner {
 
     // classify the token based on what we read
     if (word === '_')
-      this.pushChar(TokenType.Underscore)
+      this.push(TokenType.Underscore)
 
     else if (word === 'null')
       this.push(TokenType.Null)

src/utils/tree.ts

@@ -11,7 +11,8 @@ const nodeToString = (node: SyntaxNode, input: string, depth = 0): string => {
     return `${indent}${nodeName}`
   } else {
     // Only strip quotes from whole String nodes (legacy DoubleQuote), not StringFragment/EscapeSeq/CurlyString
-    const cleanText = nodeName === 'String' ? text.slice(1, -1) : text
+    let cleanText = nodeName === 'String' ? text.slice(1, -1) : text
+    if (cleanText === ' ') cleanText = '(space)'
     return cleanText ? `${indent}${nodeName} ${cleanText}` : `${indent}${nodeName}`
   }
 }