wip

src/parser/contextTracker.ts

@@ -0,0 +1,26 @@
+import { ContextTracker } from '@lezer/lr'
+import { Assignment } from '#parser/shrimp.terms'
+
+interface ParserContext {
+  definedVariables: Set<string>
+}
+
+export const contextTracker = new ContextTracker<ParserContext>({
+  start: { definedVariables: new Set() },
+
+  reduce(context, term, stack, input) {
+    console.log(`🤏 REDUCE`, termToString(term))
+    if (term !== Assignment) return context
+
+    return context
+  },
+
+  shift(context, term, stack, input) {
+    console.log(`  ⇧ SHIFT `, termToString(term))
+    return context
+  },
+})
+
+const termToString = (term: number) => {
+  return Object.entries(require('./shrimp.terms')).find(([k, v]) => v === term)?.[0] || term
+}

src/parser/old-shrimp.grammar

@@ -1,79 +0,0 @@
-@external propSource highlighting from "./highlight.js"
-@top Program { line* }
-
-line { 
-  CommandCall semi |
-  expr semi
-}
-
-@skip { space }
-
-@tokens {
-  @precedence { Number "-"}
-  space { @whitespace+ }
-  Number { "-"? $[0-9]+ ('.' $[0-9]+)? }
-  Boolean { "true" | "false" }
-  String { '\'' !["]* '\'' }
-  NamedArgPrefix { $[a-z]+ $[a-z0-9\-]* "=" }  // matches "lines=", "follow=", etc.
-
-  fn[@name=keyword] { "fn" }
-  equals[@name=operator] { "=" }
-  ":"[@name=colon]
-  "+"[@name=operator]
-  "-"[@name=operator]
-  "*"[@name=operator]
-  "/"[@name=operator]
-  leftParen[@name=paren] { "(" }
-  rightParen[@name=paren] { ")" }
-}
-
-@external tokens tokenizer from "./tokenizers" { 
-  Identifier,
-  Command,
-  CommandPartial
-}
-
-@external tokens argTokenizer from "./tokenizers" {
-  UnquotedArg
-}
-
-@external tokens insertSemicolon from "./tokenizers" { insertedSemi }
-
-@precedence {
-  multiplicative @left,
-  additive @left,
-  namedComplete @left,
-  function @right
-  assignment @right
-}
-
-expr {
-  Assignment |
-  Function |
-  BinOp |
-  atom
-}
-
-semi { insertedSemi | ";" }
-
-argValue { atom | UnquotedArg }
-
-CommandCall { (Command | CommandPartial) (NamedArg | PartialNamedArg | Arg)* }
-Arg { !namedComplete argValue }
-NamedArg { NamedArgPrefix !namedComplete argValue }  // Required atom, higher precedence
-PartialNamedArg { NamedArgPrefix }  // Just the prefix
-
-Assignment { Identifier !assignment equals expr }
-
-Function { !function fn Params ":" expr }
-Params { Identifier* }
-
-BinOp { 
-  expr !multiplicative "*" expr |
-  expr !multiplicative "/" expr |
-  expr !additive "+" expr |
-  expr !additive "-" expr
-}
-
-ParenExpr { leftParen expr rightParen }
-atom { Identifier ~command | Number | String | Boolean | ParenExpr }

src/parser/shrimp.grammar

@@ -17,7 +17,6 @@
   rightParen { ")" }
   ":"
   "fn"
-  "do"
   "end"
   "="
   "+"[@name=operator]
@@ -73,7 +72,15 @@ NamedArg {
 }
 
 FunctionDef {
-  "fn" Params ":" "do" expression "end"
+  singleLineFunctionDef | multiLineFunctionDef
+}
+
+singleLineFunctionDef {
+  "fn" Params ":" expression "end"
+}
+
+multiLineFunctionDef {
+  "fn" Params ":" newline (expression newline)* "end"
 }
 
 Params {

src/parser/shrimp.terms.ts

@@ -15,4 +15,4 @@ export const
   NamedArgPrefix = 17,
   FunctionDef = 18,
   Params = 20,
-  Assignment = 24
+  Assignment = 23

src/parser/shrimp.test.ts

@@ -192,24 +192,22 @@ describe('BinOp', () => {
 
 describe('Fn', () => {
   test('parses function no parameters', () => {
-    expect('fn: do 1 end').toMatchTree(`
+    expect('fn: 1 end').toMatchTree(`
       FunctionDef
         fn fn
         Params 
         : :
-        do do
         Number 1
         end end`)
   })
 
   test('parses function with single parameter', () => {
-    expect('fn x: do x + 1 end').toMatchTree(`
+    expect('fn x: x + 1 end').toMatchTree(`
       FunctionDef
         fn fn
         Params
           Identifier x
         : :
-        do do
         BinOp
           Identifier x
           operator +
@@ -218,23 +216,44 @@ describe('Fn', () => {
   })
 
   test('parses function with multiple parameters', () => {
-    expect('fn x y: do x * y end').toMatchTree(`
+    expect('fn x y: x * y end').toMatchTree(`
       FunctionDef
         fn fn
         Params
           Identifier x
           Identifier y
         : :
-        do do
         BinOp
           Identifier x
           operator *
           Identifier y
         end end`)
   })
+
+  test('parses multiline function with multiple statements', () => {
+    expect(`fn x y:
+  x * y
+  x + 9
+end`).toMatchTree(`
+      FunctionDef
+        fn fn
+        Params
+          Identifier x
+          Identifier y
+        : :
+        BinOp
+          Identifier x
+          operator *
+          Identifier y
+        BinOp
+          Identifier x
+          operator +
+          Number 9
+        end end`)
+  })
 })
 
-describe('abmiguity', () => {
+describe('ambiguity', () => {
   test('parses ambiguous expressions correctly', () => {
     expect('a + -3').toMatchTree(`
       BinOp
@@ -267,7 +286,7 @@ describe('Assignment', () => {
   })
 
   test('parses assignment with functions', () => {
-    expect('add = fn a b: do a + b end').toMatchTree(`
+    expect('add = fn a b: a + b end').toMatchTree(`
       Assignment
         Identifier add
         = =
@@ -277,7 +296,6 @@ describe('Assignment', () => {
             Identifier a
             Identifier b
           : :
-          do do
           BinOp
             Identifier a
             operator +

src/parser/shrimp.ts

@@ -4,16 +4,16 @@ import {tokenizer} from "./tokenizers"
 import {highlighting} from "./highlight.js"
 export const parser = LRParser.deserialize({
   version: 14,
-  states: "&YQVQTOOOnQPO'#DSO!tQUO'#DSO#OQPOOOOQO'#DR'#DRO#oQTO'#CbOOQS'#DP'#DPO#vQTO'#CnOOQO'#C|'#C|O$OQPO'#CvQVQTOOOOQS'#DO'#DOOOQS'#Ca'#CaO$TQTO'#ClOOQS'#C}'#C}OOQS'#Cw'#CwO$[QUO,58zOVQTO,59`O$lQTO,58}O$lQTO,58}O$sQPO,58|O%UQUO'#DSO%]QPO,58|OOQS'#Cx'#CxO%bQTO'#CpO%jQPO,59YOOQS,59b,59bOOQS-E6t-E6tOOQS,59W,59WOOQS-E6u-E6uOOQO1G.z1G.zOOQO'#DS'#DSOOQO1G.i1G.iO%oQPO1G.iOOQS1G.h1G.hOOQS-E6v-E6vO&WQPO1G.tO$lQTO7+$`O&]QPO<<GzOOQOAN=fAN=f",
+  states: "'UQVQTOOOnQPO'#DSO!tQUO'#DSO#OQPOOOOQO'#DR'#DRO#oQTO'#CbOOQS'#DP'#DPO#vQTO'#DUOOQO'#Cn'#CnOOQO'#C|'#C|O$OQPO'#CuQVQTOOOOQS'#DO'#DOOOQS'#Ca'#CaO$TQTO'#ClOOQS'#C}'#C}OOQS'#Cv'#CvO$[QUO,58zOVQTO,59_O$lQTO,58}O$lQTO,58}O$sQPO,58|O%UQUO'#DSO%]QPO,58|OOQS'#Cw'#CwO%bQTO'#CpO%jQPO,59pOOQS,59a,59aOOQS-E6s-E6sOOQS,59W,59WOOQS-E6t-E6tOOQO1G.y1G.yOOQO'#DS'#DSOOQO1G.i1G.iO%oQPO1G.iOOQS1G.h1G.hOOQS-E6u-E6uO&WQTO1G/[O&bQPO7+$vO&sQTO7+$wOOQO<<Hb<<HbO&}QPO'#CxO'`QTO<<HcOOQO<<Hc<<HcOOQS,59d,59dOOQS-E6v-E6vOOQOAN=}AN=}",
-  stateData: "&q~OoOS~OPQOQUO]UO^UO_UOcVOtTO~OWvXXvXYvXZvXxpX~OPZOQUO]UO^UO_UOa]OtTOWvXXvXYvXZvX~OiaOx[X~P!POWbOXbOYcOZcO~OQUO]UO^UO_UOtTO~OPeO~P#^OPgOedP~OxjO~OPZO~P#^OPZOa]OxSawSa~P#^OPoO~P#^OwrOWvXXvXYvXZvX~Ow[X~P!POwrO~OPgOedX~OetO~OWbOXbOYViZVixViwVigVi~OfuO~OWbOXbOYcOZcOgwO~O^Z~",
+  stateData: "'m~OoOS~OPQOQUO]UO^UO_UOcVOtTO~OWvXXvXYvXZvXzpX~OP[OQUO]UO^UO_UOa^OtTOWvXXvXYvXZvX~OhbOz[X~P!POWcOXcOYdOZdO~OQUO]UO^UO_UOtTO~OPfO~P#^OPhOedP~OzkO~OP[O~P#^OP[Oa^OzSawSa~P#^OPpO~P#^OwsOWvXXvXYvXZvX~Ow[X~P!POwsO~OPhOedX~OeuO~OWcOXcOYViZVizViwVifVi~OPpOzwO~P#^OWcOXcOYdOZdOfxO~OPpOf{O~P#^OWcOXcOYdOZdOz|O~OPpOf!OO~P#^O^Z~",
-  goto: "$kwPPPPx!Q!V!dPPPPxPPP!QP!mP!rPPP!mP!u!{#SPPP#Y#a#f#nP#}$[UWOYaRfTV^Q`egUOQTY]`abceu_SOTYabcuVWOYaRiVQYORkYS`QeRm`QhVRshSXOYRnaV_Q`eU[Q`eRl]^SOTYabcuXZQ]`eUPOYaQdTVobcuWROTYaQpbQqcRvu",
+  goto: "%VzPPPP{!T!Y!iPPPP{PPP!TP!tP!yPP!tP!|#S#Z#aPPP#g#n#s#{P$^$mP%Q%QUXOZbRgTV_QafkUOQTZ^abcdfuwzcSOTZbcduwzVXOZbRjVQZORlZSaQfRnaQiVRtiQzwR}zSYOZRobV`QafU]QafRm^bSOTZbcduwzX[Q^afUPOZbQeTZpcduwzWROTZbQqcQrdQvuTywzVWOZb",
-  nodeNames: "⚠ Identifier Word Program FunctionCall PositionalArg ParenExpr BinOp operator operator operator operator FunctionCallOrIdentifier String Number Boolean NamedArg NamedArgPrefix FunctionDef fn Params : do end Assignment =",
+  nodeNames: "⚠ Identifier Word Program FunctionCall PositionalArg ParenExpr BinOp operator operator operator operator FunctionCallOrIdentifier String Number Boolean NamedArg NamedArgPrefix FunctionDef fn Params : end Assignment =",
-  maxTerm: 40,
+  maxTerm: 42,
   propSources: [highlighting],
   skippedNodes: [0],
-  repeatNodeCount: 3,
+  repeatNodeCount: 4,
-  tokenData: ")c~RdYZ!apq!fwx!kxy#oyz#tz{#y{|$O}!O$T!P!Q$v!Q![$]![!]${!_!`%Q#T#W%V#W#X%e#X#Y&P#Y#Z&z#Z#h%V#h#i(s#i#o%V~~!a~!fOx~~!kOo~~!nUOr!ksw!kwx#Qx;'S!k;'S;=`#i<%lO!k~#VU]~Or!ksw!kwx#Qx;'S!k;'S;=`#i<%lO!k~#lP;=`<%l!k~#tOt~~#yOw~~$OOW~~$TOY~~$YPZ~!Q![$]~$bQ^~!O!P$h!Q![$]~$kP!Q![$n~$sP^~!Q![$n~${OX~~%QOe~~%VOi~Q%YQ!_!`%`#T#o%VQ%eOaQR%hS!_!`%`#T#c%V#c#d%t#d#o%VR%yQfP!_!`%`#T#o%VR&SS!_!`%`#T#b%V#b#c&`#c#o%VR&cS!_!`%`#T#W%V#W#X&o#X#o%VR&tQgP!_!`%`#T#o%V~&}T!_!`%`#T#U'^#U#b%V#b#c(h#c#o%V~'aS!_!`%`#T#`%V#`#a'm#a#o%V~'pS!_!`%`#T#g%V#g#h'|#h#o%V~(PS!_!`%`#T#X%V#X#Y(]#Y#o%V~(bQ_~!_!`%`#T#o%VR(mQcP!_!`%`#T#o%V~(vS!_!`%`#T#f%V#f#g)S#g#o%V~)VS!_!`%`#T#i%V#i#j'|#j#o%V",
+  tokenData: "(t~RcYZ!^pq!cwx!hxy#lyz#qz{#v{|#{}!O$Q!P!Q$s!Q![$Y![!]$x!_!`$}#T#X%S#X#Y%b#Y#Z&]#Z#h%S#h#i(U#i#o%S~~!^~!cOz~~!hOo~~!kUOr!hsw!hwx!}x;'S!h;'S;=`#f<%lO!h~#SU]~Or!hsw!hwx!}x;'S!h;'S;=`#f<%lO!h~#iP;=`<%l!h~#qOt~~#vOw~~#{OW~~$QOY~~$VPZ~!Q![$Y~$_Q^~!O!P$e!Q![$Y~$hP!Q![$k~$pP^~!Q![$k~$xOX~~$}Oe~~%SOh~Q%VQ!_!`%]#T#o%SQ%bOaQR%eS!_!`%]#T#b%S#b#c%q#c#o%SR%tS!_!`%]#T#W%S#W#X&Q#X#o%SR&VQfP!_!`%]#T#o%S~&`T!_!`%]#T#U&o#U#b%S#b#c'y#c#o%S~&rS!_!`%]#T#`%S#`#a'O#a#o%S~'RS!_!`%]#T#g%S#g#h'_#h#o%S~'bS!_!`%]#T#X%S#X#Y'n#Y#o%S~'sQ_~!_!`%]#T#o%SR(OQcP!_!`%]#T#o%S~(XS!_!`%]#T#f%S#f#g(e#g#o%S~(hS!_!`%]#T#i%S#i#j'_#j#o%S",
   tokenizers: [0, 1, tokenizer],
   topRules: {"Program":[0,3]},
-  tokenPrec: 260
+  tokenPrec: 302
 })