Merge pull request 'Cache the parsing' (#28) from less-parsing into main

Reviewed-on: #28

.gitignore

@@ -34,4 +34,6 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
 .DS_Store
 
 /tmp
-/docs
+/docs
+
+*.vsix

vscode-extension/.gitignore

@@ -1,4 +0,0 @@
-node_modules
-client/dist
-server/dist
-*.vsix

vscode-extension/server/src/diagnostics.ts

@@ -1,12 +1,12 @@
 import { TextDocument, Position } from 'vscode-languageserver-textdocument'
 import { Diagnostic, DiagnosticSeverity } from 'vscode-languageserver/node'
-import { parser } from '../../../src/parser/shrimp'
+import { Tree } from '@lezer/common'
 import { Compiler } from '../../../src/compiler/compiler'
 import { CompilerError } from '../../../src/compiler/compilerError'
 
-export const buildDiagnostics = (textDocument: TextDocument): Diagnostic[] => {
+export const buildDiagnostics = (textDocument: TextDocument, tree: Tree): Diagnostic[] => {
   const text = textDocument.getText()
-  const diagnostics = getParseErrors(textDocument)
+  const diagnostics = getParseErrors(textDocument, tree)
 
   if (diagnostics.length > 0) {
     return diagnostics
@@ -59,9 +59,7 @@ const unknownDiagnostic = (message: string): Diagnostic => {
   return diagnostic
 }
 
-const getParseErrors = (textDocument: TextDocument): Diagnostic[] => {
+const getParseErrors = (textDocument: TextDocument, tree: Tree): Diagnostic[] => {
-  const tree = parser.parse(textDocument.getText())
-
   const ranges: { start: Position; end: Position }[] = []
   tree.iterate({
     enter(n) {

vscode-extension/server/src/semanticTokens.ts

@@ -1,6 +1,6 @@
 import { parser } from '../../../src/parser/shrimp'
 import * as Terms from '../../../src/parser/shrimp.terms'
-import { SyntaxNode } from '@lezer/common'
+import { SyntaxNode, Tree } from '@lezer/common'
 import { TextDocument } from 'vscode-languageserver-textdocument'
 import {
   SemanticTokensBuilder,
@@ -28,9 +28,7 @@ export const TOKEN_MODIFIERS = [
   SemanticTokenModifiers.readonly,
 ]
 
-export function buildSemanticTokens(document: TextDocument): number[] {
+export function buildSemanticTokens(document: TextDocument, tree: Tree): number[] {
-  const text = document.getText()
-  const tree = parser.parse(text)
   const builder = new SemanticTokensBuilder()
   const scopeTracker = new EditorScopeAnalyzer(document)
 

vscode-extension/server/src/server.ts

@@ -7,6 +7,7 @@ import { PRELUDE_NAMES } from './metadata/prelude-names'
 import { parser } from '../../../src/parser/shrimp'
 import { setGlobals } from '../../../src/parser/tokenizer'
 import { Compiler } from '../../../src/compiler/compiler'
+import { Tree } from '@lezer/common'
 import {
   InitializeResult,
   TextDocuments,
@@ -14,7 +15,10 @@ import {
   createConnection,
   ProposedFeatures,
   CompletionItemKind,
+  TextDocumentChangeEvent,
 } from 'vscode-languageserver/node'
+import { setGlobals } from '../../../src/parser/tokenizer'
+import { globals } from '../../../src/prelude'
 
 // Initialize parser with prelude globals so it knows dict/list/str are in scope
 setGlobals(PRELUDE_NAMES)
@@ -23,12 +27,16 @@ const connection = createConnection(ProposedFeatures.all)
 const documents = new TextDocuments(TextDocument)
 documents.listen(connection)
 
+const documentTrees = new Map<string, Tree>()
+
 // Server capabilities
 connection.onInitialize(handleInitialize)
 
 // Language features
 connection.languages.semanticTokens.on(handleSemanticTokens)
+documents.onDidOpen(handleDocumentOpen)
 documents.onDidChangeContent(handleDocumentChange)
+documents.onDidClose(handleDocumentClose)
 connection.onCompletion(handleCompletion)
 connection.onSignatureHelp(handleSignatureHelp)
 
@@ -39,10 +47,7 @@ connection.onRequest('shrimp/bytecode', handleBytecode)
 // Start listening
 connection.listen()
 
-// ============================================================================
 // Handler implementations
-// ============================================================================
-
 function handleInitialize(): InitializeResult {
   connection.console.log('🦐 Server initialized with capabilities')
   const result: InitializeResult = {
@@ -67,22 +72,42 @@ function handleInitialize(): InitializeResult {
   return result
 }
 
+function handleDocumentOpen(event: TextDocumentChangeEvent<TextDocument>) {
+  const document = event.document
+  setGlobals(Object.keys(globals))
+  const tree = parser.parse(document.getText())
+  documentTrees.set(document.uri, tree)
+}
+
 function handleSemanticTokens(params: any) {
   const document = documents.get(params.textDocument.uri)
   if (!document) return { data: [] }
 
-  const data = buildSemanticTokens(document)
+  const tree = documentTrees.get(params.textDocument.uri)
+  if (!tree) return { data: [] }
+
+  const data = buildSemanticTokens(document, tree)
   return { data }
 }
 
-function handleDocumentChange(change: any) {
+function handleDocumentChange(change: TextDocumentChangeEvent<TextDocument>) {
-  const textDocument = change.document
+  const document = change.document
-  const diagnostics = buildDiagnostics(textDocument)
+
-  connection.sendDiagnostics({ uri: textDocument.uri, diagnostics })
+  // Parse and cache
+  setGlobals(Object.keys(globals))
+  const tree = parser.parse(document.getText())
+  documentTrees.set(document.uri, tree)
+
+  // Build diagnostics using cached tree
+  const diagnostics = buildDiagnostics(document, tree)
+  connection.sendDiagnostics({ uri: document.uri, diagnostics })
+}
+
+function handleDocumentClose(event: TextDocumentChangeEvent<TextDocument>) {
+  documentTrees.delete(event.document.uri)
 }
 
 function handleCompletion(params: any) {
-  console.log(`🌭 YOU ARE COMPLETING at position`, params.position)
   const document = documents.get(params.textDocument.uri)
   if (!document) {
     console.log('❌ No document found')
@@ -98,7 +123,10 @@ function handleCompletion(params: any) {
   const contextCompletions = provideCompletions(document, position)
   console.log(`🎯 Context completions count: ${contextCompletions.length}`)
   if (contextCompletions.length > 0) {
-    console.log(`✅ Returning ${contextCompletions.length} completions:`, contextCompletions.map(c => c.label).join(', '))
+    console.log(
+      `✅ Returning ${contextCompletions.length} completions:`,
+      contextCompletions.map((c) => c.label).join(', ')
+    )
     return contextCompletions
   }
 
@@ -129,8 +157,13 @@ function handleParseTree(params: { uri: string }) {
   const document = documents.get(params.uri)
   if (!document) return 'Document not found'
 
+  const tree = documentTrees.get(params.uri)
+  if (!tree) {
+    connection.console.error(`🦐 No cached tree for ${params.uri}`)
+    return 'No cached parse tree available'
+  }
+
   const text = document.getText()
-  const tree = parser.parse(text)
   const cursor = tree.cursor()
 
   let formatted = ''