Discard CST to preserve memory

Author: msujew

examples/requirements/test/validator.test.ts

@@ -13,13 +13,13 @@ import { NodeFileSystem } from 'langium/node';
 describe('A requirement identifier and a test identifier shall contain a number.', () => {
     test('T001_good_case', async () => {
         const services = createRequirementsAndTestsLangServices(NodeFileSystem);
-        const [mainDoc,allDocs] = await extractDocuments(
+        const [mainDoc, allDocs] = await extractDocuments(
             path.join(__dirname, 'files', 'good', 'requirements.req'),
             services.requirements
         );
         expect((mainDoc.diagnostics ?? [])).toEqual([]);
         expect(allDocs.length).toEqual(3);
-        allDocs.forEach(doc=>{
+        allDocs.forEach(doc => {
             expect((doc.diagnostics ?? [])).toEqual([]);
         });
     });
@@ -35,7 +35,7 @@ describe('A requirement identifier shall contain a number.', () => {
         expect(mainDoc.diagnostics ?? []).toEqual(expect.arrayContaining([
             expect.objectContaining({
                 message: expect.stringMatching('Requirement name ReqIdABC_reqID should container a number'),
-                range: expect.objectContaining({start:expect.objectContaining({line: 2})}) // zero based
+                range: expect.objectContaining({ start: expect.objectContaining({ line: 2 }) }) // zero based
             })
         ]));
 
@@ -45,17 +45,17 @@ describe('A requirement identifier shall contain a number.', () => {
 describe('A test identifier shall contain a number.', () => {
     test('T003_badTstId: bad case', async () => {
         const services = createRequirementsAndTestsLangServices(NodeFileSystem);
-        const [,allDocs] = await extractDocuments(
+        const [, allDocs] = await extractDocuments(
             path.join(__dirname, 'files', 'bad1', 'requirements.req'),
             services.requirements
         );
-        const doc = allDocs.find(doc=>/tests_part1.tst/.test(doc.uri.fsPath));
+        const doc = allDocs.find(doc => /tests_part1.tst/.test(doc.uri.fsPath));
         expect(doc).toBeDefined();
         if (!doc) throw new Error('impossible');
         expect(doc.diagnostics ?? []).toEqual(expect.arrayContaining([
             expect.objectContaining({
                 message: expect.stringMatching('Test name TA should container a number.'),
-                range: expect.objectContaining({start:expect.objectContaining({line: 1})}) // zero based
+                range: expect.objectContaining({ start: expect.objectContaining({ line: 1 }) }) // zero based
             })
         ]));
     });
@@ -71,7 +71,7 @@ describe('A requirement shall be covered by at least one test.', () => {
         expect(mainDoc.diagnostics ?? []).toEqual(expect.arrayContaining([
             expect.objectContaining({
                 message: expect.stringMatching('Requirement ReqId004_unicorn not covered by a test.'),
-                range: expect.objectContaining({start:expect.objectContaining({line: 4})}) // zero based
+                range: expect.objectContaining({ start: expect.objectContaining({ line: 4 }) }) // zero based
             })
         ]));
     });
@@ -80,28 +80,32 @@ describe('A requirement shall be covered by at least one test.', () => {
 describe('A referenced environment in a test must be found in one of the referenced requirements.', () => {
     test('referenced environment test', async () => {
         const services = createRequirementsAndTestsLangServices(NodeFileSystem);
-        const [,allDocs] = await extractDocuments(
+        const [, allDocs] = await extractDocuments(
             path.join(__dirname, 'files', 'bad2', 'requirements.req'),
             services.requirements
         );
-        const doc = allDocs.find(doc=>/tests_part1.tst/.test(doc.uri.fsPath));
+        const doc = allDocs.find(doc => /tests_part1.tst/.test(doc.uri.fsPath));
         expect(doc).toBeDefined();
         if (!doc) throw new Error('impossible');
         expect((doc.diagnostics ?? [])).toEqual(expect.arrayContaining([
             expect.objectContaining({
                 message: expect.stringMatching('Test T002_badReqId references environment Linux_x86 which is used in any referenced requirement.'),
-                range: expect.objectContaining({start:expect.objectContaining({
-                    line: 3,
-                    character: 65
-                })}) // zero based
+                range: expect.objectContaining({
+                    start: expect.objectContaining({
+                        line: 3,
+                        character: 65
+                    })
+                }) // zero based
             })
         ]));
         expect((doc.diagnostics ?? [])).toEqual(expect.arrayContaining([
             expect.objectContaining({
                 message: expect.stringMatching('Test T004_cov references environment Linux_x86 which is used in any referenced requirement.'),
-                range: expect.objectContaining({start:expect.objectContaining({
-                    line: 5
-                })}) // zero based
+                range: expect.objectContaining({
+                    start: expect.objectContaining({
+                        line: 5
+                    })
+                }) // zero based
             })
         ]));
 

packages/langium/src/default-module.ts

@@ -35,6 +35,7 @@ import { LangiumParserErrorMessageProvider } from './parser/langium-parser.js';
 import { DefaultAsyncParser } from './parser/async-parser.js';
 import { DefaultWorkspaceLock } from './workspace/workspace-lock.js';
 import { DefaultHydrator } from './serializer/hydrator.js';
+import { DefaultEnvironment } from './workspace/environment.js';
 
 /**
  * Context required for creating the default language-specific dependency injection module.
@@ -117,7 +118,8 @@ export function createDefaultSharedCoreModule(context: DefaultSharedCoreModuleCo
             WorkspaceManager: (services) => new DefaultWorkspaceManager(services),
             FileSystemProvider: (services) => context.fileSystemProvider(services),
             WorkspaceLock: () => new DefaultWorkspaceLock(),
-            ConfigurationProvider: (services) => new DefaultConfigurationProvider(services)
+            ConfigurationProvider: (services) => new DefaultConfigurationProvider(services),
+            Environment: () => new DefaultEnvironment()
         }
     };
 }

packages/langium/src/documentation/comment-provider.ts

@@ -28,9 +28,12 @@ export class DefaultCommentProvider implements CommentProvider {
         this.grammarConfig = () => services.parser.GrammarConfig;
     }
     getComment(node: AstNode): string | undefined {
-        if(isAstNodeWithComment(node)) {
+        if (isAstNodeWithComment(node)) {
             return node.$comment;
+        } else if (node.$segments && 'comment' in node.$segments) {
+            return node.$segments.comment;
+        } else {
+            return findCommentNode(node.$cstNode, this.grammarConfig().multilineCommentRules)?.text;
         }
-        return findCommentNode(node.$cstNode, this.grammarConfig().multilineCommentRules)?.text;
     }
 }

packages/langium/src/lsp/call-hierarchy-provider.ts

@@ -54,12 +54,12 @@ export abstract class AbstractCallHierarchyProvider implements CallHierarchyProv
             return undefined;
         }
 
-        const declarationNode = this.references.findDeclarationNode(targetNode);
+        const declarationNode = this.references.findDeclaration(targetNode);
         if (!declarationNode) {
             return undefined;
         }
 
-        return this.getCallHierarchyItems(declarationNode.astNode, document);
+        return this.getCallHierarchyItems(declarationNode, document);
     }
 
     protected getCallHierarchyItems(targetNode: AstNode, document: LangiumDocument<AstNode>): CallHierarchyItem[] | undefined {

packages/langium/src/lsp/definition-provider.ts

@@ -10,7 +10,7 @@ import type { GrammarConfig } from '../languages/grammar-config.js';
 import type { NameProvider } from '../references/name-provider.js';
 import type { References } from '../references/references.js';
 import type { LangiumServices } from './lsp-services.js';
-import type { CstNode } from '../syntax-tree.js';
+import type { AstNode, CstNode } from '../syntax-tree.js';
 import type { MaybePromise } from '../utils/promise-utils.js';
 import type { LangiumDocument } from '../workspace/documents.js';
 import { LocationLink } from 'vscode-languageserver';
@@ -37,7 +37,7 @@ export interface DefinitionProvider {
 
 export interface GoToLink {
     source: CstNode
-    target: CstNode
+    target: AstNode
     targetDocument: LangiumDocument
 }
 
@@ -67,21 +67,25 @@ export class DefaultDefinitionProvider implements DefinitionProvider {
 
     protected collectLocationLinks(sourceCstNode: CstNode, _params: DefinitionParams): MaybePromise<LocationLink[] | undefined> {
         const goToLink = this.findLink(sourceCstNode);
-        if (goToLink) {
-            return [LocationLink.create(
-                goToLink.targetDocument.textDocument.uri,
-                (goToLink.target.astNode.$cstNode ?? goToLink.target).range,
-                goToLink.target.range,
-                goToLink.source.range
-            )];
+        if (goToLink && goToLink.target.$segments) {
+            const name = this.nameProvider.getNameProperty(goToLink.target);
+            if (name) {
+                const nameSegment = goToLink.target.$segments.properties.get(name);
+                return nameSegment.map(segment => LocationLink.create(
+                    goToLink.targetDocument.textDocument.uri,
+                    goToLink.target.$segments!.full.range,
+                    segment.range,
+                    goToLink.source.range
+                ));
+            }
         }
         return undefined;
     }
 
     protected findLink(source: CstNode): GoToLink | undefined {
-        const target = this.references.findDeclarationNode(source);
-        if (target?.astNode) {
-            const targetDocument = getDocument(target.astNode);
+        const target = this.references.findDeclaration(source);
+        if (target) {
+            const targetDocument = getDocument(target);
             if (target && targetDocument) {
                 return { source, target, targetDocument };
             }

packages/langium/src/lsp/document-update-handler.ts

@@ -4,17 +4,18 @@
  * terms of the MIT License, which is available in the project root.
  ******************************************************************************/
 
-import type { TextDocumentWillSaveEvent, DidChangeWatchedFilesParams, DidChangeWatchedFilesRegistrationOptions, TextDocumentChangeEvent, TextEdit } from 'vscode-languageserver';
+import type { TextDocumentWillSaveEvent, DidChangeWatchedFilesParams, DidChangeWatchedFilesRegistrationOptions, TextDocumentChangeEvent, TextEdit, Connection } from 'vscode-languageserver';
 import { DidChangeWatchedFilesNotification, FileChangeType } from 'vscode-languageserver';
 import { stream } from '../utils/stream.js';
 import { URI } from '../utils/uri-utils.js';
 import type { DocumentBuilder } from '../workspace/document-builder.js';
-import type { TextDocument } from '../workspace/documents.js';
+import type { LangiumDocuments, TextDocument } from '../workspace/documents.js';
 import type { WorkspaceLock } from '../workspace/workspace-lock.js';
 import type { LangiumSharedServices } from './lsp-services.js';
 import type { WorkspaceManager } from '../workspace/workspace-manager.js';
 import type { ServiceRegistry } from '../service-registry.js';
 import type { MaybePromise } from '../utils/promise-utils.js';
+import { discardCst } from '../utils/cst-utils.js';
 
 /**
  * Shared service for handling text document changes and watching relevant files.
@@ -71,13 +72,17 @@ export class DefaultDocumentUpdateHandler implements DocumentUpdateHandler {
     protected readonly workspaceManager: WorkspaceManager;
     protected readonly documentBuilder: DocumentBuilder;
     protected readonly workspaceLock: WorkspaceLock;
+    protected readonly documents: LangiumDocuments;
+    protected readonly connection: Connection | undefined;
     protected readonly serviceRegistry: ServiceRegistry;
 
     constructor(services: LangiumSharedServices) {
         this.workspaceManager = services.workspace.WorkspaceManager;
         this.documentBuilder = services.workspace.DocumentBuilder;
         this.workspaceLock = services.workspace.WorkspaceLock;
         this.serviceRegistry = services.ServiceRegistry;
+        this.documents = services.workspace.LangiumDocuments;
+        this.connection = services.lsp.Connection;
 
         let canRegisterFileWatcher = false;
         services.lsp.LanguageServer.onInitialize(params => {
@@ -98,15 +103,14 @@ export class DefaultDocumentUpdateHandler implements DocumentUpdateHandler {
             .distinct()
             .toArray();
         if (fileExtensions.length > 0) {
-            const connection = services.lsp.Connection;
             const options: DidChangeWatchedFilesRegistrationOptions = {
                 watchers: [{
                     globPattern: fileExtensions.length === 1
                         ? `**/*.${fileExtensions[0]}`
                         : `**/*.{${fileExtensions.join(',')}}`
                 }]
             };
-            connection?.client.register(DidChangeWatchedFilesNotification.type, options);
+            this.connection?.client.register(DidChangeWatchedFilesNotification.type, options);
         }
     }
 
@@ -141,4 +145,18 @@ export class DefaultDocumentUpdateHandler implements DocumentUpdateHandler {
             .toArray();
         this.fireDocumentUpdate(changedUris, deletedUris);
     }
+
+    didCloseDocument(event: TextDocumentChangeEvent<TextDocument>): void {
+        const document = this.documents.getDocument(URI.parse(event.document.uri));
+        if (document) {
+            // Preserve memory by discarding the CST of the document
+            // Whenever the user reopens the document, the CST will be rebuilt
+            discardCst(document.parseResult.value);
+        }
+        // Discard the diagnostics for the closed document
+        this.connection?.sendDiagnostics({
+            uri: event.document.uri,
+            diagnostics: []
+        });
+    }
 }

packages/langium/src/lsp/language-server.ts

@@ -198,6 +198,7 @@ export class DefaultLanguageServer implements LanguageServer {
     }
 
     protected fireInitializeOnDefaultServices(params: InitializeParams): void {
+        this.services.workspace.Environment.initialize(params);
         this.services.workspace.ConfigurationProvider.initialize(params);
         this.services.workspace.WorkspaceManager.initialize(params);
     }

packages/langium/src/lsp/type-hierarchy-provider.ts

@@ -58,12 +58,12 @@ export abstract class AbstractTypeHierarchyProvider implements TypeHierarchyProv
             return undefined;
         }
 
-        const declarationNode = this.references.findDeclarationNode(targetNode);
+        const declarationNode = this.references.findDeclaration(targetNode);
         if (!declarationNode) {
             return undefined;
         }
 
-        return this.getTypeHierarchyItems(declarationNode.astNode, document);
+        return this.getTypeHierarchyItems(declarationNode, document);
     }
 
     protected getTypeHierarchyItems(targetNode: AstNode, document: LangiumDocument): TypeHierarchyItem[] | undefined {

packages/langium/src/parser/async-parser.ts

@@ -7,7 +7,7 @@
 import type { CancellationToken } from '../utils/cancellation.js';
 import type { LangiumCoreServices } from '../services.js';
 import type { AstNode } from '../syntax-tree.js';
-import type { LangiumParser, ParseResult } from './langium-parser.js';
+import type { LangiumParser, ParseResult, ParserOptions } from './langium-parser.js';
 import type { Hydrator } from '../serializer/hydrator.js';
 import type { Event } from '../utils/event.js';
 import { Deferred, OperationCancelled } from '../utils/promise-utils.js';
@@ -30,7 +30,7 @@ export interface AsyncParser {
      *
      * @throws `OperationCancelled` if the parsing process is cancelled.
      */
-    parse<T extends AstNode>(text: string, cancelToken: CancellationToken): Promise<ParseResult<T>>;
+    parse<T extends AstNode>(text: string, options: ParserOptions | undefined, cancelToken: CancellationToken): Promise<ParseResult<T>>;
 }
 
 /**
@@ -47,8 +47,8 @@ export class DefaultAsyncParser implements AsyncParser {
         this.syncParser = services.parser.LangiumParser;
     }
 
-    parse<T extends AstNode>(text: string, _cancelToken: CancellationToken): Promise<ParseResult<T>> {
-        return Promise.resolve(this.syncParser.parse<T>(text));
+    parse<T extends AstNode>(text: string, options: ParserOptions | undefined, _cancelToken: CancellationToken): Promise<ParseResult<T>> {
+        return Promise.resolve(this.syncParser.parse<T>(text, options));
     }
 }
 
@@ -89,7 +89,7 @@ export abstract class AbstractThreadedAsyncParser implements AsyncParser {
         }
     }
 
-    async parse<T extends AstNode>(text: string, cancelToken: CancellationToken): Promise<ParseResult<T>> {
+    async parse<T extends AstNode>(text: string, options: ParserOptions | undefined, cancelToken: CancellationToken): Promise<ParseResult<T>> {
         const worker = await this.acquireParserWorker(cancelToken);
         const deferred = new Deferred<ParseResult<T>>();
         let timeout: NodeJS.Timeout | undefined;
@@ -101,7 +101,7 @@ export abstract class AbstractThreadedAsyncParser implements AsyncParser {
                 this.terminateWorker(worker);
             }, this.terminationDelay);
         });
-        worker.parse(text).then(result => {
+        worker.parse(text, options).then(result => {
             const hydrated = this.hydrator.hydrate<T>(result);
             deferred.resolve(hydrated);
         }).catch(err => {
@@ -194,13 +194,13 @@ export class ParserWorker {
         this.onReadyEmitter.fire();
     }
 
-    parse(text: string): Promise<ParseResult> {
+    parse(text: string, options: ParserOptions | undefined): Promise<ParseResult> {
         if (this._parsing) {
             throw new Error('Parser worker is busy');
         }
         this._parsing = true;
         this.deferred = new Deferred();
-        this.sendMessage(text);
+        this.sendMessage([text, options]);
         return this.deferred.promise;
     }
 }

packages/langium/src/parser/cst-node-builder.ts

@@ -87,20 +87,20 @@ export class CstNodeBuilder {
         }
     }
 
-    construct(item: { $type: string | symbol | undefined, $cstNode: CstNode }): void {
+    construct(item: { $type: string | symbol | undefined, $cstNode: CstNode }): CstNode {
         const current: CstNode = this.current;
         // The specified item could be a datatype ($type is symbol) or a fragment ($type is undefined)
         // Only if the $type is a string, we actually assign the element
         if (typeof item.$type === 'string') {
             this.current.astNode = <AstNode>item;
         }
-        item.$cstNode = current;
         const node = this.nodeStack.pop();
         // Empty composite nodes are not valid
         // Simply remove the node from the tree
         if (node?.content.length === 0) {
             this.removeNode(node);
         }
+        return current;
     }
 }