perf(analyzer): move model and token analyzing to the normal analysing stage

Author: d-biehl

packages/language_server/src/robotcode/language_server/common/parts/workspace.py

@@ -53,7 +53,6 @@
 from robotcode.core.uri import Uri
 from robotcode.core.utils.dataclasses import from_dict
 from robotcode.core.utils.logging import LoggingDescriptor
-from robotcode.core.utils.path import path_is_relative_to
 from robotcode.core.workspace import ConfigBase, TConfig, WorkspaceFolder
 from robotcode.core.workspace import Workspace as CoreWorkspace
 from robotcode.jsonrpc2.protocol import rpc_method
@@ -327,21 +326,6 @@ def get_configuration_raw(
         result_future.set_result([result])
         return result_future
 
-    def get_workspace_folder(self, uri: Union[Uri, str]) -> Optional[WorkspaceFolder]:
-        if isinstance(uri, str):
-            uri = Uri(uri)
-
-        result = sorted(
-            [f for f in self.workspace_folders if path_is_relative_to(uri.to_path(), f.uri.to_path())],
-            key=lambda v1: len(v1.uri),
-            reverse=True,
-        )
-
-        if len(result) > 0:
-            return result[0]
-
-        return None
-
     @rpc_method(name="workspace/didChangeWorkspaceFolders", param_type=DidChangeWorkspaceFoldersParams)
     def _workspace_did_change_workspace_folders(
         self, event: WorkspaceFoldersChangeEvent, *args: Any, **kwargs: Any

packages/language_server/src/robotcode/language_server/robotframework/parts/diagnostics.py

@@ -1,9 +1,6 @@
-import ast
 from concurrent.futures import CancelledError
 from typing import TYPE_CHECKING, Any, List, Optional
 
-from robot.parsing.lexer.tokens import Token
-
 from robotcode.core.concurrent import check_current_task_canceled
 from robotcode.core.language import language_id
 from robotcode.core.lsp.types import (
@@ -25,12 +22,6 @@
 )
 from robotcode.robot.diagnostics.library_doc import LibraryDoc
 from robotcode.robot.diagnostics.namespace import Namespace
-from robotcode.robot.utils.ast import (
-    iter_nodes,
-    range_from_node,
-    range_from_token,
-)
-from robotcode.robot.utils.stubs import HasError, HasErrors, HeaderAndBodyBlock
 
 from ...common.parts.diagnostics import DiagnosticsCollectType, DiagnosticsResult
 
@@ -50,9 +41,6 @@ def __init__(self, parent: "RobotLanguageServerProtocol") -> None:
 
         self.parent.on_initialized.add(self._on_initialized)
 
-        self.parent.diagnostics.collect.add(self.collect_token_errors)
-        self.parent.diagnostics.collect.add(self.collect_model_errors)
-
         self.parent.diagnostics.collect.add(self.collect_namespace_diagnostics)
 
         self.parent.diagnostics.collect.add(self.collect_unused_keyword_references)
@@ -62,7 +50,6 @@ def __init__(self, parent: "RobotLanguageServerProtocol") -> None:
 
     def _on_initialized(self, sender: Any) -> None:
         self.parent.diagnostics.analyze.add(self.analyze)
-        self.parent.documents_cache.namespace_invalidated.add(self._on_namespace_invalidated)
         self.parent.documents_cache.namespace_initialized(self._on_namespace_initialized)
         self.parent.documents_cache.libraries_changed.add(self._on_libraries_changed)
         self.parent.documents_cache.variables_changed.add(self._on_variables_changed)
@@ -92,12 +79,6 @@ def _on_namespace_initialized(self, sender: Any, namespace: Namespace) -> None:
         if namespace.document is not None:
             self.parent.diagnostics.force_refresh_document(namespace.document)
 
-    @language_id("robotframework")
-    def _on_namespace_invalidated(self, sender: Any, namespace: Namespace) -> None:
-        if namespace.document is not None:
-            namespace.document.remove_cache_entry(self._collect_model_errors)
-            namespace.document.remove_cache_entry(self._collect_token_errors)
-
     @language_id("robotframework")
     def _on_get_related_documents(self, sender: Any, document: TextDocument) -> Optional[List[TextDocument]]:
         namespace = self.parent.documents_cache.get_only_initialized_namespace(document)
@@ -165,148 +146,6 @@ def collect_namespace_diagnostics(
                 ],
             )
 
-    def _create_error_from_node(
-        self,
-        node: ast.AST,
-        msg: str,
-        source: Optional[str] = None,
-        only_start: bool = True,
-    ) -> Diagnostic:
-        from robot.parsing.model.statements import Statement
-
-        if isinstance(node, HeaderAndBodyBlock):
-            if node.header is not None:
-                node = node.header
-            elif node.body:
-                stmt = next((n for n in node.body if isinstance(n, Statement)), None)
-                if stmt is not None:
-                    node = stmt
-
-        return Diagnostic(
-            range=range_from_node(node, True, only_start),
-            message=msg,
-            severity=DiagnosticSeverity.ERROR,
-            source=source if source is not None else self.source_name,
-            code="ModelError",
-        )
-
-    def _create_error_from_token(self, token: Token, source: Optional[str] = None) -> Diagnostic:
-        return Diagnostic(
-            range=range_from_token(token),
-            message=token.error if token.error is not None else "(No Message).",
-            severity=DiagnosticSeverity.ERROR,
-            source=source if source is not None else self.source_name,
-            code="TokenError",
-        )
-
-    @language_id("robotframework")
-    @_logger.call
-    def collect_token_errors(
-        self, sender: Any, document: TextDocument, diagnostics_type: DiagnosticsCollectType
-    ) -> DiagnosticsResult:
-        return document.get_cache(self._collect_token_errors)
-
-    def _collect_token_errors(self, document: TextDocument) -> DiagnosticsResult:
-        from robot.errors import VariableError
-        from robot.parsing.lexer.tokens import Token
-
-        result: List[Diagnostic] = []
-        try:
-            for token in self.parent.documents_cache.get_tokens(document):
-                check_current_task_canceled()
-
-                if token.type in [Token.ERROR, Token.FATAL_ERROR]:
-                    result.append(self._create_error_from_token(token))
-
-                try:
-                    for variable_token in token.tokenize_variables():
-                        if variable_token == token:
-                            break
-
-                        if variable_token.type in [Token.ERROR, Token.FATAL_ERROR]:
-                            result.append(self._create_error_from_token(variable_token))
-
-                except VariableError as e:
-                    result.append(
-                        Diagnostic(
-                            range=range_from_token(token),
-                            message=str(e),
-                            severity=DiagnosticSeverity.ERROR,
-                            source=self.source_name,
-                            code=type(e).__qualname__,
-                        )
-                    )
-        except (CancelledError, SystemExit, KeyboardInterrupt):
-            raise
-        except BaseException as e:
-            return DiagnosticsResult(
-                self.collect_token_errors,
-                [
-                    Diagnostic(
-                        range=Range(
-                            start=Position(line=0, character=0),
-                            end=Position(
-                                line=len(document.get_lines()),
-                                character=len((document.get_lines())[-1] or ""),
-                            ),
-                        ),
-                        message=f"Fatal: can't get token diagnostics '{e}' ({type(e).__qualname__})",
-                        severity=DiagnosticSeverity.ERROR,
-                        source=self.source_name,
-                        code=type(e).__qualname__,
-                    )
-                ],
-            )
-
-        return DiagnosticsResult(self.collect_token_errors, self.modify_diagnostics(document, result))
-
-    @language_id("robotframework")
-    @_logger.call
-    def collect_model_errors(
-        self, sender: Any, document: TextDocument, diagnostics_type: DiagnosticsCollectType
-    ) -> DiagnosticsResult:
-        return document.get_cache(self._collect_model_errors)
-
-    def _collect_model_errors(self, document: TextDocument) -> DiagnosticsResult:
-        try:
-            model = self.parent.documents_cache.get_model(document, True)
-
-            result: List[Diagnostic] = []
-            for node in iter_nodes(model):
-                check_current_task_canceled()
-
-                error = node.error if isinstance(node, HasError) else None
-                if error is not None:
-                    result.append(self._create_error_from_node(node, error))
-                errors = node.errors if isinstance(node, HasErrors) else None
-                if errors is not None:
-                    for e in errors:
-                        result.append(self._create_error_from_node(node, e))
-
-            return DiagnosticsResult(self.collect_model_errors, self.modify_diagnostics(document, result))
-
-        except (CancelledError, SystemExit, KeyboardInterrupt):
-            raise
-        except BaseException as e:
-            return DiagnosticsResult(
-                self.collect_model_errors,
-                [
-                    Diagnostic(
-                        range=Range(
-                            start=Position(line=0, character=0),
-                            end=Position(
-                                line=len(document.get_lines()),
-                                character=len((document.get_lines())[-1] or ""),
-                            ),
-                        ),
-                        message=f"Fatal: can't get model diagnostics '{e}' ({type(e).__qualname__})",
-                        severity=DiagnosticSeverity.ERROR,
-                        source=self.source_name,
-                        code=type(e).__qualname__,
-                    )
-                ],
-            )
-
     @language_id("robotframework")
     @_logger.call
     def collect_unused_keyword_references(

packages/language_server/src/robotcode/language_server/robotframework/parts/hover.py

@@ -114,7 +114,7 @@ def _hover_default(self, nodes: List[ast.AST], document: TextDocument, position:
                         ),
                     )
                 )
-
+                value = None
                 if found_range is not None:
                     highlight_range = found_range
                     if variable.has_value or variable.resolvable:

packages/robot/src/robotcode/robot/diagnostics/errors.py

@@ -1,6 +1,6 @@
 from typing import final
 
-DIAGNOSTICS_SOURCE_NAME = "robotcode.namespace"
+DIAGNOSTICS_SOURCE_NAME = "robotcode"
 
 
 @final
@@ -38,3 +38,5 @@ class Error:
     OVERRIDDEN_BY_COMMANDLINE = "OverriddenByCommandLine"
     VARIABLE_ALREADY_DEFINED = "VariableAlreadyDefined"
     VARIABLE_OVERRIDDEN = "VariableOverridden"
+    MODEL_ERROR = "ModelError"
+    TOKEN_ERROR = "TokenError"

packages/robot/src/robotcode/robot/diagnostics/model_helper.py

@@ -8,6 +8,7 @@
 from typing import (
     TYPE_CHECKING,
     Any,
+    Callable,
     Iterator,
     List,
     Optional,
@@ -18,6 +19,7 @@
     Union,
 )
 
+from robot.errors import VariableError
 from robot.parsing.lexer.tokens import Token
 from robot.utils.escaping import split_from_equals, unescape
 from robot.variables.finders import NOT_FOUND, NumberFinder
@@ -358,21 +360,30 @@ def tokenize_variables(
         ignore_errors: bool = False,
         *,
         extra_types: Optional[Set[str]] = None,
+        exception_handler: Optional[Callable[[Exception, Token], None]] = None,
     ) -> Iterator[Token]:
-        for t in tokenize_variables(token, identifiers, ignore_errors, extra_types=extra_types):
-            if t.type == Token.VARIABLE:
-                var, rest = cls.remove_index_from_variable_token(t)
-                if var is not None:
-                    yield var
-                if rest is not None:
-                    yield from cls.tokenize_variables(
-                        rest,
-                        identifiers,
-                        ignore_errors,
-                        extra_types=extra_types,
-                    )
-            else:
-                yield t
+        if exception_handler is not None:
+            ignore_errors = False
+        try:
+            for t in tokenize_variables(token, identifiers, ignore_errors, extra_types=extra_types):
+                if t.type == Token.VARIABLE:
+                    var, rest = cls.remove_index_from_variable_token(t)
+                    if var is not None:
+                        yield var
+                    if rest is not None:
+                        yield from cls.tokenize_variables(
+                            rest,
+                            identifiers,
+                            ignore_errors,
+                            extra_types=extra_types,
+                        )
+                else:
+                    yield t
+        except VariableError as e:
+            if exception_handler is not None:
+                exception_handler(e, token)
+            elif not ignore_errors:
+                raise
 
     @classmethod
     def iter_variables_from_token(

packages/robot/src/robotcode/robot/diagnostics/namespace.py

@@ -739,8 +739,6 @@ def __init__(
 
         self._in_initialize = False
 
-        self._ignored_lines: Optional[List[int]] = None
-
     @event
     def has_invalidated(sender) -> None: ...