Kotlin cache perf trials

.github/workflows/comparison-results.yml

@@ -49,7 +49,7 @@ jobs:
         with:
           python-version: '3.10'
 
-      - name: Clone standalone-monitoring-stability/main
+      - name: Clone standalone-monitoring-stability/test-kotlin-cache-changes
         uses: actions/checkout@v3
         with:
           repository: Privado-Inc/standalone-monitoring-stability

.github/workflows/comparison_results.yml

@@ -68,12 +68,12 @@ jobs:
         with:
           python-version: '3.10'
 
-      - name: Clone standalone-monitoring-stability/custom-joern-build
+      - name: Clone standalone-monitoring-stability/test-kotlin-cache-changes
         uses: actions/checkout@v3
         with:
           repository: Privado-Inc/standalone-monitoring-stability
           path: ./temp/standalone-monitoring-stability
-          ref: main
+          ref: test-kotlin-cache-changes
 
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v3

README.md

@@ -15,6 +15,7 @@ query language. Joern is developed with the goal of providing a useful
 tool for vulnerability discovery and research in static program
 analysis.
 
+
 Website: https://joern.io
 
 Documentation: https://docs.joern.io/

joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/util/PackageRootFinderTests.scala

@@ -1,7 +1,6 @@
 package io.joern.javasrc2cpg.util
 
-import io.joern.javasrc2cpg.testfixtures.SourceCodeFixture
-
+import io.joern.x2cpg.testfixtures.SourceCodeFixture
 import java.nio.file.Path
 
 class PackageRootFinderTests extends SourceCodeFixture {

joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Kotlin2Cpg.scala

@@ -1,35 +1,27 @@
 package io.joern.kotlin2cpg
 
 import better.files.File
-import io.joern.kotlin2cpg.compiler.CompilerAPI
-import io.joern.kotlin2cpg.compiler.ErrorLoggingMessageCollector
+import io.joern.kotlin2cpg.compiler.{BindingContextAnalyserPass, CompilerAPI, ErrorLoggingMessageCollector}
 import io.joern.kotlin2cpg.files.SourceFilesPicker
 import io.joern.kotlin2cpg.interop.JavaSrcInterop
 import io.joern.kotlin2cpg.jar4import.UsesService
 import io.joern.kotlin2cpg.passes.*
-import io.joern.kotlin2cpg.types.{ContentSourcesPicker, TypeInfoProvider}
-import io.joern.x2cpg.SourceFiles
-import io.joern.x2cpg.X2CpgFrontend
-import io.joern.x2cpg.X2Cpg.withNewEmptyCpg
-import io.joern.x2cpg.passes.frontend.MetaDataPass
-import io.joern.x2cpg.passes.frontend.TypeNodePass
-import io.joern.x2cpg.utils.dependency.DependencyResolver
-import io.joern.x2cpg.utils.dependency.DependencyResolverParams
-import io.joern.x2cpg.utils.dependency.GradleConfigKeys
+import io.joern.kotlin2cpg.types.{ContentSourcesPicker, ModuleInfo, TypeInfoProvider}
 import io.joern.x2cpg.SourceFiles.filterFile
-import io.shiftleft.codepropertygraph.generated.Cpg
-import io.shiftleft.codepropertygraph.generated.Languages
-import io.shiftleft.utils.IOUtils
+import io.joern.x2cpg.X2Cpg.withNewEmptyCpg
+import io.joern.x2cpg.passes.frontend.{MetaDataPass, TypeNodePass}
+import io.joern.x2cpg.utils.dependency.{DependencyResolver, DependencyResolverParams, GradleConfigKeys}
+import io.joern.x2cpg.{SourceFiles, X2CpgFrontend}
+import io.shiftleft.codepropertygraph.generated.{Cpg, Languages}
+import io.shiftleft.utils.{IOUtils, StatsLogger}
 import org.jetbrains.kotlin.cli.jvm.compiler.{KotlinCoreEnvironment, KotlinToJVMBytecodeCompiler}
 import org.jetbrains.kotlin.com.intellij.openapi.util.Disposer
 import org.jetbrains.kotlin.psi.KtFile
-import org.jetbrains.kotlin.resolve.BindingContext
+import org.jetbrains.kotlin.resolve.{BindingContext, BindingTraceContext}
 import org.slf4j.LoggerFactory
 
-import java.nio.file.Files
-import java.nio.file.Paths
-import scala.jdk.CollectionConverters.CollectionHasAsScala
-import scala.jdk.CollectionConverters.EnumerationHasAsScala
+import java.nio.file.{Files, Paths}
+import scala.jdk.CollectionConverters.{CollectionHasAsScala, EnumerationHasAsScala}
 import scala.util.Try
 import scala.util.matching.Regex
 
@@ -157,6 +149,14 @@ class Kotlin2Cpg extends X2CpgFrontend[Config] with UsesService {
     defaultContentRootJars
   }
 
+  private def identifyModules(sourceDir: String, config: Config): Seq[ModuleInfo] = {
+    val modules = ContentSourcesPicker.getModuleWiseSegregation(sourceDir, config)
+    if (modules.isEmpty) {
+      logger.warn("The list of modules is empty.")
+    }
+    modules
+  }
+
   private def gatherDirsForSourcesToCompile(sourceDir: String, config: Config): Seq[String] = {
     val dirsForSourcesToCompile = ContentSourcesPicker.dirsForRoot(sourceDir, config)
     if (dirsForSourcesToCompile.isEmpty) {
@@ -170,7 +170,7 @@ class Kotlin2Cpg extends X2CpgFrontend[Config] with UsesService {
     config: Config,
     environment: KotlinCoreEnvironment
   ): Iterable[KtFileWithMeta] = {
-    val sourceEntries = entriesForSources(environment.getSourceFiles.asScala, sourceDir)
+    val sourceEntries = entriesForSources(environment.getSourceFiles.asScala, config.inputPath)
     val sourceFiles = sourceEntries.filter(entry =>
       SourceFiles.filterFile(
         entry.filename,
@@ -204,39 +204,38 @@ class Kotlin2Cpg extends X2CpgFrontend[Config] with UsesService {
 
   def createCpg(config: Config): Try[Cpg] = {
     withNewEmptyCpg(config.outputPath, config) { (cpg, config) =>
-      val sourceDir = config.inputPath
-      logger.info(s"Starting CPG generation for input directory `$sourceDir`.")
-
-      checkSourceDir(sourceDir)
+      val originalSourceDir = config.inputPath
+      logger.info(s"Starting CPG generation for input directory `$originalSourceDir`.")
+      checkSourceDir(originalSourceDir)
       logMaxHeapSize()
-
-      val filesWithJavaExtension  = gatherFilesWithJavaExtension(sourceDir, config)
-      val mavenCoordinates        = gatherMavenCoordinates(sourceDir, config)
-      val defaultContentRootJars  = gatherDefaultContentRootJars(sourceDir, config, filesWithJavaExtension)
-      val dirsForSourcesToCompile = gatherDirsForSourcesToCompile(sourceDir, config)
-      val environment = CompilerAPI.makeEnvironment(
-        dirsForSourcesToCompile,
-        filesWithJavaExtension,
-        defaultContentRootJars,
-        new ErrorLoggingMessageCollector
-      )
-
-      val sourceFiles = gatherSourceFiles(sourceDir, config, environment)
-      val configFiles = entriesForConfigFiles(SourceFilesPicker.configFiles(sourceDir), sourceDir)
-
       new MetaDataPass(cpg, Languages.KOTLIN, config.inputPath).createAndApply()
+      // TODO: Add maven project structure handling
+      identifyModules(originalSourceDir, config).foreach { module =>
+        val sourceDir              = module.modulePathRoot
+        val filesWithJavaExtension = gatherFilesWithJavaExtension(sourceDir, config)
+        val defaultContentRootJars = gatherDefaultContentRootJars(sourceDir, config, filesWithJavaExtension)
+        logger.info(s"module directory size `${module.sourceFileDirs.size}`")
+        val environments = CompilerAPI.makeEnvironment(
+          module.sourceFileDirs,
+          filesWithJavaExtension,
+          defaultContentRootJars,
+          new ErrorLoggingMessageCollector
+        )
+        val bindingContext = BindingContextAnalyserPass(environments, config).apply()
+        val sourceFiles    = environments.flatMap(environment => gatherSourceFiles(sourceDir, config, environment))
+        val configFiles    = entriesForConfigFiles(SourceFilesPicker.configFiles(sourceDir), sourceDir)
+        val astCreator     = new AstCreationPass(sourceFiles, bindingContext, cpg)(config.schemaValidation)
+        astCreator.createAndApply()
+        environments.foreach { environment => Disposer.dispose(environment.getProjectEnvironment.getParentDisposable) }
+
+        val kotlinAstCreatorTypes = astCreator.usedTypes()
+        TypeNodePass.withRegisteredTypes(kotlinAstCreatorTypes, cpg).createAndApply()
+
+        runJavaSrcInterop(cpg, config, filesWithJavaExtension, kotlinAstCreatorTypes)
+        new ConfigPass(configFiles, cpg).createAndApply()
+      }
 
-      val bindingContext = createBindingContext(environment, config)
-      val astCreator     = new AstCreationPass(sourceFiles, bindingContext, cpg)(config.schemaValidation)
-      astCreator.createAndApply()
-
-      Disposer.dispose(environment.getProjectEnvironment.getParentDisposable)
-
-      val kotlinAstCreatorTypes = astCreator.usedTypes()
-      TypeNodePass.withRegisteredTypes(kotlinAstCreatorTypes, cpg).createAndApply()
-
-      runJavaSrcInterop(cpg, config, filesWithJavaExtension, kotlinAstCreatorTypes)
-      new ConfigPass(configFiles, cpg).createAndApply()
+      val mavenCoordinates = gatherMavenCoordinates(originalSourceDir, config)
       new DependenciesFromMavenCoordinatesPass(mavenCoordinates, cpg).createAndApply()
     }
   }
@@ -316,24 +315,4 @@ class Kotlin2Cpg extends X2CpgFrontend[Config] with UsesService {
     } yield FileContentAtPath(fileContents, relPath, fileName)
   }
 
-  private def createBindingContext(environment: KotlinCoreEnvironment, config: Config): BindingContext = {
-    try {
-      if (!config.resolveTypes) {
-        logger.info("Skipped Running Kotlin compiler analysis... due to no resolve types flag")
-        BindingContext.EMPTY
-      } else {
-        logger.info("Running Kotlin compiler analysis...")
-        val t0             = System.nanoTime()
-        val analysisResult = KotlinToJVMBytecodeCompiler.INSTANCE.analyze(environment)
-        val t1             = System.nanoTime()
-        logger.info(s"Kotlin compiler analysis finished in `${(t1 - t0) / 1000000}` ms.")
-        analysisResult.getBindingContext
-      }
-
-    } catch {
-      case exc: Exception =>
-        logger.error(s"Kotlin compiler analysis failed with exception `${exc.toString}`:`${exc.getMessage}`.", exc)
-        BindingContext.EMPTY
-    }
-  }
 }

joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstCreator.scala

@@ -1,37 +1,28 @@
 package io.joern.kotlin2cpg.ast
 
-import io.joern.kotlin2cpg.Constants
-import io.joern.kotlin2cpg.KtFileWithMeta
+import io.joern.kotlin2cpg.{Constants, KtFileWithMeta}
 import io.joern.kotlin2cpg.datastructures.Scope
-import io.joern.kotlin2cpg.types.NameRenderer
-import io.joern.kotlin2cpg.types.TypeConstants
-import io.joern.kotlin2cpg.types.TypeInfoProvider
-import io.joern.x2cpg.Ast
-import io.joern.x2cpg.AstCreatorBase
-import io.joern.x2cpg.AstNodeBuilder
-import io.joern.x2cpg.Defines
-import io.joern.x2cpg.ValidationMode
+import io.joern.kotlin2cpg.types.{NameRenderer, TypeConstants, TypeInfoProvider}
+import io.joern.x2cpg.*
 import io.joern.x2cpg.datastructures.Global
 import io.joern.x2cpg.datastructures.Stack.*
-import io.joern.x2cpg.utils.IntervalKeyPool
-import io.joern.x2cpg.utils.NodeBuilders
+import io.joern.x2cpg.utils.{IntervalKeyPool, NodeBuilders}
 import io.joern.x2cpg.utils.NodeBuilders.newMethodReturnNode
 import io.shiftleft.codepropertygraph.generated.*
 import io.shiftleft.codepropertygraph.generated.nodes.*
-import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder
 import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal
 import org.jetbrains.kotlin.com.intellij.psi.PsiElement
-import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
-import org.jetbrains.kotlin.descriptors.DescriptorVisibilities
-import org.jetbrains.kotlin.descriptors.DescriptorVisibility
-import org.jetbrains.kotlin.descriptors.FunctionDescriptor
-import org.jetbrains.kotlin.lexer.KtToken
-import org.jetbrains.kotlin.lexer.KtTokens
+import org.jetbrains.kotlin.descriptors.{
+  DeclarationDescriptor,
+  DescriptorVisibilities,
+  DescriptorVisibility,
+  FunctionDescriptor
+}
+import org.jetbrains.kotlin.lexer.{KtToken, KtTokens}
 import org.jetbrains.kotlin.psi.*
 import org.jetbrains.kotlin.resolve.BindingContext
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
+import org.slf4j.{Logger, LoggerFactory}
 
 import scala.annotation.tailrec
 import scala.collection.mutable
@@ -54,8 +45,7 @@ class AstCreator(fileWithMeta: KtFileWithMeta, bindingContext: BindingContext, g
     with AstForExpressionsCreator
     with AstNodeBuilder[PsiElement, AstCreator] {
 
-  import AstCreator.BindingInfo
-  import AstCreator.ClosureBindingDef
+  import AstCreator.{BindingInfo, ClosureBindingDef}
 
   protected val closureBindingDefQueue: mutable.ArrayBuffer[ClosureBindingDef] = mutable.ArrayBuffer.empty
   protected val bindingInfoQueue: mutable.ArrayBuffer[BindingInfo]             = mutable.ArrayBuffer.empty
@@ -73,7 +63,7 @@ class AstCreator(fileWithMeta: KtFileWithMeta, bindingContext: BindingContext, g
   protected val debugScope: mutable.Stack[KtDeclaration]      = mutable.Stack.empty[KtDeclaration]
 
   protected val nameRenderer     = new NameRenderer()
-  protected val bindingUtils     = new BindingContextUtils(bindingContext)
+  protected val bindingUtils     = BindingContextUtils(bindingContext)
   protected val typeInfoProvider = new TypeInfoProvider(bindingContext)
 
   def createAst(): DiffGraphBuilder = {

joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForDeclarationsCreator.scala

@@ -4,20 +4,12 @@ import io.joern.kotlin2cpg.Constants
 import io.joern.kotlin2cpg.psi.PsiUtils
 import io.joern.kotlin2cpg.psi.PsiUtils.nonUnderscoreDestructuringEntries
 import io.joern.kotlin2cpg.types.TypeConstants
-import io.joern.x2cpg.Ast
+import io.joern.x2cpg.{Ast, Defines, ValidationMode}
 import io.joern.x2cpg.datastructures.Stack.*
-import io.joern.x2cpg.Defines
-import io.joern.x2cpg.ValidationMode
 import io.joern.x2cpg.utils.NodeBuilders
-import io.joern.x2cpg.utils.NodeBuilders.newBindingNode
-import io.joern.x2cpg.utils.NodeBuilders.newIdentifierNode
-import io.joern.x2cpg.utils.NodeBuilders.newMethodReturnNode
-import io.joern.x2cpg.utils.NodeBuilders.newModifierNode
-import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, ModifierTypes, NodeTypes, Operators}
-import io.shiftleft.codepropertygraph.generated.nodes.NewBlock
-import io.shiftleft.codepropertygraph.generated.nodes.NewCall
-import io.shiftleft.codepropertygraph.generated.nodes.NewMethod
-import io.shiftleft.codepropertygraph.generated.nodes.NewTypeDecl
+import io.joern.x2cpg.utils.NodeBuilders.{newBindingNode, newIdentifierNode, newMethodReturnNode, newModifierNode}
+import io.shiftleft.codepropertygraph.generated.nodes.{NewBlock, NewCall, NewMethod, NewTypeDecl}
+import io.shiftleft.codepropertygraph.generated.*
 import io.shiftleft.semanticcpg.language.*
 import org.jetbrains.kotlin.descriptors.Modality
 import org.jetbrains.kotlin.psi.*
@@ -29,8 +21,7 @@ import scala.util.Random
 trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) {
   this: AstCreator =>
 
-  import AstCreator.AnonymousObjectContext
-  import AstCreator.BindingInfo
+  import AstCreator.{AnonymousObjectContext, BindingInfo}
 
   def astsForClassOrObject(
     ktClass: KtClassOrObject,