Merge 4-0-275 to backup_sync

.github/workflows/master.yml

@@ -15,8 +15,10 @@ jobs:
       - name: Set up JDK
         uses: actions/setup-java@v4
         with:
-          distribution: 'temurin'
-          java-version: '21'
+          distribution: temurin
+          java-version: 21
+          cache: sbt
+      - uses: sbt/setup-sbt@v1
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:

.github/workflows/pr.yml

@@ -13,18 +13,13 @@ jobs:
       - name: Set up JDK
         uses: actions/setup-java@v4
         with:
-          distribution: 'temurin'
-          java-version: '21'
-      - name: Install sbt
-        if: matrix.os == 'macos-latest'
-        run: brew install sbt
+          distribution: temurin
+          java-version: 21
+          cache: sbt
+      - uses: sbt/setup-sbt@v1
       - name: Install php
         if: matrix.os == 'macos-latest'
         run: brew install php
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 2.7
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
@@ -36,8 +31,6 @@ jobs:
         with:
           development: true
           swift-version: "5.10"
-      - name: Install Bundler
-        run: gem install bundler -v 2.4.22
       - name: Delete `.rustup` directory
         run: rm -rf /home/runner/.rustup # to save disk space
         if: runner.os == 'Linux'
@@ -61,8 +54,10 @@ jobs:
       - name: Set up JDK
         uses: actions/setup-java@v4
         with:
-          distribution: 'temurin'
-          java-version: '21'
+          distribution: temurin
+          java-version: 21
+          cache: sbt
+      - uses: sbt/setup-sbt@v1
       - uses: actions/cache@v4
         with:
           path: |
@@ -75,6 +70,8 @@ jobs:
         if: ${{ failure() }}
       - name: Validate CITATION.cff
         uses: dieghernan/cff-validator@v3
+        with:
+          install-r: true
 
   test-scripts:
     runs-on: ubuntu-latest
@@ -85,8 +82,10 @@ jobs:
       - name: Set up JDK
         uses: actions/setup-java@v4
         with:
-          distribution: 'temurin'
-          java-version: '21'
+          distribution: temurin
+          java-version: 21
+          cache: sbt
+      - uses: sbt/setup-sbt@v1
       - uses: actions/cache@v4
         with:
           path: |
@@ -100,7 +99,14 @@ jobs:
           ./joern --src /tmp/foo --run scan
           ./joern-scan /tmp/foo
           ./joern-scan --dump
-          ./joern-slice data-flow -o target/slice
+      - name: Joern Slice Testing
+        run: |
+          mkdir /tmp/slice
+          ./joern-slice data-flow tests/code/javasrc/SliceTest.java -o /tmp/slice/dataflow-slice-javasrc.json
+          echo "checking that the script output contains the content we expect:"
+          ./joern --script "tests/test-dataflow-slice.sc" --param sliceFile=/tmp/slice/dataflow-slice-javasrc.json | grep 'List(boolean b, b, this, s, "MALICIOUS", s, new Foo("MALICIOUS"), s, s, "SAFE", s, b, this, this, b, s, System.out)'
+      - name: SARIF Export Testing
+        run: ./tests/finding-to-sarif-test.sh
       - run: |
           cd joern-cli/target/universal/stage
           ./schema-extender/test.sh

.github/workflows/release.yml

@@ -15,8 +15,10 @@ jobs:
       - name: Set up JDK
         uses: actions/setup-java@v4
         with:
-          distribution: 'temurin'
-          java-version: '21'
+          distribution: temurin
+          java-version: 21
+          cache: sbt
+      - uses: sbt/setup-sbt@v1
       - run: sudo apt update && sudo apt install -y gnupg
       - run: echo $PGP_SECRET | base64 --decode | gpg --batch --import
         env:

.github/workflows/upgrade-deps.yml

@@ -17,8 +17,10 @@ jobs:
       - name: Set up JDK
         uses: actions/setup-java@v4
         with:
-          distribution: 'temurin'
-          java-version: '21'
+          distribution: temurin
+          java-version: 21
+          cache: sbt
+      - uses: sbt/setup-sbt@v1
       - uses: actions/cache@v4
         with:
           path: |

.gitignore

@@ -80,3 +80,5 @@ flake.lock
 **/.bsp
 
 
+/joern-cli/frontends/c2cpg/eclipse-cdt/build/
+/joern-cli/frontends/c2cpg/eclipse-cdt/org.eclipse.cdt.core-*.jar

Dockerfile

@@ -1,11 +1,11 @@
-FROM alpine:3.17.3
+FROM alpine:latest
 
 # dependencies
 RUN apk update && apk upgrade && apk add --no-cache openjdk17-jdk python3 git curl gnupg bash nss ncurses php
 RUN ln -sf python3 /usr/bin/python
 
 # sbt
-ENV SBT_VERSION 1.8.0
+ENV SBT_VERSION 1.10.3
 ENV SBT_HOME /usr/local/sbt
 ENV PATH ${PATH}:${SBT_HOME}/bin
 RUN curl -sL "https://github.com/sbt/sbt/releases/download/v$SBT_VERSION/sbt-$SBT_VERSION.tgz" | gunzip | tar -x -C /usr/local

README.md

@@ -23,6 +23,7 @@ Specification: https://cpg.joern.io
 
 ## News / Changelog
 
+- Joern v4.0.0 [migrates from overflowdb to flatgraph](changelog/4.0.0-flatgraph.md)
 - Joern v2.0.0 [upgrades from Scala2 to Scala3](changelog/2.0.0-scala3.md)
 - Joern v1.2.0 removes the `overflowdb.traversal.Traversal` class. This change is not completely backwards compatible. See [here](changelog/traversal_removal.md) for a detailed writeup.
 

build.sbt

@@ -1,8 +1,8 @@
 name                     := "joern"
 ThisBuild / organization := "io.joern"
-ThisBuild / scalaVersion := "3.4.2"
+ThisBuild / scalaVersion := "3.5.2"
 
-val cpgVersion = "1.6.16"
+val cpgVersion = "1.7.27"
 
 lazy val joerncli          = Projects.joerncli
 lazy val querydb           = Projects.querydb
@@ -45,7 +45,8 @@ ThisBuild / compile / javacOptions ++= Seq(
 ThisBuild / scalacOptions ++= Seq(
   "-deprecation", // Emit warning and location for usages of deprecated APIs.
   "--release",
-  "11"
+  "11",
+  "-Wshadow:type-parameter-shadow",
 )
 
 lazy val createDistribution = taskKey[File]("Create a complete Joern distribution")
@@ -73,7 +74,8 @@ Global / onChangedBuildSource := ReloadOnSourceChanges
 
 // publishing info for sonatype / maven central
 ThisBuild / publishTo  := sonatypePublishToBundle.value
-sonatypeCredentialHost := "s01.oss.sonatype.org"
+ThisBuild / sonatypeCredentialHost := xerial.sbt.Sonatype.sonatypeCentralHost
+
 ThisBuild / scmInfo    := Some(ScmInfo(url("https://github.com/joernio/joern"), "scm:git@github.com:joernio/joern.git"))
 ThisBuild / homepage   := Some(url("https://joern.io/"))
 ThisBuild / licenses   := List("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0"))

changelog/4.0.0-flatgraph.md

@@ -0,0 +1,43 @@
+# 4.0.x: Migration to flatgraph
+
+Joern uses the domain-specific classes from codepropertygraph, which (up to joern 2.x) were generated by overflowdb (specifically https://github.com/ShiftLeftSecurity/overflowdb and https://github.com/ShiftLeftSecurity/overflowdb-codegen). 
+As of joern 4.0.x we replaced overflowdb with it's successor, [flatgraph](https://github.com/joernio/flatgraph). The most important PRs paving the way for flatgraph are https://github.com/ShiftLeftSecurity/codepropertygraph/pull/1769 and https://github.com/joernio/joern/pull/4630. 
+
+### Why the change?
+Most importantly, flatgraph brings us about 40% less memory usage as well as faster traversals. The reduced memory footprint is achieved by flatgraph's efficient columnar layout, essentially we hold everything in few (albeit very large) arrays. 
+The faster traversals account for about 40% performance improvement for many joern use-cases, e.g. running the default passes while importing a large cpg into joern. Some numbers for Linux 4, as an example for a very large codebase. Numbers are based on my workstation and just rough measurements. 
+
+Linux 4.1.16, cpg created with c2cpg after `importCpg` into joern: 
+* 48M nodes with 630M properties (mostly `String` and `Integer`)
+* 431M edges with 115M properties (all `String`)
+
+|                                             | joern 2 (overflowdb) | joern 4 (flatgraph) |
+| --------------------------------------------|----------------------|-------------------- |
+| heap after import (after garbage collection)|                33g   | 20g                 |
+| minimum required heap (Xmx) for import      |                80g   | 30g                 |
+| time for importCpg                          |         18 minutes   |     11 minutes      |
+| file size on disk                           |              2600M   | 400m                |
+
+Linux 5 and 6 are considerably larger, so I wasn't able to import them into joern 2 on my workstation (which has 128G physical memory). With joern 4 it works just fine with `./joern -J-Xmx90g` for linux6 😀
+
+Also worth noting: one of overflowdb's features was the overflowing-to-disk mechanism. While it sounds nice to be able to handle graphs larger than the available memory, in practice it was too slow to be useful, so we didn't reimplement it in flatgraph.
+
+### API changes / upgrade guide
+We tried to minimise the joern-user-facing API changes, and depending on your usage you may not notice anything at all. That being said, if your code makes use of the `overflowdb` namespace then you will have to make some changes. In most cases, it's simply a namespace change to `flatgraph`. Since hopefully no joern user used the overflowdb api (with one exception listed below), I won't list the changes here, instead please look at the [joern migration PR](https://github.com/joernio/joern/pull/4630/files) and/or ask us on [discord](https://discord.com/channels/832209896089976854/842699104383533076). 
+
+Most relevant changes:
+1) `overflowdb.BatchedUpdate.applyDiff` -> `flatgraph.DiffGraphApplier.applyDiff`
+1) `io.shiftleft.passes.IntervalKeyPool` -> `io.joern.x2cpg.utils.IntervalKeyPool`
+
+1) `StoredNode.propertyOption` now returns an `Option` rather than a `java.util.Optional` - the API is almost identical, and there's builtin conversions both ways (`.toScala|.toJava` via `import scala.jdk.OptionConverters.*`).
+
+1) the arrow syntax for quickly constructing graphs, e.g. `v0 --- "CFG" --> v1`, quite useful for testing, doesn't exist in flatgraph yet. You'll need to create a diffgraph instead. There's plenty of examples in the [joern migration PR](https://github.com/joernio/joern/pull/4630/files).
+
+1) Edges can only have zero or one properties. Since the codepropertygraph schema never defined more than one property per edge type, this should not affect you as a joern user, unless you've extended the cpg schema...
+
+### Credits and kudos
+Flatgraph is based on [@bbrehm](https://github.com/bbrehm)'s great ideas for a memory efficient columnar layout on the jvm. He built a working prototype with very promising benchmarks that convinced us that the effort to migrate is worth-while, and that turned out to be true. 
+
+### Why did we leave out version 3?
+I'm glad you asked! Version 3 is typically a source for trouble, you know... just look at Gnome 3, Python 3 and many more. The only exception is Scala 3, of course - ymmv :)
+

console/src/main/scala/io/joern/console/BridgeBase.scala

@@ -2,18 +2,21 @@ package io.joern.console
 
 import better.files.*
 import io.shiftleft.codepropertygraph.generated.Languages
+import io.shiftleft.semanticcpg.sarif.SarifConfig
 import org.apache.commons.text.StringEscapeUtils
 import replpp.scripting.ScriptRunner
 
 import java.nio.file.{Files, Path}
+import scala.collection.mutable
 import scala.jdk.CollectionConverters.*
 import scala.util.Try
 
 case class Config(
   scriptFile: Option[Path] = None,
   command: Option[String] = None,
   params: Map[String, String] = Map.empty,
-  additionalImports: Seq[Path] = Nil,
+  predefFiles: Seq[Path] = Nil,
+  runBefore: Seq[String] = Nil,
   additionalClasspathEntries: Seq[String] = Seq.empty,
   addPlugin: Option[String] = None,
   rmPlugin: Option[String] = None,
@@ -70,8 +73,15 @@ trait BridgeBase extends InteractiveShell with ScriptExecution with PluginHandli
         .valueName("script1.sc")
         .unbounded()
         .optional()
-        .action((x, c) => c.copy(additionalImports = c.additionalImports :+ x))
-        .text("import (and run) additional script(s) on startup - may be passed multiple times")
+        .action((x, c) => c.copy(predefFiles = c.predefFiles :+ x))
+        .text("given source files will be compiled and added to classpath - this may be passed multiple times")
+
+      opt[String]("runBefore")
+        .valueName("'import Int.MaxValue'")
+        .unbounded()
+        .optional()
+        .action((x, c) => c.copy(runBefore = c.runBefore :+ x))
+        .text("given code will be executed on startup - this may be passed multiple times")
 
       opt[String]("classpathEntry")
         .valueName("path/to/classpath")
@@ -211,14 +221,21 @@ trait BridgeBase extends InteractiveShell with ScriptExecution with PluginHandli
     }
   }
 
-  protected def createPredefFile(additionalLines: Seq[String] = Nil): Path = {
-    val tmpFile = Files.createTempFile("joern-predef", "sc")
-    Files.write(tmpFile, (predefLines ++ additionalLines).asJava)
-    tmpFile.toAbsolutePath
-  }
-
   /** code that is executed on startup */
-  protected def predefLines: Seq[String]
+  protected def runBeforeCode: Seq[String]
+
+  protected def buildRunBeforeCode(config: Config): Seq[String] = {
+    val builder = Seq.newBuilder[String]
+    builder ++= runBeforeCode
+    config.cpgToLoad.foreach { cpgFile =>
+      builder += s"""importCpg("$cpgFile")"""
+    }
+    config.forInputPath.foreach { name =>
+      builder += s"""openForInputPath("$name")""".stripMargin
+    }
+    builder ++= config.runBefore
+    builder.result()
+  }
 
   protected def greeting: String
 
@@ -229,19 +246,10 @@ trait BridgeBase extends InteractiveShell with ScriptExecution with PluginHandli
 
 trait InteractiveShell { this: BridgeBase =>
   protected def startInteractiveShell(config: Config) = {
-    val replConfig = config.cpgToLoad.map { cpgFile =>
-      "importCpg(\"" + cpgFile + "\")"
-    } ++ config.forInputPath.map { name =>
-      s"""
-         |openForInputPath(\"$name\")
-         |""".stripMargin
-    }
-
-    val predefFile = createPredefFile(replConfig.toSeq)
-
     replpp.InteractiveShell.run(
       replpp.Config(
-        predefFiles = predefFile +: config.additionalImports,
+        predefFiles = config.predefFiles,
+        runBefore = buildRunBeforeCode(config),
         nocolors = config.nocolors,
         verbose = config.verbose,
         classpathConfig = replpp.Config
@@ -268,10 +276,10 @@ trait ScriptExecution { this: BridgeBase =>
     if (!Files.exists(scriptFile)) {
       Try(throw new AssertionError(s"given script file `$scriptFile` does not exist"))
     } else {
-      val predefFile = createPredefFile(importCpgCode(config))
       val scriptReturn = ScriptRunner.exec(
         replpp.Config(
-          predefFiles = predefFile +: config.additionalImports,
+          predefFiles = config.predefFiles,
+          runBefore = buildRunBeforeCode(config),
           scriptFile = Option(scriptFile),
           command = config.command,
           params = config.params,
@@ -286,18 +294,6 @@ trait ScriptExecution { this: BridgeBase =>
       scriptReturn
     }
   }
-
-  /** For the given config, generate a list of commands to import the CPG
-    */
-  private def importCpgCode(config: Config): List[String] = {
-    config.cpgToLoad.map { cpgFile =>
-      "importCpg(\"" + cpgFile + "\")"
-    }.toList ++ config.forInputPath.map { name =>
-      s"""
-         |openForInputPath(\"$name\")
-         |""".stripMargin
-    }
-  }
 }
 
 trait PluginHandling { this: BridgeBase =>
@@ -406,10 +402,9 @@ trait PluginHandling { this: BridgeBase =>
 trait ServerHandling { this: BridgeBase =>
 
   protected def startHttpServer(config: Config): Unit = {
-    val predefFile = createPredefFile(Nil)
-
     val baseConfig = replpp.Config(
-      predefFiles = predefFile +: config.additionalImports,
+      predefFiles = config.predefFiles,
+      runBefore = buildRunBeforeCode(config),
       verbose = true, // always print what's happening - helps debugging
       classpathConfig = replpp.Config
         .ForClasspath(inheritClasspath = true, dependencies = config.dependencies, resolvers = config.resolvers)

console/src/main/scala/io/joern/console/Commit.scala

@@ -3,7 +3,7 @@ package io.joern.console
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.passes.CpgPass
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions}
-import overflowdb.BatchedUpdate.DiffGraphBuilder
+import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder
 
 object Commit {
   val overlayName: String = "commit"
@@ -26,7 +26,7 @@ class Commit(opts: CommitOptions) extends LayerCreator {
         builder.absorb(opts.diffGraphBuilder)
       }
     }
-    runPass(pass, context)
+    pass.createAndApply()
     opts.diffGraphBuilder = Cpg.newDiffGraphBuilder
   }