Merge 4-0-155 to backup_sync

.github/workflows/pr.yml

@@ -21,10 +21,6 @@ jobs:
       - name: Install php
         if: matrix.os == 'macos-latest'
         run: brew install php
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 2.7
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
@@ -36,8 +32,6 @@ jobs:
         with:
           development: true
           swift-version: "5.10"
-      - name: Install Bundler
-        run: gem install bundler -v 2.4.22
       - name: Delete `.rustup` directory
         run: rm -rf /home/runner/.rustup # to save disk space
         if: runner.os == 'Linux'
@@ -100,7 +94,10 @@ jobs:
           ./joern --src /tmp/foo --run scan
           ./joern-scan /tmp/foo
           ./joern-scan --dump
-          ./joern-slice data-flow -o target/slice
+      - run: |
+          mkdir /tmp/slice
+          ./joern-slice data-flow tests/code/javasrc/SliceTest.java -o /tmp/slice/dataflow-slice-javasrc.json
+          ./joern --script "./test-dataflow-slice.sc" --param sliceFile=/tmp/slice/dataflow-slice-javasrc.json | grep -q 'List(boolean b, b, this, s, "MALICIOUS", s, new Foo("MALICIOUS"), s, s, "SAFE", s, b, this, this, b, s, System.out)'
       - run: |
           cd joern-cli/target/universal/stage
           ./schema-extender/test.sh

Dockerfile

@@ -1,11 +1,11 @@
-FROM alpine:3.17.3
+FROM alpine:3.20
 
 # dependencies
 RUN apk update && apk upgrade && apk add --no-cache openjdk17-jdk python3 git curl gnupg bash nss ncurses php
 RUN ln -sf python3 /usr/bin/python
 
 # sbt
-ENV SBT_VERSION 1.8.0
+ENV SBT_VERSION 1.10.0
 ENV SBT_HOME /usr/local/sbt
 ENV PATH ${PATH}:${SBT_HOME}/bin
 RUN curl -sL "https://github.com/sbt/sbt/releases/download/v$SBT_VERSION/sbt-$SBT_VERSION.tgz" | gunzip | tar -x -C /usr/local

README.md

@@ -23,6 +23,7 @@ Specification: https://cpg.joern.io
 
 ## News / Changelog
 
+- Joern v4.0.0 [migrates from overflowdb to flatgraph](changelog/4.0.0-flatgraph.md)
 - Joern v2.0.0 [upgrades from Scala2 to Scala3](changelog/2.0.0-scala3.md)
 - Joern v1.2.0 removes the `overflowdb.traversal.Traversal` class. This change is not completely backwards compatible. See [here](changelog/traversal_removal.md) for a detailed writeup.
 

build.sbt

@@ -1,8 +1,8 @@
 name                     := "joern"
 ThisBuild / organization := "io.joern"
-ThisBuild / scalaVersion := "3.4.2"
+ThisBuild / scalaVersion := "3.4.3"
 
-val cpgVersion = "1.6.16"
+val cpgVersion = "1.7.13"
 
 lazy val joerncli          = Projects.joerncli
 lazy val querydb           = Projects.querydb
@@ -45,7 +45,8 @@ ThisBuild / compile / javacOptions ++= Seq(
 ThisBuild / scalacOptions ++= Seq(
   "-deprecation", // Emit warning and location for usages of deprecated APIs.
   "--release",
-  "11"
+  "11",
+  "-Wshadow:type-parameter-shadow",
 )
 
 lazy val createDistribution = taskKey[File]("Create a complete Joern distribution")
@@ -73,7 +74,8 @@ Global / onChangedBuildSource := ReloadOnSourceChanges
 
 // publishing info for sonatype / maven central
 ThisBuild / publishTo  := sonatypePublishToBundle.value
-sonatypeCredentialHost := "s01.oss.sonatype.org"
+ThisBuild / sonatypeCredentialHost := xerial.sbt.Sonatype.sonatypeCentralHost
+
 ThisBuild / scmInfo    := Some(ScmInfo(url("https://github.com/joernio/joern"), "scm:[email protected]:joernio/joern.git"))
 ThisBuild / homepage   := Some(url("https://joern.io/"))
 ThisBuild / licenses   := List("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0"))

changelog/4.0.0-flatgraph.md

@@ -0,0 +1,43 @@
+# 4.0.x: Migration to flatgraph
+
+Joern uses the domain-specific classes from codepropertygraph, which (up to joern 2.x) were generated by overflowdb (specifically https://github.com/ShiftLeftSecurity/overflowdb and https://github.com/ShiftLeftSecurity/overflowdb-codegen). 
+As of joern 4.0.x we replaced overflowdb with it's successor, [flatgraph](https://github.com/joernio/flatgraph). The most important PRs paving the way for flatgraph are https://github.com/ShiftLeftSecurity/codepropertygraph/pull/1769 and https://github.com/joernio/joern/pull/4630. 
+
+### Why the change?
+Most importantly, flatgraph brings us about 40% less memory usage as well as faster traversals. The reduced memory footprint is achieved by flatgraph's efficient columnar layout, essentially we hold everything in few (albeit very large) arrays. 
+The faster traversals account for about 40% performance improvement for many joern use-cases, e.g. running the default passes while importing a large cpg into joern. Some numbers for Linux 4, as an example for a very large codebase. Numbers are based on my workstation and just rough measurements. 
+
+Linux 4.1.16, cpg created with c2cpg after `importCpg` into joern: 
+* 48M nodes with 630M properties (mostly `String` and `Integer`)
+* 431M edges with 115M properties (all `String`)
+
+|                                             | joern 2 (overflowdb) | joern 4 (flatgraph) |
+| --------------------------------------------|----------------------|-------------------- |
+| heap after import (after garbage collection)|                33g   | 20g                 |
+| minimum required heap (Xmx) for import      |                80g   | 30g                 |
+| time for importCpg                          |         18 minutes   |     11 minutes      |
+| file size on disk                           |              2600M   | 400m                |
+
+Linux 5 and 6 are considerably larger, so I wasn't able to import them into joern 2 on my workstation (which has 128G physical memory). With joern 4 it works just fine with `./joern -J-Xmx90g` for linux6 😀
+
+Also worth noting: one of overflowdb's features was the overflowing-to-disk mechanism. While it sounds nice to be able to handle graphs larger than the available memory, in practice it was too slow to be useful, so we didn't reimplement it in flatgraph.
+
+### API changes / upgrade guide
+We tried to minimise the joern-user-facing API changes, and depending on your usage you may not notice anything at all. That being said, if your code makes use of the `overflowdb` namespace then you will have to make some changes. In most cases, it's simply a namespace change to `flatgraph`. Since hopefully no joern user used the overflowdb api (with one exception listed below), I won't list the changes here, instead please look at the [joern migration PR](https://github.com/joernio/joern/pull/4630/files) and/or ask us on [discord](https://discord.com/channels/832209896089976854/842699104383533076). 
+
+Most relevant changes:
+1) `overflowdb.BatchedUpdate.applyDiff` -> `flatgraph.DiffGraphApplier.applyDiff`
+1) `io.shiftleft.passes.IntervalKeyPool` -> `io.joern.x2cpg.utils.IntervalKeyPool`
+
+1) `StoredNode.propertyOption` now returns an `Option` rather than a `java.util.Optional` - the API is almost identical, and there's builtin conversions both ways (`.toScala|.toJava` via `import scala.jdk.OptionConverters.*`).
+
+1) the arrow syntax for quickly constructing graphs, e.g. `v0 --- "CFG" --> v1`, quite useful for testing, doesn't exist in flatgraph yet. You'll need to create a diffgraph instead. There's plenty of examples in the [joern migration PR](https://github.com/joernio/joern/pull/4630/files).
+
+1) Edges can only have zero or one properties. Since the codepropertygraph schema never defined more than one property per edge type, this should not affect you as a joern user, unless you've extended the cpg schema...
+
+### Credits and kudos
+Flatgraph is based on [@bbrehm](https://github.com/bbrehm)'s great ideas for a memory efficient columnar layout on the jvm. He built a working prototype with very promising benchmarks that convinced us that the effort to migrate is worth-while, and that turned out to be true. 
+
+### Why did we leave out version 3?
+I'm glad you asked! Version 3 is typically a source for trouble, you know... just look at Gnome 3, Python 3 and many more. The only exception is Scala 3, of course - ymmv :)
+

console/src/main/scala/io/joern/console/Commit.scala

@@ -3,7 +3,7 @@ package io.joern.console
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.passes.CpgPass
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions}
-import overflowdb.BatchedUpdate.DiffGraphBuilder
+import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder
 
 object Commit {
   val overlayName: String = "commit"
@@ -26,7 +26,7 @@ class Commit(opts: CommitOptions) extends LayerCreator {
         builder.absorb(opts.diffGraphBuilder)
       }
     }
-    runPass(pass, context)
+    pass.createAndApply()
     opts.diffGraphBuilder = Cpg.newDiffGraphBuilder
   }
 

console/src/main/scala/io/joern/console/Console.scala

@@ -12,8 +12,8 @@ import io.shiftleft.codepropertygraph.cpgloading.CpgLoader
 import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.language.dotextension.ImageViewer
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext}
-import overflowdb.traversal.help.Doc
-import overflowdb.traversal.help.Table.AvailableWidthProvider
+import io.shiftleft.codepropertygraph.generated.help.Doc
+import flatgraph.help.Table.AvailableWidthProvider
 
 import scala.sys.process.Process
 import scala.util.control.NoStackTrace
@@ -349,10 +349,14 @@ class Console[T <: Project](loader: WorkspaceLoader[T], baseDir: File = File.cur
 
     val cpgDestinationPath = cpgDestinationPathOpt.get
 
-    if (CpgLoader.isLegacyCpg(cpgFile)) {
-      report("You have provided a legacy proto CPG. Attempting conversion.")
+    val isProtoFormat      = CpgLoader.isProtoFormat(cpgFile.path)
+    val isOverflowDbFormat = CpgLoader.isOverflowDbFormat(cpgFile.path)
+    if (isProtoFormat || isOverflowDbFormat) {
+      if (isProtoFormat) report("You have provided a legacy proto CPG. Attempting conversion.")
+      else if (isOverflowDbFormat) report("You have provided a legacy overflowdb CPG. Attempting conversion.")
       try {
-        CpgConverter.convertProtoCpgToOverflowDb(cpgFile.path.toString, cpgDestinationPath.toString)
+        val cpg = CpgLoader.load(cpgFile.path, cpgDestinationPath)
+        cpg.close()
       } catch {
         case exc: Exception =>
           report("Error converting legacy CPG: " + exc.getMessage)

console/src/main/scala/io/joern/console/ConsoleConfig.scala

@@ -1,6 +1,6 @@
 package io.joern.console
 
-import better.files._
+import better.files.*
 
 import scala.annotation.tailrec
 import scala.collection.mutable

console/src/main/scala/io/joern/console/CpgConverter.scala

@@ -1,14 +1,18 @@
 package io.joern.console
 
-import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig}
-import overflowdb.Config
+import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, ProtoCpgLoader}
+
+import java.nio.file.Paths
 
 object CpgConverter {
 
-  def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit = {
-    val odbConfig = Config.withDefaults.withStorageLocation(dstFilename)
-    val config    = CpgLoaderConfig.withDefaults.doNotCreateIndexesOnLoad.withOverflowConfig(odbConfig)
-    CpgLoader.load(srcFilename, config).close
+  def convertProtoCpgToFlatgraph(srcFilename: String, dstFilename: String): Unit = {
+    val cpg = ProtoCpgLoader.loadFromProtoZip(srcFilename, Option(Paths.get(dstFilename)))
+    cpg.close()
   }
 
+  @deprecated("method got renamed to `convertProtoCpgToFlatgraph, please use that instead", "joern v3")
+  def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit =
+    convertProtoCpgToFlatgraph(srcFilename, dstFilename)
+
 }

console/src/main/scala/io/joern/console/Help.scala

@@ -1,19 +1,25 @@
 package io.joern.console
 
-import overflowdb.traversal.help.DocFinder.*
-import overflowdb.traversal.help.Table.AvailableWidthProvider
-import overflowdb.traversal.help.{DocFinder, Table}
+import flatgraph.help.DocFinder.*
+import flatgraph.help.Table.{AvailableWidthProvider, Row}
+import flatgraph.help.{DocFinder, Table}
 
 object Help {
 
+  /** allows users to extend the help table with additional entries */
+  val additionalHelpEntries = Seq.newBuilder[Tuple3[String, String, String]]
+
   def overview(clazz: Class[?])(using AvailableWidthProvider): String = {
     val columnNames = List("command", "description", "example")
-    val rows = DocFinder
-      .findDocumentedMethodsOf(clazz)
-      .map { case StepDoc(_, funcName, doc) =>
-        List(funcName, doc.info, doc.example)
-      }
-      .toList ++ List(runRow)
+
+    val rows = Seq.newBuilder[Row]
+    rows += runRow
+    DocFinder.findDocumentedMethodsOf(clazz).foreach { case StepDoc(_, funcName, doc) =>
+      rows += List(funcName, doc.info, doc.example)
+    }
+    additionalHelpEntries.result().foreach { case (a, b, c) =>
+      rows += List(a, b, c)
+    }
 
     val header = formatNoQuotes("""
       |
@@ -27,7 +33,7 @@ object Help {
       |
       |
       |""".stripMargin)
-    header + "\n" + Table(columnNames, rows.sortBy(_.head)).render
+    header + "\n" + Table(columnNames, rows.result().sortBy(_.head)).render
   }
 
   def format(text: String): String = {
@@ -45,8 +51,7 @@ object Help {
     List("run", "Run analyzer on active CPG", "run.securityprofile")
 
   // Since `run` is generated dynamically, it's not picked up when looking
-  // through methods via reflection, and therefore, we are adding
-  // it manually.
+  // through methods via reflection, and therefore, we are adding it manually.
   def runLongHelp: String =
     Help.format("""
         |
@@ -60,11 +65,10 @@ object Help {
       }
       .mkString("\n")
 
-    val overview = Help.overview(clazz)
     s"""
        | class Helper() {
        |   def run: String = Help.runLongHelp
-       |   override def toString: String = \"\"\"$overview\"\"\"
+       |   override def toString: String = Help.overview(classOf[${clazz.getName}])
        |
        |  $membersCode
        | }

console/src/main/scala/io/joern/console/PluginManager.scala

@@ -1,5 +1,5 @@
 package io.joern.console
-import better.files.Dsl._
+import better.files.Dsl.*
 import better.files.File
 import better.files.File.apply
 

console/src/main/scala/io/joern/console/Run.scala

@@ -6,7 +6,7 @@ import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext}
 import org.reflections8.Reflections
 import org.reflections8.util.{ClasspathHelper, ConfigurationBuilder}
 
-import scala.jdk.CollectionConverters._
+import scala.jdk.CollectionConverters.*
 
 object Run {
 
@@ -22,7 +22,7 @@ object Run {
             query.store()(builder)
           }
         }
-        runPass(pass, context)
+        pass.createAndApply()
       }
     })
   }
@@ -64,7 +64,7 @@ object Run {
          |
          |val opts = new OptsDynamic()
          |
-         | import _root_.overflowdb.BatchedUpdate.DiffGraphBuilder
+         | import _root_.io.shiftleft.codepropertygraph.generated.DiffGraphBuilder
          | implicit def _diffGraph: DiffGraphBuilder = opts.commit.diffGraphBuilder
          | def diffGraph = _diffGraph
          |""".stripMargin
@@ -75,7 +75,7 @@ object Run {
 
     val toStringCode =
       s"""
-         |  import overflowdb.traversal.help.Table
+         |  import flatgraph.help.Table
          |  override def toString() : String = {
          |    val columnNames = List("name", "description")
          |    val rows =

console/src/main/scala/io/joern/console/cpgcreation/CpgGenerator.scala

@@ -3,7 +3,7 @@ package io.joern.console.cpgcreation
 import better.files.File
 import io.shiftleft.codepropertygraph.generated.Cpg
 
-import scala.sys.process._
+import scala.sys.process.*
 import scala.util.Try
 
 /** A CpgGenerator generates Code Property Graphs from code. Each supported language implements a Generator, e.g.,

console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala

@@ -1,11 +1,10 @@
 package io.joern.console.cpgcreation
 
-import better.files.Dsl._
+import better.files.Dsl.*
 import better.files.File
-import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig}
+import io.shiftleft.codepropertygraph.cpgloading.CpgLoader
 import io.shiftleft.codepropertygraph.generated.Languages
-import io.joern.console.ConsoleConfig
-import overflowdb.Config
+import io.joern.console.{ConsoleConfig, CpgConverter}
 
 import java.nio.file.Path
 import scala.util.Try
@@ -60,12 +59,12 @@ class CpgGeneratorFactory(config: ConsoleConfig) {
       generator.generate(inputPath, outputPath).map(File(_))
     outputFileOpt.map { outFile =>
       val parentPath = outFile.parent.path.toAbsolutePath
-      if (isZipFile(outFile)) {
+      if (CpgLoader.isProtoFormat(outFile.path)) {
         report("Creating database from bin.zip")
         val srcFilename = outFile.path.toAbsolutePath.toString
         val dstFilename = parentPath.resolve("cpg.bin").toAbsolutePath.toString
         // MemoryHelper.hintForInsufficientMemory(srcFilename).map(report)
-        convertProtoCpgToOverflowDb(srcFilename, dstFilename)
+        convertProtoCpgToFlatgraph(srcFilename, dstFilename)
       } else {
         report("moving cpg.bin.zip to cpg.bin because it is already a database file")
         val srcPath = parentPath.resolve("cpg.bin.zip")
@@ -77,18 +76,13 @@ class CpgGeneratorFactory(config: ConsoleConfig) {
     }
   }
 
-  def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit = {
-    val odbConfig = Config.withDefaults.withStorageLocation(dstFilename)
-    val config    = CpgLoaderConfig.withDefaults.doNotCreateIndexesOnLoad.withOverflowConfig(odbConfig)
-    CpgLoader.load(srcFilename, config).close
-    File(srcFilename).delete()
-  }
+  @deprecated("method got renamed to `convertProtoCpgToFlatgraph, please use that instead", "joern v3")
+  def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit =
+    convertProtoCpgToFlatgraph(srcFilename, dstFilename)
 
-  def isZipFile(file: File): Boolean = {
-    val bytes = file.bytes
-    Try {
-      bytes.next() == 'P' && bytes.next() == 'K'
-    }.getOrElse(false)
+  def convertProtoCpgToFlatgraph(srcFilename: String, dstFilename: String): Unit = {
+    CpgConverter.convertProtoCpgToFlatgraph(srcFilename, dstFilename)
+    File(srcFilename).delete()
   }
 
   private def report(str: String): Unit = System.err.println(str)