Scala 3 Upgrade (#628)

* Scala 3 Upgrade Upgraded Joern and as minimal dependencies as possible to their compatible Scala 3 variant. Note: [Mockito has no Scala 3 compatibility](https://index.scala-lang.org/mockito/mockito-scala) (yet) so we will have to disable it or find an alternative. The bad news is that this project has no clear path to [Scala 3](mockito/mockito-scala#364 (comment)) * Removing Mockito * Updated Scala format to Scala 3 * Removed Scala 2.13 spec from code quality workflow * Fixed Scala 3 compilation issues * removed snakeyaml dependency --------- Co-authored-by: “Hitesh <hitesh.bedre@privado.com>
Privado-Inc · Jul 11, 2023 · a60ce34 · a60ce34
1 parent 293aefa
commit a60ce34
Show file tree

Hide file tree

Showing 23 changed files with 60 additions and 72 deletions.
diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
@@ -18,7 +18,7 @@ jobs:
           distribution: 'temurin'
           java-version: '17'
       - name: Check formatting
-        run: sbt ++2.13.8 scalafmtCheck test:scalafmtCheck
+        run: sbt scalafmtCheck test:scalafmtCheck
       - run: echo "Previous step failed because code is not formatted. Run 'sbt scalafmt Test/scalafmt'"
         if: ${{ failure() }}
 
@@ -35,6 +35,6 @@ jobs:
           distribution: 'temurin'
           java-version: '17'
       - name: Run unit test
-        run: sbt ++2.13.8 test test:test
+        run: sbt test test:test
       - run: echo "Previous step failed because unit test failed."
         if: ${{ failure() }}
diff --git a/.scalafmt.conf b/.scalafmt.conf
@@ -1,5 +1,5 @@
 version = 3.5.1
-runner.dialect = scala213
+runner.dialect = scala3
 preset = IntelliJ
 maxColumn = 120
 align.preset = true
diff --git a/build.sbt b/build.sbt
@@ -1,60 +1,58 @@
 import sbt.Credentials
 name                     := "privado-core"
 ThisBuild / organization := "ai.privado"
-ThisBuild / scalaVersion := "2.13.8"
+ThisBuild / scalaVersion := "3.3.0"
 ThisBuild / version      := sys.env.getOrElse("BUILD_VERSION", "dev-SNAPSHOT")
 // parsed by project/Versions.scala, updated by updateDependencies.sh
 
-val cpgVersion        = "1.3.612"
-val joernVersion      = "1.2.36"
+val cpgVersion        = "1.4.1"
+val joernVersion      = "2.0.5"
 val overflowdbVersion = "1.179"
 
 //External dependency versions
-val circeVersion   = "0.14.1"
-val jacksonVersion = "2.14.3"
-val mockitoVersion = "1.17.12"
+val circeVersion   = "0.14.2"
+val jacksonVersion = "2.15.2"
+val mockitoVersion = "1.17.14"
 
 lazy val schema         = Projects.schema
 lazy val domainClasses  = Projects.domainClasses
 lazy val schemaExtender = Projects.schemaExtender
 
 dependsOn(domainClasses)
 libraryDependencies ++= Seq(
-  "com.github.pathikrit"            %% "better-files"            % "3.9.1",
-  "com.github.scopt"                %% "scopt"                   % "3.7.1",
+  "com.github.pathikrit"            %% "better-files"            % "3.9.2",
+  "com.github.scopt"                %% "scopt"                   % "4.1.0",
   "io.joern"                        %% "x2cpg"                   % Versions.joern,
   "io.joern"                        %% "javasrc2cpg"             % Versions.joern,
   "io.joern"                        %% "pysrc2cpg"               % Versions.joern,
   "io.joern"                        %% "rubysrc2cpg"             % Versions.joern,
   "io.joern"                        %% "joern-cli"               % Versions.joern,
   "io.joern"                        %% "semanticcpg"             % Versions.joern,
   "io.joern"                        %% "semanticcpg"             % Versions.joern % Test classifier "tests",
-  "org.scalatest"                   %% "scalatest"               % "3.1.1"        % Test,
-  "org.mockito"                     %% "mockito-scala"           % mockitoVersion % Test,
-  "org.mockito"                     %% "mockito-scala-scalatest" % mockitoVersion % Test,
-  "org.mockito"                     %% "mockito-scala-specs2"    % mockitoVersion % Test,
+  "org.scalatest"                   %% "scalatest"               % "3.2.16"       % Test,
   "io.circe"                        %% "circe-core"              % circeVersion,
   "io.circe"                        %% "circe-generic"           % circeVersion,
   "io.circe"                        %% "circe-parser"            % circeVersion,
-  "io.circe"                        %% "circe-yaml"              % circeVersion,
+  // NOTE: circe-yaml currently only goes until 0.14.2 (Last checked 06/07/2023)
+  "io.circe"                        %% "circe-yaml"              % circeVersion exclude("org.yaml", "snakeyaml"),
   "com.lihaoyi"                     %% "upickle"                 % "2.0.0",
   "com.lihaoyi"                     %% "requests"                % "0.7.0",
   "org.scala-lang.modules"          %% "scala-xml"               % "2.1.0",
   "commons-io"                       % "commons-io"              % "2.11.0",
   "com.networknt"                    % "json-schema-validator"   % "1.0.72",
   "com.fasterxml.jackson.module"    %% "jackson-module-scala"    % jacksonVersion,
-  "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % jacksonVersion,
+  "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % jacksonVersion exclude("org.yaml", "snakeyaml"),
   "com.github.wnameless.json"        % "json-flattener"          % "0.14.0",
   "org.apache.logging.log4j"         % "log4j-core"              % "2.19.0",
   "org.apache.logging.log4j"         % "log4j-slf4j2-impl"       % "2.19.0"       % Runtime,
   "org.apache.poi"                   % "poi-ooxml"               % "5.2.2",
   "com.github.jsqlparser"            % "jsqlparser"              % "4.6",
   "org.apache.maven"                 % "maven-model"             % "3.9.0",
   "net.sourceforge.htmlunit"         % "htmlunit"                % "2.70.0",
-  "org.yaml"                         % "snakeyaml"               % "1.29",
+  "org.yaml"                         % "snakeyaml"               % "1.33",
   "org.scala-lang"                   % "scala-reflect"           % "2.13.8",
   "org.scala-lang"                   % "scala-compiler"          % "2.13.8",
-  "com.iheart"                      %% "ficus"                   % "1.4.7" exclude ("com.typesafe", "config")
+  "com.iheart"                      %% "ficus"                   % "1.5.2" exclude ("com.typesafe", "config")
 )
 
 ThisBuild / Compile / scalacOptions ++= Seq("-feature", "-deprecation", "-language:implicitConversions")
@@ -70,7 +68,7 @@ ThisBuild / resolvers ++= Seq(
   "Sonatype OSS" at "https://oss.sonatype.org/content/repositories/public",
   "Gradle Releases" at "https://repo.gradle.org/gradle/libs-releases"
 ) ++ Resolver.sonatypeOssRepos("snapshots")
-lazy val astGenDlUrl       = "https://github.com/joernio/astgen/releases/download/v2.14.0/"
+lazy val astGenDlUrl       = "https://github.com/joernio/astgen/releases/download/v3.1.0/"
 lazy val astGenBinaryNames = Seq("astgen-linux", "astgen-macos", "astgen-win.exe", "astgen-macos-arm")
 
 lazy val astGenDlTask = taskKey[Unit](s"Download astgen binaries")

diff --git a/src/main/scala/ai/privado/dataflow/Dataflow.scala b/src/main/scala/ai/privado/dataflow/Dataflow.scala
@@ -70,13 +70,11 @@ class Dataflow(cpg: Cpg) {
     println(s"${TimeMetric.getNewTimeAndSetItToStageLast()} - --no of source nodes - ${sources.size}")
     println(s"${TimeMetric.getNewTimeAndSetItToStageLast()} - --no of sinks nodes - ${sinks.size}")
 
-    if (sources.isEmpty || sinks.isEmpty)
-      Map[String, Path]()
+    if (sources.isEmpty || sinks.isEmpty) Map[String, Path]()
     else {
       println(s"${TimeMetric.getNewTimeAndSetItToStageLast()} - --Finding flows invoked...")
       val dataflowPathsUnfiltered = {
-        if (privadoScanConfig.disable2ndLevelClosure)
-          sinks.reachableByFlows(sources).l
+        if (privadoScanConfig.disable2ndLevelClosure) sinks.reachableByFlows(sources).l
         else {
           val firstLevelSources =
             sources.or(

diff --git a/src/main/scala/ai/privado/entrypoint/CommandParser.scala b/src/main/scala/ai/privado/entrypoint/CommandParser.scala
@@ -255,7 +255,7 @@ object CommandParser {
             println(OParser.usage(parser))
             exit(1)
         }
-        commandProcessor.config = config
+        commandProcessor.withConfig(config)
         Some(commandProcessor)
       case _ =>
         println(OParser.usage(parser))

diff --git a/src/main/scala/ai/privado/entrypoint/CommandProcessor.scala b/src/main/scala/ai/privado/entrypoint/CommandProcessor.scala
@@ -23,6 +23,13 @@
 package ai.privado.entrypoint
 
 trait CommandProcessor {
-  var config: PrivadoInput
+
+  var config: PrivadoInput = null
+
+  def withConfig(value: PrivadoInput): CommandProcessor = {
+    this.config = value
+    this
+  }
+
   def process(): Either[String, Unit]
 }
diff --git a/src/main/scala/ai/privado/entrypoint/RuleValidator.scala b/src/main/scala/ai/privado/entrypoint/RuleValidator.scala
@@ -9,8 +9,6 @@ object RuleValidator extends CommandProcessor {
 
   private val logger = LoggerFactory.getLogger(this.getClass)
 
-  override var config: PrivadoInput = _
-
   override def process(): Either[String, Unit] = {
     println("Starting rule validations ...")
     validateRules() match {

diff --git a/src/main/scala/ai/privado/entrypoint/ScanProcessor.scala b/src/main/scala/ai/privado/entrypoint/ScanProcessor.scala
@@ -378,5 +378,4 @@ object ScanProcessor extends CommandProcessor {
     sourceLocation.listRecursively.count(f => f.extension(toLowerCase = true).toString.contains(".java")) > 0
   }
 
-  override var config: PrivadoInput = _
 }
diff --git a/src/main/scala/ai/privado/entrypoint/UploadProcessor.scala b/src/main/scala/ai/privado/entrypoint/UploadProcessor.scala
@@ -50,5 +50,4 @@ object UploadProcessor extends CommandProcessor {
         Left("Output file does not exist.")
     }
   }
-  override var config: PrivadoInput = _
 }
diff --git a/src/main/scala/ai/privado/exporter/ExporterUtility.scala b/src/main/scala/ai/privado/exporter/ExporterUtility.scala
@@ -122,8 +122,7 @@ object ExporterUtility {
                 currentNodeModel
             }
         }
-      } else
-        currentNodeModel
+      } else currentNodeModel
     }
   }
 
@@ -168,8 +167,7 @@ object ExporterUtility {
       }
     }
 
-    if (fileName.equals(Constants.EMPTY) || sample.equals(Constants.EMPTY))
-      None
+    if (fileName.equals(Constants.EMPTY) || sample.equals(Constants.EMPTY)) None
     else {
       val message = {
         if (Iterator(node).isCall.nonEmpty) {

diff --git a/src/main/scala/ai/privado/exporter/SinkExporter.scala b/src/main/scala/ai/privado/exporter/SinkExporter.scala
@@ -142,8 +142,7 @@ class SinkExporter(cpg: Cpg, ruleCache: RuleCache) {
                   .toArray
               }
               apiurls
-            } else
-              Array[String]()
+            } else Array[String]()
           }
           val databaseDetails = DatabaseDetailsCache.getDatabaseDetails(rule.id)
           Some(

diff --git a/src/main/scala/ai/privado/languageEngine/java/passes/read/DatabaseReadUtility.scala b/src/main/scala/ai/privado/languageEngine/java/passes/read/DatabaseReadUtility.scala
@@ -77,8 +77,7 @@ object DatabaseReadUtility {
             .l
         } else
           List()
-      } else
-        List(node)
+      } else List(node)
     }
 
     val sensitiveClassesWithMatchedRules = taggerCache.typeDeclMemberCache

diff --git a/src/main/scala/ai/privado/languageEngine/java/tagger/sink/FeignAPI.scala b/src/main/scala/ai/privado/languageEngine/java/tagger/sink/FeignAPI.scala
@@ -161,8 +161,7 @@ class FeignAPI(cpg: Cpg, ruleCache: RuleCache) {
             .getOrElse(feignFlows.head.elements.head.code.split(" ").last)
       }
       (cpg.typeDecl.name(firstArgument).l, apiLiteral)
-    } else
-      (List[TypeDecl](), "")
+    } else (List[TypeDecl](), "")
   }
 
   /** Tag all the feign api calls which have some url like thing associated with them

diff --git a/src/main/scala/ai/privado/languageEngine/javascript/tagger/sink/JSAPITagger.scala b/src/main/scala/ai/privado/languageEngine/javascript/tagger/sink/JSAPITagger.scala
@@ -41,8 +41,7 @@ class JSAPITagger(cpg: Cpg, ruleCache: RuleCache) extends APITagger(cpg, ruleCac
     scriptTags.foreach(scriptTag => {
       var newRuleIdToUse = ruleInfo.id
       val domain         = getDomainFromTemplates(scriptTag.code)
-      if (ruleInfo.id.equals(Constants.internalAPIRuleId))
-        addRuleTags(builder, scriptTag, ruleInfo, ruleCache)
+      if (ruleInfo.id.equals(Constants.internalAPIRuleId)) addRuleTags(builder, scriptTag, ruleInfo, ruleCache)
       else {
         newRuleIdToUse = ruleInfo.id + "." + domain._2
         ruleCache.setRuleInfo(ruleInfo.copy(id = newRuleIdToUse, name = ruleInfo.name + " " + domain._2))
@@ -77,8 +76,7 @@ class JSAPITagger(cpg: Cpg, ruleCache: RuleCache) extends APITagger(cpg, ruleCac
       var newRuleIdToUse = ruleInfo.id
       val domain         = getDomainFromTemplates(link.code)
       val callTag        = link.astParent
-      if (ruleInfo.id.equals(Constants.internalAPIRuleId))
-        addRuleTags(builder, callTag, ruleInfo, ruleCache)
+      if (ruleInfo.id.equals(Constants.internalAPIRuleId)) addRuleTags(builder, callTag, ruleInfo, ruleCache)
       else {
         newRuleIdToUse = ruleInfo.id + "." + domain._2
         ruleCache.setRuleInfo(ruleInfo.copy(id = newRuleIdToUse, name = ruleInfo.name + " " + domain._2))

diff --git a/src/main/scala/ai/privado/languageEngine/python/semantic/PythonSemanticGenerator.scala b/src/main/scala/ai/privado/languageEngine/python/semantic/PythonSemanticGenerator.scala
@@ -68,8 +68,7 @@ object PythonSemanticGenerator extends SemanticGenerator {
             arg.order - 1
         }
         Some(index.toString + s"${Constants.semanticDelimeter}\"${arg.argumentName.get}\"")
-      } else
-        None
+      } else None
     }.l
     val parameterSemantic = mutable.HashSet[String]()
 

diff --git a/src/main/scala/ai/privado/passes/PropertyParserPass.scala b/src/main/scala/ai/privado/passes/PropertyParserPass.scala
@@ -8,25 +8,25 @@ import io.joern.x2cpg.SourceFiles
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.NewFile
 import org.slf4j.LoggerFactory
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.*
 
 import java.io.{File, StringReader}
 import scala.io.Source
 import java.util.Properties
-import scala.jdk.CollectionConverters._
-import io.circe.parser._
-import io.circe._
+import scala.jdk.CollectionConverters.*
+import io.circe.parser.*
+import io.circe.*
 
 import scala.collection.mutable
-import com.typesafe.config._
+import com.typesafe.config.*
 
 import scala.xml.XML
 import com.github.wnameless.json.flattener.JsonFlattener
 import io.circe.yaml.parser
-import org.yaml.snakeyaml.Yaml
+import org.yaml.snakeyaml.{LoaderOptions, Yaml}
 import org.yaml.snakeyaml.nodes.{MappingNode, Node, NodeTuple, ScalarNode, SequenceNode}
 
-import scala.jdk.CollectionConverters._
+import scala.jdk.CollectionConverters.*
 import ai.privado.model.Language
 import ai.privado.tagger.PrivadoParallelCpgPass
 import org.yaml.snakeyaml.constructor.SafeConstructor
@@ -175,15 +175,15 @@ class PropertyParserPass(cpg: Cpg, projectRoot: String, ruleCache: RuleCache, la
     try {
       val yamlContent = better.files.File(file).contentAsString // Read the YAML file content as a string
 
-      val yaml                                = new Yaml(new SafeConstructor())
+      val yaml                                = new Yaml(new SafeConstructor(LoaderOptions()))
       val rootNode                            = yaml.compose(new StringReader(yamlContent))
       var result: List[(String, String, Int)] = List[(String, String, Int)]()
       processNode(rootNode, "")
 
       def processNode(node: Node, path: String): Unit = {
         node match {
           case mappingNode: MappingNode =>
-            mappingNode.getValue.asScala.foreach { nodeTuple: NodeTuple =>
+            mappingNode.getValue.asScala.foreach { (nodeTuple: NodeTuple) =>
               val keyNode   = nodeTuple.getKeyNode.asInstanceOf[ScalarNode]
               val valueNode = nodeTuple.getValueNode
               val fullPath  = if (path.isEmpty) keyNode.getValue else s"$path.${keyNode.getValue}"

diff --git a/src/main/scala/ai/privado/passes/SQLParser.scala b/src/main/scala/ai/privado/passes/SQLParser.scala
@@ -26,7 +26,7 @@ package ai.privado.passes
 import ai.privado.cache.RuleCache
 import ai.privado.model.sql.{SQLColumn, SQLQuery}
 import ai.privado.tagger.PrivadoParallelCpgPass
-import ai.privado.utility.{SQLParser, Utilities}
+import ai.privado.utility.{SQLParser => UtilitySQLParser, Utilities}
 import better.files._
 import io.joern.x2cpg.SourceFiles
 import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes}
@@ -80,7 +80,7 @@ class SQLParser(cpg: Cpg, projectRoot: String, ruleCache: RuleCache) extends Pri
         val query           = queryWthLine._1
         val queryLineNumber = queryWthLine._2
         try {
-          SQLParser.parseSqlQuery(query) match {
+          UtilitySQLParser.parseSqlQuery(query) match {
             case Some(parsedQueryList) =>
               parsedQueryList.zipWithIndex.foreach { case (parsedQueryItem: SQLQuery, queryOrder) =>
                 buildAndReturnIndividualQueryNode(

diff --git a/src/main/scala/ai/privado/rulevalidator/YamlFileValidator.scala b/src/main/scala/ai/privado/rulevalidator/YamlFileValidator.scala
@@ -73,11 +73,11 @@ object YamlFileValidator {
           .extension(toLowerCase = true)
           .toString
           .contains(".yaml")
-          ||
-            subDir
-              .extension(toLowerCase = true)
-              .toString
-              .contains(".yml")
+        ||
+        subDir
+          .extension(toLowerCase = true)
+          .toString
+          .contains(".yml")
       )
       .flatMap(ruleFile => {
         validateRuleFile(ruleFile, CommandConstants.VALIDATE) match {

diff --git a/src/main/scala/ai/privado/script/ExternalScalaScriptRunner.scala b/src/main/scala/ai/privado/script/ExternalScalaScriptRunner.scala
@@ -32,21 +32,21 @@ import java.io.File
 import scala.collection.mutable
 import scala.io.{BufferedSource, Source}
 import scala.reflect.runtime.{currentMirror, universe}
-import scala.tools.reflect.ToolBox
+import scala.tools.reflect.{FastTrack, ToolBox}
 
 abstract class ExternalScript {
   def process(cpg: Cpg, output: mutable.LinkedHashMap[String, Json]): Unit
 }
 
 case class LoadExternalScript(filePath: String) {
-  val toolbox: ToolBox[universe.type] = currentMirror.mkToolBox()
+  val toolbox: ToolBox[universe.type] = universe.runtimeMirror(getClass.getClassLoader).mkToolBox()
   val sourceFile: BufferedSource      = Source.fromFile(filePath)
   private val fileContents =
     try sourceFile.getLines().mkString("\n")
     finally sourceFile.close()
 
   // The below package import should be similar to the pacakage where ExternalScript abstract case is present
-  private val tree         = toolbox.parse(s"import ai.privado.script._;\n$fileContents")
+  private val tree         = toolbox.parse(s"import ai.privado.script.*;\n$fileContents")
   private val compiledCode = toolbox.compile(tree)
 
   def getFileReference: ExternalScript = compiledCode().asInstanceOf[ExternalScript]

diff --git a/src/main/scala/ai/privado/semantic/SemanticGenerator.scala b/src/main/scala/ai/privado/semantic/SemanticGenerator.scala
@@ -80,8 +80,7 @@ trait SemanticGenerator {
     if (semantic.signature.nonEmpty) {
       val generatedSemantic = "\"" + semantic.signature.trim + "\" " + semantic.flow
       Some(generatedSemantic.trim)
-    } else
-      None
+    } else None
   }
 
   /** Takes sequence of semantic as input and returns the unique semantic by signature which have the longest flow

diff --git a/src/main/scala/ai/privado/tagger/utility/APITaggerUtility.scala b/src/main/scala/ai/privado/tagger/utility/APITaggerUtility.scala
@@ -68,8 +68,7 @@ object APITaggerUtility {
         val apiNode     = flow.elements.last
         // Tag API's when we find a dataflow to them
         var newRuleIdToUse = ruleInfo.id
-        if (ruleInfo.id.equals(Constants.internalAPIRuleId))
-          addRuleTags(builder, apiNode, ruleInfo, ruleCache)
+        if (ruleInfo.id.equals(Constants.internalAPIRuleId)) addRuleTags(builder, apiNode, ruleInfo, ruleCache)
         else {
           val domain = getDomainFromString(literalCode)
           newRuleIdToUse = ruleInfo.id + "." + domain

diff --git a/src/main/scala/ai/privado/utility/SQLParser.scala b/src/main/scala/ai/privado/utility/SQLParser.scala
@@ -134,7 +134,7 @@ object SQLParser {
   }
 
   def getColumns(plainSelect: PlainSelect): List[SQLColumn] = {
-    plainSelect.getSelectItems.asScala.flatMap { item: SelectItem =>
+    plainSelect.getSelectItems.asScala.flatMap { (item: SelectItem) =>
       item.toString match {
         case f: String if f.contains("(") =>
           val function = CCJSqlParserUtil.parseExpression(f).asInstanceOf[Function]

diff --git a/src/main/scala/ai/privado/utility/Utilities.scala b/src/main/scala/ai/privado/utility/Utilities.scala
@@ -166,8 +166,7 @@ object Utilities {
             }
           }
           .mkString("\n")
-      } else
-        ""
+      } else ""
     } catch {
       case e: Exception =>
         logger.debug("Error : ", e)