support Spark 3.3.1 version

Signed-off-by: Pawel Leszczynski <leszczynski.pawel@gmail.com>
OpenLineage · Nov 3, 2022 · 825f50f · 825f50f
1 parent f598337
commit 825f50f
Show file tree

Hide file tree

Showing 6 changed files with 22 additions and 5 deletions.
diff --git a/.circleci/workflows/openlineage-java.yml b/.circleci/workflows/openlineage-java.yml
@@ -22,11 +22,11 @@ workflows:
       - build-integration-spark:
           matrix:
             parameters:
-              spark-version: [ '2.4.6', '3.1.3', '3.2.2', '3.3.0' ]
+              spark-version: [ '2.4.6', '3.1.3', '3.2.2', '3.3.1' ]
       - integration-test-integration-spark:
           matrix:
             parameters:
-              spark-version: [ '2.4.6', '3.1.3', '3.2.2', '3.3.0' ]
+              spark-version: [ '2.4.6', '3.1.3', '3.2.2', '3.3.1' ]
           requires:
             - build-integration-spark
       - publish-snapshot-integration-spark:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,9 @@
 # Changelog
 
 ## [Unreleased](https://github.com/OpenLineage/OpenLineage/compare/0.16.0...HEAD)
+
+* Support latest Spark 3.3.1 [`#1183`](https://github.com/OpenLineage/OpenLineage/pull/1183) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)
+
 ## [0.16.0](https://github.com/OpenLineage/OpenLineage/compare/0.15.1...0.16.0) - 2022-11-1
 ### Added
 * Airflow: add `dag_run` information to Airflow version run facet [`#1133`](https://github.com/OpenLineage/OpenLineage/pull/1133) [@fm100](https://github.com/fm100)  

diff --git a/integration/README.md b/integration/README.md
@@ -4,6 +4,6 @@
 |Apache Airflow|2.1.*, 2.2.*, 2.3.*|https://github.com/apache/airflow/releases|[README](./airflow/README.md)|Support for Airflow 1.x is deprecated and will be discontinued after September 30th, 2022|
 |Dagster|0.13.8+|https://github.com/dagster-io/dagster/releases|[README](./dagster/README.md)| |
 |dbt|0.20+, 1.3|https://github.com/dbt-labs/dbt-core/releases|[README](./dbt/README.md)| |
-|Apache Spark|2.4.6, 3.1.2, 3.2.1+, 3.3.0|https://github.com/apache/spark/tags|[README](./spark/README.md)| |
+|Apache Spark|2.4.6, 3.1.2, 3.2.1+, 3.3.1|https://github.com/apache/spark/tags|[README](./spark/README.md)| |
 |Apache Flink|1.14.5, 1.15.1|https://github.com/apache/flink/tags|[README](./spark/README.md)| Flink support is currently experimental |
 
diff --git a/integration/spark/app/build.gradle b/integration/spark/app/build.gradle
@@ -64,6 +64,13 @@ ext {
     versions = versionsMap[shortVersion]
 }
 
+configurations.all { // https://github.com/apache/spark/pull/38355 - can be remove for Spark 3.3.2
+    resolutionStrategy {
+        // https://github.com/FasterXML/jackson-databind/issues/3627
+        force "com.fasterxml.jackson:jackson-bom:$jacksonVersion"
+    }
+}
+
 dependencies {
     compileOnly "org.projectlombok:lombok:${lombokVersion}"
     annotationProcessor "org.projectlombok:lombok:${lombokVersion}"

diff --git a/integration/spark/gradle.properties b/integration/spark/gradle.properties
@@ -1,4 +1,4 @@
 jdk8.build=true
 version=0.17.0-SNAPSHOT
-spark.version=3.3.0
+spark.version=3.3.1
 org.gradle.jvmargs=-Xmx1G
diff --git a/integration/spark/spark33/build.gradle b/integration/spark/spark33/build.gradle
@@ -40,11 +40,18 @@ repositories {
 archivesBaseName='openlineage-spark-spark3'
 
 ext {
-    sparkVersion = '3.3.0'
+    sparkVersion = '3.3.1'
     jacksonVersion = '2.10.0'
     lombokVersion = '1.18.20'
 }
 
+configurations.all { // https://github.com/apache/spark/pull/38355 - can be remove for Spark 3.3.2
+    resolutionStrategy {
+        // https://github.com/FasterXML/jackson-databind/issues/3627
+        force "com.fasterxml.jackson:jackson-bom:$jacksonVersion"
+    }
+}
+
 dependencies {
     compileOnly "org.projectlombok:lombok:${lombokVersion}"
     annotationProcessor "org.projectlombok:lombok:${lombokVersion}"