[Issue 9728] Upgrade GRPC and bookkeeper (apache#9729)

* Upgraded bookkeeper to 4.13.0; grpc to 1.33 * updated LICENSE * fixed the rest of the license check * no idea why license check results on CI are different from local :/ * one more license fix * Integration tests failed to start container/service there because of java.lang.NoClassDefFoundError: io/perfmark/PerfMark * another try to get the permark.jar in * another take on perfmark.jar and integration tests * add perfmark.jar to LICENSE
dlg99 · Dec 1, 2021 · 3cc6aaf · 3cc6aaf
1 parent 59756f2
commit 3cc6aaf
Show file tree

Hide file tree

Showing 15 changed files with 131 additions and 60 deletions.
diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml
@@ -182,6 +182,12 @@
       <artifactId>grpc-all</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>io.perfmark</groupId>
+      <artifactId>perfmark-api</artifactId>
+      <scope>compile</scope>
+    </dependency>
+
     <!-- bookkeeper http server -->
     <dependency>
       <groupId>org.apache.bookkeeper.http</groupId>

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -320,7 +320,11 @@ The Apache Software License, Version 2.0
      - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.11.1.jar
      - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.11.1.jar
      - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.11.1.jar
+     - org.codehaus.jackson-jackson-core-asl-1.9.11.jar
+     - org.codehaus.jackson-jackson-mapper-asl-1.9.11.jar
  * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.6.2.jar
+ * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.1.jar
+ * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-1.17.0.jar
  * Joda -- org.joda-joda-convert-2.2.1.jar
  * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar
  * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-1.12.0.jar
@@ -396,32 +400,32 @@ The Apache Software License, Version 2.0
     - org.apache.logging.log4j-log4j-web-2.10.0.jar
  * Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar
  * BookKeeper
-    - org.apache.bookkeeper-bookkeeper-common-4.12.0.jar
-    - org.apache.bookkeeper-bookkeeper-common-allocator-4.12.0.jar
-    - org.apache.bookkeeper-bookkeeper-proto-4.12.0.jar
-    - org.apache.bookkeeper-bookkeeper-server-4.12.0.jar
-    - org.apache.bookkeeper-bookkeeper-tools-framework-4.12.0.jar
-    - org.apache.bookkeeper-circe-checksum-4.12.0.jar
-    - org.apache.bookkeeper-cpu-affinity-4.12.0.jar
-    - org.apache.bookkeeper-statelib-4.12.0.jar
-    - org.apache.bookkeeper-stream-storage-api-4.12.0.jar
-    - org.apache.bookkeeper-stream-storage-common-4.12.0.jar
-    - org.apache.bookkeeper-stream-storage-java-client-4.12.0.jar
-    - org.apache.bookkeeper-stream-storage-java-client-base-4.12.0.jar
-    - org.apache.bookkeeper-stream-storage-proto-4.12.0.jar
-    - org.apache.bookkeeper-stream-storage-server-4.12.0.jar
-    - org.apache.bookkeeper-stream-storage-service-api-4.12.0.jar
-    - org.apache.bookkeeper-stream-storage-service-impl-4.12.0.jar
-    - org.apache.bookkeeper.http-http-server-4.12.0.jar
-    - org.apache.bookkeeper.http-vertx-http-server-4.12.0.jar
-    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.12.0.jar
-    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.12.0.jar
-    - org.apache.bookkeeper.tests-stream-storage-tests-common-4.12.0.jar
-    - org.apache.distributedlog-distributedlog-common-4.12.0.jar
-    - org.apache.distributedlog-distributedlog-core-4.12.0-tests.jar
-    - org.apache.distributedlog-distributedlog-core-4.12.0.jar
-    - org.apache.distributedlog-distributedlog-protocol-4.12.0.jar
-    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.12.0.jar
+    - org.apache.bookkeeper-bookkeeper-common-4.13.0.jar
+    - org.apache.bookkeeper-bookkeeper-common-allocator-4.13.0.jar
+    - org.apache.bookkeeper-bookkeeper-proto-4.13.0.jar
+    - org.apache.bookkeeper-bookkeeper-server-4.13.0.jar
+    - org.apache.bookkeeper-bookkeeper-tools-framework-4.13.0.jar
+    - org.apache.bookkeeper-circe-checksum-4.13.0.jar
+    - org.apache.bookkeeper-cpu-affinity-4.13.0.jar
+    - org.apache.bookkeeper-statelib-4.13.0.jar
+    - org.apache.bookkeeper-stream-storage-api-4.13.0.jar
+    - org.apache.bookkeeper-stream-storage-common-4.13.0.jar
+    - org.apache.bookkeeper-stream-storage-java-client-4.13.0.jar
+    - org.apache.bookkeeper-stream-storage-java-client-base-4.13.0.jar
+    - org.apache.bookkeeper-stream-storage-proto-4.13.0.jar
+    - org.apache.bookkeeper-stream-storage-server-4.13.0.jar
+    - org.apache.bookkeeper-stream-storage-service-api-4.13.0.jar
+    - org.apache.bookkeeper-stream-storage-service-impl-4.13.0.jar
+    - org.apache.bookkeeper.http-http-server-4.13.0.jar
+    - org.apache.bookkeeper.http-vertx-http-server-4.13.0.jar
+    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.13.0.jar
+    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.13.0.jar
+    - org.apache.bookkeeper.tests-stream-storage-tests-common-4.13.0.jar
+    - org.apache.distributedlog-distributedlog-common-4.13.0.jar
+    - org.apache.distributedlog-distributedlog-core-4.13.0-tests.jar
+    - org.apache.distributedlog-distributedlog-core-4.13.0.jar
+    - org.apache.distributedlog-distributedlog-protocol-4.13.0.jar
+    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.13.0.jar
   * Apache HTTP Client
     - org.apache.httpcomponents-httpclient-4.5.5.jar
     - org.apache.httpcomponents-httpcore-4.4.9.jar
@@ -455,23 +459,32 @@ The Apache Software License, Version 2.0
  * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.3.4.jar
  * Apache Thrifth - org.apache.thrift-libthrift-0.12.0.jar
  * OkHttp
-    - com.squareup.okhttp-okhttp-2.5.0.jar
+    - com.squareup.okhttp-okhttp-2.7.4.jar
  * Okio - com.squareup.okio-okio-1.13.0.jar
  * Javassist -- org.javassist-javassist-3.25.0-GA.jar
-  * gRPC
-    - io.grpc-grpc-all-1.18.0.jar
-    - io.grpc-grpc-auth-1.18.0.jar
-    - io.grpc-grpc-context-1.18.0.jar
-    - io.grpc-grpc-core-1.18.0.jar
-    - io.grpc-grpc-netty-1.18.0.jar
-    - io.grpc-grpc-okhttp-1.18.0.jar
-    - io.grpc-grpc-protobuf-1.18.0.jar
-    - io.grpc-grpc-protobuf-lite-1.18.0.jar
-    - io.grpc-grpc-stub-1.18.0.jar
-    - io.grpc-grpc-protobuf-nano-1.18.0.jar
+ * gRPC
+    - io.grpc-grpc-all-1.33.0.jar
+    - io.grpc-grpc-auth-1.33.0.jar
+    - io.grpc-grpc-context-1.33.0.jar
+    - io.grpc-grpc-core-1.33.0.jar
+    - io.grpc-grpc-netty-1.33.0.jar
+    - io.grpc-grpc-okhttp-1.33.0.jar
+    - io.grpc-grpc-protobuf-1.33.0.jar
+    - io.grpc-grpc-protobuf-lite-1.33.0.jar
+    - io.grpc-grpc-stub-1.33.0.jar
+    - io.grpc-grpc-testing-1.33.0.jar
+    - io.grpc-grpc-alts-1.33.0.jar
+    - io.grpc-grpc-api-1.33.0.jar
+    - io.grpc-grpc-grpclb-1.33.0.jar
+    - io.grpc-grpc-netty-shaded-1.33.0.jar
+    - io.grpc-grpc-services-1.33.0.jar
+    - io.grpc-grpc-xds-1.33.0.jar
+  * Perfmark 
+    - io.perfmark-perfmark-api-0.19.0.jar
   * OpenCensus
     - io.opencensus-opencensus-api-0.18.0.jar
-    - io.opencensus-opencensus-contrib-grpc-metrics-0.18.0.jar
+    - io.opencensus-opencensus-contrib-http-util-0.24.0.jar
+    - io.opencensus-opencensus-proto-0.2.0.jar
   * Jodah
     - net.jodah-typetools-0.5.0.jar
   * Apache Avro
@@ -515,10 +528,16 @@ The Apache Software License, Version 2.0
     - io.vertx-vertx-web-common-3.9.7.jar
   * Apache ZooKeeper
     - org.apache.zookeeper-zookeeper-jute-3.5.9.jar
+  * Google HTTP Client
+    - com.google.http-client-google-http-client-jackson2-1.34.0.jar
+    - com.google.http-client-google-http-client-1.34.0.jar
+    - com.google.auto.value-auto-value-annotations-1.7.jar
+    - com.google.re2j-re2j-1.2.jar
 
 BSD 3-clause "New" or "Revised" License
  * Google auth library
-    - com.google.auth-google-auth-library-credentials-0.9.0.jar -- licenses/LICENSE-google-auth-library.txt
+    - com.google.auth-google-auth-library-credentials-0.20.0.jar -- licenses/LICENSE-google-auth-library.txt
+    - com.google.auth-google-auth-library-oauth2-http-0.20.0.jar -- licenses/LICENSE-google-auth-library.txt
  * LevelDB -- (included in org.rocksdb.*.jar) -- licenses/LICENSE-LevelDB.txt
  * JSR305 -- com.google.code.findbugs-jsr305-3.0.2.jar -- licenses/LICENSE-JSR305.txt
 
@@ -531,17 +550,13 @@ MIT License
     - org.slf4j-jul-to-slf4j-1.7.25.jar
     - org.slf4j-slf4j-api-1.7.25.jar
     - org.slf4j-jcl-over-slf4j-1.7.25.jar
- * Animal Sniffer Annotations
-    - org.codehaus.mojo-animal-sniffer-annotations-1.17.jar
  * The Checker Framework
- 	- org.checkerframework-checker-compat-qual-2.5.2.jar
     - org.checkerframework-checker-qual-3.5.0.jar
 
 Protocol Buffers License
  * Protocol Buffers
    - com.google.protobuf-protobuf-java-3.11.4.jar -- licenses/LICENSE-protobuf.txt
    - com.google.protobuf-protobuf-java-util-3.11.4.jar -- licenses/LICENSE-protobuf.txt
-   - com.google.protobuf.nano-protobuf-javanano-3.0.0-alpha-5.jar -- licenses/LICENSE-protobuf.txt
 
 CDDL-1.1 -- licenses/LICENSE-CDDL-1.1.txt
  * Java Annotations API

diff --git a/distribution/server/src/assemble/NOTICE.bin.txt b/distribution/server/src/assemble/NOTICE.bin.txt
@@ -201,7 +201,6 @@ http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html
     * Performance and memory usage improvement
 
 ------------------------------------------------------------------------------------
-- org.checkerframework-checker-compat-qual-2.5.2.jar
 
 A few parts of the Checker Framework have more permissive licenses.
 

diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml
@@ -38,6 +38,11 @@
       <type>pom</type>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>io.perfmark</groupId>
+      <artifactId>perfmark-api</artifactId>
+      <scope>runtime</scope>
+    </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>pulsar-offloader-distribution</artifactId>

diff --git a/pom.xml b/pom.xml
@@ -103,7 +103,7 @@ flexible messaging model and an intuitive client API.</description>
     <!-- apache commons -->
     <commons-compress.version>1.19</commons-compress.version>
 
-    <bookkeeper.version>4.12.0</bookkeeper.version>
+    <bookkeeper.version>4.13.0</bookkeeper.version>
     <zookeeper.version>3.5.9</zookeeper.version>
     <netty.version>4.1.60.Final</netty.version>
     <netty-tc-native.version>2.0.36.Final</netty-tc-native.version>
@@ -131,8 +131,8 @@ flexible messaging model and an intuitive client API.</description>
     <protobuf2.version>2.4.1</protobuf2.version>
     <protobuf3.version>3.11.4</protobuf3.version>
     <protoc3.version>${protobuf3.version}</protoc3.version>
-    <grpc.version>1.18.0</grpc.version>
-
+    <grpc.version>1.33.0</grpc.version>
+    <perfmark.version>0.19.0</perfmark.version>
     <protoc-gen-grpc-java.version>${grpc.version}</protoc-gen-grpc-java.version>
     <gson.version>2.8.6</gson.version>
     <sketches.version>0.8.3</sketches.version>
@@ -879,6 +879,19 @@ flexible messaging model and an intuitive client API.</description>
         <version>${grpc.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>io.perfmark</groupId>
+        <artifactId>perfmark-api</artifactId>
+        <version>${perfmark.version}</version>
+        <scope>runtime</scope>
+        <exclusions>
+          <exclusion>
+            <artifactId>error_prone_annotations</artifactId>
+            <groupId>com.google.errorprone</groupId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+
       <dependency>
 
         <groupId>io.grpc</groupId>

diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml
@@ -82,6 +82,7 @@
                   <include>net.java.dev.jna:*</include>
                   <include>com.carrotsearch:*</include>
                   <include>io.prometheus:*</include>
+                  <include>io.perfmark:*</include>
                   <include>com.github.ben-manes.caffeine:*</include>
                   <include>org.glassfish.jersey.*:*</include>
                   <include>org.rocksdb:*</include>

diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml
@@ -87,6 +87,7 @@
                   <include>org.glassfish.hk2*:*</include>
                   <include>com.fasterxml.jackson.*:*</include>
                   <include>io.grpc:*</include>
+                  <include>io.perfmark:*</include>
                   <include>com.yahoo.datasketches:*</include>
                   <include>com.squareup.*:*</include>
                   <include>com.google.*:*</include>

diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml
@@ -134,6 +134,7 @@
                   <include>org.glassfish.hk2*:*</include>
                   <include>com.fasterxml.jackson.*:*</include>
                   <include>io.grpc:*</include>
+                  <include>io.perfmark:*</include>
                   <include>com.yahoo.datasketches:*</include>
                   <include>io.netty:*</include>
                   <include>com.squareup.*:*</include>

diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml
@@ -123,6 +123,7 @@
                   <include>com.fasterxml.jackson.core:jackson-core</include>
                   <include>com.fasterxml.jackson.dataformat</include>
                   <include>io.netty:*</include>
+                  <include>io.perfmark:*</include>
                   <include>org.eclipse.jetty:*</include>
                   <include>com.yahoo.datasketches:*</include>
                   <include>commons-*:*</include>

diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml
@@ -109,6 +109,12 @@
       <artifactId>grpc-all</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>io.perfmark</groupId>
+      <artifactId>perfmark-api</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.bookkeeper</groupId>
       <artifactId>bookkeeper-common</artifactId>

diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml
@@ -132,6 +132,10 @@
                                     <pattern>io.grpc</pattern>
                                     <shadedPattern>org.apache.pulsar.functions.runtime.shaded.io.grpc</shadedPattern>
                                 </relocation>
+                                <relocation>
+                                    <pattern>io.perfmark</pattern>
+                                    <shadedPattern>org.apache.pulsar.functions.runtime.shaded.io.perfmark</shadedPattern>
+                                </relocation>
                                 <relocation>
                                     <pattern>org.joda</pattern>
                                     <shadedPattern>org.apache.pulsar.functions.runtime.shaded.org.joda</shadedPattern>

diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml
@@ -58,5 +58,12 @@
             <groupId>io.grpc</groupId>
             <artifactId>grpc-all</artifactId>
         </dependency>
+
+        <dependency>
+            <groupId>io.perfmark</groupId>
+            <artifactId>perfmark-api</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+
     </dependencies>
 </project>
diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml
@@ -61,6 +61,12 @@
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>io.perfmark</groupId>
+            <artifactId>perfmark-api</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+
     </dependencies>
     <build>
         <plugins>

diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
@@ -416,18 +416,18 @@ The Apache Software License, Version 2.0
     - async-http-client-2.12.1.jar
     - async-http-client-netty-utils-2.12.1.jar
   * Apache Bookkeeper
-    - bookkeeper-common-4.12.0.jar
-    - bookkeeper-common-allocator-4.12.0.jar
-    - bookkeeper-proto-4.12.0.jar
-    - bookkeeper-server-4.12.0.jar
-    - bookkeeper-stats-api-4.12.0.jar
-    - bookkeeper-tools-framework-4.12.0.jar
-    - circe-checksum-4.12.0.jar
-    - codahale-metrics-provider-4.12.0jar
-    - cpu-affinity-4.12.0.jar
-    - http-server-4.12.0.jar
-    - prometheus-metrics-provider-4.12.0.jar
-    - codahale-metrics-provider-4.12.0.jar
+    - bookkeeper-common-4.13.0.jar
+    - bookkeeper-common-allocator-4.13.0.jar
+    - bookkeeper-proto-4.13.0.jar
+    - bookkeeper-server-4.13.0.jar
+    - bookkeeper-stats-api-4.13.0.jar
+    - bookkeeper-tools-framework-4.13.0.jar
+    - circe-checksum-4.13.0.jar
+    - codahale-metrics-provider-4.13.0.jar
+    - cpu-affinity-4.13.0.jar
+    - http-server-4.13.0.jar
+    - prometheus-metrics-provider-4.13.0.jar
+    - codahale-metrics-provider-4.13.0.jar
   * Apache Commons
     - commons-cli-1.2.jar
     - commons-codec-1.10.jar

diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml
@@ -61,6 +61,12 @@
       <artifactId>grpc-all</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>io.perfmark</groupId>
+      <artifactId>perfmark-api</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.bookkeeper.stats</groupId>
       <artifactId>prometheus-metrics-provider</artifactId>