Merge branch 'refs/heads/master' into dev/ml_parallel_read

.github/workflows/pulsar-ci.yaml

@@ -1498,12 +1498,16 @@ jobs:
         with:
           action: wait
 
-  # This job is required for pulls to be merged.
+  # This job is required for pulls to be merged. This job is referenced by name in .asf.yaml file in the
+  # protected_branches section for master branch required_status_checks.
   # It depends on all other jobs in this workflow.
-  # It cleans up the binaries in the same job in order to not spin up another runner for basically doing nothing.
+  # This job also cleans up the binaries at the end of the workflow.
   pulsar-ci-checks-completed:
     name: "Pulsar CI checks completed"
-    if: ${{ always() && needs.preconditions.result == 'success' }}
+    # run always, but skip for other repositories than apache/pulsar when a scheduled workflow is cancelled
+    # this is to allow the workflow scheduled jobs to show as cancelled instead of failed since scheduled
+    # jobs are not enabled for other than apache/pulsar repository.
+    if: ${{ always() && !(cancelled() && github.repository != 'apache/pulsar' && github.event_name == 'schedule') }}
     runs-on: ubuntu-22.04
     timeout-minutes: 10
     needs: [
@@ -1521,10 +1525,11 @@ jobs:
     ]
     steps:
       - name: Check that all required jobs were completed successfully
-        if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
+        if: ${{ needs.preconditions.result != 'success' || needs.preconditions.outputs.docs_only != 'true'  }}
         run: |
           if [[ ! ( \
-                   "${{ needs.unit-tests.result }}" == "success" \
+                   "${{ needs.preconditions.result }}" == "success" \
+                && "${{ needs.unit-tests.result }}" == "success" \
                 && "${{ needs.integration-tests.result }}" == "success" \
                 && "${{ needs.system-tests.result }}" == "success" \
                 && "${{ needs.macos-build.result }}" == "success" \

buildtools/pom.xml

@@ -41,7 +41,7 @@
     <maven.compiler.target>1.8</maven.compiler.target>
     <surefire.version>3.1.0</surefire.version>
     <log4j2.version>2.23.1</log4j2.version>
-    <slf4j.version>1.7.32</slf4j.version>
+    <slf4j.version>2.0.13</slf4j.version>
     <testng.version>7.7.1</testng.version>
     <commons-lang3.version>3.11</commons-lang3.version>
     <license-maven-plugin.version>4.1</license-maven-plugin.version>
@@ -100,6 +100,12 @@
       <groupId>org.testng</groupId>
       <artifactId>testng</artifactId>
       <version>${testng.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
@@ -111,7 +117,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-slf4j-impl</artifactId>
+      <artifactId>log4j-slf4j2-impl</artifactId>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>

conf/broker.conf

@@ -88,6 +88,16 @@ advertisedAddress=
 # If true, the real IP addresses of consumers and producers can be obtained when getting topic statistics data.
 haProxyProtocolEnabled=false
 
+# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests.
+webServiceHaProxyProtocolEnabled=false
+
+# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false.
+webServiceTrustXForwardedFor=false
+
+# Add detailed client/remote and server/local addresses and ports to http/https request logging.
+# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled.
+webServiceLogDetailedAddresses=
+
 # Number of threads to config Netty Acceptor. Default is 1
 numAcceptorThreads=
 
@@ -1660,10 +1670,10 @@ s3ManagedLedgerOffloadBucket=
 # For Amazon S3 ledger offload, Alternative endpoint to connect to (useful for testing)
 s3ManagedLedgerOffloadServiceEndpoint=
 
-# For Amazon S3 ledger offload, Max block size in bytes. (64MB by default, 5MB minimum)
+# For Amazon S3 ledger offload, Max block size in bytes. (64MiB by default, 5MiB minimum)
 s3ManagedLedgerOffloadMaxBlockSizeInBytes=67108864
 
-# For Amazon S3 ledger offload, Read buffer size in bytes (1MB by default)
+# For Amazon S3 ledger offload, Read buffer size in bytes (1MiB by default)
 s3ManagedLedgerOffloadReadBufferSizeInBytes=1048576
 
 # For Google Cloud Storage ledger offload, region where offload bucket is located.
@@ -1673,10 +1683,11 @@ gcsManagedLedgerOffloadRegion=
 # For Google Cloud Storage ledger offload, Bucket to place offloaded ledger into
 gcsManagedLedgerOffloadBucket=
 
-# For Google Cloud Storage ledger offload, Max block size in bytes. (64MB by default, 5MB minimum)
-gcsManagedLedgerOffloadMaxBlockSizeInBytes=67108864
+# For Google Cloud Storage ledger offload, Max block size in bytes. (128MiB by default, 5MiB minimum)
+# Since JClouds limits the maximum number of blocks to 32, the maximum size of a ledger is 32 times the block size.
+gcsManagedLedgerOffloadMaxBlockSizeInBytes=134217728
 
-# For Google Cloud Storage ledger offload, Read buffer size in bytes (1MB by default)
+# For Google Cloud Storage ledger offload, Read buffer size in bytes (1MiB by default)
 gcsManagedLedgerOffloadReadBufferSizeInBytes=1048576
 
 # For Google Cloud Storage, path to json file containing service account credentials.

conf/functions_worker.yml

@@ -27,6 +27,16 @@ workerHostname: localhost
 workerPort: 6750
 workerPortTls: 6751
 
+# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests.
+webServiceHaProxyProtocolEnabled: false
+
+# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false.
+webServiceTrustXForwardedFor: false
+
+# Add detailed client/remote and server/local addresses and ports to http/https request logging.
+# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled.
+webServiceLogDetailedAddresses: null
+
 # The Configuration metadata store url
 # Examples:
 # * zk:my-zk-1:2181,my-zk-2:2181,my-zk-3:2181

conf/proxy.conf

@@ -63,6 +63,16 @@ advertisedAddress=
 # If true, the real IP addresses of consumers and producers can be obtained when getting topic statistics data.
 haProxyProtocolEnabled=false
 
+# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests.
+webServiceHaProxyProtocolEnabled=false
+
+# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false.
+webServiceTrustXForwardedFor=false
+
+# Add detailed client/remote and server/local addresses and ports to http/https request logging.
+# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled.
+webServiceLogDetailedAddresses=
+
 # Enables zero-copy transport of data across network interfaces using the splice system call.
 # Zero copy mode cannot be used when TLS is enabled or when proxyLogLevel is > 0.
 proxyZeroCopyModeEnabled=true

conf/standalone.conf

@@ -51,6 +51,16 @@ advertisedAddress=
 # If true, the real IP addresses of consumers and producers can be obtained when getting topic statistics data.
 haProxyProtocolEnabled=false
 
+# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests.
+webServiceHaProxyProtocolEnabled=false
+
+# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false.
+webServiceTrustXForwardedFor=false
+
+# Add detailed client/remote and server/local addresses and ports to http/https request logging.
+# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled.
+webServiceLogDetailedAddresses=
+
 # Number of threads to use for Netty IO. Default is set to 2 * Runtime.getRuntime().availableProcessors()
 numIOThreads=
 

conf/websocket.conf

@@ -46,6 +46,16 @@ statusFilePath=
 # Hostname or IP address the service binds on, default is 0.0.0.0.
 bindAddress=0.0.0.0
 
+# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests.
+webServiceHaProxyProtocolEnabled=false
+
+# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false.
+webServiceTrustXForwardedFor=false
+
+# Add detailed client/remote and server/local addresses and ports to http/https request logging.
+# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled.
+webServiceLogDetailedAddresses=
+
 # Name of the pulsar cluster to connect to
 clusterName=
 

distribution/licenses/LICENSE-Reactive-gRPC.txt

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2019, Salesforce.com, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

distribution/server/pom.xml

@@ -180,7 +180,7 @@
 
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-slf4j-impl</artifactId>
+      <artifactId>log4j-slf4j2-impl</artifactId>
     </dependency>
 
     <dependency>

distribution/server/src/assemble/LICENSE.bin.txt

@@ -262,7 +262,7 @@ The Apache Software License, Version 2.0
      - com.fasterxml.jackson.module-jackson-module-parameter-names-2.14.2.jar
  * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar
  * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar
- * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.9.0.jar
+ * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.17.0.jar
  * Bitbucket -- org.bitbucket.b_c-jose4j-0.9.4.jar
  * Gson
     - com.google.code.gson-gson-2.8.9.jar
@@ -350,40 +350,40 @@ The Apache Software License, Version 2.0
  * Log4J
     - org.apache.logging.log4j-log4j-api-2.23.1.jar
     - org.apache.logging.log4j-log4j-core-2.23.1.jar
-    - org.apache.logging.log4j-log4j-slf4j-impl-2.23.1.jar
+    - org.apache.logging.log4j-log4j-slf4j2-impl-2.23.1.jar
     - org.apache.logging.log4j-log4j-web-2.23.1.jar
  * Java Native Access JNA
     - net.java.dev.jna-jna-jpms-5.12.1.jar
     - net.java.dev.jna-jna-platform-jpms-5.12.1.jar
  * BookKeeper
-    - org.apache.bookkeeper-bookkeeper-common-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-proto-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-server-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.5.jar
-    - org.apache.bookkeeper-circe-checksum-4.16.5.jar
-    - org.apache.bookkeeper-cpu-affinity-4.16.5.jar
-    - org.apache.bookkeeper-statelib-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-api-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-common-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-java-client-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-java-client-base-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-proto-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-server-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-service-api-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-service-impl-4.16.5.jar
-    - org.apache.bookkeeper.http-http-server-4.16.5.jar
-    - org.apache.bookkeeper.http-vertx-http-server-4.16.5.jar
-    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.5.jar
-    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.5.jar
-    - org.apache.distributedlog-distributedlog-common-4.16.5.jar
-    - org.apache.distributedlog-distributedlog-core-4.16.5-tests.jar
-    - org.apache.distributedlog-distributedlog-core-4.16.5.jar
-    - org.apache.distributedlog-distributedlog-protocol-4.16.5.jar
-    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.5.jar
-    - org.apache.bookkeeper-native-io-4.16.5.jar
+    - org.apache.bookkeeper-bookkeeper-common-4.17.0.jar
+    - org.apache.bookkeeper-bookkeeper-common-allocator-4.17.0.jar
+    - org.apache.bookkeeper-bookkeeper-proto-4.17.0.jar
+    - org.apache.bookkeeper-bookkeeper-server-4.17.0.jar
+    - org.apache.bookkeeper-bookkeeper-tools-framework-4.17.0.jar
+    - org.apache.bookkeeper-circe-checksum-4.17.0.jar
+    - org.apache.bookkeeper-cpu-affinity-4.17.0.jar
+    - org.apache.bookkeeper-statelib-4.17.0.jar
+    - org.apache.bookkeeper-stream-storage-api-4.17.0.jar
+    - org.apache.bookkeeper-stream-storage-common-4.17.0.jar
+    - org.apache.bookkeeper-stream-storage-java-client-4.17.0.jar
+    - org.apache.bookkeeper-stream-storage-java-client-base-4.17.0.jar
+    - org.apache.bookkeeper-stream-storage-proto-4.17.0.jar
+    - org.apache.bookkeeper-stream-storage-server-4.17.0.jar
+    - org.apache.bookkeeper-stream-storage-service-api-4.17.0.jar
+    - org.apache.bookkeeper-stream-storage-service-impl-4.17.0.jar
+    - org.apache.bookkeeper.http-http-server-4.17.0.jar
+    - org.apache.bookkeeper.http-vertx-http-server-4.17.0.jar
+    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.17.0.jar
+    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.17.0.jar
+    - org.apache.distributedlog-distributedlog-common-4.17.0.jar
+    - org.apache.distributedlog-distributedlog-core-4.17.0-tests.jar
+    - org.apache.distributedlog-distributedlog-core-4.17.0.jar
+    - org.apache.distributedlog-distributedlog-protocol-4.17.0.jar
+    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.17.0.jar
+    - org.apache.bookkeeper-bookkeeper-slogger-api-4.17.0.jar
+    - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.17.0.jar
+    - org.apache.bookkeeper-native-io-4.17.0.jar
   * Apache HTTP Client
     - org.apache.httpcomponents-httpclient-4.5.13.jar
     - org.apache.httpcomponents-httpcore-4.4.15.jar
@@ -430,23 +430,23 @@ The Apache Software License, Version 2.0
      - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.8.20.jar
      - org.jetbrains-annotations-13.0.jar
  * gRPC
-    - io.grpc-grpc-all-1.55.3.jar
-    - io.grpc-grpc-auth-1.55.3.jar
-    - io.grpc-grpc-context-1.55.3.jar
-    - io.grpc-grpc-core-1.55.3.jar
-    - io.grpc-grpc-netty-1.55.3.jar
-    - io.grpc-grpc-protobuf-1.55.3.jar
-    - io.grpc-grpc-protobuf-lite-1.55.3.jar
-    - io.grpc-grpc-stub-1.55.3.jar
-    - io.grpc-grpc-alts-1.55.3.jar
-    - io.grpc-grpc-api-1.55.3.jar
-    - io.grpc-grpc-grpclb-1.55.3.jar
-    - io.grpc-grpc-netty-shaded-1.55.3.jar
-    - io.grpc-grpc-services-1.55.3.jar
-    - io.grpc-grpc-xds-1.55.3.jar
-    - io.grpc-grpc-rls-1.55.3.jar
-    - io.grpc-grpc-servlet-1.55.3.jar
-    - io.grpc-grpc-servlet-jakarta-1.55.3.jar
+    - io.grpc-grpc-all-1.56.0.jar
+    - io.grpc-grpc-auth-1.56.0.jar
+    - io.grpc-grpc-context-1.56.0.jar
+    - io.grpc-grpc-core-1.56.0.jar
+    - io.grpc-grpc-netty-1.56.0.jar
+    - io.grpc-grpc-protobuf-1.56.0.jar
+    - io.grpc-grpc-protobuf-lite-1.56.0.jar
+    - io.grpc-grpc-stub-1.56.0.jar
+    - io.grpc-grpc-alts-1.56.0.jar
+    - io.grpc-grpc-api-1.56.0.jar
+    - io.grpc-grpc-grpclb-1.56.0.jar
+    - io.grpc-grpc-netty-shaded-1.56.0.jar
+    - io.grpc-grpc-services-1.56.0.jar
+    - io.grpc-grpc-xds-1.56.0.jar
+    - io.grpc-grpc-rls-1.56.0.jar
+    - io.grpc-grpc-servlet-1.56.0.jar
+    - io.grpc-grpc-servlet-jakarta-1.56.0.jar
   * Perfmark
     - io.perfmark-perfmark-api-0.26.0.jar
   * OpenCensus
@@ -481,7 +481,12 @@ The Apache Software License, Version 2.0
   * Prometheus
     - io.prometheus-simpleclient_httpserver-0.16.0.jar
   * Oxia
-    - io.streamnative.oxia-oxia-client-0.1.0-shaded.jar
+    - io.streamnative.oxia-oxia-client-0.1.6.jar
+    - io.streamnative.oxia-oxia-client-metrics-api-0.1.6.jar
+  * OpenHFT
+    - net.openhft-zero-allocation-hashing-0.16.jar
+  * Project reactor
+    - io.projectreactor-reactor-core-3.5.2.jar
   * Java JSON WebTokens
     - io.jsonwebtoken-jjwt-api-0.11.1.jar
     - io.jsonwebtoken-jjwt-impl-0.11.1.jar
@@ -504,8 +509,8 @@ The Apache Software License, Version 2.0
   * Google HTTP Client
     - com.google.http-client-google-http-client-gson-1.41.0.jar
     - com.google.http-client-google-http-client-1.41.0.jar
-    - com.google.auto.value-auto-value-annotations-1.9.jar
-    - com.google.re2j-re2j-1.6.jar
+    - com.google.auto.value-auto-value-annotations-1.10.1.jar
+    - com.google.re2j-re2j-1.7.jar
   * Jetcd
     - io.etcd-jetcd-api-0.7.5.jar
     - io.etcd-jetcd-common-0.7.5.jar
@@ -548,15 +553,18 @@ BSD 3-clause "New" or "Revised" License
  * JSR305 -- com.google.code.findbugs-jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt
  * JLine -- jline-jline-2.14.6.jar -- ../licenses/LICENSE-JLine.txt
  * JLine3 -- org.jline-jline-3.21.0.jar -- ../licenses/LICENSE-JLine.txt
+ * Reactive gRPC
+    - com.salesforce.servicelibs-reactive-grpc-common-1.2.4.jar -- ../licenses/LICENSE-Reactive-gRPC.txt
+    - com.salesforce.servicelibs-reactor-grpc-stub-1.2.4.jar -- ../licenses/LICENSE-Reactive-gRPC.txt
 
 BSD 2-Clause License
  * HdrHistogram -- org.hdrhistogram-HdrHistogram-2.1.9.jar -- ../licenses/LICENSE-HdrHistogram.txt
 
 MIT License
  * Java SemVer -- com.github.zafarkhaja-java-semver-0.9.0.jar -- ../licenses/LICENSE-SemVer.txt
  * SLF4J -- ../licenses/LICENSE-SLF4J.txt
-    - org.slf4j-slf4j-api-1.7.32.jar
-    - org.slf4j-jcl-over-slf4j-1.7.32.jar
+    - org.slf4j-slf4j-api-2.0.13.jar
+    - org.slf4j-jcl-over-slf4j-2.0.13.jar
  * The Checker Framework
     - org.checkerframework-checker-qual-3.33.0.jar
  * oshi
@@ -566,8 +574,8 @@ MIT License
     - com.auth0-jwks-rsa-0.22.0.jar
 Protocol Buffers License
  * Protocol Buffers
-   - com.google.protobuf-protobuf-java-3.19.6.jar -- ../licenses/LICENSE-protobuf.txt
-   - com.google.protobuf-protobuf-java-util-3.19.6.jar -- ../licenses/LICENSE-protobuf.txt
+   - com.google.protobuf-protobuf-java-3.22.3.jar -- ../licenses/LICENSE-protobuf.txt
+   - com.google.protobuf-protobuf-java-util-3.22.3.jar -- ../licenses/LICENSE-protobuf.txt
 
 CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt
  * Java Annotations API