Merge branch 'website/branch-2.7.2-chapter-3' into website/branch-2.7…

….2-chapter-4 * website/branch-2.7.2-chapter-3: Update the nesting of forms in the document [pulsar-java-client] Auto-recovery after exception like out of direct memory (apache#12170) Allow to config pulsar client allocator out of memory policy (apache#12200) [Transaction] Fix bugs, Exception thrower by TB::appendBufferToTxn must be ManagedLedgerException. (apache#12376) Bumped version to 2.10.0-SNAPSHOT (apache#12285) [docs][Website] add docs of broker entry metadata (apache#12404) [C++] Use weak ref to ClientConnection for timeout task (apache#12409) fix windows test path probleam (apache#12398) [website][upgrade]feat: home page (apache#12383) [docs] [ISSUE 11526] Update the description of `topic` (apache#12375) [Docs] Add document label check robot (apache#12371) [Admin] Get schema validation enforce add applied. (apache#12349) [Doc] add scope explanations (apache#12380) Fix java doc for MultipleListenerValidator (apache#12389)
zeo1995 · Oct 20, 2021 · 987d7e8 · 987d7e8
2 parents 63e50a8 + 88ac2a0
commit 987d7e8
Show file tree

Hide file tree

Showing 189 changed files with 2,650 additions and 346 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -72,15 +72,15 @@ Check the box below and label this PR (if you have committer privilege).
 
 Need to update docs? 
 
-- [ ] doc-required 
+- [ ] `doc-required` 
 
   (If you need help on updating docs, create a doc issue)
 
-- [ ] no-need-doc 
+- [ ] `no-need-doc` 
 
   (Please explain why)
 
-- [ ] doc 
+- [ ] `doc` 
 
   (If this PR contains doc changes)
 

diff --git a/.github/workflows/ci-documentbot.yml b/.github/workflows/ci-documentbot.yml
@@ -0,0 +1,28 @@
+name: Auto Labeling
+
+on:
+  pull_request_target :
+    types:
+      - opened
+      - edited
+      - labeled
+
+
+
+# A GitHub token created for a PR coming from a fork doesn't have
+# 'admin' or 'write' permission (which is required to add labels)
+# To avoid this issue, you can use the `scheduled` event and run
+# this action on a certain interval.And check the label about the
+# document.
+
+jobs:
+  labeling:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: apache/pulsar-test-infra/doc-label-check@master
+        with:
+          github-token: ${{ secrets.DOCBOT_TOKEN }}
+          label-pattern: '- \[(.*?)\] ?`(.+?)`' # matches '- [x] `label`'
+          bot-name: 'eolivelli'
diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml
@@ -25,7 +25,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>bouncy-castle-parent</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
 

diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml
@@ -24,7 +24,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>bouncy-castle-parent</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
 

diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml
@@ -25,7 +25,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>bouncy-castle-parent</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
 

diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml
@@ -25,7 +25,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>pulsar</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
 

diff --git a/buildtools/pom.xml b/buildtools/pom.xml
@@ -31,7 +31,7 @@
 
   <groupId>org.apache.pulsar</groupId>
   <artifactId>buildtools</artifactId>
-  <version>2.9.0-SNAPSHOT</version>
+  <version>2.10.0-SNAPSHOT</version>
   <packaging>jar</packaging>
   <name>Pulsar Build Tools</name>
 

diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml
@@ -25,7 +25,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>distribution</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
 

diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml
@@ -25,7 +25,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>distribution</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
 

diff --git a/distribution/pom.xml b/distribution/pom.xml
@@ -25,7 +25,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>pulsar</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
 

diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml
@@ -25,7 +25,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>distribution</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
 

diff --git a/docker/grafana/pom.xml b/docker/grafana/pom.xml
@@ -23,7 +23,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>docker-images</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>grafana-docker-image</artifactId>

diff --git a/docker/pom.xml b/docker/pom.xml
@@ -26,7 +26,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>pulsar</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
   </parent>
   <artifactId>docker-images</artifactId>
   <name>Apache Pulsar :: Docker Images</name>

diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml
@@ -23,7 +23,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>docker-images</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>pulsar-all-docker-image</artifactId>

diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
@@ -23,7 +23,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>docker-images</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>pulsar-docker-image</artifactId>

diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml
@@ -26,7 +26,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>pulsar</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
 

diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml
@@ -26,7 +26,7 @@
     <parent>
         <artifactId>pulsar</artifactId>
         <groupId>org.apache.pulsar</groupId>
-        <version>2.9.0-SNAPSHOT</version>
+        <version>2.10.0-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 

diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml
@@ -25,7 +25,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>pulsar</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
 

diff --git a/pom.xml b/pom.xml
@@ -32,7 +32,7 @@
   <groupId>org.apache.pulsar</groupId>
   <artifactId>pulsar</artifactId>
 
-  <version>2.9.0-SNAPSHOT</version>
+  <version>2.10.0-SNAPSHOT</version>
 
   <name>Pulsar</name>
   <description>Pulsar is a distributed pub-sub messaging platform with a very
@@ -1326,6 +1326,7 @@ flexible messaging model and an intuitive client API.</description>
             -Dpulsar.allocator.pooled=true
             -Dpulsar.allocator.leak_detection=Advanced
             -Dpulsar.allocator.exit_on_oom=false
+            -Dpulsar.allocator.out_of_memory_policy=FallbackToHeap
             -Dio.netty.tryReflectionSetAccessible=true
             ${test.additional.args}
           </argLine>

diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml
@@ -26,7 +26,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>pulsar</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>pulsar-broker-auth-athenz</artifactId>

diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml
@@ -26,7 +26,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>pulsar</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>pulsar-broker-auth-sasl</artifactId>

diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml
@@ -26,7 +26,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>pulsar</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>pulsar-broker-common</artifactId>

diff --git a/...er-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java b/...er-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java
@@ -39,10 +39,11 @@ public final class MultipleListenerValidator {
 
     /**
      * Validate the configuration of `advertisedListeners`, `internalListenerName`.
-     * 2. the listener name in `advertisedListeners` must not duplicate.
-     * 3. user can not assign same 'host:port' to different listener.
-     * 4. if `internalListenerName` is absent, the first `listener` in the `advertisedListeners` will be the `internalListenerName`.
-     * 5. if pulsar do not specify `brokerServicePortTls`, should only contain one entry of `pulsar://` per listener name.
+     * 1. `advertisedListeners` consists of a comma-separated list of endpoints.
+     * 2. Each endpoint consists of a listener name and an associated address (`listener:scheme://host:port`).
+     * 3. A listener name may be repeated to define both a non-TLS and a TLS endpoint.
+     * 4. Duplicate definitions are disallowed.
+     * 5. If `internalListenerName` is absent, set it to the first listener defined in `advertisedListeners`.
      * @param config the pulsar broker configure.
      * @return
      */

diff --git a/...rc/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java b/...rc/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java
@@ -21,7 +21,6 @@
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
 
@@ -240,7 +239,7 @@ public void testAuthSecretKeyFromFile() throws Exception {
         AuthenticationProviderToken provider = new AuthenticationProviderToken();
 
         Properties properties = new Properties();
-        properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_SECRET_KEY, "file://" + secretKeyFile.toString());
+        properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_SECRET_KEY, "file:///" + secretKeyFile.toString().replace('\\', '/'));
 
         ServiceConfiguration conf = new ServiceConfiguration();
         conf.setProperties(properties);

diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml
@@ -26,7 +26,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>pulsar</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
 

diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml
@@ -25,7 +25,7 @@
   <parent>
     <groupId>org.apache.pulsar</groupId>
     <artifactId>pulsar</artifactId>
-    <version>2.9.0-SNAPSHOT</version>
+    <version>2.10.0-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
 

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
@@ -2387,10 +2387,15 @@ protected void internalSetSchemaCompatibilityStrategy(SchemaCompatibilityStrateg
                 "schemaCompatibilityStrategy");
     }
 
-    protected boolean internalGetSchemaValidationEnforced() {
+    protected boolean internalGetSchemaValidationEnforced(boolean applied) {
         validateNamespacePolicyOperation(namespaceName, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY,
                 PolicyOperation.READ);
-        return getNamespacePolicies(namespaceName).schema_validation_enforced;
+        boolean schemaValidationEnforced = getNamespacePolicies(namespaceName).schema_validation_enforced;
+        if (!schemaValidationEnforced && applied) {
+            return pulsar().getConfiguration().isSchemaValidationEnforced();
+        } else {
+            return schemaValidationEnforced;
+        }
     }
 
     protected void internalSetSchemaValidationEnforced(boolean schemaValidationEnforced) {

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
@@ -1626,9 +1626,10 @@ public void setSubscriptionTypesEnabled(
     @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"),
                             @ApiResponse(code = 404, message = "Tenants or Namespace doesn't exist") })
     public boolean getSchemaValidtionEnforced(@PathParam("tenant") String tenant,
-                                           @PathParam("namespace") String namespace) {
+                                              @PathParam("namespace") String namespace,
+                                              @QueryParam("applied") @DefaultValue("false") boolean applied) {
         validateNamespaceName(tenant, namespace);
-        return internalGetSchemaValidationEnforced();
+        return internalGetSchemaValidationEnforced(applied);
     }
 
     @POST

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -2986,6 +2986,10 @@ public void publishTxnMessage(TxnID txnID, ByteBuf headersAndPayload, PublishCon
                             decrementPendingWriteOpsAndCheck();
                         })
                         .exceptionally(throwable -> {
+                            throwable = throwable.getCause();
+                            if (!(throwable instanceof ManagedLedgerException)) {
+                                throwable = new ManagedLedgerException(throwable);
+                            }
                             addFailed((ManagedLedgerException) throwable, publishContext);
                             return null;
                         });

diff --git a/...rc/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/...rc/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java
@@ -219,7 +219,7 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) {
             @Override
             public void addFailed(ManagedLedgerException exception, Object ctx) {
                 log.error("Failed to append buffer to txn {}", txnId, exception);
-                completableFuture.completeExceptionally(new PersistenceException(exception));
+                completableFuture.completeExceptionally(exception);
             }
         }, null);
         return completableFuture;

diff --git a/...broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaValidationEnforced.java b/...broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaValidationEnforced.java
@@ -68,6 +68,15 @@ public void cleanup() throws Exception {
         super.internalCleanup();
     }
 
+    @Test
+    public void testGetSchemaValidationEnforcedApplied() throws Exception {
+        String namespace = "schema-validation-enforced/testApplied";
+        admin.namespaces().createNamespace(namespace);
+        this.conf.setSchemaValidationEnforced(true);
+        assertTrue(admin.namespaces().getSchemaValidationEnforced(namespace, true));
+        assertFalse(admin.namespaces().getSchemaValidationEnforced(namespace, false));
+    }
+
     @Test
     public void testDisableSchemaValidationEnforcedNoSchema() throws Exception {
         admin.namespaces().createNamespace("schema-validation-enforced/default-no-schema");