diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 0a3dccb25d90c..c89a1fefb6c3d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -72,15 +72,15 @@ Check the box below and label this PR (if you have committer privilege). Need to update docs? -- [ ] doc-required +- [ ] `doc-required` (If you need help on updating docs, create a doc issue) -- [ ] no-need-doc +- [ ] `no-need-doc` (Please explain why) -- [ ] doc +- [ ] `doc` (If this PR contains doc changes) diff --git a/.github/workflows/ci-documentbot.yml b/.github/workflows/ci-documentbot.yml new file mode 100644 index 0000000000000..546e85efcfd44 --- /dev/null +++ b/.github/workflows/ci-documentbot.yml @@ -0,0 +1,28 @@ +name: Auto Labeling + +on: + pull_request_target : + types: + - opened + - edited + - labeled + + + +# A GitHub token created for a PR coming from a fork doesn't have +# 'admin' or 'write' permission (which is required to add labels) +# To avoid this issue, you can use the `scheduled` event and run +# this action on a certain interval.And check the label about the +# document. + +jobs: + labeling: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: apache/pulsar-test-infra/doc-label-check@master + with: + github-token: ${{ secrets.DOCBOT_TOKEN }} + label-pattern: '- \[(.*?)\] ?`(.+?)`' # matches '- [x] `label`' + bot-name: 'eolivelli' diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 696b43fafcf6c..1f69ad6c6211e 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 07b31ebe0638d..595dd31548aa0 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 49f6737d4d325..31d1c1191a333 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index f908c0e3fde36..6d2cecb3c9286 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 7b5b02a510660..ba57042d55c5e 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT jar Pulsar Build Tools diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 86a8bbdaeb479..da2cbb628096c 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 352ed71e6d868..0e6721371e86e 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/distribution/pom.xml b/distribution/pom.xml index 381e7194a68df..ee0a3c32decd6 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 72aab71e6eb80..f5fb295cf5229 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/docker/grafana/pom.xml b/docker/grafana/pom.xml index b91947eb1fe84..5a3b85fa7fed5 100644 --- a/docker/grafana/pom.xml +++ b/docker/grafana/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 grafana-docker-image diff --git a/docker/pom.xml b/docker/pom.xml index 1d106d77247e2..610442ef7f0d8 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 11ff264b16abe..b008f5a90a552 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 06e21e9d46b02..20a9c2eaa3676 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 1214656f427ff..832a9043dc9fb 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml index 62814c939879d..e704f9ddc9b43 100644 --- a/kafka-connect-avro-converter-shaded/pom.xml +++ b/kafka-connect-avro-converter-shaded/pom.xml @@ -26,7 +26,7 @@ pulsar org.apache.pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 5bbb0448022cb..d2bb22b050b74 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index 185b391cf7daa..534440739b491 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT Pulsar Pulsar is a distributed pub-sub messaging platform with a very @@ -1326,6 +1326,7 @@ flexible messaging model and an intuitive client API. -Dpulsar.allocator.pooled=true -Dpulsar.allocator.leak_detection=Advanced -Dpulsar.allocator.exit_on_oom=false + -Dpulsar.allocator.out_of_memory_policy=FallbackToHeap -Dio.netty.tryReflectionSetAccessible=true ${test.additional.args} diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index bbec5e0084dcf..d215465000193 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-broker-auth-athenz diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index 13dd14aa8f58a..ac2c36e61e99e 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-broker-auth-sasl diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index 91179a1bd8f6f..6166fadfa6447 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-broker-common diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java index 1ba46e12739e2..82258c3170e77 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/MultipleListenerValidator.java @@ -39,10 +39,11 @@ public final class MultipleListenerValidator { /** * Validate the configuration of `advertisedListeners`, `internalListenerName`. - * 2. the listener name in `advertisedListeners` must not duplicate. - * 3. user can not assign same 'host:port' to different listener. - * 4. if `internalListenerName` is absent, the first `listener` in the `advertisedListeners` will be the `internalListenerName`. - * 5. if pulsar do not specify `brokerServicePortTls`, should only contain one entry of `pulsar://` per listener name. + * 1. `advertisedListeners` consists of a comma-separated list of endpoints. + * 2. Each endpoint consists of a listener name and an associated address (`listener:scheme://host:port`). + * 3. A listener name may be repeated to define both a non-TLS and a TLS endpoint. + * 4. Duplicate definitions are disallowed. + * 5. If `internalListenerName` is absent, set it to the first listener defined in `advertisedListeners`. * @param config the pulsar broker configure. * @return */ diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java index 6582a6ab69d89..b05ad4ca4839d 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java @@ -21,7 +21,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -240,7 +239,7 @@ public void testAuthSecretKeyFromFile() throws Exception { AuthenticationProviderToken provider = new AuthenticationProviderToken(); Properties properties = new Properties(); - properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_SECRET_KEY, "file://" + secretKeyFile.toString()); + properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_SECRET_KEY, "file:///" + secretKeyFile.toString().replace('\\', '/')); ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml index bf1e78bba0249..5f16bb20e4eaa 100644 --- a/pulsar-broker-shaded/pom.xml +++ b/pulsar-broker-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index fd11162d8ea3b..7140176630a5c 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 07c676af4854b..f2327e28b93c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -2387,10 +2387,15 @@ protected void internalSetSchemaCompatibilityStrategy(SchemaCompatibilityStrateg "schemaCompatibilityStrategy"); } - protected boolean internalGetSchemaValidationEnforced() { + protected boolean internalGetSchemaValidationEnforced(boolean applied) { validateNamespacePolicyOperation(namespaceName, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.READ); - return getNamespacePolicies(namespaceName).schema_validation_enforced; + boolean schemaValidationEnforced = getNamespacePolicies(namespaceName).schema_validation_enforced; + if (!schemaValidationEnforced && applied) { + return pulsar().getConfiguration().isSchemaValidationEnforced(); + } else { + return schemaValidationEnforced; + } } protected void internalSetSchemaValidationEnforced(boolean schemaValidationEnforced) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 53eab242d3d55..cf84374f34bc9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -1626,9 +1626,10 @@ public void setSubscriptionTypesEnabled( @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenants or Namespace doesn't exist") }) public boolean getSchemaValidtionEnforced(@PathParam("tenant") String tenant, - @PathParam("namespace") String namespace) { + @PathParam("namespace") String namespace, + @QueryParam("applied") @DefaultValue("false") boolean applied) { validateNamespaceName(tenant, namespace); - return internalGetSchemaValidationEnforced(); + return internalGetSchemaValidationEnforced(applied); } @POST diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 64e4b7ff4ed49..a1cdbd997c529 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -2986,6 +2986,10 @@ public void publishTxnMessage(TxnID txnID, ByteBuf headersAndPayload, PublishCon decrementPendingWriteOpsAndCheck(); }) .exceptionally(throwable -> { + throwable = throwable.getCause(); + if (!(throwable instanceof ManagedLedgerException)) { + throwable = new ManagedLedgerException(throwable); + } addFailed((ManagedLedgerException) throwable, publishContext); return null; }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 79f7f35732754..1b0fb6697ec3f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -219,7 +219,7 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { @Override public void addFailed(ManagedLedgerException exception, Object ctx) { log.error("Failed to append buffer to txn {}", txnId, exception); - completableFuture.completeExceptionally(new PersistenceException(exception)); + completableFuture.completeExceptionally(exception); } }, null); return completableFuture; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaValidationEnforced.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaValidationEnforced.java index 3daf920c975bc..2fb808efd6a2f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaValidationEnforced.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaValidationEnforced.java @@ -68,6 +68,15 @@ public void cleanup() throws Exception { super.internalCleanup(); } + @Test + public void testGetSchemaValidationEnforcedApplied() throws Exception { + String namespace = "schema-validation-enforced/testApplied"; + admin.namespaces().createNamespace(namespace); + this.conf.setSchemaValidationEnforced(true); + assertTrue(admin.namespaces().getSchemaValidationEnforced(namespace, true)); + assertFalse(admin.namespaces().getSchemaValidationEnforced(namespace, false)); + } + @Test public void testDisableSchemaValidationEnforcedNoSchema() throws Exception { admin.namespaces().createNamespace("schema-validation-enforced/default-no-schema"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 971f8b24bce8b..bb172b599c730 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -18,15 +18,19 @@ */ package org.apache.pulsar.broker.transaction; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl.TRANSACTION_LOG_PREFIX; import com.google.common.collect.Sets; +import io.netty.buffer.Unpooled; import java.lang.reflect.Field; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; @@ -295,4 +299,73 @@ public void testTakeSnapshotBeforeBuildTxnProducer() throws Exception { Assert.assertEquals(snapshot1.getMaxReadPositionEntryId(), 1); }); } + + + @Test + public void testAppendBufferWithNotManageLedgerExceptionCanCastToMLE() + throws Exception { + String topic = "persistent://pulsar/system/testReCreateTopic"; + admin.topics().createNonPartitionedTopic(topic); + + PersistentTopic persistentTopic = + (PersistentTopic) pulsarServiceList.get(0).getBrokerService() + .getTopic(topic, false) + .get().get(); + CountDownLatch countDownLatch = new CountDownLatch(1); + Topic.PublishContext publishContext = new Topic.PublishContext() { + + @Override + public String getProducerName() { + return "test"; + } + + public long getSequenceId() { + return 30; + } + /** + * Return the producer name for the original producer. + * + * For messages published locally, this will return the same local producer name, though in case of + * replicated messages, the original producer name will differ + */ + public String getOriginalProducerName() { + return "test"; + } + + public long getOriginalSequenceId() { + return 30; + } + + public long getHighestSequenceId() { + return 30; + } + + public long getOriginalHighestSequenceId() { + return 30; + } + + public long getNumberOfMessages() { + return 30; + } + + @Override + public void completed(Exception e, long ledgerId, long entryId) { + Assert.assertTrue(e.getCause() instanceof ManagedLedgerException.ManagedLedgerAlreadyClosedException); + countDownLatch.countDown(); + } + }; + + //Close topic manageLedger. + persistentTopic.getManagedLedger().close(); + + //Publish to a closed managerLedger to test ManagerLedgerException. + persistentTopic.publishTxnMessage(new TxnID(123L, 321L), + Unpooled.copiedBuffer("message", UTF_8), publishContext); + + //If it times out, it means that the assertTrue in publishContext.completed is failed. + Awaitility.await().until(() -> { + countDownLatch.await(); + return true; + }); + } } \ No newline at end of file diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index d7e6217fd63c0..3832d27dd53e5 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index aa61fb15d152f..d3797d4dfed99 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index 1fa512ecc1b4a..15eab4a62f0ee 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index b9f68f1b6e2a0..bd5fa2d09b04f 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java index f6f8654f2d4a7..6ea4bab025cd3 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java @@ -3539,6 +3539,7 @@ void setSchemaAutoUpdateCompatibilityStrategy(String namespace, /** * Get schema validation enforced for namespace. + * @param namespace namespace for this command. * @return the schema validation enforced flag * @throws NotAuthorizedException * Don't have admin permission @@ -3547,16 +3548,39 @@ void setSchemaAutoUpdateCompatibilityStrategy(String namespace, * @throws PulsarAdminException * Unexpected error */ - boolean getSchemaValidationEnforced(String namespace) - throws PulsarAdminException; + boolean getSchemaValidationEnforced(String namespace) throws PulsarAdminException; /** * Get schema validation enforced for namespace asynchronously. + * @param namespace namespace for this command. * * @return the schema validation enforced flag */ CompletableFuture getSchemaValidationEnforcedAsync(String namespace); + /** + * Get schema validation enforced for namespace. + * @param namespace namespace for this command. + * @param applied applied for this command. + * @return the schema validation enforced flag + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Tenant or Namespace does not exist + * @throws PulsarAdminException + * Unexpected error + */ + boolean getSchemaValidationEnforced(String namespace, boolean applied) throws PulsarAdminException; + + /** + * Get schema validation enforced for namespace asynchronously. + * @param namespace namespace for this command. + * @param applied applied for this command. + * + * @return the schema validation enforced flag + */ + CompletableFuture getSchemaValidationEnforcedAsync(String namespace, boolean applied); + /** * Set schema validation enforced for namespace. * if a producer without a schema attempts to produce to a topic with schema in this the namespace, the diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index 406f4b28d0d84..c2fb46c09cd5d 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index 2d28b12e5cbde..91f358e6345e3 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java index 8307a5f172583..610423b18c101 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java @@ -3060,11 +3060,21 @@ public void setSchemaAutoUpdateCompatibilityStrategy(String namespace, } @Override - public boolean getSchemaValidationEnforced(String namespace) + public boolean getSchemaValidationEnforced(String namespace) throws PulsarAdminException { + return getSchemaValidationEnforced(namespace, false); + } + + @Override + public CompletableFuture getSchemaValidationEnforcedAsync(String namespace) { + return getSchemaValidationEnforcedAsync(namespace, false); + } + + @Override + public boolean getSchemaValidationEnforced(String namespace, boolean applied) throws PulsarAdminException { try { - return getSchemaValidationEnforcedAsync(namespace). - get(this.readTimeoutMs, TimeUnit.MILLISECONDS); + return getSchemaValidationEnforcedAsync(namespace, applied) + .get(this.readTimeoutMs, TimeUnit.MILLISECONDS); } catch (ExecutionException e) { throw (PulsarAdminException) e.getCause(); } catch (InterruptedException e) { @@ -3076,9 +3086,10 @@ public boolean getSchemaValidationEnforced(String namespace) } @Override - public CompletableFuture getSchemaValidationEnforcedAsync(String namespace) { + public CompletableFuture getSchemaValidationEnforcedAsync(String namespace, boolean applied) { NamespaceName ns = NamespaceName.get(namespace); WebTarget path = namespacePath(ns, "schemaValidationEnforced"); + path = path.queryParam("applied", applied); final CompletableFuture future = new CompletableFuture<>(); asyncGetRequest(path, new InvocationCallback() { diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index f829386a18ed5..da87678ef43ab 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index 949901d2cdbb8..a5286859498c1 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index 5539f3bc124c2..95472851b9385 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index 087736b2e11e4..c2c34297b4943 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client-cpp/lib/ClientConnection.cc b/pulsar-client-cpp/lib/ClientConnection.cc index 6f947312f01e6..4b8a3dc4a420f 100644 --- a/pulsar-client-cpp/lib/ClientConnection.cc +++ b/pulsar-client-cpp/lib/ClientConnection.cc @@ -532,18 +532,25 @@ void ClientConnection::handleResolve(const boost::system::error_code& err, return; } - auto self = shared_from_this(); - connectTimeoutTask_->setCallback([this, self](const PeriodicTask::ErrorCode& ec) { - if (state_ != Ready) { - LOG_ERROR(cnxString_ << "Connection was not established in " << connectTimeoutTask_->getPeriodMs() - << " ms, close the socket"); + auto self = ClientConnectionWeakPtr(shared_from_this()); + + connectTimeoutTask_->setCallback([self](const PeriodicTask::ErrorCode& ec) { + ClientConnectionPtr ptr = self.lock(); + if (!ptr) { + // Connection was already destroyed + return; + } + + if (ptr->state_ != Ready) { + LOG_ERROR(ptr->cnxString_ << "Connection was not established in " + << ptr->connectTimeoutTask_->getPeriodMs() << " ms, close the socket"); PeriodicTask::ErrorCode err; - socket_->close(err); + ptr->socket_->close(err); if (err) { - LOG_WARN(cnxString_ << "Failed to close socket: " << err.message()); + LOG_WARN(ptr->cnxString_ << "Failed to close socket: " << err.message()); } } - connectTimeoutTask_->stop(); + ptr->connectTimeoutTask_->stop(); }); LOG_DEBUG(cnxString_ << "Connecting to " << endpointIterator->endpoint() << "..."); diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 45e97a7267b83..6567eb054fc7b 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index a3df727fa7ed1..9479f3ad6972f 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index efe90ceaaeada..9d4325ea99eee 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index 32befc8a95386..60fb6acf1d9a6 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -368,6 +368,9 @@ public void namespaces() throws Exception { namespaces.run(split("get-subscription-types-enabled myprop/clust/ns1")); verify(mockNamespaces).getSubscriptionTypesEnabled("myprop/clust/ns1"); + namespaces.run(split("get-schema-validation-enforce myprop/clust/ns1 -ap")); + verify(mockNamespaces).getSchemaValidationEnforced("myprop/clust/ns1", true); + namespaces .run(split("set-bookie-affinity-group myprop/clust/ns1 --primary-group test1 --secondary-group test2")); verify(mockNamespaces).setBookieAffinityGroup("myprop/clust/ns1", diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 4fc38f1336adf..4740cda51632a 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 435b5d8b45ddb..c642288383aeb 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -1877,11 +1877,14 @@ private class GetSchemaValidationEnforced extends CliCommand { @Parameter(description = "tenant/namespace", required = true) private java.util.List params; + @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the namespace") + private boolean applied = false; + @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - System.out.println(getAdmin().namespaces().getSchemaValidationEnforced(namespace)); + System.out.println(getAdmin().namespaces().getSchemaValidationEnforced(namespace, applied)); } } diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 65aef6af05a5f..3ad26159a3f4e 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java index 9f81e03e5af8a..cea567e8d687e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java @@ -66,18 +66,28 @@ public boolean add(MessageImpl msg, SendCallback callback) { } if (++numMessagesInBatch == 1) { - // some properties are common amongst the different messages in the batch, hence we just pick it up from - // the first message - messageMetadata.setSequenceId(msg.getSequenceId()); - lowestSequenceId = Commands.initBatchMessageMetadata(messageMetadata, msg.getMessageBuilder()); - this.firstCallback = callback; - batchedMessageMetadataAndPayload = PulsarByteBufAllocator.DEFAULT - .buffer(Math.min(maxBatchSize, ClientCnx.getMaxMessageSize())); - if (msg.getMessageBuilder().hasTxnidMostBits() && currentTxnidMostBits == -1) { - currentTxnidMostBits = msg.getMessageBuilder().getTxnidMostBits(); - } - if (msg.getMessageBuilder().hasTxnidLeastBits() && currentTxnidLeastBits == -1) { - currentTxnidLeastBits = msg.getMessageBuilder().getTxnidLeastBits(); + try { + // some properties are common amongst the different messages in the batch, hence we just pick it up from + // the first message + messageMetadata.setSequenceId(msg.getSequenceId()); + lowestSequenceId = Commands.initBatchMessageMetadata(messageMetadata, msg.getMessageBuilder()); + this.firstCallback = callback; + batchedMessageMetadataAndPayload = PulsarByteBufAllocator.DEFAULT + .buffer(Math.min(maxBatchSize, ClientCnx.getMaxMessageSize())); + if (msg.getMessageBuilder().hasTxnidMostBits() && currentTxnidMostBits == -1) { + currentTxnidMostBits = msg.getMessageBuilder().getTxnidMostBits(); + } + if (msg.getMessageBuilder().hasTxnidLeastBits() && currentTxnidLeastBits == -1) { + currentTxnidLeastBits = msg.getMessageBuilder().getTxnidLeastBits(); + } + } catch (Throwable e) { + log.error("construct first message failed, exception is ", e); + if (batchedMessageMetadataAndPayload != null) { + // if payload has been allocated release it + batchedMessageMetadataAndPayload.release(); + } + discard(new PulsarClientException(e)); + return false; } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java new file mode 100644 index 0000000000000..3d554871141e3 --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import org.apache.bookkeeper.common.allocator.impl.ByteBufAllocatorBuilderImpl; +import org.apache.bookkeeper.common.allocator.impl.ByteBufAllocatorImpl; +import org.apache.pulsar.client.api.CompressionType; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; +import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.testng.IObjectFactory; +import org.testng.annotations.ObjectFactory; +import org.testng.annotations.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +@PrepareForTest({ByteBufAllocatorImpl.class, ByteBufAllocatorBuilderImpl.class}) +@PowerMockIgnore({"javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*"}) +public class BatchMessageContainerImplTest { + + @ObjectFactory + public IObjectFactory getObjectFactory() { + return new org.powermock.modules.testng.PowerMockObjectFactory(); + } + + @Test + public void recoveryAfterOom() throws Exception { + final ByteBufAllocatorImpl mockAllocator = PowerMockito.mock(ByteBufAllocatorImpl.class); + PowerMockito.whenNew(ByteBufAllocatorImpl.class).withAnyArguments().thenReturn(mockAllocator); + PowerMockito.when(mockAllocator.buffer(Mockito.anyInt(), Mockito.anyInt())).thenThrow(new OutOfMemoryError("test")).thenReturn(null); + final ProducerImpl producer = Mockito.mock(ProducerImpl.class); + final ProducerConfigurationData producerConfigurationData = new ProducerConfigurationData(); + producerConfigurationData.setCompressionType(CompressionType.NONE); + Mockito.when(producer.getConfiguration()).thenReturn(producerConfigurationData); + final BatchMessageContainerImpl batchMessageContainer = new BatchMessageContainerImpl(); + batchMessageContainer.setProducer(producer); + MessageMetadata messageMetadata1 = new MessageMetadata(); + messageMetadata1.setSequenceId(1L); + messageMetadata1.setProducerName("producer1"); + messageMetadata1.setPublishTime(System.currentTimeMillis()); + ByteBuffer payload1 = ByteBuffer.wrap("payload1".getBytes(StandardCharsets.UTF_8)); + final MessageImpl message1 = MessageImpl.create(messageMetadata1, payload1, Schema.BYTES, null); + batchMessageContainer.add(message1, null); + MessageMetadata messageMetadata2 = new MessageMetadata(); + messageMetadata2.setSequenceId(1L); + messageMetadata2.setProducerName("producer1"); + messageMetadata2.setPublishTime(System.currentTimeMillis()); + ByteBuffer payload2 = ByteBuffer.wrap("payload2".getBytes(StandardCharsets.UTF_8)); + final MessageImpl message2 = MessageImpl.create(messageMetadata2, payload2, Schema.BYTES, null); + // after oom, our add can self-healing, won't throw exception + batchMessageContainer.add(message2, null); + } + +} diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 225c260c4e234..343ec1351042c 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/allocator/PulsarByteBufAllocator.java b/pulsar-common/src/main/java/org/apache/pulsar/common/allocator/PulsarByteBufAllocator.java index 5f05640a6d95f..dcb2df5ee617a 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/allocator/PulsarByteBufAllocator.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/allocator/PulsarByteBufAllocator.java @@ -27,6 +27,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.common.allocator.ByteBufAllocatorBuilder; import org.apache.bookkeeper.common.allocator.LeakDetectionPolicy; +import org.apache.bookkeeper.common.allocator.OutOfMemoryPolicy; import org.apache.bookkeeper.common.allocator.PoolingPolicy; /** @@ -39,6 +40,7 @@ public class PulsarByteBufAllocator { public static final String PULSAR_ALLOCATOR_POOLED = "pulsar.allocator.pooled"; public static final String PULSAR_ALLOCATOR_EXIT_ON_OOM = "pulsar.allocator.exit_on_oom"; public static final String PULSAR_ALLOCATOR_LEAK_DETECTION = "pulsar.allocator.leak_detection"; + public static final String PULSAR_ALLOCATOR_OUT_OF_MEMORY_POLICY = "pulsar.allocator.out_of_memory_policy"; public static final ByteBufAllocator DEFAULT; @@ -53,6 +55,7 @@ public static void registerOOMListener(Consumer listener) { static { boolean isPooled = "true".equalsIgnoreCase(System.getProperty(PULSAR_ALLOCATOR_POOLED, "true")); EXIT_ON_OOM = "true".equalsIgnoreCase(System.getProperty(PULSAR_ALLOCATOR_EXIT_ON_OOM, "false")); + OutOfMemoryPolicy outOfMemoryPolicy = OutOfMemoryPolicy.valueOf(System.getProperty(PULSAR_ALLOCATOR_OUT_OF_MEMORY_POLICY, "FallbackToHeap")); LeakDetectionPolicy leakDetectionPolicy = LeakDetectionPolicy .valueOf(System.getProperty(PULSAR_ALLOCATOR_LEAK_DETECTION, "Disabled")); @@ -85,6 +88,7 @@ public static void registerOOMListener(Consumer listener) { } else { builder.poolingPolicy(PoolingPolicy.UnpooledHeap); } + builder.outOfMemoryPolicy(outOfMemoryPolicy); DEFAULT = builder.build(); } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/allocator/PulsarByteBufAllocatorDefaultTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/allocator/PulsarByteBufAllocatorDefaultTest.java new file mode 100644 index 0000000000000..503a17dca7eeb --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/allocator/PulsarByteBufAllocatorDefaultTest.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.allocator; + +import io.netty.buffer.ByteBufAllocator; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.common.allocator.LeakDetectionPolicy; +import org.apache.bookkeeper.common.allocator.OutOfMemoryPolicy; +import org.apache.bookkeeper.common.allocator.PoolingPolicy; +import org.apache.bookkeeper.common.allocator.impl.ByteBufAllocatorBuilderImpl; +import org.apache.bookkeeper.common.allocator.impl.ByteBufAllocatorImpl; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.testng.IObjectFactory; +import org.testng.annotations.ObjectFactory; +import org.testng.annotations.Test; + +@PrepareForTest({ByteBufAllocatorImpl.class, ByteBufAllocatorBuilderImpl.class}) +@PowerMockIgnore({"javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*"}) +@Slf4j +public class PulsarByteBufAllocatorDefaultTest { + + @ObjectFactory + public IObjectFactory getObjectFactory() { + return new org.powermock.modules.testng.PowerMockObjectFactory(); + } + + @Test + public void testDefaultConfig() throws Exception { + final ByteBufAllocatorImpl mockAllocator = PowerMockito.mock(ByteBufAllocatorImpl.class); + PowerMockito.whenNew(ByteBufAllocatorImpl.class).withAnyArguments().thenReturn(mockAllocator); + final ByteBufAllocatorImpl byteBufAllocator = (ByteBufAllocatorImpl) PulsarByteBufAllocator.DEFAULT; + // use the variable, in case the compiler optimization + log.trace("{}", byteBufAllocator); + PowerMockito.verifyNew(ByteBufAllocatorImpl.class).withArguments(Mockito.any(ByteBufAllocator.class), Mockito.any(), + Mockito.eq(PoolingPolicy.PooledDirect), Mockito.any(), Mockito.eq(OutOfMemoryPolicy.FallbackToHeap), + Mockito.any(), Mockito.eq(LeakDetectionPolicy.Advanced)); + } + +} \ No newline at end of file diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/allocator/PulsarByteBufAllocatorOomThrowExceptionTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/allocator/PulsarByteBufAllocatorOomThrowExceptionTest.java new file mode 100644 index 0000000000000..a23e6899bac94 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/allocator/PulsarByteBufAllocatorOomThrowExceptionTest.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.allocator; + +import io.netty.buffer.ByteBufAllocator; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.common.allocator.LeakDetectionPolicy; +import org.apache.bookkeeper.common.allocator.OutOfMemoryPolicy; +import org.apache.bookkeeper.common.allocator.PoolingPolicy; +import org.apache.bookkeeper.common.allocator.impl.ByteBufAllocatorBuilderImpl; +import org.apache.bookkeeper.common.allocator.impl.ByteBufAllocatorImpl; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.testng.IObjectFactory; +import org.testng.annotations.ObjectFactory; +import org.testng.annotations.Test; + +@PrepareForTest({ByteBufAllocatorImpl.class, ByteBufAllocatorBuilderImpl.class}) +@PowerMockIgnore({"javax.management.*", "javax.ws.*", "org.apache.logging.log4j.*"}) +@Slf4j +public class PulsarByteBufAllocatorOomThrowExceptionTest { + + @ObjectFactory + public IObjectFactory getObjectFactory() { + return new org.powermock.modules.testng.PowerMockObjectFactory(); + } + + @Test + public void testDefaultConfig() throws Exception { + try { + System.setProperty("pulsar.allocator.out_of_memory_policy", "ThrowException"); + final ByteBufAllocatorImpl mockAllocator = PowerMockito.mock(ByteBufAllocatorImpl.class); + PowerMockito.whenNew(ByteBufAllocatorImpl.class).withAnyArguments().thenReturn(mockAllocator); + final ByteBufAllocatorImpl byteBufAllocator = (ByteBufAllocatorImpl) PulsarByteBufAllocator.DEFAULT; + // use the variable, in case the compiler optimization + log.trace("{}", byteBufAllocator); + PowerMockito.verifyNew(ByteBufAllocatorImpl.class).withArguments(Mockito.any(ByteBufAllocator.class), Mockito.any(), + Mockito.eq(PoolingPolicy.PooledDirect), Mockito.any(), Mockito.eq(OutOfMemoryPolicy.ThrowException), + Mockito.any(), Mockito.eq(LeakDetectionPolicy.Advanced)); + } finally { + System.clearProperty("pulsar.allocator.out_of_memory_policy"); + } + } + +} diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index 134e3f147ee06..ac175ffc55166 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 68901a3da9cd2..56cf9523f6e47 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-functions-api diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index c6f5524e76fb9..6c2ae1bdb4bac 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-functions-instance diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index 82735a03f0ee8..160e613b40f23 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-functions-api-examples diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index ded9d60bed82b..065f45adf3595 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index de762ec006453..4146e94bd662f 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index d5dcc34ae8475..1b06421f6825a 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-functions diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index d534b751c7de5..231f7ff1cda5f 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-functions-proto diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index d7f35e70661be..5fef9d0152fa4 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index f3cbcc21f35f4..078733e8ccd81 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-functions-runtime diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 6c56b59c9702c..fe348a62a71e0 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-functions-secrets diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index 5611acd316485..33df98530b4ae 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-functions-utils diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index f5709031047d1..be4851d028857 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java index f37d5df1fe9a0..89fb32adb5941 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java @@ -1031,8 +1031,8 @@ public void testUpdateFunctionWithUrl() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); - String filePackageUrl = "file://" + fileLocation; + String fileLocation = file.getAbsolutePath().replace('\\', '/'); + String filePackageUrl = "file:///" + fileLocation; FunctionConfig functionConfig = new FunctionConfig(); functionConfig.setOutput(outputTopic); @@ -1427,10 +1427,10 @@ public void testDownloadFunctionHttpUrl() throws Exception { public void testDownloadFunctionFile() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); + String fileLocation = file.getAbsolutePath().replace('\\', '/'); String testDir = FunctionApiV2ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); FunctionsImplV2 function = new FunctionsImplV2(() -> mockedWorkerService); - StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction("file://" + fileLocation, null).getEntity(); + StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction("file:///" + fileLocation, null).getEntity(); File pkgFile = new File(testDir, UUID.randomUUID().toString()); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); @@ -1446,8 +1446,8 @@ public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); - String filePackageUrl = "file://" + fileLocation; + String fileLocation = file.getAbsolutePath().replace('\\', '/'); + String filePackageUrl = "file:///" + fileLocation; when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); FunctionConfig functionConfig = new FunctionConfig(); @@ -1479,8 +1479,8 @@ public void testRegisterFunctionWithConflictingFields() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); - String filePackageUrl = "file://" + fileLocation; + String fileLocation = file.getAbsolutePath().replace('\\', '/'); + String filePackageUrl = "file:///" + fileLocation; when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java index d2e7aca1cfa87..837e3211368c9 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java @@ -29,7 +29,6 @@ import static org.powermock.api.mockito.PowerMockito.doThrow; import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; import com.google.common.collect.Lists; @@ -1525,7 +1524,7 @@ public void testDownloadFunctionHttpUrl() throws Exception { public void testDownloadFunctionFile() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); + String fileLocation = file.getAbsolutePath().replace('\\', '/'); String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); PulsarWorkerService worker = mock(PulsarWorkerService.class); doReturn(true).when(worker).isInitialized(); @@ -1533,7 +1532,7 @@ public void testDownloadFunctionFile() throws Exception { when(config.isAuthorizationEnabled()).thenReturn(false); when(worker.getWorkerConfig()).thenReturn(config); FunctionsImpl function = new FunctionsImpl(() -> worker); - StreamingOutput streamOutput = function.downloadFunction("file://" + fileLocation, null, null); + StreamingOutput streamOutput = function.downloadFunction("file:///" + fileLocation, null, null); File pkgFile = new File(testDir, UUID.randomUUID().toString()); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); @@ -1549,8 +1548,8 @@ public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); - String filePackageUrl = "file://" + fileLocation; + String fileLocation = file.getAbsolutePath().replace('\\', '/'); + String filePackageUrl = "file:///" + fileLocation; when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); FunctionConfig functionConfig = new FunctionConfig(); @@ -1577,8 +1576,8 @@ public void testRegisterFunctionWithConflictingFields() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); - String filePackageUrl = "file://" + fileLocation; + String fileLocation = file.getAbsolutePath().replace('\\', '/'); + String filePackageUrl = "file:///" + fileLocation; when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false); @@ -1601,8 +1600,8 @@ public void testCreateFunctionWithoutSettingRuntime() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath(); - String filePackageUrl = "file://" + fileLocation; + String fileLocation = file.getAbsolutePath().replace('\\', '/'); + String filePackageUrl = "file:///" + fileLocation; when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); FunctionConfig functionConfig = new FunctionConfig(); diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index f93329be2644a..ba1be01c0effa 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-aerospike diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index 512f0c3d0aa2c..4f8b17d37ca58 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-aws diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 98e6b8c9039fb..eba3a70ae3b0d 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-batch-data-generator diff --git a/pulsar-io/batch-discovery-triggerers/pom.xml b/pulsar-io/batch-discovery-triggerers/pom.xml index a29cf663be57f..f3d2aac6ea215 100644 --- a/pulsar-io/batch-discovery-triggerers/pom.xml +++ b/pulsar-io/batch-discovery-triggerers/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-batch-discovery-triggerers diff --git a/pulsar-io/canal/pom.xml b/pulsar-io/canal/pom.xml index de7b1f44ac820..b1dcad215eccf 100644 --- a/pulsar-io/canal/pom.xml +++ b/pulsar-io/canal/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index 9a18c5fbc4551..98f87597e10db 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-cassandra diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index 8a6adf9a38a1a..92c07b16e27a5 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-common diff --git a/pulsar-io/core/pom.xml b/pulsar-io/core/pom.xml index 250806edda186..a57ba165311be 100644 --- a/pulsar-io/core/pom.xml +++ b/pulsar-io/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-core diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index 0191c6b671969..0bd19c94cb7dc 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-data-generator diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index e004b0f421206..4453b34427e3d 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-debezium-core diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index 7c0f8bcf48f2f..75ef9ca34cb13 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-debezium-mongodb diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 911ddac11edc1..c07d168740116 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-debezium-mssql diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index f044b53e3e598..4c98164c72aff 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-debezium-mysql diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index 950c46a7ff861..9c1682a7d4315 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-debezium-oracle diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index d0fafcb4fa063..95a442c27f2d0 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-debezium diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index 69f4c93f8594d..0af7d448cfea8 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-debezium-postgres diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 03b5114883283..3b234fbf84720 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-docs diff --git a/pulsar-io/dynamodb/pom.xml b/pulsar-io/dynamodb/pom.xml index 5737b50cab58e..8ad349938523a 100644 --- a/pulsar-io/dynamodb/pom.xml +++ b/pulsar-io/dynamodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-dynamodb diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index 43a82c6f07071..a4fcc5a5bc390 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-elastic-search Pulsar IO :: ElasticSearch diff --git a/pulsar-io/file/pom.xml b/pulsar-io/file/pom.xml index 8b1af8083c1ac..3653000f542c1 100644 --- a/pulsar-io/file/pom.xml +++ b/pulsar-io/file/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-file diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index 649803a8e0e03..1669674b2b351 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-flume diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 0b25207840e19..d178cdcbe9a31 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-hbase Pulsar IO :: Hbase diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index 36358c4d5fc53..4198ea42379b2 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-hdfs2 Pulsar IO :: Hdfs2 diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index fe12a54a75b43..d2422f5358eaa 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-hdfs3 Pulsar IO :: Hdfs3 diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index 88ef951ded242..2379419c03609 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-influxdb diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index d09feea5499b0..d682ae8b41eb9 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 32a22f43f5a4a..c5cb0f56fd1f1 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index f1ba934af160e..921971f0556a2 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index 7935748eeef7d..b0824d812ba79 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-jdbc diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index 2e76f431571c1..b80dbb10f8f6d 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index 741ecc31e7817..cb05a33463e86 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 pulsar-io-jdbc-sqlite diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index bf0a5fc22bb4d..30512d8ce9683 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-kafka-connect-adaptor-nar diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 5f07d24a304e4..7d3c9a08439c0 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-kafka-connect-adaptor diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index 1e6a3869fac31..7002a73b7829e 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-kafka diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index 30b57475cbee1..faf47577a2cc2 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-kinesis diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 93f2c1443f43c..3a527c255dd57 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-mongo diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index 3a41643d65cba..9a9aa26af0f71 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-netty diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index ee4dd4150ac52..c34954f5bcc80 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-nsq diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 2e20f7934edaa..d790383de741d 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index cef4b1c4a48d0..b13e67374f8ff 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-rabbitmq diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index 30da872435bb9..4be0dffdae30e 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-redis diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 6b0c02465533e..4bed6344e3d51 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index f32034ec829f7..e945878bce4d6 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-io-twitter diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 59052d71ddba6..b289033fbb5bb 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index 32c18716e6459..3f6146f9ddc44 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index f6cfbee94a8fd..bef392f45eeac 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index 8946793f6135e..bbd4e1fbb2b75 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. 4.0.0 diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index 277f63c184e83..fe24f16001d80 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-proxy diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 296d6f1330de1..89f083a58166d 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-sql diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index a0ded27662cc5..4a80657b15579 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-presto-distribution diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index 1cad798fc96e0..203871ab7aa83 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-presto-connector diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index d62d947e54a75..ad3bbe6b1adae 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-presto-connector-original diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index 18f80a04f0be3..dcf5966a1c00c 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index b3b0e3d369a07..0b377f6b51ea5 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-transaction-common diff --git a/pulsar-transaction/coordinator/pom.xml b/pulsar-transaction/coordinator/pom.xml index b5bedab65452f..430d6468df158 100644 --- a/pulsar-transaction/coordinator/pom.xml +++ b/pulsar-transaction/coordinator/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-transaction-coordinator diff --git a/pulsar-transaction/pom.xml b/pulsar-transaction/pom.xml index 053a494039d4b..7366237f8c93a 100644 --- a/pulsar-transaction/pom.xml +++ b/pulsar-transaction/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-transaction-parent diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index 35377d7ed3c6e..36a15c5a28d7d 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml index f29e10df2ca4e..b06bd8d3b23c4 100644 --- a/pulsar-zookeeper-utils/pom.xml +++ b/pulsar-zookeeper-utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/site2/docs/client-libraries-node.md b/site2/docs/client-libraries-node.md index 78a6b3194458f..a0c4b9dd026e1 100644 --- a/site2/docs/client-libraries-node.md +++ b/site2/docs/client-libraries-node.md @@ -107,7 +107,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -137,7 +137,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar automatically generates a globally unique name. If you choose to explicitly assign a name, it needs to be unique across *all* Pulsar clusters, otherwise the creation operation throws an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer waits for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error is thrown. If you set `sendTimeoutMs` to -1, the timeout is set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/docs/developing-binary-protocol.md b/site2/docs/developing-binary-protocol.md index 33861af0da7c9..56f7aef4d4059 100644 --- a/site2/docs/developing-binary-protocol.md +++ b/site2/docs/developing-binary-protocol.md @@ -29,42 +29,55 @@ The Pulsar protocol allows for two types of commands: Simple (payload-free) commands have this basic structure: -| Component | Description | Size (in bytes) | -|:------------|:----------------------------------------------------------------------------------------|:----------------| -| totalSize | The size of the frame, counting everything that comes after it (in bytes) | 4 | -| commandSize | The size of the protobuf-serialized command | 4 | -| message | The protobuf message serialized in a raw binary format (rather than in protobuf format) | | +| Component | Description | Size (in bytes) | +|:--------------|:----------------------------------------------------------------------------------------|:----------------| +| `totalSize` | The size of the frame, counting everything that comes after it (in bytes) | 4 | +| `commandSize` | The size of the protobuf-serialized command | 4 | +| `message` | The protobuf message serialized in a raw binary format (rather than in protobuf format) | | ### Payload commands Payload commands have this basic structure: -| Component | Description | Size (in bytes) | -|:-------------|:--------------------------------------------------------------------------------------------|:----------------| -| totalSize | The size of the frame, counting everything that comes after it (in bytes) | 4 | -| commandSize | The size of the protobuf-serialized command | 4 | -| message | The protobuf message serialized in a raw binary format (rather than in protobuf format) | | -| magicNumber | A 2-byte byte array (`0x0e01`) identifying the current format | 2 | -| checksum | A [CRC32-C checksum](http://www.evanjones.ca/crc32c.html) of everything that comes after it | 4 | -| metadataSize | The size of the message [metadata](#message-metadata) | 4 | -| metadata | The message [metadata](#message-metadata) stored as a binary protobuf message | | -| payload | Anything left in the frame is considered the payload and can include any sequence of bytes | | +| Component | Required or optional| Description | Size (in bytes) | +|:-----------------------------------|:----------|:--------------------------------------------------------------------------------------------|:----------------| +| `totalSize` | Required | The size of the frame, counting everything that comes after it (in bytes) | 4 | +| `commandSize` | Required | The size of the protobuf-serialized command | 4 | +| `message` | Required | The protobuf message serialized in a raw binary format (rather than in protobuf format) | | +| `magicNumberOfBrokerEntryMetadata` | Optional | A 2-byte byte array (`0x0e02`) identifying the broker entry metadata
**Note**: `magicNumberOfBrokerEntryMetadata` , `brokerEntryMetadataSize`, and `brokerEntryMetadata` should be used **together**. | 2 | +| `brokerEntryMetadataSize` | Optional | The size of the broker entry metadata | 4 | +| `brokerEntryMetadata` | Optional | The broker entry metadata stored as a binary protobuf message | | +| `magicNumber` | Required | A 2-byte byte array (`0x0e01`) identifying the current format | 2 | +| `checksum` | Required | A [CRC32-C checksum](http://www.evanjones.ca/crc32c.html) of everything that comes after it | 4 | +| `metadataSize` | Required | The size of the message [metadata](#message-metadata) | 4 | +| `metadata` | Required | The message [metadata](#message-metadata) stored as a binary protobuf message | | +| `payload` | Required | Anything left in the frame is considered the payload and can include any sequence of bytes | | + +## Broker entry metadata + +Broker entry metadata is stored alongside the message metadata as a serialized protobuf message. +It is created by the broker when the message arrived at the broker and passed without changes to the consumer if configured. + +| Field | Required or optional | Description | +|:-------------------|:----------------|:------------------------------------------------------------------------------------------------------------------------------| +| `broker_timestamp` | Optional | The timestamp when a message arrived at the broker (`id est` as the number of milliseconds since January 1st, 1970 in UTC) | +| `index` | Optional | The index of the message. It is assigned by the broker. ## Message metadata -Message metadata is stored alongside the application-specified payload as a serialized protobuf message. Metadata is created by the producer and passed on unchanged to the consumer. +Message metadata is stored alongside the application-specified payload as a serialized protobuf message. Metadata is created by the producer and passed without changes to the consumer. -| Field | Description | -|:-------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `producer_name` | The name of the producer that published the message | -| `sequence_id` | The sequence ID of the message, assigned by producer | -| `publish_time` | The publish timestamp in Unix time (i.e. as the number of milliseconds since January 1st, 1970 in UTC) | -| `properties` | A sequence of key/value pairs (using the [`KeyValue`](https://github.com/apache/pulsar/blob/master/pulsar-common/src/main/proto/PulsarApi.proto#L32) message). These are application-defined keys and values with no special meaning to Pulsar. | -| `replicated_from` *(optional)* | Indicates that the message has been replicated and specifies the name of the [cluster](reference-terminology.md#cluster) where the message was originally published | -| `partition_key` *(optional)* | While publishing on a partition topic, if the key is present, the hash of the key is used to determine which partition to choose. Partition key is used as the message key. | -| `compression` *(optional)* | Signals that payload has been compressed and with which compression library | -| `uncompressed_size` *(optional)* | If compression is used, the producer must fill the uncompressed size field with the original payload size | -| `num_messages_in_batch` *(optional)* | If this message is really a [batch](#batch-messages) of multiple entries, this field must be set to the number of messages in the batch | +| Field | Required or optional | Description | +|:-------------------------|:----------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `producer_name` | Required | The name of the producer that published the message | +| `sequence_id` | Required | The sequence ID of the message, assigned by producer | +| `publish_time` | Required | The publish timestamp in Unix time (i.e. as the number of milliseconds since January 1st, 1970 in UTC) | +| `properties` | Required | A sequence of key/value pairs (using the [`KeyValue`](https://github.com/apache/pulsar/blob/master/pulsar-common/src/main/proto/PulsarApi.proto#L32) message). These are application-defined keys and values with no special meaning to Pulsar. | +| `replicated_from` | Optional | Indicates that the message has been replicated and specifies the name of the [cluster](reference-terminology.md#cluster) where the message was originally published | +| `partition_key` | Optional | While publishing on a partition topic, if the key is present, the hash of the key is used to determine which partition to choose. Partition key is used as the message key. | +| `compression` | Optional | Signals that payload has been compressed and with which compression library | +| `uncompressed_size` | Optional | If compression is used, the producer must fill the uncompressed size field with the original payload size | +| `num_messages_in_batch` | Optional | If this message is really a [batch](#batch-messages) of multiple entries, this field must be set to the number of messages in the batch | ### Batch messages @@ -76,19 +89,19 @@ object. For a single batch, the payload format will look like this: -| Field | Description | -|:--------------|:------------------------------------------------------------| -| metadataSizeN | The size of the single message metadata serialized Protobuf | -| metadataN | Single message metadata | -| payloadN | Message payload passed by application | +| Field | Required or optional | Description | +|:----------------|:---------------------|:-----------------------------------------------------------| +| `metadataSizeN` | Required |The size of the single message metadata serialized Protobuf | +| `metadataN` | Required |Single message metadata | +| `payloadN` | Required |Message payload passed by application | Each metadata field looks like this; -| Field | Description | -|:---------------------------|:--------------------------------------------------------| -| properties | Application-defined properties | -| partition key *(optional)* | Key to indicate the hashing to a particular partition | -| payload_size | Size of the payload for the single message in the batch | +| Field | Required or optional | Description | +|:----------------|:----------------------|:--------------------------------------------------------| +| `properties` | Required | Application-defined properties | +| `partition key` | Optional | Key to indicate the hashing to a particular partition | +| `payload_size` | Required | Size of the payload for the single message in the batch | When compression is enabled, the whole batch will be compressed at once. diff --git a/site2/docs/security-oauth2.md b/site2/docs/security-oauth2.md index 7ea9f35969397..67641d7bf7c2a 100644 --- a/site2/docs/security-oauth2.md +++ b/site2/docs/security-oauth2.md @@ -29,6 +29,7 @@ The following table lists parameters supported for the `client credentials` auth | `issuerUrl` | URL of the authentication provider which allows the Pulsar client to obtain an access token | `https://accounts.google.com` | Required | | `privateKey` | URL to a JSON credentials file | Support the following pattern formats:
  • `file:///path/to/file`
  • `file:/path/to/file`
  • `data:application/json;base64,` | Required | | `audience` | An OAuth 2.0 "resource server" identifier for the Pulsar cluster | `https://broker.example.com` | Required | +| `scope` | Scope of an access request.
    For more more information, see [access token scope](https://datatracker.ietf.org/doc/html/rfc6749#section-3.3). | api://pulsar-cluster-1/.default | Optional | The credentials file contains service account credentials used with the client authentication type. The following shows an example of a credentials file `credentials_file.json`. @@ -98,7 +99,7 @@ PulsarClient client = PulsarClient.builder() ### C++ client -The C++ client is similar to the Java client. You need to provide parameters of `issuerUrl`, `private_key` (the credentials file path), and the audience. +The C++ client is similar to the Java client. You need to provide parameters of `issuerUrl`, `private_key` (the credentials file path), and `audience`. ```c++ #include diff --git a/site2/website-next/data/connectors.js b/site2/website-next/data/connectors.js new file mode 100644 index 0000000000000..4182f2c98648c --- /dev/null +++ b/site2/website-next/data/connectors.js @@ -0,0 +1,176 @@ +module.exports = [ + { + name: 'aerospike', + longName: 'Aerospike sink', + type: 'Sink', + link: 'https://www.aerospike.com/' + }, + { + name: 'canal', + longName: 'Alibaba Canal CDC source', + type: 'Source', + link: 'https://github.com/alibaba/canal' + }, + { + name: 'cassandra', + longName: 'Apache Cassandra source and sink', + type: 'Source and sink', + link: 'https://cassandra.apache.org' + }, + { + name: 'kafka-connect-adaptor', + longName: 'Apache Kafka Connect Adaptor source and sink', + type: 'Source and sink', + link: 'http://kafka.apache.org/' + }, + { + name: 'dynamodb', + longName: 'AWS DynamoDB source', + type: 'Source', + link: 'https://aws.amazon.com/dynamodb/' + }, + { + name: 'kinesis', + longName: 'AWS Kinesis source and sink', + type: 'Source and sink', + link: 'https://aws.amazon.com/kinesis/' + }, + { + name: 'debezium-mysql', + longName: 'Debezium MySQL CDC source', + type: 'Source', + link: 'https://debezium.io/' + }, + { + name: 'debezium-postgres', + longName: 'Debezium PostgreSQL CDC source', + type: 'Source', + link: 'https://debezium.io/' + }, + { + name: 'elastic-search', + longName: 'ElasticSearch source and sink', + type: 'Source and sink', + link: 'https://www.elastic.co/' + }, + { + name: 'file', + longName: 'File source', + type: 'Source', + link: 'https://github.com/apache/pulsar/tree/master/pulsar-io/file' + }, + { + name: 'flume', + longName: 'Flume source and sink', + type: 'Source and sink', + link: 'https://flume.apache.org/' + }, + { + name: 'hbase', + longName: 'HBase sink', + type: 'Sink', + link: 'https://hbase.apache.org/' + }, + { + name: 'hdfs2', + longName: 'HDFS2 sink', + type: 'Sink', + link: 'https://hadoop.apache.org/' + }, + { + name: 'hdfs3', + longName: 'HDFS3 sink', + type: 'Sink', + link: 'https://hadoop.apache.org/' + }, + { + name: 'influxdb', + longName: 'InfluxDB sink', + type: 'Sink', + link: 'https://www.influxdata.com/' + }, + { + name: 'jdbc-clickhouse', + longName: 'JDBC ClickHouse sink', + type: 'Sink', + link: 'https://clickhouse.tech/' + }, + { + name: 'jdbc-mariadb', + longName: 'JDBC MariaDB sink', + type: 'Sink', + link: 'https://mariadb.org/' + }, + { + name: 'jdbc-postgres', + longName: 'JDBC PostgresSQL sink', + type: 'Sink', + link: 'https://www.postgresql.org/' + }, + { + name: 'jdbc-sqlite', + longName: 'JDBC SQLite sink', + type: 'Sink', + link: 'https://www.sqlite.org/' + }, + { + name: 'kafka', + longName: 'Kafka source and sink', + type: 'Source and sink', + link: 'https://kafka.apache.org/' + }, + { + name: 'mongo', + longName: 'MongoDB sink', + type: 'Sink', + link: 'https://www.mongodb.com/' + }, + { + name: 'netty', + longName: 'Netty source', + type: 'Source', + link: 'https://netty.io/' + }, + { + name: 'nsq', + longName: 'NSQ source', + type: 'Source', + link: 'https://nsq.io/', + }, + { + name: 'rabbitmq', + longName: 'RabbitMQ source and sink', + type: 'Source and sink', + link: 'https://www.rabbitmq.com/' + }, + { + name: 'redis', + longName: 'Redis sink', + type: 'Sink', + link: 'https://redis.io/' + }, + { + name: 'solr', + longName: 'Solr sink', + type: 'Sink', + link: 'https://lucene.apache.org/solr/' + }, + { + name: 'netty', + longName: 'TCP/UDP with Netty source', + type: 'Source', + link: 'https://netty.io/' + }, + { + name: 'data-generator', + longName: 'Test Data Generator source', + type: 'Source', + link: 'https://github.com/apache/pulsar/tree/master/pulsar-io/data-generator' + }, + { + name: 'twitter', + longName: 'Twitter Firehose source', + type: 'Source', + link: 'https://developer.twitter.com/en/docs' + } +] diff --git a/site2/website-next/data/resources.js b/site2/website-next/data/resources.js new file mode 100644 index 0000000000000..44a392b5c3f20 --- /dev/null +++ b/site2/website-next/data/resources.js @@ -0,0 +1,909 @@ +module.exports = { + articles: [ + { + forum: 'RT Instights', + forum_link: 'https://www.rtinsights.com/', + date: 'Oct 2021', + title: "The Critical Role Streaming Plays in a Data Stack", + link: 'https://www.rtinsights.com/the-critical-role-streaming-plays-in-a-data-stack/' + }, + { + forum: 'Mikel\'s TechBlog', + forum_link: 'https://mikeldeltio.com/', + date: 'Sept 2021', + title: "Distributed Databases: Apache Pulsar (in ESP)", + link: 'https://mikeldeltio.com/2021/09/20/distributed-databases-apache-pulsar/' + }, + { + forum: 'Pandio Blog', + forum_link: 'https://pandio.com/blog/', + date: 'Sept 2021', + title: "Apache Pulsar vs. Google PubSub", + link: 'https://pandio.com/blog/a-detailed-deep-dive-into-pulsar-and-google-pubsub-which-is-better-and-why' + }, + { + forum: 'StreamNative Blog', + forum_link: 'https://streamnative.io/', + date: 'Aug 2021', + title: "Scalable Stream Processing with Pulsar’s Key_Shared Subscription", + link: 'https://streamnative.io/en/blog/engineering/2021-08-25-scalable-stream-processing-with-pulsars-key-shared-subscription/' + }, + { + forum: 'Pandio Blog', + forum_link: 'https://pandio.com/blog/', + date: 'July 2021', + title: "Pulsar vs. Kafka", + link: 'https://pandio.com/blog/pulsar-vs-kafka/' + }, + { + forum: 'Building the Open Data Stack', + forum_link: 'https://medium.com/building-the-open-data-stack', + date: 'June 2021', + title: "5 More Reasons to Choose Apache Pulsar Over Apache Kafka", + link: 'https://medium.com/building-the-open-data-stack/5-more-reasons-to-choose-apache-pulsar-over-apache-kafka-c09b259e3691' + }, + { + forum: 'dattell', + forum_link: 'https://dattell.com/data-architecture-blog/', + date: 'Feb 2021', + title: "Apache Pulsar Subscription Types", + link: 'https://dattell.com/data-architecture-blog/subscription-types-in-apache-pulsar/' + } + + ], + + /****************************** + * Presentations + * + ******************************/ + presentations: [ + { + forum: 'Apache Pulsar Neighborhood YouTube', + forum_link: 'https://www.youtube.com/apachepulsarneighborhood', + presenter: 'Enrico Olivelli', + date: 'Sept 2021', + title: "Apache Pulsar Deep Dive- an End-to-end view of the Data Flow", + link: 'https://www.youtube.com/watch?v=oLXCCCGsrWM' + }, + { + forum: 'Apache Pulsar Neighborhood YouTube', + forum_link: 'https://www.youtube.com/apachepulsarneighborhood', + presenter: 'Simba Khadder', + date: 'Sept 2021', + title: "Apache Pulsar and Machine Learning", + link: 'https://youtu.be/K2WXDwo1y0k' + }, + { + forum: 'Apache Pulsar Neighborhood YouTube', + forum_link: 'https://www.youtube.com/apachepulsarneighborhood', + presenter: 'Enrico Olivelli', + date: 'Sept 2021', + title: "Leveraging Pulsar's Next Gen Streaming Capabilities from a JavaEE Application", + link: 'https://www.youtube.com/watch?v=0NA0BIvkQrs' + }, + { + forum: 'Devin Bost\'s YouTube', + forum_link: 'https://www.youtube.com/channel/UCz_Y86AT4G15xU3IojhX2Kw', + presenter: 'Devin Bost', + date: 'Aug 2021', + title: "Streaming Patterns and Best Practices with Apache Pulsar for Enabling Machine Learning and Analytics", + link: 'https://www.youtube.com/watch?v=Fdqre_hTnUI' + }, + + { + forum: 'Pulsar Summit NA 2021', + forum_link: 'https://pulsar-summit.org/', + presenter: '', + date: 'June 2021', + title: "Pulsar Summit NA 2021 Playlist", + link: 'https://www.youtube.com/watch?v=-Bm1h508oIQ&list=PLqRma1oIkcWjyezVodJHcp8GSypvGuc1C' + } + ], + /*************************** + * Older Articles + * + * + * *************************/ + older_articles: [ + + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + date: 'July 2020', + title: "Pulsar vs Kafka - Part 1 - A More Accurate Perspective on Performance, Architecture, and Features", + link: 'https://streamnative.io/blog/tech/pulsar-vs-kafka-part-1', + title2: "Pulsar vs Kafka - Part 2 - Adoption, Use Cases, Differentiators, and Community", + link2: 'https://streamnative.io/blog/tech/pulsar-vs-kafka-part-2' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + date: 'June 2020', + title: "Taking messaging and data ingestion systems to the next level", + link: 'https://streamnative.io/blog/tech/2020-07-08-podcast' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + date: 'June 2020', + title: "Announcing AMQP-on-Pulsar: bring native AMQP protocol support to Apache Pulsar", + link: 'https://streamnative.io/blog/tech/2020-06-15-announcing-aop-on-pulsar' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + date: 'June 2020', + title: "How to trace Pulsar messages with OpenTracing and Jaeger", + link: 'https://streamnative.io/blog/tech/2020-06-11-opentracing-instrumentation-for-pulsar' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + date: 'May 2020', + title: "Introducing StreamNative Hub — Extend Pulsar Capabilities with Rich Integrations", + link: 'https://streamnative.io/blog/tech/2020-05-26-intro-to-hub' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + title: "How to Build a Distributed Database with Apache BookKeeper — Part 1", + link: 'https://streamnative.io/blog/tech/2020-02-04-how-to-build-database', + title2: "How to Build a Distributed Database with Apache BookKeeper — Part 2", + link2: 'https://streamnative.io/blog/tech/2020-04-14-distributed-database-bk2', + title3: "How to Build a Distributed Database with Apache BookKeeper — Part 3", + link3: 'https://streamnative.io/blog/tech/2020-05-12-distributed-database-bk3' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + title: "How Apache Pulsar Helps Streamline Message System and Reduces O&M Costs at Tuya Smart", + link: 'https://streamnative.io/blog/tech/2020-05-08-tuya-tech-blog' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + title: "Why Zhaopin Chooses Pulsar SQL for Search Log Analysis", + link: 'https://streamnative.io/blog/tech/2020-05-07-zhaopin-tech-blog' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + title: "Why we moved from Apache Kafka to Apache Pulsar", + link: 'https://streamnative.io/blog/tech/2020-04-21-from-apache-kafka-to-apache-pulsar' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + title: "Announcing Kafka-on-Pulsar: bring native Kafka protocol support to Apache Pulsar", + link: 'https://streamnative.io/blog/tech/2020-03-24-bring-native-kafka-protocol-support-to-apache-pulsar' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + title: "Apache Pulsar® Helps Tencent Process Tens of Billions of Financial Transactions Efficiently with Virtually No Data Loss", + link: 'https://streamnative.io/blog/tech/2020-02-18-pulsar-help-tencent' + }, + { + forum: 'Yahoo Engineering blog', + forum_link: 'https://yahooeng.tumblr.com/', + title: 'Open sourcing Pulsar, pub-sub messaging at scale', + link: 'https://yahooeng.tumblr.com/post/150078336821/open-sourcing-pulsar-pub-sub-messaging-at-scale' + }, + { + forum: 'Streamlio blog', + forum_link: 'https://streaml.io/blog', + title: 'Introduction to Apache Pulsar', + link: 'https://streaml.io/blog/intro-to-pulsar/' + }, + { + forum: 'Streamlio blog', + forum_link: 'https://streaml.io/blog', + title: 'Why Apache Pulsar? Part 1', + link: 'https://streaml.io/blog/why-apache-pulsar/', + title2: 'Why Apache Pulsar? Part 2', + link2: 'https://streaml.io/blog/why-apache-pulsar-part-2' + }, + { + forum: "Jack Vanlightly's blog", + forum_link: 'https://jack-vanlightly.com', + title: 'Understanding How Apache Pulsar Works', + link: 'https://jack-vanlightly.com/blog/2018/10/2/understanding-how-apache-pulsar-works' + }, + { + forum: "Anuradha Prasanna's blog", + forum_link: 'https://medium.com/@anuradha.neo', + title: 'Kafka is not the best anymore - Meet Pulsar!', + link: 'https://medium.com/@anuradha.neo/kafka-is-not-the-best-anymore-meet-pulsar-9eb435c9fc0b' + }, + { + forum: 'Kafkaesque blog', + forum_link: 'https://kafkaesque.io/news/', + title: '7 Reasons We Choose Apache Pulsar over Apache Kafka', + link: 'https://kafkaesque.io/7-reasons-we-choose-apache-pulsar-over-apache-kafka/' + }, + { + forum: "Jack Vanlightly's blog", + forum_link: 'https://jack-vanlightly.com', + title: 'How to (not) lose messages on an Apache Pulsar Cluster', + link: 'https://jack-vanlightly.com/blog/2018/10/21/how-to-not-lose-messages-on-an-apache-pulsar-cluster' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + title: "How Orange Financial combats financial fraud in over 50M transactions a day using Apache Pulsar", + link: 'https://streamnative.io/blog/tech/2019-11-11-how-orange-finaacial-combats-financial-fraud-over-50m-transactions-a-day-useing-apache-pulsar' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + title: "Powering Tencent Billing Platform with Apache Pulsar", + link: 'https://streamnative.io/blog/tech/2019-10-22-powering-tencent-billing-platform-with-apache-pulsar/' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + title: "Use Apache SkyWalking to Trace Apache Pulsar Messages", + link: 'https://streamnative.io/blog/tech/2019-10-10-use-apache-skywalking-to-trace-apache-pulsar/' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + title: "StreamNative open sourced and contributed Apache Pulsar Manager to ASF", + link: 'https://streamnative.io/blog/tech/2019-09-24-streamnative-opensourced-pulsar-manager/' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + title: "Apache Pulsar Adoption Story in ActorCloud (IoT Platform)", + link: 'https://streamnative.io/blog/tech/2019-09-09-apache-pulsar-adoption-in-actorcloud/' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + title: "Apache Pulsar at Yahoo!JAPAN", + link: 'https://streamnative.io/blog/tech/2019-09-06-pular-at-yahoo-japan/' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + title: "Build a Priority-based Push Notification System Using Apache Pulsar at GeTui", + link: 'https://streamnative.io/blog/tech/2019-07-23-build-a-priority-based-push-notification-system-using-apache-pulsar-at-getui/' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://streamnative.io/', + title: "Use Apache Pulsar as Streaming Table with 8 Lines of Code", + link: 'https://streamnative.io/blog/tech/2019-08-28-use-apache-pulsar-as-streaming-table-with-8-lines-of-code/' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://medium.com/streamnative/', + title: 'Apache Pulsar as One Unified Storage System for Real Time and Historical Analysis', + link: 'https://medium.com/streamnative/apache-pulsar-as-one-storage-455222c59017' + }, + { + forum: 'StreamNative blog', + forum_link: 'https://medium.com/streamnative/', + title: "What’s New in Apache Pulsar 2.4.0", + link: 'https://medium.com/streamnative/whats-new-in-apache-pulsar-2-4-0-d646f6727642' + }, + { + forum: "Penghui Li's blog", + forum_link: 'https://medium.com/@codelipenghui', + title: "Simplifying Zhaopin's Event Center Using Apache Pulsar", + link: 'https://medium.com/@codelipenghui/simplifying-zhaopins-event-center-with-apache-pulsar-9784b73bead1' + }, + { + forum: "Jowanza Joseph's blog", + forum_link: "https://www.jowanza.com/blog", + title: 'Efficient Stream Processing With Pulsar Functions', + link: 'https://www.jowanza.com/blog/2019/3/9/efficient-stream-processing-with-pulsar-functions' + }, + { + forum: "Jesse Anderson's blog", + forum_link: 'http://www.jesse-anderson.com/', + title: 'Creating Work Queues with Apache Kafka and Apache Pulsar', + link: 'http://www.jesse-anderson.com/2018/08/creating-work-queues-with-apache-kafka-and-apache-pulsar/' + }, + { + forum: 'Streamlio blog', + forum_link: 'https://streaml.io/blog', + title: 'Building Data Driven Applications at STICORP using Apache Pulsar', + link: 'https://streaml.io/blog/building-data-driven-applications-with-apache-pulsar-at-sticorp' + }, + { + forum: 'The Hut Group tech blog', + forum_link: 'https://medium.com/thg-tech-blog', + title: 'A feather in the caps', + link: 'https://medium.com/thg-tech-blog/a-feather-in-their-caps-4c4083cf9f46' + }, + { + forum: "Karthikeyan Palanivelu's blog", + forum_link: 'https://medium.com/@pckeyan', + title: 'Apache Pulsar — One Cluster for the entire enterprise using Multi-tenancy', + link: 'https://medium.com/@pckeyan/apache-pulsar-one-cluster-for-the-entire-enterprise-using-multi-tenancy-c37e2ee350ef' + }, + { + forum: "Capital One tech blog", + forum_link: "https://medium.com/capital-one-tech", + title: 'Apache Pulsar — A Gentle Introduction to Apache’s Newest Pub-Sub Messaging Platform', + link: 'https://medium.com/capital-one-tech/apache-pulsar-apaches-newest-pub-sub-messaging-platform-1c1ba1a6c673' + }, + { + forum: "Karthikeyan Palanivelu's blog", + forum_link: 'https://medium.com/@pckeyan', + title: 'Apache Pulsar: Geo-replication', + link: 'https://medium.com/@pckeyan/apache-pulsar-geo-replication-ad4f0ca3224b' + }, + { + forum: "Karthikeyan Palanivelu's blog", + forum_link: 'https://medium.com/@pckeyan', + title: 'Apache Pulsar: Geo-replication — Synchronous Replication : Hybrid Deployment Model', + link: 'https://medium.com/@pckeyan/apache-pulsar-geo-replication-synchronous-replication-hybrid-deployment-model-3282013e9ae5' + }, + { + forum: "Streamlio blog", + forum_link: 'https://streaml.io/blog', + title: 'Pulsar topic compaction', + link: 'https://streaml.io/blog/pulsar-topic-compaction' + }, + { + forum: "Streamlio blog", + forum_link: 'https://streaml.io/blog', + title: 'Tiered Storage in Apache Pulsar', + link: 'https://streaml.io/blog/tiered-storage-in-apache-pulsar' + }, + { + forum: "Streamlio blog", + forum_link: 'https://streaml.io/blog', + title: 'Introducing Pulsar IO', + link: 'https://streaml.io/blog/introducing-pulsar-io' + }, + { + forum: "Streamlio blog", + forum_link: 'https://streaml.io/blog', + title: 'Pulsar topic compaction', + link: 'https://streaml.io/blog/pulsar-topic-compaction' + }, + + { + forum: "Jesse Anderson's blog", + forum_link: 'https://www.jesse-anderson.com', + title: 'Reducing Operational Overhead with Pulsar Functions', + link: 'https://www.jesse-anderson.com/2019/05/reducing-operational-overhead-with-pulsar-functions/' + }, + { + forum: "Streamlio blog", + forum_link: 'https://streaml.io/blog', + title: 'Querying Data Streams with Apache Pulsar SQL', + link: 'https://streaml.io/blog/querying-data-streams-with-apache-pulsar-sql' + }, + { + forum: "Streamlio blog", + forum_link: 'https://streaml.io/blog', + title: 'Configuring Apache Pulsar Tiered Storage with Amazon S3', + link: 'https://streaml.io/blog/configuring-apache-pulsar-tiered-storage-with-amazon-s3' + }, + { + forum: "Streamlio blog", + forum_link: 'https://streaml.io/blog', + title: 'Comparing LogDevice and Apache Pulsar', + link: 'https://streaml.io/blog/comparing-logdevice-and-apache-pulsar' + }, + { + forum: "Streamlio blog", + forum_link: 'https://streaml.io/blog', + title: 'Pulsar’s Multi-Layer System Architecture', + link: 'https://streaml.io/blog/apache-pulsar-architecture-designing-for-streaming-performance-and-scalability' + }, + { + forum: "Streamlio blog", + forum_link: 'https://streaml.io/blog', + title: 'Debugging Pulsar Functions in Java', + link: 'https://streaml.io/blog/debugging-pulsar-functions-in-java' + }, + { + forum: "Streamlio blog", + forum_link: 'https://streaml.io/blog', + title: 'Sentiment Analysis of Tweets using Apache Pulsar', + link: 'https://streaml.io/blog/sentiment-analysis-of-tweets-using-apache-pulsar' + }, + { + forum: 'Apache Flink blog', + forum_link: 'https://flink.apache.org/blog/', + title: 'When Flink & Pulsar Come Together', + link: 'https://flink.apache.org/2019/05/03/pulsar-flink.html' + }, + { + forum: 'Debezium blog', + forum_link: 'https://debezium.io/blog', + title: 'Tutorial for Using Debezium Connectors With Apache Pulsar', + link: 'https://debezium.io/blog/2019/05/23/tutorial-using-debezium-connectors-with-apache-pulsar/' + }, + { + forum: 'Pulsar in Action', + forum_link: 'https://www.manning.com/books/pulsar-in-action', + title: "Pulsar in Action", + link: 'https://www.manning.com/books/pulsar-in-action' + } + ], + /*************************** + * Older Presentations + * + * + * *************************/ + older_presentations: [ + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Ludwig Pummer, Joe Francis', + date: 'June 2020', + title: "Five Years of Operating a Large Scale Globally Replicated Pulsar Installation", + link: 'https://www.slideshare.net/streamnative/five-years-of-operating-a-large-scale-globally-replicated-pulsar-installation' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Alexandre Duval', + date: 'June 2020', + title: "Building a FaaS with Pulsar", + link: 'https://www.slideshare.net/streamnative/building-a-faas-with-pulsar' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Greg Methvin', + date: 'June 2020', + title: "Scaling customer engagement with Apache Pulsar", + link: 'https://www.slideshare.net/streamnative/scaling-customer-engagement-with-apache-pulsar' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'William MaLane', + date: 'June 2020', + title: "Finding your pulse for a global enterprise communications nervous system", + link: 'https://www.slideshare.net/streamnative/finding-your-pulse-for-a-global-enterprise-communications-nervous-systemwilliam-mclane' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'David Kjerrumgaard', + date: 'June 2020', + title: "Using Apache Pulsar to Provide Real-Time IoT Analytics on the Edge", + link: 'https://www.slideshare.net/streamnative/using-apache-pulsar-to-provide-realtime-iot-analytics-on-the-edgedavid' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Neng Lv', + date: 'June 2020', + title: "Stream or segment: what is the best way to access your events in Pulsar", + link: 'https://www.slideshare.net/streamnative/stream-or-segment-what-is-the-best-way-to-access-your-events-in-pulsarneng' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Simba Khadder', + date: 'June 2020', + title: "Feature Stories: Building Machine Learning Infrastructure on Apache Pulsar", + link: 'https://www.slideshare.net/streamnative/feature-stores-building-machine-learning-infrastructure-on-apache-pulsarsimba-khadder' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Sanjeev Kulkarni', + date: 'June 2020', + title: "Pulsar Functions Deep Dive", + link: 'https://www.slideshare.net/streamnative/pulsar-functions-deep-divesanjeev-kulkarni' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Jesse Anderson', + date: 'June 2020', + title: "Pulsar for Kafka People", + link: 'https://www.slideshare.net/streamnative/pulsar-for-kafka-peoplejesse-anderson' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Jerry Peng', + date: 'June 2020', + title: "Interactive querying of streams using Apache Pulsar", + link: 'https://www.slideshare.net/streamnative/interactive-querying-of-streams-using-apache-pulsarjerry-peng' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Anup Ghatage, Ankit Jain', + date: 'June 2020', + title: "The first step to Multi AZ architecture for Apache BookKeeper", + link: 'https://www.slideshare.net/streamnative/the-first-step-to-multi-az-architecture-for-apache-bookkeeperanup-ghatage' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Carolyn King, Matteo Merli, Sijie Guo', + date: 'June 2020', + title: "Messaging & Streaming Everywhere", + link: 'https://www.slideshare.net/streamnative/open-keynotecarolynmatteosijie' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Joe Francis, Rajan Dhabalia', + date: 'June 2020', + title: "Pulsar Storage on BookKeeper - Seamless Evolution", + link: 'https://www.slideshare.net/streamnative/pulsar-storage-on-bookkeeper-seamless-evolution' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Ningguo Chen', + date: 'June 2020', + title: "How Apache Pulsar Helps Tencent Process Tens of Billions of Transactions Efficiently", + link: 'https://www.slideshare.net/streamnative/how-apache-pulsar-helps-tencent-process-tens-of-billions-of-transactions-efficientlyningguo-chen' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Vincent Xie, Jia Zhai', + date: 'June 2020', + title: "Unify Storage Backend for Batch and Streaming Computation with Apache Pulsar", + link: 'https://www.slideshare.net/streamnative/unify-storage-backend-for-batch-and-streaming-computation-with-apache-pulsarvincent' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Seth Wiesman', + date: 'June 2020', + title: "Unified Data Processing with Apache Flink and Apache Pulsar", + link: 'https://www.slideshare.net/streamnative/unified-data-processing-with-apache-flink-and-apache-pulsarseth-wiesman' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Paige Roberts', + date: 'June 2020', + title: "Architecting Production IoT Analytics", + link: 'https://www.slideshare.net/streamnative/architecting-production-iot-analyticspaige-roberts' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Nozomi Kurihara', + date: 'June 2020', + title: "Large scale log pipeline using Apache Pulsar", + link: 'https://www.slideshare.net/streamnative/large-scale-log-pipeline-using-apache-pulsarnozomi' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Robert van Mölken', + date: 'June 2020', + title: "Ten reasons to choose Apache Pulsar over Apache Kafka for Event Sourcing", + link: 'https://www.slideshare.net/streamnative/ten-reasons-to-choose-apache-pulsar-over-apache-kafka-for-event-sourcingrobert-van-mlken' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Simon Crosby', + date: 'June 2020', + title: "Easily Build a Smart Pulsar Stream Processor", + link: 'https://www.slideshare.net/streamnative/easily-build-a-smart-pulsar-stream-processorsimon-crosby' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Pranav Dharma', + date: 'June 2020', + title: "How Splunk Mission Control leverages various Pulsar subscription types", + link: 'https://www.slideshare.net/streamnative/how-splunk-mission-control-leverages-various-pulsar-subscription-typespranav-dharma' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Caito Scherr', + date: 'June 2020', + title: "Streaming, Fast and Slow", + link: 'https://www.slideshare.net/streamnative/streaming-fast-and-slowcaito-scherr' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Karthik Ramasamy', + date: 'June 2020', + title: "Why Splunk Chose Pulsar", + link: 'https://www.slideshare.net/streamnative/why-splunk-chose-pulsarkarthik-ramasamy' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Chris Kellogg', + date: 'June 2020', + title: "Securing your Pulsar Cluster with Vault", + link: 'https://www.slideshare.net/streamnative/securing-your-pulsar-cluster-with-vaultchris-kellogg-236137352' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Enrico Olivelli', + date: 'June 2020', + title: "Introducing HerdDB - a distributed JVM embeddable database built upon Apache BookKeeper", + link: 'https://www.slideshare.net/streamnative/introducing-herddb-a-distributed-jvm-embeddable-database-built-upon-apache-bookkeeperenrico-olivelli' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Pierre Zemb', + date: 'June 2020', + title: "Building a Messaging Solutions for OVHcloud with Apache Pulsar", + link: 'https://www.slideshare.net/streamnative/building-a-messaging-solutions-for-ovhcloud-with-apache-pulsarpierre-zemb' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Devin Bost', + date: 'June 2020', + title: "Pulsar Architectural Patterns for CI/CD Automation and Self-Service", + link: 'https://www.slideshare.net/streamnative/pulsar-architectural-patterns-for-cicd-automation-and-selfservicedevin-bost-236083234' + }, + { + forum: 'Pulsar Summit 2020', + forum_link: 'https://pulsar-summit.org/', + presenter: 'Addison Higham', + date: 'June 2020', + title: "Getting Pulsar Spinning", + link: 'https://www.slideshare.net/streamnative/getting-pulsar-spinningaddison-higham' + }, + { + forum: 'StreamNative Academy', + forum_link: 'https://streamnative.io/academy', + presenter: 'Shivji Jha', + date: 'May 2020', + title: "Lessons from managing a Pulsar cluster (Nutanix)", + link: 'https://www.slideshare.net/streamnative/lessons-from-managing-a-pulsar-cluster-nutanix' + }, + { + forum: '', + forum_link: '', + presenter: 'Sijie Guo', + date: 'May 2020', + title: "Building event streaming pipelines using Apache Pulsar", + link: 'https://www.slideshare.net/streamnative/building-event-streaming-pipelines-using-apache-pulsar' + }, + { + forum: 'TGIPulsar', + forum_link: 'https://streamnative.io/resource#tgip', + presenter: 'Sijie Guo', + date: 'April 2020', + title: "Lifecycle of a Pulsar message", + link: 'https://www.slideshare.net/streamnative/tgipulsar-ep-006-lifecycle-of-a-pulsar-message' + }, + { + forum: '', + forum_link: '', + presenter: 'Sijie Guo, Pierre Zemb', + date: 'March 2020', + title: "Introducing Kafka-on-Pulsar: bring native Kafka protocol support to Apache Pulsar", + link: 'https://www.slideshare.net/streamnative/introducing-kafkaonpulsar-bring-native-kafka-protocol-support-to-apache-pulsar' + }, + { + forum: '2019 China Technical Communication Forum', + forum_link: 'http://www.tc-china.org/event/tcweekly/7454/', + presenter: 'Yu Liu', + date: 'December 2019', + title: "Code the docs", + link: 'https://www.slideshare.net/streamnative/code-the-docsyu-liu' + }, + { + forum: 'Google DevFest 2019 in Beijing', + forum_link: 'https://devfest.withgoogle.com/?utm_source=devsite&utm_medium=events&utm_campaign=past&utm_content=DevSite_HPP', + presenter: 'Xiaolong Ran', + date: 'December 2019', + title: "Serverless Event Streaming with Pulsar Functions", + link: 'https://www.slideshare.net/streamnative/google-devfest-about-pulsar-by-xiaolong' + }, + { + forum: 'Apache Pulsar Meetup x PingCAP Infra Meetup | Beijing', + forum_link: 'https://www.huodongxing.com/event/7520647658000', + presenter: 'Xiaolong Ran', + date: 'December 2019', + title: "Apache Pulsar and GitHub", + link: 'https://www.slideshare.net/streamnative/apache-pulsar-and-githubxiaolong' + }, + { + forum: 'Apache Pulsar Meetup x PingCAP Infra Meetup | Beijing', + forum_link: 'https://www.huodongxing.com/event/7520647658000', + presenter: 'Penghui Li, Yong Zhang', + date: 'December 2019', + title: "Transaction preview of Apache Pulsar", + link: 'https://www.slideshare.net/streamnative/transaction-preview-of-apache-pulsar' + }, + { + forum: 'Apache Pulsar Meetup | Shanghai', + forum_link: 'https://www.huodongxing.com/event/5515876233300', + presenter: 'Xiaolong Ran, Guangning E', + date: 'November 2019', + title: "Pulsarctl & Pulsar Manager", + link: 'https://www.slideshare.net/streamnative/pulsarctl-pulsar-manager' + }, + { + forum: 'Apache Pulsar Meetup | Shanghai', + forum_link: 'https://www.huodongxing.com/event/5515876233300', + presenter: 'Jennifer Huang', + date: 'November 2019', + title: "Apache Pulsar Community", + link: 'https://www.slideshare.net/streamnative/apache-pulsar-communityjennifer' + }, + { + forum: 'Apache Pulsar Meetup | Shanghai', + forum_link: 'https://www.huodongxing.com/event/5515876233300', + presenter: 'Penghui Li, Bo Cong, Jia Zhai', + date: 'November 2019', + title: "Preview of Apache Pulsar 2.5.0", + link: 'https://www.slideshare.net/streamnative/preview-of-apache-pulsar-250' + }, + { + forum: 'COSCon\'19', + forum_link: 'https://kaiyuanshe.cn/activity/summit/coscon-2019/', + presenter: 'Jennifer Huang', + date: 'November, 2019', + title: "Apache Pulsar: A borderless community", + link: 'https://www.slideshare.net/streamnative/apache-pulsaraborderlesscommunityjennifer' + }, + { + forum: 'COSCon\'19', + forum_link: 'https://kaiyuanshe.cn/activity/summit/coscon-2019/', + presenter: 'Yu Liu', + date: 'November, 2019', + title: "Cos con19 about Pulsar", + link: 'https://www.slideshare.net/streamnative/cos-con19-pulsaryuliu' + }, + { + forum: 'Flink Forward Europe', + forum_link: 'https://europe-2019.flink-forward.org/', + presenter: 'Sijie Guo', + date: 'October 2019', + title: "Query Pulsar Streams using Apache Flink", + link: 'https://www.slideshare.net/streamnative/query-pulsar-streams-using-apache-flink' + }, + { + forum: 'Strata Data Conference | New York', + forum_link: 'https://conferences.oreilly.com/strata/strata-ny', + presenter: 'Vincent Xie, Jia Zhai', + date: 'September 2019', + title: "How Orange Financial combat financial frauds over 50M transactions a day using Apache Pulsar", + link: 'https://www.slideshare.net/streamnative/how-orange-financial-combat-financial-frauds-over-50m-transactions-a-day-using-apache-pulsar-176284080' + }, + { + forum: 'ApacheCon NA 2019', + forum_link: 'https://www.apachecon.com/acna19/', + presenter: 'Xiaolong Ran', + date: 'September 2019', + title: "Serverless Event Streaming with Pulsar Functions", + link: 'https://www.slideshare.net/streamnative/serverless-event-streaming-with-pulsar-functions-171597848' + }, + { + forum: 'ApacheCon NA 2019', + forum_link: 'https://www.apachecon.com/acna19/', + presenter: 'Penghui Li, Jia Zhai', + date: 'September 2019', + title: "Building Zhaopin's enterprise event center on Apache Pulsar", + link: 'https://www.slideshare.net/streamnative/building-zhaopins-enterprise-event-center-on-apache-pulsar' + }, + { + forum: '', + forum_link: '', + presenter: 'Sijie Guo', + date: 'August 2019', + title: "When Apache Pulsar meets Apache Flink", + link: 'https://www.slideshare.net/streamnative/when-apache-pulsar-meets-apache-flink' + }, + { + forum: 'Apache Pulsar Meetup | Beijing', + forum_link: 'https://www.huodongxing.com/event/1502359221000', + presenter: 'Sijie Guo, Yong Zhang', + date: 'August 2019', + title: "Transaction Support in Pulsar 2.5.0", + link: 'https://www.slideshare.net/streamnative/transaction-support-in-pulsar-250' + }, + { + forum: 'Apache Pulsar Meetup | Beijing', + forum_link: 'https://www.huodongxing.com/event/1502359221000', + presenter: 'Yijie Shen', + date: 'August 2019', + title: "Integrating Apache Pulsar with Big Data Ecosystem", + link: 'https://www.slideshare.net/streamnative/8-integrate-apachepulsarwithbigdataecosystem' + }, + { + forum: 'Apache Pulsar Meetup | Beijing', + forum_link: 'https://www.huodongxing.com/event/1502359221000', + presenter: 'Penghui Li, Bo Cong', + date: 'August 2019', + title: "How Zhaopin contributes to Pulsar community", + link: 'https://www.slideshare.net/streamnative/3-zhaopin-inpulsarcommunity-165096725' + }, + { + forum: 'Apache Pulsar Meetup | Beijing', + forum_link: 'https://www.huodongxing.com/event/1502359221000', + presenter: 'Jia Zhai', + date: 'August 2019', + title: "Kafka on Pulsar(KOP)", + link: 'https://www.slideshare.net/streamnative/2-kafkaonpulsarjia' + }, + { + forum: 'Apache Pulsar Meetup | Beijing', + forum_link: 'https://www.huodongxing.com/event/1502359221000', + presenter: 'Nozomi Kurihara', + date: 'August 2019', + title: "Apache Pulsar at Yahoo!JAPAN", + link: 'https://www.slideshare.net/streamnative/1-apache-pulsaratyahoojapan-165095533' + }, + { + forum: '', + forum_link: '', + presenter: 'Sijie Guo', + date: 'June 2019', + title: "What's new in apache pulsar 2.4.0", + link: 'https://www.slideshare.net/streamnative/whats-new-in-apache-pulsar-240' + }, + { + forum: 'Apache Pulsar Meetup | Shenzhen', + forum_link: 'https://www.huodongxing.com/event/9495713659500', + presenter: 'Yijie Shen', + date: 'June 2019', + title: 'A Unified Platform for Real-time Storage and Processing - Apache Pulsar as Stream Storage, Apache Spark for Processing as an example', + link: 'https://www.slideshare.net/streamnative/a-unified-platform-for-realtime-storage-and-processing' + }, + { + forum: 'Ray Forward Beijing Meetup', + forum_link: 'https://tech.antfin.com/community/activities/698', + presenter: 'Sijie Guo', + date: 'June 2019', + title: 'Serverless Event Streaming with Pulsar Functions', + link: 'https://www.slideshare.net/streamnative/serverless-event-streaming-with-pulsar-functions' + }, + { + forum: 'Flink Forward San Francisco 2019', + forum_link: 'https://sf-2019.flink-forward.org/', + presenter: 'Sijie Guo', + date: 'April 2019', + title: 'Elastic Data Processing with Apache Flink and Apache Pulsar', + link: 'https://www.slideshare.net/streamnative/elastic-data-processing-with-apache-flink-and-apache-pulsar' + }, + { + forum: 'Strata Data San Francisco 2019', + forum_link: '', + presenter: 'Penghui Li, Sijie Guo', + date: 'March 2019', + title: 'How Zhaopin built its Event Center using Apache Pulsar', + link: 'https://www.slideshare.net/streamnative/how-zhaopin-built-its-event-center-using-apache-pulsar-152691364' + }, + { + forum: 'Strata San Jose', + forum_link: 'https://conferences.oreilly.com/strata/strata-ca', + presenter: 'Matteo Merli', + date: 'March 2018', + title: 'Effectively-once semantics in Apache Pulsar', + link: 'https://www.slideshare.net/merlimat/effectivelyonce-semantics-in-apache-pulsar' + }, + { + forum: '', + forum_link: '', + presenter: 'Matteo Merli', + date: 'November 2016', + title: 'Pulsar: a distributed pub-sub platform', + link: 'https://www.slideshare.net/merlimat/pulsar-distributed-pubsub-platform' + }, + { + forum: 'Bay Area Hadoop Meetup', + forum_link: 'https://www.meetup.com/hadoop', + presenter: 'Matteo Merli', + date: 'October 2016', + title: 'Pulsar: a highly scalable, low-latency pub-sub messaging system', + link: 'https://www.slideshare.net/ydn/october-2016-hug-pulsar-a-highly-scalable-low-latency-pubsub-messaging-system' + } + ] +} diff --git a/site2/website-next/data/team.js b/site2/website-next/data/team.js new file mode 100644 index 0000000000000..2253e7ccaa893 --- /dev/null +++ b/site2/website-next/data/team.js @@ -0,0 +1,264 @@ +module.exports = { + committers: [ + { + name: 'Ali Ahmed', + apacheId: 'aahmed', + roles: 'Committer' + }, + { + name: 'Addison Higham', + apacheId: 'addisonj', + org: 'StreamNative', + roles: 'Committer' + }, + { + name: 'Bo Cong', + apacheId: 'bogong', + org: 'StreamNative', + roles: 'Committer' + }, + { + name: 'Boyang Jerry Peng', + apacheId: 'jerrypeng', + org: 'Streamlio', + roles: 'Committer, PMC' + }, + { + name: 'Brad McMillen', + apacheId: 'bradtm', + org: 'Yahoo', + roles: 'Committer, PMC', + }, + { + name: 'Chris Kellogg', + apacheId: 'cckellogg', + org: 'Splunk', + roles: 'Committer' + }, + { + name: 'Daniel Blankensteiner', + apacheId: 'blankensteiner', + roles: 'Committer' + }, + { + name: 'David Fisher', + apacheId: 'wave', + roles: 'Committer, PMC' + }, + { + name: 'Enrico Olivelli', + apacheId: 'eolivelli', + roles: 'Committer, PMC' + }, + { + name: 'Fangbin Sun', + apacheId: 'sunfangbin', + roles: 'Committer' + }, + { + name: 'Francis Christopher Liu', + apacheId: 'toffer', + roles: 'Committer, PMC' + }, + { + name: 'Guangning E', + apacheId: 'guangning', + roles: 'Committer' + }, + { + name: 'Hang Chen', + apacheId: 'chenhang', + org: 'BIGO Technology', + roles: 'Committer, PMC' + }, + { + name: 'Hiroyuki Sakai', + apacheId: 'hrsakai', + org: 'Yahoo Japan Corporation', + roles: 'Committer, PMC' + }, + { + name: 'Huanli Meng', + apacheId: 'huanlimeng', + org: 'StreamNative', + roles: 'Committer' + }, + { + name: 'Ivan Brendan Kelly', + apacheId: 'ivank', + org: 'Streamlio', + roles: 'Committer, PMC' + }, + { + name: 'Jai Asher', + apacheId: 'jai1', + org: 'Yahoo', + roles: 'Committer, PMC' + }, + { + name: 'Jennifer Huang', + apacheId: 'hjf', + org: 'StreamNative', + roles: 'Committer, PMC' + }, + { + name: 'Jia Zhai', + apacheId: 'zhaijia', + org: 'StreamNative', + roles: 'Committer, PMC' + }, + { + name: 'Jim Jagielski', + apacheId: 'jim', + roles: 'Committer, PMC' + }, + { + name: 'Joe Francis', + apacheId: 'joef', + org: 'Yahoo', + roles: 'Committer, PMC' + }, + { + name: 'Lari Hotari', + apacheId: 'lhotari', + org: 'DataStax', + roles: 'Committer' + }, + { + name: 'Lin Lin', + apacheId: 'linlin', + org: 'Tencent', + roles: 'Committer, PMC' + }, + { + name: 'Ludwig Pummer', + apacheId: 'ludwigp', + org: 'Yahoo', + roles: 'Committer, PMC' + }, + { + name: 'Masahiro Sakamoto', + apacheId: 'massakam', + org: 'Yahoo Japan Corporation', + roles: 'Committer, PMC' + }, + { + name: 'Masakazu Kitajo', + apacheId: 'maskit', + org: '', + roles: 'Committer, PMC' + }, + { + name: 'Matteo Merli', + apacheId: 'mmerli', + org: 'Streamlio', + roles: 'Committer, PMC' + }, + { + name: 'Nozomi Kurihara', + apacheId: 'nkurihar', + org: 'Yahoo Japan Corporation', + roles: 'Committer, PMC' + }, + { + name: 'P. Taylor Goetz', + apacheId: 'ptgoetz', + roles: 'Committer, PMC' + }, + { + name: 'Penghui Li', + apacheId: 'penghui', + roles: 'Committer, PMC' + }, + { + name: 'Rajan Dhabalia', + apacheId: 'rdhabalia', + org: 'Yahoo', + roles: 'Committer, PMC' + }, + { + name: 'Ran Gao', + apacheId: 'rgao', + org: 'StreamNative', + roles: 'Committer' + }, + { + name: 'Rui Fu', + apacheId: 'rfu', + org: 'StreamNative', + roles: 'Committer' + }, + { + name: 'Sahaya Andrews', + apacheId: 'andrews', + org: 'Yahoo', + roles: 'Committer, PMC' + }, + { + name: 'Sanjeev Kulkarni', + apacheId: 'sanjeevrk', + org: 'Streamlio', + roles: 'Committer, PMC' + }, + { + name: 'Sebastián Schepens', + apacheId: 'sschepens', + org: 'MercadoLibre', + roles: 'Committer, PMC' + }, + { + name: 'Siddharth Boobna', + apacheId: 'sboobna', + org: 'Salesforce', + roles: 'Committer, PMC' + }, + { + name: 'Sijie Guo', + apacheId: 'sijie', + org: 'StreamNative', + roles: 'Committer, PMC' + }, + { + name: 'Xiaolong Ran', + apacheId: 'rxl', + org: 'StreamNative', + roles: 'Committer' + }, + { + name: 'Yijie Shen', + apacheId: 'yjshen', + org: 'KUAISHOU', + roles: 'Committer' + }, + { + name: 'Yu Liu', + apacheId: 'liuyu', + org: 'StreamNative', + roles: 'Committer' + }, + { + name: 'Yuki Shiga', + apacheId: 'yushiga', + org: 'Yahoo Japan Corporation', + roles: 'Committer, PMC' + }, + { + name: 'Yunze Xu', + apacheId: 'xyz', + org: 'StreamNative', + roles: 'Committer' + }, + { + name: 'Yuto Furuta', + apacheId: 'k2la', + org: 'Yahoo Japan Corporation', + roles: 'Committer' + }, + { + name: 'Yong Zhang', + apacheId: 'yong', + org: 'StreamNative', + roles: 'Committer' + } + ] +} diff --git a/site2/website-next/data/users.js b/site2/website-next/data/users.js new file mode 100644 index 0000000000000..da1f2c293e438 --- /dev/null +++ b/site2/website-next/data/users.js @@ -0,0 +1,339 @@ +module.exports = [ + { + name: 'Kingsoft Cloud', + url: 'https://www.ksyun.com/', + logo: 'https://fe.ksyun.com/header/images/pc/logo.png?a048d39f', + logo_white: true + }, + { + name: 'Narvar', + url: 'https://corp.narvar.com/', + logo: 'https://assets-global.website-files.com/5d278e4646ba9e55d6c67475/5d49ce69c03a64b2d8466816_Logo%20-%20Colored.svg', + }, + { + name: 'Giggso', + url: 'https://giggso.com/', + logo: 'https://www.giggso.com/wp-content/uploads/2021/06/logo-giggs.png', + }, + { + name: 'STICorp', + url: 'https://www.sticorp.com.br/', + logo: 'https://sticorp.com.br/wp-content/uploads/2021/02/logo-sticorp.png', + }, + { + name: 'Verizon Media', + url: 'https://www.verizonmedia.com/', + logo: 'https://upload.wikimedia.org/wikipedia/commons/thumb/0/09/Verizon_Media_2019_logo.svg/1200px-Verizon_Media_2019_logo.svg.png', + featured: 1 + }, + { + name: 'Yahoo! JAPAN', + url: 'https://www.yahoo.co.jp/', + logo: 'https://s.yimg.jp/images/top/sp2/cmn/logo-170307.png', + featured: 2 + }, + { + name: 'Zhaopin', + url: 'https://www.zhaopin.com/', + logo: 'https://fecdn5.zhaopin.cn/www/assets/zhaopin.4659478df1ec9a2e6ed76997a64884f6.png' + }, + { + name: 'Onde', + url: 'https://onde.app/', + logo: 'https://pbs.twimg.com/profile_images/1142024893229346818/JlCmThRZ_200x200.png', + }, + { + name: 'Overstock', + url: 'https://www.overstock.com/', + logo: 'https://ak1.ostkcdn.com/img/mxc/OSTK_MAIN_LOGO_20181127KCH.jpg' + }, + { + name: 'Ndustrial.io', + url: 'https://www.ndustrial.io/', + logo: 'https://images.squarespace-cdn.com/content/v1/56df41f2f699bb94c7a2c6fc/1552657060081-WX86GJXHX7UK81RGE6YU/ke17ZwdGBToddI8pDm48kKCofG1A85AnHnyK5g0bGkJZw-zPPgdn4jUwVcJE1ZvWQUxwkmyExglNqGp0IvTJZUJFbgE-7XRK3dMEBRBhUpzRPjxD400fzchYtVcY_asIzZyz67rHxPkx39rBmDbrxrXjNbFEDrPRRKbjQDhrWWQ/ndustrial.png' + }, + { + name: 'Nutanix', + url: 'https://www.nutanix.com/', + logo: 'https://www.zuora.com/wp-content/uploads/2017/05/nutanix.png', + featured: 4 + }, + { + name: 'The Hut Group', + url: 'https://www.thg.com', + logo: 'https://1000logos.net/wp-content/uploads/2020/09/THG-Logo-1024x640.png' + }, + { + name: 'Turtlequeue', + url: 'https://turtlequeue.com', + logo: 'https://turtlequeue.com/logo_black.png' + }, + { + name: 'ProxyClick', + url: 'https://www.proxyclick.com/', + logo: 'https://www.proxyclick.com/hubfs/assets/images/logo-positive.svg' + }, + { + name: 'Toast', + url: 'https://toasttab.com', + logo: 'https://cdn2.hubspot.net/hubfs/412971/toast-restaurant-pos.png' + }, + { + name: 'Quantel AI', + url: 'http://quantel.ai/', + logo: 'https://static.wixstatic.com/media/7c19eb_41a090c255c94d19bc37cca9777c1d90~mv2.jpg/v1/fill/w_314,h_270,al_c,q_80,usm_0.66_1.00_0.01/Quantel-final%20white%20bg.webp' + }, + { + name: 'Splunk', + url: 'https://splunk.com/', + logo: 'https://www.splunk.com/content/dam/splunk-blogs/images/2017/02/splunk-logo.png', + featured: 5 + }, + { + name: 'StreamNative', + url: 'https://streamnative.io/', + logo: 'https://streamnative.io/assets/static/sn-logo-vertical-dark.c516648.a34f97839dee38e7499aa0c21d0b2255.png', + }, + { + name: 'EMQ', + url: 'https://www.emqx.io/', + logo: 'https://www.emqx.io/images/logo.png', + }, + { + name: 'Tencent', + url: 'https://www.tencent.com', + logo: 'http://pluspng.com/img-png/tencent-png-tencent-logo-logotype-emblem-2-7537.png', + featured: 3 + }, + { + name: 'Comcast', + url: 'https://corporate.comcast.com/', + logo: 'https://corporate.comcast.com/resources/corporate/assets/images/svg/logo-desktop.svg' + }, + { + name: 'China Mobile', + url: 'https://www.chinamobileltd.com/', + logo: 'https://www.chinamobileltd.com/en/images/global/logo.svg' + }, + { + name: 'VipKid', + url: 'https://t.vipkid.com.cn/', + logo: 'https://s.vipkidstatic.com/fe-static/teacher/mkt-pc/img/e74f189.svg' + }, + { + name: 'Vivy', + url: 'https://www.vivy.com/de/startseite/', + logo: 'https://www.vivy.com/fileadmin/vivy-startpage/vivy_logo.svg' + }, + { + name: 'IoTium', + url: 'https://www.iotium.io/', + logo: 'https://www.iotium.io/site/wp-content/uploads/2018/04/IoTium-logo.jpg' + }, + { + name: 'One Click Retail', + url: 'https://oneclickretail.com/', + logo: 'https://www.ascentialedge.com/themes/contrib/custom/images/edge-by-ascential.svg', + logo_white: true + }, + { + name: 'Tuya', + url: 'https://en.tuya.com/', + logo: 'https://static1.tuyaus.com/static/portal_intl/img/e25eefa.png' + }, + { + name: 'China Telecom', + url: 'https://www.chinatelecom-h.com/', + logo: 'https://www.chinatelecom-h.com/en/images/global/logo.png' + }, + { + name: 'Kafkaesque', + url: 'https://kafkaesque.io/', + logo: 'https://kafkaesque.io/wp-content/uploads/2019/04/color_logo_with_background.svg' + }, + { + name: 'OKCoin', + url: 'https://www.okcoin.com/', + logo: 'https://i0.wp.com/blog.okcoin.com/wp-content/uploads/2019/08/OKCoin-Logo-Spring2019-250x60px-01.png?fit=251%2C62&ssl=1', + }, + { + name: 'BestPay', + url: 'https://www.bestpay.com.cn/', + logo: 'https://www.bestpay.com.cn/global/oig/images/logo.png' + }, + { + name: '360', + url: 'https://www.360.cn/', + logo: 'http://p2.ssl.qhimg.com/t01d91636862957f76e.png' + }, + { + name: 'Lakala', + url: 'http://www.lakala.com/', + logo: 'http://www.lakala.com/templets/default/image/temp/logo.png' + }, + { + name: 'Huya', + url: 'https://www.huya.com/', + logo: 'https://a.msstatic.com/huya/main/img/logo2.png', + logo_white: true + }, + { + name: 'CTWing', + url: 'https://www.ctwing.cn/', + logo: 'https://www.ctwing.cn/r/cms/www/default/images/public/logo.png' + }, + { + name: 'Globalegrow E-Commerce', + url: 'https://www.globalegrow.com/', + logo: 'https://static.globalegrow.com/dist/images/domeimg/ch/logo.jpg' + }, + { + name: 'YunZhiHuLian', + url: 'https://www.weihudashi.com', + logo: 'https://www.weihudashi.com/image/header-logo.png' + }, + { + name: 'XiaoDuoAI', + url: 'https://www.xiaoduoai.com/', + logo: 'https://cdn.xiaoduoai.com/official-site/dist/assets/img/indexPage/xiaoduo_logo_en_white.843927.png', + logo_white: true + }, + { + name: 'Shihong', + url: 'https://shihong.net.cn', + logo: 'https://www.shihong.net.cn/website/static/src/img/shihong1.png' + }, + { + name: 'CleverCloud', + url: 'https://www.clever-cloud.com/en/', + logo: 'https://www.clever-cloud.com/images/brand-assets/logos/v2/logo_on_white.svg' + }, + { + name: 'OVHcloud', + url: 'https://www.ovhcloud.com/', + logo: 'https://www.ovh.com/world/images/logo/ovhLogoColor.svg' + }, + { + name: 'BrandsEye', + url: 'https://www.brandseye.com/', + logo: 'https://www.brandseye.com/theme-images/brandseye-logo-v2.png' + }, + { + name: 'Max Kelsen', + url: 'https://maxkelsen.com/', + logo: 'https://maxkelsen.com/imgs/logo/Max-Kelsen-logo-black.png' + }, + { + name: 'HSL Developer Community of Helsinki Regional Transport Authority', + url: 'https://dev.hsl.fi/', + logo: 'https://pbs.twimg.com/profile_images/2507371771/9toyr6r5ngcozfwg42zv_400x400.png' + }, + // { + // name: 'Ottoo', + // url: 'https://ottoo.com', + // logo: 'https://ottoo.com/assets/images/ottoo-logo.svg' + // }, + { + name: 'MercadoLibre', + url: 'https://www.mercadolibre.com/', + logo: 'https://d1.awsstatic.com/case-studies/LATAM/mercadolibre.1d9d3766bb06e60948f1c5ce8851ce0eac63b349.png' + }, + { + name: 'BIGO', + url: 'https://www.bigo.sg/', + logo: 'https://static-web.bigolive.tv/as/bigo-static/official_website/pc/img/about/logo-blue-big.png', + }, + { + name: 'Keytop', + url: 'https://www.ikeytop.com/', + logo: 'https://static.wixstatic.com/media/ecbdea_434ae22f85ed4e93ba3fdc39c7141c3f~mv2.png/v1/fill/w_424,h_116,al_c,usm_0.66_1.00_0.01/Keytop_logo.png', + }, + { + name: 'Zhihu', + url: 'https://www.zhihu.com/', + logo: 'https://pic4.zhimg.com/80/v2-a47051e92cf74930bedd7469978e6c08_hd.png', + }, + { + name: 'Newland', + url: 'http://www.nlsoft.com.cn/', + logo: 'http://www.nlsoft.com.cn/images/common/logo.png', + }, + { + name: 'PalFish', + url: 'https://www.ipalfish.com/', + logo: 'https://s04.cdn.ipalfish.com/kid/img/logo.ad4731cb.png', + }, + { + name: 'Dada Group', + url: 'https://about.imdada.cn', + logo: 'https://fe.imdada.cn/alaskan/static/logo_color-aaed10c6738b703ad2b54c17f4c3f114.png', + }, + { + name: 'Huawei Cloud', + url: 'https://www.huaweicloud.com', + logo: 'https://res.hc-cdn.com/cnpm-common-resource/2.0.2/base/header/components/images/logo.png', + }, + { + name: 'Su Ning', + url: 'https://www.suning.com', + logo: 'https://image1.suning.cn/uimg/cms/img/159642507148437980.png', + }, + { + name: 'Pingan Securites', + url: 'https://stock.pingan.com', + logo: 'https://stock.pingan.com/static/uploads/softdowm/2018-10-9/upload_c9ac4929e293f5109623b541740618ba.png', + }, + { + name: 'Ge Tui', + url: 'https://www.getui.com', + logo: 'https://gt-static.getui.com/getui_pc/client/img/e33ecd9.svg', + logo_white: true + }, + { + name: 'DataStax', + url: 'https://datastax.com', + logo: 'https://www.datastax.com/sites/default/files/inline-images/datastax-logotype-negative.png', + logo_white: false + }, + { + name: 'KAISA', + url: 'https://www.kaisafin.com', + logo: 'https://www.kaisafin.com/images/logo.svg', + }, + { + name: 'Bairong.Inc', + url: 'http://www.brgroup.com', + logo: 'http://www.brgroup.com/_nuxt/img/d2a976e.png', + logo_white: true + }, + { + name: 'Itcast.cn', + url: 'http://www.itcast.cn', + logo: 'http://www.itcast.cn/2018czgw/images/logo2.png', + }, + { + name: 'Appen', + url: 'https://www.appen.com.cn/', + logo: 'https://resources.appen.com/wp-content/uploads/2020/03/no-tagline.svg', + }, + { + name: 'Pandio', + url: 'https://pandio.com', + logo: 'https://pandio-public-assets.s3-us-west-2.amazonaws.com/pandio_225-05.png', + }, + { + name: 'Deposit-Solutions', + url: 'https://www.deposit-solutions.com/', + logo: 'https://media-exp1.licdn.com/dms/image/C4E0BAQGWAAW9iYFSwA/company-logo_200_200/0/1625219600895?e=1638403200&v=beta&t=iGNhEWlAFQDJOHAScjumctjY5M2eSU4fL4X-9kgpR2M' + }, + { + name: 'Macrometa', + url: 'https://macrometa.com', + logo: 'https://uploads-ssl.webflow.com/5fa9e94bc848ae335afdd627/602ae50ae801b44f15185683_logo-moc-blue.png', + }, + { + name: 'Softtech', + url: 'https://softtech.com.tr/en/homepage/', + logo: 'https://softtech.com.tr/wp-content/uploads/2017/12/SOFTTECH-LOGO.png', + } +].sort((a, b) => (a.name > b.name) ? 1 : -1); diff --git a/site2/website-next/docs/client-libraries-node.md b/site2/website-next/docs/client-libraries-node.md index 794cb72c5e37d..d16a4840df7c0 100644 --- a/site2/website-next/docs/client-libraries-node.md +++ b/site2/website-next/docs/client-libraries-node.md @@ -156,7 +156,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar automatically generates a globally unique name. If you choose to explicitly assign a name, it needs to be unique across *all* Pulsar clusters, otherwise the creation operation throws an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer waits for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error is thrown. If you set `sendTimeoutMs` to -1, the timeout is set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website-next/docusaurus.config.js b/site2/website-next/docusaurus.config.js index f0b98631281ff..36b40b73cf0b9 100644 --- a/site2/website-next/docusaurus.config.js +++ b/site2/website-next/docusaurus.config.js @@ -98,6 +98,9 @@ module.exports = { favicon: "img/favicon.ico", organizationName: "apache", projectName: "pulsar", + customFields: { + githubUrl, + }, themeConfig: { navbar: { title: "", @@ -145,26 +148,7 @@ module.exports = { }, footer: { style: "dark", - links: [ - { - title: "More", - items: [ - { - label: "Docs", - to: "/docs", - }, - { - label: "Blog", - to: "/blog", - }, - { - label: "GitHub", - href: "https://github.com/apache/pulsar", - }, - ], - }, - ], - copyright: `Copyright © ${new Date().getFullYear()} Apache Pulsar, Inc.`, + copyright: `Inc.Copyright © ${new Date().getFullYear()} The Apache Software Foundation. All Rights Reserved. Apache, Apache Pulsar and the Apache feather logo are trademarks of The Apache Software Foundation.`, }, prism: { // theme: lightCodeTheme, @@ -210,5 +194,6 @@ module.exports = { fromExtensions: ["md"], }, ], + "./postcss-tailwind-loader", ], }; diff --git a/site2/website-next/package.json b/site2/website-next/package.json index 833240c621999..dfe0459bda87e 100644 --- a/site2/website-next/package.json +++ b/site2/website-next/package.json @@ -17,13 +17,20 @@ "@docusaurus/core": "2.0.0-beta.6", "@docusaurus/plugin-client-redirects": "^2.0.0-beta.6", "@docusaurus/preset-classic": "2.0.0-beta.6", + "@emotion/react": "^11.5.0", + "@emotion/styled": "^11.3.0", "@mdx-js/react": "^1.6.22", + "@mui/material": "^5.0.4", "@svgr/webpack": "^5.5.0", "clsx": "^1.1.1", "file-loader": "^6.2.0", + "install": "^0.13.0", + "npm": "^8.1.0", "prism-react-renderer": "^1.2.1", "react": "^17.0.2", "react-dom": "^17.0.2", + "react-markdown": "^7.0.1", + "react-svg": "^14.0.15", "remark-linkify-regex": "^1.0.0", "replace-in-file": "^6.2.0", "url-loader": "^4.1.1" @@ -39,5 +46,14 @@ "last 1 firefox version", "last 1 safari version" ] + }, + "devDependencies": { + "autoprefixer": "^10.3.7", + "postcss": "^8.3.9", + "postcss-cli": "^9.0.1", + "postcss-import": "^14.0.2", + "postcss-nested": "^5.0.6", + "postcss-preset-env": "^6.7.0", + "tailwindcss": "^2.2.17" } } diff --git a/site2/website-next/postcss-tailwind-loader.js b/site2/website-next/postcss-tailwind-loader.js new file mode 100644 index 0000000000000..9cce0d4c525b1 --- /dev/null +++ b/site2/website-next/postcss-tailwind-loader.js @@ -0,0 +1,14 @@ +module.exports = function (context, options) { + return { + name: "postcss-tailwindcss-loader", + configurePostCss(postcssOptions) { + postcssOptions.plugins.push( + require("postcss-import"), + require("tailwindcss"), + require("postcss-nested"), + require("autoprefixer") + ); + return postcssOptions; + }, + }; +}; diff --git a/site2/website-next/src/components/HomepageFeatures.js b/site2/website-next/src/components/HomepageFeatures.js index 68ac5253d5f8f..c2497cce1ff90 100644 --- a/site2/website-next/src/components/HomepageFeatures.js +++ b/site2/website-next/src/components/HomepageFeatures.js @@ -1,12 +1,15 @@ import React from "react"; import clsx from "clsx"; import styles from "./HomepageFeatures.module.css"; +import ReactMarkdown from "react-markdown"; +import { docUrl } from "../utils/index"; -const FeatureList = [ +const FeatureList = (language) => [ { - title: "Pulsar Functions", + // title: "Pulsar Functions", + title: `[Pulsar Functions](${docUrl("functions-overview", language)})`, // Svg: require("../../static/img/undraw_docusaurus_mountain.svg").default, - description: ( + content: ( <> Easy to deploy, lightweight compute process, developer-friendly APIs, no need to run your own stream processing engine. @@ -14,9 +17,12 @@ const FeatureList = [ ), }, { - title: "Proven in production", + title: `[Proven in production](${docUrl( + "concepts-architecture-overview", + language + )})`, // Svg: require("../../static/img/undraw_docusaurus_tree.svg").default, - description: ( + content: ( <> Run in production at Yahoo! scale for over 5 years, with millions of messages per second across millions of topics. @@ -24,32 +30,71 @@ const FeatureList = [ ), }, { - title: "Horizontally scalable", + title: `[Horizontally scalable](${docUrl( + "concepts-architecture-overview", + language + )})`, // Svg: require("../../static/img/undraw_docusaurus_react.svg").default, - description: <>Expand capacity seamlessly to hundreds of nodes., + content: <>Expand capacity seamlessly to hundreds of nodes., + }, + { + content: + "Low publish latency (< 5ms) at scale with strong durability guarantees.", + title: `[Low latency with durability](${docUrl( + "concepts-architecture-overview", + language + )})`, + }, + { + content: + "Configurable replication between data centers across multiple geographic regions.", + title: `[Geo-replication](${docUrl("administration-geo", language)})`, + }, + { + content: + "Built from the ground up as a multi-tenant system. Supports isolation, authentication, authorization and quotas.", + title: `[Multi-tenancy](${docUrl("concepts-multi-tenancy", language)})`, + }, + { + content: + "Persistent message storage based on Apache BookKeeper. IO-level isolation between write and read operations.", + title: `[Persistent storage](${docUrl( + "concepts-architecture-overview#persistent-storage", + language + )})`, + }, + { + content: + "Flexible messaging models with high-level APIs for Java, Go, Python, C++, Node.js, WebSocket and C#.", + title: `[Client libraries](${docUrl("client-libraries", language)})`, + }, + { + content: + "REST Admin API for provisioning, administration, tools and monitoring. Can be deployed on bare metal, Kubernetes, Amazon Web Services(AWS), and DataCenter Operating System(DC/OS).", + title: `[Operability](${docUrl("admin-api-overview", language)})`, }, ]; -function Feature({ Svg, title, description }) { +function Feature({ Svg, title, content }) { return ( -
    +
    {/*
    */}
    -

    {title}

    -

    {description}

    + +

    {content}

    ); } -export default function HomepageFeatures() { +export default function HomepageFeatures(props) { return (
    - {FeatureList.map((props, idx) => ( + {FeatureList(props.language || "").map((props, idx) => ( ))}
    diff --git a/site2/website-next/src/components/Svg.js b/site2/website-next/src/components/Svg.js new file mode 100644 index 0000000000000..3d3b1a23f8a9c --- /dev/null +++ b/site2/website-next/src/components/Svg.js @@ -0,0 +1,24 @@ +import React from "react"; +import { ReactSVG } from "react-svg"; +import useBaseUrl from "@docusaurus/useBaseUrl"; + +export default function Svg(props) { + return ( + { + svg.setAttribute( + "style", + "" + + (!isNaN(parseFloat(props.width)) + ? "width: " + props.width + ";" + : "") + + (!isNaN(parseFloat(props.height)) + ? "height: " + props.height + ";" + : "") + ); + svg.setAttribute("class", props.className); + }} + /> + ); +} diff --git a/site2/website-next/src/css/custom.css b/site2/website-next/src/css/custom.css index ff2d075a961a3..a15083b9dabcb 100644 --- a/site2/website-next/src/css/custom.css +++ b/site2/website-next/src/css/custom.css @@ -5,7 +5,22 @@ * work well for content-centric websites. */ -/* You can override the default Infima variables here. */ +.tailwind { + @tailwind base; + @tailwind components; + @tailwind utilities; + @tailwind screens; +} + +/* .section { + @apply max-w-screen-xl xl:max-w-screen-xl px-8 container; +} */ + +svg { + width: inherit; + height: inherit; +} + :root { --ifm-color-primary: #198fff; --ifm-color-primary-dark: rgb(33, 175, 144); @@ -15,6 +30,8 @@ --ifm-color-primary-lighter: rgb(102, 212, 189); --ifm-color-primary-lightest: rgb(146, 224, 208); --ifm-code-font-size: 95%; + --ifm-container-width: 1280px; + --ifm-container-width-xl: 1536px; } .docusaurus-highlight-code-line { @@ -24,6 +41,6 @@ padding: 0 var(--ifm-pre-padding); } -html[data-theme='dark'] .docusaurus-highlight-code-line { +html[data-theme="dark"] .docusaurus-highlight-code-line { background-color: rgba(0, 0, 0, 0.3); } diff --git a/site2/website-next/src/pages/index.js b/site2/website-next/src/pages/index.js index 61d476ddc0dcd..a412a18ce118d 100644 --- a/site2/website-next/src/pages/index.js +++ b/site2/website-next/src/pages/index.js @@ -1,40 +1,73 @@ -import React from 'react'; -import clsx from 'clsx'; -import Layout from '@theme/Layout'; -import Link from '@docusaurus/Link'; -import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; -import styles from './index.module.css'; -import HomepageFeatures from '../components/HomepageFeatures'; +import React from "react"; +import clsx from "clsx"; +import Layout from "@theme/Layout"; +import useDocusaurusContext from "@docusaurus/useDocusaurusContext"; +import styles from "./index.module.css"; +import HomepageFeatures from "../components/HomepageFeatures"; +import Svg from "../components/Svg"; +import { docUrl, githubUrl } from "../utils/index"; +import Stack from "@mui/material/Stack"; +import Button from "@mui/material/Button"; -function HomepageHeader() { - const {siteConfig} = useDocusaurusContext(); +function HomepageHeader(props) { + const { siteConfig } = useDocusaurusContext(); return ( -
    -
    -

    {siteConfig.title}

    -

    {siteConfig.tagline}

    -
    - - Read️ the Docs - -
    +
    +
    + +

    {siteConfig.tagline}

    + + + +
    ); } export default function Home() { - const {siteConfig} = useDocusaurusContext(); + const { siteConfig } = useDocusaurusContext(); return ( + description="Description will go into a meta tag in " + > -
    - +
    +
    + +
    +

    + + + Companies Powered by Pulsar + + +

    +
    +

    + Apache Pulsar is available under the{" "} + + Apache License, version 2.0 + + . +

    +
    ); } diff --git a/site2/website-next/src/theme/Root.js b/site2/website-next/src/theme/Root.js new file mode 100644 index 0000000000000..2d27da6638084 --- /dev/null +++ b/site2/website-next/src/theme/Root.js @@ -0,0 +1,14 @@ +import React from "react"; +import { ThemeProvider } from "@mui/material/styles"; +import theme from "./material-theme"; + +// Default implementation, that you can customize +function Root({ children }) { + return ( + <> + {children} + + ); +} + +export default Root; diff --git a/site2/website-next/src/theme/material-theme.js b/site2/website-next/src/theme/material-theme.js new file mode 100644 index 0000000000000..20c9ce2434730 --- /dev/null +++ b/site2/website-next/src/theme/material-theme.js @@ -0,0 +1,11 @@ +import { createTheme } from "@mui/material/styles"; + +export default createTheme({ + palette: { + primary: { + light: "#198fff", + main: "#198fff", + dark: "#198fff", + }, + }, +}); diff --git a/site2/website-next/src/utils/index.js b/site2/website-next/src/utils/index.js new file mode 100644 index 0000000000000..adbdf80c51140 --- /dev/null +++ b/site2/website-next/src/utils/index.js @@ -0,0 +1,21 @@ +const users = require(`../../data/users.js`); +const featuredUsers = users.filter((x) => x.hasOwnProperty("featured")); +featuredUsers.sort((a, b) => (a.featured > b.featured ? 1 : -1)); + +const siteConfig = require(`../../docusaurus.config.js`); + +export function imgUrl(img) { + return siteConfig.baseUrl + "img/" + img; +} + +export function docUrl(doc, language) { + return siteConfig.baseUrl + "docs/" + (language ? language + "/" : "") + doc; +} + +export function pageUrl(page, language) { + return siteConfig.baseUrl + (language ? language + "/" : "") + page; +} + +export function githubUrl() { + return siteConfig.customFields.githubUrl; +} diff --git a/site2/website-next/static/img/pulsar.svg b/site2/website-next/static/img/pulsar.svg new file mode 100644 index 0000000000000..bd063c735f882 --- /dev/null +++ b/site2/website-next/static/img/pulsar.svg @@ -0,0 +1 @@ +Asset 2 \ No newline at end of file diff --git a/site2/website-next/tailwind.config.js b/site2/website-next/tailwind.config.js new file mode 100644 index 0000000000000..60fdd35bc20f7 --- /dev/null +++ b/site2/website-next/tailwind.config.js @@ -0,0 +1,46 @@ +const plugin = require("tailwindcss/plugin"); +const colors = require("tailwindcss/colors"); + +module.exports = { + purge: ["./src/**/*.html", "./src/**/*.js", "./src/**/*.tsx"], + darkMode: false, + theme: { + colors: { + ...colors, + primary: "#198fff", + }, + }, + variants: { + extend: {}, + }, + plugins: [ + plugin(function ({ addBase, config }) { + addBase({ + h1: { + fontSize: config("theme.fontSize.6xl"), + fontWeight: config("theme.fontWeight.bold"), + }, + h2: { + fontSize: config("theme.fontSize.5xl"), + fontWeight: config("theme.fontWeight.bold"), + }, + h3: { + fontSize: config("theme.fontSize.4xl"), + fontWeight: config("theme.fontWeight.bold"), + }, + h4: { + fontSize: config("theme.fontSize.3xl"), + fontWeight: config("theme.fontWeight.bold"), + }, + h5: { + fontSize: config("theme.fontSize.2xl"), + fontWeight: config("theme.fontWeight.bold"), + }, + h6: { + fontSize: config("theme.fontSize.xl"), + fontWeight: config("theme.fontWeight.bold"), + }, + }); + }), + ], +}; diff --git a/site2/website-next/versioned_docs/version-2.7.2/schema-evolution-compatibility.md b/site2/website-next/versioned_docs/version-2.7.2/schema-evolution-compatibility.md index 60a436cb4b720..a2bb99922254f 100644 --- a/site2/website-next/versioned_docs/version-2.7.2/schema-evolution-compatibility.md +++ b/site2/website-next/versioned_docs/version-2.7.2/schema-evolution-compatibility.md @@ -1,7 +1,7 @@ --- id: schema-evolution-compatibility title: Schema evolution and compatibility -sidebar_label: Schema evolution and compatibility +sidebar_label: "Schema evolution and compatibility" original_id: schema-evolution-compatibility --- @@ -59,19 +59,19 @@ Suppose that you have a topic containing three schemas (V1, V2, and V3), V1 is t | --- | --- | --- | --- | --- | | `ALWAYS_COMPATIBLE` | Disable schema compatibility check. | All changes are allowed | All previous versions | Any order | | `ALWAYS_INCOMPATIBLE` | Disable schema evolution. | All changes are disabled | None | None | -| `BACKWARD` | Consumers using the schema V3 can process data written by producers using the schema V3 or V2. | * Add optional fields * Delete fields | Latest version | Consumers | -| `BACKWARD_TRANSITIVE` | Consumers using the schema V3 can process data written by producers using the schema V3, V2 or V1. | * Add optional fields * Delete fields | All previous versions | Consumers | -| `FORWARD` | Consumers using the schema V3 or V2 can process data written by producers using the schema V3. | * Add fields * Delete optional fields | Latest version | Producers | -| `FORWARD_TRANSITIVE` | Consumers using the schema V3, V2 or V1 can process data written by producers using the schema V3. | * Add fields * Delete optional fields | All previous versions | Producers | -| `FULL` | Backward and forward compatible between the schema V3 and V2. | * Modify optional fields | Latest version | Any order | -| `FULL_TRANSITIVE` | Backward and forward compatible among the schema V3, V2, and V1. | * Modify optional fields | All previous versions | Any order | +| `BACKWARD` | Consumers using the schema V3 can process data written by producers using the schema V3 or V2. |
  • Add optional fields
  • Delete fields
    • | Latest version | Consumers | +| `BACKWARD_TRANSITIVE` | Consumers using the schema V3 can process data written by producers using the schema V3, V2 or V1. |
  • Add optional fields
  • Delete fields
    • | All previous versions | Consumers | +| `FORWARD` | Consumers using the schema V3 or V2 can process data written by producers using the schema V3. |
  • Add fields
  • Delete optional fields
    • | Latest version | Producers | +| `FORWARD_TRANSITIVE` | Consumers using the schema V3, V2 or V1 can process data written by producers using the schema V3. |
  • Add fields
  • Delete optional fields
    • | All previous versions | Producers | +| `FULL` | Backward and forward compatible between the schema V3 and V2. |
  • Modify optional fields
    • | Latest version | Any order | +| `FULL_TRANSITIVE` | Backward and forward compatible among the schema V3, V2, and V1. |
  • Modify optional fields
    • | All previous versions | Any order | ### ALWAYS_COMPATIBLE and ALWAYS_INCOMPATIBLE | Compatibility check strategy | Definition | Note | | --- | --- | --- | | `ALWAYS_COMPATIBLE` | Disable schema compatibility check. | None | -| `ALWAYS_INCOMPATIBLE` | Disable schema evolution, that is, any schema change is rejected. | * For all schema types except Avro and JSON, the default schema compatibility check strategy is `ALWAYS_INCOMPATIBLE`. * For Avro and JSON, the default schema compatibility check strategy is `FULL`. | +| `ALWAYS_INCOMPATIBLE` | Disable schema evolution, that is, any schema change is rejected. |
  • For all schema types except Avro and JSON, the default schema compatibility check strategy is `ALWAYS_INCOMPATIBLE`.
  • For Avro and JSON, the default schema compatibility check strategy is `FULL`.
    • | #### Example @@ -151,7 +151,7 @@ Suppose that you have a topic containing three schemas (V1, V2, and V3), V1 is t | Compatibility check strategy | Definition | Description | Note | | --- | --- | --- | --- | -| `FULL` | Schemas are both backward and forward compatible, which means: Consumers using the last schema can process data written by producers using the new schema. AND Consumers using the new schema can process data written by producers using the last schema. | Consumers using the schema V3 can process data written by producers using the schema V3 or V2. AND Consumers using the schema V3 or V2 can process data written by producers using the schema V3. | * For Avro and JSON, the default schema compatibility check strategy is `FULL`. * For all schema types except Avro and JSON, the default schema compatibility check strategy is `ALWAYS_INCOMPATIBLE`. | +| `FULL` | Schemas are both backward and forward compatible, which means: Consumers using the last schema can process data written by producers using the new schema. AND Consumers using the new schema can process data written by producers using the last schema. | Consumers using the schema V3 can process data written by producers using the schema V3 or V2. AND Consumers using the schema V3 or V2 can process data written by producers using the schema V3. |
  • For Avro and JSON, the default schema compatibility check strategy is `FULL`.
  • For all schema types except Avro and JSON, the default schema compatibility check strategy is `ALWAYS_INCOMPATIBLE`.
    • | | `FULL_TRANSITIVE` | The new schema is backward and forward compatible with all previously registered schemas. | Consumers using the schema V3 can process data written by producers using the schema V3, V2 or V1. AND Consumers using the schema V3, V2 or V1 can process data written by producers using the schema V3. | None | #### Example @@ -196,9 +196,9 @@ For example, the producers using schemas to write data to Pulsar and the consume | --- | --- | --- | | `ALWAYS_COMPATIBLE` | Any order | The compatibility check is disabled. Consequently, you can upgrade the producers and consumers in **any order**. | | `ALWAYS_INCOMPATIBLE` | None | The schema evolution is disabled. | -| * `BACKWARD` * `BACKWARD_TRANSITIVE` | Consumers | There is no guarantee that consumers using the old schema can read data produced using the new schema. Consequently, **upgrade all consumers first**, and then start producing new data. | -| * `FORWARD` * `FORWARD_TRANSITIVE` | Producers | There is no guarantee that consumers using the new schema can read data produced using the old schema. Consequently, **upgrade all producers first** to use the new schema and ensure that the data already produced using the old schemas are not available to consumers, and then upgrade the consumers. | -| * `FULL` * `FULL_TRANSITIVE` | Any order | There is no guarantee that consumers using the old schema can read data produced using the new schema and consumers using the new schema can read data produced using the old schema. Consequently, you can upgrade the producers and consumers in **any order**. | +|
  • `BACKWARD`
  • `BACKWARD_TRANSITIVE`
    • | Consumers | There is no guarantee that consumers using the old schema can read data produced using the new schema. Consequently, **upgrade all consumers first**, and then start producing new data. | +|
  • `FORWARD`
  • `FORWARD_TRANSITIVE`
    • | Producers | There is no guarantee that consumers using the new schema can read data produced using the old schema. Consequently, **upgrade all producers first**
  • to use the new schema and ensure that the data already produced using the old schemas are not available to consumers, and then upgrade the consumers.
    • | +|
  • `FULL`
  • `FULL_TRANSITIVE`
    • | Any order | There is no guarantee that consumers using the old schema can read data produced using the new schema and consumers using the new schema can read data produced using the old schema. Consequently, you can upgrade the producers and consumers in **any order**. | diff --git a/site2/website-next/versioned_docs/version-2.7.2/schema-get-started.md b/site2/website-next/versioned_docs/version-2.7.2/schema-get-started.md index 496fe9077e491..c8193ab572c69 100644 --- a/site2/website-next/versioned_docs/version-2.7.2/schema-get-started.md +++ b/site2/website-next/versioned_docs/version-2.7.2/schema-get-started.md @@ -1,7 +1,7 @@ --- id: schema-get-started title: Get started -sidebar_label: Get started +sidebar_label: "Get started" original_id: schema-get-started --- diff --git a/site2/website-next/versioned_docs/version-2.7.2/schema-manage.md b/site2/website-next/versioned_docs/version-2.7.2/schema-manage.md index aa42485736939..e95369f0da08a 100644 --- a/site2/website-next/versioned_docs/version-2.7.2/schema-manage.md +++ b/site2/website-next/versioned_docs/version-2.7.2/schema-manage.md @@ -1,7 +1,7 @@ --- id: schema-manage title: Manage schema -sidebar_label: Manage schema +sidebar_label: "Manage schema" original_id: schema-manage --- @@ -158,9 +158,9 @@ To manage schemas, you can use one of the following methods. | Method | Description | | --- | --- | -| **Admin CLI** | You can use the `pulsar-admin` tool to manage Pulsar schemas, brokers, clusters, sources, sinks, topics, tenants and so on. For more information about how to use the `pulsar-admin` tool, see [here](reference-pulsar-admin). | -| **REST API** | Pulsar exposes schema related management API in Pulsar’s admin RESTful API. You can access the admin RESTful endpoint directly to manage schemas. For more information about how to use the Pulsar REST API, see [here](http://pulsar.apache.org/admin-rest-api/). | -| **Java Admin API** | Pulsar provides Java admin library. | +| **Admin CLI**
  • | You can use the `pulsar-admin` tool to manage Pulsar schemas, brokers, clusters, sources, sinks, topics, tenants and so on. For more information about how to use the `pulsar-admin` tool, see [here](reference-pulsar-admin). | +| **REST API**
  • | Pulsar exposes schema related management API in Pulsar’s admin RESTful API. You can access the admin RESTful endpoint directly to manage schemas. For more information about how to use the Pulsar REST API, see [here](http://pulsar.apache.org/admin-rest-api/). | +| **Java Admin API**
  • | Pulsar provides Java admin library. | ### Upload a schema @@ -210,7 +210,7 @@ The `schema-definition-file` includes the following fields: | Field | Description | | --- | --- | | `type` | The schema type. | -| `schema` | The schema definition data, which is encoded in UTF 8 charset. * If the schema is a **primitive** schema, this field should be blank. * If the schema is a **struct** schema, this field should be a JSON string of the Avro schema definition. | +| `schema` | The schema definition data, which is encoded in UTF 8 charset.
  • If the schema is a
    • **primitive**
  • schema, this field should be blank.
  • If the schema is a
    • **struct**
  • schema, this field should be a JSON string of the Avro schema definition.
    • | | `properties` | The additional properties associated with the schema. | Here are examples of the `schema-definition-file` for a JSON schema. @@ -263,7 +263,7 @@ The post payload includes the following fields: | Field | Description | | --- | --- | | `type` | The schema type. | -| `schema` | The schema definition data, which is encoded in UTF 8 charset. * If the schema is a **primitive** schema, this field should be blank. * If the schema is a **struct** schema, this field should be a JSON string of the Avro schema definition. | +| `schema` | The schema definition data, which is encoded in UTF 8 charset.
  • If the schema is a
    • **primitive**
  • schema, this field should be blank.
  • If the schema is a
    • **struct**
  • schema, this field should be a JSON string of the Avro schema definition.
    • | | `properties` | The additional properties associated with the schema. | @@ -280,7 +280,7 @@ The `PostSchemaPayload` includes the following fields: | Field | Description | | --- | --- | | `type` | The schema type. | -| `schema` | The schema definition data, which is encoded in UTF 8 charset. * If the schema is a **primitive** schema, this field should be blank. * If the schema is a **struct** schema, this field should be a JSON string of the Avro schema definition. | +| `schema` | The schema definition data, which is encoded in UTF 8 charset.
  • If the schema is a
    • **primitive**
  • schema, this field should be blank.
  • If the schema is a
    • **struct**
  • schema, this field should be a JSON string of the Avro schema definition.
    • | | `properties` | The additional properties associated with the schema. | Here is an example of `PostSchemaPayload`: @@ -368,7 +368,7 @@ The response includes the following fields: | `version` | The schema version, which is a long number. | | `type` | The schema type. | | `timestamp` | The timestamp of creating this version of schema. | -| `data` | The schema definition data, which is encoded in UTF 8 charset. * If the schema is a **primitive** schema, this field should be blank. * If the schema is a **struct** schema, this field should be a JSON string of the Avro schema definition. | +| `data` | The schema definition data, which is encoded in UTF 8 charset.
  • If the schema is a
    • **primitive**
  • schema, this field should be blank.
  • If the schema is a
    • **struct**
  • schema, this field should be a JSON string of the Avro schema definition.
    • | | `properties` | The additional properties associated with the schema. | @@ -386,7 +386,7 @@ The `SchemaInfo` includes the following fields: | --- | --- | | `name` | The schema name. | | `type` | The schema type. | -| `schema` | A byte array of the schema definition data, which is encoded in UTF 8 charset. * If the schema is a **primitive** schema, this byte array should be empty. * If the schema is a **struct** schema, this field should be a JSON string of the Avro schema definition converted to a byte array. | +| `schema` | A byte array of the schema definition data, which is encoded in UTF 8 charset.
  • If the schema is a
    • **primitive**
  • schema, this byte array should be empty.
  • If the schema is a
    • **struct**
  • schema, this field should be a JSON string of the Avro schema definition converted to a byte array.
    • | | `properties` | The additional properties associated with the schema. | Here is an example of `SchemaInfo`: @@ -460,7 +460,7 @@ The response includes the following fields: | `version` | The schema version, which is a long number. | | `type` | The schema type. | | `timestamp` | The timestamp of creating this version of schema. | -| `data` | The schema definition data, which is encoded in UTF 8 charset. * If the schema is a **primitive** schema, this field should be blank. * If the schema is a **struct** schema, this field should be a JSON string of the Avro schema definition. | +| `data` | The schema definition data, which is encoded in UTF 8 charset.
  • If the schema is a
    • **primitive**
  • schema, this field should be blank.
  • If the schema is a
    • **struct**
  • schema, this field should be a JSON string of the Avro schema definition.
    • | | `properties` | The additional properties associated with the schema. | @@ -478,7 +478,7 @@ The `SchemaInfo` includes the following fields: | --- | --- | | `name` | The schema name. | | `type` | The schema type. | -| `schema` | A byte array of the schema definition data, which is encoded in UTF 8. * If the schema is a **primitive** schema, this byte array should be empty. * If the schema is a **struct** schema, this field should be a JSON string of the Avro schema definition converted to a byte array. | +| `schema` | A byte array of the schema definition data, which is encoded in UTF 8.
  • If the schema is a
    • **primitive**
  • schema, this byte array should be empty.
  • If the schema is a
    • **struct**
  • schema, this field should be a JSON string of the Avro schema definition converted to a byte array.
    • | | `properties` | The additional properties associated with the schema. | Here is an example of `SchemaInfo`: @@ -528,7 +528,6 @@ To delete a schema for a topic, you can use one of the following methods. :::note - In any case, the **delete** action deletes **all versions** of a schema registered for a topic. ::: @@ -648,7 +647,6 @@ public interface SchemaStorage { :::tip - For a complete example of **schema storage** implementation, see [BookKeeperSchemaStorage](https://github.com/apache/pulsar/blob/master/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java) class. ::: @@ -668,7 +666,6 @@ public interface SchemaStorageFactory { :::tip - For a complete example of **schema storage factory** implementation, see [BookKeeperSchemaStorageFactory](https://github.com/apache/pulsar/blob/master/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorageFactory.java) class. ::: diff --git a/site2/website-next/versioned_docs/version-2.7.2/schema-understand.md b/site2/website-next/versioned_docs/version-2.7.2/schema-understand.md index 7de41e6c3838f..87822c55810b4 100644 --- a/site2/website-next/versioned_docs/version-2.7.2/schema-understand.md +++ b/site2/website-next/versioned_docs/version-2.7.2/schema-understand.md @@ -1,7 +1,7 @@ --- id: schema-understand title: Understand schema -sidebar_label: Understand schema +sidebar_label: "Understand schema" original_id: schema-understand --- @@ -22,7 +22,7 @@ A `SchemaInfo` consists of the following fields: | Field | Description | | --- | --- | | `name` | Schema name (a string). | -| `type` | Schema type, which determines how to interpret the schema data. * Predefined schema: see [here](schema-understand.md#schema-type). * Customized schema: it is left as an empty string. | +| `type` | Schema type, which determines how to interpret the schema data.
  • Predefined schema: see [here](schema-understand.md#schema-type).
  • Customized schema: it is left as an empty string.
    • | | `schema`（`payload`) | Schema data, which is a sequence of 8-bit unsigned bytes and schema-type specific. | | `properties` | It is a user defined properties as a string/string map. Applications can use this bag for carrying any application specific logics. Possible properties might be the Git hash associated with the schema, an environment string like `dev` or `prod`. | @@ -403,8 +403,8 @@ The table below lists the possible scenarios when this connection attempt occurs | Scenario | What happens | | --- | --- | -| * No schema exists for the topic. | (1) The producer is created using the given schema. (2) Since no existing schema is compatible with the `SensorReading` schema, the schema is transmitted to the broker and stored. (3) Any consumer created using the same schema or topic can consume messages from the `sensor-data` topic. | -| * A schema already exists. * The producer connects using the same schema that is already stored. | (1) The schema is transmitted to the broker. (2) The broker determines that the schema is compatible. (3) The broker attempts to store the schema in [BookKeeper](concepts-architecture-overview.md#persistent-storage) but then determines that it's already stored, so it is used to tag produced messages. | * A schema already exists. * The producer connects using a new schema that is compatible. | (1) The schema is transmitted to the broker. (2) The broker determines that the schema is compatible and stores the new schema as the current version (with a new version number). | +|
  • No schema exists for the topic.
    • | (1) The producer is created using the given schema. (2) Since no existing schema is compatible with the `SensorReading` schema, the schema is transmitted to the broker and stored. (3) Any consumer created using the same schema or topic can consume messages from the `sensor-data` topic. | +|
  • A schema already exists.
  • The producer connects using the same schema that is already stored.
    • | (1) The schema is transmitted to the broker. (2) The broker determines that the schema is compatible. (3) The broker attempts to store the schema in [BookKeeper](concepts-architecture-overview.md#persistent-storage) but then determines that it's already stored, so it is used to tag produced messages. |
  • A schema already exists.
  • The producer connects using a new schema that is compatible.
    • | (1) The schema is transmitted to the broker. (2) The broker determines that the schema is compatible and stores the new schema as the current version (with a new version number). | ## How does schema work diff --git a/site2/website-next/versioned_docs/version-2.7.3/client-libraries-node.md b/site2/website-next/versioned_docs/version-2.7.3/client-libraries-node.md index 3ba0057a0f27c..9b8ecd41d1307 100644 --- a/site2/website-next/versioned_docs/version-2.7.3/client-libraries-node.md +++ b/site2/website-next/versioned_docs/version-2.7.3/client-libraries-node.md @@ -124,7 +124,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -153,7 +153,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer will publish messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar will automatically generate a globally unique name. If you choose to explicitly assign a name, it will need to be unique across *all* Pulsar clusters, otherwise the creation operation will throw an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer will wait for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error will be thrown. If you set `sendTimeoutMs` to -1, the timeout will be set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website-next/versioned_docs/version-2.8.0/client-libraries-node.md b/site2/website-next/versioned_docs/version-2.8.0/client-libraries-node.md index 83fb78d36b0a1..b1ef130b8b8c8 100644 --- a/site2/website-next/versioned_docs/version-2.8.0/client-libraries-node.md +++ b/site2/website-next/versioned_docs/version-2.8.0/client-libraries-node.md @@ -126,7 +126,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -157,7 +157,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar automatically generates a globally unique name. If you choose to explicitly assign a name, it needs to be unique across *all* Pulsar clusters, otherwise the creation operation throws an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer waits for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error is thrown. If you set `sendTimeoutMs` to -1, the timeout is set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.5.0/client-libraries-node.md b/site2/website/versioned_docs/version-2.5.0/client-libraries-node.md index 2e9c0397869a2..40f1f1ca9b753 100644 --- a/site2/website/versioned_docs/version-2.5.0/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.5.0/client-libraries-node.md @@ -101,7 +101,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -129,7 +129,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer will publish messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar will automatically generate a globally unique name. If you choose to explicitly assign a name, it will need to be unique across *all* Pulsar clusters, otherwise the creation operation will throw an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer will wait for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error will be thrown. If you set `sendTimeoutMs` to -1, the timeout will be set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.5.1/client-libraries-node.md b/site2/website/versioned_docs/version-2.5.1/client-libraries-node.md index 2d10ce487f475..1ba8cbf2fb416 100644 --- a/site2/website/versioned_docs/version-2.5.1/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.5.1/client-libraries-node.md @@ -103,7 +103,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -131,7 +131,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer will publish messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar will automatically generate a globally unique name. If you choose to explicitly assign a name, it will need to be unique across *all* Pulsar clusters, otherwise the creation operation will throw an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer will wait for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error will be thrown. If you set `sendTimeoutMs` to -1, the timeout will be set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.5.2/client-libraries-node.md b/site2/website/versioned_docs/version-2.5.2/client-libraries-node.md index 2ceae503e4ed8..0ee2c9ed29dcf 100644 --- a/site2/website/versioned_docs/version-2.5.2/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.5.2/client-libraries-node.md @@ -103,7 +103,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -131,7 +131,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer will publish messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar will automatically generate a globally unique name. If you choose to explicitly assign a name, it will need to be unique across *all* Pulsar clusters, otherwise the creation operation will throw an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer will wait for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error will be thrown. If you set `sendTimeoutMs` to -1, the timeout will be set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.6.0/client-libraries-node.md b/site2/website/versioned_docs/version-2.6.0/client-libraries-node.md index d7f0ca2b1c365..55c4c26b66604 100644 --- a/site2/website/versioned_docs/version-2.6.0/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.6.0/client-libraries-node.md @@ -103,7 +103,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -131,7 +131,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer will publish messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar will automatically generate a globally unique name. If you choose to explicitly assign a name, it will need to be unique across *all* Pulsar clusters, otherwise the creation operation will throw an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer will wait for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error will be thrown. If you set `sendTimeoutMs` to -1, the timeout will be set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.6.1/client-libraries-node.md b/site2/website/versioned_docs/version-2.6.1/client-libraries-node.md index 2d15f6ecb9fb1..08f9d2a6e2dfd 100644 --- a/site2/website/versioned_docs/version-2.6.1/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.6.1/client-libraries-node.md @@ -103,7 +103,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -131,7 +131,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer will publish messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar will automatically generate a globally unique name. If you choose to explicitly assign a name, it will need to be unique across *all* Pulsar clusters, otherwise the creation operation will throw an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer will wait for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error will be thrown. If you set `sendTimeoutMs` to -1, the timeout will be set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.6.2/client-libraries-node.md b/site2/website/versioned_docs/version-2.6.2/client-libraries-node.md index fee4b27cc434a..b4bcc878d0210 100644 --- a/site2/website/versioned_docs/version-2.6.2/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.6.2/client-libraries-node.md @@ -103,7 +103,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -131,7 +131,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer will publish messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar will automatically generate a globally unique name. If you choose to explicitly assign a name, it will need to be unique across *all* Pulsar clusters, otherwise the creation operation will throw an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer will wait for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error will be thrown. If you set `sendTimeoutMs` to -1, the timeout will be set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.6.3/client-libraries-node.md b/site2/website/versioned_docs/version-2.6.3/client-libraries-node.md index b7d0409c6d542..0b44e94a34d4f 100644 --- a/site2/website/versioned_docs/version-2.6.3/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.6.3/client-libraries-node.md @@ -103,7 +103,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -131,7 +131,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer will publish messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar will automatically generate a globally unique name. If you choose to explicitly assign a name, it will need to be unique across *all* Pulsar clusters, otherwise the creation operation will throw an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer will wait for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error will be thrown. If you set `sendTimeoutMs` to -1, the timeout will be set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.6.4/client-libraries-node.md b/site2/website/versioned_docs/version-2.6.4/client-libraries-node.md index 4bc21ea547b7a..aed8028c9d894 100644 --- a/site2/website/versioned_docs/version-2.6.4/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.6.4/client-libraries-node.md @@ -103,7 +103,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -131,7 +131,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer will publish messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar will automatically generate a globally unique name. If you choose to explicitly assign a name, it will need to be unique across *all* Pulsar clusters, otherwise the creation operation will throw an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer will wait for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error will be thrown. If you set `sendTimeoutMs` to -1, the timeout will be set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.7.0/client-libraries-node.md b/site2/website/versioned_docs/version-2.7.0/client-libraries-node.md index 324ee5457797b..f508c4a7554b0 100644 --- a/site2/website/versioned_docs/version-2.7.0/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.7.0/client-libraries-node.md @@ -106,7 +106,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -134,7 +134,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer will publish messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar will automatically generate a globally unique name. If you choose to explicitly assign a name, it will need to be unique across *all* Pulsar clusters, otherwise the creation operation will throw an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer will wait for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error will be thrown. If you set `sendTimeoutMs` to -1, the timeout will be set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.7.1/client-libraries-node.md b/site2/website/versioned_docs/version-2.7.1/client-libraries-node.md index e8c4b95064d35..c1708a176ddcb 100644 --- a/site2/website/versioned_docs/version-2.7.1/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.7.1/client-libraries-node.md @@ -106,7 +106,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -134,7 +134,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer will publish messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar will automatically generate a globally unique name. If you choose to explicitly assign a name, it will need to be unique across *all* Pulsar clusters, otherwise the creation operation will throw an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer will wait for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error will be thrown. If you set `sendTimeoutMs` to -1, the timeout will be set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.7.2/client-libraries-node.md b/site2/website/versioned_docs/version-2.7.2/client-libraries-node.md index 3b1137b9df7d4..a067139622b8f 100644 --- a/site2/website/versioned_docs/version-2.7.2/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.7.2/client-libraries-node.md @@ -106,7 +106,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -134,7 +134,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer will publish messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar will automatically generate a globally unique name. If you choose to explicitly assign a name, it will need to be unique across *all* Pulsar clusters, otherwise the creation operation will throw an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer will wait for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error will be thrown. If you set `sendTimeoutMs` to -1, the timeout will be set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.7.3/client-libraries-node.md b/site2/website/versioned_docs/version-2.7.3/client-libraries-node.md index dde8a32ee9dd1..1d36814ea0e7c 100644 --- a/site2/website/versioned_docs/version-2.7.3/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.7.3/client-libraries-node.md @@ -134,7 +134,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer will publish messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar will automatically generate a globally unique name. If you choose to explicitly assign a name, it will need to be unique across *all* Pulsar clusters, otherwise the creation operation will throw an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer will wait for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error will be thrown. If you set `sendTimeoutMs` to -1, the timeout will be set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.8.0/client-libraries-node.md b/site2/website/versioned_docs/version-2.8.0/client-libraries-node.md index 65997c1be8260..b9940712fb71d 100644 --- a/site2/website/versioned_docs/version-2.8.0/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.8.0/client-libraries-node.md @@ -138,7 +138,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar automatically generates a globally unique name. If you choose to explicitly assign a name, it needs to be unique across *all* Pulsar clusters, otherwise the creation operation throws an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer waits for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error is thrown. If you set `sendTimeoutMs` to -1, the timeout is set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.8.1/client-libraries-node.md b/site2/website/versioned_docs/version-2.8.1/client-libraries-node.md index ff41c2fc6e571..03c828f3945cf 100644 --- a/site2/website/versioned_docs/version-2.8.1/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.8.1/client-libraries-node.md @@ -138,7 +138,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar automatically generates a globally unique name. If you choose to explicitly assign a name, it needs to be unique across *all* Pulsar clusters, otherwise the creation operation throws an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer waits for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error is thrown. If you set `sendTimeoutMs` to -1, the timeout is set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/site2/website/versioned_docs/version-2.8.2/client-libraries-node.md b/site2/website/versioned_docs/version-2.8.2/client-libraries-node.md index 62ebe4ea38a58..a5072faf0ded0 100644 --- a/site2/website/versioned_docs/version-2.8.2/client-libraries-node.md +++ b/site2/website/versioned_docs/version-2.8.2/client-libraries-node.md @@ -108,7 +108,7 @@ Here is an example: ```JavaScript const producer = await client.createProducer({ - topic: 'my-topic', + topic: 'my-topic', // or 'my-tenant/my-namespace/my-topic' to specify topic's tenant and namespace }); await producer.send({ @@ -138,7 +138,7 @@ Pulsar Node.js producers have the following methods available: | Parameter | Description | Default | | :-------- | :---------- | :------ | -| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. | | +| `topic` | The Pulsar [topic](reference-terminology.md#topic) to which the producer publishes messages. The topic format is `` or `//`. For example, `sample/ns1/my-topic`. | | | `producerName` | A name for the producer. If you do not explicitly assign a name, Pulsar automatically generates a globally unique name. If you choose to explicitly assign a name, it needs to be unique across *all* Pulsar clusters, otherwise the creation operation throws an error. | | | `sendTimeoutMs` | When publishing a message to a topic, the producer waits for an acknowledgment from the responsible Pulsar [broker](reference-terminology.md#broker). If a message is not acknowledged within the threshold set by this parameter, an error is thrown. If you set `sendTimeoutMs` to -1, the timeout is set to infinity (and thus removed). Removing the send timeout is recommended when using Pulsar's [message de-duplication](cookbooks-deduplication.md) feature. | 30000 | | `initialSequenceId` | The initial sequence ID of the message. When producer send message, add sequence ID to message. The ID is increased each time to send. | | diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index cb882523fbafe..dc026a4f9104e 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/testmocks/pom.xml b/testmocks/pom.xml index 0d199f1ccb273..2dea92fa863a5 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT testmocks diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index 50d1883cc07dd..8692fe61804e1 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT bc_2_0_0 diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index 10830353ee6d4..1d4a0c544aaf9 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT bc_2_0_1 diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index 2a81b22e597d5..bca6b8c68495d 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index 72dff476c39b0..ce1c2ff4ede6a 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 java-test-functions diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 7dd26d8040ebb..0e9aee1aa3ab1 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 java-test-image diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 8d0aba77ad47f..697772b5dfee4 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 latest-version-image diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index f68e7d4dbbde7..8466911a8e7c8 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT docker-images Apache Pulsar :: Tests :: Docker Images diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 78382c292984c..66f3872e7961f 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT integration diff --git a/tests/pom.xml b/tests/pom.xml index df6f2c1b411c2..f237e653264a1 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT org.apache.pulsar.tests tests-parent diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index 75b31765f40a4..6dba6c58291a9 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-client-admin-shade-test diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index c5b84a933e83a..2904fd4b70789 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-client-all-shade-test diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index 438fd758474ee..4671a483f71f0 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT pulsar-client-shade-test diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index c261a9f0b7b01..590d1f4ad703f 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index 265743c7e376e..ea9e51783fd42 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT .. diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml index fb04ef4c4805c..08767068cb1e8 100644 --- a/tiered-storage/pom.xml +++ b/tiered-storage/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ..