Skip to content

Commit

Permalink
Merge branch 'up/master' into website/branch-2.6.1-part-5
Browse files Browse the repository at this point in the history
* up/master:
  [Doc] add explanations for REST (apache#12918)
  [Doc] add explanations for permissions (apache#12920)
  Apply executable file permissions to py scripts. (apache#12852)
  Fix log level config for pulsar-admin, pulsar-client and pulsar-perf (apache#12915)
  Abstract some common methods to AbstractMetadataStore (apache#12916)
  Build website using https://pulsar.apache.org (apache#12901)
  • Loading branch information
Yan Zhang committed Nov 23, 2021
2 parents 0e42d46 + 92009de commit dfa93e7
Show file tree
Hide file tree
Showing 25 changed files with 264 additions and 28 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci-license.yaml
Expand Up @@ -86,4 +86,4 @@ jobs:

- name: license check
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
run: src/check-binary-license ./distribution/server/target/apache-pulsar-*-bin.tar.gz
run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz
2 changes: 2 additions & 0 deletions bin/pulsar-admin-common.sh
Expand Up @@ -98,11 +98,13 @@ OPTS="$OPTS $PULSAR_EXTRA_OPTS"
# log directory & file
PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"}
PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"}
PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"info"}
PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"}
PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"}

#Configure log configuration system properties
OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_ROOT_LEVEL"
OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT"
2 changes: 2 additions & 0 deletions bin/pulsar-client
Expand Up @@ -101,12 +101,14 @@ OPTS="$OPTS $PULSAR_EXTRA_OPTS"
# log directory & file
PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"}
PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"}
PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"info"}
PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"}

#Configure log configuration system properties
OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_ROOT_LEVEL"

#Change to PULSAR_HOME to support relative paths
cd "$PULSAR_HOME"
Expand Down
4 changes: 4 additions & 0 deletions bin/pulsar-perf
Expand Up @@ -138,9 +138,13 @@ OPTS="$OPTS $PULSAR_EXTRA_OPTS"
PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"}
PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"}
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-perftest.log"}
PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"info"}
PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"}

#Configure log configuration system properties
OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_ROOT_LEVEL"
OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
OPTS="$OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE"

Expand Down
2 changes: 1 addition & 1 deletion distribution/io/pom.xml
Expand Up @@ -111,7 +111,7 @@
<goal>exec</goal>
</goals>
<configuration>
<executable>${project.basedir}/../../src/pulsar-io-gen</executable>
<executable>${project.basedir}/../../src/pulsar-io-gen.sh</executable>
<outputFile>${project.basedir}/target/pulsar-io-gen.output</outputFile>
<arguments>
<argument>conf</argument>
Expand Down
Expand Up @@ -41,6 +41,7 @@
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.metadata.api.GetResult;
import org.apache.pulsar.metadata.api.MetadataCache;
import org.apache.pulsar.metadata.api.MetadataSerde;
import org.apache.pulsar.metadata.api.MetadataStoreException;
Expand Down Expand Up @@ -141,6 +142,21 @@ public <T> MetadataCache<T> getMetadataCache(MetadataSerde<T> serde) {
return metadataCache;
}

@Override
public CompletableFuture<Optional<GetResult>> get(String path) {
if (!isValidPath(path)) {
return FutureUtil.failedFuture(new MetadataStoreException.InvalidPathException(path));
}
return storeGet(path);
}

protected abstract CompletableFuture<Optional<GetResult>> storeGet(String path);

@Override
public CompletableFuture<Stat> put(String path, byte[] value, Optional<Long> expectedVersion) {
return put(path, value, expectedVersion, EnumSet.noneOf(CreateOption.class));
}

@Override
public final CompletableFuture<List<String>> getChildren(String path) {
if (!isValidPath(path)) {
Expand Down
Expand Up @@ -90,10 +90,7 @@ public LocalMemoryMetadataStore(String metadataURL, MetadataStoreConfig metadata
}

@Override
public CompletableFuture<Optional<GetResult>> get(String path) {
if (!isValidPath(path)) {
return FutureUtil.failedFuture(new MetadataStoreException.InvalidPathException(path));
}
public CompletableFuture<Optional<GetResult>> storeGet(String path) {
synchronized (map) {
Value v = map.get(path);
if (v != null) {
Expand Down Expand Up @@ -140,11 +137,6 @@ public CompletableFuture<Boolean> existsFromStore(String path) {
}
}

@Override
public CompletableFuture<Stat> put(String path, byte[] value, Optional<Long> expectedVersion) {
return put(path, value, expectedVersion, EnumSet.noneOf(CreateOption.class));
}

@Override
public CompletableFuture<Stat> storePut(String path, byte[] data, Optional<Long> optExpectedVersion,
EnumSet<CreateOption> options) {
Expand Down
Expand Up @@ -128,7 +128,7 @@ protected void receivedSessionEvent(SessionEvent event) {
}

@Override
public CompletableFuture<Optional<GetResult>> get(String path) {
public CompletableFuture<Optional<GetResult>> storeGet(String path) {
CompletableFuture<Optional<GetResult>> future = new CompletableFuture<>();

try {
Expand Down Expand Up @@ -201,11 +201,6 @@ public CompletableFuture<Boolean> existsFromStore(String path) {
return future;
}

@Override
public CompletableFuture<Stat> put(String path, byte[] value, Optional<Long> optExpectedVersion) {
return put(path, value, optExpectedVersion, EnumSet.noneOf(CreateOption.class));
}

@Override
protected CompletableFuture<Stat> storePut(String path, byte[] value, Optional<Long> optExpectedVersion,
EnumSet<CreateOption> options) {
Expand Down
9 changes: 7 additions & 2 deletions site2/docs/admin-api-permissions.md
Expand Up @@ -14,8 +14,13 @@ sidebar_label: Permissions
>
> - For the latest and complete information about `Java admin API`, including classes, methods, descriptions, and more, see [Java admin API doc](https://pulsar.apache.org/api/admin/).
Permissions in Pulsar are managed at the [namespace](reference-terminology.md#namespace) level
(that is, within [tenants](reference-terminology.md#tenant) and [clusters](reference-terminology.md#cluster)).
Pulsar allows you to grant namespace-level or topic-level permission to users.

- If you grant a namespace-level permission to a user, then the user can access all the topics under the namespace.

- If you grant a topic-level permission to a user, then the user can access only the topic.

The chapters below demonstrate how to grant namespace-level permissions to users. For how to grant topic-level permissions to users, see [manage topics](admin-api-topics.md/#grant-permission).

## Grant permissions

Expand Down
122 changes: 122 additions & 0 deletions site2/docs/client-libraries-rest.md
@@ -0,0 +1,122 @@
---
id: client-libraries-rest
title: Pulsar REST
sidebar_label: REST
---

Pulsar not only provides REST endpoints to manage resources in Pulsar clusters, but also provides methods to query the state for those resources. In addition, Pulsar REST provides a simple way to interact with Pulsar **without using client libraries**, which is convenient for applications to use HTTP to interact with Pulsar.

## Connection

To connect to Pulsar, you need to specify a URL.

- Produce messages to non-partitioned or partitioned topics

```
brokerUrl:{8080/8081}/topics/{persistent/non-persistent}/{my-tenant}/{my-namespace}/{my-topic}
```

- Produce messages to specific partitions of partitioned topics

```
brokerUrl:{8080/8081}/topics/{persistent/non-persistent}/{my-tenant}/{my-namespace}/{my-topic}/partitions/{partition-number}
```

## Producer

Currently, you can produce messages to the following destinations with tools like cURL or Postman via REST.

- Non-partitioned or partitioned topics

- Specific partitions of partitioned topics

> **Note**
>
> You can only produce messages to **topics that already exist** in Pulsar via REST.
Consuming and reading messages via REST will be supported in the future.

### Message

- Below is the structure of a request payload.

Parameter|Required?|Description
|---|---|---
`schemaVersion`|No| Schema version of existing schema used for this message </br></br>You need provide one of the following: <br/><br/> - `schemaVersion` <br/> - `keySchema`/`valueSchema`<br/><br/>If both of them are provided, then `schemaVersion` is used
`keySchema/valueSchema`|No|Key schema / Value schema used for this message
`producerName`|No|Producer name
`Messages[] SingleMessage`|Yes|Messages to be sent

- Below is the structure of a message.

Parameter|Required?|Type|Description
|---|---|---|---
`payload`|Yes|`String`|Actual message payload </br></br>Messages are sent in strings and encoded with given schemas on the server side
`properties`|No|`Map<String, String>`|Custom properties
`key`|No|`String`|Partition key
`replicationClusters`|No|`List<String>`|Clusters to which messages replicate
`eventTime`|No|`String`|Message event time
`sequenceId`|No|`long`|Message sequence ID
`disableReplication`|No|`boolean`|Whether to disable replication of messages
`deliverAt`|No|`long`|Deliver messages only at or after specified absolute timestamp
`deliverAfterMs`|No|`long`|Deliver messages only after specified relative delay (in milliseconds)

### Schema

- Currently, Primitive, Avro, JSON, and KeyValue schemas are supported.

- For Primitive, Avro and JSON schemas, schemas should be provided as the full schema encoded as a string.

- If the schema is not set, messages are encoded with string schema.

### Example

Below is an example of sending messages to topics using JSON schema via REST.

Assume that you send messages representing the following class.

```java
class Seller {
public String state;
public String street;
public long zipCode;
}

class PC {
public String brand;
public String model;
public int year;
public GPU gpu;
public Seller seller;
}
```

Send messages to topics with JSON schema using the command below.

```shell
curl --location --request POST 'brokerUrl:{8080/8081}/topics/{persistent/non-persistent}/{my-tenant}/{my-namespace}/{my-topic}' \
--header 'Content-Type: application/json' \
--data-raw '{
"valueSchema": "{\"name\":\"\",\"schema\":\"eyJ0eXBlIjoicmVjb3JkIiwibmFtZSI6IlBDIiwibmFtZXNwYWNlIjoib3JnLmFwYWNoZS5wdWxzYXIuYnJva2VyLmFkbWluLlRvcGljc1Rlc3QiLCJmaWVsZHMiOlt7Im5hbWUiOiJicmFuZCIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJncHUiLCJ0eXBlIjpbIm51bGwiLHsidHlwZSI6ImVudW0iLCJuYW1lIjoiR1BVIiwic3ltYm9scyI6WyJBTUQiLCJOVklESUEiXX1dLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJtb2RlbCIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJzZWxsZXIiLCJ0eXBlIjpbIm51bGwiLHsidHlwZSI6InJlY29yZCIsIm5hbWUiOiJTZWxsZXIiLCJmaWVsZHMiOlt7Im5hbWUiOiJzdGF0ZSIsInR5cGUiOlsibnVsbCIsInN0cmluZyJdLCJkZWZhdWx0IjpudWxsfSx7Im5hbWUiOiJzdHJlZXQiLCJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXSwiZGVmYXVsdCI6bnVsbH0seyJuYW1lIjoiemlwQ29kZSIsInR5cGUiOiJsb25nIn1dfV0sImRlZmF1bHQiOm51bGx9LHsibmFtZSI6InllYXIiLCJ0eXBlIjoiaW50In1dfQ==\",\"type\":\"JSON\",\"properties\":{\"__jsr310ConversionEnabled\":\"false\",\"__alwaysAllowNull\":\"true\"},\"schemaDefinition\":\"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"PC\\\",\\\"namespace\\\":\\\"org.apache.pulsar.broker.admin.TopicsTest\\\",\\\"fields\\\":[{\\\"name\\\":\\\"brand\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"gpu\\\",\\\"type\\\":[\\\"null\\\",{\\\"type\\\":\\\"enum\\\",\\\"name\\\":\\\"GPU\\\",\\\"symbols\\\":[\\\"AMD\\\",\\\"NVIDIA\\\"]}],\\\"default\\\":null},{\\\"name\\\":\\\"model\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"seller\\\",\\\"type\\\":[\\\"null\\\",{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"Seller\\\",\\\"fields\\\":[{\\\"name\\\":\\\"state\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"street\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"zipCode\\\",\\\"type\\\":\\\"long\\\"}]}],\\\"default\\\":null},{\\\"name\\\":\\\"year\\\",\\\"type\\\":\\\"int\\\"}]}\"}",
// Schema data is just the base 64 encoded schemaDefinition.
"producerName": "rest-producer",
"messages": [
{
"key":"my-key",
"payload":"{\"brand\":\"dell\",\"model\":\"alienware\",\"year\":2021,\"gpu\":\"AMD\",\"seller\":{\"state\":\"WA\",\"street\":\"main street\",\"zipCode\":98004}}",
"eventTime":1603045262772,
"sequenceId":1
},
{
"key":"my-key",
"payload":"{\"brand\":\"asus\",\"model\":\"rog\",\"year\":2020,\"gpu\":\"NVIDIA\",\"seller\":{\"state\":\"CA\",\"street\":\"back street\",\"zipCode\":90232}}",
"eventTime":1603045262772,
"sequenceId":2
}
]
}
`
// Sample message
```
2 changes: 1 addition & 1 deletion site2/website-next/docusaurus.config.js
Expand Up @@ -3,7 +3,7 @@ const darkCodeTheme = require("prism-react-renderer/themes/dracula");

const linkifyRegex = require("./plugins/remark-linkify-regex");

const url = "https://pulsar.incubator.apache.org";
const url = "https://pulsar.apache.org";
const javadocUrl = url + "/api";
const restApiUrl = url + "/admin-rest-api";
const functionsApiUrl = url + "/functions-rest-api";
Expand Down
3 changes: 2 additions & 1 deletion site2/website/sidebars.json
Expand Up @@ -117,7 +117,8 @@
"client-libraries-cpp",
"client-libraries-node",
"client-libraries-websocket",
"client-libraries-dotnet"
"client-libraries-dotnet",
"client-libraries-rest"
],
"Admin API": [
"admin-api-overview",
Expand Down
2 changes: 1 addition & 1 deletion site2/website/siteConfig.js
Expand Up @@ -88,7 +88,7 @@ const renderEndpoint = (initializedPlugin, baseUrl, keyparts, restUrl) => {
return rendered_content;
};

const url = 'https://pulsar.incubator.apache.org';
const url = 'https://pulsar.apache.org';
const javadocUrl = url + '/api';
const restApiUrl = url + "/admin-rest-api";
const functionsApiUrl = url + "/functions-rest-api";
Expand Down
10 changes: 10 additions & 0 deletions site2/website/versioned_docs/version-2.8.2/admin-api-brokers.md
Expand Up @@ -5,6 +5,16 @@ sidebar_label: Brokers
original_id: admin-api-brokers
---

> **Important**
>
> This page only shows **some frequently used operations**.
>
> - For the latest and complete information about `Pulsar admin`, including commands, flags, descriptions, and more, see [Pulsar admin doc](https://pulsar.apache.org/tools/pulsar-admin/).
>
> - For the latest and complete information about `REST API`, including parameters, responses, samples, and more, see {@inject: rest:REST:/} API doc.
>
> - For the latest and complete information about `Java admin API`, including classes, methods, descriptions, and more, see [Java admin API doc](https://pulsar.apache.org/api/admin/).
Pulsar brokers consist of two components:

1. An HTTP server exposing a {@inject: rest:REST:/} interface administration and [topic](reference-terminology.md#topic) lookup.
Expand Down
10 changes: 10 additions & 0 deletions site2/website/versioned_docs/version-2.8.2/admin-api-clusters.md
Expand Up @@ -5,6 +5,16 @@ sidebar_label: Clusters
original_id: admin-api-clusters
---

> **Important**
>
> This page only shows **some frequently used operations**.
>
> - For the latest and complete information about `Pulsar admin`, including commands, flags, descriptions, and more, see [Pulsar admin doc](https://pulsar.apache.org/tools/pulsar-admin/).
>
> - For the latest and complete information about `REST API`, including parameters, responses, samples, and more, see {@inject: rest:REST:/} API doc.
>
> - For the latest and complete information about `Java admin API`, including classes, methods, descriptions, and more, see [Java admin API doc](https://pulsar.apache.org/api/admin/).
Pulsar clusters consist of one or more Pulsar [brokers](reference-terminology.md#broker), one or more [BookKeeper](reference-terminology.md#bookkeeper)
servers (aka [bookies](reference-terminology.md#bookie)), and a [ZooKeeper](https://zookeeper.apache.org) cluster that provides configuration and coordination management.

Expand Down
10 changes: 10 additions & 0 deletions site2/website/versioned_docs/version-2.8.2/admin-api-functions.md
Expand Up @@ -5,6 +5,16 @@ sidebar_label: Functions
original_id: admin-api-functions
---

> **Important**
>
> This page only shows **some frequently used operations**.
>
> - For the latest and complete information about `Pulsar admin`, including commands, flags, descriptions, and more, see [Pulsar admin doc](https://pulsar.apache.org/tools/pulsar-admin/).
>
> - For the latest and complete information about `REST API`, including parameters, responses, samples, and more, see {@inject: rest:REST:/} API doc.
>
> - For the latest and complete information about `Java admin API`, including classes, methods, descriptions, and more, see [Java admin API doc](https://pulsar.apache.org/api/admin/).
**Pulsar Functions** are lightweight compute processes that

* consume messages from one or more Pulsar topics
Expand Down
10 changes: 10 additions & 0 deletions site2/website/versioned_docs/version-2.8.2/admin-api-namespaces.md
Expand Up @@ -5,6 +5,16 @@ sidebar_label: Namespaces
original_id: admin-api-namespaces
---

> **Important**
>
> This page only shows **some frequently used operations**.
>
> - For the latest and complete information about `Pulsar admin`, including commands, flags, descriptions, and more, see [Pulsar admin doc](https://pulsar.apache.org/tools/pulsar-admin/).
>
> - For the latest and complete information about `REST API`, including parameters, responses, samples, and more, see {@inject: rest:REST:/} API doc.
>
> - For the latest and complete information about `Java admin API`, including classes, methods, descriptions, and more, see [Java admin API doc](https://pulsar.apache.org/api/admin/).
Pulsar [namespaces](reference-terminology.md#namespace) are logical groupings of [topics](reference-terminology.md#topic).

Namespaces can be managed via:
Expand Down
19 changes: 15 additions & 4 deletions site2/website/versioned_docs/version-2.8.2/admin-api-overview.md
Expand Up @@ -9,16 +9,27 @@ The Pulsar admin interface enables you to manage all important entities in a Pul

You can interact with the admin interface via:

- HTTP calls, which are made against the admin {@inject: rest:REST:/} API provided by Pulsar brokers. For some RESTful APIs, they might be redirected to the owner brokers for serving with [`307 Temporary Redirect`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/307), hence the HTTP callers should handle `307 Temporary Redirect`. If you use `curl` commands, you should specify `-L` to handle redirections.
- A Java client interface.
- The `pulsar-admin` CLI tool, which is available in the `bin` folder of your Pulsar installation:

```shell
$ bin/pulsar-admin
bin/pulsar-admin
```

For complete commands of `pulsar-admin` tool, see [Pulsar admin snapshot](https://pulsar.apache.org/tools/pulsar-admin/).
> **Important**
>
> For the latest and complete information about `Pulsar admin`, including commands, flags, descriptions, and more, see [Pulsar admin doc](https://pulsar.apache.org/tools/pulsar-admin/).
- HTTP calls, which are made against the admin {@inject: rest:REST:/} API provided by Pulsar brokers. For some RESTful APIs, they might be redirected to the owner brokers for serving with [`307 Temporary Redirect`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/307), hence the HTTP callers should handle `307 Temporary Redirect`. If you use `curl` commands, you should specify `-L` to handle redirections.

> **Important**
>
> For the latest and complete information about `REST API`, including parameters, responses, samples, and more, see {@inject: rest:REST:/} API doc.
- A Java client interface.

> **Important**
>
> For the latest and complete information about `Java admin API`, including classes, methods, descriptions, and more, see [Java admin API doc](https://pulsar.apache.org/api/admin/).
> **The REST API is the admin interface**. Both the `pulsar-admin` CLI tool and the Java client use the REST API. If you implement your own admin interface client, you should use the REST API.
Expand Down

0 comments on commit dfa93e7

Please sign in to comment.