diff --git a/.ci/clusters/compute_v1alpha1_function.yaml b/.ci/clusters/compute_v1alpha1_function.yaml
index dc4096ed2..8a7b5441a 100644
--- a/.ci/clusters/compute_v1alpha1_function.yaml
+++ b/.ci/clusters/compute_v1alpha1_function.yaml
@@ -4,7 +4,7 @@ metadata:
name: function-sample
namespace: default
spec:
- image: streamnative/pulsar-functions-java-sample:2.8.0.13
+ image: streamnative/pulsar-functions-java-sample:2.8.0.14
className: org.apache.pulsar.functions.api.examples.ExclamationFunction
forwardSourceMessageProperty: true
MaxPendingAsyncRequests: 1000
diff --git a/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml b/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml
index 459f558ea..67cc68b8b 100644
--- a/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml
+++ b/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml
@@ -4,7 +4,7 @@ metadata:
name: function-builtin-hpa-sample
namespace: default
spec:
- image: streamnative/pulsar-functions-java-sample:2.8.0.13
+ image: streamnative/pulsar-functions-java-sample:2.8.0.14
className: org.apache.pulsar.functions.api.examples.ExclamationFunction
forwardSourceMessageProperty: true
MaxPendingAsyncRequests: 1000
diff --git a/.ci/clusters/compute_v1alpha1_function_hpa.yaml b/.ci/clusters/compute_v1alpha1_function_hpa.yaml
index 0ac7c6cfb..7d1d0bdbb 100644
--- a/.ci/clusters/compute_v1alpha1_function_hpa.yaml
+++ b/.ci/clusters/compute_v1alpha1_function_hpa.yaml
@@ -4,7 +4,7 @@ metadata:
name: function-hpa-sample
namespace: default
spec:
- image: streamnative/pulsar-functions-java-sample:2.8.0.13
+ image: streamnative/pulsar-functions-java-sample:2.8.0.14
className: org.apache.pulsar.functions.api.examples.ExclamationFunction
forwardSourceMessageProperty: true
MaxPendingAsyncRequests: 1000
diff --git a/.ci/clusters/compute_v1alpha1_functionmesh.yaml b/.ci/clusters/compute_v1alpha1_functionmesh.yaml
index 5e693258d..54d15b7b9 100644
--- a/.ci/clusters/compute_v1alpha1_functionmesh.yaml
+++ b/.ci/clusters/compute_v1alpha1_functionmesh.yaml
@@ -5,7 +5,7 @@ metadata:
spec:
functions:
- name: java-function
- image: streamnative/pulsar-functions-java-sample:2.8.0.13
+ image: streamnative/pulsar-functions-java-sample:2.8.0.14
className: org.apache.pulsar.functions.api.examples.ExclamationFunction
replicas: 1
maxReplicas: 1
@@ -31,7 +31,7 @@ spec:
memory: 1.1G
clusterName: test
- name: golang-function
- image: streamnative/pulsar-functions-go-sample:2.8.0.13
+ image: streamnative/pulsar-functions-go-sample:2.8.0.14
replicas: 1
maxReplicas: 1
input:
@@ -56,7 +56,7 @@ spec:
memory: 1.1G
clusterName: test
- name: python-function
- image: streamnative/pulsar-functions-python-sample:2.8.0.13
+ image: streamnative/pulsar-functions-python-sample:2.8.0.14
className: exclamation_function.ExclamationFunction
replicas: 1
maxReplicas: 1
diff --git a/.ci/clusters/compute_v1alpha1_go_function.yaml b/.ci/clusters/compute_v1alpha1_go_function.yaml
index 29ba59339..6d6a22a86 100644
--- a/.ci/clusters/compute_v1alpha1_go_function.yaml
+++ b/.ci/clusters/compute_v1alpha1_go_function.yaml
@@ -4,7 +4,7 @@ metadata:
name: go-function-sample
namespace: default
spec:
- image: streamnative/pulsar-functions-go-sample:2.8.0.13
+ image: streamnative/pulsar-functions-go-sample:2.8.0.14
forwardSourceMessageProperty: true
MaxPendingAsyncRequests: 1000
replicas: 1
diff --git a/.ci/clusters/compute_v1alpha1_py_function.yaml b/.ci/clusters/compute_v1alpha1_py_function.yaml
index ed9c5e83f..4ad156b38 100644
--- a/.ci/clusters/compute_v1alpha1_py_function.yaml
+++ b/.ci/clusters/compute_v1alpha1_py_function.yaml
@@ -4,7 +4,7 @@ metadata:
name: py-function-sample
namespace: default
spec:
- image: streamnative/pulsar-functions-python-sample:2.8.0.13
+ image: streamnative/pulsar-functions-python-sample:2.8.0.14
className: exclamation_function.ExclamationFunction
forwardSourceMessageProperty: true
MaxPendingAsyncRequests: 1000
diff --git a/.ci/clusters/values.yaml b/.ci/clusters/values.yaml
index 3d879fcbe..f181c77c6 100644
--- a/.ci/clusters/values.yaml
+++ b/.ci/clusters/values.yaml
@@ -42,22 +42,22 @@ monitoring:
images:
zookeeper:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
bookie:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
autorecovery:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
broker:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
functions:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
proxy:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
zookeeper:
replicaCount: 1
@@ -67,7 +67,7 @@ bookkeeper:
metadata:
image:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
configData:
# `BOOKIE_MEM` is used for `bookie shell`
BOOKIE_MEM: >
@@ -97,7 +97,7 @@ bookkeeper:
pulsar_metadata:
image:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
broker:
replicaCount: 1
diff --git a/.ci/clusters/values_cluster.yaml b/.ci/clusters/values_cluster.yaml
index 3d879fcbe..f181c77c6 100644
--- a/.ci/clusters/values_cluster.yaml
+++ b/.ci/clusters/values_cluster.yaml
@@ -42,22 +42,22 @@ monitoring:
images:
zookeeper:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
bookie:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
autorecovery:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
broker:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
functions:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
proxy:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
zookeeper:
replicaCount: 1
@@ -67,7 +67,7 @@ bookkeeper:
metadata:
image:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
configData:
# `BOOKIE_MEM` is used for `bookie shell`
BOOKIE_MEM: >
@@ -97,7 +97,7 @@ bookkeeper:
pulsar_metadata:
image:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
broker:
replicaCount: 1
diff --git a/.ci/clusters/values_custom_runner_images.yaml b/.ci/clusters/values_custom_runner_images.yaml
index 569d1c420..3e2fb4bfe 100644
--- a/.ci/clusters/values_custom_runner_images.yaml
+++ b/.ci/clusters/values_custom_runner_images.yaml
@@ -41,27 +41,27 @@ monitoring:
images:
zookeeper:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
pullPolicy: IfNotPresent
bookie:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
pullPolicy: IfNotPresent
autorecovery:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
pullPolicy: IfNotPresent
broker:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
pullPolicy: IfNotPresent
proxy:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
pullPolicy: IfNotPresent
functions:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
pullPolicy: IfNotPresent
zookeeper:
@@ -102,7 +102,7 @@ bookkeeper:
pulsar_metadata:
image:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
pullPolicy: IfNotPresent
broker:
@@ -148,9 +148,9 @@ functions:
narExtractionDirectory: ""
functionRuntimeFactoryConfigs:
functionDockerImages:
- JAVA: "streamnative/pulsar-functions-java-runner:2.8.0.13"
- PYTHON: "streamnative/pulsar-functions-python-runner:2.8.0.13"
- GO: "streamnative/pulsar-functions-go-runner:2.8.0.13"
+ JAVA: "streamnative/pulsar-functions-java-runner:2.8.0.14"
+ PYTHON: "streamnative/pulsar-functions-python-runner:2.8.0.14"
+ GO: "streamnative/pulsar-functions-go-runner:2.8.0.14"
proxy:
replicaCount: 1
diff --git a/.ci/clusters/values_mesh_worker_service.yaml b/.ci/clusters/values_mesh_worker_service.yaml
index a13899f7f..cd1b1691a 100644
--- a/.ci/clusters/values_mesh_worker_service.yaml
+++ b/.ci/clusters/values_mesh_worker_service.yaml
@@ -113,6 +113,8 @@ broker:
managedLedgerDefaultEnsembleSize: "1"
managedLedgerDefaultWriteQuorum: "1"
managedLedgerDefaultAckQuorum: "1"
+ enablePackagesManagement: "true"
+ PULSAR_PREFIX_enablePackagesManagement: "true"
proxy:
replicaCount: 1
diff --git a/.ci/clusters/values_runner_images.yaml b/.ci/clusters/values_runner_images.yaml
index 80ed7afbd..e21035935 100644
--- a/.ci/clusters/values_runner_images.yaml
+++ b/.ci/clusters/values_runner_images.yaml
@@ -40,27 +40,27 @@ monitoring:
images:
zookeeper:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
pullPolicy: IfNotPresent
bookie:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
pullPolicy: IfNotPresent
autorecovery:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
pullPolicy: IfNotPresent
broker:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
pullPolicy: IfNotPresent
proxy:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
pullPolicy: IfNotPresent
functions:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
pullPolicy: IfNotPresent
zookeeper:
@@ -101,7 +101,7 @@ bookkeeper:
pulsar_metadata:
image:
repository: streamnative/pulsar-all
- tag: 2.8.0.13
+ tag: 2.8.0.14
pullPolicy: IfNotPresent
broker:
@@ -161,9 +161,9 @@ functions:
disk: 1048576000
functionRuntimeFactoryConfigs:
functionDockerImages:
- JAVA: "streamnative/pulsar-functions-java-runner:2.8.0.13"
- PYTHON: "streamnative/pulsar-functions-python-runner:2.8.0.13"
- GO: "streamnative/pulsar-functions-go-runner:2.8.0.13"
+ JAVA: "streamnative/pulsar-functions-java-runner:2.8.0.14"
+ PYTHON: "streamnative/pulsar-functions-python-runner:2.8.0.14"
+ GO: "streamnative/pulsar-functions-go-runner:2.8.0.14"
functionInstanceMinResources:
cpu: 0.1
ram: 10485760
diff --git a/.ci/helm.sh b/.ci/helm.sh
index 6084974e0..a588b59ca 100644
--- a/.ci/helm.sh
+++ b/.ci/helm.sh
@@ -286,6 +286,7 @@ function ci::verify_mesh_worker_service_pulsar_admin() {
fi
${KUBECTL} get pods -n ${NAMESPACE}
sleep 120
+ ${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-source"
WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-sink" | wc -l)
while [[ ${WC} -lt 1 ]]; do
echo ${WC};
@@ -293,6 +294,7 @@ function ci::verify_mesh_worker_service_pulsar_admin() {
${KUBECTL} get pods -n ${NAMESPACE}
WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-sink" | wc -l)
done
+ ${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-source"
WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-source" | wc -l)
while [[ ${WC} -lt 1 ]]; do
echo ${WC};
@@ -300,6 +302,7 @@ function ci::verify_mesh_worker_service_pulsar_admin() {
${KUBECTL} get pods -n ${NAMESPACE}
WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-source" | wc -l)
done
+ sleep 120
${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin sinks status --name data-generator-sink
RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin sinks status --name data-generator-sink)
if [[ $RET != *"true"* ]]; then
@@ -342,3 +345,107 @@ function ci::verify_mesh_worker_service_pulsar_admin() {
${KUBECTL} get pods -n ${NAMESPACE}
}
+
+function ci::upload_java_package() {
+
+ RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages upload function://public/default/java-function@1.0 --path /pulsar/examples/api-examples.jar --description java-function@1.0)
+ if [[ $RET != *"successfully"* ]]; then
+ echo "${RET}"
+ ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0
+ sleep 60
+ return 1
+ fi
+ RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages download function://public/default/java-function@1.0 --path /pulsar/api-examples.jar)
+ if [[ $RET != *"successfully"* ]]; then
+ echo "${RET}"
+ ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0
+ sleep 60
+ return 1
+ fi
+}
+
+function ci::verify_java_package() {
+ RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --jar function://public/default/java-function@1.0 --name package-java-fn --className org.apache.pulsar.functions.api.examples.ExclamationFunction --inputs persistent://public/default/package-java-fn-input --cpu 0.1)
+ ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0
+ sleep 15
+ echo $RET
+ ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0
+ sleep 15
+ ${KUBECTL} get pods -A
+ sleep 5
+ WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-java-fn" | wc -l)
+ while [[ ${WC} -lt 1 ]]; do
+ echo ${WC};
+ sleep 20
+ ${KUBECTL} get pods -n ${NAMESPACE}
+ ${KUBECTL} describe pod package-java-fn-function-0
+ WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-java-fn" | wc -l)
+ done
+ echo "java function test done"
+ RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions delete --name package-java-fn)
+ echo "${RET}"
+}
+
+function ci::upload_python_package() {
+ RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages upload function://public/default/python-function@1.0 --path /pulsar/examples/python-examples/exclamation_function.py --description python-function@1.0)
+ if [[ $RET != *"successfully"* ]]; then
+ echo "${RET}"
+ return 1
+ fi
+}
+
+function ci::verify_python_package() {
+ RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --py function://public/default/python-function@1.0 --name package-python-fn --classname exclamation_function.ExclamationFunction --inputs persistent://public/default/package-python-fn-input --cpu 0.1)
+ if [[ $RET != *"successfully"* ]]; then
+ echo "${RET}"
+ return 1
+ fi
+ sleep 15
+ ${KUBECTL} get pods -A
+ sleep 5
+ WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-python-fn" | wc -l)
+ while [[ ${WC} -lt 1 ]]; do
+ echo ${WC};
+ sleep 20
+ ${KUBECTL} get pods -n ${NAMESPACE}
+ ${KUBECTL} describe pod package-python-fn-function-0
+ WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-python-fn" | wc -l)
+ done
+ echo "python function test done"
+ RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions delete --name package-python-fn)
+ echo "${RET}"
+}
+
+function ci::upload_go_package() {
+ ${KUBECTL} cp "${FUNCTION_MESH_HOME}/.ci/examples/go-examples" "${NAMESPACE}/${CLUSTER}-pulsar-broker-0:/pulsar/examples"
+ sleep 1
+ ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- ls -l /pulsar/examples/go-examples
+ sleep 1
+ RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages upload function://public/default/go-function@1.0 --path /pulsar/examples/go-examples/exclamationFunc --description go-function@1.0)
+ if [[ $RET != *"successfully"* ]]; then
+ echo "${RET}"
+ return 1
+ fi
+}
+
+function ci::verify_go_package() {
+ RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --go function://public/default/go-function@1.0 --name package-go-fn --inputs persistent://public/default/package-go-fn-input --cpu 0.1)
+ if [[ $RET != *"successfully"* ]]; then
+ echo "${RET}"
+ return 1
+ fi
+ sleep 15
+ ${KUBECTL} get pods -A
+ sleep 5
+ WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-go-fn" | wc -l)
+ while [[ ${WC} -lt 1 ]]; do
+ echo ${WC};
+ sleep 20
+ ${KUBECTL} get pods -n ${NAMESPACE}
+ ${KUBECTL} describe pod package-go-fn-function-0
+ WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-go-fn" | wc -l)
+ done
+ echo "go function test done"
+ RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions delete --name package-go-fn)
+ echo "${RET}"
+}
diff --git a/.ci/verify_package_management_service.sh b/.ci/verify_package_management_service.sh
new file mode 100755
index 000000000..f121c6eb2
--- /dev/null
+++ b/.ci/verify_package_management_service.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+BINDIR=`dirname "$0"`
+PULSAR_HOME=`cd ${BINDIR}/..;pwd`
+TLS=${TLS:-"false"}
+SYMMETRIC=${SYMMETRIC:-"false"}
+FUNCTION=${FUNCTION:-"false"}
+
+source ${PULSAR_HOME}/.ci/helm.sh
+
+ci::upload_java_package
+ci::verify_java_package
+
+ci::upload_python_package
+ci::verify_python_package
+
+ci::upload_go_package
+ci::verify_go_package
diff --git a/.github/workflows/test-e2e-crypto.yml b/.github/workflows/test-e2e-crypto.yml
index c912042c9..191f9cc14 100644
--- a/.github/workflows/test-e2e-crypto.yml
+++ b/.github/workflows/test-e2e-crypto.yml
@@ -34,7 +34,7 @@ jobs:
- name: Build runner images
run: |
- PULSAR_IMAGE_TAG=2.8.0.13 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh
+ PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh
- name: Install operator-sdk
run: |
diff --git a/.github/workflows/test-function-key-based-batcher.yml b/.github/workflows/test-function-key-based-batcher.yml
index 6447b1de1..00e5b4edb 100644
--- a/.github/workflows/test-function-key-based-batcher.yml
+++ b/.github/workflows/test-function-key-based-batcher.yml
@@ -34,7 +34,7 @@ jobs:
- name: Build runner images
run: |
- PULSAR_IMAGE_TAG=2.8.0.13 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh
+ PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh
- name: Install operator-sdk
run: |
diff --git a/.github/workflows/test-function-runner.yml b/.github/workflows/test-function-runner.yml
index d0bde6711..082b55c77 100644
--- a/.github/workflows/test-function-runner.yml
+++ b/.github/workflows/test-function-runner.yml
@@ -34,7 +34,7 @@ jobs:
- name: Build runner images
run: |
- PULSAR_IMAGE_TAG=2.8.0.13 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh
+ PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh
- name: Verify function runner
run: |
diff --git a/.github/workflows/test-integration-kind-samples.yml b/.github/workflows/test-integration-kind-samples.yml
index 7ad65542b..6e2b6bc14 100644
--- a/.github/workflows/test-integration-kind-samples.yml
+++ b/.github/workflows/test-integration-kind-samples.yml
@@ -35,8 +35,8 @@ jobs:
- name: Build runner images
run: |
- PULSAR_IMAGE_TAG=2.8.0.13 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh
- PULSAR_IMAGE_TAG=2.8.0.13 KIND_PUSH=true images/samples/build.sh
+ PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh
+ PULSAR_IMAGE_TAG=2.8.0.14 KIND_PUSH=true images/samples/build.sh
- name: Install operator-sdk
run: |
diff --git a/.github/workflows/test-integration-mesh-worker-service.yml b/.github/workflows/test-integration-mesh-worker-service.yml
index ec063ebbf..d4303fda9 100644
--- a/.github/workflows/test-integration-mesh-worker-service.yml
+++ b/.github/workflows/test-integration-mesh-worker-service.yml
@@ -78,6 +78,14 @@ jobs:
make install
nohup make run &
+ - name: Build runner images
+ run: |
+ PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh
+
- name: verify mesh-worker-service
run: |
.ci/verify_mesh_worker_service.sh
+
+ - name: verify package management service
+ run: |
+ .ci/verify_package_management_service.sh
diff --git a/controllers/spec/common.go b/controllers/spec/common.go
index cba687f5b..193dc0ff7 100644
--- a/controllers/spec/common.go
+++ b/controllers/spec/common.go
@@ -35,7 +35,7 @@ import (
const (
EnvShardID = "SHARD_ID"
FunctionsInstanceClasspath = "pulsar.functions.instance.classpath"
- DefaultRunnerTag = "2.8.0.13"
+ DefaultRunnerTag = "2.8.0.14"
DefaultRunnerPrefix = "streamnative/"
DefaultRunnerImage = DefaultRunnerPrefix + "pulsar-all:" + DefaultRunnerTag
DefaultJavaRunnerImage = DefaultRunnerPrefix + "pulsar-functions-java-runner:" + DefaultRunnerTag
diff --git a/images/pulsar-functions-base-runner/Dockerfile b/images/pulsar-functions-base-runner/Dockerfile
index 69a400ad6..30b143c46 100644
--- a/images/pulsar-functions-base-runner/Dockerfile
+++ b/images/pulsar-functions-base-runner/Dockerfile
@@ -1,7 +1,7 @@
ARG PULSAR_IMAGE
ARG PULSAR_IMAGE_TAG
FROM ${PULSAR_IMAGE}:${PULSAR_IMAGE_TAG} as pulsar
-FROM openjdk:11-jre-slim as functions-runner
+FROM ubuntu:20.04 as functions-runner
RUN mkdir -p /pulsar/bin/
RUN mkdir -p /pulsar/lib/
@@ -18,6 +18,14 @@ COPY --from=pulsar /pulsar/lib /pulsar/lib
RUN chmod -R g=u /pulsar
+RUN apt-get update \
+ && apt-get -y dist-upgrade \
+ && apt-get -y install openjdk-11-jdk-headless netcat dnsutils less procps iputils-ping \
+ && apt-get -y --purge autoremove \
+ && apt-get autoclean \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE
WORKDIR /pulsar
\ No newline at end of file
diff --git a/images/pulsar-functions-python-runner/Dockerfile b/images/pulsar-functions-python-runner/Dockerfile
index bb38a6f8a..b11692583 100644
--- a/images/pulsar-functions-python-runner/Dockerfile
+++ b/images/pulsar-functions-python-runner/Dockerfile
@@ -15,15 +15,15 @@ RUN if [ -d "/tmp/pulsar/cpp-client" ]; then mv /tmp/pulsar/cpp-client /pulsar/c
# Install some utilities
RUN apt-get update \
- && DEBIAN_FRONTEND=noninteractive apt-get install -y python3.7 python3.7-dev python3-setuptools python3-yaml python3-kazoo \
+ && DEBIAN_FRONTEND=noninteractive apt-get install -y python3 python3-dev python3-setuptools python3-yaml python3-kazoo \
libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
curl \
&& apt-get clean autoclean && apt-get autoremove --yes && rm -rf /var/lib/apt/lists/*
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-RUN python3.7 get-pip.py
+RUN python3 get-pip.py
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 10
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
RUN if [ -d "/pulsar/cpp-client" ]; then apt-get update \
&& apt install -y /pulsar/cpp-client/*.deb \
@@ -32,3 +32,4 @@ RUN if [ -d "/pulsar/cpp-client" ]; then apt-get update \
WORKDIR /pulsar
RUN if [ -f "/pulsar/bin/install-pulsar-client-37.sh" ]; then /pulsar/bin/install-pulsar-client-37.sh ; fi
+RUN if [ -f "/pulsar/bin/install-pulsar-client.sh" ]; then /pulsar/bin/install-pulsar-client.sh ; fi
diff --git a/mesh-worker-service/integration-tests/docker/Dockerfile b/mesh-worker-service/integration-tests/docker/Dockerfile
index 6f6f22ad7..1c7b7c229 100644
--- a/mesh-worker-service/integration-tests/docker/Dockerfile
+++ b/mesh-worker-service/integration-tests/docker/Dockerfile
@@ -17,7 +17,7 @@
# under the License.
#
-FROM streamnative/pulsar-all:2.8.0.13
+FROM streamnative/pulsar:2.8.0.14
COPY ./target/mesh-worker-service*.nar /pulsar/mesh-worker-service.nar
COPY ./integration-tests/docker/connectors.yaml /pulsar/conf/connectors.yaml
RUN mkdir -p /pulsar-nar
diff --git a/mesh-worker-service/integration-tests/docker/connectors.yaml b/mesh-worker-service/integration-tests/docker/connectors.yaml
index b4df87cf0..d16fdb043 100644
--- a/mesh-worker-service/integration-tests/docker/connectors.yaml
+++ b/mesh-worker-service/integration-tests/docker/connectors.yaml
@@ -24,6 +24,6 @@
sourceConfigClass: org.apache.pulsar.io.datagenerator.DataGeneratorSourceConfig
sinkClass: org.apache.pulsar.io.datagenerator.DataGeneratorPrintSink
imageRepository: streamnative/pulsar-io-data-generator
- version: 2.8.0.13
- imageTag: 2.8.0.13
+ version: 2.8.0.14
+ imageTag: 2.8.0.14
typeClassName: org.apache.pulsar.client.impl.schema.JSONSchema
diff --git a/mesh-worker-service/pom.xml b/mesh-worker-service/pom.xml
index 764798b6e..a14240887 100644
--- a/mesh-worker-service/pom.xml
+++ b/mesh-worker-service/pom.xml
@@ -29,7 +29,7 @@
v0.1.7-rc3
- 2.8.0.13
+ 2.8.0.14
1.18.16
2.14.0
12.0.1
diff --git a/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java b/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java
index 08da59231..edb2b329f 100644
--- a/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java
+++ b/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java
@@ -35,6 +35,7 @@
import io.functionmesh.compute.models.CustomRuntimeOptions;
import io.kubernetes.client.custom.Quantity;
import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
import org.apache.pulsar.client.admin.PulsarAdminException;
@@ -267,7 +268,7 @@ public static V1alpha1Function createV1alpha1FunctionFromFunctionConfig(String k
}
Class>[] typeArgs = null;
if (componentPackageFile != null) {
- typeArgs = extractTypeArgs(functionConfig, componentPackageFile);
+ typeArgs = extractTypeArgs(functionConfig, componentPackageFile, worker.getWorkerConfig().isForwardSourceMessageProperty());
}
if (StringUtils.isNotEmpty(functionConfig.getJar())) {
V1alpha1FunctionSpecJava v1alpha1FunctionSpecJava = new V1alpha1FunctionSpecJava();
@@ -564,14 +565,22 @@ private static File downloadPackageFile(MeshWorkerService worker, String package
// use the Nar extraction directory as a temporary directory for downloaded files
tempDirectory = Paths.get(worker.getWorkerConfig().getNarExtractionDirectory());
}
- File file = Files.createTempFile(tempDirectory, "function", ".tmp").toFile();
- worker.getBrokerAdmin().packages().download(packageName, file.toString());
- return file;
+ if (Files.notExists(tempDirectory)) {
+ Files.createDirectories(tempDirectory);
+ }
+ String fileName = String.format("function-%s.tmp", RandomStringUtils.random(5, true, true).toLowerCase());
+ Path filePath = Paths.get(tempDirectory.toString(), fileName);
+ Files.deleteIfExists(filePath);
+ worker.getBrokerAdmin().packages().download(packageName, filePath.toString());
+ return filePath.toFile();
}
private static Class>[] extractTypeArgs(final FunctionConfig functionConfig,
- final File componentPackageFile) {
+ final File componentPackageFile,
+ final boolean isForwardSourceMessageProperty) {
Class>[] typeArgs = null;
+ FunctionConfigUtils.inferMissingArguments(
+ functionConfig, isForwardSourceMessageProperty);
if (componentPackageFile == null) {
return null;
}