From f00586521a658419f6f40b5415c27e480da53cc4 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 25 Aug 2021 09:49:46 +0800 Subject: [PATCH 01/18] add package management service tests --- .ci/clusters/values_mesh_worker_service.yaml | 2 + .ci/helm.sh | 92 +++++++++++++++++++ .ci/verify_package_management_service.sh | 38 ++++++++ .../test-integration-mesh-worker-service.yml | 4 + 4 files changed, 136 insertions(+) create mode 100755 .ci/verify_package_management_service.sh diff --git a/.ci/clusters/values_mesh_worker_service.yaml b/.ci/clusters/values_mesh_worker_service.yaml index a13899f7f..cd1b1691a 100644 --- a/.ci/clusters/values_mesh_worker_service.yaml +++ b/.ci/clusters/values_mesh_worker_service.yaml @@ -113,6 +113,8 @@ broker: managedLedgerDefaultEnsembleSize: "1" managedLedgerDefaultWriteQuorum: "1" managedLedgerDefaultAckQuorum: "1" + enablePackagesManagement: "true" + PULSAR_PREFIX_enablePackagesManagement: "true" proxy: replicaCount: 1 diff --git a/.ci/helm.sh b/.ci/helm.sh index 6084974e0..6ff2d0c76 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -342,3 +342,95 @@ function ci::verify_mesh_worker_service_pulsar_admin() { ${KUBECTL} get pods -n ${NAMESPACE} } + +function ci::upload_java_package() { + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages upload function://public/default/java-function@1.0 --path /pulsar/examples/api-examples.jar --description java-function@1.0) + if [[ $RET != *"successfully"* ]]; then + echo "${RET}" + return 1 + fi +} + +function ci::verify_java_package() { + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --jar function://public/default/java-function@1.0 --name package-java-fn --className org.apache.pulsar.functions.api.examples.ExclamationFunction --inputs persistent://public/default/package-java-fn-input --cpu 0.1) + if [[ $RET != *"successfully"* ]]; then + echo "${RET}" + return 1 + fi + sleep 15 + ${KUBECTL} get pods -A + sleep 5 + WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-java-fn" | wc -l) + while [[ ${WC} -lt 1 ]]; do + echo ${WC}; + sleep 20 + ${KUBECTL} get pods -n ${NAMESPACE} + ${KUBECTL} describe pod package-java-fn-function-0 + WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-java-fn" | wc -l) + done + echo "java function test done" + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions delete --name package-java-fn) + echo "${RET}" +} + +function ci::upload_python_package() { + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages upload function://public/default/python-function@1.0 --path /pulsar/examples/python-examples/exclamation_function.py --description python-function@1.0) + if [[ $RET != *"successfully"* ]]; then + echo "${RET}" + return 1 + fi +} + +function ci::verify_python_package() { + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --py function://public/default/python-function@1.0 --name package-python-fn --classname exclamation_function.ExclamationFunction --inputs persistent://public/default/package-python-fn-input --cpu 0.1) + if [[ $RET != *"successfully"* ]]; then + echo "${RET}" + return 1 + fi + sleep 15 + ${KUBECTL} get pods -A + sleep 5 + WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-python-fn" | wc -l) + while [[ ${WC} -lt 1 ]]; do + echo ${WC}; + sleep 20 + ${KUBECTL} get pods -n ${NAMESPACE} + ${KUBECTL} describe pod package-python-fn-function-0 + WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-python-fn" | wc -l) + done + echo "python function test done" + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions delete --name package-python-fn) + echo "${RET}" +} + +function ci::upload_go_package() { + ${KUBECTL} cp "${FUNCTION_MESH_HOME}/.ci/examples/go-examples" "${NAMESPACE}/${CLUSTER}-pulsar-broker-0:/pulsar/examples" + sleep 1 + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages upload function://public/default/go-function@1.0 --path /pulsar/examples/go-examples --description go-function@1.0) + if [[ $RET != *"successfully"* ]]; then + echo "${RET}" + return 1 + fi +} + +function ci::verify_go_package() { + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --go function://public/default/go-function@1.0 --name package-go-fn --inputs persistent://public/default/package-go-fn-input --cpu 0.1) + if [[ $RET != *"successfully"* ]]; then + echo "${RET}" + return 1 + fi + sleep 15 + ${KUBECTL} get pods -A + sleep 5 + WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-go-fn" | wc -l) + while [[ ${WC} -lt 1 ]]; do + echo ${WC}; + sleep 20 + ${KUBECTL} get pods -n ${NAMESPACE} + ${KUBECTL} describe pod package-go-fn-function-0 + WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-go-fn" | wc -l) + done + echo "go function test done" + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions delete --name package-go-fn) + echo "${RET}" +} diff --git a/.ci/verify_package_management_service.sh b/.ci/verify_package_management_service.sh new file mode 100755 index 000000000..9637574cd --- /dev/null +++ b/.ci/verify_package_management_service.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +BINDIR=`dirname "$0"` +PULSAR_HOME=`cd ${BINDIR}/..;pwd` +TLS=${TLS:-"false"} +SYMMETRIC=${SYMMETRIC:-"false"} +FUNCTION=${FUNCTION:-"false"} + +source ${PULSAR_HOME}/.ci/helm.sh + +ci::upload_java_package +ci::verify_java_package + +ci::upload_python_package +ci::verify_python_package + +ci::upload_go_package +ci::verify_go_package \ No newline at end of file diff --git a/.github/workflows/test-integration-mesh-worker-service.yml b/.github/workflows/test-integration-mesh-worker-service.yml index ec063ebbf..73bcacc60 100644 --- a/.github/workflows/test-integration-mesh-worker-service.yml +++ b/.github/workflows/test-integration-mesh-worker-service.yml @@ -81,3 +81,7 @@ jobs: - name: verify mesh-worker-service run: | .ci/verify_mesh_worker_service.sh + + - name: verify package management service + run: | + .ci/verify_package_management_service.sh From 8cabf083171d3923d1f4c80847aaf4a6a8da96bc Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 25 Aug 2021 10:26:55 +0800 Subject: [PATCH 02/18] more logs --- .ci/helm.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.ci/helm.sh b/.ci/helm.sh index 6ff2d0c76..47c32d39f 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -349,12 +349,18 @@ function ci::upload_java_package() { echo "${RET}" return 1 fi + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages download function://public/default/java-function@1.0 --path /pulsar/api-examples.jar) + if [[ $RET != *"successfully"* ]]; then + echo "${RET}" + return 1 + fi } function ci::verify_java_package() { RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --jar function://public/default/java-function@1.0 --name package-java-fn --className org.apache.pulsar.functions.api.examples.ExclamationFunction --inputs persistent://public/default/package-java-fn-input --cpu 0.1) if [[ $RET != *"successfully"* ]]; then echo "${RET}" + ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 return 1 fi sleep 15 From 1f5fb40946d5b99b2ad5f9c48016bead4a9279d7 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 25 Aug 2021 10:52:12 +0800 Subject: [PATCH 03/18] fix CI --- .ci/helm.sh | 8 +++----- .ci/verify_package_management_service.sh | 8 ++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.ci/helm.sh b/.ci/helm.sh index 47c32d39f..3ae579879 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -358,11 +358,9 @@ function ci::upload_java_package() { function ci::verify_java_package() { RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --jar function://public/default/java-function@1.0 --name package-java-fn --className org.apache.pulsar.functions.api.examples.ExclamationFunction --inputs persistent://public/default/package-java-fn-input --cpu 0.1) - if [[ $RET != *"successfully"* ]]; then - echo "${RET}" - ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 - return 1 - fi + sleep 15 + echo "${RET}" + ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 sleep 15 ${KUBECTL} get pods -A sleep 5 diff --git a/.ci/verify_package_management_service.sh b/.ci/verify_package_management_service.sh index 9637574cd..c0ef5d6b1 100755 --- a/.ci/verify_package_management_service.sh +++ b/.ci/verify_package_management_service.sh @@ -31,8 +31,8 @@ source ${PULSAR_HOME}/.ci/helm.sh ci::upload_java_package ci::verify_java_package -ci::upload_python_package -ci::verify_python_package +#ci::upload_python_package +#ci::verify_python_package -ci::upload_go_package -ci::verify_go_package \ No newline at end of file +#ci::upload_go_package +#ci::verify_go_package \ No newline at end of file From 3a2828de84ee758321290d9ea467352c16ba1a11 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 25 Aug 2021 13:38:37 +0800 Subject: [PATCH 04/18] fix CI --- .ci/helm.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.ci/helm.sh b/.ci/helm.sh index 3ae579879..a5c2d3b11 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -357,9 +357,8 @@ function ci::upload_java_package() { } function ci::verify_java_package() { - RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --jar function://public/default/java-function@1.0 --name package-java-fn --className org.apache.pulsar.functions.api.examples.ExclamationFunction --inputs persistent://public/default/package-java-fn-input --cpu 0.1) + ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --jar function://public/default/java-function@1.0 --name package-java-fn --className org.apache.pulsar.functions.api.examples.ExclamationFunction --inputs persistent://public/default/package-java-fn-input --cpu 0.1 sleep 15 - echo "${RET}" ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 sleep 15 ${KUBECTL} get pods -A From 52aa6c2c551c4c32a350b47743211a0b5cee5346 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 25 Aug 2021 14:19:21 +0800 Subject: [PATCH 05/18] fix CI --- .../main/java/io/functionmesh/compute/util/FunctionsUtil.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java b/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java index 08da59231..981d69d54 100644 --- a/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java +++ b/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java @@ -564,6 +564,9 @@ private static File downloadPackageFile(MeshWorkerService worker, String package // use the Nar extraction directory as a temporary directory for downloaded files tempDirectory = Paths.get(worker.getWorkerConfig().getNarExtractionDirectory()); } + if (Files.notExists(tempDirectory)) { + Files.createDirectories(tempDirectory); + } File file = Files.createTempFile(tempDirectory, "function", ".tmp").toFile(); worker.getBrokerAdmin().packages().download(packageName, file.toString()); return file; From d59a61181f8e62186901f48432da927c518c6a76 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 25 Aug 2021 15:30:32 +0800 Subject: [PATCH 06/18] fix download --- .../java/io/functionmesh/compute/util/FunctionsUtil.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java b/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java index 981d69d54..9dc89a320 100644 --- a/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java +++ b/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java @@ -35,6 +35,7 @@ import io.functionmesh.compute.models.CustomRuntimeOptions; import io.kubernetes.client.custom.Quantity; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -568,7 +569,10 @@ private static File downloadPackageFile(MeshWorkerService worker, String package Files.createDirectories(tempDirectory); } File file = Files.createTempFile(tempDirectory, "function", ".tmp").toFile(); - worker.getBrokerAdmin().packages().download(packageName, file.toString()); + String fileName = String.format("function-%s.tmp", RandomStringUtils.random(5, true, true).toLowerCase()); + Path filePath = Paths.get(tempDirectory.toString(), fileName); + Files.deleteIfExists(filePath); + worker.getBrokerAdmin().packages().download(packageName, filePath.toString()); return file; } From e5cf88427f1aee00c0eda5bfa1ce9da8e11eb613 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 25 Aug 2021 16:35:41 +0800 Subject: [PATCH 07/18] fix extractType --- .../java/io/functionmesh/compute/util/FunctionsUtil.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java b/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java index 9dc89a320..5524e1df7 100644 --- a/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java +++ b/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java @@ -268,7 +268,7 @@ public static V1alpha1Function createV1alpha1FunctionFromFunctionConfig(String k } Class[] typeArgs = null; if (componentPackageFile != null) { - typeArgs = extractTypeArgs(functionConfig, componentPackageFile); + typeArgs = extractTypeArgs(functionConfig, componentPackageFile, worker.getWorkerConfig().isForwardSourceMessageProperty()); } if (StringUtils.isNotEmpty(functionConfig.getJar())) { V1alpha1FunctionSpecJava v1alpha1FunctionSpecJava = new V1alpha1FunctionSpecJava(); @@ -577,8 +577,11 @@ private static File downloadPackageFile(MeshWorkerService worker, String package } private static Class[] extractTypeArgs(final FunctionConfig functionConfig, - final File componentPackageFile) { + final File componentPackageFile, + final boolean isForwardSourceMessageProperty) { Class[] typeArgs = null; + FunctionConfigUtils.inferMissingArguments( + functionConfig, isForwardSourceMessageProperty); if (componentPackageFile == null) { return null; } From 5ef15b914c901c99ab5ce3eae7a663495a7a10cc Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 25 Aug 2021 17:11:12 +0800 Subject: [PATCH 08/18] fix download --- .../main/java/io/functionmesh/compute/util/FunctionsUtil.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java b/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java index 5524e1df7..edb2b329f 100644 --- a/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java +++ b/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java @@ -568,12 +568,11 @@ private static File downloadPackageFile(MeshWorkerService worker, String package if (Files.notExists(tempDirectory)) { Files.createDirectories(tempDirectory); } - File file = Files.createTempFile(tempDirectory, "function", ".tmp").toFile(); String fileName = String.format("function-%s.tmp", RandomStringUtils.random(5, true, true).toLowerCase()); Path filePath = Paths.get(tempDirectory.toString(), fileName); Files.deleteIfExists(filePath); worker.getBrokerAdmin().packages().download(packageName, filePath.toString()); - return file; + return filePath.toFile(); } private static Class[] extractTypeArgs(final FunctionConfig functionConfig, From f8b4b2df77e2fd3ebfc12a6523f7d55db7a62762 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Fri, 27 Aug 2021 08:41:17 +0800 Subject: [PATCH 09/18] bump to 2.8.0.14 to verify package management service --- .ci/clusters/compute_v1alpha1_function.yaml | 2 +- ...compute_v1alpha1_function_builtin_hpa.yaml | 2 +- .../compute_v1alpha1_function_hpa.yaml | 2 +- .../compute_v1alpha1_functionmesh.yaml | 6 +++--- .../compute_v1alpha1_go_function.yaml | 2 +- .../compute_v1alpha1_py_function.yaml | 2 +- .ci/clusters/values.yaml | 16 +++++++-------- .ci/clusters/values_cluster.yaml | 16 +++++++-------- .ci/clusters/values_custom_runner_images.yaml | 20 +++++++++---------- .ci/clusters/values_runner_images.yaml | 20 +++++++++---------- .github/workflows/test-e2e-crypto.yml | 2 +- .../test-function-key-based-batcher.yml | 2 +- .github/workflows/test-function-runner.yml | 2 +- .../test-integration-kind-samples.yml | 4 ++-- controllers/spec/common.go | 2 +- .../integration-tests/docker/Dockerfile | 2 +- .../integration-tests/docker/connectors.yaml | 4 ++-- mesh-worker-service/pom.xml | 2 +- 18 files changed, 54 insertions(+), 54 deletions(-) diff --git a/.ci/clusters/compute_v1alpha1_function.yaml b/.ci/clusters/compute_v1alpha1_function.yaml index dc4096ed2..8a7b5441a 100644 --- a/.ci/clusters/compute_v1alpha1_function.yaml +++ b/.ci/clusters/compute_v1alpha1_function.yaml @@ -4,7 +4,7 @@ metadata: name: function-sample namespace: default spec: - image: streamnative/pulsar-functions-java-sample:2.8.0.13 + image: streamnative/pulsar-functions-java-sample:2.8.0.14 className: org.apache.pulsar.functions.api.examples.ExclamationFunction forwardSourceMessageProperty: true MaxPendingAsyncRequests: 1000 diff --git a/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml b/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml index 459f558ea..67cc68b8b 100644 --- a/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml +++ b/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml @@ -4,7 +4,7 @@ metadata: name: function-builtin-hpa-sample namespace: default spec: - image: streamnative/pulsar-functions-java-sample:2.8.0.13 + image: streamnative/pulsar-functions-java-sample:2.8.0.14 className: org.apache.pulsar.functions.api.examples.ExclamationFunction forwardSourceMessageProperty: true MaxPendingAsyncRequests: 1000 diff --git a/.ci/clusters/compute_v1alpha1_function_hpa.yaml b/.ci/clusters/compute_v1alpha1_function_hpa.yaml index 0ac7c6cfb..7d1d0bdbb 100644 --- a/.ci/clusters/compute_v1alpha1_function_hpa.yaml +++ b/.ci/clusters/compute_v1alpha1_function_hpa.yaml @@ -4,7 +4,7 @@ metadata: name: function-hpa-sample namespace: default spec: - image: streamnative/pulsar-functions-java-sample:2.8.0.13 + image: streamnative/pulsar-functions-java-sample:2.8.0.14 className: org.apache.pulsar.functions.api.examples.ExclamationFunction forwardSourceMessageProperty: true MaxPendingAsyncRequests: 1000 diff --git a/.ci/clusters/compute_v1alpha1_functionmesh.yaml b/.ci/clusters/compute_v1alpha1_functionmesh.yaml index 5e693258d..54d15b7b9 100644 --- a/.ci/clusters/compute_v1alpha1_functionmesh.yaml +++ b/.ci/clusters/compute_v1alpha1_functionmesh.yaml @@ -5,7 +5,7 @@ metadata: spec: functions: - name: java-function - image: streamnative/pulsar-functions-java-sample:2.8.0.13 + image: streamnative/pulsar-functions-java-sample:2.8.0.14 className: org.apache.pulsar.functions.api.examples.ExclamationFunction replicas: 1 maxReplicas: 1 @@ -31,7 +31,7 @@ spec: memory: 1.1G clusterName: test - name: golang-function - image: streamnative/pulsar-functions-go-sample:2.8.0.13 + image: streamnative/pulsar-functions-go-sample:2.8.0.14 replicas: 1 maxReplicas: 1 input: @@ -56,7 +56,7 @@ spec: memory: 1.1G clusterName: test - name: python-function - image: streamnative/pulsar-functions-python-sample:2.8.0.13 + image: streamnative/pulsar-functions-python-sample:2.8.0.14 className: exclamation_function.ExclamationFunction replicas: 1 maxReplicas: 1 diff --git a/.ci/clusters/compute_v1alpha1_go_function.yaml b/.ci/clusters/compute_v1alpha1_go_function.yaml index 29ba59339..6d6a22a86 100644 --- a/.ci/clusters/compute_v1alpha1_go_function.yaml +++ b/.ci/clusters/compute_v1alpha1_go_function.yaml @@ -4,7 +4,7 @@ metadata: name: go-function-sample namespace: default spec: - image: streamnative/pulsar-functions-go-sample:2.8.0.13 + image: streamnative/pulsar-functions-go-sample:2.8.0.14 forwardSourceMessageProperty: true MaxPendingAsyncRequests: 1000 replicas: 1 diff --git a/.ci/clusters/compute_v1alpha1_py_function.yaml b/.ci/clusters/compute_v1alpha1_py_function.yaml index ed9c5e83f..4ad156b38 100644 --- a/.ci/clusters/compute_v1alpha1_py_function.yaml +++ b/.ci/clusters/compute_v1alpha1_py_function.yaml @@ -4,7 +4,7 @@ metadata: name: py-function-sample namespace: default spec: - image: streamnative/pulsar-functions-python-sample:2.8.0.13 + image: streamnative/pulsar-functions-python-sample:2.8.0.14 className: exclamation_function.ExclamationFunction forwardSourceMessageProperty: true MaxPendingAsyncRequests: 1000 diff --git a/.ci/clusters/values.yaml b/.ci/clusters/values.yaml index 3d879fcbe..f181c77c6 100644 --- a/.ci/clusters/values.yaml +++ b/.ci/clusters/values.yaml @@ -42,22 +42,22 @@ monitoring: images: zookeeper: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 bookie: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 autorecovery: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 broker: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 functions: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 proxy: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 zookeeper: replicaCount: 1 @@ -67,7 +67,7 @@ bookkeeper: metadata: image: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 configData: # `BOOKIE_MEM` is used for `bookie shell` BOOKIE_MEM: > @@ -97,7 +97,7 @@ bookkeeper: pulsar_metadata: image: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 broker: replicaCount: 1 diff --git a/.ci/clusters/values_cluster.yaml b/.ci/clusters/values_cluster.yaml index 3d879fcbe..f181c77c6 100644 --- a/.ci/clusters/values_cluster.yaml +++ b/.ci/clusters/values_cluster.yaml @@ -42,22 +42,22 @@ monitoring: images: zookeeper: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 bookie: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 autorecovery: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 broker: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 functions: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 proxy: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 zookeeper: replicaCount: 1 @@ -67,7 +67,7 @@ bookkeeper: metadata: image: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 configData: # `BOOKIE_MEM` is used for `bookie shell` BOOKIE_MEM: > @@ -97,7 +97,7 @@ bookkeeper: pulsar_metadata: image: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 broker: replicaCount: 1 diff --git a/.ci/clusters/values_custom_runner_images.yaml b/.ci/clusters/values_custom_runner_images.yaml index 569d1c420..3e2fb4bfe 100644 --- a/.ci/clusters/values_custom_runner_images.yaml +++ b/.ci/clusters/values_custom_runner_images.yaml @@ -41,27 +41,27 @@ monitoring: images: zookeeper: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent bookie: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent autorecovery: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent broker: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent proxy: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent functions: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent zookeeper: @@ -102,7 +102,7 @@ bookkeeper: pulsar_metadata: image: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent broker: @@ -148,9 +148,9 @@ functions: narExtractionDirectory: "" functionRuntimeFactoryConfigs: functionDockerImages: - JAVA: "streamnative/pulsar-functions-java-runner:2.8.0.13" - PYTHON: "streamnative/pulsar-functions-python-runner:2.8.0.13" - GO: "streamnative/pulsar-functions-go-runner:2.8.0.13" + JAVA: "streamnative/pulsar-functions-java-runner:2.8.0.14" + PYTHON: "streamnative/pulsar-functions-python-runner:2.8.0.14" + GO: "streamnative/pulsar-functions-go-runner:2.8.0.14" proxy: replicaCount: 1 diff --git a/.ci/clusters/values_runner_images.yaml b/.ci/clusters/values_runner_images.yaml index 80ed7afbd..e21035935 100644 --- a/.ci/clusters/values_runner_images.yaml +++ b/.ci/clusters/values_runner_images.yaml @@ -40,27 +40,27 @@ monitoring: images: zookeeper: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent bookie: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent autorecovery: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent broker: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent proxy: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent functions: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent zookeeper: @@ -101,7 +101,7 @@ bookkeeper: pulsar_metadata: image: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent broker: @@ -161,9 +161,9 @@ functions: disk: 1048576000 functionRuntimeFactoryConfigs: functionDockerImages: - JAVA: "streamnative/pulsar-functions-java-runner:2.8.0.13" - PYTHON: "streamnative/pulsar-functions-python-runner:2.8.0.13" - GO: "streamnative/pulsar-functions-go-runner:2.8.0.13" + JAVA: "streamnative/pulsar-functions-java-runner:2.8.0.14" + PYTHON: "streamnative/pulsar-functions-python-runner:2.8.0.14" + GO: "streamnative/pulsar-functions-go-runner:2.8.0.14" functionInstanceMinResources: cpu: 0.1 ram: 10485760 diff --git a/.github/workflows/test-e2e-crypto.yml b/.github/workflows/test-e2e-crypto.yml index c912042c9..191f9cc14 100644 --- a/.github/workflows/test-e2e-crypto.yml +++ b/.github/workflows/test-e2e-crypto.yml @@ -34,7 +34,7 @@ jobs: - name: Build runner images run: | - PULSAR_IMAGE_TAG=2.8.0.13 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh - name: Install operator-sdk run: | diff --git a/.github/workflows/test-function-key-based-batcher.yml b/.github/workflows/test-function-key-based-batcher.yml index 6447b1de1..00e5b4edb 100644 --- a/.github/workflows/test-function-key-based-batcher.yml +++ b/.github/workflows/test-function-key-based-batcher.yml @@ -34,7 +34,7 @@ jobs: - name: Build runner images run: | - PULSAR_IMAGE_TAG=2.8.0.13 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh - name: Install operator-sdk run: | diff --git a/.github/workflows/test-function-runner.yml b/.github/workflows/test-function-runner.yml index d0bde6711..082b55c77 100644 --- a/.github/workflows/test-function-runner.yml +++ b/.github/workflows/test-function-runner.yml @@ -34,7 +34,7 @@ jobs: - name: Build runner images run: | - PULSAR_IMAGE_TAG=2.8.0.13 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh - name: Verify function runner run: | diff --git a/.github/workflows/test-integration-kind-samples.yml b/.github/workflows/test-integration-kind-samples.yml index 7ad65542b..6e2b6bc14 100644 --- a/.github/workflows/test-integration-kind-samples.yml +++ b/.github/workflows/test-integration-kind-samples.yml @@ -35,8 +35,8 @@ jobs: - name: Build runner images run: | - PULSAR_IMAGE_TAG=2.8.0.13 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh - PULSAR_IMAGE_TAG=2.8.0.13 KIND_PUSH=true images/samples/build.sh + PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=2.8.0.14 KIND_PUSH=true images/samples/build.sh - name: Install operator-sdk run: | diff --git a/controllers/spec/common.go b/controllers/spec/common.go index cba687f5b..193dc0ff7 100644 --- a/controllers/spec/common.go +++ b/controllers/spec/common.go @@ -35,7 +35,7 @@ import ( const ( EnvShardID = "SHARD_ID" FunctionsInstanceClasspath = "pulsar.functions.instance.classpath" - DefaultRunnerTag = "2.8.0.13" + DefaultRunnerTag = "2.8.0.14" DefaultRunnerPrefix = "streamnative/" DefaultRunnerImage = DefaultRunnerPrefix + "pulsar-all:" + DefaultRunnerTag DefaultJavaRunnerImage = DefaultRunnerPrefix + "pulsar-functions-java-runner:" + DefaultRunnerTag diff --git a/mesh-worker-service/integration-tests/docker/Dockerfile b/mesh-worker-service/integration-tests/docker/Dockerfile index 6f6f22ad7..e0ee4ce75 100644 --- a/mesh-worker-service/integration-tests/docker/Dockerfile +++ b/mesh-worker-service/integration-tests/docker/Dockerfile @@ -17,7 +17,7 @@ # under the License. # -FROM streamnative/pulsar-all:2.8.0.13 +FROM streamnative/pulsar-all:2.8.0.14 COPY ./target/mesh-worker-service*.nar /pulsar/mesh-worker-service.nar COPY ./integration-tests/docker/connectors.yaml /pulsar/conf/connectors.yaml RUN mkdir -p /pulsar-nar diff --git a/mesh-worker-service/integration-tests/docker/connectors.yaml b/mesh-worker-service/integration-tests/docker/connectors.yaml index b4df87cf0..d16fdb043 100644 --- a/mesh-worker-service/integration-tests/docker/connectors.yaml +++ b/mesh-worker-service/integration-tests/docker/connectors.yaml @@ -24,6 +24,6 @@ sourceConfigClass: org.apache.pulsar.io.datagenerator.DataGeneratorSourceConfig sinkClass: org.apache.pulsar.io.datagenerator.DataGeneratorPrintSink imageRepository: streamnative/pulsar-io-data-generator - version: 2.8.0.13 - imageTag: 2.8.0.13 + version: 2.8.0.14 + imageTag: 2.8.0.14 typeClassName: org.apache.pulsar.client.impl.schema.JSONSchema diff --git a/mesh-worker-service/pom.xml b/mesh-worker-service/pom.xml index 764798b6e..a14240887 100644 --- a/mesh-worker-service/pom.xml +++ b/mesh-worker-service/pom.xml @@ -29,7 +29,7 @@ v0.1.7-rc3 - 2.8.0.13 + 2.8.0.14 1.18.16 2.14.0 12.0.1 From fb597d62ab3ebaae1aa140518a277f0d2092b54b Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Fri, 27 Aug 2021 09:03:28 +0800 Subject: [PATCH 10/18] fix python runner --- images/pulsar-functions-python-runner/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/images/pulsar-functions-python-runner/Dockerfile b/images/pulsar-functions-python-runner/Dockerfile index bb38a6f8a..b11692583 100644 --- a/images/pulsar-functions-python-runner/Dockerfile +++ b/images/pulsar-functions-python-runner/Dockerfile @@ -15,15 +15,15 @@ RUN if [ -d "/tmp/pulsar/cpp-client" ]; then mv /tmp/pulsar/cpp-client /pulsar/c # Install some utilities RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y python3.7 python3.7-dev python3-setuptools python3-yaml python3-kazoo \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y python3 python3-dev python3-setuptools python3-yaml python3-kazoo \ libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \ curl \ && apt-get clean autoclean && apt-get autoremove --yes && rm -rf /var/lib/apt/lists/* RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -RUN python3.7 get-pip.py +RUN python3 get-pip.py -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 10 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 RUN if [ -d "/pulsar/cpp-client" ]; then apt-get update \ && apt install -y /pulsar/cpp-client/*.deb \ @@ -32,3 +32,4 @@ RUN if [ -d "/pulsar/cpp-client" ]; then apt-get update \ WORKDIR /pulsar RUN if [ -f "/pulsar/bin/install-pulsar-client-37.sh" ]; then /pulsar/bin/install-pulsar-client-37.sh ; fi +RUN if [ -f "/pulsar/bin/install-pulsar-client.sh" ]; then /pulsar/bin/install-pulsar-client.sh ; fi From 116e3c9e3944b80cc13b6a3a256afb65f1ad905b Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Fri, 27 Aug 2021 09:17:57 +0800 Subject: [PATCH 11/18] fix runner images --- images/pulsar-functions-base-runner/Dockerfile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/images/pulsar-functions-base-runner/Dockerfile b/images/pulsar-functions-base-runner/Dockerfile index 69a400ad6..30b143c46 100644 --- a/images/pulsar-functions-base-runner/Dockerfile +++ b/images/pulsar-functions-base-runner/Dockerfile @@ -1,7 +1,7 @@ ARG PULSAR_IMAGE ARG PULSAR_IMAGE_TAG FROM ${PULSAR_IMAGE}:${PULSAR_IMAGE_TAG} as pulsar -FROM openjdk:11-jre-slim as functions-runner +FROM ubuntu:20.04 as functions-runner RUN mkdir -p /pulsar/bin/ RUN mkdir -p /pulsar/lib/ @@ -18,6 +18,14 @@ COPY --from=pulsar /pulsar/lib /pulsar/lib RUN chmod -R g=u /pulsar +RUN apt-get update \ + && apt-get -y dist-upgrade \ + && apt-get -y install openjdk-11-jdk-headless netcat dnsutils less procps iputils-ping \ + && apt-get -y --purge autoremove \ + && apt-get autoclean \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE WORKDIR /pulsar \ No newline at end of file From 2577b130cd19d9765fbc668db7e924ded6b15155 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Fri, 27 Aug 2021 10:09:12 +0800 Subject: [PATCH 12/18] add python and go test --- .ci/verify_package_management_service.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/verify_package_management_service.sh b/.ci/verify_package_management_service.sh index c0ef5d6b1..f121c6eb2 100755 --- a/.ci/verify_package_management_service.sh +++ b/.ci/verify_package_management_service.sh @@ -31,8 +31,8 @@ source ${PULSAR_HOME}/.ci/helm.sh ci::upload_java_package ci::verify_java_package -#ci::upload_python_package -#ci::verify_python_package +ci::upload_python_package +ci::verify_python_package -#ci::upload_go_package -#ci::verify_go_package \ No newline at end of file +ci::upload_go_package +ci::verify_go_package From b2cacc68c513bdcac020ab2e3a46b2870e02403e Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Fri, 27 Aug 2021 10:54:30 +0800 Subject: [PATCH 13/18] push images --- .github/workflows/test-integration-mesh-worker-service.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-integration-mesh-worker-service.yml b/.github/workflows/test-integration-mesh-worker-service.yml index 73bcacc60..d4303fda9 100644 --- a/.github/workflows/test-integration-mesh-worker-service.yml +++ b/.github/workflows/test-integration-mesh-worker-service.yml @@ -78,6 +78,10 @@ jobs: make install nohup make run & + - name: Build runner images + run: | + PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh + - name: verify mesh-worker-service run: | .ci/verify_mesh_worker_service.sh From c5fc2407dbad1bc5c06993c13cba33af6cbef59f Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Fri, 27 Aug 2021 15:40:51 +0800 Subject: [PATCH 14/18] fix CI --- .ci/helm.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.ci/helm.sh b/.ci/helm.sh index a5c2d3b11..4df29c747 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -286,6 +286,7 @@ function ci::verify_mesh_worker_service_pulsar_admin() { fi ${KUBECTL} get pods -n ${NAMESPACE} sleep 120 + ${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-source" WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-sink" | wc -l) while [[ ${WC} -lt 1 ]]; do echo ${WC}; @@ -293,6 +294,7 @@ function ci::verify_mesh_worker_service_pulsar_admin() { ${KUBECTL} get pods -n ${NAMESPACE} WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-sink" | wc -l) done + ${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-source" WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-source" | wc -l) while [[ ${WC} -lt 1 ]]; do echo ${WC}; @@ -300,6 +302,7 @@ function ci::verify_mesh_worker_service_pulsar_admin() { ${KUBECTL} get pods -n ${NAMESPACE} WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-source" | wc -l) done + sleep 120 ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin sinks status --name data-generator-sink RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin sinks status --name data-generator-sink) if [[ $RET != *"true"* ]]; then From e7eb6ddb25c2a2bd08ef43509b901f2a33d2c926 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Sun, 29 Aug 2021 19:51:41 +0800 Subject: [PATCH 15/18] fix CI --- .ci/helm.sh | 4 +++- .ci/verify_package_management_service.sh | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.ci/helm.sh b/.ci/helm.sh index 4df29c747..fe91a417c 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -360,8 +360,10 @@ function ci::upload_java_package() { } function ci::verify_java_package() { - ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --jar function://public/default/java-function@1.0 --name package-java-fn --className org.apache.pulsar.functions.api.examples.ExclamationFunction --inputs persistent://public/default/package-java-fn-input --cpu 0.1 + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --jar function://public/default/java-function@1.0 --name package-java-fn --className org.apache.pulsar.functions.api.examples.ExclamationFunction --inputs persistent://public/default/package-java-fn-input --cpu 0.1) + ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 sleep 15 + echo $RET ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 sleep 15 ${KUBECTL} get pods -A diff --git a/.ci/verify_package_management_service.sh b/.ci/verify_package_management_service.sh index f121c6eb2..095145758 100755 --- a/.ci/verify_package_management_service.sh +++ b/.ci/verify_package_management_service.sh @@ -28,6 +28,9 @@ FUNCTION=${FUNCTION:-"false"} source ${PULSAR_HOME}/.ci/helm.sh +ci::ensure_mesh_worker_service_role +ci::ensure_function_mesh_config + ci::upload_java_package ci::verify_java_package From 9efbd3d3b9d4f699e84f057bfd94494c0ab824dd Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Sun, 29 Aug 2021 20:59:52 +0800 Subject: [PATCH 16/18] fix CI --- .ci/verify_package_management_service.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/.ci/verify_package_management_service.sh b/.ci/verify_package_management_service.sh index 095145758..f121c6eb2 100755 --- a/.ci/verify_package_management_service.sh +++ b/.ci/verify_package_management_service.sh @@ -28,9 +28,6 @@ FUNCTION=${FUNCTION:-"false"} source ${PULSAR_HOME}/.ci/helm.sh -ci::ensure_mesh_worker_service_role -ci::ensure_function_mesh_config - ci::upload_java_package ci::verify_java_package From 1ee763e77b37747ab17bcea91504c82d2bd169f7 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Mon, 30 Aug 2021 08:55:08 +0800 Subject: [PATCH 17/18] fix CI --- .ci/helm.sh | 5 +++++ mesh-worker-service/integration-tests/docker/Dockerfile | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.ci/helm.sh b/.ci/helm.sh index fe91a417c..e612511a5 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -347,14 +347,19 @@ function ci::verify_mesh_worker_service_pulsar_admin() { } function ci::upload_java_package() { + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages upload function://public/default/java-function@1.0 --path /pulsar/examples/api-examples.jar --description java-function@1.0) if [[ $RET != *"successfully"* ]]; then echo "${RET}" + ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 + sleep 60 return 1 fi RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages download function://public/default/java-function@1.0 --path /pulsar/api-examples.jar) if [[ $RET != *"successfully"* ]]; then echo "${RET}" + ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 + sleep 60 return 1 fi } diff --git a/mesh-worker-service/integration-tests/docker/Dockerfile b/mesh-worker-service/integration-tests/docker/Dockerfile index e0ee4ce75..1c7b7c229 100644 --- a/mesh-worker-service/integration-tests/docker/Dockerfile +++ b/mesh-worker-service/integration-tests/docker/Dockerfile @@ -17,7 +17,7 @@ # under the License. # -FROM streamnative/pulsar-all:2.8.0.14 +FROM streamnative/pulsar:2.8.0.14 COPY ./target/mesh-worker-service*.nar /pulsar/mesh-worker-service.nar COPY ./integration-tests/docker/connectors.yaml /pulsar/conf/connectors.yaml RUN mkdir -p /pulsar-nar From 6a662494db43e7b1255d88c8e45b62fe381cb228 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Mon, 30 Aug 2021 09:39:21 +0800 Subject: [PATCH 18/18] fix go tests --- .ci/helm.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/helm.sh b/.ci/helm.sh index e612511a5..a588b59ca 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -419,7 +419,9 @@ function ci::verify_python_package() { function ci::upload_go_package() { ${KUBECTL} cp "${FUNCTION_MESH_HOME}/.ci/examples/go-examples" "${NAMESPACE}/${CLUSTER}-pulsar-broker-0:/pulsar/examples" sleep 1 - RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages upload function://public/default/go-function@1.0 --path /pulsar/examples/go-examples --description go-function@1.0) + ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- ls -l /pulsar/examples/go-examples + sleep 1 + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages upload function://public/default/go-function@1.0 --path /pulsar/examples/go-examples/exclamationFunc --description go-function@1.0) if [[ $RET != *"successfully"* ]]; then echo "${RET}" return 1