diff --git a/.ci/clusters/compute_v1alpha1_function.yaml b/.ci/clusters/compute_v1alpha1_function.yaml index dc4096ed2..8a7b5441a 100644 --- a/.ci/clusters/compute_v1alpha1_function.yaml +++ b/.ci/clusters/compute_v1alpha1_function.yaml @@ -4,7 +4,7 @@ metadata: name: function-sample namespace: default spec: - image: streamnative/pulsar-functions-java-sample:2.8.0.13 + image: streamnative/pulsar-functions-java-sample:2.8.0.14 className: org.apache.pulsar.functions.api.examples.ExclamationFunction forwardSourceMessageProperty: true MaxPendingAsyncRequests: 1000 diff --git a/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml b/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml index 459f558ea..67cc68b8b 100644 --- a/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml +++ b/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml @@ -4,7 +4,7 @@ metadata: name: function-builtin-hpa-sample namespace: default spec: - image: streamnative/pulsar-functions-java-sample:2.8.0.13 + image: streamnative/pulsar-functions-java-sample:2.8.0.14 className: org.apache.pulsar.functions.api.examples.ExclamationFunction forwardSourceMessageProperty: true MaxPendingAsyncRequests: 1000 diff --git a/.ci/clusters/compute_v1alpha1_function_hpa.yaml b/.ci/clusters/compute_v1alpha1_function_hpa.yaml index 0ac7c6cfb..7d1d0bdbb 100644 --- a/.ci/clusters/compute_v1alpha1_function_hpa.yaml +++ b/.ci/clusters/compute_v1alpha1_function_hpa.yaml @@ -4,7 +4,7 @@ metadata: name: function-hpa-sample namespace: default spec: - image: streamnative/pulsar-functions-java-sample:2.8.0.13 + image: streamnative/pulsar-functions-java-sample:2.8.0.14 className: org.apache.pulsar.functions.api.examples.ExclamationFunction forwardSourceMessageProperty: true MaxPendingAsyncRequests: 1000 diff --git a/.ci/clusters/compute_v1alpha1_functionmesh.yaml b/.ci/clusters/compute_v1alpha1_functionmesh.yaml index 5e693258d..54d15b7b9 100644 --- a/.ci/clusters/compute_v1alpha1_functionmesh.yaml +++ b/.ci/clusters/compute_v1alpha1_functionmesh.yaml @@ -5,7 +5,7 @@ metadata: spec: functions: - name: java-function - image: streamnative/pulsar-functions-java-sample:2.8.0.13 + image: streamnative/pulsar-functions-java-sample:2.8.0.14 className: org.apache.pulsar.functions.api.examples.ExclamationFunction replicas: 1 maxReplicas: 1 @@ -31,7 +31,7 @@ spec: memory: 1.1G clusterName: test - name: golang-function - image: streamnative/pulsar-functions-go-sample:2.8.0.13 + image: streamnative/pulsar-functions-go-sample:2.8.0.14 replicas: 1 maxReplicas: 1 input: @@ -56,7 +56,7 @@ spec: memory: 1.1G clusterName: test - name: python-function - image: streamnative/pulsar-functions-python-sample:2.8.0.13 + image: streamnative/pulsar-functions-python-sample:2.8.0.14 className: exclamation_function.ExclamationFunction replicas: 1 maxReplicas: 1 diff --git a/.ci/clusters/compute_v1alpha1_go_function.yaml b/.ci/clusters/compute_v1alpha1_go_function.yaml index 29ba59339..6d6a22a86 100644 --- a/.ci/clusters/compute_v1alpha1_go_function.yaml +++ b/.ci/clusters/compute_v1alpha1_go_function.yaml @@ -4,7 +4,7 @@ metadata: name: go-function-sample namespace: default spec: - image: streamnative/pulsar-functions-go-sample:2.8.0.13 + image: streamnative/pulsar-functions-go-sample:2.8.0.14 forwardSourceMessageProperty: true MaxPendingAsyncRequests: 1000 replicas: 1 diff --git a/.ci/clusters/compute_v1alpha1_py_function.yaml b/.ci/clusters/compute_v1alpha1_py_function.yaml index ed9c5e83f..4ad156b38 100644 --- a/.ci/clusters/compute_v1alpha1_py_function.yaml +++ b/.ci/clusters/compute_v1alpha1_py_function.yaml @@ -4,7 +4,7 @@ metadata: name: py-function-sample namespace: default spec: - image: streamnative/pulsar-functions-python-sample:2.8.0.13 + image: streamnative/pulsar-functions-python-sample:2.8.0.14 className: exclamation_function.ExclamationFunction forwardSourceMessageProperty: true MaxPendingAsyncRequests: 1000 diff --git a/.ci/clusters/values.yaml b/.ci/clusters/values.yaml index 3d879fcbe..f181c77c6 100644 --- a/.ci/clusters/values.yaml +++ b/.ci/clusters/values.yaml @@ -42,22 +42,22 @@ monitoring: images: zookeeper: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 bookie: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 autorecovery: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 broker: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 functions: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 proxy: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 zookeeper: replicaCount: 1 @@ -67,7 +67,7 @@ bookkeeper: metadata: image: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 configData: # `BOOKIE_MEM` is used for `bookie shell` BOOKIE_MEM: > @@ -97,7 +97,7 @@ bookkeeper: pulsar_metadata: image: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 broker: replicaCount: 1 diff --git a/.ci/clusters/values_cluster.yaml b/.ci/clusters/values_cluster.yaml index 3d879fcbe..f181c77c6 100644 --- a/.ci/clusters/values_cluster.yaml +++ b/.ci/clusters/values_cluster.yaml @@ -42,22 +42,22 @@ monitoring: images: zookeeper: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 bookie: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 autorecovery: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 broker: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 functions: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 proxy: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 zookeeper: replicaCount: 1 @@ -67,7 +67,7 @@ bookkeeper: metadata: image: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 configData: # `BOOKIE_MEM` is used for `bookie shell` BOOKIE_MEM: > @@ -97,7 +97,7 @@ bookkeeper: pulsar_metadata: image: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 broker: replicaCount: 1 diff --git a/.ci/clusters/values_custom_runner_images.yaml b/.ci/clusters/values_custom_runner_images.yaml index 569d1c420..3e2fb4bfe 100644 --- a/.ci/clusters/values_custom_runner_images.yaml +++ b/.ci/clusters/values_custom_runner_images.yaml @@ -41,27 +41,27 @@ monitoring: images: zookeeper: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent bookie: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent autorecovery: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent broker: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent proxy: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent functions: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent zookeeper: @@ -102,7 +102,7 @@ bookkeeper: pulsar_metadata: image: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent broker: @@ -148,9 +148,9 @@ functions: narExtractionDirectory: "" functionRuntimeFactoryConfigs: functionDockerImages: - JAVA: "streamnative/pulsar-functions-java-runner:2.8.0.13" - PYTHON: "streamnative/pulsar-functions-python-runner:2.8.0.13" - GO: "streamnative/pulsar-functions-go-runner:2.8.0.13" + JAVA: "streamnative/pulsar-functions-java-runner:2.8.0.14" + PYTHON: "streamnative/pulsar-functions-python-runner:2.8.0.14" + GO: "streamnative/pulsar-functions-go-runner:2.8.0.14" proxy: replicaCount: 1 diff --git a/.ci/clusters/values_mesh_worker_service.yaml b/.ci/clusters/values_mesh_worker_service.yaml index a13899f7f..cd1b1691a 100644 --- a/.ci/clusters/values_mesh_worker_service.yaml +++ b/.ci/clusters/values_mesh_worker_service.yaml @@ -113,6 +113,8 @@ broker: managedLedgerDefaultEnsembleSize: "1" managedLedgerDefaultWriteQuorum: "1" managedLedgerDefaultAckQuorum: "1" + enablePackagesManagement: "true" + PULSAR_PREFIX_enablePackagesManagement: "true" proxy: replicaCount: 1 diff --git a/.ci/clusters/values_runner_images.yaml b/.ci/clusters/values_runner_images.yaml index 80ed7afbd..e21035935 100644 --- a/.ci/clusters/values_runner_images.yaml +++ b/.ci/clusters/values_runner_images.yaml @@ -40,27 +40,27 @@ monitoring: images: zookeeper: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent bookie: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent autorecovery: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent broker: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent proxy: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent functions: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent zookeeper: @@ -101,7 +101,7 @@ bookkeeper: pulsar_metadata: image: repository: streamnative/pulsar-all - tag: 2.8.0.13 + tag: 2.8.0.14 pullPolicy: IfNotPresent broker: @@ -161,9 +161,9 @@ functions: disk: 1048576000 functionRuntimeFactoryConfigs: functionDockerImages: - JAVA: "streamnative/pulsar-functions-java-runner:2.8.0.13" - PYTHON: "streamnative/pulsar-functions-python-runner:2.8.0.13" - GO: "streamnative/pulsar-functions-go-runner:2.8.0.13" + JAVA: "streamnative/pulsar-functions-java-runner:2.8.0.14" + PYTHON: "streamnative/pulsar-functions-python-runner:2.8.0.14" + GO: "streamnative/pulsar-functions-go-runner:2.8.0.14" functionInstanceMinResources: cpu: 0.1 ram: 10485760 diff --git a/.ci/helm.sh b/.ci/helm.sh index 6084974e0..a588b59ca 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -286,6 +286,7 @@ function ci::verify_mesh_worker_service_pulsar_admin() { fi ${KUBECTL} get pods -n ${NAMESPACE} sleep 120 + ${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-source" WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-sink" | wc -l) while [[ ${WC} -lt 1 ]]; do echo ${WC}; @@ -293,6 +294,7 @@ function ci::verify_mesh_worker_service_pulsar_admin() { ${KUBECTL} get pods -n ${NAMESPACE} WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-sink" | wc -l) done + ${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-source" WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-source" | wc -l) while [[ ${WC} -lt 1 ]]; do echo ${WC}; @@ -300,6 +302,7 @@ function ci::verify_mesh_worker_service_pulsar_admin() { ${KUBECTL} get pods -n ${NAMESPACE} WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "data-generator-source" | wc -l) done + sleep 120 ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin sinks status --name data-generator-sink RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin sinks status --name data-generator-sink) if [[ $RET != *"true"* ]]; then @@ -342,3 +345,107 @@ function ci::verify_mesh_worker_service_pulsar_admin() { ${KUBECTL} get pods -n ${NAMESPACE} } + +function ci::upload_java_package() { + + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages upload function://public/default/java-function@1.0 --path /pulsar/examples/api-examples.jar --description java-function@1.0) + if [[ $RET != *"successfully"* ]]; then + echo "${RET}" + ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 + sleep 60 + return 1 + fi + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages download function://public/default/java-function@1.0 --path /pulsar/api-examples.jar) + if [[ $RET != *"successfully"* ]]; then + echo "${RET}" + ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 + sleep 60 + return 1 + fi +} + +function ci::verify_java_package() { + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --jar function://public/default/java-function@1.0 --name package-java-fn --className org.apache.pulsar.functions.api.examples.ExclamationFunction --inputs persistent://public/default/package-java-fn-input --cpu 0.1) + ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 + sleep 15 + echo $RET + ${KUBECTL} logs -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 + sleep 15 + ${KUBECTL} get pods -A + sleep 5 + WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-java-fn" | wc -l) + while [[ ${WC} -lt 1 ]]; do + echo ${WC}; + sleep 20 + ${KUBECTL} get pods -n ${NAMESPACE} + ${KUBECTL} describe pod package-java-fn-function-0 + WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-java-fn" | wc -l) + done + echo "java function test done" + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions delete --name package-java-fn) + echo "${RET}" +} + +function ci::upload_python_package() { + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages upload function://public/default/python-function@1.0 --path /pulsar/examples/python-examples/exclamation_function.py --description python-function@1.0) + if [[ $RET != *"successfully"* ]]; then + echo "${RET}" + return 1 + fi +} + +function ci::verify_python_package() { + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --py function://public/default/python-function@1.0 --name package-python-fn --classname exclamation_function.ExclamationFunction --inputs persistent://public/default/package-python-fn-input --cpu 0.1) + if [[ $RET != *"successfully"* ]]; then + echo "${RET}" + return 1 + fi + sleep 15 + ${KUBECTL} get pods -A + sleep 5 + WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-python-fn" | wc -l) + while [[ ${WC} -lt 1 ]]; do + echo ${WC}; + sleep 20 + ${KUBECTL} get pods -n ${NAMESPACE} + ${KUBECTL} describe pod package-python-fn-function-0 + WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-python-fn" | wc -l) + done + echo "python function test done" + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions delete --name package-python-fn) + echo "${RET}" +} + +function ci::upload_go_package() { + ${KUBECTL} cp "${FUNCTION_MESH_HOME}/.ci/examples/go-examples" "${NAMESPACE}/${CLUSTER}-pulsar-broker-0:/pulsar/examples" + sleep 1 + ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- ls -l /pulsar/examples/go-examples + sleep 1 + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin packages upload function://public/default/go-function@1.0 --path /pulsar/examples/go-examples/exclamationFunc --description go-function@1.0) + if [[ $RET != *"successfully"* ]]; then + echo "${RET}" + return 1 + fi +} + +function ci::verify_go_package() { + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --go function://public/default/go-function@1.0 --name package-go-fn --inputs persistent://public/default/package-go-fn-input --cpu 0.1) + if [[ $RET != *"successfully"* ]]; then + echo "${RET}" + return 1 + fi + sleep 15 + ${KUBECTL} get pods -A + sleep 5 + WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-go-fn" | wc -l) + while [[ ${WC} -lt 1 ]]; do + echo ${WC}; + sleep 20 + ${KUBECTL} get pods -n ${NAMESPACE} + ${KUBECTL} describe pod package-go-fn-function-0 + WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep "package-go-fn" | wc -l) + done + echo "go function test done" + RET=$(${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions delete --name package-go-fn) + echo "${RET}" +} diff --git a/.ci/verify_package_management_service.sh b/.ci/verify_package_management_service.sh new file mode 100755 index 000000000..f121c6eb2 --- /dev/null +++ b/.ci/verify_package_management_service.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +BINDIR=`dirname "$0"` +PULSAR_HOME=`cd ${BINDIR}/..;pwd` +TLS=${TLS:-"false"} +SYMMETRIC=${SYMMETRIC:-"false"} +FUNCTION=${FUNCTION:-"false"} + +source ${PULSAR_HOME}/.ci/helm.sh + +ci::upload_java_package +ci::verify_java_package + +ci::upload_python_package +ci::verify_python_package + +ci::upload_go_package +ci::verify_go_package diff --git a/.github/workflows/test-e2e-crypto.yml b/.github/workflows/test-e2e-crypto.yml index c912042c9..191f9cc14 100644 --- a/.github/workflows/test-e2e-crypto.yml +++ b/.github/workflows/test-e2e-crypto.yml @@ -34,7 +34,7 @@ jobs: - name: Build runner images run: | - PULSAR_IMAGE_TAG=2.8.0.13 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh - name: Install operator-sdk run: | diff --git a/.github/workflows/test-function-key-based-batcher.yml b/.github/workflows/test-function-key-based-batcher.yml index 6447b1de1..00e5b4edb 100644 --- a/.github/workflows/test-function-key-based-batcher.yml +++ b/.github/workflows/test-function-key-based-batcher.yml @@ -34,7 +34,7 @@ jobs: - name: Build runner images run: | - PULSAR_IMAGE_TAG=2.8.0.13 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh - name: Install operator-sdk run: | diff --git a/.github/workflows/test-function-runner.yml b/.github/workflows/test-function-runner.yml index d0bde6711..082b55c77 100644 --- a/.github/workflows/test-function-runner.yml +++ b/.github/workflows/test-function-runner.yml @@ -34,7 +34,7 @@ jobs: - name: Build runner images run: | - PULSAR_IMAGE_TAG=2.8.0.13 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh - name: Verify function runner run: | diff --git a/.github/workflows/test-integration-kind-samples.yml b/.github/workflows/test-integration-kind-samples.yml index 7ad65542b..6e2b6bc14 100644 --- a/.github/workflows/test-integration-kind-samples.yml +++ b/.github/workflows/test-integration-kind-samples.yml @@ -35,8 +35,8 @@ jobs: - name: Build runner images run: | - PULSAR_IMAGE_TAG=2.8.0.13 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh - PULSAR_IMAGE_TAG=2.8.0.13 KIND_PUSH=true images/samples/build.sh + PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=2.8.0.14 KIND_PUSH=true images/samples/build.sh - name: Install operator-sdk run: | diff --git a/.github/workflows/test-integration-mesh-worker-service.yml b/.github/workflows/test-integration-mesh-worker-service.yml index ec063ebbf..d4303fda9 100644 --- a/.github/workflows/test-integration-mesh-worker-service.yml +++ b/.github/workflows/test-integration-mesh-worker-service.yml @@ -78,6 +78,14 @@ jobs: make install nohup make run & + - name: Build runner images + run: | + PULSAR_IMAGE_TAG=2.8.0.14 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh + - name: verify mesh-worker-service run: | .ci/verify_mesh_worker_service.sh + + - name: verify package management service + run: | + .ci/verify_package_management_service.sh diff --git a/controllers/spec/common.go b/controllers/spec/common.go index cba687f5b..193dc0ff7 100644 --- a/controllers/spec/common.go +++ b/controllers/spec/common.go @@ -35,7 +35,7 @@ import ( const ( EnvShardID = "SHARD_ID" FunctionsInstanceClasspath = "pulsar.functions.instance.classpath" - DefaultRunnerTag = "2.8.0.13" + DefaultRunnerTag = "2.8.0.14" DefaultRunnerPrefix = "streamnative/" DefaultRunnerImage = DefaultRunnerPrefix + "pulsar-all:" + DefaultRunnerTag DefaultJavaRunnerImage = DefaultRunnerPrefix + "pulsar-functions-java-runner:" + DefaultRunnerTag diff --git a/images/pulsar-functions-base-runner/Dockerfile b/images/pulsar-functions-base-runner/Dockerfile index 69a400ad6..30b143c46 100644 --- a/images/pulsar-functions-base-runner/Dockerfile +++ b/images/pulsar-functions-base-runner/Dockerfile @@ -1,7 +1,7 @@ ARG PULSAR_IMAGE ARG PULSAR_IMAGE_TAG FROM ${PULSAR_IMAGE}:${PULSAR_IMAGE_TAG} as pulsar -FROM openjdk:11-jre-slim as functions-runner +FROM ubuntu:20.04 as functions-runner RUN mkdir -p /pulsar/bin/ RUN mkdir -p /pulsar/lib/ @@ -18,6 +18,14 @@ COPY --from=pulsar /pulsar/lib /pulsar/lib RUN chmod -R g=u /pulsar +RUN apt-get update \ + && apt-get -y dist-upgrade \ + && apt-get -y install openjdk-11-jdk-headless netcat dnsutils less procps iputils-ping \ + && apt-get -y --purge autoremove \ + && apt-get autoclean \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE WORKDIR /pulsar \ No newline at end of file diff --git a/images/pulsar-functions-python-runner/Dockerfile b/images/pulsar-functions-python-runner/Dockerfile index bb38a6f8a..b11692583 100644 --- a/images/pulsar-functions-python-runner/Dockerfile +++ b/images/pulsar-functions-python-runner/Dockerfile @@ -15,15 +15,15 @@ RUN if [ -d "/tmp/pulsar/cpp-client" ]; then mv /tmp/pulsar/cpp-client /pulsar/c # Install some utilities RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y python3.7 python3.7-dev python3-setuptools python3-yaml python3-kazoo \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y python3 python3-dev python3-setuptools python3-yaml python3-kazoo \ libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \ curl \ && apt-get clean autoclean && apt-get autoremove --yes && rm -rf /var/lib/apt/lists/* RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -RUN python3.7 get-pip.py +RUN python3 get-pip.py -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 10 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 RUN if [ -d "/pulsar/cpp-client" ]; then apt-get update \ && apt install -y /pulsar/cpp-client/*.deb \ @@ -32,3 +32,4 @@ RUN if [ -d "/pulsar/cpp-client" ]; then apt-get update \ WORKDIR /pulsar RUN if [ -f "/pulsar/bin/install-pulsar-client-37.sh" ]; then /pulsar/bin/install-pulsar-client-37.sh ; fi +RUN if [ -f "/pulsar/bin/install-pulsar-client.sh" ]; then /pulsar/bin/install-pulsar-client.sh ; fi diff --git a/mesh-worker-service/integration-tests/docker/Dockerfile b/mesh-worker-service/integration-tests/docker/Dockerfile index 6f6f22ad7..1c7b7c229 100644 --- a/mesh-worker-service/integration-tests/docker/Dockerfile +++ b/mesh-worker-service/integration-tests/docker/Dockerfile @@ -17,7 +17,7 @@ # under the License. # -FROM streamnative/pulsar-all:2.8.0.13 +FROM streamnative/pulsar:2.8.0.14 COPY ./target/mesh-worker-service*.nar /pulsar/mesh-worker-service.nar COPY ./integration-tests/docker/connectors.yaml /pulsar/conf/connectors.yaml RUN mkdir -p /pulsar-nar diff --git a/mesh-worker-service/integration-tests/docker/connectors.yaml b/mesh-worker-service/integration-tests/docker/connectors.yaml index b4df87cf0..d16fdb043 100644 --- a/mesh-worker-service/integration-tests/docker/connectors.yaml +++ b/mesh-worker-service/integration-tests/docker/connectors.yaml @@ -24,6 +24,6 @@ sourceConfigClass: org.apache.pulsar.io.datagenerator.DataGeneratorSourceConfig sinkClass: org.apache.pulsar.io.datagenerator.DataGeneratorPrintSink imageRepository: streamnative/pulsar-io-data-generator - version: 2.8.0.13 - imageTag: 2.8.0.13 + version: 2.8.0.14 + imageTag: 2.8.0.14 typeClassName: org.apache.pulsar.client.impl.schema.JSONSchema diff --git a/mesh-worker-service/pom.xml b/mesh-worker-service/pom.xml index 764798b6e..a14240887 100644 --- a/mesh-worker-service/pom.xml +++ b/mesh-worker-service/pom.xml @@ -29,7 +29,7 @@ v0.1.7-rc3 - 2.8.0.13 + 2.8.0.14 1.18.16 2.14.0 12.0.1 diff --git a/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java b/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java index 08da59231..edb2b329f 100644 --- a/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java +++ b/mesh-worker-service/src/main/java/io/functionmesh/compute/util/FunctionsUtil.java @@ -35,6 +35,7 @@ import io.functionmesh.compute.models.CustomRuntimeOptions; import io.kubernetes.client.custom.Quantity; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -267,7 +268,7 @@ public static V1alpha1Function createV1alpha1FunctionFromFunctionConfig(String k } Class[] typeArgs = null; if (componentPackageFile != null) { - typeArgs = extractTypeArgs(functionConfig, componentPackageFile); + typeArgs = extractTypeArgs(functionConfig, componentPackageFile, worker.getWorkerConfig().isForwardSourceMessageProperty()); } if (StringUtils.isNotEmpty(functionConfig.getJar())) { V1alpha1FunctionSpecJava v1alpha1FunctionSpecJava = new V1alpha1FunctionSpecJava(); @@ -564,14 +565,22 @@ private static File downloadPackageFile(MeshWorkerService worker, String package // use the Nar extraction directory as a temporary directory for downloaded files tempDirectory = Paths.get(worker.getWorkerConfig().getNarExtractionDirectory()); } - File file = Files.createTempFile(tempDirectory, "function", ".tmp").toFile(); - worker.getBrokerAdmin().packages().download(packageName, file.toString()); - return file; + if (Files.notExists(tempDirectory)) { + Files.createDirectories(tempDirectory); + } + String fileName = String.format("function-%s.tmp", RandomStringUtils.random(5, true, true).toLowerCase()); + Path filePath = Paths.get(tempDirectory.toString(), fileName); + Files.deleteIfExists(filePath); + worker.getBrokerAdmin().packages().download(packageName, filePath.toString()); + return filePath.toFile(); } private static Class[] extractTypeArgs(final FunctionConfig functionConfig, - final File componentPackageFile) { + final File componentPackageFile, + final boolean isForwardSourceMessageProperty) { Class[] typeArgs = null; + FunctionConfigUtils.inferMissingArguments( + functionConfig, isForwardSourceMessageProperty); if (componentPackageFile == null) { return null; }