From 259c698bdba20d68917d14591740f151084a1433 Mon Sep 17 00:00:00 2001 From: ran Date: Tue, 14 Sep 2021 08:43:37 +0800 Subject: [PATCH] [Branch-2.7][Build] Use ubuntu 20.04 as docker image base (#12017) ### Motivation Currently, the docker is from `openjdk:8-jdk-slim `, it has a new release a few days ago and the Linux version was changed to `Impish`, it didn't support install `python3.7` by apt-get tool, the minimum Python version is 3.9, so we need to change the python version or use a different Linux release, I cherry-pick some commits from branch master to build docker image from the `ubuntu:20.04`. Mainly related PRs: 1. https://github.com/apache/pulsar/pull/11026 2. https://github.com/apache/pulsar/pull/11623 3. https://github.com/apache/pulsar/pull/11862 ### Modifications Build docker image from the `ubuntu:20.04`. upgrade pulsar-go-client from `0.2.0` to `0.6.0` and rever #9124 from branch-2.7 --- .github/workflows/ci-docker-build.yaml | 90 +++++++++++++++++++ docker/pulsar-standalone/Dockerfile | 13 ++- docker/pulsar/Dockerfile | 27 +++--- docker/pulsar/pom.xml | 4 +- ...-client-37.sh => install-pulsar-client.sh} | 5 +- pulsar-function-go/go.mod | 2 +- pulsar-function-go/pf/instance.go | 11 --- 7 files changed, 119 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/ci-docker-build.yaml rename docker/pulsar/scripts/{install-pulsar-client-37.sh => install-pulsar-client.sh} (79%) diff --git a/.github/workflows/ci-docker-build.yaml b/.github/workflows/ci-docker-build.yaml new file mode 100644 index 0000000000000..e69b732c64396 --- /dev/null +++ b/.github/workflows/ci-docker-build.yaml @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: CI - Docker Build +on: + pull_request: + branches: + - master + - branch-* + push: + branches: + - branch-* + +env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 + +jobs: + + sql: + name: + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + with: + fetch-depth: 25 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Check if this pull request only changes documentation + id: docs + uses: apache/pulsar-test-infra/diff-only@master + with: + args: site2 deployment .asf.yaml .ci ct.yaml + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + if: steps.docs.outputs.changed_only == 'no' + with: + java-version: 1.8 + + - name: clean disk + if: steps.docs.outputs.changed_only == 'no' + run: | + sudo swapoff -a + sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt clean + docker rmi $(docker images -q) -f + df -h + + - name: run install by skip tests + if: steps.docs.outputs.changed_only == 'no' + run: mvn -q -B -ntp clean install -DskipTests + + - name: build pulsar image + if: steps.docs.outputs.changed_only == 'no' + run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true + + - name: build pulsar-all image + if: steps.docs.outputs.changed_only == 'no' + run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true + + - name: build artifacts and docker pulsar latest test image + if: steps.docs.outputs.changed_only == 'no' + run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker -DskipTests diff --git a/docker/pulsar-standalone/Dockerfile b/docker/pulsar-standalone/Dockerfile index 0035de5e33d1b..600f747421860 100644 --- a/docker/pulsar-standalone/Dockerfile +++ b/docker/pulsar-standalone/Dockerfile @@ -24,16 +24,21 @@ FROM apachepulsar/pulsar-all:latest as pulsar FROM apachepulsar/pulsar-dashboard:latest as dashboard # Restart from -FROM openjdk:8-jdk +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get -y install wget gnupg && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +RUN bash -c "echo deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main >> /etc/apt/sources.list.d/pgdg.list" # Note that the libpq-dev package is needed here in order to install # the required python psycopg2 package (for postgresql) later RUN apt-get update \ - && apt-get -y install python3.7 python3.7-dev python3-pip postgresql sudo nginx supervisor libpq-dev + && apt-get -y install openjdk-11-jdk-headless python3 python3-dev python3-pip postgresql-11 sudo nginx supervisor libpq-dev RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 RUN update-ca-certificates -RUN ln -s /usr/bin/pip3 /usr/bin/pip # Postgres configuration COPY --from=dashboard /etc/postgresql/11/main/postgresql.conf /etc/postgresql/11/main/postgresql.conf @@ -73,7 +78,7 @@ RUN pip install -r /pulsar/django/requirements.txt COPY --from=dashboard /pulsar/init-postgres.sh /pulsar/django/init-postgres.sh RUN mkdir /data RUN /pulsar/django/init-postgres.sh -RUN sudo -u postgres /etc/init.d/postgresql stop +RUN sudo -u postgres /etc/init.d/postgresql stop # Add postgresql to supervisord. Redirect logs to stdout RUN echo "\n[program:postgresql]\n\ command = /etc/init.d/postgresql start\n\ diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 37a022d965ff2..c5c20bde0b519 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -32,36 +32,37 @@ COPY scripts/generate-zookeeper-config.sh /pulsar/bin COPY scripts/pulsar-zookeeper-ruok.sh /pulsar/bin COPY scripts/watch-znode.py /pulsar/bin COPY scripts/set_python_version.sh /pulsar/bin -COPY scripts/install-pulsar-client-37.sh /pulsar/bin +COPY scripts/install-pulsar-client.sh /pulsar/bin -### Create 2nd stage from OpenJDK image -### and add Python dependencies (for Pulsar functions) +### Create 2nd stage from Ubuntu image +### and add OpenJDK and Python dependencies (for Pulsar functions) -FROM openjdk:8-jdk-slim +FROM ubuntu:20.04 -# Install software-properties-common -RUN apt update \ - && apt install software-properties-common \ - && add-apt-repository ppa:deadsnakes/ppa +ARG DEBIAN_FRONTEND=noninteractive # Install some utilities RUN apt-get update \ - && apt-get install -y netcat dnsutils less procps iputils-ping \ - python3.7 python3.7-dev python3-setuptools python3-yaml python3-kazoo \ + && apt-get -y dist-upgrade \ + && apt-get -y install openjdk-11-jdk-headless netcat dnsutils less procps iputils-ping \ + python3 python3-dev python3-setuptools python3-yaml python3-kazoo \ libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \ curl \ + && apt-get -y --purge autoremove \ + && apt-get autoclean \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -RUN python3.7 get-pip.py +RUN python3 get-pip.py RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 ADD target/python-client/ /pulsar/pulsar-client ADD target/cpp-client/ /pulsar/cpp-client -RUN echo networkaddress.cache.ttl=1 >> $JAVA_HOME/jre/lib/security/java.security +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 +RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/java-11-openjdk-amd64/conf/security/java.security RUN apt-get update \ && apt install -y /pulsar/cpp-client/*.deb \ && apt-get clean \ @@ -75,4 +76,4 @@ ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE COPY --from=pulsar /pulsar /pulsar WORKDIR /pulsar -RUN /pulsar/bin/install-pulsar-client-37.sh +RUN /pulsar/bin/install-pulsar-client.sh diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 1cc58615b6575..58f13904b4da7 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -63,8 +63,8 @@ ${project.basedir}/target ${project.basedir}/../../pulsar-client-cpp/docker/build-wheels.sh - - 3.7 cp37-cp37m + + 3.8 cp38-cp38 diff --git a/docker/pulsar/scripts/install-pulsar-client-37.sh b/docker/pulsar/scripts/install-pulsar-client.sh similarity index 79% rename from docker/pulsar/scripts/install-pulsar-client-37.sh rename to docker/pulsar/scripts/install-pulsar-client.sh index 9d3714c0a979a..56702b10db3d8 100755 --- a/docker/pulsar/scripts/install-pulsar-client-37.sh +++ b/docker/pulsar/scripts/install-pulsar-client.sh @@ -20,5 +20,6 @@ set -x -WHEEL_FILE=$(ls /pulsar/pulsar-client | grep cp37) -pip3.7 install /pulsar/pulsar-client/${WHEEL_FILE} +PYTHON_MAJOR_MINOR=$(python3 -V | sed -E 's/.* ([[:digit:]]+)\.([[:digit:]]+).*/\1\2/') +WHEEL_FILE=$(ls /pulsar/pulsar-client | grep "cp${PYTHON_MAJOR_MINOR}") +pip3 install /pulsar/pulsar-client/${WHEEL_FILE}[all] diff --git a/pulsar-function-go/go.mod b/pulsar-function-go/go.mod index 560834533aad7..6170ae97699b6 100644 --- a/pulsar-function-go/go.mod +++ b/pulsar-function-go/go.mod @@ -3,7 +3,7 @@ module github.com/apache/pulsar/pulsar-function-go go 1.13 require ( - github.com/apache/pulsar-client-go v0.2.0 + github.com/apache/pulsar-client-go v0.6.0 github.com/golang/protobuf v1.4.2 github.com/prometheus/client_golang v1.7.1 github.com/prometheus/client_model v0.2.0 diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index afbf2992aa429..d3a45e08d3641 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -224,17 +224,6 @@ func (gi *goInstance) getProducer(topicName string) (pulsar.Producer, error) { gi.context.instanceConf.funcDetails.Namespace, gi.context.instanceConf.funcDetails.Name), gi.context.instanceConf.instanceID) - batchBuilderType := pulsar.DefaultBatchBuilder - - if gi.context.instanceConf.funcDetails.Sink.ProducerSpec != nil { - batchBuilder := gi.context.instanceConf.funcDetails.Sink.ProducerSpec.BatchBuilder - if batchBuilder != "" { - if batchBuilder == "KEY_BASED" { - batchBuilderType = pulsar.KeyBasedBatchBuilder - } - } - } - producer, err := gi.client.CreateProducer(pulsar.ProducerOptions{ Topic: topicName, Properties: properties,