diff --git a/.github/workflows/ci-docker-build.yaml b/.github/workflows/ci-docker-build.yaml new file mode 100644 index 00000000000000..e69b732c64396d --- /dev/null +++ b/.github/workflows/ci-docker-build.yaml @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: CI - Docker Build +on: + pull_request: + branches: + - master + - branch-* + push: + branches: + - branch-* + +env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 + +jobs: + + sql: + name: + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + with: + fetch-depth: 25 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Check if this pull request only changes documentation + id: docs + uses: apache/pulsar-test-infra/diff-only@master + with: + args: site2 deployment .asf.yaml .ci ct.yaml + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + if: steps.docs.outputs.changed_only == 'no' + with: + java-version: 1.8 + + - name: clean disk + if: steps.docs.outputs.changed_only == 'no' + run: | + sudo swapoff -a + sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt clean + docker rmi $(docker images -q) -f + df -h + + - name: run install by skip tests + if: steps.docs.outputs.changed_only == 'no' + run: mvn -q -B -ntp clean install -DskipTests + + - name: build pulsar image + if: steps.docs.outputs.changed_only == 'no' + run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true + + - name: build pulsar-all image + if: steps.docs.outputs.changed_only == 'no' + run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true + + - name: build artifacts and docker pulsar latest test image + if: steps.docs.outputs.changed_only == 'no' + run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker -DskipTests diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 15159c69d6a2a8..c5c20bde0b5197 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -17,8 +17,7 @@ # under the License. # -# First create a stage with the Pulsar tarball, the scripts, the python client, -# the cpp client, and the data directory. Then ensure correct file permissions. +# First create a stage with just the Pulsar tarball and scripts FROM busybox as pulsar ARG PULSAR_TARBALL @@ -35,32 +34,21 @@ COPY scripts/watch-znode.py /pulsar/bin COPY scripts/set_python_version.sh /pulsar/bin COPY scripts/install-pulsar-client.sh /pulsar/bin -COPY target/python-client/ /pulsar/pulsar-client -COPY target/cpp-client/ /pulsar/cpp-client -RUN mkdir /pulsar/data +### Create 2nd stage from Ubuntu image +### and add OpenJDK and Python dependencies (for Pulsar functions) -# In order to support running this docker image as a container on OpenShift -# the final image needs to give the root group enough permission. -# The file permissions are maintained when copied into the target image. -RUN chmod -R g=u /pulsar +FROM ubuntu:20.04 -### Create 2nd stage from OpenJDK image -### and add Python dependencies (for Pulsar functions) +ARG DEBIAN_FRONTEND=noninteractive -FROM eclipse-temurin:11.0.15_10-jdk - -# Create the pulsar group and user to make docker container run as a non root user by default -RUN groupadd -g 10001 pulsar -RUN adduser -u 10000 --gid 10001 --disabled-login --disabled-password --gecos '' pulsar - -# Install upgrades and some utilities -RUN DEBIAN_FRONTEND=noninteractive apt-get update \ +# Install some utilities +RUN apt-get update \ && apt-get -y dist-upgrade \ - && apt-get install -y netcat dnsutils less procps iputils-ping \ + && apt-get -y install openjdk-11-jdk-headless netcat dnsutils less procps iputils-ping \ python3 python3-dev python3-setuptools python3-yaml python3-kazoo \ - libreadline-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \ - curl vim net-tools unzip \ + libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \ + curl \ && apt-get -y --purge autoremove \ && apt-get autoclean \ && apt-get clean \ @@ -71,26 +59,21 @@ RUN python3 get-pip.py RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 -# The pulsar directory is staged correctly in the first stage, above. -# The chown and chmod ensure proper permissions for running as a non root user and non root group -# as well as running on OpenShift with a random user that is part of the root group -RUN mkdir /pulsar && chown pulsar:0 /pulsar && chmod g=u /pulsar -COPY --from=pulsar --chown=pulsar:0 /pulsar /pulsar - -RUN echo networkaddress.cache.ttl=1 >> $JAVA_HOME/conf/security/java.security - +ADD target/python-client/ /pulsar/pulsar-client +ADD target/cpp-client/ /pulsar/cpp-client +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 +RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/java-11-openjdk-amd64/conf/security/java.security RUN apt-get update \ && apt install -y /pulsar/cpp-client/*.deb \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +VOLUME ["/pulsar/conf", "/pulsar/data"] + ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE + +COPY --from=pulsar /pulsar /pulsar WORKDIR /pulsar -# This script is intentionally run as the root user to make the dependencies -# available to the root user and the pulsar user RUN /pulsar/bin/install-pulsar-client.sh - -# Switch to the pulsar user to ensure container defaults to run as a non root user -USER 10000 diff --git a/pulsar-function-go/go.mod b/pulsar-function-go/go.mod index 560834533aad73..6170ae97699b6c 100644 --- a/pulsar-function-go/go.mod +++ b/pulsar-function-go/go.mod @@ -3,7 +3,7 @@ module github.com/apache/pulsar/pulsar-function-go go 1.13 require ( - github.com/apache/pulsar-client-go v0.2.0 + github.com/apache/pulsar-client-go v0.6.0 github.com/golang/protobuf v1.4.2 github.com/prometheus/client_golang v1.7.1 github.com/prometheus/client_model v0.2.0