Thumbor 7

.semaphore/semaphore.yml

@@ -26,8 +26,12 @@ blocks:
           - echo "Docker build ..."
           - "./build && echo Build done"
           - bash -c "for filename in tests/*.bats; do sudo bats --tap \$filename || exit 1; done"
+          - >-
+            ( [ "$SEMAPHORE_GIT_BRANCH" == "thumbor-7" ] && \
+              [ "${SEMAPHORE_GIT_REF_TYPE}" != "pull-request" ] && \
+              ./push $SEMAPHORE_GIT_BRANCH && echo Push $SEMAPHORE_GIT_BRANCH done ) || echo "Not pushed $SEMAPHORE_GIT_BRANCH"
           - echo "Docker push (only master)..."
           - >-
             ( [ "$SEMAPHORE_GIT_BRANCH" == "master" ] && \
               [ "${SEMAPHORE_GIT_REF_TYPE}" != "pull-request" ] && \
-              ./push && echo Push done ) || echo "Not pushed"
+              ./push $SEMAPHORE_GIT_BRANCH && echo Push $SEMAPHORE_GIT_BRANCH done ) || echo "Not pushed $SEMAPHORE_GIT_BRANCH"

benchmarks/locust/Dockerfile

@@ -1,6 +1,6 @@
-FROM python:3.6
+FROM python:3.8
 
-RUN python3.6 -m pip install locustio==0.11.0
+RUN python3.8 -m pip install locustio==0.14.5
 
 EXPOSE 8089
 EXPOSE 5557

benchmarks/locust/docker-compose.yml

@@ -1,11 +1,10 @@
 version: "3"
 services:
   thumbor:
-    image: minimalcompact/thumbor
+    image: minimalcompact/thumbor:7
     environment:
       # we can control the number of processes and threads via ENV
       - THUMBOR_NUM_PROCESSES=${THUMBOR_NUM_PROCESSES:-1}
-      - ENGINE_THREADPOOL_SIZE=${ENGINE_THREADPOOL_SIZE:-0}
       # this would allow CORS from any origin (you can restrict to specific origins if you want)
       - CORS_ALLOW_ORIGIN=*
       # returns a webp image if browser Accept headers match
@@ -27,7 +26,6 @@ services:
     environment:
       - THUMBOR_DOCKER_PROCS=${THUMBOR_DOCKER_PROCS:-1}
       - THUMBOR_NUM_PROCESSES=${THUMBOR_NUM_PROCESSES:-1}
-      - ENGINE_THREADPOOL_SIZE=${ENGINE_THREADPOOL_SIZE:-0}
     volumes:
       - ./scripts:/opt/scripts
       - ./reports:/opt/reports
@@ -41,7 +39,7 @@ services:
               --master
               --expect-slaves 3
               --only-summary
-              --csv=/opt/reports/${THUMBOR_DOCKER_PROCS}-${THUMBOR_NUM_PROCESSES}-${ENGINE_THREADPOOL_SIZE}"
+              --csv=/opt/reports/${THUMBOR_DOCKER_PROCS}-${THUMBOR_NUM_PROCESSES}"
   # this is the locust slave. We can control the number of slaves using docker-compose scale
   locust-slave:
     build:

benchmarks/locust/run_benchmark

@@ -4,37 +4,18 @@ docker-compose build
 docker-compose down
 
 for procs in 1 3 6 9; do
-  for threads in 0 3 6; do
-    export THUMBOR_DOCKER_PROCS=1
+  for docker_procs in 1 3 6 9; do
+    export THUMBOR_DOCKER_PROCS=$docker_procs
     export THUMBOR_NUM_PROCESSES=$procs
-    export ENGINE_THREADPOOL_SIZE=$threads
 
     echo
-    echo "***************** BEGIN BENCHMARK WITH ${THUMBOR_DOCKER_PROCS} Docker processes, ${THUMBOR_NUM_PROCESSES} thumbor processes and ${ENGINE_THREADPOOL_SIZE} threads ********************"
+    echo "***************** BEGIN BENCHMARK WITH ${THUMBOR_DOCKER_PROCS} Docker processes, ${THUMBOR_NUM_PROCESSES} thumbor processes ********************"
     echo
     docker-compose up -d --scale locust-slave=3 --scale thumbor=${THUMBOR_DOCKER_PROCS} --force-recreate --always-recreate-deps --renew-anon-volumes
     docker-compose logs -f locust
     docker-compose down
     echo
-    echo "***************** END BENCHMARK WITH ${THUMBOR_DOCKER_PROCS} Docker processes, ${THUMBOR_NUM_PROCESSES} thumbor processes and ${ENGINE_THREADPOOL_SIZE} threads ********************"
-    echo
-  done
-done
-
-for procs in 3 6 9; do
-  for threads in 0 3 6; do
-    export THUMBOR_DOCKER_PROCS=$procs
-    export THUMBOR_NUM_PROCESSES=1
-    export ENGINE_THREADPOOL_SIZE=$threads
-
-    echo
-    echo "***************** BEGIN BENCHMARK WITH ${THUMBOR_DOCKER_PROCS} Docker processes, ${THUMBOR_NUM_PROCESSES} thumbor processes and ${ENGINE_THREADPOOL_SIZE} threads ********************"
-    echo
-    docker-compose up -d --scale locust-slave=3 --scale thumbor=${THUMBOR_DOCKER_PROCS} --force-recreate --always-recreate-deps --renew-anon-volumes
-    docker-compose logs -f locust
-    docker-compose down
-    echo
-    echo "***************** END BENCHMARK WITH ${THUMBOR_DOCKER_PROCS} Docker processes, ${THUMBOR_NUM_PROCESSES} thumbor processes and ${ENGINE_THREADPOOL_SIZE} threads ********************"
+    echo "***************** END BENCHMARK WITH ${THUMBOR_DOCKER_PROCS} Docker processes, ${THUMBOR_NUM_PROCESSES} thumbor processes ********************"
     echo
   done
 done

benchmarks/locust/scripts/benchmark.py

@@ -1,4 +1,4 @@
-from locust import HttpLocust, TaskSet, task
+from locust import HttpLocust, TaskSet, task, between
 
 
 class HealthTaskSet(TaskSet):
@@ -17,3 +17,4 @@ def fit_in(self):
 
 class Benchmark(HttpLocust):
     task_set = SimpleTaskSet
+    wait_time = between(0, 0.1)

push

@@ -1,29 +1,30 @@
 #!/bin/bash
 
+BRANCH=$1
 # introspecting version by running the built image
 THUMBOR_VERSION=`docker run --rm minimalcompact/thumbor bash -c "pip freeze |grep ^thumbor== | cut -d= -f 3"`
 echo "THUMBOR VERSION: $THUMBOR_VERSION"
 
 echo "--> pushing minimalcompact/thumbor:simdsse4"
-docker push minimalcompact/thumbor:latest-simd-sse4
+[ "$BRANCH" == "master" ] && docker push minimalcompact/thumbor:latest-simd-sse4
 docker push minimalcompact/thumbor:$THUMBOR_VERSION-simd-sse4
 
 echo "--> pushing minimalcompact/thumbor:simdavx2"
-docker push minimalcompact/thumbor:latest-simd-avx2
+[ "$BRANCH" == "master" ] && docker push minimalcompact/thumbor:latest-simd-avx2
 docker push minimalcompact/thumbor:$THUMBOR_VERSION-simd-avx2
 
 echo "--> pushing minimalcompact/thumbor-nginx-proxy (DEPRECATED)"
-docker push minimalcompact/thumbor-nginx-proxy:latest
+[ "$BRANCH" == "master" ] && docker push minimalcompact/thumbor-nginx-proxy:latest
 docker push minimalcompact/thumbor-nginx-proxy:$THUMBOR_VERSION
 
 echo "--> pushing minimalcompact/thumbor-nginx-proxy-cache"
-docker push minimalcompact/thumbor-nginx-proxy-cache:latest
+[ "$BRANCH" == "master" ] && docker push minimalcompact/thumbor-nginx-proxy-cache:latest
 docker push minimalcompact/thumbor-nginx-proxy-cache:$THUMBOR_VERSION
 
 echo "--> pushing minimalcompact/remotecv"
-docker push minimalcompact/remotecv:latest
+[ "$BRANCH" == "master" ] && docker push minimalcompact/remotecv:latest
 docker push minimalcompact/remotecv:$THUMBOR_VERSION
 
 echo "--> pushing minimalcompact/thumbor"
-docker push minimalcompact/thumbor:latest
+[ "$BRANCH" == "master" ] && docker push minimalcompact/thumbor:latest
 docker push minimalcompact/thumbor:$THUMBOR_VERSION

thumbor/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:2
+FROM python:3.8
 
 LABEL maintainer="MinimalCompact"
 
@@ -30,46 +30,31 @@ ENV SHELL bash
 ENV WORKON_HOME /app
 WORKDIR /app
 
+RUN pip install thumbor==7.0.0
 COPY requirements.txt /app/requirements.txt
-RUN pip install --trusted-host None --no-cache-dir \
-   -r /app/requirements.txt
+RUN pip install --trusted-host None --no-cache-dir -r /app/requirements.txt
+RUN pip install --no-dependencies tc-aws==6.2.15
+RUN pip install --no-dependencies tc-core==0.4.1
+RUN pip install --no-dependencies tc-shortener==0.2.2
 
 COPY conf/thumbor.conf.tpl /app/thumbor.conf.tpl
 
-ADD conf/circus.ini.tpl /etc/
-RUN mkdir  /etc/circus.d /etc/setup.d
-ADD conf/thumbor-circus.ini.tpl /etc/circus.d/
-
 ARG SIMD_LEVEL
-# workaround for https://github.com/python-pillow/Pillow/issues/3441
-# https://github.com/thumbor/thumbor/issues/1102
-RUN PILLOW_VERSION=$(python -c 'import PIL; print(PIL.__version__)') ; \
-    if [ -z "$SIMD_LEVEL" ]; then \
-      CC="cc" && PILLOW_PACKET="pillow" && PILLOW_VERSION_SUFFIX="" ;\
-    else \
-      CC="cc -m$SIMD_LEVEL" && PILLOW_PACKET="pillow-SIMD" && PILLOW_VERSION_SUFFIX=".post99" ;\
-      # hardcoding to overcome https://github.com/MinimalCompact/thumbor/pull/37#issuecomment-514771902
-      PILLOW_VERSION="5.2.0" ; \
-    fi ; \
 
-    pip uninstall -y pillow || true && \
-    CC=$CC \
-    # https://github.com/python-pillow/Pillow/pull/3241
-    LIB=/usr/lib/x86_64-linux-gnu/ \
-    # https://github.com/python-pillow/Pillow/pull/3237 or https://github.com/python-pillow/Pillow/pull/3245
-    INCLUDE=/usr/include/x86_64-linux-gnu/ \
-    pip install --no-cache-dir -U --force-reinstall --no-binary=:all: "${PILLOW_PACKET}<=${PILLOW_VERSION}${PILLOW_VERSION_SUFFIX}" \
-    # --global-option="build_ext" --global-option="--debug" \
-    --global-option="build_ext" --global-option="--enable-lcms" \
-    --global-option="build_ext" --global-option="--enable-zlib" \
-    --global-option="build_ext" --global-option="--enable-jpeg" \
-    --global-option="build_ext" --global-option="--enable-tiff"
+RUN PILLOW_VERSION=$(python -c 'import PIL; print(PIL.__version__)') ; \
+    if [ "$SIMD_LEVEL" ]; then \
+      pip uninstall -y pillow || true && \
+      CC="cc -m$SIMD_LEVEL" pip install --no-cache-dir -U --force-reinstall --no-binary=:all: "pillow-SIMD<=${PILLOW_VERSION}.post99" \
+      # --global-option="build_ext" --global-option="--debug" \
+      --global-option="build_ext" --global-option="--enable-lcms" \
+      --global-option="build_ext" --global-option="--enable-zlib" \
+      --global-option="build_ext" --global-option="--enable-jpeg" \
+      --global-option="build_ext" --global-option="--enable-tiff" ; \
+    fi ;
 
 COPY docker-entrypoint.sh /
 ENTRYPOINT ["/docker-entrypoint.sh"]
 
-# running thumbor multiprocess via circus by default
-# to override and run thumbor solo, set THUMBOR_NUM_PROCESSES=1 or unset it
-CMD ["circus"]
+CMD ["thumbor", "--port=$THUMBOR_PORT", "-l ${LOG_LEVEL:-debug}", "--processes=${THUMBOR_NUM_PROCESSES:-1}"]
 
 EXPOSE 80 8888

thumbor/conf/thumbor.conf.tpl

@@ -365,6 +365,8 @@ STORES_CRYPTO_KEY_FOR_EACH_IMAGE = {{ STORES_CRYPTO_KEY_FOR_EACH_IMAGE | default
 ## Defaults to: /tmp/thumbor/storage
 FILE_STORAGE_ROOT_PATH = '{{ FILE_STORAGE_ROOT_PATH | default('/data/storage') }}'
 
+RESULT_STORAGE_FILE_STORAGE_ROOT_PATH = '{{ RESULT_STORAGE_FILE_STORAGE_ROOT_PATH | default('/data/result-storage') }}'
+
 ################################################################################
 
 

thumbor/docker-entrypoint.sh

@@ -3,9 +3,6 @@
 # To disable warning libdc1394 error: Failed to initialize libdc1394
 ln -s /dev/null /dev/raw1394
 
-envtpl /etc/circus.ini.tpl  --allow-missing --keep-template
-envtpl /etc/circus.d/thumbor-circus.ini.tpl  --allow-missing --keep-template
-
 if [ ! -f /app/thumbor.conf ]; then
   envtpl /app/thumbor.conf.tpl  --allow-missing --keep-template
 fi
@@ -20,14 +17,9 @@ if [ -z ${THUMBOR_PORT+x} ]; then
     THUMBOR_PORT=80
 fi
 
-if [ "$1" = 'thumbor' ] || [ "$1" = 'circus' ]; then
-    if [ "${THUMBOR_NUM_PROCESSES:-1}" -gt "1" ]; then
-        echo "---> Starting thumbor circus with ${THUMBOR_NUM_PROCESSES} processes..."
-        exec /usr/local/bin/circusd /etc/circus.ini
-    else
-        echo "---> Starting thumbor solo..."
-        exec python -m thumbor/server --port=$THUMBOR_PORT --conf=/app/thumbor.conf $LOG_PARAMETER
-    fi
+if [ "$1" = 'thumbor' ]; then
+    echo "---> Starting thumbor with ${THUMBOR_NUM_PROCESSES:-1} processes..."
+    exec thumbor --port=$THUMBOR_PORT --conf=/app/thumbor.conf $LOG_PARAMETER --processes=${THUMBOR_NUM_PROCESSES:-1}
 fi
 
 exec "$@"

thumbor/requirements.txt

@@ -1,24 +1,11 @@
 envtpl==0.6.0
-circus==0.15.0
-circus-web==1.0.0
-boto==2.48.0
-botocore==1.12.253
-dateutils==0.6.6
-numpy==1.14.3
-remotecv==2.2.2
-opencv-engine==1.0.1
-thumbor==6.7.0
+remotecv==3.0.0
+boto==2.49.0
 tornado-botocore==1.5.0
-tc-aws==6.2.10
-tc-core==0.4.0
-tc-shortener==0.2.2
-raven==6.7.0
-thumbor-memcached==5.1.0
-tc-mongodb==5.1.0
-tc-redis==1.0.1
-thumbor-cloud-storage==3.0.0
-pgmagick==0.7.4
-graphicsmagick-engine==0.1.1
+python-dateutil
+dateutils==0.6.12
+shortuuid==1.0.8
+redis==3.5.3
+raven==6.10.0
 cairosvg==1.0.22
-opencv-python==4.1.0.25
-tc_prometheus
+pycurl==7.44.1