Upgrade celery worker options for compatibility with celery 5

- reorder --app to come after the `celery` command - drop --without-* which are buggy currently (and not really useful for this environment). See celery/celery#6365.
SoftwareHeritage · Sep 25, 2020 · ec803e5 · ec803e5
1 parent dc8d410
commit ec803e5
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 21 deletions.
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -324,6 +324,7 @@ services:
     command: worker
     env_file:
       - ./env/common_python.env
+      - ./env/workers.env
     environment:
       SWH_CONFIG_FILENAME: /cooker.yml
     depends_on:

diff --git a/docker/services/swh-indexer-worker/entrypoint.sh b/docker/services/swh-indexer-worker/entrypoint.sh
@@ -19,14 +19,13 @@ case "$1" in
         wait_pgsql
 
         echo Starting swh-indexer worker
-        exec python -m celery worker \
+        exec python -m celery \
              --app=swh.scheduler.celery_backend.config.app \
+             worker \
              --pool=prefork --events \
              --concurrency=${CONCURRENCY} \
-             --maxtasksperchild=${MAX_TASKS_PER_CHILD} \
-             -Ofair --loglevel=${LOGLEVEL} --without-gossip \
-             --without-mingle \
-             --heartbeat-interval 10 \
+             --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \
+             -Ofair --loglevel=${LOGLEVEL} \
              --hostname "${SWH_WORKER_INSTANCE}@%h"
     ;;
 esac
diff --git a/docker/services/swh-listers-worker/entrypoint.sh b/docker/services/swh-listers-worker/entrypoint.sh
@@ -34,14 +34,13 @@ case "$1" in
         wait-for-it amqp:5672 -s --timeout=0
 
         echo Starting the swh-lister Celery worker for ${SWH_WORKER_INSTANCE}
-        exec python -m celery worker \
+        exec python -m celery \
                     --app=swh.scheduler.celery_backend.config.app \
+                    worker \
                     --pool=prefork --events \
                     --concurrency=${CONCURRENCY} \
-                    --maxtasksperchild=${MAX_TASKS_PER_CHILD} \
-                    -Ofair --loglevel=${LOGLEVEL} --without-gossip \
-                    --without-mingle \
-                    --heartbeat-interval 10 \
+                    --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \
+                    -Ofair --loglevel=${LOGLEVEL} \
                     --hostname "${SWH_WORKER_INSTANCE}@%h"
         ;;
 esac
diff --git a/docker/services/swh-vault/entrypoint.sh b/docker/services/swh-vault/entrypoint.sh
@@ -13,14 +13,14 @@ case "$1" in
         exec bash -i
         ;;
     "worker")
-        echo Starting the swh-vault Celery worker for
-        exec python -m celery worker \
+        echo Starting the swh-vault Celery worker
+        exec python -m celery \
                     --app=swh.scheduler.celery_backend.config.app \
+                    worker \
                     --pool=prefork --events \
-                    --concurrency=${CONCURRENCY:-1} \
-                    --maxtasksperchild=${MAX_TASKS_PER_CHILD:-10} \
-                    -Ofair --loglevel=${LOGLEVEL:-INFO} --without-gossip \
-                    --without-mingle --without-heartbeat \
+                    --concurrency=${CONCURRENCY} \
+                    --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \
+                    -Ofair --loglevel=${LOGLEVEL} \
                     --hostname "vault@%h"
         ;;
     "server")

diff --git a/docker/services/swh-worker/entrypoint.sh b/docker/services/swh-worker/entrypoint.sh
@@ -18,14 +18,13 @@ case "$1" in
         swh scheduler -C ${SWH_CONFIG_FILENAME} task-type register
 
         echo Starting the swh Celery worker for ${SWH_WORKER_INSTANCE}
-        exec python -m celery worker \
+        exec python -m celery \
                     --app=swh.scheduler.celery_backend.config.app \
+                    worker \
                     --pool=prefork --events \
                     --concurrency=${CONCURRENCY} \
-                    --maxtasksperchild=${MAX_TASKS_PER_CHILD} \
-                    -Ofair --loglevel=${LOGLEVEL} --without-gossip \
-                    --without-mingle \
-                    --heartbeat-interval 10 \
+                    --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \
+                    -Ofair --loglevel=${LOGLEVEL} \
                     --hostname "${SWH_WORKER_INSTANCE}@%h"
         ;;
 esac