Substituição do Redis por RabbitMQ como broker de mensageria do Celery

.envs/.local/.django

@@ -8,6 +8,7 @@ REDIS_URL=redis://redis:6379/0
 
 # Celery
 # ------------------------------------------------------------------------------
+CELERY_BROKER_URL=amqp://guest:guest@rabbitmq:5672//
 
 # Flower
 CELERY_FLOWER_USER=PhFRdLexbrsBvrrbSXxjcMMOcVOavCrZ

.envs/.production/.django

@@ -6,8 +6,14 @@ IPYTHONDIR=/app/.ipython
 # ------------------------------------------------------------------------------
 REDIS_URL=redis://redis:6379/0
 
+# RabbitMQ
+# ------------------------------------------------------------------------------
+RABBITMQ_DEFAULT_USER=guest
+RABBITMQ_DEFAULT_PASS=guest
+
 # Celery
 # ------------------------------------------------------------------------------
+CELERY_BROKER_URL=amqp://guest:guest@rabbitmq:5672//
 
 # Flower
 CELERY_FLOWER_USER=PhFRdLexbrsBvrrbSXxjcMMOcVOavCrZ

compose/production/django/entrypoint

@@ -6,10 +6,6 @@ set -o nounset
 
 
 
-# N.B. If only .env files supported variable expansion...
-export CELERY_BROKER_URL="${REDIS_URL}"
-
-
 if [ -z "${POSTGRES_USER}" ]; then
     base_postgres_image_default_user='postgres'
     export POSTGRES_USER="${base_postgres_image_default_user}"

config/settings/base.py

@@ -366,7 +366,7 @@
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_url
 CELERY_BROKER_URL = env("CELERY_BROKER_URL")
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-result_backend
-CELERY_RESULT_BACKEND = CELERY_BROKER_URL
+CELERY_RESULT_BACKEND = env("CELERY_RESULT_BACKEND", default="redis://redis:6379/0")
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-accept_content
 CELERY_ACCEPT_CONTENT = ["json"]
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-task_serializer
@@ -415,10 +415,8 @@
 # Celery Results
 # ------------------------------------------------------------------------------
 # https://django-celery-results.readthedocs.io/en/latest/getting_started.html
-# NOTA: Não usar "django-db" como result backend em produção.
-# O result backend já está configurado como Redis (CELERY_BROKER_URL) acima.
-# Manter "django-db" aqui causaria escritas extras no PostgreSQL a cada tarefa concluída.
-# CELERY_RESULT_BACKEND = "django-db"  # REMOVIDO: já usa Redis acima
+# O result backend usa Redis (via CELERY_RESULT_BACKEND) para armazenar resultados.
+# O broker de mensageria usa RabbitMQ (via CELERY_BROKER_URL).
 CELERY_CACHE_BACKEND = "django-cache"
 CELERY_RESULT_EXTENDED = True
 

docs/conf.py

@@ -22,7 +22,7 @@
 else:
     sys.path.insert(0, os.path.abspath("/app"))
 os.environ["DATABASE_URL"] = "sqlite:///readthedocs.db"
-os.environ["CELERY_BROKER_URL"] = os.getenv("REDIS_URL", "redis://redis:6379")
+os.environ["CELERY_BROKER_URL"] = os.getenv("CELERY_BROKER_URL", "amqp://guest:guest@rabbitmq:5672//")
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
 django.setup()
 

local.yml

@@ -8,6 +8,7 @@ services:
     image: scielo_core_local_django
     container_name: scielo_core_local_django
     depends_on:
+      - rabbitmq
       - redis
       - postgres
       - mailhog
@@ -83,12 +84,19 @@ services:
     ports:
       - "6399:6379"
 
+  rabbitmq:
+    image: rabbitmq:3-management
+    container_name: scielo_core_local_rabbitmq
+    ports:
+      - "5672:5672"
+      - "15672:15672"
 
   celeryworker:
     <<: *django
     image: scielo_core_local_celeryworker
     container_name: scielo_core_local_celeryworker
     depends_on:
+      - rabbitmq
       - redis
       - postgres
       - mailhog
@@ -100,6 +108,7 @@ services:
     image: scielo_core_local_celerybeat
     container_name: scielo_core_local_celerybeat
     depends_on:
+      - rabbitmq
       - redis
       - postgres
       - mailhog

production.yml

@@ -9,6 +9,7 @@ services:
     platform: linux/x86_64
     depends_on:
       - postgres
+      - rabbitmq
       - redis
       - solr
     links:
@@ -61,10 +62,18 @@ services:
     ports:
       - "6379:6379"
 
+  rabbitmq:
+    image: rabbitmq:3
+    ports:
+      - "5672:5672"
+    env_file:
+      - ./.envs/.production/.django
+
   celeryworker:
     <<: *django
     image: core_production_celeryworker
     depends_on:
+      - rabbitmq
       - redis
       - postgres
     command: /start-celeryworker
@@ -73,6 +82,7 @@ services:
     <<: *django
     image: core_production_celerybeat
     depends_on:
+      - rabbitmq
       - redis
       - postgres
     command: /start-celerybeat
@@ -87,9 +97,9 @@ services:
     restart: always
     ports:
      - 9808:9808
-    command: ["--broker-url=redis://redis:6379/0"]
+    command: ["--broker-url=amqp://guest:guest@rabbitmq:5672//"]
     depends_on:
-     - redis
+     - rabbitmq
 
   victoriametrics:
     image: victoriametrics/victoria-metrics

start-dev.sh

@@ -10,7 +10,7 @@ export IP=$(/sbin/ip -o -4 addr list $ethernet | awk '{print $4}' | cut -d/ -f1)
 #export IP=$(ifconfig $ethernet | grep inet | grep -v inet6 | awk '{print $2}')
 
 export DATABASE_URL=postgres://GVRFlLmcCNfGLhsFvSnCioYOPJPYpyfj:BQ4hSUL4rdj5WZLdR8ilDLRQMvCtzo0caMaXDO0olGsmycQjlcZlTVK9DepZR8kk@$IP:5439/scielo_core
-export CELERY_BROKER_URL=redis://$IP:6399/0
+export CELERY_BROKER_URL=amqp://guest:guest@$IP:5672//
 export USE_DOCKER=no
 export IPYTHONDIR=/app/.ipython
 export REDIS_URL=redis://$IP:6399/0