django + celery + rabbitmq + docker-compose 셋팅하기

1. 프로젝트 생성

 - django-admin.py startproject conf .

 - conf 폴더에서 __init__.py에 코드 추가

 - celery.py 파일 생성

 - 디렉터리 구조

 

conf/

         __init__.py *(수정)

         settings.py

         urls.py

         wsgi.py

         celery.py *(생성)

   manage.py

   docker-compose.yml

   Dockerfile

   venv/

 

# requirements.txt
amqp==2.6.1
asgiref==3.3.1
billiard==3.6.3.0
celery==4.4.7
certifi==2020.12.5
chardet==4.0.0
Django==3.1.5
django-celery-beat==2.1.0
django-celery-results==2.0.0
django-cors-headers==3.6.0
django-timezone-field==4.1.1
djangorestframework==3.11.0
flake8==3.8.4
flower==0.9.7
humanize==3.2.0
idna==2.10
kombu==4.6.11
mccabe==0.6.1
Pillow==8.1.0
prometheus-client==0.8.0
psycopg2==2.8.6
pycodestyle==2.6.0
pyflakes==2.2.0
PyJWT==2.0.0
python-crontab==2.5.1
python-dateutil==2.8.1
pytz==2020.5
requests==2.25.1
six==1.15.0
sqlparse==0.4.1
tornado==6.1
urllib3==1.26.2
vine==1.3.0

 

# __init__.py
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app

__all__ = ('celery_app',)



# celery.py
import os

from celery import Celery

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')

app = Celery('config')

# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
#   should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')

app.conf.beat_schedule = {
    'every-15-second': {
        'task': 'polls.tasks.say_hello',
        'schedule': 15,
#        'args': (,)
    }
}
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()


@app.task(bind=True)
def debug_task(self):
    print(f'Request: {self.request!r}')
    
    
# polls/tasks.py
from celery import shared_task
import time

@shared_task
def add(x, y):
    time.sleep(15)
    return x + y


@shared_task
def mul(x, y):
    return x * y


@shared_task
def xsum(numbers):
    return sum(numbers)


@shared_task
def say_hello():
    print('hwanil nim hihihi')

 

2. docker-compose.yml 작성

# docker-compose.yml

version: '3.7'

networks:
  default:
    driver: bridge
    name: devopts

services:
  backend: &python
    build:
      context: .
      dockerfile: Dockerfile
    container_name: 'backend'
    volumes:
      - .:/code/backend
    ports:
      - "8000:8000"
    expose:
      - "8000"
    restart: on-failure
    command: python manage.py runserver 0.0.0.0:8000
    depends_on:
      - db
      - rabbitmq
      - celery_worker
  db:
      image: postgres:10-alpine
      environment:
          - POSTGRES_DB=app
          - POSTGRES_USER=postgres
          - POSTGRES_PASSWORD=supersecretpassword
  # rabbitmq message queue
  rabbitmq:
    image: rabbitmq:3.7-alpine
    ports:
      - 5672:5672
      - 15672:15672
    expose:
      - "15672"
  celery_worker:
    <<: *python
    container_name: 'backend_celery_worker'
    command: celery -A config worker --loglevel=info
    ports: []
    depends_on:
      - rabbitmq
      - db
  celery_beat:
    <<: *python
    container_name: 'backend_celery_beat'
    command: celery -A config beat --loglevel=info
    ports: []
    depends_on:
      - rabbitmq
      - db
  flower:
    image: mher/flower
    environment:
      - CELERY_BROKER_URL=amqp://guest:guest@rabbitmq
#      - CELERY_BROKER_URL=amqp://rabbitmq
      - FLOWER_PORT=5555
    ports:
      - 5555:5555


# Dockerfile
FROM python:3.6.9

# 환경 변수 설정
ENV PYTHONDONTWRITEBYTECODE 1
ENV PYTHONUNBUFFERED 1

# Avoiding user interaction with tzdata
ENV DEBIAN_FRONTEND=noninteractive

#dependencies를 위한 apt-get update
RUN apt-get update && apt-get -y install \
    libpq-dev --no-install-recommends apt-utils

COPY . .

WORKDIR /code/backend/

COPY requirements.txt .

RUN pip install --upgrade pip && pip install -r requirements.txt

 

# 참고

velog.io/@khh180cm/celery-RabbitMQ-2

 

heodolf.tistory.com/50

 

github.com/gyeongdo/django-celery-rabbitmq-docker