Celery 3.1 with Django

Install

$ pip install celery

# for Amazon SQS
$ pip install boto

# for IronMQ
$ pip install iron-mq iron_celery==0.4.0

ref:
http://docs.celeryproject.org/

Configuration

in init.py(跟 settings.py 放在一起)

from __future__ import absolute_import

# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app

in celery.py(跟 settings.py 放在一起)

from __future__ import absolute_import
import os

from celery import Celery

from django.conf import settings

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dps.settings')

app = Celery('dps')

# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

in settings.py

ref:
http://docs.celeryproject.org/en/latest/getting-started/brokers/sqs.html#broker-sqs
https://github.com/iron-io/iron_celery

# for Amazon SQS
AWS_ACCESS_KEY_ID_ESCAPED = urllib.quote_plus(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY_ESCAPED = urllib.quote_plus(AWS_SECRET_ACCESS_KEY)
BROKER_URL = 'sqs://%s:%[email protected]' % (AWS_ACCESS_KEY_ID_ESCAPED, AWS_SECRET_ACCESS_KEY_ESCAPED)
BROKER_TRANSPORT_OPTIONS = {
    'region': 'ap-northeast-1',
    'queue_name_prefix': 'dps-celery-',
}

# for IronMQ
BROKER_URL = 'ironmq://project_id:[email protected]'

CELERYD_CONCURRENCY = 2

CELERY_ACCEPT_CONTENT = ['json', ]
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'

Usage

in your_app/tasks.py

from __future__ import absolute_import

from celery import shared_task


@shared_task
def add(x, y):
    return x + y

Commands

# 在 project root 執行這個指令即可
# goeasytaxi 就是 celery app 的名稱
$ celery -A goeasytaxi worker -l info

Upstart

in /etc/init/celeryd.conf

description "Celery"

start on runlevel [2345]
stop on runlevel [!2345]

script
    chdir /home/vinta/goeasytaxi
    exec /home/vinta/.virtualenvs/goeasytaxi/bin/celery -A goeasytaxi worker -l info
end script

respawn
$ sudo service celeryd restart