Sentry (Raven) with Django and Celery

Sentry is an error logging and aggregation platform, Raven is its client which supports Python, Ruby, PHP, Go, JavaScript, etc.

ref:
https://app.getsentry.com/docs/platforms/

Installation

First, create an account on https://getsentry.com/.

$ pip install raven

Configuration

in settings.py

INSTALLED_APPS = (
    ...
    'raven.contrib.django.raven_compat',
    ...
)

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse',
        },
        'require_debug_true': {
            '()': 'django.utils.log.RequireDebugTrue',
        },
    },
    'formatters': {
        'verbose': {
            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s',
        },
        'simple': {
            'format': '%(levelname)s %(name)s %(message)s',
        },
        'clear': {
            'format': '%(message)s',
        },
    },
    'handlers': {
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'clear',
            'filters': ['require_debug_true', ],
        },
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler',
            'filters': ['require_debug_false', ],
        },
        'sentry': {
            'level': 'WARNING',
            'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
        },
    },
    'loggers': {
        'celery.worker': {
            'level': 'WARNING',
            'handlers': ['sentry', ],
        },
        'django.request': {
            'level': 'WARNING',
            'handlers': ['console', ],
        },
        'log_to_console': {
            'level': 'DEBUG',
            'handlers': ['console', ],
        },
        'log_to_sentry': {
            'level': 'WARNING',
            'handlers': ['sentry', ],
        },
        'raven': {
            'level': 'INFO',
            'handlers': ['console', ],
        },
        'sentry.errors': {
            'level': 'INFO',
            'handlers': ['console', ],
        },
    },
}

CELERYD_HIJACK_ROOT_LOGGER = False

in settings_prod.py

RAVEN_CONFIG = {
    'dsn': 'https://xxx:[email protected]/123',
}

LOGGING['root'] = {
    'level': 'WARNING',
    'handlers': ['sentry', ],
}

in settings_dev.py

RAVEN_CONFIG = {
    'dsn': '',
}

Usage

raven.py

import logging


# send a log to Sentry manually
sentry_logger = logging.getLogger('log_to_sentry')

try:
    do_something()
except:
    # If you're actually catching an exception, use `exc_info=True`
    sentry_logger.warning('TEST 1', exc_info=True)

# If you don't have an exception, but still want to capture a stacktrace, use the `stack` arg
sentry_logger.warning('TEST 2', extra={'stack': True})

# with extra information
logger.warning('TEST 3', extra={
    'stack': True,
    'data': {
        'your_data': 'any',
    },
})

raven.js

<script src="//cdnjs.cloudflare.com/ajax/libs/raven.js/x.x.x/raven.min.js"></script>
<script>
var options = {
  logger: 'js',
  ignoreUrls: [
    /graph\.facebook\.com/i
  ],
  includePaths: [
    /https?:\/\/(www\.)?getsentry\.com/,
    /https?:\/\/d3nslu0hdya83q\.cloudfront\.net/
  ]
};
Raven.config('https://[email protected]/123', options).install();
</script>
// record a simple message
Raven.captureMessage('Hello World!')

// capture an exception
try {
    errorThrowingCode();
} catch(err) {
    Raven.captureException(err);
}

ref:
http://raven-js.readthedocs.org/en/latest/
http://raven-js.readthedocs.org/en/latest/usage/index.html

Celery 3.1 with Django

Install

$ pip install celery

# for Amazon SQS
$ pip install boto

# for IronMQ
$ pip install iron-mq iron_celery==0.4.0

ref:
http://docs.celeryproject.org/

Configuration

in init.py(跟 settings.py 放在一起)

from __future__ import absolute_import

# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app

in celery.py(跟 settings.py 放在一起)

from __future__ import absolute_import
import os

from celery import Celery

from django.conf import settings

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dps.settings')

app = Celery('dps')

# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

in settings.py

ref:
http://docs.celeryproject.org/en/latest/getting-started/brokers/sqs.html#broker-sqs
https://github.com/iron-io/iron_celery

# for Amazon SQS
AWS_ACCESS_KEY_ID_ESCAPED = urllib.quote_plus(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY_ESCAPED = urllib.quote_plus(AWS_SECRET_ACCESS_KEY)
BROKER_URL = 'sqs://%s:%s@' % (AWS_ACCESS_KEY_ID_ESCAPED, AWS_SECRET_ACCESS_KEY_ESCAPED)
BROKER_TRANSPORT_OPTIONS = {
    'region': 'ap-northeast-1',
    'queue_name_prefix': 'dps-celery-',
}

# for IronMQ
BROKER_URL = 'ironmq://project_id:token@'

CELERYD_CONCURRENCY = 2

CELERY_ACCEPT_CONTENT = ['json', ]
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'

Usage

in your_app/tasks.py

from __future__ import absolute_import

from celery import shared_task


@shared_task
def add(x, y):
    return x + y

Commands

# 在 project root 執行這個指令即可
# goeasytaxi 就是 celery app 的名稱
$ celery -A goeasytaxi worker -l info

Upstart

in /etc/init/celeryd.conf

description "Celery"

start on runlevel [2345]
stop on runlevel [!2345]

script
    chdir /home/vinta/goeasytaxi
    exec /home/vinta/.virtualenvs/goeasytaxi/bin/celery -A goeasytaxi worker -l info
end script

respawn
$ sudo service celeryd restart