用 Redis 做來排行榜統計

熱門藝人
熱門歌曲
熱門專輯
熱門播放清單
即時、每日、每週、每月、所有時間

from datetime import date, datetime, timedelta

from pytz import timezone

from svapp.redis import rdb


def get_last_iso_year_and_week(the_date):
    """
    計算 ISO 定義下的上一週
    """

    d = the_date - timedelta(weeks=1)
    year = d.isocalendar()[0]
    week = d.isocalendar()[1]

    return year, week


def get_weekday_start_and_end(iso_year, iso_week):
    """
    ISO 的定義是每週的第一天是禮拜一
    The Jan 4th must be in week 1 according to ISO
    """

    d = date(iso_year, 1, 4)
    start_date = d + timedelta(weeks=(iso_week - 1), days=-d.weekday())
    end_date = start_date + timedelta(days=6)

    return start_date, end_date


def ndays_later(n, timezone_str='Asia/Taipei'):
    """
    n 天後的日期

    n = 0 表示今天
    n = 1 表示明天(一天後)
    n = 2 表示後天(兩天後)
    n = -1 表示昨天
    """

    tz = timezone(timezone_str)
    the_date = datetime.now(tz) + timedelta(n)

    return the_date


def date_key(n=0):
    """
    默認回傳今天的 date_key

    格式 20140602
    """

    the_date = ndays_later(n)
    key = the_date.strftime('%Y%m%d')

    return key


"""
各 counter type 的加權參數
TODO: 這些數值要再想一下
"""
COUNTER_TYPES = {
    'view': 0.5,
    'play': 1,
    'play_full': 2,
    'embed_play': 1,
    'embed_play_full': 2,
    'like': 3,
    'add_to_playlist': 4,
}


def check_counter_type(counter_type):
    if counter_type not in COUNTER_TYPES.keys():
        raise RuntimeError


def increase_song_counter(song, counter_type):
    """
    用 hash 儲存歌曲的各種 counter

    範例:
    counter:song:978
    {
        'view': 1244.5,
        'play': 456,
        'play_full', 211,
        ...
    }
    """

    check_counter_type(counter_type)

    key = 'counter:song:%s' % (song.id)
    rdb.hincrby(key, counter_type, 1)


def update_song_leaderboard(song, counter_type):
    """
    用 sorted set 儲存當日的歌曲排行榜
    這個排行榜是即時更新的
    """

    inc = COUNTER_TYPES[counter_type]
    style_ids = (0, song.style)
    for style_id in style_ids:
        key = 'leaderboard:song:style:%s:realtime' % (style_id)
        rdb.zincrby(key, song.id, inc)

        # 這個資料應該每天 achive 存進 mysql
        key = 'leaderboard:song:style:%s:%s' % (style_id, date_key())
        rdb.zincrby(key, song.id, inc)


def trim_leaderboard(key, topN):
    """
    只保留 sorted set 的 top 100
    rdb.zremrangebyrank(key, 0, -101)
    """

    value = -abs(topN) - 1
    rdb.zremrangebyrank(key, 0, value)


def get_song_leaderboard(when='realtime', style_id=0, topN=20):
    if isinstance(when, (date, datetime)):
        when = when.strftime('%Y%m%d')

    key = 'leaderboard:song:style:%s:%s' % (style_id, when)

    result = rdb.zrevrange(key, 0, topN - 1, withscores=True)

    return result


def get_song_leaderboard_weekly(year, week, style_id=0, topN=20):
    """
    TODO:
    1. 檢查 week_key 是否已存在
    2. 在 update_song_leaderboard() 的時候就直接建 week 和 month 的 key,這樣就不需要 zunionstore
    """

    week_key = 'leaderboard.week:song:style:%s:%s%02d' % (style_id, year, week)

    start_date, end_date = get_weekday_start_and_end(year, week)
    date_range = [start_date + timedelta(days=n) for n in xrange(0, 6)]
    date_range_keys = ['leaderboard:song:style:%s:%s' % (style_id, d.strftime('%Y%m%d')) for d in date_range]

    rdb.zunionstore(week_key, date_range_keys)

    result = rdb.zrevrange(week_key, 0, topN - 1, withscores=True)

    return result


def get_song_leaderboard_monthly(year, month, style_id=0, topN=20):
    """
    TODO: 檢查 month_key 是否已存在
    """

    # leaderboard.monthly:song:style:0:201406
    month_key = 'leaderboard.month:song:style:%s:%s%02d' % (style_id, year, month)

    search = 'leaderboard:song:style:%s:%s%02d??' % (style_id, year, month)
    day_keys = rdb.keys(search)
    rdb.zunionstore(month_key, day_keys)

    result = rdb.zrevrange(month_key, 0, topN - 1, withscores=True)

    return result

豆瓣圖書 API

Oauth 2.0

http://developers.douban.com/wiki/?title=oauth2

權限不同,token 就會不同

Configuration

with python-social-auth

SOCIAL_AUTH_DOUBAN_OAUTH2_KEY = 'YOUR_KEY'
SOCIAL_AUTH_DOUBAN_OAUTH2_SECRET = 'YOUR_SECRET'
SOCIAL_AUTH_DOUBAN_OAUTH2_SCOPE = [
    'douban_basic_common,book_basic_r,book_basic_w'
]

Usage

http://developers.douban.com/wiki/?title=api_v2
http://developers.douban.com/wiki/?title=book_v2

headers = {
    'Authorization': 'Bearer YOUR_TOKEN',
}

for isbn in (item['isbn_13'], item['isbn_10']):
    book_isbn_url = 'https://api.douban.com/v2/book/isbn/%s' % (isbn)
    r = requests.get(book_isbn_url, headers=headers)
    if r.ok:
        book = json.loads(r.content)
        book_id = book['id']

        book_collect_url = 'https://api.douban.com/v2/book/%s/collection' % (book_id)
        data = {
            'status': item['status'],
        }
        if 'comment' in item:
            data['comment'] = item['comment']
        r2 = requests.post(book_collect_url, data=data, headers=headers)
        if not r2.ok:
            if r2.status_code == 409:
                # 已收藏,略過
                pass
            else:
                log.msg('post fail: %s' % (book_name), level=log.ERROR)
                log.msg('post fail: %s' % (r2.status_code), level=log.ERROR)

        break
    else:
        log.msg('isbn not found: %s' % (book_name), level=log.ERROR)
        log.msg('isbn not found: %s' % (isbn), level=log.ERROR)

Compile nginx with sticky-module on Ubuntu 12.04

# 先移除用 apt-get 安裝的 nginx
$ sudo apt-get autoremove nginx

開始編譯

$ sudo su root

$ cd /usr/src/
$ apt-get source nginx

$ cd nginx-1.6.0/debian/modules
$ wget https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng/get/1.2.5.tar.gz
$ tar zxvf 1.2.5.tar.gz
$ mv nginx-goodies-nginx-sticky-module-ng-bd312d586752/ nginx-goodies-nginx-sticky-module-ng/

$ vim /usr/src/nginx-1.6.0/debian/rules
# 在 full_configure_flags 底下加上:
# --add-module=$(MODULESDIR)/ngx_http_substitutions_filter_module \
# --add-module=$(MODULESDIR)/nginx-goodies-nginx-sticky-module-ng

$ cd /usr/src/nginx-1.6.0/
$ aptitude build-dep nginx
$ aptitude install liblua5.1-0-dev init-system-helpers
$ dpkg-buildpackage -b

$ cd /usr/src
$ dpkg -i nginx-common_1.6.0-1+precise0_all.deb
$ dpkg -i nginx-full_1.6.0-1+precise0_amd64.deb

# 驗證一下
$ nginx -V

ref:
http://gravitronic.com/compiling-the-nginx-sticky-session-module-in-ubuntu/