Login

Error rate limiter

Author:
s29
Posted:
October 25, 2010
Language:
Python
Version:
Not specified
Score:
2 (after 2 ratings)

Prevents error flooding on high traffic production sites. Particularly useful to prevent clogging email servers etc.

See discussion and closed ticket.

Uses memcache if it can, or falls back to local, in-process memory where unavailable (down, etc).

Tweak to your needs using setting ERROR_RATE_LIMIT (seconds).

Requires Django 1.3+ or trunk r13981+

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
In ratelimit.py --


import traceback
try:
    from hashlib import md5
except ImportError:
    from md5 import md5
from datetime import datetime, timedelta


class RateLimitFilter(object):
    
    _errors = {}
        
    def filter(self, record):
        from django.conf import settings
        from django.core.cache import cache                               
        
        tb = '\n'.join(traceback.format_exception(*record.exc_info))

        # Track duplicate errors
        duplicate = False
        rate = getattr(settings, 'ERROR_RATE_LIMIT', 10)  # seconds
        if rate > 0:
            key = md5(tb).hexdigest()
            prefix = getattr(settings, 'ERROR_RATE_CACHE_PREFIX', 'ERROR_RATE')
            
            # Test if the cache works
            cache_key = '%s_%s' % (prefix, key)
            try:
                cache.set(prefix, 1, 1)
                use_cache = cache.get(prefix) == 1
            except:
                use_cache = False
            
            if use_cache:
                duplicate = cache.get(cache_key) == 1
                cache.set(cache_key, 1, rate)
            else:
                min_date = datetime.now() - timedelta(seconds=rate)
                max_keys = getattr(settings, 'ERROR_RATE_KEY_LIMIT', 100)
                duplicate = (key in self._errors and self._errors[key] >= min_date)
                self._errors = dict(filter(lambda x: x[1] >= min_date, 
                                          sorted(self._errors.items(), 
                                                 key=lambda x: x[1]))[0-max_keys:])
                if not duplicate:
                    self._errors[key] = datetime.now()

        return not duplicate




Example LOGGING config in your settings.py --

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler',
            'filters': ['ratelimit'],
        }
    },
    'filters': {
        'ratelimit': {
            '()': 'ratelimit.RateLimitFilter',
        }
    },
    'loggers': {
        'django.request':{
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    },
}

More like this

  1. Month / Year SelectDateWidget based on django SelectDateWidget by pierreben 3 weeks, 4 days ago
  2. Python Django CRUD Example Tutorial by tuts_station 1 month, 1 week ago
  3. Browser-native date input field by kytta 2 months, 3 weeks ago
  4. Generate and render HTML Table by LLyaudet 3 months ago
  5. My firs Snippets by GutemaG 3 months, 1 week ago

Comments

Please login first before commenting.