Login

Error rate limiter

Author:
s29
Posted:
October 25, 2010
Language:
Python
Version:
Not specified
Score:
2 (after 2 ratings)

Prevents error flooding on high traffic production sites. Particularly useful to prevent clogging email servers etc.

See discussion and closed ticket.

Uses memcache if it can, or falls back to local, in-process memory where unavailable (down, etc).

Tweak to your needs using setting ERROR_RATE_LIMIT (seconds).

Requires Django 1.3+ or trunk r13981+ 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
In ratelimit.py --


import traceback
try:
    from hashlib import md5
except ImportError:
    from md5 import md5
from datetime import datetime, timedelta


class RateLimitFilter(object):
    
    _errors = {}
        
    def filter(self, record):
        from django.conf import settings
        from django.core.cache import cache                               
        
        tb = '\n'.join(traceback.format_exception(*record.exc_info))

        # Track duplicate errors
        duplicate = False
        rate = getattr(settings, 'ERROR_RATE_LIMIT', 10)  # seconds
        if rate > 0:
            key = md5(tb).hexdigest()
            prefix = getattr(settings, 'ERROR_RATE_CACHE_PREFIX', 'ERROR_RATE')
            
            # Test if the cache works
            cache_key = '%s_%s' % (prefix, key)
            try:
                cache.set(prefix, 1, 1)
                use_cache = cache.get(prefix) == 1
            except:
                use_cache = False
            
            if use_cache:
                duplicate = cache.get(cache_key) == 1
                cache.set(cache_key, 1, rate)
            else:
                min_date = datetime.now() - timedelta(seconds=rate)
                max_keys = getattr(settings, 'ERROR_RATE_KEY_LIMIT', 100)
                duplicate = (key in self._errors and self._errors[key] >= min_date)
                self._errors = dict(filter(lambda x: x[1] >= min_date, 
                                          sorted(self._errors.items(), 
                                                 key=lambda x: x[1]))[0-max_keys:])
                if not duplicate:
                    self._errors[key] = datetime.now()

        return not duplicate




Example LOGGING config in your settings.py --

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler',
            'filters': ['ratelimit'],
        }
    },
    'filters': {
        'ratelimit': {
            '()': 'ratelimit.RateLimitFilter',
        }
    },
    'loggers': {
        'django.request':{
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    },
}

More like this

  1. Template tag - list punctuation for a list of items by shapiromatron 3 months, 2 weeks ago
  2. JSONRequestMiddleware adds a .json() method to your HttpRequests by cdcarter 3 months, 3 weeks ago
  3. Serializer factory with Django Rest Framework by julio 10 months, 2 weeks ago
  4. Image compression before saving the new model / work with JPG, PNG by Schleidens 11 months, 1 week ago
  5. Help text hyperlinks by sa2812 12 months ago

Comments

Please login first before commenting.