From 3132bfa7f4ff05c507c96c0a62cd14e602f86737 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 30 Oct 2015 13:15:07 -0700 Subject: [PATCH] cache: add a simple RedisCache implementation (alongside local and uwsgi) proxy_ip_resolver: add option to use RedisCache if redis_cache_key set in config proxy_ip_resolver: add 'delete' option to delete ip from cache, closes #145 --- pywb/framework/cache.py | 29 ++++++++++++++++++++++++++++- pywb/framework/proxy_resolvers.py | 15 ++++++++++----- tests/test_proxy_http_ip.py | 11 +++++++++++ 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/pywb/framework/cache.py b/pywb/framework/cache.py index 3582dedc..618baedd 100644 --- a/pywb/framework/cache.py +++ b/pywb/framework/cache.py @@ -5,6 +5,9 @@ except ImportError: uwsgi_cache = False +from redis import StrictRedis + + #================================================================= class UwsgiCache(object): # pragma: no cover def __setitem__(self, item, value): @@ -27,7 +30,31 @@ class DefaultCache(dict): #================================================================= -def create_cache(): +class RedisCache(object): + def __init__(self, redis_url): + # must be of the form redis://host:port/db/key + redis_url, key = redis_url.rsplit('/', 1) + self.redis = StrictRedis.from_url(redis_url) + self.key = key + + def __setitem__(self, item, value): + self.redis.hset(self.key, item, value) + + def __getitem__(self, item): + return self.redis.hget(self.key, item) + + def __contains__(self, item): + return self.redis.hexists(self.key, item) + + def __delitem__(self, item): + self.redis.hdel(self.key, item) + + +#================================================================= +def create_cache(redis_url_key=None): + if redis_url_key: + return RedisCache(redis_url_key) + if uwsgi_cache: # pragma: no cover return UwsgiCache() else: diff --git a/pywb/framework/proxy_resolvers.py b/pywb/framework/proxy_resolvers.py index 46d53f15..44c32144 100644 --- a/pywb/framework/proxy_resolvers.py +++ b/pywb/framework/proxy_resolvers.py @@ -114,7 +114,7 @@ class ProxyAuthResolver(BaseCollResolver): class IPCacheResolver(BaseCollResolver): def __init__(self, routes, config): super(IPCacheResolver, self).__init__(routes, config) - self.cache = create_cache() + self.cache = create_cache(config.get('redis_cache_key')) self.magic_name = config['magic_name'] def _get_ip(self, env): @@ -131,17 +131,22 @@ class IPCacheResolver(BaseCollResolver): def get_proxy_coll_ts(self, env): ip = env['REMOTE_ADDR'] qs = env.get('pywb.proxy_query') + if qs: res = urlparse.parse_qs(qs) if 'ip' in res: ip = res['ip'][0] - if 'coll' in res: - self.cache[ip + ':c'] = res['coll'][0] + if 'delete' in res: + del self.cache[ip + ':c'] + del self.cache[ip + ':t'] + else: + if 'coll' in res: + self.cache[ip + ':c'] = res['coll'][0] - if 'ts' in res: - self.cache[ip + ':t'] = res['ts'][0] + if 'ts' in res: + self.cache[ip + ':t'] = res['ts'][0] coll = self.cache[ip + ':c'] ts = self.cache[ip + ':t'] diff --git a/tests/test_proxy_http_ip.py b/tests/test_proxy_http_ip.py index 30127fe6..6b62228c 100644 --- a/tests/test_proxy_http_ip.py +++ b/tests/test_proxy_http_ip.py @@ -81,3 +81,14 @@ class TestProxyIPResolver(BaseIntegration): resp = self.get_url('http://www.iana.org/', '127.0.0.3') self._assert_basic_html(resp) assert '"20140127171238"' in resp.body + + def test_proxy_ip_delete_ip(self): + resp = self.get_url('http://info.pywb.proxy/') + assert resp.json == {'ip': '127.0.0.1', 'coll': 'all', 'ts': '1996'} + + resp = self.get_url('http://info.pywb.proxy/set?delete=true') + assert resp.json == {'ip': '127.0.0.1', 'coll': None, 'ts': None} + + resp = self.get_url('http://info.pywb.proxy/') + assert resp.json == {'ip': '127.0.0.1', 'coll': None, 'ts': None} +