backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

cache: add a simple RedisCache implementation (alongside local and uwsgi)

Browse Source 
proxy_ip_resolver: add option to use RedisCache if redis_cache_key set in config
proxy_ip_resolver: add 'delete' option to delete ip from cache, closes #145
This commit is contained in:
Ilya Kreymer 2015-10-30 13:15:07 -07:00
parent 16cf997a07
commit 3132bfa7f4
3 changed files with 49 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
pywb/framework/cache.py
View File

@ -5,6 +5,9 @@ except ImportError:
uwsgi_cache = False uwsgi_cache = False
from redis import StrictRedis
#================================================================= #=================================================================
class UwsgiCache(object): # pragma: no cover class UwsgiCache(object): # pragma: no cover
def __setitem__(self, item, value): def __setitem__(self, item, value):
@ -27,7 +30,31 @@ class DefaultCache(dict):
#================================================================= #=================================================================
def create_cache(): class RedisCache(object):
def __init__(self, redis_url):
# must be of the form redis://host:port/db/key
redis_url, key = redis_url.rsplit('/', 1)
self.redis = StrictRedis.from_url(redis_url)
self.key = key
def __setitem__(self, item, value):
self.redis.hset(self.key, item, value)
def __getitem__(self, item):
return self.redis.hget(self.key, item)
def __contains__(self, item):
return self.redis.hexists(self.key, item)
def __delitem__(self, item):
self.redis.hdel(self.key, item)
#=================================================================
def create_cache(redis_url_key=None):
if redis_url_key:
return RedisCache(redis_url_key)
if uwsgi_cache: # pragma: no cover if uwsgi_cache: # pragma: no cover
return UwsgiCache() return UwsgiCache()
else: else:

15
pywb/framework/proxy_resolvers.py
View File

@ -114,7 +114,7 @@ class ProxyAuthResolver(BaseCollResolver):
class IPCacheResolver(BaseCollResolver): class IPCacheResolver(BaseCollResolver):
def __init__(self, routes, config): def __init__(self, routes, config):
super(IPCacheResolver, self).__init__(routes, config) super(IPCacheResolver, self).__init__(routes, config)
self.cache = create_cache() self.cache = create_cache(config.get('redis_cache_key'))
self.magic_name = config['magic_name'] self.magic_name = config['magic_name']
def _get_ip(self, env): def _get_ip(self, env):
@ -131,17 +131,22 @@ class IPCacheResolver(BaseCollResolver):
def get_proxy_coll_ts(self, env): def get_proxy_coll_ts(self, env):
ip = env['REMOTE_ADDR'] ip = env['REMOTE_ADDR']
qs = env.get('pywb.proxy_query') qs = env.get('pywb.proxy_query')
if qs: if qs:
res = urlparse.parse_qs(qs) res = urlparse.parse_qs(qs)
if 'ip' in res: if 'ip' in res:
ip = res['ip'][0] ip = res['ip'][0]
if 'coll' in res: if 'delete' in res:
self.cache[ip + ':c'] = res['coll'][0] del self.cache[ip + ':c']
del self.cache[ip + ':t']
else:
if 'coll' in res:
self.cache[ip + ':c'] = res['coll'][0]
if 'ts' in res: if 'ts' in res:
self.cache[ip + ':t'] = res['ts'][0] self.cache[ip + ':t'] = res['ts'][0]
coll = self.cache[ip + ':c'] coll = self.cache[ip + ':c']
ts = self.cache[ip + ':t'] ts = self.cache[ip + ':t']

11
tests/test_proxy_http_ip.py
View File

@ -81,3 +81,14 @@ class TestProxyIPResolver(BaseIntegration):
resp = self.get_url('http://www.iana.org/', '127.0.0.3') resp = self.get_url('http://www.iana.org/', '127.0.0.3')
self._assert_basic_html(resp) self._assert_basic_html(resp)
assert '"20140127171238"' in resp.body assert '"20140127171238"' in resp.body
def test_proxy_ip_delete_ip(self):
resp = self.get_url('http://info.pywb.proxy/')
assert resp.json == {'ip': '127.0.0.1', 'coll': 'all', 'ts': '1996'}
resp = self.get_url('http://info.pywb.proxy/set?delete=true')
assert resp.json == {'ip': '127.0.0.1', 'coll': None, 'ts': None}
resp = self.get_url('http://info.pywb.proxy/')
assert resp.json == {'ip': '127.0.0.1', 'coll': None, 'ts': None}