From 8fea623c52a27e1efce3074838ffa330869d0084 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Mon, 21 Aug 2017 22:29:13 -0700 Subject: [PATCH] optimization: redisindexsource scan_keys: use cached key list, if available bump requirements to gevent 1.2.2 --- pywb/warcserver/index/indexsource.py | 8 ++++++-- requirements.txt | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pywb/warcserver/index/indexsource.py b/pywb/warcserver/index/indexsource.py index 9ef408f2..66ca7216 100644 --- a/pywb/warcserver/index/indexsource.py +++ b/pywb/warcserver/index/indexsource.py @@ -292,8 +292,12 @@ class RedisIndexSource(BaseIndexSource): key = res_template(member_key, params) - keys = self.redis.smembers(key) - params['scan:' + key] = keys + scan_key = 'scan:' + key + # check if already have keys to avoid extra smembers call + keys = params.get(scan_key) + if not keys: + keys = self.redis.smembers(key) + params[scan_key] = keys match_templ = match_templ.encode('utf-8') diff --git a/requirements.txt b/requirements.txt index c32c542d..175a4fc6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,6 @@ brotlipy pyyaml werkzeug webencodings -gevent==1.2.1 +gevent==1.2.2 webassets==0.12.1 portalocker