From aa80cd6881169f1cf60fd4caee0347728284ceb3 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Mon, 21 Mar 2016 11:50:01 -0700 Subject: [PATCH] recorder: add simple recorder config indexing to redis --- recorder/test/rec.ini | 17 +++++++++++++++++ recorder/test/simplerec.py | 23 +++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 recorder/test/rec.ini create mode 100644 recorder/test/simplerec.py diff --git a/recorder/test/rec.ini b/recorder/test/rec.ini new file mode 100644 index 00000000..06a5f8ea --- /dev/null +++ b/recorder/test/rec.ini @@ -0,0 +1,17 @@ +[uwsgi] +if-not-env = PORT +http-socket = :8010 +endif = + +master = true +buffer-size = 65536 +die-on-term = true + +if-env = VIRTUAL_ENV +venv = $(VIRTUAL_ENV) +endif = + +gevent = 100 +#gevent-early-monkey-patch = + +wsgi = recorder.test.simplerec diff --git a/recorder/test/simplerec.py b/recorder/test/simplerec.py new file mode 100644 index 00000000..84f83736 --- /dev/null +++ b/recorder/test/simplerec.py @@ -0,0 +1,23 @@ +from gevent import monkey; monkey.patch_all() + +from recorder.recorderapp import RecorderApp +from recorder.redisindexer import WritableRedisIndexer + +from recorder.warcwriter import MultiFileWARCWriter +from recorder.filters import SkipDupePolicy + +upstream_url = 'http://localhost:8080' + +target = './_recordings/' + +dedup_index = WritableRedisIndexer('redis://localhost/2/rec:cdxj', + file_key_template='rec:warc', + rel_path_template=target, + dupe_policy=SkipDupePolicy()) + +recorder_app = RecorderApp(upstream_url, + MultiFileWARCWriter(target, dedup_index=dedup_index), + accept_colls='live') + +application = recorder_app +