From a974ec86fa6d37c0ae2db15be3273a3154555fcd Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 17 Jan 2018 15:33:41 -0800 Subject: [PATCH] fixes to make tests pass --- tests/test_writer.py | 33 ++++++++++++++++----------------- warcprox/main.py | 5 +++-- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/test_writer.py b/tests/test_writer.py index 9c37424..0a18c33 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -93,16 +93,15 @@ def test_special_dont_write_prefix(): logging.debug('cd %s', tmpdir) os.chdir(tmpdir) - inq = warcprox.TimestampedQueue(maxsize=1) - outq = warcprox.TimestampedQueue(maxsize=1) - wwt = warcprox.writerthread.WarcWriterThread( - inq, outq, Options(prefix='-')) + wwt = warcprox.writerthread.WarcWriterThread(Options(prefix='-')) + wwt.inq = warcprox.TimestampedQueue(maxsize=1) + wwt.outq = warcprox.TimestampedQueue(maxsize=1) try: wwt.start() # not to be written due to default prefix recorder = ProxyingRecorder(io.BytesIO(b'some payload'), None) recorder.read() - inq.put(RecordedUrl( + wwt.inq.put(RecordedUrl( url='http://example.com/no', content_type='text/plain', status=200, client_ip='127.0.0.2', request_data=b'abc', response_recorder=recorder, remote_ip='127.0.0.3', @@ -111,31 +110,31 @@ def test_special_dont_write_prefix(): # to be written due to warcprox-meta prefix recorder = ProxyingRecorder(io.BytesIO(b'some payload'), None) recorder.read() - inq.put(RecordedUrl( + wwt.inq.put(RecordedUrl( url='http://example.com/yes', content_type='text/plain', status=200, client_ip='127.0.0.2', request_data=b'abc', response_recorder=recorder, remote_ip='127.0.0.3', timestamp=datetime.utcnow(), payload_digest=recorder.block_digest, warcprox_meta={'warc-prefix': 'normal-warc-prefix'})) - recorded_url = outq.get(timeout=10) + recorded_url = wwt.outq.get(timeout=10) assert not recorded_url.warc_records - recorded_url = outq.get(timeout=10) + recorded_url = wwt.outq.get(timeout=10) assert recorded_url.warc_records - assert outq.empty() + assert wwt.outq.empty() finally: wwt.stop.set() wwt.join() - inq = warcprox.TimestampedQueue(maxsize=1) - outq = warcprox.TimestampedQueue(maxsize=1) - wwt = warcprox.writerthread.WarcWriterThread(inq, outq) + wwt = warcprox.writerthread.WarcWriterThread() + wwt.inq = warcprox.TimestampedQueue(maxsize=1) + wwt.outq = warcprox.TimestampedQueue(maxsize=1) try: wwt.start() # to be written due to default prefix recorder = ProxyingRecorder(io.BytesIO(b'some payload'), None) recorder.read() - inq.put(RecordedUrl( + wwt.inq.put(RecordedUrl( url='http://example.com/yes', content_type='text/plain', status=200, client_ip='127.0.0.2', request_data=b'abc', response_recorder=recorder, remote_ip='127.0.0.3', @@ -144,18 +143,18 @@ def test_special_dont_write_prefix(): # not to be written due to warcprox-meta prefix recorder = ProxyingRecorder(io.BytesIO(b'some payload'), None) recorder.read() - inq.put(RecordedUrl( + wwt.inq.put(RecordedUrl( url='http://example.com/no', content_type='text/plain', status=200, client_ip='127.0.0.2', request_data=b'abc', response_recorder=recorder, remote_ip='127.0.0.3', timestamp=datetime.utcnow(), payload_digest=recorder.block_digest, warcprox_meta={'warc-prefix': '-'})) - recorded_url = outq.get(timeout=10) + recorded_url = wwt.outq.get(timeout=10) assert recorded_url.warc_records - recorded_url = outq.get(timeout=10) + recorded_url = wwt.outq.get(timeout=10) assert not recorded_url.warc_records - assert outq.empty() + assert wwt.outq.empty() finally: wwt.stop.set() wwt.join() diff --git a/warcprox/main.py b/warcprox/main.py index 860e816..beeed35 100644 --- a/warcprox/main.py +++ b/warcprox/main.py @@ -311,7 +311,8 @@ def ensure_rethinkdb_tables(argv=None): svcreg = doublethink.ServiceRegistry(rr, table=parsed.table) did_something = True if args.rethinkdb_stats_url: - stats_db = warcprox.stats.RethinkStatsDb(options=options) + stats_db = warcprox.stats.RethinkStatsProcessor(options=options) + stats_db._ensure_db_table() did_something = True if args.rethinkdb_dedup_url: dedup_db = warcprox.dedup.RethinkDedupDb(options=options) @@ -322,7 +323,7 @@ def ensure_rethinkdb_tables(argv=None): if args.rethinkdb_trough_db_url: dedup_db = warcprox.dedup.TroughDedupDb(options) logging.warn( - 'trough it responsible for creating most of the rethinkdb ' + 'trough is responsible for creating most of the rethinkdb ' 'tables that it uses') did_something = True