backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

fixes to make tests pass

Browse Source
This commit is contained in:
Noah Levitt 2018-01-17 15:33:41 -08:00
parent 9c5a5eda99
commit a974ec86fa
2 changed files with 19 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
tests/test_writer.py
View File

@ -93,16 +93,15 @@ def test_special_dont_write_prefix():
logging.debug('cd %s', tmpdir) logging.debug('cd %s', tmpdir)
os.chdir(tmpdir) os.chdir(tmpdir)
inq = warcprox.TimestampedQueue(maxsize=1) wwt = warcprox.writerthread.WarcWriterThread(Options(prefix='-'))
outq = warcprox.TimestampedQueue(maxsize=1) wwt.inq = warcprox.TimestampedQueue(maxsize=1)
wwt = warcprox.writerthread.WarcWriterThread( wwt.outq = warcprox.TimestampedQueue(maxsize=1)
inq, outq, Options(prefix='-'))
try: try:
wwt.start() wwt.start()
# not to be written due to default prefix # not to be written due to default prefix
recorder = ProxyingRecorder(io.BytesIO(b'some payload'), None) recorder = ProxyingRecorder(io.BytesIO(b'some payload'), None)
recorder.read() recorder.read()
inq.put(RecordedUrl( wwt.inq.put(RecordedUrl(
url='http://example.com/no', content_type='text/plain', url='http://example.com/no', content_type='text/plain',
status=200, client_ip='127.0.0.2', request_data=b'abc', status=200, client_ip='127.0.0.2', request_data=b'abc',
response_recorder=recorder, remote_ip='127.0.0.3', response_recorder=recorder, remote_ip='127.0.0.3',
@ -111,31 +110,31 @@ def test_special_dont_write_prefix():
# to be written due to warcprox-meta prefix # to be written due to warcprox-meta prefix
recorder = ProxyingRecorder(io.BytesIO(b'some payload'), None) recorder = ProxyingRecorder(io.BytesIO(b'some payload'), None)
recorder.read() recorder.read()
inq.put(RecordedUrl( wwt.inq.put(RecordedUrl(
url='http://example.com/yes', content_type='text/plain', url='http://example.com/yes', content_type='text/plain',
status=200, client_ip='127.0.0.2', request_data=b'abc', status=200, client_ip='127.0.0.2', request_data=b'abc',
response_recorder=recorder, remote_ip='127.0.0.3', response_recorder=recorder, remote_ip='127.0.0.3',
timestamp=datetime.utcnow(), timestamp=datetime.utcnow(),
payload_digest=recorder.block_digest, payload_digest=recorder.block_digest,
warcprox_meta={'warc-prefix': 'normal-warc-prefix'})) warcprox_meta={'warc-prefix': 'normal-warc-prefix'}))
recorded_url = outq.get(timeout=10) recorded_url = wwt.outq.get(timeout=10)
assert not recorded_url.warc_records assert not recorded_url.warc_records
recorded_url = outq.get(timeout=10) recorded_url = wwt.outq.get(timeout=10)
assert recorded_url.warc_records assert recorded_url.warc_records
assert outq.empty() assert wwt.outq.empty()
finally: finally:
wwt.stop.set() wwt.stop.set()
wwt.join() wwt.join()
inq = warcprox.TimestampedQueue(maxsize=1) wwt = warcprox.writerthread.WarcWriterThread()
outq = warcprox.TimestampedQueue(maxsize=1) wwt.inq = warcprox.TimestampedQueue(maxsize=1)
wwt = warcprox.writerthread.WarcWriterThread(inq, outq) wwt.outq = warcprox.TimestampedQueue(maxsize=1)
try: try:
wwt.start() wwt.start()
# to be written due to default prefix # to be written due to default prefix
recorder = ProxyingRecorder(io.BytesIO(b'some payload'), None) recorder = ProxyingRecorder(io.BytesIO(b'some payload'), None)
recorder.read() recorder.read()
inq.put(RecordedUrl( wwt.inq.put(RecordedUrl(
url='http://example.com/yes', content_type='text/plain', url='http://example.com/yes', content_type='text/plain',
status=200, client_ip='127.0.0.2', request_data=b'abc', status=200, client_ip='127.0.0.2', request_data=b'abc',
response_recorder=recorder, remote_ip='127.0.0.3', response_recorder=recorder, remote_ip='127.0.0.3',
@ -144,18 +143,18 @@ def test_special_dont_write_prefix():
# not to be written due to warcprox-meta prefix # not to be written due to warcprox-meta prefix
recorder = ProxyingRecorder(io.BytesIO(b'some payload'), None) recorder = ProxyingRecorder(io.BytesIO(b'some payload'), None)
recorder.read() recorder.read()
inq.put(RecordedUrl( wwt.inq.put(RecordedUrl(
url='http://example.com/no', content_type='text/plain', url='http://example.com/no', content_type='text/plain',
status=200, client_ip='127.0.0.2', request_data=b'abc', status=200, client_ip='127.0.0.2', request_data=b'abc',
response_recorder=recorder, remote_ip='127.0.0.3', response_recorder=recorder, remote_ip='127.0.0.3',
timestamp=datetime.utcnow(), timestamp=datetime.utcnow(),
payload_digest=recorder.block_digest, payload_digest=recorder.block_digest,
warcprox_meta={'warc-prefix': '-'})) warcprox_meta={'warc-prefix': '-'}))
recorded_url = outq.get(timeout=10) recorded_url = wwt.outq.get(timeout=10)
assert recorded_url.warc_records assert recorded_url.warc_records
recorded_url = outq.get(timeout=10) recorded_url = wwt.outq.get(timeout=10)
assert not recorded_url.warc_records assert not recorded_url.warc_records
assert outq.empty() assert wwt.outq.empty()
finally: finally:
wwt.stop.set() wwt.stop.set()
wwt.join() wwt.join()

5
warcprox/main.py
View File

@ -311,7 +311,8 @@ def ensure_rethinkdb_tables(argv=None):
svcreg = doublethink.ServiceRegistry(rr, table=parsed.table) svcreg = doublethink.ServiceRegistry(rr, table=parsed.table)
did_something = True did_something = True
if args.rethinkdb_stats_url: if args.rethinkdb_stats_url:
stats_db = warcprox.stats.RethinkStatsDb(options=options) stats_db = warcprox.stats.RethinkStatsProcessor(options=options)
stats_db._ensure_db_table()
did_something = True did_something = True
if args.rethinkdb_dedup_url: if args.rethinkdb_dedup_url:
dedup_db = warcprox.dedup.RethinkDedupDb(options=options) dedup_db = warcprox.dedup.RethinkDedupDb(options=options)
@ -322,7 +323,7 @@ def ensure_rethinkdb_tables(argv=None):
if args.rethinkdb_trough_db_url: if args.rethinkdb_trough_db_url:
dedup_db = warcprox.dedup.TroughDedupDb(options) dedup_db = warcprox.dedup.TroughDedupDb(options)
logging.warn( logging.warn(
'trough it responsible for creating most of the rethinkdb ' 'trough is responsible for creating most of the rethinkdb '
'tables that it uses') 'tables that it uses')
did_something = True did_something = True