backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

trough dedup - handle case of no warc records written

Browse Source
This commit is contained in:
Noah Levitt 2017-11-30 12:55:39 -08:00
parent 61a7c234e8
commit c5f33bda7a
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
warcprox/dedup.py
View File

@ -292,11 +292,10 @@ class TroughDedupDb(object):
return None return None
def notify(self, recorded_url, records): def notify(self, recorded_url, records):
if (records[0].get_header(warctools.WarcRecord.TYPE) == warctools.WarcRecord.RESPONSE if (records and records[0].type == b'response'
and recorded_url.response_recorder.payload_size() > 0): and recorded_url.response_recorder.payload_size() > 0):
digest_key = warcprox.digest_str( digest_key = warcprox.digest_str(
recorded_url.payload_digest, recorded_url.payload_digest, self.options.base32)
self.options.base32)
if recorded_url.warcprox_meta and 'captures-bucket' in recorded_url.warcprox_meta: if recorded_url.warcprox_meta and 'captures-bucket' in recorded_url.warcprox_meta:
self.save( self.save(
digest_key, records[0], digest_key, records[0],