backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

WARCPROX_WRITE_RECORD respect buffer size setting

Browse Source
This commit is contained in:
Noah Levitt 2018-08-20 11:09:53 -07:00
parent e4befeec14
commit 2e71d86072
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
warcprox/warcproxy.py
View File

@ -287,9 +287,11 @@ class WarcProxyHandler(warcprox.mitmproxy.MitmProxyHandler):
and (warc_type or 'WARC-Type' in self.headers)): and (warc_type or 'WARC-Type' in self.headers)):
timestamp = datetime.datetime.utcnow() timestamp = datetime.datetime.utcnow()
request_data = tempfile.SpooledTemporaryFile(max_size=524288) request_data = tempfile.SpooledTemporaryFile(
max_size=self.options.tmp_file_max_memory_size)
payload_digest = hashlib.new(self.server.digest_algorithm) payload_digest = hashlib.new(self.server.digest_algorithm)
# XXX we don't support chunked uploads for now
length = int(self.headers['Content-Length']) length = int(self.headers['Content-Length'])
buf = self.rfile.read(min(65536, length - request_data.tell())) buf = self.rfile.read(min(65536, length - request_data.tell()))
while buf != b'': while buf != b'':