From 2e71d86072c0d75eabb19105a57a7a7822aa5cf4 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 20 Aug 2018 11:09:53 -0700 Subject: [PATCH] WARCPROX_WRITE_RECORD respect buffer size setting --- warcprox/warcproxy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/warcprox/warcproxy.py b/warcprox/warcproxy.py index 2ccfa13..17e0ebc 100644 --- a/warcprox/warcproxy.py +++ b/warcprox/warcproxy.py @@ -287,9 +287,11 @@ class WarcProxyHandler(warcprox.mitmproxy.MitmProxyHandler): and (warc_type or 'WARC-Type' in self.headers)): timestamp = datetime.datetime.utcnow() - request_data = tempfile.SpooledTemporaryFile(max_size=524288) + request_data = tempfile.SpooledTemporaryFile( + max_size=self.options.tmp_file_max_memory_size) payload_digest = hashlib.new(self.server.digest_algorithm) + # XXX we don't support chunked uploads for now length = int(self.headers['Content-Length']) buf = self.rfile.read(min(65536, length - request_data.tell())) while buf != b'':