From 3ea5c36e7f2db301cf5161e8c8c02d1c3a4f9840 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 9 Jan 2019 15:10:37 -0800 Subject: [PATCH 1/4] add idna as dep with acceptable to other deps because my understanding is that pip cannot fully resolve version constraints for indirect dependencies. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 27dde45..b5f91e1 100755 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ deps = [ 'requests>=2.0.1', 'PySocks>=1.6.8', 'cryptography>=2.3', + 'idna>=2.5,<2.8', ] try: import concurrent.futures @@ -40,7 +41,7 @@ except: setuptools.setup( name='warcprox', - version='2.4b3.dev192', + version='2.4b3.dev193', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', From e07ee3630ebf4ef62778183a8d5bbadd0bc9d41c Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 9 Jan 2019 15:15:37 -0800 Subject: [PATCH 2/4] 2.4b3 for pypi --- setup.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index b5f91e1..e97d899 100755 --- a/setup.py +++ b/setup.py @@ -25,13 +25,13 @@ import setuptools deps = [ 'certauth==1.1.6', - 'warctools>=4.10.0', - 'urlcanon>=0.1.dev16', - 'doublethink>=0.2.0.dev87', - 'urllib3>=1.23', - 'requests>=2.0.1', - 'PySocks>=1.6.8', - 'cryptography>=2.3', + 'warctools>=4.10.0,<=4.10.0', + 'urlcanon>=0.1.dev16,<=0.1.dev23', + 'doublethink>=0.2.0.dev87,<=0.2.0.dev88', + 'urllib3>=1.23,<=1.23', + 'requests>=2.0.1,<=2.19.1', + 'PySocks>=1.6.8,<=1.6.8', + 'cryptography>=2.3,<=2.3.1', 'idna>=2.5,<2.8', ] try: @@ -41,7 +41,7 @@ except: setuptools.setup( name='warcprox', - version='2.4b3.dev193', + version='2.4b3', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', @@ -67,6 +67,7 @@ setuptools.setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: Proxy Servers', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Python Modules', From a780f1774c0e2943062c714e131e9d1dcde529c7 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 17 Jan 2019 17:15:33 -0800 Subject: [PATCH 3/4] back to dev version number --- setup.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index e97d899..80b70d2 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ ''' setup.py - setuptools installation configuration for warcprox -Copyright (C) 2013-2018 Internet Archive +Copyright (C) 2013-2019 Internet Archive This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -26,13 +26,13 @@ import setuptools deps = [ 'certauth==1.1.6', 'warctools>=4.10.0,<=4.10.0', - 'urlcanon>=0.1.dev16,<=0.1.dev23', - 'doublethink>=0.2.0.dev87,<=0.2.0.dev88', - 'urllib3>=1.23,<=1.23', - 'requests>=2.0.1,<=2.19.1', - 'PySocks>=1.6.8,<=1.6.8', - 'cryptography>=2.3,<=2.3.1', - 'idna>=2.5,<2.8', + 'urlcanon>=0.1.dev16', + 'doublethink>=0.2.0.dev87', + 'urllib3>=1.23', + 'requests>=2.0.1', + 'PySocks>=1.6.8', + 'cryptography>=2.3', + 'idna>=2.5', ] try: import concurrent.futures @@ -41,7 +41,7 @@ except: setuptools.setup( name='warcprox', - version='2.4b3', + version='2.4b4.dev194', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', From cb72af015a7fc014ccfb58b78913fe6b758f78cb Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 21 Jan 2019 10:37:09 -0800 Subject: [PATCH 4/4] fix idle rollover --- setup.py | 2 +- warcprox/writerthread.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 80b70d2..ceb9886 100755 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ except: setuptools.setup( name='warcprox', - version='2.4b4.dev194', + version='2.4b4.dev195', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/writerthread.py b/warcprox/writerthread.py index f6ac277..b929a7f 100644 --- a/warcprox/writerthread.py +++ b/warcprox/writerthread.py @@ -57,6 +57,7 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor): self.writer_pool.close_for_prefix(prefix) except queue.Empty: break + self.writer_pool.maybe_idle_rollover() super()._get_process_put() def close_for_prefix(self, prefix=None):