backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Merge branch 'master' into crawl-log

Browse Source 
* master:
  no SIGQUIT on windows, so no SIGQUIT handler
  https://github.com/internetarchive/warcprox/pull/32 warrants a version bump
  fix --size option (https://github.com/internetarchive/warcprox/issues/31)
  fix --playback-port option (https://github.com/internetarchive/warcprox/issues/29)
This commit is contained in:
Noah Levitt 2017-09-07 12:28:07 -07:00
commit a3f84097ee
3 changed files with 11 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -49,7 +49,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.1b1.dev94', version='2.2b1.dev98',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

16
warcprox/main.py
View File

@ -81,9 +81,9 @@ def _build_arg_parser(prog=os.path.basename(sys.argv[0])):
help='write gzip-compressed warc records') help='write gzip-compressed warc records')
arg_parser.add_argument('-n', '--prefix', dest='prefix', arg_parser.add_argument('-n', '--prefix', dest='prefix',
default='WARCPROX', help='WARC filename prefix') default='WARCPROX', help='WARC filename prefix')
arg_parser.add_argument('-s', '--size', dest='size', arg_parser.add_argument(
default=1000*1000*1000, type=int, '-s', '--size', dest='rollover_size', default=1000*1000*1000,
help='WARC file rollover size threshold in bytes') type=int, help='WARC file rollover size threshold in bytes')
arg_parser.add_argument('--rollover-idle-time', arg_parser.add_argument('--rollover-idle-time',
dest='rollover_idle_time', default=None, type=int, dest='rollover_idle_time', default=None, type=int,
help="WARC file rollover idle time threshold in seconds (so that Friday's last open WARC doesn't sit there all weekend waiting for more data)") help="WARC file rollover idle time threshold in seconds (so that Friday's last open WARC doesn't sit there all weekend waiting for more data)")
@ -225,9 +225,7 @@ def init_controller(args):
playback_index_db = warcprox.playback.PlaybackIndexDb( playback_index_db = warcprox.playback.PlaybackIndexDb(
args.playback_index_db_file, options=options) args.playback_index_db_file, options=options)
playback_proxy = warcprox.playback.PlaybackProxy( playback_proxy = warcprox.playback.PlaybackProxy(
server_address=(args.address, args.playback_port), ca=ca, ca=ca, playback_index_db=playback_index_db, options=options)
playback_index_db=playback_index_db, warcs_dir=args.directory,
options=options)
listeners.append(playback_index_db) listeners.append(playback_index_db)
else: else:
playback_index_db = None playback_index_db = None
@ -306,7 +304,11 @@ def main(argv=sys.argv):
signal.signal(signal.SIGTERM, lambda a,b: controller.stop.set()) signal.signal(signal.SIGTERM, lambda a,b: controller.stop.set())
signal.signal(signal.SIGINT, lambda a,b: controller.stop.set()) signal.signal(signal.SIGINT, lambda a,b: controller.stop.set())
signal.signal(signal.SIGQUIT, dump_state) try:
signal.signal(signal.SIGQUIT, dump_state)
except AttributeError:
# SIGQUIT does not exist on some platforms (windows)
pass
controller.run_until_shutdown() controller.run_until_shutdown()

3
warcprox/playback.py
View File

@ -210,7 +210,6 @@ class PlaybackProxyHandler(MitmProxyHandler):
class PlaybackProxy(socketserver.ThreadingMixIn, http_server.HTTPServer): class PlaybackProxy(socketserver.ThreadingMixIn, http_server.HTTPServer):
logger = logging.getLogger("warcprox.playback.PlaybackProxy") logger = logging.getLogger("warcprox.playback.PlaybackProxy")
def __init__(self, ca=None, playback_index_db=None, options=warcprox.Options()): def __init__(self, ca=None, playback_index_db=None, options=warcprox.Options()):
server_address = (options.address or 'localhost', options.playback_port if options.playback_port is not None else 8001) server_address = (options.address or 'localhost', options.playback_port if options.playback_port is not None else 8001)
http_server.HTTPServer.__init__(self, server_address, PlaybackProxyHandler, bind_and_activate=True) http_server.HTTPServer.__init__(self, server_address, PlaybackProxyHandler, bind_and_activate=True)
@ -231,7 +230,7 @@ class PlaybackProxy(socketserver.ThreadingMixIn, http_server.HTTPServer):
class PlaybackIndexDb(object): class PlaybackIndexDb(object):
logger = logging.getLogger("warcprox.playback.PlaybackIndexDb") logger = logging.getLogger("warcprox.playback.PlaybackIndexDb")
def __init__(self, file='./warcprox.sqlite'): def __init__(self, file='./warcprox.sqlite', options=warcprox.Options()):
self.file = file self.file = file
self._lock = threading.RLock() self._lock = threading.RLock()