backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Merge branch 'master' into travis-py37

Browse Source 
* master:
  account for surt fix in urlcanon 0.3.0
  every change is a point release now
  Upgrade PyYAML to >=5.1
  Use YAML instead of JSON
  Add option to load logging conf from JSON file
This commit is contained in:
Noah Levitt 2019-03-21 13:48:58 -07:00
commit f2eebae641
3 changed files with 15 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
setup.py
View File

@ -26,13 +26,14 @@ import setuptools
deps = [ deps = [
'certauth==1.1.6', 'certauth==1.1.6',
'warctools>=4.10.0', 'warctools>=4.10.0',
'urlcanon>=0.1.dev16', 'urlcanon>=0.3.0',
'doublethink>=0.2.0.dev87', 'doublethink>=0.2.0.dev87',
'urllib3>=1.14', 'urllib3>=1.14',
'requests>=2.0.1', 'requests>=2.0.1',
'PySocks>=1.6.8', 'PySocks>=1.6.8',
'cryptography>=2.3', 'cryptography>=2.3',
'idna>=2.5', 'idna>=2.5',
'PyYAML>=5.1',
] ]
try: try:
import concurrent.futures import concurrent.futures
@ -41,7 +42,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.4b7.dev197', version='2.4.0',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

4
tests/test_warcprox.py
View File

@ -965,12 +965,12 @@ def test_block_rules(http_daemon, https_daemon, warcprox_, archiving_proxies):
}, },
{ {
"url_match": "SURT_MATCH", "url_match": "SURT_MATCH",
"value": "http://(localhost:%s,)/fuh/" % (http_daemon.server_port), "value": "http://(localhost,:%s)/fuh/" % (http_daemon.server_port),
}, },
{ {
"url_match": "SURT_MATCH", "url_match": "SURT_MATCH",
# this rule won't match because of http scheme, https port # this rule won't match because of http scheme, https port
"value": "http://(localhost:%s,)/fuh/" % (https_daemon.server_port), "value": "http://(localhost,:%s)/fuh/" % (https_daemon.server_port),
}, },
{ {
"domain": "bad.domain.com", "domain": "bad.domain.com",

10
warcprox/main.py
View File

@ -30,6 +30,7 @@ except ImportError:
import Queue as queue import Queue as queue
import logging import logging
import logging.config
import sys import sys
import hashlib import hashlib
import argparse import argparse
@ -39,6 +40,7 @@ import traceback
import signal import signal
import threading import threading
import certauth.certauth import certauth.certauth
import yaml
import warcprox import warcprox
import doublethink import doublethink
import cryptography.hazmat.backends.openssl import cryptography.hazmat.backends.openssl
@ -239,6 +241,9 @@ def _build_arg_parser(prog='warcprox', show_hidden=False):
arg_parser.add_argument( arg_parser.add_argument(
'--trace', dest='trace', action='store_true', '--trace', dest='trace', action='store_true',
help='very verbose logging') help='very verbose logging')
arg_parser.add_argument(
'--logging-conf-file', dest='logging_conf_file', default=None,
help=('reads logging configuration from a YAML file'))
arg_parser.add_argument( arg_parser.add_argument(
'--version', action='version', '--version', action='version',
version="warcprox {}".format(warcprox.__version__)) version="warcprox {}".format(warcprox.__version__))
@ -302,6 +307,11 @@ def main(argv=None):
'%(asctime)s %(process)d %(levelname)s %(threadName)s ' '%(asctime)s %(process)d %(levelname)s %(threadName)s '
'%(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')) '%(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s'))
if args.logging_conf_file:
with open(args.logging_conf_file, 'r') as fd:
conf = yaml.safe_load(fd)
logging.config.dictConfig(conf)
# see https://github.com/pyca/cryptography/issues/2911 # see https://github.com/pyca/cryptography/issues/2911
cryptography.hazmat.backends.openssl.backend.activate_builtin_random() cryptography.hazmat.backends.openssl.backend.activate_builtin_random()