backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Merge branch 'master' into qa

Browse Source 
* master:
  fix logging config which trough interfered with
  bump version after merge
  bump version after merge
  change trough dedup `date` type to varchar
This commit is contained in:
Noah Levitt 2020-01-07 15:21:56 -08:00
commit 968ea7c273
4 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
setup.py
View File

@ -2,7 +2,7 @@
''' '''
setup.py - setuptools installation configuration for warcprox setup.py - setuptools installation configuration for warcprox
Copyright (C) 2013-2019 Internet Archive Copyright (C) 2013-2020 Internet Archive
This program is free software; you can redistribute it and/or This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License modify it under the terms of the GNU General Public License
@ -44,7 +44,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.4.21', version='2.4.24',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

2
warcprox/controller.py
View File

@ -111,7 +111,7 @@ class Factory:
assert hasattr(plugin, 'notify') ^ hasattr(plugin, '_startup') assert hasattr(plugin, 'notify') ^ hasattr(plugin, '_startup')
return plugin return plugin
except Exception as e: except Exception as e:
logging.fatal('problem with plugin class %r: %s', qualname, e) logging.fatal('problem with plugin class %r', qualname, exc_info=1)
sys.exit(1) sys.exit(1)
@staticmethod @staticmethod

4
warcprox/dedup.py
View File

@ -26,7 +26,6 @@ import os
import json import json
from hanzo import warctools from hanzo import warctools
import warcprox import warcprox
import trough.client
import sqlite3 import sqlite3
import doublethink import doublethink
import datetime import datetime
@ -500,13 +499,14 @@ class TroughDedupDb(DedupDb, DedupableMixin):
SCHEMA_SQL = ('create table dedup (\n' SCHEMA_SQL = ('create table dedup (\n'
' digest_key varchar(100) primary key,\n' ' digest_key varchar(100) primary key,\n'
' url varchar(2100) not null,\n' ' url varchar(2100) not null,\n'
' date datetime not null,\n' ' date varchar(100) not null,\n'
' id varchar(100));\n') # warc record id ' id varchar(100));\n') # warc record id
WRITE_SQL_TMPL = ('insert or ignore into dedup\n' WRITE_SQL_TMPL = ('insert or ignore into dedup\n'
'(digest_key, url, date, id)\n' '(digest_key, url, date, id)\n'
'values (%s, %s, %s, %s);') 'values (%s, %s, %s, %s);')
def __init__(self, options=warcprox.Options()): def __init__(self, options=warcprox.Options()):
import trough.client
DedupableMixin.__init__(self, options) DedupableMixin.__init__(self, options)
self.options = options self.options = options
self._trough_cli = trough.client.TroughClient( self._trough_cli = trough.client.TroughClient(

1
warcprox/main.py
View File

@ -302,6 +302,7 @@ def main(argv=None):
else: else:
loglevel = logging.INFO loglevel = logging.INFO
logging.root.handlers = []
logging.basicConfig( logging.basicConfig(
stream=sys.stdout, level=loglevel, format=( stream=sys.stdout, level=loglevel, format=(
'%(asctime)s %(process)d %(levelname)s %(threadName)s ' '%(asctime)s %(process)d %(levelname)s %(threadName)s '