backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

hidden argument --rethinkdb-big-table-name

Browse Source
This commit is contained in:
Noah Levitt 2017-07-20 12:53:59 -07:00
parent b1a8fecd9d
commit 13ee68ce4a
2 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -50,7 +50,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.1b1.dev93', version='2.1b1.dev94',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

6
warcprox/main.py
View File

@ -113,6 +113,9 @@ def _build_arg_parser(prog=os.path.basename(sys.argv[0])):
arg_parser.add_argument('--rethinkdb-big-table', arg_parser.add_argument('--rethinkdb-big-table',
dest='rethinkdb_big_table', action='store_true', default=False, dest='rethinkdb_big_table', action='store_true', default=False,
help='use a big rethinkdb table called "captures", instead of a small table called "dedup"; table is suitable for use as index for playback (ignored unless --rethinkdb-servers is specified)') help='use a big rethinkdb table called "captures", instead of a small table called "dedup"; table is suitable for use as index for playback (ignored unless --rethinkdb-servers is specified)')
arg_parser.add_argument(
'--rethinkdb-big-table-name', dest='rethinkdb_big_table_name',
default='captures', help=argparse.SUPPRESS)
arg_parser.add_argument('--kafka-broker-list', dest='kafka_broker_list', arg_parser.add_argument('--kafka-broker-list', dest='kafka_broker_list',
default=None, help='kafka broker list for capture feed') default=None, help='kafka broker list for capture feed')
arg_parser.add_argument('--kafka-capture-feed-topic', dest='kafka_capture_feed_topic', arg_parser.add_argument('--kafka-capture-feed-topic', dest='kafka_capture_feed_topic',
@ -169,7 +172,8 @@ def init_controller(args):
rr = doublethink.Rethinker( rr = doublethink.Rethinker(
args.rethinkdb_servers.split(","), args.rethinkdb_db) args.rethinkdb_servers.split(","), args.rethinkdb_db)
if args.rethinkdb_big_table: if args.rethinkdb_big_table:
captures_db = warcprox.bigtable.RethinkCaptures(rr, options=options) captures_db = warcprox.bigtable.RethinkCaptures(
rr, table=args.rethinkdb_big_table_name, options=options)
dedup_db = warcprox.bigtable.RethinkCapturesDedup( dedup_db = warcprox.bigtable.RethinkCapturesDedup(
captures_db, options=options) captures_db, options=options)
listeners.append(captures_db) listeners.append(captures_db)