From ad33dc67282559ce8337eee12a1f721421c5d879 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Tue, 23 May 2017 09:19:09 -0700 Subject: [PATCH] refactor: webagg -> warcserver rename - ResAggApp -> BaseWarcServer - AutoApp -> WarcServer - move index related files to warcserver.index package, tests to warcserver.index.test - move resource loading related files to warcserver.resource package, tests to warcserver.resource.test - pywb.cdx -> pywb.warcserver.index - split pywb.warc -> pywb.warcserver.resource or pywb.indexer (for cdx generation) - bump to 0.51.0 for now! - tests for pywb.warcserver should be working --- pywb/__init__.py | 2 +- pywb/{cdx => indexer}/__init__.py | 0 pywb/{warc => indexer}/archiveindexer.py | 0 pywb/{warc => indexer}/cdxindexer.py | 0 pywb/{warc => indexer}/test/test_indexing.py | 0 pywb/{webagg => warcserver}/Dockerfile | 0 pywb/{webagg => warcserver}/README.rst | 0 pywb/{warc => warcserver}/__init__.py | 0 .../app.py => warcserver/basewarcserver.py} | 5 +- pywb/{webagg => warcserver}/handlers.py | 7 ++- pywb/{webagg => warcserver/index}/__init__.py | 0 .../index}/aggregator.py | 9 ++- pywb/{cdx => warcserver/index}/cdxobject.py | 0 pywb/{cdx => warcserver/index}/cdxops.py | 8 +-- .../index}/fuzzymatcher.py | 0 .../index}/indexsource.py | 11 ++-- pywb/{cdx => warcserver/index}/query.py | 2 +- .../index/test}/__init__.py | 0 .../index}/test/test_cdxobject.py | 2 +- .../index}/test/test_cdxops.py | 2 +- .../index}/test/test_dir_agg.py | 10 ++-- .../index}/test/test_indexsource.py | 8 +-- .../index}/test/test_lazy_ops.py | 4 +- .../index}/test/test_memento_agg.py | 19 +++--- .../index}/test/test_redis_agg.py | 4 +- .../index}/test/test_timeouts.py | 8 +-- .../index}/test/test_zipnum.py | 4 +- pywb/{webagg => warcserver/index}/zipnum.py | 11 ++-- pywb/{webagg => warcserver}/inputrequest.py | 0 .../test => warcserver/resource}/__init__.py | 0 .../resource/blockrecordloader.py | 2 +- .../resource/pathresolvers.py | 2 +- .../resource/resolvingloader.py | 2 +- .../resource}/responseloader.py | 10 ++-- .../resource}/test/__init__.py | 0 .../resource/test/test_loading.py | 8 +-- .../resource/test/test_pathresolvers.py | 7 ++- pywb/warcserver/test/__init__.py | 0 pywb/{webagg => warcserver}/test/live.ini | 0 pywb/{webagg => warcserver}/test/live.py | 0 .../test/test_configwarcserver.py} | 16 ++--- .../test/test_handlers.py | 58 +++++++++---------- .../test/test_inputreq.py | 2 +- .../test/test_upstream.py | 14 +++-- .../test/test_warcserver_config.yaml} | 0 pywb/{webagg => warcserver}/test/testutils.py | 11 ++-- .../upstreamindexsource.py | 12 ++-- pywb/{webagg => warcserver}/utils.py | 0 .../autoapp.py => warcserver/warcserver.py} | 23 ++++---- 49 files changed, 146 insertions(+), 137 deletions(-) rename pywb/{cdx => indexer}/__init__.py (100%) rename pywb/{warc => indexer}/archiveindexer.py (100%) rename pywb/{warc => indexer}/cdxindexer.py (100%) rename pywb/{warc => indexer}/test/test_indexing.py (100%) rename pywb/{webagg => warcserver}/Dockerfile (100%) rename pywb/{webagg => warcserver}/README.rst (100%) rename pywb/{warc => warcserver}/__init__.py (100%) rename pywb/{webagg/app.py => warcserver/basewarcserver.py} (97%) rename pywb/{webagg => warcserver}/handlers.py (96%) rename pywb/{webagg => warcserver/index}/__init__.py (100%) rename pywb/{webagg => warcserver/index}/aggregator.py (97%) rename pywb/{cdx => warcserver/index}/cdxobject.py (100%) rename pywb/{cdx => warcserver/index}/cdxops.py (97%) rename pywb/{webagg => warcserver/index}/fuzzymatcher.py (100%) rename pywb/{webagg => warcserver/index}/indexsource.py (98%) rename pywb/{cdx => warcserver/index}/query.py (98%) rename pywb/{webagg/resource => warcserver/index/test}/__init__.py (100%) rename pywb/{cdx => warcserver/index}/test/test_cdxobject.py (94%) rename pywb/{webagg => warcserver/index}/test/test_cdxops.py (99%) rename pywb/{webagg => warcserver/index}/test/test_dir_agg.py (94%) rename pywb/{webagg => warcserver/index}/test/test_indexsource.py (96%) rename pywb/{cdx => warcserver/index}/test/test_lazy_ops.py (94%) rename pywb/{webagg => warcserver/index}/test/test_memento_agg.py (92%) rename pywb/{webagg => warcserver/index}/test/test_redis_agg.py (89%) rename pywb/{webagg => warcserver/index}/test/test_timeouts.py (91%) rename pywb/{webagg => warcserver/index}/test/test_zipnum.py (98%) rename pywb/{webagg => warcserver/index}/zipnum.py (97%) rename pywb/{webagg => warcserver}/inputrequest.py (100%) rename pywb/{webagg/resource/test => warcserver/resource}/__init__.py (100%) rename pywb/{webagg => warcserver}/resource/blockrecordloader.py (96%) rename pywb/{webagg => warcserver}/resource/pathresolvers.py (98%) rename pywb/{webagg => warcserver}/resource/resolvingloader.py (99%) rename pywb/{webagg => warcserver/resource}/responseloader.py (98%) rename pywb/{webagg => warcserver/resource}/test/__init__.py (100%) rename pywb/{webagg => warcserver}/resource/test/test_loading.py (98%) rename pywb/{webagg => warcserver}/resource/test/test_pathresolvers.py (91%) create mode 100644 pywb/warcserver/test/__init__.py rename pywb/{webagg => warcserver}/test/live.ini (100%) rename pywb/{webagg => warcserver}/test/live.py (100%) rename pywb/{webagg/test/test_autoapp.py => warcserver/test/test_configwarcserver.py} (87%) rename pywb/{webagg => warcserver}/test/test_handlers.py (91%) rename pywb/{webagg => warcserver}/test/test_inputreq.py (96%) rename pywb/{webagg => warcserver}/test/test_upstream.py (89%) rename pywb/{webagg/test/test_autoapp.yaml => warcserver/test/test_warcserver_config.yaml} (100%) rename pywb/{webagg => warcserver}/test/testutils.py (93%) rename pywb/{webagg => warcserver}/upstreamindexsource.py (86%) rename pywb/{webagg => warcserver}/utils.py (100%) rename pywb/{webagg/autoapp.py => warcserver/warcserver.py} (90%) diff --git a/pywb/__init__.py b/pywb/__init__.py index 061a9bcc..96dca75f 100644 --- a/pywb/__init__.py +++ b/pywb/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.50.0' +__version__ = '0.51.0' DEFAULT_CONFIG = 'pywb/default_config.yaml' diff --git a/pywb/cdx/__init__.py b/pywb/indexer/__init__.py similarity index 100% rename from pywb/cdx/__init__.py rename to pywb/indexer/__init__.py diff --git a/pywb/warc/archiveindexer.py b/pywb/indexer/archiveindexer.py similarity index 100% rename from pywb/warc/archiveindexer.py rename to pywb/indexer/archiveindexer.py diff --git a/pywb/warc/cdxindexer.py b/pywb/indexer/cdxindexer.py similarity index 100% rename from pywb/warc/cdxindexer.py rename to pywb/indexer/cdxindexer.py diff --git a/pywb/warc/test/test_indexing.py b/pywb/indexer/test/test_indexing.py similarity index 100% rename from pywb/warc/test/test_indexing.py rename to pywb/indexer/test/test_indexing.py diff --git a/pywb/webagg/Dockerfile b/pywb/warcserver/Dockerfile similarity index 100% rename from pywb/webagg/Dockerfile rename to pywb/warcserver/Dockerfile diff --git a/pywb/webagg/README.rst b/pywb/warcserver/README.rst similarity index 100% rename from pywb/webagg/README.rst rename to pywb/warcserver/README.rst diff --git a/pywb/warc/__init__.py b/pywb/warcserver/__init__.py similarity index 100% rename from pywb/warc/__init__.py rename to pywb/warcserver/__init__.py diff --git a/pywb/webagg/app.py b/pywb/warcserver/basewarcserver.py similarity index 97% rename from pywb/webagg/app.py rename to pywb/warcserver/basewarcserver.py index c2368c57..d6e62530 100644 --- a/pywb/webagg/app.py +++ b/pywb/warcserver/basewarcserver.py @@ -1,4 +1,5 @@ -from pywb.webagg.inputrequest import DirectWSGIInputRequest, POSTInputRequest +from pywb.warcserver.inputrequest import DirectWSGIInputRequest, POSTInputRequest + from werkzeug.routing import Map, Rule from werkzeug.exceptions import HTTPException @@ -13,7 +14,7 @@ JSON_CT = 'application/json; charset=utf-8' #============================================================================= -class ResAggApp(object): +class BaseWarcServer(object): def __init__(self, *args, **kwargs): self.route_dict = {} self.debug = kwargs.get('debug', False) diff --git a/pywb/webagg/handlers.py b/pywb/warcserver/handlers.py similarity index 96% rename from pywb/webagg/handlers.py rename to pywb/warcserver/handlers.py index c1b7df74..5b490fa2 100644 --- a/pywb/webagg/handlers.py +++ b/pywb/warcserver/handlers.py @@ -1,10 +1,11 @@ -from pywb.webagg.responseloader import WARCPathLoader, LiveWebLoader, VideoLoader -from pywb.webagg.utils import MementoUtils from pywb.utils.wbexception import BadRequestException, WbException from pywb.utils.wbexception import NotFoundException + from warcio.recordloader import ArchiveLoadFailed -from pywb.webagg.fuzzymatcher import FuzzyMatcher +from pywb.warcserver.index.fuzzymatcher import FuzzyMatcher +from pywb.warcserver.resource.responseloader import WARCPathLoader, LiveWebLoader, VideoLoader +from pywb.warcserver.utils import MementoUtils import six diff --git a/pywb/webagg/__init__.py b/pywb/warcserver/index/__init__.py similarity index 100% rename from pywb/webagg/__init__.py rename to pywb/warcserver/index/__init__.py diff --git a/pywb/webagg/aggregator.py b/pywb/warcserver/index/aggregator.py similarity index 97% rename from pywb/webagg/aggregator.py rename to pywb/warcserver/index/aggregator.py index 656ac4cf..fb80f02f 100644 --- a/pywb/webagg/aggregator.py +++ b/pywb/warcserver/index/aggregator.py @@ -7,17 +7,16 @@ import os from warcio.timeutils import timestamp_now -from pywb.cdx.cdxops import process_cdx -from pywb.cdx.query import CDXQuery - from heapq import merge from collections import deque from itertools import chain -from pywb.webagg.indexsource import FileIndexSource, RedisIndexSource from pywb.utils.wbexception import NotFoundException, WbException +from pywb.warcserver.utils import ParamFormatter, res_template -from pywb.webagg.utils import ParamFormatter, res_template +from pywb.warcserver.index.indexsource import FileIndexSource, RedisIndexSource +from pywb.warcserver.index.cdxops import process_cdx +from pywb.warcserver.index.query import CDXQuery import six import glob diff --git a/pywb/cdx/cdxobject.py b/pywb/warcserver/index/cdxobject.py similarity index 100% rename from pywb/cdx/cdxobject.py rename to pywb/warcserver/index/cdxobject.py diff --git a/pywb/cdx/cdxops.py b/pywb/warcserver/index/cdxops.py similarity index 97% rename from pywb/cdx/cdxops.py rename to pywb/warcserver/index/cdxops.py index 42a1ac30..d04c1d96 100644 --- a/pywb/cdx/cdxops.py +++ b/pywb/warcserver/index/cdxops.py @@ -1,8 +1,8 @@ -from pywb.cdx.cdxobject import CDXObject, IDXObject -from pywb.cdx.cdxobject import TIMESTAMP, STATUSCODE, MIMETYPE, DIGEST -from pywb.cdx.cdxobject import OFFSET, LENGTH, FILENAME +from pywb.warcserver.index.cdxobject import CDXObject, IDXObject +from pywb.warcserver.index.cdxobject import TIMESTAMP, STATUSCODE, MIMETYPE, DIGEST +from pywb.warcserver.index.cdxobject import OFFSET, LENGTH, FILENAME -from pywb.cdx.query import CDXQuery +from pywb.warcserver.index.query import CDXQuery from warcio.timeutils import timestamp_to_sec, pad_timestamp from warcio.timeutils import PAD_14_DOWN, PAD_14_UP diff --git a/pywb/webagg/fuzzymatcher.py b/pywb/warcserver/index/fuzzymatcher.py similarity index 100% rename from pywb/webagg/fuzzymatcher.py rename to pywb/warcserver/index/fuzzymatcher.py diff --git a/pywb/webagg/indexsource.py b/pywb/warcserver/index/indexsource.py similarity index 98% rename from pywb/webagg/indexsource.py rename to pywb/warcserver/index/indexsource.py index c2a7f673..133d37e5 100644 --- a/pywb/webagg/indexsource.py +++ b/pywb/warcserver/index/indexsource.py @@ -1,13 +1,14 @@ from pywb.utils.binsearch import iter_range -from warcio.timeutils import timestamp_to_http_date, http_date_to_timestamp -from warcio.timeutils import timestamp_now from pywb.utils.canonicalize import canonicalize from pywb.utils.wbexception import NotFoundException -from pywb.cdx.cdxobject import CDXObject +from warcio.timeutils import timestamp_to_http_date, http_date_to_timestamp +from warcio.timeutils import timestamp_now -from pywb.webagg.utils import ParamFormatter, res_template -from pywb.webagg.utils import MementoUtils +from pywb.warcserver.index.cdxobject import CDXObject + +from pywb.warcserver.utils import ParamFormatter, res_template +from pywb.warcserver.utils import MementoUtils import redis diff --git a/pywb/cdx/query.py b/pywb/warcserver/index/query.py similarity index 98% rename from pywb/cdx/query.py rename to pywb/warcserver/index/query.py index 6f7e1bcd..e88ddfbf 100644 --- a/pywb/cdx/query.py +++ b/pywb/warcserver/index/query.py @@ -1,5 +1,5 @@ from six.moves.urllib.parse import urlencode -from pywb.cdx.cdxobject import CDXException +from pywb.warcserver.index.cdxobject import CDXException from pywb.utils.canonicalize import calc_search_range diff --git a/pywb/webagg/resource/__init__.py b/pywb/warcserver/index/test/__init__.py similarity index 100% rename from pywb/webagg/resource/__init__.py rename to pywb/warcserver/index/test/__init__.py diff --git a/pywb/cdx/test/test_cdxobject.py b/pywb/warcserver/index/test/test_cdxobject.py similarity index 94% rename from pywb/cdx/test/test_cdxobject.py rename to pywb/warcserver/index/test/test_cdxobject.py index 75b425c7..c26ef288 100644 --- a/pywb/cdx/test/test_cdxobject.py +++ b/pywb/warcserver/index/test/test_cdxobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from pywb.cdx.cdxobject import CDXObject, IDXObject, CDXException +from pywb.warcserver.index.cdxobject import CDXObject, IDXObject, CDXException from pytest import raises def test_empty_cdxobject(): diff --git a/pywb/webagg/test/test_cdxops.py b/pywb/warcserver/index/test/test_cdxops.py similarity index 99% rename from pywb/webagg/test/test_cdxops.py rename to pywb/warcserver/index/test/test_cdxops.py index ef5b3699..ec82ec75 100644 --- a/pywb/webagg/test/test_cdxops.py +++ b/pywb/warcserver/index/test/test_cdxops.py @@ -159,7 +159,7 @@ org,iana)/domains/root/db 20140126200928 http://www.iana.org/domains/root/db tex """ #================================================================= -from pywb.webagg.autoapp import init_index_agg +from pywb.warcserver.warcserver import init_index_agg import os import sys diff --git a/pywb/webagg/test/test_dir_agg.py b/pywb/warcserver/index/test/test_dir_agg.py similarity index 94% rename from pywb/webagg/test/test_dir_agg.py rename to pywb/warcserver/index/test/test_dir_agg.py index 0b1c521c..b56e6806 100644 --- a/pywb/webagg/test/test_dir_agg.py +++ b/pywb/warcserver/index/test/test_dir_agg.py @@ -3,15 +3,15 @@ import os import shutil import json -from .testutils import to_path, to_json_list, TempDirTests, BaseTestClass, TEST_CDX_PATH +from pywb.warcserver.test.testutils import to_path, to_json_list, TempDirTests, BaseTestClass, TEST_CDX_PATH from mock import patch import time -from pywb.webagg.aggregator import DirectoryIndexSource, CacheDirectoryIndexSource -from pywb.webagg.aggregator import SimpleAggregator -from pywb.webagg.indexsource import MementoIndexSource +from pywb.warcserver.index.aggregator import DirectoryIndexSource, CacheDirectoryIndexSource +from pywb.warcserver.index.aggregator import SimpleAggregator +from pywb.warcserver.index.indexsource import MementoIndexSource #============================================================================= @@ -114,7 +114,7 @@ class TestDirAgg(TempDirTests, BaseTestClass): assert(to_json_list(res) == exp) assert(errs == {}) - @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', mock_link_header) + @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', mock_link_header) def test_agg_dir_and_memento(self): sources = {'ia': MementoIndexSource.from_timegate_url('http://web.archive.org/web/'), 'local': self.dir_loader} diff --git a/pywb/webagg/test/test_indexsource.py b/pywb/warcserver/index/test/test_indexsource.py similarity index 96% rename from pywb/webagg/test/test_indexsource.py rename to pywb/warcserver/index/test/test_indexsource.py index 80ad8f87..c0c91e3b 100644 --- a/pywb/webagg/test/test_indexsource.py +++ b/pywb/warcserver/index/test/test_indexsource.py @@ -1,11 +1,11 @@ -from pywb.webagg.indexsource import FileIndexSource, RemoteIndexSource, MementoIndexSource, RedisIndexSource -from pywb.webagg.indexsource import LiveIndexSource +from pywb.warcserver.index.indexsource import FileIndexSource, RemoteIndexSource, MementoIndexSource, RedisIndexSource +from pywb.warcserver.index.indexsource import LiveIndexSource -from pywb.webagg.aggregator import SimpleAggregator +from pywb.warcserver.index.aggregator import SimpleAggregator from warcio.timeutils import timestamp_now -from .testutils import key_ts_res, TEST_CDX_PATH +from pywb.warcserver.test.testutils import key_ts_res, TEST_CDX_PATH import pytest import os diff --git a/pywb/cdx/test/test_lazy_ops.py b/pywb/warcserver/index/test/test_lazy_ops.py similarity index 94% rename from pywb/cdx/test/test_lazy_ops.py rename to pywb/warcserver/index/test/test_lazy_ops.py index d31f27b6..166b7b61 100644 --- a/pywb/cdx/test/test_lazy_ops.py +++ b/pywb/warcserver/index/test/test_lazy_ops.py @@ -1,6 +1,6 @@ from pywb.utils.wbexception import AccessException -from pywb.cdx.cdxops import cdx_load -from pywb.cdx.query import CDXQuery +from pywb.warcserver.index.cdxops import cdx_load +from pywb.warcserver.index.query import CDXQuery from pytest import raises diff --git a/pywb/webagg/test/test_memento_agg.py b/pywb/warcserver/index/test/test_memento_agg.py similarity index 92% rename from pywb/webagg/test/test_memento_agg.py rename to pywb/warcserver/index/test/test_memento_agg.py index e1e5673b..0c28ee71 100644 --- a/pywb/webagg/test/test_memento_agg.py +++ b/pywb/warcserver/index/test/test_memento_agg.py @@ -1,10 +1,11 @@ from gevent import monkey; monkey.patch_all(thread=False) -from pywb.webagg.aggregator import SimpleAggregator, GeventTimeoutAggregator -from pywb.webagg.aggregator import BaseAggregator +from pywb.warcserver.index.aggregator import SimpleAggregator, GeventTimeoutAggregator +from pywb.warcserver.index.aggregator import BaseAggregator -from pywb.webagg.indexsource import FileIndexSource, RemoteIndexSource, MementoIndexSource -from .testutils import to_json_list, to_path, TEST_CDX_PATH, MementoOverrideTests, BaseTestClass +from pywb.warcserver.index.indexsource import FileIndexSource, RemoteIndexSource, MementoIndexSource + +from pywb.warcserver.test.testutils import to_json_list, to_path, TEST_CDX_PATH, MementoOverrideTests, BaseTestClass import json import pytest @@ -13,7 +14,7 @@ import six from mock import patch -from pywb.webagg.handlers import IndexHandler +from pywb.warcserver.handlers import IndexHandler # Aggregator Mappings @@ -40,7 +41,7 @@ agg_nf = {'simple': SimpleAggregator(nf), # ============================================================================ class TestMemAgg(MementoOverrideTests, BaseTestClass): @pytest.mark.parametrize("agg", list(aggs.values()), ids=list(aggs.keys())) - @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('agg_test_1')) + @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('agg_test_1')) def test_mem_agg_index_1(self, agg): url = 'http://iana.org/' res, errs = agg(dict(url=url, closest='20140126000000', limit=5)) @@ -58,7 +59,7 @@ class TestMemAgg(MementoOverrideTests, BaseTestClass): @pytest.mark.parametrize("agg", list(aggs.values()), ids=list(aggs.keys())) - @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('agg_test_2')) + @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('agg_test_2')) def test_mem_agg_index_2(self, agg): url = 'http://example.com/' res, errs = agg(dict(url=url, closest='20100512', limit=6)) @@ -76,7 +77,7 @@ class TestMemAgg(MementoOverrideTests, BaseTestClass): @pytest.mark.parametrize("agg", list(aggs.values()), ids=list(aggs.keys())) - @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('agg_test_3')) + @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('agg_test_3')) def test_mem_agg_index_3(self, agg): url = 'http://vvork.com/' res, errs = agg(dict(url=url, closest='20141001', limit=5)) @@ -92,7 +93,7 @@ class TestMemAgg(MementoOverrideTests, BaseTestClass): @pytest.mark.parametrize("agg", list(aggs.values()), ids=list(aggs.keys())) - @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('agg_test_4')) + @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('agg_test_4')) def test_mem_agg_index_4(self, agg): url = 'http://vvork.com/' res, errs = agg(dict(url=url, closest='20141001', limit=2, sources='rhiz,ait')) diff --git a/pywb/webagg/test/test_redis_agg.py b/pywb/warcserver/index/test/test_redis_agg.py similarity index 89% rename from pywb/webagg/test/test_redis_agg.py rename to pywb/warcserver/index/test/test_redis_agg.py index 9aadf1df..abd9264f 100644 --- a/pywb/webagg/test/test_redis_agg.py +++ b/pywb/warcserver/index/test/test_redis_agg.py @@ -1,5 +1,5 @@ -from pywb.webagg.aggregator import RedisMultiKeyIndexSource -from .testutils import to_path, to_json_list, FakeRedisTests, BaseTestClass, TEST_CDX_PATH +from pywb.warcserver.index.aggregator import RedisMultiKeyIndexSource +from pywb.warcserver.test.testutils import to_path, to_json_list, FakeRedisTests, BaseTestClass, TEST_CDX_PATH class TestRedisAgg(FakeRedisTests, BaseTestClass): diff --git a/pywb/webagg/test/test_timeouts.py b/pywb/warcserver/index/test/test_timeouts.py similarity index 91% rename from pywb/webagg/test/test_timeouts.py rename to pywb/warcserver/index/test/test_timeouts.py index 980af85e..f50e83d0 100644 --- a/pywb/webagg/test/test_timeouts.py +++ b/pywb/warcserver/index/test/test_timeouts.py @@ -1,11 +1,11 @@ from gevent import monkey; monkey.patch_all(thread=False) import time -from pywb.webagg.indexsource import FileIndexSource +from pywb.warcserver.index.indexsource import FileIndexSource -from pywb.webagg.aggregator import SimpleAggregator, TimeoutMixin -from pywb.webagg.aggregator import GeventTimeoutAggregator, GeventTimeoutAggregator +from pywb.warcserver.index.aggregator import SimpleAggregator, TimeoutMixin +from pywb.warcserver.index.aggregator import GeventTimeoutAggregator, GeventTimeoutAggregator -from .testutils import to_json_list, TEST_CDX_PATH +from pywb.warcserver.test.testutils import to_json_list, TEST_CDX_PATH class TimeoutFileSource(FileIndexSource): diff --git a/pywb/webagg/test/test_zipnum.py b/pywb/warcserver/index/test/test_zipnum.py similarity index 98% rename from pywb/webagg/test/test_zipnum.py rename to pywb/warcserver/index/test/test_zipnum.py index 1f07e179..b5a5863c 100644 --- a/pywb/webagg/test/test_zipnum.py +++ b/pywb/warcserver/index/test/test_zipnum.py @@ -122,9 +122,9 @@ Exception: No Locations Found for: foo2 """ -from pywb.webagg.test.test_cdxops import cdx_ops_test, cdx_ops_test_data from pywb import get_test_dir -from pywb.webagg.autoapp import init_index_agg +from pywb.warcserver.index.test.test_cdxops import cdx_ops_test, cdx_ops_test_data +from pywb.warcserver.warcserver import init_index_agg import shutil import tempfile diff --git a/pywb/webagg/zipnum.py b/pywb/warcserver/index/zipnum.py similarity index 97% rename from pywb/webagg/zipnum.py rename to pywb/warcserver/index/zipnum.py index 6b57b446..607ebfd0 100644 --- a/pywb/webagg/zipnum.py +++ b/pywb/warcserver/index/zipnum.py @@ -9,13 +9,14 @@ import six from six.moves import map -#from pywb.cdx.cdxsource import CDXSource -from pywb.webagg.indexsource import BaseIndexSource -from pywb.cdx.cdxobject import IDXObject, CDXException, CDXObject -from pywb.cdx.query import CDXQuery +from warcio.bufferedreaders import gzip_decompressor + +#from pywb.warcserver.index.cdxsource import CDXSource +from pywb.warcserver.index.indexsource import BaseIndexSource +from pywb.warcserver.index.cdxobject import IDXObject, CDXException, CDXObject +from pywb.warcserver.index.query import CDXQuery from pywb.utils.loaders import BlockLoader, read_last_line -from warcio.bufferedreaders import gzip_decompressor from pywb.utils.binsearch import iter_range, linearsearch, search diff --git a/pywb/webagg/inputrequest.py b/pywb/warcserver/inputrequest.py similarity index 100% rename from pywb/webagg/inputrequest.py rename to pywb/warcserver/inputrequest.py diff --git a/pywb/webagg/resource/test/__init__.py b/pywb/warcserver/resource/__init__.py similarity index 100% rename from pywb/webagg/resource/test/__init__.py rename to pywb/warcserver/resource/__init__.py diff --git a/pywb/webagg/resource/blockrecordloader.py b/pywb/warcserver/resource/blockrecordloader.py similarity index 96% rename from pywb/webagg/resource/blockrecordloader.py rename to pywb/warcserver/resource/blockrecordloader.py index e5804589..91d9ebff 100644 --- a/pywb/webagg/resource/blockrecordloader.py +++ b/pywb/warcserver/resource/blockrecordloader.py @@ -2,7 +2,7 @@ from warcio.bufferedreaders import DecompressingBufferedReader from warcio.recordloader import ArcWarcRecordLoader from pywb.utils.loaders import BlockLoader -from pywb.webagg.utils import BUFF_SIZE +from pywb.warcserver.utils import BUFF_SIZE #================================================================= diff --git a/pywb/webagg/resource/pathresolvers.py b/pywb/warcserver/resource/pathresolvers.py similarity index 98% rename from pywb/webagg/resource/pathresolvers.py rename to pywb/warcserver/resource/pathresolvers.py index e475ecce..ff815348 100644 --- a/pywb/webagg/resource/pathresolvers.py +++ b/pywb/warcserver/resource/pathresolvers.py @@ -3,7 +3,7 @@ import redis from pywb.utils.binsearch import iter_exact from pywb.utils.loaders import to_native_str -from pywb.webagg.indexsource import RedisIndexSource +from pywb.warcserver.index.indexsource import RedisIndexSource from six.moves.urllib.request import url2pathname import six diff --git a/pywb/webagg/resource/resolvingloader.py b/pywb/warcserver/resource/resolvingloader.py similarity index 99% rename from pywb/webagg/resource/resolvingloader.py rename to pywb/warcserver/resource/resolvingloader.py index 676997cf..a8fc455f 100644 --- a/pywb/webagg/resource/resolvingloader.py +++ b/pywb/warcserver/resource/resolvingloader.py @@ -1,7 +1,7 @@ from warcio.recordloader import ArchiveLoadFailed from warcio.timeutils import iso_date_to_timestamp -from pywb.webagg.resource.blockrecordloader import BlockArcWarcRecordLoader +from pywb.warcserver.resource.blockrecordloader import BlockArcWarcRecordLoader from pywb.utils.wbexception import NotFoundException diff --git a/pywb/webagg/responseloader.py b/pywb/warcserver/resource/responseloader.py similarity index 98% rename from pywb/webagg/responseloader.py rename to pywb/warcserver/resource/responseloader.py index f8e7840c..22321b62 100644 --- a/pywb/webagg/responseloader.py +++ b/pywb/warcserver/resource/responseloader.py @@ -1,6 +1,3 @@ -from pywb.webagg.utils import MementoUtils, StreamIter, compress_gzip_iter -from pywb.webagg.utils import ParamFormatter - from warcio.timeutils import timestamp_to_datetime, datetime_to_timestamp from warcio.timeutils import iso_date_to_datetime, datetime_to_iso_date from warcio.timeutils import http_date_to_datetime, datetime_to_http_date @@ -10,8 +7,11 @@ from warcio.statusandheaders import StatusAndHeaders, StatusAndHeadersParser from pywb.utils.wbexception import LiveResourceException, WbException -from pywb.webagg.resource.resolvingloader import ResolvingLoader -from pywb.webagg.resource.pathresolvers import DefaultResolverMixin +from pywb.warcserver.utils import MementoUtils, StreamIter, compress_gzip_iter +from pywb.warcserver.utils import ParamFormatter + +from pywb.warcserver.resource.resolvingloader import ResolvingLoader +from pywb.warcserver.resource.pathresolvers import DefaultResolverMixin from six.moves.urllib.parse import urlsplit, quote, unquote diff --git a/pywb/webagg/test/__init__.py b/pywb/warcserver/resource/test/__init__.py similarity index 100% rename from pywb/webagg/test/__init__.py rename to pywb/warcserver/resource/test/__init__.py diff --git a/pywb/webagg/resource/test/test_loading.py b/pywb/warcserver/resource/test/test_loading.py similarity index 98% rename from pywb/webagg/resource/test/test_loading.py rename to pywb/warcserver/resource/test/test_loading.py index cff6c8a0..6322b255 100644 --- a/pywb/webagg/resource/test/test_loading.py +++ b/pywb/warcserver/resource/test/test_loading.py @@ -294,11 +294,11 @@ import six from warcio.recordloader import ArcWarcRecordLoader, ArchiveLoadFailed -from pywb.webagg.resource.blockrecordloader import BlockArcWarcRecordLoader -from pywb.webagg.resource.resolvingloader import ResolvingLoader -from pywb.webagg.resource.pathresolvers import DefaultResolverMixin +from pywb.warcserver.resource.blockrecordloader import BlockArcWarcRecordLoader +from pywb.warcserver.resource.resolvingloader import ResolvingLoader +from pywb.warcserver.resource.pathresolvers import DefaultResolverMixin -from pywb.cdx.cdxobject import CDXObject +from pywb.warcserver.index.cdxobject import CDXObject from pywb import get_test_dir from mock import patch diff --git a/pywb/webagg/resource/test/test_pathresolvers.py b/pywb/warcserver/resource/test/test_pathresolvers.py similarity index 91% rename from pywb/webagg/resource/test/test_pathresolvers.py rename to pywb/warcserver/resource/test/test_pathresolvers.py index b9ab20cb..4566d511 100644 --- a/pywb/webagg/resource/test/test_pathresolvers.py +++ b/pywb/warcserver/resource/test/test_pathresolvers.py @@ -1,8 +1,9 @@ from pywb import get_test_dir -from pywb.webagg.resource.pathresolvers import PrefixResolver, PathIndexResolver, RedisResolver -from pywb.webagg.resource.pathresolvers import DefaultResolverMixin from pywb.utils.loaders import to_file_url -from pywb.cdx.cdxobject import CDXObject + +from pywb.warcserver.resource.pathresolvers import PrefixResolver, PathIndexResolver, RedisResolver +from pywb.warcserver.resource.pathresolvers import DefaultResolverMixin +from pywb.warcserver.index.cdxobject import CDXObject import os diff --git a/pywb/warcserver/test/__init__.py b/pywb/warcserver/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pywb/webagg/test/live.ini b/pywb/warcserver/test/live.ini similarity index 100% rename from pywb/webagg/test/live.ini rename to pywb/warcserver/test/live.ini diff --git a/pywb/webagg/test/live.py b/pywb/warcserver/test/live.py similarity index 100% rename from pywb/webagg/test/live.py rename to pywb/warcserver/test/live.py diff --git a/pywb/webagg/test/test_autoapp.py b/pywb/warcserver/test/test_configwarcserver.py similarity index 87% rename from pywb/webagg/test/test_autoapp.py rename to pywb/warcserver/test/test_configwarcserver.py index 984ebdd0..ab101f7f 100644 --- a/pywb/webagg/test/test_autoapp.py +++ b/pywb/warcserver/test/test_configwarcserver.py @@ -1,17 +1,17 @@ from .testutils import TempDirTests, BaseTestClass -from pywb.webagg.autoapp import AutoConfigApp +from pywb.warcserver.warcserver import WarcServer import os -from pywb.webagg.indexsource import RemoteIndexSource, LiveIndexSource, MementoIndexSource, FileIndexSource -from pywb.webagg.handlers import ResourceHandler, HandlerSeq -from pywb.webagg.aggregator import BaseSourceListAggregator, DirectoryIndexSource +from pywb.warcserver.index.indexsource import RemoteIndexSource, LiveIndexSource, MementoIndexSource, FileIndexSource +from pywb.warcserver.index.aggregator import BaseSourceListAggregator, DirectoryIndexSource +from pywb.warcserver.handlers import ResourceHandler, HandlerSeq # ============================================================================ -class TestAutoConfigApp(TempDirTests, BaseTestClass): +class TestWarcServer(TempDirTests, BaseTestClass): @classmethod def setup_class(cls): - super(TestAutoConfigApp, cls).setup_class() + super(TestWarcServer, cls).setup_class() cls.orig_cwd = os.getcwd() os.chdir(cls.root_dir) os.mkdir('./local') @@ -30,12 +30,12 @@ class TestAutoConfigApp(TempDirTests, BaseTestClass): with open(os.path.join('local', 'indexes', 'file.loc'), 'a') as fh: fh.write('foo') - cls.loader = AutoConfigApp(os.path.join(cls.get_curr_dir(), 'test_autoapp.yaml')) + cls.loader = WarcServer(os.path.join(cls.get_curr_dir(), 'test_warcserver_config.yaml')) @classmethod def teardown_class(cls): os.chdir(cls.orig_cwd) - super(TestAutoConfigApp, cls).teardown_class() + super(TestWarcServer, cls).teardown_class() @staticmethod def get_curr_dir(): diff --git a/pywb/webagg/test/test_handlers.py b/pywb/warcserver/test/test_handlers.py similarity index 91% rename from pywb/webagg/test/test_handlers.py rename to pywb/warcserver/test/test_handlers.py index 912874a5..3c9e2a05 100644 --- a/pywb/webagg/test/test_handlers.py +++ b/pywb/warcserver/test/test_handlers.py @@ -1,22 +1,6 @@ -#from gevent import monkey; monkey.patch_all(thread=False) - +from .testutils import to_path, MementoOverrideTests, FakeRedisTests, BaseTestClass, TEST_CDX_PATH, TEST_WARC_PATH from collections import OrderedDict -from pywb.webagg.handlers import DefaultResourceHandler, HandlerSeq - -from pywb.webagg.indexsource import MementoIndexSource, FileIndexSource, LiveIndexSource -from pywb.webagg.indexsource import RemoteIndexSource - -from pywb.webagg.aggregator import GeventTimeoutAggregator, SimpleAggregator -from pywb.webagg.aggregator import DirectoryIndexSource - -from pywb.webagg.app import ResAggApp -from pywb.webagg.utils import MementoUtils - -from warcio.recordloader import ArcWarcRecordLoader -from warcio.statusandheaders import StatusAndHeadersParser -from warcio.bufferedreaders import ChunkedDataReader - from io import BytesIO from six.moves.urllib.parse import urlencode @@ -24,10 +8,24 @@ import webtest from fakeredis import FakeStrictRedis from mock import patch -from .testutils import to_path, MementoOverrideTests, FakeRedisTests, BaseTestClass, TEST_CDX_PATH, TEST_WARC_PATH - import json +from warcio.recordloader import ArcWarcRecordLoader +from warcio.statusandheaders import StatusAndHeadersParser +from warcio.bufferedreaders import ChunkedDataReader + +from pywb.warcserver.handlers import DefaultResourceHandler, HandlerSeq + +from pywb.warcserver.index.indexsource import MementoIndexSource, FileIndexSource, LiveIndexSource +from pywb.warcserver.index.indexsource import RemoteIndexSource + +from pywb.warcserver.index.aggregator import GeventTimeoutAggregator, SimpleAggregator +from pywb.warcserver.index.aggregator import DirectoryIndexSource + +from pywb.warcserver.basewarcserver import BaseWarcServer +from pywb.warcserver.utils import MementoUtils + + sources = { 'local': DirectoryIndexSource(TEST_CDX_PATH), 'ia': MementoIndexSource.from_timegate_url('http://web.archive.org/web/'), @@ -43,14 +41,14 @@ ia_cdx = { -class TestResAgg(MementoOverrideTests, FakeRedisTests, BaseTestClass): +class TestBaseWarcServer(MementoOverrideTests, FakeRedisTests, BaseTestClass): @classmethod def setup_class(cls): - super(TestResAgg, cls).setup_class() + super(TestBaseWarcServer, cls).setup_class() live_source = SimpleAggregator({'live': LiveIndexSource()}) live_handler = DefaultResourceHandler(live_source) - app = ResAggApp() + app = BaseWarcServer() app.add_route('/live', live_handler) source1 = GeventTimeoutAggregator(sources) @@ -178,7 +176,7 @@ class TestResAgg(MementoOverrideTests, FakeRedisTests, BaseTestClass): assert 'ResErrors' not in resp.headers - @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_mem_1')) + @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_mem_1')) def test_agg_select_mem_1(self): resp = self.testapp.get('/many/resource?url=http://vvork.com/&closest=20141001') @@ -193,7 +191,7 @@ class TestResAgg(MementoOverrideTests, FakeRedisTests, BaseTestClass): assert 'ResErrors' not in resp.headers - @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_mem_2')) + @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_mem_2')) def test_agg_select_mem_2(self): resp = self.testapp.get('/many/resource?url=http://vvork.com/&closest=20151231') @@ -219,7 +217,7 @@ class TestResAgg(MementoOverrideTests, FakeRedisTests, BaseTestClass): assert record.http_headers.get_statuscode() == '302' assert record.http_headers.get_header('Location') == 'https://www.iana.org/' - @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_live')) + @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_live')) def test_agg_select_live(self): resp = self.testapp.get('/many/resource?url=http://vvork.com/&closest=now') @@ -232,7 +230,7 @@ class TestResAgg(MementoOverrideTests, FakeRedisTests, BaseTestClass): assert 'ResErrors' not in resp.headers - @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_local')) + @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_local')) def test_agg_select_local(self): resp = self.testapp.get('/many/resource?url=http://iana.org/&closest=20140126200624') @@ -245,7 +243,7 @@ class TestResAgg(MementoOverrideTests, FakeRedisTests, BaseTestClass): assert json.loads(resp.headers['ResErrors']) == {"rhiz": "NotFoundException('http://webenact.rhizome.org/vvork/http://iana.org/',)"} - @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_local_postreq')) + @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_local_postreq')) def test_agg_select_local_postreq(self): req_data = """\ GET / HTTP/1.1 @@ -265,7 +263,7 @@ Host: iana.org assert json.loads(resp.headers['ResErrors']) == {"rhiz": "NotFoundException('http://webenact.rhizome.org/vvork/http://iana.org/',)"} - @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_live_postreq')) + @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_live_postreq')) def test_agg_live_postreq(self): req_data = """\ GET /get?foo=bar HTTP/1.1 @@ -449,7 +447,7 @@ host: www.youtube.com\ assert resp.text == resp.headers['ResErrors'] - @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_local_revisit')) + @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_local_revisit')) def test_agg_local_revisit(self): resp = self.testapp.get('/many/resource?url=http://www.example.com/&closest=20140127171251&sources=local') @@ -476,7 +474,7 @@ host: www.youtube.com\ assert resp.json == {'message': 'output=foobar not supported'} assert resp.text == resp.headers['ResErrors'] - @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_not_found')) + @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_not_found')) def test_error_local_not_found(self): resp = self.testapp.get('/many/resource?url=http://not-found.error/&sources=local', status=404) diff --git a/pywb/webagg/test/test_inputreq.py b/pywb/warcserver/test/test_inputreq.py similarity index 96% rename from pywb/webagg/test/test_inputreq.py rename to pywb/warcserver/test/test_inputreq.py index 95024d8c..edd55eb3 100644 --- a/pywb/webagg/test/test_inputreq.py +++ b/pywb/warcserver/test/test_inputreq.py @@ -1,4 +1,4 @@ -from pywb.webagg.inputrequest import DirectWSGIInputRequest, POSTInputRequest +from pywb.warcserver.inputrequest import DirectWSGIInputRequest, POSTInputRequest from werkzeug.routing import Map, Rule import webtest diff --git a/pywb/webagg/test/test_upstream.py b/pywb/warcserver/test/test_upstream.py similarity index 89% rename from pywb/webagg/test/test_upstream.py rename to pywb/warcserver/test/test_upstream.py index 30367ba9..6b3b689d 100644 --- a/pywb/webagg/test/test_upstream.py +++ b/pywb/warcserver/test/test_upstream.py @@ -3,21 +3,23 @@ from gevent import monkey; monkey.patch_all(thread=False) import webtest from io import BytesIO -from pywb.webagg.app import ResAggApp import requests -from pywb.webagg.handlers import DefaultResourceHandler -from pywb.webagg.aggregator import SimpleAggregator -from pywb.webagg.upstreamindexsource import UpstreamMementoIndexSource, UpstreamAggIndexSource - from warcio.recordloader import ArcWarcRecordLoader +from pywb.warcserver.handlers import DefaultResourceHandler +from pywb.warcserver.basewarcserver import BaseWarcServer + +from pywb.warcserver.index.aggregator import SimpleAggregator + +from pywb.warcserver.upstreamindexsource import UpstreamMementoIndexSource, UpstreamAggIndexSource + from .testutils import LiveServerTests, BaseTestClass class TestUpstream(LiveServerTests, BaseTestClass): def setup(self): - app = ResAggApp() + app = BaseWarcServer() base_url = 'http://localhost:{0}'.format(self.server.port) app.add_route('/upstream', diff --git a/pywb/webagg/test/test_autoapp.yaml b/pywb/warcserver/test/test_warcserver_config.yaml similarity index 100% rename from pywb/webagg/test/test_autoapp.yaml rename to pywb/warcserver/test/test_warcserver_config.yaml diff --git a/pywb/webagg/test/testutils.py b/pywb/warcserver/test/testutils.py similarity index 93% rename from pywb/webagg/test/testutils.py rename to pywb/warcserver/test/testutils.py index 05fdfc8f..ef8d44e7 100644 --- a/pywb/webagg/test/testutils.py +++ b/pywb/warcserver/test/testutils.py @@ -8,10 +8,11 @@ import time from fakeredis import FakeStrictRedis from mock import patch -from pywb.webagg.aggregator import SimpleAggregator -from pywb.webagg.app import ResAggApp -from pywb.webagg.handlers import DefaultResourceHandler -from pywb.webagg.indexsource import LiveIndexSource, MementoIndexSource +from pywb.warcserver.basewarcserver import BaseWarcServer +from pywb.warcserver.handlers import DefaultResourceHandler + +from pywb.warcserver.index.aggregator import SimpleAggregator +from pywb.warcserver.index.indexsource import LiveIndexSource, MementoIndexSource from pywb.urlrewrite.geventserver import GeventServer @@ -141,7 +142,7 @@ class LiveServerTests(object): @staticmethod def make_live_app(): - app = ResAggApp() + app = BaseWarcServer() app.add_route('/live', DefaultResourceHandler(SimpleAggregator( {'live': LiveIndexSource()}) diff --git a/pywb/webagg/upstreamindexsource.py b/pywb/warcserver/upstreamindexsource.py similarity index 86% rename from pywb/webagg/upstreamindexsource.py rename to pywb/warcserver/upstreamindexsource.py index 9b0ba05d..9c9d46e7 100644 --- a/pywb/webagg/upstreamindexsource.py +++ b/pywb/warcserver/upstreamindexsource.py @@ -1,10 +1,12 @@ -from pywb.cdx.cdxobject import CDXObject -from pywb.utils.wbexception import NotFoundException -from pywb.webagg.indexsource import BaseIndexSource, RemoteIndexSource -from pywb.webagg.responseloader import LiveWebLoader -from pywb.webagg.utils import ParamFormatter, res_template from warcio.timeutils import timestamp_now +from pywb.utils.wbexception import NotFoundException + +from pywb.warcserver.index.cdxobject import CDXObject +from pywb.warcserver.index.indexsource import BaseIndexSource, RemoteIndexSource +from pywb.warcserver.resource.responseloader import LiveWebLoader +from pywb.warcserver.utils import ParamFormatter, res_template + #============================================================================= class UpstreamAggIndexSource(RemoteIndexSource): diff --git a/pywb/webagg/utils.py b/pywb/warcserver/utils.py similarity index 100% rename from pywb/webagg/utils.py rename to pywb/warcserver/utils.py diff --git a/pywb/webagg/autoapp.py b/pywb/warcserver/warcserver.py similarity index 90% rename from pywb/webagg/autoapp.py rename to pywb/warcserver/warcserver.py index b9106469..46a60aa4 100644 --- a/pywb/webagg/autoapp.py +++ b/pywb/warcserver/warcserver.py @@ -1,16 +1,17 @@ -from pywb.webagg.app import ResAggApp -from pywb.webagg.utils import load_config from pywb.utils.loaders import load_yaml_config -from pywb.webagg.aggregator import CacheDirectoryIndexSource, RedisMultiKeyIndexSource -from pywb.webagg.aggregator import GeventTimeoutAggregator, SimpleAggregator +from pywb.warcserver.basewarcserver import BaseWarcServer +from pywb.warcserver.utils import load_config -from pywb.webagg.handlers import DefaultResourceHandler, HandlerSeq +from pywb.warcserver.index.aggregator import CacheDirectoryIndexSource, RedisMultiKeyIndexSource +from pywb.warcserver.index.aggregator import GeventTimeoutAggregator, SimpleAggregator -from pywb.webagg.indexsource import FileIndexSource, RemoteIndexSource -from pywb.webagg.indexsource import MementoIndexSource, RedisIndexSource -from pywb.webagg.indexsource import LiveIndexSource -from pywb.webagg.zipnum import ZipNumIndexSource +from pywb.warcserver.handlers import DefaultResourceHandler, HandlerSeq + +from pywb.warcserver.index.indexsource import FileIndexSource, RemoteIndexSource +from pywb.warcserver.index.indexsource import MementoIndexSource, RedisIndexSource +from pywb.warcserver.index.indexsource import LiveIndexSource +from pywb.warcserver.index.zipnum import ZipNumIndexSource from pywb import DEFAULT_CONFIG @@ -30,7 +31,7 @@ SOURCE_LIST = [LiveIndexSource, # ============================================================================ -class AutoConfigApp(ResAggApp): +class WarcServer(BaseWarcServer): AUTO_DIR_INDEX_PATH = '{coll}/indexes/' AUTO_DIR_ARCHIVE_PATH = '{coll}/archive/' @@ -51,7 +52,7 @@ class AutoConfigApp(ResAggApp): custom_config['collections'].update(config['collections']) config.update(custom_config) - super(AutoConfigApp, self).__init__(debug=config.get('debug', False)) + super(WarcServer, self).__init__(debug=config.get('debug', False)) self.config = config if self.config.get('enable_auto_colls', True):