diff --git a/pywb/recorder/filters.py b/pywb/recorder/filters.py
index 1a1e7fd4..a001741c 100644
--- a/pywb/recorder/filters.py
+++ b/pywb/recorder/filters.py
@@ -63,7 +63,7 @@ class SkipNothingFilter(object):
def skip_request(self, path, req_headers):
return False
- def skip_response(self, path, req_headers, resp_headers):
+ def skip_response(self, path, req_headers, resp_headers, params):
return False
@@ -85,7 +85,7 @@ class CollectionFilter(SkipNothingFilter):
return False
- def skip_response(self, path, req_headers, resp_headers):
+ def skip_response(self, path, req_headers, resp_headers, params):
if resp_headers.get('Recorder-Skip') == '1':
return True
diff --git a/pywb/recorder/multifilewarcwriter.py b/pywb/recorder/multifilewarcwriter.py
index 9c206213..1adfdf60 100644
--- a/pywb/recorder/multifilewarcwriter.py
+++ b/pywb/recorder/multifilewarcwriter.py
@@ -138,8 +138,8 @@ class MultiFileWARCWriter(BaseWARCWriter):
to_rec.rec_headers.add_header(name, header)
def _do_write_req_resp(self, req, resp, params):
- self._copy_header(resp, req, 'WARC-Recorded-From-URI')
- self._copy_header(resp, req, 'WARC-Recorded-On-Date')
+ self._copy_header(resp, req, 'WARC-Source-URI')
+ self._copy_header(resp, req, 'WARC-Creation-Date')
resp = self._check_revisit(resp, params)
if not resp:
diff --git a/pywb/recorder/recorderapp.py b/pywb/recorder/recorderapp.py
index b1fdecb5..4febed12 100644
--- a/pywb/recorder/recorderapp.py
+++ b/pywb/recorder/recorderapp.py
@@ -302,7 +302,8 @@ class RespWrapper(Wrapper):
else:
skipping = any(x.skip_response(self.path,
self.req.headers,
- self.headers)
+ self.headers,
+ self.params)
for x in self.skip_filters)
if not skipping:
diff --git a/pywb/webagg/aggregator.py b/pywb/webagg/aggregator.py
index 656ac4cf..f7d9db75 100644
--- a/pywb/webagg/aggregator.py
+++ b/pywb/webagg/aggregator.py
@@ -95,8 +95,8 @@ class BaseAggregator(object):
raise NotImplemented()
def get_source_list(self, params):
- srcs = self._iter_sources(params)
- result = [(name, str(value)) for name, value in srcs]
+ sources = self._iter_sources(params)
+ result = [(name, str(value)) for name, value in sources]
result = {'sources': dict(result)}
return result
@@ -105,30 +105,51 @@ class BaseAggregator(object):
class BaseSourceListAggregator(BaseAggregator):
def __init__(self, sources, **kwargs):
self.sources = sources
+ self.sources_key = kwargs.get('sources_key', 'sources')
+ self.invert_sources = kwargs.get('invert_sources', False)
def get_all_sources(self, params):
return self.sources
def _iter_sources(self, params):
+ invert_sources = self.invert_sources
+ sel_sources = params.get(self.sources_key)
+ if sel_sources and sel_sources[0] == '!':
+ invert_sources = True
+ sel_sources = sel_sources[1:]
+
+ if not sel_sources or sel_sources == '*':
+ if not invert_sources:
+ return six.iteritems(self.get_all_sources(params))
+ else:
+ return iter([])
+
+ if not invert_sources:
+ return self.yield_sources(sel_sources, params)
+ else:
+ return self.yield_invert_sources(sel_sources, params)
+
+ def yield_sources(self, sel_sources, params):
sources = self.get_all_sources(params)
- srcs_list = params.get('sources')
- if not srcs_list or srcs_list == '*':
- return sources.items()
+ sel_sources = tuple(sel_sources.split(','))
+ for name in sel_sources:
+ if name in sources:
+ yield (name, sources[name])
- sel_sources = tuple(srcs_list.split(','))
-
- def yield_sources(sources, sel_sources, params):
- for name in sel_sources:
+ elif ':' in name:
+ name, param = name.split(':', 1)
if name in sources:
+ params['param.' + name + '.src_coll'] = param
yield (name, sources[name])
- elif ':' in name:
- name, param = name.split(':', 1)
- if name in sources:
- params['param.' + name + '.src_coll'] = param
- yield (name, sources[name])
+ def yield_invert_sources(self, sel_sources, params):
+ sources = self.get_all_sources(params)
+ sel_sources = tuple([src.split(':', 1)[0]
+ for src in sel_sources.split(',')])
- return yield_sources(sources, sel_sources, params)
+ for name in six.iterkeys(sources):
+ if name not in sel_sources:
+ yield (name, sources[name])
#=============================================================================
diff --git a/pywb/webagg/responseloader.py b/pywb/webagg/responseloader.py
index 9153a93d..374c2934 100644
--- a/pywb/webagg/responseloader.py
+++ b/pywb/webagg/responseloader.py
@@ -438,8 +438,8 @@ class LiveWebLoader(BaseLoader):
if not cdx.get('is_live'):
now = datetime.datetime.utcnow()
- warc_headers['WARC-Recorded-From-URI'] = cdx.get('load_url')
- warc_headers['WARC-Recorded-On-Date'] = datetime_to_iso_date(now)
+ warc_headers['WARC-Source-URI'] = cdx.get('load_url')
+ warc_headers['WARC-Creation-Date'] = datetime_to_iso_date(now)
if remote_ip:
warc_headers['WARC-IP-Address'] = remote_ip
diff --git a/pywb/webagg/test/test_memento_agg.py b/pywb/webagg/test/test_memento_agg.py
index e1e5673b..ef08d78e 100644
--- a/pywb/webagg/test/test_memento_agg.py
+++ b/pywb/webagg/test/test_memento_agg.py
@@ -29,6 +29,10 @@ aggs = {'simple': SimpleAggregator(sources),
'gevent': GeventTimeoutAggregator(sources, timeout=5.0),
}
+aggs_inv = {'simple': SimpleAggregator(sources, invert_sources=True),
+ 'gevent': GeventTimeoutAggregator(sources, invert_sources=True, timeout=5.0),
+ }
+
agg_tm = {'gevent': GeventTimeoutAggregator(sources, timeout=0.0)}
nf = {'notfound': FileIndexSource(to_path('testdata/not-found-x'))}
@@ -104,6 +108,30 @@ class TestMemAgg(MementoOverrideTests, BaseTestClass):
assert(errs == {})
+ @pytest.mark.parametrize("agg", list(aggs.values()), ids=list(aggs.keys()))
+ @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('agg_test_5'))
+ def test_mem_agg_index_5(self, agg):
+ url = 'http://vvork.com/'
+ res, errs = agg(dict(url=url, closest='20141001', limit=2, sources='!rhiz,ait'))
+
+
+ exp = [{'timestamp': '20141018133107', 'load_url': 'http://web.archive.org/web/20141018133107id_/http://vvork.com/', 'source': 'ia'}]
+
+ assert(to_json_list(res) == exp)
+ assert(errs == {'bl': "NotFoundException('http://www.webarchive.org.uk/wayback/archive/http://vvork.com/',)"})
+
+ @pytest.mark.parametrize("agg", list(aggs_inv.values()), ids=list(aggs_inv.keys()))
+ @patch('pywb.webagg.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('agg_test_5'))
+ def test_mem_agg_index_5_inverse_preset(self, agg):
+ url = 'http://vvork.com/'
+ res, errs = agg(dict(url=url, closest='20141001', limit=2, sources='rhiz,ait'))
+
+
+ exp = [{'timestamp': '20141018133107', 'load_url': 'http://web.archive.org/web/20141018133107id_/http://vvork.com/', 'source': 'ia'}]
+
+ assert(to_json_list(res) == exp)
+ assert(errs == {'bl': "NotFoundException('http://www.webarchive.org.uk/wayback/archive/http://vvork.com/',)"})
+
@pytest.mark.parametrize("agg", list(agg_nf.values()), ids=list(agg_nf.keys()))
def test_mem_agg_not_found(self, agg):
url = 'http://vvork.com/'
diff --git a/sample_archive/text_content/link_headers.yaml b/sample_archive/text_content/link_headers.yaml
index c629f587..3216193b 100644
--- a/sample_archive/text_content/link_headers.yaml
+++ b/sample_archive/text_content/link_headers.yaml
@@ -28,6 +28,11 @@ agg_test_4:
'http://webenact.rhizome.org/vvork/{url}': '; rel="memento"; datetime="Mon, 06 Oct 2014 18:43:57 GMT", ; rel="original", ; rel="timemap"; type="application/link-format"'
+
+agg_test_5:
+ 'http://web.archive.org/web/{url}': '; rel="original", ; rel="memento"; datetime="Sat, 18 Oct 2014 13:31:07 GMT", ; rel="timemap"; type="application/link-format"'
+
+
select_mem_1:
'http://web.archive.org/web/{url}': '; rel="original", ; rel="timemap"; type="application/link-format", ; rel="first memento"; datetime="Sat, 27 Jul 2002 09:13:31 GMT", ; rel="prev memento"; datetime="Wed, 06 Aug 2014 16:12:28 GMT", ; rel="memento"; datetime="Sat, 18 Oct 2014 13:31:07 GMT", ; rel="next memento"; datetime="Mon, 20 Oct 2014 16:12:43 GMT", ; rel="last memento"; datetime="Thu, 27 Oct 2016 00:13:53 GMT"'