From c93d7ecafcab5d426bc43ed041a568286bf6ce0a Mon Sep 17 00:00:00 2001
From: Ilya Kreymer <ikreymer@gmail.com>
Date: Thu, 4 Aug 2016 16:53:24 -0400
Subject: [PATCH] webagg: Fix loading of url-lookup (url agnostic) revisits,
 ensure all params passed to cdx lookup, add tests for url-agnostic revisit
 lookup

---
 testdata/example-url-agnostic-orig.warc.gz    | Bin 0 -> 1354 bytes
 testdata/example-url-agnostic-revisit.warc.gz | Bin 0 -> 946 bytes
 testdata/url-agnost-example.cdxj              |   2 ++
 webagg/responseloader.py                      |  21 +++++++++++++-----
 webagg/test/test_handlers.py                  |  16 +++++++++++++
 5 files changed, 33 insertions(+), 6 deletions(-)
 create mode 100644 testdata/example-url-agnostic-orig.warc.gz
 create mode 100644 testdata/example-url-agnostic-revisit.warc.gz
 create mode 100644 testdata/url-agnost-example.cdxj

diff --git a/testdata/example-url-agnostic-orig.warc.gz b/testdata/example-url-agnostic-orig.warc.gz
new file mode 100644
index 0000000000000000000000000000000000000000..987003732a24124cc5cbbbe0cd15a8028316a63f
GIT binary patch
literal 1354
zcmV-Q1-1GgiwFP!0000018tGtYJ)Ho#_t8+AukY3{L|Jj2GXvqFxJruh2BVvNeyTs
zF}B;Deo<t*Spwnk@qL{0JGtbiZHNQpdly&1;;ZKbJg8w(3B8CrYMmG%oIn&qloTi>
zI3baXA#`0#d)abDyCn^^kvE9u6HG#I`(BJE&a|oFVMoBzXcQSoS(D!a;d-5JL<Sk6
z940AC;3isA$g`AIE5vvbXV1R3)yi^Zmw9R5w~|pSwF>9%Lh`mA{%mht-E89+lGyir
zZ_vUzpG!&#AH0H0cb!&4;E3S)J^whr9t-GzV;rG1ilY_Iz{{sKlv!<jPiWIoOTfef
zCUCHy6)-Q7X*C&!LKDzf+mkS?WTnO~sFVnV38&cnl!x=Nj5aAk<(-?sjgQOLEqk!F
zp~41zEqt#Zt5yyTHv}9UjZxf!+^|O8x$9-P^Vk3XPL$&7ggf0I{HdTZ=m7u#ABzY8
z000000050u+j84B5Pj#2{$b0$Br}3|(T$=hO%>Ux)!MaU%W;!8h*(L8NrELn$(m07
zz2HT+L=Tl_NN^W>cK7Tq@G^d!SWe4s7%zH7=DQSq$WW#tk+>JqY=w$EKTdsEE0y||
zg>N)YBW!V+(3Z$nR=xF_DsAuDPMg?$V!Lx^;CDN|?S8E{%;+u>l#y$`LaBU^YwGwP
z+`IY9%k25#%lNkcJe_^&e|?^P>U_Tc@br1o?u~2P>5PmS)4@p}2JM#Pby}{|YdOxK
z-tdT_$QYSk`w-+=;^#SMerPW|>NzfPsN;}M$Q&|oyaDNYi_pVGuhZ=>M-5{l5`~GX
z8K7wz@sKJm601o03SzlZS~Wa!v$wdzWTn<VINeUaVKj`}`FvJTs%fy{;m?LK4ns^8
zd8CQX*@wkWq5RRI!Z)Qo#nBanYnn+^!{_-&;x&wl(s_<euwD2gk4k!BPb~a?KW`W}
zbGq`u?0a_4c1dU1BOSJAI}3Nv_b{~0hH*!wBKLyvB}b<5)_D`$E{xNx#_Rb`+jrX)
z-V<gU%zU_+Ji$BpEwnlxCpYgK#@}Q@!?lj*b~?YnfZMlAX@X%voeyCo3aT1L@Cy@R
zK_C=Kj*MXcLdr%4=wG0?ig0vObrxKUnDQjBDydOziBZu5KW46$#52JfohXZ95&S3T
zo1wW6uW)NZSbYu63shf1N@$EjbHj0)icFo@Z#h%zA;S$1k(3`z;0afpMnr}*!l847
zl4_@uDLbcf97<_cx`kjn__th^pEP`3Wg<@)2}L9{`M)l0{dcbpTVg$kT(fE)MJZ*B
zC#y5rQb$UbH0IGx_rybr2|Uq6noTfoG1{Qgoeh6sj#_dMG+~_S44NSIRw6Ou%OhgB
z_+CC4-<$+prtd?~w$rzt<LqJ#G*{w6xZN{@OSvyk{4exaV@5K{c&;bN!T9PVB<e>z
zd2Q<VhD)xHodmdtYkM8fJ3%aJL0EBK(5h1tk>DxlZ!t1X;ayDMNS$TB$nyPZ$<8bI
ze=_-gSe+AjLgqPRXLY}z`+*w!TzlaAK-n>f>KRyNi;jx55m?nc8bOho9#0))-N<X_
zr_t1+N|%o2Yc7G6a?mS%w9`ez<yzN530NTHQZJVx10F@WR0ZoBg!&qhdVaKZn8$j+
zWDCBCT<5$ij6iARs_|;{<Y=|!YAtdFX~s2jC^Qv3DeDRPo61+6dg(O9Eap;bMJuq<
zL)3vjFEU&X&H7f`Znwu9t@0=LSDiV!7a3~z8ZRz=v|1XBj!MA5Dwztbnm4OD-inj&
MKkR-NH)00>0BMMoIRF3v

literal 0
HcmV?d00001

diff --git a/testdata/example-url-agnostic-revisit.warc.gz b/testdata/example-url-agnostic-revisit.warc.gz
new file mode 100644
index 0000000000000000000000000000000000000000..3770ed0a8aceb2567092233be67b6daac53c2db3
GIT binary patch
literal 946
zcmV;j15NxNiwFP!0000018tFAYlAQphVKRcL;gTCM*CqHgVL_7FxJruh2BWaM-6Bq
zF}B-&{i4WrvjoE7<$XElIk{|3+c=3(7+ib>i?2=)@Sv6{EzK<MsPiI5Nd{30QC1{5
z!70H$hR}63?NuW*ZRa$!;tsWVpy@=T6y3gO>q#(UIXvtLcp9uG!=O0%E!GRZSP7X!
zRB;S5nl4~Pvn9;4C1WxdDbM9|7;KGpLc4il+I5X$)G4FmslQaB;r*ZOP3W7eJw=v=
zK^XK#I`8vIMd3qKkeRkMx{o{)Jil)~&acM;dgL_0XoXU|Omgt@=`3Z=*f5aBw$u?Y
z@_-Q?>?Z|Gt7JxxhQ2Ta)XsGzjw@B^p^YjnBWdF?Ha+Fxd@S*LflzrDw)f*FW#iY~
zJ8r3TQO9K%bVJpsz802%gQu}tIIyv-R(IihS>gTs|NlvFb{%oY`vZXnsdnlC001A0
z2ng^e7E=HNb#iQ9VP|e{b963uVRB;tZIWMan=ll`-;wwZR-ZO$4%iU>xa+huNtbMm
znx>@L`etm0FcBLW(>33ICMDh0P9T9U_a2|~<8DT)v4;k}>ul^B%nr36K#Rw+Db1-y
zbs-E~ujT~ojH!v|8PS-};)(9-N(`9VdwuJK8as}C6hJ?QIL%N>hREIzeZ!I-Dn&V*
zl!cWbpxIGGF7b7?*(}#LKSr~7Jzw6&59{S^c)z&3yB`P9sGprL;fQlBnudTh7@#B^
zU=$4yN>AciZOcl$xRk=wwq58)F;Kejeo@W3tX`kbYjp`}Ays2a21_oaebNfny-9vF
zd7_|O$!t7MKqX{hc71&%wn8^BQv`f$wIpp@a>D#Pp$WneQG{T~ID#okQaDU<mWVtG
zhw=8)Yk~iPJ~1w-q{j?zhW<tF<0JBr`1s+al_|~PeDb$J9HiI}g9rx0B!OX$Im~hF
z!!$&ghMXsqBWrN1q_I@`wV-ufSwN;#((C-4gMCvN`{}|`u9a-WpNZ=uBWo;<UB`81
zS+?xZTcbX>`Q|z!#zbx4ib~6vfc#)r>h*a>JQ;h?zB&hNM|C62#X7r!iR(OvUMTqi
z;1qmqt0Mq#+BWcIk-5$^qXhwHapFflhT%4XA<qMp<1|hL^Uqx8nl=V56fd`B_g}MJ
zYxRa;RO?<ZA}Fw|p7rj!MbZRJ$9Lde^OJeu)p+{eb-u$fWjpI<HqYJyKo;w<Dh2Dv
U5x^?dk$S#<16J@dxqt%z0GFu8`2YX_

literal 0
HcmV?d00001

diff --git a/testdata/url-agnost-example.cdxj b/testdata/url-agnost-example.cdxj
new file mode 100644
index 00000000..6eebd255
--- /dev/null
+++ b/testdata/url-agnost-example.cdxj
@@ -0,0 +1,2 @@
+com,example)/ 20130729195151 {"url": "http://test@example.com/", "mime": "warc/revisit", "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A", "length": "591", "offset": "355", "filename": "example-url-agnostic-revisit.warc.gz"}
+org,iana,example)/ 20130702195402 {"url": "http://example.iana.org/", "mime": "text/html", "status": "200", "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A", "length": "1001", "offset": "353", "filename": "example-url-agnostic-orig.warc.gz"}
diff --git a/webagg/responseloader.py b/webagg/responseloader.py
index 0afe6442..ed3c72ba 100644
--- a/webagg/responseloader.py
+++ b/webagg/responseloader.py
@@ -166,10 +166,6 @@ class WARCPathLoader(BaseLoader):
 
         self.cdx_source = cdx_source
 
-    def cdx_index_source(self, *args, **kwargs):
-        cdx_iter, errs = self.cdx_source(*args, **kwargs)
-        return cdx_iter
-
     def _make_resolver(self, path):
         if hasattr(path, '__call__'):
             return path
@@ -188,13 +184,26 @@ class WARCPathLoader(BaseLoader):
             return None
 
         orig_source = cdx.get('source', '').split(':')[0]
-        cdx._formatter = ParamFormatter(params, orig_source)
+        formatter = ParamFormatter(params, orig_source)
+        cdx._formatter = formatter
+
+        def local_index_query(local_params):
+            for n, v in six.iteritems(params):
+                if n.startswith('param.'):
+                    local_params[n] = v
+
+            cdx_iter, errs = self.cdx_source(local_params)
+            for cdx in cdx_iter:
+                cdx._formatter = formatter
+                yield cdx
+
+            return cdx_iter
 
         failed_files = []
         headers, payload = (self.resolve_loader.
                              load_headers_and_payload(cdx,
                                                       failed_files,
-                                                      self.cdx_index_source))
+                                                      local_index_query))
 
         if cdx.get('status', '').startswith('3'):
             status_headers = self.headers_parser.parse(payload.stream)
diff --git a/webagg/test/test_handlers.py b/webagg/test/test_handlers.py
index 70b239e2..6fb5c8d8 100644
--- a/webagg/test/test_handlers.py
+++ b/webagg/test/test_handlers.py
@@ -63,6 +63,9 @@ class TestResAgg(FakeRedisTests, BaseTestClass):
         app.add_route('/empty', HandlerSeq([]))
         app.add_route('/invalid', DefaultResourceHandler([SimpleAggregator({'invalid': 'should not be a callable'})]))
 
+        url_agnost = SimpleAggregator({'url-agnost': FileIndexSource(to_path('testdata/url-agnost-example.cdxj'))})
+        app.add_route('/urlagnost', DefaultResourceHandler(url_agnost, 'redis://localhost/2/test:{arg}:warc'))
+
         cls.testapp = webtest.TestApp(app)
 
     def _check_uri_date(self, resp, uri, dt):
@@ -85,6 +88,7 @@ class TestResAgg(FakeRedisTests, BaseTestClass):
                                        '/posttest', '/posttest/postreq',
                                        '/seq', '/seq/postreq',
                                        '/allredis', '/allredis/postreq',
+                                       '/urlagnost', '/urlagnost/postreq',
                                        '/invalid', '/invalid/postreq'])
 
         assert res['/fallback'] == {'modes': ['list_sources', 'index', 'resource']}
@@ -331,6 +335,18 @@ foo=bar&test=abc"""
 
         assert resp.headers['WebAgg-Source-Coll'] == 'example'
 
+    def test_url_agnost(self):
+        f = FakeStrictRedis.from_url('redis://localhost/2')
+        f.hset('test:foo:warc', 'example-url-agnostic-revisit.warc.gz', './testdata/example-url-agnostic-revisit.warc.gz')
+        f.hset('test:foo:warc', 'example-url-agnostic-orig.warc.gz', './testdata/example-url-agnostic-orig.warc.gz')
+
+        resp = self.testapp.get('/urlagnost/resource?url=http://example.com/&param.arg=foo')
+
+        assert resp.status_int == 200
+        assert resp.headers['Link'] == MementoUtils.make_link('http://test@example.com/', 'original')
+        assert resp.headers['WebAgg-Source-Coll'] == 'url-agnost'
+        assert resp.headers['Memento-Datetime'] == 'Mon, 29 Jul 2013 19:51:51 GMT'
+
     def test_live_video_loader(self):
         params = {'url': 'http://www.youtube.com/v/BfBgWtAIbRc',
                   'content_type': 'application/vnd.youtube-dl_formats+json'