diff --git a/pywb/apps/rewriterapp.py b/pywb/apps/rewriterapp.py
index cfc5c649..1186db75 100644
--- a/pywb/apps/rewriterapp.py
+++ b/pywb/apps/rewriterapp.py
@@ -358,7 +358,11 @@ class RewriterApp(object):
                 error = ''
 
             details = dict(args=kwargs, error=error)
-            raise UpstreamException(r.status_code, url=wb_url.url, details=details)
+            if r.status_code == 404:
+                raise NotFoundException(url=wb_url.url, msg=details)
+
+            else:
+                raise UpstreamException(r.status_code, url=wb_url.url, details=details)
 
         cdx = CDXObject(r.headers.get('Warcserver-Cdx').encode('utf-8'))
 
diff --git a/tests/test_integration.py b/tests/test_integration.py
index 807bc266..792bff11 100644
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -405,10 +405,13 @@ class TestWbIntegration(BaseConfigTest):
 
     def test_replay_not_found(self, fmod):
         fmod_slash = fmod + '/' if fmod else ''
-        resp = self.head('/pywb/{0}http://not-exist.example.com/', fmod_slash, status=404)
+        resp = self.get('/pywb/{0}http://not-exist.example.com/path?A=B', fmod_slash, status=404)
         assert resp.content_type == 'text/html'
         assert resp.status_int == 404
 
+        assert 'Url Not Found' in resp.text
+        assert 'The url <b>http://not-exist.example.com/path?A=B</b> could not be found in this collection.' in resp.text
+
     def test_static_content(self):
         resp = self.testapp.get('/static/default_banner.css')
         assert resp.status_int == 200