youtube-dl: remove from dependency, installation is optional. Return 404 if attempting live

proxy of videos and youtube-dl is not available (the only use case).
HTTPParser wrapping logic no longer needed in latest versions
Modify tests to only run if youtube-dl is installed in cases where it is not available #118

pywb/webapp/live_rewrite_handler.py

@@ -225,6 +225,11 @@ class RewriteHandler(SearchPageWbUrlHandler):
             cache_key = self._get_cache_key('v:', video_url)
 
         info = self.youtubedl.extract_info(video_url)
+        if info is None:
+            msg = ('youtube-dl is not installed, pip install youtube-dl to ' +
+                   'enable improved video proxy')
+
+            return WbResponse.text_response(msg=msg, status='404 Not Found')
 
         #if info and info.formats and len(info.formats) == 1:
 
@@ -251,35 +256,22 @@ class RewriteHandler(SearchPageWbUrlHandler):
 
 #=================================================================
 class YoutubeDLWrapper(object):
-    """ Used to wrap youtubedl import, since youtubedl currently overrides
+    """ YoutubeDL wrapper, inits youtubee-dil if it is available
-    global HTMLParser.locatestarttagend regex with a different regex
-    that doesn't quite work.
-
-    This wrapper ensures that this regex is only set for YoutubeDL and unset
-    otherwise
     """
     def __init__(self):
-        import HTMLParser as htmlparser
+        try:
-        self.htmlparser = htmlparser
+            from youtube_dl import YoutubeDL as YoutubeDL
-
+        except ImportError:
-        self.orig_tagregex = htmlparser.locatestarttagend
+            self.ydl = None
-
+            pass
-        from youtube_dl import YoutubeDL as YoutubeDL
-
-        self.ydl_tagregex = htmlparser.locatestarttagend
-
-        htmlparser.locatestarttagend = self.orig_tagregex
 
         self.ydl = YoutubeDL(dict(simulate=True,
                                   youtube_include_dash_manifest=False))
         self.ydl.add_default_info_extractors()
 
     def extract_info(self, url):
-        info = None
+        if not self.ydl:
-        try:
+            return None
-            self.htmlparser.locatestarttagend = self.ydl_tagregex
-            info = self.ydl.extract_info(url)
-        finally:
-            self.htmlparser.locatestarttagend = self.orig_tagregex
 
+        info = self.ydl.extract_info(url)
         return info

pywb/webapp/views.py

@@ -76,7 +76,6 @@ class FileOnlyPackageLoader(PackageLoader):
 class RelEnvironment(Environment):
     """Override join_path() to enable relative template paths."""
     def join_path(self, template, parent):
-        print(parent)
         return os.path.join(os.path.dirname(parent), template)
 
 

setup.py

@@ -76,7 +76,6 @@ setup(
         'jinja2',
         'surt',
         'pyyaml',
-        'youtube_dl',
         'waitress',
         'watchdog'
        ],

tests/test_live_proxy.py

@@ -9,6 +9,7 @@ from pywb.webapp.pywb_init import create_wb_router
 from pywb.framework.wsgi_wrappers import init_app
 import webtest
 import shutil
+import pytest
 
 import pywb.webapp.live_rewrite_handler
 
@@ -162,6 +163,8 @@ class TestProxyLiveRewriter:
         assert len(self.cache) == 1
 
     def test_echo_proxy_video_info(self):
+        yt = pytest.importorskip('youtube_dl')
+
         resp = self.testapp.get('/rewrite/vi_/https://www.youtube.com/watch?v=DjFZyFWSt1M')
         assert resp.status_int == 200
         assert resp.content_type == RewriteHandler.YT_DL_TYPE, resp.content_type
@@ -178,6 +181,8 @@ class TestProxyLiveRewriter:
         assert RewriteHandler.create_cache_key('v:', 'https://www.youtube.com/watch?v=DjFZyFWSt1M') in self.cache
 
     def test_echo_proxy_video_with_referrer(self):
+        yt = pytest.importorskip('youtube_dl')
+
         headers = [('Range', 'bytes=1000-2000'), ('Referer', 'http://localhost:80/rewrite/https://example.com/')]
         resp = self.testapp.get('/rewrite/http://www.youtube.com/watch?v=DjFZyFWSt1M', headers=headers)
 

tests/test_live_rewriter.py

@@ -2,6 +2,8 @@ from pywb.webapp.live_rewrite_handler import RewriteHandler
 from pywb.apps.cli import LiveCli
 from pywb.framework.wsgi_wrappers import init_app
 import webtest
+import pytest
+
 
 class TestLiveRewriter:
     def setup(self):
@@ -39,6 +41,7 @@ class TestLiveRewriter:
         assert resp.status_int == 400
 
     def test_live_video_info(self):
+        yt = pytest.importorskip('youtube_dl')
         resp = self.testapp.get('/live/vi_/https://www.youtube.com/watch?v=DjFZyFWSt1M')
         assert resp.status_int == 200
         assert resp.content_type == RewriteHandler.YT_DL_TYPE, resp.content_type