From f0359877f083ad3985ebcb2e27cc6a08cdd6a17e Mon Sep 17 00:00:00 2001
From: Ilya Kreymer <ikreymer@gmail.com>
Date: Sat, 27 Jun 2015 16:11:59 -0700
Subject: [PATCH] youtube-dl: remove from dependency, installation is optional.
 Return 404 if attempting live proxy of videos and youtube-dl is not available
 (the only use case). HTTPParser wrapping logic no longer needed in latest
 versions Modify tests to only run if youtube-dl is installed in cases where
 it is not available #118

---
 pywb/webapp/live_rewrite_handler.py | 36 +++++++++++------------------
 pywb/webapp/views.py                |  1 -
 setup.py                            |  1 -
 tests/test_live_proxy.py            |  5 ++++
 tests/test_live_rewriter.py         |  3 +++
 5 files changed, 22 insertions(+), 24 deletions(-)

diff --git a/pywb/webapp/live_rewrite_handler.py b/pywb/webapp/live_rewrite_handler.py
index 598d44ec..02033e6e 100644
--- a/pywb/webapp/live_rewrite_handler.py
+++ b/pywb/webapp/live_rewrite_handler.py
@@ -225,6 +225,11 @@ class RewriteHandler(SearchPageWbUrlHandler):
             cache_key = self._get_cache_key('v:', video_url)
 
         info = self.youtubedl.extract_info(video_url)
+        if info is None:
+            msg = ('youtube-dl is not installed, pip install youtube-dl to ' +
+                   'enable improved video proxy')
+
+            return WbResponse.text_response(msg=msg, status='404 Not Found')
 
         #if info and info.formats and len(info.formats) == 1:
 
@@ -251,35 +256,22 @@ class RewriteHandler(SearchPageWbUrlHandler):
 
 #=================================================================
 class YoutubeDLWrapper(object):
-    """ Used to wrap youtubedl import, since youtubedl currently overrides
-    global HTMLParser.locatestarttagend regex with a different regex
-    that doesn't quite work.
-
-    This wrapper ensures that this regex is only set for YoutubeDL and unset
-    otherwise
+    """ YoutubeDL wrapper, inits youtubee-dil if it is available
     """
     def __init__(self):
-        import HTMLParser as htmlparser
-        self.htmlparser = htmlparser
-
-        self.orig_tagregex = htmlparser.locatestarttagend
-
-        from youtube_dl import YoutubeDL as YoutubeDL
-
-        self.ydl_tagregex = htmlparser.locatestarttagend
-
-        htmlparser.locatestarttagend = self.orig_tagregex
+        try:
+            from youtube_dl import YoutubeDL as YoutubeDL
+        except ImportError:
+            self.ydl = None
+            pass
 
         self.ydl = YoutubeDL(dict(simulate=True,
                                   youtube_include_dash_manifest=False))
         self.ydl.add_default_info_extractors()
 
     def extract_info(self, url):
-        info = None
-        try:
-            self.htmlparser.locatestarttagend = self.ydl_tagregex
-            info = self.ydl.extract_info(url)
-        finally:
-            self.htmlparser.locatestarttagend = self.orig_tagregex
+        if not self.ydl:
+            return None
 
+        info = self.ydl.extract_info(url)
         return info
diff --git a/pywb/webapp/views.py b/pywb/webapp/views.py
index d6ce9b30..45b59968 100644
--- a/pywb/webapp/views.py
+++ b/pywb/webapp/views.py
@@ -76,7 +76,6 @@ class FileOnlyPackageLoader(PackageLoader):
 class RelEnvironment(Environment):
     """Override join_path() to enable relative template paths."""
     def join_path(self, template, parent):
-        print(parent)
         return os.path.join(os.path.dirname(parent), template)
 
 
diff --git a/setup.py b/setup.py
index 1d06ae99..cf34c0ed 100755
--- a/setup.py
+++ b/setup.py
@@ -76,7 +76,6 @@ setup(
         'jinja2',
         'surt',
         'pyyaml',
-        'youtube_dl',
         'waitress',
         'watchdog'
        ],
diff --git a/tests/test_live_proxy.py b/tests/test_live_proxy.py
index 7bd86706..e5cfc106 100644
--- a/tests/test_live_proxy.py
+++ b/tests/test_live_proxy.py
@@ -9,6 +9,7 @@ from pywb.webapp.pywb_init import create_wb_router
 from pywb.framework.wsgi_wrappers import init_app
 import webtest
 import shutil
+import pytest
 
 import pywb.webapp.live_rewrite_handler
 
@@ -162,6 +163,8 @@ class TestProxyLiveRewriter:
         assert len(self.cache) == 1
 
     def test_echo_proxy_video_info(self):
+        yt = pytest.importorskip('youtube_dl')
+
         resp = self.testapp.get('/rewrite/vi_/https://www.youtube.com/watch?v=DjFZyFWSt1M')
         assert resp.status_int == 200
         assert resp.content_type == RewriteHandler.YT_DL_TYPE, resp.content_type
@@ -178,6 +181,8 @@ class TestProxyLiveRewriter:
         assert RewriteHandler.create_cache_key('v:', 'https://www.youtube.com/watch?v=DjFZyFWSt1M') in self.cache
 
     def test_echo_proxy_video_with_referrer(self):
+        yt = pytest.importorskip('youtube_dl')
+
         headers = [('Range', 'bytes=1000-2000'), ('Referer', 'http://localhost:80/rewrite/https://example.com/')]
         resp = self.testapp.get('/rewrite/http://www.youtube.com/watch?v=DjFZyFWSt1M', headers=headers)
 
diff --git a/tests/test_live_rewriter.py b/tests/test_live_rewriter.py
index e77ab1ae..741d527f 100644
--- a/tests/test_live_rewriter.py
+++ b/tests/test_live_rewriter.py
@@ -2,6 +2,8 @@ from pywb.webapp.live_rewrite_handler import RewriteHandler
 from pywb.apps.cli import LiveCli
 from pywb.framework.wsgi_wrappers import init_app
 import webtest
+import pytest
+
 
 class TestLiveRewriter:
     def setup(self):
@@ -39,6 +41,7 @@ class TestLiveRewriter:
         assert resp.status_int == 400
 
     def test_live_video_info(self):
+        yt = pytest.importorskip('youtube_dl')
         resp = self.testapp.get('/live/vi_/https://www.youtube.com/watch?v=DjFZyFWSt1M')
         assert resp.status_int == 200
         assert resp.content_type == RewriteHandler.YT_DL_TYPE, resp.content_type