build process: simplify build process by moving essential deps to requirements.txt, and extras to extra_requirements.txt

setup.py just loads from requirements.txt
Dockerfile pip installs requirements, then extra requirements for improved cacheing
travis runs setup install, then installs extra requirements

.travis.yml

@@ -18,10 +18,10 @@ cache:
 sudo: false
 
 install:
-  - "pip install 'argparse>=1.2.1' --allow-all-external"
-  - pip install boto certauth youtube-dl
-  - pip install git+https://github.com/esnme/ultrajson.git
+  #- pip install boto certauth youtube-dl
+  #- pip install git+https://github.com/esnme/ultrajson.git
   - python setup.py -q install
+  - pip install -r extra_requirements.txt
   - pip install coverage pytest-cov coveralls
   - npm install
 

Dockerfile

@@ -5,14 +5,22 @@ MAINTAINER Ilya Kreymer <ikreymer at gmail.com>
 RUN mkdir /uwsgi
 COPY uwsgi.ini /uwsgi/
 
-RUN pip install gevent==1.1.2 certauth youtube-dl boto uwsgi urllib3
-RUN pip install git+https://github.com/t0m/pyamf.git@python3
-RUN pip install webassets pyyaml brotlipy
-RUN pip install six chardet 'requests<2.12' redis jinja2 'surt>=0.3.0' webencodings portalocker
+#RUN pip install gevent==1.1.2 certauth youtube-dl boto uwsgi urllib3
+#RUN pip install git+https://github.com/t0m/pyamf.git@python3
+#RUN pip install webassets pyyaml brotlipy
+#RUN pip install six chardet 'requests<2.12' redis jinja2 'surt>=0.3.0' webencodings portalocker
 
-RUN mkdir /pywb
-ADD . /pywb
-RUN cd pywb; python setup.py install
+#RUN mkdir /pywb
+WORKDIR /pywb
+
+ADD requirements.txt .
+RUN pip install -r requirements.txt
+
+ADD extra_requirements.txt .
+RUN pip install -r extra_requirements.txt
+
+ADD . .
+RUN python setup.py install
 
 RUN mkdir /webarchive
 COPY config.yaml /webarchive/

extra_requirements.txt

@@ -0,0 +1,8 @@
+certauth
+youtube-dl
+boto
+uwsgi
+urllib3
+git+https://github.com/t0m/pyamf.git@python3
+git+https://github.com/esnme/ultrajson.git
+

requirements.txt

@@ -0,0 +1,13 @@
+six
+warcio
+chardet
+requests
+redis
+jinja2<2.9
+surt>=0.3.0
+brotlipy
+pyyaml
+webencodings
+gevent==1.1.2
+webassets==0.12.1
+portalocker

setup.py

@@ -31,6 +31,12 @@ class PyTest(TestCommand):
 
         sys.exit(errcode)
 
+
+def load_requirements(filename):
+    with open(filename, 'rt') as fh:
+        return fh.read().rstrip().split('\n')
+
+
 setup(
     name='pywb',
     version=__version__,
@@ -69,22 +75,7 @@ setup(
         ('sample_archive/text_content',
             glob.glob('sample_archive/text_content/*')),
         ],
-    install_requires=[
-        'six',
-        'warcio',
-        'chardet',
-        'requests',
-        'redis',
-        'jinja2<2.9',
-        'surt>=0.3.0',
-        'brotlipy',
-        'pyyaml',
-        'webencodings',
-        'gevent==1.1.2',
-        'webassets==0.12.1',
-        'portalocker'
-        #'pyamf'
-    ],
+    install_requires=load_requirements('requirements.txt'),
     dependency_links=[
         #'git+https://github.com/t0m/pyamf.git@python3#egg=pyamf-0.8.0'
     ],