Docker Improvements (#446)

* Misc improvements, including fixes from @funkyfuture:
- Dockerfile: Reduces number of created layers and source contents
- Support for automatic collection creation if INIT_COLLECTION is defined
- Add entry point script docker-entrypoint.sh
- update to latest python (3.7.2 currently)
- additions to .dockerignore
- setup.py and requirements cleanup (just use plain 'gevent' requirement)

* docker-entrypoint.sh improvements:
- before running cmd, match uid/gid to that of volume dir (specified via $VOLUME_DIR, defaulting to /webarchive)
- if volume is owned by root (default if none mounted), just run as root
- if volume is owned by different user, create/update user 'archivist' to match the uid/gid of $VOLUME_DIR, then run cmd as 'su archivist'

.dockerignore

@@ -1,11 +1,33 @@
 build/
 dist/
+karma-tests/
+sample_archive/
+tests/
+tests_disabled/
+venv/
+collections/
+
 .cache/
 .eggs/
 .git/
+.github/
+.venv
+.travis
+.pytest_cache
+
+.coveragerc
+.dockerignore
+.editorconfig
+.gitattributes
+.gitignore
+.travis.yml
+appveyor.yml
+package.json
+run-*
+Vagrantfile
+Dockerfile
+
 **/*.egg
 **/*.egg-info
 **/__pycache__
 **/*.pyc
-collections/
-

Dockerfile

@@ -1,34 +1,34 @@
-ARG PYTHON=python:3.5.3
+ARG PYTHON=python:3.7.2
 
 FROM $PYTHON
 
-RUN mkdir /uwsgi
-COPY uwsgi.ini /uwsgi/
-
 WORKDIR /pywb
 
-ADD requirements.txt .
+COPY requirements.txt extra_requirements.txt ./
-RUN pip install -r requirements.txt
 
-ADD extra_requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt -r extra_requirements.txt
-RUN pip install -r extra_requirements.txt
 
-ADD . .
+COPY . ./
-RUN python setup.py install
 
-RUN mkdir /webarchive
+RUN python setup.py install \
-COPY config.yaml /webarchive/
+ && mv ./docker-entrypoint.sh / \
-
+ && mkdir /uwsgi && mv ./uwsgi.ini /uwsgi/ \
-VOLUME /webarchive
+ && mkdir /webarchive && mv ./config.yaml /webarchive/
 
 WORKDIR /webarchive
 
+# auto init collection
+ENV INIT_COLLECTION ''
+
+ENV VOLUME_DIR /webarchive
+
+#USER archivist
+COPY docker-entrypoint.sh ./
+
+# volume and port
+VOLUME /webarchive
 EXPOSE 8080
 
+ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD ["uwsgi", "/uwsgi/uwsgi.ini"]
 
-RUN useradd -ms /bin/bash -u 1000 archivist
-
-USER archivist
-
-

docker-entrypoint.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+set -e
+
+# Get UID/GID from volume dir
+VOLUME_UID=$(stat -c '%u' $VOLUME_DIR)
+VOLUME_GID=$(stat -c '%g' $VOLUME_DIR)
+
+MY_UID=$(id -u)
+MY_GID=$(id -g)
+
+# Run as custom user
+if [ "$MY_GID" != "$VOLUME_GID" ] || [ "$MY_UID" != "$VOLUME_UID" ]; then
+    # create or modify user and group to match expected uid/gid
+    groupadd --gid $VOLUME_GID archivist || groupmod -o --gid $VOLUME_GID archivist
+    useradd -ms /bin/bash -u $VOLUME_UID -g $VOLUME_GID archivist || usermod -o -u $VOLUME_UID archivist
+
+    # initialize a collection if defined and not present
+    if [ -n "$INIT_COLLECTION" ] && [ ! -d $VOLUME_DIR/collections/$INIT_COLLECTION ]; then
+        su archivist -c "wb-manager init $INIT_COLLECTION"
+    fi
+
+    cmd="cd $PWD; $@"
+
+    # run process as new archivist user
+    su archivist -c "$cmd"
+
+# run as current user (root)
+else
+    # initialize a collection if defined and not present
+    if [ -n "$INIT_COLLECTION" ] && [ ! -d $VOLUME_DIR/collections/$INIT_COLLECTION ]; then
+        wb-manager init $INIT_COLLECTION
+    fi
+
+    # run process directly
+    exec $@
+fi
+

requirements.txt

@@ -9,7 +9,7 @@ brotlipy
 pyyaml
 werkzeug
 webencodings
-gevent>=1.3[dnspython]
+gevent
 webassets==0.12.1
 portalocker
 wsgiprox>=1.5.1

setup.py

@@ -10,7 +10,10 @@ import sys
 from pywb import __version__
 
 
-long_description = open('README.rst').read()
+def get_ldecription():
+    with open('README.rst', 'r') as fh:
+        long_description = fh.read()
+    return long_description
 
 
 class PyTest(TestCommand):
@@ -58,7 +61,7 @@ def generate_git_hash_py(pkg, filename='git_hash.py'):
 def load_requirements(filename):
     with open(filename, 'rt') as fh:
         requirements = fh.read().rstrip().split('\n')
-    if (sys.version_info > (3, 0)):
+    if sys.version_info > (3, 0):
         requirements.append("py3AMF")
     else:
         requirements.append("pyAMF")
@@ -87,7 +90,7 @@ setup(
     author='Ilya Kreymer',
     author_email='ikreymer@gmail.com',
     description='Pywb Webrecorder web archive replay and capture tools',
-    long_description=long_description,
+    long_description=get_ldecription(),
     license='GPL',
     packages=find_packages(exclude=['tests_disabled']),
     zip_safe=True,