From a64a12289ef5642b2a6d00ebc3fa181046dbef8d Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 18 Oct 2017 15:21:53 -0700 Subject: [PATCH] in travis-ci, run trough in another docker container, so that its version of python can be independent of the one used to run the warcprox tests --- .travis.yml | 19 ++++--------------- tests/run-trough.sh | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 15 deletions(-) create mode 100644 tests/run-trough.sh diff --git a/.travis.yml b/.travis.yml index 70245ef..a2e00f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,21 +27,10 @@ services: before_install: - sudo service docker restart ; sleep 10 # https://github.com/travis-ci/travis-ci/issues/4778 -- docker run -d --publish=28015:28015 rethinkdb -- docker run -d --publish=8020:8020 --publish=50070:50070 --publish=50010:50010 --publish=50020:50020 --publish=50075:50075 chalimartines/cdh5-pseudo-distributed -- virtualenv -p python3 /opt/trough-ve3 -- . /opt/trough-ve3/bin/activate -- pip install git+https://github.com/jkafader/snakebite@feature/python3-version-string -- pip install git+https://github.com/nlevitt/trough.git@toward-warcprox-dedup -- 'sync.py >>/tmp/trough-sync-local.out 2>&1 &' -- sleep 5 -- python -c "import doublethink ; from trough.settings import settings ; rr = doublethink.Rethinker(settings['RETHINKDB_HOSTS']) ; rr.db('trough_configuration').wait().run()" -- 'sync.py --server >>/tmp/trough-sync-server.out 2>&1 &' -- 'uwsgi --venv=/opt/trough-ve3 --http :6222 --master --processes=2 --harakiri=240 --max-requests=50000 --vacuum --die-on-term --wsgi-file /opt/trough-ve3/bin/writer.py >>/tmp/trough-write.out 2>&1 &' -- 'uwsgi --venv=/opt/trough-ve3 --http :6112 --master --processes=2 --harakiri=20 --max-requests=50000 --vacuum --die-on-term --wsgi-file /opt/trough-ve3/bin/write_provisioner_local.py >>/tmp/trough-write-provisioner-local.out 2>&1 &' -- 'uwsgi --venv=/opt/trough-ve3 --http :6111 --master --processes=2 --harakiri=20 --max-requests=50000 --vacuum --die-on-term --wsgi-file /opt/trough-ve3/bin/write_provisioner_server.py >>/tmp/trough-write-provisioner-server.out 2>&1 &' -- 'uwsgi --venv=/opt/trough-ve3 --http :6444 --master --processes=2 --harakiri=3200 --socket-timeout=3200 --max-requests=50000 --vacuum --die-on-term --wsgi-file /opt/trough-ve3/bin/reader.py >>/tmp/trough-read.out 2>&1 &' -- deactivate +- docker network create --driver=bridge trough +- docker run --rm --detach --network=trough --name=rethinkdb rethinkdb +- docker run --rm --detach --network=trough --name=hadoop chalimartines/cdh5-pseudo-distributed +- docker run --rm --detach --volume="$PWD/tests/run-trough.sh:/run-trough.sh" --publish=6111:6111 --publish=6112:6112 --publish=6222:6222 --publish=6444:6444 python:3 bash /run-trough.sh install: - pip install . pytest requests warcio diff --git a/tests/run-trough.sh b/tests/run-trough.sh new file mode 100644 index 0000000..d5fbcb7 --- /dev/null +++ b/tests/run-trough.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# this is used by .travis.yml +# + +pip install git+https://github.com/jkafader/snakebite@feature/python3-version-string +pip install git+https://github.com/nlevitt/trough.git@9ae5f477a28f22 + +mkdir /etc/trough + +# hello docker user-defined bridge networking +echo ' +HDFS_HOST: hadoop +RETHINKDB_HOSTS: +- rethinkdb +' > /etc/trough/settings.yml + +sync.py >>/tmp/trough-sync-local.out 2>&1 & + +sleep 5 +python -c " +import doublethink +from trough.settings import settings +rr = doublethink.Rethinker(settings['RETHINKDB_HOSTS']) +rr.db('trough_configuration').wait().run()" + +sync.py --server >>/tmp/trough-sync-server.out 2>&1 & +uwsgi --http :6222 --master --processes=2 --harakiri=240 --max-requests=50000 --vacuum --die-on-term --wsgi-file /usr/local/bin/writer.py >>/tmp/trough-write.out 2>&1 & +uwsgi --http :6112 --master --processes=2 --harakiri=20 --max-requests=50000 --vacuum --die-on-term --wsgi-file /usr/local/bin/write_provisioner_local.py >>/tmp/trough-write-provisioner-local.out 2>&1 & +uwsgi --http :6111 --master --processes=2 --harakiri=20 --max-requests=50000 --vacuum --die-on-term --wsgi-file /usr/local/bin/write_provisioner_server.py >>/tmp/trough-write-provisioner-server.out 2>&1 & +uwsgi --http :6444 --master --processes=2 --harakiri=3200 --socket-timeout=3200 --max-requests=50000 --vacuum --die-on-term --wsgi-file /usr/local/bin/reader.py >>/tmp/trough-read.out 2>&1 & + +wait