1
0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-15 00:03:28 +01:00
pywb/setup.py
John Berlin ec0df7b9ae Refactor of auto-fetch worker system with support for proxy mode, fixes https://github.com/webrecorder/pywb/issues/371: (#379)
- Split wombat and auto-fetch worker into two files (proxy mode and non-proxy mode)
- Renamed preservationWorker to autoFetchWorker in order to better convey what it does
- Root config file control over including wombat and auto-fetch worker in proxy or non-proxy mode
- Added additional proxy mode + auto-fetch worker only route for fetching the auto-fetch worker code nicely for CORS
- templateview: add 'tobool' formatter to more cleanly format python bools to JS 'true'/'false'
- proxy options: config and command line: 
  'use_auto_fetch_worker' and '--proxy-with-auto-fetch'
  'use_wombat' and '--proxy-with-wombat'
- head_insert.html: only include wombat in proxy mode when use_wombat or use_auto_fetch_worker are set.
- wombatProxyMode.js: slimmed down wombat for proxy mode only including auto-fetch support.
- more consistent naming: rename 'preserveWorker' and 'autoArchive' to 'auto-fetch'

Updated tests:
- test_wbrequestresponse.py: added tests covering constructor defaults, _init_derived, options_response, json_response, encode_stream, text_stream
- test_auto_colls.py: fixed broken test test_more_custom_templates, reason using ujson now not json so spacing was off
- test_proxy.py: updated existing tests to reflect splitting wombat into proxy and non-proxy mode, added tests covering auto-fetch worker specific endpoints in proxy mode
removed duplicate addons key in .travis.yml
- test_cli.py: updated to properly test the cli with these changes
added ultrajon dep to tests_require in setup.py to reflect its usage by wbrequestresponse.py

Fully documented:
- cli.py
- frontendapp.py
- templateview.py
- wbrequestresponse.py

Removed duplicate addons key in .travis.yml
Added ultrajson dependency to tests_require in setup.py to reflect its usage by wbrequestresponse.py

Fixes #371
2018-10-03 16:27:49 -04:00

153 lines
4.6 KiB
Python
Executable File

#!/usr/bin/env python
# vim: set sw=4 et:
from setuptools import setup, find_packages
from setuptools.command.test import test as TestCommand
import glob
import os
import sys
from pywb import __version__
long_description = open('README.rst').read()
class PyTest(TestCommand):
user_options = []
def finalize_options(self):
TestCommand.finalize_options(self)
self.test_suite = ' '
def run_tests(self):
from gevent.monkey import patch_all; patch_all()
import pytest
import os
os.environ.pop('PYWB_CONFIG_FILE', None)
cmdline = '--cov-config .coveragerc --cov pywb'
cmdline += ' -v --doctest-module ./pywb/ tests/'
errcode = pytest.main(cmdline.split(' '))
sys.exit(errcode)
def get_git_short_hash():
import subprocess
try:
hash_id = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).rstrip()
if sys.version_info >= (3, 0):
hash_id = hash_id.decode('utf-8')
return hash_id
except:
return ''
def generate_git_hash_py(pkg, filename='git_hash.py'):
try:
git_hash = get_git_short_hash()
with open(os.path.join(pkg, filename), 'wt') as fh:
fh.write('git_hash = "{0}"\n'.format(git_hash))
except:
pass
def load_requirements(filename):
with open(filename, 'rt') as fh:
requirements = fh.read().rstrip().split('\n')
if (sys.version_info > (3, 0)):
requirements.append("py3AMF")
else:
requirements.append("pyAMF")
return requirements
def get_package_data():
pkgs = ['static/*.*',
'templates/*',
'*.yaml']
for root, dirs, files in os.walk(os.path.join('pywb', 'static')):
for dir_ in dirs:
pkgs.append(os.path.relpath(os.path.join(root, dir_, '*'), 'pywb'))
return pkgs
generate_git_hash_py('pywb')
setup(
name='pywb',
version=__version__,
url='https://github.com/webrecorder/pywb',
author='Ilya Kreymer',
author_email='ikreymer@gmail.com',
description='Pywb Webrecorder web archive replay and capture tools',
long_description=long_description,
license='GPL',
packages=find_packages(exclude=['tests_disabled']),
zip_safe=True,
package_data={
'pywb': get_package_data(),
},
data_files=[
('sample_archive/cdx', glob.glob('sample_archive/cdx/*')),
('sample_archive/cdxj', glob.glob('sample_archive/cdxj/*')),
('sample_archive/non-surt-cdx', glob.glob('sample_archive/non-surt-cdx/*')),
('sample_archive/zipcdx', glob.glob('sample_archive/zipcdx/*')),
('sample_archive/warcs', glob.glob('sample_archive/warcs/*')),
('sample_archive/text_content',
glob.glob('sample_archive/text_content/*')),
],
install_requires=load_requirements('requirements.txt'),
tests_require=[
'pytest',
'WebTest',
'pytest-cov',
'fakeredis',
'mock',
'urllib3',
'werkzeug',
'httpbin==0.5.0',
'ujson'
],
cmdclass={'test': PyTest},
test_suite='',
entry_points="""
[console_scripts]
pywb = pywb.apps.cli:wayback
wayback = pywb.apps.cli:wayback
cdx-server = pywb.apps.cli:cdx_server
live-rewrite-server = pywb.apps.cli:live_rewrite_server
cdx-indexer = pywb.indexer.cdxindexer:main
wb-manager = pywb.manager.manager:main_wrap_exc
warcserver = pywb.apps.cli:warcserver
""",
classifiers=[
'Development Status :: 4 - Beta',
'Environment :: Web Environment',
'License :: OSI Approved :: GNU General Public License (GPL)',
'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Topic :: Internet :: Proxy Servers',
'Topic :: Internet :: WWW/HTTP',
'Topic :: Internet :: WWW/HTTP :: WSGI',
'Topic :: Internet :: WWW/HTTP :: WSGI :: Application',
'Topic :: Internet :: WWW/HTTP :: WSGI :: Middleware',
'Topic :: Internet :: WWW/HTTP :: WSGI :: Server',
'Topic :: Software Development :: Libraries :: Python Modules',
'Topic :: System :: Archiving',
'Topic :: System :: Archiving :: Backup',
'Topic :: Utilities',
])