From cda7705075afab145ac3ca315b5e625d2df7d8e6 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Sun, 29 Mar 2015 17:38:57 -0700 Subject: [PATCH] split and refactor: remove certauth.py / test_certauth.py and instead use this functionality from 'certauth' package. Also remove `proxy-cert-auth` cli as the 'certauth' tool superceeds this functionality. (#90). To use https proxy mode, 'pip install certauth' is required. (update travis config) --- .travis.yml | 2 +- README.rst | 11 +- pywb/framework/certauth.py | 248 --------------------------- pywb/framework/proxy.py | 21 ++- pywb/framework/test/test_certauth.py | 58 ------- setup.py | 1 - 6 files changed, 18 insertions(+), 323 deletions(-) delete mode 100644 pywb/framework/certauth.py delete mode 100644 pywb/framework/test/test_certauth.py diff --git a/.travis.yml b/.travis.yml index ffc54581..17e3f2af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ sudo: false install: - "pip install 'argparse>=1.2.1' --allow-all-external" - - pip install pyopenssl + - pip install certauth - python setup.py -q install - pip install coverage pytest-cov coveralls --use-mirrors diff --git a/README.rst b/README.rst index 3fb69c6f..90d4c94d 100644 --- a/README.rst +++ b/README.rst @@ -189,10 +189,6 @@ number of useful command-line and web server tools. The tools should be availabl for an updated documentation on the latest query api. -* ``proxy-cert-auth`` -- a utility to support proxy mode. It can be used in CA root certificate, or per-host certificate with an existing root cert. - - - Latest Changes -------------- @@ -217,9 +213,10 @@ Running in HTTP/HTTPS Proxy Mode **pywb** can also be used as an actual HTTP and/or HTTPS proxy server. See `pywb Proxy Mode Usage `_ for more details on configuring proxy mode. -To run as an HTTPS proxy server, pywb provides a facility for generating a custom self-signed root certificate, which can be used to replay HTTPS content from the archive. -(The certificate should be used with caution within a controlled setting). -Using these features requiring an extra dependency: the pyopenssl library must be installed via ``pip install pyopenssl`` +To run as an HTTPS proxy server, pywb uses the `certauth `_ tool for generating a custom self-signed root certificate, which can be used to replay HTTPS content from the archive. (The certificate should be used with caution within a controlled setting). + +Using these features requiring an extra dependency: installing *certauth* with ``pip install certauth``. (This will also install the ``pyOpenSSL`` package which is used to handle the +ssl functionality). For more info, see `Proxy Mode Usage `_ diff --git a/pywb/framework/certauth.py b/pywb/framework/certauth.py deleted file mode 100644 index cadf6bd1..00000000 --- a/pywb/framework/certauth.py +++ /dev/null @@ -1,248 +0,0 @@ -import logging -import os -openssl_avail = False -try: - from OpenSSL import crypto - from OpenSSL.SSL import FILETYPE_PEM - openssl_avail = True -except ImportError: # pragma: no cover - pass - -import random -from argparse import ArgumentParser - - -#================================================================= -# Duration of 10 years -CERT_DURATION = 10 * 365 * 24 * 60 * 60 - -CERTS_DIR = './ca/certs/' - -CERT_NAME = 'pywb https proxy replay CA' - -CERT_CA_FILE = './ca/pywb-ca.pem' - - -#================================================================= -class CertificateAuthority(object): - """ - Utility class for signing individual certificate - with a root cert. - - Static generate_ca_root() method for creating the root cert - - All certs saved on filesystem. Individual certs are stored - in specified certs_dir and reused if previously created. - """ - - def __init__(self, ca_file, certs_dir): - if not ca_file: - ca_file = CERT_CA_FILE - - if not certs_dir: - certs_dir = CERTS_DIR - - self.ca_file = ca_file - self.certs_dir = certs_dir - - # read previously created root cert - self.cert, self.key = self.read_pem(ca_file) - - if not os.path.exists(certs_dir): - os.mkdir(certs_dir) - - def get_cert_for_host(self, host, overwrite=False, wildcard=False): - host_filename = os.path.join(self.certs_dir, host) + '.pem' - - if not overwrite and os.path.exists(host_filename): - return False, host_filename - - self.generate_host_cert(host, self.cert, self.key, host_filename, - wildcard) - - return True, host_filename - - def get_wildcard_cert(self, cert_host): - host_parts = cert_host.split('.', 1) - if len(host_parts) == 2 and '.' in host_parts[1]: - cert_host = host_parts[1] - - created, certfile = self.get_cert_for_host(cert_host, - wildcard=True) - - return certfile - - def get_root_PKCS12(self): - p12 = crypto.PKCS12() - p12.set_certificate(self.cert) - p12.set_privatekey(self.key) - return p12.export() - - @staticmethod - def _make_cert(certname): - cert = crypto.X509() - cert.set_version(2) - cert.set_serial_number(random.randint(0, 2 ** 64 - 1)) - cert.get_subject().CN = certname - - cert.gmtime_adj_notBefore(0) - cert.gmtime_adj_notAfter(CERT_DURATION) - return cert - - @staticmethod - def generate_ca_root(ca_file, certname, overwrite=False): - if not certname: - certname = CERT_NAME - - if not ca_file: - ca_file = CERT_CA_FILE - - if not overwrite and os.path.exists(ca_file): - cert, key = CertificateAuthority.read_pem(ca_file) - return False, cert, key - - # Generate key - key = crypto.PKey() - key.generate_key(crypto.TYPE_RSA, 2048) - - # Generate cert - cert = CertificateAuthority._make_cert(certname) - - cert.set_issuer(cert.get_subject()) - cert.set_pubkey(key) - cert.add_extensions([ - crypto.X509Extension(b"basicConstraints", - True, - b"CA:TRUE, pathlen:0"), - - crypto.X509Extension(b"keyUsage", - True, - b"keyCertSign, cRLSign"), - - crypto.X509Extension(b"subjectKeyIdentifier", - False, - b"hash", - subject=cert), - ]) - cert.sign(key, "sha1") - - # Write cert + key - CertificateAuthority.write_pem(ca_file, cert, key) - return True, cert, key - - @staticmethod - def generate_host_cert(host, root_cert, root_key, host_filename, - wildcard=False): - # Generate key - key = crypto.PKey() - key.generate_key(crypto.TYPE_RSA, 2048) - - # Generate CSR - req = crypto.X509Req() - req.get_subject().CN = host - req.set_pubkey(key) - req.sign(key, 'sha1') - - # Generate Cert - cert = CertificateAuthority._make_cert(host) - - cert.set_issuer(root_cert.get_subject()) - cert.set_pubkey(req.get_pubkey()) - - if wildcard: - DNS = 'DNS:' - alt_hosts = [DNS + host, - DNS + '*.' + host] - - alt_hosts = ', '.join(alt_hosts) - - cert.add_extensions([ - crypto.X509Extension('subjectAltName', - False, - alt_hosts)]) - - cert.sign(root_key, 'sha1') - - # Write cert + key - CertificateAuthority.write_pem(host_filename, cert, key) - return cert, key - - @staticmethod - def write_pem(filename, cert, key): - with open(filename, 'wb+') as f: - f.write(crypto.dump_privatekey(FILETYPE_PEM, key)) - - f.write(crypto.dump_certificate(FILETYPE_PEM, cert)) - - @staticmethod - def read_pem(filename): - with open(filename, 'r') as f: - cert = crypto.load_certificate(FILETYPE_PEM, f.read()) - f.seek(0) - key = crypto.load_privatekey(FILETYPE_PEM, f.read()) - - return cert, key - - -#================================================================= -def main(args=None): - parser = ArgumentParser(description='Cert Auth Cert Maker') - - parser.add_argument('output_pem_file', help='path to cert .pem file') - - parser.add_argument('-r', '--use-root', - help=('use specified root cert (.pem file) ' + - 'to create signed cert')) - - parser.add_argument('-n', '--name', action='store', default=CERT_NAME, - help='name for root certificate') - - parser.add_argument('-d', '--certs-dir', default=CERTS_DIR) - - parser.add_argument('-f', '--force', action='store_true') - - parser.add_argument('-w', '--wildcard_cert', action='store_true', - help='add wildcard SAN to host: *., ') - - result = parser.parse_args(args=args) - - overwrite = result.force - - # Create a new signed certificate using specified root - if result.use_root: - certs_dir = result.certs_dir - wildcard = result.wildcard_cert - ca = CertificateAuthority(ca_file=result.use_root, - certs_dir=result.certs_dir) - - created, host_filename = ca.get_cert_for_host(result.output_pem_file, - overwrite, wildcard) - - if created: - print ('Created new cert "' + host_filename + - '" signed by root cert ' + - result.use_root) - return 0 - - else: - print ('Cert "' + host_filename + '" already exists,' + - ' use -f to overwrite') - return 1 - - # Create new root certificate - else: - created, c, k = (CertificateAuthority. - generate_ca_root(result.output_pem_file, - result.name, - overwrite)) - - if created: - print 'Created new root cert: "' + result.output_pem_file + '"' - return 0 - else: - print ('Root cert "' + result.output_pem_file + - '" already exists,' + ' use -f to overwrite') - return 1 - -if __name__ == "__main__": - main() diff --git a/pywb/framework/proxy.py b/pywb/framework/proxy.py index 6d557dee..cc9a6679 100644 --- a/pywb/framework/proxy.py +++ b/pywb/framework/proxy.py @@ -57,6 +57,10 @@ class ProxyRouter(object): CERT_DL_PEM = '/pywb-ca.pem' CERT_DL_P12 = '/pywb-ca.p12' + CA_ROOT_FILE = './ca/pywb-ca.pem' + CA_ROOT_NAME = 'pywb https proxy replay CA' + CA_CERTS_DIR = './ca/certs/' + EXTRA_HEADERS = {'cache-control': 'no-cache', 'p3p': 'CP="NOI ADM DEV COM NAV OUR STP"'} @@ -90,24 +94,25 @@ class ProxyRouter(object): self.ca = None return - from certauth import CertificateAuthority, openssl_avail - - if not openssl_avail: # pragma: no cover - print('HTTPS proxy not available as pyopenssl is not installed') - print('Please install via "pip install pyopenssl" ' + + try: + from certauth.certauth import CertificateAuthority + except ImportError: #pragma: no cover + print('HTTPS proxy is not available as the "certauth" module ' + + 'is not installed') + print('Please install via "pip install certauth" ' + 'to enable HTTPS support') self.ca = None return # HTTPS Only Options - ca_file = proxy_options.get('root_ca_file') + ca_file = proxy_options.get('root_ca_file', self.CA_ROOT_FILE) # attempt to create the root_ca_file if doesn't exist # (generally recommended to create this seperately) - certname = proxy_options.get('root_ca_name') + certname = proxy_options.get('root_ca_name', self.CA_ROOT_NAME) CertificateAuthority.generate_ca_root(ca_file, certname) - certs_dir = proxy_options.get('certs_dir') + certs_dir = proxy_options.get('certs_dir', self.CA_CERTS_DIR) self.ca = CertificateAuthority(ca_file=ca_file, certs_dir=certs_dir) diff --git a/pywb/framework/test/test_certauth.py b/pywb/framework/test/test_certauth.py deleted file mode 100644 index 472dc37e..00000000 --- a/pywb/framework/test/test_certauth.py +++ /dev/null @@ -1,58 +0,0 @@ -import pytest - -import os -import shutil - -from pywb.framework.certauth import main, CertificateAuthority - -TEST_CA_DIR = os.path.join('.', 'pywb', 'framework', 'test', 'pywb_test_ca_certs') -TEST_CA_ROOT = os.path.join('.', 'pywb', 'framework', 'test', 'pywb_test_ca.pem') - -def setup_module(): - openssl_support = pytest.importorskip("OpenSSL") - pass - -def test_create_root(): - ret = main([TEST_CA_ROOT, '-n', 'Test Root Cert']) - assert ret == 0 - -def test_create_host_cert(): - ret = main(['example.com', '-r', TEST_CA_ROOT, '-d', TEST_CA_DIR]) - assert ret == 0 - certfile = os.path.join(TEST_CA_DIR, 'example.com.pem') - assert os.path.isfile(certfile) - #os.remove(certfile) - -def test_create_wildcard_host_cert_force_overwrite(): - ret = main(['example.com', '-r', TEST_CA_ROOT, '-d', TEST_CA_DIR, '-w', '-f']) - assert ret == 0 - certfile = os.path.join(TEST_CA_DIR, 'example.com.pem') - assert os.path.isfile(certfile) - -def test_explicit_wildcard(): - ca = CertificateAuthority(TEST_CA_ROOT, TEST_CA_DIR) - filename = ca.get_wildcard_cert('test.example.proxy') - certfile = os.path.join(TEST_CA_DIR, 'example.proxy.pem') - assert filename == certfile - assert os.path.isfile(certfile) - os.remove(certfile) - -def test_create_already_exists(): - ret = main(['example.com', '-r', TEST_CA_ROOT, '-d', TEST_CA_DIR, '-w']) - assert ret == 1 - certfile = os.path.join(TEST_CA_DIR, 'example.com.pem') - assert os.path.isfile(certfile) - # remove now - os.remove(certfile) - -def test_create_root_already_exists(): - ret = main([TEST_CA_ROOT]) - # not created, already exists - assert ret == 1 - # remove now - os.remove(TEST_CA_ROOT) - -def test_delete_files(): - shutil.rmtree(TEST_CA_DIR) - assert not os.path.isdir(TEST_CA_DIR) - assert not os.path.isfile(TEST_CA_ROOT) diff --git a/setup.py b/setup.py index d820b922..1d06ae99 100755 --- a/setup.py +++ b/setup.py @@ -95,7 +95,6 @@ setup( cdx-server = pywb.apps.cli:cdx_server live-rewrite-server = pywb.apps.cli:live_rewrite_server cdx-indexer = pywb.warc.cdxindexer:main - proxy-cert-auth = pywb.framework.certauth:main wb-manager = pywb.manager.manager:main_wrap_exc """, classifiers=[