backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

offer WarcproxController to plugin constructors

Browse Source 
because plugin needs to get at stuff, especially the warc writer
processor, for this close api to be useful
This commit is contained in:
Noah Levitt 2019-01-09 22:47:04 +00:00
parent 150c1e67c6
commit 8fd1af1d04
2 changed files with 17 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
warcprox/__init__.py
View File

@ -81,11 +81,14 @@ class RequestBlockedByRule(Exception):
class BasePostfetchProcessor(threading.Thread):
logger = logging.getLogger("warcprox.BasePostfetchProcessor")
def __init__(self, options=Options()):
def __init__(self, options=Options(), controller=None, **kwargs):
threading.Thread.__init__(self, name=self.__class__.__name__)
self.options = options
self.controller = controller
self.stop = threading.Event()
# these should be set before thread is started
# these should be set by the caller before thread is started
self.inq = None
self.outq = None
self.profiler = None
@ -205,8 +208,8 @@ class BaseBatchPostfetchProcessor(BasePostfetchProcessor):
raise Exception('not implemented')
class ListenerPostfetchProcessor(BaseStandardPostfetchProcessor):
def __init__(self, listener, options=Options()):
BaseStandardPostfetchProcessor.__init__(self, options)
def __init__(self, listener, options=Options(), controller=None, **kwargs):
BaseStandardPostfetchProcessor.__init__(self, options, controller, **kwargs)
self.listener = listener
self.name = listener.__class__.__name__

16
warcprox/controller.py
View File

@ -93,15 +93,19 @@ class Factory:
return None
@staticmethod
def plugin(qualname, options):
def plugin(qualname, options, controller=None):
try:
(module_name, class_name) = qualname.rsplit('.', 1)
module_ = importlib.import_module(module_name)
class_ = getattr(module_, class_name)
try: # new plugins take `options` argument
plugin = class_(options)
except: # backward-compatibility
plugin = class_()
try:
# new plugins take `options` and `controller` arguments
plugin = class_(options, controller)
except:
try: # medium plugins take `options` argument
plugin = class_(options)
except: # old plugins take no arguments
plugin = class_()
# check that this is either a listener or a batch processor
assert hasattr(plugin, 'notify') ^ hasattr(plugin, '_startup')
return plugin
@ -229,7 +233,7 @@ class WarcproxController(object):
crawl_logger, self.options))
for qualname in self.options.plugins or []:
plugin = Factory.plugin(qualname, self.options)
plugin = Factory.plugin(qualname, self.options, self)
if hasattr(plugin, 'notify'):
self._postfetch_chain.append(
warcprox.ListenerPostfetchProcessor(