diff --git a/hc_spider/worker.py b/hc_spider/worker.py index 1336df3..f53fe61 100644 --- a/hc_spider/worker.py +++ b/hc_spider/worker.py @@ -82,17 +82,11 @@ class Worker(multiprocessing.Process): _shutdown_event: threading.Event _job_queue: multiprocessing.Queue _shared_objects: SharedObjects - _blacklisted_urls: list - _blacklisted_domains: list - _otc_base_url: str _current_url: str _url_processor: URLProcessor def __init__(self, **kwargs) -> None: self._shared_objects = SharedObjects(**kwargs) - self._blacklisted_urls = self._shared_objects.config.get("blacklisted_urls", []) - self._blacklisted_domains = self._shared_objects.config.get("blacklisted_domains", []) - self._otc_base_url = self._shared_objects.config.get("otc_base_url", "") self._url_processor = URLProcessor(**kwargs) super().__init__() self.daemon = False