infra/hc-spider
16
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Removed not required members

Browse Source
This commit is contained in:
Szirtesi,Tamás Gábor 2023-10-09 14:29:54 +02:00
parent e201da9a4b
commit c2f15865b1
1 changed files with 0 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hc_spider/worker.py
View File

@ -82,17 +82,11 @@ class Worker(multiprocessing.Process):
_shutdown_event: threading.Event _shutdown_event: threading.Event
_job_queue: multiprocessing.Queue _job_queue: multiprocessing.Queue
_shared_objects: SharedObjects _shared_objects: SharedObjects
_blacklisted_urls: list
_blacklisted_domains: list
_otc_base_url: str
_current_url: str _current_url: str
_url_processor: URLProcessor _url_processor: URLProcessor
def __init__(self, **kwargs) -> None: def __init__(self, **kwargs) -> None:
self._shared_objects = SharedObjects(**kwargs) self._shared_objects = SharedObjects(**kwargs)
self._blacklisted_urls = self._shared_objects.config.get("blacklisted_urls", [])
self._blacklisted_domains = self._shared_objects.config.get("blacklisted_domains", [])
self._otc_base_url = self._shared_objects.config.get("otc_base_url", "")
self._url_processor = URLProcessor(**kwargs) self._url_processor = URLProcessor(**kwargs)
super().__init__() super().__init__()
self.daemon = False self.daemon = False