infra/hc-spider
16
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Removed deepcopy of config items

Browse Source
This commit is contained in:
Szirtesi,Tamás Gábor 2023-10-09 14:30:14 +02:00
parent c2f15865b1
commit a8d6a31d07
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hc_spider/worker.py
View File

@ -23,9 +23,9 @@ class URLProcessor:
def __init__(self, **kwargs) -> None: def __init__(self, **kwargs) -> None:
self._shared_objects = SharedObjects(**kwargs) self._shared_objects = SharedObjects(**kwargs)
self._blacklisted_urls = copy.deepcopy(self._shared_objects.config.get("blacklisted_urls")) self._blacklisted_urls = self._shared_objects.config.get("blacklisted_urls", [])
self._blacklisted_domains = copy.deepcopy(self._shared_objects.config.get("blacklisted_domains")) self._blacklisted_domains = self._shared_objects.config.get("blacklisted_domains", [])
self._otc_base_url = copy.deepcopy(self._shared_objects.config.get("otc_base_url")) self._otc_base_url = self._shared_objects.config.get("otc_base_url", "")
self._scraper = BeautifulSoupScraper(shared_objects=self._shared_objects) self._scraper = BeautifulSoupScraper(shared_objects=self._shared_objects)
def _process_children(self) -> None: def _process_children(self) -> None: