diff --git a/hc_spider/worker.py b/hc_spider/worker.py index f53fe61..7dbd37b 100644 --- a/hc_spider/worker.py +++ b/hc_spider/worker.py @@ -23,9 +23,9 @@ class URLProcessor: def __init__(self, **kwargs) -> None: self._shared_objects = SharedObjects(**kwargs) - self._blacklisted_urls = copy.deepcopy(self._shared_objects.config.get("blacklisted_urls")) - self._blacklisted_domains = copy.deepcopy(self._shared_objects.config.get("blacklisted_domains")) - self._otc_base_url = copy.deepcopy(self._shared_objects.config.get("otc_base_url")) + self._blacklisted_urls = self._shared_objects.config.get("blacklisted_urls", []) + self._blacklisted_domains = self._shared_objects.config.get("blacklisted_domains", []) + self._otc_base_url = self._shared_objects.config.get("otc_base_url", "") self._scraper = BeautifulSoupScraper(shared_objects=self._shared_objects) def _process_children(self) -> None: