diff --git a/hc_spider/json_logger.py b/hc_spider/json_logger.py index 417cce3..d038294 100644 --- a/hc_spider/json_logger.py +++ b/hc_spider/json_logger.py @@ -4,17 +4,20 @@ import threading import time import warnings from typing import Callable, Union +import datetime from hc_spider.model import SharedObjects class JSONLogger(threading.Thread): - logging_interval: int + _logging_interval: int + _timestamp: str _shared_objects: SharedObjects def __init__(self, **kwargs) -> None: self._shared_objects = SharedObjects(**kwargs) - self.logging_interval = self._shared_objects.config.get("logging_interval") + self._logging_interval = self._shared_objects.config.get("logging_interval") + self._timestamp = f"{datetime.datetime.now():%Y%m%d_%H%M%S}" super().__init__() self.daemon = True self.name = "JSONLogger" @@ -28,7 +31,7 @@ class JSONLogger(threading.Thread): print(f"[{self.name}] is starting") while self._shared_objects.shutdown_event.is_set() is False: - time.sleep(self.logging_interval) + time.sleep(self._logging_interval) try: self._dump_logs() @@ -58,7 +61,7 @@ class JSONLogger(threading.Thread): def _dump_data_to_file(self, data: list | dict, filename: str, sort_key: Union[Callable, None]) -> None: if self._shared_objects.shutdown_event.is_set() is True: data.sort(key=sort_key) - with open(f"{self._shared_objects.config.get('log_dir')}/{filename}.json", "w") as f: + with open(f"{self._shared_objects.config.get('log_dir')}/{filename}_{self._timestamp}.json", "w") as f: f.write(json.dumps(data, indent=4)) def _dump_not_valid_urls(self) -> None: