From 838821bee5289a016f41e505b798efd134a64036 Mon Sep 17 00:00:00 2001 From: Tamas Szirtesi Date: Mon, 9 Oct 2023 16:06:05 +0200 Subject: [PATCH] Added timestamp to logging --- hc_spider/json_logger.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/hc_spider/json_logger.py b/hc_spider/json_logger.py index 417cce3..d038294 100644 --- a/hc_spider/json_logger.py +++ b/hc_spider/json_logger.py @@ -4,17 +4,20 @@ import threading import time import warnings from typing import Callable, Union +import datetime from hc_spider.model import SharedObjects class JSONLogger(threading.Thread): - logging_interval: int + _logging_interval: int + _timestamp: str _shared_objects: SharedObjects def __init__(self, **kwargs) -> None: self._shared_objects = SharedObjects(**kwargs) - self.logging_interval = self._shared_objects.config.get("logging_interval") + self._logging_interval = self._shared_objects.config.get("logging_interval") + self._timestamp = f"{datetime.datetime.now():%Y%m%d_%H%M%S}" super().__init__() self.daemon = True self.name = "JSONLogger" @@ -28,7 +31,7 @@ class JSONLogger(threading.Thread): print(f"[{self.name}] is starting") while self._shared_objects.shutdown_event.is_set() is False: - time.sleep(self.logging_interval) + time.sleep(self._logging_interval) try: self._dump_logs() @@ -58,7 +61,7 @@ class JSONLogger(threading.Thread): def _dump_data_to_file(self, data: list | dict, filename: str, sort_key: Union[Callable, None]) -> None: if self._shared_objects.shutdown_event.is_set() is True: data.sort(key=sort_key) - with open(f"{self._shared_objects.config.get('log_dir')}/{filename}.json", "w") as f: + with open(f"{self._shared_objects.config.get('log_dir')}/{filename}_{self._timestamp}.json", "w") as f: f.write(json.dumps(data, indent=4)) def _dump_not_valid_urls(self) -> None: