Merge pull request #6 from FerdinandZhong/develop

add verbose and update version

examples/simple.py → examples/simple_sample.py

@@ -18,7 +18,9 @@ def worker(text):
     formatter = logging.Formatter(
         "%(asctime)s - %(name)s - %(filename)s- [%(levelname)s]: %(message)s"
     )
-    register_handler(client_logger, logging_level=logging.INFO, formatter=formatter)
+    register_handler(
+        client_logger, logging_level=logging.INFO, formatter=formatter, verbose=True
+    )
     client_logger.setLevel(logging.INFO)
     pid = os.getpid()
     for i in range(TEST_ROUND):
@@ -47,7 +49,7 @@ def sample_running(text):
 
     server_handler = ServerHandler(LOG_FILE, maxBytes=MAX_BYTES)
     server_handler.setLevel(logging.INFO)
-    server = Server(server_handler, batch_size=BATCH_SIZE)
+    server = Server(server_handler, batch_size=BATCH_SIZE, verbose=True)
     server.start()
 
     sample_running(args.text)

setup.py

@@ -9,13 +9,13 @@
 
 setup(
     name="mp-socket-logging",
-    version="0.1.0",
+    version="0.1.1",
     description="A small toolkit for python multiprocessing file logging",
     long_description=readme,
     long_description_content_type="text/markdown",
     author="Zhong Qishuai",
     author_email="[email protected]",
-    url="https://git.garena.com/shopee-ds/ml-platform/sre-utils",
+    url="https://github.com/FerdinandZhong/mp-socket-logging",
     packages=find_packages(exclude=["tests*", "example*"]),
     classifiers=[
         "Programming Language :: Python :: 3",

socket_logging/client.py

@@ -15,15 +15,15 @@ class Client(logging.Handler):
         logging (str): socket address to connect to server
     """
 
-    def __init__(self, socket_addr) -> None:
+    def __init__(self, socket_addr, verbose) -> None:
         super().__init__()
         self.socket = socket.socket(socket.AF_UNIX)
         self.socket.settimeout(1)
         self.logger = logging.getLogger(__name__)
-        register_logger(self.logger)
+        register_logger(self.logger, verbose)
         try:
             self.socket.connect(socket_addr)
-            self.logger.info(f"socket connect to {socket_addr}")
+            self.logger.debug(f"socket connect to {socket_addr}")
         except BrokenPipeError as err:
             self.logger.warning(f"socket is broken: {err}")
         except FileNotFoundError as err:
@@ -57,6 +57,7 @@ def register_handler(
     logging_level,
     formatter: logging.Formatter = None,
     socket_addr="/tmp/socket",
+    verbose=False,
 ):
     """Register the Client instance to the logger
 
@@ -71,7 +72,7 @@ def register_handler(
         formatter = logging.Formatter(
             "%(asctime)s - %(name)s - [%(levelname)s]: %(message)s"
         )
-    client = Client(socket_addr)
+    client = Client(socket_addr=socket_addr, verbose=verbose)
     client.setFormatter(formatter)
     client.setLevel(logging_level)
     target_logger.addHandler(client)

socket_logging/server.py

@@ -99,6 +99,7 @@ def __init__(
         defined_handler: logging.Handler,
         socket_address="/tmp/socket",
         batch_size=20000,
+        verbose=False,
     ) -> None:
         """Server for receiving logs from client and do writting logs in batch to files.
 
@@ -112,12 +113,12 @@ def __init__(
         except OSError:
             pass
         self.logger = logging.getLogger(__name__)
-        register_logger(self.logger)
+        register_logger(self.logger, verbose)
 
         self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
         self.socket.bind(socket_address)
         self.socket.listen()
-        self.logger.info(f"listening on {socket_address}")
+        self.logger.debug(f"listening on {socket_address}")
         self.socket.setblocking(False)
 
         self.sel = selectors.DefaultSelector()
@@ -131,7 +132,7 @@ def __init__(
 
     def accept(self, sock):
         conn, addr = sock.accept()  # Should be ready to read
-        self.logger.info(f"accepted connection from {addr}")
+        self.logger.debug(f"accepted connection from {addr}")
         conn.setblocking(False)
         events = selectors.EVENT_READ
         self.sel.register(conn, events, data=self.read_data)
@@ -152,7 +153,7 @@ def read_data(self, conn):
             else:
                 self.bytes += LINE_BREAK
         if self.termination.is_set():
-            self.logger.info(f"current total length {self.total_length}")
+            self.logger.debug(f"rest log record total length {self.total_length}")
             self.handler.emit(self.bytes.decode())
             self.bytes = bytearray()
             self.sel.unregister(conn)
@@ -179,7 +180,7 @@ def run(self):
                 callback = key.data
                 callback(key.fileobj)
             if self.termination.is_set():
-                self.logger.info("termination is set")
+                self.logger.debug("termination is set")
                 break
         self.socket.close()
         self.sel.close()

socket_logging/utils.py

@@ -27,12 +27,12 @@ def format(self, record):
         return formatter.format(record)
 
 
-def register_logger(logger=None):
+def register_logger(logger=None, verbose=False):
     """register colorful debug log"""
     if not logger:
         logger = logging.getLogger()
     if not logger.hasHandlers():
         handler = logging.StreamHandler(sys.stderr)
         handler.setFormatter(ColorfulFormatter())
-        logger.setLevel(logging.DEBUG)
+        logger.setLevel(logging.DEBUG) if verbose else logger.setLevel(logging.INFO)
         logger.addHandler(handler)