From 3c304511faa9f5343e07f3fb43bdcb0d8ac43b7e Mon Sep 17 00:00:00 2001 From: Qishuai Zhong Date: Wed, 10 Nov 2021 16:59:18 +0800 Subject: [PATCH 1/2] add verbose and update version --- examples/{simple.py => simple_sample.py} | 4 ++-- setup.py | 4 ++-- socket_logging/client.py | 9 +++++---- socket_logging/server.py | 11 ++++++----- socket_logging/utils.py | 4 ++-- 5 files changed, 17 insertions(+), 15 deletions(-) rename examples/{simple.py => simple_sample.py} (92%) diff --git a/examples/simple.py b/examples/simple_sample.py similarity index 92% rename from examples/simple.py rename to examples/simple_sample.py index 074ed53..27c0b1a 100644 --- a/examples/simple.py +++ b/examples/simple_sample.py @@ -18,7 +18,7 @@ def worker(text): formatter = logging.Formatter( "%(asctime)s - %(name)s - %(filename)s- [%(levelname)s]: %(message)s" ) - register_handler(client_logger, logging_level=logging.INFO, formatter=formatter) + register_handler(client_logger, logging_level=logging.INFO, formatter=formatter, verbose=True) client_logger.setLevel(logging.INFO) pid = os.getpid() for i in range(TEST_ROUND): @@ -47,7 +47,7 @@ def sample_running(text): server_handler = ServerHandler(LOG_FILE, maxBytes=MAX_BYTES) server_handler.setLevel(logging.INFO) - server = Server(server_handler, batch_size=BATCH_SIZE) + server = Server(server_handler, batch_size=BATCH_SIZE, verbose=True) server.start() sample_running(args.text) diff --git a/setup.py b/setup.py index cecae85..376777b 100644 --- a/setup.py +++ b/setup.py @@ -9,13 +9,13 @@ setup( name="mp-socket-logging", - version="0.1.0", + version="0.1.1", description="A small toolkit for python multiprocessing file logging", long_description=readme, long_description_content_type="text/markdown", author="Zhong Qishuai", author_email="ferdinandzhong@gmail.com", - url="https://git.garena.com/shopee-ds/ml-platform/sre-utils", + url="https://github.com/FerdinandZhong/mp-socket-logging", packages=find_packages(exclude=["tests*", "example*"]), classifiers=[ "Programming Language :: Python :: 3", diff --git a/socket_logging/client.py b/socket_logging/client.py index 8ac0e04..62c7cb5 100644 --- a/socket_logging/client.py +++ b/socket_logging/client.py @@ -15,15 +15,15 @@ class Client(logging.Handler): logging (str): socket address to connect to server """ - def __init__(self, socket_addr) -> None: + def __init__(self, socket_addr, verbose) -> None: super().__init__() self.socket = socket.socket(socket.AF_UNIX) self.socket.settimeout(1) self.logger = logging.getLogger(__name__) - register_logger(self.logger) + register_logger(self.logger, verbose) try: self.socket.connect(socket_addr) - self.logger.info(f"socket connect to {socket_addr}") + self.logger.debug(f"socket connect to {socket_addr}") except BrokenPipeError as err: self.logger.warning(f"socket is broken: {err}") except FileNotFoundError as err: @@ -57,6 +57,7 @@ def register_handler( logging_level, formatter: logging.Formatter = None, socket_addr="/tmp/socket", + verbose=False, ): """Register the Client instance to the logger @@ -71,7 +72,7 @@ def register_handler( formatter = logging.Formatter( "%(asctime)s - %(name)s - [%(levelname)s]: %(message)s" ) - client = Client(socket_addr) + client = Client(socket_addr=socket_addr, verbose=verbose) client.setFormatter(formatter) client.setLevel(logging_level) target_logger.addHandler(client) diff --git a/socket_logging/server.py b/socket_logging/server.py index 4fba728..5ac0e9d 100644 --- a/socket_logging/server.py +++ b/socket_logging/server.py @@ -99,6 +99,7 @@ def __init__( defined_handler: logging.Handler, socket_address="/tmp/socket", batch_size=20000, + verbose=False, ) -> None: """Server for receiving logs from client and do writting logs in batch to files. @@ -112,12 +113,12 @@ def __init__( except OSError: pass self.logger = logging.getLogger(__name__) - register_logger(self.logger) + register_logger(self.logger, verbose) self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.socket.bind(socket_address) self.socket.listen() - self.logger.info(f"listening on {socket_address}") + self.logger.debug(f"listening on {socket_address}") self.socket.setblocking(False) self.sel = selectors.DefaultSelector() @@ -131,7 +132,7 @@ def __init__( def accept(self, sock): conn, addr = sock.accept() # Should be ready to read - self.logger.info(f"accepted connection from {addr}") + self.logger.debug(f"accepted connection from {addr}") conn.setblocking(False) events = selectors.EVENT_READ self.sel.register(conn, events, data=self.read_data) @@ -152,7 +153,7 @@ def read_data(self, conn): else: self.bytes += LINE_BREAK if self.termination.is_set(): - self.logger.info(f"current total length {self.total_length}") + self.logger.debug(f"rest log record total length {self.total_length}") self.handler.emit(self.bytes.decode()) self.bytes = bytearray() self.sel.unregister(conn) @@ -179,7 +180,7 @@ def run(self): callback = key.data callback(key.fileobj) if self.termination.is_set(): - self.logger.info("termination is set") + self.logger.debug("termination is set") break self.socket.close() self.sel.close() diff --git a/socket_logging/utils.py b/socket_logging/utils.py index 6ff72a3..8c4462f 100644 --- a/socket_logging/utils.py +++ b/socket_logging/utils.py @@ -27,12 +27,12 @@ def format(self, record): return formatter.format(record) -def register_logger(logger=None): +def register_logger(logger=None, verbose=False): """register colorful debug log""" if not logger: logger = logging.getLogger() if not logger.hasHandlers(): handler = logging.StreamHandler(sys.stderr) handler.setFormatter(ColorfulFormatter()) - logger.setLevel(logging.DEBUG) + logger.setLevel(logging.DEBUG) if verbose else logger.setLevel(logging.INFO) logger.addHandler(handler) From 065a02f4b80692192a70f4e9c4b1ffb6b813b285 Mon Sep 17 00:00:00 2001 From: Qishuai Zhong Date: Wed, 10 Nov 2021 17:04:41 +0800 Subject: [PATCH 2/2] reformatting --- examples/simple_sample.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/simple_sample.py b/examples/simple_sample.py index 27c0b1a..0712b89 100644 --- a/examples/simple_sample.py +++ b/examples/simple_sample.py @@ -18,7 +18,9 @@ def worker(text): formatter = logging.Formatter( "%(asctime)s - %(name)s - %(filename)s- [%(levelname)s]: %(message)s" ) - register_handler(client_logger, logging_level=logging.INFO, formatter=formatter, verbose=True) + register_handler( + client_logger, logging_level=logging.INFO, formatter=formatter, verbose=True + ) client_logger.setLevel(logging.INFO) pid = os.getpid() for i in range(TEST_ROUND):