From 240a95c6a61a5ff86b6fe17905c9570669eece81 Mon Sep 17 00:00:00 2001 From: nir0s Date: Thu, 8 Mar 2018 20:40:10 +0200 Subject: [PATCH] Add syslog and file handlers with env var configs based on best effort --- README.md | 86 +++++++++++++++++++++++++++++++++++------- wryte.py | 110 +++++++++++++++++++++++++++++++++--------------------- 2 files changed, 140 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 35e96c8..13725c8 100644 --- a/README.md +++ b/README.md @@ -458,32 +458,92 @@ The idea behind this is three-fold: ### Using Environment Variables to configure logging handlers +NOTE: This is WIP, so things may break / be broken. To truly be able to use this feature, Wryte will have to support logger-name-based env vars (e.g. `WRYTE_HANDLERS_logger_name_*`). + +NOTE: DO NOT use this feature if you have multiple loggers in the same service unless you explicitly intend to have all loggers log to all handlers configured. + One of Wryte's goals is to provide a simple way to configure loggers. Much like Grafana and Fabio, Wryte aims to be completely env-var configurable. -A POC currently exists for using environment variables to enable certain handlers: +On top of having two default `console` and `json` handlers which indicate the formatting and both log to stdout, you can utilize built-in and 3rd party handlers quite easily. -```bash -export WRYTE_FILE_PATH=PATH_TO_OUTPUT_FILE -export WRYTE_LOGZIO_TOKEN=YOUR_LOGZIO_TOKEN +#### File Handler + +Wryte supports both the rotating and watching file handlers (on Windows, FileHandler replaces WatchingFileHandler if not rotating). -export WRYTE_ELASTICSEARCH_HOSTS=localhost:9200,elasticsearch.service.consul:9200 -export WRYTE_ELASTICSEARCH_INDEX (defaults to `logs`) ``` +WRYTE_HANDLERS_FILE_ENABLED=true # If set, enables the handler. -Will automatically append `json` formatted handlers to any logger you instantiate. -Of course, this should be configurable on a logger level so, when this is done, it should provide something like: +WRYTE_HANDLERS_FILE_PATH=FILE_TO_LOG_TO # (Required) Absolute path to the file logs should be written to +WRYTE_HANDLERS_FILE_NAME='file' # The logger's name +WRYTE_HANDLERS_FILE_LEVEL='info' # The logger's default level +WRYTE_HANDLERS_FLIE_FORMATTER='json' # The logger format to use + +WRYTE_HANDLERS_FILE_ROTATE=false # Rotate the files? Defaults to false in favor of explicitness so that people who use logrotate won't double-rotate by accident. +WRYTE_HANDLERS_FILE_MAX_BYTES=13107200 # Size of each file in bytes if rotating +WRYTE_HANDLERS_FILE_BACKUP_COUNT=7 # Amount of logs files to keep ``` -export WRYTE_logger_name_FILE_PATH=... -export WRYTE_logger_name_LOGZIO_TOKEN=... + +#### Syslog Handler + +Allows to emit logs to a Syslog server + ``` +WRYTE_HANDLERS_SYSLOG_ENABLED=true # If set, enables the handler. + +WRYTE_HANDLERS_SYSLOG_NAME='syslog' # The logger's name +WRYTE_HANDLERS_SYSLOG_LEVEL='info' # The logger's default level +WRYTE_HANDLERS_SYSLOG_FORMATTER='json' # The logger format to use + +WRYTE_HANDLERS_SYSLOG_HOST='localhost:514' # Colon seprated syslog host string +WRYTE_HANDLERS_SYSLOG_SOCKET_TYPE='udp' # udp/tcp +WRYTE_HANDLERS_SYSLOG_FACILITY='LOG_USER' # Syslog facility to use (see https://success.trendmicro.com/solution/TP000086250-What-are-Syslog-Facilities-and-Levels) +``` + +#### Elasticsearch Handler + +While it may be useful to send your messages through logstash, you may also log to Elasticsearch directly. + +Wryte utilizes the [CMRESHandler](https://github.com/cmanaha/python-elasticsearch-logger) for this. +Currently, only the hosts can be supplied. SSL, index name pattern, etc.. will be added later. + +To install the handler, run `pip install wryte[elasticsearch]`. + +``` +WRYTE_HANDLERS_ELASTICSEARCH_ENABLED=true # If set, enables the handler. + +WRYTE_HANDLERS_ELASTICSEARCH_NAME='elasticsearch' # The logger's name +WRYTE_HANDLERS_ELASTICSEARCH_LEVEL='info' # The logger's default level +WRYTE_HANDLERS_ELASTICSEARCH_FORMATTER='json' # The logger format to use + +WRYTE_HANDLERS_ELASTICSEARCH_HOST=http://es.dc1.service.consul:9200,http://es.dc1.service.consul:9200 # (Required) A comma-separated list of host:port pairs to use. +``` + +#### Logzio Handler + +You can also directly send your logs to logzio via the official [logzio handler](https://github.com/logzio/logzio-python-handler). + +To install the handler, run `pip install wryte[logzio]`. + +``` +WRYTE_HANDLERS_LOGZIO_ENABLED=true # If set, enables the handler. + +WRYTE_HANDLERS_LOGZIO_NAME='logzio' # The logger's name +WRYTE_HANDLERS_LOGZIO_LEVEL='info' # The logger's default level +WRYTE_HANDLERS_LOGZIO_FORMATTER='json' # The logger format to use + +WRYTE_HANDLERS_LOGZIO_TOKEN=oim12o3i3ou2itj3jkdng3bgjs1gbg # (Required) Your logzio API token +``` + +See https://github.com/nir0s/wryte/issues/10 and https://github.com/nir0s/wryte/issues/18 for more info. -See https://github.com/nir0s/wryte/issues/10 for more info. +#### Examples -Example: +Logging to file: ``` -$ export WRYTE_FILE_PATH=log.file +$ export WRYTE_HANDLERS_FILE_ENABLED=true +$ export WRYTE_HANDLERS_FILE_PATH=log.file $ python wryte.py 2018-02-18T08:56:27.921500 - Wryte - INFO - Logging an error level message: diff --git a/wryte.py b/wryte.py index cc2b864..b74a374 100644 --- a/wryte.py +++ b/wryte.py @@ -263,13 +263,16 @@ def _configure_handlers(self, level, jsonify=False): else: self.add_default_json_handler(level) - if self._env('LOGZIO_TOKEN'): + if self._env('HANDLERS_SYSLOG_ENABLED'): + self.add_syslog_handler() + + if self._env('HANDLERS_LOGZIO_ENABLED'): self.add_logzio_handler() - if self._env('FILE_PATH'): + if self._env('HANDLERS_FILE_ENABLED'): self.add_file_handler() - if self._env('ELASTICSEARCH_HOST'): + if self._env('HANDLERS_ELASTICSEARCH_ENABLED'): self.add_elasticsearch_handler() def add_handler(self, @@ -335,20 +338,23 @@ def add_default_console_handler(self, level='debug'): level=level) def add_file_handler(self, **kwargs): - name = self._env('FILE_NAME', default='file') - level = self._env('FILE_LEVEL', default='debug') - formatter = self._env('FILE_FORMATTER', default='json') + assert self._env('HANDLERS_FILE_PATH') + + name = self._env('HANDLERS_FILE_NAME', default='file') + level = self._env('HANDLERS_FILE_LEVEL', default='info') + formatter = self._env('HANDLERS_FILE_FORMATTER', default='json') - if self._env('FILE_ROTATE'): + if self._env('HANDLERS_FILE_ROTATE'): handler = logging.handlers.RotatingFileHandler( - self._env('FILE_PATH'), - maxBytes=int(self._env('FILE_MAX_BYTES', default=13107200)), - backupCount=int(self._env('FILE_BACKUP_COUNT', default=7))) + self._env('HANDLERS_FILE_PATH'), + maxBytes=int( + self._env('HANDLERS_FILE_MAX_BYTES', default=13107200)), + backupCount=int(self._env('HANDLERS_FILE_BACKUP_COUNT', default=7))) elif os.name == 'nt': - handler = logging.FileHandler(self._env('FILE_PATH')) + handler = logging.FileHandler(self._env('HANDLERS_FILE_PATH')) else: handler = logging.handlers.WatchedFileHandler( - self._env('FILE_PATH')) + self._env('HANDLERS_FILE_PATH')) self.add_handler( handler=handler, @@ -357,12 +363,13 @@ def add_file_handler(self, **kwargs): level=level) def add_syslog_handler(self, **kwargs): - name = self._env('SYSLOG_NAME', default='syslog') - level = self._env('SYSLOG_LEVEL', default='info') - formatter = 'json' + name = self._env('HANDLERS_SYSLOG_NAME', default='syslog') + level = self._env('HANDLERS_SYSLOG_LEVEL', default='info') + formatter = self._env('HANDLERS_SYSLOG_FORMATTER', default='json') - syslog_host = self._env('SYSLOG_HOST', default='localhost:514') - syslog_host = syslog_host.split(':') + syslog_host = self._env('HANDLERS_SYSLOG_HOST', + default='localhost:514') + syslog_host = syslog_host.split(':', 1) if len(syslog_host) == 2: # Syslog listener @@ -372,9 +379,14 @@ def add_syslog_handler(self, **kwargs): # Unix socket or otherwise address = syslog_host + socket_type = self._env('HANDLERS_SYSLOG_SOCKET_TYPE', default='udp') + assert socket_type in ('tcp', 'udp') + handler = logging.handlers.SysLogHandler( address=address, - facility=self._env('SYSLOG_FACILITY', default='LOG_USER')) + facility=self._env('HANDLERS_SYSLOG_FACILITY', default='LOG_USER'), + socktype=socket.SOCK_STREAM if socket_type == 'tcp' + else socket.SOCK_DGRAM) self.add_handler( handler=handler, @@ -384,10 +396,12 @@ def add_syslog_handler(self, **kwargs): def add_logzio_handler(self, **kwargs): if LOGZIO_INSTALLED: - name = self._env('LOGZIO_NAME', default='logzio-python') - level = self._env('LOGZIO_LEVEL', default='info') - formatter = 'json' - handler = LogzioHandler(self._env('LOGZIO_TOKEN')) + assert self._env('HANDLERS_LOGZIO_TOKEN') + + name = self._env('HANDLERS_LOGZIO_NAME', default='logzio') + level = self._env('HANDLERS_LOGZIO_LEVEL', default='info') + formatter = self._env('HANDLERS_LOGZIO_FORMATTER', default='json') + handler = LogzioHandler(self._env('HANDLERS_LOGZIO_TOKEN')) self.add_handler( handler=handler, @@ -401,29 +415,39 @@ def add_logzio_handler(self, **kwargs): 'wryte[logzio]`') def add_elasticsearch_handler(self, **kwargs): - name = self._env('ELASTICSEARCH_NAME', default='elasticsearch') - level = self._env('ELASTICSEARCH_LEVEL', default='info') - formatter = 'json' - - hosts = [] - es_hosts = self._env('ELASTICSEARCH_HOST') - es_hosts = es_hosts.split(',') - for es_host in es_hosts: - host, port = es_host.split(':', 1) - hosts.append({'host': host, 'port': port}) - - handler_args = { - 'hosts': hosts, - 'auth_type': CMRESHandler.AuthType.NO_AUTH - } + if ELASTICSEARCH_INSTALLED: + assert self._env('HANDLERS_ELASTICSEARCH_HOST') - handler = CMRESHandler(**handler_args) + name = self._env('HANDLERS_ELASTICSEARCH_NAME', + default='elasticsearch') + level = self._env('HANDLERS_ELASTICSEARCH_LEVEL', default='info') + formatter = self._env( + 'HANDLER_ELASTICSEARCH_FORMATTER', default='json') - self.add_handler( - handler=handler, - name=name, - formatter=formatter, - level=level) + hosts = [] + es_hosts = self._env('HANDLERS_ELASTICSEARCH_HOST') + es_hosts = es_hosts.split(',') + for es_host in es_hosts: + host, port = es_host.split(':', 1) + hosts.append({'host': host, 'port': port}) + + handler_args = { + 'hosts': hosts, + 'auth_type': CMRESHandler.AuthType.NO_AUTH + } + + handler = CMRESHandler(**handler_args) + + self.add_handler( + handler=handler, + name=name, + formatter=formatter, + level=level) + else: + raise WryteError( + 'It seems that the elasticsearch handler is not installed. ' + 'You can install it by running `pip install ' + 'wryte[elasticsearch]`') def set_level(self, level): # TODO: Consider removing this check and letting the user