Merge pull request #24 from Worvast/master

New version 3

CHANGELOG.md

@@ -4,6 +4,42 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [3.0.0] - 2020-04-28
+#### Added 
+ * Faker
+     * Faker time periods for probability and frequency
+     * SimulationGenerator
+     * New providers
+     * New and splitted documentation of use
+     * Examples for scripts and CLI shell
+     * Generators/Faker now accept custom providers, custom functions 
+     from base python or custom scripts for use in templates
+     * CLI Shell has now all necessary flags for use all generators 
+
+#### Changed
+ * General
+    * Modified base devo-sender from version 3.0.x to 3.3.0
+    * Update dependencies to a new majors versions
+    * Set dependencies to fixed versions (more maintenance, but much more security and reliability)
+ * Sorter
+    * file_sorted_join moved from devoutils.sorter to devoutils.fileio
+    * Remove Python 2 compatibility
+ * Faker
+    * Faker cli add verbose mode to show the events in the console
+    * Add file_name param to define a file to store events in batch mode or 
+     for testing, store in a file but do not send it.
+    * Generator template are moved for Template
+    * Generator date_generator are moved for Template
+    * Generators name changed, example: BatchSender for BatchFakerGenerator
+    * Now you can make functions callable in each line of Faker Jinja2 Template
+    * null_date_generator (Default date generator) isn't a generator now, its a normal function
+    * Template.process() now not create new DateGenerator in each call
+    * Numbers_providers has PEP8 variable names
+    * `freq` var/flag in all code are now `frequency`
+    * `prob` var/flag in all code are now `probability`
+    * CLI Shell --config file now require all vars in "faker" object, and all vars for Sender in "sender" object
+
+
 ## [2.0.0] - 2019-07-02
 #### Changed
  * Modified base devo-sender from version 2.x to >=3.0.1
@@ -28,4 +64,4 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [1.0.0] - 2019-01-16
 #### Added
- * On the ninth month of the year, a Devo programmer finally decided to publish a part of the Utils and rest in peace.
+ * On the ninth month of the year, a Devo programmer finally decided to publish a part of the Utils and rest in peace.

README.md

@@ -1,6 +1,6 @@
 [![relese-next Build Status](https://travis-ci.com/DevoInc/python-utils.svg?branch=master)](https://travis-ci.com/DevoInc/python-utils) [![LICENSE](https://img.shields.io/dub/l/vibe-d.svg)](https://github.com/DevoInc/python-utils/blob/master/LICENSE)
 
-[![wheel](https://img.shields.io/badge/wheel-yes-brightgreen.svg)](https://pypi.org/project/devo-utils/) [![version](https://img.shields.io/badge/version-2.0.1-blue.svg)](https://pypi.org/project/devo-utils/) [![python](https://img.shields.io/badge/python-3.5%20%7C%203.6%20%7C%203.7-blue.svg)](https://pypi.org/project/devo-utils/)
+[![wheel](https://img.shields.io/badge/wheel-yes-brightgreen.svg)](https://pypi.org/project/devo-utils/) [![version](https://img.shields.io/badge/version-3.0.0-blue.svg)](https://pypi.org/project/devo-utils/) [![python](https://img.shields.io/badge/python-3.5%20%7C%203.6%20%7C%203.7%20%7C%203.8-blue.svg)](https://pypi.org/project/devo-utils/)
 
 
 # Devo Python Utils
@@ -41,8 +41,18 @@ You can use sources files, clonning the project too:
 
 You have specific documentation in _[docs](docs)_ folder for each part of SDK:
 * [Faker: fake data](docs/faker.md)
-* [File IO](docs/io/fileio.md)
-* [Sorting data](docs/sorter.md)
+* [File IO](docs/fileio.md)
+* [Sorting data](docs/sorter.md)If you wrap a line with the block creator
+
+{% - set .... -%}
+
+Faker will not treat that line as a send line, if a variable is created without the hyphens, for example:
+
+{% set ....%}
+
+Faker will believe that it is a line to send, so you have to keep this in mind
+
+And examples in `examples` folder.
 
 
 ## Contributing

devoutils/__version__.py

@@ -1,7 +1,7 @@
 __description__ = 'Devo Python Library.'
 __url__ = 'http://www.devo.com'
-__version__ = "2.0.1"
+__version__ = "3.0.0"
 __author__ = 'Devo'
 __author_email__ = '[email protected]'
 __license__ = 'MIT'
-__copyright__ = 'Copyright 2019 Devo'
+__copyright__ = 'Copyright 2020 Devo'

devoutils/faker/__init__.py

@@ -1,5 +1,6 @@
-from .senders.batch_sender import BatchSender
-from .senders.file_sender import FileSender
-from .senders.syslog_sender import SyslogSender
-from .senders.syslog_raw_sender import SyslogRawSender
-from .senders.template_parser import TemplateParser
+from .generators.batch_fake_generator import BatchFakeGenerator
+from .generators.file_fake_generator import FileFakeGenerator
+from .generators.syslog_fake_generator import SyslogFakeGenerator
+from .generators.syslog_raw_fake_generator import SyslogRawFakeGenerator
+from .generators.template_parser import TemplateParser
+from .generators.simulation_fake_generator import SimulationFakeGenerator

devoutils/faker/generators/__init__.py

@@ -0,0 +1,5 @@
+from .batch_fake_generator import BatchFakeGenerator
+from .file_fake_generator import FileFakeGenerator
+from .syslog_fake_generator import SyslogFakeGenerator
+from .syslog_raw_fake_generator import SyslogRawFakeGenerator
+from .template_parser import TemplateParser

devoutils/faker/generators/base_fake_generator.py

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+"""Base Sender, father of all other providers"""
+import random
+import threading
+import time
+from datetime import datetime, timedelta
+from pycron import is_now
+from .template_parser import TemplateParser
+
+
+class BaseFakeGenerator(threading.Thread):
+    """Base provider main class"""
+
+    def __init__(self, engine=None, template=None, **kwargs):
+        """
+        Init BaseFakeGenerator, all generator use this class as father
+
+        :param engine (Sender): Sender object, if neccesary, for send data
+        :param template (str, bytes): Template, read, in a variable, not the
+        path to a file
+        :param kwargs: List of accepted variables below
+            :param time_rules (dict): Time rules as crondates format for
+            variable frequeny and probability
+            :param frequency (list): Fixed shipping frequency list (min, max)
+            in seconds
+            :param probability (int): Fixed shipping probability
+            from 0 to 100(%)
+            :param interactive (bool): Interactive mode
+            :param simulation (bool): Simulation mode, not send/write if true
+            :param verbose (bool): Verbose mode
+            :param date_generator (object): Alternative date generator
+            for templates
+            :param providers (dict): Custom providers dict
+        """
+
+        threading.Thread.__init__(self)
+        self.engine = engine
+
+        if kwargs.get('time_rules', None) is not None:
+            self.time_rules = kwargs.get("time_rules")
+            self.set_last_time_rule()
+
+            self.cron_child = threading.Thread(
+                target=self.check_time_rules
+            )
+            self.cron_child.setDaemon(True)
+            self.cron_child.start()
+        else:
+            self.prob = kwargs.get('probability', 100)
+            self.freq = kwargs.get('frequency', (1, 1))
+
+        self.interactive = kwargs.get('interactive', False)
+        self.simulation = kwargs.get('simulation', False)
+        self.verbose = kwargs.get('verbose', False)
+        self.parser = TemplateParser(template=str(template),
+                                     providers=kwargs.get('providers', {}),
+                                     date_generator=
+                                     kwargs.get('date_generator', None))
+
+    def process(self, date_generator=None, **kwargs):
+        """Process template"""
+        return self.parser.process(date_generator=date_generator,
+                                   **kwargs)
+
+    def wait(self):
+        """Time to wait between events"""
+        # freq[0] is the minimum
+        # freq[1] is the maximum
+        if self.freq[0] == self.freq[1]:
+            secs = self.freq[0]
+        elif self.freq[1] < self.freq[0]:
+            secs = random.uniform(self.freq[1], self.freq[0])
+        else:
+            secs = random.uniform(self.freq[0], self.freq[1])
+        time.sleep(secs)
+
+    def set_last_time_rule(self):
+        """
+        When crondate time rules are used, it is found which one should have
+        been executed last and the probability and frequency are set
+        :return None
+        """
+        now = datetime.now()
+        try:
+            for _ in range(0, 525600):
+                for item in self.time_rules:
+                    if is_now(item["rule"], now):
+                        self.prob = item['probability']
+                        self.freq = item['frequency']
+                        raise Exception('found')
+                now = now - timedelta(minutes=1)
+        except Exception as inst:
+            if inst.args[0] != "found":
+                raise Exception(inst)
+
+    def check_time_rules(self):
+        """
+        Controls every minute if a change in frequency and probability
+        is to be made based on the specified rules
+        :return:
+        """
+        while True:
+            for item in self.time_rules:
+                if is_now(item["rule"]):
+                    self.prob = item['probability']
+                    self.freq = item['frequency']
+                    break
+            time.sleep(60)
+
+    def probability(self):
+        """Calculate probability"""
+        k = random.randint(0, 100)
+        if k <= int(self.prob):
+            return True
+        return False
+
+    def run(self):
+        """Run example (for override)"""
+        while True:
+            if self.probability():
+                # Do something
+                pass
+            self.wait()

devoutils/faker/generators/batch_fake_generator.py

@@ -0,0 +1,104 @@
+"""Batch sender provider"""
+import datetime
+import os.path
+import random
+import sys
+import click
+
+from .base_fake_generator import BaseFakeGenerator
+
+
+class BatchFakeGenerator(BaseFakeGenerator):
+    """
+    Generates a lot of events in a single batch without any waits, it
+    supports generating events from the past
+    """
+
+    GENERATION_ENDED_TOKEN = '###END###'
+
+    def __init__(self, template=None, start_date=None, end_date=None,
+                 **kwargs):
+
+        BaseFakeGenerator.__init__(self, template=template, **kwargs)
+
+        self._start_date = start_date
+        self._end_date = end_date
+        self.date_format = kwargs.get('date_format', "%Y-%m-%d %H:%M:%S.%f")
+        self.dont_remove_microseconds = kwargs.get('dont_remove_microseconds',
+                                                   False)
+        self.file_name = kwargs.get("file_name") if kwargs.get("file_name") \
+            else "eventbatch.log"
+
+    @staticmethod
+    def date_range(start_date=None, end_date=None, frequency=None,
+                   probability=None,
+                   date_format="%Y-%m-%d %H:%M:%S.%f",
+                   dont_remove_microseconds=True):
+        """
+        Generates date range
+        :param start_date:me objects between a start date and an end date with
+        some given frequency,
+        Some events can be piled up in the same millisecond, this happens
+        sometimes with real data.
+        :param end_date:
+        :param start_date:
+        :param frequency:
+        :param probability:
+        :param date_format:
+        :param dont_remove_microseconds:
+        :return:
+        """
+
+        # Control how fast events are spaced between them with a
+        # frequency (--frequency argument)
+        millis = 0
+        idx = 0
+        while True:
+
+            millis_increment = random.uniform(frequency[0],
+                                              frequency[1]) * 1000
+
+            # Add some randomness to the event generation with
+            # the --probability argument
+            k = random.randint(0, 100)
+            if k <= int(probability):
+                millis += millis_increment
+
+            idx += 1
+            next_date = start_date + datetime.timedelta(milliseconds=millis)
+
+            if next_date > end_date:
+                yield BatchFakeGenerator.GENERATION_ENDED_TOKEN
+            else:
+                if dont_remove_microseconds:
+                    yield next_date.strftime(date_format)
+                else:
+                    yield next_date.strftime(date_format)[:-3]
+
+    def run(self):
+        """Run function for cli or call function"""
+        if os.path.exists(self.file_name):
+            raise ValueError('an {} file already exists, remove it before'
+                             ' generating new events'.format(self.file_name))
+
+        counter = 0
+        date_generator = self.date_range(
+            self._start_date, self._end_date, self.freq, self.prob,
+            self.date_format, self.dont_remove_microseconds)
+        with open(self.file_name, "w") as f:
+            while True:
+                lines = self.process(date_generator=date_generator).split('\n')
+                for line in lines:
+                    if BatchFakeGenerator.GENERATION_ENDED_TOKEN in line:
+                        print("Wrote {} events in {}".format(counter,
+                                                             self.file_name),
+                              file=sys.stdout)
+                        return
+                    if counter % 100 == 0:
+                        click.echo('Wrote {} lines'.format(counter),
+                                   file=sys.stderr)
+                        click.echo(line[:40], file=sys.stderr)
+                    if self.verbose:
+                        print(line, file=sys.stdout)
+                    f.write(line+"\n")
+                    counter += 1

devoutils/faker/generators/file_fake_generator.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+"""File Sender provider"""
+from datetime import datetime
+from .realtime_fake_generator import RealtimeFakeGenerator
+
+
+class FileFakeGenerator(RealtimeFakeGenerator):
+    """Generate a lot of events from file"""
+    def __init__(self, template=None, **kwargs):
+        RealtimeFakeGenerator.__init__(self, template=template, **kwargs)
+        self.last_flush = int(datetime.now().timestamp())
+        self.file_name = kwargs.get('file_name', "safestream.log")
+        self.file = None
+
+    def write_row(self, message=None):
+        """Write row to file"""
+        self.file.write(message)
+        self.file.write('\n')
+        now = int(datetime.now().timestamp())
+        # flush every 5 secs
+        if now - self.last_flush > 5:
+            self.last_flush = now
+            self.file.flush()
+
+    def run(self):
+        """Run function for cli or call function"""
+        with open(self.file_name, "a") as self.file:
+            self.realtime_iteration(write_function=self.write_row)

devoutils/faker/generators/realtime_fake_generator.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""realtime base sender provider"""
+from datetime import datetime
+from .base_fake_generator import BaseFakeGenerator
+
+
+class RealtimeFakeGenerator(BaseFakeGenerator):
+    """Realtime fake data generation"""
+    def __init__(self, engine=None, template=None, **kwargs):
+        """Realtime fake data generation"""
+        BaseFakeGenerator.__init__(self, engine=engine, template=template,
+                                   **kwargs)
+
+    def realtime_iteration(self, write_function=None):
+        """Realtime function for parse and send/show generated data"""
+        while True:
+            lines = self.process().split('\n')
+            for line in lines:
+                if self.probability():
+                    if not self.simulation:
+                        write_function(line)
+                    now = datetime.utcnow().ctime()
+                    if self.verbose:
+                        print('{0} => {1}'.format(now, str(line)))
+                else:
+                    now = datetime.utcnow().ctime()
+                    if self.verbose:
+                        print('{0} => Skipped by prob.'.format(now))
+
+                if self.interactive:
+                    input("» Press Enter for next iteration «")
+                else:
+                    self.wait()