diff --git a/.github/workflows/package.yaml b/.github/workflows/package.yaml index 8cf272b..3f0b037 100644 --- a/.github/workflows/package.yaml +++ b/.github/workflows/package.yaml @@ -3,7 +3,7 @@ on: pull_request: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] @@ -11,7 +11,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} architecture: x64 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 94e0bd1..76775d4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,7 +3,7 @@ on: pull_request: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] diff --git a/CHANGELOG.md b/CHANGELOG.md index 0082aa1..582523a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +## [5.2.1] - 2023-11-09 + +- PPPSYS-44518 Converted int values like `gt_ms` to strings +- DEVOPS-5661 Missing timeouts for gha workflow ## [5.2.0] - 2022-07-11 diff --git a/piwik_pro_log_analytics/import_logs.py b/piwik_pro_log_analytics/import_logs.py index 3bec8a3..0a713c9 100755 --- a/piwik_pro_log_analytics/import_logs.py +++ b/piwik_pro_log_analytics/import_logs.py @@ -239,7 +239,6 @@ def remove_ignored_groups(self, groups): class W3cExtendedFormat(RegexFormat): - FIELDS_LINE_PREFIX = "#Fields: " REGEX_UNKNOWN_FIELD = r'(?:".*?"|\S+)' @@ -336,7 +335,6 @@ def create_regex(self, file): self.regex = re.compile(full_regex) def _configure_expected_fields(self): - expected_fields = type( self ).fields.copy() # turn custom field mapping into field => regex mapping @@ -384,7 +382,6 @@ def _is_time_taken_milli(self): class IisFormat(W3cExtendedFormat): - fields = W3cExtendedFormat.fields.copy() fields.update( { @@ -405,7 +402,6 @@ def __init__(self): class IncapsulaW3CFormat(W3cExtendedFormat): - # use custom unknown field regex to make resulting regex much simpler REGEX_UNKNOWN_FIELD = r'".*?"' @@ -438,7 +434,6 @@ def get(self, key): class ShoutcastFormat(W3cExtendedFormat): - fields = W3cExtendedFormat.fields.copy() fields.update( { @@ -461,7 +456,6 @@ def get(self, key): class AmazonCloudFrontFormat(W3cExtendedFormat): - fields = W3cExtendedFormat.fields.copy() fields.update( { @@ -2154,7 +2148,7 @@ def execute(self, args_config, initial_args=None): args = initial_args or {} if args_config.hit.generation_time_milli > 0: - args["gt_ms"] = int(args_config.hit.generation_time_milli) + args["gt_ms"] = str(int(args_config.hit.generation_time_milli)) if args_config.hit.event_category and args_config.hit.event_action: args["e_c"] = args_config.hit.event_category @@ -2164,7 +2158,7 @@ def execute(self, args_config, initial_args=None): args["e_n"] = args_config.hit.event_name if args_config.hit.length: - args["bw_bytes"] = args_config.hit.length + args["bw_bytes"] = str(args_config.hit.length) # convert custom variable args to JSON if "cvar" in args and not isinstance(args["cvar"], str): @@ -2372,7 +2366,6 @@ def _record_hits(self, hits, single=False): if hit_count > 0: try: - response = piwik.call( config.options.piwik_tracker_endpoint_path, args=args, @@ -2713,7 +2706,6 @@ def filtered_line(line, reason): logging.debug("Filtered line out (%s): %s" % (reason, line)) def _get_file_and_filename(self, filename): - if filename == "-": return "(stdin)", sys.stdin else: @@ -2741,7 +2733,6 @@ def _get_file_and_filename(self, filename): # Returns True if format was configured def _configure_format(self, file): - if config.format: # The format was explicitly specified. format = config.format @@ -2996,7 +2987,7 @@ def parse(self, filename): # noqa C901 if config.options.seconds_to_add_to_date: for param in ["_idts", "_viewts", "_ects", "_refts"]: if param in hit.args: - hit.args[param] = ( + hit.args[param] = str( int(hit.args[param]) + config.options.seconds_to_add_to_date ) diff --git a/tests/test_main.py b/tests/test_main.py index 966883c..c62afc5 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -119,7 +119,6 @@ def _test_ipv6(self, format_name, log_file=None): assert groups["ip"] == "0:0:0:0:0:ffff:7b2d:4359" def test_format_detection(self): - for format_name in import_logs.FORMATS.keys(): # w3c extended tested by iis and netscaler log files; amazon cloudfront tested later if ( @@ -259,20 +258,20 @@ def test_replay_tracking_seconds_to_add_to_date(): hits = [hit.args for hit in import_logs.Recorder.recorders] - assert hits[0]["_idts"] == 1360047661 + 3600 - assert hits[0]["_viewts"] == 1360047661 + 3600 - assert hits[0]["_refts"] == 1360047661 + 3600 - assert hits[0]["_ects"] == 1360047634 + 3600 + assert hits[0]["_idts"] == str(1360047661 + 3600) + assert hits[0]["_viewts"] == str(1360047661 + 3600) + assert hits[0]["_refts"] == str(1360047661 + 3600) + assert hits[0]["_ects"] == str(1360047634 + 3600) - assert hits[1]["_idts"] == 1360047661 + 3600 - assert hits[1]["_viewts"] == 1360047661 + 3600 - assert hits[1]["_refts"] == 1360047661 + 3600 - assert hits[1]["_ects"] == 1360047534 + 3600 + assert hits[1]["_idts"] == str(1360047661 + 3600) + assert hits[1]["_viewts"] == str(1360047661 + 3600) + assert hits[1]["_refts"] == str(1360047661 + 3600) + assert hits[1]["_ects"] == str(1360047534 + 3600) - assert hits[2]["_idts"] == 1360047661 + 3600 - assert hits[2]["_viewts"] == 1360047661 + 3600 - assert hits[2]["_refts"] == 1360047661 + 3600 - assert hits[2]["_ects"] == 1360047614 + 3600 + assert hits[2]["_idts"] == str(1360047661 + 3600) + assert hits[2]["_viewts"] == str(1360047661 + 3600) + assert hits[2]["_refts"] == str(1360047661 + 3600) + assert hits[2]["_ects"] == str(1360047614 + 3600) def test_replay_tracking_arguments(): @@ -714,7 +713,7 @@ def test_shoutcast_parsing(): assert hits[0]["is_download"] is False assert hits[0]["referrer"] == "" assert hits[0]["args"] == {} - assert hits[0]["generation_time_milli"] == 1000.0 + assert hits[0]["generation_time_milli"] == 1000 assert hits[0]["host"] == "foo" assert hits[0]["filename"] == "logs/shoutcast.log" assert hits[0]["is_redirect"] is False @@ -754,7 +753,7 @@ def test_splitted_date_and_time_parsing(): assert hits[0]["is_download"] is False assert hits[0]["referrer"] == "" assert hits[0]["args"] == {} - assert hits[0]["generation_time_milli"] == 1000.0 + assert hits[0]["generation_time_milli"] == 1000 assert hits[0]["host"] == "foo" assert hits[0]["filename"] == "logs/splitted_date_and_time.log" assert hits[0]["is_redirect"] is False @@ -1458,7 +1457,6 @@ def test_bz2_parsing(): def test_static_resolver_with_idsite(): - import_logs.piwik = Piwik() import_logs.stats = import_logs.Statistics() import_logs.resolver = import_logs.StaticResolver("194edb22-394a-48e5-aed8-0797ab29d2ae")