From 4d5321b8bbd545c8967bde2b1a04709f027f4a89 Mon Sep 17 00:00:00 2001 From: Andreas Florath Date: Fri, 3 Jul 2020 07:16:29 +0000 Subject: [PATCH] Added support for python 3.8 Signed-off-by: Andreas Florath --- .travis.yml | 21 ++- COPYING | 3 +- doc/release_notes/25.rst | 2 + rmtoo/lib/xmlutils/xmlcmp.py | 20 ++- .../{gantt2.xml => _ALT_001_gantt2.xml} | 0 .../result_should/_ALT_002_gantt2.xml | 16 ++ .../{gantt2.xml => _ALT_001_gantt2.xml} | 0 .../result_should/_ALT_002_gantt2.xml | 167 ++++++++++++++++++ .../{gantt2.xml => _ALT_001_gantt2.xml} | 0 .../result_should/_ALT_002_gantt2.xml | 114 ++++++++++++ .../{gantt2.xml => _ALT_001_gantt2.xml} | 0 .../result_should/_ALT_002_gantt2.xml | 114 ++++++++++++ .../{gantt2.xml => _ALT_001_gantt2.xml} | 0 .../result_should/_ALT_002_gantt2.xml | 116 ++++++++++++ .../{gantt2.xml => _ALT_001_gantt2.xml} | 0 .../result_should/_ALT_002_gantt2.xml | 114 ++++++++++++ rmtoo/tests/lib/BBHelper.py | 144 ++++++++++++--- tox.ini | 21 ++- 18 files changed, 825 insertions(+), 27 deletions(-) rename rmtoo/tests/RMTTest-Blackbox/RMTTest-BB001/result_should/{gantt2.xml => _ALT_001_gantt2.xml} (100%) create mode 100644 rmtoo/tests/RMTTest-Blackbox/RMTTest-BB001/result_should/_ALT_002_gantt2.xml rename rmtoo/tests/RMTTest-Blackbox/RMTTest-BB002/result_should/{gantt2.xml => _ALT_001_gantt2.xml} (100%) create mode 100644 rmtoo/tests/RMTTest-Blackbox/RMTTest-BB002/result_should/_ALT_002_gantt2.xml rename rmtoo/tests/RMTTest-Blackbox/RMTTest-BB003/result_should/{gantt2.xml => _ALT_001_gantt2.xml} (100%) create mode 100644 rmtoo/tests/RMTTest-Blackbox/RMTTest-BB003/result_should/_ALT_002_gantt2.xml rename rmtoo/tests/RMTTest-Blackbox/RMTTest-BB004/result_should/{gantt2.xml => _ALT_001_gantt2.xml} (100%) create mode 100644 rmtoo/tests/RMTTest-Blackbox/RMTTest-BB004/result_should/_ALT_002_gantt2.xml rename rmtoo/tests/RMTTest-Blackbox/RMTTest-BB010/result_should/{gantt2.xml => _ALT_001_gantt2.xml} (100%) create mode 100644 rmtoo/tests/RMTTest-Blackbox/RMTTest-BB010/result_should/_ALT_002_gantt2.xml rename rmtoo/tests/RMTTest-Blackbox/RMTTest-BB014/result_should/{gantt2.xml => _ALT_001_gantt2.xml} (100%) create mode 100644 rmtoo/tests/RMTTest-Blackbox/RMTTest-BB014/result_should/_ALT_002_gantt2.xml diff --git a/.travis.yml b/.travis.yml index a06461d8..ec869d5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,22 @@ +# (c) 2020 by flonatel GmbH & Co. KG / Andreas Florath +# +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of rmtoo. +# +# rmtoo is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# rmtoo is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rmtoo. If not, see . + language: python git: @@ -9,6 +28,7 @@ python: - "3.5" - "3.6" - "3.7" + - "3.8" # Required to run the respective python env in tox install: pip install tox-travis @@ -17,4 +37,3 @@ install: pip install tox-travis script: - tox - tox -e pep8 - diff --git a/COPYING b/COPYING index 8c22db37..26eb344a 100644 --- a/COPYING +++ b/COPYING @@ -1,9 +1,10 @@ The provided material is -copyright 2010-2012,2017 by +copyright 2010-2012,2017,2020 by flonatel GmbH & Co. KG +Andreas Florath Raerener Strasse 10b D-52076 Aachen rmtoo@florath.net diff --git a/doc/release_notes/25.rst b/doc/release_notes/25.rst index 1f4d6f3b..90a93e66 100644 --- a/doc/release_notes/25.rst +++ b/doc/release_notes/25.rst @@ -25,6 +25,8 @@ User visible changes -------------------- * Drop support for python 2 - which is EOL +* Include support / tests for all currently supported + python versions: 3.5, 3.6, 3.7, 3.8 * Effort estimation: introduce flag to allow any value for the effort estimation. diff --git a/rmtoo/lib/xmlutils/xmlcmp.py b/rmtoo/lib/xmlutils/xmlcmp.py index 9dbb09a5..b19746e6 100644 --- a/rmtoo/lib/xmlutils/xmlcmp.py +++ b/rmtoo/lib/xmlutils/xmlcmp.py @@ -16,9 +16,25 @@ 'no they are not the same' (maybe with a hint of the difference's location) is needed. - (c) 2011,2017 by flonatel GmbH & Co. KG - + (c) 2011,2017,2020 by flonatel GmbH & Co. KG For licensing details see COPYING + + SPDX-License-Identifier: GPL-3.0-or-later + + This file is part of rmtoo. + + rmtoo is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + rmtoo is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with rmtoo. If not, see . ''' from __future__ import unicode_literals diff --git a/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB001/result_should/gantt2.xml b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB001/result_should/_ALT_001_gantt2.xml similarity index 100% rename from rmtoo/tests/RMTTest-Blackbox/RMTTest-BB001/result_should/gantt2.xml rename to rmtoo/tests/RMTTest-Blackbox/RMTTest-BB001/result_should/_ALT_001_gantt2.xml diff --git a/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB001/result_should/_ALT_002_gantt2.xml b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB001/result_should/_ALT_002_gantt2.xml new file mode 100644 index 00000000..04d3ec15 --- /dev/null +++ b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB001/result_should/_ALT_002_gantt2.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + diff --git a/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB002/result_should/gantt2.xml b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB002/result_should/_ALT_001_gantt2.xml similarity index 100% rename from rmtoo/tests/RMTTest-Blackbox/RMTTest-BB002/result_should/gantt2.xml rename to rmtoo/tests/RMTTest-Blackbox/RMTTest-BB002/result_should/_ALT_001_gantt2.xml diff --git a/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB002/result_should/_ALT_002_gantt2.xml b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB002/result_should/_ALT_002_gantt2.xml new file mode 100644 index 00000000..113d53e3 --- /dev/null +++ b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB002/result_should/_ALT_002_gantt2.xml @@ -0,0 +1,167 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB003/result_should/gantt2.xml b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB003/result_should/_ALT_001_gantt2.xml similarity index 100% rename from rmtoo/tests/RMTTest-Blackbox/RMTTest-BB003/result_should/gantt2.xml rename to rmtoo/tests/RMTTest-Blackbox/RMTTest-BB003/result_should/_ALT_001_gantt2.xml diff --git a/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB003/result_should/_ALT_002_gantt2.xml b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB003/result_should/_ALT_002_gantt2.xml new file mode 100644 index 00000000..f33c68ab --- /dev/null +++ b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB003/result_should/_ALT_002_gantt2.xml @@ -0,0 +1,114 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB004/result_should/gantt2.xml b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB004/result_should/_ALT_001_gantt2.xml similarity index 100% rename from rmtoo/tests/RMTTest-Blackbox/RMTTest-BB004/result_should/gantt2.xml rename to rmtoo/tests/RMTTest-Blackbox/RMTTest-BB004/result_should/_ALT_001_gantt2.xml diff --git a/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB004/result_should/_ALT_002_gantt2.xml b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB004/result_should/_ALT_002_gantt2.xml new file mode 100644 index 00000000..e6d39e27 --- /dev/null +++ b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB004/result_should/_ALT_002_gantt2.xml @@ -0,0 +1,114 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB010/result_should/gantt2.xml b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB010/result_should/_ALT_001_gantt2.xml similarity index 100% rename from rmtoo/tests/RMTTest-Blackbox/RMTTest-BB010/result_should/gantt2.xml rename to rmtoo/tests/RMTTest-Blackbox/RMTTest-BB010/result_should/_ALT_001_gantt2.xml diff --git a/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB010/result_should/_ALT_002_gantt2.xml b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB010/result_should/_ALT_002_gantt2.xml new file mode 100644 index 00000000..643f6e71 --- /dev/null +++ b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB010/result_should/_ALT_002_gantt2.xml @@ -0,0 +1,116 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB014/result_should/gantt2.xml b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB014/result_should/_ALT_001_gantt2.xml similarity index 100% rename from rmtoo/tests/RMTTest-Blackbox/RMTTest-BB014/result_should/gantt2.xml rename to rmtoo/tests/RMTTest-Blackbox/RMTTest-BB014/result_should/_ALT_001_gantt2.xml diff --git a/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB014/result_should/_ALT_002_gantt2.xml b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB014/result_should/_ALT_002_gantt2.xml new file mode 100644 index 00000000..4254e2b9 --- /dev/null +++ b/rmtoo/tests/RMTTest-Blackbox/RMTTest-BB014/result_should/_ALT_002_gantt2.xml @@ -0,0 +1,114 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rmtoo/tests/lib/BBHelper.py b/rmtoo/tests/lib/BBHelper.py index cfc2c55e..055cefc9 100644 --- a/rmtoo/tests/lib/BBHelper.py +++ b/rmtoo/tests/lib/BBHelper.py @@ -4,16 +4,34 @@ Blackbox helper - (c) 2010,2017 by flonatel GmbH & Co. KG - + (c) 2010,2017,2020 by flonatel GmbH & Co. KG For licensing details see COPYING + + SPDX-License-Identifier: GPL-3.0-or-later + + This file is part of rmtoo. + + rmtoo is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + rmtoo is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with rmtoo. If not, see . ''' + from __future__ import print_function import io import os import shutil import difflib +import sys import zipfile import time @@ -39,18 +57,19 @@ def find(mdir): return r -def unified_diff(mdir, fname, sorted_diff=False, artifacts_dir=None): +def unified_diff(mdir, fname_is, fname_should, + sorted_diff=False, artifacts_dir=None): if artifacts_dir is None: artifacts_path = os.environ["rmtoo_test_dir"] else: artifacts_path = os.path.join(os.environ["rmtoo_test_dir"], artifacts_dir) - with io.open(os.path.join(artifacts_path, fname), "r", + with io.open(os.path.join(artifacts_path, fname_is), "r", encoding="utf-8") as fa: a = fa.readlines() - with io.open(os.path.join(mdir, "result_should", fname), "r", + with io.open(os.path.join(mdir, "result_should", fname_should), "r", encoding="utf-8") as fb: b = fb.readlines() @@ -69,22 +88,104 @@ def unified_diff(mdir, fname, sorted_diff=False, artifacts_dir=None): # This implements the compare_xml with the help of the xmldiff # package. -def compare_xml(mdir, fname): - if fname == "reqspricing.ods-extracted/content.xml": +def compare_xml(mdir, fname_is, fname_should): + if fname_is == "reqspricing.ods-extracted/content.xml": # Skip this (output from oomodule) return True - file1 = os.path.join(os.environ["rmtoo_test_dir"], fname) - file2 = os.path.join(mdir, "result_should", fname) + file1 = os.path.join(os.environ["rmtoo_test_dir"], fname_is) + file2 = os.path.join(mdir, "result_should", fname_should) r, s = xmlcmp_files(file1, file2) if not r: - print("XMLCmp difference: file [%s] diff [%s]" % (fname, s)) + print("XMLCmp difference: file [%s]-[%s] diff [%s]" % + (fname_is, fname_should, s)) return r +def search_matches(fis, files_should): + '''Search matched of the fis in the files_should''' + + normal_match = None + alt_match_set = set() + + for fshould in files_should: + if fis == fshould: + assert normal_match is None + normal_match = fis + continue + + if fshould.startswith("_ALT_") and fshould.endswith(fis): + alt_match_set.add(fshould) + + return normal_match, alt_match_set + + +def extract_alternative_ids(alt_match_set): + '''Extract the alternative ids''' + res_map = {} + for amatch in alt_match_set: + res_map[amatch[5:8]] = amatch + return res_map + + +def choose_alternative_id(alt_ids): + '''Choose the correct alternative from a give set of ids''' + if set(alt_ids.keys()) == set(['001', '002']): + if sys.version_info.major != 3: + print("Invalid alternative set") + assert False + if sys.version_info.minor <= 7: + return '001' + return '002' + + print("Not implemented alternative id set") + assert False + + +def handle_alternative_result_files(files_is, files_should): + '''Try to work around some result subtilities. + + From time to time even python internal things change. This is + e.g. the case in the XML writer which changed from python 3.7 to + 3.8. Both ways of the XML are (more or less) correct, but result + in a different object tree. + + This function tries to handle this by picking the correct result + file. + ''' + + res_missing_files = set() + res_files_mapping = {} + matched_files = set() + for fis in files_is: + normal_match, alt_match_set = search_matches(fis, files_should) + + if normal_match and alt_match_set: + print("Normal and alternative match: error in test case") + assert False + + if not normal_match and not alt_match_set: + res_missing_files.add(fis) + continue + + if normal_match: + res_files_mapping[fis] = fis + matched_files.add(fis) + continue + + alt_ids = extract_alternative_ids(alt_match_set) + alt_id = choose_alternative_id(alt_ids) + res_files_mapping[fis] = alt_ids[alt_id] + matched_files = matched_files.union(alt_match_set) + + res_additional_files = files_should - matched_files + + return res_additional_files, res_missing_files, res_files_mapping + + # This returns a trippel: # missing files in result_is # additional files in result_is @@ -101,26 +202,25 @@ def compare_results(mdir, relaxed=False, artifacts_dir=None): files_is = find(artifacts_path) files_should = find(os.path.join(mdir, "result_should")) - missing_files = files_is - files_should - additional_files = files_should - files_is - - files_to_compare = files_is.intersection(files_should) + additional_files, missing_files, files_mapping \ + = handle_alternative_result_files(files_is, files_should) r = {} - for df in files_to_compare: + for df_is, df_should in files_mapping.items(): # XML files must be handled differently: they might be # different when compared with diff but might have the same # semantic. - if df.endswith(".xml"): - if not compare_xml(mdir, df): - r[df] = "XML files differ" + if df_is.endswith(".xml"): + if not compare_xml(mdir, df_is, df_should): + r[df_is] = "XML files differ" else: sorted_diff = relaxed and \ - df in ['stderr', 'makefile_deps', 'req-graph1.dot', - 'reqsprios.tex'] - ud = unified_diff(mdir, df, sorted_diff, artifacts_dir) + df_is in ['stderr', 'makefile_deps', 'req-graph1.dot', + 'reqsprios.tex'] + ud = unified_diff(mdir, df_is, df_should, + sorted_diff, artifacts_dir) if ud is not None: - r[df] = ud + r[df_is] = ud return missing_files, additional_files, r diff --git a/tox.ini b/tox.ini index c8f66114..fb087186 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,24 @@ +# (c) 2020 by flonatel GmbH & Co. KG / Andreas Florath +# +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of rmtoo. +# +# rmtoo is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# rmtoo is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rmtoo. If not, see . + [tox] -envlist = py35,py36,py37,pep8 +envlist = py35,py36,py37,py38,pep8 minversion = 1.6 skipsdist = True