From e0923a24472c3c473bd10e0bfc2cb23814b5fb09 Mon Sep 17 00:00:00 2001 From: anxdpanic Date: Wed, 6 Feb 2019 13:19:35 -0500 Subject: [PATCH 1/2] add --exclude-file and --exclude-file-ext - allows user to exclude files/folders/file extensions from permissions and file extension checks --- README.md | 2 ++ kodi_addon_checker/__main__.py | 4 +++ kodi_addon_checker/check_addon.py | 5 ++-- kodi_addon_checker/check_files.py | 44 ++++++++++++++++++++++++++----- tests/test_check_addon.py | 2 ++ tests/test_check_files.py | 28 +++++++++++++++++--- 6 files changed, 74 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index a1e5c79c..ab2d1112 100644 --- a/README.md +++ b/README.md @@ -93,4 +93,6 @@ You can use the tool with the following options: --branch name of the branch the tool is to run on --PR only when the tool is running on a pull request --allow-folder-id-mismatch allow the addon's folder name and id to mismatch +--exclude-file files/folders to exclude from permissions and file extension checks +--exclude-file-ext file extensions to exclude from permissions and file extension checks ``` diff --git a/kodi_addon_checker/__main__.py b/kodi_addon_checker/__main__.py index 3990a62f..a2752433 100644 --- a/kodi_addon_checker/__main__.py +++ b/kodi_addon_checker/__main__.py @@ -76,6 +76,10 @@ def main(): parser.add_argument("--PR", help="Tell if tool is to run on a pull requests or not", action='store_true') parser.add_argument("--allow-folder-id-mismatch", help="Allow the addon's folder name and id to mismatch", action="store_true") + parser.add_argument("--exclude-file", nargs="*", default=[], + help="Files/folders to exclude from permissions and file extension checks") + parser.add_argument("--exclude-file-ext", nargs="*", default=[], + help="File extensions to exclude from permissions and file extension checks") ConfigManager.fill_cmd_args(parser) args = parser.parse_args() diff --git a/kodi_addon_checker/check_addon.py b/kodi_addon_checker/check_addon.py index 74cf0d36..33ddcd50 100644 --- a/kodi_addon_checker/check_addon.py +++ b/kodi_addon_checker/check_addon.py @@ -58,7 +58,7 @@ def start(addon_path, branch_name, all_repo_addons, args, config=None): check_dependencies.check_reverse_dependencies(addon_report, addon_id, branch_name, all_repo_addons) - check_files.check_file_permission(addon_report, file_index) + check_files.check_file_permission(addon_report, file_index, args.exclude_file, args.exclude_file_ext) check_files.check_for_invalid_xml_files(addon_report, file_index) @@ -96,7 +96,8 @@ def start(addon_path, branch_name, all_repo_addons, args, config=None): # General blacklist check_string.find_blacklisted_strings(addon_report, addon_path, [], [], []) - check_files.check_file_whitelist(addon_report, file_index, addon_path) + check_files.check_file_whitelist(addon_report, file_index, addon_path, + args.exclude_file, args.exclude_file_ext) else: addon_report.add( Record(INFORMATION, "Addon marked as broken - skipping")) diff --git a/kodi_addon_checker/check_files.py b/kodi_addon_checker/check_files.py index 7cc48512..12718ccc 100644 --- a/kodi_addon_checker/check_files.py +++ b/kodi_addon_checker/check_files.py @@ -103,23 +103,37 @@ def check_for_legacy_language_path(report: Report, addon_path: str): break -def check_file_whitelist(report: Report, file_index: list, addon_path: str): +def check_file_whitelist(report: Report, file_index: list, addon_path: str, + whitelisted_files: list, whitelisted_exts: list): """check whether the files present in addon are in whitelist or not It ignores README.md and .gitignore file :file_index: list having names and path of all the files present in addon :addon_path: path to the addon folder + :whitelisted_files: list of files/folders to whitelist provided by argument --exclude-file + :whitelisted_exts: list of file extensions to whitelist provided by argument --exclude-file-ext """ if ".module." in addon_path: report.add(Record(INFORMATION, "Module skipping whitelist")) return + whitelisted_exts = "|".join(set([re.escape(ext.lstrip(".")) for ext in whitelisted_exts])) + if whitelisted_exts: + whitelisted_exts = "|" + whitelisted_exts + whitelist = ( r"\.?(py|xml|gif|png|jpg|jpeg|md|txt|po|json|gitignore|markdown|yml|" r"rst|ini|flv|wav|mp4|html|css|lst|pkla|g|template|in|cfg|xsd|directory|" - r"help|list|mpeg|pls|info|ttf|xsp|theme|yaml|dict|crt)?$" + r"help|list|mpeg|pls|info|ttf|xsp|theme|yaml|dict|crt{})?$".format(whitelisted_exts) ) + whitelisted_files = set(whitelisted_files) + for file in file_index: + full_path = os.path.join(file["path"], file["name"]) + if any(f == file["name"] or f == full_path or (os.path.isdir(f) and file["path"].startswith(f)) + for f in whitelisted_files): + continue + file_parts = file["name"].rsplit(".") if len(file_parts) > 1: file_ending = "." + file_parts[len(file_parts) - 1] @@ -129,13 +143,31 @@ def check_file_whitelist(report: Report, file_index: list, addon_path: str): relative_path(os.path.join(file["path"], file["name"])))) -def check_file_permission(report: Report, file_index: list): +def check_file_permission(report: Report, file_index: list, whitelisted_files: list, whitelisted_exts: list): """Check whether the files present in addon are marked executable or not :file_index: list having names and path of all the files present in addon + :whitelisted_files: list of files/folders to whitelist provided by argument --exclude-file + :whitelisted_exts: list of file extensions to whitelist provided by argument --exclude-file-ext """ + whitelisted_exts = "|".join(set([re.escape(ext.lstrip(".")) for ext in whitelisted_exts])) + whitelist = r"\.?({})?$".format(whitelisted_exts) + + whitelisted_files = set(whitelisted_files) + for file in file_index: - file = os.path.join(file["path"], file["name"]) - if os.path.isfile(file) and os.access(str(file), os.X_OK): - report.add(Record(PROBLEM, "%s is marked as stand-alone executable" % relative_path(str(file)))) + full_path = os.path.join(file["path"], file["name"]) + if any(f == file["name"] or f == full_path or (os.path.isdir(f) and file["path"].startswith(f)) + for f in whitelisted_files): + continue + + if whitelisted_exts: + file_parts = file["name"].rsplit(".") + if len(file_parts) > 1: + file_ending = "." + file_parts[len(file_parts) - 1] + if re.match(whitelist, file_ending, re.IGNORECASE): + continue + + if os.path.isfile(full_path) and os.access(str(full_path), os.X_OK): + report.add(Record(PROBLEM, "%s is marked as stand-alone executable" % relative_path(str(full_path)))) diff --git a/tests/test_check_addon.py b/tests/test_check_addon.py index 590db318..823ff356 100644 --- a/tests/test_check_addon.py +++ b/tests/test_check_addon.py @@ -10,6 +10,8 @@ class Args(object): PR = False allow_folder_id_mismatch = False + exclude_file = [] + exclude_file_ext = [] class TestCheckAddon(unittest.TestCase): diff --git a/tests/test_check_files.py b/tests/test_check_files.py index 042ebefb..b7bdfd81 100644 --- a/tests/test_check_files.py +++ b/tests/test_check_files.py @@ -25,9 +25,10 @@ def setUp(self): def test_check_file_permission_is_true(self): self.path = join(HERE, 'test_data', 'Executable file') - self.string = "ERROR: .{path}/file_permission.py is marked as stand-alone executable".format(path=self.path) + self.string = "ERROR: {path} is marked as stand-alone executable"\ + .format(path=relative_path(join(self.path, "file_permission.py"))) file_index = create_file_index(self.path) - check_file_permission(self.report, file_index) + check_file_permission(self.report, file_index, [], []) records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports] flag = any(s == self.string for s in records) self.assertTrue(flag) @@ -35,4 +36,25 @@ def test_check_file_permission_is_true(self): def test_check_file_permission_is_None(self): self.path = join(HERE, 'test_data', 'Non-Executable file') file_index = create_file_index(self.path) - self.assertIsNone(check_file_permission(self.report, file_index)) + self.assertIsNone(check_file_permission(self.report, file_index, [], [])) + + def test_check_file_permission_is_excluded_by_name(self): + self.path = join(HERE, 'test_data', 'Executable file') + self.string = "ERROR: {path} is marked as stand-alone executable"\ + .format(path=relative_path(join(self.path, "file_permission.py"))) + file_index = create_file_index(self.path) + check_file_permission(self.report, file_index, ["file_permission.py"], []) + records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports] + flag = any(s == self.string for s in records) + self.assertFalse(flag) + + def test_check_file_permission_is_excluded_by_ext(self): + self.path = join(HERE, 'test_data', 'Executable file') + self.string = "ERROR: {path} is marked as stand-alone executable"\ + .format(path=relative_path(join(self.path, "file_permission.py"))) + file_index = create_file_index(self.path) + check_file_permission(self.report, file_index, [], ["py"]) + records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports] + flag = any(s == self.string for s in records) + self.assertFalse(flag) + From 7f8b2799aa20d418c12907a608718744ad0e99c2 Mon Sep 17 00:00:00 2001 From: anxdpanic Date: Wed, 6 Feb 2019 13:44:28 -0500 Subject: [PATCH 2/2] add --whitelist-file and --whitelist-file-ext - allows user to exclude files/folders/file extensions from file extension checks --- README.md | 2 ++ kodi_addon_checker/__main__.py | 4 +++ kodi_addon_checker/check_addon.py | 5 ++-- kodi_addon_checker/check_files.py | 4 +-- tests/test_check_addon.py | 2 ++ tests/test_check_files.py | 30 +++++++++++++++++++ .../File whitelist/file_whitelist.ext | 1 + 7 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 tests/test_data/File whitelist/file_whitelist.ext diff --git a/README.md b/README.md index ab2d1112..0a4f240b 100644 --- a/README.md +++ b/README.md @@ -95,4 +95,6 @@ You can use the tool with the following options: --allow-folder-id-mismatch allow the addon's folder name and id to mismatch --exclude-file files/folders to exclude from permissions and file extension checks --exclude-file-ext file extensions to exclude from permissions and file extension checks +--whitelist-file files/folders to exclude from file extension checks +--whitelist-file-ext file extensions to exclude from file extension checks ``` diff --git a/kodi_addon_checker/__main__.py b/kodi_addon_checker/__main__.py index a2752433..c9443346 100644 --- a/kodi_addon_checker/__main__.py +++ b/kodi_addon_checker/__main__.py @@ -80,6 +80,10 @@ def main(): help="Files/folders to exclude from permissions and file extension checks") parser.add_argument("--exclude-file-ext", nargs="*", default=[], help="File extensions to exclude from permissions and file extension checks") + parser.add_argument("--whitelist-file", nargs="*", default=[], + help="Files/folders to exclude from file extension checks") + parser.add_argument("--whitelist-file-ext", nargs="*", default=[], + help="File extensions to exclude from file extension checks") ConfigManager.fill_cmd_args(parser) args = parser.parse_args() diff --git a/kodi_addon_checker/check_addon.py b/kodi_addon_checker/check_addon.py index 33ddcd50..2e126f01 100644 --- a/kodi_addon_checker/check_addon.py +++ b/kodi_addon_checker/check_addon.py @@ -96,8 +96,9 @@ def start(addon_path, branch_name, all_repo_addons, args, config=None): # General blacklist check_string.find_blacklisted_strings(addon_report, addon_path, [], [], []) - check_files.check_file_whitelist(addon_report, file_index, addon_path, - args.exclude_file, args.exclude_file_ext) + whitelist_files = args.exclude_file + args.whitelist_file + whitelist_exts = args.exclude_file_ext + args.whitelist_file_ext + check_files.check_file_whitelist(addon_report, file_index, addon_path, whitelist_files, whitelist_exts) else: addon_report.add( Record(INFORMATION, "Addon marked as broken - skipping")) diff --git a/kodi_addon_checker/check_files.py b/kodi_addon_checker/check_files.py index 12718ccc..9b899e54 100644 --- a/kodi_addon_checker/check_files.py +++ b/kodi_addon_checker/check_files.py @@ -109,8 +109,8 @@ def check_file_whitelist(report: Report, file_index: list, addon_path: str, It ignores README.md and .gitignore file :file_index: list having names and path of all the files present in addon :addon_path: path to the addon folder - :whitelisted_files: list of files/folders to whitelist provided by argument --exclude-file - :whitelisted_exts: list of file extensions to whitelist provided by argument --exclude-file-ext + :whitelisted_files: list of files/folders to whitelist provided by --exclude-file + --whitelist-file + :whitelisted_exts: list of file extensions to whitelist provided by --exclude-file-ext + --whitelist-file-ext """ if ".module." in addon_path: report.add(Record(INFORMATION, "Module skipping whitelist")) diff --git a/tests/test_check_addon.py b/tests/test_check_addon.py index 823ff356..8cf8a58f 100644 --- a/tests/test_check_addon.py +++ b/tests/test_check_addon.py @@ -12,6 +12,8 @@ class Args(object): allow_folder_id_mismatch = False exclude_file = [] exclude_file_ext = [] + whitelist_file = [] + whitelist_file_ext = [] class TestCheckAddon(unittest.TestCase): diff --git a/tests/test_check_files.py b/tests/test_check_files.py index b7bdfd81..1088f3ec 100644 --- a/tests/test_check_files.py +++ b/tests/test_check_files.py @@ -4,6 +4,7 @@ from os.path import abspath, dirname, join from kodi_addon_checker.check_files import check_file_permission +from kodi_addon_checker.check_files import check_file_whitelist from kodi_addon_checker.handle_files import create_file_index from kodi_addon_checker.common import load_plugins @@ -58,3 +59,32 @@ def test_check_file_permission_is_excluded_by_ext(self): flag = any(s == self.string for s in records) self.assertFalse(flag) + def test_check_file_whitelist_is_true(self): + self.path = join(HERE, 'test_data', 'File whitelist') + self.string = "WARN: Found non whitelisted file ending in filename {path}"\ + .format(path=relative_path(join(self.path, "file_whitelist.ext"))) + file_index = create_file_index(self.path) + check_file_whitelist(self.report, file_index, self.path, [], []) + records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports] + flag = any(s == self.string for s in records) + self.assertTrue(flag) + + def test_check_file_whitelist_is_excluded_by_name(self): + self.path = join(HERE, 'test_data', 'File whitelist') + self.string = "WARN: Found non whitelisted file ending in filename {path}"\ + .format(path=relative_path(join(self.path, "file_whitelist.ext"))) + file_index = create_file_index(self.path) + check_file_whitelist(self.report, file_index, self.path, ["file_whitelist.ext"], []) + records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports] + flag = any(s == self.string for s in records) + self.assertFalse(flag) + + def test_check_file_whitelist_is_excluded_by_ext(self): + self.path = join(HERE, 'test_data', 'File whitelist') + self.string = "WARN: Found non whitelisted file ending in filename {path}"\ + .format(path=relative_path(join(self.path, "file_whitelist.ext"))) + file_index = create_file_index(self.path) + check_file_whitelist(self.report, file_index, self.path, [], ["ext"]) + records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports] + flag = any(s == self.string for s in records) + self.assertFalse(flag) diff --git a/tests/test_data/File whitelist/file_whitelist.ext b/tests/test_data/File whitelist/file_whitelist.ext new file mode 100644 index 00000000..354d77d2 --- /dev/null +++ b/tests/test_data/File whitelist/file_whitelist.ext @@ -0,0 +1 @@ +Non whitelisted file ending