diff --git a/README.md b/README.md index a1e5c79c..0a4f240b 100644 --- a/README.md +++ b/README.md @@ -93,4 +93,8 @@ You can use the tool with the following options: --branch name of the branch the tool is to run on --PR only when the tool is running on a pull request --allow-folder-id-mismatch allow the addon's folder name and id to mismatch +--exclude-file files/folders to exclude from permissions and file extension checks +--exclude-file-ext file extensions to exclude from permissions and file extension checks +--whitelist-file files/folders to exclude from file extension checks +--whitelist-file-ext file extensions to exclude from file extension checks ``` diff --git a/kodi_addon_checker/__main__.py b/kodi_addon_checker/__main__.py index 3990a62f..c9443346 100644 --- a/kodi_addon_checker/__main__.py +++ b/kodi_addon_checker/__main__.py @@ -76,6 +76,14 @@ def main(): parser.add_argument("--PR", help="Tell if tool is to run on a pull requests or not", action='store_true') parser.add_argument("--allow-folder-id-mismatch", help="Allow the addon's folder name and id to mismatch", action="store_true") + parser.add_argument("--exclude-file", nargs="*", default=[], + help="Files/folders to exclude from permissions and file extension checks") + parser.add_argument("--exclude-file-ext", nargs="*", default=[], + help="File extensions to exclude from permissions and file extension checks") + parser.add_argument("--whitelist-file", nargs="*", default=[], + help="Files/folders to exclude from file extension checks") + parser.add_argument("--whitelist-file-ext", nargs="*", default=[], + help="File extensions to exclude from file extension checks") ConfigManager.fill_cmd_args(parser) args = parser.parse_args() diff --git a/kodi_addon_checker/check_addon.py b/kodi_addon_checker/check_addon.py index 74cf0d36..2e126f01 100644 --- a/kodi_addon_checker/check_addon.py +++ b/kodi_addon_checker/check_addon.py @@ -58,7 +58,7 @@ def start(addon_path, branch_name, all_repo_addons, args, config=None): check_dependencies.check_reverse_dependencies(addon_report, addon_id, branch_name, all_repo_addons) - check_files.check_file_permission(addon_report, file_index) + check_files.check_file_permission(addon_report, file_index, args.exclude_file, args.exclude_file_ext) check_files.check_for_invalid_xml_files(addon_report, file_index) @@ -96,7 +96,9 @@ def start(addon_path, branch_name, all_repo_addons, args, config=None): # General blacklist check_string.find_blacklisted_strings(addon_report, addon_path, [], [], []) - check_files.check_file_whitelist(addon_report, file_index, addon_path) + whitelist_files = args.exclude_file + args.whitelist_file + whitelist_exts = args.exclude_file_ext + args.whitelist_file_ext + check_files.check_file_whitelist(addon_report, file_index, addon_path, whitelist_files, whitelist_exts) else: addon_report.add( Record(INFORMATION, "Addon marked as broken - skipping")) diff --git a/kodi_addon_checker/check_files.py b/kodi_addon_checker/check_files.py index 7cc48512..9b899e54 100644 --- a/kodi_addon_checker/check_files.py +++ b/kodi_addon_checker/check_files.py @@ -103,23 +103,37 @@ def check_for_legacy_language_path(report: Report, addon_path: str): break -def check_file_whitelist(report: Report, file_index: list, addon_path: str): +def check_file_whitelist(report: Report, file_index: list, addon_path: str, + whitelisted_files: list, whitelisted_exts: list): """check whether the files present in addon are in whitelist or not It ignores README.md and .gitignore file :file_index: list having names and path of all the files present in addon :addon_path: path to the addon folder + :whitelisted_files: list of files/folders to whitelist provided by --exclude-file + --whitelist-file + :whitelisted_exts: list of file extensions to whitelist provided by --exclude-file-ext + --whitelist-file-ext """ if ".module." in addon_path: report.add(Record(INFORMATION, "Module skipping whitelist")) return + whitelisted_exts = "|".join(set([re.escape(ext.lstrip(".")) for ext in whitelisted_exts])) + if whitelisted_exts: + whitelisted_exts = "|" + whitelisted_exts + whitelist = ( r"\.?(py|xml|gif|png|jpg|jpeg|md|txt|po|json|gitignore|markdown|yml|" r"rst|ini|flv|wav|mp4|html|css|lst|pkla|g|template|in|cfg|xsd|directory|" - r"help|list|mpeg|pls|info|ttf|xsp|theme|yaml|dict|crt)?$" + r"help|list|mpeg|pls|info|ttf|xsp|theme|yaml|dict|crt{})?$".format(whitelisted_exts) ) + whitelisted_files = set(whitelisted_files) + for file in file_index: + full_path = os.path.join(file["path"], file["name"]) + if any(f == file["name"] or f == full_path or (os.path.isdir(f) and file["path"].startswith(f)) + for f in whitelisted_files): + continue + file_parts = file["name"].rsplit(".") if len(file_parts) > 1: file_ending = "." + file_parts[len(file_parts) - 1] @@ -129,13 +143,31 @@ def check_file_whitelist(report: Report, file_index: list, addon_path: str): relative_path(os.path.join(file["path"], file["name"])))) -def check_file_permission(report: Report, file_index: list): +def check_file_permission(report: Report, file_index: list, whitelisted_files: list, whitelisted_exts: list): """Check whether the files present in addon are marked executable or not :file_index: list having names and path of all the files present in addon + :whitelisted_files: list of files/folders to whitelist provided by argument --exclude-file + :whitelisted_exts: list of file extensions to whitelist provided by argument --exclude-file-ext """ + whitelisted_exts = "|".join(set([re.escape(ext.lstrip(".")) for ext in whitelisted_exts])) + whitelist = r"\.?({})?$".format(whitelisted_exts) + + whitelisted_files = set(whitelisted_files) + for file in file_index: - file = os.path.join(file["path"], file["name"]) - if os.path.isfile(file) and os.access(str(file), os.X_OK): - report.add(Record(PROBLEM, "%s is marked as stand-alone executable" % relative_path(str(file)))) + full_path = os.path.join(file["path"], file["name"]) + if any(f == file["name"] or f == full_path or (os.path.isdir(f) and file["path"].startswith(f)) + for f in whitelisted_files): + continue + + if whitelisted_exts: + file_parts = file["name"].rsplit(".") + if len(file_parts) > 1: + file_ending = "." + file_parts[len(file_parts) - 1] + if re.match(whitelist, file_ending, re.IGNORECASE): + continue + + if os.path.isfile(full_path) and os.access(str(full_path), os.X_OK): + report.add(Record(PROBLEM, "%s is marked as stand-alone executable" % relative_path(str(full_path)))) diff --git a/tests/test_check_addon.py b/tests/test_check_addon.py index 590db318..8cf8a58f 100644 --- a/tests/test_check_addon.py +++ b/tests/test_check_addon.py @@ -10,6 +10,10 @@ class Args(object): PR = False allow_folder_id_mismatch = False + exclude_file = [] + exclude_file_ext = [] + whitelist_file = [] + whitelist_file_ext = [] class TestCheckAddon(unittest.TestCase): diff --git a/tests/test_check_files.py b/tests/test_check_files.py index 042ebefb..1088f3ec 100644 --- a/tests/test_check_files.py +++ b/tests/test_check_files.py @@ -4,6 +4,7 @@ from os.path import abspath, dirname, join from kodi_addon_checker.check_files import check_file_permission +from kodi_addon_checker.check_files import check_file_whitelist from kodi_addon_checker.handle_files import create_file_index from kodi_addon_checker.common import load_plugins @@ -25,9 +26,10 @@ def setUp(self): def test_check_file_permission_is_true(self): self.path = join(HERE, 'test_data', 'Executable file') - self.string = "ERROR: .{path}/file_permission.py is marked as stand-alone executable".format(path=self.path) + self.string = "ERROR: {path} is marked as stand-alone executable"\ + .format(path=relative_path(join(self.path, "file_permission.py"))) file_index = create_file_index(self.path) - check_file_permission(self.report, file_index) + check_file_permission(self.report, file_index, [], []) records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports] flag = any(s == self.string for s in records) self.assertTrue(flag) @@ -35,4 +37,54 @@ def test_check_file_permission_is_true(self): def test_check_file_permission_is_None(self): self.path = join(HERE, 'test_data', 'Non-Executable file') file_index = create_file_index(self.path) - self.assertIsNone(check_file_permission(self.report, file_index)) + self.assertIsNone(check_file_permission(self.report, file_index, [], [])) + + def test_check_file_permission_is_excluded_by_name(self): + self.path = join(HERE, 'test_data', 'Executable file') + self.string = "ERROR: {path} is marked as stand-alone executable"\ + .format(path=relative_path(join(self.path, "file_permission.py"))) + file_index = create_file_index(self.path) + check_file_permission(self.report, file_index, ["file_permission.py"], []) + records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports] + flag = any(s == self.string for s in records) + self.assertFalse(flag) + + def test_check_file_permission_is_excluded_by_ext(self): + self.path = join(HERE, 'test_data', 'Executable file') + self.string = "ERROR: {path} is marked as stand-alone executable"\ + .format(path=relative_path(join(self.path, "file_permission.py"))) + file_index = create_file_index(self.path) + check_file_permission(self.report, file_index, [], ["py"]) + records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports] + flag = any(s == self.string for s in records) + self.assertFalse(flag) + + def test_check_file_whitelist_is_true(self): + self.path = join(HERE, 'test_data', 'File whitelist') + self.string = "WARN: Found non whitelisted file ending in filename {path}"\ + .format(path=relative_path(join(self.path, "file_whitelist.ext"))) + file_index = create_file_index(self.path) + check_file_whitelist(self.report, file_index, self.path, [], []) + records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports] + flag = any(s == self.string for s in records) + self.assertTrue(flag) + + def test_check_file_whitelist_is_excluded_by_name(self): + self.path = join(HERE, 'test_data', 'File whitelist') + self.string = "WARN: Found non whitelisted file ending in filename {path}"\ + .format(path=relative_path(join(self.path, "file_whitelist.ext"))) + file_index = create_file_index(self.path) + check_file_whitelist(self.report, file_index, self.path, ["file_whitelist.ext"], []) + records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports] + flag = any(s == self.string for s in records) + self.assertFalse(flag) + + def test_check_file_whitelist_is_excluded_by_ext(self): + self.path = join(HERE, 'test_data', 'File whitelist') + self.string = "WARN: Found non whitelisted file ending in filename {path}"\ + .format(path=relative_path(join(self.path, "file_whitelist.ext"))) + file_index = create_file_index(self.path) + check_file_whitelist(self.report, file_index, self.path, [], ["ext"]) + records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports] + flag = any(s == self.string for s in records) + self.assertFalse(flag) diff --git a/tests/test_data/File whitelist/file_whitelist.ext b/tests/test_data/File whitelist/file_whitelist.ext new file mode 100644 index 00000000..354d77d2 --- /dev/null +++ b/tests/test_data/File whitelist/file_whitelist.ext @@ -0,0 +1 @@ +Non whitelisted file ending