Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updated validation of value tests to better handle value classes based on regex #1015

Merged
merged 3 commits into from
Sep 22, 2024
Merged

Updated validation of value tests to better handle value classes based on regex #1015

Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c93d310
Added SCORE 2.0.0 to schema cache
VisLab Sep 13, 2024
a5ddb21
Fixed value tests
VisLab Sep 21, 2024
5f05c97
Added support for alternate codes in the spec_tests
VisLab Sep 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Added support for alternate codes in the spec_tests
  • Loading branch information
@VisLab
VisLab committed Sep 21, 2024
commit 5f05c97edced4c5a7d4d599a453d559609afa0d8
48 changes: 27 additions & 21 deletions spec_tests/test_errors.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@

skip_tests = {
"VERSION_DEPRECATED": "Not applicable",
"tag-extension-invalid-bad-node-name": "Part of character invalid checking/didn't get to it yet",
# "tag-extension-invalid-bad-node-name": "Part of character invalid checking/didn't get to it yet",
"curly-braces-has-no-hed": "Need to fix issue #1006",
"character-invalid-non-printing appears": "Need to recheck how this is verified for textClass",
# "character-invalid-non-printing appears": "Need to recheck how this is verified for textClass",
"invalid-character-name-value-class-deprecated": "Removing support for 8.2.0 or earlier name classes"
}

Expand All @@ -43,6 +43,7 @@ def run_single_test(self, test_file):
test_info = json.load(fp)
for info in test_info:
error_code = info['error_code']
all_codes = [error_code] + info.get('alt_codes', [])
if error_code in skip_tests:
print(f"Skipping {error_code} test because: {skip_tests[error_code]}")
continue
Expand All @@ -65,30 +66,32 @@ def run_single_test(self, test_file):
if not issues:
issues += [{"code": e.code,
"message": e.message}]
self.report_result("fails", issues, error_code, description, name, "dummy", "Schema")
self.report_result("fails", issues, error_code, all_codes, description, name, "dummy", "Schema")
# self.fail_count.append(name)
continue
definitions = info.get('definitions', None)
def_dict = DefinitionDict(definitions, schema)
self.assertFalse(def_dict.issues)

else:
def_dict = DefinitionDict()
for section_name, section in info["tests"].items():
if section_name == "string_tests":
self._run_single_string_test(section, schema, def_dict, error_code,
self._run_single_string_test(section, schema, def_dict, error_code, all_codes,
description, name, error_handler)
if section_name == "sidecar_tests":
self._run_single_sidecar_test(section, schema, def_dict, error_code,
self._run_single_sidecar_test(section, schema, def_dict, error_code, all_codes,
description, name, error_handler)
if section_name == "event_tests":
self._run_single_events_test(section, schema, def_dict, error_code,
self._run_single_events_test(section, schema, def_dict, error_code, all_codes,
description, name, error_handler)
if section_name == "combo_tests":
self._run_single_combo_test(section, schema, def_dict, error_code, description, name, error_handler)
self._run_single_combo_test(section, schema, def_dict, error_code, all_codes,
description, name, error_handler)
if section_name == "schema_tests":
self._run_single_schema_test(section, error_code, description, name, error_handler)
self._run_single_schema_test(section, error_code, all_codes, description, name, error_handler)

def report_result(self, expected_result, issues, error_code, description, name, test, test_type):
def report_result(self, expected_result, issues, error_code, all_codes, description, name, test, test_type):
# Filter out pre-release warnings, we don't care about them.
issues = [issue for issue in issues if issue["code"] != SchemaWarnings.SCHEMA_PRERELEASE_VERSION_USED]
if expected_result == "fails":
Expand All @@ -98,9 +101,9 @@ def report_result(self, expected_result, issues, error_code, description, name,
print(get_printable_issue_string(issues))
self.fail_count.append(name)
else:
if any(issue['code'] == error_code for issue in issues):
if any(issue['code'] in all_codes for issue in issues):
return
print(f"{error_code}: {description}")
print(f"{error_code}: {description} all codes:[{str(all_codes)}]")
print(f"Failed '{test_type}' (unexpected errors found) '{name}': {test}")
print(get_printable_issue_string(issues))
self.fail_count.append(name)
Expand All @@ -111,7 +114,7 @@ def report_result(self, expected_result, issues, error_code, description, name,
print(get_printable_issue_string(issues))
self.fail_count.append(name)

def _run_single_string_test(self, info, schema, def_dict, error_code, description, name, error_handler):
def _run_single_string_test(self, info, schema, def_dict, error_code, all_codes, description, name, error_handler):
string_validator = HedValidator(hed_schema=schema, def_dicts=def_dict)
for result, tests in info.items():
for test in tests:
Expand All @@ -120,18 +123,18 @@ def _run_single_string_test(self, info, schema, def_dict, error_code, descriptio
issues = string_validator.run_basic_checks(test_string, False)
issues += string_validator.run_full_string_checks(test_string)
error_handler.add_context_and_filter(issues)
self.report_result(result, issues, error_code, description, name, test, "string_test")
self.report_result(result, issues, error_code, all_codes, description, name, test, "string_test")

def _run_single_sidecar_test(self, info, schema, def_dict, error_code, description, name, error_handler):
def _run_single_sidecar_test(self, info, schema, def_dict, error_code, all_codes, description, name, error_handler):
for result, tests in info.items():
for test in tests:
# print(f"{error_code}: {name}")
buffer = io.BytesIO(json.dumps(test).encode("utf-8"))
sidecar = Sidecar(buffer)
issues = sidecar.validate(hed_schema=schema, extra_def_dicts=def_dict, error_handler=error_handler)
self.report_result(result, issues, error_code, description, name, test, "sidecar_test")
self.report_result(result, issues, error_code, all_codes, description, name, test, "sidecar_test")

def _run_single_events_test(self, info, schema, def_dict, error_code, description, name, error_handler):
def _run_single_events_test(self, info, schema, def_dict, error_code, all_codes, description, name, error_handler):
from hed import TabularInput
for result, tests in info.items():
for test in tests:
Expand All @@ -150,9 +153,9 @@ def _run_single_events_test(self, info, schema, def_dict, error_code, descriptio

file = TabularInput(file_obj, sidecar=self.default_sidecar)
issues = file.validate(hed_schema=schema, extra_def_dicts=def_dict, error_handler=error_handler)
self.report_result(result, issues, error_code, description, name, test, "events_test")
self.report_result(result, issues, error_code, all_codes, description, name, test, "events_test")

def _run_single_combo_test(self, info, schema, def_dict, error_code, description, name, error_handler):
def _run_single_combo_test(self, info, schema, def_dict, error_code, all_codes, description, name, error_handler):
from hed import TabularInput
for result, tests in info.items():
for test in tests:
Expand Down Expand Up @@ -182,9 +185,9 @@ def _run_single_combo_test(self, info, schema, def_dict, error_code, description
print(f"This is probably a missing set of square brackets.")
continue
issues += file.validate(hed_schema=schema, extra_def_dicts=def_dict, error_handler=error_handler)
self.report_result(result, issues, error_code, description, name, test, "combo_tests")
self.report_result(result, issues, error_code, all_codes, description, name, test, "combo_tests")

def _run_single_schema_test(self, info, error_code, description, name, error_handler):
def _run_single_schema_test(self, info, error_code, all_codes, description, name, error_handler):
for result, tests in info.items():
for test in tests:
schema_string = "\n".join(test)
Expand All @@ -200,15 +203,18 @@ def _run_single_schema_test(self, info, error_code, description, name, error_han
except urllib.error.HTTPError:
issues += [{"code": "Http_error",
"message": "HTTP error in testing, probably due to rate limiting for local testing."}]
self.report_result(result, issues, error_code, description, name, test, "schema_tests")
self.report_result(result, issues, error_code, all_codes, description, name, test, "schema_tests")

def test_errors(self):
for test_file in self.test_files:
self.run_single_test(test_file)
#test_file = './temp.json'
self.run_single_test(test_file)
print(f"{len(self.fail_count)} tests got an unexpected result")
print("\n".join(self.fail_count))
self.assertEqual(len(self.fail_count), 0)


if __name__ == '__main__':

unittest.main()
Loading