Support checking if token value is in specified set

Also, read token configuration from YAML file using new option --config

docs/index.rst

@@ -283,6 +283,34 @@ For example:
     Test cases with invalid tags: 1 (10.00%)
     Total number of tests with parsing issues: 1 (10.00%)
 
+validate-values command
++++++++++++++++++++++++
+
+Checks that tokens of tests in given path have only allowed possible values.
+E.g. you can specify that ``Tier`` token can only be ``tier1``, ``tier2`` or
+``tier3``. Any other value or missing token will be reported at all.
+
+Allowed values are specified in a yaml file you configure with
+``--token-values`` option and which have structure like this:
+
+.. code-block:: console
+
+    ---
+    Tier:
+        - tier1
+        - tier2
+        - tier3
+
+To run the command, you need to specify tokens to check:
+
+.. code-block:: console
+
+   $ testimony --tokens="Tier" --minimum-tokens="Tier" --token-values=tests/validate-values.yaml validate-values tests/
+   [...]
+   * Token tier have unexpected value of tier0
+   [...]
+
+
 Misc Options
 ++++++++++++
 

setup.py

@@ -16,7 +16,7 @@
     author='Suresh Thirugn',
     author_email='[email protected]',
     packages=find_packages(),
-    install_requires=['Click', 'termcolor', 'docutils'],
+    install_requires=['Click', 'termcolor', 'docutils', 'pyyaml'],
     entry_points='''
         [console_scripts]
         testimony=testimony.cli:testimony

testimony/__init__.py

@@ -16,13 +16,15 @@
     CLR_ERR,
     CLR_GOOD,
     PRINT_INVALID_DOC,
+    PRINT_INVALID_VALUE,
     PRINT_NO_DOC,
     PRINT_NO_MINIMUM_DOC_TC,
     PRINT_REPORT,
     PRINT_RST_PARSING_ISSUE,
     PRINT_TOTAL_TC,
     SUMMARY_REPORT,
     VALIDATE_DOCSTRING_REPORT,
+    VALIDATE_VALUES_REPORT,
 )
 from testimony.parser import DocstringParser
 
@@ -252,6 +254,8 @@ def main(report, paths, json_output, nocolor):
         report_function = print_report
     elif report == VALIDATE_DOCSTRING_REPORT:
         report_function = validate_docstring_report
+    elif report == VALIDATE_VALUES_REPORT:
+        report_function = validate_values_report
     sys.exit(report_function(get_testcases(paths)))
 
 
@@ -444,6 +448,93 @@ def validate_docstring_report(testcases):
         return -1
 
 
+def validate_values_report(testcases):
+    """Check that given token contains one of allowed values."""
+    result = {}
+    invalid_docstring_count = 0
+    invalid_token_value_count = 0
+    missing_docstring_count = 0
+    missing_token_count = 0
+    testcase_count = 0
+    for path, tests in testcases.items():
+        testcase_count += len(tests)
+        for testcase in tests:
+            issues = []
+            missing_token_count_local = 0
+            if not testcase.docstring:
+                issues.append('Missing docstring.')
+                missing_docstring_count += 1
+            for token in SETTINGS['tokens']:
+                token = token.lower()
+                token_config = SETTINGS['token_configs'][token]
+                if token not in testcase.tokens:
+                    missing_token_count_local += 1
+                    missing_token_count += 1
+                if token in testcase.tokens \
+                   and not token_config.validate(testcase.tokens[token]):
+                    issues.append(
+                        'Token {} have unexpected value of {}'.format(
+                            token, testcase.tokens[token])
+                    )
+                    invalid_token_value_count += 1
+            if missing_token_count_local > 0:
+                issues.append(
+                    'Docstring is missing {} token(s)'.format(
+                        missing_token_count_local)
+                )
+            if issues:
+                title = testcase_title(testcase)
+                result.setdefault(
+                    path, collections.OrderedDict())[title] = issues
+                invalid_docstring_count += 1
+
+    if SETTINGS['json']:
+        print(json.dumps(result))
+        return
+
+    for path, testcases in result.items():
+        print('{0}\n{1}\n'.format(path, '=' * len(path)))
+        for testcase, issues in testcases.items():
+            print('{0}\n{1}\n'.format(testcase, '-' * len(testcase)))
+            print(
+                '\n'.join(['* {0}'.format(issue) for issue in issues]) + '\n')
+
+    if invalid_docstring_count == 0:
+        color = CLR_GOOD
+    else:
+        color = CLR_ERR
+    print('{}: {}'.format(
+        PRINT_TOTAL_TC,
+        testcase_count,
+    ))
+    print('{}: {} ({:05.02f}%)'.format(
+        PRINT_INVALID_DOC,
+        colored(invalid_docstring_count, color, attrs=['bold']),
+        float(invalid_docstring_count)/testcase_count * 100
+    ))
+    if invalid_token_value_count == 0:
+        color = CLR_GOOD
+    else:
+        color = CLR_ERR
+    print('{}: {} ({:05.02f}%)'.format(
+        PRINT_INVALID_VALUE,
+        colored(invalid_token_value_count, color, attrs=['bold']),
+        float(invalid_token_value_count)/testcase_count * 100
+    ))
+    if missing_docstring_count == 0:
+        color = CLR_GOOD
+    else:
+        color = CLR_ERR
+    print('{}: {} ({:.02f}%)'.format(
+        PRINT_NO_DOC.strip(),
+        colored(missing_docstring_count, color, attrs=['bold']),
+        float(missing_docstring_count)/testcase_count * 100
+    ))
+
+    if len(result) > 0:
+        return -1
+
+
 def get_testcases(paths):
     """Walk each path in ``paths`` and return the test cases found.
 

testimony/cli.py

@@ -2,7 +2,7 @@
 """Testimony CLI utilities."""
 import click
 
-from testimony import SETTINGS, constants, main
+from testimony import SETTINGS, config, constants, main
 
 
 @click.command()
@@ -12,15 +12,19 @@
 @click.option('--tokens', help='Comma separated list of expected tokens')
 @click.option(
     '--minimum-tokens', help='Comma separated list of minimum expected tokens')
+@click.option(
+    '--token-configs', help='Yaml file with allowed values per token')
 @click.argument('report', type=click.Choice(constants.REPORT_TAGS))
 @click.argument('path', nargs=-1, type=click.Path(exists=True))
 def testimony(
-        json, nocolor, tokens, minimum_tokens, report, path):
+        json, nocolor, tokens, minimum_tokens, token_configs, report, path):
     """Inspect and report on the Python test cases."""
     if tokens:
         SETTINGS['tokens'] = [
             token.strip().lower() for token in tokens.split(',')]
     if minimum_tokens:
         SETTINGS['minimum_tokens'] = [
             token.strip().lower() for token in minimum_tokens.split(',')]
+    if token_configs:
+        SETTINGS['token_configs'] = config.parse_config(token_configs)
     main(report, path, json, nocolor)

testimony/config.py

@@ -0,0 +1,63 @@
+# -*- coding: UTF-8 -*-
+
+"""
+Parse config for testimony.
+
+Example:
+
+    ---
+    Status:
+        type: choice
+        choices:
+            - Manual
+            - Automated
+    ...
+
+Currently only supported type is 'choice', but in the future more can be added.
+"""
+
+import sys
+import yaml
+
+TOKEN_TYPES = ['choice']
+
+
+def parse_config(filename):
+    """Parse the config."""
+    try:
+        with open(filename, 'r') as fp:
+            data = yaml.load(fp, Loader=yaml.SafeLoader)
+            return {k.lower(): TokenConfig(k, v) for k, v in data.items()}
+    except FileNotFoundError:
+        print('ERROR: File {} not found'.format(filename))
+        sys.exit(1)
+
+
+class TokenConfig(object):
+    """
+    Represent config for one token.
+
+    Curently only chacks for value.
+    """
+
+    def __init__(self, name, config):
+        """
+        Initialize token config object.
+
+        Takes name of the token and actual config structure as a param
+        """
+        self.name = name.lower()
+
+        assert 'type' in config
+        assert config['type'] in TOKEN_TYPES
+        self.token_type = config['type']
+
+        if self.token_type == 'choice':
+            assert 'choices' in config
+            assert isinstance(config['choices'], list)
+            self.choices = [i.lower() for i in config['choices']]
+
+    def validate(self, what):
+        """Ensure that 'what' meets value validation criteria."""
+        if self.token_type == 'choice':
+            return what.lower() in self.choices

testimony/constants.py

@@ -9,11 +9,13 @@
 PRINT_REPORT = 'print'
 SUMMARY_REPORT = 'summary'
 VALIDATE_DOCSTRING_REPORT = 'validate'
+VALIDATE_VALUES_REPORT = 'validate-values'
 
 REPORT_TAGS = (
     PRINT_REPORT,
     SUMMARY_REPORT,
     VALIDATE_DOCSTRING_REPORT,
+    VALIDATE_VALUES_REPORT,
 )
 
 DEFAULT_TOKENS = (
@@ -35,6 +37,8 @@
 )
 
 PRINT_INVALID_DOC = 'Total number of invalid docstrings'
+PRINT_INVALID_VALUE = \
+    'Total number of docstrings with unexpected token values'
 PRINT_NO_DOC = 'Test cases with no docstrings'
 PRINT_NO_MINIMUM_DOC_TC = 'Test cases missing minimal docstrings'
 PRINT_TOTAL_TC = 'Total number of tests'

tests/sample_output.txt

@@ -273,3 +273,62 @@ Test cases with no docstrings: 1 (10.00%)
 Test cases missing minimal docstrings: 3 (30.00%)
 Test cases with invalid tags: 1 (10.00%)
 Total number of tests with parsing issues: 1 (10.00%)
+
+==========================
+= validate_values report =
+==========================
+
+tests/test_sample.py
+====================
+
+test_outside_class:8
+--------------------
+
+* Token feature have unexpected value of Test functions
+* Docstring is missing 1 token(s)
+
+Testsample1::test_positive_login_1:27
+-------------------------------------
+
+* Docstring is missing 2 token(s)
+
+Testsample1::test_positive_login_2:49
+-------------------------------------
+
+* Missing docstring.
+* Docstring is missing 2 token(s)
+
+Testsample1::test_positive_login_3:54
+-------------------------------------
+
+* Docstring is missing 1 token(s)
+
+Testsample1::test_negative_login_5:87
+-------------------------------------
+
+* Docstring is missing 1 token(s)
+
+Testsample3::test_negative_login_7:133
+--------------------------------------
+
+* Docstring is missing 1 token(s)
+
+RSTFormattingTestCase::test_invalid_list_style:150
+--------------------------------------------------
+
+* Token feature have unexpected value of RST Parsing Issues
+* Docstring is missing 1 token(s)
+
+tests/sample_pkg/test_sample2.py
+================================
+
+TestPackage1::test_positive_login_3:15
+--------------------------------------
+
+* Token feature have unexpected value of Package Feature
+* Docstring is missing 1 token(s)
+
+Total number of tests: 10
+Total number of invalid docstrings: 8 (80.00%)
+Total number of docstrings with unexpected token values: 3 (30.00%)
+Test cases with no docstrings: 1 (10.00%)

tests/test_testimony.sh

@@ -17,3 +17,10 @@ echo "= validate_docstring report ="
 echo "============================="
 echo
 testimony -n validate tests
+
+echo
+echo "=========================="
+echo "= validate_values report ="
+echo "=========================="
+echo
+testimony -n --tokens="Status,Feature" --minimum-tokens="Status,Feature" --token-configs=tests/validate-values.yaml validate-values tests

tests/validate-values.yaml

@@ -0,0 +1,12 @@
+---
+Status:
+    type: choice
+    choices:
+        - Manual
+        - Automated
+Feature:
+    type: choice
+    choices:
+        - Login - Positive
+        - Login - Negative
+...