corrupt resolution result

This commit introduces a new type of resolution result for resolver. It adds the ability for resolvers to inform the user about files which might be a test, but there is something wrong inside them. If the resolver marks the reference as corrupted, the avocado will write a warning message to the user about it. Reference: avocado-framework#5710 Signed-off-by: Jan Richter <[email protected]>
richtja · Nov 16, 2023 · 1a69bf4 · 1a69bf4
1 parent 1e8315e
commit 1a69bf4
Show file tree

Hide file tree

Showing 7 changed files with 84 additions and 31 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -376,12 +376,12 @@ jobs:
          cd examples/plugins/tests/magic
          python3 setup.py develop --user
          cd ../../../../
-         python3 -m avocado -V list -- pass fail | grep "magic: 2"
+         python3 -m avocado -V list -- magic:pass magic:fail | grep "magic: 2"
          podman pull quay.io/avocado-framework/avocado-ci-magic
-         python3 -m avocado run --spawner=podman --spawner-podman-image=quay.io/avocado-framework/avocado-ci-magic -- pass
-         tail -n1 ~/avocado/job-results/latest/results.tap | grep "ok 1 pass"
-         python3 -m avocado run --spawner=podman --spawner-podman-image=quay.io/avocado-framework/avocado-ci-magic -- fail || true
-         tail -n1 ~/avocado/job-results/latest/results.tap | grep "not ok 1 fail"
+         python3 -m avocado run --spawner=podman --spawner-podman-image=quay.io/avocado-framework/avocado-ci-magic -- magic:pass
+         tail -n1 ~/avocado/job-results/latest/results.tap | grep "ok 1 magic:pass"
+         python3 -m avocado run --spawner=podman --spawner-podman-image=quay.io/avocado-framework/avocado-ci-magic -- magic:fail || true
+         tail -n1 ~/avocado/job-results/latest/results.tap | grep "not ok 1 magic:fail"
 
   fedora_develop_install_uninstall_task:
     name: Fedora develop install/uninstall task

diff --git a/avocado/core/resolver.py b/avocado/core/resolver.py
@@ -22,6 +22,7 @@
 
 from avocado.core.enabled_extension_manager import EnabledExtensionManager
 from avocado.core.exceptions import JobTestSuiteReferenceResolutionError
+from avocado.core.output import LOG_UI
 
 
 class ReferenceResolutionAssetType(Enum):
@@ -38,6 +39,8 @@ class ReferenceResolutionAssetType(Enum):
 class ReferenceResolutionResult(Enum):
     #: Given test reference was properly resolved
     SUCCESS = object()
+    #: Given test reference might be resolved, but it is corrupted.
+    CORRUPT = object()
     #: Given test reference was not properly resolved
     NOTFOUND = object()
     #: Internal error in the resolution process
@@ -108,6 +111,7 @@ class Resolver(EnabledExtensionManager):
 
     DEFAULT_POLICY = {
         ReferenceResolutionResult.SUCCESS: ReferenceResolutionAction.RETURN,
+        ReferenceResolutionResult.CORRUPT: ReferenceResolutionAction.RETURN,
         ReferenceResolutionResult.NOTFOUND: ReferenceResolutionAction.CONTINUE,
         ReferenceResolutionResult.ERROR: ReferenceResolutionAction.CONTINUE,
     }
@@ -267,6 +271,14 @@ def resolve(references, hint=None, ignore_missing=True, config=None):
         discoverer = Discoverer(config)
         resolutions.extend(discoverer.discover())
 
+    for res in resolutions:
+        if res.result == ReferenceResolutionResult.CORRUPT:
+            LOG_UI.warning(
+                "Reference %s might be resolved by %s resolver, but the file is corrupted: %s",
+                res.reference,
+                res.origin,
+                res.info or "",
+            )
     # This came up from a previous method and can be refactored to improve
     # performance since that we could merge with the loop above.
     if not ignore_missing:

diff --git a/avocado/plugins/list.py b/avocado/plugins/list.py
@@ -95,6 +95,7 @@ def _display_extra(suite, verbose=True):
 
             mapping = {
                 ReferenceResolutionResult.SUCCESS: TERM_SUPPORT.healthy_str,
+                ReferenceResolutionResult.CORRUPT: TERM_SUPPORT.warn_header_str,
                 ReferenceResolutionResult.NOTFOUND: TERM_SUPPORT.fail_header_str,
                 ReferenceResolutionResult.ERROR: TERM_SUPPORT.fail_header_str,
             }

diff --git a/docs/source/guides/contributor/chapters/plugins.rst b/docs/source/guides/contributor/chapters/plugins.rst
@@ -357,60 +357,70 @@ Resolving magic tests
 ---------------------
 
 Resolving the "pass" and "fail" references that the magic plugin knows about
-can be seen by running ``avocado list pass fail``::
+can be seen by running ``avocado list magic:pass magic:fail``::
 
-  magic pass
-  magic fail
+  magic magic:pass
+  magic magic:fail
 
 And you may get more insight into the resolution results, by adding a
 verbose parameter and another reference.  Try running ``avocado -V
-list pass fail something-else``::
+list magic:pass magic:fail magic:foo something-else``::
 
-  Type  Test Tag(s)
-  magic pass
-  magic fail
+  Reference magic:foo might be resolved by magic resolver, but the file is corrupted: Word "magic:foo" is magic type but the foo is not a valid magic word
+  Type  Test       Tag(s)
+  magic magic:pass
+  magic magic:fail
 
   Resolver             Reference      Info
-  avocado-instrumented pass           File "pass" does not end with ".py"
-  exec-test            pass           File "pass" does not exist or is not a executable file
-  golang               pass
-  avocado-instrumented fail           File "fail" does not end with ".py"
-  exec-test            fail           File "fail" does not exist or is not a executable file
-  golang               fail
+  avocado-instrumented magic:pass     File "magic" does not end with ".py"
+  golang               magic:pass     go binary not found
+  avocado-instrumented magic:fail     File "magic" does not end with ".py"
+  golang               magic:fail     go binary not found
+  avocado-instrumented magic:foo    File "magic" does not end with ".py"
+  golang               magic:foo    go binary not found
+  magic                magic:foo    Word "magic:foo" is magic type but the foo is not a valid magic word
   avocado-instrumented something-else File "something-else" does not end with ".py"
-  exec-test            something-else File "something-else" does not exist or is not a executable file
-  golang               something-else
+  golang               something-else go binary not found
   magic                something-else Word "something-else" is not a valid magic word
   python-unittest      something-else File "something-else" does not end with ".py"
   robot                something-else File "something-else" does not end with ".robot"
+  rogue                something-else Word "something-else" is not the magic word
+  exec-test            something-else File "something-else" does not exist or is not a executable file
   tap                  something-else File "something-else" does not exist or is not a executable file
 
   TEST TYPES SUMMARY
   ==================
   magic: 2
 
 It's worth realizing that magic (and other plugins) were asked to
-resolve the ``something-else`` reference, but couldn't::
+resolve the ``magic:foo`` and ``something-else`` references, but couldn't::
 
   Resolver             Reference      Info
   ...
+  magic                magic:foo    Word "magic:foo" is magic type but the foo is not a valid magic word
+  ...
   magic                something-else Word "something-else" is not a valid magic word
   ...
 
+We can see that the reference "magic:foo" resembles the magic words by type but it is not magic words ``pass`` or ``fail``.
+Consequently, the resolver can provide the user with information about potentially corrupted references.
+This can assist the user in identifying typos or reference mistakes. As the creator of the resolver,
+you can use the "ReferenceResolutionResult.CORRUPT" variable to notify the user of such a situation.
+
 Running magic tests
 -------------------
 
 The common way of running Avocado tests is to run them through
 ``avocado run``.  To run both the ``pass`` and ``fail`` magic tests,
-you'd run ``avocado run -- pass fail``::
+you'd run ``avocado run -- magic:pass magic:fail``::
 
-  $ avocado run -- pass fail
+  $ avocado run -- magic:pass magic:fail
   JOB ID     : 86fd45f8c1f2fe766c252eefbcac2704c2106db9
   JOB LOG    : $HOME/avocado/job-results/job-2021-02-05T12.43-86fd45f/job.log
-   (1/2) pass: STARTED
-   (1/2) pass: PASS (0.00 s)
-   (2/2) fail: STARTED
-   (2/2) fail: FAIL (0.00 s)
+   (1/2) magic:pass: STARTED
+   (1/2) magic:pass: PASS (0.00 s)
+   (2/2) magic:fail: STARTED
+   (2/2) magic:fail: FAIL (0.00 s)
   RESULTS    : PASS 1 | ERROR 0 | FAIL 1 | SKIP 0 | WARN 0 | INTERRUPT 0 | CANCEL 0
   JOB HTML   : $HOME/avocado/job-results/job-2021-02-05T12.43-86fd45f/results.html
   JOB TIME   : 1.83 s
diff --git a/examples/plugins/tests/magic/avocado_magic/resolver.py b/examples/plugins/tests/magic/avocado_magic/resolver.py
@@ -31,13 +31,26 @@ class MagicResolver(Resolver):
 
     @staticmethod
     def resolve(reference):  # pylint: disable=W0221
-        if reference not in VALID_MAGIC_WORDS:
+        try:
+            key_word, magic_word = reference.split(":", 1)
+        except (ValueError):
+            key_word = None
+            magic_word = reference
+        if key_word != "magic":
             return ReferenceResolution(
                 reference,
                 ReferenceResolutionResult.NOTFOUND,
                 info=f'Word "{reference}" is not a valid magic word',
             )
 
+        if magic_word not in VALID_MAGIC_WORDS:
+            return ReferenceResolution(
+                reference,
+                ReferenceResolutionResult.CORRUPT,
+                [Runnable("magic", reference)],
+                info=f'Word "{reference}" is magic type but the {magic_word} is not a valid magic word',
+            )
+
         return ReferenceResolution(
             reference, ReferenceResolutionResult.SUCCESS, [Runnable("magic", reference)]
         )

diff --git a/examples/plugins/tests/magic/avocado_magic/runner.py b/examples/plugins/tests/magic/avocado_magic/runner.py
@@ -26,8 +26,8 @@ class MagicRunner(BaseRunner):
 
     def run(self, runnable):
         yield StartedMessage.get()
-        if runnable.uri in ["pass", "fail"]:
-            result = runnable.uri
+        if runnable.uri in ["magic:pass", "magic:fail"]:
+            result = runnable.uri.split(":")[1]
         else:
             result = "error"
         yield FinishedMessage.get(result)

diff --git a/selftests/functional/resolver.py b/selftests/functional/resolver.py
@@ -8,7 +8,7 @@
 # is also the same
 from selftests.functional.list import AVOCADO_TEST_OK as AVOCADO_INSTRUMENTED_TEST
 from selftests.functional.list import EXEC_TEST
-from selftests.utils import AVOCADO, BASEDIR
+from selftests.utils import AVOCADO, BASEDIR, python_module_available
 
 
 class ResolverFunctional(unittest.TestCase):
@@ -106,6 +106,23 @@ def test_recursive_by_default(self):
         )
         self.assertEqual("avocado-instrumented: 10", lines[-1])
 
+    @unittest.skipUnless(
+        python_module_available("magic"), "avocado-magic not available"
+    )
+    def test_corrupted_reference(self):
+        cmd_line = f"{AVOCADO} list magic:foo"
+        result = process.run(cmd_line)
+        self.assertIn(
+            "Reference magic:foo might be resolved by magic resolver, but the file is corrupted:",
+            result.stderr_text,
+        )
+        cmd_line = f"{AVOCADO} run magic:foo"
+        result = process.run(cmd_line, ignore_status=True)
+        self.assertIn(
+            "Reference magic:foo might be resolved by magic resolver, but the file is corrupted:",
+            result.stderr_text,
+        )
+
 
 if __name__ == "__main__":
     unittest.main()