upstream ci: Avoid scheduling tests that will not be executed.

Currently, all tests are scheduled to execution, even those that are
not executed due to being absent from the list of enabled tests
configured in the IPA_ENABLED_* variables. The tests that are not
executed are marked 'skipped'.

This patch change this behavior by not scheduling tests that are not
configured to be executed. It means that tests not the IPA_DISABLED_*
lists are not skipped anymore, but not scheduled to be executed. If
any test is in IPA_ENABLED_* lists, only those tests are marked for
execution. A side effect is that there is no visual feedback on which
tests were not executed, as disabled tests are not evaluated anymore.

Also, when IPA_SERVER_HOST was not set, all tests were skipped, but
an error should raised in this case, as there are no hosts to run the
tests against.

This patch modifies this behavior to fail the test with an exception if
IPA_SERVER_HOST is not set.

tests/test_playbook_runs.py

@@ -24,7 +24,10 @@ import functools
 
 from unittest import TestCase
 
-from utils import get_test_playbooks, get_skip_conditions, run_playbook
+from utils import (
+    get_test_playbooks, get_server_host, run_playbook, get_enabled_test,
+    get_disabled_test
+)
 
 
 def prepare_test(testname, testpath):
@@ -47,6 +50,9 @@ def prepare_test(testname, testpath):
     return decorator
 
 
+if not get_server_host():
+    raise RuntimeError("IPA_SERVER_HOST is not set.")
+
 # Dynamically create the TestCase classes with respective
 #   test_* methods.
 for test_dir_name, playbooks_in_dir in get_test_playbooks().items():
@@ -56,16 +62,17 @@ for test_dir_name, playbooks_in_dir in get_test_playbooks().items():
         test_name = playbook["name"].replace("-", "_")
         test_path = playbook["path"]
 
-        skip = get_skip_conditions(test_dir_name, test_name) or {}
-
-        # pylint: disable=W0621,W0640,W0613
-        @pytest.mark.skipif(**skip)
-        @pytest.mark.playbook
-        @prepare_test(test_name, test_path)
-        def method(self, test_path, test_name):
-            run_playbook(test_path)
-        # pylint: enable=W0621,W0640,W0613
-
-        _tests[test_name] = method
+        if (
+            get_enabled_test(test_dir_name, test_name)
+            and not get_disabled_test(test_dir_name, test_name)
+        ):
+            # pylint: disable=W0621,W0640,W0613
+            @pytest.mark.playbook
+            @prepare_test(test_name, test_path)
+            def method(self, test_path, test_name):
+                run_playbook(test_path)
+            # pylint: enable=W0621,W0640,W0613
+
+            _tests[test_name] = method
 
     globals()[test_dir_name] = type(test_dir_name, tuple([TestCase]), _tests,)

tests/utils.py

@@ -84,30 +84,6 @@ def get_enabled_test(group_name, test_name):
     return group_enabled or test_enabled
 
 
-def get_skip_conditions(group_name, test_name):
-    """
-    Check tests that need to be skipped.
-
-    The return is a dict containing `condition` and `reason`. For the test
-    to be skipped, `condition` must be True, if it is `False`, the test is
-    to be skipped. Although "reason" must be always provided, it can be
-    `None` if `condition` is True.
-    """
-    if not get_server_host():
-        return {
-            "condition": True,
-            "reason": "Environment variable IPA_SERVER_HOST must be set",
-        }
-
-    if not get_enabled_test(group_name, test_name):
-        return {"condition": True, "reason": "Test not configured to run"}
-
-    if get_disabled_test(group_name, test_name):
-        return {"condition": True, "reason": "Test configured to not run"}
-
-    return {"condition": False, "reason": "Test will run."}
-
-
 def get_inventory_content():
     """Create the content of an inventory file for a test run."""
     ipa_server_host = get_server_host()