diff --git a/tests/azure/templates/galaxy_script.yml b/tests/azure/templates/galaxy_script.yml
index b669481891432e0cf9722d763f5a5ba53e017ac8..6d20bbd60e490265d81f56d6e0a9c5ef7504f7e8 100644
--- a/tests/azure/templates/galaxy_script.yml
+++ b/tests/azure/templates/galaxy_script.yml
@@ -25,6 +25,7 @@ jobs:
timeoutInMinutes: 120
variables:
- template: variables.yaml
+ - template: variables_${{ parameters.scenario }}.yaml
steps:
- task: UsePythonVersion@0
inputs:
@@ -49,6 +50,10 @@ jobs:
env:
ANSIBLE_LIBRARY: ./molecule
+ - script: |
+ python utils/check_test_configuration.py ${{ parameters.scenario }}
+ displayName: Check scenario test configuration
+
- script: |
cd ~/.ansible/collections/ansible_collections/freeipa/ansible_freeipa
pytest \
diff --git a/tests/azure/templates/playbook_tests.yml b/tests/azure/templates/playbook_tests.yml
index 3fb15ad3a3d04667bed36864c846713f81bc11b2..4001a647d9e40ad393a51e7a66e4558b2ec17c04 100644
--- a/tests/azure/templates/playbook_tests.yml
+++ b/tests/azure/templates/playbook_tests.yml
@@ -24,6 +24,7 @@ jobs:
timeoutInMinutes: 120
variables:
- template: variables.yaml
+ - template: variables_${{ parameters.scenario }}.yaml
steps:
- task: UsePythonVersion@0
inputs:
@@ -51,6 +52,10 @@ jobs:
env:
ANSIBLE_LIBRARY: ./molecule
+ - script: |
+ python utils/check_test_configuration.py ${{ parameters.scenario }}
+ displayName: Check scenario test configuration
+
- script: |
pytest \
-m "playbook" \
diff --git a/tests/azure/templates/pytest_tests.yml b/tests/azure/templates/pytest_tests.yml
index f25ce2a7dcae80c90e72d7318d0c6725a1570d37..6afd58905a526d7a4d911602d8c030423410eac9 100644
--- a/tests/azure/templates/pytest_tests.yml
+++ b/tests/azure/templates/pytest_tests.yml
@@ -18,6 +18,7 @@ jobs:
timeoutInMinutes: 120
variables:
- template: variables.yaml
+ - template: variables_${{ parameters.scenario }}.yaml
steps:
- task: UsePythonVersion@0
inputs:
diff --git a/tests/azure/templates/variables.yaml b/tests/azure/templates/variables.yaml
index b6e1686806c728654b5e2ff21a7d5e58f0e6f127..3cdadc03ed8ada14f3ff4601542c4acaa965d890 100644
--- a/tests/azure/templates/variables.yaml
+++ b/tests/azure/templates/variables.yaml
@@ -3,6 +3,9 @@
# For easier management of items to enable/disable,
# use one test/module on each line, followed by a comma.
#
+# If no variable is to be set, add 'empty: true', as the
+# 'variables' dict cannot be empty.
+#
# Example:
#
# disabled_modules: >-
@@ -12,9 +15,8 @@
#
---
variables:
+ empty: true
# ipa_enabled_modules: >-
# ipa_enabled_tests: >-
- ipa_disabled_modules: >-
- dnsconfig,
- ipa_disabled_tests: >-
- test_dnsconfig_forwarders_ports
+ # ipa_disabled_modules: >-
+ # ipa_disabled_tests: >-
diff --git a/tests/azure/templates/variables_c8s.yaml b/tests/azure/templates/variables_c8s.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..586d5ecb2404924285a9f19ed2082bcf35be0a38
--- /dev/null
+++ b/tests/azure/templates/variables_c8s.yaml
@@ -0,0 +1,19 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# disabled_modules: >-
+# dnsconfig,
+# group,
+# hostgroup
+#
+---
+variables:
+ empty: true
+# ipa_enabled_modules: >-
+# ipa_enabled_tests: >-
+# ipa_disabled_modules: >-
+# ipa_disabled_tests: >-
diff --git a/tests/azure/templates/variables_c9s.yaml b/tests/azure/templates/variables_c9s.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..586d5ecb2404924285a9f19ed2082bcf35be0a38
--- /dev/null
+++ b/tests/azure/templates/variables_c9s.yaml
@@ -0,0 +1,19 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# disabled_modules: >-
+# dnsconfig,
+# group,
+# hostgroup
+#
+---
+variables:
+ empty: true
+# ipa_enabled_modules: >-
+# ipa_enabled_tests: >-
+# ipa_disabled_modules: >-
+# ipa_disabled_tests: >-
diff --git a/tests/azure/templates/variables_centos-7.yaml b/tests/azure/templates/variables_centos-7.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..586d5ecb2404924285a9f19ed2082bcf35be0a38
--- /dev/null
+++ b/tests/azure/templates/variables_centos-7.yaml
@@ -0,0 +1,19 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# disabled_modules: >-
+# dnsconfig,
+# group,
+# hostgroup
+#
+---
+variables:
+ empty: true
+# ipa_enabled_modules: >-
+# ipa_enabled_tests: >-
+# ipa_disabled_modules: >-
+# ipa_disabled_tests: >-
diff --git a/tests/azure/templates/variables_fedora-latest.yaml b/tests/azure/templates/variables_fedora-latest.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..04fb6d291c24b356eca5c376e08dff27f934883a
--- /dev/null
+++ b/tests/azure/templates/variables_fedora-latest.yaml
@@ -0,0 +1,20 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# disabled_modules: >-
+# dnsconfig,
+# group,
+# hostgroup
+#
+---
+variables:
+ # ipa_enabled_modules: >-
+ # ipa_enabled_tests: >-
+ ipa_disabled_modules: >-
+ dnsforwardzone,
+ ipa_disabled_tests: >-
+ test_dnsconfig_forwarders_ports
diff --git a/tests/azure/templates/variables_fedora-rawhide.yaml b/tests/azure/templates/variables_fedora-rawhide.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d828bccb91772bf0e4083000a6e61d76e7971b65
--- /dev/null
+++ b/tests/azure/templates/variables_fedora-rawhide.yaml
@@ -0,0 +1,19 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# disabled_modules: >-
+# dnsconfig,
+# group,
+# hostgroup
+#
+---
+variables:
+ empty: true
+ # ipa_enabled_modules: >-
+ # ipa_enabled_tests: >-
+ # ipa_disabled_modules: >-
+ # ipa_disabled_tests: >-
diff --git a/tests/sanity/ignore-2.12.txt b/tests/sanity/ignore-2.12.txt
index ee85fb0adeef464dbdafb50c39904cd088d318e6..73afcd395b84653448846a7a67197d410a148b66 100644
--- a/tests/sanity/ignore-2.12.txt
+++ b/tests/sanity/ignore-2.12.txt
@@ -50,6 +50,7 @@ utils/ansible-ipa-server-install shebang!skip
utils/build-galaxy-release.sh shebang!skip
utils/build-srpm.sh shebang!skip
utils/changelog shebang!skip
+utils/check_test_configuration.py shebang!skip
utils/galaxyfy-README.py shebang!skip
utils/galaxyfy-module-EXAMPLES.py shebang!skip
utils/galaxyfy-playbook.py shebang!skip
diff --git a/tests/test_playbook_runs.py b/tests/test_playbook_runs.py
index 174a10f9d549965e4cb8959c31fe17a6d4603ce0..a67444d6ac38aae9ffc59110a18170004377b475 100644
--- a/tests/test_playbook_runs.py
+++ b/tests/test_playbook_runs.py
@@ -24,7 +24,10 @@ import functools
from unittest import TestCase
-from utils import get_test_playbooks, get_skip_conditions, run_playbook
+from utils import (
+ get_test_playbooks, get_server_host, run_playbook, get_enabled_test,
+ get_disabled_test
+)
def prepare_test(testname, testpath):
@@ -47,6 +50,9 @@ def prepare_test(testname, testpath):
return decorator
+if not get_server_host():
+ raise RuntimeError("IPA_SERVER_HOST is not set.")
+
# Dynamically create the TestCase classes with respective
# test_* methods.
for test_dir_name, playbooks_in_dir in get_test_playbooks().items():
@@ -56,16 +62,17 @@ for test_dir_name, playbooks_in_dir in get_test_playbooks().items():
test_name = playbook["name"].replace("-", "_")
test_path = playbook["path"]
- skip = get_skip_conditions(test_dir_name, test_name) or {}
-
- # pylint: disable=W0621,W0640,W0613
- @pytest.mark.skipif(**skip)
- @pytest.mark.playbook
- @prepare_test(test_name, test_path)
- def method(self, test_path, test_name):
- run_playbook(test_path)
- # pylint: enable=W0621,W0640,W0613
-
- _tests[test_name] = method
+ if (
+ get_enabled_test(test_dir_name, test_name)
+ and not get_disabled_test(test_dir_name, test_name)
+ ):
+ # pylint: disable=W0621,W0640,W0613
+ @pytest.mark.playbook
+ @prepare_test(test_name, test_path)
+ def method(self, test_path, test_name):
+ run_playbook(test_path)
+ # pylint: enable=W0621,W0640,W0613
+
+ _tests[test_name] = method
globals()[test_dir_name] = type(test_dir_name, tuple([TestCase]), _tests,)
diff --git a/tests/utils.py b/tests/utils.py
index c7e630a5fd6f2099e53c7a22344310a5226ad22f..36dac9cf0561de8407f5db2376c430a51b82284d 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -84,30 +84,6 @@ def get_enabled_test(group_name, test_name):
return group_enabled or test_enabled
-def get_skip_conditions(group_name, test_name):
- """
- Check tests that need to be skipped.
-
- The return is a dict containing `condition` and `reason`. For the test
- to be skipped, `condition` must be True, if it is `False`, the test is
- to be skipped. Although "reason" must be always provided, it can be
- `None` if `condition` is True.
- """
- if not get_server_host():
- return {
- "condition": True,
- "reason": "Environment variable IPA_SERVER_HOST must be set",
- }
-
- if not get_enabled_test(group_name, test_name):
- return {"condition": True, "reason": "Test not configured to run"}
-
- if get_disabled_test(group_name, test_name):
- return {"condition": True, "reason": "Test configured to not run"}
-
- return {"condition": False, "reason": "Test will run."}
-
-
def get_inventory_content():
"""Create the content of an inventory file for a test run."""
ipa_server_host = get_server_host()
diff --git a/utils/check_test_configuration.py b/utils/check_test_configuration.py
new file mode 100755
index 0000000000000000000000000000000000000000..1500e1dcb8893bc02b1747adbe575fceae612e56
--- /dev/null
+++ b/utils/check_test_configuration.py
@@ -0,0 +1,110 @@
+#!/usr/bin/python
+
+"""Check which tests are scheduled to be executed."""
+
+import sys
+import re
+import os
+import yaml
+
+
+RE_IS_TEST = re.compile(r"(.*/)?test_.*\.yml")
+RE_IS_VARS = re.compile(r"(.*/)?variables(_.*)?\.yaml")
+REPO_ROOT = os.path.join(os.path.dirname(__file__), "..")
+
+
+def get_tests():
+ """Retrieve a list of modules and its tests."""
+ def get_module(root):
+ if root != _test_dir:
+ while True:
+ module = os.path.basename(root)
+ root = os.path.dirname(root)
+ if root == _test_dir:
+ return module
+ return "."
+
+ _result = {}
+ _test_dir = os.path.join(REPO_ROOT, "tests")
+ for root, _dirs, files in os.walk(_test_dir):
+ module = get_module(root)
+ _result[module] = [
+ os.path.splitext(test)[0]
+ for test in files
+ if RE_IS_TEST.search(test)
+ ]
+
+ return _result
+
+
+def get_test_config(scenarios):
+ template_path = os.path.join(REPO_ROOT, "tests/azure/templates")
+ _result = {}
+ for _root, _dirs, files in os.walk(template_path):
+ for filename in [x for x in files if RE_IS_VARS.search(x)]:
+ _templates, *scenario = os.path.basename(
+ os.path.splitext(filename)[0]
+ ).split("_", 1)
+ scenario = scenario[0] if scenario else "All"
+ _result[scenario] = {}
+ # only process selected scenarios
+ if scenario not in scenarios and len(scenarios) > 1:
+ continue
+ with open(os.path.join(template_path, filename), "rt") as inp:
+ data = yaml.safe_load(inp)
+ if not data["variables"].get("empty", False):
+ variables = data["variables"]
+ for key, value in variables.items():
+ variables[key] = [
+ x.strip() for x in value.split(",") if x.strip()
+ ]
+ _result[scenario] = variables
+
+ return _result
+
+
+def print_configuration(scenario, disabled, enabled):
+ """Print the test configuration for a scenario."""
+ print(f"\nScenario: {scenario}")
+ for test_cfg, title in [(disabled, "Disabled"), (enabled, "Enabled")]:
+ print(f" {title} tests:")
+ if test_cfg:
+ for module, tests in test_cfg.items():
+ print(f" {module}:")
+ for test in tests:
+ print(f" - {test}")
+ else:
+ print(" No custom configuration.")
+
+
+def main():
+ if any(item in sys.argv for item in ["-h", "--help"]):
+ print("usage: check_test_config.py [-h|--help] [SCENARIO...]")
+ return
+
+ scenarios = ["All"] + sys.argv[1:]
+ all_tests = get_tests()
+ test_config = get_test_config(scenarios)
+
+ print("Test configuration:")
+ for scenario in sorted(test_config.keys()):
+ if scenario not in scenarios and len(scenarios) > 1:
+ continue
+ # extract scenario configuration
+ config = test_config[scenario]
+ disabled = {}
+ enabled = {}
+ for res, state in [(disabled, "disabled"), (enabled, "enabled")]:
+ for module in config.get(f"ipa_{state}_modules", []):
+ res[module] = set(all_tests[module])
+ for test in config.get(f"ipa_{state}_tests", []):
+ for module, tests in all_tests.items():
+ if test in tests:
+ mod = res.setdefault(module, set())
+ mod.add(test)
+ tests.remove(test)
+ print_configuration(scenario, disabled, enabled)
+
+
+if __name__ == "__main__":
+ main()