diff --git a/tests/README.md b/tests/README.md index 631a37c83f8930bd76fae3a853e9f0c70fb62808..65ba97627f4cf75064fe0221e77aed9d290c3565 100644 --- a/tests/README.md +++ b/tests/README.md @@ -63,6 +63,24 @@ IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest -rs For a complete list of options check `pytest --help`. +### Disabling and enabling playbook tests + +Sometimes it is useful to enable or disable specific playbook tests. To only run a subset of modules or tests, use the variables IPA_ENABLED_MODULES and IPA ENABLED_TESTS, to define a comma-separated list of modules or tests to be enabled. Any test or module not in the list will not be executed. For example, to run only `sudorule` and `sudocmd` tests: + +``` +IPA_ENABLE_MODULES="sudorule,sudocmd" IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest +``` + +If all but a few selected tests are to be executed, use the IPA_DISABLED_MODULES or IPA_DISABLED_TESTS. For example, to run all, but "test_service_certificate" test: + +``` +IPA_DISABLED_TESTS=test_service_certificate IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest +``` + +If none of this variables are defined, all tests will be executed. + +To configure the tests that will run for your pull request, add a TEMP commit, with the configuration defined in the file `tests/azure/templates/variables.yml`. Set the variables `ipa_enable_modules`, `ipa_enable_tests`, `ipa_disable_modules`, and `ipa_disable_tests`, in the same way as the equivalent environment variables. + ### Types of tests #### Playbook tests @@ -119,6 +137,7 @@ molecule destroy -s c8s See [Running the tests](#running-the-tests) section for more information on available options. + ## Upcoming/desired improvements: * A script to pre-config the complete test environment using virsh. diff --git a/tests/azure/templates/playbook_tests.yml b/tests/azure/templates/playbook_tests.yml index 097750457976d2e52775feaf0b646dd361880deb..c838cbc3df5a3775419bb31265bea833da63b0c0 100644 --- a/tests/azure/templates/playbook_tests.yml +++ b/tests/azure/templates/playbook_tests.yml @@ -18,11 +18,12 @@ parameters: - name: build_number type: string - jobs: - job: Test_Group${{ parameters.group_number }} displayName: Run playbook tests ${{ parameters.scenario }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }}) timeoutInMinutes: 120 + variables: + - template: variables.yaml steps: - task: UsePythonVersion@0 inputs: @@ -63,6 +64,10 @@ jobs: env: IPA_SERVER_HOST: ${{ parameters.scenario }} RUN_TESTS_IN_DOCKER: true + IPA_DISABLED_MODULES: ${{ variables.disabled_modules }} + IPA_DISABLED_TESTS: ${{ variables.disabled_tests }} + IPA_ENABLED_MODULES: ${{ variables.enabled_modules }} + IPA_ENABLED_TESTS: ${{ variables.enabled_tests }} - task: PublishTestResults@2 inputs: diff --git a/tests/azure/templates/pytest_tests.yml b/tests/azure/templates/pytest_tests.yml index 92783c52e02bbccb6facac88a64213f2e9f8c68d..48a8cb0f3236e0596fc18fcf6f7b19d4caa46a6b 100644 --- a/tests/azure/templates/pytest_tests.yml +++ b/tests/azure/templates/pytest_tests.yml @@ -53,6 +53,10 @@ jobs: env: IPA_SERVER_HOST: ${{ parameters.scenario }} RUN_TESTS_IN_DOCKER: true + IPA_DISABLED_MODULES: ${{ variables.disabled_modules }} + IPA_DISABLED_TESTS: ${{ variables.disabled_tests }} + IPA_ENABLED_MODULES: ${{ variables.enabled_modules }} + IPA_ENABLED_TESTS: ${{ variables.enabled_tests }} - task: PublishTestResults@2 inputs: diff --git a/tests/azure/templates/variables.yaml b/tests/azure/templates/variables.yaml new file mode 100644 index 0000000000000000000000000000000000000000..109ec96de97a372b5412858b8a8ed6346b325541 --- /dev/null +++ b/tests/azure/templates/variables.yaml @@ -0,0 +1,18 @@ +# +# Variables must be defined as comma separated lists. +# For easier management of items to enable/disable, +# use one test/module on each line, followed by a comma. +# +# Example: +# +# disabled_modules: >- +# dnsconfig, +# group, +# hostgroup +# +--- +variables: + # ipa_enabled_modules: >- + # ipa_enabled_tests: >- + # ipa_disabled_modules: >- + # ipa_disabled_tests: >- diff --git a/tests/test_playbook_runs.py b/tests/test_playbook_runs.py index 4a9c24b773480bc170ddf53d355fff27022560b5..174a10f9d549965e4cb8959c31fe17a6d4603ce0 100644 --- a/tests/test_playbook_runs.py +++ b/tests/test_playbook_runs.py @@ -24,11 +24,12 @@ import functools from unittest import TestCase -from utils import get_test_playbooks, get_server_host, run_playbook +from utils import get_test_playbooks, get_skip_conditions, run_playbook -def prepare_test(test_name, test_path): - """Decorator for the tests generated automatically from playbooks. +def prepare_test(testname, testpath): + """ + Decorate tests generated automatically from playbooks. Injects 2 arguments to the test (`test_path` and `test_name`) and name the test method using test name (to ensure test reports are useful). @@ -36,13 +37,13 @@ def prepare_test(test_name, test_path): def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): - kwargs["test_path"] = test_path - kwargs["test_name"] = test_name + kwargs["test_path"] = testpath + kwargs["test_name"] = testname return func(*args, **kwargs) return wrapper - decorator.__name__ = test_name + decorator.__name__ = testname return decorator @@ -50,18 +51,21 @@ def prepare_test(test_name, test_path): # test_* methods. for test_dir_name, playbooks_in_dir in get_test_playbooks().items(): _tests = {} + for playbook in playbooks_in_dir: test_name = playbook["name"].replace("-", "_") test_path = playbook["path"] - @pytest.mark.skipif( - not get_server_host(), - reason="Environment variable IPA_SERVER_HOST must be set", - ) + skip = get_skip_conditions(test_dir_name, test_name) or {} + + # pylint: disable=W0621,W0640,W0613 + @pytest.mark.skipif(**skip) @pytest.mark.playbook @prepare_test(test_name, test_path) def method(self, test_path, test_name): run_playbook(test_path) + # pylint: enable=W0621,W0640,W0613 _tests[test_name] = method + globals()[test_dir_name] = type(test_dir_name, tuple([TestCase]), _tests,) diff --git a/tests/utils.py b/tests/utils.py index d681b0d68bb76553422e421e78622b3fae22fbc1..ae7cd48d208145d0f21fc5c45f989ea14fe076ec 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -45,6 +45,68 @@ def get_server_host(): return os.getenv("IPA_SERVER_HOST") +def get_disabled_test(group_name, test_name): + disabled_modules = [ + disabled.strip() + for disabled in os.environ.get("IPA_DISABLED_MODULES", "").split(",") + ] + disabled_tests = [ + disabled.strip() + for disabled in os.environ.get("IPA_DISABLED_TESTS", "").split(",") + if disabled.strip() + ] + + if not any([disabled_modules, disabled_tests]): + return False + + return group_name in disabled_modules or test_name in disabled_tests + + +def get_enabled_test(group_name, test_name): + enabled_modules = [ + enabled.strip() + for enabled in os.environ.get("IPA_ENABLED_MODULES", "").split(":") + if enabled.strip() + ] + enabled_tests = [ + enabled.strip() + for enabled in os.environ.get("IPA_ENABLED_TESTS", "").split(":") + if enabled.strip() + ] + + if not any([enabled_modules, enabled_tests]): + return True + + group_enabled = group_name in enabled_modules + test_enabled = test_name in enabled_tests + + return group_enabled or test_enabled + + +def get_skip_conditions(group_name, test_name): + """ + Check tests that need to be skipped. + + The return is a dict containing `condition` and `reason`. For the test + to be skipped, `condition` must be True, if it is `False`, the test is + to be skipped. Although "reason" must be always provided, it can be + `None` if `condition` is True. + """ + if not get_server_host(): + return { + "condition": True, + "reason": "Environment variable IPA_SERVER_HOST must be set", + } + + if not get_enabled_test(group_name, test_name): + return {"condition": True, "reason": "Test not configured to run"} + + if get_disabled_test(group_name, test_name): + return {"condition": True, "reason": "Test configured to not run"} + + return {"condition": False, "reason": "Test will run."} + + def get_inventory_content(): """Create the content of an inventory file for a test run.""" ipa_server_host = get_server_host()