diff --git a/tests/README.md b/tests/README.md index 631a37c83f8930bd76fae3a853e9f0c70fb62808..65ba97627f4cf75064fe0221e77aed9d290c3565 100644 --- a/tests/README.md +++ b/tests/README.md @@ -63,6 +63,24 @@ IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest -rs For a complete list of options check `pytest --help`. +### Disabling and enabling playbook tests + +Sometimes it is useful to enable or disable specific playbook tests. To only run a subset of modules or tests, use the variables IPA_ENABLED_MODULES and IPA ENABLED_TESTS, to define a comma-separated list of modules or tests to be enabled. Any test or module not in the list will not be executed. For example, to run only `sudorule` and `sudocmd` tests: + +``` +IPA_ENABLE_MODULES="sudorule,sudocmd" IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest +``` + +If all but a few selected tests are to be executed, use the IPA_DISABLED_MODULES or IPA_DISABLED_TESTS. For example, to run all, but "test_service_certificate" test: + +``` +IPA_DISABLED_TESTS=test_service_certificate IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest +``` + +If none of this variables are defined, all tests will be executed. + +To configure the tests that will run for your pull request, add a TEMP commit, with the configuration defined in the file `tests/azure/templates/variables.yml`. Set the variables `ipa_enable_modules`, `ipa_enable_tests`, `ipa_disable_modules`, and `ipa_disable_tests`, in the same way as the equivalent environment variables. + ### Types of tests #### Playbook tests @@ -119,6 +137,7 @@ molecule destroy -s c8s See [Running the tests](#running-the-tests) section for more information on available options. + ## Upcoming/desired improvements: * A script to pre-config the complete test environment using virsh. diff --git a/tests/azure/templates/playbook_tests.yml b/tests/azure/templates/playbook_tests.yml index 097750457976d2e52775feaf0b646dd361880deb..3fb15ad3a3d04667bed36864c846713f81bc11b2 100644 --- a/tests/azure/templates/playbook_tests.yml +++ b/tests/azure/templates/playbook_tests.yml @@ -18,11 +18,12 @@ parameters: - name: build_number type: string - jobs: - job: Test_Group${{ parameters.group_number }} displayName: Run playbook tests ${{ parameters.scenario }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }}) timeoutInMinutes: 120 + variables: + - template: variables.yaml steps: - task: UsePythonVersion@0 inputs: @@ -63,6 +64,10 @@ jobs: env: IPA_SERVER_HOST: ${{ parameters.scenario }} RUN_TESTS_IN_DOCKER: true + IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }} + IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }} + IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }} + IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }} - task: PublishTestResults@2 inputs: diff --git a/tests/azure/templates/pytest_tests.yml b/tests/azure/templates/pytest_tests.yml index 92783c52e02bbccb6facac88a64213f2e9f8c68d..f25ce2a7dcae80c90e72d7318d0c6725a1570d37 100644 --- a/tests/azure/templates/pytest_tests.yml +++ b/tests/azure/templates/pytest_tests.yml @@ -16,6 +16,8 @@ jobs: - job: Test_PyTests displayName: Run pytests on ${{ parameters.scenario }} timeoutInMinutes: 120 + variables: + - template: variables.yaml steps: - task: UsePythonVersion@0 inputs: @@ -53,6 +55,10 @@ jobs: env: IPA_SERVER_HOST: ${{ parameters.scenario }} RUN_TESTS_IN_DOCKER: true + IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }} + IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }} + IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }} + IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }} - task: PublishTestResults@2 inputs: diff --git a/tests/azure/templates/variables.yaml b/tests/azure/templates/variables.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f200416bf27feff11ac1a4972d19acf7de617364 --- /dev/null +++ b/tests/azure/templates/variables.yaml @@ -0,0 +1,20 @@ +# +# Variables must be defined as comma separated lists. +# For easier management of items to enable/disable, +# use one test/module on each line, followed by a comma. +# +# Example: +# +# disabled_modules: >- +# dnsconfig, +# group, +# hostgroup +# +--- +variables: + # ipa_enabled_modules: >- + # ipa_enabled_tests: >- + ipa_disabled_modules: >- + dnsconfig, + dnsforwardzone, + # ipa_disabled_tests: >- diff --git a/tests/sanity/ignore-2.12.txt b/tests/sanity/ignore-2.12.txt index b08a6ffa83adedcaf6155f07f8cc19105c2b32c9..ee85fb0adeef464dbdafb50c39904cd088d318e6 100644 --- a/tests/sanity/ignore-2.12.txt +++ b/tests/sanity/ignore-2.12.txt @@ -43,7 +43,6 @@ tests/sanity/sanity.sh shebang!skip tests/user/users.sh shebang!skip tests/user/users_absent.sh shebang!skip tests/utils.py pylint:ansible-format-automatic-specification -tests/utils.py pylint:subprocess-run-check utils/ansible-doc-test shebang!skip utils/ansible-ipa-client-install shebang!skip utils/ansible-ipa-replica-install shebang!skip diff --git a/tests/test_playbook_runs.py b/tests/test_playbook_runs.py index 4a9c24b773480bc170ddf53d355fff27022560b5..174a10f9d549965e4cb8959c31fe17a6d4603ce0 100644 --- a/tests/test_playbook_runs.py +++ b/tests/test_playbook_runs.py @@ -24,11 +24,12 @@ import functools from unittest import TestCase -from utils import get_test_playbooks, get_server_host, run_playbook +from utils import get_test_playbooks, get_skip_conditions, run_playbook -def prepare_test(test_name, test_path): - """Decorator for the tests generated automatically from playbooks. +def prepare_test(testname, testpath): + """ + Decorate tests generated automatically from playbooks. Injects 2 arguments to the test (`test_path` and `test_name`) and name the test method using test name (to ensure test reports are useful). @@ -36,13 +37,13 @@ def prepare_test(test_name, test_path): def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): - kwargs["test_path"] = test_path - kwargs["test_name"] = test_name + kwargs["test_path"] = testpath + kwargs["test_name"] = testname return func(*args, **kwargs) return wrapper - decorator.__name__ = test_name + decorator.__name__ = testname return decorator @@ -50,18 +51,21 @@ def prepare_test(test_name, test_path): # test_* methods. for test_dir_name, playbooks_in_dir in get_test_playbooks().items(): _tests = {} + for playbook in playbooks_in_dir: test_name = playbook["name"].replace("-", "_") test_path = playbook["path"] - @pytest.mark.skipif( - not get_server_host(), - reason="Environment variable IPA_SERVER_HOST must be set", - ) + skip = get_skip_conditions(test_dir_name, test_name) or {} + + # pylint: disable=W0621,W0640,W0613 + @pytest.mark.skipif(**skip) @pytest.mark.playbook @prepare_test(test_name, test_path) def method(self, test_path, test_name): run_playbook(test_path) + # pylint: enable=W0621,W0640,W0613 _tests[test_name] = method + globals()[test_dir_name] = type(test_dir_name, tuple([TestCase]), _tests,) diff --git a/tests/utils.py b/tests/utils.py index d681b0d68bb76553422e421e78622b3fae22fbc1..db22f9735fdea6939e00d45ff71912c77d5f934a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -45,6 +45,68 @@ def get_server_host(): return os.getenv("IPA_SERVER_HOST") +def get_disabled_test(group_name, test_name): + disabled_modules = [ + disabled.strip() + for disabled in os.environ.get("IPA_DISABLED_MODULES", "").split(",") + ] + disabled_tests = [ + disabled.strip() + for disabled in os.environ.get("IPA_DISABLED_TESTS", "").split(",") + if disabled.strip() + ] + + if not any([disabled_modules, disabled_tests]): + return False + + return group_name in disabled_modules or test_name in disabled_tests + + +def get_enabled_test(group_name, test_name): + enabled_modules = [ + enabled.strip() + for enabled in os.environ.get("IPA_ENABLED_MODULES", "").split(":") + if enabled.strip() + ] + enabled_tests = [ + enabled.strip() + for enabled in os.environ.get("IPA_ENABLED_TESTS", "").split(":") + if enabled.strip() + ] + + if not any([enabled_modules, enabled_tests]): + return True + + group_enabled = group_name in enabled_modules + test_enabled = test_name in enabled_tests + + return group_enabled or test_enabled + + +def get_skip_conditions(group_name, test_name): + """ + Check tests that need to be skipped. + + The return is a dict containing `condition` and `reason`. For the test + to be skipped, `condition` must be True, if it is `False`, the test is + to be skipped. Although "reason" must be always provided, it can be + `None` if `condition` is True. + """ + if not get_server_host(): + return { + "condition": True, + "reason": "Environment variable IPA_SERVER_HOST must be set", + } + + if not get_enabled_test(group_name, test_name): + return {"condition": True, "reason": "Test not configured to run"} + + if get_disabled_test(group_name, test_name): + return {"condition": True, "reason": "Test configured to not run"} + + return {"condition": False, "reason": "Test will run."} + + def get_inventory_content(): """Create the content of an inventory file for a test run.""" ipa_server_host = get_server_host() @@ -112,6 +174,7 @@ def _run_playbook(playbook): inventory_file.name, playbook, ] + # pylint: disable=subprocess-run-check process = subprocess.run( cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) @@ -238,11 +301,13 @@ class AnsibleFreeIPATestCase(TestCase): host_connection_info, ssh_identity_file=ssh_identity_file, ) - def run_playbook(self, playbook, allow_failures=False): + @staticmethod + def run_playbook(playbook, allow_failures=False): return run_playbook(playbook, allow_failures) - def run_playbook_with_exp_msg(self, playbook, expected_msg): - result = self.run_playbook(playbook, allow_failures=True) + @staticmethod + def run_playbook_with_exp_msg(playbook, expected_msg): + result = run_playbook(playbook, allow_failures=True) assert ( expected_msg in result.stdout.decode("utf8") or