Skip to content
Snippets Groups Projects
Unverified Commit fdfea1b6 authored by Thomas Woerner's avatar Thomas Woerner Committed by GitHub
Browse files

Merge pull request #354 from rjeffman/tests_ignore_tests

 Add support to define which playbook tests to execute with pytest.
parents 4ff5aaa1 ac92ed14
No related branches found
No related tags found
No related merge requests found
......@@ -63,6 +63,24 @@ IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest -rs
For a complete list of options check `pytest --help`.
### Disabling and enabling playbook tests
Sometimes it is useful to enable or disable specific playbook tests. To only run a subset of modules or tests, use the variables IPA_ENABLED_MODULES and IPA ENABLED_TESTS, to define a comma-separated list of modules or tests to be enabled. Any test or module not in the list will not be executed. For example, to run only `sudorule` and `sudocmd` tests:
```
IPA_ENABLE_MODULES="sudorule,sudocmd" IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest
```
If all but a few selected tests are to be executed, use the IPA_DISABLED_MODULES or IPA_DISABLED_TESTS. For example, to run all, but "test_service_certificate" test:
```
IPA_DISABLED_TESTS=test_service_certificate IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest
```
If none of this variables are defined, all tests will be executed.
To configure the tests that will run for your pull request, add a TEMP commit, with the configuration defined in the file `tests/azure/templates/variables.yml`. Set the variables `ipa_enable_modules`, `ipa_enable_tests`, `ipa_disable_modules`, and `ipa_disable_tests`, in the same way as the equivalent environment variables.
### Types of tests
#### Playbook tests
......@@ -119,6 +137,7 @@ molecule destroy -s c8s
See [Running the tests](#running-the-tests) section for more information on available options.
## Upcoming/desired improvements:
* A script to pre-config the complete test environment using virsh.
......
......@@ -18,11 +18,12 @@ parameters:
- name: build_number
type: string
jobs:
- job: Test_Group${{ parameters.group_number }}
displayName: Run playbook tests ${{ parameters.scenario }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }})
timeoutInMinutes: 120
variables:
- template: variables.yaml
steps:
- task: UsePythonVersion@0
inputs:
......@@ -63,6 +64,10 @@ jobs:
env:
IPA_SERVER_HOST: ${{ parameters.scenario }}
RUN_TESTS_IN_DOCKER: true
IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }}
IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }}
IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }}
IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }}
- task: PublishTestResults@2
inputs:
......
......@@ -16,6 +16,8 @@ jobs:
- job: Test_PyTests
displayName: Run pytests on ${{ parameters.scenario }}
timeoutInMinutes: 120
variables:
- template: variables.yaml
steps:
- task: UsePythonVersion@0
inputs:
......@@ -53,6 +55,10 @@ jobs:
env:
IPA_SERVER_HOST: ${{ parameters.scenario }}
RUN_TESTS_IN_DOCKER: true
IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }}
IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }}
IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }}
IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }}
- task: PublishTestResults@2
inputs:
......
#
# Variables must be defined as comma separated lists.
# For easier management of items to enable/disable,
# use one test/module on each line, followed by a comma.
#
# Example:
#
# disabled_modules: >-
# dnsconfig,
# group,
# hostgroup
#
---
variables:
# ipa_enabled_modules: >-
# ipa_enabled_tests: >-
ipa_disabled_modules: >-
dnsconfig,
dnsforwardzone,
# ipa_disabled_tests: >-
......@@ -43,7 +43,6 @@ tests/sanity/sanity.sh shebang!skip
tests/user/users.sh shebang!skip
tests/user/users_absent.sh shebang!skip
tests/utils.py pylint:ansible-format-automatic-specification
tests/utils.py pylint:subprocess-run-check
utils/ansible-doc-test shebang!skip
utils/ansible-ipa-client-install shebang!skip
utils/ansible-ipa-replica-install shebang!skip
......
......@@ -24,11 +24,12 @@ import functools
from unittest import TestCase
from utils import get_test_playbooks, get_server_host, run_playbook
from utils import get_test_playbooks, get_skip_conditions, run_playbook
def prepare_test(test_name, test_path):
"""Decorator for the tests generated automatically from playbooks.
def prepare_test(testname, testpath):
"""
Decorate tests generated automatically from playbooks.
Injects 2 arguments to the test (`test_path` and `test_name`) and
name the test method using test name (to ensure test reports are useful).
......@@ -36,13 +37,13 @@ def prepare_test(test_name, test_path):
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
kwargs["test_path"] = test_path
kwargs["test_name"] = test_name
kwargs["test_path"] = testpath
kwargs["test_name"] = testname
return func(*args, **kwargs)
return wrapper
decorator.__name__ = test_name
decorator.__name__ = testname
return decorator
......@@ -50,18 +51,21 @@ def prepare_test(test_name, test_path):
# test_* methods.
for test_dir_name, playbooks_in_dir in get_test_playbooks().items():
_tests = {}
for playbook in playbooks_in_dir:
test_name = playbook["name"].replace("-", "_")
test_path = playbook["path"]
@pytest.mark.skipif(
not get_server_host(),
reason="Environment variable IPA_SERVER_HOST must be set",
)
skip = get_skip_conditions(test_dir_name, test_name) or {}
# pylint: disable=W0621,W0640,W0613
@pytest.mark.skipif(**skip)
@pytest.mark.playbook
@prepare_test(test_name, test_path)
def method(self, test_path, test_name):
run_playbook(test_path)
# pylint: enable=W0621,W0640,W0613
_tests[test_name] = method
globals()[test_dir_name] = type(test_dir_name, tuple([TestCase]), _tests,)
......@@ -45,6 +45,68 @@ def get_server_host():
return os.getenv("IPA_SERVER_HOST")
def get_disabled_test(group_name, test_name):
disabled_modules = [
disabled.strip()
for disabled in os.environ.get("IPA_DISABLED_MODULES", "").split(",")
]
disabled_tests = [
disabled.strip()
for disabled in os.environ.get("IPA_DISABLED_TESTS", "").split(",")
if disabled.strip()
]
if not any([disabled_modules, disabled_tests]):
return False
return group_name in disabled_modules or test_name in disabled_tests
def get_enabled_test(group_name, test_name):
enabled_modules = [
enabled.strip()
for enabled in os.environ.get("IPA_ENABLED_MODULES", "").split(":")
if enabled.strip()
]
enabled_tests = [
enabled.strip()
for enabled in os.environ.get("IPA_ENABLED_TESTS", "").split(":")
if enabled.strip()
]
if not any([enabled_modules, enabled_tests]):
return True
group_enabled = group_name in enabled_modules
test_enabled = test_name in enabled_tests
return group_enabled or test_enabled
def get_skip_conditions(group_name, test_name):
"""
Check tests that need to be skipped.
The return is a dict containing `condition` and `reason`. For the test
to be skipped, `condition` must be True, if it is `False`, the test is
to be skipped. Although "reason" must be always provided, it can be
`None` if `condition` is True.
"""
if not get_server_host():
return {
"condition": True,
"reason": "Environment variable IPA_SERVER_HOST must be set",
}
if not get_enabled_test(group_name, test_name):
return {"condition": True, "reason": "Test not configured to run"}
if get_disabled_test(group_name, test_name):
return {"condition": True, "reason": "Test configured to not run"}
return {"condition": False, "reason": "Test will run."}
def get_inventory_content():
"""Create the content of an inventory file for a test run."""
ipa_server_host = get_server_host()
......@@ -112,6 +174,7 @@ def _run_playbook(playbook):
inventory_file.name,
playbook,
]
# pylint: disable=subprocess-run-check
process = subprocess.run(
cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
......@@ -238,11 +301,13 @@ class AnsibleFreeIPATestCase(TestCase):
host_connection_info, ssh_identity_file=ssh_identity_file,
)
def run_playbook(self, playbook, allow_failures=False):
@staticmethod
def run_playbook(playbook, allow_failures=False):
return run_playbook(playbook, allow_failures)
def run_playbook_with_exp_msg(self, playbook, expected_msg):
result = self.run_playbook(playbook, allow_failures=True)
@staticmethod
def run_playbook_with_exp_msg(playbook, expected_msg):
result = run_playbook(playbook, allow_failures=True)
assert (
expected_msg in result.stdout.decode("utf8")
or
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment