run-tests: Run tests locally with upstream CI images

This patch allows local execution of playbook tests using ustream CI
testing images. Either 'podman' or 'docker' can be used to execute the
tests.

.ansible-lint

@@ -10,6 +10,8 @@ exclude_paths:
   - molecule/
   - tests/azure/
   - meta/runtime.yml
+  - requirements-docker.yml
+  - requirements-podman.yml
 
 kinds:
   - playbook: '**/tests/**/test_*.yml'

requirements-docker.yml

+---
+collections:
+  - name: community.docker

requirements-podman.yml

+---
+collections:
+  - name: containers.podman

tests/README.md

@@ -138,6 +138,35 @@ molecule destroy -s c8s
 See [Running the tests](#running-the-tests) section for more information on available options.
 
 
+## Running local tests with upstream CI images
+
+To run tests locally using the same images used by upstream CI use `utils/run-tests.sh`.
+
+```
+utils/run-tests.sh tests/config/test_config.yml
+```
+
+To run all tests for a single plugin, use the `-s` option with plugin directory name. This will search, recursively for playbooks named with the pattern `test_*.yml`. To run all playbook tests for `ipauser`:
+
+```
+utils/run-tests.sh -s user
+```
+
+When executed, `utils/run-tests.sh` will create a container (either using `docker` or `podman`) using one of the testing ansible-freeipa images (https://quay.io/repository/ansible-freeipa/upstream-tests?tab=tags), run the selected tests against the container, and remove the container after tests are executed. If a test fails the container is not removed, so the failure can be investigated.
+
+It is possible to keep the container after the execution, even if tests succeed, using the `-C` option:
+
+```
+utils/run-tests.sh -s config -C
+```
+
+By default the tests are executed against the latest version of the Fedora image (`fedora-latest`). The testing image can be selected with the `-i` option. Use `-l` to list the available image names.
+
+```
+utils/run-tests.sh -i c9s tests/host/test_host.yml
+```
+
+
 ## Upcoming/desired improvements:
 
 * A script to pre-config the complete test environment using virsh.

tests/utils.py

@@ -32,10 +32,11 @@ from unittest import TestCase
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
-def is_docker_env():
-    if os.getenv("RUN_TESTS_IN_DOCKER", "0") == "0":
-        return False
-    return True
+def get_docker_env():
+    docker_env = os.getenv("RUN_TESTS_IN_DOCKER", None)
+    if docker_env in ["1", "True", "true", "yes", True]:
+        docker_env = "docker"
+    return docker_env
 
 
 def get_ssh_password():
@@ -88,8 +89,9 @@ def get_inventory_content():
     """Create the content of an inventory file for a test run."""
     ipa_server_host = get_server_host()
 
-    if is_docker_env():
-        ipa_server_host += " ansible_connection=docker"
+    container_engine = get_docker_env()
+    if container_engine is not None:
+        ipa_server_host += f" ansible_connection={container_engine}"
 
     sshpass = get_ssh_password()
     if sshpass:
@@ -145,12 +147,11 @@ def _run_playbook(playbook):
     with tempfile.NamedTemporaryFile() as inventory_file:
         inventory_file.write(get_inventory_content())
         inventory_file.flush()
-        cmd = [
-            "ansible-playbook",
-            "-i",
-            inventory_file.name,
-            playbook,
-        ]
+        cmd_options = ["-i", inventory_file.name]
+        verbose = os.environ.get("IPA_VERBOSITY", None)
+        if verbose is not None:
+            cmd_options.append(verbose)
+        cmd = ["ansible-playbook"] + cmd_options + [playbook]
         # pylint: disable=subprocess-run-check
         process = subprocess.run(
             cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE, stderr=subprocess.PIPE
@@ -257,8 +258,9 @@ def kdestroy(host):
 
 class AnsibleFreeIPATestCase(TestCase):
     def setUp(self):
-        if is_docker_env():
-            protocol = "docker://"
+        container_engine = get_docker_env()
+        if container_engine:
+            protocol = f"{container_engine}://"
             user = ""
             ssh_identity_file = None
         else:

utils/run-tests.sh

+#!/bin/bash -eu
+
+trap interrupt_exception SIGINT
+
+RST="\033[0m"
+RED="\033[31m"
+# BRIGHTRED="\033[31;1m"
+# GREEN="\033[32m"
+BRIGHTGREEN="\033[32;1m"
+# BROWN="\033[33m"
+YELLOW="\033[33;1m"
+# NAVY="\033[34m"
+BLUE="\033[34;1m"
+# MAGENTA="\033[35m"
+# BRIGHTMAGENTA="\033[35;1m"
+# DARKCYAN="\033[36m"
+# CYAN="\033[36;1m"
+# BLACK="\033[30m"
+# DARKGRAY="\033[30;1m"
+# GRAY="\033[37m"
+WHITE="\033[37;1m"
+
+TOPDIR="$(readlink -f "$(dirname "$0")/..")"
+
+interrupt_exception() {
+    trap - SIGINT
+    log warn "User interrupted test execution."
+    cleanup
+    exit 1
+}
+
+usage() {
+    local prog="${0##*/}"
+    cat <<EOF
+usage: ${prog} [-h] [-l] [-e] [-g] [-s TESTS_SUITE] [-i IMAGE] [TEST...]
+    ${prog} runs playbook(s) TEST using an ansible-freeipa testing image.
+
+EOF
+}
+
+help() {
+    usage
+    echo -e "$(cat <<EOF
+positional arguments:
+  TEST                A list of playbook tests to be executed.
+                      Either a TEST or a MODULE must be provided.
+
+optional arguments:
+  -h              display this message and exit
+  -c CONTAINER    use container CONTAINER to run tests
+  -K              keep container, even if tests succeed
+  -l              list available images
+  -e              force recreation of the virtual environment
+  -i              select image to run the tests (default: fedora-latest)
+  -m              container memory, in GiB (default: 3)
+  -s TEST_SUITE   run all playbooks for test suite, which is a directory
+                  under ${WHITE}tests${RST}
+EOF
+)"
+}
+
+log() {
+    local level="${1^^}" message="${*:2}"
+    case "${level}" in
+        ERROR) COLOR="${RED}" ;;
+        WARN)  COLOR="${YELLOW}" ;;
+        DEBUG) COLOR="${BLUE}" ;;
+        INFO) COLOR="${WHITE}" ;;
+        SUCCESS) COLOR="${BRIGHTGREEN}" ;;
+        *) COLOR="${RST}" ;;
+    esac
+    echo -en "${COLOR}"
+    [ "${level}" == "ERROR" ] && echo -en "${level}:"
+    echo -e "${message}${RST}"
+}
+
+quiet() {
+     "$@" >/dev/null 2>&1
+}
+
+in_python_virtualenv() {
+    local script
+    read -r -d "" script <<EOS
+import sys;
+base = getattr(sys, "base_prefix", ) or getattr(sys, "real_prefix", ) or sys.prefix
+print('yes' if sys.prefix != base else 'no')
+EOS
+    test "$(python -c "${script}")" == "yes"
+}
+
+run_inline_playbook() {
+    local playbook
+    local err
+    quiet mkdir -p "${test_env}/playbooks"
+    playbook=$(mktemp "${test_env}/playbooks/ansible-freeipa-test-playbook_ipa.XXXXXXXX")
+    cat - >"${playbook}"
+    ansible-playbook -i "${inventory}" "${playbook}"
+    err=$?
+    rm "${playbook}"
+    return ${err}
+}
+
+die() {
+    usg="N"
+    if [ "${1}" == "-u" ]
+    then
+       usg="Y"
+       shift 1
+    fi
+    log error "${*}"
+    STOP_CONTAINER="N"
+    cleanup
+    [ "${usg}" == "Y" ] && usage
+    exit 1
+}
+
+make_inventory() {
+    local scenario=$1 engine=${2:-podman}
+    inventory="${test_env}/inventory"
+    log info "Inventory file: ${inventory}"
+    cat << EOF > "${inventory}"
+[ipaserver]
+${scenario} ansible_connection=${engine}
+[ipaserver:vars]
+ipaserver_domain = test.local
+ipaserver_realm = TEST.LOCAL
+EOF
+}
+
+stop_container() {
+    local scenario=${1} engine=${2:-podman}
+    echo "Stopping container..."
+    quiet "${engine}" stop "${scenario}"
+    echo "Removing container..."
+    quiet "${engine}" rm "${scenario}"
+}
+
+cleanup() {
+    if [ $# -gt 0 ]
+    then
+        if [ "${STOP_CONTAINER}" != "N" ]
+        then
+            stop_container "${1}" "${2}"
+            rm "${inventory}"
+        else
+            log info "Keeping container: $(podman ps --format "{{.Names}} - {{.ID}}" --filter "name=${1}")"
+        fi
+    fi
+    if [ "${STOP_VIRTUALENV}" == "Y" ]
+    then
+        echo "Deactivating virtual environment"
+        deactivate
+    fi
+}
+
+list_images() {
+    local quay_api="https://quay.io/api/v1/repository/ansible-freeipa/upstream-tests/tag"
+    echo -e "${WHITE}Available images:"
+    curl --silent -L "${quay_api}" | jq '.tags[]|.name' | tr -d '"'| sort | uniq | sed "s/.*/    &/"
+    echo -e "${RST}"
+}
+
+# Defaults
+
+ANSIBLE_VERSION=${ANSIBLE_VERSION:-'ansible-core>=2.12,<2.13'}
+verbose=""
+FORCE_ENV="N"
+CONTINUE_ON_ERROR=""
+STOP_CONTAINER="Y"
+STOP_VIRTUALENV="N"
+declare -a ENABLED_MODULES
+declare -a ENABLED_TESTS
+ENABLED_MODULES=()
+ENABLED_TESTS=()
+test_env="${TESTENV_DIR:-${VIRTUAL_ENV:-/tmp/ansible-freeipa-tests}}"
+engine="podman"
+IMAGE_REPO="quay.io/ansible-freeipa/upstream-tests"
+IMAGE_TAG="fedora-latest"
+scenario=""
+MEMORY=3
+hostname="ipaserver.test.local"
+
+ANSIBLE_COLLECTIONS=${ANSIBLE_COLLECTIONS:-"containers.podman"}
+
+# Process command options
+
+while getopts ":hc:ei:Klms:v" option
+do
+    case "$option" in
+        h) help && exit 0 ;;
+        c) scenario="${OPTARG}" ;;
+        e) FORCE_ENV="Y" ;;
+        i) IMAGE_TAG="${OPTARG}" ;;
+        K) STOP_CONTAINER="N" ;;
+        l) list_images && exit 0 || exit 1;;
+        m) MEMORY="${OPTARG}" ;;
+        s)
+           if [ -d "${TOPDIR}/tests/${OPTARG}" ]
+           then
+               ENABLED_MODULES+=("${OPTARG}")
+           else
+               log error "Invalid suite: ${OPTARG}"
+           fi
+           ;;
+        v) verbose=${verbose:--}${option} ;;
+        *) die -u "Invalid option: ${OPTARG}" ;;
+    esac
+done
+
+for test in "${@:${OPTIND}}"
+do
+    # shellcheck disable=SC2207
+    if stat "$test" >/dev/null 2>&1
+    then
+        ENABLED_TESTS+=($(basename "${test}" .yml))
+    else
+        log error "Test not found: ${test}"
+    fi
+done
+
+[ ${#ENABLED_MODULES[@]} -eq 0 ] && [ ${#ENABLED_TESTS[@]} -eq 0 ] && die -u "No test defined."
+
+# Prepare virtual environment
+VENV=$(in_python_virtualenv && echo Y || echo N)
+
+if [ "${FORCE_ENV}" == "Y" ]
+then
+    [ "${VENV}" == "Y" ] && deactivate
+    VENV="N"
+    rm -rf "$test_env"
+    log info "Virtual environment will be (re)created."
+fi
+
+if [ "$VENV" == "N" ]
+then
+    log info "Preparing virtual environment: ${test_env}"
+    if [ ! -d "${test_env}" ]
+    then
+        log info "Creating virtual environment: ${test_env}..."
+        if ! python3 -m venv "${test_env}"
+        then
+            die "Cannot create virtual environment."
+        fi
+    fi
+    if [ -f "${test_env}/bin/activate" ]
+    then
+        log info "Starting virtual environment: ${test_env}"
+        # shellcheck disable=SC1091
+        . "${test_env}/bin/activate" || die "Cannot activate environment."
+        STOP_VIRTUALENV="Y"
+    else
+        die "Cannot activate environment."
+    fi
+    log info "Installing required tools."
+    log none "Upgrading: pip setuptools wheel"
+    pip install --quiet --upgrade pip setuptools wheel
+    log info "Installing dependencies from 'requirements-tests.txt'"
+    pip install --quiet -r "${TOPDIR}/requirements-tests.txt"
+    log info "Installing Ansible: ${ANSIBLE_VERSION}"
+    pip install --quiet "${ANSIBLE_VERSION}"
+    log debug "Ansible version: $(ansible --version | sed -n "1p")${RST}"
+    if [ -n "${ANSIBLE_COLLECTIONS}" ]
+    then
+        log warn "Installed collections will not be removed after execution."
+        log none "Installing: Ansible Collection ${ANSIBLE_COLLECTIONS}"
+        # shellcheck disable=SC2086
+        quiet ansible-galaxy collection install ${ANSIBLE_COLLECTIONS} || die "Failed to install Ansible collections."
+    fi
+else
+   log info "Using current virtual environment."
+fi
+
+# Ansible configuration
+export ANSIBLE_ROLES_PATH="${TOPDIR}/roles"
+export ANSIBLE_LIBRARY="${TOPDIR}/plugins:${TOPDIR}/molecule"
+export ANSIBLE_MODULE_UTILS="${TOPDIR}/plugins/module_utils"
+
+# Prepare container
+container_id=""
+container_status=("-f" "status=created" "-f" "status=running")
+[ -n "${scenario}" ] && container_id="$(${engine} ps --all -q -f "name=${scenario}" "${container_status[@]}")"
+if [ -z "${container_id}" ]
+then
+    # Retrieve image and start container.
+    log info "Pulling FreeIPA image '${IMAGE_REPO}:${IMAGE_TAG}'..."
+    img_id=$(${engine} pull -q "${IMAGE_REPO}:${IMAGE_TAG}")
+    log info "Creating container..."
+    CONFIG="--hostname ${hostname} --memory ${MEMORY}g --memory-swap -1 --dns none --add-host ipaserver.test.local:127.0.0.1"
+    [ -n "${scenario}" ] && CONFIG="${CONFIG} --name ${scenario}"
+    # shellcheck disable=SC2086
+    container_id=$(${engine} create ${CONFIG} "${img_id}" || die "Cannot create container")
+    echo "CONTAINER: ${container_id}"
+fi
+scenario="${scenario:-$(${engine} ps -q --format "{{.Names}}" --filter "id=${container_id}" "${container_status[@]}")}"
+log debug "Using container: ${scenario}"
+
+# Start container
+make_inventory "${scenario}"
+log info "Starting container for ${scenario}..."
+quiet ${engine} start "${scenario}"
+
+# create /etc/resolve.conf
+run_inline_playbook <<EOF || die "Failed to create /etc/resolv.conf"
+---
+- name: Create /etc/resolv.conf
+  hosts: ipaserver
+  gather_facts: no
+  become: yes
+  tasks:
+  - name: Create /etc/resolv.conf
+    ansible.builtin.copy:
+      dest: /etc/resolv.conf
+      mode: 0644
+      content: |
+        search test.local
+        nameserver 127.0.0.1
+...
+EOF
+
+# wait for FreeIPA services to be available
+run_inline_playbook <<EOF || die "Failed to verify IPA or KDC services."
+---
+- name: Wait for IPA services to be available
+  hosts: ipaserver
+  gather_facts: no
+  tasks:
+  - name: Wait for IPA to be started.
+    ansible.builtin.systemd:
+      name: ipa
+      state: started
+  - name: Wait for Kerberos KDC to be started.
+    ansible.builtin.systemd:
+      name: krb5kdc
+      state: started
+    register: result
+    until: not result.failed
+    retries: 30
+    delay: 5
+  - name: Check if TGT is available for admin.
+    ansible.builtin.shell:
+      cmd: echo SomeADMINpassword | kinit -c ansible_freeipa_cache admin
+    register: result
+    until: not result.failed
+    retries: 30
+    delay: 5
+  - name: Cleanup TGT.
+    ansible.builtin.shell:
+      cmd: kdestroy -c ansible_freeipa_cache -A
+...
+EOF
+
+# check image software versions.
+run_inline_playbook <<EOF || die "Failed to verify software installation."
+---
+- name: Software environment.
+  hosts: ipaserver
+  become: yes
+  gather_facts: no
+  tasks:
+  - name: Retrieve versions.
+    shell:
+      cmd: |
+        rpm -q freeipa-server freeipa-client ipa-server ipa-client 389-ds-base pki-ca krb5-server
+        cat /etc/redhat-release
+        uname -a
+    register: result
+  - name: Testing environment.
+    debug:
+      var: result.stdout_lines
+EOF
+
+
+# run tests
+RESULT=0
+# shellcheck disable=SC2086
+export RUN_TESTS_IN_DOCKER=${engine}
+export IPA_SERVER_HOST="${scenario}"
+joined="$(printf "%s," "${ENABLED_MODULES[@]}")"
+# shelcheck disable=SC2178
+IPA_ENABLED_MODULES="${joined%,}"
+joined="$(printf "%s," "${ENABLED_TESTS[@]}")"
+# shelcheck disable=SC2178
+IPA_ENABLED_TESTS="${joined%,}"
+export IPA_ENABLED_MODULES IPA_ENABLED_TESTS
+[ -n "${IPA_ENABLED_MODULES}" ] && log info "Test suites: ${IPA_ENABLED_MODULES}"
+[ -n "${IPA_ENABLED_TESTS}" ] && log info "Individual tests: ${IPA_ENABLED_TESTS}"
+
+IPA_VERBOSITY="${verbose}"
+[ -n "${IPA_VERBOSITY}" ] && export IPA_VERBOSITY
+
+if ! pytest -m "playbook" --verbose --color=yes
+then
+    RESULT=2
+    log error "Container not stopped for verification: ${scenario}"
+    log info "Container: $(podman ps -f "id=${container_id}" --format "{{.Names}} - {{.ID}}")"
+fi
+[ -z "${CONTINUE_ON_ERROR}" ] && [ $RESULT -ne 0 ] && die "Stopping on test failure."
+
+# cleanup environment
+cleanup "${scenario}" "${engine}"