Merge pull request #1300 from rjeffman/update_upstream_ci

Fix upstream CI and remove molecule

.ansible-lint

@@ -7,7 +7,6 @@ exclude_paths:
   - .tox/
   - .venv/
   - .yamllint
-  - molecule/
   - tests/azure/
   - meta/runtime.yml
   - requirements-docker.yml

tests/azure/azure-pipelines.yml → infra/azure/azure-pipelines.yml

@@ -15,8 +15,9 @@ stages:
   - template: templates/group_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
-      scenario: fedora-latest
+      distro: fedora-latest
       ansible_version: "-core"
+      skip_git_test: true
 
 # Fedora
 
@@ -26,8 +27,9 @@ stages:
   - template: templates/group_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
-      scenario: fedora-latest
-      ansible_version: "-core <2.14"
+      distro: fedora-latest
+      ansible_version: "-core <2.16"
+      skip_git_test: true
 
 # Fedora
 
@@ -37,19 +39,22 @@ stages:
   - template: templates/group_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
-      scenario: fedora-latest
-      ansible_version: "-core >=2.14,<2.15"
+      distro: fedora-latest
+      skip_git_test: true
+      ansible_version: "-core >=2.16,<2.17"
 
 # Galaxy on Fedora
 
 - stage: Galaxy_Fedora_Latest
   dependsOn: []
   jobs:
-  - template: templates/galaxy_tests.yml
+  - template: templates/group_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
-      scenario: fedora-latest
-      ansible_version: "-core >=2.14,<2.15"
+      distro: fedora-latest
+      ansible_version: "-core >=2.16,<2.17"
+      skip_git_test: true
+      test_galaxy: true
 
 # CentOS 9 Stream
 
@@ -59,8 +64,9 @@ stages:
   - template: templates/group_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
-      scenario: c9s
-      ansible_version: "-core >=2.14,<2.15"
+      distro: c9s
+      ansible_version: "-core >=2.16,<2.17"
+      skip_git_test: true
 
 # CentOS 8 Stream
 
@@ -70,16 +76,6 @@ stages:
   - template: templates/group_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
-      scenario: c8s
-      ansible_version: "-core >=2.14,<2.15"
-
-# CentOS 7
-
-- stage: CentOS_7
-  dependsOn: []
-  jobs:
-  - template: templates/group_tests.yml
-    parameters:
-      build_number: $(Build.BuildNumber)
-      scenario: centos-7
-      ansible_version: "-core >=2.14,<2.15"
+      distro: c8s
+      ansible_version: "-core <2.17"
+      skip_git_test: true

tests/azure/build-containers.yml → infra/azure/build-containers.yml

@@ -11,20 +11,10 @@ schedules:
 trigger: none
 
 pool:
-  vmImage: 'ubuntu-22.04'
+  vmImage: 'ubuntu-24.04'
 
 stages:
 
-# Currently, it's not possible to use CentOS container
-#
-# - stage: CentOS_7
-#   dependsOn: []
-#   jobs:
-#   - template: templates/build_container.yml
-#     parameters:
-#       job_name_suffix: Centos7
-#       distro: centos-7
-
 - stage: CentOS_8_Stream
   dependsOn: []
   jobs:
@@ -43,6 +33,14 @@ stages:
       job_name_suffix: C9S
       distro: c9s
 
+- stage: CentOS_10_Stream
+  dependsOn: []
+  jobs:
+  - template: templates/build_container.yml
+    parameters:
+      job_name_suffix: C10S
+      distro: c10s
+
 - stage: Fedora_Latest
   dependsOn: []
   jobs:

infra/azure/nightly.yml

+---
+schedules:
+- cron: "0 19 * * *"
+  displayName: Nightly Builds
+  branches:
+    include:
+    - master
+  always: true
+
+trigger: none
+
+pool:
+  vmImage: 'ubuntu-20.04'
+
+stages:
+
+# Fedora
+
+- stage: fedora_latest_Ansible_Core_2_15
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: fedora-latest
+      ansible_version: "-core >=2.15,<2.16"
+      skip_git_test: true
+
+- stage: fedora_latest_Ansible_Core_2_16
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: fedora-latest
+      ansible_version: ""
+      skip_git_test: true
+
+- stage: fedora_latest_Ansible_Core_2_17
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: fedora-latest
+      ansible_version: "-core >=2.17"
+      skip_git_test: true
+
+# Galaxy on Fedora
+
+- stage: Galaxy_fedora_latest_Ansible_Core_2_17
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: fedora-latest
+      ansible_version: "-core >=2.17,<2.18"
+      skip_git_test: true
+      test_galaxy: true
+
+# Fedora Rawhide
+
+- stage: fedora_rawhide_Ansible_Core_2_17
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: fedora-rawhide
+      ansible_version: "-core >=2.17,<2.18"
+      skip_git_test: true
+
+# CentoOS 9 Stream
+
+- stage: c9s_Ansible_Core_2_15
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: c9s
+      ansible_version: "-core >=2.15,<2.16"
+      skip_git_test: true
+
+- stage: c9s_Ansible_Core_2_16
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: c9s
+      ansible_version: "-core >=2.16,<2.17"
+      skip_git_test: true
+
+- stage: c9s_Ansible_Core_2_17
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: c9s
+      ansible_version: "-core >=2.17,<2.18"
+      skip_git_test: true
+
+# CentOS 8 Stream only works up to ansible-core 2.16.z
+
+- stage: c8s_Ansible_Core_2_15
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: c8s
+      ansible_version: "-core >=2.15,<2.16"
+      skip_git_test: true
+
+- stage: c8s_Ansible_Core_2_16
+  dependsOn: []
+  jobs:
+  - template: templates/group_tests.yml
+    parameters:
+      build_number: $(Build.BuildNumber)
+      distro: c8s
+      ansible_version: "-core >=2.16,<2.17"
+      skip_git_test: true

tests/azure/pr-pipeline.yml → infra/azure/pr-pipeline.yml

@@ -12,67 +12,58 @@ stages:
 - stage: Fedora_Latest
   dependsOn: []
   jobs:
-  - template: templates/fast_tests.yml
+  - template: templates/run_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
       distro: fedora-latest
       ansible_version: "-core >=2.15,<2.16"
+      skip_git_test: false
 
 # Galaxy on Fedora
 
 - stage: Galaxy_Fedora_Latest
   dependsOn: []
   jobs:
-  - template: templates/fast_tests.yml
+  - template: templates/run_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
       distro: fedora-latest
       ansible_version: "-core >=2.15,<2.16"
+      skip_git_test: false
+      test_galaxy: true
 
 # CentOS 9 Stream
 
 - stage: CentOS_9_Stream
   dependsOn: []
   jobs:
-  - template: templates/fast_tests.yml
+  - template: templates/run_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
       distro: c9s
       ansible_version: "-core >=2.15,<2.16"
+      skip_git_test: false
 
 # CentOS 8 Stream
 
 - stage: CentOS_8_Stream
   dependsOn: []
   jobs:
-  - template: templates/fast_tests.yml
+  - template: templates/run_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
       distro: c8s
       ansible_version: "-core >=2.15,<2.16"
-      target_python: "/usr/libexec/platform-python"
-
-# CentOS 7 cannot be used with current systemd
-#
-# CentOS 7
-#
-# - stage: CentOS_7
-#   dependsOn: []
-#   jobs:
-#   - template: templates/fast_tests.yml
-#     parameters:
-#       build_number: $(Build.BuildNumber)
-#       distro: centos-7
-#       ansible_version: "-core >=2.15,<2.16"
-#       target_python: "/usr/bin/python2"
+      skip_git_test: false
 
 # Rawhide
 
 - stage: Fedora_Rawhide
   dependsOn: []
   jobs:
-  - template: templates/fast_tests.yml
+  - template: templates/run_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
       distro: fedora-rawhide
       ansible_version: "-core >=2.15,<2.16"
+      skip_git_test: false

tests/azure/templates/build_container.yml → infra/azure/templates/build_container.yml

@@ -22,18 +22,15 @@ jobs:
     retryCountOnTaskFailure: 5
     displayName: Install tools
 
-  - script: |
-      rm -rf ~/.ansible
-      mkdir -p ~/.ansible
-      ln -snf $(readlink -f roles) ~/.ansible/roles
-      ln -snf $(readlink -f plugins) ~/.ansible/plugins
-    displayName: Setup ansible-freeipa using Git repository
-
   - script: ansible-galaxy collection install containers.podman
     displayName: Install Ansible Galaxy collections
 
   - script: infra/image/build.sh -s ${{ parameters.distro }}
     displayName: Build ${{ parameters.distro }} base image
+    env:
+      ANSIBLE_ROLES_PATH: "${PWD}/roles"
+      ANSIBLE_LIBRARY: "${PWD}/plugins/modules"
+      ANSIBLE_MODULE_UTILS: "${PWD}/plugins/module_utils"
 
   - script: podman login -u="$QUAY_ROBOT_USERNAME" -p="$QUAY_ROBOT_TOKEN" quay.io
     displayName: Registry login

infra/azure/templates/group_tests.yml

+---
+parameters:
+  - name: distro
+    type: string
+    default: fedora-latest
+  - name: build_number
+    type: string
+  - name: ansible_version
+    type: string
+    default: ""
+  - name: skip_git_test
+    type: boolean
+    default: false
+  - name: test_galaxy
+    type: boolean
+    default: false
+
+jobs:
+
+- ${{ each group in split('1,2,3', ',') }}:
+  - template: run_tests.yml
+    parameters:
+      group_number: ${{ group }}
+      number_of_groups: 3
+      build_number: ${{ parameters.build_number }}
+      distro: ${{ parameters.distro }}
+      ansible_version: ${{ parameters.ansible_version }}
+      python_version: '< 3.12'
+      skip_git_test: ${{ parameters.skip_git_test }}
+      test_galaxy: ${{ parameters.test_galaxy }}

infra/azure/templates/prepare_environment.yaml

+---
+parameters:
+  - name: distro
+    type: string
+    default: fedora-latest
+  - name: ansible_version
+    type: string
+    default: ""
+  - name: python_version
+    type: string
+    default: 3.x
+  - name: build_number
+    type: string
+
+steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '${{ parameters.python_version }}'
+
+  - script: |
+      pip install "ansible${{ parameters.ansible_version }}" -r requirements-tests.txt
+    retryCountOnTaskFailure: 5
+    displayName: Install test dependencies
+
+  - script: ansible-galaxy collection install -r requirements-podman.yml
+    retryCountOnTaskFailure: 5
+    displayName: Install Ansible collections
+
+  - script: infra/image/start.sh ${{ parameters.distro }}-server
+    displayName: Setup target container for ${{ parameters.distro }}

tests/azure/templates/playbook_fast.yml → infra/azure/templates/run_tests.yml

@@ -17,9 +17,15 @@ parameters:
     default: 3.x
   - name: build_number
     type: string
-  - name: target_python
+  - name: skip_git_test
+    type: boolean
+    default: true
+  - name: test_type
     type: string
-    default: "/usr/bin/python3"
+    default: "playbook"
+  - name: test_galaxy
+    type: boolean
+    default: false
 
 jobs:
 - job: Test_Group${{ parameters.group_number }}
@@ -29,53 +35,61 @@ jobs:
   - template: variables.yaml
   - template: variables_${{ parameters.distro }}.yaml
   steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '${{ parameters.python_version }}'
-
-  - script: |
-      pip install "ansible${{ parameters.ansible_version }}"
-    retryCountOnTaskFailure: 5
-    displayName: Install Ansible
-
-  - script: ansible-galaxy collection install containers.podman
-    retryCountOnTaskFailure: 5
-    displayName: Install Ansible collections
+  - template: prepare_environment.yaml
+    parameters:
+      build_number: ${{ parameters.build_number }}
+      distro: ${{ parameters.distro }}
+      ansible_version: ${{ parameters.ansible_version }}
+      python_version: ${{ parameters.python_version }}
 
-  - script: pip install -r requirements-tests.txt
-    retryCountOnTaskFailure: 5
-    displayName: Install dependencies
+  - bash: echo "##vso[task.setvariable variable=TOPDIR]${PWD}"
+    displayName: Set repo rootdir
 
   - script: |
-      . utils/set_test_modules
+      . "${TOPDIR}/utils/set_test_modules"
       python3 utils/check_test_configuration.py ${{ parameters.distro }}
     displayName: Check test configuration
+    env:
+       SKIP_GIT_TEST: ${{ parameters.skip_git_test }}
 
   - script: |
-      utils/setup_test_container.sh \
-        -e podman \
-        -a \
-        -m 4 \
-        -n "ipaserver.test.local" \
-        -p ${{ parameters.target_python }} \
-        -i ${{ parameters.distro }}-server \
-        ${{ parameters.distro }}-test
-    displayName: Setup target container
+      git fetch --unshallow
+      utils/build-galaxy-release.sh -i
+    retryCountOnTaskFailure: 5
+    displayName: Build Galaxy release
+    condition: ${{ parameters.test_galaxy }}
 
   - script: |
-      . utils/set_test_modules
-      pytest -m "playbook" --verbose --color=yes --suppress-no-test-exit-code --junit-xml=TEST-results-pr-check.xml
+      echo "PWD: ${PWD}"
+      echo "TOPDIR: ${TOPDIR}"
+      echo "ROLES: ${ANSIBLE_ROLES_PATH}"
+      echo "LIBRARY: ${ANSIBLE_LIBRARY}"
+      echo "MODULE_UTILS: ${ANSIBLE_MODULE_UTILS}"
+      . "${TOPDIR}/utils/set_test_modules"
+      [ "${{ parameters.test_galaxy }}" == "True"  ] && cd ~/.ansible/collections/ansible_collections/freeipa/ansible_freeipa
+      pytest \
+          -m "${{ parameters.test_type }}" \
+          --verbose \
+          --color=yes \
+          --splits=${{ parameters.number_of_groups }} \
+          --group=${{ parameters.group_number }} \
+          --randomly-seed=$(date "+%Y%m%d") \
+          --suppress-no-test-exit-code \
+          --junit-xml=TEST-results-pr-check.xml
     displayName: Run playbook tests
     env:
+      SKIP_GIT_TEST: ${{ parameters.skip_git_test }}
+      ${{ if not(parameters.test_galaxy)  }}:
         ANSIBLE_ROLES_PATH: "${PWD}/roles"
         ANSIBLE_LIBRARY: "${PWD}/plugins"
         ANSIBLE_MODULE_UTILS: "${PWD}/plugins/module_utils"
-      IPA_SERVER_HOST: ${{ parameters.distro }}-test
+      IPA_SERVER_HOST: ansible-freeipa-tests
       RUN_TESTS_IN_DOCKER: podman
       IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }}
       IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }}
+      IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }}
+      IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }}
       IPA_VERBOSITY: "-vvv"
-      IPA_PYTHON_PATH: ${{ parameters.target_python }}
 
   - task: PublishTestResults@2
     inputs:

infra/image/build.sh

@@ -15,7 +15,7 @@ valid_distro() {
 usage() {
     local prog="${0##*/}"
     cat << EOF
-usage: ${prog} [-h] [-s] distro
+usage: ${prog} [-h] [-p] [-c HOSTNAME] [-s] distro
     ${prog} build a container image to test ansible-freeipa.
 EOF
 }
@@ -29,23 +29,26 @@ positional arguments:
 
 optional arguments:
 
+    -c HOSTNAME   Container hostname
+    -p            Give extended privileges to the container
     -s            Deploy IPA server
-
 EOF
 }
 
 name="ansible-freeipa-image-builder"
 hostname="ipaserver.test.local"
-# Number of cpus is not available in usptream CI (Ubuntu 22.04).
-# cpus="2"
+cpus="2"
 memory="3g"
 quayname="quay.io/ansible-freeipa/upstream-tests"
 deploy_server="N"
+privileged=""
 
-while getopts ":hs" option
+while getopts ":hc:ps" option
 do
     case "${option}" in
         h) help && exit 0 ;;
+        c) hostname="${OPTARG}" ;;
+        p) privileged="privileged" ;;
         s) deploy_server="Y" ;;
         *) die -u "Invalid option: ${option}" ;;
     esac
@@ -82,13 +85,28 @@ container_remove_image_if_exists "${tag}"
     container_remove_image_if_exists "${server_tag}"
 
 container_build "${tag}" "${BASEDIR}/dockerfile/${distro}" "${BASEDIR}"
-container_create "${name}" "${tag}" "${hostname}" "${memory}"
+container_create "${name}" "${tag}" \
+    "hostname=${hostname}" \
+    "memory=${memory}" \
+    "cpus=${cpus}" \
+    "${privileged}"
 container_commit "${name}" "${quayname}:${tag}"
 
 if [ "${deploy_server}" == "Y" ]
 then
     deployed=false
 
+    # Set path to ansible-freeipa roles
+    [ -z "${ANSIBLE_ROLES_PATH:-""}" ] && export ANSIBLE_ROLES_PATH="${TOPDIR}/roles"
+
+    # Install collection containers.podman if not available
+    if [ -z "$(ansible-galaxy collection list containers.podman)" ]
+    then
+        tmpdir="$(mktemp -d)"
+        export ANSIBLE_COLLECTIONS_PATH="${tmpdir}"
+        ansible-galaxy collection install -p "${tmpdir}" containers.podman
+    fi
+
     [ "${container_state}" != "running" ] && container_start "${name}"
 
     container_wait_for_journald "${name}"

infra/image/shcontainer

@@ -9,24 +9,35 @@ TOPDIR="$(readlink -f "${SCRIPTDIR}/../..")"
 container_create() {
     local name=${1}
     local image=${2}
-    local hostname=${3}
-    local memory=${4:-"3g"}
-    local cpus=${5:-"2"}
+    shift 2
+    declare -a extra_opts=()
+    for opt in "$@"
+    do
+        [ -z "${opt}" ] && continue
+        case "${opt}" in
+            hostname=*) extra_opts+=("--${opt}") ;;
+            cpus=*) extra_opts+=("--${opt}") ;;
+            memory=*) extra_opts+=("--${opt}") ;;
+            privileged) extra_opts+=("--${opt}") ;;
+            *) log error "container_create: Invalid option: ${opt}" ;;
+        esac
+    done
 
-    [ -n "${hostname}" ] || die "No hostname given"
+    # ensure default values are set
+    [[ " ${extra_opts[*]} " =~ " --cpus=" ]] || extra_opts+=("--cpus=2")
+    [[ " ${extra_opts[*]} " =~ " --hostname=" ]] \
+        || extra_opts+=("--hostname=ipaserver.test.local")
 
     log info "= Creating ${name} ="
     podman create \
            --security-opt label=disable \
-           --name "${name}" \
-           --hostname "${hostname}" \
            --network bridge:interface_name=eth0 \
            --systemd true \
-           --cpus "${cpus}" \
-           --memory "${memory}" \
+           --name "${name}" \
            --memory-swap -1 \
            --no-hosts \
            --replace \
+           "${extra_opts[@]}" \
            "${image}"
     echo
 }

infra/image/start.sh

@@ -11,7 +11,7 @@ TOPDIR="$(readlink -f "${BASEDIR}/../..")"
 usage() {
     local prog="${0##*/}"
     cat << EOF
-usage: ${prog} [-h] [-l] image
+usage: ${prog} [-h] [-l] [-n HOSTNAME ] image
     ${prog} start a prebuilt ansible-freeipa test container image.
 EOF
 }
@@ -24,7 +24,14 @@ positional arguments:
 
 optional arguments:
 
+    -h            Show this message
     -l            Try to use local image first, if not found download.
+    -n HOSTNAME   Set container hostname
+
+NOTE:
+    - The hostname must be the same as the hostname of the container
+      when FreeIPA was deployed. Use only if you built the image and
+      defined its hostname.
 
 EOF
 }
@@ -45,11 +52,12 @@ name="ansible-freeipa-tests"
 hostname="ipaserver.test.local"
 try_local_first="N"
 
-while getopts ":hl" option
+while getopts ":hln:" option
 do
     case "${option}" in
         h) help && exit 0 ;;
         l) try_local_first="Y" ;;
+        n) hostname="${OPTARG}" ;;
         *) die -u "Invalid option: ${option}" ;;
     esac
 done
@@ -79,7 +87,7 @@ fi
 
 [ -z "${local_image}" ] && die "Image '${image}' is not valid"
 
-container_create "${name}" "${local_image}" "${hostname}"
+container_create "${name}" "${local_image}" "hostname=${hostname}"
 container_start "${name}"
 container_wait_for_journald "${name}"
 container_wait_up "${name}"

molecule/c8s-build/Dockerfile

-FROM quay.io/centos/centos:stream8
-ENV container=docker
-
-RUN rm -fv /var/cache/dnf/metadata_lock.pid; \
-dnf makecache; \
-dnf --assumeyes install \
-    /usr/bin/python3 \
-    /usr/bin/python3-config \
-    /usr/bin/dnf-3 \
-    sudo \
-    bash \
-    systemd \
-    procps-ng \
-    iproute && \
-dnf clean all; \
-(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \
-rm -f /lib/systemd/system/multi-user.target.wants/*;\
-rm -f /etc/systemd/system/*.wants/*;\
-rm -f /lib/systemd/system/local-fs.target.wants/*; \
-rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
-rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
-rm -f /lib/systemd/system/basic.target.wants/*;\
-rm -f /lib/systemd/system/anaconda.target.wants/*; \
-rm -rf /var/cache/dnf/;
-
-STOPSIGNAL RTMIN+3
-
-VOLUME ["/sys/fs/cgroup"]
-
-CMD ["/usr/sbin/init"]

molecule/c8s-build/molecule.yml

----
-driver:
-  name: docker
-platforms:
-  - name: c8s-build
-    image: "quay.io/centos/centos:stream8"
-    dockerfile: Dockerfile
-    hostname: ipaserver.test.local
-    dns_servers:
-      - 8.8.8.8
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
-    command: /usr/sbin/init
-    privileged: true
-provisioner:
-  name: ansible
-  playbooks:
-    prepare: ../resources/playbooks/prepare-build.yml
-prerun: false