Merge pull request #1273 from t-woerner/new_image_build_with_podman

New image builder without molecule using podman

.github/workflows/lint.yml

@@ -88,3 +88,5 @@ jobs:
           fetch-depth: 1
       - name: Run ShellCheck
         uses: ludeeus/action-shellcheck@master
+        env:
+          SHELLCHECK_OPTS: -x

.pre-commit-config.yaml

@@ -54,4 +54,7 @@ repos:
     name: ShellCheck
     language: system
     entry: shellcheck
-    files: \.sh$
+    args: ['-x']
+    files: >
+      \.sh$
+      utils/sh*$

infra/image/build.sh

+#!/bin/bash -eu
+
+BASEDIR="$(readlink -f "$(dirname "$0")")"
+TOPDIR="$(readlink -f "${BASEDIR}/../..")"
+
+. "${TOPDIR}/utils/shfun"
+
+valid_distro() {
+    find "${BASEDIR}/dockerfile" -type f -printf "%f\n" | tr "\n" " "
+}
+
+usage() {
+    local prog="${0##*/}"
+    cat << EOF
+usage: ${prog} [-h] [i] distro
+    ${prog} build a container image to test ansible-freeipa.
+EOF
+}
+
+help() {
+    cat << EOF
+positional arguments:
+
+    distro    The base distro to build the test container.
+              Availble distros: $(valid_distro)
+
+optional arguments:
+
+    -s  Deploy IPA server
+
+EOF
+}
+
+name="ansible-freeipa-image-builder"
+hostname="ipaserver.test.local"
+# Number of cpus is not available in usptream CI (Ubuntu 22.04).
+# cpus="2"
+memory="4g"
+quayname="quay.io/ansible-freeipa/upstream-tests"
+deploy_server="N"
+
+while getopts ":hs" option
+do
+    case "${option}" in
+        h) help && exit 0 ;;
+        s) deploy_server="Y" ;;
+        *) die -u "Invalid option: ${option}" ;;
+    esac
+done
+
+shift $((OPTIND - 1))
+distro=${1:-}
+
+[ -n "${distro}" ] || die "Distro needs to be given.\nUse one of: $(valid_distro)"
+
+[ -f "${BASEDIR}/dockerfile/${distro}" ] \
+  || die "${distro} is not a valid distro target.\nUse one of: $(valid_distro)"
+
+[ -n "$(command -v "podman")" ] || die "podman is required."
+
+if [ "${deploy_server}" == "Y" ]
+then
+    [ -n "$(command -v "ansible-playbook")" ] || die "ansible-playbook is required to install FreeIPA."
+
+    deploy_playbook="${TOPDIR}/playbooks/install-server.yml"
+    [ -f "${deploy_playbook}" ] || die "Can't find playbook '${deploy_playbook}'"
+
+    inventory_file="${BASEDIR}/inventory"
+    [ -f "${inventory_file}" ] || die "Can't find inventory '${inventory_file}'"
+fi
+
+container_state="$(podman ps -q --all --format "{{.State}}" --filter "name=${name}")"
+
+tag="${distro}-base"
+server_tag="${distro}-server"
+
+# in older (as in Ubuntu 22.04) podman versions,
+# 'podman image rm --force' fails if the image
+# does not exist.
+remove_image_if_exists()
+{
+    local tag_to_remove
+    tag_to_remove="${1}"
+    if podman image exists "${tag_to_remove}"
+    then
+        log info "= Cleanup ${tag_to_remove} ="
+        podman image rm "${tag_to_remove}" --force
+        echo
+    fi
+}
+
+remove_image_if_exists "${tag}"
+[ "${deploy_server}" == "Y" ] && remove_image_if_exists "${server_tag}"
+
+
+log info "= Building ${tag} ="
+podman build -t "${tag}" -f "${BASEDIR}/dockerfile/${distro}" \
+       "${BASEDIR}"
+echo
+
+log info "= Creating ${name} ="
+podman create --privileged --name "${name}" --hostname "${hostname}" \
+    --network bridge:interface_name=eth0 --systemd true \
+    --memory "${memory}" --memory-swap -1 --no-hosts \
+    --replace "${tag}"
+echo
+
+log info "= Committing \"${quayname}:${tag}\" ="
+podman commit "${name}" "${quayname}:${tag}"
+echo
+
+if [ "${deploy_server}" == "Y" ]
+then
+    deployed=false
+
+    log info "= Starting ${name} ="
+    [ "${container_state}" == "running" ] || podman start "${name}"
+    echo
+
+    log info "= Deploying IPA ="
+    if ansible-playbook -i "${inventory_file}" "${deploy_playbook}"
+    then
+        deployed=true
+    fi
+    echo
+
+    if $deployed; then
+        log info "= Enabling additional services ="
+        podman exec "${name}" systemctl enable fixnet
+        podman exec "${name}" systemctl enable fixipaip
+        echo
+    fi
+    
+    log info "= Stopping container ${name} ="
+    podman stop "${name}"
+    echo
+
+    $deployed || die "Deployment failed"
+
+    log info "= Committing \"${quayname}:${server_tag}\" ="
+    podman commit "${name}" "${quayname}:${server_tag}"
+    echo
+fi
+
+log info "= DONE: Image created. ="
+
+# For tests:
+# podman start "${name}"
+# while [ -n "$(podman exec ansible-test systemctl list-jobs | grep -vi "no jobs running")" ]; do echo "waiting.."; sleep 5; done
+# # Run tests
+# podman stop "${name}"

infra/image/dockerfile/c10s

+FROM quay.io/centos/centos:stream10-development
+ENV container=podman
+
+RUN rm -fv /var/cache/dnf/metadata_lock.pid; \
+dnf makecache; \
+dnf --assumeyes install \
+    /usr/bin/python3 \
+    /usr/bin/dnf-3 \
+    sudo \
+    bash \
+    systemd \
+    procps-ng \
+    iproute; \
+rm -rf /var/cache/dnf/;
+
+COPY system-service/fixnet.sh /root/
+COPY system-service/fixipaip.sh /root/
+COPY system-service/fixnet.service /etc/systemd/system/
+COPY system-service/fixipaip.service /etc/systemd/system/
+RUN chmod +x /root/fixnet.sh /root/fixipaip.sh
+
+STOPSIGNAL RTMIN+3
+
+VOLUME ["/sys/fs/cgroup"]
+
+CMD ["/usr/sbin/init"]

infra/image/dockerfile/c8s

+FROM quay.io/centos/centos:stream8
+ENV container=podman
+
+RUN rm -fv /var/cache/dnf/metadata_lock.pid; \
+sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo; \
+sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo; \
+sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo; \
+dnf makecache; \
+dnf --assumeyes install \
+    /usr/bin/python3 \
+    /usr/bin/python3-config \
+    /usr/bin/dnf-3 \
+    sudo \
+    bash \
+    systemd \
+    procps-ng \
+    iproute; \
+dnf clean all; \
+rm -rf /var/cache/dnf/;
+
+COPY system-service/fixnet.sh /root/
+COPY system-service/fixipaip.sh /root/
+COPY system-service/fixnet.service /etc/systemd/system/
+COPY system-service/fixipaip.service /etc/systemd/system/
+RUN chmod +x /root/fixnet.sh /root/fixipaip.sh
+
+STOPSIGNAL RTMIN+3
+
+VOLUME ["/sys/fs/cgroup"]
+
+CMD ["/usr/sbin/init"]
+

infra/image/dockerfile/c9s

+FROM quay.io/centos/centos:stream9
+ENV container=podman
+
+RUN rm -fv /var/cache/dnf/metadata_lock.pid; \
+dnf makecache; \
+dnf --assumeyes install \
+    /usr/bin/python3 \
+    /usr/bin/dnf-3 \
+    sudo \
+    bash \
+    systemd \
+    procps-ng \
+    iproute; \
+rm -rf /var/cache/dnf/;
+
+COPY system-service/fixnet.sh /root/
+COPY system-service/fixipaip.sh /root/
+COPY system-service/fixnet.service /etc/systemd/system/
+COPY system-service/fixipaip.service /etc/systemd/system/
+RUN chmod +x /root/fixnet.sh /root/fixipaip.sh
+
+STOPSIGNAL RTMIN+3
+
+VOLUME ["/sys/fs/cgroup"]
+
+CMD ["/usr/sbin/init"]

infra/image/dockerfile/fedora-latest

+FROM fedora:latest
+ENV container=podman
+
+RUN rm -fv /var/cache/dnf/metadata_lock.pid; \
+dnf makecache; \
+dnf --assumeyes install \
+    /usr/bin/python3 \
+    /usr/bin/python3-config \
+    /usr/bin/dnf-3 \
+    sudo \
+    bash \
+    systemd \
+    procps-ng \
+    iproute; \
+dnf clean all; \
+rm -rf /var/cache/dnf/;
+
+COPY system-service/fixnet.sh /root/
+COPY system-service/fixipaip.sh /root/
+COPY system-service/fixnet.service /etc/systemd/system/
+COPY system-service/fixipaip.service /etc/systemd/system/
+RUN chmod +x /root/fixnet.sh /root/fixipaip.sh
+
+STOPSIGNAL RTMIN+3
+
+VOLUME ["/sys/fs/cgroup"]
+
+CMD ["/usr/sbin/init"]

infra/image/dockerfile/fedora-rawhide

+FROM fedora:rawhide
+ENV container=podman
+
+RUN rm -fv /var/cache/dnf/metadata_lock.pid; \
+dnf makecache; \
+dnf --assumeyes install \
+    /usr/bin/python3 \
+    /usr/bin/python3-config \
+    /usr/bin/dnf-3 \
+    python3-libdnf5 \
+    sudo \
+    bash \
+    systemd \
+    procps-ng \
+    iproute; \
+dnf clean all; \
+rm -rf /var/cache/dnf/;
+
+COPY system-service/fixnet.sh /root/
+COPY system-service/fixipaip.sh /root/
+COPY system-service/fixnet.service /etc/systemd/system/
+COPY system-service/fixipaip.service /etc/systemd/system/
+RUN chmod +x /root/fixnet.sh /root/fixipaip.sh
+
+STOPSIGNAL RTMIN+3
+
+VOLUME ["/sys/fs/cgroup"]
+
+CMD ["/usr/sbin/init"]

infra/image/inventory

+[ipaserver]
+ansible-freeipa-image-builder ansible_connection=podman ansible_python_interpreter=/usr/bin/python3
+
+[ipaserver:vars]
+ipaadmin_password=SomeADMINpassword
+ipadm_password=SomeDMpassword
+ipaserver_domain=test.local
+ipaserver_realm=TEST.LOCAL
+ipaserver_setup_dns=true
+ipaserver_auto_forwarders=true
+ipaserver_no_dnssec_validation=true
+ipaserver_auto_reverse=true
+ipaserver_setup_kra=true
+ipaserver_setup_firewalld=false
+ipaclient_no_ntp=true

infra/image/system-service/fixipaip.service

+[Unit]
+Description=Fix IPA server IP in IPA Server
+After=multi-user.target
+
+[Service]
+Type=oneshot
+ExecStart=/root/fixipaip.sh
+
+[Install]
+WantedBy=default.target

infra/image/system-service/fixipaip.sh

+#!/bin/bash -eu
+
+HOSTNAME=$(hostname)
+IP=$(hostname -I | cut -d " " -f 1)
+
+if [ -z "${HOSTNAME}" ]; then
+    echo "ERROR: Failed to retrieve hostname."
+    exit 1
+fi
+if [ -z "${IP}" ]; then
+    echo "ERROR: Failed to retrieve IP address."
+    exit 1
+fi
+
+if ! echo "SomeADMINpassword" | kinit -c ansible_freeipa_cache admin
+then
+    echo "ERROR: Failed to obtain Kerberos ticket"
+    exit 1
+fi
+KRB5CCNAME=ansible_freeipa_cache \
+    ipa dnsrecord-mod test.local "${HOSTNAME%%.*}" --a-rec="$IP"
+KRB5CCNAME=ansible_freeipa_cache \
+    ipa dnsrecord-mod test.local ipa-ca --a-rec="$IP"
+kdestroy -c ansible_freeipa_cache -A
+
+exit 0

infra/image/system-service/fixnet.service

+[Unit]
+Description=Fix server IP in IPA Server
+Wants=network.target
+After=network.target
+Before=ipa.service
+
+[Service]
+Type=oneshot
+ExecStart=/root/fixnet.sh
+
+[Install]
+WantedBy=ipa.service

infra/image/system-service/fixnet.sh

+#!/bin/bash -eu
+
+HOSTNAME=$(hostname)
+IP=$(hostname -I | cut -d " " -f 1)
+
+if [ -z "${HOSTNAME}" ]; then
+    echo "ERROR: Failed to retrieve hostname."
+    exit 1
+fi
+if [ -z "${IP}" ]; then
+    echo "ERROR: Failed to retrieve IP address."
+    exit 1
+fi
+
+# shellcheck disable=SC2143
+if [ -n "$(grep -P "[[:space:]]${HOSTNAME}" /etc/hosts)" ]; then
+    sed -ie "s/.*${HOSTNAME}/${IP}\t${HOSTNAME}/" /etc/hosts
+else
+    echo -e "$IP\t${HOSTNAME}" >> /etc/hosts
+fi
+
+echo "nameserver 127.0.0.1" > /etc/resolv.conf
+
+exit 0

tests/azure/build-containers.yml

@@ -11,18 +11,19 @@ schedules:
 trigger: none
 
 pool:
-  vmImage: 'ubuntu-20.04'
+  vmImage: 'ubuntu-22.04'
 
 stages:
 
-- stage: CentOS_7
-  dependsOn: []
-  jobs:
-  - template: templates/build_container.yml
-    parameters:
-      job_name_suffix: Centos7
-      container_name: centos-7
-      build_scenario_name: centos-7-build
+# Currently, it's not possible to use CentOS container
+#
+# - stage: CentOS_7
+#   dependsOn: []
+#   jobs:
+#   - template: templates/build_container.yml
+#     parameters:
+#       job_name_suffix: Centos7
+#       distro: centos-7
 
 - stage: CentOS_8_Stream
   dependsOn: []
@@ -30,8 +31,9 @@ stages:
   - template: templates/build_container.yml
     parameters:
       job_name_suffix: C8S
-      container_name: c8s
-      build_scenario_name: c8s-build
+      distro: c8s
+      # ansible-core 2.17+ cannot be used to deploy on CentOS 8 Stream.
+      ansible_core_version: "<2.17"
 
 - stage: CentOS_9_Stream
   dependsOn: []
@@ -39,8 +41,7 @@ stages:
   - template: templates/build_container.yml
     parameters:
       job_name_suffix: C9S
-      container_name: c9s
-      build_scenario_name: c9s-build
+      distro: c9s
 
 - stage: Fedora_Latest
   dependsOn: []
@@ -48,8 +49,7 @@ stages:
   - template: templates/build_container.yml
     parameters:
       job_name_suffix: FedoraLatest
-      container_name: fedora-latest
-      build_scenario_name: fedora-latest-build
+      distro: fedora-latest
 
 - stage: Fedora_Rawhide
   dependsOn: []
@@ -57,5 +57,4 @@ stages:
   - template: templates/build_container.yml
     parameters:
       job_name_suffix: FedoraRawhide
-      container_name: fedora-rawhide
-      build_scenario_name: fedora-rawhide-build
+      distro: fedora-rawhide

tests/azure/templates/build_container.yml

@@ -2,42 +2,49 @@
 parameters:
   - name: job_name_suffix
     type: string
-  - name: container_name
-    type: string
-  - name: build_scenario_name
+  - name: distro
     type: string
   - name: python_version
     type: string
     default: 3.x
+  - name: ansible_core_version
+    default: ""
 
 jobs:
 - job: BuildTestImage${{ parameters.job_name_suffix }}
-  displayName: Build ${{ parameters.container_name }} test container
+  displayName: Build ${{ parameters.distro }} test container
   steps:
   - task: UsePythonVersion@0
     inputs:
       versionSpec: '${{ parameters.python_version }}'
 
-  - script: python -m pip install --upgrade pip setuptools wheel ansible
+  - script: python -m pip install --upgrade pip "ansible-core${{ parameters.ansible_core_version }}"
     retryCountOnTaskFailure: 5
     displayName: Install tools
 
-  - script: pip install molecule-plugins[docker] "requests<2.29"
-    retryCountOnTaskFailure: 5
-    displayName: Install molecule
+  - script: |
+      rm -rf ~/.ansible
+      mkdir -p ~/.ansible
+      ln -snf $(readlink -f roles) ~/.ansible/roles
+      ln -snf $(readlink -f plugins) ~/.ansible/plugins
+    displayName: Setup ansible-freeipa using Git repository
 
-  - script: molecule create -s ${{ parameters.build_scenario_name }}
-    retryCountOnTaskFailure: 5
-    displayName: Create test container
-    env:
-      ANSIBLE_LIBRARY: ./molecule
+  - script: ansible-galaxy collection install containers.podman
+    displayName: Install Ansible Galaxy collections
 
-  - script: |
-      docker stop ${{ parameters.build_scenario_name }}
-      docker commit ${{ parameters.build_scenario_name }} quay.io/ansible-freeipa/upstream-tests:${{ parameters.container_name }}
-      docker login -u="$QUAY_ROBOT_USERNAME" -p="$QUAY_ROBOT_TOKEN" quay.io
-      docker push quay.io/ansible-freeipa/upstream-tests:${{ parameters.container_name }}
-    displayName: Save image and upload
+  - script: infra/image/build.sh -s ${{ parameters.distro }}
+    displayName: Build ${{ parameters.distro }} base image
+
+  - script: podman login -u="$QUAY_ROBOT_USERNAME" -p="$QUAY_ROBOT_TOKEN" quay.io
+    displayName: Registry login
     env:
       # Secrets needs to be mapped as env vars to work properly
       QUAY_ROBOT_TOKEN: $(QUAY_ROBOT_TOKEN)
+
+  - script: |
+      podman push quay.io/ansible-freeipa/upstream-tests:${{parameters.distro}}-base quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-base
+    displayName: Push base image
+
+  - script: |
+      podman push quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-server quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-server
+    displayName: Push server image

utils/shfun

+#!/bin/bash -eu
+# This file is meant to be source'd by shell scripts
+
+SCRIPTDIR="$(dirname -- "$(readlink -f "${BASH_SOURCE[0]}")")"
+
+. "${SCRIPTDIR}/shlog"
+
+[ -n "$(command -v python3)" ] && python="$(command -v python3)" || python="$(command -v python2)"
+export python
+
+trap interrupt_exception SIGINT
+
+interrupt_exception() {
+    trap - SIGINT
+    log warn "User interrupted test execution."
+    # shellcheck disable=SC2119
+    cleanup "${scenario:+${scenario}}"
+    exit 1
+}
+
+run_if_exists() {
+    cmd="${1}"
+    shift
+    [ -n "$(command -v "${cmd}")" ] && "${cmd}" "${@}"
+}
+
+# shellcheck disable=SC2120
+cleanup() {
+    local container container_engine
+    container="${1:-${scenario:+${scenario}}}"
+    container_engine="${2:-${engine:-"podman"}}"
+    if [ "${STOP_CONTAINER:-"Y"}" == "Y" ] && [ -n "${container}" ]
+    then
+	    run_if_exists stop_container "${container}" "${container_engine}"
+        [ -f "${inventory:-}" ] && rm "${inventory}"
+    else
+        if [ -n "${container}" ]
+        then
+            log info "Keeping container: $(${container_engine} ps --format "{{.Names}} - {{.ID}}" --filter "name=${container}")"
+        fi
+    fi
+    if [ "${STOP_VIRTUALENV:-"N"}" == "Y" ]
+    then
+        echo "Deactivating virtual environment"
+        run_if_exists deactivate
+    fi
+}
+
+start_virtual_environment() {
+    # options -f
+    local FORCE_ENV VENV envdirectory
+    FORCE_ENV="N"
+    while getopts ":f" option
+    do
+        case "$option" in
+            f) FORCE_ENV="Y" ;;
+            *) die "prepare_virtual_environment: Invalid option: ${option}" ;;
+        esac
+    done
+    envdirectory="${test_env:-/tmp/ansible-freeipa-tests}"
+
+    # Prepare virtual environment
+    VENV=$(in_python_virtualenv && echo Y || echo N)
+
+    if [ "${FORCE_ENV}" == "Y" ]
+    then
+        run_if_exists deactivate
+        VENV="N"
+        rm -rf "$test_env"
+        log info "Virtual environment will be (re)created."
+    fi
+
+    if [ "$VENV" == "N" ]
+    then
+        log info "Preparing virtual environment: ${envdirectory}"
+        if [ ! -d "${envdirectory}" ] || [ ! -f "${envdirectory}/bin/activate" ]
+        then
+            log info "Creating virtual environment: ${envdirectory}..."
+	    log warn "RUN: ${python} -m venv ${envdirectory}"
+	    ${python} -m venv "${envdirectory}" || die "Cannot create virtual environment."
+	fi
+	log info "Starting virtual environment: ${envdirectory}"
+	[ -f "${envdirectory}/bin/activate" ] || die "Failed to create virtual environment."
+	# shellcheck disable=SC1091
+	. "${envdirectory}/bin/activate" || die "Cannot activate virtual environment."
+	export STOP_VIRTUALENV="Y"
+        log info "Installing required tools."
+        log none "Upgrading: pip setuptools wheel"
+        pip install --quiet --upgrade pip setuptools wheel
+    else
+       log info "Using current virtual environment."
+    fi
+}
+
+die() {
+    usg="N"
+    if [ "${1}" == "-u" ]
+    then
+       usg="Y"
+       shift 1
+    fi
+    log error "${*}"
+    STOP_CONTAINER="N"
+    cleanup "${scenario:+${scenario}}"
+    [ "${usg}" == "Y" ] && run_if_exists usage
+    exit 1
+}
+
+in_python_virtualenv() {
+    local script
+    read -r -d "" script <<EOS
+import sys;
+base = getattr(sys, "base_prefix", ) or getattr(sys, "real_prefix", ) or sys.prefix
+print('yes' if sys.prefix != base else 'no')
+EOS
+    test "$(${python} -c "${script}")" == "yes"
+}
+

utils/shlog

+#!/bin/bash -eu
+# This file is meant to be source'd by shell scripts
+
+# shellcheck disable=SC2034
+RST="\033[0m"
+# shellcheck disable=SC2034
+RED="\033[31m"
+# shellcheck disable=SC2034
+BRIGHTRED="\033[31;1m"
+# shellcheck disable=SC2034
+GREEN="\033[32m"
+# shellcheck disable=SC2034
+BRIGHTGREEN="\033[32;1m"
+# shellcheck disable=SC2034
+BROWN="\033[33m"
+# shellcheck disable=SC2034
+YELLOW="\033[33;1m"
+# shellcheck disable=SC2034
+NAVY="\033[34m"
+# shellcheck disable=SC2034
+BLUE="\033[34;1m"
+# shellcheck disable=SC2034
+MAGENTA="\033[35m"
+# shellcheck disable=SC2034
+BRIGHTMAGENTA="\033[35;1m"
+# shellcheck disable=SC2034
+DARKCYAN="\033[36m"
+# shellcheck disable=SC2034
+CYAN="\033[36;1m"
+# shellcheck disable=SC2034
+BLACK="\033[30m"
+# shellcheck disable=SC2034
+DARKGRAY="\033[30;1m"
+# shellcheck disable=SC2034
+GRAY="\033[37m"
+# shellcheck disable=SC2034
+WHITE="\033[37;1m"
+
+log() {
+    local level="${1^^}" message="${*:2}"
+    case "${level}" in
+        ERROR) COLOR="${RED}" ;;
+        WARN)  COLOR="${YELLOW}" ;;
+        DEBUG) COLOR="${BLUE}" ;;
+        INFO) COLOR="${WHITE}" ;;
+        SUCCESS) COLOR="${BRIGHTGREEN}" ;;
+        *) COLOR="${RST}" ;;
+    esac
+    echo -en "${COLOR}"
+    [ "${level}" == "ERROR" ] && echo -en "${level}:"
+    echo -e "${message}${RST}"
+}
+
+quiet() {
+     "$@" >/dev/null 2>&1
+}
+