From a0587e0b8ee5962ad279e7c0abdf063d4e68093a Mon Sep 17 00:00:00 2001
From: Antoine Legrand <2t.antoine@gmail.com>
Date: Mon, 1 Jul 2024 12:25:36 +0200
Subject: [PATCH] CI: rework pipeline: short/extended based on labels (#11324)
* CI: reduce VM resources requests to improve scheduling
* CI: Reduce default jobs; add labels(ci-full/extended) to run more test
* CI: use jobs dependencies instead of stages
* precommit one-job
* CI: Use Kubevirt VM to run Molecule and Vagrant jobs
---
.gitlab-ci.yml | 30 +-
.gitlab-ci/build.yml | 2 +-
.gitlab-ci/lint.yml | 43 +--
.gitlab-ci/molecule.yml | 64 ++--
.gitlab-ci/packet.yml | 362 ++++++++----------
.gitlab-ci/pre-commit-dynamic-stub.yml | 2 +-
.gitlab-ci/terraform.yml | 22 +-
.gitlab-ci/vagrant.yml | 41 +-
Vagrantfile | 1 +
docs/developers/ci-setup.md | 4 +-
pipeline.Dockerfile | 5 +-
tests/Makefile | 3 +-
.../roles/packet-ci/defaults/main.yml | 5 +-
.../roles/packet-ci/templates/vm.yml.j2 | 6 +-
.../packet_almalinux8-calico-ha-ebpf.yml | 2 +-
...malinux8-calico-nodelocaldns-secondary.yml | 2 +-
tests/files/packet_almalinux8-calico.yml | 2 +-
tests/files/packet_almalinux8-docker.yml | 2 +-
tests/files/packet_almalinux8-kube-ovn.yml | 2 +-
tests/files/packet_rockylinux8-calico.yml | 2 +-
tests/files/packet_rockylinux9-calico.yml | 2 +-
tests/files/packet_rockylinux9-cilium.yml | 2 +-
.../packet_ubuntu22-all-in-one-docker.yml | 2 +-
.../packet_ubuntu22-calico-all-in-one.yml | 2 +-
.../packet_ubuntu24-all-in-one-docker.yml | 2 +-
.../packet_ubuntu24-calico-all-in-one.yml | 2 +-
.../packet_ubuntu24-calico-etcd-datastore.yml | 2 +-
tests/files/vagrant_ubuntu20-flannel.rb | 1 -
tests/files/vagrant_ubuntu20-flannel.yml | 1 +
tests/scripts/vagrant-install.sh | 122 ++++++
30 files changed, 419 insertions(+), 321 deletions(-)
create mode 100755 tests/scripts/vagrant-install.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b838c6fc2..71ed0f37e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,12 +1,9 @@
---
stages:
- build
- - unit-tests
- - moderator
+ - test
- deploy-part1
- - deploy-part2
- - deploy-part3
- - deploy-special
+ - deploy-extended
variables:
KUBESPRAY_VERSION: v2.25.0
@@ -43,15 +40,26 @@ before_script:
.job: &job
tags:
- - packet
+ - ffci
image: $PIPELINE_IMAGE
artifacts:
when: always
paths:
- cluster-dump/
+ needs:
+ - pipeline-image
+
+.job-moderated:
+ extends: .job
+ needs:
+ - pipeline-image
+ - ci-not-authorized
+ - check-galaxy-version # lint
+ - pre-commit # lint
+ - vagrant-validate # lint
.testcases: &testcases
- <<: *job
+ extends: .job-moderated
retry: 1
interruptible: true
before_script:
@@ -61,12 +69,12 @@ before_script:
script:
- ./tests/scripts/testcases_run.sh
after_script:
- - chronic ./tests/scripts/testcases_cleanup.sh
+ - ./tests/scripts/testcases_cleanup.sh
# For failfast, at least 1 job must be defined in .gitlab-ci.yml
# Premoderated with manual actions
ci-not-authorized:
- stage: moderator
+ stage: build
before_script: []
after_script: []
rules:
@@ -87,8 +95,8 @@ ci-not-authorized:
script:
- exit $CI_OK_TO_TEST
tags:
- - light
-
+ - ffci
+ needs: []
include:
- .gitlab-ci/build.yml
diff --git a/.gitlab-ci/build.yml b/.gitlab-ci/build.yml
index 74acfdef4..89cbd196c 100644
--- a/.gitlab-ci/build.yml
+++ b/.gitlab-ci/build.yml
@@ -5,7 +5,7 @@
paths:
- image-cache
tags:
- - packet
+ - ffci
stage: build
image:
name: gcr.io/kaniko-project/executor:debug
diff --git a/.gitlab-ci/lint.yml b/.gitlab-ci/lint.yml
index 00f381f4e..cf81c5517 100644
--- a/.gitlab-ci/lint.yml
+++ b/.gitlab-ci/lint.yml
@@ -1,40 +1,35 @@
---
-generate-pre-commit:
- image: 'mikefarah/yq@sha256:bcb889a1f9bdb0613c8a054542d02360c2b1b35521041be3e1bd8fbd0534d411'
- stage: build
- before_script: []
+pre-commit:
+ stage: test
+ tags:
+ - ffci
+ image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef'
+ variables:
+ PRE_COMMIT_HOME: /pre-commit-cache
script:
- - >
- yq -r < .pre-commit-config.yaml '.repos[].hooks[].id' |
- sed 's/^/ - /' |
- cat .gitlab-ci/pre-commit-dynamic-stub.yml - > pre-commit-generated.yml
- artifacts:
+ - pre-commit run --all-files
+ cache:
+ key: pre-commit-all
paths:
- - pre-commit-generated.yml
-
-run-pre-commit:
- stage: unit-tests
- trigger:
- include:
- - artifact: pre-commit-generated.yml
- job: generate-pre-commit
- strategy: depend
+ - /pre-commit-cache
+ needs: []
vagrant-validate:
extends: .job
- stage: unit-tests
- tags: [light]
+ stage: test
+ tags: [ffci]
variables:
VAGRANT_VERSION: 2.3.7
script:
- - ./tests/scripts/vagrant-validate.sh
+ - ./tests/scripts/vagrant-validate.sh
except: ['triggers', 'master']
# TODO: convert to pre-commit hook
check-galaxy-version:
- stage: unit-tests
- tags: [light]
+ needs: []
+ stage: test
+ tags: [ffci]
image: python:3
script:
- - tests/scripts/check_galaxy_version.sh
+ - tests/scripts/check_galaxy_version.sh
diff --git a/.gitlab-ci/molecule.yml b/.gitlab-ci/molecule.yml
index 94e5a5606..c5fd4d4b2 100644
--- a/.gitlab-ci/molecule.yml
+++ b/.gitlab-ci/molecule.yml
@@ -1,30 +1,42 @@
---
-
.molecule:
- tags: [c3.small.x86]
+ tags: [ffci-vm-med]
only: [/^pr-.*$/]
except: ['triggers']
- image: $PIPELINE_IMAGE
+ image: quay.io/kubespray/vm-kubespray-ci:v6
services: []
stage: deploy-part1
+ needs: []
+ # - ci-not-authorized
+ variables:
+ VAGRANT_DEFAULT_PROVIDER: "libvirt"
before_script:
- - tests/scripts/rebase.sh
- - ./tests/scripts/vagrant_clean.sh
+ - groups
+ - python3 -m venv citest
+ - source citest/bin/activate
+ - vagrant plugin expunge --reinstall --force --no-tty
+ - vagrant plugin install vagrant-libvirt
+ - pip install --no-compile --no-cache-dir pip -U
+ - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt
+ - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
+ - ./tests/scripts/rebase.sh
+ - ./tests/scripts/vagrant_clean.sh
script:
- - ./tests/scripts/molecule_run.sh
+ - ./tests/scripts/molecule_run.sh
after_script:
- - chronic ./tests/scripts/molecule_logs.sh
+ - ./tests/scripts/molecule_logs.sh
artifacts:
when: always
paths:
- - molecule_logs/
+ - molecule_logs/
# CI template for periodic CI jobs
# Enabled when PERIODIC_CI_ENABLED var is set
+
.molecule_periodic:
only:
variables:
- - $PERIODIC_CI_ENABLED
+ - $PERIODIC_CI_ENABLED
allow_failure: true
extends: .molecule
@@ -34,50 +46,50 @@ molecule_full:
molecule_no_container_engines:
extends: .molecule
script:
- - ./tests/scripts/molecule_run.sh -e container-engine
+ - ./tests/scripts/molecule_run.sh -e container-engine
when: on_success
molecule_docker:
extends: .molecule
script:
- - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd
+ - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd
when: on_success
molecule_containerd:
extends: .molecule
script:
- - ./tests/scripts/molecule_run.sh -i container-engine/containerd
+ - ./tests/scripts/molecule_run.sh -i container-engine/containerd
when: on_success
molecule_cri-o:
extends: .molecule
- stage: deploy-part2
+ stage: deploy-part1
script:
- - ./tests/scripts/molecule_run.sh -i container-engine/cri-o
+ - ./tests/scripts/molecule_run.sh -i container-engine/cri-o
allow_failure: true
when: on_success
-# Stage 3 container engines don't get as much attention so allow them to fail
-molecule_kata:
- extends: .molecule
- stage: deploy-part3
- script:
- - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers
- when: manual
-# FIXME: this test is broken (perma-failing)
+# # Stage 3 container engines don't get as much attention so allow them to fail
+# molecule_kata:
+# extends: .molecule
+# stage: deploy-extended
+# script:
+# - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers
+# when: manual
+# # FIXME: this test is broken (perma-failing)
molecule_gvisor:
extends: .molecule
- stage: deploy-part3
+ stage: deploy-extended
script:
- - ./tests/scripts/molecule_run.sh -i container-engine/gvisor
+ - ./tests/scripts/molecule_run.sh -i container-engine/gvisor
when: manual
# FIXME: this test is broken (perma-failing)
molecule_youki:
extends: .molecule
- stage: deploy-part3
+ stage: deploy-extended
script:
- - ./tests/scripts/molecule_run.sh -i container-engine/youki
+ - ./tests/scripts/molecule_run.sh -i container-engine/youki
when: manual
# FIXME: this test is broken (perma-failing)
diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml
index 1f22599d9..235c7fe4d 100644
--- a/.gitlab-ci/packet.yml
+++ b/.gitlab-ci/packet.yml
@@ -6,13 +6,55 @@
CI_PLATFORM: packet
SSH_USER: kubespray
tags:
- - packet
- except: [triggers]
+ - ffci
+ needs:
+ - pipeline-image
+ - ci-not-authorized
# CI template for PRs
.packet_pr:
- only: [/^pr-.*$/]
+ stage: deploy-part1
+ rules:
+ - if: $PR_LABELS =~ /.*ci-short.*/
+ when: manual
+ allow_failure: true
+ - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+ when: on_success
+ - when: manual
+ allow_failure: true
+ extends: .packet
+
+ ## Uncomment this to have multiple stages
+ # needs:
+ # - packet_ubuntu20-calico-all-in-one
+
+.packet_pr_short:
+ stage: deploy-part1
extends: .packet
+ rules:
+ - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+ when: on_success
+ - when: manual
+ allow_failure: true
+
+.packet_pr_manual:
+ extends: .packet_pr
+ stage: deploy-extended
+ rules:
+ - if: $PR_LABELS =~ /.*ci-full.*/
+ when: on_success
+ # Else run as manual
+ - when: manual
+ allow_failure: true
+
+.packet_pr_extended:
+ extends: .packet_pr
+ stage: deploy-extended
+ rules:
+ - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
+ when: on_success
+ - when: manual
+ allow_failure: true
# CI template for periodic CI jobs
# Enabled when PERIODIC_CI_ENABLED var is set
@@ -34,118 +76,87 @@ packet_cleanup_old:
# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
packet_ubuntu20-calico-all-in-one:
stage: deploy-part1
- extends: .packet_pr
- when: on_success
+ extends: .packet_pr_short
variables:
RESET_CHECK: "true"
# ### PR JOBS PART2
-packet_ubuntu20-all-in-one-docker:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
+packet_ubuntu20-crio:
+ extends: .packet_pr_manual
-packet_ubuntu20-calico-all-in-one-hardening:
- stage: deploy-part2
+packet_ubuntu22-calico-all-in-one:
extends: .packet_pr
- when: on_success
-packet_ubuntu22-all-in-one-docker:
- stage: deploy-part2
+packet_ubuntu24-calico-etcd-datastore:
extends: .packet_pr
- when: on_success
-packet_ubuntu22-calico-all-in-one:
- stage: deploy-part2
+packet_almalinux8-crio:
extends: .packet_pr
- when: on_success
-packet_ubuntu24-all-in-one-docker:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
-packet_ubuntu24-calico-all-in-one:
- stage: deploy-part2
+packet_almalinux8-kube-ovn:
extends: .packet_pr
- when: on_success
-packet_ubuntu24-calico-etcd-datastore:
- stage: deploy-part2
+packet_debian10-docker:
extends: .packet_pr
- when: on_success
-packet_centos7-flannel-addons-ha:
+packet_debian11-calico:
extends: .packet_pr
- stage: deploy-part2
- when: on_success
-packet_almalinux8-crio:
- extends: .packet_pr
- stage: deploy-part2
- when: on_success
- allow_failure: true
-packet_ubuntu20-crio:
+packet_debian12-cilium:
extends: .packet_pr
- stage: deploy-part2
- when: manual
-packet_fedora37-crio:
+packet_rockylinux8-calico:
extends: .packet_pr
- stage: deploy-part2
- when: manual
-packet_ubuntu20-flannel-ha:
- stage: deploy-part2
+packet_rockylinux9-cilium:
extends: .packet_pr
- when: manual
+ variables:
+ RESET_CHECK: "true"
-packet_debian10-cilium-svc-proxy:
- stage: deploy-part2
- extends: .packet_periodic
- when: on_success
+packet_amazon-linux-2-all-in-one:
+ extends: .packet_pr
-packet_debian10-calico:
- stage: deploy-part2
+packet_fedora38-docker-weave:
extends: .packet_pr
- when: on_success
+ allow_failure: true
-packet_debian10-docker:
- stage: deploy-part2
+packet_opensuse-docker-cilium:
extends: .packet_pr
- when: on_success
-packet_debian11-calico:
- stage: deploy-part2
+
+packet_ubuntu20-cilium-sep:
extends: .packet_pr
- when: on_success
+## Extended
packet_debian11-docker:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
+ extends: .packet_pr_extended
+
+packet_debian12-docker:
+ extends: .packet_pr_extended
packet_debian12-calico:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
+ extends: .packet_pr_extended
-packet_debian12-docker:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
+packet_almalinux8-calico-remove-node:
+ extends: .packet_pr_extended
+ variables:
+ REMOVE_NODE_CHECK: "true"
+ REMOVE_NODE_NAME: "instance-3"
-packet_debian12-cilium:
- stage: deploy-part2
- extends: .packet_periodic
- when: on_success
+packet_rockylinux9-calico:
+ extends: .packet_pr_extended
+
+packet_almalinux8-calico:
+ extends: .packet_pr_extended
+
+packet_almalinux8-docker:
+ extends: .packet_pr_extended
packet_centos7-calico-ha-once-localhost:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
+ extends: .packet_pr_extended
variables:
# This will instruct Docker not to start over TLS.
DOCKER_TLS_CERTDIR: ""
@@ -153,196 +164,127 @@ packet_centos7-calico-ha-once-localhost:
services:
- docker:26.1.4-dind
-packet_almalinux8-kube-ovn:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
+packet_centos7-flannel-addons-ha:
+ extends: .packet_pr_extended
-packet_almalinux8-calico:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
+packet_debian10-calico:
+ extends: .packet_pr_extended
-packet_rockylinux8-calico:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
+packet_ubuntu20-calico-all-in-one-hardening:
+ extends: .packet_pr_extended
-packet_rockylinux9-calico:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
+packet_ubuntu24-calico-all-in-one:
+ extends: .packet_pr_extended
-packet_rockylinux9-cilium:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
- variables:
- RESET_CHECK: "true"
+packet_ubuntu20-calico-etcd-kubeadm:
+ extends: .packet_pr_extended
-packet_almalinux8-docker:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
+packet_ubuntu24-all-in-one-docker:
+ extends: .packet_pr_extended
-packet_amazon-linux-2-all-in-one:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
+packet_ubuntu22-all-in-one-docker:
+ extends: .packet_pr_extended
-packet_fedora38-docker-weave:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
- allow_failure: true
+# ### MANUAL JOBS
+packet_centos7-calico-ha:
+ extends: .packet_pr_manual
-packet_opensuse-docker-cilium:
- stage: deploy-part2
- extends: .packet_pr
- when: on_success
+packet_fedora37-crio:
+ extends: .packet_pr_manual
-# ### MANUAL JOBS
+packet_ubuntu20-flannel-ha:
+ extends: .packet_pr_manual
-packet_ubuntu20-docker-weave-sep:
- stage: deploy-part2
- extends: .packet_pr
- when: manual
+packet_ubuntu20-all-in-one-docker:
+ extends: .packet_pr_manual
-packet_ubuntu20-cilium-sep:
- stage: deploy-special
- extends: .packet_pr
- when: manual
+packet_ubuntu20-docker-weave-sep:
+ extends: .packet_pr_manual
packet_ubuntu20-flannel-ha-once:
- stage: deploy-part2
- extends: .packet_pr
- when: manual
+ extends: .packet_pr_manual
+
+packet_fedora37-calico-swap-selinux:
+ extends: .packet_pr_manual
-# Calico HA eBPF
packet_almalinux8-calico-ha-ebpf:
- stage: deploy-part2
- extends: .packet_pr
- when: manual
+ extends: .packet_pr_manual
-packet_debian10-macvlan:
- stage: deploy-part2
- extends: .packet_pr
- when: manual
+packet_almalinux8-calico-nodelocaldns-secondary:
+ extends: .packet_pr_manual
-packet_centos7-calico-ha:
- stage: deploy-part2
- extends: .packet_pr
- when: manual
+packet_debian10-macvlan:
+ extends: .packet_pr_manual
packet_centos7-multus-calico:
- stage: deploy-part2
- extends: .packet_pr
- when: manual
+ extends: .packet_pr_manual
+
+packet_debian11-custom-cni:
+ extends: .packet_pr_manual
+
+packet_debian11-kubelet-csr-approver:
+ extends: .packet_pr_manual
+
+packet_debian12-custom-cni-helm:
+ extends: .packet_pr_manual
+
+packet_ubuntu20-calico-ha-wireguard:
+ extends: .packet_pr_manual
+# PERIODIC
packet_fedora38-docker-calico:
- stage: deploy-part2
+ stage: deploy-extended
extends: .packet_periodic
- when: on_success
variables:
RESET_CHECK: "true"
packet_fedora37-calico-selinux:
- stage: deploy-part2
+ stage: deploy-extended
extends: .packet_periodic
- when: on_success
-
-packet_fedora37-calico-swap-selinux:
- stage: deploy-part2
- extends: .packet_pr
- when: manual
-
-packet_almalinux8-calico-nodelocaldns-secondary:
- stage: deploy-part2
- extends: .packet_pr
- when: manual
packet_fedora38-kube-ovn:
- stage: deploy-part2
+ stage: deploy-extended
extends: .packet_periodic
- when: on_success
-
-packet_debian11-custom-cni:
- stage: deploy-part2
- extends: .packet_pr
- when: manual
-
-packet_debian11-kubelet-csr-approver:
- stage: deploy-part2
- extends: .packet_pr
- when: manual
-
-packet_debian12-custom-cni-helm:
- stage: deploy-part2
- extends: .packet_pr
- when: manual
-
-# ### PR JOBS PART3
-# Long jobs (45min+)
packet_centos7-weave-upgrade-ha:
- stage: deploy-part3
+ stage: deploy-extended
extends: .packet_periodic
- when: on_success
variables:
UPGRADE_TEST: basic
packet_ubuntu20-calico-etcd-kubeadm-upgrade-ha:
- stage: deploy-part3
+ stage: deploy-extended
extends: .packet_periodic
- when: on_success
variables:
UPGRADE_TEST: basic
-# Calico HA Wireguard
-packet_ubuntu20-calico-ha-wireguard:
- stage: deploy-part2
- extends: .packet_pr
- when: manual
-
-packet_debian11-calico-upgrade:
- stage: deploy-part3
- extends: .packet_pr
- when: on_success
- variables:
- UPGRADE_TEST: graceful
-
-packet_almalinux8-calico-remove-node:
- stage: deploy-part3
- extends: .packet_pr
- when: on_success
- variables:
- REMOVE_NODE_CHECK: "true"
- REMOVE_NODE_NAME: "instance-3"
-
-packet_ubuntu20-calico-etcd-kubeadm:
- stage: deploy-part3
- extends: .packet_pr
- when: on_success
packet_debian11-calico-upgrade-once:
- stage: deploy-part3
+ stage: deploy-extended
extends: .packet_periodic
- when: on_success
variables:
UPGRADE_TEST: graceful
packet_ubuntu20-calico-ha-recover:
- stage: deploy-part3
+ stage: deploy-extended
extends: .packet_periodic
- when: on_success
variables:
RECOVER_CONTROL_PLANE_TEST: "true"
RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
packet_ubuntu20-calico-ha-recover-noquorum:
- stage: deploy-part3
+ stage: deploy-extended
extends: .packet_periodic
- when: on_success
variables:
RECOVER_CONTROL_PLANE_TEST: "true"
RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[1:]:kube_control_plane[1:]"
+
+packet_debian10-cilium-svc-proxy:
+ stage: deploy-extended
+ extends: .packet_periodic
+
+packet_debian11-calico-upgrade:
+ stage: deploy-extended
+ extends: .packet_periodic
+ variables:
+ UPGRADE_TEST: graceful
diff --git a/.gitlab-ci/pre-commit-dynamic-stub.yml b/.gitlab-ci/pre-commit-dynamic-stub.yml
index 55f81b513..f91b5e810 100644
--- a/.gitlab-ci/pre-commit-dynamic-stub.yml
+++ b/.gitlab-ci/pre-commit-dynamic-stub.yml
@@ -7,7 +7,7 @@ pre-commit:
variables:
PRE_COMMIT_HOME: /pre-commit-cache
script:
- - pre-commit run -a $HOOK_ID
+ - pre-commit run --all-files
cache:
key: pre-commit-$HOOK_ID
paths:
diff --git a/.gitlab-ci/terraform.yml b/.gitlab-ci/terraform.yml
index 32991776c..223522815 100644
--- a/.gitlab-ci/terraform.yml
+++ b/.gitlab-ci/terraform.yml
@@ -2,6 +2,10 @@
# Tests for contrib/terraform/
.terraform_install:
extends: .job
+ needs:
+ - ci-not-authorized
+ - pipeline-image
+ stage: deploy-part1
before_script:
- update-alternatives --install /usr/bin/python python /usr/bin/python3 1
- ./tests/scripts/rebase.sh
@@ -24,17 +28,19 @@
.terraform_validate:
extends: .terraform_install
- stage: unit-tests
- tags: [light]
+ tags: [ffci]
only: ['master', /^pr-.*$/]
script:
- terraform -chdir="contrib/terraform/$PROVIDER" validate
- terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
+ stage: test
+ needs:
+ - pipeline-image
.terraform_apply:
extends: .terraform_install
- tags: [light]
- stage: deploy-part3
+ tags: [ffci]
+ stage: deploy-extended
when: manual
only: [/^pr-.*$/]
artifacts:
@@ -51,7 +57,7 @@
- tests/scripts/testcases_run.sh
after_script:
# Cleanup regardless of exit code
- - chronic ./tests/scripts/testcases_cleanup.sh
+ - ./tests/scripts/testcases_cleanup.sh
tf-validate-openstack:
extends: .terraform_validate
@@ -146,8 +152,7 @@ tf-validate-nifcloud:
TF_VAR_router_id: "ab95917c-41fb-4881-b507-3a6dfe9403df"
tf-elastx_cleanup:
- stage: unit-tests
- tags: [light]
+ tags: [ffci]
image: python
variables:
<<: *elastx_variables
@@ -155,10 +160,11 @@ tf-elastx_cleanup:
- pip install -r scripts/openstack-cleanup/requirements.txt
script:
- ./scripts/openstack-cleanup/main.py
+ allow_failure: true
tf-elastx_ubuntu20-calico:
extends: .terraform_apply
- stage: deploy-part3
+ stage: deploy-part1
when: on_success
allow_failure: true
variables:
diff --git a/.gitlab-ci/vagrant.yml b/.gitlab-ci/vagrant.yml
index a02e39835..2d9fc9e3c 100644
--- a/.gitlab-ci/vagrant.yml
+++ b/.gitlab-ci/vagrant.yml
@@ -1,64 +1,73 @@
---
-
.vagrant:
extends: .testcases
+ needs:
+ - ci-not-authorized
variables:
CI_PLATFORM: "vagrant"
SSH_USER: "vagrant"
VAGRANT_DEFAULT_PROVIDER: "libvirt"
KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb
- tags: [c3.small.x86]
- only: [/^pr-.*$/]
- except: ['triggers']
- image: $PIPELINE_IMAGE
+ DOCKER_NAME: vagrant
+ VAGRANT_ANSIBLE_TAGS: facts
+ tags: [ffci-vm-large]
+ # only: [/^pr-.*$/]
+ # except: ['triggers']
+ image: quay.io/kubespray/vm-kubespray-ci:v6
services: []
before_script:
+ - echo $USER
+ - python3 -m venv citest
+ - source citest/bin/activate
+ - vagrant plugin expunge --reinstall --force --no-tty
+ - vagrant plugin install vagrant-libvirt
+ - pip install --no-compile --no-cache-dir pip -U
+ - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt
+ - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
- ./tests/scripts/vagrant_clean.sh
script:
- ./tests/scripts/testcases_run.sh
- after_script:
- - chronic ./tests/scripts/testcases_cleanup.sh
vagrant_ubuntu20-calico-dual-stack:
- stage: deploy-part2
+ stage: deploy-extended
extends: .vagrant
when: manual
# FIXME: this test if broken (perma-failing)
vagrant_ubuntu20-weave-medium:
- stage: deploy-part2
+ stage: deploy-extended
extends: .vagrant
when: manual
vagrant_ubuntu20-flannel:
- stage: deploy-part2
+ stage: deploy-part1
extends: .vagrant
when: on_success
allow_failure: false
vagrant_ubuntu20-flannel-collection:
- stage: deploy-part2
+ stage: deploy-extended
extends: .vagrant
- when: on_success
+ when: manual
vagrant_ubuntu20-kube-router-sep:
- stage: deploy-part2
+ stage: deploy-extended
extends: .vagrant
when: manual
# Service proxy test fails connectivity testing
vagrant_ubuntu20-kube-router-svc-proxy:
- stage: deploy-part2
+ stage: deploy-extended
extends: .vagrant
when: manual
vagrant_fedora37-kube-router:
- stage: deploy-part2
+ stage: deploy-extended
extends: .vagrant
when: manual
# FIXME: this test if broken (perma-failing)
vagrant_centos7-kube-router:
- stage: deploy-part2
+ stage: deploy-extended
extends: .vagrant
when: manual
diff --git a/Vagrantfile b/Vagrantfile
index 4d8caff85..9733105b0 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -278,6 +278,7 @@ Vagrant.configure("2") do |config|
"local_path_provisioner_enabled": "#{$local_path_provisioner_enabled}",
"local_path_provisioner_claim_root": "#{$local_path_provisioner_claim_root}",
"ansible_ssh_user": SUPPORTED_OS[$os][:user],
+ "ansible_ssh_private_key_file": File.join(Dir.home, ".vagrant.d", "insecure_private_key"),
"unsafe_show_logs": "True"
}
diff --git a/docs/developers/ci-setup.md b/docs/developers/ci-setup.md
index a9d380eab..ae52415c7 100644
--- a/docs/developers/ci-setup.md
+++ b/docs/developers/ci-setup.md
@@ -5,8 +5,8 @@
1. build: build a docker image to be used in the pipeline
2. unit-tests: fast jobs for fast feedback (linting, etc...)
3. deploy-part1: small number of jobs to test if the PR works with default settings
-4. deploy-part2: slow jobs testing different platforms, OS, settings, CNI, etc...
-5. deploy-part3: very slow jobs (upgrades, etc...)
+4. deploy-extended: slow jobs testing different platforms, OS, settings, CNI, etc...
+5. deploy-extended: very slow jobs (upgrades, etc...)
## Runners
diff --git a/pipeline.Dockerfile b/pipeline.Dockerfile
index ccc752e79..49d00ae4f 100644
--- a/pipeline.Dockerfile
+++ b/pipeline.Dockerfile
@@ -4,7 +4,7 @@ FROM ubuntu:jammy-20230308
# Pip needs this as well at the moment to install ansible
# (and potentially other packages)
# See: https://github.com/pypa/pip/issues/10219
-ENV VAGRANT_VERSION=2.3.7 \
+ENV VAGRANT_VERSION=2.4.1 \
VAGRANT_DEFAULT_PROVIDER=libvirt \
VAGRANT_ANSIBLE_TAGS=facts \
LANG=C.UTF-8 \
@@ -31,6 +31,8 @@ RUN apt update -q \
unzip \
libvirt-clients \
qemu-utils \
+ qemu-kvm \
+ dnsmasq \
&& curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
&& add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
&& apt update -q \
@@ -46,6 +48,7 @@ ADD ./roles/kubespray-defaults/defaults/main/main.yml /kubespray/roles/kubespray
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
&& pip install --no-compile --no-cache-dir pip -U \
&& pip install --no-compile --no-cache-dir -r tests/requirements.txt \
+ && pip install --no-compile --no-cache-dir -r requirements.txt \
&& KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main/main.yml) \
&& curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
&& echo $(curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
diff --git a/tests/Makefile b/tests/Makefile
index 81d2d9b32..b7d64f19d 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -35,8 +35,7 @@ cleanup-packet:
create-vagrant:
vagrant up
- find / -name vagrant_ansible_inventory
- cp /builds/kargo-ci/kubernetes-sigs-kubespray/inventory/sample/vagrant_ansible_inventory $(INVENTORY)
+ cp $(CI_PROJECT_DIR)/inventory/sample/vagrant_ansible_inventory $(INVENTORY)
delete-vagrant:
vagrant destroy -f
diff --git a/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml b/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml
index 59ef0f8fe..265d54576 100644
--- a/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml
+++ b/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml
@@ -4,14 +4,13 @@
vm_cpu_cores: 2
vm_cpu_sockets: 1
vm_cpu_threads: 2
-vm_memory: 2048Mi
+vm_memory: 2048
# Replace invalid characters so that we can use the branch name in kubernetes labels
branch_name_sane: "{{ branch | regex_replace('/', '-') }}"
# Request/Limit allocation settings
-
-cpu_allocation_ratio: 0.5
+cpu_allocation_ratio: 0.25
memory_allocation_ratio: 1
# Default path for inventory
diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2
index 6a8e02721..371656b73 100644
--- a/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2
+++ b/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2
@@ -4,6 +4,8 @@ kind: VirtualMachine
metadata:
name: "instance-{{ vm_id }}"
namespace: "{{ test_name }}"
+ annotations:
+ kubespray.com/ci.template-path: "tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2"
labels:
kubevirt.io/os: {{ cloud_image }}
spec:
@@ -34,10 +36,10 @@ spec:
threads: {{ vm_cpu_threads }}
resources:
requests:
- memory: {{ vm_memory * memory_allocation_ratio }}
+ memory: "{{ vm_memory * memory_allocation_ratio }}Mi"
cpu: {{ vm_cpu_cores * cpu_allocation_ratio }}
limits:
- memory: {{ vm_memory }}
+ memory: "{{ vm_memory }}Mi"
cpu: {{ vm_cpu_cores }}
networks:
- name: default
diff --git a/tests/files/packet_almalinux8-calico-ha-ebpf.yml b/tests/files/packet_almalinux8-calico-ha-ebpf.yml
index e4f4bb658..21b7d8e01 100644
--- a/tests/files/packet_almalinux8-calico-ha-ebpf.yml
+++ b/tests/files/packet_almalinux8-calico-ha-ebpf.yml
@@ -2,7 +2,7 @@
# Instance settings
cloud_image: almalinux-8
mode: ha
-vm_memory: 3072Mi
+vm_memory: 3072
# Kubespray settings
calico_bpf_enabled: true
diff --git a/tests/files/packet_almalinux8-calico-nodelocaldns-secondary.yml b/tests/files/packet_almalinux8-calico-nodelocaldns-secondary.yml
index 52ef86984..626825863 100644
--- a/tests/files/packet_almalinux8-calico-nodelocaldns-secondary.yml
+++ b/tests/files/packet_almalinux8-calico-nodelocaldns-secondary.yml
@@ -2,7 +2,7 @@
# Instance settings
cloud_image: almalinux-8
mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
# Kubespray settings
enable_nodelocaldns_secondary: true
diff --git a/tests/files/packet_almalinux8-calico.yml b/tests/files/packet_almalinux8-calico.yml
index 648a72d13..37a5deb5d 100644
--- a/tests/files/packet_almalinux8-calico.yml
+++ b/tests/files/packet_almalinux8-calico.yml
@@ -2,7 +2,7 @@
# Instance settings
cloud_image: almalinux-8
mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
# Kubespray settings
metrics_server_enabled: true
diff --git a/tests/files/packet_almalinux8-docker.yml b/tests/files/packet_almalinux8-docker.yml
index bcc69cd53..17917a097 100644
--- a/tests/files/packet_almalinux8-docker.yml
+++ b/tests/files/packet_almalinux8-docker.yml
@@ -2,7 +2,7 @@
# Instance settings
cloud_image: almalinux-8
mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
# Use docker
container_manager: docker
diff --git a/tests/files/packet_almalinux8-kube-ovn.yml b/tests/files/packet_almalinux8-kube-ovn.yml
index 15dbabbd2..1514ccc87 100644
--- a/tests/files/packet_almalinux8-kube-ovn.yml
+++ b/tests/files/packet_almalinux8-kube-ovn.yml
@@ -2,7 +2,7 @@
# Instance settings
cloud_image: almalinux-8
mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
# Kubespray settings
kube_network_plugin: kube-ovn
diff --git a/tests/files/packet_rockylinux8-calico.yml b/tests/files/packet_rockylinux8-calico.yml
index b47511215..e71c0615e 100644
--- a/tests/files/packet_rockylinux8-calico.yml
+++ b/tests/files/packet_rockylinux8-calico.yml
@@ -2,7 +2,7 @@
# Instance settings
cloud_image: rockylinux-8
mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
# Kubespray settings
metrics_server_enabled: true
diff --git a/tests/files/packet_rockylinux9-calico.yml b/tests/files/packet_rockylinux9-calico.yml
index 17e6ae58b..5f3fcc5b0 100644
--- a/tests/files/packet_rockylinux9-calico.yml
+++ b/tests/files/packet_rockylinux9-calico.yml
@@ -2,7 +2,7 @@
# Instance settings
cloud_image: rockylinux-9
mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
# Kubespray settings
metrics_server_enabled: true
diff --git a/tests/files/packet_rockylinux9-cilium.yml b/tests/files/packet_rockylinux9-cilium.yml
index 033fde1b8..2f759f0f5 100644
--- a/tests/files/packet_rockylinux9-cilium.yml
+++ b/tests/files/packet_rockylinux9-cilium.yml
@@ -2,7 +2,7 @@
# Instance settings
cloud_image: rockylinux-9
mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
# Kubespray settings
kube_network_plugin: cilium
diff --git a/tests/files/packet_ubuntu22-all-in-one-docker.yml b/tests/files/packet_ubuntu22-all-in-one-docker.yml
index ddd320a29..16ae45986 100644
--- a/tests/files/packet_ubuntu22-all-in-one-docker.yml
+++ b/tests/files/packet_ubuntu22-all-in-one-docker.yml
@@ -2,7 +2,7 @@
# Instance settings
cloud_image: ubuntu-2204
mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
# Kubespray settings
auto_renew_certificates: true
diff --git a/tests/files/packet_ubuntu22-calico-all-in-one.yml b/tests/files/packet_ubuntu22-calico-all-in-one.yml
index 614cbeabc..2c666f8e3 100644
--- a/tests/files/packet_ubuntu22-calico-all-in-one.yml
+++ b/tests/files/packet_ubuntu22-calico-all-in-one.yml
@@ -2,7 +2,7 @@
# Instance settings
cloud_image: ubuntu-2204
mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
# Kubespray settings
auto_renew_certificates: true
diff --git a/tests/files/packet_ubuntu24-all-in-one-docker.yml b/tests/files/packet_ubuntu24-all-in-one-docker.yml
index b8318d49a..d4a0adccb 100644
--- a/tests/files/packet_ubuntu24-all-in-one-docker.yml
+++ b/tests/files/packet_ubuntu24-all-in-one-docker.yml
@@ -2,7 +2,7 @@
# Instance settings
cloud_image: ubuntu-2404
mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
# Kubespray settings
auto_renew_certificates: true
diff --git a/tests/files/packet_ubuntu24-calico-all-in-one.yml b/tests/files/packet_ubuntu24-calico-all-in-one.yml
index 286034b27..4b9e403ca 100644
--- a/tests/files/packet_ubuntu24-calico-all-in-one.yml
+++ b/tests/files/packet_ubuntu24-calico-all-in-one.yml
@@ -2,7 +2,7 @@
# Instance settings
cloud_image: ubuntu-2404
mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
# Kubespray settings
auto_renew_certificates: true
diff --git a/tests/files/packet_ubuntu24-calico-etcd-datastore.yml b/tests/files/packet_ubuntu24-calico-etcd-datastore.yml
index ec7747de2..2805fa731 100644
--- a/tests/files/packet_ubuntu24-calico-etcd-datastore.yml
+++ b/tests/files/packet_ubuntu24-calico-etcd-datastore.yml
@@ -2,7 +2,7 @@
# Instance settings
cloud_image: ubuntu-2404
mode: node-etcd-client
-vm_memory: 1600Mi
+vm_memory: 1600
# Kubespray settings
auto_renew_certificates: true
diff --git a/tests/files/vagrant_ubuntu20-flannel.rb b/tests/files/vagrant_ubuntu20-flannel.rb
index 55daa19e3..d8bf563a3 100644
--- a/tests/files/vagrant_ubuntu20-flannel.rb
+++ b/tests/files/vagrant_ubuntu20-flannel.rb
@@ -2,7 +2,6 @@ $os = "ubuntu2004"
# For CI we are not worries about data persistence across reboot
$libvirt_volume_cache = "unsafe"
-
# Checking for box update can trigger API rate limiting
# https://www.vagrantup.com/docs/vagrant-cloud/request-limits.html
$box_check_update = false
diff --git a/tests/files/vagrant_ubuntu20-flannel.yml b/tests/files/vagrant_ubuntu20-flannel.yml
index 6f8916feb..43ec54c6a 100644
--- a/tests/files/vagrant_ubuntu20-flannel.yml
+++ b/tests/files/vagrant_ubuntu20-flannel.yml
@@ -1,3 +1,4 @@
---
# Kubespray settings
kube_network_plugin: flannel
+ansible_ssh_private_key: .vagrant.d/insecure_private_key
diff --git a/tests/scripts/vagrant-install.sh b/tests/scripts/vagrant-install.sh
new file mode 100755
index 000000000..a2d1f0a3c
--- /dev/null
+++ b/tests/scripts/vagrant-install.sh
@@ -0,0 +1,122 @@
+#!/bin/bash
+
+# install_vagrant() {
+# sudo apt install vagrant-libvirt vagrant -y
+# sudo vagrant plugin install vagrant-libvirt
+# }
+
+# prep(){
+# sudo apt-get update -y
+# sudo apt-get install ca-certificates curl libvirt-daemon-system\
+# libvirt-clients qemu-utils qemu-kvm htop atop -y
+
+# sudo install -m 0755 -d /etc/apt/keyrings
+# }
+# install_docker() {
+# VERSION_STRING=5:26.1.0-1~ubuntu.24.04~noble
+# sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+# sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+# # Add the repository to Apt sources:
+# echo \
+# "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+# $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+# sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+# sudo apt-get update -y
+
+# sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y
+# }
+# install_docker_auto () {
+# curl -fsSL https://get.docker.com -o get-docker.sh
+# sudo sh ./get-docker.sh --dry-run
+# }
+
+
+
+VAGRANT_VERSION=2.4.1
+VAGRANT_DEFAULT_PROVIDER=libvirt
+VAGRANT_ANSIBLE_TAGS=facts
+LANG=C.UTF-8
+DEBIAN_FRONTEND=noninteractive
+PYTHONDONTWRITEBYTECODE=1
+KUBE_VERSION=1.29.5
+pipeline_install() {
+ cp /etc/apt/sources.list /etc/apt/sources.list."$(date +"%F")"
+ sed -i -e '/^# deb-src.*universe$/s/# //g' /etc/apt/sources.list
+ sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources
+
+ apt update
+ # libssl-dev \
+ # python3-dev \
+ # # jq \
+ # moreutils \
+ # libvirt-dev \
+ # # rsync \
+ # git \
+ # # htop \
+ # gpg \
+ # atop
+
+ # gnupg2 \
+# software-properties-common
+#
+ apt install --no-install-recommends -y \
+ git \
+ make \
+ python3-pip \
+ sshpass \
+ apt-transport-https \
+ openssh-client \
+ ca-certificates \
+ curl \
+ libfuse2 \
+ unzip \
+ qemu-utils \
+ libvirt-daemon-system \
+ libvirt-clients \
+ qemu-kvm \
+ ebtables libguestfs-tools \
+ ruby-fog-libvirt \
+ libvirt-dev \
+ gcc \
+ build-essential \
+ ruby-libvirt \
+ libxslt-dev libxml2-dev zlib1g-dev \
+ python3-venv python3-full \
+ dnsmasq
+
+ apt-get build-dep -y ruby-libvirt ruby-dev
+ ### VAGRANT ###
+ # apt-get install -y unzip
+ curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_linux_amd64.zip
+ unzip vagrant_${VAGRANT_VERSION}_linux_amd64.zip
+ mv vagrant /usr/local/bin/vagrant
+ chmod a+x /usr/local/bin/vagrant
+ # ls -la /usr/local/bin/vagrant
+ /usr/local/bin/vagrant plugin install vagrant-libvirt
+ usermod -aG kvm kubespray
+ usermod -aG libvirt kubespray
+
+ ### DOCKER ###
+ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+ add-apt-repository -y "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+ apt update
+ apt install --no-install-recommends -y docker-ce
+ apt autoremove -y --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/*
+
+ ### KUBECTL ###
+ curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+ mv kubectl /usr/local/bin/kubectl
+ chmod a+x /usr/local/bin/kubectl
+ systemctl restart libvirtd
+ # Install Vagrant
+ # apt update -y
+ # echo apt-get install -y unzip libfuse2 vagrant vagrant-libvirt
+ # apt --fix-broken install -y
+ # dpkg --configure -a -y
+
+
+}
+# wrapped up in a function so that we have some protection against only getting
+# half the file during "curl | sh"
+pipeline_install
--
GitLab