From 5df48ef8fd86fc953a0f08fae0b11038375d8836 Mon Sep 17 00:00:00 2001
From: Maxime Guyot <Miouge1@users.noreply.github.com>
Date: Tue, 17 Dec 2019 16:07:59 +0100
Subject: [PATCH] [docs] Add CI matrix and script (#5461)

* Rename CI jobs from ubuntu to ubuntu16

* Add CI matrix and script
---
 .gitlab-ci/packet.yml                         | 16 +--
 docs/_sidebar.md                              |  1 +
 docs/ci.md                                    | 39 ++++++++
 ...l => packet_centos7-weave-kubeadm-sep.yml} |  0
 ...e-7-canal.yml => packet_oracle7-canal.yml} |  0
 ...al-ha.yml => packet_ubuntu16-canal-ha.yml} |  0
 ....yml => packet_ubuntu16-canal-kubeadm.yml} |  0
 ...sep.yml => packet_ubuntu16-contiv-sep.yml} |  0
 ...-ha.yml => packet_ubuntu16-flannel-ha.yml} |  0
 ...ml => packet_ubuntu16-kube-router-sep.yml} |  0
 ...-sep.yml => packet_ubuntu16-weave-sep.yml} |  0
 tests/scripts/md-table/main.py                | 97 +++++++++++++++++++
 tests/scripts/md-table/requirements.txt       |  4 +
 tests/scripts/md-table/table.md.j2            | 15 +++
 tests/scripts/md-table/test.sh                | 11 +++
 tests/scripts/md-table/tmp.md                 | 30 ++++++
 16 files changed, 205 insertions(+), 8 deletions(-)
 create mode 100644 docs/ci.md
 rename tests/files/{packet_centos-weave-kubeadm-sep.yml => packet_centos7-weave-kubeadm-sep.yml} (100%)
 rename tests/files/{packet_oracle-7-canal.yml => packet_oracle7-canal.yml} (100%)
 rename tests/files/{packet_ubuntu-canal-ha.yml => packet_ubuntu16-canal-ha.yml} (100%)
 rename tests/files/{packet_ubuntu-canal-kubeadm.yml => packet_ubuntu16-canal-kubeadm.yml} (100%)
 rename tests/files/{packet_ubuntu-contiv-sep.yml => packet_ubuntu16-contiv-sep.yml} (100%)
 rename tests/files/{packet_ubuntu-flannel-ha.yml => packet_ubuntu16-flannel-ha.yml} (100%)
 rename tests/files/{packet_ubuntu-kube-router-sep.yml => packet_ubuntu16-kube-router-sep.yml} (100%)
 rename tests/files/{packet_ubuntu-weave-sep.yml => packet_ubuntu16-weave-sep.yml} (100%)
 create mode 100755 tests/scripts/md-table/main.py
 create mode 100644 tests/scripts/md-table/requirements.txt
 create mode 100644 tests/scripts/md-table/table.md.j2
 create mode 100755 tests/scripts/md-table/test.sh
 create mode 100644 tests/scripts/md-table/tmp.md

diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml
index 92440ac70..9aa398ee3 100644
--- a/.gitlab-ci/packet.yml
+++ b/.gitlab-ci/packet.yml
@@ -23,37 +23,37 @@ packet_centos7-flannel-addons:
 
 # ### MANUAL JOBS
 
-packet_centos-weave-kubeadm-sep:
+packet_centos7-weave-kubeadm-sep:
   stage: deploy-part2
   extends: .packet
   when: on_success
   variables:
     UPGRADE_TEST: basic
 
-packet_ubuntu-weave-sep:
+packet_ubuntu16-weave-sep:
   stage: deploy-part2
   extends: .packet
   when: manual
 
 # # More builds for PRs/merges (manual) and triggers (auto)
 
-packet_ubuntu-canal-ha:
+packet_ubuntu16-canal-ha:
   stage: deploy-special
   extends: .packet
   when: manual
 
-packet_ubuntu-canal-kubeadm:
+packet_ubuntu16-canal-kubeadm:
   stage: deploy-part2
   extends: .packet
   when: on_success
 
-packet_ubuntu-flannel-ha:
+packet_ubuntu16-flannel-ha:
   stage: deploy-part2
   extends: .packet
   when: manual
 
 # Contiv does not work in k8s v1.16
-# packet_ubuntu-contiv-sep:
+# packet_ubuntu16-contiv-sep:
 #   stage: deploy-part2
 #   extends: .packet
 #   when: on_success
@@ -110,12 +110,12 @@ packet_opensuse-canal:
   extends: .packet
   when: manual
 
-packet_oracle-7-canal:
+packet_oracle7-canal:
   stage: deploy-part2
   extends: .packet
   when: manual
 
-packet_ubuntu-kube-router-sep:
+packet_ubuntu16-kube-router-sep:
   stage: deploy-part2
   extends: .packet
   when: manual
diff --git a/docs/_sidebar.md b/docs/_sidebar.md
index 34121a7a0..f3b727e6f 100644
--- a/docs/_sidebar.md
+++ b/docs/_sidebar.md
@@ -36,4 +36,5 @@
 * Developers
   * [Test cases](docs/test_cases.md)
   * [Vagrant](docs/vagrant.md)
+  * [CI Matrix](docs/ci.md)
 * [Roadmap](docs/roadmap.md)
diff --git a/docs/ci.md b/docs/ci.md
new file mode 100644
index 000000000..4e9c9fab2
--- /dev/null
+++ b/docs/ci.md
@@ -0,0 +1,39 @@
+# CI test coverage
+
+To generate this Matrix run `./tests/scripts/md-table/main.py`
+
+## docker
+
+| OS / CNI | calico | canal | cilium | contiv | flannel | kube-ovn | kube-router | macvlan | weave |
+|---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
+centos7 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: |
+coreos |  :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :white_check_mark: |
+debian10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
+opensuse |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+oracle |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+oracle7 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rhel7 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
+ubuntu |  :x: | :white_check_mark: | :x: | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: |
+ubuntu16 |  :x: | :white_check_mark: | :x: | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: |
+ubuntu18 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
+
+## containerd
+
+| OS / CNI | calico | canal | cilium | contiv | flannel | kube-ovn | kube-router | macvlan | weave |
+|---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos7 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
+coreos |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian10 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+oracle |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rhel7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu18 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
diff --git a/tests/files/packet_centos-weave-kubeadm-sep.yml b/tests/files/packet_centos7-weave-kubeadm-sep.yml
similarity index 100%
rename from tests/files/packet_centos-weave-kubeadm-sep.yml
rename to tests/files/packet_centos7-weave-kubeadm-sep.yml
diff --git a/tests/files/packet_oracle-7-canal.yml b/tests/files/packet_oracle7-canal.yml
similarity index 100%
rename from tests/files/packet_oracle-7-canal.yml
rename to tests/files/packet_oracle7-canal.yml
diff --git a/tests/files/packet_ubuntu-canal-ha.yml b/tests/files/packet_ubuntu16-canal-ha.yml
similarity index 100%
rename from tests/files/packet_ubuntu-canal-ha.yml
rename to tests/files/packet_ubuntu16-canal-ha.yml
diff --git a/tests/files/packet_ubuntu-canal-kubeadm.yml b/tests/files/packet_ubuntu16-canal-kubeadm.yml
similarity index 100%
rename from tests/files/packet_ubuntu-canal-kubeadm.yml
rename to tests/files/packet_ubuntu16-canal-kubeadm.yml
diff --git a/tests/files/packet_ubuntu-contiv-sep.yml b/tests/files/packet_ubuntu16-contiv-sep.yml
similarity index 100%
rename from tests/files/packet_ubuntu-contiv-sep.yml
rename to tests/files/packet_ubuntu16-contiv-sep.yml
diff --git a/tests/files/packet_ubuntu-flannel-ha.yml b/tests/files/packet_ubuntu16-flannel-ha.yml
similarity index 100%
rename from tests/files/packet_ubuntu-flannel-ha.yml
rename to tests/files/packet_ubuntu16-flannel-ha.yml
diff --git a/tests/files/packet_ubuntu-kube-router-sep.yml b/tests/files/packet_ubuntu16-kube-router-sep.yml
similarity index 100%
rename from tests/files/packet_ubuntu-kube-router-sep.yml
rename to tests/files/packet_ubuntu16-kube-router-sep.yml
diff --git a/tests/files/packet_ubuntu-weave-sep.yml b/tests/files/packet_ubuntu16-weave-sep.yml
similarity index 100%
rename from tests/files/packet_ubuntu-weave-sep.yml
rename to tests/files/packet_ubuntu16-weave-sep.yml
diff --git a/tests/scripts/md-table/main.py b/tests/scripts/md-table/main.py
new file mode 100755
index 000000000..72cd12b8c
--- /dev/null
+++ b/tests/scripts/md-table/main.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+import argparse
+import sys
+import glob
+from pathlib import Path
+import yaml
+from pydblite import Base
+import re
+import jinja2
+import sys
+
+from pprint import pprint
+
+
+parser = argparse.ArgumentParser(description='Generate a Markdown table representing the CI test coverage')
+parser.add_argument('--dir', default='tests/files/', help='folder with test yml files')
+
+
+args = parser.parse_args()
+p = Path(args.dir)
+
+env = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath=sys.path[0]))
+
+# Data represents CI coverage data matrix
+class Data:
+    def __init__(self):
+        self.db = Base(':memory:')
+        self.db.create('container_manager', 'network_plugin', 'operating_system')
+
+    
+    def set(self, container_manager, network_plugin, operating_system):
+        self.db.insert(container_manager=container_manager, network_plugin=network_plugin, operating_system=operating_system)
+        self.db.commit()
+    def exists(self, container_manager, network_plugin, operating_system):
+        return len((self.db("container_manager") == container_manager) & (self.db("network_plugin") == network_plugin) & (self.db("operating_system") == operating_system)) > 0
+
+    def jinja(self):
+        template = env.get_template('table.md.j2')
+        container_engines = list(self.db.get_unique_ids('container_manager'))
+        network_plugins = list(self.db.get_unique_ids("network_plugin"))
+        operating_systems = list(self.db.get_unique_ids("operating_system"))
+
+        container_engines.sort()
+        container_engines.reverse() # reverse sort container_engines to get Docker first in the list
+        network_plugins.sort()
+        operating_systems.sort()
+
+        return template.render(
+            container_engines=container_engines,
+            network_plugins=network_plugins,
+            operating_systems=operating_systems,
+            exists=self.exists
+        )
+
+    def markdown(self):
+        out = ''
+        for container_manager in self.db.get_unique_ids('container_manager'):
+            # Prepare the headers
+            out += "# " + container_manager + "\n"
+            headers = '|OS / CNI| '
+            underline = '|----|'
+            for network_plugin in self.db.get_unique_ids("network_plugin"):
+                headers += network_plugin + ' | '
+                underline += '----|'
+            out += headers + "\n" + underline + "\n"
+            for operating_system in self.db.get_unique_ids("operating_system"):
+                out += '| ' + operating_system + ' | '
+                for network_plugin in self.db.get_unique_ids("network_plugin"):
+                    if self.exists(container_manager, network_plugin, operating_system):
+                        emoji = ':white_check_mark:'
+                    else:
+                        emoji = ':x:'
+                    out += emoji + ' | '
+                out += "\n"
+
+        pprint(self.db.get_unique_ids('operating_system'))
+        pprint(self.db.get_unique_ids('network_plugin'))
+        return out
+
+
+
+if not p.is_dir():
+    print("Path is not a directory")
+    sys.exit(2)
+
+data = Data()
+files = p.glob('*.yml')
+for f in files:
+    y = yaml.load(f.open(), Loader=yaml.FullLoader)
+
+    container_manager = y.get('container_manager', 'docker')
+    network_plugin = y.get('kube_network_plugin', 'calico')
+    x = re.match(r"^[a-z-]+_([a-z0-9]+).*", f.name)
+    operating_system = x.group(1)
+    data.set(container_manager=container_manager, network_plugin=network_plugin, operating_system=operating_system)
+#print(data.markdown())
+print(data.jinja())
\ No newline at end of file
diff --git a/tests/scripts/md-table/requirements.txt b/tests/scripts/md-table/requirements.txt
new file mode 100644
index 000000000..754da38d8
--- /dev/null
+++ b/tests/scripts/md-table/requirements.txt
@@ -0,0 +1,4 @@
+pyaml
+jinja2
+pathlib
+pydblite
\ No newline at end of file
diff --git a/tests/scripts/md-table/table.md.j2 b/tests/scripts/md-table/table.md.j2
new file mode 100644
index 000000000..7e8f4cae1
--- /dev/null
+++ b/tests/scripts/md-table/table.md.j2
@@ -0,0 +1,15 @@
+# CI test coverage
+
+To generate this Matrix run `./tests/scripts/md-table/main.py` 
+
+{%- for container_engine in container_engines %}
+
+## {{ container_engine }}
+
+| OS / CNI |{% for cni in network_plugins %} {{ cni }} |{% endfor %}
+|---|{% for cni in network_plugins %} --- |{% endfor %}
+{%- for os in operating_systems %}
+{{ os }} | {% for cni in network_plugins %} {{ ':white_check_mark:' if exists(container_engine, cni, os) else ':x:' }} |{% endfor %}
+{%- endfor %}
+
+{%- endfor %}
\ No newline at end of file
diff --git a/tests/scripts/md-table/test.sh b/tests/scripts/md-table/test.sh
new file mode 100755
index 000000000..a67ad895f
--- /dev/null
+++ b/tests/scripts/md-table/test.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+set -euxo pipefail
+
+echo "Install requirements..."
+pip install -r ./tests/scripts/md-table/requirements.txt
+
+echo "Generate current file..."
+./tests/scripts/md-table/main.py > tmp.md
+
+echo "Compare docs/ci.md with actual tests in tests/files/*.yml ..."
+cmp docs/ci.md tmp.md
\ No newline at end of file
diff --git a/tests/scripts/md-table/tmp.md b/tests/scripts/md-table/tmp.md
new file mode 100644
index 000000000..81bee7bb2
--- /dev/null
+++ b/tests/scripts/md-table/tmp.md
@@ -0,0 +1,30 @@
+# CI test coverage
+
+## docker
+
+| OS / CNI | canal | cilium | weave | contiv | kube-router | calico | macvlan | flannel | kube-ovn |
+|---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+amazon |  :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
+centos7 |  :x: | :x: | :white_check_mark: | :x: | :white_check_mark: | :white_check_mark: | :x: | :x: | :white_check_mark: |
+coreos |  :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
+ubuntu18 |  :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
+debian9 |  :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :white_check_mark: | :x: | :x: |
+opensuse |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu16 |  :white_check_mark: | :x: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :white_check_mark: | :x: |
+oracle7 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+
+## containerd
+
+| OS / CNI | canal | cilium | weave | contiv | kube-router | calico | macvlan | flannel | kube-ovn |
+|---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
+coreos |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu18 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
+debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian10 |  :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
+--
-- 
GitLab