Add complete test integration

.travis.yml

-sudo: required
-dist: trusty
-language: python
-python: "2.7"
+sudo: false
 
-addons:
-  hosts:
-    - node1
+git:
+  depth: 5
 
 env:
-  - SITE=cluster.yml ANSIBLE_VERSION=2.0.0
+  global:
+    GCE_USER=travis
+    SSH_USER=$GCE_USER
+    TEST_ID=$TRAVIS_JOB_NUMBER
+    CONTAINER_ENGINE=docker
+    PRIVATE_KEY=$GCE_PRIVATE_KEY
+    ANSIBLE_KEEP_REMOTE_FILES=1
+  matrix:
+    # Debian Jessie
+    - >-
+      KUBE_NETWORK_PLUGIN=flannel
+      CLOUD_IMAGE=debian-8
+      CLOUD_REGION=europe-west1-b
+    - >-
+      KUBE_NETWORK_PLUGIN=calico
+      CLOUD_IMAGE=debian-8
+      CLOUD_REGION=europe-west1-b
 
-install:
+    # Centos 7
+    - >-
+      KUBE_NETWORK_PLUGIN=flannel
+      CLOUD_IMAGE=centos-7-sudo
+      CLOUD_REGION=us-central1-c
+
+    # - >-
+    #   KUBE_NETWORK_PLUGIN=calico
+    #   CLOUD_IMAGE=centos-7-sudo
+    #   CLOUD_REGION=us-central1-c
+
+
+   # # Redhat 7
+   #  - >-
+   #    KUBE_NETWORK_PLUGIN=flannel
+   #    CLOUD_IMAGE=rhel-7-sudo
+   #    CLOUD_REGION=us-east1-d
+   #    SSH_ARGS='--ssh-extra-args="-t"'
+   #  - >-
+   #    KUBE_NETWORK_PLUGIN=calico
+   #    CLOUD_IMAGE=rhel-7-sudo
+   #    CLOUD_REGION=us-east1-d
+
+    # Ubuntu 14.04
+    - >-
+      KUBE_NETWORK_PLUGIN=flannel
+      CLOUD_IMAGE=ubuntu-1404-trusty
+      CLOUD_REGION=europe-west1-c
+    - >-
+      KUBE_NETWORK_PLUGIN=calico
+      CLOUD_IMAGE=ubuntu-1404-trusty
+      CLOUD_REGION=europe-west1-c
+
+    # Ubuntu 15.10
+    - >-
+      KUBE_NETWORK_PLUGIN=flannel
+      CLOUD_IMAGE=ubuntu-1510-wily
+      CLOUD_REGION=us-central1-a
+    - >-
+      KUBE_NETWORK_PLUGIN=calico
+      CLOUD_IMAGE=ubuntu-1510-wily
+      CLOUD_REGION=us-central1-a
+
+
+matrix:
+  allow_failures:
+    - env: KUBE_NETWORK_PLUGIN=flannel CLOUD_IMAGE=centos-7-sudo CLOUD_REGION=us-central1-c
+
+before_install:
   # Install Ansible.
-  - sudo -H pip install ansible==${ANSIBLE_VERSION}
-  - sudo -H pip install netaddr
+  - pip install --user boto -U
+  - pip install --user ansible
+  - pip install --user netaddr
+  - pip install --user apache-libcloud
 
 cache:
-  directories:
-    - $HOME/releases
+  - directories:
     - $HOME/.cache/pip
+    - $HOME/.local
 
 before_script:
-  - export PATH=$PATH:/usr/local/bin
+  - echo "RUN $TRAVIS_JOB_NUMBER $KUBE_NETWORK_PLUGIN $CONTAINER_ENGINE "
+  - mkdir -p $HOME/.ssh
+  - echo $PRIVATE_KEY | base64 -d > $HOME/.ssh/id_rsa
+  - echo $GCE_PEM_FILE | base64 -d > $HOME/.ssh/gce
+  - chmod 400 $HOME/.ssh/id_rsa
+  - chmod 755 $HOME/.local/bin/ansible-playbook
+  - $HOME/.local/bin/ansible-playbook --version
+  - cp tests/ansible.cfg .
+#  - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}'  $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml"
 
 script:
-  # Check the role/playbook's syntax.
-  - "sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --syntax-check"
+  - >
+    $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/create-gce.yml -i tests/local_inventory/hosts -c local $LOG_LEVEL
+    -e test_id=${TEST_ID}
+    -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+    -e gce_project_id=${GCE_PROJECT_ID}
+    -e gce_service_account_email=${GCE_ACCOUNT}
+    -e gce_pem_file=${HOME}/.ssh/gce
+    -e cloud_image=${CLOUD_IMAGE}
+    -e inventory_path=${PWD}/inventory/inventory.ini
+    -e cloud_region=${CLOUD_REGION}
 
-  # Run the role/playbook with ansible-playbook.
-  - "sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --connection=local"
+    # Create cluster
+  - "$HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}'  $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} cluster.yml"
+    # Tests Cases
+    ## Test Master API
+  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/testcases/010_check-apiserver.yml $LOG_LEVEL
+    ## Create a POD
+  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL
+    ## Ping the between 2 pod
+  - $HOME/.local/bin/ansible-playbook -i setup-kubernetes/inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
 
-  # Run the role/playbook again, checking to make sure it's idempotent.
+after_script:
   - >
-    sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --connection=local
-    | tee /dev/stderr | grep -q 'changed=0.*failed=0'
-    && (echo 'Idempotence test: pass' && exit 0)
-    || (echo 'Idempotence test: fail' && exit 1)
+    $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local  $LOG_LEVEL
+    -e test_id=${TEST_ID}
+    -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+    -e gce_project_id=${GCE_PROJECT_ID}
+    -e gce_service_account_email=${GCE_ACCOUNT}
+    -e gce_pem_file=${HOME}/.ssh/gce
+    -e cloud_image=${CLOUD_IMAGE}
+    -e inventory_path=${PWD}/inventory/inventory.ini
+    -e cloud_region=${CLOUD_REGION}

ansible.cfg

+[ssh_connection]
+pipelining=True
+[defaults] 
+host_key_checking=False

tests/README.md

+# k8s-integration-tests
+
+*Work In Progress*
+
+## Test environment variables
+
+### Common
+
+Variable              | Description                            | Required   | Default
+--------------------- | -------------------------------------- | ---------- | --------
+`TEST_ID`             | A unique execution ID for this test    | Yes        |
+`KUBE_NETWORK_PLUGIN` | The network plugin (calico or flannel) | Yes        |
+`PRIVATE_KEY_FILE`    | The path to the SSH private key file   | No         |
+
+### AWS Tests
+
+Variable              | Description                                     | Required   | Default
+--------------------- | ----------------------------------------------- | ---------- | ---------
+`AWS_ACCESS_KEY`      | The Amazon Access Key ID                        | Yes        |
+`AWS_SECRET_KEY`      | The Amazon Secret Access Key                    | Yes        |
+`AWS_AMI_ID`          | The AMI ID to deploy                            | Yes        |
+`AWS_KEY_PAIR_NAME`   | The name of the EC2 key pair to use             | Yes        |
+`AWS_SECURITY_GROUP`  | The EC2 Security Group to use                   | No         | default
+`AWS_REGION`          | The EC2 region                                  | No         | eu-central-1
+
+#### Use private ssh key
+
+##### Key
+
+```bash
+openssl pkcs12 -in gce-secure.p12 -passin pass:notasecret -nodes -nocerts | openssl rsa -out gce-secure.pem
+cat gce-secure.pem |base64 -w0 > GCE_PEM_FILE`
+```

tests/ansible.cfg

+[ssh_connection]
+pipelining=True
+[defaults] 
+host_key_checking=False

tests/cloud_playbooks/create-aws.yml

+---
+- hosts: localhost
+  sudo: False
+  gather_facts: False
+
+  tasks:
+  - name: Provision a set of instances
+    ec2:
+      key_name: "{{ aws.key_name }}"
+      aws_access_key: "{{ aws.access_key }}"
+      aws_secret_key: "{{ aws.secret_key }}"
+      region: "{{ aws.region }}"
+      group_id: "{{ aws.group}}"
+      instance_type: "{{ aws.instance_type}}"
+      image: "{{ aws.ami_id }}"
+      wait: true
+      count: "{{ aws.count }}"
+      instance_tags: "{{ aws.tags }}"
+    register: ec2
+
+  - name: Template the inventory
+    template:
+      src: templates/inventory.ini.j2
+      dest: "{{ inventory_path }}"
+
+  - name: Wait until SSH is available
+    local_action:
+      module: wait_for
+      host: "{{ item.public_ip }}"
+      port: 22
+      timeout: 300
+      state: started
+    with_items: ec2.instances

tests/cloud_playbooks/create-gce.yml

+---
+- hosts: localhost
+  sudo: False
+  gather_facts: no
+  vars:
+    cloud_machine_type: g1-small
+
+  tasks:
+    - name: replace_test_id
+      set_fact:
+        test_name: "{{test_id |regex_replace('\\.', '-')}}"
+
+    - name: Create gce instances
+      gce:
+        instance_names: "k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3"
+        machine_type: "{{ cloud_machine_type }}"
+        image: "{{ cloud_image }}"
+        service_account_email: "{{ gce_service_account_email }}"
+        pem_file: "{{ gce_pem_file }}"
+        project_id: "{{ gce_project_id }}"
+        zone: "{{cloud_region}}"
+        metadata: '{"test_id": "{{test_id}}", "network": "{{kube_network_plugin}}"}'
+        tags: "build-{{test_name}},{{kube_network_plugin}}"
+      register: gce
+
+    - name: Template the inventory
+      template:
+        src: ../templates/inventory-gce.j2
+        dest: "{{ inventory_path }}"
+
+    - name: Wait for SSH to come up
+      wait_for: host={{item.public_ip}} port=22 delay=10 timeout=180 state=started
+      with_items: gce.instance_data

tests/cloud_playbooks/delete-aws.yml

+---
+- hosts: kube-node
+  sudo: False
+
+  tasks:
+  - name: Gather EC2 facts
+    action: ec2_facts
+
+  - name: Terminate EC2 instances
+    local_action:
+      module: ec2
+      state: absent
+      instance_ids: "{{ ansible_ec2_instance_id }}"
+      region: "{{ ansible_ec2_placement_region }}"
+      wait: True

tests/cloud_playbooks/delete-gce.yml

+---
+- hosts: localhost
+  sudo: False
+  gather_facts: no
+  vars:
+    cloud_machine_type: f1-micro
+
+  tasks:
+    - name: replace_test_id
+      set_fact:
+        test_name: "{{test_id |regex_replace('\\.', '-')}}"
+
+    - name: delete gce instances
+      gce:
+        instance_names: "k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3"
+        machine_type: "{{ cloud_machine_type }}"
+        image: "{{ cloud_image }}"
+        service_account_email: "{{ gce_service_account_email }}"
+        pem_file: "{{ gce_pem_file }}"
+        project_id: "{{ gce_project_id }}"
+        zone: "{{cloud_region | default('europe-west1-b')}}"
+        metadata: '{"test_id": "{{test_id}}", "network": "{{kube_network_plugin}}"}'
+        state: 'absent'
+      register: gce

tests/local_inventory/host_vars/localhost

+aws:
+  key_name: "{{ key_name | default('ansibl8s') }}"
+  access_key: "{{ aws_access_key }}"
+  secret_key: "{{ aws_secret_key }}"
+  region: "{{ aws_region | default('eu-west-1') }}" # default to eu-west-1
+  group: "{{ aws_security_group | default ('default')}}"
+  instance_type: t2.micro
+  ami_id: "{{ aws_ami_id | default('ami-02724d1f') }}" # default to Debian Jessie
+  count: 3
+  tags:
+    test_id: "{{ test_id }}"
+    network_plugin: "{{ kube_network_plugin }}"

tests/local_inventory/hosts.cfg

+localhost ansible_connection=localx
\ No newline at end of file

tests/run-tests.sh

+#! /bin/bash
+
+# curl -# -C - -o shebang-unit https://raw.github.com/arpinum-oss/shebang-unit/master/releases/shebang-unit
+# chmod +x shebang-unit
+
+now=$(date +"%Y%m%d%H%M%S")
+mkdir -p ${PWD}/tests-results
+./shebang-unit --reporters=simple,junit --output-file=${PWD}/tests-results/junit_report-${now}.xml tests

tests/scripts/ansibl8s_test.sh

+#! /bin/bash
+
+global_setup() {
+  git clone https://github.com/ansibl8s/setup-kubernetes.git setup-kubernetes
+  private_key=""
+  if [ ! -z ${PRIVATE_KEY_FILE} ]
+  then
+    private_key="--private-key=${PRIVATE_KEY_FILE}"
+  fi
+  ansible-playbook create.yml -i hosts -u admin -s \
+      -e test_id=${TEST_ID} \
+      -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} \
+      -e aws_access_key=${AWS_ACCESS_KEY} \
+      -e aws_secret_key=${AWS_SECRET_KEY} \
+      -e aws_ami_id=${AWS_AMI_ID} \
+      -e aws_security_group=${AWS_SECURITY_GROUP} \
+      -e key_name=${AWS_KEY_PAIR_NAME} \
+      -e inventory_path=${PWD}/inventory.ini \
+      -e aws_region=${AWS_REGION}
+}
+
+global_teardown() {
+  if [ -f inventory.ini ];
+  then
+    ansible-playbook -i inventory.ini -u admin delete.yml
+  fi
+  rm -rf ${PWD}/setup-kubernetes
+}
+
+should_deploy_cluster() {
+  ansible-playbook -i inventory.ini -s ${private_key} -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml
+
+  assertion__status_code_is_success $?
+}
+
+should_api_server_respond() {
+  ansible-playbook -i inventory.ini ${private_key} testcases/check-apiserver.yml
+
+  assertion__status_code_is_success $?
+}
+
+should_create_pod() {
+  ansible-playbook -i inventory.ini -s ${private_key} testcases/check-create-pod.yml -vv
+
+  assertion__status_code_is_success $?
+}
+
+should_pod_be_in_expected_subnet() {
+  ansible-playbook -i inventory.ini -s ${private_key} testcases/check-network.yml -vv
+
+  assertion__status_code_is_success $?
+}

tests/templates/inventory-gce.j2

+node1 ansible_ssh_host={{gce.instance_data[0].public_ip}}
+node2 ansible_ssh_host={{gce.instance_data[1].public_ip}}
+node3 ansible_ssh_host={{gce.instance_data[2].public_ip}}
+
+[kube-master]
+node1
+node2
+
+[kube-node]
+node1
+node2
+node3
+
+[etcd]
+node1
+node2
+
+[k8s-cluster:children]
+kube-node
+kube-master

tests/templates/inventory.ini.j2

+node1 ansible_ssh_host={{ec2.instances[0].public_ip}} ansible_ssh_user=admin
+node2 ansible_ssh_host={{ec2.instances[1].public_ip}} ansible_ssh_user=admin
+node3 ansible_ssh_host={{ec2.instances[2].public_ip}} ansible_ssh_user=admin
+
+[kube-master]
+node1
+node2
+
+[kube-node]
+node1
+node2
+node3
+
+[etcd]
+node1
+node2
+
+[k8s-cluster:children]
+kube-node
+kube-master

tests/testcases/010_check-apiserver.yml

+---
+- hosts: kube-master
+
+  tasks:
+  - name: Check the API servers are responding
+    local_action:
+      module: uri
+      url: https://{{ansible_ssh_host}}/api/v1
+      user: kube
+      password: changeme
+      validate_certs: no
+      status_code: 200

tests/testcases/020_check-create-pod.yml

+---
+- hosts: node1
+
+  tasks:
+  - name: Run a replica controller composed of 2 pods
+    shell: "kubectl run test --image=busybox --replicas=2 --command -- tail -f /dev/null"
+
+  - name: Pods are running
+    shell: "kubectl get pods --no-headers -o json"
+    register: run_pods_log
+    until: (run_pods_log.stdout | from_json)['items'] | map(attribute = 'status.phase') | join(',') == "Running,Running"
+    retries: 24
+    delay: 5

tests/testcases/030_check-network.yml

+---
+- hosts: node1
+
+  tasks:
+
+  - name: Get pod names
+    shell: "kubectl get pods -o json"
+    register: pods
+
+  - set_fact:
+      pod_names: "{{ (pods.stdout | from_json)['items'] | map(attribute = 'metadata.name') | list }}"
+      pod_ips: "{{ (pods.stdout | from_json)['items'] | map(attribute = 'status.podIP') | list }}"
+
+  - name: Check pods IP are in correct network
+    assert:
+      that: item | ipaddr(kube_pods_subnet)
+    with_items: pod_ips
+
+
+  - name: Ping between pods is working
+    shell: "kubectl exec {{pod_names[0]}} -- ping -c 4 {{ pod_ips[1] }}"