diff --git a/.gitignore b/.gitignore index e7bbe0bea718db25503b5c828c6df731b115718e..f4c7d990a0695d187797ea51f4e87eb69d5d7cd5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ temp .idea .tox .cache +*.bak *.egg-info *.pyc *.pyo diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2a0106162f505fc3a330bf6b03b3f1e5f51c257e..be43c4f06a355fb98899b33289212deaaa59e0e8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,7 +18,7 @@ variables: # us-west1-a before_script: - - pip install ansible==2.2.1.0 + - pip install ansible==2.3.0 - pip install netaddr - pip install apache-libcloud==0.20.1 - pip install boto==2.9.0 @@ -74,7 +74,7 @@ before_script: - $HOME/.cache before_script: - docker info - - pip install ansible==2.2.1.0 + - pip install ansible==2.3.0 - pip install netaddr - pip install apache-libcloud==0.20.1 - pip install boto==2.9.0 @@ -137,7 +137,7 @@ before_script: if [ "${UPGRADE_TEST}" != "false" ]; then test "${UPGRADE_TEST}" == "basic" && PLAYBOOK="cluster.yml"; test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml"; - pip install ansible==2.2.1.0; + pip install ansible==2.3.0; git checkout "${CI_BUILD_REF}"; ansible-playbook -i inventory/inventory.ini -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER ${SSH_ARGS} @@ -596,6 +596,7 @@ syntax-check: - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root -b --become-user=root cluster.yml -vvv --syntax-check - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root -b --become-user=root upgrade-cluster.yml -vvv --syntax-check - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root -b --become-user=root reset.yml -vvv --syntax-check + - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root -b --become-user=root extra_playbooks/upgrade-only-k8s.yml -vvv --syntax-check except: ['triggers', 'master'] tox-inventory-builder: diff --git a/.travis.yml.bak b/.travis.yml.bak deleted file mode 100644 index 7b948dcfe6e0e77774675333ad37278529c4d466..0000000000000000000000000000000000000000 --- a/.travis.yml.bak +++ /dev/null @@ -1,161 +0,0 @@ -sudo: required - -services: - - docker - -git: - depth: 5 - -env: - global: - GCE_USER=travis - SSH_USER=$GCE_USER - TEST_ID=$TRAVIS_JOB_NUMBER - CONTAINER_ENGINE=docker - PRIVATE_KEY=$GCE_PRIVATE_KEY - GS_ACCESS_KEY_ID=$GS_KEY - GS_SECRET_ACCESS_KEY=$GS_SECRET - ANSIBLE_KEEP_REMOTE_FILES=1 - CLUSTER_MODE=default - BOOTSTRAP_OS=none - matrix: - # Debian Jessie - - >- - KUBE_NETWORK_PLUGIN=canal - CLOUD_IMAGE=debian-8-kubespray - CLOUD_REGION=asia-east1-a - CLUSTER_MODE=ha - - >- - KUBE_NETWORK_PLUGIN=calico - CLOUD_IMAGE=debian-8-kubespray - CLOUD_REGION=europe-west1-c - CLUSTER_MODE=default - - # Centos 7 - - >- - KUBE_NETWORK_PLUGIN=flannel - CLOUD_IMAGE=centos-7 - CLOUD_REGION=asia-northeast1-c - CLUSTER_MODE=default - - >- - KUBE_NETWORK_PLUGIN=calico - CLOUD_IMAGE=centos-7 - CLOUD_REGION=us-central1-b - CLUSTER_MODE=ha - - # Redhat 7 - - >- - KUBE_NETWORK_PLUGIN=weave - CLOUD_IMAGE=rhel-7 - CLOUD_REGION=us-east1-c - CLUSTER_MODE=default - - # CoreOS stable - #- >- - # KUBE_NETWORK_PLUGIN=weave - # CLOUD_IMAGE=coreos-stable - # CLOUD_REGION=europe-west1-b - # CLUSTER_MODE=ha - # BOOTSTRAP_OS=coreos - - >- - KUBE_NETWORK_PLUGIN=canal - CLOUD_IMAGE=coreos-stable - CLOUD_REGION=us-west1-b - CLUSTER_MODE=default - BOOTSTRAP_OS=coreos - - # Extra cases for separated roles - - >- - KUBE_NETWORK_PLUGIN=canal - CLOUD_IMAGE=rhel-7 - CLOUD_REGION=asia-northeast1-b - CLUSTER_MODE=separate - - >- - KUBE_NETWORK_PLUGIN=weave - CLOUD_IMAGE=ubuntu-1604-xenial - CLOUD_REGION=europe-west1-d - CLUSTER_MODE=separate - - >- - KUBE_NETWORK_PLUGIN=calico - CLOUD_IMAGE=coreos-stable - CLOUD_REGION=us-central1-f - CLUSTER_MODE=separate - BOOTSTRAP_OS=coreos - -matrix: - allow_failures: - - env: KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=coreos-stable CLOUD_REGION=europe-west1-b CLUSTER_MODE=ha BOOTSTRAP_OS=coreos - -before_install: - # Install Ansible. - - pip install --user ansible - - pip install --user netaddr - # W/A https://github.com/ansible/ansible-modules-core/issues/5196#issuecomment-253766186 - - pip install --user apache-libcloud==0.20.1 - - pip install --user boto==2.9.0 -U - # Load cached docker images - - if [ -d /var/tmp/releases ]; then find /var/tmp/releases -type f -name "*.tar" | xargs -I {} sh -c "zcat {} | docker load"; fi - -cache: - - directories: - - $HOME/.cache/pip - - $HOME/.local - - /var/tmp/releases - -before_script: - - echo "RUN $TRAVIS_JOB_NUMBER $KUBE_NETWORK_PLUGIN $CONTAINER_ENGINE " - - mkdir -p $HOME/.ssh - - echo $PRIVATE_KEY | base64 -d > $HOME/.ssh/id_rsa - - echo $GCE_PEM_FILE | base64 -d > $HOME/.ssh/gce - - chmod 400 $HOME/.ssh/id_rsa - - chmod 755 $HOME/.local/bin/ansible-playbook - - $HOME/.local/bin/ansible-playbook --version - - cp tests/ansible.cfg . - - export PYPATH=$([ $BOOTSTRAP_OS = none ] && echo /usr/bin/python || echo /opt/bin/python) -# - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}' $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml" - -script: - - > - $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/create-gce.yml -i tests/local_inventory/hosts.cfg -c local $LOG_LEVEL - -e mode=${CLUSTER_MODE} - -e test_id=${TEST_ID} - -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} - -e gce_project_id=${GCE_PROJECT_ID} - -e gce_service_account_email=${GCE_ACCOUNT} - -e gce_pem_file=${HOME}/.ssh/gce - -e cloud_image=${CLOUD_IMAGE} - -e inventory_path=${PWD}/inventory/inventory.ini - -e cloud_region=${CLOUD_REGION} - - # Create cluster with netchecker app deployed - - > - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS - -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} - -e bootstrap_os=${BOOTSTRAP_OS} - -e ansible_python_interpreter=${PYPATH} - -e download_run_once=true - -e download_localhost=true - -e local_release_dir=/var/tmp/releases - -e deploy_netchecker=true - cluster.yml - - # Tests Cases - ## Test Master API - - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/010_check-apiserver.yml $LOG_LEVEL - ## Ping the between 2 pod - - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL - ## Advanced DNS checks - - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/040_check-network-adv.yml $LOG_LEVEL - -after_script: - - > - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local $LOG_LEVEL - -e mode=${CLUSTER_MODE} - -e test_id=${TEST_ID} - -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} - -e gce_project_id=${GCE_PROJECT_ID} - -e gce_service_account_email=${GCE_ACCOUNT} - -e gce_pem_file=${HOME}/.ssh/gce - -e cloud_image=${CLOUD_IMAGE} - -e inventory_path=${PWD}/inventory/inventory.ini - -e cloud_region=${CLOUD_REGION} diff --git a/README.md b/README.md index 5395f5d461c42d960c22e66fffec47b1ae2e9caf..94ba1716d84f62e4d822ca7cd2534c7a12676579 100644 --- a/README.md +++ b/README.md @@ -50,13 +50,14 @@ Note: Upstart/SysV init based OS types are not supported. Versions of supported components -------------------------------- + [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.6.4 <br> [etcd](https://github.com/coreos/etcd/releases) v3.0.17 <br> [flanneld](https://github.com/coreos/flannel/releases) v0.6.2 <br> [calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.23.0 <br> [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) <br> [weave](http://weave.works/) v1.8.2 <br> -[docker](https://www.docker.com/) v1.12.5 <br> +[docker](https://www.docker.com/) v1.13.1 <br> [rkt](https://coreos.com/rkt/docs/latest/) v1.21.0 <br> Note: rkt support as docker alternative is limited to control plane (etcd and @@ -67,9 +68,9 @@ plugins can be deployed for a given single cluster. Requirements -------------- -* **Ansible v2.2 (or newer) and python-netaddr is installed on the machine +* **Ansible v2.3 (or newer) and python-netaddr is installed on the machine that will run Ansible commands** -* **Jinja 2.8 (or newer) is required to run the Ansible Playbooks** +* **Jinja 2.9 (or newer) is required to run the Ansible Playbooks** * The target servers must have **access to the Internet** in order to pull docker images. * The target servers are configured to allow **IPv4 forwarding**. * **Your ssh key must be copied** to all the servers part of your inventory. diff --git a/contrib/terraform/aws/README.md b/contrib/terraform/aws/README.md index 03bc4e23eee0293c48d60d1ba1480112d8c066ce..de858b2a9ca5cf97468979467968ec699b10157f 100644 --- a/contrib/terraform/aws/README.md +++ b/contrib/terraform/aws/README.md @@ -14,20 +14,42 @@ This project will create: **How to Use:** -- Export the variables for your AWS credentials or edit credentials.tfvars: +- Export the variables for your AWS credentials or edit `credentials.tfvars`: ``` -export aws_access_key="xxx" -export aws_secret_key="yyy" -export aws_ssh_key_name="zzz" +export AWS_ACCESS_KEY_ID="www" +export AWS_SECRET_ACCESS_KEY ="xxx" +export AWS_SSH_KEY_NAME="yyy" +export AWS_DEFAULT_REGION="zzz" ``` +- Rename `contrib/terraform/aws/terraform.tfvars.example` to `terraform.tfvars` -- Update contrib/terraform/aws/terraform.tfvars with your data +- Update `contrib/terraform/aws/terraform.tfvars` with your data + - Allocate new AWS Elastic IPs: Depending on # of Availability Zones used (2 for each AZ) + - Create an AWS EC2 SSH Key -- Run with `terraform apply -var-file="credentials.tfvars"` or `terraform apply` depending if you exported your AWS credentials + +- Run with `terraform apply --var-file="credentials.tfvars"` or `terraform apply` depending if you exported your AWS credentials + +- Terraform automatically creates an Ansible Inventory file called `hosts` with the created infrastructure in the directory `inventory` - Once the infrastructure is created, you can run the kargo playbooks and supply inventory/hosts with the `-i` flag. +**Troubleshooting** + +***Remaining AWS IAM Instance Profile***: + +If the cluster was destroyed without using Terraform it is possible that +the AWS IAM Instance Profiles still remain. To delete them you can use +the `AWS CLI` with the following command: +``` +aws iam delete-instance-profile --region <region_name> --instance-profile-name <profile_name> +``` + +***Ansible Inventory doesnt get created:*** + +It could happen that Terraform doesnt create an Ansible Inventory file automatically. If this is the case copy the output after `inventory=` and create a file named `hosts`in the directory `inventory` and paste the inventory into the file. + **Architecture** Pictured is an AWS Infrastructure created with this Terraform project distributed over two Availability Zones. diff --git a/contrib/terraform/aws/create-infrastructure.tf b/contrib/terraform/aws/create-infrastructure.tf index 14da95492558497c198d8b86ccdb1603b6f2f682..781edea86a504c27620db700b603f1cf71066d72 100644 --- a/contrib/terraform/aws/create-infrastructure.tf +++ b/contrib/terraform/aws/create-infrastructure.tf @@ -173,6 +173,7 @@ data "template_file" "inventory" { list_etcd = "${join("\n",aws_instance.k8s-etcd.*.tags.Name)}" elb_api_fqdn = "apiserver_loadbalancer_domain_name=\"${module.aws-elb.aws_elb_api_fqdn}\"" elb_api_port = "loadbalancer_apiserver.port=${var.aws_elb_api_port}" + kube_insecure_apiserver_address = "kube_apiserver_insecure_bind_address: ${var.kube_insecure_apiserver_address}" } } diff --git a/contrib/terraform/aws/output.tf b/contrib/terraform/aws/output.tf index fbe74f2622c7fb697bbecffd9a05c6a738c7b7b7..fabc0d218562bc7e3e7174fc7b558f5b91265b06 100644 --- a/contrib/terraform/aws/output.tf +++ b/contrib/terraform/aws/output.tf @@ -18,3 +18,7 @@ output "etcd" { output "aws_elb_api_fqdn" { value = "${module.aws-elb.aws_elb_api_fqdn}:${var.aws_elb_api_port}" } + +output "inventory" { + value = "${data.template_file.inventory.rendered}" +} diff --git a/contrib/terraform/aws/templates/inventory.tpl b/contrib/terraform/aws/templates/inventory.tpl index 4140aa768e5a029234d815c5d1d56207140499d3..8d5afd1cfe33dad0a95fbae3aa3c2d99e994cc94 100644 --- a/contrib/terraform/aws/templates/inventory.tpl +++ b/contrib/terraform/aws/templates/inventory.tpl @@ -25,3 +25,4 @@ kube-master [k8s-cluster:vars] ${elb_api_fqdn} ${elb_api_port} +${kube_insecure_apiserver_address} diff --git a/contrib/terraform/aws/terraform.tfvars.example b/contrib/terraform/aws/terraform.tfvars.example index 214ef89db2fd84a6002f3fd0c237278dbf2a65c9..666b21db2268d60c214a02db7ecd70e5c3393719 100644 --- a/contrib/terraform/aws/terraform.tfvars.example +++ b/contrib/terraform/aws/terraform.tfvars.example @@ -1,6 +1,5 @@ #Global Vars aws_cluster_name = "devtest" -aws_region = "eu-central-1" #VPC Vars aws_vpc_cidr_block = "10.250.192.0/18" @@ -28,5 +27,6 @@ aws_cluster_ami = "ami-903df7ff" #Settings AWS ELB -aws_elb_api_port = 443 -k8s_secure_api_port = 443 +aws_elb_api_port = 6443 +k8s_secure_api_port = 6443 +kube_insecure_apiserver_address = 0.0.0.0 diff --git a/contrib/terraform/aws/variables.tf b/contrib/terraform/aws/variables.tf index 82e2fb018e0a95d513a7be226f3050cc4de5c5c1..c740e647211a35190b5a2af7fdb11a0a9589eb32 100644 --- a/contrib/terraform/aws/variables.tf +++ b/contrib/terraform/aws/variables.tf @@ -95,3 +95,7 @@ variable "aws_elb_api_port" { variable "k8s_secure_api_port" { description = "Secure Port of K8S API Server" } + +variable "kube_insecure_apiserver_address" { + description= "Bind Address for insecure Port of K8s API Server" +} diff --git a/contrib/terraform/openstack/README.md b/contrib/terraform/openstack/README.md index 9df7abd9f7b9bbbe6aaf44d7033cf69f4c6618a4..e98b8068a3b2790fbe3b0c88acca0ae9ee4ca37a 100644 --- a/contrib/terraform/openstack/README.md +++ b/contrib/terraform/openstack/README.md @@ -86,7 +86,7 @@ This will provision one VM as master using a floating ip, two additional masters Additionally, now the terraform based installation supports provisioning of a GlusterFS shared file system based on a separate set of VMs, running either a Debian or RedHat based set of VMs. To enable this, you need to add to your `my-terraform-vars.tfvars` the following variables: ``` -# Flavour depends on your openstack installation, you can get available flavours through `nova list-flavors` +# Flavour depends on your openstack installation, you can get available flavours through `nova flavor-list` flavor_gfs_node = "af659280-5b8a-42b5-8865-a703775911da" # This is the name of an image already available in your openstack installation. image_gfs = "Ubuntu 15.10" diff --git a/docs/ansible.md b/docs/ansible.md index eb8a607695183f28098f69c73ee3a97fe0555db9..4da6edb48cdd97d286fe9aef18761184c7652d31 100644 --- a/docs/ansible.md +++ b/docs/ansible.md @@ -27,7 +27,7 @@ not _kube-node_. There are also two special groups: -* **calico-rr** : explained for [advanced Calico networking cases](docs/calico.md) +* **calico-rr** : explained for [advanced Calico networking cases](calico.md) * **bastion** : configure a bastion host if your nodes are not directly reachable Below is a complete inventory example: diff --git a/docs/coreos.md b/docs/coreos.md index 7c9b2c8a68ed79daf40eeab4d8f5dc660f8cf929..546ad0e89137cd52f2283d4a80523427e1f350b2 100644 --- a/docs/coreos.md +++ b/docs/coreos.md @@ -11,6 +11,10 @@ Or with Ansible: Before running the cluster playbook you must satisfy the following requirements: -* On each CoreOS nodes a writable directory **/opt/bin** (~400M disk space) +General CoreOS Pre-Installation Notes: +- You should set the bootstrap_os variable to `coreos` +- Ensure that the bin_dir is set to `/opt/bin` +- ansible_python_interpreter should be `/opt/bin/python`. This will be laid down by the bootstrap task. +- The default resolvconf_mode setting of `docker_dns` **does not** work for CoreOS. This is because we do not edit the systemd service file for docker on CoreOS nodes. Instead, just use the `host_resolvconf` mode. It should work out of the box. Then you can proceed to [cluster deployment](#run-deployment) diff --git a/docs/getting-started.md b/docs/getting-started.md index caf4485aef992024ff1174e5c3ff28b934c6d1cc..5c61ef7649760a5182d48deb99ba0b156cc86590 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -38,7 +38,7 @@ Example inventory generator usage: ``` cp -r inventory my_inventory declare -a IPS=(10.10.1.3 10.10.1.4 10.10.1.5) -CONFIG_FILE=my_inventory/inventory.cfg python3 contrib/inventory_builder/inventory.py ${IPS} +CONFIG_FILE=my_inventory/inventory.cfg python3 contrib/inventory_builder/inventory.py ${IPS[@]} ``` Starting custom deployment diff --git a/docs/upgrades.md b/docs/upgrades.md index c37cad54af9f386c0cec85b892ca938cc11ec759..cb431d4c071547749d43fb011b960b85f4d425a6 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -44,7 +44,15 @@ deployed. ``` git fetch origin git checkout origin/master -ansible-playbook upgrade-cluster.yml -b -i inventory/inventory.cfg +ansible-playbook upgrade-cluster.yml -b -i inventory/inventory.cfg -e kube_version=v1.6.0 +``` + +After a successul upgrade, the Server Version should be updated: + +``` +$ kubectl version +Client Version: version.Info{Major:"1", Minor:"6", GitVersion:"v1.6.0", GitCommit:"fff5156092b56e6bd60fff75aad4dc9de6b6ef37", GitTreeState:"clean", BuildDate:"2017-03-28T19:15:41Z", GoVersion:"go1.8", Compiler:"gc", Platform:"darwin/amd64"} +Server Version: version.Info{Major:"1", Minor:"6", GitVersion:"v1.6.0+coreos.0", GitCommit:"8031716957d697332f9234ddf85febb07ac6c3e3", GitTreeState:"clean", BuildDate:"2017-03-29T04:33:09Z", GoVersion:"go1.7.5", Compiler:"gc", Platform:"linux/amd64"} ``` #### Upgrade order diff --git a/docs/vars.md b/docs/vars.md index 966b3ffc831e77ec52f243efaaa14de9e4d12655..603a614b269d11aa4be00ae6984b90de83757cce 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -98,6 +98,20 @@ Stack](https://github.com/kubernetes-incubator/kargo/blob/master/docs/dns-stack. loaded by preinstall kubernetes processes. For example, ceph and rbd backed volumes. Set this variable to true to let kubelet load kernel modules. +##### Custom flags for Kube Components +For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. Example: +``` +kubelet_custom_flags: + - "--eviction-hard=memory.available<100Mi" + - "--eviction-soft-grace-period=memory.available=30s" + - "--eviction-soft=memory.available<300Mi" +``` +The possible vars are: +* *apiserver_custom_flags* +* *controller_mgr_custom_flags* +* *scheduler_custom_flags* +* *kubelet_custom_flags* + #### User accounts Kargo sets up two Kubernetes accounts by default: ``root`` and ``kube``. Their diff --git a/extra_playbooks/inventory b/extra_playbooks/inventory new file mode 120000 index 0000000000000000000000000000000000000000..e09e1addd372d547609f948193efba3aedda6ef9 --- /dev/null +++ b/extra_playbooks/inventory @@ -0,0 +1 @@ +../inventory \ No newline at end of file diff --git a/extra_playbooks/roles b/extra_playbooks/roles new file mode 120000 index 0000000000000000000000000000000000000000..d8c4472ca1b65cea039252e137ff3b4ab5d3a555 --- /dev/null +++ b/extra_playbooks/roles @@ -0,0 +1 @@ +../roles \ No newline at end of file diff --git a/extra_playbooks/upgrade-only-k8s.yml b/extra_playbooks/upgrade-only-k8s.yml new file mode 100644 index 0000000000000000000000000000000000000000..f10259b0770d3817249f632072ce3f392c3d601d --- /dev/null +++ b/extra_playbooks/upgrade-only-k8s.yml @@ -0,0 +1,60 @@ +### NOTE: This playbook cannot be used to deploy any new nodes to the cluster. +### Additional information: +### * Will not upgrade etcd +### * Will not upgrade network plugins +### * Will not upgrade Docker +### * Currently does not support Vault deployment. +### +### In most cases, you probably want to use upgrade-cluster.yml playbook and +### not this one. + +- hosts: localhost + gather_facts: False + roles: + - { role: kargo-defaults} + - { role: bastion-ssh-config, tags: ["localhost", "bastion"]} + +- hosts: k8s-cluster:etcd:calico-rr + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + gather_facts: false + vars: + # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining + # fail. bootstrap-os fixes this on these systems, so in later plays it can be enabled. + ansible_ssh_pipelining: false + roles: + - { role: kargo-defaults} + - { role: bootstrap-os, tags: bootstrap-os} + +- hosts: k8s-cluster:etcd:calico-rr + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + vars: + ansible_ssh_pipelining: true + gather_facts: true + +- hosts: k8s-cluster:etcd:calico-rr + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + roles: + - { role: kargo-defaults} + - { role: kubernetes/preinstall, tags: preinstall } + +#Handle upgrades to master components first to maintain backwards compat. +- hosts: kube-master + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + serial: 1 + roles: + - { role: kargo-defaults} + - { role: upgrade/pre-upgrade, tags: pre-upgrade } + - { role: kubernetes/node, tags: node } + - { role: kubernetes/master, tags: master } + - { role: upgrade/post-upgrade, tags: post-upgrade } + +#Finally handle worker upgrades, based on given batch size +- hosts: kube-node:!kube-master + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + serial: "{{ serial | default('20%') }}" + roles: + - { role: kargo-defaults} + - { role: upgrade/pre-upgrade, tags: pre-upgrade } + - { role: kubernetes/node, tags: node } + - { role: upgrade/post-upgrade, tags: post-upgrade } + - { role: kargo-defaults} diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index a216de7120cf4eb29297bae20ddb057520c1cd81..ef5e363dc357775887259229a060ebd159bfe0e2 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -98,7 +98,7 @@ cluster_name: cluster.local # Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods ndots: 2 # Can be dnsmasq_kubedns, kubedns or none -dns_mode: dnsmasq_kubedns +dns_mode: kubedns # Can be docker_dns, host_resolvconf or none resolvconf_mode: docker_dns # Deploy netchecker app to verify DNS resolve as an HTTP service diff --git a/requirements.txt b/requirements.txt index ccf58ea3a38269e3fe0ee41c658ebafe2cc6ee4c..6458113ac548ecfba541c9124c1c0eaf30bcacf2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,3 @@ -ansible==2.2.1.0 +ansible>=2.3.0 netaddr -# Ansible 2.2.1 requires jinja2<2.9, see <https://github.com/ansible/ansible/blob/v2.2.1.0-1/setup.py#L25>, -# but without explicit limiting upper jinja2 version here pip ignores -# Ansible requirements and installs latest available jinja2 -# (pip is not very smart here), which is incompatible with with -# Ansible 2.2.1. -# With incompatible jinja2 version "ansible-vault create" (and probably other parts) -# fails with: -# ERROR! Unexpected Exception: The 'jinja2<2.9' distribution was not found -# and is required by ansible -# This upper limit should be removed in 2.2.2 release, see: -# <https://github.com/ansible/ansible/commit/978311bf3f91dae5806ab72b665b0937adce38ad> -jinja2>=2.8,<2.9 +jinja2>=2.9.6 diff --git a/roles/bootstrap-os/tasks/bootstrap-centos.yml b/roles/bootstrap-os/tasks/bootstrap-centos.yml index b8cf126c16396657e396f02a0d062a9c55e7de8d..c9233dfb1285b07df0ea48720d20852f24c24059 100644 --- a/roles/bootstrap-os/tasks/bootstrap-centos.yml +++ b/roles/bootstrap-os/tasks/bootstrap-centos.yml @@ -13,3 +13,6 @@ line: "enabled=0" state: present when: fastestmirror.stat.exists + +- name: Install packages requirements for bootstrap + raw: yum -y install libselinux-python diff --git a/roles/dnsmasq/templates/dnsmasq-autoscaler.yml b/roles/dnsmasq/templates/dnsmasq-autoscaler.yml index ca65c2dab92d3ffec412b1f06f1fd127d857ab4f..4e5e2ddcc82e665c2672a217c0e5ef2ed1178ad2 100644 --- a/roles/dnsmasq/templates/dnsmasq-autoscaler.yml +++ b/roles/dnsmasq/templates/dnsmasq-autoscaler.yml @@ -41,7 +41,7 @@ spec: - /cluster-proportional-autoscaler - --namespace=kube-system - --configmap=dnsmasq-autoscaler - - --target=ReplicationController/dnsmasq + - --target=Deployment/dnsmasq # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate. # If using small nodes, "nodesPerReplica" should dominate. - --default-params={"linear":{"nodesPerReplica":{{ dnsmasq_nodes_per_replica }},"preventSinglePointFailure":true}} diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index b58efec058321f25de2ca5983e282055a3aad1ee..9284fbbdff4ff437336862323b539a7085eb62a0 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -22,8 +22,8 @@ kube_version: v1.6.4 etcd_version: v3.0.17 #TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download -calico_version: "v1.1.0-rc8" -calico_cni_version: "v1.5.6" +calico_version: "v1.1.3" +calico_cni_version: "v1.7.0" calico_policy_version: "v0.5.4" weave_version: 1.8.2 flannel_version: v0.6.2 @@ -50,10 +50,8 @@ calico_cni_image_repo: "calico/cni" calico_cni_image_tag: "{{ calico_cni_version }}" calico_policy_image_repo: "calico/kube-policy-controller" calico_policy_image_tag: "{{ calico_policy_version }}" -# TODO(adidenko): switch to "calico/routereflector" when -# https://github.com/projectcalico/calico-bird/pull/27 is merged -calico_rr_image_repo: "quay.io/l23network/routereflector" -calico_rr_image_tag: "v0.1" +calico_rr_image_repo: "quay.io/calico/routereflector" +calico_rr_image_tag: "v0.3.0" exechealthz_version: 1.1 exechealthz_image_repo: "gcr.io/google_containers/exechealthz-amd64" exechealthz_image_tag: "{{ exechealthz_version }}" @@ -61,9 +59,11 @@ hyperkube_image_repo: "quay.io/coreos/hyperkube" hyperkube_image_tag: "{{ kube_version }}_coreos.0" pod_infra_image_repo: "gcr.io/google_containers/pause-amd64" pod_infra_image_tag: "{{ pod_infra_version }}" -netcheck_tag: "v1.0" +netcheck_version: "v1.0" netcheck_agent_img_repo: "quay.io/l23network/k8s-netchecker-agent" +netcheck_agent_tag: "{{ netcheck_version }}" netcheck_server_img_repo: "quay.io/l23network/k8s-netchecker-server" +netcheck_server_tag: "{{ netcheck_version }}" weave_kube_image_repo: "weaveworks/weave-kube" weave_kube_image_tag: "{{ weave_version }}" weave_npc_image_repo: "weaveworks/weave-npc" @@ -103,13 +103,13 @@ downloads: netcheck_server: container: true repo: "{{ netcheck_server_img_repo }}" - tag: "{{ netcheck_tag }}" + tag: "{{ netcheck_server_tag }}" sha256: "{{ netcheck_server_digest_checksum|default(None) }}" enabled: "{{ deploy_netchecker|bool }}" netcheck_agent: container: true repo: "{{ netcheck_agent_img_repo }}" - tag: "{{ netcheck_tag }}" + tag: "{{ netcheck_agent_tag }}" sha256: "{{ netcheck_agent_digest_checksum|default(None) }}" enabled: "{{ deploy_netchecker|bool }}" etcd: diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index 37c72462e46ce313464fdfbcd2c5bafbe773e626..24d1b5bcabd24405d49172c28f592cf9705b50ae 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -2,14 +2,18 @@ - name: downloading... debug: msg: "{{ download.url }}" - when: "{{ download.enabled|bool and not download.container|bool }}" + when: + - download.enabled|bool + - not download.container|bool - name: Create dest directories file: path: "{{local_release_dir}}/{{download.dest|dirname}}" state: directory recurse: yes - when: "{{ download.enabled|bool and not download.container|bool }}" + when: + - download.enabled|bool + - not download.container|bool tags: bootstrap-os - name: Download items @@ -23,7 +27,9 @@ until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg" retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: "{{ download.enabled|bool and not download.container|bool }}" + when: + - download.enabled|bool + - not download.container|bool - name: Extract archives unarchive: @@ -32,7 +38,11 @@ owner: "{{ download.owner|default(omit) }}" mode: "{{ download.mode|default(omit) }}" copy: no - when: "{{ download.enabled|bool and not download.container|bool and download.unarchive is defined and download.unarchive == True }}" + when: + - download.enabled|bool + - not download.container|bool + - download.unarchive is defined + - download.unarchive == True - name: Fix permissions file: @@ -40,7 +50,10 @@ path: "{{local_release_dir}}/{{download.dest}}" owner: "{{ download.owner|default(omit) }}" mode: "{{ download.mode|default(omit) }}" - when: "{{ download.enabled|bool and not download.container|bool and (download.unarchive is not defined or download.unarchive == False) }}" + when: + - download.enabled|bool + - not download.container|bool + - (download.unarchive is not defined or download.unarchive == False) - set_fact: download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}" @@ -53,13 +66,15 @@ recurse: yes mode: 0755 owner: "{{ansible_ssh_user|default(ansible_user_id)}}" - when: "{{ download.enabled|bool and download.container|bool }}" + when: + - download.enabled|bool + - download.container|bool tags: bootstrap-os # This is required for the download_localhost delegate to work smooth with Container Linux by CoreOS cluster nodes - name: Hack python binary path for localhost raw: sh -c "mkdir -p /opt/bin; ln -sf /usr/bin/python /opt/bin/python" - when: "{{ download_delegate == 'localhost' }}" + when: download_delegate == 'localhost' delegate_to: localhost failed_when: false run_once: true @@ -73,12 +88,18 @@ delegate_to: localhost become: false run_once: true - when: "{{ download_run_once|bool and download.enabled|bool and download.container|bool and download_delegate == 'localhost' }}" + when: + - download_run_once|bool + - download.enabled|bool + - download.container|bool + - download_delegate == 'localhost' tags: localhost - name: Make download decision if pull is required by tag or sha256 include: set_docker_image_facts.yml - when: "{{ download.enabled|bool and download.container|bool }}" + when: + - download.enabled|bool + - download.container|bool delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}" run_once: "{{ download_run_once|bool }}" tags: facts @@ -86,7 +107,9 @@ - name: pulling... debug: msg: "{{ pull_args }}" - when: "{{ download.enabled|bool and download.container|bool }}" + when: + - download.enabled|bool + - download.container|bool #NOTE(bogdando) this brings no docker-py deps for nodes - name: Download containers if pull is required or told to always pull @@ -95,7 +118,10 @@ until: pull_task_result|succeeded retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: "{{ download.enabled|bool and download.container|bool and pull_required|bool|default(download_always_pull) }}" + when: + - download.enabled|bool + - download.container|bool + - pull_required|bool|default(download_always_pull) delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}" run_once: "{{ download_run_once|bool }}" @@ -110,7 +136,10 @@ - name: "Update the 'container_changed' fact" set_fact: container_changed: "{{ pull_required|bool|default(false) or not 'up to date' in pull_task_result.stdout }}" - when: "{{ download.enabled|bool and download.container|bool and pull_required|bool|default(download_always_pull) }}" + when: + - download.enabled|bool + - download.container|bool + - pull_required|bool|default(download_always_pull) delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}" run_once: "{{ download_run_once|bool }}" tags: facts @@ -120,7 +149,10 @@ path: "{{fname}}" register: img changed_when: false - when: "{{ download.enabled|bool and download.container|bool and download_run_once|bool }}" + when: + - download.enabled|bool + - download.container|bool + - download_run_once|bool delegate_to: "{{ download_delegate }}" become: false run_once: true @@ -131,7 +163,12 @@ delegate_to: "{{ download_delegate }}" register: saved run_once: true - when: (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or download_delegate == "localhost") and download_run_once|bool and download.enabled|bool and download.container|bool and (container_changed|bool or not img.stat.exists) + when: + - (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or download_delegate == "localhost") + - download_run_once|bool + - download.enabled|bool + - download.container|bool + - (container_changed|bool or not img.stat.exists) - name: Download | copy container images to ansible host synchronize: @@ -140,7 +177,14 @@ mode: pull delegate_to: localhost become: false - when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and inventory_hostname == groups['kube-master'][0] and download_delegate != "localhost" and download_run_once|bool and download.enabled|bool and download.container|bool and saved.changed + when: + - not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] + - inventory_hostname == groups['kube-master'][0] + - download_delegate != "localhost" + - download_run_once|bool + - download.enabled|bool + - download.container|bool + - saved.changed - name: Download | upload container images to nodes synchronize: @@ -153,10 +197,21 @@ until: get_task|succeeded retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and inventory_hostname != groups['kube-master'][0] or download_delegate == "localhost") and download_run_once|bool and download.enabled|bool and download.container|bool + when: + - (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and + inventory_hostname != groups['kube-master'][0] or + download_delegate == "localhost") + - download_run_once|bool + - download.enabled|bool + - download.container|bool tags: [upload, upgrade] - name: Download | load container images shell: "{{ docker_bin_dir }}/docker load < {{ fname }}" - when: (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and inventory_hostname != groups['kube-master'][0] or download_delegate == "localhost") and download_run_once|bool and download.enabled|bool and download.container|bool + when: + - (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and + inventory_hostname != groups['kube-master'][0] or download_delegate == "localhost") + - download_run_once|bool + - download.enabled|bool + - download.container|bool tags: [upload, upgrade] diff --git a/roles/etcd/tasks/gen_certs_vault.yml b/roles/etcd/tasks/gen_certs_vault.yml index 144e3b6585a40ff447ec1929e577a344789663f3..e45b2d02d95667313060d92c99a697ebd5f12ff1 100644 --- a/roles/etcd/tasks/gen_certs_vault.yml +++ b/roles/etcd/tasks/gen_certs_vault.yml @@ -1,4 +1,12 @@ --- +- include: sync_etcd_master_certs.yml + when: inventory_hostname in groups.etcd + tags: etcd-secrets + +- include: sync_etcd_node_certs.yml + when: inventory_hostname in etcd_node_cert_hosts + tags: etcd-secrets + - name: gen_certs_vault | Read in the local credentials command: cat /etc/vault/roles/etcd/userpass @@ -15,7 +23,7 @@ url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ etcd_vault_creds.username }}" headers: Accept: application/json - Content-Type: application/json + Content-Type: application/json method: POST body_format: json body: @@ -37,7 +45,7 @@ issue_cert_copy_ca: "{{ item == etcd_master_certs_needed|first }}" issue_cert_file_group: "{{ etcd_cert_group }}" issue_cert_file_owner: kube - issue_cert_headers: "{{ etcd_vault_headers }}" + issue_cert_headers: "{{ etcd_vault_headers }}" issue_cert_hosts: "{{ groups.etcd }}" issue_cert_ip_sans: >- [ @@ -60,7 +68,7 @@ issue_cert_copy_ca: "{{ item == etcd_node_certs_needed|first }}" issue_cert_file_group: "{{ etcd_cert_group }}" issue_cert_file_owner: kube - issue_cert_headers: "{{ etcd_vault_headers }}" + issue_cert_headers: "{{ etcd_vault_headers }}" issue_cert_hosts: "{{ etcd_node_cert_hosts }}" issue_cert_ip_sans: >- [ @@ -75,3 +83,5 @@ with_items: "{{ etcd_node_certs_needed|d([]) }}" when: inventory_hostname in etcd_node_cert_hosts notify: set etcd_secret_changed + + diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index d917b56acadabdd7d012ba719f3b6461d33dc473..afd5fa88364cbb827a65c9256c50a31d6d6c4f6d 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -7,20 +7,7 @@ when: cert_management == "script" tags: [etcd-secrets, facts] -- include: gen_certs_script.yml - when: cert_management == "script" - tags: etcd-secrets - -- include: sync_etcd_master_certs.yml - when: cert_management == "vault" and inventory_hostname in groups.etcd - tags: etcd-secrets - -- include: sync_etcd_node_certs.yml - when: cert_management == "vault" and inventory_hostname in etcd_node_cert_hosts - tags: etcd-secrets - -- include: gen_certs_vault.yml - when: cert_management == "vault" and (etcd_master_certs_needed|d() or etcd_node_certs_needed|d()) +- include: "gen_certs_{{ cert_management }}.yml" tags: etcd-secrets - include: "install_{{ etcd_deployment_type }}.yml" diff --git a/roles/kargo-defaults/defaults/main.yaml b/roles/kargo-defaults/defaults/main.yaml index 36b8146145bff8ef1be824d535d4d7ad7d01baca..f0323d4796c202d6705d4d4cbeb6ddf6918bd744 100644 --- a/roles/kargo-defaults/defaults/main.yaml +++ b/roles/kargo-defaults/defaults/main.yaml @@ -42,9 +42,6 @@ kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" kube_manifest_dir: "{{ kube_config_dir }}/manifests" system_namespace: kube-system -# Logging directory (sysvinit systems) -kube_log_dir: "/var/log/kubernetes" - # This is where all the cert scripts and certs will be located kube_cert_dir: "{{ kube_config_dir }}/ssl" diff --git a/roles/kernel-upgrade/tasks/reboot.yml b/roles/kernel-upgrade/tasks/reboot.yml index 5e01dd8fcf9297acb6f84f60388030001775a2bc..87748f3f6c9e1e49e6047092931f0a1ea24da5b0 100644 --- a/roles/kernel-upgrade/tasks/reboot.yml +++ b/roles/kernel-upgrade/tasks/reboot.yml @@ -17,7 +17,7 @@ - set_fact: wait_for_delegate: "{{hostvars['bastion']['ansible_ssh_host']}}" - when: "{{ 'bastion' in groups['all'] }}" + when: "'bastion' in groups['all']" - name: wait for bastion to come back wait_for: @@ -27,7 +27,7 @@ timeout: 300 become: false delegate_to: localhost - when: "is_bastion" + when: is_bastion - name: waiting for server to come back (using bastion if necessary) wait_for: @@ -37,4 +37,4 @@ timeout: 300 become: false delegate_to: "{{ wait_for_delegate }}" - when: "not is_bastion" + when: not is_bastion diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 6d0562fc9f99e684acab19442a93fa55575a2aaf..89bdd4277dfad603bcbfabe7b078bcb94d031297 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -24,8 +24,8 @@ deploy_netchecker: false netchecker_port: 31081 agent_report_interval: 15 netcheck_namespace: default -agent_img: "{{ netcheck_agent_img_repo }}:{{ netcheck_tag }}" -server_img: "{{ netcheck_server_img_repo }}:{{ netcheck_tag }}" +agent_img: "{{ netcheck_agent_img_repo }}:{{ netcheck_agent_tag }}" +server_img: "{{ netcheck_server_img_repo }}:{{ netcheck_server_tag }}" # Limits for netchecker apps netchecker_agent_cpu_limit: 30m diff --git a/roles/kubernetes-apps/ansible/tasks/netchecker.yml b/roles/kubernetes-apps/ansible/tasks/netchecker.yml index b9047a1e2b0ce6ead37d69ebc0dc840acb3d7616..aae75d0914fff2dd2ab19fd00769a78cb07dafa7 100644 --- a/roles/kubernetes-apps/ansible/tasks/netchecker.yml +++ b/roles/kubernetes-apps/ansible/tasks/netchecker.yml @@ -5,7 +5,7 @@ with_items: - {file: netchecker-agent-ds.yml.j2, type: ds, name: netchecker-agent} - {file: netchecker-agent-hostnet-ds.yml.j2, type: ds, name: netchecker-agent-hostnet} - - {file: netchecker-server-pod.yml.j2, type: po, name: netchecker-server} + - {file: netchecker-server-deployment.yml.j2, type: po, name: netchecker-server} - {file: netchecker-server-svc.yml.j2, type: svc, name: netchecker-service} register: manifests when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml index 65dee527f284bd7787a05b310e614c426e7beebc..c0f519e2c578706f3f9807745fa36292c8aee44a 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml +++ b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml @@ -42,7 +42,7 @@ spec: - --namespace=kube-system - --configmap=kubedns-autoscaler # Should keep target in sync with cluster/addons/dns/kubedns-controller.yaml.base - - --target=replicationcontroller/kubedns + - --target=Deployment/kubedns - --default-params={"linear":{"nodesPerReplica":{{ kubedns_nodes_per_replica }},"min":{{ kubedns_min_replicas }}}} - --logtostderr=true - --v=2 diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml index 1bae177d39ece1a6e74380447ef89ec037674e86..a2150cc70ee4cb5d477fe321dfe9f6d81722009b 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml +++ b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml @@ -83,6 +83,7 @@ spec: {% if kube_log_level == '4' %} - --log-queries {% endif %} + - --local=/{{ bogus_domains }} ports: - containerPort: 53 name: dns diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 index 1394987332a9ca292eaf592589ccd34a4de3d4de..df0b8ba90d7ab7c742042a1f8ce1a27792d07fcb 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 @@ -20,6 +20,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName args: - "-v=5" - "-alsologtostderr=true" diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.j2 index 13a966c806cee7657130b42bf55de54eeb1e1912..10a74da846c1dcae6b97c019c65e12f154f1eeb0 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.j2 @@ -24,6 +24,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName args: - "-v=5" - "-alsologtostderr=true" diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 index 13a966c806cee7657130b42bf55de54eeb1e1912..10a74da846c1dcae6b97c019c65e12f154f1eeb0 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 @@ -24,6 +24,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName args: - "-v=5" - "-alsologtostderr=true" diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..6c52352fb5523199c0212cec447851a890967ced --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 @@ -0,0 +1,33 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: netchecker-server +spec: + replicas: 1 + template: + metadata: + name: netchecker-server + labels: + app: netchecker-server + namespace: {{ netcheck_namespace }} + spec: + containers: + - name: netchecker-server + image: "{{ server_img }}" + env: + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + limits: + cpu: {{ netchecker_server_cpu_limit }} + memory: {{ netchecker_server_memory_limit }} + requests: + cpu: {{ netchecker_server_cpu_requests }} + memory: {{ netchecker_server_memory_requests }} + ports: + - containerPort: 8081 + hostPort: 8081 + args: + - "-v=5" + - "-logtostderr" + - "-kubeproxyinit" + - "-endpoint=0.0.0.0:8081" diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml.j2 deleted file mode 100644 index 06aea406ad1f752ad01ce858b790b82b8b78ed12..0000000000000000000000000000000000000000 --- a/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml.j2 +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: netchecker-server - labels: - app: netchecker-server - namespace: {{ netcheck_namespace }} -spec: - containers: - - name: netchecker-server - image: "{{ server_img }}" - env: - imagePullPolicy: {{ k8s_image_pull_policy }} - resources: - limits: - cpu: {{ netchecker_server_cpu_limit }} - memory: {{ netchecker_server_memory_limit }} - requests: - cpu: {{ netchecker_server_cpu_requests }} - memory: {{ netchecker_server_memory_requests }} - ports: - - containerPort: 8081 - hostPort: 8081 - args: - - "-v=5" - - "-logtostderr" - - "-kubeproxyinit" - - "-endpoint=0.0.0.0:8081" diff --git a/roles/kubernetes-apps/helm/defaults/main.yml b/roles/kubernetes-apps/helm/defaults/main.yml index ae139556d68426ee87683463b783d1c7298ab0c5..b1b2dfca97295639d872c4fb7603904607acc14b 100644 --- a/roles/kubernetes-apps/helm/defaults/main.yml +++ b/roles/kubernetes-apps/helm/defaults/main.yml @@ -1 +1,4 @@ helm_enabled: false + +# specify a dir and attach it to helm for HELM_HOME. +helm_home_dir: "/root/.helm" diff --git a/roles/kubernetes-apps/helm/tasks/main.yml b/roles/kubernetes-apps/helm/tasks/main.yml index 907cc7c20374bd9bf6b1715149f0a429a278860b..1d50f8b9b996e999037588856ce278014987ec86 100644 --- a/roles/kubernetes-apps/helm/tasks/main.yml +++ b/roles/kubernetes-apps/helm/tasks/main.yml @@ -1,4 +1,7 @@ --- +- name: Helm | Make sure HELM_HOME directory exists + file: path={{ helm_home_dir }} state=directory + - name: Helm | Set up helm launcher template: src: helm-container.j2 @@ -8,7 +11,7 @@ register: helm_container - name: Helm | Install/upgrade helm - command: "helm init --upgrade --tiller-image={{ tiller_image_repo }}:{{ tiller_image_tag }}" + command: "{{ bin_dir }}/helm init --upgrade --tiller-image={{ tiller_image_repo }}:{{ tiller_image_tag }}" when: helm_container.changed - name: Helm | Set up bash completion diff --git a/roles/kubernetes-apps/helm/templates/helm-container.j2 b/roles/kubernetes-apps/helm/templates/helm-container.j2 index 598daa73a0b3c15e70b217341dbc9c25e6c37551..68210ea30ac85b03f34d9672451c611960695b0b 100644 --- a/roles/kubernetes-apps/helm/templates/helm-container.j2 +++ b/roles/kubernetes-apps/helm/templates/helm-container.j2 @@ -3,6 +3,7 @@ --net=host \ --name=helm \ -v /etc/ssl:/etc/ssl:ro \ + -v {{ helm_home_dir }}:{{ helm_home_dir }}:rw \ {% for dir in ssl_ca_dirs -%} -v {{ dir }}:{{ dir }}:ro \ {% endfor -%} diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 016df0c647734bf597a5aa78b747372eb276c7c6..785ef43afd62eee7020fdce6c46611e607e3586d 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -36,6 +36,13 @@ kube_apiserver_cpu_limit: 800m kube_apiserver_memory_requests: 256M kube_apiserver_cpu_requests: 100m +# Admission control plug-ins +kube_apiserver_admission_control: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - DefaultStorageClass + - ResourceQuota ## Enable/Disable Kube API Server Authentication Methods kube_basic_auth: true @@ -51,3 +58,10 @@ kube_oidc_auth: false # kube_oidc_ca_file: {{ kube_cert_dir }}/ca.pem # kube_oidc_username_claim: sub # kube_oidc_groups_claim: groups + +##Variables for custom flags +apiserver_custom_flags: [] + +controller_mgr_custom_flags: [] + +scheduler_custom_flags: [] \ No newline at end of file diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index 2c669c46dd98ea65feec9a8cf131c71de13726d7..dadef4bf5dc9c238f3f540d19f1ab34b7e3ec280 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -34,9 +34,9 @@ - meta: flush_handlers -- name: copy kube system namespace manifest - copy: - src: namespace.yml +- name: Write kube system namespace manifest + template: + src: namespace.j2 dest: "{{kube_config_dir}}/{{system_namespace}}-ns.yml" run_once: yes when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 967f0a9cbf5455079a745b26a28539d8a3afa313..b0f1a2f5319ae7509167136ef34cf0f85879ba3b 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -9,7 +9,7 @@ metadata: spec: hostNetwork: true {% if kube_version | version_compare('v1.6', '>=') %} - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: ClusterFirst {% endif %} containers: - name: kube-apiserver @@ -33,7 +33,7 @@ spec: - --etcd-keyfile={{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem - --insecure-bind-address={{ kube_apiserver_insecure_bind_address }} - --apiserver-count={{ kube_apiserver_count }} - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota + - --admission-control={{ kube_apiserver_admission_control | join(',') }} - --service-cluster-ip-range={{ kube_service_addresses }} - --service-node-port-range={{ kube_apiserver_node_port_range }} - --client-ca-file={{ kube_cert_dir }}/ca.pem @@ -80,6 +80,13 @@ spec: {% endif %} {% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=') %} - --anonymous-auth={{ kube_api_anonymous_auth }} +{% endif %} +{% if apiserver_custom_flags is string %} + - {{ apiserver_custom_flags }} +{% else %} +{% for flag in apiserver_custom_flags %} + - {{ flag }} +{% endfor %} {% endif %} livenessProbe: httpGet: diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 477d6a64fc86fe3a7ce16357c97ebc861b192722..d3f8a23a5d0003a841839b96db221154a48dd5aa 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -8,7 +8,7 @@ metadata: spec: hostNetwork: true {% if kube_version | version_compare('v1.6', '>=') %} - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: ClusterFirst {% endif %} containers: - name: kube-controller-manager @@ -45,6 +45,13 @@ spec: - --allocate-node-cidrs=true - --configure-cloud-routes=true - --cluster-cidr={{ kube_pods_subnet }} +{% endif %} +{% if controller_mgr_custom_flags is string %} + - {{ controller_mgr_custom_flags }} +{% else %} +{% for flag in controller_mgr_custom_flags %} + - {{ flag }} +{% endfor %} {% endif %} livenessProbe: httpGet: diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index 7431ddf3d7eb6ac81acf3b8b302befe3f821d7f2..441f991eb7f26bf59f9716677168cf24bb83e88f 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -8,7 +8,7 @@ metadata: spec: hostNetwork: true {% if kube_version | version_compare('v1.6', '>=') %} - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: ClusterFirst {% endif %} containers: - name: kube-scheduler @@ -27,6 +27,13 @@ spec: - --leader-elect=true - --master={{ kube_apiserver_endpoint }} - --v={{ kube_log_level }} +{% if scheduler_custom_flags is string %} + - {{ scheduler_custom_flags }} +{% else %} +{% for flag in scheduler_custom_flags %} + - {{ flag }} +{% endfor %} +{% endif %} livenessProbe: httpGet: host: 127.0.0.1 diff --git a/roles/kubernetes/master/files/namespace.yml b/roles/kubernetes/master/templates/namespace.j2 similarity index 100% rename from roles/kubernetes/master/files/namespace.yml rename to roles/kubernetes/master/templates/namespace.j2 diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 95221417992a5a70b013f13eb986899753f22194..7ef6d01e04c8a44febb63945d1c4c8afad301ea3 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -1,3 +1,6 @@ +# Valid options: docker (default), rkt, or host +kubelet_deployment_type: docker + # change to 0.0.0.0 to enable insecure access from anywhere (not recommended) kube_apiserver_insecure_bind_address: 127.0.0.1 @@ -45,3 +48,6 @@ etcd_config_dir: /etc/ssl/etcd kube_apiserver_node_port_range: "30000-32767" kubelet_load_modules: false + +##Support custom flags to be passed to kubelet +kubelet_custom_flags: [] diff --git a/roles/kubernetes/node/tasks/install_host.yml b/roles/kubernetes/node/tasks/install_host.yml new file mode 100644 index 0000000000000000000000000000000000000000..e80b204986cb4a8bf5d16f34ab401e9b8e327d52 --- /dev/null +++ b/roles/kubernetes/node/tasks/install_host.yml @@ -0,0 +1,10 @@ +--- +- name: install | Copy kubelet from hyperkube container + command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -f /hyperkube /systembindir/kubelet" + register: kubelet_task_result + until: kubelet_task_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false + tags: [hyperkube, upgrade] + notify: restart kubelet diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 324e38867125d99ca88e5f9c731c55d1441900b7..f09845f762f0ba90c8b8aa943c88853733565bce 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -7,6 +7,12 @@ - include: pre_upgrade.yml tags: kubelet +- name: Ensure /var/lib/cni exists + file: + path: /var/lib/cni + state: directory + mode: 0755 + - include: install.yml tags: kubelet diff --git a/roles/kubernetes/node/templates/kubelet-container.j2 b/roles/kubernetes/node/templates/kubelet-container.j2 index 75d067cf634da55645721297ac62db127ca67130..94c7f79a5a2fef6090489f7f29d9b2dd0a40c26a 100644 --- a/roles/kubernetes/node/templates/kubelet-container.j2 +++ b/roles/kubernetes/node/templates/kubelet-container.j2 @@ -25,6 +25,7 @@ -v /var/lib/cni:/var/lib/cni:shared \ -v /var/run:/var/run:rw \ -v {{kube_config_dir}}:{{kube_config_dir}}:ro \ + -v /etc/os-release:/etc/os-release:ro \ {{ hyperkube_image_repo }}:{{ hyperkube_image_tag}} \ ./hyperkube kubelet \ "$@" diff --git a/roles/kubernetes/node/templates/kubelet.docker.service.j2 b/roles/kubernetes/node/templates/kubelet.docker.service.j2 index e3bf408782a116138e12dcf12697aca235ada76f..cf79f6fa41b6420d2cc2d4d8e31b065a84b6adca 100644 --- a/roles/kubernetes/node/templates/kubelet.docker.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.docker.service.j2 @@ -23,10 +23,11 @@ ExecStart={{ bin_dir }}/kubelet \ $DOCKER_SOCKET \ $KUBELET_NETWORK_PLUGIN \ $KUBELET_CLOUDPROVIDER -ExecStartPre=-{{ docker_bin_dir }}/docker rm -f kubelet -ExecReload={{ docker_bin_dir }}/docker restart kubelet Restart=always RestartSec=10s +ExecStartPre=-{{ docker_bin_dir }}/docker rm -f kubelet +ExecReload={{ docker_bin_dir }}/docker restart kubelet + [Install] WantedBy=multi-user.target diff --git a/roles/kubernetes/node/templates/kubelet.host.service.j2 b/roles/kubernetes/node/templates/kubelet.host.service.j2 new file mode 100644 index 0000000000000000000000000000000000000000..71a9da8c3b1eef0610ee4d0866e5b9ac1e4cbf05 --- /dev/null +++ b/roles/kubernetes/node/templates/kubelet.host.service.j2 @@ -0,0 +1,30 @@ +[Unit] +Description=Kubernetes Kubelet Server +Documentation=https://github.com/GoogleCloudPlatform/kubernetes +{% if kube_network_plugin is defined and kube_network_plugin == "calico" %} +After=docker.service docker.socket calico-node.service +Wants=docker.socket calico-node.service +{% else %} +After=docker.service +Wants=docker.socket +{% endif %} + +[Service] +EnvironmentFile={{kube_config_dir}}/kubelet.env +ExecStart={{ bin_dir }}/kubelet \ + $KUBE_LOGTOSTDERR \ + $KUBE_LOG_LEVEL \ + $KUBELET_API_SERVER \ + $KUBELET_ADDRESS \ + $KUBELET_PORT \ + $KUBELET_HOSTNAME \ + $KUBE_ALLOW_PRIV \ + $KUBELET_ARGS \ + $DOCKER_SOCKET \ + $KUBELET_NETWORK_PLUGIN \ + $KUBELET_CLOUDPROVIDER +Restart=always +RestartSec=10s + +[Install] +WantedBy=multi-user.target diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index ba02e5eb93b6fc4628dd05ec7b3ff745d26ef24a..d2959b8a64fa399edd439d193bc906c427345396 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -19,13 +19,13 @@ KUBELET_HOSTNAME="--hostname-override={{ ansible_hostname }}" {# DNS settings for kubelet #} {% if dns_mode == 'kubedns' %} -{% set kubelet_args_cluster_dns %}--cluster_dns={{ skydns_server }}{% endset %} +{% set kubelet_args_cluster_dns %}--cluster-dns={{ skydns_server }}{% endset %} {% elif dns_mode == 'dnsmasq_kubedns' %} -{% set kubelet_args_cluster_dns %}--cluster_dns={{ dns_server }}{% endset %} +{% set kubelet_args_cluster_dns %}--cluster-dns={{ dns_server }}{% endset %} {% else %} {% set kubelet_args_cluster_dns %}{% endset %} {% endif %} -{% set kubelet_args_dns %}{{ kubelet_args_cluster_dns }} --cluster_domain={{ dns_domain }} --resolv-conf={{ kube_resolv_conf }}{% endset %} +{% set kubelet_args_dns %}{{ kubelet_args_cluster_dns }} --cluster-domain={{ dns_domain }} --resolv-conf={{ kube_resolv_conf }}{% endset %} {# Location of the apiserver #} {% set kubelet_args_kubeconfig %}--kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --require-kubeconfig{% endset %} @@ -44,7 +44,7 @@ KUBELET_HOSTNAME="--hostname-override={{ ansible_hostname }}" {% set node_labels %}--node-labels=node-role.kubernetes.io/node=true{% endset %} {% endif %} -KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }}" +KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave", "canal"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %} diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 index be8a13dbfddbbee5313393fa336547a8d420df5d..5f83514587414ce92d507666eec0795c06fa1cf4 100644 --- a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 @@ -20,6 +20,7 @@ ExecStartPre=-/bin/mkdir -p /var/lib/kubelet EnvironmentFile={{kube_config_dir}}/kubelet.env # stage1-fly mounts /proc /sys /dev so no need to duplicate the mounts ExecStart=/usr/bin/rkt run \ + --volume os-release,kind=host,source=/etc/os-release,readOnly=true \ --volume dns,kind=host,source=/etc/resolv.conf \ --volume etc-kubernetes,kind=host,source={{ kube_config_dir }},readOnly=false \ --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ @@ -39,6 +40,7 @@ ExecStart=/usr/bin/rkt run \ --mount volume=opt-cni,target=/opt/cni \ --mount volume=var-lib-cni,target=/var/lib/cni \ {% endif %} + --mount volume=os-release,target=/etc/os-release \ --mount volume=dns,target=/etc/resolv.conf \ --mount volume=etc-kubernetes,target={{ kube_config_dir }} \ --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index 745c671d8eb8cfa09b53443282369cbe0c6badb7..9b7d5385752fc0e70db9577355c9c63f47078ba9 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -8,7 +8,7 @@ metadata: spec: hostNetwork: true {% if kube_version | version_compare('v1.6', '>=') %} - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: ClusterFirst {% endif %} containers: - name: kube-proxy diff --git a/roles/kubernetes/preinstall/handlers/main.yml b/roles/kubernetes/preinstall/handlers/main.yml index f7e309e92fda68aeee861971eb4140b6dca1b37d..35fec7d94d2f9823b5549933dc12bad9bc34ad8f 100644 --- a/roles/kubernetes/preinstall/handlers/main.yml +++ b/roles/kubernetes/preinstall/handlers/main.yml @@ -45,5 +45,5 @@ when: inventory_hostname in groups['kube-master'] and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' - name: Preinstall | restart kube-controller-manager - shell: "docker ps -f name=k8s-controller-manager* -q | xargs --no-run-if-empty docker rm -f" + shell: "docker ps -f name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty docker rm -f" when: inventory_hostname in groups['kube-master'] and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' and kube_controller_set.stat.exists diff --git a/roles/kubernetes/preinstall/tasks/etchosts.yml b/roles/kubernetes/preinstall/tasks/etchosts.yml index 181fbcb0f400b25e522fc02c2b1cd586e263b648..df330be088c344181cc3560b9fdf5382497146a7 100644 --- a/roles/kubernetes/preinstall/tasks/etchosts.yml +++ b/roles/kubernetes/preinstall/tasks/etchosts.yml @@ -17,7 +17,10 @@ line: "{{ loadbalancer_apiserver.address }} {{ apiserver_loadbalancer_domain_name| default('lb-apiserver.kubernetes.local') }}" state: present backup: yes - when: loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined and apiserver_loadbalancer_domain_name is defined + when: + - loadbalancer_apiserver is defined + - loadbalancer_apiserver.address is defined + - apiserver_loadbalancer_domain_name is defined - name: Hosts | localhost ipv4 in hosts file lineinfile: diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 3ae785d6eaec712ec58547e3b12e7115c1895d1c..2f5bff2290d2ed7c1c4be97415bd218622559014 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -43,7 +43,7 @@ path: "{{ kube_config_dir }}" state: directory owner: kube - when: "{{ inventory_hostname in groups['k8s-cluster'] }}" + when: inventory_hostname in groups['k8s-cluster'] tags: [kubelet, k8s-secrets, kube-controller-manager, kube-apiserver, bootstrap-os, apps, network, master, node] - name: Create kubernetes script directory @@ -51,7 +51,7 @@ path: "{{ kube_script_dir }}" state: directory owner: kube - when: "{{ inventory_hostname in groups['k8s-cluster'] }}" + when: "inventory_hostname in groups['k8s-cluster']" tags: [k8s-secrets, bootstrap-os] - name: Create kubernetes manifests directory @@ -59,17 +59,21 @@ path: "{{ kube_manifest_dir }}" state: directory owner: kube - when: "{{ inventory_hostname in groups['k8s-cluster'] }}" + when: "inventory_hostname in groups['k8s-cluster']" tags: [kubelet, bootstrap-os, master, node] - name: check cloud_provider value fail: msg: "If set the 'cloud_provider' var must be set either to 'generic', 'gce', 'aws', 'azure', 'openstack' or 'vsphere'" - when: cloud_provider is defined and cloud_provider not in ['generic', 'gce', 'aws', 'azure', 'openstack', 'vsphere'] + when: + - cloud_provider is defined + - cloud_provider not in ['generic', 'gce', 'aws', 'azure', 'openstack', 'vsphere'] tags: [cloud-provider, facts] - include: "{{ cloud_provider }}-credential-check.yml" - when: cloud_provider is defined and cloud_provider in [ 'openstack', 'azure', 'vsphere' ] + when: + - cloud_provider is defined + - cloud_provider in [ 'openstack', 'azure', 'vsphere' ] tags: [cloud-provider, facts] - name: Create cni directories @@ -80,7 +84,9 @@ with_items: - "/etc/cni/net.d" - "/opt/cni/bin" - when: kube_network_plugin in ["calico", "weave", "canal"] and "{{ inventory_hostname in groups['k8s-cluster'] }}" + when: + - kube_network_plugin in ["calico", "weave", "canal"] + - inventory_hostname in groups['k8s-cluster'] tags: [network, calico, weave, canal, bootstrap-os] - name: Update package management cache (YUM) @@ -91,7 +97,9 @@ until: yum_task_result|succeeded retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: ansible_pkg_mgr == 'yum' and not is_atomic + when: + - ansible_pkg_mgr == 'yum' + - not is_atomic tags: bootstrap-os - name: Install latest version of python-apt for Debian distribs @@ -109,14 +117,17 @@ until: dnf_task_result|succeeded retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: ansible_distribution == "Fedora" and - ansible_distribution_major_version > 21 + when: + - ansible_distribution == "Fedora" + - ansible_distribution_major_version > 21 changed_when: False tags: bootstrap-os - name: Install epel-release on RedHat/CentOS shell: rpm -qa | grep epel-release || rpm -ivh {{ epel_rpm_download_url }} - when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic + when: + - ansible_distribution in ["CentOS","RedHat"] + - not is_atomic register: epel_task_result until: epel_task_result|succeeded retries: 4 @@ -149,7 +160,9 @@ selinux: policy: targeted state: permissive - when: ansible_os_family == "RedHat" and slc.stat.exists == True + when: + - ansible_os_family == "RedHat" + - slc.stat.exists == True changed_when: False tags: bootstrap-os @@ -159,7 +172,9 @@ line: "precedence ::ffff:0:0/96 100" state: present backup: yes - when: disable_ipv6_dns and not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] + when: + - disable_ipv6_dns + - not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] tags: bootstrap-os - name: set default sysctl file path @@ -176,7 +191,9 @@ - name: Change sysctl file path to link source if linked set_fact: sysctl_file_path: "{{sysctl_file_stat.stat.lnk_source}}" - when: sysctl_file_stat.stat.islnk is defined and sysctl_file_stat.stat.islnk + when: + - sysctl_file_stat.stat.islnk is defined + - sysctl_file_stat.stat.islnk tags: bootstrap-os - name: Enable ip forwarding @@ -193,22 +210,33 @@ dest: "{{ kube_config_dir }}/cloud_config" group: "{{ kube_cert_group }}" mode: 0640 - when: inventory_hostname in groups['k8s-cluster'] and cloud_provider is defined and cloud_provider in [ 'openstack', 'azure', 'vsphere' ] + when: + - inventory_hostname in groups['k8s-cluster'] + - cloud_provider is defined + - cloud_provider in [ 'openstack', 'azure', 'vsphere' ] tags: [cloud-provider] - include: etchosts.yml tags: [bootstrap-os, etchosts] - include: resolvconf.yml - when: dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' + when: + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' tags: [bootstrap-os, resolvconf] - include: dhclient-hooks.yml - when: dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' and not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] + when: + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] tags: [bootstrap-os, resolvconf] - include: dhclient-hooks-undo.yml - when: dns_mode != 'none' and resolvconf_mode != 'host_resolvconf' and not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] + when: + - dns_mode != 'none' + - resolvconf_mode != 'host_resolvconf' + - not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] tags: [bootstrap-os, resolvconf] - name: Check if we are running inside a Azure VM @@ -218,7 +246,7 @@ tags: bootstrap-os - include: growpart-azure-centos-7.yml - when: azure_check.stat.exists and - ansible_distribution in ["CentOS","RedHat"] + when: + - azure_check.stat.exists + - ansible_distribution in ["CentOS","RedHat"] tags: bootstrap-os - diff --git a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml index ffea74b40b9db1c8727cd795bffc95fe3340ae20..18728faa79716a738e2729f10c41283c66747fe7 100644 --- a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml @@ -16,7 +16,13 @@ {{dns_domain}}.{{d}}./{{d}}.{{d}}./com.{{d}}./ {%- endfor %} default_resolver: >- - {%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%} + {%- if cloud_provider is defined and cloud_provider == 'gce' -%} + 169.254.169.254 + {%- elif cloud_provider is defined and cloud_provider == 'aws' -%} + 169.254.169.253 + {%- else -%} + 8.8.8.8 + {%- endif -%} - name: check if kubelet is configured stat: diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh index 3cea6d79ee94414cfdc9565c224933adb65191af..55ea13d1e0dee106e011ea6f55b4556448654555 100755 --- a/roles/kubernetes/secrets/files/make-ssl.sh +++ b/roles/kubernetes/secrets/files/make-ssl.sh @@ -85,7 +85,7 @@ if [ -n "$MASTERS" ]; then cn="${host%%.*}" # admin key openssl genrsa -out admin-${host}-key.pem 2048 > /dev/null 2>&1 - openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=kube-admin-${cn}" > /dev/null 2>&1 + openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=kube-admin-${cn}/O=system:masters" > /dev/null 2>&1 openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days 3650 > /dev/null 2>&1 done fi diff --git a/roles/kubernetes/secrets/tasks/gen_certs_vault.yml b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml index 5a7c4827bf1e2d4f6c62340cb381bba19ef3a141..ac8e128b4427b68f36cdea6be8c130948cdd98bb 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs_vault.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml @@ -1,4 +1,11 @@ --- +- include: sync_kube_master_certs.yml + when: inventory_hostname in groups['kube-master'] + tags: k8s-secrets + +- include: sync_kube_node_certs.yml + when: inventory_hostname in groups['k8s-cluster'] + tags: k8s-secrets - name: gen_certs_vault | Read in the local credentials command: cat /etc/vault/roles/kube/userpass @@ -15,7 +22,7 @@ url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ kube_vault_creds.username }}" headers: Accept: application/json - Content-Type: application/json + Content-Type: application/json method: POST body_format: json body: @@ -54,7 +61,7 @@ }} issue_cert_file_group: "{{ kube_cert_group }}" issue_cert_file_owner: kube - issue_cert_headers: "{{ kube_vault_headers }}" + issue_cert_headers: "{{ kube_vault_headers }}" issue_cert_hosts: "{{ groups['kube-master'] }}" issue_cert_ip_sans: >- [ @@ -75,7 +82,7 @@ issue_cert_copy_ca: "{{ item == kube_node_certs_needed|first }}" issue_cert_file_group: "{{ kube_cert_group }}" issue_cert_file_owner: kube - issue_cert_headers: "{{ kube_vault_headers }}" + issue_cert_headers: "{{ kube_vault_headers }}" issue_cert_hosts: "{{ groups['k8s-cluster'] }}" issue_cert_path: "{{ item }}" issue_cert_role: kube diff --git a/roles/kubernetes/secrets/tasks/main.yml b/roles/kubernetes/secrets/tasks/main.yml index 919ed0df76450c03aea3fc857e8ad659245e7121..fb4c38f3801f6ee13551f5fb5d36256e79721df9 100644 --- a/roles/kubernetes/secrets/tasks/main.yml +++ b/roles/kubernetes/secrets/tasks/main.yml @@ -74,13 +74,5 @@ - include: "gen_certs_{{ cert_management }}.yml" tags: k8s-secrets -- include: sync_kube_master_certs.yml - when: cert_management == "vault" and inventory_hostname in groups['kube-master'] - tags: k8s-secrets - -- include: sync_kube_node_certs.yml - when: cert_management == "vault" and inventory_hostname in groups['k8s-cluster'] - tags: k8s-secrets - - include: gen_tokens.yml tags: k8s-secrets diff --git a/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml index 0561d65811ecc2b9b78a19a883c7e86712232133..b02120ccb92a1623b8fedb41a7f228709286b9a7 100644 --- a/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml +++ b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml @@ -6,7 +6,7 @@ with_items: "{{ groups['kube-master'] }}" - include: ../../../vault/tasks/shared/sync_file.yml - vars: + vars: sync_file: "{{ item }}" sync_file_dir: "{{ kube_cert_dir }}" sync_file_group: "{{ kube_cert_group }}" @@ -38,7 +38,7 @@ set_fact: kube_api_certs_needed: "{{ item.path }}" with_items: "{{ sync_file_results|d([]) }}" - when: "{{ item.no_srcs }}" + when: item.no_srcs - name: sync_kube_master_certs | Unset sync_file_results after apiserver cert set_fact: @@ -46,7 +46,7 @@ - include: ../../../vault/tasks/shared/sync_file.yml - vars: + vars: sync_file: ca.pem sync_file_dir: "{{ kube_cert_dir }}" sync_file_group: "{{ kube_cert_group }}" diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml index 2f3096bf3dc3c2e8b4367b10e34ffecd8c715867..eda9c2934540dfe4039734244858826e82a8111e 100644 --- a/roles/network_plugin/calico/tasks/main.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -56,7 +56,7 @@ retries: 4 delay: "{{ retry_stagger | random + 3 }}" changed_when: false - when: "{{ overwrite_hyperkube_cni|bool }}" + when: overwrite_hyperkube_cni|bool tags: [hyperkube, upgrade] - name: Calico | Set cni directory permissions diff --git a/roles/reset/defaults/main.yml b/roles/reset/defaults/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..7d4dbfdff9dbabd2ae60cc8232c0dafee5ee7461 --- /dev/null +++ b/roles/reset/defaults/main.yml @@ -0,0 +1,2 @@ +--- +flush_iptables: true diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index f5f749647f72cb8e2d62bf7f3185206056bec223..96984f92b9b67d70fe390905d3a9b3af3171d9a0 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -8,6 +8,7 @@ - kubelet - etcd failed_when: false + tags: ['services'] - name: reset | remove services file: @@ -17,6 +18,7 @@ - kubelet - etcd register: services_removed + tags: ['services'] - name: reset | remove docker dropins file: @@ -26,6 +28,7 @@ - docker-dns.conf - docker-options.conf register: docker_dropins_removed + tags: ['docker'] - name: reset | systemctl daemon-reload command: systemctl daemon-reload @@ -33,25 +36,31 @@ - name: reset | remove all containers shell: "{{ docker_bin_dir }}/docker ps -aq | xargs -r docker rm -fv" + tags: ['docker'] - name: reset | restart docker if needed service: name: docker state: restarted when: docker_dropins_removed.changed + tags: ['docker'] - name: reset | gather mounted kubelet dirs shell: mount | grep /var/lib/kubelet | awk '{print $3}' | tac check_mode: no register: mounted_dirs + tags: ['mounts'] - name: reset | unmount kubelet dirs command: umount {{item}} with_items: '{{ mounted_dirs.stdout_lines }}' + tags: ['mounts'] - name: flush iptables iptables: flush: yes + when: flush_iptables|bool + tags: ['iptables'] - name: reset | delete some files and directories file: @@ -74,6 +83,8 @@ - /etc/dhcp/dhclient.d/zdnsupdate.sh - /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate - "{{ bin_dir }}/kubelet" + tags: ['files'] + - name: reset | remove dns settings from dhclient.conf blockinfile: @@ -85,6 +96,7 @@ with_items: - /etc/dhclient.conf - /etc/dhcp/dhclient.conf + tags: ['files', 'dns'] - name: reset | remove host entries from /etc/hosts blockinfile: @@ -92,6 +104,7 @@ state: absent follow: yes marker: "# Ansible inventory hosts {mark}" + tags: ['files', 'dns'] - name: reset | Restart network service: @@ -103,3 +116,4 @@ {%- endif %} state: restarted when: ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] + tags: ['services', 'network'] diff --git a/roles/vault/tasks/bootstrap/start_vault_temp.yml b/roles/vault/tasks/bootstrap/start_vault_temp.yml index eeaaad53501ea4595e375e3129447d0b5ccdea32..4a5e6bc5ed0c2006a4f1bf326eab14004042eca2 100644 --- a/roles/vault/tasks/bootstrap/start_vault_temp.yml +++ b/roles/vault/tasks/bootstrap/start_vault_temp.yml @@ -3,7 +3,7 @@ - name: bootstrap/start_vault_temp | Ensure vault-temp isn't already running shell: if docker rm -f {{ vault_temp_container_name }} 2>&1 1>/dev/null;then echo true;else echo false;fi register: vault_temp_stop_check - changed_when: "{{ 'true' in vault_temp_stop_check.stdout }}" + changed_when: "'true' in vault_temp_stop_check.stdout" - name: bootstrap/start_vault_temp | Start single node Vault with file backend command: > @@ -13,6 +13,10 @@ -v /etc/vault:/etc/vault {{ vault_image_repo }}:{{ vault_version }} server +#FIXME(mattymo): Crashes on first start with aufs docker storage. See hashicorp/docker-vault#19 +- name: bootstrap/start_vault_temp | Start again single node Vault with file backend + command: docker start {{ vault_temp_container_name }} + - name: bootstrap/start_vault_temp | Initialize vault-temp uri: url: "http://localhost:{{ vault_port }}/v1/sys/init"