Adding kubelet in rkt

.gitlab-ci.yml

@@ -49,6 +49,8 @@ before_script:
   ANSIBLE_KEEP_REMOTE_FILES: "1"
   BOOTSTRAP_OS: none
   LOG_LEVEL: "-vv"
+  ETCD_DEPLOYMENT: "docker"
+  KUBELET_DEPLOYMENT: "docker"
 
 .gce: &gce
   <<: *job
@@ -102,6 +104,8 @@ before_script:
       -e download_localhost=true
       -e deploy_netchecker=true
       -e local_release_dir=${PWD}/downloads
+      -e etcd_deployment_type=${ETCD_DEPLOYMENT}
+      -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
       cluster.yml
 
 
@@ -202,6 +206,15 @@ before_script:
   CLUSTER_MODE: ha
   BOOTSTRAP_OS: coreos
 
+.ubuntu_rkt_sep_variables: &ubuntu_rkt_sep_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: flannel
+  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_REGION: us-central1-b
+  CLUSTER_MODE: separated
+  ETCD_DEPLOYMENT: rkt
+  KUBELET_DEPLOYMENT: rkt
+
 # Builds for PRs only (auto) and triggers (auto)
 coreos-calico-sep:
   stage: deploy-gce-part1
@@ -405,6 +418,17 @@ coreos-alpha-weave-ha:
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
+ubuntu-rkt-sep:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_rkt_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
 syntax-check:
   <<: *job
   stage: unit-tests

roles/download/defaults/main.yml

@@ -115,13 +115,13 @@ downloads:
     version: "{{etcd_version}}"
     dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
     sha256: >-
-      {%- if etcd_deployment_type == 'docker' -%}{{etcd_digest_checksum|default(None)}}{%- else -%}{{etcd_checksum}}{%- endif -%}
+      {%- if etcd_deployment_type in [ 'docker', 'rkt' ] -%}{{etcd_digest_checksum|default(None)}}{%- else -%}{{etcd_checksum}}{%- endif -%}
     source_url: "{{ etcd_download_url }}"
     url: "{{ etcd_download_url }}"
     unarchive: true
     owner: "etcd"
     mode: "0755"
-    container: "{{ etcd_deployment_type }} in [ 'docker', 'rkt' ]"
+    container: "{{ etcd_deployment_type in [ 'docker', 'rkt' ] }}"
     repo: "{{ etcd_image_repo }}"
     tag: "{{ etcd_image_tag }}"
   hyperkube:

roles/etcd/templates/etcd-rkt.service.j2

@@ -9,7 +9,7 @@ RestartSec=10s
 TimeoutStartSec=0
 LimitNOFILE=40000
 
-ExecStart={{ rkt_bin_dir | default("/usr/bin") }}/rkt run \
+ExecStart=/usr/bin/rkt run \
 --uuid-file-save=/var/run/etcd.uuid \
 --volume=etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
 --mount=volume=etc-ssl-certs,target=/etc/ssl/certs \

roles/kubernetes/node/tasks/install.yml

 ---
+- name: Trust kubelet container
+  command: >-
+    /usr/bin/rkt trust
+    --skip-fingerprint-review
+    --root
+    {{ item }}
+  register: kubelet_rkt_trust_result
+  until: kubelet_rkt_trust_result.rc == 0
+  with_items:
+    - "https://quay.io/aci-signing-key"
+    - "https://coreos.com/dist/pubkeys/aci-pubkeys.gpg"
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false
+  when: kubelet_deployment_type == "rkt"
+
+- name: create kubelet working directory
+  file:
+    state: directory
+    path: /var/lib/kubelet
+  when: kubelet_deployment_type == "rkt"
+
 - name: install | Write kubelet systemd init file
-  template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes
+  template: "src=kubelet.{{ kubelet_deployment_type }}.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes"
   notify: restart kubelet
 
 - name: install | Install kubelet launch script
   template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes
   notify: restart kubelet
+  when: kubelet_deployment_type == "docker"

roles/kubernetes/node/templates/kubelet.rkt.service.j2

+[Unit]
+Description=Kubernetes Kubelet Server
+Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
+After=calico-node.service
+Wants=network.target calico-node.service
+{% else %}
+Wants=network.target
+{% endif %}
+
+[Service]
+Restart=on-failure
+RestartSec=10s
+TimeoutStartSec=0
+LimitNOFILE=40000
+
+ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet.uuid
+ExecStartPre=-/bin/mkdir -p /var/lib/kubelet
+
+EnvironmentFile={{kube_config_dir}}/kubelet.env
+# stage1-fly mounts /proc /sys /dev so no need to duplicate the mounts
+ExecStart=/usr/bin/rkt run \
+        --volume var-log,kind=host,source=/var/log \
+        --volume dns,kind=host,source=/etc/resolv.conf \
+        --volume etc-kubernetes,kind=host,source={{ kube_config_dir }},readOnly=false \
+        --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
+        --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
+        --volume var-lib-docker,kind=host,source={{ docker_daemon_graph }},readOnly=false \
+	--volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=false \
+        --volume run,kind=host,source=/run,readOnly=false \
+        --mount volume=var-log,target=/var/log \
+        --mount volume=dns,target=/etc/resolv.conf \
+        --mount volume=etc-kubernetes,target={{ kube_config_dir }} \
+        --mount volume=etc-ssl-certs,target=/etc/ssl/certs \
+        --mount volume=usr-share-certs,target=/usr/share/ca-certificates \
+        --mount volume=var-lib-docker,target=/var/lib/docker \
+        --mount volume=var-lib-kubelet,target=/var/lib/kubelet \
+        --mount volume=run,target=/run \
+        --stage1-from-dir=stage1-fly.aci \
+        {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} \
+        --uuid-file-save=/var/run/kubelet.uuid \
+        --debug --exec=/kubelet -- \
+                $KUBE_LOGTOSTDERR \
+                $KUBE_LOG_LEVEL \
+                $KUBELET_API_SERVER \
+                $KUBELET_ADDRESS \
+                $KUBELET_PORT \
+                $KUBELET_HOSTNAME \
+                $KUBE_ALLOW_PRIV \
+                $KUBELET_ARGS \
+                $DOCKER_SOCKET \
+                $KUBELET_REGISTER_NODE \
+                $KUBELET_NETWORK_PLUGIN
+
+ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet.uuid
+
+[Install]
+WantedBy=multi-user.target