Vault security hardening and role isolation

.gitlab-ci.yml

@@ -54,6 +54,7 @@ before_script:
   LOG_LEVEL: "-vv"
   ETCD_DEPLOYMENT: "docker"
   KUBELET_DEPLOYMENT: "docker"
+  VAULT_DEPLOYMENT: "docker"
   WEAVE_CPU_LIMIT: "100m"
   MAGIC: "ci check this"
 
@@ -106,6 +107,7 @@ before_script:
       -e ansible_python_interpreter=${PYPATH}
       -e ansible_ssh_user=${SSH_USER} 
       -e bootstrap_os=${BOOTSTRAP_OS}
+      -e cert_management=${CERT_MGMT:-script}
       -e cloud_provider=gce
       -e deploy_netchecker=true
       -e download_localhost=true
@@ -115,6 +117,7 @@ before_script:
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
       -e local_release_dir=${PWD}/downloads
       -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
       cluster.yml
 
 
@@ -292,6 +295,14 @@ before_script:
   ETCD_DEPLOYMENT: rkt
   KUBELET_DEPLOYMENT: rkt
 
+.ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: canal
+  CERT_MGMT: vault
+  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_REGION: us-central1-b
+  CLUSTER_MODE: separate
+
 # Builds for PRs only (premoderated by unit-tests step) and triggers (auto)
 coreos-calico-sep:
   stage: deploy-gce-part1
@@ -506,6 +517,17 @@ ubuntu-rkt-sep:
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
+ubuntu-vault-sep:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_vault_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
 # Premoderated with manual actions
 ci-authorized:
   <<: *job

cluster.yml

@@ -28,14 +28,21 @@
   roles:
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker }
-    - { role: rkt, tags: rkt, when: "'rkt' in [ etcd_deployment_type, kubelet_deployment_type ]" }
+    - role: rkt
+      tags: rkt
+      when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
 
-- hosts: all
+- hosts: etcd:k8s-cluster:vault
   any_errors_fatal: true
   roles:
     - { role: vault, tags: vault, vault_bootstrap: true, when: "cert_management == 'vault'" }
 
-- hosts: etcd:k8s-cluster
+- hosts: etcd:!k8s-cluster
+  any_errors_fatal: true
+  roles:
+    - { role: etcd, tags: etcd }
+
+- hosts: k8s-cluster
   any_errors_fatal: true
   roles:
     - { role: etcd, tags: etcd }

docs/vault.md

+Hashicorp Vault Role
+====================
+
+Overview
+--------
+
+The Vault role is a two-step process:
+
+1. Bootstrap
+
+You cannot start your certificate management service securely with SSL (and 
+the datastore behind it) without having the certificates in-hand already. This
+presents an unfortunate chicken and egg scenario, with one requiring the other.
+To solve for this, the Bootstrap step was added.
+
+This step spins up a temporary instance of Vault to issue certificates for
+Vault itself. It then leaves the temporary instance running, so that the Etcd
+role can generate certs for itself as well. Eventually, this may be improved
+to allow alternate backends (such as Consul), but currently the tasks are
+hardcoded to only create a Vault role for Etcd.
+
+2. Cluster
+
+This step is where the long-term Vault cluster is started and configured. Its
+first task, is to stop any temporary instances of Vault, to free the port for
+the long-term. At the end of this task, the entire Vault cluster should be up
+and read to go.
+
+
+Keys to the Kingdom
+-------------------
+
+The two most important security pieces of Vault are the ``root_token``
+and ``unsealing_keys``. Both of these values are given exactly once, during
+the initialization of the Vault cluster. For convenience, they are saved
+to the ``vault_secret_dir`` (default: /etc/vault/secrets) of every host in the
+vault group.
+
+It is *highly* recommended that these secrets are removed from the servers after
+your cluster has been deployed, and kept in a safe location of your choosing.
+Naturally, the seriousness of the situation depends on what you're doing with
+your Kargo cluster, but with these secrets, an attacker will have the ability
+to authenticate to almost everything in Kubernetes and decode all private
+(HTTPS) traffic on your network signed by Vault certificates.
+
+For even greater security, you may want to remove and store elsewhere any
+CA keys generated as well (e.g. /etc/vault/ssl/ca-key.pem). 
+
+Vault by default encrypts all traffic to and from the datastore backend, all
+resting data, and uses TLS for its TCP listener. It is recommended that you
+do not change the Vault config to disable TLS, unless you absolutely have to.
+
+
+Usage
+-----
+
+To get the Vault role running, you must to do two things at a minimum:
+
+1. Assign the ``vault`` group to at least 1 node in your inventory
+2. Change ``cert_management`` to be ``vault`` instead of ``script``
+
+Nothing else is required, but customization is possible. Check
+``roles/vault/defaults/main.yml`` for the different variables that can be
+overridden, most common being ``vault_config``, ``vault_port``, and
+``vault_deployment_type``.
+
+Also, if you intend to use a Root or Intermediate CA generated elsewhere,
+you'll need to copy the certificate and key to the hosts in the vault group
+prior to running the vault role. By default, they'll be located at
+``/etc/vault/ssl/ca.pem`` and ``/etc/vault/ssl/ca-key.pem``, respectively.
+
+Additional Notes:
+
+- ``groups.vault|first`` is considered the source of truth for Vault variables
+- ``vault_leader_url`` is used as pointer for the current running Vault
+- Each service should have its own role and credentials. Currently those 
+  credentials are saved to ``/etc/vault/roles/<role>/``. The service will
+  need to read in those credentials, if they want to interact with Vault.
+
+
+Potential Work
+--------------
+
+- Change the Vault role to not run certain tasks when ``root_token`` and
+  ``unseal_keys`` are not present. Alternatively, allow user input for these
+  values when missing.
+- Add the ability to start temp Vault with Host, Rkt, or Docker
+- Add a dynamic way to change out the backend role creation during Bootstrap,
+  so other services can be used (such as Consul)
+- Segregate Server Cert generation from Auth Cert generation (separate CAs).
+  This work was partially started with the `auth_cert_backend` tasks, but would
+  need to be further applied to all roles (particularly Etcd and Kubernetes).

inventory/group_vars/all.yml

@@ -204,6 +204,7 @@ kpm_packages: []
 rkt_version: 1.21.0
 etcd_deployment_type: docker
 kubelet_deployment_type: docker
+vault_deployment_type: docker
 
 efk_enabled: false
 

roles/adduser/defaults/main.yml

@@ -14,14 +14,6 @@ addusers:
     system: yes
     group: "{{ kube_cert_group }}"
     createhome: no
-  vault:
-    comment: "Hashicorp Vault user"
-    createhome: no
-    name: vault
-    shell: /sbin/nologin
-    system: yes
-
-
 
 adduser:
   name: "{{ user.name }}"

roles/download/defaults/main.yml

@@ -26,7 +26,6 @@ calico_cni_version: "v1.5.5"
 weave_version: 1.8.2
 flannel_version: v0.6.2
 pod_infra_version: 3.0
-vault_version: 0.6.3
 
 # Download URL's
 etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd"

roles/download/tasks/main.yml

@@ -94,7 +94,7 @@
 
 - name: "Set default value for 'container_changed' to false"
   set_fact:
-    container_changed: "{{pull_required|bool|default(false)}}"
+    container_changed: "{{pull_required|default(false)|bool}}"
 
 - name: "Update the 'container_changed' fact"
   set_fact:

roles/etcd/defaults/main.yml

@@ -2,7 +2,6 @@
 etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/"
 
 etcd_config_dir: /etc/ssl/etcd
-# Role vault.boostrap has an implicit requirement on this var. It should be set at a higher level (inventory+)
 etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
 etcd_cert_group: root
 
@@ -16,3 +15,5 @@ etcd_memory_limit: 512M
 
 # Uncomment to set CPU share for etcd
 #etcd_cpu_limit: 300m
+
+etcd_node_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.get('calico-rr', [])) }}"

roles/etcd/meta/main.yml

@@ -6,3 +6,5 @@ dependencies:
   - role: download
     file: "{{ downloads.etcd }}"
     tags: download
+
+# NOTE: Dynamic task dependency on Vault Role if cert_management == "vault"

roles/etcd/tasks/gen_certs_vault.yml

+---
+
+- name: gen_certs_vault | Read in the local credentials
+  command: cat /etc/vault/roles/etcd/userpass
+  register: etcd_vault_creds_cat
+  when: inventory_hostname == groups.etcd|first
+
+- name: gen_certs_vault | Set facts for read Vault Creds
+  set_fact:
+    etcd_vault_creds: "{{ hostvars[groups.etcd|first]['etcd_vault_creds_cat']['stdout']|from_json }}"
+  when: inventory_hostname == groups.etcd|first
+
+- name: gen_certs_vault | Log into Vault and obtain an token
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ etcd_vault_creds.username }}"
+    headers:
+      Accept: application/json
+      Content-Type: application/json 
+    method: POST
+    body_format: json
+    body:
+      password: "{{ etcd_vault_creds.password }}"
+  register: etcd_vault_login_result
+  when: inventory_hostname == groups.etcd|first
+
+- name: gen_certs_vault | Set fact for Vault API token
+  set_fact:
+    etcd_vault_headers:
+        Accept: application/json
+        Content-Type: application/json
+        X-Vault-Token: "{{ hostvars[groups.etcd|first]['etcd_vault_login_result']['json']['auth']['client_token'] }}"
+
+# Issue master certs to Etcd nodes
+- include: ../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_alt_names: "{{ groups.etcd + ['localhost'] }}"
+    issue_cert_copy_ca: "{{ item == etcd_master_certs_needed|first }}"
+    issue_cert_file_group: "{{ etcd_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ etcd_vault_headers }}" 
+    issue_cert_hosts: "{{ groups.etcd }}"
+    issue_cert_ip_sans: >-
+        [
+        {%- for host in groups.etcd  -%}
+        "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+        {%- endfor -%}
+        "127.0.0.1","::1"
+        ]
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: etcd
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ etcd_master_certs_needed|d([]) }}"
+  when: inventory_hostname in groups.etcd
+  notify: set etcd_secret_changed
+
+# Issue node certs to everyone else
+- include: ../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_alt_names: "{{ etcd_node_cert_hosts }}"
+    issue_cert_copy_ca: "{{ item == etcd_node_certs_needed|first }}"
+    issue_cert_file_group: "{{ etcd_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ etcd_vault_headers }}" 
+    issue_cert_hosts: "{{ etcd_node_cert_hosts }}"
+    issue_cert_ip_sans: >-
+        [
+        {%- for host in etcd_node_cert_hosts -%}
+        "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+        {%- endfor -%}
+        "127.0.0.1","::1"
+        ]
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: etcd
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ etcd_node_certs_needed|d([]) }}"
+  when: inventory_hostname in etcd_node_cert_hosts
+  notify: set etcd_secret_changed

roles/etcd/tasks/main.yml

 ---
 - include: pre_upgrade.yml
   tags: etcd-pre-upgrade
+
 - include: check_certs.yml
+  when: cert_management == "script"
   tags: [etcd-secrets, facts]
-- include: gen_certs.yml
+- include: gen_certs_script.yml
+  when: cert_management == "script"
+  tags: etcd-secrets
+
+- include: sync_etcd_master_certs.yml
+  when: cert_management == "vault" and inventory_hostname in groups.etcd
   tags: etcd-secrets
+- include: sync_etcd_node_certs.yml
+  when: cert_management == "vault" and inventory_hostname in etcd_node_cert_hosts 
+  tags: etcd-secrets
+- include: gen_certs_vault.yml
+  when: cert_management == "vault" and (etcd_master_certs_needed|d() or etcd_node_certs_needed|d())
+  tags: etcd-secrets
+
 - include: "install_{{ etcd_deployment_type }}.yml"
   when: is_etcd_master
   tags: upgrade

roles/vault/tasks/bootstrap/sync_etcd_certs.yml → roles/etcd/tasks/sync_etcd_master_certs.yml

 ---
 
-- name: bootstrap/sync_etcd_certs | Create list of certs needing creation
+- name: sync_etcd_master_certs | Create list of master certs needing creation
   set_fact: 
-    vault_etcd_cert_list: >-
-        {{ vault_etcd_cert_list|default([]) +  [
+    etcd_master_cert_list: >-
+        {{ etcd_master_cert_list|default([]) +  [
         "admin-" + item + ".pem",
         "member-" + item + ".pem"
         ] }}
   with_items: "{{ groups.etcd }}"
 
-- include: ../sync_file.yml
+- include: ../../vault/tasks/shared/sync_file.yml
   vars: 
     sync_file: "{{ item }}"
     sync_file_dir: "{{ etcd_cert_dir }}"
     sync_file_hosts: "{{ groups.etcd }}"
     sync_file_is_cert: true
-  with_items: "{{ vault_etcd_cert_list|default([]) }}"
+  with_items: "{{ etcd_master_cert_list|d([]) }}"
 
-- name: bootstrap/sync_etcd_certs | Set facts for etcd sync_file results
+- name: sync_etcd_certs | Set facts for etcd sync_file results
   set_fact:
-    vault_etcd_certs_needed: "{{ vault_etcd_certs_needed|default([]) + [item.path] }}"
-  with_items: "{{ sync_file_results }}"
+    etcd_master_certs_needed: "{{ etcd_master_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results|d([]) }}"
   when: item.no_srcs|bool
 
-- name: bootstrap/sync_etcd_certs | Unset sync_file_results after etcd certs sync
+- name: sync_etcd_certs | Unset sync_file_results after etcd certs sync
   set_fact:
     sync_file_results: []
 
-- include: ../sync_file.yml
+- include: ../../vault/tasks/shared/sync_file.yml
   vars:
     sync_file: ca.pem
     sync_file_dir: "{{ etcd_cert_dir }}"
     sync_file_hosts: "{{ groups.etcd }}"
 
-- name: bootstrap/sync_etcd_certs | Unset sync_file_results after ca.pem sync
+- name: sync_etcd_certs | Unset sync_file_results after ca.pem sync
   set_fact:
     sync_file_results: []

roles/etcd/tasks/sync_etcd_node_certs.yml

+---
+
+- name: sync_etcd_node_certs | Create list of node certs needing creation
+  set_fact: 
+    etcd_node_cert_list: "{{ etcd_node_cert_list|default([]) +  ['node-' + item + '.pem'] }}"
+  with_items: "{{ etcd_node_cert_hosts }}"
+
+- include: ../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ etcd_cert_dir }}"
+    sync_file_hosts: "{{ etcd_node_cert_hosts }}"
+    sync_file_is_cert: true
+  with_items: "{{ etcd_node_cert_list|d([]) }}"
+
+- name: sync_etcd_node_certs | Set facts for etcd sync_file results
+  set_fact:
+    etcd_node_certs_needed: "{{ etcd_node_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results|d([]) }}"
+  when: item.no_srcs|bool
+
+- name: sync_etcd_node_certs | Unset sync_file_results after etcd node certs
+  set_fact:
+    sync_file_results: []
+
+- include: ../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: ca.pem
+    sync_file_dir: "{{ etcd_cert_dir }}"
+    sync_file_hosts: "{{ etcd_node_cert_hosts }}"
+
+- name: sync_etcd_node_certs | Unset sync_file_results after ca.pem
+  set_fact:
+    sync_file_results: []

roles/kubernetes/node/tasks/install.yml

@@ -25,6 +25,20 @@
   template: "src=kubelet.{{ kubelet_deployment_type }}.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes"
   notify: restart kubelet
 
+- name: install | Set SSL CA directories
+  set_fact:
+    ssl_ca_dirs: "[
+      {% if ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] -%}
+      '/usr/share/ca-certificates',
+      {% elif ansible_os_family == 'RedHat' -%}
+      '/etc/pki/tls',
+      '/etc/pki/ca-trust',
+      {% elif ansible_os_family == 'Debian' -%}
+      '/usr/share/ca-certificates',
+      {% endif -%}
+    ]"
+  tags: facts
+
 - name: install | Install kubelet launch script
   template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes
   notify: restart kubelet

roles/kubernetes/preinstall/defaults/main.yml

@@ -10,6 +10,7 @@ common_required_pkgs:
   - rsync
   - bash-completion
   - socat
+  - unzip
 
 # Set to true if your network does not support IPv6
 # This maybe necessary for pulling Docker images from

roles/kubernetes/secrets/meta/main.yml

+---
+# NOTE: Dynamic task dependency on Vault Role if cert_management == "vault"

roles/kubernetes/secrets/tasks/gen_certs.yml → roles/kubernetes/secrets/tasks/gen_certs_script.yml

@@ -160,20 +160,6 @@
       {%- endif %}
   tags: facts
 
-- name: SSL CA directories | Set SSL CA directories
-  set_fact:
-    ssl_ca_dirs: "[
-      {% if ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] -%}
-      '/usr/share/ca-certificates',
-      {% elif ansible_os_family == 'RedHat' -%}
-      '/etc/pki/tls',
-      '/etc/pki/ca-trust',
-      {% elif ansible_os_family == 'Debian' -%}
-      '/usr/share/ca-certificates',
-      {% endif -%}
-    ]"
-  tags: facts
-
 - name: Gen_certs | add CA to trusted CA dir
   copy:
     src: "{{ kube_cert_dir }}/ca.pem"

roles/kubernetes/secrets/tasks/gen_certs_vault.yml

+---
+
+- name: gen_certs_vault | Read in the local credentials
+  command: cat /etc/vault/roles/kube/userpass
+  register: kube_vault_creds_cat
+  when: inventory_hostname == groups['k8s-cluster']|first
+
+- name: gen_certs_vault | Set facts for read Vault Creds
+  set_fact:
+    kube_vault_creds: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_creds_cat']['stdout'] | from_json }}"
+  when: inventory_hostname == groups['k8s-cluster']|first
+
+- name: gen_certs_vault | Log into Vault and obtain an token
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ kube_vault_creds.username }}"
+    headers:
+      Accept: application/json
+      Content-Type: application/json 
+    method: POST
+    body_format: json
+    body:
+      password: "{{ kube_vault_creds.password }}"
+  register: kube_vault_login_result
+  when: inventory_hostname == groups['k8s-cluster']|first
+
+- name: gen_certs_vault | Set fact for Vault API token
+  set_fact:
+    kube_vault_headers:
+        Accept: application/json
+        Content-Type: application/json
+        X-Vault-Token: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_login_result']['json']['auth']['client_token'] }}"
+
+# Issue certs to kube-master nodes
+- include: ../../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_copy_ca: "{{ item == kube_master_certs_needed|first }}"
+    issue_cert_file_group: "{{ kube_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ kube_vault_headers }}"
+    issue_cert_hosts: "{{ groups['kube-master'] }}"
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: kube
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ kube_master_certs_needed|d([]) }}"
+  when: inventory_hostname in groups['kube-master']
+
+- include: ../../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_alt_names: >-
+        {{
+        groups['kube-master'] +
+        ['kubernetes.default.svc.cluster.local', 'kubernetes.default.svc', 'kubernetes.default', 'kubernetes'] +
+        ['localhost']
+        }}
+    issue_cert_file_group: "{{ kube_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ kube_vault_headers }}" 
+    issue_cert_hosts: "{{ groups['kube-master'] }}"
+    issue_cert_ip_sans: >-
+        [
+        {%- for host in groups['kube-master']  -%}
+        "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+        {%- endfor -%}
+        "127.0.0.1","::1","{{ kube_apiserver_ip }}"
+        ]
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: kube
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ kube_api_certs_needed|d([]) }}"
+  when: inventory_hostname in groups['kube-master']
+
+# Issue node certs to k8s-cluster nodes
+- include: ../../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_copy_ca: "{{ item == kube_node_certs_needed|first }}"
+    issue_cert_file_group: "{{ kube_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ kube_vault_headers }}" 
+    issue_cert_hosts: "{{ groups['k8s-cluster'] }}"
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: kube
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ kube_node_certs_needed|d([]) }}"
+  when: inventory_hostname in groups['k8s-cluster']

roles/kubernetes/secrets/tasks/main.yml

@@ -70,7 +70,19 @@
   delegate_to: "{{groups['kube-master'][0]}}"
   when: gen_tokens|default(false)
 
-- include: gen_certs.yml
+- include: gen_certs_script.yml
+  when: cert_management == "script"
   tags: k8s-secrets
+
+- include: sync_kube_master_certs.yml
+  when: cert_management == "vault" and inventory_hostname in groups['kube-master']
+  tags: k8s-secrets
+- include: sync_kube_node_certs.yml
+  when: cert_management == "vault" and inventory_hostname in groups['k8s-cluster']
+  tags: k8s-secrets
+- include: gen_certs_vault.yml
+  when: cert_management == "vault"
+  tags: k8s-secrets
+
 - include: gen_tokens.yml
   tags: k8s-secrets