diff --git a/roles/etcd/tasks/join_etcd-events_member.yml b/roles/etcd/tasks/join_etcd-events_member.yml
index d5df065f92fa887099ae38f801048acd33e8b297..b75460c418fa6acd7529f93b0ef91eb0a1a7ab96 100644
--- a/roles/etcd/tasks/join_etcd-events_member.yml
+++ b/roles/etcd/tasks/join_etcd-events_member.yml
@@ -15,7 +15,7 @@
etcd_events_peer_addresses: >-
{% for host in groups['etcd'] -%}
{%- if hostvars[host]['etcd_events_member_in_cluster'].rc == 0 -%}
- {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2382,
+ {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(fallback_ips[host])) }}:2382,
{%- endif -%}
{%- if loop.last -%}
{{ etcd_member_name }}={{ etcd_events_peer_url }}
diff --git a/roles/etcd/tasks/join_etcd_member.yml b/roles/etcd/tasks/join_etcd_member.yml
index 0aad020493de15eddf305d347ae903595ea35c5d..d512eb78a3b238b0725b86253cb0adf645364b7f 100644
--- a/roles/etcd/tasks/join_etcd_member.yml
+++ b/roles/etcd/tasks/join_etcd_member.yml
@@ -16,7 +16,7 @@
etcd_peer_addresses: >-
{% for host in groups['etcd'] -%}
{%- if hostvars[host]['etcd_member_in_cluster'].rc == 0 -%}
- {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2380,
+ {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(fallback_ips[host])) }}:2380,
{%- endif -%}
{%- if loop.last -%}
{{ etcd_member_name }}={{ etcd_peer_url }}
diff --git a/roles/etcd/templates/openssl.conf.j2 b/roles/etcd/templates/openssl.conf.j2
index 402417827c1ae7a9e32d706bf9c00188bdd97575..f6681a145daa2586f5b7bdc3d0f15ddc3a69b1d4 100644
--- a/roles/etcd/templates/openssl.conf.j2
+++ b/roles/etcd/templates/openssl.conf.j2
@@ -37,7 +37,7 @@ DNS.{{ counter["dns"] }} = {{ etcd_alt_name }}{{ increment(counter, 'dns') }}
{% if hostvars[host]['access_ip'] is defined %}
IP.{{ counter["ip"] }} = {{ hostvars[host]['access_ip'] }}{{ increment(counter, 'ip') }}
{% endif %}
-IP.{{ counter["ip"] }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}{{ increment(counter, 'ip') }}
+IP.{{ counter["ip"] }} = {{ hostvars[host]['ip'] | default(fallback_ips[host]) }}{{ increment(counter, 'ip') }}
{% endfor %}
{% for cert_alt_ip in etcd_cert_alt_ips %}
IP.{{ counter["ip"] }} = {{ cert_alt_ip }}{{ increment(counter, 'ip') }}
diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml
index 125a4de4bfc8d1926d14fc47645e51d22fdfc008..31067522ad90f35379815079a48695768d0bcd78 100644
--- a/roles/kubernetes/master/tasks/kubeadm-setup.yml
+++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml
@@ -61,7 +61,7 @@
{%- if hostvars[host]['access_ip'] is defined %}
{{ hostvars[host]['access_ip'] }}
{% endif %}
- {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+ {{ hostvars[host]['ip'] | default(fallback_ips[host]) }}
{%- endfor %}
{%- if supplementary_addresses_in_ssl_keys is defined -%}
{% for addr in supplementary_addresses_in_ssl_keys %}
diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2
index f496c3808f9499a3303d87377efb8ca5f967d42e..f6138dd6b5cfc1003eecabfe2689a373eac43ba8 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2
@@ -5,7 +5,7 @@ api:
controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}
bindPort: {{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
{% else %}
- advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+ advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }}
bindPort: {{ kube_apiserver_port }}
{% endif %}
etcd:
diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
index eca666aaa15f0ba8ce3ea1be720fe9efad3d0912..79fe63dbdec99dd83f2c23e63e86d870d60e46ee 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
@@ -5,7 +5,7 @@ api:
controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}
bindPort: {{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
{% else %}
- advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+ advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }}
bindPort: {{ kube_apiserver_port }}
{% endif %}
etcd:
diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2
index 3e79eeeeeaa0dd65f40905aeec03c5867ba5dce8..befdaa1afecb36c35dc371a0eacf347af2a39c35 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2
@@ -1,7 +1,7 @@
apiVersion: kubeadm.k8s.io/v1alpha3
kind: InitConfiguration
apiEndpoint:
- advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+ advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }}
bindPort: {{ kube_apiserver_port }}
nodeRegistration:
{% if kube_override_hostname|default('') %}
@@ -40,7 +40,7 @@ kubernetesVersion: {{ kube_version }}
{% if kubeadm_config_api_fqdn is defined %}
controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
{% else %}
-controlPlaneEndpoint: {{ ip | default(ansible_default_ipv4.address) }}:{{ kube_apiserver_port }}
+controlPlaneEndpoint: {{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }}
{% endif %}
apiServerCertSANs:
{% for san in apiserver_sans.split() | unique %}
diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2
index b7b343e58d4e709bd38c0f5a0661c91d13c64249..2d7daccd6f521b899925192a11b8c01075ef58f6 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2
@@ -1,7 +1,7 @@
apiVersion: kubeadm.k8s.io/v1beta1
kind: InitConfiguration
localAPIEndpoint:
- advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+ advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }}
bindPort: {{ kube_apiserver_port }}
nodeRegistration:
{% if kube_override_hostname|default('') %}
@@ -40,7 +40,7 @@ kubernetesVersion: {{ kube_version }}
{% if kubeadm_config_api_fqdn is defined %}
controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
{% else %}
-controlPlaneEndpoint: {{ ip | default(ansible_default_ipv4.address) }}:{{ kube_apiserver_port }}
+controlPlaneEndpoint: {{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }}
{% endif %}
certificatesDir: {{ kube_cert_dir }}
imageRepository: {{ kube_image_repo }}
diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml
index 3500dc44b6bf05d6e3cd664fe5940fa8f176949d..e3f20d4a67f6cc0e17c5afbdb9ca58b07199187b 100644
--- a/roles/kubernetes/node/defaults/main.yml
+++ b/roles/kubernetes/node/defaults/main.yml
@@ -3,7 +3,7 @@
kube_apiserver_insecure_bind_address: 127.0.0.1
# advertised host IP for kubelet. This affects network plugin config. Take caution
-kubelet_address: "{{ ip | default(ansible_default_ipv4['address']) }}"
+kubelet_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}"
# bind address for kubelet. Set to 0.0.0.0 to listen on all interfaces
kubelet_bind_address: "{{ ip | default('0.0.0.0') }}"
diff --git a/roles/kubernetes/node/templates/nginx.conf.j2 b/roles/kubernetes/node/templates/nginx.conf.j2
index 86984a10132410beddd89a6b5d77bc56cef1fbb7..99a48d65d0d470f20650dfca71f061ce1904d460 100644
--- a/roles/kubernetes/node/templates/nginx.conf.j2
+++ b/roles/kubernetes/node/templates/nginx.conf.j2
@@ -11,7 +11,7 @@ stream {
upstream kube_apiserver {
least_conn;
{% for host in groups['kube-master'] -%}
- server {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }};
+ server {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(fallback_ips[host])) }}:{{ kube_apiserver_port }};
{% endfor %}
}
diff --git a/roles/kubernetes/preinstall/tasks/0090-etchosts.yml b/roles/kubernetes/preinstall/tasks/0090-etchosts.yml
index 48eadf5524c74188835ab8ed78d761ffabda8fff..7979568c8ce1394241909c3fe99d319f1c815b30 100644
--- a/roles/kubernetes/preinstall/tasks/0090-etchosts.yml
+++ b/roles/kubernetes/preinstall/tasks/0090-etchosts.yml
@@ -3,7 +3,11 @@
blockinfile:
path: /etc/hosts
block: |-
- {% for item in (groups['k8s-cluster'] + groups['etcd'] + groups['calico-rr']|default([]))|unique -%}{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}{% if (item != hostvars[item]['ansible_hostname']) %} {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] }}{% endif %} {{ item }} {{ item }}.{{ dns_domain }}
+ {% for item in (groups['k8s-cluster'] + groups['etcd'] + groups['calico-rr']|default([]))|unique -%}
+ {% if 'access_ip' in hostvars[item] or 'ip' in hostvars[item] or fallback_ips[item] != "skip" -%}
+ {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(fallback_ips[item]))}}
+ {%- if ('ansible_hostname' in hostvars[item] and item != hostvars[item]['ansible_hostname']) %} {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] }}{% endif %} {{ item }} {{ item }}.{{ dns_domain }}
+ {% endif %}
{% endfor %}
state: present
create: yes
diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index b259bc57fc730083a0784cde22a8f6a87e98a535..16fa09233f6d6b7fa0af64f9ed920d52c3bc09c8 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -358,6 +358,14 @@ contiv_peer_with_uplink_leaf: false
contiv_global_as: "65002"
contiv_global_neighbor_as: "500"
+# Set 127.0.0.1 as fallback IP if we do not have host facts for host
+fallback_ips_base: |
+ ---
+ {% for item in groups['k8s-cluster'] + groups['etcd'] + groups['calico-rr']|default([])|unique %}
+ {{item}}: "{{ hostvars[item].get('ansible_default_ipv4', {'address': '127.0.0.1'})['address'] }}"
+ {% endfor %}
+fallback_ips: "{{ fallback_ips_base | from_yaml }}"
+
## Set no_proxy to all assigned cluster IPs and hostnames
no_proxy: >-
{%- if http_proxy is defined or https_proxy is defined %}
@@ -366,8 +374,8 @@ no_proxy: >-
{{ loadbalancer_apiserver.address | default('') }},
{%- endif -%}
{%- for item in (groups['k8s-cluster'] + groups['etcd'] + groups['calico-rr']|default([]))|unique -%}
- {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }},
- {%- if (item != hostvars[item]['ansible_hostname']) -%}
+ {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(fallback_ips[item])) }},
+ {%- if item != hostvars[item].get('ansible_hostname', "") -%}
{{ hostvars[item]['ansible_hostname'] }},
{{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }},
{%- endif -%}
@@ -399,9 +407,9 @@ ssl_ca_dirs: >-
# Vars for pointing to kubernetes api endpoints
is_kube_master: "{{ inventory_hostname in groups['kube-master'] }}"
kube_apiserver_count: "{{ groups['kube-master'] | length }}"
-kube_apiserver_address: "{{ ip | default(ansible_default_ipv4['address']) }}"
+kube_apiserver_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}"
kube_apiserver_access_address: "{{ access_ip | default(kube_apiserver_address) }}"
-first_kube_master: "{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}"
+first_kube_master: "{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(fallback_ips[groups['kube-master'][0]])) }}"
loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}"
# applied if only external loadbalancer_apiserver is defined, otherwise ignored
apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local"
@@ -425,7 +433,7 @@ etcd_events_cluster_enabled: false
# Vars for pointing to etcd endpoints
is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}"
-etcd_address: "{{ ip | default(ansible_default_ipv4['address']) }}"
+etcd_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}"
etcd_access_address: "{{ access_ip | default(etcd_address) }}"
etcd_events_access_address: "{{ access_ip | default(etcd_address) }}"
etcd_peer_url: "https://{{ etcd_access_address }}:2380"
@@ -434,12 +442,12 @@ etcd_events_peer_url: "https://{{ etcd_events_access_address }}:2382"
etcd_events_client_url: "https://{{ etcd_events_access_address }}:2381"
etcd_access_addresses: |-
{% for item in groups['etcd'] -%}
- https://{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}
+ https://{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(fallback_ips[item])) }}:2379{% if not loop.last %},{% endif %}
{%- endfor %}
etcd_events_access_addresses_list: |-
[
{% for item in groups['etcd'] -%}
- 'https://{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}:2381'{% if not loop.last %},{% endif %}
+ 'https://{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(fallback_ips[item])) }}:2381'{% if not loop.last %},{% endif %}
{%- endfor %}
]
etcd_events_access_addresses: "{{etcd_events_access_addresses_list | join(',')}}"
@@ -451,11 +459,11 @@ etcd_member_name: |-
{% endfor %}
etcd_peer_addresses: |-
{% for item in groups['etcd'] -%}
- {{ hostvars[item].etcd_member_name | default("etcd" + loop.index|string) }}=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %}
+ {{ hostvars[item].etcd_member_name | default("etcd" + loop.index|string) }}=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(fallback_ips[item])) }}:2380{% if not loop.last %},{% endif %}
{%- endfor %}
etcd_events_peer_addresses: |-
{% for item in groups['etcd'] -%}
- {{ hostvars[item].etcd_member_name | default("etcd" + loop.index|string) }}-events=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2382{% if not loop.last %},{% endif %}
+ {{ hostvars[item].etcd_member_name | default("etcd" + loop.index|string) }}-events=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(fallback_ips[item])) }}:2382{% if not loop.last %},{% endif %}
{%- endfor %}
podsecuritypolicy_enabled: false
diff --git a/roles/network_plugin/calico/rr/tasks/main.yml b/roles/network_plugin/calico/rr/tasks/main.yml
index 4a53a6cf627d4405f52830f1007feaac076e197c..1c41fdf7a51495d9358e1c1d6cf48fd21a09b17e 100644
--- a/roles/network_plugin/calico/rr/tasks/main.yml
+++ b/roles/network_plugin/calico/rr/tasks/main.yml
@@ -5,7 +5,7 @@
- name: Calico-rr | Set IP fact
set_fact:
- rr_ip: "{{ calico_rr_ip | default(ip) | default(ansible_default_ipv4.address) }}"
+ rr_ip: "{{ calico_rr_ip | default(ip) | default(fallback_ips[inventory_hostname]) }}"
- name: Calico-rr | Create calico certs directory
file:
diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml
index 8fe6528014dd7de3956319d2a439045421ab0532..a80cd9e25698055a0e78a7d6a10c971b9fccc7fc 100644
--- a/roles/network_plugin/calico/tasks/install.yml
+++ b/roles/network_plugin/calico/tasks/install.yml
@@ -297,12 +297,12 @@
"apiVersion": "projectcalico.org/v3",
"kind": "BGPPeer",
"metadata": {
- "name": "{{ inventory_hostname }}-{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}"
+ "name": "{{ inventory_hostname }}-{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}"
},
"spec": {
"asNumber": "{{ local_as | default(global_as_num)}}",
"node": "{{ inventory_hostname }}",
- "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}"
+ "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}"
}}' | {{ bin_dir }}/calicoctl create --skip-exists -f -
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
@@ -322,7 +322,7 @@
"apiVersion": "v1",
"metadata": {"node": "{{ inventory_hostname }}",
"scope": "node",
- "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}"}
+ "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}"}
}'
| {{ bin_dir }}/calicoctl create --skip-exists -f -
retries: 4
diff --git a/roles/network_plugin/contiv/defaults/main.yml b/roles/network_plugin/contiv/defaults/main.yml
index 82316357ceeed65868e14d595d605a15c5f6359d..f4c51ae584ac253f500d227f65b80b9911db6ccd 100644
--- a/roles/network_plugin/contiv/defaults/main.yml
+++ b/roles/network_plugin/contiv/defaults/main.yml
@@ -14,7 +14,7 @@ contiv_etcd_listen_port: 6666
contiv_etcd_peer_port: 6667
contiv_etcd_endpoints: |-
{% for host in groups['kube-master'] -%}
- contiv_etcd{{ loop.index }}=http://{{ hostvars[host]['ip'] | default(hostvars[host].ansible_default_ipv4['address']) }}:{{ contiv_etcd_peer_port }}{% if not loop.last %},{% endif %}
+ contiv_etcd{{ loop.index }}=http://{{ hostvars[host]['ip'] | default(fallback_ips[host]) }}:{{ contiv_etcd_peer_port }}{% if not loop.last %},{% endif %}
{%- endfor %}
# Parameters for Contiv api-proxy
diff --git a/roles/network_plugin/flannel/defaults/main.yml b/roles/network_plugin/flannel/defaults/main.yml
index e48a9475a7311e690ee0c054877552b041c0c8d2..f7f773fdc351f781b4eff4d9dc251744b7b9d4ae 100644
--- a/roles/network_plugin/flannel/defaults/main.yml
+++ b/roles/network_plugin/flannel/defaults/main.yml
@@ -2,7 +2,7 @@
# Flannel public IP
# The address that flannel should advertise as how to access the system
# Disabled until https://github.com/coreos/flannel/issues/712 is fixed
-# flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address)) }}"
+# flannel_public_ip: "{{ access_ip|default(ip|default(fallback_ips[inventory_hostname])) }}"
## interface that should be used for flannel operations
## This is actually an inventory cluster-level item
@@ -25,4 +25,4 @@ flannel_memory_requests: 64M
flannel_cpu_requests: 150m
# Legacy directory, will be removed if found.
-flannel_cert_dir: /etc/flannel/certs
\ No newline at end of file
+flannel_cert_dir: /etc/flannel/certs