diff --git a/tests/files/custom_cni/cilium.yaml b/tests/files/custom_cni/cilium.yaml
index 9bd3bfbe46f511d45bfde7cf39b0845ec90c6554..c89ae15ebf9bb12e789614f3278c38495c2eee50 100644
--- a/tests/files/custom_cni/cilium.yaml
+++ b/tests/files/custom_cni/cilium.yaml
@@ -6,6 +6,13 @@ metadata:
name: "cilium"
namespace: kube-system
---
+# Source: cilium/templates/cilium-envoy/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: "cilium-envoy"
+ namespace: kube-system
+---
# Source: cilium/templates/cilium-operator/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
@@ -36,9 +43,6 @@ data:
identity-gc-interval: "15m0s"
cilium-endpoint-gc-interval: "5m0s"
nodes-gc-interval: "5m0s"
- skip-cnp-status-startup-clean: "false"
- # Disable the usage of CiliumEndpoint CRD
- disable-endpoint-crd: "false"
# If you want to run cilium in debug mode change this value to true
debug: "false"
@@ -47,6 +51,13 @@ data:
# default, always and never.
# https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes
enable-policy: "default"
+ policy-cidr-match-mode: ""
+ # If you want metrics enabled in cilium-operator, set the port for
+ # which the Cilium Operator will have their metrics exposed.
+ # NOTE that this will open the port on the nodes where Cilium operator pod
+ # is scheduled.
+ operator-prometheus-serve-addr: ":9963"
+ enable-metrics: "true"
# Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
# address.
@@ -58,7 +69,7 @@ data:
# Users who wish to specify their own custom CNI configuration file must set
# custom-cni-conf to "true", otherwise Cilium may overwrite the configuration.
custom-cni-conf: "false"
- enable-bpf-clock-probe: "true"
+ enable-bpf-clock-probe: "false"
# If you want cilium monitor to aggregate tracing for packets, set this level
# to "low", "medium", or "maximum". The higher the level, the less packets
# that will be seen in monitor output.
@@ -86,6 +97,10 @@ data:
bpf-lb-map-max: "65536"
bpf-lb-external-clusterip: "false"
+ bpf-events-drop-enabled: "true"
+ bpf-events-policy-verdict-enabled: "true"
+ bpf-events-trace-enabled: "true"
+
# Pre-allocation of map entries allows per-packet latency to be reduced, at
# the expense of up-front memory allocation for the entries in the maps. The
# default value below will minimize memory usage in the default installation;
@@ -103,10 +118,6 @@ data:
# 1.4 or later, then it may cause one-time disruptions during the upgrade.
preallocate-bpf-maps: "false"
- # Regular expression matching compatible Istio sidecar istio-proxy
- # container image names
- sidecar-istio-proxy-image: "cilium/istio_proxy"
-
# Name of the cluster. Only relevant when building a mesh of clusters.
cluster-name: default
# Unique ID of the cluster. Must be unique across all conneted clusters and
@@ -118,63 +129,444 @@ data:
# - disabled
# - vxlan (default)
# - geneve
- tunnel: "vxlan"
+ # Default case
+ routing-mode: "tunnel"
+ tunnel-protocol: "vxlan"
+ service-no-backend-response: "reject"
# Enables L7 proxy for L7 policy enforcement and visibility
enable-l7-proxy: "true"
enable-ipv4-masquerade: "true"
+ enable-ipv4-big-tcp: "false"
enable-ipv6-big-tcp: "false"
enable-ipv6-masquerade: "true"
+ enable-tcx: "true"
+ datapath-mode: "veth"
+ enable-masquerade-to-route-source: "false"
enable-xt-socket-fallback: "true"
- install-iptables-rules: "true"
install-no-conntrack-iptables-rules: "false"
auto-direct-node-routes: "false"
+ direct-routing-skip-unreachable: "false"
enable-local-redirect-policy: "false"
+ enable-runtime-device-detection: "true"
- kube-proxy-replacement: "disabled"
+ kube-proxy-replacement: "false"
+ kube-proxy-replacement-healthz-bind-address: ""
bpf-lb-sock: "false"
+ bpf-lb-sock-terminate-pod-connections: "false"
+ enable-host-port: "false"
+ enable-external-ips: "false"
+ enable-node-port: "false"
+ nodeport-addresses: ""
enable-health-check-nodeport: "true"
+ enable-health-check-loadbalancer-ip: "false"
node-port-bind-protection: "true"
enable-auto-protect-node-port-range: "true"
+ bpf-lb-acceleration: "disabled"
enable-svc-source-range-check: "true"
enable-l2-neigh-discovery: "true"
arping-refresh-period: "30s"
+ k8s-require-ipv4-pod-cidr: "false"
+ k8s-require-ipv6-pod-cidr: "false"
+ enable-k8s-networkpolicy: "true"
+ # Tell the agent to generate and write a CNI configuration file
+ write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist
+ cni-exclusive: "true"
+ cni-log-file: "/var/run/cilium/cilium-cni.log"
enable-endpoint-health-checking: "true"
enable-health-checking: "true"
enable-well-known-identities: "false"
- enable-remote-node-identity: "true"
+ enable-node-selector-labels: "false"
synchronize-k8s-nodes: "true"
operator-api-serve-addr: "127.0.0.1:9234"
ipam: "cluster-pool"
+ ipam-cilium-node-update-rate: "15s"
cluster-pool-ipv4-cidr: "{{ kube_pods_subnet }}"
cluster-pool-ipv4-mask-size: "24"
- disable-cnp-status-updates: "true"
+ egress-gateway-reconciliation-trigger-interval: "1s"
enable-vtep: "false"
vtep-endpoint: ""
vtep-cidr: ""
vtep-mask: ""
vtep-mac: ""
- enable-bgp-control-plane: "false"
procfs: "/host/proc"
bpf-root: "/sys/fs/bpf"
cgroup-root: "/run/cilium/cgroupv2"
enable-k8s-terminating-endpoint: "true"
enable-sctp: "false"
+
+ k8s-client-qps: "10"
+ k8s-client-burst: "20"
remove-cilium-node-taints: "true"
+ set-cilium-node-taints: "true"
set-cilium-is-up-condition: "true"
unmanaged-pod-watcher-interval: "15"
+ # default DNS proxy to transparent mode in non-chaining modes
+ dnsproxy-enable-transparent-mode: "true"
+ dnsproxy-socket-linger-timeout: "10"
tofqdns-dns-reject-response-code: "refused"
tofqdns-enable-dns-compression: "true"
tofqdns-endpoint-max-ip-per-hostname: "50"
tofqdns-idle-connection-grace-period: "0s"
tofqdns-max-deferred-connection-deletes: "10000"
- tofqdns-min-ttl: "3600"
tofqdns-proxy-response-max-delay: "100ms"
agent-not-ready-taint-key: "node.cilium.io/agent-not-ready"
+
+ mesh-auth-enabled: "true"
+ mesh-auth-queue-size: "1024"
+ mesh-auth-rotated-identities-queue-size: "1024"
+ mesh-auth-gc-interval: "5m0s"
+
+ proxy-xff-num-trusted-hops-ingress: "0"
+ proxy-xff-num-trusted-hops-egress: "0"
+ proxy-connect-timeout: "2"
+ proxy-max-requests-per-connection: "0"
+ proxy-max-connection-duration-seconds: "0"
+ proxy-idle-timeout-seconds: "60"
+
+ external-envoy-proxy: "true"
+ envoy-base-id: "0"
+
+ envoy-keep-cap-netbindservice: "false"
+ max-connected-clusters: "255"
+ clustermesh-enable-endpoint-sync: "false"
+ clustermesh-enable-mcs-api: "false"
+
+ nat-map-stats-entries: "32"
+ nat-map-stats-interval: "30s"
+
+# Extra config allows adding arbitrary properties to the cilium config.
+# By putting it at the end of the ConfigMap, it's also possible to override existing properties.
+---
+# Source: cilium/templates/cilium-envoy/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: cilium-envoy-config
+ namespace: kube-system
+data:
+ bootstrap-config.json: |
+ {
+ "node": {
+ "id": "host~127.0.0.1~no-id~localdomain",
+ "cluster": "ingress-cluster"
+ },
+ "staticResources": {
+ "listeners": [
+ {
+ "name": "envoy-prometheus-metrics-listener",
+ "address": {
+ "socket_address": {
+ "address": "0.0.0.0",
+ "port_value": 9964
+ }
+ },
+ "filter_chains": [
+ {
+ "filters": [
+ {
+ "name": "envoy.filters.network.http_connection_manager",
+ "typed_config": {
+ "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
+ "stat_prefix": "envoy-prometheus-metrics-listener",
+ "route_config": {
+ "virtual_hosts": [
+ {
+ "name": "prometheus_metrics_route",
+ "domains": [
+ "*"
+ ],
+ "routes": [
+ {
+ "name": "prometheus_metrics_route",
+ "match": {
+ "prefix": "/metrics"
+ },
+ "route": {
+ "cluster": "/envoy-admin",
+ "prefix_rewrite": "/stats/prometheus"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "http_filters": [
+ {
+ "name": "envoy.filters.http.router",
+ "typed_config": {
+ "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
+ }
+ }
+ ],
+ "stream_idle_timeout": "0s"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "envoy-health-listener",
+ "address": {
+ "socket_address": {
+ "address": "127.0.0.1",
+ "port_value": 9878
+ }
+ },
+ "filter_chains": [
+ {
+ "filters": [
+ {
+ "name": "envoy.filters.network.http_connection_manager",
+ "typed_config": {
+ "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
+ "stat_prefix": "envoy-health-listener",
+ "route_config": {
+ "virtual_hosts": [
+ {
+ "name": "health",
+ "domains": [
+ "*"
+ ],
+ "routes": [
+ {
+ "name": "health",
+ "match": {
+ "prefix": "/healthz"
+ },
+ "route": {
+ "cluster": "/envoy-admin",
+ "prefix_rewrite": "/ready"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "http_filters": [
+ {
+ "name": "envoy.filters.http.router",
+ "typed_config": {
+ "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
+ }
+ }
+ ],
+ "stream_idle_timeout": "0s"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ],
+ "clusters": [
+ {
+ "name": "ingress-cluster",
+ "type": "ORIGINAL_DST",
+ "connectTimeout": "2s",
+ "lbPolicy": "CLUSTER_PROVIDED",
+ "typedExtensionProtocolOptions": {
+ "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
+ "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
+ "commonHttpProtocolOptions": {
+ "idleTimeout": "60s",
+ "maxConnectionDuration": "0s",
+ "maxRequestsPerConnection": 0
+ },
+ "useDownstreamProtocolConfig": {}
+ }
+ },
+ "cleanupInterval": "2.500s"
+ },
+ {
+ "name": "egress-cluster-tls",
+ "type": "ORIGINAL_DST",
+ "connectTimeout": "2s",
+ "lbPolicy": "CLUSTER_PROVIDED",
+ "typedExtensionProtocolOptions": {
+ "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
+ "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
+ "commonHttpProtocolOptions": {
+ "idleTimeout": "60s",
+ "maxConnectionDuration": "0s",
+ "maxRequestsPerConnection": 0
+ },
+ "upstreamHttpProtocolOptions": {},
+ "useDownstreamProtocolConfig": {}
+ }
+ },
+ "cleanupInterval": "2.500s",
+ "transportSocket": {
+ "name": "cilium.tls_wrapper",
+ "typedConfig": {
+ "@type": "type.googleapis.com/cilium.UpstreamTlsWrapperContext"
+ }
+ }
+ },
+ {
+ "name": "egress-cluster",
+ "type": "ORIGINAL_DST",
+ "connectTimeout": "2s",
+ "lbPolicy": "CLUSTER_PROVIDED",
+ "typedExtensionProtocolOptions": {
+ "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
+ "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
+ "commonHttpProtocolOptions": {
+ "idleTimeout": "60s",
+ "maxConnectionDuration": "0s",
+ "maxRequestsPerConnection": 0
+ },
+ "useDownstreamProtocolConfig": {}
+ }
+ },
+ "cleanupInterval": "2.500s"
+ },
+ {
+ "name": "ingress-cluster-tls",
+ "type": "ORIGINAL_DST",
+ "connectTimeout": "2s",
+ "lbPolicy": "CLUSTER_PROVIDED",
+ "typedExtensionProtocolOptions": {
+ "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
+ "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
+ "commonHttpProtocolOptions": {
+ "idleTimeout": "60s",
+ "maxConnectionDuration": "0s",
+ "maxRequestsPerConnection": 0
+ },
+ "upstreamHttpProtocolOptions": {},
+ "useDownstreamProtocolConfig": {}
+ }
+ },
+ "cleanupInterval": "2.500s",
+ "transportSocket": {
+ "name": "cilium.tls_wrapper",
+ "typedConfig": {
+ "@type": "type.googleapis.com/cilium.UpstreamTlsWrapperContext"
+ }
+ }
+ },
+ {
+ "name": "xds-grpc-cilium",
+ "type": "STATIC",
+ "connectTimeout": "2s",
+ "loadAssignment": {
+ "clusterName": "xds-grpc-cilium",
+ "endpoints": [
+ {
+ "lbEndpoints": [
+ {
+ "endpoint": {
+ "address": {
+ "pipe": {
+ "path": "/var/run/cilium/envoy/sockets/xds.sock"
+ }
+ }
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "typedExtensionProtocolOptions": {
+ "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
+ "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
+ "explicitHttpConfig": {
+ "http2ProtocolOptions": {}
+ }
+ }
+ }
+ },
+ {
+ "name": "/envoy-admin",
+ "type": "STATIC",
+ "connectTimeout": "2s",
+ "loadAssignment": {
+ "clusterName": "/envoy-admin",
+ "endpoints": [
+ {
+ "lbEndpoints": [
+ {
+ "endpoint": {
+ "address": {
+ "pipe": {
+ "path": "/var/run/cilium/envoy/sockets/admin.sock"
+ }
+ }
+ }
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ },
+ "dynamicResources": {
+ "ldsConfig": {
+ "apiConfigSource": {
+ "apiType": "GRPC",
+ "transportApiVersion": "V3",
+ "grpcServices": [
+ {
+ "envoyGrpc": {
+ "clusterName": "xds-grpc-cilium"
+ }
+ }
+ ],
+ "setNodeOnFirstMessageOnly": true
+ },
+ "resourceApiVersion": "V3"
+ },
+ "cdsConfig": {
+ "apiConfigSource": {
+ "apiType": "GRPC",
+ "transportApiVersion": "V3",
+ "grpcServices": [
+ {
+ "envoyGrpc": {
+ "clusterName": "xds-grpc-cilium"
+ }
+ }
+ ],
+ "setNodeOnFirstMessageOnly": true
+ },
+ "resourceApiVersion": "V3"
+ }
+ },
+ "bootstrapExtensions": [
+ {
+ "name": "envoy.bootstrap.internal_listener",
+ "typed_config": {
+ "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener"
+ }
+ }
+ ],
+ "layeredRuntime": {
+ "layers": [
+ {
+ "name": "static_layer_0",
+ "staticLayer": {
+ "overload": {
+ "global_downstream_max_connections": 50000
+ }
+ }
+ }
+ ]
+ },
+ "admin": {
+ "address": {
+ "pipe": {
+ "path": "/var/run/cilium/envoy/sockets/admin.sock"
+ }
+ }
+ }
+ }
---
# Source: cilium/templates/cilium-agent/clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1
@@ -228,6 +620,9 @@ rules:
resources:
- ciliumloadbalancerippools
- ciliumbgppeeringpolicies
+ - ciliumbgpnodeconfigs
+ - ciliumbgpadvertisements
+ - ciliumbgppeerconfigs
- ciliumclusterwideenvoyconfigs
- ciliumclusterwidenetworkpolicies
- ciliumegressgatewaypolicies
@@ -239,6 +634,9 @@ rules:
- ciliumnetworkpolicies
- ciliumnodes
- ciliumnodeconfigs
+ - ciliumcidrgroups
+ - ciliuml2announcementpolicies
+ - ciliumpodippools
verbs:
- list
- watch
@@ -275,10 +673,10 @@ rules:
- apiGroups:
- cilium.io
resources:
- - ciliumnetworkpolicies/status
- - ciliumclusterwidenetworkpolicies/status
- ciliumendpoints/status
- ciliumendpoints
+ - ciliuml2announcementpolicies/status
+ - ciliumbgpnodeconfigs/status
verbs:
- patch
---
@@ -301,6 +699,15 @@ rules:
# to automatically delete [core|kube]dns pods so that are starting to being
# managed by Cilium
- delete
+- apiGroups:
+ - ""
+ resources:
+ - configmaps
+ resourceNames:
+ - cilium-config
+ verbs:
+ # allow patching of the configmap to set annotations
+ - patch
- apiGroups:
- ""
resources:
@@ -416,6 +823,9 @@ rules:
resources:
- ciliumendpointslices
- ciliumenvoyconfigs
+ - ciliumbgppeerconfigs
+ - ciliumbgpadvertisements
+ - ciliumbgpnodeconfigs
verbs:
- create
- update
@@ -442,6 +852,11 @@ rules:
resourceNames:
- ciliumloadbalancerippools.cilium.io
- ciliumbgppeeringpolicies.cilium.io
+ - ciliumbgpclusterconfigs.cilium.io
+ - ciliumbgppeerconfigs.cilium.io
+ - ciliumbgpadvertisements.cilium.io
+ - ciliumbgpnodeconfigs.cilium.io
+ - ciliumbgpnodeconfigoverrides.cilium.io
- ciliumclusterwideenvoyconfigs.cilium.io
- ciliumclusterwidenetworkpolicies.cilium.io
- ciliumegressgatewaypolicies.cilium.io
@@ -454,14 +869,27 @@ rules:
- ciliumnetworkpolicies.cilium.io
- ciliumnodes.cilium.io
- ciliumnodeconfigs.cilium.io
+ - ciliumcidrgroups.cilium.io
+ - ciliuml2announcementpolicies.cilium.io
+ - ciliumpodippools.cilium.io
- apiGroups:
- cilium.io
resources:
- ciliumloadbalancerippools
+ - ciliumpodippools
+ - ciliumbgppeeringpolicies
+ - ciliumbgpclusterconfigs
+ - ciliumbgpnodeconfigoverrides
verbs:
- get
- list
- watch
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumpodippools
+ verbs:
+ - create
- apiGroups:
- cilium.io
resources:
@@ -550,6 +978,31 @@ subjects:
name: "cilium"
namespace: kube-system
---
+# Source: cilium/templates/cilium-envoy/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: cilium-envoy
+ namespace: kube-system
+ annotations:
+ prometheus.io/scrape: "true"
+ prometheus.io/port: "9964"
+ labels:
+ k8s-app: cilium-envoy
+ app.kubernetes.io/name: cilium-envoy
+ app.kubernetes.io/part-of: cilium
+ io.cilium/app: proxy
+spec:
+ clusterIP: None
+ type: ClusterIP
+ selector:
+ k8s-app: cilium-envoy
+ ports:
+ - name: envoy-metrics
+ port: 9964
+ protocol: TCP
+ targetPort: envoy-metrics
+---
# Source: cilium/templates/cilium-agent/daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
@@ -571,21 +1024,17 @@ spec:
template:
metadata:
annotations:
- # Set app AppArmor's profile to "unconfined". The value of this annotation
- # can be modified as long users know which profiles they have available
- # in AppArmor.
- container.apparmor.security.beta.kubernetes.io/cilium-agent: "unconfined"
- container.apparmor.security.beta.kubernetes.io/clean-cilium-state: "unconfined"
- container.apparmor.security.beta.kubernetes.io/mount-cgroup: "unconfined"
- container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: "unconfined"
labels:
k8s-app: cilium
app.kubernetes.io/name: cilium-agent
app.kubernetes.io/part-of: cilium
spec:
+ securityContext:
+ appArmorProfile:
+ type: Unconfined
containers:
- name: cilium-agent
- image: "quay.io/cilium/cilium:v1.13.0@sha256:6544a3441b086a2e09005d3e21d1a4afb216fae19c5a60b35793c8a9438f8f68"
+ image: "quay.io/cilium/cilium:v1.16.3@sha256:62d2a09bbef840a46099ac4c69421c90f84f28d018d479749049011329aa7f28"
imagePullPolicy: IfNotPresent
command:
- cilium-agent
@@ -603,6 +1052,7 @@ spec:
failureThreshold: 105
periodSeconds: 2
successThreshold: 1
+ initialDelaySeconds: 5
livenessProbe:
httpGet:
host: "127.0.0.1"
@@ -642,26 +1092,38 @@ spec:
fieldPath: metadata.namespace
- name: CILIUM_CLUSTERMESH_CONFIG
value: /var/lib/cilium/clustermesh/
- - name: CILIUM_CNI_CHAINING_MODE
- valueFrom:
- configMapKeyRef:
- name: cilium-config
- key: cni-chaining-mode
- optional: true
- - name: CILIUM_CUSTOM_CNI_CONF
+ - name: GOMEMLIMIT
valueFrom:
- configMapKeyRef:
- name: cilium-config
- key: custom-cni-conf
- optional: true
+ resourceFieldRef:
+ resource: limits.memory
+ divisor: '1'
lifecycle:
postStart:
exec:
command:
- - "/cni-install.sh"
- - "--enable-debug=false"
- - "--cni-exclusive=true"
- - "--log-file=/var/run/cilium/cilium-cni.log"
+ - "bash"
+ - "-c"
+ - |
+ set -o errexit
+ set -o pipefail
+ set -o nounset
+
+ # When running in AWS ENI mode, it's likely that 'aws-node' has
+ # had a chance to install SNAT iptables rules. These can result
+ # in dropped traffic, so we should attempt to remove them.
+ # We do it using a 'postStart' hook since this may need to run
+ # for nodes which might have already been init'ed but may still
+ # have dangling rules. This is safe because there are no
+ # dependencies on anything that is part of the startup script
+ # itself, and can be safely run multiple times per node (e.g. in
+ # case of a restart).
+ if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]];
+ then
+ echo 'Deleting iptables rules created by the AWS CNI VPC plugin'
+ iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore
+ fi
+ echo 'Done!'
+
preStop:
exec:
command:
@@ -688,6 +1150,9 @@ spec:
- ALL
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
+ - name: envoy-sockets
+ mountPath: /var/run/cilium/envoy/sockets
+ readOnly: false
# Unprivileged containers need to mount /proc/sys/net from the host
# to have write access
- mountPath: /host/proc/sys/net
@@ -705,8 +1170,6 @@ spec:
mountPropagation: HostToContainer
- name: cilium-run
mountPath: /var/run/cilium
- - name: cni-path
- mountPath: /host/opt/cni/bin
- name: etc-cni-netd
mountPath: /host/etc/cni/net.d
- name: clustermesh-secrets
@@ -722,10 +1185,10 @@ spec:
mountPath: /tmp
initContainers:
- name: config
- image: "quay.io/cilium/cilium:v1.13.0@sha256:6544a3441b086a2e09005d3e21d1a4afb216fae19c5a60b35793c8a9438f8f68"
+ image: "quay.io/cilium/cilium:v1.16.3@sha256:62d2a09bbef840a46099ac4c69421c90f84f28d018d479749049011329aa7f28"
imagePullPolicy: IfNotPresent
command:
- - cilium
+ - cilium-dbg
- build-config
env:
- name: K8S_NODE_NAME
@@ -745,7 +1208,7 @@ spec:
# Required to mount cgroup2 filesystem on the underlying Kubernetes node.
# We use nsenter command with host's cgroup and mount namespaces enabled.
- name: mount-cgroup
- image: "quay.io/cilium/cilium:v1.13.0@sha256:6544a3441b086a2e09005d3e21d1a4afb216fae19c5a60b35793c8a9438f8f68"
+ image: "quay.io/cilium/cilium:v1.16.3@sha256:62d2a09bbef840a46099ac4c69421c90f84f28d018d479749049011329aa7f28"
imagePullPolicy: IfNotPresent
env:
- name: CGROUP_ROOT
@@ -782,7 +1245,7 @@ spec:
drop:
- ALL
- name: apply-sysctl-overwrites
- image: "quay.io/cilium/cilium:v1.13.0@sha256:6544a3441b086a2e09005d3e21d1a4afb216fae19c5a60b35793c8a9438f8f68"
+ image: "quay.io/cilium/cilium:v1.16.3@sha256:62d2a09bbef840a46099ac4c69421c90f84f28d018d479749049011329aa7f28"
imagePullPolicy: IfNotPresent
env:
- name: BIN_PATH
@@ -820,7 +1283,7 @@ spec:
# from a privileged container because the mount propagation bidirectional
# only works from privileged containers.
- name: mount-bpf-fs
- image: "quay.io/cilium/cilium:v1.13.0@sha256:6544a3441b086a2e09005d3e21d1a4afb216fae19c5a60b35793c8a9438f8f68"
+ image: "quay.io/cilium/cilium:v1.16.3@sha256:62d2a09bbef840a46099ac4c69421c90f84f28d018d479749049011329aa7f28"
imagePullPolicy: IfNotPresent
args:
- 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf'
@@ -836,7 +1299,7 @@ spec:
mountPath: /sys/fs/bpf
mountPropagation: Bidirectional
- name: clean-cilium-state
- image: "quay.io/cilium/cilium:v1.13.0@sha256:6544a3441b086a2e09005d3e21d1a4afb216fae19c5a60b35793c8a9438f8f68"
+ image: "quay.io/cilium/cilium:v1.16.3@sha256:62d2a09bbef840a46099ac4c69421c90f84f28d018d479749049011329aa7f28"
imagePullPolicy: IfNotPresent
command:
- /init-container.sh
@@ -853,6 +1316,12 @@ spec:
name: cilium-config
key: clean-cilium-bpf-state
optional: true
+ - name: WRITE_CNI_CONF_WHEN_READY
+ valueFrom:
+ configMapKeyRef:
+ name: cilium-config
+ key: write-cni-conf-when-ready
+ optional: true
terminationMessagePolicy: FallbackToLogsOnError
securityContext:
seLinuxOptions:
@@ -874,15 +1343,32 @@ spec:
mountPath: /run/cilium/cgroupv2
mountPropagation: HostToContainer
- name: cilium-run
- mountPath: /var/run/cilium
+ mountPath: /var/run/cilium # wait-for-kube-proxy
+ # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent
+ - name: install-cni-binaries
+ image: "quay.io/cilium/cilium:v1.16.3@sha256:62d2a09bbef840a46099ac4c69421c90f84f28d018d479749049011329aa7f28"
+ imagePullPolicy: IfNotPresent
+ command:
+ - "/install-plugin.sh"
resources:
requests:
cpu: 100m
- memory: 100Mi # wait-for-kube-proxy
+ memory: 10Mi
+ securityContext:
+ seLinuxOptions:
+ level: s0
+ type: spc_t
+ capabilities:
+ drop:
+ - ALL
+ terminationMessagePolicy: FallbackToLogsOnError
+ volumeMounts:
+ - name: cni-path
+ mountPath: /host/opt/cni/bin # .Values.cni.install
restartPolicy: Always
priorityClassName: system-node-critical
- serviceAccount: "cilium"
serviceAccountName: "cilium"
+ automountServiceAccountToken: true
terminationGracePeriodSeconds: 1
hostNetwork: true
affinity:
@@ -910,7 +1396,7 @@ spec:
hostPath:
path: /sys/fs/bpf
type: DirectoryOrCreate
- # To mount cgroup2 filesystem on the host
+ # To mount cgroup2 filesystem on the host or apply sysctlfix
- name: hostproc
hostPath:
path: /proc
@@ -939,13 +1425,48 @@ spec:
hostPath:
path: /run/xtables.lock
type: FileOrCreate
+ # Sharing socket with Cilium Envoy on the same node by using a host path
+ - name: envoy-sockets
+ hostPath:
+ path: "/var/run/cilium/envoy/sockets"
+ type: DirectoryOrCreate
# To read the clustermesh configuration
- name: clustermesh-secrets
- secret:
- secretName: cilium-clustermesh
+ projected:
# note: the leading zero means this number is in octal representation: do not remove it
defaultMode: 0400
- optional: true
+ sources:
+ - secret:
+ name: cilium-clustermesh
+ optional: true
+ # note: items are not explicitly listed here, since the entries of this secret
+ # depend on the peers configured, and that would cause a restart of all agents
+ # at every addition/removal. Leaving the field empty makes each secret entry
+ # to be automatically projected into the volume as a file whose name is the key.
+ - secret:
+ name: clustermesh-apiserver-remote-cert
+ optional: true
+ items:
+ - key: tls.key
+ path: common-etcd-client.key
+ - key: tls.crt
+ path: common-etcd-client.crt
+ - key: ca.crt
+ path: common-etcd-client-ca.crt
+ # note: we configure the volume for the kvstoremesh-specific certificate
+ # regardless of whether KVStoreMesh is enabled or not, so that it can be
+ # automatically mounted in case KVStoreMesh gets subsequently enabled,
+ # without requiring an agent restart.
+ - secret:
+ name: clustermesh-apiserver-local-cert
+ optional: true
+ items:
+ - key: tls.key
+ path: local-etcd-client.key
+ - key: tls.crt
+ path: local-etcd-client.crt
+ - key: ca.crt
+ path: local-etcd-client-ca.crt
- name: host-proc-sys-net
hostPath:
path: /proc/sys/net
@@ -955,6 +1476,174 @@ spec:
path: /proc/sys/kernel
type: Directory
---
+# Source: cilium/templates/cilium-envoy/daemonset.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: cilium-envoy
+ namespace: kube-system
+ labels:
+ k8s-app: cilium-envoy
+ app.kubernetes.io/part-of: cilium
+ app.kubernetes.io/name: cilium-envoy
+ name: cilium-envoy
+spec:
+ selector:
+ matchLabels:
+ k8s-app: cilium-envoy
+ updateStrategy:
+ rollingUpdate:
+ maxUnavailable: 2
+ type: RollingUpdate
+ template:
+ metadata:
+ annotations:
+ labels:
+ k8s-app: cilium-envoy
+ name: cilium-envoy
+ app.kubernetes.io/name: cilium-envoy
+ app.kubernetes.io/part-of: cilium
+ spec:
+ securityContext:
+ appArmorProfile:
+ type: Unconfined
+ containers:
+ - name: cilium-envoy
+ image: "quay.io/cilium/cilium-envoy:v1.29.9-1728346947-0d05e48bfbb8c4737ec40d5781d970a550ed2bbd@sha256:42614a44e508f70d03a04470df5f61e3cffd22462471a0be0544cf116f2c50ba"
+ imagePullPolicy: IfNotPresent
+ command:
+ - /usr/bin/cilium-envoy-starter
+ args:
+ - '--'
+ - '-c /var/run/cilium/envoy/bootstrap-config.json'
+ - '--base-id 0'
+ - '--log-level info'
+ - '--log-format [%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v'
+ startupProbe:
+ httpGet:
+ host: "127.0.0.1"
+ path: /healthz
+ port: 9878
+ scheme: HTTP
+ failureThreshold: 105
+ periodSeconds: 2
+ successThreshold: 1
+ initialDelaySeconds: 5
+ livenessProbe:
+ httpGet:
+ host: "127.0.0.1"
+ path: /healthz
+ port: 9878
+ scheme: HTTP
+ periodSeconds: 30
+ successThreshold: 1
+ failureThreshold: 10
+ timeoutSeconds: 5
+ readinessProbe:
+ httpGet:
+ host: "127.0.0.1"
+ path: /healthz
+ port: 9878
+ scheme: HTTP
+ periodSeconds: 30
+ successThreshold: 1
+ failureThreshold: 3
+ timeoutSeconds: 5
+ env:
+ - name: K8S_NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ - name: CILIUM_K8S_NAMESPACE
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: metadata.namespace
+ ports:
+ - name: envoy-metrics
+ containerPort: 9964
+ hostPort: 9964
+ protocol: TCP
+ securityContext:
+ seLinuxOptions:
+ level: s0
+ type: spc_t
+ capabilities:
+ add:
+ - NET_ADMIN
+ - SYS_ADMIN
+ drop:
+ - ALL
+ terminationMessagePolicy: FallbackToLogsOnError
+ volumeMounts:
+ - name: envoy-sockets
+ mountPath: /var/run/cilium/envoy/sockets
+ readOnly: false
+ - name: envoy-artifacts
+ mountPath: /var/run/cilium/envoy/artifacts
+ readOnly: true
+ - name: envoy-config
+ mountPath: /var/run/cilium/envoy/
+ readOnly: true
+ - name: bpf-maps
+ mountPath: /sys/fs/bpf
+ mountPropagation: HostToContainer
+ restartPolicy: Always
+ priorityClassName: system-node-critical
+ serviceAccountName: "cilium-envoy"
+ automountServiceAccountToken: true
+ terminationGracePeriodSeconds: 1
+ hostNetwork: true
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: cilium.io/no-schedule
+ operator: NotIn
+ values:
+ - "true"
+ podAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchLabels:
+ k8s-app: cilium
+ topologyKey: kubernetes.io/hostname
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchLabels:
+ k8s-app: cilium-envoy
+ topologyKey: kubernetes.io/hostname
+ nodeSelector:
+ kubernetes.io/os: linux
+ tolerations:
+ - operator: Exists
+ volumes:
+ - name: envoy-sockets
+ hostPath:
+ path: "/var/run/cilium/envoy/sockets"
+ type: DirectoryOrCreate
+ - name: envoy-artifacts
+ hostPath:
+ path: "/var/run/cilium/envoy/artifacts"
+ type: DirectoryOrCreate
+ - name: envoy-config
+ configMap:
+ name: cilium-envoy-config
+ # note: the leading zero means this number is in octal representation: do not remove it
+ defaultMode: 0400
+ items:
+ - key: bootstrap-config.json
+ path: bootstrap-config.json
+ # To keep state between restarts / upgrades
+ # To keep state between restarts / upgrades for bpf maps
+ - name: bpf-maps
+ hostPath:
+ path: /sys/fs/bpf
+ type: DirectoryOrCreate
+---
# Source: cilium/templates/cilium-operator/deployment.yaml
apiVersion: apps/v1
kind: Deployment
@@ -974,14 +1663,20 @@ spec:
matchLabels:
io.cilium/app: operator
name: cilium-operator
+ # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case
+ # of one replica and no user configured Recreate strategy.
+ # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the
+ # podAntiAffinity which prevents deployments of multiple operator replicas on the same node.
strategy:
rollingUpdate:
- maxSurge: 1
- maxUnavailable: 1
+ maxSurge: 25%
+ maxUnavailable: 50%
type: RollingUpdate
template:
metadata:
annotations:
+ prometheus.io/port: "9963"
+ prometheus.io/scrape: "true"
labels:
io.cilium/app: operator
name: cilium-operator
@@ -990,7 +1685,7 @@ spec:
spec:
containers:
- name: cilium-operator
- image: "quay.io/cilium/operator-generic:v1.13.0@sha256:4b58d5b33e53378355f6e8ceb525ccf938b7b6f5384b35373f1f46787467ebf5"
+ image: "quay.io/cilium/operator-generic:v1.16.3@sha256:6e2925ef47a1c76e183c48f95d4ce0d34a1e5e848252f910476c3e11ce1ec94b"
imagePullPolicy: IfNotPresent
command:
- cilium-operator-generic
@@ -1014,6 +1709,11 @@ spec:
key: debug
name: cilium-config
optional: true
+ ports:
+ - name: prometheus
+ containerPort: 9963
+ hostPort: 9963
+ protocol: TCP
livenessProbe:
httpGet:
host: "127.0.0.1"
@@ -1023,6 +1723,16 @@ spec:
initialDelaySeconds: 60
periodSeconds: 10
timeoutSeconds: 3
+ readinessProbe:
+ httpGet:
+ host: "127.0.0.1"
+ path: /healthz
+ port: 9234
+ scheme: HTTP
+ initialDelaySeconds: 0
+ periodSeconds: 5
+ timeoutSeconds: 3
+ failureThreshold: 5
volumeMounts:
- name: cilium-config-path
mountPath: /tmp/cilium/config-map
@@ -1031,8 +1741,8 @@ spec:
hostNetwork: true
restartPolicy: Always
priorityClassName: system-cluster-critical
- serviceAccount: "cilium-operator"
serviceAccountName: "cilium-operator"
+ automountServiceAccountToken: true
# In HA mode, cilium-operator pods must not be scheduled on the same
# node as they will clash with each other.
affinity:
@@ -1051,6 +1761,3 @@ spec:
- name: cilium-config-path
configMap:
name: cilium-config
----
-# Source: cilium/templates/cilium-secrets-namespace.yaml
-# Only create the namespace if it's different from Ingress secret namespace or Ingress is not enabled.
diff --git a/tests/files/custom_cni/values.yaml b/tests/files/custom_cni/values.yaml
index bba8cf7444aeeb7157192e9a42d0c410a352d6cc..047a9001885cd6c6d6826e303cc7a5df2ecd569c 100644
--- a/tests/files/custom_cni/values.yaml
+++ b/tests/files/custom_cni/values.yaml
@@ -8,4 +8,4 @@ hubble:
ipam:
operator:
# Set the appropriate pods subnet
- clusterPoolIPv4PodCIDR: "{{ kube_pods_subnet }}"
+ clusterPoolIPv4PodCIDRList: ["{{ kube_pods_subnet }}"]
diff --git a/tests/files/packet_debian11-custom-cni.yml b/tests/files/packet_debian11-custom-cni.yml
index 407423e384bb5153adde7cc4e9f884ae5d9910a7..9eb717b22cb9e25e7e16673e3538b068e1ded337 100644
--- a/tests/files/packet_debian11-custom-cni.yml
+++ b/tests/files/packet_debian11-custom-cni.yml
@@ -4,6 +4,7 @@ cloud_image: debian-11
mode: default
# Kubespray settings
+kube_owner: root
kube_network_plugin: custom_cni
custom_cni_manifests:
- "{{ playbook_dir }}/../tests/files/custom_cni/cilium.yaml"
diff --git a/tests/files/packet_debian12-custom-cni-helm.yml b/tests/files/packet_debian12-custom-cni-helm.yml
index 0ed919828c749f68e73ae085149c838dcdc1331e..107c44dd3a4f09b219ec268b77d59eb19f3511c1 100644
--- a/tests/files/packet_debian12-custom-cni-helm.yml
+++ b/tests/files/packet_debian12-custom-cni-helm.yml
@@ -11,7 +11,7 @@ custom_cni_chart_release_name: cilium
custom_cni_chart_repository_name: cilium
custom_cni_chart_repository_url: https://helm.cilium.io
custom_cni_chart_ref: cilium/cilium
-custom_cni_chart_version: 1.14.3
+custom_cni_chart_version: 1.16.3
custom_cni_chart_values:
cluster:
name: kubespray