bootstrapping kube-proxy

kube-proxy とは

kube-proxy とは各worker nodeで動作するネットワークプロキシを実現するコンポーネントです。具体的にはSertviceリソースで作成されるCluster IPやNode Portの管理とそのルーティングテーブルの管理、またnginx ingress controllerを利用したIngressリソースではPodへの負荷分散にkube-proxyを活用指定たりするそうです(With NGINX, we’ll use the DNS name or virtual IP address to identify the service, and rely on kube-proxy to perform the internal load-balancing across the pool of pods.)

手順

Dockerfile_kube-proxy.armhf を作成する

Dockerfile_kube-proxy.armhf

cat << 'EOF' > Dockerfile_kube-proxy.armhf
FROM arm64v8/ubuntu:bionic

ARG VERSION="v1.30.1"
ARG ARCH="arm64"

RUN set -ex \
  && apt update \
  && apt install -y wget \
  && apt clean \
  && wget -P /usr/bin/ https://dl.k8s.io/$VERSION/bin/linux/$ARCH/kube-proxy \
  && chmod +x /usr/bin/kube-proxy \
  && install -o root -g root -m 755 -d /var/lib/kube-proxy \
  && install -o root -g root -m 755 -d /etc/kubernetes/config

COPY kube-proxy.kubeconfig /var/lib//kube-proxy/kubeconfig

ENTRYPOINT ["/usr/bin/kube-proxy"]
EOF

image build

sudo nerdctl build --namespace k8s.io -f Dockerfile_kube-proxy.armhf -t k8s-kube-proxy ./

kernel parameter

cat <<EOF | sudo tee /etc/sysctl.d/kubelet.conf
# kube-proxy
net.ipv4.conf.all.route_localnet = 1
net.netfilter.nf_conntrack_max = 131072
net.netfilter.nf_conntrack_tcp_timeout_established = 86400
net.netfilter.nf_conntrack_tcp_timeout_close_wait = 3600
EOF

sudo sysctl --system

cat <<EOF | sudo tee /etc/modprobe.d/kube-proxy.conf
options nf_conntrack hashsize=32768
EOF

sudo /sbin/modprobe nf_conntrack hashsize=32768

pod manifestsを /etc/kubernetes/manifests/ へ作成する

/etc/kubernetes/manifests/kube-proxy.yaml

cluster_cidr="10.200.0.0/16"
sudo mkdir -p /etc/kubernetes/manifests

cat << EOF | sudo tee /etc/kubernetes/manifests/kube-proxy.yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
  labels:
    app: kube-proxy
  name: kube-proxy-configuration
  namespace: kube-system
data:
  config.conf: |-
    ---
    apiVersion: kubeproxy.config.k8s.io/v1alpha1
    kind: KubeProxyConfiguration
    clientConnection:
      kubeconfig: "/var/lib/kube-proxy/kubeconfig"
    mode: "iptables"
    clusterCIDR: "${cluster_cidr}"

    # https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/
    # metricsBindAddress: 127.0.0.1:10249
    metricsBindAddress: 0.0.0.0:10249
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-proxy
  namespace: kube-system
  labels:
    component: kube-proxy
    # TODO
    # master nodeにaddon-managerを導入したらコメント外す
    # addonmanager.kubernetes.io/mode=Reconcile
spec:
  selector:
    matchLabels:
      name: kube-proxy
  # https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#performing-a-rolling-update
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  template:
    # template 以下はpod templates
    #   (apiVersionやkindをもたないことを除いては、Podのテンプレートと同じスキーマ)
    #   https://kubernetes.io/ja/docs/concepts/workloads/controllers/daemonset/
    metadata:
      labels:
        name: kube-proxy
    spec:
      # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
      priorityClassName: system-node-critical
      hostNetwork: true
      containers:
        - name: kube-proxy
          image: k8s-kube-proxy:latest
          securityContext:
            capabilities:
              add:
                - SYS_ADMIN
                - NET_ADMIN
                - NET_RAW
          command:
            - /usr/bin/kube-proxy
            - --config=/var/lib/kube-proxy/kube-proxy-config.yaml
          imagePullPolicy: IfNotPresent
          resources:
            requests:
              cpu: "256m"
          volumeMounts:
          - name: kube-proxy-configuration
            mountPath: /var/lib/kube-proxy/kube-proxy-config.yaml
          - name: conntrack-command
            mountPath: /usr/sbin/conntrack
          - name: iptables-command
            mountPath: /usr/sbin/iptables
          - name: iptables-restore-command
            mountPath: /usr/sbin/iptables-restore
          - name: iptables-save-command
            mountPath: /usr/sbin/iptables-save
          - name: xtables-lock-file
            mountPath: /run/xtables.lock
          - name: usr-lib-dir
            mountPath: /usr/lib
          - name: lib-dir
            mountPath: /lib
          - name: sys-dir
            mountPath: /sys
      volumes:
      - name: kube-proxy-configuration
        configMap:
          name: kube-proxy-configuration
      - name: conntrack-command
        hostPath:
          path: /usr/sbin/conntrack
      - name: iptables-command
        hostPath:
          path: /usr/sbin/iptables
      - name: iptables-restore-command
        hostPath:
          path: /usr/sbin/iptables-restore
      - name: iptables-save-command
        hostPath:
          path: /usr/sbin/iptables-save
      - name: xtables-lock-file
        hostPath:
          path: /run/xtables.lock
      - name: usr-lib-dir
        hostPath:
          path: /usr/lib
      - name: lib-dir
        hostPath:
          path: /lib
      - name: sys-dir
        hostPath:
          path: /sys
EOF

podをデプロイする

kubectl apply -f /etc/kubernetes/manifests/kube-proxy.yaml