bootstrapping kube-apiserver

手順

Dockerfile_kube-apiserver.armhf を作成する

Dockerfile_kube-apiserver.armhf

cat << 'EOF' > Dockerfile_kube-apiserver.armhf
FROM arm64v8/ubuntu:bionic

ARG VERSION="v1.30.1"
ARG ARCH="arm64"

RUN set -ex \
  && apt update \
  && apt install -y wget \
  && apt clean \
  && wget --quiet -P /usr/bin/ https://dl.k8s.io/$VERSION/bin/linux/$ARCH/kube-apiserver \
  && chmod +x /usr/bin/kube-apiserver \
  && install -o root -g root -m 755 -d /var/lib/kubernetes \
  && install -o root -g root -m 755 -d /etc/kubernetes/config \
  && install -o root -g root -m 755 -d /etc/kubernetes/webhook

COPY ca.pem \
     ca-key.pem \
     kubernetes-key.pem \
     kubernetes.pem \
     service-account-key.pem \
     service-account.pem \
     encryption-config.yaml \
     front-proxy-ca.pem \
     front-proxy.pem \
     front-proxy-key.pem \
     /var/lib/kubernetes/

COPY authorization-config.yaml /etc/kubernetes/webhook/

EXPOSE 6443

ENTRYPOINT ["/usr/bin/kube-apiserver"]
EOF

encryption-provider-config を作成する
- --encryption-provider-config オプションで指定してsecretリソースを作成する際に暗号化するための鍵を定義する
  - https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/#encrypting-your-data
  - https://access.redhat.com/documentation/ja-jp/openshift_container_platform/3.11/html/cluster_administration/admin-guide-encrypting-data-at-datastore
    encryption-config.yaml
    ENCRYPTION_KEY=$(head -c 32 /dev/urandom | base64) cat << EOF > encryption-config.yaml --- kind: EncryptionConfig apiVersion: v1 resources: - resources: - secrets providers: - aescbc: keys: - name: key1 secret: ${ENCRYPTION_KEY} - identity: {} EOF

webbhook configファイルを作成する

--authorization-webhook-config-file で指定するファイル

authorization-config.yaml

KUBE_API_SERVER_ADDRESS=k8s-master

cat << EOF > authorization-config.yaml
---
apiVersion: v1
# kind of the API object
kind: Config
# clusters refers to the remote service.
clusters:
  - name: kubernetes
    cluster:
      certificate-authority: /var/lib/kubernetes/ca.pem       # CA for verifying the remote service.
      server: https://${KUBE_API_SERVER_ADDRESS}:6443/authenticate # URL of remote service to query. Must use 'https'.

# users refers to the API server's webhook configuration.
users:
  - name: api-server-webhook
    user:
      client-certificate: /var/lib/kubernetes/kubernetes.pem  # cert for the webhook plugin to use
      client-key: /var/lib/kubernetes/kubernetes-key.pem      # key matching the cert

# kubeconfig files require a context. Provide one for the API server.
current-context: webhook
contexts:
- context:
    cluster: kubernetes
    user: api-server-webhook
  name: webhook
EOF

image build

sudo nerdctl build --namespace k8s.io -t k8s-kube-apiserver --file=Dockerfile_kube-apiserver.armhf ./

pod manifestsを /etc/kubelet.d へ作成する

--advertise-address オプションはIPアドレスで指定する必要がある(hostnameでは起動しなかった)

/etc/kubelet.d/kube-api-server.yaml

KUBE_API_SERVER_ADDRESS=192.168.3.50

cat << EOF | sudo tee /etc/kubelet.d/kube-api-server.yaml
---
apiVersion: v1
kind: Pod
metadata:
  name: kube-apiserver
  namespace: kube-system
  annotations:
    seccomp.security.alpha.kubernetes.io/pod: runtime/default
  labels:
    tier: control-plane
    component: kube-apiserver

spec:
  # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
  priorityClassName: system-node-critical
  hostNetwork: true
  containers:
    - name: kube-apiserver
      image: k8s-kube-apiserver:latest
      imagePullPolicy: IfNotPresent
      resources:
        requests:
          memory: "512Mi"
        limits:
          memory: "1024Mi"
      command:
        - /usr/bin/kube-apiserver
        - --advertise-address=k8s-master
        - --allow-privileged=true
        - --anonymous-auth=false
        - --apiserver-count=1
        - --audit-log-maxage=30
        - --audit-log-maxbackup=3
        - --audit-log-maxsize=100
        - --audit-log-path=/var/log/audit.log
        - --authorization-mode=Node,RBAC,Webhook
        - --authorization-webhook-config-file=/etc/kubernetes/webhook/authorization-config.yaml
        - --authentication-token-webhook-cache-ttl=2m
        - --authentication-token-webhook-version=v1
        - --bind-address=0.0.0.0
        - --client-ca-file=/var/lib/kubernetes/ca.pem
        - --enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,RuntimeClass
        - --etcd-cafile=/var/lib/kubernetes/ca.pem
        - --etcd-certfile=/var/lib/kubernetes/kubernetes.pem
        - --etcd-keyfile=/var/lib/kubernetes/kubernetes-key.pem
        - --etcd-servers=https://k8s-master:2379
        - --event-ttl=1h
        - --encryption-provider-config=/var/lib/kubernetes/encryption-config.yaml
        - --kubelet-certificate-authority=/var/lib/kubernetes/ca.pem
        - --kubelet-client-certificate=/var/lib/kubernetes/kubernetes.pem
        - --kubelet-client-key=/var/lib/kubernetes/kubernetes-key.pem
        - --runtime-config=authentication.k8s.io/v1beta1=true
        - --feature-gates=APIPriorityAndFairness=false
        - --service-account-key-file=/var/lib/kubernetes/service-account.pem
        - --service-account-signing-key-file=/var/lib/kubernetes/service-account-key.pem
        - --service-account-issuer=api
        - --service-account-api-audiences=api
        - --service-cluster-ip-range=10.32.0.0/24
        - --service-node-port-range=30000-32767
        - --tls-cert-file=/var/lib/kubernetes/kubernetes.pem
        - --tls-private-key-file=/var/lib/kubernetes/kubernetes-key.pem
        - --http2-max-streams-per-connection=3000
        - --max-requests-inflight=3000
        - --max-mutating-requests-inflight=1000
        - --enable-aggregator-routing=true
        - --requestheader-client-ca-file=/var/lib/kubernetes/front-proxy-ca.pem
        - --requestheader-allowed-names=front-proxy-ca
        - --requestheader-extra-headers-prefix=X-Remote-Extra
        - --requestheader-group-headers=X-Remote-Group
        - --requestheader-username-headers=X-Remote-User
        - --proxy-client-cert-file=/var/lib/kubernetes/front-proxy.pem
        - --proxy-client-key-file=/var/lib/kubernetes/front-proxy-key.pem
        - --v=2
EOF

crictl でコンテナ起動を確認する

$ sudo crictl ps --name kube-apiserver
CONTAINER           IMAGE                                                              CREATED             STATE               NAME                ATTEMPT             POD ID
82c371fd9d99e       83e685a0b921ef5dd91eb3cdf208ba70690c1dd7decfc39bb3903be6ede752e6   24 seconds ago      Running             kube-apiserver      0                   6af4d1b99fa37

master nodeにPodがscheduleされないようにする
- taintを設定する
  - https://kubernetes.io/ja/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm/
  - https://kubernetes.io/ja/docs/concepts/scheduling-eviction/taint-and-toleration/
```
kubectl taint nodes k8s-master node-role.kubernetes.io/master:NoSchedule
```
```
$ kubectl get node k8s-master -o=jsonpath='{.spec.taints}'
[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]
```

エラー事例

failed creating mandatory flowcontrol settings: failed getting mandatory FlowSchema exempt due to the server was unable to return a response in the time allotted, but may still be processing the request
- https://github.com/kubernetes/kubernetes/issues/97525#issuecomment-753022219
  - kube-apiserver起動オプションに以下2つを付加する
    - --feature-gates=APIPriorityAndFairness=false
    - --runtime-config=flowcontrol.apiserver.k8s.io/v1beta1=false
failed to run Kubelet: no client provided, cannot use webhook authentication
- kubeletがWebhook認証を期待しているのにkube-api-serverでWebhook認証が有効でない場合
Failed creating a mirror pod for "kube-scheduler-k8s-master_kube-system(a4a914cd05761a5a4335e2510ca075aa)": pods "kube-scheduler-k8s-master" is forbidden: PodSecurityPolicy: no providers available to validate pod request
- StaticPodを起動した際にkubeletからkube-apiserverへmirror pod情報を登録しようとしてPodSecurityPolicyにより拒否された

参考文献

old

defaultのPodSecurityPolicy(PSP)を作成する

staticPod を作成する際にkubeletからmirror pod作成リクエストが拒否されないようにします (参考)

PSP / ClusterRole / ClusterRoleBinding

cat << EOF | kubectl apply --kubeconfig admin.kubeconfig -f -
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
  annotations:
    apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
    apparmor.security.beta.kubernetes.io/defaultProfileName:  'runtime/default'
    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default'
    seccomp.security.alpha.kubernetes.io/defaultProfileName:  'docker/default'
  name: default
spec:
  # allowedCapabilities: []  # default set of capabilities are implicitly allowed
  allowedCapabilities:
    - '*'
    # - NET_ADMIN
    # - NET_RAW
    # - SYS_ADMIN
  fsGroup:
    rule: 'MustRunAs'
    ranges:
      # Forbid adding the root group.
      - min: 1
        max: 65535
  hostIPC: true
  hostNetwork: true
  hostPID: true
  privileged: true
  allowPrivilegeEscalation: true
  readOnlyRootFilesystem: true
  runAsUser:
    rule: 'MustRunAsNonRoot'
  seLinux:
    rule: 'RunAsNonRoot'
  supplementalGroups:
    rule: 'RunAsNonRoot'
    ranges:
      # Forbid adding the root group.
      - min: 1
        max: 65535
  volumes:
  - 'configMap'
  - 'downwardAPI'
  - 'emptyDir'
  - 'persistentVolumeClaim'
  - 'projected'
  - 'secret'
  - 'hostPath'
  hostNetwork: true
  runAsUser:
    rule: 'RunAsAny'
  seLinux:
    rule: 'RunAsAny'
  supplementalGroups:
    rule: 'RunAsAny'
  fsGroup:
    rule: 'RunAsAny'

---

# Cluster role which grants access to the default pod security policy
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: default-psp
rules:
- apiGroups:
  - policy
  resourceNames:
  - default
  resources:
  - podsecuritypolicies
  verbs:
  - use

---

# Cluster role binding for default pod security policy granting all authenticated users access
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: default-psp
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: default-psp
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:authenticated
EOF

$ cat <<EOF | kubectl apply --kubeconfig admin.kubeconfig -f -

  <省略>

podsecuritypolicy.policy/default created
clusterrole.rbac.authorization.k8s.io/default-psp created
clusterrolebinding.rbac.authorization.k8s.io/default-psp created