Tencent cloud uses the system disk as the default kubelet working directory in / var/lib/kubelet. The system disk is generally not large enough and inconvenient for expansion, so you need to modify the kubelet working directory. The general process is as follows:
-
Preparation
-
Expulsion node
-
Modify configuration
-
Restart service
-
Unsealing node
- Validation service
Preparation
Find the kubelet startup file. The general configuration information is in the startup file
[root@VM_8_9_centos ~]# locate kubelet.service
/etc/systemd/system/multi-user.target.wants/kubelet.service
/usr/lib/systemd/system/kubelet.service
[root@VM_8_9_centos ~]# ls -lh /usr/lib/systemd/system/kubelet.service
[root@VM_8_9_centos ~]# ls -lh /etc/systemd/system/multi-user.target.wants/kubelet.service
lrwxrwxrwx 1 root root 39 Sep 25 11:59 /etc/systemd/system/multi-user.target.wants/kubelet.service -> /usr/lib/systemd/system/kubelet.service
[root@VM_8_9_centos ~]# cat /usr/lib/systemd/system/kubelet.service
[Unit]
Description=kubelet
[Service]
Environment=QCLOUD_NORM_URL=
EnvironmentFile=-/etc/kubernetes/kubelet
ExecStart=/usr/bin/kubelet ${CNI_BIN_DIR} ${KUBE_RESERVED} ${MAX_PODS} ${CLOUD_PROVIDER} ${CLOUD_CONFIG} ${POD_INFRA_CONTAINER_IMAGE} ${CLUSTER_DOMAIN} ${ALLOW_PRIVILEGED} ${REGISTER_SCHEDULABLE} ${FAIL_SWAP_ON} ${ANONYMOUS_AUTH} ${IMAGE_PULL_PROGRESS_DEADLINE} ${HOSTNAME_OVERRIDE} ${EVICTION_HARD} ${AUTHENTICATION_TOKEN_WEBHOOK} ${CLIENT_CA_FILE} ${AUTHORIZATION_MODE} ${CLUSTER_DNS} ${NON_MASQUERADE_CIDR} ${NETWORK_PLUGIN} ${KUBECONFIG} ${V}
ExecStartPost=-/bin/bash /etc/kubernetes/deny-tcp-port-10250.sh
Restart=always
RestartSec=10
LimitNOFILE=65536
[Install]
WantedBy=multi-user.targetDiscover environment file in kubelet according to startup file
[root@VM_8_9_centos ~]# cat /etc/kubernetes/kubelet CNI_BIN_DIR="--cni-bin-dir=/opt/cni/bin/" KUBE_RESERVED="--kube-reserved=cpu=90m,memory=1830Mi" MAX_PODS="--max-pods=253" CLOUD_PROVIDER="--cloud-provider=qcloud" CLOUD_CONFIG="--cloud-config=/etc/kubernetes/qcloud.conf" POD_INFRA_CONTAINER_IMAGE="--pod-infra-container-image=ccr.ccs.tencentyun.com/library/pause:latest" CLUSTER_DOMAIN="--cluster-domain=cluster.local" ALLOW_PRIVILEGED="--allow-privileged=true" REGISTER_SCHEDULABLE="--register-schedulable=true" FAIL_SWAP_ON="--fail-swap-on=false" ANONYMOUS_AUTH="--anonymous-auth=false" IMAGE_PULL_PROGRESS_DEADLINE="--image-pull-progress-deadline=10m0s" HOSTNAME_OVERRIDE="--hostname-override=192.168.8.9" EVICTION_HARD="--eviction-hard=nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi" AUTHENTICATION_TOKEN_WEBHOOK="--authentication-token-webhook=true" CLIENT_CA_FILE="--client-ca-file=/etc/kubernetes/cluster-ca.crt" AUTHORIZATION_MODE="--authorization-mode=Webhook" CLUSTER_DNS="--cluster-dns=172.16.255.48" NON_MASQUERADE_CIDR="--non-masquerade-cidr=0.0.0.0/0" NETWORK_PLUGIN="--network-plugin=cni" KUBECONFIG="--kubeconfig=/etc/kubernetes/kubelet-kubeconfig" V="--v=2"
Expulsion node
Eviction will automatically block the node (evicting the pod in the node to other nodes in the cluster will not affect the use of the cluster, except for the DaemonSet pod). After eviction, the node will be checked as non schedulable
[root@VM_8_9_centos ~]# kubectl drain 192.168.8.9 node/192.168.8.9 cordoned error: unable to drain node "192.168.8.9", aborting command... There are pending nodes to be drained: 192.168.8.9 error: DaemonSet-managed pods (use --ignore-daemonsets to ignore): rancher-logging-fluentd-tlqnq, rancher-logging-log-aggregator-m5g69, registry-proxy-xkmk5, exporter-node-cluster-monitoring-t5jj7, cattle-node-agent-j8wcn, ccs-log-collector-x57f8, ip-masq-agent-xt9q2, tke-bridge-agent-8rxgl, tke-cni-agent-lvv98; pods with local storage (use --delete-local-data to override): coupon-service-1, yuedi-passenger-api-1, coupon-service-1, rental-api-1, passenger-api-0, eureka-1, service-9777bd6ff-bsnj7, rental-service-0, passenger-api-0, swift-778d866966-c6nk7 [root@VM_8_9_centos ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION 192.168.8.14 Ready <none> 89d v1.12.4-tke.10 192.168.8.15 Ready <none> 181d v1.12.4-tke.3 192.168.8.3 Ready <none> 181d v1.12.4-tke.3 192.168.8.4 Ready <none> 181d v1.12.4-tke.3 192.168.8.9 Ready,SchedulingDisabled <none> 89d v1.12.4-tke.10 [root@VM_8_9_centos ~]# ls /data/ docker kubelet lost+found
Modify configuration
Modify the first line of the kubelet configuration file to add the kubelet directory parameter
[root@VM_8_9_centos ~]# head -n 1 /etc/kubernetes/kubelet ROOT_DIR="--root-dir=/data/kubelet"
Modify the startup profile to add the root? Dir parameter
[root@VM_8_9_centos ~]# vim /usr/lib/systemd/system/kubelet.service
ExecStart=/usr/bin/kubelet ${ROOT_DIR} ${CNI_BIN_DIR} ${KUBE_RESERVED} ${MAX_PODS} ${CLOUD_PROVIDER} ${CLOUD_CONFIG} ${POD_INFRA_CONTAINER_IMAGE} ${CLUSTER_DOMAIN} ${ALLOW_PRIVILEGED} ${REGISTER_SCHEDULABLE} ${FAIL_SWAP_ON} ${ANONYMOUS_AUTH} ${IMAGE_PULL_PROGRESS_DEADLINE} ${HOSTNAME_OVERRIDE} ${EVICTION_HARD} ${AUTHENTICATION_TOKEN_WEBHOOK} ${CLIENT_CA_FILE} ${AUTHORIZATION_MODE} ${CLUSTER_DNS} ${NON_MASQUERADE_CIDR} ${NETWORK_PLUGIN} ${KUBECONFIG} ${V}Restart service
[root@VM_8_9_centos ~]# systemctl daemon-reload
[root@VM_8_9_centos ~]# systemctl restart kubelet
[root@VM_8_9_centos ~]# systemctl status kubelet -l
● kubelet.service - kubelet
Loaded: loaded (/usr/lib/systemd/system/kubelet.service; enabled; vendor preset: disabled)ls /
Active: active (running) since Mon 2019-12-23 16:38:04 CST; 1min 2s ago
Process: 24745 ExecStartPost=/bin/bash /etc/kubernetes/deny-tcp-port-10250.sh (code=exited, status=0/SUCCESS)
Main PID: 24744 (kubelet)
Tasks: 26
Memory: 49.2M
CGroup: /system.slice/kubelet.service
└─24744 /usr/bin/kubelet --root-dir=/data/kubelet --cni-bin-dir=/opt/cni/bin/ --kube-reserved=cpu=90m,memory=1830Mi --max-pods=253 --cloud-provider=qcloud --cloud-config=/etc/kubernetes/qcloud.conf --pod-infra-container-image=ccr.ccs.tencentyun.com/library/pause:latest --cluster-domain=cluster.local --allow-privileged=true --register-schedulable=true --fail-swap-on=false --anonymous-auth=false --image-pull-progress-deadline=10m0s --hostname-override=192.168.8.9 --eviction-hard=nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi --authentication-token-webhook=true --client-ca-file=/etc/kubernetes/cluster-ca.crt --authorization-mode=Webhook --cluster-dns=172.16.255.48 --non-masquerade-cidr=0.0.0.0/0 --network-plugin=cni --kubeconfig=/etc/kubernetes/kubelet-kubeconfig --v=2Cancel the blockade
[root@VM_8_9_centos ~]# kubectl uncordon 192.168.8.9 node/192.168.8.9 uncordoned
Validation effect
Viewing node information is normal
[root@VM_8_9_centos ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION 192.168.8.14 Ready <none> 89d v1.12.4-tke.10 192.168.8.15 Ready <none> 181d v1.12.4-tke.3 192.168.8.3 Ready <none> 181d v1.12.4-tke.3 192.168.8.4 Ready <none> 181d v1.12.4-tke.3 192.168.8.9 Ready <none> 89d v1.12.4-tke.10
Create a new kubelet directory under data
[root@VM_8_9_centos ~]# ls /data/kubelet/ cpu_manager_state plugin-containers plugins pods [root@VM_8_9_centos ~]# du -sh /data/kubelet/ 480K /data/kubelet/