$ kk apply -f pod_volume_emptydir.yaml
pod/busybox created
$ kk get pod                           
NAME      READY   STATUS    RESTARTS   AGE
busybox   2/2     Running   0          2m8s
$ kk logs busybox read
hellok8s!

 

 # 在master节点查看 pod uid
$ kk get pod busybox  -o jsonpath='{.metadata.uid}'
b6abc3f2-d9b3-4297-b636-e33f06d0278d

# 在node1查看具体位置
[root@k8s-node1 ~]# ls /var/lib/kubelet/pods/b6abc3f2-d9b3-4297-b636-e33f06d0278d/volumes/
kubernetes.io~empty-dir  kubernetes.io~projected
[root@k8s-node1 ~]# ls /var/lib/kubelet/pods/b6abc3f2-d9b3-4297-b636-e33f06d0278d/volumes/kubernetes.io~empty-dir/
temp-dir

 

 volumes:
  - name: cache-volume
    emptyDir:
      medium: Memory

 

 # 必须提前在node1上创建目录（当hostPath.type为空时）
[root@k8s-node1 ~]# mkdir /home/host-temp-dir

# 在master上启动Pod
$ kk apply -f pod_volume_hostpath.yaml
$ kk get pod                            
NAME               READY   STATUS    RESTARTS   AGE
busybox            2/2     Running   0          26m
busybox-hostpath   1/1     Running   0          11s

# 在node1上查看数据
[root@k8s-node1 ~]# cat /home/host-temp-dir/data 
hellok8s!

 

 $ kk apply -f pod_use_pvc.yaml
persistentvolume/pv-hostpath created
persistentvolumeclaim/pvc-hostpath created
pod/busybox-use-pvc configured

$ kk get pv,pvc,pod
NAME                           CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                  STORAGECLASS   REASON   AGE
persistentvolume/pv-hostpath   1Gi        RWX            Retain           Bound    default/pvc-hostpath   node-local              112s

NAME                                 STATUS   VOLUME        CAPACITY   ACCESS MODES   STORAGECLASS   AGE
persistentvolumeclaim/pvc-hostpath   Bound    pv-hostpath   1Gi        RWX            node-local     112s

NAME                   READY   STATUS    RESTARTS   AGE
pod/busybox            2/2     Running   0          5h31m
pod/busybox-hostpath   1/1     Running   0          4h38m
pod/busybox-use-pvc    1/1     Running   0          2m48s

# 在node1查看数据写入
[root@k8s-node1 ~]# cat /home/host-pv-dir/data 
hellok8s, pvc used!

 

 # 修改pvc中的storage为大于pv中容量的数字，比如5000Gi
$ kk get pv,pvc,pod
NAME                           CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM   STORAGECLASS   REASON   AGE
persistentvolume/pv-hostpath   1Gi        RWX            Retain           Available           node-local              6s

NAME                                 STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS   AGE
persistentvolumeclaim/pvc-hostpath   Pending                                      node-local     6s

NAME                   READY   STATUS    RESTARTS   AGE
pod/busybox            2/2     Running   0          5h49m
pod/busybox-hostpath   1/1     Running   0          4h56m
pod/busybox-use-pvc    0/1     Pending   0          6s

$ kk describe pvc pvc-hostpath 
Name:          pvc-hostpath
Namespace:     default
StorageClass:  node-local
Status:        Pending
Volume:        
Labels:        <none>
Annotations:   <none>
Finalizers:    [kubernetes.io/pvc-protection]
Capacity:      
Access Modes:  
VolumeMode:    Filesystem
Used By:       busybox-use-pvc
Events:
  Type     Reason              Age   From                         Message
  ----     ------              ----  ----                         -------
  Warning  ProvisioningFailed  11s   persistentvolume-controller  storageclass.storage.k8s.io "node-local" not found

 

 $ kk apply -f pod-hostpath.yaml 
persistentvolumeclaim/pvc-hostpath-2 created

$ kk describe pvc pvc-hostpath-2
Name:          pvc-hostpath-2
Namespace:     default
StorageClass:  node-local
Status:        Pending
Volume:        
Labels:        <none>
Annotations:   <none>
Finalizers:    [kubernetes.io/pvc-protection]
Capacity:      
Access Modes:  
VolumeMode:    Filesystem
Used By:       <none>
Events:
  Type     Reason              Age   From                         Message
  ----     ------              ----  ----                         -------
  Warning  ProvisioningFailed  8s    persistentvolume-controller  storageclass.storage.k8s.io "node-local" not found

 

 $ kk delete pvc pvc-hostpath --force
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
persistentvolumeclaim "pvc-hostpath" force deleted

$ kk get pv,pvc,pod                 
NAME                           CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                  STORAGECLASS   REASON   AGE
persistentvolume/pv-hostpath   1Ti        RWX            Retain           Bound    default/pvc-hostpath   node-local              21m

NAME                                 STATUS        VOLUME        CAPACITY   ACCESS MODES   STORAGECLASS   AGE
persistentvolumeclaim/pvc-hostpath   Terminating   pv-hostpath   1Ti        RWX            node-local     70s

NAME                   READY   STATUS    RESTARTS   AGE
pod/busybox            2/2     Running   0          6h39m
pod/busybox-hostpath   1/1     Running   0          5h46m
pod/busybox-use-pvc    1/1     Running   0          70s

 

 apiVersion: v1
kind: PersistentVolume
metadata:
  name: foo-pv
spec:
  storageClassName: ""
  claimRef:
    name: foo-pvc # 预留pvc的名称
    namespace: foo
...

 

 apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: example-nfs
provisioner: example.com/external-nfs
parameters:
  server: nfs-server.example.com
  path: /share
  readOnly: "false"

 

 apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: slow
provisioner: kubernetes.io/aws-ebs
parameters:
  type: io1
  iopsPerGB: "10"
  fsType: ext4

 

 $ kk apply -f stateful-svc.yaml
service/stateful-svc created
statefulset.apps/statefulset created
storageclass.storage.k8s.io/sc-hostpath created
persistentvolume/pv-hostpath-0 created
persistentvolume/pv-hostpath-1 created

  # sts是statefulset的缩写
$ kk get svc,sc,pv,pvc,pod,sts -o wide
NAME                   TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)    AGE     SELECTOR
service/kubernetes     ClusterIP   20.1.0.1     <none>        443/TCP    44h     <none>
service/stateful-svc   ClusterIP   None         <none>        8080/TCP   2m24s   app=stateful

NAME                                      PROVISIONER                    RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
storageclass.storage.k8s.io/sc-hostpath   kubernetes.io/no-provisioner   Delete          WaitForFirstConsumer   false                  2m24s

NAME                             CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                                 STORAGECLASS   REASON   AGE     VOLUMEMODE
persistentvolume/pv-hostpath-0   500Gi      RWX            Retain           Bound    default/stateful-data-statefulset-0   sc-hostpath             2m24s   Filesystem
persistentvolume/pv-hostpath-1   500Gi      RWX            Retain           Bound    default/stateful-data-statefulset-1   sc-hostpath             2m24s   Filesystem

NAME                                                STATUS   VOLUME          CAPACITY   ACCESS MODES   STORAGECLASS   AGE    VOLUMEMODE
persistentvolumeclaim/stateful-data-statefulset-0   Bound    pv-hostpath-0   500Gi      RWX            sc-hostpath    103s   Filesystem
persistentvolumeclaim/stateful-data-statefulset-1   Bound    pv-hostpath-1   500Gi      RWX            sc-hostpath    101s   Filesystem

NAME                READY   STATUS    RESTARTS            AGE    IP             NODE         NOMINATED NODE   READINESS GATES
pod/statefulset-0   1/1     Running   0                   103s   20.2.36.83     k8s-node1    <none>           <none>
pod/statefulset-1   1/1     Running   0                   4s     20.2.235.196   k8s-master   <none>           <none>

NAME                           READY   AGE     CONTAINERS            IMAGES
statefulset.apps/statefulset   2/2     2m24s   python-svc-stateful   python:4.7

 

 # 在master执行
[root@k8s-master ~]# cat /home/host-sts-pv-dir/data             
This host is statefulset-1!

# 在node1上执行
[root@k8s-node1 ~]# cat /home/host-sts-pv-dir/data 
This host is statefulset-0!

# 在master启动cURL Pod，访问无头Service
$ kk apply -f pod_cURL.yaml           
pod/cURL created

$ kk exec -it cURL --  sh     
/ # nslookup stateful-svc
nslookup: can't resolve '(null)': Name does not resolve

Name:      stateful-svc
Address 1: 20.2.235.201 statefulset-1.stateful-svc.default.svc.cluster.local
Address 2: 20.2.36.84 statefulset-0.stateful-svc.default.svc.cluster.local

/ # cURL statefulset-0.stateful-svc.default.svc.cluster.local
<p> The host is statefulset-0</p>
/ # cURL statefulset-1.stateful-svc.default.svc.cluster.local
<p> The host is statefulset-1</p>

 

 $ kk delete pod statefulset-0        
pod "statefulset-0" deleted

# 稍等几秒 Pod 进行重建，由于之前 0号Pod是在node1上允许的，所以在node1上查看重建时写入的数据                                                                                                                                                                               
[root@k8s-node1 ~]# cat /home/host-sts-pv-dir/data                                
This host is statefulset-0!
This host is statefulset-0!

 

 $ kk apply -f pod_limitResource.yaml       
pod/test-limit-resource created

$ kk get pod                         
NAME                      READY   STATUS    RESTARTS         AGE
test-limit-resource       1/1     Running   0                10s

# 等候20s 观察到Pod由于内存使用超额被终止
$ kk get pod
NAME                      READY   STATUS      RESTARTS         AGE
test-limit-resource       0/1     OOMKilled   0                35s

# 再等候几秒，观察到无法再次启动
$ kk get pod                           
NAME                  READY   STATUS             RESTARTS        AGE
test-limit-resource   0/1     CrashLoopBackOff   3 (6d17h ago)   2m32s

 

 # namespace的缩写是ns
$ kk create namespace test-namespace
namespace/test-namespace created

 

 apiVersion: v1
kind: Namespace
metadata:
  name: test-namespace2

 

 $ kk get namespace
NAME              STATUS   AGE
default           Active   12d  # 所有未指定namespace属性的对象会分配到这里
kube-node-lease   Active   12d  # 存放各节点的Lease对象，用于节点的心跳检测
kube-public       Active   12d  # 此空间下的所有资源可以被所有人访问
kube-system       Active   12d  # 存放集群内建资源
test-namespace    Active   5m
test-namespace2   Active   86s

 

 apiVersion: v1
kind: Pod
metadata:
  name: go-http
  namespace: test-namespace
...

 

 $ kk api-resources --namespaced=true
NAME                        SHORTNAMES   APIVERSION                     NAMESPACED   KIND
bindings                                 v1                             true         Binding
configmaps                  cm           v1                             true         ConfigMap
endpoints                   ep           v1                             true         Endpoints
events                      ev           v1                             true         Event
limitranges                 limits       v1                             true         LimitRange
persistentvolumeclaims      pvc          v1                             true         PersistentVolumeClaim
pods                        po           v1                             true         Pod
podtemplates                             v1                             true         PodTemplate
replicationcontrollers      rc           v1                             true         ReplicationController
resourcequotas              quota        v1                             true         ResourceQuota
secrets                                  v1                             true         Secret
serviceaccounts             sa           v1                             true         ServiceAccount
services                    svc          v1                             true         Service
controllerrevisions                      apps/v1                        true         ControllerRevision
daemonsets                  ds           apps/v1                        true         DaemonSet
deployments                 deploy       apps/v1                        true         Deployment
replicasets                 rs           apps/v1                        true         ReplicaSet
statefulsets                sts          apps/v1                        true         StatefulSet
localsubjectaccessreviews                authorization.k8s.io/v1        true         LocalSubjectAccessReview
horizontalpodautoscalers    hpa          autoscaling/v2                 true         HorizontalPodAutoscaler
cronjobs                    cj           batch/v1                       true         CronJob
jobs                                     batch/v1                       true         Job
leases                                   coordination.k8s.io/v1         true         Lease
networkpolicies                          crd.projectcalico.org/v1       true         NetworkPolicy
networksets                              crd.projectcalico.org/v1       true         NetworkSet
endpointslices                           discovery.k8s.io/v1            true         EndpointSlice
events                      ev           events.k8s.io/v1               true         Event
ingresses                   ing          networking.k8s.io/v1           true         Ingress
networkpolicies             netpol       networking.k8s.io/v1           true         NetworkPolicy
poddisruptionbudgets        pdb          policy/v1                      true         PodDisruptionBudget
rolebindings                             rbac.authorization.k8s.io/v1   true         RoleBinding
roles                                    rbac.authorization.k8s.io/v1   true         Role
csistoragecapacities                     storage.k8s.io/v1              true         CSIStorageCapacity

 

 kk delete namespace <namespace-name>

 

 $ kk apply -f resource-quota.yaml
  resourcequota/quota-default created

# 查看空间下的资源配额，可以看到创建的 quota-default 对象配置
$ kk describe namespace default
Name:         default
Labels:       kubernetes.io/metadata.name=default
Annotations:  <none>
Status:       Active

  Resource Quotas
Name:                   quota-default
  Resource                Used   Hard
  --------                ---    ---
  configmaps              2      10
  cpu                     50m    10
  limits.cpu              100m   10
  limits.memory           100Mi  1Gi
  memory                  50Mi   500Mi
  persistentvolumeclaims  0      10
  pods                    1      50
  replicationcontrollers  0      20
  requests.cpu            50m    10
  requests.memory         50Mi   500Mi
  requests.storage        0      10Gi
  resourcequotas          1      5
  secrets                 2      10
  services                1      10
  services.loadbalancers  0      5
  services.nodeports      0      5

  No LimitRange resource.

 

 $ kk describe limits  # limits是limitrange的缩写
Name:       limitrange-for-container
Namespace:  default
Type        Resource  Min    Max    Default Request  Default Limit  Max Limit/Request Ratio
----        --------  ---    ---    ---------------  -------------  -----------------------
Container   cpu       100m   200m   100m             150m           2
Container   memory    100Mi  300Mi  100Mi            200Mi          2


Name:       limitrange-for-pod
Namespace:  default
Type        Resource  Min    Max    Default Request  Default Limit  Max Limit/Request Ratio
----        --------  ---    ---    ---------------  -------------  -----------------------
Pod         memory    100Mi  300Mi  -                -              1
Pod         cpu       100m   200m   -                -              1


Name:                  limitrange-for-pvc
Namespace:             default
Type                   Resource  Min    Max  Default Request  Default Limit  Max Limit/Request Ratio
----                   --------  ---    ---  ---------------  -------------  -----------------------
PersistentVolumeClaim  storage   100Mi  1Gi  -                -              -

 

 metadata:
  labels:
    k1: v1
    k2: v2
    ...

 

 # 添加
kubectl label <资源类型> <资源名称> <key>=<value>

# 删除
kubectl label <资源类型> <资源名称> <key>-

 

 # 如果表达式包含空格，则需要放在单引号内
$ kubectl get pods -l env!=dev,tier=frontend

 

 $ kubectl get pods -l 'env in (prod,dev), tier notin (frontend,backend)'

 

 $ kubectl get pods -l 'tier, !env' # 带tier标签 且不带env标签的资源

 

 # 查询标签键包含app、tier、role的资源，并且在输出中包含标签信息
$ kubectl get pods -Lapp -Ltier -Lrole
NAME                           READY  STATUS    RESTARTS   AGE   APP         TIER       ROLE
guestbook-fe-4nlpb             1/1    Running   0          1m    guestbook   frontend   <none>
guestbook-fe-ght6d             1/1    Running   0          1m    guestbook   frontend   <none>
guestbook-fe-jpy62             1/1    Running   0          1m    guestbook   frontend   <none>
guestbook-redis-master-5pg3b   1/1    Running   0          1m    guestbook   backend    master
guestbook-redis-replica-2q2yf  1/1    Running   0          1m    guestbook   backend    replica
guestbook-redis-replica-qgazl  1/1    Running   0          1m    guestbook   backend    replica
my-nginx-divi2                 1/1    Running   0          29m   nginx       <none>     <none>
my-nginx-o0ef1                 1/1    Running   0          29m   nginx       <none>     <none>

 

 $ kubectl label pods -l app=nginx tier=fe
pod/my-nginx-2035384211-j5fhi labeled
pod/my-nginx-2035384211-u2c7e labeled
pod/my-nginx-2035384211-u3t6x labeled

 

 selector:
  component: redis  # 等价于 component=redis 或 component in (redis)

 

 selector:
  # matchLabels 和 matchExpressions 需要同时满足
  matchLabels:
    component: redis
  matchExpressions:
    # operator 支持 In、NotIn、Exists 和 DoesNotExist
    - { key: tier, operator: In, values: [ cache ] }
    - { key: environment, operator: NotIn, values: [ dev ] }

 

 metadata:
  annotations:
    key1: "value1_string"
    # 允许跨行，实际内容不含换行符，可不加双引号
    key2: Annotations is a set of key value pairs that
      give extra information about the rule
    # yaml允许使用管道符，它会忽略开头结尾的换行符，但保留内容中的换行符，实际内容是 "123\n456"
    key3: |-
      123
      456

 

 annotations:
description: "Web server serving the main application."

 

 annotations:
  prometheus.io/scrape: "true"
  prometheus.io/port: "8080"

 

 annotations:
  pipeline/build-number: "123"
  pipeline/deployed-by: "jenkins"

 

 annotations:
  build-timestamp: "2023-11-10T14:30:00Z"
  git-branch: "main"
  image-hash: "sha256:abcdef123456"

 

 annotations:
  contact-person: "John Doe (john@example.com)"

 

 $ kk describe pod cURL                                      
Name:             cURL
Namespace:        default
Priority:         0
Service Account:  default
Node:             k8s-node1/10.0.0.3
Start Time:       Fri, 03 Nov 2023 12:38:14 +0800
Labels:           app=cURL
Annotations:      cni.projectcalico.org/containerID: afb1e9f94f02d8f293b48fabe028623063159b7b7cd35ccd20726ee7e19ed63b
                  cni.projectcalico.org/podIP: 20.2.36.91/32
                  cni.projectcalico.org/podIPs: 20.2.36.91/32
                  description:
                    The `cURL` command is a powerful tool used to make HTTP requests from the command line. It is versatile and supports various protocols, in...
                  key1: value1
...
                                                                                                                                                                                       
$ kubectl get pod cURL -o=jsonpath='{.metadata.annotations}'     
{"cni.projectcalico.org/containerID":"afb1e9f94f02d8f293b48fabe028623063159b7b7cd35ccd20726ee7e19ed63b","cni.projectcalico.org/podIP":"20.2.36.91/32","cni.projectcalico.org/podIPs":"20.2.36.91/32","description":"The `cURL` command is a powerful tool used to make HTTP requests from the command line. It is versatile and supports various protocols, including HTTP, HTTPS, FTP, FTPS, and more.","key1":"value1","kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"kind\":\"Pod\",\"metadata\":{\"annotations\":{\"description\":\"The `cURL` command is a powerful tool used to make HTTP requests from the command line. It is versatile and supports various protocols, including HTTP, HTTPS, FTP, FTPS, and more.\",\"key1\":\"value1\"},\"labels\":{\"app\":\"cURL\"},\"name\":\"cURL\",\"namespace\":\"default\"},\"spec\":{\"containers\":[{\"command\":[\"sh\",\"-c\",\"sleep 1h\"],\"image\":\"appropriate/cURL\",\"name\":\"cURL-container\"}]}}\n"}#

$ kubectl get pod cURL -o=jsonpath='{.metadata.annotations.key1}'
value1

$ kubectl get pod cURL -o=jsonpath='{.metadata.annotations.description}'
The `cURL` command is a powerful tool used to make HTTP requests from the command line. It is versatile and supports various protocols, including HTTP, HTTPS, FTP, FTPS, and more.

 

 $ kubectl get pod cURL -o=jsonpath='{.metadata.uid}'                    
37ea632b-2adc-4c0c-9133-5c2229480206

 

 kubectl get pods --field-selector status.phase=Running

 

 $ wget https://hub.gitmirror.com/?q=https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml -O metrics-server.yaml

$ vi metrics-server.yaml
# 在Deployment中添加启动参数 --kubelet-insecure-tls 以禁用与kubelet通信时的tls检查（若不添加此参数则metrics-server的Pod无法达到Running状态）
#    spec:
#      containers:
#      - args:
#        - --cert-dir=/tmp
#        - --secure-port=4443
#        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
#        - --kubelet-use-node-status-port
#        - --metric-resolution=15s
#        - --kubelet-insecure-tls

$ kk apply -f metrics-server.yaml

 

 # 在node1上手动拉取该镜像的docker仓库备份
ctr -n k8s.io image pull docker.io/anjia0532/google-containers.metrics-server.metrics-server:v0.6.4

# tag
ctr -n k8s.io i tag docker.io/anjia0532/google-containers.metrics-server.metrics-server:v0.6.4 registry.k8s.io/metrics-server/metrics-server:v0.6.4

# 查看下载的镜像
ctr -n k8s.io i ls |grep metrics
crictl images |grep metrics

# 然后删除metrics-server Pod触发重建即可

 

 $ kk top pod
NAME                                CPU(cores)   MEMORY(bytes)   
busybox                             0m           0Mi             
hellok8s-go-http-66459fdc45-jrm2t   1m           4Mi             
hellok8s-go-http-66459fdc45-q6s7l   0m           0Mi             
nginx-hpa-test-85694d4764-855x2     0m           2Mi 

$ kk top node
NAME         CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%   
k8s-master   288m         14%    1165Mi          62%       
k8s-node1    57m          2%     710Mi           35%

 

 # 创建deployment对象
$ kk apply -f pod_nginx_svc.yaml               
deployment.apps/nginx-hpa-test created
service/nginx-hpa-test created

# 创建绑定deployment对象的hpa对象
$ kk apply -f hpa_nginx.yaml    
horizontalpodautoscaler.autoscaling/nginx-hpa-test created

# 查看hpa状态
# 若Metric Server成功安装，则可以在 TARGETS 列看到所监控的 Deployment对象下所有Pod当前的 平均CPU利用率/hpa设置的预期值
# - REPLICAS 列表示当前的Pod副本数
$ kk get hpa
NAME             REFERENCE                   TARGETS   MINPODS   MAXPODS   REPLICAS   AGE
nginx-hpa-test   Deployment/nginx-hpa-test   0%/50%    1         5         1          2d3h

 

 # 创建一个Pod，并使用wget不断访问目标Service，以模拟Pod的CPU负载（稍后按ctrl+C停止）
$ kk run -i --tty load-generator --rm --image=busybox --restart=Never -- /bin/sh -c "while sleep 0.001; do wget -q -O- http://nginx-hpa-test; done"

# 打开另一个终端，持续监控当前的Deployment对象的平均CPU利用率以及Pod副本数
$ kk get hpa nginx-hpa-test --watch

# 在笔者的环境中大概过了30s左右，hpa显示的平均CPU负载达到58%（可能与你的环境有些差别），超过了预设值50%，此时发生了扩容
# 并且看到Pod副本数从1个扩容到3个，同时可以在describe中看到相应的扩容事件
$ kk describe hpa 
...
│ Conditions:                                                                                                                                                                   │
│   Type            Status  Reason               Message                                                                                                                        │
│   ----            ------  ------               -------                                                                                                                        │
│   AbleToScale     True    SucceededGetScale    the HPA controller was able to get the target's current scale                                                                  │
│   ScalingActive   True    FailedGetPodsMetric  ...                                                                                                       │
│   ScalingLimited  False   DesiredWithinRange   the desired count is within the acceptable range
│ Events:                                                                                                                                                                       │
│   Type     Reason                        Age    From                       Message                                                                                            │
│   ----     ------                        ----   ----                       -------                                                                                            │
│   Normal   SuccessfulRescale             115s   horizontal-pod-autoscaler  New size: 2; reason: cpu resource utilization (percentage of request) above target                 │
│   Normal   SuccessfulRescale             100s   horizontal-pod-autoscaler  New size: 3; reason: cpu resource utilization (percentage of request) above target

 

 package framework

import "context"

/*
省略部分。。
*/

// ScorePlugin is an interface that must be implemented by "Score" plugins to rank
// nodes that passed the filtering phase.
type ScorePlugin interface {
	Plugin
	// Score is called on each filtered node. It must return success and an integer
	// indicating the rank of the node. All scoring plugins must return success or
	// the pod will be rejected.
	Score(ctx context.Context, state *CycleState, p *v1.Pod, nodeName string) (int64, *Status)

	// ScoreExtensions returns a ScoreExtensions interface if it implements one, or nil if does not.
	ScoreExtensions() ScoreExtensions
}

 

 # ...省略部分
spec:
  containers:
    - name: go-http
      image: leigg/hellok8s:v1
  nodeSelector:
    disktype: ssd

 

 # 1. 首先设置master节点标签
$ kk label nodes k8s-master disktype=ssd
node/k8s-master labeled

# 2. 查看master节点标签，确认标记成功
$ kk describe node k8s-master |grep Labels -A 8
Labels:             beta.kubernetes.io/arch=amd64
                    beta.kubernetes.io/os=linux
                    disktype=ssd
                    kubernetes.io/arch=amd64
                    kubernetes.io/hostname=k8s-master
                    kubernetes.io/os=linux
                    node-role.kubernetes.io/control-plane=
                    node.kubernetes.io/exclude-from-external-load-balancers=
# 2.1 换个命令查看标签
$ kubectl get nodes --show-labels |grep k8s-master
k8s-master   Ready    control-plane   14d   v1.27.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,disktype=ssd,kubernetes.io/arch=amd64,kubernetes.io/hostname=k8s-master,kubernetes.io/os=linux,node-role.kubernetes.io/control-plane=,node.kubernetes.io/exclude-from-external-load-balancers=

# 3. 删除master节点 NoSchedule的污点，省略

# 4. 部署和观察
$ kk apply -f pod_nodeLabel.yaml 
pod/go-http created
$ kk get po go-http -o wide     
NAME      READY   STATUS    RESTARTS   AGE   IP             NODE         NOMINATED NODE   READINESS GATES
go-http   1/1     Running   0          3s    20.2.235.209   k8s-master   <none>           <none>

# 删除pod
$ kk delete po go-http

 

 # ...省略部分
spec:
  containers:
    - name: go-http
      image: leigg/hellok8s:v1
  nodeName: k8s-master

 

 $ kk apply -f pod_nodeName.yaml
pod/go-http created

$ kk get po go-http -o wide
NAME      READY   STATUS    RESTARTS   AGE     IP             NODE         NOMINATED NODE   READINESS GATES
go-http   1/1     Running   0          2m16s   20.2.235.205   k8s-master   <none>           <none>

  # 删除pod
$ kk delete po go-http

 

 $ kk apply -f pods_diff_labels.yaml  
pod/go-http-master created
pod/go-http-node1 created

$ kk get po -o wide
NAME             READY   STATUS    RESTARTS   AGE     IP             NODE         NOMINATED NODE   READINESS GATES
go-http-master   1/1     Running   0          12s     20.2.235.211   k8s-master   <none>           <none>
go-http-node1    1/1     Running   0          12s     20.2.36.68     k8s-node1    <none>           <none>

$ kk apply -f pod_affinityPod.yaml 
pod/go-http-podaffinity created
$ kk get po -o wide               
NAME                  READY   STATUS    RESTARTS   AGE   IP             NODE         NOMINATED NODE   READINESS GATES
go-http-master        1/1     Running   0          40m   20.2.235.211   k8s-master   <none>           <none>
go-http-node1         1/1     Running   0          31m   20.2.36.68     k8s-node1    <none>           <none>
go-http-podaffinity   1/1     Running   0          3s    20.2.235.212   k8s-master   <none>           <none>

$ kk delete -f pod_affinityPod.yaml && kk delete -f pods_diff_labels.yaml


 

 # 查看污点，这个是master自带的污点，表示不接受常规Pod调度
$ kk get node k8s-master -o=jsonpath='{.spec.taints}'                         
[{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane"}]

# 添加污点
$ kubectl taint nodes k8s-master role/log:NoSchedule
node/k8s-master tainted

$ kk get node k8s-master -o=jsonpath='{.spec.taints}'
[{"effect":"NoSchedule","key":"role/log"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane"}]

# 删除污点
$ kubectl taint nodes k8s-master role/log-                    
node/k8s-master untainted                                                                                                                     

 

 spec:
  tolerations:
    - key: "key1"
      operator: "Equal"
      value: "value1"
      effect: "NoSchedule"
    - key: "key1"
      operator: "Equal"
      value: "value1"
      effect: "NoExecute"
      tolerationSeconds: 3600

 

 # 确认master节点（删除其他污点）
$ kk taint nodes k8s-master role/log:NoExecute       
node/k8s-master tainted
$ kk get node k8s-master -o=jsonpath='{.spec.taints}'
[{"effect":"NoExecute","key":"role/log"}]

# 创建Pod，并且能够在master上运行
$ kk apply -f pod_tolerance.yaml 
pod/go-http-tolerance created
$ kk get po -o wide             
NAME                READY   STATUS    RESTARTS   AGE   IP           NODE        NOMINATED NODE   READINESS GATES
go-http-tolerance   1/1     Running   0          6s    20.2.36.98   k8s-master   <none>           <none>

# 给master添加新的污点
$ kk taint nodes k8s-master gpu:NoExecute
node/k8s-master tainted

# 观察到pod从master上被驱逐（并且不会调度到node1，所以直接从集群中消失了）
$ kk get po -o wide                     
No resources found in default namespace.

# 再次创建Pod
$ kk apply -f pod_tolerance.yaml
pod/go-http-tolerance created

# 可以看到被直接调度到node1
$ kk get po -o wide             
NAME                READY   STATUS    RESTARTS   AGE   IP            NODE        NOMINATED NODE   READINESS GATES
go-http-tolerance   1/1     Running   0          2s    20.2.36.122   k8s-node1   <none>           <none>

 

 $ kk apply -f pod_priorityClass.yaml                                                
priorityclass.scheduling.k8s.io/high-priority created
pod/nginx created

$ kk get po                      
NAME    READY   STATUS    RESTARTS   AGE
nginx   1/1     Running   0          33s

$ kubectl get pod nginx -o=jsonpath='{.spec.priorityClassName}'
high-priority

$ kk describe po nginx |grep Priority
Priority:             1000000
Priority Class Name:  high-priority

$ kk get pc high-priority            
NAME            VALUE     GLOBAL-DEFAULT   AGE
high-priority   1000000   false            7m19s

 

 $ kk get pc system-cluster-critical                                 
NAME                      VALUE        GLOBAL-DEFAULT   AGE
system-cluster-critical   2000000000   false            15d
$ kk get pc system-node-critical                      
NAME                   VALUE        GLOBAL-DEFAULT   AGE
system-node-critical   2000001000   false            15d

 

 apiVersion: apiserver.config.k8s.io/v1
kind: AdmissionConfiguration
plugins:
  - name: "ResourceQuota"
    configuration:
      apiVersion: apiserver.config.k8s.io/v1
      kind: ResourceQuotaConfiguration
      limitedResources:
        - resource: pods
          matchScopes:
            - scopeName: PriorityClass
              operator: In
              values: [ "cluster-services" ]

 

 apiVersion: v1
kind: ResourceQuota
metadata:
  name: pods-cluster-services
  namespace: kube-system
spec:
  scopeSelector:
    matchExpressions:
      - operator: In
        scopeName: PriorityClass
        values: [ "cluster-services" ]

 

 # 在master节点执行
$ kubectl proxy --port 8080
Starting to serve on 127.0.0.1:8080

 

 # 你也可以单独定义 pod_nginx.json，在cURL中通过 --data-binary @pod_nginx.json 使用
$ cURL localhost:8080/api/v1/namespaces/default/pods -X POST -H "Content-Type: application/json" -d '{
  "apiVersion": "v1",
  "kind": "Pod",
  "metadata": {
    "name": "nginx",
    "labels": {
      "app": "nginx"
    }
  },
  "spec": {
    "containers": [
      {
        "name": "nginx-container",
        "image": "nginx"
      }
    ]
  }
}'

# 返回创建成功的Pod完整JSON
{
  "kind": "Pod",
  "apiVersion": "v1",
  "metadata": {
    "name": "nginx",
    "namespace": "default",
    "uid": "06e71b18-b592-446e-a446-6f6de80a3cf8",
...

 

 $ cURL localhost:8080/api/v1/namespaces/default/pods/nginx
{
  "kind": "Pod",
  "apiVersion": "v1",
  "metadata": {
    "name": "nginx",
    "namespace": "default",
    "uid": "06e71b18-b592-446e-a446-6f6de80a3cf8",
...

 

 $ kk get po                
NAME    READY   STATUS    RESTARTS   AGE
nginx   1/1     Running   0          2m

 

 # 在master节点访问
$ cURL --insecure https://localhost:6443/api/v1/namespaces/default/pods/nginx
{
  "kind": "Status",
  "apiVersion": "v1",
  "metadata": {},
  "status": "Failure",
  "message": "pods \"nginx\" is forbidden: User \"system:anonymous\" cannot get resource \"pods\" in API group \"\" in the namespace \"default\"",
  "reason": "Forbidden",
  "details": {
    "name": "nginx",
    "kind": "pods"
  },
  "code": 403
}

 

 # 1. 生成根证书私钥
#openssl genrsa -out ca.key 2048
# 2. 基于根证书私钥生成证书文件 （-days 设置证书有效期）
#openssl req -x509 -new -nodes -key ca.key -subj "/CN=<master-ip>" -days 1000 -out ca.crt

-- 前两步可省略，因为安装集群时已经提供了默认的ca证书以及key文件在 /etc/kubernetes/pki/ 下面

# 3. 生成client证书私钥
openssl genrsa -out client.key 2048

# 4. 基于client证书私钥生成client证书的csr文件（证书签名请求），CN是用户名，O是组名
openssl req -new -key client.key -out client.csr -subj "/CN=user2/O=app1/O=app2"

# 5. 基于client两个文件和k8s的ca私钥生成client证书
openssl x509 -req -in client.csr -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key -CAcreateserial -out client.crt -days 365

# 检查client证书
openssl x509 -in client.crt -text -noout

 

 # 首先从kubeconfig文件中获取API Server的根证书（当然也可以使用 --insecure 选项禁用服务器证书校验）
$ echo $(grep certificate-authority-data /etc/kubernetes/admin.conf |cut -d" " -f 6) |base64 -d > apiserver-ca.crt

# 然后使用client证书、client密钥以及服务器根证书进行访问
# - 注意访问地址的host部分必须是apiserver的暴露ip（与kubeconfig中的集群地址一致），否则不能通过服务器证书校验
# - 根据response可见，能够识别到 user2 用户，但由于没授权，所以还不能访问资源
# - 若不能识别，则显示 system:anonymous 用户
# - （这里有一个cURL的坑要注意，必须在client证书和密钥前加上 ./ 否则无法正常识别，原因未知）
$ curl --cert ./client.crt --key ./client.key --cacert apiserver-ca.crt \
      https://10.0.0.2:6443/api/v1/namespaces/default/pods/nginx
{
  "kind": "Status",
  "apiVersion": "v1",
  "metadata": {},
  "status": "Failure",
  "message": "pods \"nginx\" is forbidden: User \"user2\" cannot get resource \"pods\" in API group \"\" in the namespace \"default\"",
  "reason": "Forbidden",
  "details": {
    "name": "nginx",
    "kind": "pods"
  },
  "code": 403
}

 

 # 1. 设置用户凭据
kubectl config set-credentials user2 --embed-certs=true --client-certificate=client.crt --client-key=client.key
# 2. 设置上下文（将 cluster 和 credentials进行组合成访问集群的上下文）
kubectl config set-context user2@kubernetes --cluster=kubernetes --user=user2
# 3. 切换上下文
kubectl config use-context user2@kubernetes
# 4. 访问资源
# - 成功识别到user2用户
$ kubectl get pods
Error from server (Forbidden): pods is forbidden: User "user2" cannot list resource "pods" in API group "" in the namespace "default"

# 最后切换回admin
kubectl config use-context kubernetes-admin@kubernetes
# 可以携带上下文参数来指定用户
# kubectl --context=user2@kubernetes get pods

 

 # vi k8s_account_tokens.csv
# 分别是token，用户名，用户id，所属用户组
nlZtQeHoS8k0Pvbe,user3,3
nxdt123445k0P21d,user4,4,"group1,group2"

 

 # kube-apiserver pod从固定挂载的几个目录读取文件，所以我们需要将文件移动到其中的一个目录下才能被读取到
$ mv k8s_account_tokens.csv /etc/kubernetes/pki/

# 修改 kube-apiserver.yaml
$ vi /etc/kubernetes/manifests/kube-apiserver.yaml
#spec:
#  containers:
#  - command:
#    - kube-apiserver
#    - --token-auth-file=/etc/kubernetes/pki/k8s_account_tokens.csv   #  添加这行
...

 

 $ curl --insecure https://localhost:6443/api/v1/namespaces/default/pods -H "Authorization:Bearer nlZtQeHoS8k0Pvbe"
{
  "kind": "Status",
  "apiVersion": "v1",
  "metadata": {},
  "status": "Failure",
  "message": "pods is forbidden: User \"user3\" cannot list resource \"pods\" in API group \"\" in the namespace \"default\"",
  "reason": "Forbidden",
  "details": {
    "kind": "pods"
  },
  "code": 403
}

 

 $ kk get sa       
NAME      SECRETS   AGE
default   0         175m

$ kk describe sa default                                        
Name:                default
Namespace:           default
Labels:              <none>
...

$ kubectl get pod nginx -o jsonpath='{.spec.serviceAccountName}'
default

$ kubectl get pod nginx -o jsonpath='{.spec.containers[*].volumeMounts}' | jq .
[
  {
    "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
    "name": "kube-api-access-fzjrb",
    "readOnly": true
  }
]

# jq用于格式化json
$ kubectl get pod nginx -o jsonpath='{.spec.volumes}' | jq .
[
  {
    "name": "kube-api-access-fzjrb",
    "projected": {
      "defaultMode": 420,
      "sources": [
        {
          "serviceAccountToken": {
            "expirationSeconds": 3607,
            "path": "token"
          }
        },
        {
          "configMap": {
            "items": [
              {
                "key": "ca.crt",
                "path": "ca.crt"
              }
            ],
            "name": "kube-root-ca.crt"
          }
        },
        {
          "downwardAPI": {
            "items": [
              {
                "fieldRef": {
                  "apiVersion": "v1",
                  "fieldPath": "metadata.namespace"
                },
                "path": "namespace"
              }
            ]
          }
        }
...

 

 $ kk exec -it nginx -- bash                  
root@nginx:/# ls /var/run/secrets/kubernetes.io/serviceaccount/
ca.crt	namespace  token

 

 root@nginx:/# TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
root@nginx:/# CACERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt

# kubernetes.default.svc.cluster.local 是API Server的host，在创建每个pod时自动注入
# 允许访问公开API
root@nginx:/# curl --cacert $CACERT --header "Authorization: Bearer $TOKEN" -X GET https://kubernetes.default.svc.cluster.local/version 
{
  "major": "1",
  "minor": "27",
  "gitVersion": "v1.27.0",
  "gitCommit": "1b4df30b3cdfeaba6024e81e559a6cd09a089d65",
  "gitTreeState": "clean",
  "buildDate": "2023-04-11T17:04:24Z",
  "goVersion": "go1.20.3",
  "compiler": "gc",
  "platform": "linux/amd64"
}
# 默认服务账号没有访问集群资源的权限
root@nginx:/# curl --cacert $CACERT --header "Authorization: Bearer $TOKEN" https://kubernetes.default.svc.cluster.local/api/v1/namespaces/default/pods
{
  "kind": "Status",
  "apiVersion": "v1",
  "metadata": {},
  "status": "Failure",
  "message": "pods is forbidden: User \"system:serviceaccount:default:default\" cannot list resource \"pods\" in API group \"\" in the namespace \"default\"",
  "reason": "Forbidden",
  "details": {
    "kind": "pods"
  },
  "code": 403
}

 

 $ kubectl drain mynode --as=superman --as-group=system:masters

 

 # 定义伪装权限的角色不能在Role和RoleBinding中使用
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: limited-impersonator
rules:
  # 可以伪装成用户 "jane.doe@example.com"
  - apiGroups: [ "" ]   # 核心API组包含 authentication.k8s.io
    resources: [ "users" ]
    verbs: [ "impersonate" ]
    resourceNames: [ "jane.doe@example.com" ]

  # 可以伪装成用户组 "developers" 和 "admins"
  - apiGroups: [ "" ]
    resources: [ "groups" ]
    verbs: [ "impersonate" ]
    resourceNames: [ "developers","admins" ]

  # 可以将附加字段 "scopes" 伪装成 "view" 和 "development"
  - apiGroups: [ "authentication.k8s.io" ]
    resources: [ "userextras/scopes" ]
    verbs: [ "impersonate" ]
    resourceNames: [ "view", "development" ]

  # 可以伪装 UID "06f6ce97-e2c5-4ab8-7ba5-7654dd08d52b"
  - apiGroups: [ "authentication.k8s.io" ]
    resources: [ "uids" ]
    verbs: [ "impersonate" ]
    resourceNames: [ "06f6ce97-e2c5-4ab8-7ba5-7654dd08d52b" ]

 

 # 得到yes或no
$ kubectl auth can-i create deployments --namespace dev
yes

# --as用来检查其他用户有无对应权限（但不会告诉你用户是否存在）
$ kubectl auth can-i list secrets --namespace dev --as dave
no

# 检查命名空间 dev 里的 dev-sa 服务账户是否可以列举命名空间 target 里的 Pod
$ kubectl auth can-i list pods \
	--namespace target \
	--as system:serviceaccount:dev:dev-sa
no

 

 $ kk apply -f rbac_role.yaml 
role.rbac.authorization.k8s.io/pod-reader created

$ kk apply -f rbac_rolebinding.yaml
rolebinding.rbac.authorization.k8s.io/read-pods created

$ kk get role                          
NAME         CREATED AT
pod-reader   2023-11-20T00:13:17Z

$ kk get rolebinding
NAME        ROLE              AGE
read-pods   Role/pod-reader   21s

$ kk describe rolebinding read-pods 
Name:         read-pods
Labels:       <none>
Annotations:  <none>
Role:
  Kind:  Role
  Name:  pod-reader
Subjects:
  Kind            Name  Namespace
  ----            ----  ---------
  User            user2
  User            user3  
  ServiceAccount  sa1

 

 # 1. 通过x509证书创建的user2
$ curl --cert ./client.crt --key ./client.key --cacert apiserver-ca.crt \
      https://10.0.0.2:6443/api/v1/namespaces/default/pods/nginx
{
  "kind": "PodList",
  "apiVersion": "v1",
  "metadata": {
    "resourceVersion": "128733"
  },
...内容较长，省略

# 2. 通过静态令牌文件创建的user3
$ curl --insecure https://localhost:6443/api/v1/namespaces/default/pods -H "Authorization:Bearer nlZtQeHoS8k0Pvbe"
...省略

# 3. 通过服务账号sa1
$ kk exec -it nginx-sa-longtime -- bash
root@nginx-sa-longtime:/# TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
root@nginx-sa-longtime:/# CACERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
root@nginx-sa-longtime:/# curl --cacert $CACERT --header "Authorization: Bearer $TOKEN" https://kubernetes.default.svc.cluster.local/api/v1/namespaces/default/pods
...省略

 

 $ kk apply -f rbac_clusterrole.yaml 
clusterrole.rbac.authorization.k8s.io/secret-reader created
$ kk apply -f rbac_clusterrolebinding.yaml 
clusterrolebinding.rbac.authorization.k8s.io/read-secrets-global created

# 可以访问kube-public空间的secrets
$ curl --cert ./client.crt --key ./client.key --cacert apiserver-ca.crt \
      https://10.0.0.2:6443/api/v1/namespaces/kube-public/secrets
{
  "kind": "SecretList",
  "apiVersion": "v1",
  "metadata": {
    "resourceVersion": "211074"
  },
...省略

# 但没有delete权限
$ curl --cert ./client.crt --key ./client.key --cacert apiserver-ca.crt -X DELETE \
      https://192.168.31.2:6443/api/v1/namespaces/kube-public/secrets/some-secret 
{
  "kind": "Status",
  "apiVersion": "v1",
  "metadata": {},
  "status": "Failure",
  "message": "secrets \"some-secret\" is forbidden: User \"user2\" cannot delete resource \"secrets\" in API group \"\" in the namespace \"kube-public\"",
  "reason": "Forbidden",
  "details": {
    "name": "some-secret",
    "kind": "secrets"
  },
  "code": 403
}

 

 kk get clusterrole <aggregate-clusterole-name> -o jsonpath='{.rules}'

 

 $ kk get clusterrole system:discovery -o yaml

 

 $ ps aux | grep kube-apiserver |grep admission-plugins

 

 # 创建CRD
$ kk apply -f customize_resource_define.yaml 
customresourcedefinition.apiextensions.k8s.io/crontabs.stable.example.com created

# 查看CRD（CRD本身属于集群作用域）
$ kk get crd                                
NAME                                  CREATED AT
crontabs.stable.example.com           2023-11-26T07:11:37Z

$ kk apply -f customize_object.yaml           
crontab.stable.example.com/my-new-cron-object created

# 查看自定义对象
$ kk get ct
NAME                 AGE
my-new-cron-object   31s

 

 # 1. 修改kubelet启动参数
# 进入任意节点shell（建议从普通非关键节点开始）
# 通过 service kubelet status 得到kubelet的systemctl启动配置文件位置
# 一般是 /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
# 再通过这个配置文件内容了解到kubelet启动时会读取 /etc/sysconfig/kubelet 中的内容作为环境变量覆盖
# 现在修改 /etc/sysconfig/kubelet
$ vi /etc/sysconfig/kubelet
# 添加以下内容，保存退出
# 添加多个特性开关使用逗号分割："--feature-gates=A=true,B=true"
# KUBELET_EXTRA_ARGS="--feature-gates=CPUManagerPolicyAlphaOptions=true"

# 重启kubelet
$ service kubelet restart

# 检查启动命令中是否包含添加到标志
# - 如果启动成功且包含标志说明修改成功，则对其他所有节点如法炮制
# - 如果修改失败，则快速还原修改内容再启动kubelet，避免对节点产生影响。然后查看日志：journalctl -u kubelet -f --lines=10
$ ps -aux |grep kubelet                                       
root       3924  4.8  3.0 1331704 60388 ?       Ssl  19:17   0:00 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9 --feature-gates=CPUManagerPolicyAlphaOptions=true

# Tips
# 在较新的K8s版本中，--feature-gates标志已被废弃，推荐使用 --config 标志来配置kubelet（特性配置只是其中一部分）
# 参考 https://kubernetes.io/zh-cn/docs/tasks/administer-cluster/kubelet-config-file/
# 具体来说，就是修改 /var/lib/kubelet/config.yaml，在其中添加featureGates部分内容
# - 注意：通过这个方式修改是无法通过 ps aux 命令看到效果的
# - 如果你的修改包含错误，kubelet启动时会报错，导致kubelet无法正常启动，通过journalctl查看错误日志
#apiVersion: kubelet.config.k8s.io/v1beta1
#featureGates:
#  CPUManagerPolicyAlphaOptions: true

# kube-proxy组件与kubelet不同，它是以Pod方式运行，并且引用的配置存放在configmap中
# 查看kube-proxy pod引用的configmap内容
kubectl get cm -n kube-system kube-proxy -o yaml

# configmap中的config.conf就是kube-proxy的启动配置（另外还有一个kubeconfig.conf用来访问API服务器）
# 这个config.conf的规范来自以下链接
# https://kubernetes.io/zh-cn/docs/reference/config-api/kube-proxy-config.v1alpha1/#kubeproxy-config-k8s-io-v1alpha1-KubeProxyConfiguration
# 其中，有一个featureGates字段，用来控制kube-proxy的特性开关。
# 修改前建议备份configmap
$ kk get cm kube-proxy -nkube-system -oyaml > kube-proxy-configmap.yaml

# 编辑configmap: kk edit cm kube-proxy -nkube-system
# 添加featureGates字段如下
#apiVersion: v1
#data:
#  config.conf: |-
#    apiVersion: kubeproxy.config.k8s.io/v1alpha1
#    featureGates:  # <---
#      CPUManagerPolicyAlphaOptions: true

# 修改后删除一个kube-proxy pod进行观察能否正常重启（kube-proxy是DaemonSet控制器类型的Pod）
# 若能够正常重启，则说明配置生效；否则请还原配置后再检查配置
$ kk delete po kube-proxy-xxx -nkube-system

 

 # 已将 githubusercontent 替换为 gitmirror，加速下载yaml
$ wget https://raw.gitmirror.com/kubernetes/dashboard/v3.0.0-alpha0/charts/kubernetes-dashboard.yaml

 

 # 注意这里用到了gitmirror加速，问号后面的部分才是原始链接（同时在master和node1上执行）
$ wget https://hub.gitmirror.com/?q=https://github.com/cert-manager/cert-manager/releases/download/v1.13.2/cert-manager.yaml -O cert-manager.yaml
$ cat cert-manager.yaml |grep image:     
          image: "quay.io/jetstack/cert-manager-cainjector:v1.13.2"
          image: "quay.io/jetstack/cert-manager-controller:v1.13.2"
          image: "quay.io/jetstack/cert-manager-webhook:v1.13.2"

# 提前拉取cert-manager镜像（在node1上执行）
$ grep -oP 'image:\s*\K[^[:space:]]+' cert-manager.yaml | xargs -n 1 ctr -n k8s.io image pull

# 部署（回到master）
$ kubectl apply -f cert-manager.yaml

# 查看，cert-manager部署的资源对象都在一个新的命名空间【cert-manager】里面
# 注意：kk get all 只会列出pod/deployment/service/replicaset类型的对象，还有其他对象需要单独查询，建议直接查看yaml文件
$ kk get all -n cert-manager
NAME                                           READY   STATUS    RESTARTS   AGE
pod/cert-manager-6954d7bbbf-wtxd8              1/1     Running   0          9s
pod/cert-manager-cainjector-84bdff4846-cxfkb   1/1     Running   0          9s
pod/cert-manager-webhook-85b6b76d9b-psr8h      1/1     Running   0          9s

NAME                           TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)    AGE
service/cert-manager           ClusterIP   20.1.60.27    <none>        9402/TCP   9s
service/cert-manager-webhook   ClusterIP   20.1.38.212   <none>        443/TCP    9s

NAME                                      READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/cert-manager              1/1     1            1           9s
deployment.apps/cert-manager-cainjector   1/1     1            1           9s
deployment.apps/cert-manager-webhook      1/1     1            1           9s

NAME                                                 DESIRED   CURRENT   READY   AGE
replicaset.apps/cert-manager-6954d7bbbf              1         1         1       9s
replicaset.apps/cert-manager-cainjector-84bdff4846   1         1         1       9s
replicaset.apps/cert-manager-webhook-85b6b76d9b      1         1         1       9s

 

 # 部署
$ kk apply -f kubernetes-dashboard.yaml

# 查看主要部署的对象，都位于kubernetes-dashboard空间中
$ kk get all,ingress -n kubernetes-dashboard 
NAME                                                        READY   STATUS    RESTARTS   AGE
pod/kubernetes-dashboard-api-776f7d4b87-wgf7q               1/1     Running   0          43s
pod/kubernetes-dashboard-metrics-scraper-6b85f74cd5-6wng4   1/1     Running   0          43s
pod/kubernetes-dashboard-web-685bf6fd94-26z6v               1/1     Running   0          43s

NAME                                           TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)    AGE
service/kubernetes-dashboard-api               ClusterIP   20.1.54.147   <none>        9000/TCP   43s
service/kubernetes-dashboard-metrics-scraper   ClusterIP   20.1.45.125   <none>        8000/TCP   43s
service/kubernetes-dashboard-web               ClusterIP   20.1.29.80    <none>        8000/TCP   43s

NAME                                                   READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/kubernetes-dashboard-api               1/1     1            1           43s
deployment.apps/kubernetes-dashboard-metrics-scraper   1/1     1            1           43s
deployment.apps/kubernetes-dashboard-web               1/1     1            1           43s

NAME                                                              DESIRED   CURRENT   READY   AGE
replicaset.apps/kubernetes-dashboard-api-776f7d4b87               1         1         1       43s
replicaset.apps/kubernetes-dashboard-metrics-scraper-6b85f74cd5   1         1         1       43s
replicaset.apps/kubernetes-dashboard-web-685bf6fd94               1         1         1       43s

NAME                                             CLASS   HOSTS       ADDRESS   PORTS     AGE
ingress.networking.k8s.io/kubernetes-dashboard   nginx   localhost             80, 443   43s

 

 kind: Ingress
...
spec:
  ingressClassName: nginx
  tls:
    - hosts:
        - localhost
      secretName: kubernetes-dashboard-certs
  rules:
    - host: localhost
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: kubernetes-dashboard-web
                port:
                  name: web
          - path: /api
            pathType: Prefix
            backend:
              service:
                name: kubernetes-dashboard-api
                port:
                  name: api

 

 # -k 禁用服务器证书不安全警告
# 30415端口是ingress控制器映射到集群节点的端口，对应HTTPS的443端口。通过 kk get svc -n ingress-nginx 获取
$ curl -k https://localhost:30415
<!--
Copyright 2017 The Kubernetes Authors.
...
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
--><!DOCTYPE html><html lang="en" dir="ltr"><head>
  <meta charset="utf-8">
  <title>Kubernetes Dashboard</title>
...
</body></html>

 

 ...省略
---

kind: Ingress
apiVersion: networking.k8s.io/v1
metadata:
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
  labels:
    app.kubernetes.io/name: nginx-ingress
    app.kubernetes.io/part-of: kubernetes-dashboard
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    cert-manager.io/issuer: selfsigned
spec:
  ingressClassName: nginx
  tls:
    - hosts:
        - localhost
      secretName: kubernetes-dashboard-certs
  rules:
    #    - host: localhost  # <--- 注释对域名的限制，就可以通过HTTPS+IP方式访问
    - http:
  ...省略

 

 kk apply -f kubernetes-dashboard.yaml

 

 # 浏览器访问以下地址
# - 这个IP可以是任何一个集群节点IP
https://10.0.0.2:30415

 

 # 1. 创建新的sa并绑定角色
kk apply -f kubernetes-dashboard-role.yaml

# 2. 将新用户的token信息设置到kubeconfig文件中
dashboard_token=$(kk get secret dashboard-admin -n kubernetes-dashboard -ojsonpath={.data.token} |base64 -d)
kk config set-credentials dashboard-admin --token=$dashboard_token

# 3. 添加新的context
kk config set-context dashboard-admin@kubernetes --cluster=kubernetes --user=dashboard-admin

# 4. 使用新的context测试访问
kk --context=dashboard-admin@kubernetes get po

# 5. 使用新建的context导出一份新的kubeconfig文件
kk --context=dashboard-admin@kubernetes config view --minify --raw > my-kubeconfig

 

 $ kk apply -f kubernetes-dashboard-role.yaml                              
serviceaccount/dashboard-admin created
secret/dashboard-admin created
clusterrolebinding.rbac.authorization.k8s.io/dashboard-admin created

# 获取token明文
$ kk describe secret dashboard-admin -n kubernetes-dashboard
...
Data
====
ca.crt:     1099 bytes
namespace:  20 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6IllBVHZuRUEx...

 

 $ wget https://hub.gitmirror.com/?q=https://github.com/derailed/k9s/releases/download/v0.28.2/k9s_Linux_amd64.tar.gz -O k9s_Linux_amd64.tar.gz

$ tar xvf k9s_Linux_amd64.tar.gz           
LICENSE
README.md
k9s

$ cp k9s /usr/local/sbin

# 启动面板, Ctrl+C退出
$ k9s 

 

 k9s help

# 查看k9s的落地信息（配置、日志文件等）
k9s info

# 指定K8s namespace启动面板
k9s -n mycoolns

# 指定K8s context启动面板
k9s --context dashboard-admin@kubernetes

# 只读模式启动面板，启动后不支持修改指令
k9s --readonly

 

 # 如果你的环境无法高速访问外网，最好是手动下载二进制包
$ curl -SLO https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz

$ tar -zxvf helm-v3.13.3-linux-amd64.tar.gz
linux-amd64/
linux-amd64/helm
linux-amd64/LICENSE
linux-amd64/README.md
       
$ mv linux-amd64/helm /usr/local/bin/helm
$ helm -h                
The Kubernetes package manager

Common actions for Helm:

- helm search:    search for charts
- helm pull:      download a chart to your local directory to view
- helm install:   upload the chart to Kubernetes
- helm list:      list releases of charts
...省略

### MacOS(m系列) 安装
curl -SLO https://get.helm.sh/helm-v3.13.3-darwin-arm64.tar.gz
tar -zxvf helm-v3.13.3-darwin-arm64.tar.gz
mkdir -p ~/bin
cp darwin-arm64/helm ~/bin/
chmod +x ~/bin/helm
export PATH="$HOME/bin:$PATH" # 这是临时性的env配置。若要永久有效，需要写入 ~/.bash_profile 或 ~/.zshrc

 

 $ helm repo add bitnami https://charts.bitnami.com/bitnami
"bitnami" has been added to your repositories

# 查看仓库中所有的Charts
$ helm search repo bitnami
NAME                                        	CHART VERSION	APP VERSION  	DESCRIPTION                                       
bitnami/airflow                             	17.1.6       	2.7.3        	Apache Airflow is a tool to express and execute...
bitnami/apache                              	10.2.3       	2.4.58       	Apache HTTP Server is an open-source HTTP serve...
bitnami/apisix                              	2.2.7        	4.7.0        	Apache APISIX is high-performance, real-time AP...
...省略

 

 # 查看已添加的仓库列表
# 删除仓库 helm repo remove <repo-name>
$ helm repo list
NAME URL
bitnami https://charts.bitnami.com/bitnami
microsoft http://mirror.azure.cn/kubernetes/charts/

# 在已添加的仓库中的搜索包名，其中App Version是Mysql版本
# - helm使用模糊字符串匹配算法
$ helm search repo microsoft/mysql
NAME CHART VERSION APP VERSION DESCRIPTION
microsoft/mysql 1.6.9 5.7.30 DEPRECATED - Fast, reliable, scalable, and easy...
microsoft/mysqldump 2.6.2 2.4.1 DEPRECATED! - A Helm chart to help backup MySQL...

# 先查看Chart简介
$ helm show chart microsoft/mysql
apiVersion: v1
appVersion: 5.7.30
deprecated: true
description: DEPRECATED - Fast, reliable, scalable, and easy to use open-source relational
database system.
home: https://www.mysql.com/
icon: https://www.mysql.com/common/logos/logo-mysql-170x115.png
keywords:
- mysql
- database
- sql
name: mysql
sources:
- https://github.com/kubernetes/charts
- https://github.com/docker-library/mysql
version: 1.6.9

# 下载安装包至本地
$ helm pull microsoft/mysql
# 解压
$ tar xzf mysql-1.6.9.tgz
# Chart.yaml是Helm模板，values.yaml是应用的可修改的动态配置部分（安装时填充到CHart.yaml），如镜像版本、数据库密码等
$ ls mysql/
Chart.yaml README.md templates values.yaml
$ vi mysql/values.yaml
# 修改其中的部分信息
#mysqlRootPassword: "123"
#service:
#  annotations: {}
#  type: NodePort
#  port: 3306
#  nodePort: 32000

# 使用Chart包发布（即安装mysql）
# helm install...
# 这个命令会使用当前目录下的所有文件 在helm空间下部署一个名为helm-mysql的helm应用，helm空间若不存在会自动创建
# - 注意：部署后命令行会输出有关应用的一些辅助信息
# - 此命令会立即返回，部署将在后台进行，可使用 helm status helm-mysql 查看状态、
# - --namespace参数是可选的，若没有则安装到default空间下。使用helm部署应用时建议使用。
$ helm install helm-mysql mysql/ --namespace helm --create-namespace
WARNING: This chart is deprecated
NAME: helm-mysql
LAST DEPLOYED: Mon Nov 27 08:43:05 2023
NAMESPACE: helm
STATUS: deployed
REVISION: 1
NOTES:
MySQL can be accessed via port 3306 on the following DNS name from within your cluster:
helm-mysql.helm.svc.cluster.local
...省略

# 查看部署（不能看到Pod状态），-A表示显示所有K8s命名空间下的部署
$ helm ls -A
NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
helm-mysql helm 1 2023-11-27 08:43:05.787331394 +0800 CST deployed mysql-1.6.9 5.7.30

# 查看具体的Pod状态
$ kk get po -nhelm
NAME READY STATUS RESTARTS AGE
helm-mysql-5d8bd6c54f-c44dd 0/1 Pending 0 4m10s


 

 # 查看部署状态
helm status <helm-app-name>

# 添加 --keep-history 标志以保留版本历史
helm uninstall <helm-app-name> --keep-history

# 获取命令帮助
helm get -h