Kubernetes调度管理优先级和抢占机制详解

apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: high-priority
value: 1000000
globalDefault: false
description: "This priority class should be used for high priority service pods only."

apiVersion: v1
kind: Pod
metadata:
  name: nginx
  labels:
    env: test
spec:
  containers:
  - name: nginx
    image: nginx
    imagePullPolicy: IfNotPresent
  priorityClassName: high-priority

$ kubectl label nodes kk-node01 env=uat

apiVersion: v1
kind: Pod
metadata:
  name: pod-nodeselector
spec:
  containers:
  - name: myapp
    image: ikubernetes/myapp:v1
  nodeSelector:
    env: uat

apiVersion: v1
kind: Pod
metadata:
  name: pod-nodename
spec:
  containers:
  - name: myapp
    image: ikubernetes/myapp:v1
  nodeName: kk-node01     # 节点名字

apiVersion: v1
kind: Pod
metadata:
  name: pod-nodeaffinity-preferred
spec:
  containers:
  - name: myapp
    image: ikubernetes/myapp:v1
  affinity:
    nodeAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
      - preference:
          matchExpressions:
          - key: disktype
            operator: In
            values: ["ssd", "harddisk"]
        weight: 60

apiVersion: v1
kind: Pod
metadata:
  name: pod-nodeaffinity-required
spec:
  containers:
  - name: myapp
    image: ikubernetes/myapp:v1
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: disktype
            operator: In
            values: ["ssd", "harddisk"]

apiVersion: v1
kind: Pod
metadata:
  name: fronted
  labels:
    app: myapp
    row: fronted
spec:
  containers:
  - name: myapp
    image: ikubernetes/myapp:v1
---
apiVersion: v1
kind: Pod
metadata:
  name: backend
  labels:
    app: db
    row: backend
spec:
  containers:
  - name: db
    image: busybox
    imagePullPolicy: IfNotPresent
    command:
    - "/bin/sh"
    - "-c"
    - "sleep 3600"
  affinity:
    podAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
      - labelSelector:
          matchExpressions:
          - key: app
            operator: In
            values: ["myapp"]
        topologyKey: kubernetes.io/hostname

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - nginx
            topologyKey: kubernetes.io/hostname
      containers:
      - name: nginx
        image: nginx

$ kubectl explain node.spec.taints
KIND:     Node
VERSION:  v1
RESOURCE: taints <[]Object>
DESCRIPTION:
     If specified, the node's taints.
     The node this Taint is attached to has the "effect" on any pod that does
     not tolerate the Taint.
FIELDS:
   effect       <string> -required-
     Required. The effect of the taint on pods that do not tolerate the taint.
     Valid effects are NoSchedule, PreferNoSchedule and NoExecute.
     Possible enum values:
     - `"NoExecute"` Evict any already-running pods that do not tolerate the
     taint. Currently enforced by NodeController.
     - `"NoSchedule"` Do not allow new pods to schedule onto the node unless
     they tolerate the taint, but allow all pods submitted to Kubelet without
     going through the scheduler to start, and allow all already-running pods to
     continue running. Enforced by the scheduler.
     - `"PreferNoSchedule"` Like TaintEffectNoSchedule, but the scheduler tries
     not to schedule new pods onto the node, rather than prohibiting new pods
     from scheduling onto the node entirely. Enforced by the scheduler.
   key  <string> -required-
     Required. The taint key to be applied to a node.
   timeAdded    <string>
     TimeAdded represents the time at which the taint was added. It is only
     written for NoExecute taints.
   value        <string>
     The taint value corresponding to the taint key.

$ kubectl taint nodes kk-node01 node-type=dev:NoSchedule

$ kubectl taint nodes kk-node01 node-type=dev:NoSchedule-

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
  labels:
    app: nginx
spec:
  replicas: 2
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:1.7.9
        imagePullPolicy: IfNotPresent 
        ports:
        - containerPort: 80
      tolerations:
      - key: "node-type"
        operator: Equal
        value: dev
        effect: NoSchedule
        tolerationSeconds: 20

tolerations:
- key: "key1"
  operator: "Exists"
  effect: "NoSchedule"

$ kubectl taint nodes node1 key1=value1:NoSchedule
$ kubectl taint nodes node1 key1=value1:NoExecute
$ kubectl taint nodes node1 key2=value2:NoSchedule

tolerations:
- key: "key1"
  operator: "Equal"
  value: "value1"
  effect: "NoSchedule"
- key: "key1"
  operator: "Equal"
  value: "value1"
  effect: "NoExecute"

tolerations:
- key: "key1"
  operator: "Equal"
  value: "value1"
  effect: "NoExecute"
  tolerationSeconds: 3600

$ wget https://github.com/kubernetes-sigs/descheduler/releases/download/descheduler-helm-chart-0.24.0/descheduler-0.24.0.tgz

$ helm install descheduler .

image:
  repository: registry.cn-hangzhou.aliyuncs.com/coolops/descheduler 
  # Overrides the image tag whose default is the chart version
  tag: "v0.24.0"
  pullPolicy: IfNotPresent

apiVersion: v1
data:
  policy.yaml: |
    apiVersion: "descheduler/v1alpha1"
    kind: "DeschedulerPolicy"
    strategies:
      LowNodeUtilization:
        enabled: true
        params:
          nodeResourceUtilizationThresholds:
            targetThresholds:
              cpu: 50
              memory: 50
              pods: 50
            thresholds:
              cpu: 20
              memory: 20
              pods: 20
      RemoveDuplicates:
        enabled: true
      RemovePodsViolatingInterPodAntiAffinity:
        enabled: true
      RemovePodsViolatingNodeAffinity:
        enabled: true
        params:
          nodeAffinityType:
          - requiredDuringSchedulingIgnoredDuringExecution
      RemovePodsViolatingNodeTaints:
        enabled: true
kind: ConfigMap
metadata:
  annotations:
    meta.helm.sh/release-name: descheduler
    meta.helm.sh/release-namespace: default
  creationTimestamp: "2022-08-02T03:06:57Z"
  labels:
    app.kubernetes.io/instance: descheduler
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: descheduler
    app.kubernetes.io/version: 0.24.0
    helm.sh/chart: descheduler-0.24.0
  name: descheduler
  namespace: default
  resourceVersion: "894636"
  uid: 4ab2e628-9404-4e52-bd88-615f5e096d90

apiVersion: batch/v1
kind: CronJob
metadata:
  annotations:
    meta.helm.sh/release-name: descheduler
    meta.helm.sh/release-namespace: default
  creationTimestamp: "2022-08-02T03:06:57Z"
  generation: 1
  labels:
    app.kubernetes.io/instance: descheduler
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: descheduler
    app.kubernetes.io/version: 0.24.0
    helm.sh/chart: descheduler-0.24.0
  name: descheduler
  namespace: default
  resourceVersion: "898221"
  uid: e209e498-71cb-413f-97a9-372aea5442bc
spec:
  concurrencyPolicy: Forbid
  failedJobsHistoryLimit: 1
  jobTemplate:
    metadata:
      creationTimestamp: null
    spec:
      template:
        metadata:
          annotations:
            checksum/config: 5efec14c3638fa4028e25f3fa067758f13dcae442fe711439c7d0b2e9913d41e
          creationTimestamp: null
          labels:
            app.kubernetes.io/instance: descheduler
            app.kubernetes.io/name: descheduler
          name: descheduler
        spec:
          containers:
          - args:
            - --policy-config-file
            - /policy-dir/policy.yaml
            - --v
            - "3"
            command:
            - /bin/descheduler
            image: registry.cn-hangzhou.aliyuncs.com/coolops/descheduler:v0.24.0
            imagePullPolicy: IfNotPresent
            livenessProbe:
              failureThreshold: 3
              httpGet:
                path: /healthz
                port: 10258
                scheme: HTTPS
              initialDelaySeconds: 3
              periodSeconds: 10
              successThreshold: 1
              timeoutSeconds: 1
            name: descheduler
            resources:
              requests:
                cpu: 500m
                memory: 256Mi
            securityContext:
              allowPrivilegeEscalation: false
              capabilities:
                drop:
                - ALL
              privileged: false
              readOnlyRootFilesystem: true
              runAsNonRoot: true
            terminationMessagePath: /dev/termination-log
            terminationMessagePolicy: File
            volumeMounts:
            - mountPath: /policy-dir
              name: policy-volume
          dnsPolicy: ClusterFirst
          priorityClassName: system-cluster-critical
          restartPolicy: Never
          schedulerName: default-scheduler
          securityContext: {}
          serviceAccount: descheduler
          serviceAccountName: descheduler
          terminationGracePeriodSeconds: 30
          volumes:
          - configMap:
              defaultMode: 420
              name: descheduler
            name: policy-volume
  schedule: '*/2 * * * *'
  successfulJobsHistoryLimit: 3
  suspend: false
status:
  lastScheduleTime: "2022-08-02T03:28:00Z"
  lastSuccessfulTime: "2022-08-02T03:28:03Z"