K8s nginx-ingress 灰度流量控制

1,881次阅读

共计 7966 个字符，预计需要花费 20 分钟才能阅读完成。

K8s nginx-ingress 灰度流量控制

nginx-ingress的灰度可以使用注解来完成 nginx.ingress.kubernetes.io/canary: "true"。目前常见的灰度方式有：

基于权重灰度：例如20%流量切到灰度版本
基于header灰度
- 基于 header 键：例如只要存在header 键为“canary-header”
- 基于 header 值：例如存在header键为“canary-header”，其值为：“canary-v1”
基于cookie灰度：例如cookie值为 “h5-v1”

准备测试资源

旧版资源

# 旧版deployment
[root@node1 demo]# cat nginx-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-canary
  labels:
    app: nginx-canary
spec:
  replicas: 10
  selector:
    matchLabels:
      app: nginx-canary
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 10%
      maxSurge: 10%
  template:
    metadata:
      labels:
        app: nginx-canary
    spec:
      initContainers:
      - name: init-container
        image: busybox:latest
        imagePullPolicy: IfNotPresent
        command: ["sh"]
        env:
#        - name: MY_POD_NAME
#          valueFrom:
#            fieldRef:
#              fieldPath: metadata.name
         - name: MY_POD_IP
           valueFrom:
             fieldRef:
               fieldPath: status.podIP
        args:
          [
            "-c",
            "echo ${HOSTNAME} ${MY_POD_IP} canary-v1 > /wwwroot/index.html",
          ]
        volumeMounts:
        - name: wwwroot
          mountPath: "/wwwroot"
      containers:
      - name: nginx
        image: nginx:latest
        ports:
        - containerPort: 80
          protocol: TCP
        volumeMounts:
        - name: wwwroot
          mountPath: /usr/share/nginx/html/index.html
          subPath: index.html
      volumes:
        - name: wwwroot
          emptyDir: {}

# 旧版service
[root@node1 demo]# cat nginx-canary-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: nginx-canary
  labels:
    app: nginx-canary
spec:
  ports:
  - port: 80
    targetPort: 80
    name: nginx-canary
  selector:
    app: nginx-canary

# 旧版ingress
[root@node1 demo]# cat canary-weight-ingress-old.yaml
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  name: gray-release
  namespace: default
spec:
  rules:
  - host: 'canary.xadocker.cn'
    http:
      paths:
      - path: /
        backend:
          serviceName: nginx-canary
          servicePort: 80


[root@node1 demo]# kubectl apply -f nginx-deployment.yaml
[root@node1 demo]# kubectl apply -f nginx-canary-service.yaml
[root@node1 demo]# kubectl apply -f canary-weight-ingress-old.yaml

新版资源

# 新版deployment
[root@node1 demo]# cat nginx-deployment-new.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-canary-new
  labels:
    app: nginx-canary-new
spec:
  replicas: 10
  selector:
    matchLabels:
      app: nginx-canary-new
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 10%
      maxSurge: 10%
  template:
    metadata:
      labels:
        app: nginx-canary-new
    spec:
      initContainers:
      - name: init-container
        image: busybox:latest
        imagePullPolicy: IfNotPresent
        command: ["sh"]
        env:
#        - name: MY_POD_NAME
#          valueFrom:
#            fieldRef:
#              fieldPath: metadata.name
         - name: MY_POD_IP
           valueFrom:
             fieldRef:
               fieldPath: status.podIP
        args:
          [
            "-c",
            "echo ${HOSTNAME} ${MY_POD_IP} canary-v2 > /wwwroot/index.html",
          ]
        volumeMounts:
        - name: wwwroot
          mountPath: "/wwwroot"
      containers:
      - name: nginx
        image: nginx:latest
        ports:
        - containerPort: 80
          protocol: TCP
        volumeMounts:
        - name: wwwroot
          mountPath: /usr/share/nginx/html/index.html
          subPath: index.html
      volumes:
        - name: wwwroot
          emptyDir: {}

# 新版service
[root@node1 demo]# cat nginx-canary-service-new.yaml
apiVersion: v1
kind: Service
metadata:
  name: nginx-canary-new
  labels:
    app: nginx-canary-new
spec:
  ports:
  - port: 80
    targetPort: 80
    name: nginx-canary-new
  selector:
    app: nginx-canary-new


[root@node1 demo]# kubectl apply -f nginx-deployment-new.yaml
[root@node1 demo]# kubectl apply -f nginx-canary-service-new.yaml

基于权重的灰度

# 创建权重灰度
[root@node1 demo]# cat canary-weight-ingress-new.yaml
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  name: canary-new
  namespace: default
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/canary: "true"
    nginx.ingress.kubernetes.io/canary-weight: "20"
spec:
  rules:
  - host: canary.xadocker.cn
    http:
      paths:
      - path: /
        backend:
          serviceName: nginx-canary-new
          servicePort: 80

[root@node1 demo]# kubectl apply -f canary-weight-ingress-new.yaml

测试验证

[root@node1 demo]# kubectl get ingress
NAME                   CLASS    HOSTS                      ADDRESS   PORTS     AGE
canary-new             <none>   canary.xadocker.cn                   80        18m
gray-release           <none>   canary.xadocker.cn                   80        17m

[root@node1 demo]# for i in `seq 1000`;do curl -H "HOST:canary.xadocker.cn" 127.0.0.1 -s >> test.txt;done
[root@node1 demo]# cat test.txt | awk '{print $3}' | sort -r | uniq -c
    191 canary-v2
    809 canary-v1

基于header的灰度

基于header键的灰度

# 
[root@node1 demo]# kubectl delete -f canary-weight-ingress-new.yaml
[root@node1 demo]# cat >canary-weight-ingress-new.yaml<<'EOF'
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  name: canary-new
  namespace: default
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/canary: "true"
    nginx.ingress.kubernetes.io/canary-by-header: "canary-v2"
spec:
  rules:
  - host: canary.xadocker.cn
    http:
      paths:
      - path: /
        backend:
          serviceName: nginx-canary-new
          servicePort: 80
EOF
[root@node1 demo]# kubectl apply -f canary-weight-ingress-new.yaml

# 测试的时候其键值必须为always，否则都会访问旧版
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" 127.0.0.1
nginx-canary-7fc7ccb7bd-tspvm 10.100.166.160 canary-v1
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-v2: always" 127.0.0.1
nginx-canary-new-87b757b7d-2csns 10.100.166.169 canary-v2
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-v2: as" 127.0.0.1
nginx-canary-7fc7ccb7bd-rwscr 10.100.166.157 canary-v1

基于header键值的灰度

# 配置灰度header为canary-release，其值为v2
[root@node1 demo]# kubectl delete -f canary-weight-ingress-new.yaml
[root@node1 demo]# cat >canary-weight-ingress-new.yaml<<'EOF'
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  name: canary-new
  namespace: default
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/canary: "true"
    nginx.ingress.kubernetes.io/canary-by-header: "canary-release"
    nginx.ingress.kubernetes.io/canary-by-header-value: "v2"
spec:
  rules:
  - host: canary.xadocker.cn
    http:
      paths:
      - path: /
        backend:
          serviceName: nginx-canary-new
          servicePort: 80
EOF
[root@node1 demo]# kubectl apply -f canary-weight-ingress-new.yaml

# 测试
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-: always" 127.0.0.1
nginx-canary-7fc7ccb7bd-tspvm 10.100.166.160 canary-v1
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-: always" 127.0.0.1
nginx-canary-7fc7ccb7bd-rz2f4 10.100.166.159 canary-v1
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-release: always" 127.0.0.1
nginx-canary-7fc7ccb7bd-tspvm 10.100.166.160 canary-v1
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-release: v1" 127.0.0.1
nginx-canary-7fc7ccb7bd-8vn7c 10.100.166.162 canary-v1
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-release: v2" 127.0.0.1
nginx-canary-new-87b757b7d-wqcpw 10.100.166.165 canary-v2
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-release: v2" 127.0.0.1

# 基于header的键值正则匹配
[root@node1 demo]# kubectl delete -f canary-weight-ingress-new.yaml
[root@node1 demo]# cat >canary-weight-ingress-new.yaml<<'EOF'
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  name: canary-new
  namespace: default
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/canary: "true"
    nginx.ingress.kubernetes.io/canary-by-header: "canary-release"
    nginx.ingress.kubernetes.io/canary-by-header-pattern: "v2|v3"
spec:
  rules:
  - host: canary.xadocker.cn
    http:
      paths:
      - path: /
        backend:
          serviceName: nginx-canary-new
          servicePort: 80
EOF

[root@node1 demo]# kubectl apply -f canary-weight-ingress-new.yaml
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-release: v2" 127.0.0.1
nginx-canary-new-87b757b7d-fxrq4 10.100.166.164 canary-v2
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-release: v3" 127.0.0.1
nginx-canary-new-87b757b7d-2csns 10.100.166.169 canary-v2
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-release: v1" 127.0.0.1
nginx-canary-7fc7ccb7bd-rvfcz 10.100.166.161 canary-v1

基于cookie的灰度

# 其cookie值必须为always才能切流到新版
[root@node1 demo]# kubectl delete -f canary-weight-ingress-new.yaml
[root@node1 demo]# cat canary-weight-ingress-new.yaml
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  name: canary-new
  namespace: default
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/canary: "true"
    nginx.ingress.kubernetes.io/canary-by-cookie: "userid12345678"
spec:
  rules:
  - host: canary.xadocker.cn
    http:
      paths:
      - path: /
        backend:
          serviceName: nginx-canary-new
          servicePort: 80

[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" --cookie "userid12345678" 127.0.0.1
nginx-canary-7fc7ccb7bd-rvfcz 10.100.166.161 canary-v1
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" --cookie "userid12345678=always" 127.0.0.1
nginx-canary-new-87b757b7d-z2z87 10.100.166.167 canary-v2
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" --cookie "userid1234567sdf8=always" 127.0.0.1
nginx-canary-7fc7ccb7bd-rz2f4 10.100.166.159 canary-v1
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn"  127.0.0.1
nginx-canary-7fc7ccb7bd-6qdhb 10.100.166.163 canary-v1

正文完

ingress k8s 灰度发布

发表至： kubernetes 云生原金色传说

2021-10-02

294

版权声明：本站原创文章，由 xadocker 2021-10-02发表，共计7966字。

转载说明：除特殊说明外本站文章皆由CC-4.0协议发布，转载请注明出处。

Prometheus-operator对etcd的监控

使用Thanos来扩展prometheus的高可用

k8s prometheus-operator

K8s 持久化存储-nfs

k8s调度器亲和性与反亲和性

k8s中一些常用运维命令

评论（没有评论）