k8s部署中间件-redis

共计 10036 个字符，预计需要花费 26 分钟才能阅读完成。

因为后面内部资源平台逐渐向k8s转换，很多东西都需要改造，所以此处记录下这个中间件的部署方式

redis standalone方式

准备PV/PVC

[root@k8s-master ~]# kubectl get sc
NAME          PROVISIONER   RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
nfs-storage   nfs-client    Delete          Immediate           false                  27d

[root@k8s-master redis]# cat >redis-pvc.yaml<<-'EOF'
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: redis-pvc
spec:
  storageClassName: nfs-storage
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 512Mi
EOF
[root@k8s-master redis]# kubectl apply -f redis-pvc.yaml
persistentvolumeclaim/redis-pvc created
[root@k8s-master redis]# kubectl get -f redis-pvc.yaml
NAME        STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
redis-pvc   Bound    pvc-2f6cd73a-39c8-42e1-921f-205d4565450b   512Mi      RWO            nfs-storage    4s

准备redis configmap

[root@k8s-master redis]# cat >redis-configmap.yaml<<-'EOF'
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-cm
data:
  redis.conf: |+
    requirepass 123456
    protected-mode no
    port 6379
    tcp-backlog 511
    timeout 0
    tcp-keepalive 300
    daemonize no
    supervised no
    pidfile /var/run/redis_6379.pid
    loglevel notice
    logfile ""
    databases 16
    always-show-logo yes
    save 900 1
    save 300 10
    save 60 10000
    stop-writes-on-bgsave-error yes
    rdbcompression yes
    rdbchecksum yes
    dbfilename dump.rdb
    dir /data
    slave-serve-stale-data yes
    slave-read-only yes
    repl-diskless-sync no
    repl-diskless-sync-delay 5
    repl-disable-tcp-nodelay no
    slave-priority 100
    lazyfree-lazy-eviction no
    lazyfree-lazy-expire no
    lazyfree-lazy-server-del no
    slave-lazy-flush no
    appendonly yes
    appendfilename "appendonly.aof"
    appendfsync everysec
    no-appendfsync-on-rewrite no
    auto-aof-rewrite-percentage 100
    auto-aof-rewrite-min-size 64mb
    aof-load-truncated yes
    aof-use-rdb-preamble no
    lua-time-limit 5000
    slowlog-log-slower-than 10000
    slowlog-max-len 128
    latency-monitor-threshold 0
    notify-keyspace-events Ex
    hash-max-ziplist-entries 512
    hash-max-ziplist-value 64
    list-max-ziplist-size -2
    list-compress-depth 0
    set-max-intset-entries 512
    zset-max-ziplist-entries 128
    zset-max-ziplist-value 64
    hll-sparse-max-bytes 3000
    activerehashing yes
    client-output-buffer-limit normal 0 0 0
    client-output-buffer-limit slave 256mb 64mb 60
    client-output-buffer-limit pubsub 32mb 8mb 60
    hz 10
    aof-rewrite-incremental-fsync yes
EOF

[root@k8s-master redis]# kubectl apply -f redis-configmap.yaml
configmap/redis-cm created

使用deploy方式部署redis

[root@k8s-master redis]# cat >redis-deploy.yaml<<-'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
  name: redis
  labels:
    app: redis
spec:
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: redis
  template:
    metadata:
      labels:
        app: redis
    spec:
      containers:
      - name: redis
        image: redis
        imagePullPolicy: IfNotPresent
        command: ["redis-server","/etc/redis/redis.conf"]
        securityContext:
          runAsUser: 2000
          runAsNonRoot: true
          runAsGroup: 2000
        ports:
        - containerPort: 6379
        volumeMounts:
        - name: redis-config
          mountPath: /etc/redis/redis.conf
          subPath: redis.conf
        - name: redis-persistent-storage
          mountPath: /data
      volumes:
      - name: redis-config
        configMap:
          name: redis-cm
          items:
          - key: redis.conf
            path: redis.conf
      - name: redis-persistent-storage
        persistentVolumeClaim:
          claimName: redis-pvc
EOF

[root@k8s-master redis]# kubectl apply -f redis-deploy.yaml
[root@k8s-master redis]# kubectl get -f redis-deploy.yaml
NAME    READY   UP-TO-DATE   AVAILABLE   AGE
redis   1/1     1            1           23s

创建redis SVC服务

[root@k8s-master redis]# cat >redis-svc.yaml<<-'EOF'
kind: Service
apiVersion: v1
metadata:
  name: redis-svc
spec:
  type: NodePort
  selector:
    app: redis
  ports:
  - port: 6379
    targetPort: 6379
EOF
[root@k8s-master redis]# kubectl apply -f redis-svc.yaml
service/redis-svc created
[root@k8s-master redis]# kubectl get -f redis-svc.yaml
NAME        TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
redis-svc   NodePort   10.96.247.249   <none>        6379:32576/TCP   5s

测试验证连接

[root@k8s-master redis]# redis-cli -h 10.96.247.249
10.96.247.249:6379> auth 123456
OK
10.96.247.249:6379> ping
PONG
10.96.247.249:6379> exit

redis普通主从

配置entrypoint

上面standalone方式使用无状态deploy部署，现在主从我们需要采取有状态集来部署，因为我们可用通过有状态集的固定网络标志来确定master角色，接下来我们来改造下redis的entrypoint脚本

# 这是redis原本的entrypoint
root@redis-6457c894d6-pnkft:/usr/local/bin# cat docker-entrypoint.sh
#!/bin/sh
set -e

# first arg is `-f` or `--some-option`
# or first arg is `something.conf`
if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then
        set -- redis-server "$@"
fi

# allow the container to be started with `--user`
if [ "$1" = 'redis-server' -a "$(id -u)" = '0' ]; then
        find . \! -user redis -exec chown redis '{}' +
        exec gosu redis "$0" "$@"
fi

# set an appropriate umask (if one isn't set already)
# - https://github.com/docker-library/redis/issues/305
# - https://github.com/redis/redis/blob/bb875603fb7ff3f9d19aad906bd45d7db98d9a39/utils/systemd-redis_server.service#L37
um="$(umask)"
if [ "$um" = '0022' ]; then
        umask 0077
fi

exec "$@"

# 改造后entrypoint
# StatefulSet会给每个POD分配一个固定的主机网络标识符号，所以这里可以通过主机名去判断主节点
[root@k8s-master redis-ms]# cat >docker-entrypoint.sh<<-'EOF'
#!/bin/bash
cp /etc/redis/redis.conf.ini /etc/redis/redis.conf
if  [ "${HOSTNAME}" != "redis-0" ]; then
        echo "slaveof redis-0.redis-svc 6379" >> /etc/redis/redis.conf
#        until nc -zv redis-0.redis-svc  6379 -w 1; do echo 'waiting for redis-master nc'; sleep 1; done
fi
# first arg is `-f` or `--some-option`
# or first arg is `something.conf`
if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then
        set -- redis-server "$@"
fi

# allow the container to be started with `--user`
if [ "$1" = 'redis-server' -a "$(id -u)" = '0' ]; then
        find . \! -user redis -exec chown redis '{}' +
        exec gosu redis "$0" "$@"
fi

# set an appropriate umask (if one isn't set already)
# - https://github.com/docker-library/redis/issues/305
# - https://github.com/redis/redis/blob/bb875603fb7ff3f9d19aad906bd45d7db98d9a39/utils/systemd-redis_server.service#L37
um="$(umask)"
if [ "$um" = '0022' ]; then
        umask 0077
fi

exec "$@"


[root@k8s-master redis-ms]# kubectl create configmap redis-entrypoint-sh --from-file=docker-entrypoint.sh=docker-entrypoint.sh

创建configmap

[root@k8s-master redis-ms]# cat >redis-configmap.yaml<<-'EOF'
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-ms
data:
  redis.conf.ini: |+
    requirepass 123456
    masterauth 123456
    protected-mode no
    port 6379
    tcp-backlog 511
    timeout 0
    tcp-keepalive 300
    daemonize no
    supervised no
    pidfile /var/run/redis_6379.pid
    loglevel notice
    logfile ""
    databases 16
    always-show-logo yes
    save 900 1
    save 300 10
    save 60 10000
    stop-writes-on-bgsave-error yes
    rdbcompression yes
    rdbchecksum yes
    dbfilename dump.rdb
    dir /data
    slave-serve-stale-data yes
    slave-read-only yes
    repl-diskless-sync no
    repl-diskless-sync-delay 5
    repl-disable-tcp-nodelay no
    slave-priority 100
    lazyfree-lazy-eviction no
    lazyfree-lazy-expire no
    lazyfree-lazy-server-del no
    slave-lazy-flush no
    appendonly yes
    appendfilename "appendonly.aof"
    appendfsync everysec
    no-appendfsync-on-rewrite no
    auto-aof-rewrite-percentage 100
    auto-aof-rewrite-min-size 64mb
    aof-load-truncated yes
    aof-use-rdb-preamble no
    lua-time-limit 5000
    slowlog-log-slower-than 10000
    slowlog-max-len 128
    latency-monitor-threshold 0
    notify-keyspace-events Ex
    hash-max-ziplist-entries 512
    hash-max-ziplist-value 64
    list-max-ziplist-size -2
    list-compress-depth 0
    set-max-intset-entries 512
    zset-max-ziplist-entries 128
    zset-max-ziplist-value 64
    hll-sparse-max-bytes 3000
    activerehashing yes
    client-output-buffer-limit normal 0 0 0
    client-output-buffer-limit slave 256mb 64mb 60
    client-output-buffer-limit pubsub 32mb 8mb 60
    hz 10
    aof-rewrite-incremental-fsync yes
EOF

[root@k8s-master redis-ms]# kubectl apply -f redis-configmap.yaml

创建svc

[root@k8s-master redis-ms]# cat >redis-svc.yaml<<-'EOF'
apiVersion: v1
kind: Service
metadata:
  name: redis-svc
  labels:
    app: redis
spec:
  ports:
  - name: redis-port
    port: 6379
  clusterIP: None
  selector:
    app: redis
EOF
[root@k8s-master redis-ms]# kubectl apply -f redis-svc.yaml

创建statfulset

[root@k8s-master redis-ms]# cat >redis-sts.yaml<<-'EOF'
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
spec:
  serviceName: "redis-svc"
  replicas: 3
  selector:
    matchLabels:
      app: redis
  template:
    metadata:
      labels:
        app: redis
    spec:
      containers:
      - name: redis
        image: redis
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 6379
          name: redis
        volumeMounts:
        - name: data
          mountPath: /data
        - name: redis-entrypoint-sh
          mountPath: /usr/local/bin/docker-entrypoint.sh
          subPath: docker-entrypoint.sh
        - name: redis-config
          mountPath: /etc/redis/redis.conf.ini
          subPath: redis.conf.ini
        args:
        - redis-server
        - /etc/redis/redis.conf
      volumes:
      - name: redis-entrypoint-sh
        configMap:
          name: redis-entrypoint-sh
          items:
          - key: docker-entrypoint.sh
            path: docker-entrypoint.sh
          defaultMode: 0777
      - name: redis-config
        configMap:
          name: redis-ms
          items:
          - key: redis.conf.ini
            path: redis.conf.ini
  volumeClaimTemplates:
  - metadata:
      name: data
      annotations:
        volume.beta.kubernetes.io/storage-class: "nfs-storage"
    spec:
      accessModes: ["ReadWriteOnce"]
      resources:
        requests:
          storage: 1Gi
EOF
[root@k8s-master redis-ms]# kubectl apply -f redis-sts.yaml
[root@k8s-master redis-ms]# kubectl get -f redis-sts.yaml
NAME    READY   AGE
redis   3/3     4m33s
[root@k8s-master redis-ms]# kubectl get pods
NAME            READY   STATUS    RESTARTS   AGE
dapi-test-pod   1/1     Running   0          59m
redis-0         1/1     Running   0          4m37s
redis-1         1/1     Running   0          4m35s
redis-2         1/1     Running   0          4m33s

测试验证主从同步

[root@k8s-master redis-ms]# kubectl get pods -o wide -l app=redis
NAME      READY   STATUS    RESTARTS   AGE     IP               NODE         NOMINATED NODE   READINESS GATES
redis-0   1/1     Running   0          5m50s   10.100.235.193   k8s-master   <none>           <none>
redis-1   1/1     Running   0          5m48s   10.100.235.195   k8s-master   <none>           <none>
redis-2   1/1     Running   0          5m46s   10.100.235.217   k8s-master   <none>           <none>

[root@k8s-master redis-ms]# redis-cli -h 10.100.235.193
10.100.235.193:6379> auth 123456
OK
10.100.235.193:6379> info replication
# Replication
role:master
connected_slaves:2
slave0:ip=10.100.235.195,port=6379,state=online,offset=518,lag=0
slave1:ip=10.100.235.217,port=6379,state=online,offset=518,lag=0
master_failover_state:no-failover
master_replid:a317b6500ba443f98c0e18e9c2c9d400ac163fcc
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:518
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:518
10.100.235.193:6379> set name xadocker
OK
10.100.235.193:6379> get name
"xadocker"
10.100.235.193:6379> exit
[root@k8s-master redis-ms]# redis-cli -h 10.100.235.195
10.100.235.195:6379> auth 123456
OK
10.100.235.195:6379> get name
"xadocker"
10.100.235.195:6379> exit
[root@k8s-master redis-ms]# redis-cli -h 10.100.235.217
10.100.235.217:6379> auth 123456
OK
10.100.235.217:6379> get name
"xadocker"
10.100.235.217:6379> info replication
# Replication
role:slave
master_host:redis-0.redis-svc
master_port:6379
master_link_status:up
master_last_io_seconds_ago:5
master_sync_in_progress:0
slave_read_repl_offset:648
slave_repl_offset:648
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:a317b6500ba443f98c0e18e9c2c9d400ac163fcc
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:648
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:648
10.100.235.217:6379> exit

redis哨兵模式