共计 9578 个字符,预计需要花费 24 分钟才能阅读完成。

以往在linux中使用awk/sed来处理临时数据采集的工作,但在处理json这种结构数据时显得有点力不从心,但是有个jq命令却可以方便对json格式数据做处理,以前没怎么用,最近数据采集处理多了特此记录下
jq使用
jq安装
[ | ]|
[ | ]|
{ | |
"foo": 0 | |
} |
jq基本使用
样例数据
[root-master ~]# kubectl get ns monitoring -o json | |
{ | |
"apiVersion": "v1", | |
"kind": "Namespace", | |
"metadata": { | |
"annotations": { | |
"kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{},\"name\":\"monitoring\"}}\n" | |
}, | |
"creationTimestamp": "2023-03-16T15:58:48Z", | |
"labels": { | |
"kubernetes.io/metadata.name": "monitoring" | |
}, | |
"name": "monitoring", | |
"resourceVersion": "49152", | |
"uid": "fc5c167b-9e1e-49dd-aa00-47c3e30d475f" | |
}, | |
"spec": { | |
"finalizers": [ | |
"kubernetes" | |
] | |
}, | |
"status": { | |
"phase": "Active" | |
} | |
} |
jq默认格式化后缩进为2个空格
[root-master ~]# kubectl get ns monitoring -o json | jq . | |
{ | |
"apiVersion": "v1", | |
"kind": "Namespace", | |
"metadata": { | |
"annotations": { | |
"kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{},\"name\":\"monitoring\"}}\n" | |
}, | |
"creationTimestamp": "2023-03-16T15:58:48Z", | |
"labels": { | |
"kubernetes.io/metadata.name": "monitoring" | |
}, | |
"name": "monitoring", | |
"resourceVersion": "49152", | |
"uid": "fc5c167b-9e1e-49dd-aa00-47c3e30d475f" | |
}, | |
"spec": { | |
"finalizers": [ | |
"kubernetes" | |
] | |
}, | |
"status": { | |
"phase": "Active" | |
} | |
} |
jq获取指定字段
[root-master ~]# kubectl get ns monitoring -o json | jq .metadata | |
{ | |
"annotations": { | |
"kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{},\"name\":\"monitoring\"}}\n" | |
}, | |
"creationTimestamp": "2023-03-16T15:58:48Z", | |
"labels": { | |
"kubernetes.io/metadata.name": "monitoring" | |
}, | |
"name": "monitoring", | |
"resourceVersion": "49152", | |
"uid": "fc5c167b-9e1e-49dd-aa00-47c3e30d475f" | |
} | |
[root-master ~]# kubectl get ns monitoring -o json | jq .metadata.name | |
"monitoring" |
jq获取多个字段
[root@k8s-master ~]# kubectl get ns monitoring -o json | jq ".metadata.name,.status" | |
"monitoring" | |
{ | |
"phase": "Active" | |
} |
jq迭代处理
jq处理数组或列表
[root-master ~]# echo '["a","b","c","d","e"]' | jq | |
[ | |
"a", | |
"b", | |
"c", | |
"d", | |
"e" | |
] | |
[root-master ~]# echo '["a","b","c","d","e"]' | jq '.[2:3]' | |
[ | |
"c" | |
] | |
[root-master ~]# echo '["a","b","c","d","e"]' | jq '.[2:4]' | |
[ | |
"c", | |
"d" | |
] | |
[root-master ~]# echo '["a","b","c","d","e"]' | jq '.[2:]' | |
[ | |
"c", | |
"d", | |
"e" | |
] | |
[root-master ~]# echo '["a","b","c","d","e"]' | jq '.[:4]' | |
[ | |
"a", | |
"b", | |
"c", | |
"d" | |
] | |
[root-master ~]# echo '["a","b","c","d","e"]' | jq '.[-2:]' | |
[ | |
"d", | |
"e" | |
] | |
[root-master ~]# echo '["a","b","c","d","e"]' | jq '.[2,4]' | |
"c" | |
"e" | |
[root-master ~]# echo '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]' | jq | |
[ | |
{ | |
"name": "JSON", | |
"good": true | |
}, | |
{ | |
"name": "XML", | |
"good": false | |
} | |
] | |
[root-master ~]# echo '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]' | jq ".[].name" | |
"JSON" | |
"XML" | |
[root-master ~]# echo '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]' | jq ".[0].name" | |
"JSON" | |
[root-master ~]# | |
[root-master ~]# echo '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]' | jq ".[1].name" | |
"XML" | |
# 获取ns信息 | |
[root-master ~]# kubectl get ns -o json | jq '.items[].metadata.name' | |
"calico-apiserver" | |
"calico-system" | |
"default" | |
"kong" | |
"kube-node-lease" | |
"kube-public" | |
"kube-system" | |
"monitoring" | |
"tigera-operator" | |
[root-master ~]# kubectl get ns -o json | jq -r '.items[].metadata.name' | |
calico-apiserver | |
calico-system | |
default | |
kong | |
kube-node-lease | |
kube-public | |
kube-system | |
monitoring | |
tigera-operator | |
jq管道
管道将命令的输出给后面命令作为输入,和系统命令行的管道类似
[root@k8s-master ~]# kubectl get ns -o json | jq -r '.items[] | .metadata.name' | |
calico-apiserver | |
calico-system | |
default | |
kong | |
kube-node-lease | |
kube-public | |
kube-system | |
monitoring | |
tigera-operator | |
[root@k8s-master ~]# kubectl get ns -o json | jq -r '.items[] | .metadata.name,.metadata.creationTimestamp' | |
calico-apiserver | |
2023-03-14T18:58:51Z | |
calico-system | |
2023-03-14T18:55:48Z | |
default | |
2023-03-14T18:53:41Z | |
kong | |
2023-03-15T15:24:29Z | |
kube-node-lease | |
2023-03-14T18:53:39Z | |
kube-public | |
2023-03-14T18:53:39Z | |
kube-system | |
2023-03-14T18:53:39Z | |
monitoring | |
2023-03-16T15:58:48Z | |
tigera-operator | |
2023-03-14T18:55:07Z |
jq运算
[root@k8s-master ~] | |
15 | |
20 | |
25 | |
30 | |
35 | |
[root@k8s-master ~] | |
[ | |
1, | |
2, | |
3, | |
4 | |
] | |
[root@k8s-master ~] | |
[ | |
"json" | |
] |
jq输出新对象
[ | ]|
{ | |
"metadata": { | |
"creationTimestamp": "2023-03-15T15:24:29Z", | |
"labels": { | |
"kubernetes.io/metadata.name": "kong" | |
}, | |
"name": "kong", | |
"resourceVersion": "20554", | |
"uid": "64776950-e34f-4be0-94cc-3ff56ca0010f" | |
}, | |
"test": { | |
"phase": "Active" | |
} | |
} |
jq一些内置函数
length获取长度
[root@k8s-master ~]# kubectl get ns -o json | jq '.items | length' | |
9 |
key获取键
[root@k8s-master ~]# kubectl get ns kong -o json | jq 'keys' | |
[ | |
"apiVersion", | |
"kind", | |
"metadata", | |
"spec", | |
"status" | |
] | |
[root@k8s-master ~]# echo '[4,3,2,1]' | jq 'keys' | |
[ | |
0, | |
1, | |
2, | |
3 | |
] | |
select判断
[root@k8s-master ~]# echo '[1,5,3,0,7]' | jq 'map(select(. >= 2))' | |
[ | |
5, | |
3, | |
7 | |
] | |
[root@k8s-master ~]# kubectl get pods -A -o json | jq '.items[] | select(.status.qosClass == "BestEffort") | .metadata.name' | |
"calico-apiserver-868f4c547b-pw452" | |
"calico-apiserver-868f4c547b-zn96k" | |
"calico-kube-controllers-77b99b58ff-g8qz6" | |
"calico-node-srsmd" | |
"calico-typha-6fb7f79d46-nkqd2" | |
"echo-5fc5b5bc84-fjkcb" | |
"echo-5fc5b5bc84-h5smd" | |
"echo-5fc5b5bc84-hdpdl" | |
"echo-5fc5b5bc84-rv9jk" | |
"echo-5fc5b5bc84-sgsrj" | |
"kong-kong-7959784879-2zlrm" | |
"kong-kong-init-migrations--1-8r4td" | |
"kong-kong-pre-upgrade-migrations--1-t5b8d" | |
"konga-7c477b867b-wfj4j" | |
"kube-proxy-5pk9h" | |
"nfs-client-provisioner-6949899dd7-gfvnc" | |
"tigera-operator-66f6445597-7zgfp" |
add相加
[root@k8s-master ~]# echo '["a","b","c"]' | jq 'add' | |
"abc" | |
[root@k8s-master ~]# echo '[2,3,4]' | jq 'add' | |
9 |
split分割
[root@k8s-master ~]# echo '"a, b,c,d, e, "' | jq -c ' split(", ")' | |
["a","b,c,d","e",""] | |
[root@k8s-master ~]# echo '"a-b-c-d-e-"' | jq -c ' split("-")' | |
["a","b","c","d","e",""] | |
join联合
[root@k8s-master ~] | |
"a,b,c,d,e" | |
[root@k8s-master ~] | |
[ | |
"a", | |
"b", | |
"c", | |
"d", | |
"e" | |
] |
json转换
[root-master ~]# echo '[1, "foo", ["foo"]]' | jq '[.[]|tostring]' | |
[ | |
"1", | |
"foo", | |
"[\"foo\"]" | |
] | |
[root-master ~]# echo '[1, "foo", ["foo"]]' | jq '[.[]|tojson]' | |
[ | |
"1", | |
"\"foo\"", | |
"[\"foo\"]" | |
] | |
[root-master ~]# kubectl get ns monitoring -o json | jq '.metadata.annotations | ."kubectl.kubernetes.io/last-applied-configuration"' | |
"{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{},\"name\":\"monitoring\"}}\n" | |
[root-master ~]# kubectl get ns monitoring -o json | jq '.metadata.annotations | ."kubectl.kubernetes.io/last-applied-configuration" | fromjson' | |
{ | |
"apiVersion": "v1", | |
"kind": "Namespace", | |
"metadata": { | |
"annotations": {}, | |
"name": "monitoring" | |
} | |
} | |
jq示例
获取容器的resources
[root-master ~]# kubectl get pods -n monitoring -o json | jq -r '.items[] | .metadata.namespace as $ns | .metadata.name as $pod | .spec.containers[] | [$ns, $pod, .name, ((.resources.requests.cpu // "0m")| if endswith("m") then sub("m$"; "") else .| tonumber * 1000 end), ((.resources.limits.cpu // "0m") | if endswith("m") then sub("m$"; "") else .| tonumber * 1000 end), ((.resources.requests.memory // "0Mi")| if endswith("Mi") then sub("Mi$"; "") elif endswith("Gi") then sub("Gi$"; "") | tonumber * 1024 else . end), ((.resources.limits.memory // "0Mi") |if endswith("Mi") then sub("Mi$"; "") elif endswith("Gi") then sub("Gi$"; "") | tonumber * 1024 else . end )] | @tsv' | column -t | |
monitoring alertmanager-main-0 alertmanager 4 100 100 100 | |
monitoring alertmanager-main-0 config-reloader 100 100 50 50 | |
monitoring alertmanager-main-1 alertmanager 4 100 100 100 | |
monitoring alertmanager-main-1 config-reloader 100 100 50 50 | |
monitoring alertmanager-main-2 alertmanager 4 100 100 100 | |
monitoring alertmanager-main-2 config-reloader 100 100 50 50 | |
monitoring blackbox-exporter-5c545d55d6-v6fnw blackbox-exporter 10 20 20 40 | |
monitoring blackbox-exporter-5c545d55d6-v6fnw module-configmap-reloader 10 20 20 40 | |
monitoring blackbox-exporter-5c545d55d6-v6fnw kube-rbac-proxy 10 20 20 40 | |
monitoring grafana-7945df75ff-lgg59 grafana 100 200 100 200 | |
monitoring kube-state-metrics-54bd6b479c-9vtv2 kube-state-metrics 10 100 190 250 | |
monitoring kube-state-metrics-54bd6b479c-9vtv2 kube-rbac-proxy-main 20 40 20 40 | |
monitoring kube-state-metrics-54bd6b479c-9vtv2 kube-rbac-proxy-self 10 20 20 40 | |
monitoring node-exporter-npcpd node-exporter 102 250 180 180 | |
monitoring node-exporter-npcpd kube-rbac-proxy 10 20 20 40 | |
monitoring prometheus-adapter-7bf7ff5b67-gn66n prometheus-adapter 102 250 180 180 | |
monitoring prometheus-adapter-7bf7ff5b67-k8gzg prometheus-adapter 102 250 180 180 | |
monitoring prometheus-k8s-0 prometheus 0 0 400 0 | |
monitoring prometheus-k8s-0 config-reloader 100 100 50 50 | |
monitoring prometheus-k8s-1 prometheus 0 0 400 0 | |
monitoring prometheus-k8s-1 config-reloader 100 100 50 50 | |
monitoring prometheus-operator-54dd69bbf6-6kfql prometheus-operator 100 200 100 200 | |
monitoring prometheus-operator-54dd69bbf6-6kfql kube-rbac-proxy 10 20 20 40 |
获取容器
[root-master ~]# kubectl get pods -n monitoring -o json | jq -r '.items[] | .metadata.namespace as $ns | .metadata.name as $pod | .spec.containers[].name | [$ns, $pod, .] | ' | column -t | |
monitoring alertmanager-main-0 alertmanager | |
monitoring alertmanager-main-0 config-reloader | |
monitoring alertmanager-main-1 alertmanager | |
monitoring alertmanager-main-1 config-reloader | |
monitoring alertmanager-main-2 alertmanager | |
monitoring alertmanager-main-2 config-reloader | |
monitoring blackbox-exporter-5c545d55d6-v6fnw blackbox-exporter | |
monitoring blackbox-exporter-5c545d55d6-v6fnw module-configmap-reloader | |
monitoring blackbox-exporter-5c545d55d6-v6fnw kube-rbac-proxy | |
monitoring grafana-7945df75ff-lgg59 grafana | |
monitoring kube-state-metrics-54bd6b479c-9vtv2 kube-state-metrics | |
monitoring kube-state-metrics-54bd6b479c-9vtv2 kube-rbac-proxy-main | |
monitoring kube-state-metrics-54bd6b479c-9vtv2 kube-rbac-proxy-self | |
monitoring node-exporter-npcpd node-exporter | |
monitoring node-exporter-npcpd kube-rbac-proxy | |
monitoring prometheus-adapter-7bf7ff5b67-gn66n prometheus-adapter | |
monitoring prometheus-adapter-7bf7ff5b67-k8gzg prometheus-adapter | |
monitoring prometheus-k8s-0 prometheus | |
monitoring prometheus-k8s-0 config-reloader | |
monitoring prometheus-k8s-1 prometheus | |
monitoring prometheus-k8s-1 config-reloader | |
monitoring prometheus-operator-54dd69bbf6-6kfql prometheus-operator | |
monitoring prometheus-operator-54dd69bbf6-6kfql kube-rbac-proxy |
正文完