fio测试脚本

共计 5299 个字符，预计需要花费 14 分钟才能阅读完成。

前面在搞fio测试，为了测试数据的直观和方便收集，用python脚本处理下输出

fio测试脚本

tabulate库

[root@worker-0 ~]# pip3 install tabulate

tabulate基本使用

[root@worker-0 ~]# python3
Python 3.6.8 (default, Jun 20 2023, 11:53:23)
[GCC 4.8.5 20150623 (Red Hat 4.8.5-44)] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> from tabulate import tabulate

# 解析列表
>>> table = [['First Name', 'Last Name', 'Age'],
... ['John', 'Smith', 39],
... ['Mary', 'Jane', 25],
... ['Jennifer', 'Doe', 28]]
>>> print(tabulate(table))
----------  ---------  ---
First Name  Last Name  Age
John        Smith      39
Mary        Jane       25
Jennifer    Doe        28
----------  ---------  ---
>>> print(tabulate(table, headers='firstrow'))
First Name    Last Name      Age
------------  -----------  -----
John          Smith           39
Mary          Jane            25
Jennifer      Doe             28
>>> print(tabulate(table, headers='firstrow', tablefmt='grid'))
+--------------+-------------+-------+
| First Name   | Last Name   |   Age |
+==============+=============+=======+
| John         | Smith       |    39 |
+--------------+-------------+-------+
| Mary         | Jane        |    25 |
+--------------+-------------+-------+
| Jennifer     | Doe         |    28 |
+--------------+-------------+-------+
>>> print(tabulate(table, headers='firstrow', tablefmt='fancy_grid'))
╒══════════════╤═════════════╤═══════╕
│ First Name   │ Last Name   │   Age │
╞══════════════╪═════════════╪═══════╡
│ John         │ Smith       │    39 │
├──────────────┼─────────────┼───────┤
│ Mary         │ Jane        │    25 │
├──────────────┼─────────────┼───────┤
│ Jennifer     │ Doe         │    28 │
╘══════════════╧═════════════╧═══════╛

# 解析字典
>>> info = {'First Name': ['John', 'Mary', 'Jennifer'],
... 'Last Name': ['Smith', 'Jane', 'Doe'],
... 'Age': [39, 25, 28]}
>>> print(tabulate(info, headers='keys'))
First Name    Last Name      Age
------------  -----------  -----
John          Smith           39
Mary          Jane            25
Jennifer      Doe             28

# 输出index
>>> print(tabulate(info, headers='keys', showindex=True))
    First Name    Last Name      Age
--  ------------  -----------  -----
 0  John          Smith           39
 1  Mary          Jane            25
 2  Jennifer      Doe             28

fio测试脚本

# 安装fio和libaio
[root@worker-0 fio-test]# yum install fio libaio -y

# 测试脚本
[root@blog fio-test]# cat >fio-test.py<<'EOF'
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse
import time
import subprocess
import traceback
import json
from tabulate import tabulate

table_data = []
def fio_result_format(fio_json_result):
    with open(fio_json_result, 'r') as file:
        json_data = json.load(file)
        for job in json_data['jobs']:
            job_name = job['jobname']
            job_read_bw = job['read']['bw_bytes'] / (1024 * 1024)  # 将字节转换为兆字节
            job_read_bw_str = f"{job_read_bw:.2f} MB/s"  # 格式化带宽值并添加单位
            job_read_iops = job['read']['iops']
            job_write_bw = job['write']['bw_bytes'] / (1024 * 1024)  # 将字节转换为兆字节
            job_write_bw_str = f"{job_write_bw:.2f} MB/s"  # 格式化带宽值并添加单位
            job_write_iops = job['write']['iops']
            table_data.append([job_name, job_read_bw_str, job_read_iops, job_write_bw_str, job_write_iops])
    table_str = tabulate(table_data, headers=['Job', 'Read BW', 'Read IOPS', 'Write BW', 'Write IOPS'])

    return table_str


def main():
    parser = argparse.ArgumentParser(description='fio  benchmark  ')
    parser.add_argument('-f', '--config', type=str, dest='fio_config', help='hep info')
    options = parser.parse_args()
    print(options)
    if not options.fio_config:
        print("请指定压测配置文件,可以参考 fio_rw_example.cfg ")
        exit(-1)
    else:
        fio_config = options.fio_config

    current_time = time.strftime("%Y%m%d%H%M%S", time.localtime())
    fio_json_result = "/tmp/fio_io_bench_{}.json".format(current_time)
    FIO_CMD = 'fio {0} --output-format=json --output={1}'.format(fio_config, fio_json_result)
    try:
        process = subprocess.Popen(FIO_CMD, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        process.wait()
    except subprocess.SubprocessError as e:
        print(traceback.format_exc(e))
    print("fio_cmd {}".format(FIO_CMD))
    print("fio result file : {}".format(fio_json_result))
    table_str = fio_result_format(fio_json_result)
    print(table_str)


if __name__ == '__main__':
    main()
EOF

# fio配置文件样例
[root@blog fio-test]# cat >fio_rw_example.cfg<<'EOF'
[global]
ioengine=libaio
size=1024m
direct=1
thread=1
filename=/tmp/testfio
numjobs=1
iodepth=128

[4k-randread]
bs=4k
rw=randread
stonewall

[4k-randwrite]
bs=4k
rw=randwrite
stonewall

[bandwidth-read]
bs=1m
rw=read
stonewall

[bandwidth-write]
bs=1m
rw=write
stonewall
EOF

测试运行结果

# 阿里云服务器，40G的ESSD
[root@worker-0 fio-test]# python3 fio-test.py --config fio_rw_example.cfg
Namespace(fio_config='fio_rw_example.cfg')
fio_cmd fio fio_rw_example.cfg --output-format=json --output=/tmp/fio_io_bench_20230708103332.json
fio result file : /tmp/fio_io_bench_20230708103332.json
Job              Read BW        Read IOPS  Write BW       Write IOPS
---------------  -----------  -----------  -----------  ------------
4k-randread      8.99 MB/s       2300.98   0.00 MB/s           0
4k-randwrite     0.00 MB/s          0      8.92 MB/s        2284.36
bandwidth-read   124.51 MB/s      124.514  0.00 MB/s           0
bandwidth-write  0.00 MB/s          0      113.82 MB/s       113.816

# 如果拿来做nfs后的性能
[root@worker-0 fio-test]# python3 fio-test.py --config fio_rw_example.cfg
Namespace(fio_config='fio_rw_example.cfg')
fio_cmd fio fio_rw_example.cfg --output-format=json --output=/tmp/fio_io_bench_20230708104323.json
fio result file : /tmp/fio_io_bench_20230708104323.json
Job              Read BW         Read IOPS  Write BW       Write IOPS
---------------  ------------  -----------  -----------  ------------
4k-randread      234.32 MB/s      59987.2   0.00 MB/s           0
4k-randwrite     0.00 MB/s            0     7.16 MB/s        1833.34
bandwidth-read   1309.46 MB/s      1309.46  0.00 MB/s           0
bandwidth-write  0.00 MB/s            0     124.24 MB/s       124.242

# 试了下自己本地电脑的nvme磁盘（装了vm虚拟机）
[root@blog fio-test]# python3 fio-test.py --config fio_rw_example.cfg
Namespace(fio_config='fio_rw_example.cfg')

fio_cmd fio fio_rw_example.cfg --output-format=json --output=/tmp/fio_io_bench_20230708105449.json
fio result file : /tmp/fio_io_bench_20230708105449.json
Job              Read BW         Read IOPS  Write BW        Write IOPS
---------------  ------------  -----------  ------------  ------------
4k-randread      54.20 MB/s       13874.5   0.00 MB/s             0
4k-randwrite     0.00 MB/s            0     40.66 MB/s        10409.1
bandwidth-read   1250.31 MB/s      1250.31  0.00 MB/s             0
bandwidth-write  0.00 MB/s            0     1197.66 MB/s       1197.66