Add additional fio argument check & unit tests

Author: portante

agent/bench-scripts/gold/pbench-fio/test-05.txt

@@ -0,0 +1,107 @@
++++ Running test-14 pbench-fio
+[error][1900-01-01T00:00:00.000000] Unsupported --test-types=foo,bar encountered
+The following options are available:
+
+	-t str[,str] --test-types=str[,str]
+		one or more of read,write,rw,randread,randwrite,randrw
+
+	--direct=[0/1]
+		1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled
+
+	--sync=[0/1]
+		1 = O_SYNC enabled, 0 = O_SYNC disabled (default)
+
+	--rate-iops=int
+		do not exceeed this IOP rate (per job, per client)
+
+	-r int --runtime=int
+		runtime in seconds (default is )
+
+	--ramptime=int
+		time in seconds to warm up test before taking measurements (default is )
+
+	-b int[,int] --block-sizes=str[,str] (default is 4,64,1024)
+		one or more block sizes in KiB
+
+	-s int[,int] --file-size=str[,str] (no default)
+		file sizes in MiB
+
+	-d str[,str] --targets=str[,str]
+		one or more directories or block devices (default is /tmp/fio)
+		(persistent names for devices highly recommended)
+
+	-j str --job-mode=str    str=[serial|concurrent]  (default is 'concurrent')
+		directs how --targets parameter(s) is/are used; with 'serial' mode all combinations
+		of fio job parameters are run against each target one at a time, while with 'concurrent'
+		mode all target devices are used at the same time.
+
+	--ioengine=str           str= any ioengine fio supports (default is )
+
+	--iodepth=<int>		Set the iodepth config variable in the fio job file
+
+	-c str[,str] --clients=str[,str]      str= a list of one or more host names (hosta,hostb,hostc) where you want fio to run
+		If no clients are specified, fio is run locally
+		Note: the pbench-agent must be installed on each of the client systems already.
+
+	--client-file=str        str= file (with absolute path) which contains 1 client per line
+
+	--config=str
+		name of the test configuration
+
+	--tool-group=str
+
+	--postprocess-only=[y|n]
+		use this only if you want to postprocess an existing result again
+		you must use --run-dir option with this
+
+	--run-dir=<path>
+		provide the path of an existig result (typically somewhere in /var/tmp/pbench-test-bench/pbench-agent
+
+	--numjobs=<int>
+		number of jobs to run, if not given then fio default of numjobs=1 will be used
+
+	--job-file=<path>
+		provide the path of a fio job config file, (default is /var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/templates/fio.job)
+
+	--pre-iteration-script=str
+		use executable script/program to prepare the system for test iteration
+		example: --pre-iteration-script=$HOME/drop-cache.sh
+
+	--samples=<int>
+		number of samples to use per test iteration (default is 5)
+
+	--max-stddev=<int>
+		the maximum percent stddev allowed to pass
+
+	--max-failures=<int>
+		the maximum number of failures to get below stddev
+
+	--install
+		install only (default is False)
+
+	--histogram-interval-sec=<int>
+		set the histogram logging interval in seconds (default 10)
+
+	--sysinfo=str            str= comma separated values of sysinfo to be collected
+		available: 
+--- Finished test-14 pbench-fio (status=1)
++++ pbench tree state
+/var/tmp/pbench-test-bench/pbench-agent
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-14_1900.01.01T00.00.00
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-14_1900.01.01T00.00.00/metadata.log
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log
+/var/tmp/pbench-test-bench/pbench-agent/tmp
+/var/tmp/pbench-test-bench/pbench-agent/tools-default
+/var/tmp/pbench-test-bench/pbench-agent/tools-default/mpstat
+/var/tmp/pbench-test-bench/pbench-agent/tools-default/sar
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[error][1900-01-01T00:00:00.000000] Unsupported --test-types=foo,bar encountered
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --options
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --sysinfo=default --check
+--- test-execution.log file contents
++++ fio_test-14_1900.01.01T00.00.00/metadata.log file contents
+[pbench]
+--- fio_test-14_1900.01.01T00.00.00/metadata.log file contents

agent/bench-scripts/gold/pbench-fio/test-06.txt

@@ -0,0 +1,109 @@
++++ Running test-16 pbench-fio
+verifying clients have fio installed
+Created the following job file (/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/fio.job):
+[global]
+bs=42k
+runtime=30
+ioengine=libaio
+iodepth=32
+direct=1
+sync=0
+time_based=1
+clocksource=gettimeofday
+ramp_time=5
+write_bw_log=fio
+write_iops_log=fio
+write_lat_log=fio
+log_avg_msec=1000
+write_hist_log=fio
+log_hist_msec=10000
+
+[job-/dev/sda0]
+filename=/dev/sda0
+rw=rw
+size=4096M
+numjobs=1
+
+[job-/dev/sda1]
+filename=/dev/sda1
+rw=rw
+size=4096M
+numjobs=1
+
+running fio job: /var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/fio.job (sample1)
+[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/192.168.1.1/fio_clat_hist.*.log*) to process for histograms
+[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/*/fio_clat_hist.*.log*) to process for histograms
+fio job complete
+--- Finished test-16 pbench-fio (status=0)
++++ pbench tree state
+/var/tmp/pbench-test-bench/pbench-agent
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/fio.job
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/process-iteration-samples.cmd
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/sample1
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/sample1/clients
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/sample1/clients/192.168.1.1
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/sample1/fio-postprocess.cmd
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/sample1/fio-result.txt
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/sample1/fio.cmd
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/sample1/tools-default
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/fio-client.file
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/generate-benchmark-summary.cmd
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/metadata.log
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log
+/var/tmp/pbench-test-bench/pbench-agent/tmp
+/var/tmp/pbench-test-bench/pbench-agent/tmp/fio_test-16_1900.01.01T00.00.00.iterations
+/var/tmp/pbench-test-bench/pbench-agent/tools-default
+/var/tmp/pbench-test-bench/pbench-agent/tools-default/mpstat
+/var/tmp/pbench-test-bench/pbench-agent/tools-default/sar
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[debug][1900-01-01T00:00:00.000000] [check_install_rpm] attempting to install pbench-fio-3.3
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[debug][1900-01-01T00:00:00.000000] [pbench-fio]pbench-fio 3.3 is installed
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[debug][1900-01-01T00:00:00.000000] verifying clients have fio installed
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[debug][1900-01-01T00:00:00.000000] fio_device_check() /dev/sda0,/dev/sda1 192.168.1.1
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[debug][1900-01-01T00:00:00.000000] checking to see if /dev/sda0 exists on client 192.168.1.1
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[debug][1900-01-01T00:00:00.000000] checking to see if /dev/sda1 exists on client 192.168.1.1
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[debug][1900-01-01T00:00:00.000000] creating directories on the clients
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[debug][1900-01-01T00:00:00.000000] opening port 8765 on firewall on the clients
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[debug][1900-01-01T00:00:00.000000] killing any old fio process on the clients
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[debug][1900-01-01T00:00:00.000000] starting new fio process on the clients
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[debug][1900-01-01T00:00:00.000000] waiting for fio process(server) to start on clients
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[debug][1900-01-01T00:00:00.000000] fio: Going to run [/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/bm --output-format=json /var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/fio.job  --client=/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/fio-client.file --max-jobs=2]
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/192.168.1.1/fio_clat_hist.*.log*) to process for histograms
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[warn][1900-01-01T00:00:00.000000] log_hist_msec specified in job file but failed to find any log files (.../clients/*/fio_clat_hist.*.log*) to process for histograms
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[debug][1900-01-01T00:00:00.000000] post-processing fio result
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/postprocess/fio-postprocess /var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/sample1 fio- default
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/postprocess/generate-benchmark-summary fio --config=test-16 --job-mode=concurrent -c 192.168.1.1 --test-types=rw --samples=1 --block-sizes=42 --targets=/dev/sda0,/dev/sda1 /var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/postprocess/process-iteration-samples /var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB readwrite_IOPS 5 0 6 y y
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/bm --client=/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/fio-client.file --max-jobs=2 --output-format=json /var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/fio.job
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00 --sysinfo=default beg
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00 --sysinfo=default end
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --sysinfo=default --check
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-metadata-log --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00 beg
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-metadata-log --group=default --dir=/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00 end
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-postprocess-tools --group=default --iteration=1-rw-42KiB --dir=/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/sample1
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-start-tools --group=default --iteration=1-rw-42KiB --dir=/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/sample1
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-stop-tools --group=default --iteration=1-rw-42KiB --dir=/var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/sample1
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/ssh -o StrictHostKeyChecking=no 192.168.1.1 /var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/pbench-fio --install
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/ssh -o StrictHostKeyChecking=no 192.168.1.1 firewall-cmd --add-port=8765/tcp >/dev/null
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/ssh -o StrictHostKeyChecking=no 192.168.1.1 if [ -L /dev/sda0 ]; then dev=/dev/; fi; test -b /dev/sda0
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/ssh -o StrictHostKeyChecking=no 192.168.1.1 if [ -L /dev/sda1 ]; then dev=/dev/; fi; test -b /dev/sda1
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/ssh -o StrictHostKeyChecking=no 192.168.1.1 killall fio >/dev/null 2>&1
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/ssh -o StrictHostKeyChecking=no 192.168.1.1 mkdir -p /var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/sample1
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/ssh -o StrictHostKeyChecking=no 192.168.1.1 pushd /var/tmp/pbench-test-bench/pbench-agent/fio_test-16_1900.01.01T00.00.00/1-rw-42KiB/sample1 >/dev/null; screen -dmS fio-server bash -c ''/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/bm' --server 2>&1 >client-result.txt'
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/timeout --kill-after=1 60 bash -c until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done 192.168.1.1 8765
+--- test-execution.log file contents
++++ fio_test-16_1900.01.01T00.00.00/metadata.log file contents
+[pbench]
+
+[iterations/1-rw-42KiB]
+iteration_number = 1
+test_type = rw
+block_size_kib = 42
+iteration_name = 1-rw-42KiB
+
+--- fio_test-16_1900.01.01T00.00.00/metadata.log file contents

agent/bench-scripts/gold/pbench-fio/test-14.txt

@@ -0,0 +1,107 @@
++++ Running test-17 pbench-fio
+[error][1900-01-01T00:00:00.000000] Unsupported --job-mode=bad encountered
+The following options are available:
+
+	-t str[,str] --test-types=str[,str]
+		one or more of read,write,rw,randread,randwrite,randrw
+
+	--direct=[0/1]
+		1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled
+
+	--sync=[0/1]
+		1 = O_SYNC enabled, 0 = O_SYNC disabled (default)
+
+	--rate-iops=int
+		do not exceeed this IOP rate (per job, per client)
+
+	-r int --runtime=int
+		runtime in seconds (default is )
+
+	--ramptime=int
+		time in seconds to warm up test before taking measurements (default is )
+
+	-b int[,int] --block-sizes=str[,str] (default is 4,64,1024)
+		one or more block sizes in KiB
+
+	-s int[,int] --file-size=str[,str] (no default)
+		file sizes in MiB
+
+	-d str[,str] --targets=str[,str]
+		one or more directories or block devices (default is /tmp/fio)
+		(persistent names for devices highly recommended)
+
+	-j str --job-mode=str    str=[serial|concurrent]  (default is 'concurrent')
+		directs how --targets parameter(s) is/are used; with 'serial' mode all combinations
+		of fio job parameters are run against each target one at a time, while with 'concurrent'
+		mode all target devices are used at the same time.
+
+	--ioengine=str           str= any ioengine fio supports (default is )
+
+	--iodepth=<int>		Set the iodepth config variable in the fio job file
+
+	-c str[,str] --clients=str[,str]      str= a list of one or more host names (hosta,hostb,hostc) where you want fio to run
+		If no clients are specified, fio is run locally
+		Note: the pbench-agent must be installed on each of the client systems already.
+
+	--client-file=str        str= file (with absolute path) which contains 1 client per line
+
+	--config=str
+		name of the test configuration
+
+	--tool-group=str
+
+	--postprocess-only=[y|n]
+		use this only if you want to postprocess an existing result again
+		you must use --run-dir option with this
+
+	--run-dir=<path>
+		provide the path of an existig result (typically somewhere in /var/tmp/pbench-test-bench/pbench-agent
+
+	--numjobs=<int>
+		number of jobs to run, if not given then fio default of numjobs=1 will be used
+
+	--job-file=<path>
+		provide the path of a fio job config file, (default is /var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/templates/fio.job)
+
+	--pre-iteration-script=str
+		use executable script/program to prepare the system for test iteration
+		example: --pre-iteration-script=$HOME/drop-cache.sh
+
+	--samples=<int>
+		number of samples to use per test iteration (default is 5)
+
+	--max-stddev=<int>
+		the maximum percent stddev allowed to pass
+
+	--max-failures=<int>
+		the maximum number of failures to get below stddev
+
+	--install
+		install only (default is False)
+
+	--histogram-interval-sec=<int>
+		set the histogram logging interval in seconds (default 10)
+
+	--sysinfo=str            str= comma separated values of sysinfo to be collected
+		available: 
+--- Finished test-17 pbench-fio (status=1)
++++ pbench tree state
+/var/tmp/pbench-test-bench/pbench-agent
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-17_1900.01.01T00.00.00
+/var/tmp/pbench-test-bench/pbench-agent/fio_test-17_1900.01.01T00.00.00/metadata.log
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log
+/var/tmp/pbench-test-bench/pbench-agent/tmp
+/var/tmp/pbench-test-bench/pbench-agent/tools-default
+/var/tmp/pbench-test-bench/pbench-agent/tools-default/mpstat
+/var/tmp/pbench-test-bench/pbench-agent/tools-default/sar
+--- pbench tree state
++++ pbench.log file contents
+/var/tmp/pbench-test-bench/pbench-agent/pbench.log:[error][1900-01-01T00:00:00.000000] Unsupported --job-mode=bad encountered
+--- pbench.log file contents
++++ test-execution.log file contents
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --options
+/var/tmp/pbench-test-bench/test-execution.log:/var/tmp/pbench-test-bench/opt/pbench-agent/unittest-scripts/pbench-collect-sysinfo --sysinfo=default --check
+--- test-execution.log file contents
++++ fio_test-17_1900.01.01T00.00.00/metadata.log file contents
+[pbench]
+--- fio_test-17_1900.01.01T00.00.00/metadata.log file contents