From 4b3018feaac4d029dcfac945ca2306f6cdc561d1 Mon Sep 17 00:00:00 2001 From: Jaeyun Jung Date: Wed, 22 Nov 2023 20:12:04 +0900 Subject: [PATCH] [Test] tensorflow-lite version in testcase Fix version of tensorflow-lite. Both version 2.x and 1.x can be used in testcase if tensor-filter property framework is tensorflow-lite. Default version is v2.x, and it has higher priority. Also, added minor patch to fix indent. Signed-off-by: Jaeyun Jung --- tests/meson.build | 2 +- .../nnstreamer_filter_tensorflow2_lite/runTest.sh | 6 ++-- tests/nnstreamer_filter_tensorflow_lite/runTest.sh | 33 +++++++++++----------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/tests/meson.build b/tests/meson.build index e40cbc8..e3492a2 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -391,7 +391,7 @@ if get_option('install-test') install_subdir('nnstreamer_decoder_pose', install_dir: unittest_install_dir) install_subdir('nnstreamer_demux', install_dir: unittest_install_dir) install_subdir('nnstreamer_filter_custom', install_dir: unittest_install_dir) - if tflite_support_is_available + if tflite_support_is_available or tflite2_support_is_available install_subdir('nnstreamer_filter_tensorflow_lite', install_dir: unittest_install_dir) endif if tflite2_support_is_available diff --git a/tests/nnstreamer_filter_tensorflow2_lite/runTest.sh b/tests/nnstreamer_filter_tensorflow2_lite/runTest.sh index 2cd25ef..d696885 100644 --- a/tests/nnstreamer_filter_tensorflow2_lite/runTest.sh +++ b/tests/nnstreamer_filter_tensorflow2_lite/runTest.sh @@ -206,15 +206,15 @@ cat info | grep "accl = ${auto_accl}$" testResult $? 2-17 "accelerator set test" 0 1 # Dimension declaration test cases -gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MODEL} ! \"other/tensors,num_tensors=1,dimensions=1001:1:1\" !filesink location=tensorfilter.out.log" 3 0 0 $PERFORMANCE +gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MODEL} ! \"other/tensors,num_tensors=1,dimensions=1001:1:1\" ! filesink location=tensorfilter.out.log" 3 0 0 $PERFORMANCE python3 checkLabel.py tensorfilter.out.log ${PATH_TO_LABEL} orange testResult $? 3 "Golden test comparison" 0 1 -gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MODEL} ! \"other/tensors,num_tensors=1,dimensions=(string)1001:1\" !filesink location=tensorfilter.out.log" 4 0 0 $PERFORMANCE +gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MODEL} ! \"other/tensors,num_tensors=1,dimensions=(string)1001:1\" ! filesink location=tensorfilter.out.log" 4 0 0 $PERFORMANCE python3 checkLabel.py tensorfilter.out.log ${PATH_TO_LABEL} orange testResult $? 4 "Golden test comparison" 0 1 -gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MODEL} ! \"other/tensors,num_tensors=1,dimensions=(string)1001\" !filesink location=tensorfilter.out.log" 5 0 0 $PERFORMANCE +gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MODEL} ! \"other/tensors,num_tensors=1,dimensions=(string)1001\" ! filesink location=tensorfilter.out.log" 5 0 0 $PERFORMANCE python3 checkLabel.py tensorfilter.out.log ${PATH_TO_LABEL} orange testResult $? 5 "Golden test comparison" 0 1 diff --git a/tests/nnstreamer_filter_tensorflow_lite/runTest.sh b/tests/nnstreamer_filter_tensorflow_lite/runTest.sh index 8e1a4cf..cff9fa6 100644 --- a/tests/nnstreamer_filter_tensorflow_lite/runTest.sh +++ b/tests/nnstreamer_filter_tensorflow_lite/runTest.sh @@ -19,14 +19,15 @@ fi testInit $1 # NNStreamer and plugins path for test +# These testcases set tensor-filter property 'framework=tensorflow-lite', it means tensor-filter finds tensorflow-lite library which version is ver 2.x or 1.x. (ver2.x has higher priority.) PATH_TO_PLUGIN="../../build" if [[ -d $PATH_TO_PLUGIN ]]; then ini_path="${PATH_TO_PLUGIN}/ext/nnstreamer/tensor_filter" if [[ -d ${ini_path} ]]; then - check=$(ls ${ini_path} | grep tensorflow2-lite.so) + check=$(ls ${ini_path} | grep [tensorflow,tensorflow2]-lite.so) if [[ ! $check ]]; then - echo "Cannot find tensorflow2-lite shared lib" + echo "Cannot find tensorflow-lite shared lib" report exit fi @@ -47,9 +48,9 @@ else fi if [[ -d ${value} ]]; then - check=$(ls ${value} | grep tensorflow2-lite.so) + check=$(ls ${value} | grep [tensorflow,tensorflow2]-lite.so) if [[ ! $check ]]; then - echo "Cannot find tensorflow2-lite shared lib" + echo "Cannot find tensorflow-lite shared lib" report exit fi @@ -69,35 +70,35 @@ PATH_TO_MODEL="../test_models/models/mobilenet_v1_1.0_224_quant.tflite" PATH_TO_LABEL="../test_models/labels/labels.txt" PATH_TO_IMAGE="../test_models/data/orange.png" -gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MODEL} ! filesink location=tensorfilter.out.log" 1 0 0 $PERFORMANCE +gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MODEL} ! filesink location=tensorfilter.out.log" 1 0 0 $PERFORMANCE python3 checkLabel.py tensorfilter.out.log ${PATH_TO_LABEL} orange testResult $? 1 "Golden test comparison" 0 1 # Fail test for invalid input properties -gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MODEL} input=7:1 inputtype=float32 ! filesink location=tensorfilter.out.log" 2F_n 0 1 $PERFORMANCE +gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MODEL} input=7:1 inputtype=float32 ! filesink location=tensorfilter.out.log" 2F_n 0 1 $PERFORMANCE # Fail test for invalid output properties -gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MODEL} output=1:7 outputtype=int8 ! filesink location=tensorfilter.out.log" 3F_n 0 1 $PERFORMANCE +gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MODEL} output=1:7 outputtype=int8 ! filesink location=tensorfilter.out.log" 3F_n 0 1 $PERFORMANCE PATH_TO_MULTI_TENSOR_OUTPUT_MODEL="../test_models/models/multi_person_mobilenet_v1_075_float.tflite" # Simple tests for multi-tensor output model # This should emit error because of invalid width and height size -gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num_buffers=4 ! videoconvert ! videoscale ! video/x-raw,format=RGB,width=353,height=257 ! tensor_converter ! tensor_transform mode=arithmetic option=typecast:float32,add:-127.5,div:127.5 ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MULTI_TENSOR_OUTPUT_MODEL} ! fakesink" 4_n 0 1 $PERFORMANCE +gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num_buffers=4 ! videoconvert ! videoscale ! video/x-raw,format=RGB,width=353,height=257 ! tensor_converter ! tensor_transform mode=arithmetic option=typecast:float32,add:-127.5,div:127.5 ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MULTI_TENSOR_OUTPUT_MODEL} ! fakesink" 4_n 0 1 $PERFORMANCE # This won't fail, but not much meaningful -gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num_buffers=4 ! videoconvert ! videoscale ! video/x-raw,format=RGB,width=257,height=353 ! tensor_converter ! tensor_transform mode=arithmetic option=typecast:float32,add:-127.5,div:127.5 ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MULTI_TENSOR_OUTPUT_MODEL} ! fakesink" 5 0 0 $PERFORMANCE +gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num_buffers=4 ! videoconvert ! videoscale ! video/x-raw,format=RGB,width=257,height=353 ! tensor_converter ! tensor_transform mode=arithmetic option=typecast:float32,add:-127.5,div:127.5 ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MULTI_TENSOR_OUTPUT_MODEL} ! fakesink" 5 0 0 $PERFORMANCE # Input and output combination test -gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc pattern=13 num-buffers=1 ! videoconvert ! video/x-raw,width=640,height=480,framerate=30/1 ! tensor_converter ! tee name=t t. ! queue ! mux.sink_0 t. ! queue ! filesink location=combi.dummy.golden buffer-mode=unbuffered sync=false async=false filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! mux.sink_1 tensor_mux name=mux ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MODEL} input-combination=1 output-combination=i0,o0 ! tensor_demux name=demux demux.src_0 ! filesink location=tensorfilter.combi.in.log buffer-mode=unbuffered sync=false async=false demux.src_1 ! filesink location=tensorfilter.combi.out.log buffer-mode=unbuffered sync=false async=false" 7 0 0 $PERFORMANCE +gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc pattern=13 num-buffers=1 ! videoconvert ! video/x-raw,width=640,height=480,framerate=30/1 ! tensor_converter ! tee name=t t. ! queue ! mux.sink_0 t. ! queue ! filesink location=combi.dummy.golden buffer-mode=unbuffered sync=false async=false filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! mux.sink_1 tensor_mux name=mux ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MODEL} input-combination=1 output-combination=i0,o0 ! tensor_demux name=demux demux.src_0 ! filesink location=tensorfilter.combi.in.log buffer-mode=unbuffered sync=false async=false demux.src_1 ! filesink location=tensorfilter.combi.out.log buffer-mode=unbuffered sync=false async=false" 7 0 0 $PERFORMANCE callCompareTest combi.dummy.golden tensorfilter.combi.in.log 7_0 "Output Combination Golden Test 7-0" 1 0 python3 checkLabel.py tensorfilter.combi.out.log ${PATH_TO_LABEL} orange testResult $? 1 "Golden test comparison" 0 1 -# Test the backend setting done with tensorflow2-lite +# Test the backend setting done with tensorflow-lite # This also performs tests for generic backend configuration parsing function run_pipeline() { - gst-launch-1.0 --gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MODEL} accelerator=$1 ! filesink location=tensorfilter.out.log 2>info + gst-launch-1.0 --gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MODEL} accelerator=$1 ! filesink location=tensorfilter.out.log 2>info } arch=$(uname -m) @@ -195,20 +196,20 @@ cat info | grep "accl = ${auto_accl}$" testResult $? 2-17 "accelerator set test" 0 1 # Property reading test for accelerator before setting the framework (analogous test is 2-3) -gst-launch-1.0 --gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter accelerator=true:!npu,gpu framework=tensorflow2-lite model=${PATH_TO_MODEL} ! filesink location=tensorfilter.out.log 2>info +gst-launch-1.0 --gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter accelerator=true:!npu,gpu framework=tensorflow-lite model=${PATH_TO_MODEL} ! filesink location=tensorfilter.out.log 2>info cat info | grep "accl = gpu$" testResult $? 2-18 "accelerator set test" 0 1 # Dimension declaration test cases -gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MODEL} ! \"other/tensors,num_tensors=1,dimensions=1001:1:1\" ! filesink location=tensorfilter.out.log" 3 0 0 $PERFORMANCE +gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MODEL} ! \"other/tensors,num_tensors=1,dimensions=1001:1:1\" ! filesink location=tensorfilter.out.log" 3 0 0 $PERFORMANCE python3 checkLabel.py tensorfilter.out.log ${PATH_TO_LABEL} orange testResult $? 3 "Golden test comparison" 0 1 -gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MODEL} ! \"other/tensors,num_tensors=1,dimensions=(string)1001:1\" ! filesink location=tensorfilter.out.log" 4 0 0 $PERFORMANCE +gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MODEL} ! \"other/tensors,num_tensors=1,dimensions=(string)1001:1\" ! filesink location=tensorfilter.out.log" 4 0 0 $PERFORMANCE python3 checkLabel.py tensorfilter.out.log ${PATH_TO_LABEL} orange testResult $? 4 "Golden test comparison" 0 1 -gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow2-lite model=${PATH_TO_MODEL} ! \"other/tensors,num_tensors=1,dimensions=(string)1001\" ! filesink location=tensorfilter.out.log" 5 0 0 $PERFORMANCE +gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MODEL} ! \"other/tensors,num_tensors=1,dimensions=(string)1001\" ! filesink location=tensorfilter.out.log" 5 0 0 $PERFORMANCE python3 checkLabel.py tensorfilter.out.log ${PATH_TO_LABEL} orange testResult $? 5 "Golden test comparison" 0 1 -- 2.7.4