projects / platform / upstream / dldt.git / commitdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | patch | inline | side by side (parent: 85dab59)
Shellcheck tests for bash scripts (#2532)
authorIlya Lavrenov <ilya.lavrenov@intel.com>
Mon, 5 Oct 2020 20:37:50 +0000 (23:37 +0300)
committerGitHub <noreply@github.com>
Mon, 5 Oct 2020 20:37:50 +0000 (23:37 +0300)
* Added bash scripts checks with shellcheck

* Fixed indentations

* Fixed setupvars.sh with shellcheck

* Fixed MO scripts

* Compilation with newer versions of shellcheck

CMakeLists.txt patch | blob | history
cmake/coverage/coverage_clean.cmake patch | blob | history
cmake/developer_package.cmake patch | blob | history
cmake/shellcheck/shellcheck.cmake [new file with mode: 0644] patch | blob
cmake/shellcheck/shellcheck_process.cmake [new file with mode: 0644] patch | blob
inference-engine/scripts/dependencies.sh patch | blob | history
install_dependencies.sh patch | blob | history
model-optimizer/install_prerequisites/install_prerequisites.sh patch | blob | history
scripts/setupvars/setupvars.sh patch | blob | history

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 30d0749..9035509 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -22,8 +22,7 @@ include(features)
 # include developer package
 include(developer_package)
 
-# These options are shared with 3rdparty plugins
-# by means of developer package
+# These options are shared with 3rdparty plugins by means of developer package
 include(check_features)
 include(dependencies)
 
@@ -40,6 +39,10 @@ message (STATUS "CMAKE_BUILD_TYPE ...................... " ${CMAKE_BUILD_TYPE})
 file(REMOVE "${CMAKE_BINARY_DIR}/targets_developer.cmake")
 file(REMOVE "${CMAKE_BINARY_DIR}/targets.cmake")
 
+#
+# Build
+#
+
 function(build_ngraph)
     function(ngraph_set option value)
         if(NOT DEFINED ${option})
@@ -130,7 +133,6 @@ function(openvino_developer_export_targets)
         "Paths to extra Inference Engine plugins" FORCE)
 endfunction()
 
-
 add_subdirectory(openvino)
 
 build_ngraph()
@@ -139,7 +141,26 @@ add_subdirectory(inference-engine)
 
 add_subdirectory(docs)
 
+#
+# Shellcheck
+#
+
+ie_shellcheck_process(DIRECTORY "${OpenVINO_MAIN_SOURCE_DIR}"
+                      SKIP "${OpenVINO_MAIN_SOURCE_DIR}/bin"
+                           "${OpenVINO_MAIN_SOURCE_DIR}/build"
+                           "${IE_MAIN_SOURCE_DIR}/tests/ie_test_utils/common_test_utils/gtest"
+                           "${IE_MAIN_SOURCE_DIR}/samples/thirdparty"
+                           "${IE_MAIN_SOURCE_DIR}/thirdparty"
+                           "${IE_MAIN_SOURCE_DIR}/temp"
+                           # TODO fix and enable back:
+                           "${OpenVINO_MAIN_SOURCE_DIR}/scripts/install_dependencies"
+                           "${OpenVINO_MAIN_SOURCE_DIR}/scripts/demo"
+                           "${OpenVINO_MAIN_SOURCE_DIR}/ngraph"
+                           "${IE_MAIN_SOURCE_DIR}/scripts")
+
+#
 # cpack
+#
 
 # install setupvars
 
diff --git a/cmake/coverage/coverage_clean.cmake b/cmake/coverage/coverage_clean.cmake
index 137801d..94c1727 100644 (file)
--- a/cmake/coverage/coverage_clean.cmake
+++ b/cmake/coverage/coverage_clean.cmake
@@ -3,28 +3,28 @@
 #
 
 if(NOT DEFINED IE_COVERAGE_REPORTS)
-       message(FATAL_ERROR "IE_COVERAGE_REPORTS variable is not defined")
-       return()
+    message(FATAL_ERROR "IE_COVERAGE_REPORTS variable is not defined")
+    return()
 endif()
 
 file(REMOVE_RECURSE "${IE_COVERAGE_REPORTS}")
 
 if(NOT DEFINED IE_COVERAGE_DIRECTORY)
-       message(FATAL_ERROR "IE_COVERAGE_DIRECTORY variable is not defined")
-       return()
+    message(FATAL_ERROR "IE_COVERAGE_DIRECTORY variable is not defined")
+    return()
 endif()
 
 # remove .gcno files which are kept from the previous build
 
 file(GLOB_RECURSE gcno_files "${IE_COVERAGE_DIRECTORY}/*.gcno")
 foreach(file IN LISTS gcno_files)
-       string(REPLACE ".gcno" "" temp_file "${file}")
-       string(REGEX REPLACE "CMakeFiles/.+dir/" "" temp_file "${temp_file}")
-       string(REPLACE "${CMAKE_BINARY_DIRECTORY}" "${CMAKE_SOURCE_DIRECTORY}" source_file "${temp_file}")
+    string(REPLACE ".gcno" "" temp_file "${file}")
+    string(REGEX REPLACE "CMakeFiles/.+dir/" "" temp_file "${temp_file}")
+    string(REPLACE "${CMAKE_BINARY_DIRECTORY}" "${CMAKE_SOURCE_DIRECTORY}" source_file "${temp_file}")
 
-       if(NOT EXISTS "${source_file}")
-               file(REMOVE "${file}")
-               string(REPLACE "${CMAKE_BINARY_DIRECTORY}/" "" file "${file}")
-               message("Removing ${file}")
-       endif()
+    if(NOT EXISTS "${source_file}")
+        file(REMOVE "${file}")
+        string(REPLACE "${CMAKE_BINARY_DIRECTORY}/" "" file "${file}")
+        message("Removing ${file}")
+    endif()
 endforeach()
diff --git a/cmake/developer_package.cmake b/cmake/developer_package.cmake
index 60b6b6b..0c6a857 100644 (file)
--- a/cmake/developer_package.cmake
+++ b/cmake/developer_package.cmake
@@ -108,6 +108,7 @@ function(set_temp_directory temp_variable source_tree_dir)
 endfunction()
 
 include(coverage/coverage)
+include(shellcheck/shellcheck)
 
 # External dependencies
 find_package(Threads)
@@ -129,13 +130,13 @@ elseif(X86)
 endif()
 
 if(OS_FOLDER)
-       message ("**** OS FOLDER IS: [${OS_FOLDER}]")
-       if("${OS_FOLDER}" STREQUAL "ON")
-               message ("**** USING OS FOLDER: [${CMAKE_SYSTEM_NAME}]")
-               set(BIN_FOLDER "bin/${CMAKE_SYSTEM_NAME}/${ARCH_FOLDER}")
-       else()
-               set(BIN_FOLDER "bin/${OS_FOLDER}/${ARCH_FOLDER}")
-       endif()
+    message ("**** OS FOLDER IS: [${OS_FOLDER}]")
+    if("${OS_FOLDER}" STREQUAL "ON")
+        message ("**** USING OS FOLDER: [${CMAKE_SYSTEM_NAME}]")
+        set(BIN_FOLDER "bin/${CMAKE_SYSTEM_NAME}/${ARCH_FOLDER}")
+    else()
+        set(BIN_FOLDER "bin/${OS_FOLDER}/${ARCH_FOLDER}")
+    endif()
 else()
     set(BIN_FOLDER "bin/${ARCH_FOLDER}")
 endif()
diff --git a/cmake/shellcheck/shellcheck.cmake b/cmake/shellcheck/shellcheck.cmake
new file mode 100644 (file)
index 0000000..8ee7259
--- /dev/null
+++ b/cmake/shellcheck/shellcheck.cmake
@@ -0,0 +1,49 @@
+# Copyright (C) 2018-2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+include(CMakeParseArguments)
+
+find_host_program(shellcheck_PROGRAM NAMES shellcheck DOC "Path to shellcheck tool")
+
+function(ie_shellcheck_process)
+    if(NOT shellcheck_PROGRAM)
+        message(WARNING "shellcheck tool is not found")
+        return()
+    endif()
+
+    cmake_parse_arguments(IE_SHELLCHECK "" "DIRECTORY" "SKIP" ${ARGN})
+
+    set(IE_SHELLCHECK_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/cmake/shellcheck/shellcheck_process.cmake")
+    file(GLOB_RECURSE scripts "${IE_SHELLCHECK_DIRECTORY}/*.sh")
+    foreach(script IN LISTS scripts)
+        # check if we need to skip scripts
+        unset(skip_script)
+        foreach(skip_directory IN LISTS IE_SHELLCHECK_SKIP)
+            if(script MATCHES "${skip_directory}/*")
+                set(skip_script ON)
+            endif()
+        endforeach()
+        if(skip_script)
+            continue()
+        endif()
+
+        get_filename_component(dir_name "${script}" DIRECTORY)
+        string(REPLACE "${IE_SHELLCHECK_DIRECTORY}" "${CMAKE_BINARY_DIR}/shellcheck" output_file ${script})
+        set(output_file "${output_file}.txt")
+        get_filename_component(script_name "${script}" NAME)
+
+        add_custom_command(OUTPUT ${output_file} 
+                           COMMAND ${CMAKE_COMMAND}
+                             -D IE_SHELLCHECK_PROGRAM=${shellcheck_PROGRAM}
+                             -D IE_SHELL_SCRIPT=${script}
+                             -D IE_SHELLCHECK_OUTPUT=${output_file}
+                             -P ${IE_SHELLCHECK_SCRIPT}
+                           DEPENDS ${script}
+                           COMMENT "Check script ${script_name}"
+                           VERBATIM)
+        list(APPEND outputs ${output_file})
+    endforeach()
+
+    add_custom_target(ie_shellcheck DEPENDS ${outputs})
+endfunction()
diff --git a/cmake/shellcheck/shellcheck_process.cmake b/cmake/shellcheck/shellcheck_process.cmake
new file mode 100644 (file)
index 0000000..d483ff6
--- /dev/null
+++ b/cmake/shellcheck/shellcheck_process.cmake
@@ -0,0 +1,27 @@
+# Copyright (C) 2018-2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+if(NOT DEFINED IE_SHELLCHECK_PROGRAM)
+    message(FATAL_ERROR "IE_SHELLCHECK_PROGRAM is not defined")
+endif()
+
+if(NOT DEFINED IE_SHELL_SCRIPT)
+    message(FATAL_ERROR "IE_SHELL_SCRIPT is not defined")
+endif()
+
+if(NOT DEFINED IE_SHELLCHECK_OUTPUT)
+    message(FATAL_ERROR "IE_SHELLCHECK_OUTPUT is not defined")
+endif()
+
+set(rules "SC1091,SC2236,SC2164,SC2086,SC2162,SC1090")
+execute_process(COMMAND ${IE_SHELLCHECK_PROGRAM} --exclude=${rules} ${IE_SHELL_SCRIPT}
+                OUTPUT_VARIABLE error_message
+                RESULT_VARIABLE exit_code
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+file(WRITE "${IE_SHELLCHECK_OUTPUT}" "${error_message}")
+
+if(NOT exit_code EQUAL 0)
+    message(FATAL_ERROR "${error_message}")
+endif()
diff --git a/inference-engine/scripts/dependencies.sh b/inference-engine/scripts/dependencies.sh
index b71f188..5f15c83 100755 (executable)
--- a/inference-engine/scripts/dependencies.sh
+++ b/inference-engine/scripts/dependencies.sh
@@ -3,38 +3,38 @@
 # SPDX-License-Identifier: Apache-2.0
 
 if [ "$1" = "" ]; then
-               dep_arch=64
-       else
-               dep_arch=$1
+        dep_arch=64
+    else
+        dep_arch=$1
 fi
 
 item_path=""
 add_path() {
-       component=$1
-       item_path=""
+    component=$1
+    item_path=""
     echo "Read file: dependencies_${dep_arch}.txt"
     grep_component="\b${component}\b"
 
-       if [[ $(grep -m 1 "$grep_component" "dependencies_${dep_arch}.txt") ]];then
-               archive_path=$(grep -m 1 "$grep_component" "dependencies_${dep_arch}.txt" | sed -E "s/${component}=//g")
-               library_rpath=$(grep -m 1 "$grep_component" "ld_library_rpath_${dep_arch}.txt" | sed -E "s/${component}=//g")
-               filename=$(basename "$archive_path")
-               if [[ (! -d "$DL_SDK_TEMP/test_dependencies/$component/$filename") ||
-                           (-d "$DL_SDK_TEMP/test_dependencies/$component/$filename"  &&
-                               ! $(ls -A "$DL_SDK_TEMP/test_dependencies/$component/$filename")) ]]; then
-                       mkdir -p "$DL_SDK_TEMP/test_dependencies/$component/$filename"
-                       wget -q "$archive_path" -O "$DL_SDK_TEMP/test_dependencies/$filename"
-                       if [[ $filename == *.zip ]]; then
-                               unzip "$DL_SDK_TEMP/test_dependencies/$filename" -d "$DL_SDK_TEMP/test_dependencies/$component/$filename"
-                       elif [[ $filename == *.7z ]]; then
-                           7za x -y "$DL_SDK_TEMP/test_dependencies/$filename" -o "$DL_SDK_TEMP/test_dependencies/$component/$filename"
-                       else
-                               tar xf "$DL_SDK_TEMP/test_dependencies/$filename" -C "$DL_SDK_TEMP/test_dependencies/$component/$filename"
-                       fi
-                       rm "$DL_SDK_TEMP/test_dependencies/$filename"
-               fi
-               item_path=$component/$filename/$library_rpath
-       fi
+    if [[ $(grep -m 1 "$grep_component" "dependencies_${dep_arch}.txt") ]];then
+        archive_path=$(grep -m 1 "$grep_component" "dependencies_${dep_arch}.txt" | sed -E "s/${component}=//g")
+        library_rpath=$(grep -m 1 "$grep_component" "ld_library_rpath_${dep_arch}.txt" | sed -E "s/${component}=//g")
+        filename=$(basename "$archive_path")
+        if [[ (! -d "$DL_SDK_TEMP/test_dependencies/$component/$filename") ||
+                (-d "$DL_SDK_TEMP/test_dependencies/$component/$filename"  &&
+                    ! $(ls -A "$DL_SDK_TEMP/test_dependencies/$component/$filename")) ]]; then
+            mkdir -p "$DL_SDK_TEMP/test_dependencies/$component/$filename"
+            wget -q "$archive_path" -O "$DL_SDK_TEMP/test_dependencies/$filename"
+            if [[ $filename == *.zip ]]; then
+                unzip "$DL_SDK_TEMP/test_dependencies/$filename" -d "$DL_SDK_TEMP/test_dependencies/$component/$filename"
+            elif [[ $filename == *.7z ]]; then
+                7za x -y "$DL_SDK_TEMP/test_dependencies/$filename" -o "$DL_SDK_TEMP/test_dependencies/$component/$filename"
+            else
+                tar xf "$DL_SDK_TEMP/test_dependencies/$filename" -C "$DL_SDK_TEMP/test_dependencies/$component/$filename"
+            fi
+            rm "$DL_SDK_TEMP/test_dependencies/$filename"
+        fi
+        item_path=$component/$filename/$library_rpath
+    fi
 }
 
 runtimes=(MKL CLDNN MYRIAD GNA DLIA OPENCV VPU_FIRMWARE_USB-MA2450 VPU_FIRMWARE_USB-MA2X8X HDDL OMP TBB AOCL_RTE LIBUSB)
diff --git a/install_dependencies.sh b/install_dependencies.sh
index ca31972..8e9ea32 100755 (executable)
--- a/install_dependencies.sh
+++ b/install_dependencies.sh
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-params=$@
+params=$1
 
 yes_or_no() {
     if [ "$params" == "-y" ]; then
@@ -23,7 +23,7 @@ yes_or_no() {
 if [ -f /etc/lsb-release ]; then
     # Ubuntu
     host_cpu=$(uname -m)
-    if [ $host_cpu = x86_64 ]; then
+    if [ "$host_cpu" = "x86_64" ]; then
         x86_64_specific_packages="gcc-multilib g++-multilib"
     else
         x86_64_specific_packages=""
diff --git a/model-optimizer/install_prerequisites/install_prerequisites.sh b/model-optimizer/install_prerequisites/install_prerequisites.sh
index c7db058..74a9445 100755 (executable)
--- a/model-optimizer/install_prerequisites/install_prerequisites.sh
+++ b/model-optimizer/install_prerequisites/install_prerequisites.sh
@@ -31,14 +31,14 @@ V_ENV=0
 for ((i=1;i <= $#;i++)) {
     case "${!i}" in
         caffe|tf|tf2|mxnet|kaldi|onnx)
-            postfix="_"$1""
+            postfix="_$1"
             ;;
         "venv")
             V_ENV=1
             ;;
         *)
             if [[ "$1" != "" ]]; then
-                echo "\""${!i}"\" is unsupported parameter"
+                echo "\"${!i}\" is unsupported parameter"
                 echo $"Usage: $0 {caffe|tf|tf2|mxnet|kaldi|onnx} {venv}"
                 exit 1
             fi
@@ -80,18 +80,18 @@ fi
 
 
 if [[ $V_ENV -eq 1 ]]; then
-    $python_binary -m venv $SCRIPTDIR/../venv
-    source $SCRIPTDIR/../venv/bin/activate
-    $SCRIPTDIR/../venv/bin/$python_binary -m pip install -r $SCRIPTDIR/../requirements${postfix}.txt
+    "$python_binary" -m venv "$SCRIPTDIR/../venv"
+    source "$SCRIPTDIR/../venv/bin/activate"
+    "$SCRIPTDIR/../venv/bin/$python_binary" -m pip install -r "$SCRIPTDIR/../requirements${postfix}.txt"
     echo
     echo "Before running the Model Optimizer, please activate virtualenv environment by running \"source ${SCRIPTDIR}/../venv/bin/activate\""
 else
     if [[ "$OSTYPE" == "darwin"* ]]; then
-        python3 -m pip install -r $SCRIPTDIR/../requirements${postfix}.txt
+        python3 -m pip install -r "$SCRIPTDIR/../requirements${postfix}.txt"
     else
-        sudo -E $python_binary -m pip install -r $SCRIPTDIR/../requirements${postfix}.txt
+        sudo -E $python_binary -m pip install -r "$SCRIPTDIR/../requirements${postfix}.txt"
     fi
-    echo [WARNING] All Model Optimizer dependencies are installed globally.
-    echo [WARNING] If you want to keep Model Optimizer in separate sandbox
-    echo [WARNING] run install_prerequisites.sh venv "{caffe|tf|tf2|mxnet|kaldi|onnx}"
+    echo "[WARNING] All Model Optimizer dependencies are installed globally."
+    echo "[WARNING] If you want to keep Model Optimizer in separate sandbox"
+    echo "[WARNING] run install_prerequisites.sh venv \"{caffe|tf|tf2|mxnet|kaldi|onnx}\""
 fi
diff --git a/scripts/setupvars/setupvars.sh b/scripts/setupvars/setupvars.sh
index ae5990b..ec444b5 100755 (executable)
--- a/scripts/setupvars/setupvars.sh
+++ b/scripts/setupvars/setupvars.sh
@@ -40,9 +40,9 @@ esac
 shift
 done
 
-if [ -e $INSTALLDIR/deployment_tools/inference_engine ]; then
+if [ -e "$INSTALLDIR/deployment_tools/inference_engine" ]; then
     export InferenceEngine_DIR=$INTEL_OPENVINO_DIR/deployment_tools/inference_engine/share
-    system_type=$(\ls $INTEL_OPENVINO_DIR/deployment_tools/inference_engine/lib/)
+    system_type=$(ls "$INTEL_OPENVINO_DIR/deployment_tools/inference_engine/lib/")
     IE_PLUGINS_PATH=$INTEL_OPENVINO_DIR/deployment_tools/inference_engine/lib/$system_type
 
     if [[ -e ${IE_PLUGINS_PATH}/arch_descriptions ]]; then
@@ -59,12 +59,12 @@ if [ -e $INSTALLDIR/deployment_tools/inference_engine ]; then
 
     HDDL_UNITE_DIR=$INSTALLDIR/deployment_tools/inference_engine/external/hddl_unite
 
-    if [ -e $HDDL_UNITE_DIR ]; then
+    if [ -e "$HDDL_UNITE_DIR" ]; then
         export LD_LIBRARY_PATH=$HDDL_UNITE_DIR/lib:$HDDL_UNITE_DIR/thirdparty/XLink/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
     fi
 fi
 
-if [ -e $INSTALLDIR/deployment_tools/ngraph ]; then
+if [ -e "$INSTALLDIR/deployment_tools/ngraph" ]; then
     export LD_LIBRARY_PATH=$INSTALLDIR/deployment_tools/ngraph/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
     export ngraph_DIR=$INSTALLDIR/deployment_tools/ngraph/cmake
 fi
@@ -88,11 +88,11 @@ export PATH="$INTEL_OPENVINO_DIR/deployment_tools/model_optimizer${PATH:+:$PATH}
 export PYTHONPATH="$INTEL_OPENVINO_DIR/deployment_tools/model_optimizer${PYTHONPATH:+:$PYTHONPATH}"
 
 
-if [ -e $INTEL_OPENVINO_DIR/deployment_tools/open_model_zoo/tools/accuracy_checker ]; then
+if [ -e "$INTEL_OPENVINO_DIR/deployment_tools/open_model_zoo/tools/accuracy_checker" ]; then
     export PYTHONPATH="$INTEL_OPENVINO_DIR/deployment_tools/open_model_zoo/tools/accuracy_checker:$PYTHONPATH"
 fi
 
-if [ -e $INTEL_OPENVINO_DIR/deployment_tools/tools/post_training_optimization_toolkit ]; then
+if [ -e "$INTEL_OPENVINO_DIR/deployment_tools/tools/post_training_optimization_toolkit" ]; then
     export PYTHONPATH="$INTEL_OPENVINO_DIR/deployment_tools/tools/post_training_optimization_toolkit:$PYTHONPATH"
 fi
 
@@ -107,7 +107,7 @@ fi
 
 python_bitness=$(python3 -c 'import sys; print(64 if sys.maxsize > 2**32 else 32)')   
 if [ "$python_bitness" != "" ] && [ "$python_bitness" != "64" ] && [ "$OS_NAME" != "Raspbian" ]; then
-    echo "[setupvars.sh] 64 bitness for Python" $python_version "is requred"
+    echo "[setupvars.sh] 64 bitness for Python $python_version is required"
 fi
 
 MINIMUM_REQUIRED_PYTHON_VERSION="3.6"
Domain: Machine Learning / ML Framework;