compiler/common-artifacts/CMakeLists.txt

   1 #[[ Generate common python virtual enviornment ]]
   2 find_package(PythonInterp 3 QUIET)
   3 find_package(PythonLibs 3 QUIET)
   4
   5 if(NOT ${PYTHONINTERP_FOUND})
   6   message(STATUS "Build common-artifacts: FALSE (Python3 is missing)")
   7   return()
   8 endif()
   9
  10 if(${PYTHON_VERSION_MINOR} LESS 3)
  11   message(STATUS "Build common-artifacts: FALSE (You need to install Python version higher than 3.3)")
  12   return()
  13 endif()
  14
  15 # Create python virtual environment with tensorflow 1.13.2
  16 set(VIRTUALENV_OVERLAY_TF_1_13_2 "${NNCC_OVERLAY_DIR}/venv_1_13_2")
  17
  18 # Create python virtual environment with tensorflow 2.3.0
  19 set(VIRTUALENV_OVERLAY_TF_2_3_0 "${NNCC_OVERLAY_DIR}/venv_2_3_0")
  20
  21 add_custom_command(
  22   OUTPUT ${VIRTUALENV_OVERLAY_TF_1_13_2}
  23   COMMAND ${PYTHON_EXECUTABLE} -m venv ${VIRTUALENV_OVERLAY_TF_1_13_2}
  24 )
  25
  26 add_custom_command(
  27   OUTPUT ${VIRTUALENV_OVERLAY_TF_2_3_0}
  28   COMMAND ${PYTHON_EXECUTABLE} -m venv ${VIRTUALENV_OVERLAY_TF_2_3_0}
  29 )
  30
  31 # Create requirements.txt and install required pip packages
  32 set(REQUIREMENTS_FILE "requirements.txt")
  33 set(REQUIREMENTS_OVERLAY_PATH_TF_1_13_2 "${VIRTUALENV_OVERLAY_TF_1_13_2}/${REQUIREMENTS_FILE}")
  34 set(REQUIREMENTS_OVERLAY_PATH_TF_2_3_0 "${VIRTUALENV_OVERLAY_TF_2_3_0}/${REQUIREMENTS_FILE}")
  35
  36 add_custom_command(
  37   OUTPUT ${REQUIREMENTS_OVERLAY_PATH_TF_1_13_2}
  38   COMMAND ${CMAKE_COMMAND} -E echo "tensorflow==1.13.2" > ${REQUIREMENTS_OVERLAY_PATH_TF_1_13_2}
  39   COMMAND ${VIRTUALENV_OVERLAY_TF_1_13_2}/bin/python -m pip --default-timeout=1000 install --upgrade pip setuptools
  40   COMMAND ${VIRTUALENV_OVERLAY_TF_1_13_2}/bin/python -m pip --default-timeout=1000 install -r ${REQUIREMENTS_OVERLAY_PATH_TF_1_13_2} --upgrade
  41   DEPENDS ${VIRTUALENV_OVERLAY_TF_1_13_2}
  42 )
  43
  44 add_custom_command(
  45   OUTPUT ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
  46   COMMAND ${CMAKE_COMMAND} -E remove -f ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
  47   COMMAND ${CMAKE_COMMAND} -E echo "tensorflow-cpu==2.3.0" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
  48   COMMAND ${CMAKE_COMMAND} -E echo "flatbuffers==1.12" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
  49   COMMAND ${VIRTUALENV_OVERLAY_TF_2_3_0}/bin/python -m pip --default-timeout=1000 install --upgrade pip setuptools
  50   COMMAND ${VIRTUALENV_OVERLAY_TF_2_3_0}/bin/python -m pip --default-timeout=1000 install -r ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0} --upgrade
  51   DEPENDS ${VIRTUALENV_OVERLAY_TF_2_3_0}
  52 )
  53
  54 add_custom_target(common_artifacts_python_deps ALL
  55   DEPENDS ${VIRTUALENV_OVERLAY_TF_1_13_2} ${VIRTUALENV_OVERLAY_TF_2_3_0} ${REQUIREMENTS_OVERLAY_PATH_TF_1_13_2} ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
  56 )
  57
  58 #[[ Generate common resources ]]
  59 # TODO add pbtxt
  60 nnas_find_package(HDF5 QUIET)
  61
  62 if(NOT HDF5_FOUND)
  63   message(STATUS "Build common-artifacts: FAILED (missing HDF5)")
  64   return()
  65 endif(NOT HDF5_FOUND)
  66
  67 set(SOURCES src/TestDataGenerator.cpp)
  68
  69 add_executable(testDataGenerator ${SOURCES})
  70 target_include_directories(testDataGenerator PRIVATE ${HDF5_INCLUDE_DIRS})
  71 target_link_libraries(testDataGenerator PRIVATE ${HDF5_CXX_LIBRARIES})
  72 target_link_libraries(testDataGenerator PRIVATE arser)
  73 target_link_libraries(testDataGenerator PRIVATE foder)
  74 target_link_libraries(testDataGenerator PRIVATE luci_import)
  75 target_link_libraries(testDataGenerator PRIVATE luci_interpreter)
  76 target_link_libraries(testDataGenerator PRIVATE mio_circle)
  77 target_link_libraries(testDataGenerator PRIVATE safemain)
  78
  79 unset(TEST_DEPS)
  80
  81 # Include recipe repo
  82 nncc_find_resource(TensorFlowLiteRecipes)
  83 nncc_find_resource(CircleRecipes)
  84 set(TFLITE_RECIPE_REPO "${TensorFlowLiteRecipes_DIR}")
  85 set(CIRCLE_RECIPE_REPO "${CircleRecipes_DIR}")
  86 set(TEST_RECIPE_FILENAME "test.recipe")
  87 set(TEST_RULE_FILENAME "test.rule")
  88
  89 set(MODEL2NNPKG "${NNAS_PROJECT_SOURCE_DIR}/tools/nnpackage_tool/model2nnpkg/model2nnpkg.sh")
  90 # Get test case list
  91 unset(RECIPES)
  92 file(GLOB TFLITE_SUBDIR RELATIVE ${TFLITE_RECIPE_REPO} ${TFLITE_RECIPE_REPO}/*)
  93 foreach(DIR IN ITEMS ${TFLITE_SUBDIR})
  94   if(IS_DIRECTORY ${TFLITE_RECIPE_REPO}/${DIR})
  95     list(APPEND RECIPES ${DIR})
  96   endif()
  97 endforeach()
  98 file(GLOB CIRCLE_SUBDIR RELATIVE ${CIRCLE_RECIPE_REPO} ${CIRCLE_RECIPE_REPO}/*)
  99 foreach(DIR IN ITEMS ${CIRCLE_SUBDIR})
 100   if(IS_DIRECTORY ${CIRCLE_RECIPE_REPO}/${DIR})
 101     list(APPEND RECIPES ${DIR})
 102   endif()
 103 endforeach()
 104
 105 macro(circlize NAME)
 106   set(NO_CIRCLIZE_${NAME} TRUE)
 107   set(NO_OPTIMIZE_${NAME} TRUE)
 108   set(NO_TCGEN_${NAME} TRUE)
 109 endmacro()
 110 macro(optimize NAME)
 111   set(NO_OPTIMIZE_${NAME} TRUE)
 112 endmacro()
 113 macro(tcgenerate NAME)
 114   set(NO_TCGEN_${NAME} TRUE)
 115 endmacro()
 116
 117 include("exclude.lst")
 118
 119 foreach(RECIPE IN ITEMS ${RECIPES})
 120   unset(OPT_FORMAT)
 121   unset(MODEL_FORMAT)
 122
 123   set(RECIPE_FILE "${RECIPE}.recipe")
 124   set(RULE_FILE "${RECIPE}.rule")
 125   set(TFLITE_RECIPE_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_RECIPE_FILENAME}")
 126   set(CIRCLE_RECIPE_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_RECIPE_FILENAME}")
 127
 128   if(NOT EXISTS "${TFLITE_RECIPE_SOURCE_PATH}")
 129     if(NOT EXISTS "${CIRCLE_RECIPE_SOURCE_PATH}")
 130       message(FATAL_ERROR "Missing recipe of '${RECIPE}' test")
 131     else()
 132       # circle recipe
 133       set(MODEL_FORMAT "circle")
 134       set(RECIPE_SOURCE_PATH ${CIRCLE_RECIPE_SOURCE_PATH})
 135     endif()
 136   else()
 137     # tflite recipe
 138     set(MODEL_FORMAT "tflite")
 139     set(RECIPE_SOURCE_PATH ${TFLITE_RECIPE_SOURCE_PATH})
 140   endif()
 141
 142   set(TFLITE_RULE_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_RULE_FILENAME}")
 143   set(CIRCLE_RULE_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_RULE_FILENAME}")
 144
 145   unset(RULE_SOURCE_PATH)
 146   if(EXISTS "${TFLITE_RULE_SOURCE_PATH}")
 147     set(RULE_SOURCE_PATH ${TFLITE_RULE_SOURCE_PATH})
 148   endif()
 149   if(EXISTS "${CIRCLE_RULE_SOURCE_PATH}")
 150     set(RULE_SOURCE_PATH ${CIRCLE_RULE_SOURCE_PATH})
 151   endif()
 152
 153   set(RECIPE_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RECIPE_FILE}")
 154   set(RULE_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RULE_FILE}")
 155
 156   set(TFLITE_FILE "${RECIPE}.tflite")
 157   set(TFLITE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TFLITE_FILE}")
 158   set(CIRCLE_FILE "${RECIPE}.circle")
 159   set(CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${CIRCLE_FILE}")
 160
 161   # Copy .recipe
 162   add_custom_command(OUTPUT ${RECIPE_BINARY_PATH}
 163     COMMAND ${CMAKE_COMMAND} -E copy "${RECIPE_SOURCE_PATH}" "${RECIPE_BINARY_PATH}"
 164     DEPENDS ${RECIPE_SOURCE_PATH}
 165     COMMENT "Generate ${RECIPE_FILE}"
 166   )
 167   list(APPEND TEST_DEPS ${RECIPE_BINARY_PATH})
 168
 169   if(DEFINED RULE_SOURCE_PATH)
 170     # Copy .rule
 171     add_custom_command(OUTPUT ${RULE_BINARY_PATH}
 172     COMMAND ${CMAKE_COMMAND} -E copy "${RULE_SOURCE_PATH}" "${RULE_BINARY_PATH}"
 173     DEPENDS ${RULE_SOURCE_PATH}
 174     COMMENT "Generate ${RULE_FILE}"
 175     )
 176     list(APPEND TEST_DEPS ${RULE_BINARY_PATH})
 177   endif()
 178
 179   if(${MODEL_FORMAT} STREQUAL "tflite")
 180     # Generate .tflite
 181     add_custom_command(OUTPUT ${TFLITE_OUTPUT_PATH}
 182       COMMAND $<TARGET_FILE:tflchef-file> ${RECIPE_BINARY_PATH} ${TFLITE_OUTPUT_PATH}
 183       DEPENDS $<TARGET_FILE:tflchef-file> ${RECIPE_BINARY_PATH}
 184       COMMENT "Generate ${TFLITE_FILE}"
 185     )
 186     list(APPEND TEST_DEPS ${TFLITE_OUTPUT_PATH})
 187
 188     if(NOT DEFINED NO_CIRCLIZE_${RECIPE})
 189     # Generate .circle
 190     add_custom_command(OUTPUT ${CIRCLE_OUTPUT_PATH}
 191       COMMAND $<TARGET_FILE:tflite2circle> ${TFLITE_OUTPUT_PATH} ${CIRCLE_OUTPUT_PATH}
 192       DEPENDS $<TARGET_FILE:tflite2circle> ${TFLITE_OUTPUT_PATH}
 193       COMMENT "Generate ${CIRCLE_FILE}"
 194     )
 195     set(MODEL_FORMAT "circle")
 196     list(APPEND TEST_DEPS ${CIRCLE_OUTPUT_PATH})
 197     endif()
 198   else()
 199     # Generate .circle
 200     add_custom_command(OUTPUT ${CIRCLE_OUTPUT_PATH}
 201     COMMAND $<TARGET_FILE:circlechef-file> ${RECIPE_BINARY_PATH} ${CIRCLE_OUTPUT_PATH}
 202     DEPENDS $<TARGET_FILE:circlechef-file> ${RECIPE_BINARY_PATH}
 203     COMMENT "Generate ${CIRCLE_FILE}"
 204     )
 205     list(APPEND TEST_DEPS ${CIRCLE_OUTPUT_PATH})
 206   endif()
 207
 208   set(OPT_CIRCLE_FILE "${RECIPE}.opt.circle")
 209   set(OPT_CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${OPT_CIRCLE_FILE}")
 210
 211   if(NOT DEFINED NO_OPTIMIZE_${RECIPE})
 212     # Generate optimized .circle
 213     add_custom_command(OUTPUT ${OPT_CIRCLE_OUTPUT_PATH}
 214       COMMAND $<TARGET_FILE:circle2circle> --all ${CIRCLE_OUTPUT_PATH} ${OPT_CIRCLE_OUTPUT_PATH}
 215       DEPENDS $<TARGET_FILE:circle2circle>  ${CIRCLE_OUTPUT_PATH}
 216       COMMENT "Generate ${OPT_CIRCLE_FILE}"
 217     )
 218     set(OPT_FORMAT ".opt")
 219     list(APPEND TEST_DEPS ${OPT_CIRCLE_OUTPUT_PATH})
 220   endif()
 221
 222   set(MODEL_FILE "${RECIPE}${OPT_FORMAT}.${MODEL_FORMAT}")
 223   set(MODEL_PATH "${CMAKE_CURRENT_BINARY_DIR}/${MODEL_FILE}")
 224   set(NNPKG_FILE "${RECIPE}${OPT_FORMAT}")
 225   set(NNPKG_PATH "${CMAKE_CURRENT_BINARY_DIR}/${NNPKG_FILE}")
 226
 227   add_custom_command(OUTPUT ${NNPKG_PATH}
 228     COMMAND ${MODEL2NNPKG} ${MODEL_PATH}
 229     DEPENDS ${MODEL2NNPKG} ${MODEL_PATH}
 230     COMMENT "Generate ${RECIPE} nnpackage"
 231   )
 232   list(APPEND TEST_DEPS ${NNPKG_PATH})
 233
 234   set(INPUT_HDF5_FILE "${RECIPE}${OPT_FORMAT}.input.h5")
 235   set(INPUT_BIN_PATH "${CMAKE_CURRENT_BINARY_DIR}/${INPUT_HDF5_FILE}")
 236
 237   set(EXPECTED_HDF5_FILE "${RECIPE}${OPT_FORMAT}.expected.h5")
 238   set(EXPECTED_BIN_PATH "${CMAKE_CURRENT_BINARY_DIR}/${EXPECTED_HDF5_FILE}")
 239
 240   if(NOT DEFINED NO_TCGEN_${RECIPE})
 241     # Generate input.h5, expected.h5
 242     add_custom_command(OUTPUT ${INPUT_BIN_PATH} ${EXPECTED_BIN_PATH}
 243       COMMAND $<TARGET_FILE:testDataGenerator> ${MODEL_FILE}
 244       DEPENDS $<TARGET_FILE:testDataGenerator> ${MODEL_FILE}
 245       COMMENT "Generate ${INPUT_BIN_PATH} and ${EXPECTED_BIN_PATH}"
 246     )
 247
 248     # Generate test directory
 249     set(TC_DIRECTORY "${NNPKG_PATH}/metadata/tc")
 250     add_custom_command(OUTPUT ${TC_DIRECTORY}
 251       COMMAND ${CMAKE_COMMAND} -E make_directory ${TC_DIRECTORY}
 252       DEPENDS ${NNPKG_PATH}
 253       COMMENT "Generate ${RECIPE} nnpackage test directory"
 254     )
 255
 256     # Move input hdf5 file to test directory
 257     set(INPUT_NNPKG_PATH "${TC_DIRECTORY}/input.h5")
 258     add_custom_command(OUTPUT ${INPUT_NNPKG_PATH}
 259       COMMAND ${CMAKE_COMMAND} -E rename ${INPUT_BIN_PATH} ${INPUT_NNPKG_PATH}
 260       DEPENDS ${INPUT_BIN_PATH} ${TC_DIRECTORY}
 261       COMMENT "Move ${INPUT_HDF5_FILE} to nnpackage"
 262     )
 263
 264     # Move expected hdf5 file to test directory
 265     set(EXPECTED_NNPKG_PATH "${TC_DIRECTORY}/expected.h5")
 266     add_custom_command(OUTPUT ${EXPECTED_NNPKG_PATH}
 267       COMMAND ${CMAKE_COMMAND} -E rename ${EXPECTED_BIN_PATH} ${EXPECTED_NNPKG_PATH}
 268       DEPENDS ${EXPECTED_BIN_PATH} ${TC_DIRECTORY}
 269       COMMENT "Move ${EXPECTED_HDF5_FILE} to nnpackage"
 270     )
 271     list(APPEND TEST_DEPS ${TC_DIRECTORY} ${INPUT_BIN_PATH} ${EXPECTED_BIN_PATH}
 272                           ${INPUT_NNPKG_PATH} ${EXPECTED_NNPKG_PATH})
 273   endif()
 274 endforeach()
 275
 276 add_custom_target(common_artifacts_deps ALL DEPENDS ${TEST_DEPS})