compiler/common-artifacts/CMakeLists.txt

   1 #[[ Generate common python virtual enviornment ]]
   2 find_package(PythonInterp 3 QUIET)
   3 find_package(PythonLibs 3 QUIET)
   4
   5 if(NOT ${PYTHONINTERP_FOUND})
   6   message(STATUS "Build common-artifacts: FALSE (Python3 is missing)")
   7   return()
   8 endif()
   9
  10 if(${PYTHON_VERSION_MINOR} LESS 3)
  11   message(STATUS "Build common-artifacts: FALSE (You need to install Python version higher than 3.3)")
  12   return()
  13 endif()
  14
  15 # Create python virtual environment with tensorflow 1.13.2
  16 set(VIRTUALENV_OVERLAY_TF_1_13_2 "${NNCC_OVERLAY_DIR}/venv_1_13_2")
  17
  18 # Create python virtual environment with tensorflow 2.3.0
  19 set(VIRTUALENV_OVERLAY_TF_2_3_0 "${NNCC_OVERLAY_DIR}/venv_2_3_0")
  20
  21 add_custom_command(
  22   OUTPUT ${VIRTUALENV_OVERLAY_TF_1_13_2}
  23   COMMAND ${PYTHON_EXECUTABLE} -m venv ${VIRTUALENV_OVERLAY_TF_1_13_2}
  24 )
  25
  26 add_custom_command(
  27   OUTPUT ${VIRTUALENV_OVERLAY_TF_2_3_0}
  28   COMMAND ${PYTHON_EXECUTABLE} -m venv ${VIRTUALENV_OVERLAY_TF_2_3_0}
  29 )
  30
  31 # Create requirements.txt and install required pip packages
  32 set(REQUIREMENTS_FILE "requirements.txt")
  33 set(REQUIREMENTS_OVERLAY_PATH_TF_1_13_2 "${VIRTUALENV_OVERLAY_TF_1_13_2}/${REQUIREMENTS_FILE}")
  34 set(REQUIREMENTS_OVERLAY_PATH_TF_2_3_0 "${VIRTUALENV_OVERLAY_TF_2_3_0}/${REQUIREMENTS_FILE}")
  35
  36 # TODO remove version number of '--upgrade pip==20.2.1 setuptools==49.3.0'
  37 # NOTE adding version is for temporary hotfix of setuptools 50.x.y version
  38 add_custom_command(
  39   OUTPUT ${REQUIREMENTS_OVERLAY_PATH_TF_1_13_2}
  40   COMMAND ${CMAKE_COMMAND} -E echo "tensorflow==1.13.2" > ${REQUIREMENTS_OVERLAY_PATH_TF_1_13_2}
  41   COMMAND ${VIRTUALENV_OVERLAY_TF_1_13_2}/bin/python -m pip --default-timeout=1000 install --upgrade pip==20.2.1 setuptools==49.3.0
  42   COMMAND ${VIRTUALENV_OVERLAY_TF_1_13_2}/bin/python -m pip --default-timeout=1000 install -r ${REQUIREMENTS_OVERLAY_PATH_TF_1_13_2} --upgrade
  43   DEPENDS ${VIRTUALENV_OVERLAY_TF_1_13_2}
  44 )
  45
  46 add_custom_command(
  47   OUTPUT ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
  48   COMMAND ${CMAKE_COMMAND} -E remove -f ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
  49   COMMAND ${CMAKE_COMMAND} -E echo "tensorflow-cpu==2.3.0" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
  50   COMMAND ${CMAKE_COMMAND} -E echo "flatbuffers==1.12" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
  51   COMMAND ${VIRTUALENV_OVERLAY_TF_2_3_0}/bin/python -m pip --default-timeout=1000 install --upgrade pip==20.2.1 setuptools==49.3.0
  52   COMMAND ${VIRTUALENV_OVERLAY_TF_2_3_0}/bin/python -m pip --default-timeout=1000 install -r ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0} --upgrade
  53   DEPENDS ${VIRTUALENV_OVERLAY_TF_2_3_0}
  54 )
  55
  56 add_custom_target(common_artifacts_python_deps ALL
  57   DEPENDS ${VIRTUALENV_OVERLAY_TF_1_13_2} ${VIRTUALENV_OVERLAY_TF_2_3_0} ${REQUIREMENTS_OVERLAY_PATH_TF_1_13_2} ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
  58 )
  59
  60 #[[ Generate common resources ]]
  61 # TODO add pbtxt
  62 nnas_find_package(HDF5 QUIET)
  63
  64 if(NOT HDF5_FOUND)
  65   message(STATUS "Build common-artifacts: FAILED (missing HDF5)")
  66   return()
  67 endif(NOT HDF5_FOUND)
  68
  69 set(SOURCES src/TestDataGenerator.cpp)
  70
  71 add_executable(testDataGenerator ${SOURCES})
  72 target_include_directories(testDataGenerator PRIVATE ${HDF5_INCLUDE_DIRS})
  73 target_link_libraries(testDataGenerator PRIVATE ${HDF5_CXX_LIBRARIES})
  74 target_link_libraries(testDataGenerator PRIVATE arser)
  75 target_link_libraries(testDataGenerator PRIVATE foder)
  76 target_link_libraries(testDataGenerator PRIVATE luci_import)
  77 target_link_libraries(testDataGenerator PRIVATE luci_interpreter)
  78 target_link_libraries(testDataGenerator PRIVATE mio_circle)
  79 target_link_libraries(testDataGenerator PRIVATE safemain)
  80
  81 unset(TEST_DEPS)
  82
  83 # Include recipe repo
  84 nncc_find_resource(TensorFlowLiteRecipes)
  85 nncc_find_resource(CircleRecipes)
  86 set(TFLITE_RECIPE_REPO "${TensorFlowLiteRecipes_DIR}")
  87 set(CIRCLE_RECIPE_REPO "${CircleRecipes_DIR}")
  88 set(TEST_RECIPE_FILENAME "test.recipe")
  89 set(TEST_RULE_FILENAME "test.rule")
  90
  91 set(MODEL2NNPKG "${NNAS_PROJECT_SOURCE_DIR}/tools/nnpackage_tool/model2nnpkg/model2nnpkg.sh")
  92 # Get test case list
  93 unset(RECIPES)
  94 file(GLOB TFLITE_SUBDIR RELATIVE ${TFLITE_RECIPE_REPO} ${TFLITE_RECIPE_REPO}/*)
  95 foreach(DIR IN ITEMS ${TFLITE_SUBDIR})
  96   if(IS_DIRECTORY ${TFLITE_RECIPE_REPO}/${DIR})
  97     list(APPEND RECIPES ${DIR})
  98   endif()
  99 endforeach()
 100 file(GLOB CIRCLE_SUBDIR RELATIVE ${CIRCLE_RECIPE_REPO} ${CIRCLE_RECIPE_REPO}/*)
 101 foreach(DIR IN ITEMS ${CIRCLE_SUBDIR})
 102   if(IS_DIRECTORY ${CIRCLE_RECIPE_REPO}/${DIR})
 103     list(APPEND RECIPES ${DIR})
 104   endif()
 105 endforeach()
 106
 107 macro(circlize NAME)
 108   set(NO_CIRCLIZE_${NAME} TRUE)
 109   set(NO_OPTIMIZE_${NAME} TRUE)
 110   set(NO_TCGEN_${NAME} TRUE)
 111 endmacro()
 112 macro(optimize NAME)
 113   set(NO_OPTIMIZE_${NAME} TRUE)
 114 endmacro()
 115 macro(tcgenerate NAME)
 116   set(NO_TCGEN_${NAME} TRUE)
 117 endmacro()
 118
 119 include("exclude.lst")
 120
 121 foreach(RECIPE IN ITEMS ${RECIPES})
 122   unset(OPT_FORMAT)
 123   unset(MODEL_FORMAT)
 124
 125   set(RECIPE_FILE "${RECIPE}.recipe")
 126   set(RULE_FILE "${RECIPE}.rule")
 127   set(TFLITE_RECIPE_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_RECIPE_FILENAME}")
 128   set(CIRCLE_RECIPE_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_RECIPE_FILENAME}")
 129
 130   if(NOT EXISTS "${TFLITE_RECIPE_SOURCE_PATH}")
 131     if(NOT EXISTS "${CIRCLE_RECIPE_SOURCE_PATH}")
 132       message(FATAL_ERROR "Missing recipe of '${RECIPE}' test")
 133     else()
 134       # circle recipe
 135       set(MODEL_FORMAT "circle")
 136       set(RECIPE_SOURCE_PATH ${CIRCLE_RECIPE_SOURCE_PATH})
 137     endif()
 138   else()
 139     # tflite recipe
 140     set(MODEL_FORMAT "tflite")
 141     set(RECIPE_SOURCE_PATH ${TFLITE_RECIPE_SOURCE_PATH})
 142   endif()
 143
 144   set(TFLITE_RULE_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_RULE_FILENAME}")
 145   set(CIRCLE_RULE_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_RULE_FILENAME}")
 146
 147   unset(RULE_SOURCE_PATH)
 148   if(EXISTS "${TFLITE_RULE_SOURCE_PATH}")
 149     set(RULE_SOURCE_PATH ${TFLITE_RULE_SOURCE_PATH})
 150   endif()
 151   if(EXISTS "${CIRCLE_RULE_SOURCE_PATH}")
 152     set(RULE_SOURCE_PATH ${CIRCLE_RULE_SOURCE_PATH})
 153   endif()
 154
 155   set(RECIPE_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RECIPE_FILE}")
 156   set(RULE_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RULE_FILE}")
 157
 158   set(TFLITE_FILE "${RECIPE}.tflite")
 159   set(TFLITE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TFLITE_FILE}")
 160   set(CIRCLE_FILE "${RECIPE}.circle")
 161   set(CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${CIRCLE_FILE}")
 162
 163   # Copy .recipe
 164   add_custom_command(OUTPUT ${RECIPE_BINARY_PATH}
 165     COMMAND ${CMAKE_COMMAND} -E copy "${RECIPE_SOURCE_PATH}" "${RECIPE_BINARY_PATH}"
 166     DEPENDS ${RECIPE_SOURCE_PATH}
 167     COMMENT "Generate ${RECIPE_FILE}"
 168   )
 169   list(APPEND TEST_DEPS ${RECIPE_BINARY_PATH})
 170
 171   if(DEFINED RULE_SOURCE_PATH)
 172     # Copy .rule
 173     add_custom_command(OUTPUT ${RULE_BINARY_PATH}
 174     COMMAND ${CMAKE_COMMAND} -E copy "${RULE_SOURCE_PATH}" "${RULE_BINARY_PATH}"
 175     DEPENDS ${RULE_SOURCE_PATH}
 176     COMMENT "Generate ${RULE_FILE}"
 177     )
 178     list(APPEND TEST_DEPS ${RULE_BINARY_PATH})
 179   endif()
 180
 181   if(${MODEL_FORMAT} STREQUAL "tflite")
 182     # Generate .tflite
 183     add_custom_command(OUTPUT ${TFLITE_OUTPUT_PATH}
 184       COMMAND $<TARGET_FILE:tflchef-file> ${RECIPE_BINARY_PATH} ${TFLITE_OUTPUT_PATH}
 185       DEPENDS $<TARGET_FILE:tflchef-file> ${RECIPE_BINARY_PATH}
 186       COMMENT "Generate ${TFLITE_FILE}"
 187     )
 188     list(APPEND TEST_DEPS ${TFLITE_OUTPUT_PATH})
 189
 190     if(NOT DEFINED NO_CIRCLIZE_${RECIPE})
 191     # Generate .circle
 192     add_custom_command(OUTPUT ${CIRCLE_OUTPUT_PATH}
 193       COMMAND $<TARGET_FILE:tflite2circle> ${TFLITE_OUTPUT_PATH} ${CIRCLE_OUTPUT_PATH}
 194       DEPENDS $<TARGET_FILE:tflite2circle> ${TFLITE_OUTPUT_PATH}
 195       COMMENT "Generate ${CIRCLE_FILE}"
 196     )
 197     set(MODEL_FORMAT "circle")
 198     list(APPEND TEST_DEPS ${CIRCLE_OUTPUT_PATH})
 199     endif()
 200   else()
 201     # Generate .circle
 202     add_custom_command(OUTPUT ${CIRCLE_OUTPUT_PATH}
 203     COMMAND $<TARGET_FILE:circlechef-file> ${RECIPE_BINARY_PATH} ${CIRCLE_OUTPUT_PATH}
 204     DEPENDS $<TARGET_FILE:circlechef-file> ${RECIPE_BINARY_PATH}
 205     COMMENT "Generate ${CIRCLE_FILE}"
 206     )
 207     list(APPEND TEST_DEPS ${CIRCLE_OUTPUT_PATH})
 208   endif()
 209
 210   set(OPT_CIRCLE_FILE "${RECIPE}.opt.circle")
 211   set(OPT_CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${OPT_CIRCLE_FILE}")
 212
 213   if(NOT DEFINED NO_OPTIMIZE_${RECIPE})
 214     # Generate optimized .circle
 215     add_custom_command(OUTPUT ${OPT_CIRCLE_OUTPUT_PATH}
 216       COMMAND $<TARGET_FILE:circle2circle> --all ${CIRCLE_OUTPUT_PATH} ${OPT_CIRCLE_OUTPUT_PATH}
 217       DEPENDS $<TARGET_FILE:circle2circle>  ${CIRCLE_OUTPUT_PATH}
 218       COMMENT "Generate ${OPT_CIRCLE_FILE}"
 219     )
 220     set(OPT_FORMAT ".opt")
 221     list(APPEND TEST_DEPS ${OPT_CIRCLE_OUTPUT_PATH})
 222   endif()
 223
 224   set(MODEL_FILE "${RECIPE}${OPT_FORMAT}.${MODEL_FORMAT}")
 225   set(MODEL_PATH "${CMAKE_CURRENT_BINARY_DIR}/${MODEL_FILE}")
 226   set(NNPKG_FILE "${RECIPE}${OPT_FORMAT}")
 227   set(NNPKG_PATH "${CMAKE_CURRENT_BINARY_DIR}/${NNPKG_FILE}")
 228
 229   add_custom_command(OUTPUT ${NNPKG_PATH}
 230     COMMAND ${MODEL2NNPKG} ${MODEL_PATH}
 231     DEPENDS ${MODEL2NNPKG} ${MODEL_PATH}
 232     COMMENT "Generate ${RECIPE} nnpackage"
 233   )
 234   list(APPEND TEST_DEPS ${NNPKG_PATH})
 235
 236   set(INPUT_HDF5_FILE "${RECIPE}${OPT_FORMAT}.input.h5")
 237   set(INPUT_BIN_PATH "${CMAKE_CURRENT_BINARY_DIR}/${INPUT_HDF5_FILE}")
 238
 239   set(EXPECTED_HDF5_FILE "${RECIPE}${OPT_FORMAT}.expected.h5")
 240   set(EXPECTED_BIN_PATH "${CMAKE_CURRENT_BINARY_DIR}/${EXPECTED_HDF5_FILE}")
 241
 242   if(NOT DEFINED NO_TCGEN_${RECIPE})
 243     # Generate input.h5, expected.h5
 244     add_custom_command(OUTPUT ${INPUT_BIN_PATH} ${EXPECTED_BIN_PATH}
 245       COMMAND $<TARGET_FILE:testDataGenerator> ${MODEL_FILE}
 246       DEPENDS $<TARGET_FILE:testDataGenerator> ${MODEL_FILE}
 247       COMMENT "Generate ${INPUT_BIN_PATH} and ${EXPECTED_BIN_PATH}"
 248     )
 249
 250     # Generate test directory
 251     set(TC_DIRECTORY "${NNPKG_PATH}/metadata/tc")
 252     add_custom_command(OUTPUT ${TC_DIRECTORY}
 253       COMMAND ${CMAKE_COMMAND} -E make_directory ${TC_DIRECTORY}
 254       DEPENDS ${NNPKG_PATH}
 255       COMMENT "Generate ${RECIPE} nnpackage test directory"
 256     )
 257
 258     # Move input hdf5 file to test directory
 259     set(INPUT_NNPKG_PATH "${TC_DIRECTORY}/input.h5")
 260     add_custom_command(OUTPUT ${INPUT_NNPKG_PATH}
 261       COMMAND ${CMAKE_COMMAND} -E rename ${INPUT_BIN_PATH} ${INPUT_NNPKG_PATH}
 262       DEPENDS ${INPUT_BIN_PATH} ${TC_DIRECTORY}
 263       COMMENT "Move ${INPUT_HDF5_FILE} to nnpackage"
 264     )
 265
 266     # Move expected hdf5 file to test directory
 267     set(EXPECTED_NNPKG_PATH "${TC_DIRECTORY}/expected.h5")
 268     add_custom_command(OUTPUT ${EXPECTED_NNPKG_PATH}
 269       COMMAND ${CMAKE_COMMAND} -E rename ${EXPECTED_BIN_PATH} ${EXPECTED_NNPKG_PATH}
 270       DEPENDS ${EXPECTED_BIN_PATH} ${TC_DIRECTORY}
 271       COMMENT "Move ${EXPECTED_HDF5_FILE} to nnpackage"
 272     )
 273     list(APPEND TEST_DEPS ${TC_DIRECTORY} ${INPUT_BIN_PATH} ${EXPECTED_BIN_PATH}
 274                           ${INPUT_NNPKG_PATH} ${EXPECTED_NNPKG_PATH})
 275   endif()
 276 endforeach()
 277
 278 add_custom_target(common_artifacts_deps ALL DEPENDS ${TEST_DEPS})