compiler/common-artifacts/CMakeLists.txt

   1 #[[ Generate common python virtual enviornment ]]
   2 # NOTE find_package try to use at least python3.8 as follows depending on platform version
   3 #   Ubuntu18.04; explictly installed python3.8 (default is python3.6)
   4 #   Ubuntu20.04; default python3.8
   5 #   Ubuntu22.04; default python3.10
   6 #   refer https://github.com/Samsung/ONE/issues/9962
   7 find_package(PythonInterp 3.8 QUIET)
   8 find_package(PythonLibs 3.8 QUIET)
   9
  10 if(NOT ${PYTHONINTERP_FOUND})
  11   message(STATUS "Build common-artifacts: FALSE (Python3 is missing)")
  12   return()
  13 endif()
  14
  15 if(${PYTHON_VERSION_MINOR} LESS 8)
  16   message(STATUS "Build common-artifacts: FALSE (You need to install Python version higher than 3.8)")
  17   return()
  18 endif()
  19
  20 # Create python virtual environment with tensorflow 2.12.1
  21 set(VIRTUALENV_OVERLAY_TF_2_12_1 "${NNCC_OVERLAY_DIR}/venv_2_12_1")
  22
  23 add_custom_command(
  24   OUTPUT ${VIRTUALENV_OVERLAY_TF_2_12_1}
  25   COMMAND ${PYTHON_EXECUTABLE} -m venv ${VIRTUALENV_OVERLAY_TF_2_12_1}
  26 )
  27
  28 # Create requirements.txt and install required pip packages
  29 set(REQUIREMENTS_FILE "requirements.txt")
  30 set(REQUIREMENTS_OVERLAY_PATH_TF_2_12_1 "${VIRTUALENV_OVERLAY_TF_2_12_1}/${REQUIREMENTS_FILE}")
  31
  32 set(PYTHON_OVERLAY python3)
  33 if(PYTHON_EXECUTABLE MATCHES python3.8)
  34   set(PYTHON_OVERLAY python3.8)
  35 endif()
  36
  37 # NOTE when using behind proxy with self signed certificate, need to set '--trusted-host' options
  38 set(PIP_OPTION_TRUSTED_HOST )
  39 if(DEFINED ENV{ONE_PIP_OPTION_TRUST_HOST})
  40   set(PIP_OPTION_TRUSTED_HOST --trusted-host pypi.python.org --trusted-host files.pythonhosted.org --trusted-host pypi.org)
  41 endif()
  42
  43 if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
  44   # NOTE `tensorflow-cpu` package is not available for aarch64, so we use `tensorflow` package.
  45   add_custom_command(
  46     OUTPUT ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
  47     COMMAND ${CMAKE_COMMAND} -E remove -f ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
  48     COMMAND ${CMAKE_COMMAND} -E echo "tensorflow==2.12.1" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
  49     COMMAND ${CMAKE_COMMAND} -E echo "flatbuffers==23.5.26" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
  50     COMMAND ${CMAKE_COMMAND} -E echo "protobuf==4.23.3" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
  51     COMMAND ${CMAKE_COMMAND} -E echo "pydot==1.4.2" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
  52     COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
  53             ${PIP_OPTION_TRUSTED_HOST} install --upgrade pip setuptools
  54     COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
  55             ${PIP_OPTION_TRUSTED_HOST} install -r ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1} --upgrade
  56     DEPENDS ${VIRTUALENV_OVERLAY_TF_2_12_1}
  57   )
  58 else(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
  59   add_custom_command(
  60     OUTPUT ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
  61     COMMAND ${CMAKE_COMMAND} -E remove -f ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
  62     COMMAND ${CMAKE_COMMAND} -E echo "tensorflow-cpu==2.12.1" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
  63     COMMAND ${CMAKE_COMMAND} -E echo "flatbuffers==23.5.26" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
  64     COMMAND ${CMAKE_COMMAND} -E echo "protobuf==4.23.3" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
  65     COMMAND ${CMAKE_COMMAND} -E echo "pydot==1.4.2" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
  66     COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
  67             ${PIP_OPTION_TRUSTED_HOST} install --upgrade pip setuptools
  68     COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
  69             ${PIP_OPTION_TRUSTED_HOST} install -r ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1} --upgrade
  70     DEPENDS ${VIRTUALENV_OVERLAY_TF_2_12_1}
  71   )
  72 endif(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
  73
  74 add_custom_target(common_artifacts_python_deps ALL
  75   DEPENDS ${VIRTUALENV_OVERLAY_TF_2_12_1}
  76           ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
  77 )
  78
  79 #[[ Generate common resources ]]
  80 # TODO add pbtxt
  81 nnas_find_package(HDF5 QUIET)
  82
  83 if(NOT HDF5_FOUND)
  84   message(STATUS "Build common-artifacts: FAILED (missing HDF5)")
  85   return()
  86 endif(NOT HDF5_FOUND)
  87
  88 set(SOURCES src/TestDataGenerator.cpp)
  89
  90 add_executable(testDataGenerator ${SOURCES})
  91 target_include_directories(testDataGenerator PRIVATE ${HDF5_INCLUDE_DIRS})
  92 target_link_libraries(testDataGenerator PRIVATE ${HDF5_CXX_LIBRARIES})
  93 target_link_libraries(testDataGenerator PRIVATE arser)
  94 target_link_libraries(testDataGenerator PRIVATE foder)
  95 target_link_libraries(testDataGenerator PRIVATE luci_import)
  96 target_link_libraries(testDataGenerator PRIVATE luci_interpreter)
  97 target_link_libraries(testDataGenerator PRIVATE safemain)
  98
  99 unset(TEST_DEPS)
 100
 101 # Include recipe repo
 102 nncc_find_resource(TensorFlowLiteRecipes)
 103 nncc_find_resource(CircleRecipes)
 104 set(TFLITE_RECIPE_REPO "${TensorFlowLiteRecipes_DIR}")
 105 set(CIRCLE_RECIPE_REPO "${CircleRecipes_DIR}")
 106 set(TEST_RECIPE_FILENAME "test.recipe")
 107 set(TEST_RULE_FILENAME "test.rule")
 108 set(TEST_QCONFIG_FILENAME "test.qconf.json")
 109
 110 set(MODEL2NNPKG "${NNAS_PROJECT_SOURCE_DIR}/tools/nnpackage_tool/model2nnpkg/model2nnpkg.py")
 111 # Get test case list
 112 unset(RECIPES)
 113 file(GLOB TFLITE_SUBDIR RELATIVE ${TFLITE_RECIPE_REPO} ${TFLITE_RECIPE_REPO}/*)
 114 foreach(DIR IN ITEMS ${TFLITE_SUBDIR})
 115   if(IS_DIRECTORY ${TFLITE_RECIPE_REPO}/${DIR})
 116     list(APPEND RECIPES ${DIR})
 117   endif()
 118 endforeach()
 119 file(GLOB CIRCLE_SUBDIR RELATIVE ${CIRCLE_RECIPE_REPO} ${CIRCLE_RECIPE_REPO}/*)
 120 foreach(DIR IN ITEMS ${CIRCLE_SUBDIR})
 121   if(IS_DIRECTORY ${CIRCLE_RECIPE_REPO}/${DIR})
 122     list(APPEND RECIPES ${DIR})
 123   endif()
 124 endforeach()
 125
 126 macro(circlize NAME)
 127   set(NO_CIRCLIZE_${NAME} TRUE)
 128   set(NO_OPTIMIZE_${NAME} TRUE)
 129   set(NO_TCGEN_${NAME} TRUE)
 130 endmacro()
 131 macro(optimize NAME)
 132   set(NO_OPTIMIZE_${NAME} TRUE)
 133 endmacro()
 134 macro(tcgenerate NAME)
 135   set(NO_TCGEN_${NAME} TRUE)
 136 endmacro()
 137
 138 include("exclude.lst")
 139
 140 # TODO revise using variadic arguments
 141 macro(tcgenerate_option NAME OPTION ARG1 ARG2 ARG3)
 142   set(TCGEN_OPT_${NAME} ${OPTION} ${ARG1} ${ARG2} ${ARG3})
 143 endmacro()
 144
 145 include("options.lst")
 146
 147 foreach(RECIPE IN ITEMS ${RECIPES})
 148   unset(OPT_FORMAT)
 149   unset(MODEL_FORMAT)
 150
 151   set(RECIPE_FILE "${RECIPE}.recipe")
 152   set(RULE_FILE "${RECIPE}.rule")
 153   set(QCONFIG_FILE "${RECIPE}.qconf.json")
 154   set(TFLITE_RECIPE_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_RECIPE_FILENAME}")
 155   set(CIRCLE_RECIPE_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_RECIPE_FILENAME}")
 156
 157   if(NOT EXISTS "${TFLITE_RECIPE_SOURCE_PATH}")
 158     if(NOT EXISTS "${CIRCLE_RECIPE_SOURCE_PATH}")
 159       message(FATAL_ERROR "Missing recipe of '${RECIPE}' test")
 160     else()
 161       # circle recipe
 162       set(MODEL_FORMAT "circle")
 163       set(RECIPE_SOURCE_PATH ${CIRCLE_RECIPE_SOURCE_PATH})
 164     endif()
 165   else()
 166     # tflite recipe
 167     set(MODEL_FORMAT "tflite")
 168     set(RECIPE_SOURCE_PATH ${TFLITE_RECIPE_SOURCE_PATH})
 169   endif()
 170
 171   set(TFLITE_RULE_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_RULE_FILENAME}")
 172   set(CIRCLE_RULE_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_RULE_FILENAME}")
 173
 174   unset(RULE_SOURCE_PATH)
 175   if(EXISTS "${TFLITE_RULE_SOURCE_PATH}")
 176     set(RULE_SOURCE_PATH ${TFLITE_RULE_SOURCE_PATH})
 177   endif()
 178   if(EXISTS "${CIRCLE_RULE_SOURCE_PATH}")
 179     set(RULE_SOURCE_PATH ${CIRCLE_RULE_SOURCE_PATH})
 180   endif()
 181
 182   set(TFLITE_QCONFIG_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_QCONFIG_FILENAME}")
 183   set(CIRCLE_QCONFIG_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_QCONFIG_FILENAME}")
 184
 185   unset(QCONFIG_SOURCE_PATH)
 186   if(EXISTS "${TFLITE_QCONFIG_SOURCE_PATH}")
 187     set(QCONFIG_SOURCE_PATH ${TFLITE_QCONFIG_SOURCE_PATH})
 188   endif()
 189   if(EXISTS "${CIRCLE_QCONFIG_SOURCE_PATH}")
 190     set(QCONFIG_SOURCE_PATH ${CIRCLE_QCONFIG_SOURCE_PATH})
 191   endif()
 192
 193   set(RECIPE_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RECIPE_FILE}")
 194   set(RULE_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RULE_FILE}")
 195   set(QCONFIG_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${QCONFIG_FILE}")
 196
 197   set(TFLITE_FILE "${RECIPE}.tflite")
 198   set(TFLITE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TFLITE_FILE}")
 199   set(CIRCLE_FILE "${RECIPE}.circle")
 200   set(CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${CIRCLE_FILE}")
 201
 202   # Copy .recipe
 203   add_custom_command(OUTPUT ${RECIPE_BINARY_PATH}
 204     COMMAND ${CMAKE_COMMAND} -E copy "${RECIPE_SOURCE_PATH}" "${RECIPE_BINARY_PATH}"
 205     DEPENDS ${RECIPE_SOURCE_PATH}
 206     COMMENT "Generate ${RECIPE_FILE}"
 207   )
 208   list(APPEND TEST_DEPS ${RECIPE_BINARY_PATH})
 209
 210   if(DEFINED RULE_SOURCE_PATH)
 211     # Copy .rule
 212     add_custom_command(OUTPUT ${RULE_BINARY_PATH}
 213       COMMAND ${CMAKE_COMMAND} -E copy "${RULE_SOURCE_PATH}" "${RULE_BINARY_PATH}"
 214       DEPENDS ${RULE_SOURCE_PATH}
 215       COMMENT "Generate ${RULE_FILE}"
 216     )
 217     list(APPEND TEST_DEPS ${RULE_BINARY_PATH})
 218   endif()
 219
 220   if(DEFINED QCONFIG_SOURCE_PATH)
 221     # Copy .qconf.json
 222     add_custom_command(OUTPUT ${QCONFIG_BINARY_PATH}
 223       COMMAND ${CMAKE_COMMAND} -E copy "${QCONFIG_SOURCE_PATH}" "${QCONFIG_BINARY_PATH}"
 224       DEPENDS ${QCONFIG_SOURCE_PATH}
 225       COMMENT "Generate ${QCONFIG_FILE}"
 226     )
 227     list(APPEND TEST_DEPS ${QCONFIG_BINARY_PATH})
 228   endif()
 229
 230   if(${MODEL_FORMAT} STREQUAL "tflite")
 231     # Generate .tflite
 232     add_custom_command(OUTPUT ${TFLITE_OUTPUT_PATH}
 233       COMMAND $<TARGET_FILE:tflchef-file> ${RECIPE_BINARY_PATH} ${TFLITE_OUTPUT_PATH}
 234       DEPENDS $<TARGET_FILE:tflchef-file> ${RECIPE_BINARY_PATH}
 235       COMMENT "Generate ${TFLITE_FILE}"
 236     )
 237     list(APPEND TEST_DEPS ${TFLITE_OUTPUT_PATH})
 238
 239     if(NOT DEFINED NO_CIRCLIZE_${RECIPE})
 240       # Generate .circle
 241       add_custom_command(OUTPUT ${CIRCLE_OUTPUT_PATH}
 242         COMMAND $<TARGET_FILE:tflite2circle> ${TFLITE_OUTPUT_PATH} ${CIRCLE_OUTPUT_PATH}
 243         DEPENDS $<TARGET_FILE:tflite2circle> ${TFLITE_OUTPUT_PATH}
 244         COMMENT "Generate ${CIRCLE_FILE}"
 245       )
 246       set(MODEL_FORMAT "circle")
 247       list(APPEND TEST_DEPS ${CIRCLE_OUTPUT_PATH})
 248     endif()
 249   else()
 250     # Generate .circle
 251     add_custom_command(OUTPUT ${CIRCLE_OUTPUT_PATH}
 252       COMMAND $<TARGET_FILE:circlechef-file> ${RECIPE_BINARY_PATH} ${CIRCLE_OUTPUT_PATH}
 253       DEPENDS $<TARGET_FILE:circlechef-file> ${RECIPE_BINARY_PATH}
 254       COMMENT "Generate ${CIRCLE_FILE}"
 255     )
 256     list(APPEND TEST_DEPS ${CIRCLE_OUTPUT_PATH})
 257   endif()
 258
 259   set(OPT_CIRCLE_FILE "${RECIPE}.opt.circle")
 260   set(OPT_CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${OPT_CIRCLE_FILE}")
 261
 262   if(NOT DEFINED NO_OPTIMIZE_${RECIPE})
 263     # Generate optimized .circle
 264     add_custom_command(OUTPUT ${OPT_CIRCLE_OUTPUT_PATH}
 265       # NOTE --resolve_customop_add is just to added for old -O1, no particular meaning
 266       #      --fold_dequantize is added to fold Tensor(FLOAT16) + DEQUANTIZE (Net_Dequantize_Add)
 267       #      model. FLOAT16 in general is NOT supported but only Tensor(FLOAT16) + DEQUANTIZE
 268       #      sequence accepted as folded to Tensor(FLOAT32).
 269       # TODO revise giving options from the list file
 270       COMMAND $<TARGET_FILE:circle2circle> --resolve_customop_add --fold_dequantize --fold_densify
 271               ${CIRCLE_OUTPUT_PATH} ${OPT_CIRCLE_OUTPUT_PATH}
 272       DEPENDS $<TARGET_FILE:circle2circle>  ${CIRCLE_OUTPUT_PATH}
 273       COMMENT "Generate ${OPT_CIRCLE_FILE}"
 274     )
 275     set(OPT_FORMAT ".opt")
 276     list(APPEND TEST_DEPS ${OPT_CIRCLE_OUTPUT_PATH})
 277   endif()
 278
 279   set(MODEL_FILE "${RECIPE}${OPT_FORMAT}.${MODEL_FORMAT}")
 280   set(MODEL_PATH "${CMAKE_CURRENT_BINARY_DIR}/${MODEL_FILE}")
 281   set(NNPKG_FILE "${RECIPE}${OPT_FORMAT}")
 282   set(NNPKG_DIR "${CMAKE_CURRENT_BINARY_DIR}/${NNPKG_FILE}")
 283   set(NNPKG_MODEL "${NNPKG_DIR}/${MODEL_FILE}")
 284
 285   # Generate nnpackage directory
 286   add_custom_command(OUTPUT ${NNPKG_DIR}
 287       COMMAND ${CMAKE_COMMAND} -E make_directory ${NNPKG_DIR}
 288       DEPENDS ${MODEL_PATH}
 289       COMMENT "Generate ${RECIPE} nnpackage directory"
 290     )
 291   list(APPEND TEST_DEPS ${NNPKG_DIR})
 292
 293   add_custom_command(OUTPUT ${NNPKG_MODEL}
 294     COMMAND ${PYTHON_EXECUTABLE} ${MODEL2NNPKG} -m ${MODEL_PATH}
 295     DEPENDS ${MODEL2NNPKG} ${MODEL_PATH}
 296     COMMENT "Generate ${RECIPE} nnpackage"
 297   )
 298   list(APPEND TEST_DEPS ${NNPKG_MODEL})
 299
 300   if(NOT DEFINED NO_TCGEN_${RECIPE})
 301     # Generate test directory
 302     set(TC_DIRECTORY "${NNPKG_DIR}/metadata/tc")
 303     add_custom_command(OUTPUT ${TC_DIRECTORY}
 304       COMMAND ${CMAKE_COMMAND} -E make_directory ${TC_DIRECTORY}
 305       COMMENT "Generate ${RECIPE} nnpackage test directory"
 306     )
 307     list(APPEND TEST_DEPS ${TC_DIRECTORY})
 308
 309     # set ADDITIONAL_OPTIONS as empty (one space before closing is intentional)
 310     set(ADDITIONAL_OPTIONS )
 311     if(DEFINED TCGEN_OPT_${RECIPE})
 312       set(ADDITIONAL_OPTIONS ${ADDITIONAL_OPTIONS} ${TCGEN_OPT_${RECIPE}})
 313     endif()
 314
 315     # Generate input.h5, expected.h5
 316     set(INPUT_HDF5_FILE "${TC_DIRECTORY}/input.h5")
 317     set(EXPECTED_HDF5_FILE "${TC_DIRECTORY}/expected.h5")
 318     add_custom_command(OUTPUT ${INPUT_HDF5_FILE} ${EXPECTED_HDF5_FILE}
 319       COMMAND $<TARGET_FILE:testDataGenerator>
 320               --input_data ${INPUT_HDF5_FILE}
 321               --expected_data ${EXPECTED_HDF5_FILE}
 322               ${ADDITIONAL_OPTIONS}
 323               ${MODEL_FILE}
 324       DEPENDS $<TARGET_FILE:testDataGenerator> ${MODEL_FILE} ${TC_DIRECTORY}
 325       COMMENT "Generate input.h5 and expected.h5 in ${NNPKG_FILE}/metadata/tc"
 326     )
 327     list(APPEND TEST_DEPS ${INPUT_HDF5_FILE} ${EXPECTED_HDF5_FILE})
 328   endif()
 329 endforeach()
 330
 331 add_custom_target(common_artifacts_deps ALL DEPENDS ${TEST_DEPS})