compiler/common-artifacts/CMakeLists.txt

   1 #[[ Generate common python virtual enviornment ]]
   2 # NOTE find_package try to use at least python3.8 as follows depending on platform version
   3 #   Ubuntu18.04; explictly installed python3.8 (default is python3.6)
   4 #   Ubuntu20.04; default python3.8
   5 #   Ubuntu22.04; default python3.10
   6 #   refer https://github.com/Samsung/ONE/issues/9962
   7 find_package(PythonInterp 3.8 QUIET)
   8 find_package(PythonLibs 3.8 QUIET)
   9
  10 if(NOT ${PYTHONINTERP_FOUND})
  11   message(STATUS "Build common-artifacts: FALSE (Python3 is missing)")
  12   return()
  13 endif()
  14
  15 if(${PYTHON_VERSION_MINOR} LESS 8)
  16   message(STATUS "Build common-artifacts: FALSE (You need to install Python version higher than 3.8)")
  17   return()
  18 endif()
  19
  20 # Create python virtual environment with tensorflow 2.8.0
  21 set(VIRTUALENV_OVERLAY_TF_2_8_0 "${NNCC_OVERLAY_DIR}/venv_2_8_0")
  22 # TensorFlow 2.10.1 for Ubuntu22.04
  23 if(ONE_UBUNTU_CODENAME_JAMMY)
  24   set(VIRTUALENV_OVERLAY_TF_2_10_1 "${NNCC_OVERLAY_DIR}/venv_2_10_1")
  25 endif(ONE_UBUNTU_CODENAME_JAMMY)
  26
  27 add_custom_command(
  28   OUTPUT ${VIRTUALENV_OVERLAY_TF_2_8_0}
  29   COMMAND ${PYTHON_EXECUTABLE} -m venv ${VIRTUALENV_OVERLAY_TF_2_8_0}
  30 )
  31
  32 # Create requirements.txt and install required pip packages
  33 set(REQUIREMENTS_FILE "requirements.txt")
  34 set(REQUIREMENTS_OVERLAY_PATH_TF_2_8_0 "${VIRTUALENV_OVERLAY_TF_2_8_0}/${REQUIREMENTS_FILE}")
  35
  36 if(ONE_UBUNTU_CODENAME_JAMMY)
  37   add_custom_command(
  38     OUTPUT ${VIRTUALENV_OVERLAY_TF_2_10_1}
  39     COMMAND ${PYTHON_EXECUTABLE} -m venv ${VIRTUALENV_OVERLAY_TF_2_10_1}
  40   )
  41   set(REQUIREMENTS_FILE "requirements.txt")
  42   set(REQUIREMENTS_OVERLAY_PATH_TF_2_10_1 "${VIRTUALENV_OVERLAY_TF_2_10_1}/${REQUIREMENTS_FILE}")
  43 endif(ONE_UBUNTU_CODENAME_JAMMY)
  44
  45 set(PYTHON_OVERLAY python3)
  46 if(PYTHON_EXECUTABLE MATCHES python3.8)
  47   set(PYTHON_OVERLAY python3.8)
  48 endif()
  49
  50 # NOTE when using behind proxy with self signed certificate, need to set '--trusted-host' options
  51 set(PIP_OPTION_TRUSTED_HOST )
  52 if(DEFINED ENV{ONE_PIP_OPTION_TRUST_HOST})
  53   set(PIP_OPTION_TRUSTED_HOST --trusted-host pypi.python.org --trusted-host files.pythonhosted.org --trusted-host pypi.org)
  54 endif()
  55
  56 # NOTE refer https://github.com/protocolbuffers/protobuf/issues/10051
  57 # TODO remove protobuf==3.20.1 when issue is resolved
  58 add_custom_command(
  59   OUTPUT ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0}
  60   COMMAND ${CMAKE_COMMAND} -E remove -f ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0}
  61   COMMAND ${CMAKE_COMMAND} -E echo "tensorflow-cpu==2.8.0" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0}
  62   COMMAND ${CMAKE_COMMAND} -E echo "flatbuffers==1.12" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0}
  63   COMMAND ${CMAKE_COMMAND} -E echo "protobuf==3.20.1" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0}
  64   COMMAND ${CMAKE_COMMAND} -E echo "pydot==1.4.2" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0}
  65   COMMAND ${VIRTUALENV_OVERLAY_TF_2_8_0}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
  66           ${PIP_OPTION_TRUSTED_HOST} install --upgrade pip setuptools
  67   COMMAND ${VIRTUALENV_OVERLAY_TF_2_8_0}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
  68           ${PIP_OPTION_TRUSTED_HOST} install -r ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0} --upgrade
  69   DEPENDS ${VIRTUALENV_OVERLAY_TF_2_8_0}
  70 )
  71
  72 add_custom_target(common_artifacts_python_deps ALL
  73   DEPENDS ${VIRTUALENV_OVERLAY_TF_2_8_0}
  74           ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0}
  75 )
  76
  77 if(ONE_UBUNTU_CODENAME_JAMMY)
  78   add_custom_command(
  79     OUTPUT ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1}
  80     COMMAND ${CMAKE_COMMAND} -E remove -f ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1}
  81     COMMAND ${CMAKE_COMMAND} -E echo "tensorflow-cpu==2.10.1" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1}
  82     COMMAND ${CMAKE_COMMAND} -E echo "flatbuffers==23.1.21" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1}
  83     COMMAND ${CMAKE_COMMAND} -E echo "protobuf==3.19.6" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1}
  84     COMMAND ${CMAKE_COMMAND} -E echo "pydot==1.4.2" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1}
  85     COMMAND ${VIRTUALENV_OVERLAY_TF_2_10_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
  86             ${PIP_OPTION_TRUSTED_HOST} install --upgrade pip setuptools
  87     COMMAND ${VIRTUALENV_OVERLAY_TF_2_10_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
  88             ${PIP_OPTION_TRUSTED_HOST} install -r ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1} --upgrade
  89     DEPENDS ${VIRTUALENV_OVERLAY_TF_2_10_1}
  90   )
  91
  92   add_custom_target(common_artifacts_python_u22_deps ALL
  93     DEPENDS ${VIRTUALENV_OVERLAY_TF_2_10_1}
  94             ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1}
  95   )
  96 endif(ONE_UBUNTU_CODENAME_JAMMY)
  97
  98 #[[ Generate common resources ]]
  99 # TODO add pbtxt
 100 nnas_find_package(HDF5 QUIET)
 101
 102 if(NOT HDF5_FOUND)
 103   message(STATUS "Build common-artifacts: FAILED (missing HDF5)")
 104   return()
 105 endif(NOT HDF5_FOUND)
 106
 107 set(SOURCES src/TestDataGenerator.cpp)
 108
 109 add_executable(testDataGenerator ${SOURCES})
 110 target_include_directories(testDataGenerator PRIVATE ${HDF5_INCLUDE_DIRS})
 111 target_link_libraries(testDataGenerator PRIVATE ${HDF5_CXX_LIBRARIES})
 112 target_link_libraries(testDataGenerator PRIVATE arser)
 113 target_link_libraries(testDataGenerator PRIVATE foder)
 114 target_link_libraries(testDataGenerator PRIVATE luci_import)
 115 target_link_libraries(testDataGenerator PRIVATE luci_interpreter)
 116 target_link_libraries(testDataGenerator PRIVATE safemain)
 117
 118 unset(TEST_DEPS)
 119
 120 # Include recipe repo
 121 nncc_find_resource(TensorFlowLiteRecipes)
 122 nncc_find_resource(CircleRecipes)
 123 set(TFLITE_RECIPE_REPO "${TensorFlowLiteRecipes_DIR}")
 124 set(CIRCLE_RECIPE_REPO "${CircleRecipes_DIR}")
 125 set(TEST_RECIPE_FILENAME "test.recipe")
 126 set(TEST_RULE_FILENAME "test.rule")
 127 set(TEST_QCONFIG_FILENAME "test.qconf.json")
 128
 129 set(MODEL2NNPKG "${NNAS_PROJECT_SOURCE_DIR}/tools/nnpackage_tool/model2nnpkg/model2nnpkg.py")
 130 # Get test case list
 131 unset(RECIPES)
 132 file(GLOB TFLITE_SUBDIR RELATIVE ${TFLITE_RECIPE_REPO} ${TFLITE_RECIPE_REPO}/*)
 133 foreach(DIR IN ITEMS ${TFLITE_SUBDIR})
 134   if(IS_DIRECTORY ${TFLITE_RECIPE_REPO}/${DIR})
 135     list(APPEND RECIPES ${DIR})
 136   endif()
 137 endforeach()
 138 file(GLOB CIRCLE_SUBDIR RELATIVE ${CIRCLE_RECIPE_REPO} ${CIRCLE_RECIPE_REPO}/*)
 139 foreach(DIR IN ITEMS ${CIRCLE_SUBDIR})
 140   if(IS_DIRECTORY ${CIRCLE_RECIPE_REPO}/${DIR})
 141     list(APPEND RECIPES ${DIR})
 142   endif()
 143 endforeach()
 144
 145 macro(circlize NAME)
 146   set(NO_CIRCLIZE_${NAME} TRUE)
 147   set(NO_OPTIMIZE_${NAME} TRUE)
 148   set(NO_TCGEN_${NAME} TRUE)
 149 endmacro()
 150 macro(optimize NAME)
 151   set(NO_OPTIMIZE_${NAME} TRUE)
 152 endmacro()
 153 macro(tcgenerate NAME)
 154   set(NO_TCGEN_${NAME} TRUE)
 155 endmacro()
 156
 157 include("exclude.lst")
 158
 159 # TODO revise using variadic arguments
 160 macro(tcgenerate_option NAME OPTION ARG1 ARG2 ARG3)
 161   set(TCGEN_OPT_${NAME} ${OPTION} ${ARG1} ${ARG2} ${ARG3})
 162 endmacro()
 163
 164 include("options.lst")
 165
 166 foreach(RECIPE IN ITEMS ${RECIPES})
 167   unset(OPT_FORMAT)
 168   unset(MODEL_FORMAT)
 169
 170   set(RECIPE_FILE "${RECIPE}.recipe")
 171   set(RULE_FILE "${RECIPE}.rule")
 172   set(QCONFIG_FILE "${RECIPE}.qconf.json")
 173   set(TFLITE_RECIPE_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_RECIPE_FILENAME}")
 174   set(CIRCLE_RECIPE_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_RECIPE_FILENAME}")
 175
 176   if(NOT EXISTS "${TFLITE_RECIPE_SOURCE_PATH}")
 177     if(NOT EXISTS "${CIRCLE_RECIPE_SOURCE_PATH}")
 178       message(FATAL_ERROR "Missing recipe of '${RECIPE}' test")
 179     else()
 180       # circle recipe
 181       set(MODEL_FORMAT "circle")
 182       set(RECIPE_SOURCE_PATH ${CIRCLE_RECIPE_SOURCE_PATH})
 183     endif()
 184   else()
 185     # tflite recipe
 186     set(MODEL_FORMAT "tflite")
 187     set(RECIPE_SOURCE_PATH ${TFLITE_RECIPE_SOURCE_PATH})
 188   endif()
 189
 190   set(TFLITE_RULE_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_RULE_FILENAME}")
 191   set(CIRCLE_RULE_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_RULE_FILENAME}")
 192
 193   unset(RULE_SOURCE_PATH)
 194   if(EXISTS "${TFLITE_RULE_SOURCE_PATH}")
 195     set(RULE_SOURCE_PATH ${TFLITE_RULE_SOURCE_PATH})
 196   endif()
 197   if(EXISTS "${CIRCLE_RULE_SOURCE_PATH}")
 198     set(RULE_SOURCE_PATH ${CIRCLE_RULE_SOURCE_PATH})
 199   endif()
 200
 201   set(TFLITE_QCONFIG_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_QCONFIG_FILENAME}")
 202   set(CIRCLE_QCONFIG_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_QCONFIG_FILENAME}")
 203
 204   unset(QCONFIG_SOURCE_PATH)
 205   if(EXISTS "${TFLITE_QCONFIG_SOURCE_PATH}")
 206     set(QCONFIG_SOURCE_PATH ${TFLITE_QCONFIG_SOURCE_PATH})
 207   endif()
 208   if(EXISTS "${CIRCLE_QCONFIG_SOURCE_PATH}")
 209     set(QCONFIG_SOURCE_PATH ${CIRCLE_QCONFIG_SOURCE_PATH})
 210   endif()
 211
 212   set(RECIPE_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RECIPE_FILE}")
 213   set(RULE_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RULE_FILE}")
 214   set(QCONFIG_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${QCONFIG_FILE}")
 215
 216   set(TFLITE_FILE "${RECIPE}.tflite")
 217   set(TFLITE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TFLITE_FILE}")
 218   set(CIRCLE_FILE "${RECIPE}.circle")
 219   set(CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${CIRCLE_FILE}")
 220
 221   # Copy .recipe
 222   add_custom_command(OUTPUT ${RECIPE_BINARY_PATH}
 223     COMMAND ${CMAKE_COMMAND} -E copy "${RECIPE_SOURCE_PATH}" "${RECIPE_BINARY_PATH}"
 224     DEPENDS ${RECIPE_SOURCE_PATH}
 225     COMMENT "Generate ${RECIPE_FILE}"
 226   )
 227   list(APPEND TEST_DEPS ${RECIPE_BINARY_PATH})
 228
 229   if(DEFINED RULE_SOURCE_PATH)
 230     # Copy .rule
 231     add_custom_command(OUTPUT ${RULE_BINARY_PATH}
 232       COMMAND ${CMAKE_COMMAND} -E copy "${RULE_SOURCE_PATH}" "${RULE_BINARY_PATH}"
 233       DEPENDS ${RULE_SOURCE_PATH}
 234       COMMENT "Generate ${RULE_FILE}"
 235     )
 236     list(APPEND TEST_DEPS ${RULE_BINARY_PATH})
 237   endif()
 238
 239   if(DEFINED QCONFIG_SOURCE_PATH)
 240     # Copy .qconf.json
 241     add_custom_command(OUTPUT ${QCONFIG_BINARY_PATH}
 242       COMMAND ${CMAKE_COMMAND} -E copy "${QCONFIG_SOURCE_PATH}" "${QCONFIG_BINARY_PATH}"
 243       DEPENDS ${QCONFIG_SOURCE_PATH}
 244       COMMENT "Generate ${QCONFIG_FILE}"
 245     )
 246     list(APPEND TEST_DEPS ${QCONFIG_BINARY_PATH})
 247   endif()
 248
 249   if(${MODEL_FORMAT} STREQUAL "tflite")
 250     # Generate .tflite
 251     add_custom_command(OUTPUT ${TFLITE_OUTPUT_PATH}
 252       COMMAND $<TARGET_FILE:tflchef-file> ${RECIPE_BINARY_PATH} ${TFLITE_OUTPUT_PATH}
 253       DEPENDS $<TARGET_FILE:tflchef-file> ${RECIPE_BINARY_PATH}
 254       COMMENT "Generate ${TFLITE_FILE}"
 255     )
 256     list(APPEND TEST_DEPS ${TFLITE_OUTPUT_PATH})
 257
 258     if(NOT DEFINED NO_CIRCLIZE_${RECIPE})
 259       # Generate .circle
 260       add_custom_command(OUTPUT ${CIRCLE_OUTPUT_PATH}
 261         COMMAND $<TARGET_FILE:tflite2circle> ${TFLITE_OUTPUT_PATH} ${CIRCLE_OUTPUT_PATH}
 262         DEPENDS $<TARGET_FILE:tflite2circle> ${TFLITE_OUTPUT_PATH}
 263         COMMENT "Generate ${CIRCLE_FILE}"
 264       )
 265       set(MODEL_FORMAT "circle")
 266       list(APPEND TEST_DEPS ${CIRCLE_OUTPUT_PATH})
 267     endif()
 268   else()
 269     # Generate .circle
 270     add_custom_command(OUTPUT ${CIRCLE_OUTPUT_PATH}
 271       COMMAND $<TARGET_FILE:circlechef-file> ${RECIPE_BINARY_PATH} ${CIRCLE_OUTPUT_PATH}
 272       DEPENDS $<TARGET_FILE:circlechef-file> ${RECIPE_BINARY_PATH}
 273       COMMENT "Generate ${CIRCLE_FILE}"
 274     )
 275     list(APPEND TEST_DEPS ${CIRCLE_OUTPUT_PATH})
 276   endif()
 277
 278   set(OPT_CIRCLE_FILE "${RECIPE}.opt.circle")
 279   set(OPT_CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${OPT_CIRCLE_FILE}")
 280
 281   if(NOT DEFINED NO_OPTIMIZE_${RECIPE})
 282     # Generate optimized .circle
 283     add_custom_command(OUTPUT ${OPT_CIRCLE_OUTPUT_PATH}
 284       # NOTE --resolve_customop_add is just to added for old -O1, no particular meaning
 285       #      --fold_dequantize is added to fold Tensor(FLOAT16) + DEQUANTIZE (Net_Dequantize_Add)
 286       #      model. FLOAT16 in general is NOT supported but only Tensor(FLOAT16) + DEQUANTIZE
 287       #      sequence accepted as folded to Tensor(FLOAT32).
 288       # TODO revise giving options from the list file
 289       COMMAND $<TARGET_FILE:circle2circle> --resolve_customop_add --fold_dequantize --fold_densify
 290               ${CIRCLE_OUTPUT_PATH} ${OPT_CIRCLE_OUTPUT_PATH}
 291       DEPENDS $<TARGET_FILE:circle2circle>  ${CIRCLE_OUTPUT_PATH}
 292       COMMENT "Generate ${OPT_CIRCLE_FILE}"
 293     )
 294     set(OPT_FORMAT ".opt")
 295     list(APPEND TEST_DEPS ${OPT_CIRCLE_OUTPUT_PATH})
 296   endif()
 297
 298   set(MODEL_FILE "${RECIPE}${OPT_FORMAT}.${MODEL_FORMAT}")
 299   set(MODEL_PATH "${CMAKE_CURRENT_BINARY_DIR}/${MODEL_FILE}")
 300   set(NNPKG_FILE "${RECIPE}${OPT_FORMAT}")
 301   set(NNPKG_DIR "${CMAKE_CURRENT_BINARY_DIR}/${NNPKG_FILE}")
 302   set(NNPKG_MODEL "${NNPKG_DIR}/${MODEL_FILE}")
 303
 304   # Generate nnpackage directory
 305   add_custom_command(OUTPUT ${NNPKG_DIR}
 306       COMMAND ${CMAKE_COMMAND} -E make_directory ${NNPKG_DIR}
 307       DEPENDS ${MODEL_PATH}
 308       COMMENT "Generate ${RECIPE} nnpackage directory"
 309     )
 310   list(APPEND TEST_DEPS ${NNPKG_DIR})
 311
 312   add_custom_command(OUTPUT ${NNPKG_MODEL}
 313     COMMAND ${PYTHON_EXECUTABLE} ${MODEL2NNPKG} -m ${MODEL_PATH}
 314     DEPENDS ${MODEL2NNPKG} ${MODEL_PATH}
 315     COMMENT "Generate ${RECIPE} nnpackage"
 316   )
 317   list(APPEND TEST_DEPS ${NNPKG_MODEL})
 318
 319   if(NOT DEFINED NO_TCGEN_${RECIPE})
 320     # Generate test directory
 321     set(TC_DIRECTORY "${NNPKG_DIR}/metadata/tc")
 322     add_custom_command(OUTPUT ${TC_DIRECTORY}
 323       COMMAND ${CMAKE_COMMAND} -E make_directory ${TC_DIRECTORY}
 324       COMMENT "Generate ${RECIPE} nnpackage test directory"
 325     )
 326     list(APPEND TEST_DEPS ${TC_DIRECTORY})
 327
 328     # set ADDITIONAL_OPTIONS as empty (one space before closing is intentional)
 329     set(ADDITIONAL_OPTIONS )
 330     if(DEFINED TCGEN_OPT_${RECIPE})
 331       set(ADDITIONAL_OPTIONS ${ADDITIONAL_OPTIONS} ${TCGEN_OPT_${RECIPE}})
 332     endif()
 333
 334     # Generate input.h5, expected.h5
 335     set(INPUT_HDF5_FILE "${TC_DIRECTORY}/input.h5")
 336     set(EXPECTED_HDF5_FILE "${TC_DIRECTORY}/expected.h5")
 337     add_custom_command(OUTPUT ${INPUT_HDF5_FILE} ${EXPECTED_HDF5_FILE}
 338       COMMAND $<TARGET_FILE:testDataGenerator>
 339               --input_data ${INPUT_HDF5_FILE}
 340               --expected_data ${EXPECTED_HDF5_FILE}
 341               ${ADDITIONAL_OPTIONS}
 342               ${MODEL_FILE}
 343       DEPENDS $<TARGET_FILE:testDataGenerator> ${MODEL_FILE} ${TC_DIRECTORY}
 344       COMMENT "Generate input.h5 and expected.h5 in ${NNPKG_FILE}/metadata/tc"
 345     )
 346     list(APPEND TEST_DEPS ${INPUT_HDF5_FILE} ${EXPECTED_HDF5_FILE})
 347   endif()
 348 endforeach()
 349
 350 add_custom_target(common_artifacts_deps ALL DEPENDS ${TEST_DEPS})