1 #[[ Generate common python virtual enviornment ]]
2 # NOTE find_package try to use at least python3.8 as follows depending on platform version
3 # Ubuntu18.04; explictly installed python3.8 (default is python3.6)
4 # Ubuntu20.04; default python3.8
5 # Ubuntu22.04; default python3.10
6 # refer https://github.com/Samsung/ONE/issues/9962
7 find_package(PythonInterp 3.8 QUIET)
8 find_package(PythonLibs 3.8 QUIET)
10 if(NOT ${PYTHONINTERP_FOUND})
11 message(STATUS "Build common-artifacts: FALSE (Python3 is missing)")
15 if(${PYTHON_VERSION_MINOR} LESS 8)
16 message(STATUS "Build common-artifacts: FALSE (You need to install Python version higher than 3.8)")
20 # Create python virtual environment with tensorflow 2.12.1
21 set(VIRTUALENV_OVERLAY_TF_2_12_1 "${NNCC_OVERLAY_DIR}/venv_2_12_1")
24 OUTPUT ${VIRTUALENV_OVERLAY_TF_2_12_1}
25 COMMAND ${PYTHON_EXECUTABLE} -m venv ${VIRTUALENV_OVERLAY_TF_2_12_1}
28 # Create requirements.txt and install required pip packages
29 set(REQUIREMENTS_FILE "requirements.txt")
30 set(REQUIREMENTS_OVERLAY_PATH_TF_2_12_1 "${VIRTUALENV_OVERLAY_TF_2_12_1}/${REQUIREMENTS_FILE}")
32 set(PYTHON_OVERLAY python3)
33 if(PYTHON_EXECUTABLE MATCHES python3.8)
34 set(PYTHON_OVERLAY python3.8)
37 # NOTE when using behind proxy with self signed certificate, need to set '--trusted-host' options
38 set(PIP_OPTION_TRUSTED_HOST )
39 if(DEFINED ENV{ONE_PIP_OPTION_TRUST_HOST})
40 set(PIP_OPTION_TRUSTED_HOST --trusted-host pypi.python.org --trusted-host files.pythonhosted.org --trusted-host pypi.org)
43 if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
44 # NOTE `tensorflow-cpu` package is not available for aarch64, so we use `tensorflow` package.
46 OUTPUT ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
47 COMMAND ${CMAKE_COMMAND} -E remove -f ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
48 COMMAND ${CMAKE_COMMAND} -E echo "tensorflow==2.12.1" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
49 COMMAND ${CMAKE_COMMAND} -E echo "flatbuffers==23.5.26" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
50 COMMAND ${CMAKE_COMMAND} -E echo "protobuf==4.23.3" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
51 COMMAND ${CMAKE_COMMAND} -E echo "pydot==1.4.2" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
52 COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
53 ${PIP_OPTION_TRUSTED_HOST} install --upgrade pip setuptools
54 COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
55 ${PIP_OPTION_TRUSTED_HOST} install -r ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1} --upgrade
56 DEPENDS ${VIRTUALENV_OVERLAY_TF_2_12_1}
58 else(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
60 OUTPUT ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
61 COMMAND ${CMAKE_COMMAND} -E remove -f ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
62 COMMAND ${CMAKE_COMMAND} -E echo "tensorflow-cpu==2.12.1" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
63 COMMAND ${CMAKE_COMMAND} -E echo "flatbuffers==23.5.26" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
64 COMMAND ${CMAKE_COMMAND} -E echo "protobuf==4.23.3" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
65 COMMAND ${CMAKE_COMMAND} -E echo "pydot==1.4.2" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
66 COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
67 ${PIP_OPTION_TRUSTED_HOST} install --upgrade pip setuptools
68 COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
69 ${PIP_OPTION_TRUSTED_HOST} install -r ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1} --upgrade
70 DEPENDS ${VIRTUALENV_OVERLAY_TF_2_12_1}
72 endif(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
74 add_custom_target(common_artifacts_python_deps ALL
75 DEPENDS ${VIRTUALENV_OVERLAY_TF_2_12_1}
76 ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
79 #[[ Generate common resources ]]
81 nnas_find_package(HDF5 QUIET)
84 message(STATUS "Build common-artifacts: FAILED (missing HDF5)")
88 set(SOURCES src/TestDataGenerator.cpp)
90 add_executable(testDataGenerator ${SOURCES})
91 target_include_directories(testDataGenerator PRIVATE ${HDF5_INCLUDE_DIRS})
92 target_link_libraries(testDataGenerator PRIVATE ${HDF5_CXX_LIBRARIES})
93 target_link_libraries(testDataGenerator PRIVATE arser)
94 target_link_libraries(testDataGenerator PRIVATE foder)
95 target_link_libraries(testDataGenerator PRIVATE luci_import)
96 target_link_libraries(testDataGenerator PRIVATE luci_interpreter)
97 target_link_libraries(testDataGenerator PRIVATE safemain)
101 # Include recipe repo
102 nncc_find_resource(TensorFlowLiteRecipes)
103 nncc_find_resource(CircleRecipes)
104 set(TFLITE_RECIPE_REPO "${TensorFlowLiteRecipes_DIR}")
105 set(CIRCLE_RECIPE_REPO "${CircleRecipes_DIR}")
106 set(TEST_RECIPE_FILENAME "test.recipe")
107 set(TEST_RULE_FILENAME "test.rule")
108 set(TEST_QCONFIG_FILENAME "test.qconf.json")
110 set(MODEL2NNPKG "${NNAS_PROJECT_SOURCE_DIR}/tools/nnpackage_tool/model2nnpkg/model2nnpkg.py")
113 file(GLOB TFLITE_SUBDIR RELATIVE ${TFLITE_RECIPE_REPO} ${TFLITE_RECIPE_REPO}/*)
114 foreach(DIR IN ITEMS ${TFLITE_SUBDIR})
115 if(IS_DIRECTORY ${TFLITE_RECIPE_REPO}/${DIR})
116 list(APPEND RECIPES ${DIR})
119 file(GLOB CIRCLE_SUBDIR RELATIVE ${CIRCLE_RECIPE_REPO} ${CIRCLE_RECIPE_REPO}/*)
120 foreach(DIR IN ITEMS ${CIRCLE_SUBDIR})
121 if(IS_DIRECTORY ${CIRCLE_RECIPE_REPO}/${DIR})
122 list(APPEND RECIPES ${DIR})
127 set(NO_CIRCLIZE_${NAME} TRUE)
128 set(NO_OPTIMIZE_${NAME} TRUE)
129 set(NO_TCGEN_${NAME} TRUE)
132 set(NO_OPTIMIZE_${NAME} TRUE)
134 macro(tcgenerate NAME)
135 set(NO_TCGEN_${NAME} TRUE)
138 include("exclude.lst")
140 # TODO revise using variadic arguments
141 macro(tcgenerate_option NAME OPTION ARG1 ARG2 ARG3)
142 set(TCGEN_OPT_${NAME} ${OPTION} ${ARG1} ${ARG2} ${ARG3})
145 include("options.lst")
147 foreach(RECIPE IN ITEMS ${RECIPES})
151 set(RECIPE_FILE "${RECIPE}.recipe")
152 set(RULE_FILE "${RECIPE}.rule")
153 set(QCONFIG_FILE "${RECIPE}.qconf.json")
154 set(TFLITE_RECIPE_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_RECIPE_FILENAME}")
155 set(CIRCLE_RECIPE_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_RECIPE_FILENAME}")
157 if(NOT EXISTS "${TFLITE_RECIPE_SOURCE_PATH}")
158 if(NOT EXISTS "${CIRCLE_RECIPE_SOURCE_PATH}")
159 message(FATAL_ERROR "Missing recipe of '${RECIPE}' test")
162 set(MODEL_FORMAT "circle")
163 set(RECIPE_SOURCE_PATH ${CIRCLE_RECIPE_SOURCE_PATH})
167 set(MODEL_FORMAT "tflite")
168 set(RECIPE_SOURCE_PATH ${TFLITE_RECIPE_SOURCE_PATH})
171 set(TFLITE_RULE_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_RULE_FILENAME}")
172 set(CIRCLE_RULE_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_RULE_FILENAME}")
174 unset(RULE_SOURCE_PATH)
175 if(EXISTS "${TFLITE_RULE_SOURCE_PATH}")
176 set(RULE_SOURCE_PATH ${TFLITE_RULE_SOURCE_PATH})
178 if(EXISTS "${CIRCLE_RULE_SOURCE_PATH}")
179 set(RULE_SOURCE_PATH ${CIRCLE_RULE_SOURCE_PATH})
182 set(TFLITE_QCONFIG_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_QCONFIG_FILENAME}")
183 set(CIRCLE_QCONFIG_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_QCONFIG_FILENAME}")
185 unset(QCONFIG_SOURCE_PATH)
186 if(EXISTS "${TFLITE_QCONFIG_SOURCE_PATH}")
187 set(QCONFIG_SOURCE_PATH ${TFLITE_QCONFIG_SOURCE_PATH})
189 if(EXISTS "${CIRCLE_QCONFIG_SOURCE_PATH}")
190 set(QCONFIG_SOURCE_PATH ${CIRCLE_QCONFIG_SOURCE_PATH})
193 set(RECIPE_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RECIPE_FILE}")
194 set(RULE_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RULE_FILE}")
195 set(QCONFIG_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${QCONFIG_FILE}")
197 set(TFLITE_FILE "${RECIPE}.tflite")
198 set(TFLITE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TFLITE_FILE}")
199 set(CIRCLE_FILE "${RECIPE}.circle")
200 set(CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${CIRCLE_FILE}")
203 add_custom_command(OUTPUT ${RECIPE_BINARY_PATH}
204 COMMAND ${CMAKE_COMMAND} -E copy "${RECIPE_SOURCE_PATH}" "${RECIPE_BINARY_PATH}"
205 DEPENDS ${RECIPE_SOURCE_PATH}
206 COMMENT "Generate ${RECIPE_FILE}"
208 list(APPEND TEST_DEPS ${RECIPE_BINARY_PATH})
210 if(DEFINED RULE_SOURCE_PATH)
212 add_custom_command(OUTPUT ${RULE_BINARY_PATH}
213 COMMAND ${CMAKE_COMMAND} -E copy "${RULE_SOURCE_PATH}" "${RULE_BINARY_PATH}"
214 DEPENDS ${RULE_SOURCE_PATH}
215 COMMENT "Generate ${RULE_FILE}"
217 list(APPEND TEST_DEPS ${RULE_BINARY_PATH})
220 if(DEFINED QCONFIG_SOURCE_PATH)
222 add_custom_command(OUTPUT ${QCONFIG_BINARY_PATH}
223 COMMAND ${CMAKE_COMMAND} -E copy "${QCONFIG_SOURCE_PATH}" "${QCONFIG_BINARY_PATH}"
224 DEPENDS ${QCONFIG_SOURCE_PATH}
225 COMMENT "Generate ${QCONFIG_FILE}"
227 list(APPEND TEST_DEPS ${QCONFIG_BINARY_PATH})
230 if(${MODEL_FORMAT} STREQUAL "tflite")
232 add_custom_command(OUTPUT ${TFLITE_OUTPUT_PATH}
233 COMMAND $<TARGET_FILE:tflchef-file> ${RECIPE_BINARY_PATH} ${TFLITE_OUTPUT_PATH}
234 DEPENDS $<TARGET_FILE:tflchef-file> ${RECIPE_BINARY_PATH}
235 COMMENT "Generate ${TFLITE_FILE}"
237 list(APPEND TEST_DEPS ${TFLITE_OUTPUT_PATH})
239 if(NOT DEFINED NO_CIRCLIZE_${RECIPE})
241 add_custom_command(OUTPUT ${CIRCLE_OUTPUT_PATH}
242 COMMAND $<TARGET_FILE:tflite2circle> ${TFLITE_OUTPUT_PATH} ${CIRCLE_OUTPUT_PATH}
243 DEPENDS $<TARGET_FILE:tflite2circle> ${TFLITE_OUTPUT_PATH}
244 COMMENT "Generate ${CIRCLE_FILE}"
246 set(MODEL_FORMAT "circle")
247 list(APPEND TEST_DEPS ${CIRCLE_OUTPUT_PATH})
251 add_custom_command(OUTPUT ${CIRCLE_OUTPUT_PATH}
252 COMMAND $<TARGET_FILE:circlechef-file> ${RECIPE_BINARY_PATH} ${CIRCLE_OUTPUT_PATH}
253 DEPENDS $<TARGET_FILE:circlechef-file> ${RECIPE_BINARY_PATH}
254 COMMENT "Generate ${CIRCLE_FILE}"
256 list(APPEND TEST_DEPS ${CIRCLE_OUTPUT_PATH})
259 set(OPT_CIRCLE_FILE "${RECIPE}.opt.circle")
260 set(OPT_CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${OPT_CIRCLE_FILE}")
262 if(NOT DEFINED NO_OPTIMIZE_${RECIPE})
263 # Generate optimized .circle
264 add_custom_command(OUTPUT ${OPT_CIRCLE_OUTPUT_PATH}
265 # NOTE --resolve_customop_add is just to added for old -O1, no particular meaning
266 # --fold_dequantize is added to fold Tensor(FLOAT16) + DEQUANTIZE (Net_Dequantize_Add)
267 # model. FLOAT16 in general is NOT supported but only Tensor(FLOAT16) + DEQUANTIZE
268 # sequence accepted as folded to Tensor(FLOAT32).
269 # TODO revise giving options from the list file
270 COMMAND $<TARGET_FILE:circle2circle> --resolve_customop_add --fold_dequantize --fold_densify
271 ${CIRCLE_OUTPUT_PATH} ${OPT_CIRCLE_OUTPUT_PATH}
272 DEPENDS $<TARGET_FILE:circle2circle> ${CIRCLE_OUTPUT_PATH}
273 COMMENT "Generate ${OPT_CIRCLE_FILE}"
275 set(OPT_FORMAT ".opt")
276 list(APPEND TEST_DEPS ${OPT_CIRCLE_OUTPUT_PATH})
279 set(MODEL_FILE "${RECIPE}${OPT_FORMAT}.${MODEL_FORMAT}")
280 set(MODEL_PATH "${CMAKE_CURRENT_BINARY_DIR}/${MODEL_FILE}")
281 set(NNPKG_FILE "${RECIPE}${OPT_FORMAT}")
282 set(NNPKG_DIR "${CMAKE_CURRENT_BINARY_DIR}/${NNPKG_FILE}")
283 set(NNPKG_MODEL "${NNPKG_DIR}/${MODEL_FILE}")
285 # Generate nnpackage directory
286 add_custom_command(OUTPUT ${NNPKG_DIR}
287 COMMAND ${CMAKE_COMMAND} -E make_directory ${NNPKG_DIR}
288 DEPENDS ${MODEL_PATH}
289 COMMENT "Generate ${RECIPE} nnpackage directory"
291 list(APPEND TEST_DEPS ${NNPKG_DIR})
293 add_custom_command(OUTPUT ${NNPKG_MODEL}
294 COMMAND ${PYTHON_EXECUTABLE} ${MODEL2NNPKG} -m ${MODEL_PATH}
295 DEPENDS ${MODEL2NNPKG} ${MODEL_PATH}
296 COMMENT "Generate ${RECIPE} nnpackage"
298 list(APPEND TEST_DEPS ${NNPKG_MODEL})
300 if(NOT DEFINED NO_TCGEN_${RECIPE})
301 # Generate test directory
302 set(TC_DIRECTORY "${NNPKG_DIR}/metadata/tc")
303 add_custom_command(OUTPUT ${TC_DIRECTORY}
304 COMMAND ${CMAKE_COMMAND} -E make_directory ${TC_DIRECTORY}
305 COMMENT "Generate ${RECIPE} nnpackage test directory"
307 list(APPEND TEST_DEPS ${TC_DIRECTORY})
309 # set ADDITIONAL_OPTIONS as empty (one space before closing is intentional)
310 set(ADDITIONAL_OPTIONS )
311 if(DEFINED TCGEN_OPT_${RECIPE})
312 set(ADDITIONAL_OPTIONS ${ADDITIONAL_OPTIONS} ${TCGEN_OPT_${RECIPE}})
315 # Generate input.h5, expected.h5
316 set(INPUT_HDF5_FILE "${TC_DIRECTORY}/input.h5")
317 set(EXPECTED_HDF5_FILE "${TC_DIRECTORY}/expected.h5")
318 add_custom_command(OUTPUT ${INPUT_HDF5_FILE} ${EXPECTED_HDF5_FILE}
319 COMMAND $<TARGET_FILE:testDataGenerator>
320 --input_data ${INPUT_HDF5_FILE}
321 --expected_data ${EXPECTED_HDF5_FILE}
322 ${ADDITIONAL_OPTIONS}
324 DEPENDS $<TARGET_FILE:testDataGenerator> ${MODEL_FILE} ${TC_DIRECTORY}
325 COMMENT "Generate input.h5 and expected.h5 in ${NNPKG_FILE}/metadata/tc"
327 list(APPEND TEST_DEPS ${INPUT_HDF5_FILE} ${EXPECTED_HDF5_FILE})
331 add_custom_target(common_artifacts_deps ALL DEPENDS ${TEST_DEPS})