1 #[[ Generate common python virtual enviornment ]]
2 # NOTE find_package try to use at least python3.8 as follows depending on platform version
3 # Ubuntu18.04; explictly installed python3.8 (default is python3.6)
4 # Ubuntu20.04; default python3.8
5 # Ubuntu22.04; default python3.10
6 # refer https://github.com/Samsung/ONE/issues/9962
7 find_package(PythonInterp 3.8 QUIET)
8 find_package(PythonLibs 3.8 QUIET)
10 if(NOT ${PYTHONINTERP_FOUND})
11 message(STATUS "Build common-artifacts: FALSE (Python3 is missing)")
15 if(${PYTHON_VERSION_MINOR} LESS 8)
16 message(STATUS "Build common-artifacts: FALSE (You need to install Python version higher than 3.8)")
20 # Create python virtual environment with tensorflow 2.8.0
21 set(VIRTUALENV_OVERLAY_TF_2_8_0 "${NNCC_OVERLAY_DIR}/venv_2_8_0")
22 # TensorFlow 2.10.1 for Ubuntu22.04
23 if(ONE_UBUNTU_CODENAME_JAMMY)
24 set(VIRTUALENV_OVERLAY_TF_2_10_1 "${NNCC_OVERLAY_DIR}/venv_2_10_1")
25 endif(ONE_UBUNTU_CODENAME_JAMMY)
28 OUTPUT ${VIRTUALENV_OVERLAY_TF_2_8_0}
29 COMMAND ${PYTHON_EXECUTABLE} -m venv ${VIRTUALENV_OVERLAY_TF_2_8_0}
32 # Create requirements.txt and install required pip packages
33 set(REQUIREMENTS_FILE "requirements.txt")
34 set(REQUIREMENTS_OVERLAY_PATH_TF_2_8_0 "${VIRTUALENV_OVERLAY_TF_2_8_0}/${REQUIREMENTS_FILE}")
36 if(ONE_UBUNTU_CODENAME_JAMMY)
38 OUTPUT ${VIRTUALENV_OVERLAY_TF_2_10_1}
39 COMMAND ${PYTHON_EXECUTABLE} -m venv ${VIRTUALENV_OVERLAY_TF_2_10_1}
41 set(REQUIREMENTS_FILE "requirements.txt")
42 set(REQUIREMENTS_OVERLAY_PATH_TF_2_10_1 "${VIRTUALENV_OVERLAY_TF_2_10_1}/${REQUIREMENTS_FILE}")
43 endif(ONE_UBUNTU_CODENAME_JAMMY)
45 set(PYTHON_OVERLAY python3)
46 if(PYTHON_EXECUTABLE MATCHES python3.8)
47 set(PYTHON_OVERLAY python3.8)
50 # NOTE when using behind proxy with self signed certificate, need to set '--trusted-host' options
51 set(PIP_OPTION_TRUSTED_HOST )
52 if(DEFINED ENV{ONE_PIP_OPTION_TRUST_HOST})
53 set(PIP_OPTION_TRUSTED_HOST --trusted-host pypi.python.org --trusted-host files.pythonhosted.org --trusted-host pypi.org)
56 # NOTE refer https://github.com/protocolbuffers/protobuf/issues/10051
57 # TODO remove protobuf==3.20.1 when issue is resolved
59 OUTPUT ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0}
60 COMMAND ${CMAKE_COMMAND} -E remove -f ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0}
61 COMMAND ${CMAKE_COMMAND} -E echo "tensorflow-cpu==2.8.0" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0}
62 COMMAND ${CMAKE_COMMAND} -E echo "flatbuffers==1.12" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0}
63 COMMAND ${CMAKE_COMMAND} -E echo "protobuf==3.20.1" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0}
64 COMMAND ${CMAKE_COMMAND} -E echo "pydot==1.4.2" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0}
65 COMMAND ${VIRTUALENV_OVERLAY_TF_2_8_0}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
66 ${PIP_OPTION_TRUSTED_HOST} install --upgrade pip setuptools
67 COMMAND ${VIRTUALENV_OVERLAY_TF_2_8_0}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
68 ${PIP_OPTION_TRUSTED_HOST} install -r ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0} --upgrade
69 DEPENDS ${VIRTUALENV_OVERLAY_TF_2_8_0}
72 add_custom_target(common_artifacts_python_deps ALL
73 DEPENDS ${VIRTUALENV_OVERLAY_TF_2_8_0}
74 ${REQUIREMENTS_OVERLAY_PATH_TF_2_8_0}
77 if(ONE_UBUNTU_CODENAME_JAMMY)
79 OUTPUT ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1}
80 COMMAND ${CMAKE_COMMAND} -E remove -f ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1}
81 COMMAND ${CMAKE_COMMAND} -E echo "tensorflow-cpu==2.10.1" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1}
82 COMMAND ${CMAKE_COMMAND} -E echo "flatbuffers==23.1.21" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1}
83 COMMAND ${CMAKE_COMMAND} -E echo "protobuf==3.19.6" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1}
84 COMMAND ${CMAKE_COMMAND} -E echo "pydot==1.4.2" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1}
85 COMMAND ${VIRTUALENV_OVERLAY_TF_2_10_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
86 ${PIP_OPTION_TRUSTED_HOST} install --upgrade pip setuptools
87 COMMAND ${VIRTUALENV_OVERLAY_TF_2_10_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
88 ${PIP_OPTION_TRUSTED_HOST} install -r ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1} --upgrade
89 DEPENDS ${VIRTUALENV_OVERLAY_TF_2_10_1}
92 add_custom_target(common_artifacts_python_u22_deps ALL
93 DEPENDS ${VIRTUALENV_OVERLAY_TF_2_10_1}
94 ${REQUIREMENTS_OVERLAY_PATH_TF_2_10_1}
96 endif(ONE_UBUNTU_CODENAME_JAMMY)
98 #[[ Generate common resources ]]
100 nnas_find_package(HDF5 QUIET)
103 message(STATUS "Build common-artifacts: FAILED (missing HDF5)")
105 endif(NOT HDF5_FOUND)
107 set(SOURCES src/TestDataGenerator.cpp)
109 add_executable(testDataGenerator ${SOURCES})
110 target_include_directories(testDataGenerator PRIVATE ${HDF5_INCLUDE_DIRS})
111 target_link_libraries(testDataGenerator PRIVATE ${HDF5_CXX_LIBRARIES})
112 target_link_libraries(testDataGenerator PRIVATE arser)
113 target_link_libraries(testDataGenerator PRIVATE foder)
114 target_link_libraries(testDataGenerator PRIVATE luci_import)
115 target_link_libraries(testDataGenerator PRIVATE luci_interpreter)
116 target_link_libraries(testDataGenerator PRIVATE safemain)
120 # Include recipe repo
121 nncc_find_resource(TensorFlowLiteRecipes)
122 nncc_find_resource(CircleRecipes)
123 set(TFLITE_RECIPE_REPO "${TensorFlowLiteRecipes_DIR}")
124 set(CIRCLE_RECIPE_REPO "${CircleRecipes_DIR}")
125 set(TEST_RECIPE_FILENAME "test.recipe")
126 set(TEST_RULE_FILENAME "test.rule")
127 set(TEST_QCONFIG_FILENAME "test.qconf.json")
129 set(MODEL2NNPKG "${NNAS_PROJECT_SOURCE_DIR}/tools/nnpackage_tool/model2nnpkg/model2nnpkg.py")
132 file(GLOB TFLITE_SUBDIR RELATIVE ${TFLITE_RECIPE_REPO} ${TFLITE_RECIPE_REPO}/*)
133 foreach(DIR IN ITEMS ${TFLITE_SUBDIR})
134 if(IS_DIRECTORY ${TFLITE_RECIPE_REPO}/${DIR})
135 list(APPEND RECIPES ${DIR})
138 file(GLOB CIRCLE_SUBDIR RELATIVE ${CIRCLE_RECIPE_REPO} ${CIRCLE_RECIPE_REPO}/*)
139 foreach(DIR IN ITEMS ${CIRCLE_SUBDIR})
140 if(IS_DIRECTORY ${CIRCLE_RECIPE_REPO}/${DIR})
141 list(APPEND RECIPES ${DIR})
146 set(NO_CIRCLIZE_${NAME} TRUE)
147 set(NO_OPTIMIZE_${NAME} TRUE)
148 set(NO_TCGEN_${NAME} TRUE)
151 set(NO_OPTIMIZE_${NAME} TRUE)
153 macro(tcgenerate NAME)
154 set(NO_TCGEN_${NAME} TRUE)
157 include("exclude.lst")
159 # TODO revise using variadic arguments
160 macro(tcgenerate_option NAME OPTION ARG1 ARG2 ARG3)
161 set(TCGEN_OPT_${NAME} ${OPTION} ${ARG1} ${ARG2} ${ARG3})
164 include("options.lst")
166 foreach(RECIPE IN ITEMS ${RECIPES})
170 set(RECIPE_FILE "${RECIPE}.recipe")
171 set(RULE_FILE "${RECIPE}.rule")
172 set(QCONFIG_FILE "${RECIPE}.qconf.json")
173 set(TFLITE_RECIPE_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_RECIPE_FILENAME}")
174 set(CIRCLE_RECIPE_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_RECIPE_FILENAME}")
176 if(NOT EXISTS "${TFLITE_RECIPE_SOURCE_PATH}")
177 if(NOT EXISTS "${CIRCLE_RECIPE_SOURCE_PATH}")
178 message(FATAL_ERROR "Missing recipe of '${RECIPE}' test")
181 set(MODEL_FORMAT "circle")
182 set(RECIPE_SOURCE_PATH ${CIRCLE_RECIPE_SOURCE_PATH})
186 set(MODEL_FORMAT "tflite")
187 set(RECIPE_SOURCE_PATH ${TFLITE_RECIPE_SOURCE_PATH})
190 set(TFLITE_RULE_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_RULE_FILENAME}")
191 set(CIRCLE_RULE_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_RULE_FILENAME}")
193 unset(RULE_SOURCE_PATH)
194 if(EXISTS "${TFLITE_RULE_SOURCE_PATH}")
195 set(RULE_SOURCE_PATH ${TFLITE_RULE_SOURCE_PATH})
197 if(EXISTS "${CIRCLE_RULE_SOURCE_PATH}")
198 set(RULE_SOURCE_PATH ${CIRCLE_RULE_SOURCE_PATH})
201 set(TFLITE_QCONFIG_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_QCONFIG_FILENAME}")
202 set(CIRCLE_QCONFIG_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_QCONFIG_FILENAME}")
204 unset(QCONFIG_SOURCE_PATH)
205 if(EXISTS "${TFLITE_QCONFIG_SOURCE_PATH}")
206 set(QCONFIG_SOURCE_PATH ${TFLITE_QCONFIG_SOURCE_PATH})
208 if(EXISTS "${CIRCLE_QCONFIG_SOURCE_PATH}")
209 set(QCONFIG_SOURCE_PATH ${CIRCLE_QCONFIG_SOURCE_PATH})
212 set(RECIPE_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RECIPE_FILE}")
213 set(RULE_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RULE_FILE}")
214 set(QCONFIG_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${QCONFIG_FILE}")
216 set(TFLITE_FILE "${RECIPE}.tflite")
217 set(TFLITE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TFLITE_FILE}")
218 set(CIRCLE_FILE "${RECIPE}.circle")
219 set(CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${CIRCLE_FILE}")
222 add_custom_command(OUTPUT ${RECIPE_BINARY_PATH}
223 COMMAND ${CMAKE_COMMAND} -E copy "${RECIPE_SOURCE_PATH}" "${RECIPE_BINARY_PATH}"
224 DEPENDS ${RECIPE_SOURCE_PATH}
225 COMMENT "Generate ${RECIPE_FILE}"
227 list(APPEND TEST_DEPS ${RECIPE_BINARY_PATH})
229 if(DEFINED RULE_SOURCE_PATH)
231 add_custom_command(OUTPUT ${RULE_BINARY_PATH}
232 COMMAND ${CMAKE_COMMAND} -E copy "${RULE_SOURCE_PATH}" "${RULE_BINARY_PATH}"
233 DEPENDS ${RULE_SOURCE_PATH}
234 COMMENT "Generate ${RULE_FILE}"
236 list(APPEND TEST_DEPS ${RULE_BINARY_PATH})
239 if(DEFINED QCONFIG_SOURCE_PATH)
241 add_custom_command(OUTPUT ${QCONFIG_BINARY_PATH}
242 COMMAND ${CMAKE_COMMAND} -E copy "${QCONFIG_SOURCE_PATH}" "${QCONFIG_BINARY_PATH}"
243 DEPENDS ${QCONFIG_SOURCE_PATH}
244 COMMENT "Generate ${QCONFIG_FILE}"
246 list(APPEND TEST_DEPS ${QCONFIG_BINARY_PATH})
249 if(${MODEL_FORMAT} STREQUAL "tflite")
251 add_custom_command(OUTPUT ${TFLITE_OUTPUT_PATH}
252 COMMAND $<TARGET_FILE:tflchef-file> ${RECIPE_BINARY_PATH} ${TFLITE_OUTPUT_PATH}
253 DEPENDS $<TARGET_FILE:tflchef-file> ${RECIPE_BINARY_PATH}
254 COMMENT "Generate ${TFLITE_FILE}"
256 list(APPEND TEST_DEPS ${TFLITE_OUTPUT_PATH})
258 if(NOT DEFINED NO_CIRCLIZE_${RECIPE})
260 add_custom_command(OUTPUT ${CIRCLE_OUTPUT_PATH}
261 COMMAND $<TARGET_FILE:tflite2circle> ${TFLITE_OUTPUT_PATH} ${CIRCLE_OUTPUT_PATH}
262 DEPENDS $<TARGET_FILE:tflite2circle> ${TFLITE_OUTPUT_PATH}
263 COMMENT "Generate ${CIRCLE_FILE}"
265 set(MODEL_FORMAT "circle")
266 list(APPEND TEST_DEPS ${CIRCLE_OUTPUT_PATH})
270 add_custom_command(OUTPUT ${CIRCLE_OUTPUT_PATH}
271 COMMAND $<TARGET_FILE:circlechef-file> ${RECIPE_BINARY_PATH} ${CIRCLE_OUTPUT_PATH}
272 DEPENDS $<TARGET_FILE:circlechef-file> ${RECIPE_BINARY_PATH}
273 COMMENT "Generate ${CIRCLE_FILE}"
275 list(APPEND TEST_DEPS ${CIRCLE_OUTPUT_PATH})
278 set(OPT_CIRCLE_FILE "${RECIPE}.opt.circle")
279 set(OPT_CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${OPT_CIRCLE_FILE}")
281 if(NOT DEFINED NO_OPTIMIZE_${RECIPE})
282 # Generate optimized .circle
283 add_custom_command(OUTPUT ${OPT_CIRCLE_OUTPUT_PATH}
284 # NOTE --resolve_customop_add is just to added for old -O1, no particular meaning
285 # --fold_dequantize is added to fold Tensor(FLOAT16) + DEQUANTIZE (Net_Dequantize_Add)
286 # model. FLOAT16 in general is NOT supported but only Tensor(FLOAT16) + DEQUANTIZE
287 # sequence accepted as folded to Tensor(FLOAT32).
288 # TODO revise giving options from the list file
289 COMMAND $<TARGET_FILE:circle2circle> --resolve_customop_add --fold_dequantize --fold_densify
290 ${CIRCLE_OUTPUT_PATH} ${OPT_CIRCLE_OUTPUT_PATH}
291 DEPENDS $<TARGET_FILE:circle2circle> ${CIRCLE_OUTPUT_PATH}
292 COMMENT "Generate ${OPT_CIRCLE_FILE}"
294 set(OPT_FORMAT ".opt")
295 list(APPEND TEST_DEPS ${OPT_CIRCLE_OUTPUT_PATH})
298 set(MODEL_FILE "${RECIPE}${OPT_FORMAT}.${MODEL_FORMAT}")
299 set(MODEL_PATH "${CMAKE_CURRENT_BINARY_DIR}/${MODEL_FILE}")
300 set(NNPKG_FILE "${RECIPE}${OPT_FORMAT}")
301 set(NNPKG_DIR "${CMAKE_CURRENT_BINARY_DIR}/${NNPKG_FILE}")
302 set(NNPKG_MODEL "${NNPKG_DIR}/${MODEL_FILE}")
304 # Generate nnpackage directory
305 add_custom_command(OUTPUT ${NNPKG_DIR}
306 COMMAND ${CMAKE_COMMAND} -E make_directory ${NNPKG_DIR}
307 DEPENDS ${MODEL_PATH}
308 COMMENT "Generate ${RECIPE} nnpackage directory"
310 list(APPEND TEST_DEPS ${NNPKG_DIR})
312 add_custom_command(OUTPUT ${NNPKG_MODEL}
313 COMMAND ${PYTHON_EXECUTABLE} ${MODEL2NNPKG} -m ${MODEL_PATH}
314 DEPENDS ${MODEL2NNPKG} ${MODEL_PATH}
315 COMMENT "Generate ${RECIPE} nnpackage"
317 list(APPEND TEST_DEPS ${NNPKG_MODEL})
319 if(NOT DEFINED NO_TCGEN_${RECIPE})
320 # Generate test directory
321 set(TC_DIRECTORY "${NNPKG_DIR}/metadata/tc")
322 add_custom_command(OUTPUT ${TC_DIRECTORY}
323 COMMAND ${CMAKE_COMMAND} -E make_directory ${TC_DIRECTORY}
324 COMMENT "Generate ${RECIPE} nnpackage test directory"
326 list(APPEND TEST_DEPS ${TC_DIRECTORY})
328 # set ADDITIONAL_OPTIONS as empty (one space before closing is intentional)
329 set(ADDITIONAL_OPTIONS )
330 if(DEFINED TCGEN_OPT_${RECIPE})
331 set(ADDITIONAL_OPTIONS ${ADDITIONAL_OPTIONS} ${TCGEN_OPT_${RECIPE}})
334 # Generate input.h5, expected.h5
335 set(INPUT_HDF5_FILE "${TC_DIRECTORY}/input.h5")
336 set(EXPECTED_HDF5_FILE "${TC_DIRECTORY}/expected.h5")
337 add_custom_command(OUTPUT ${INPUT_HDF5_FILE} ${EXPECTED_HDF5_FILE}
338 COMMAND $<TARGET_FILE:testDataGenerator>
339 --input_data ${INPUT_HDF5_FILE}
340 --expected_data ${EXPECTED_HDF5_FILE}
341 ${ADDITIONAL_OPTIONS}
343 DEPENDS $<TARGET_FILE:testDataGenerator> ${MODEL_FILE} ${TC_DIRECTORY}
344 COMMENT "Generate input.h5 and expected.h5 in ${NNPKG_FILE}/metadata/tc"
346 list(APPEND TEST_DEPS ${INPUT_HDF5_FILE} ${EXPECTED_HDF5_FILE})
350 add_custom_target(common_artifacts_deps ALL DEPENDS ${TEST_DEPS})