From d4a1b52d6440ba305f51bc64cdb4a11cee43a304 Mon Sep 17 00:00:00 2001 From: Yongjoo Ahn Date: Mon, 13 Jan 2025 15:44:02 +0900 Subject: [PATCH] Initialize hal backend for ml - This implements hal backends for hal-api-ml. Signed-off-by: Yongjoo Ahn --- CMakeLists.txt | 61 +++ LICENSE | 204 +++++++++++ packaging/hal-backend-ml.manifest | 5 + packaging/hal-backend-ml.spec | 93 +++++ src/hal-backend-ml-snpe.cc | 489 +++++++++++++++++++++++++ src/hal-backend-ml-test-passthrough.cc | 124 +++++++ src/hal-backend-ml-util.cc | 185 ++++++++++ src/hal-backend-ml-util.h | 26 ++ src/hal-backend-ml-vivante.cc | 230 ++++++++++++ 9 files changed, 1417 insertions(+) create mode 100644 CMakeLists.txt create mode 100644 LICENSE create mode 100644 packaging/hal-backend-ml.manifest create mode 100644 packaging/hal-backend-ml.spec create mode 100644 src/hal-backend-ml-snpe.cc create mode 100644 src/hal-backend-ml-test-passthrough.cc create mode 100644 src/hal-backend-ml-util.cc create mode 100644 src/hal-backend-ml-util.h create mode 100644 src/hal-backend-ml-vivante.cc diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..0f62002 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,61 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +PROJECT(hal-backend-ml CXX) +INCLUDE(GNUInstallDirs) + +option(ENABLE_VIVANTE "Enable vivante backend" OFF) +option(ENABLE_SNPE "Enable snpe backend" OFF) + +SET(HAL_LIBDIR ${CMAKE_HAL_LIBDIR_PREFIX}) +SET(HAL_LICENSEDIR ${CMAKE_HAL_LICENSEDIR_PREFIX}) + +INCLUDE(FindPkgConfig) +pkg_check_modules(hal_rootstrap_pkgs REQUIRED + hal-rootstrap +) + +FOREACH(flag ${hal_rootstrap_pkgs_CFLAGS}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${flag}") +ENDFOREACH(flag) + +# Common Options +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -O2 -fomit-frame-pointer -std=gnu++0x") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdata-sections -ffunction-sections") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") + +SET(UTIL_SRCS + ${PROJECT_SOURCE_DIR}/src/hal-backend-ml-util.cc +) + +# pass-through +SET(PASS_SRCS + ${PROJECT_SOURCE_DIR}/src/hal-backend-ml-test-passthrough.cc +) + +SET(PASS_LIBRARY_NAME "hal-backend-ml-pass") +ADD_LIBRARY(${PASS_LIBRARY_NAME} SHARED ${PASS_SRCS} ${UTIL_SRCS}) +TARGET_LINK_LIBRARIES(${PASS_LIBRARY_NAME} ${hal_rootstrap_pkgs_LDFLAGS}) +INSTALL(TARGETS ${PASS_LIBRARY_NAME} DESTINATION ${HAL_LIBDIR} COMPONENT RuntimeLibraries) + +# vivante +IF(ENABLE_VIVANTE) +SET(VIVANTE_SRCS +${PROJECT_SOURCE_DIR}/src/hal-backend-ml-vivante.cc +) + +SET(VIVANTE_LIBRARY_NAME "hal-backend-ml-vivante") +ADD_LIBRARY(${VIVANTE_LIBRARY_NAME} SHARED ${VIVANTE_SRCS} ${UTIL_SRCS}) +TARGET_LINK_LIBRARIES(${VIVANTE_LIBRARY_NAME} ${hal_rootstrap_pkgs_LDFLAGS}) +INSTALL(TARGETS ${VIVANTE_LIBRARY_NAME} DESTINATION ${HAL_LIBDIR} COMPONENT RuntimeLibraries) +ENDIF() + +# snpe +IF(ENABLE_SNPE) +SET(SNPE_SRCS +${PROJECT_SOURCE_DIR}/src/hal-backend-ml-snpe.cc +) + +SET(SNPE_LIBRARY_NAME "hal-backend-ml-snpe") +ADD_LIBRARY(${SNPE_LIBRARY_NAME} SHARED ${SNPE_SRCS} ${UTIL_SRCS}) +TARGET_LINK_LIBRARIES(${SNPE_LIBRARY_NAME} ${hal_rootstrap_pkgs_LDFLAGS}) +INSTALL(TARGETS ${SNPE_LIBRARY_NAME} DESTINATION ${HAL_LIBDIR} COMPONENT RuntimeLibraries) +ENDIF() diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8f17f50 --- /dev/null +++ b/LICENSE @@ -0,0 +1,204 @@ +Copyright (c) 2000 - 2011 Samsung Electronics Co., Ltd. All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/packaging/hal-backend-ml.manifest b/packaging/hal-backend-ml.manifest new file mode 100644 index 0000000..97e8c31 --- /dev/null +++ b/packaging/hal-backend-ml.manifest @@ -0,0 +1,5 @@ + + + + + diff --git a/packaging/hal-backend-ml.spec b/packaging/hal-backend-ml.spec new file mode 100644 index 0000000..210ffa2 --- /dev/null +++ b/packaging/hal-backend-ml.spec @@ -0,0 +1,93 @@ +Name: hal-backend-ml +Summary: ML HAL backend drivers for various targets +Version: 0.0.1 +Release: 0 +Group: Machine Learning/ML Framework +License: Apache-2.0 +Source0: %{name}-%{version}.tar.gz + +BuildRequires: cmake +BuildRequires: pkgconfig(hal-rootstrap) + + +# For DA +%if 0%{?_with_da_profile} + +## For meson board +%if 0%{?_with_meson64} +%define vivante_support 1 +BuildRequires: pkgconfig(ovxlib) +BuildRequires: pkgconfig(amlogic-vsi-npu-sdk) +%endif + +## For qrb board +%if 0%{?_with_qrb4210} +%define snpe_support 1 +BuildRequires: snpe-devel +%endif + +%endif # For DA + + +%description +ML HAL backend drivers for various targets + + +# Config vivante +%if 0%{?vivante_support} +%package vivante +Summary: hal-backend-ml for vivante +%description vivante +%define enable_vivante -DENABLE_VIVANTE=ON +%endif + +# Config snpe +%if 0%{?snpe_support} +%package snpe +Summary: hal-backend-ml for snpe +Requires: snpe +%description snpe +%define enable_snpe -DENABLE_SNPE=ON +%endif + + +%prep +%setup -q + +%build +%cmake \ + -DCMAKE_HAL_LIBDIR_PREFIX=%{_hal_libdir} \ + -DCMAKE_HAL_LICENSEDIR_PREFIX=%{_hal_licensedir} \ + %{?enable_vivante} \ + %{?enable_snpe} \ + . + +make %{?_smp_mflags} + +%install +%make_install + +%post +/sbin/ldconfig + +%postun +/sbin/ldconfig + +%files +%manifest packaging/hal-backend-ml.manifest +%license LICENSE +%{_hal_libdir}/libhal-backend-ml-*.so* + +%if 0%{?vivante_support} +%files vivante +%manifest packaging/hal-backend-ml.manifest +%license LICENSE +%{_hal_libdir}/libhal-backend-ml-vivante.so +%endif + +%if 0%{?snpe_support} +%files snpe +%manifest packaging/hal-backend-ml.manifest +%license LICENSE +%{_hal_libdir}/libhal-backend-ml-snpe.so +%endif diff --git a/src/hal-backend-ml-snpe.cc b/src/hal-backend-ml-snpe.cc new file mode 100644 index 0000000..5e9bef0 --- /dev/null +++ b/src/hal-backend-ml-snpe.cc @@ -0,0 +1,489 @@ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hal-backend-ml-util.h" + + +typedef struct _snpe_handle_s +{ + char *model_path; + GstTensorsInfo inputInfo; /**< Input tensors metadata */ + GstTensorsInfo outputInfo; /**< Output tensors metadata */ + + Snpe_SNPE_Handle_t snpe_h; + Snpe_UserBufferMap_Handle_t inputMap_h; + Snpe_UserBufferMap_Handle_t outputMap_h; + std::vector user_buffers; +} snpe_handle_s; + +static int ml_snpe_init(void **backend_private) +{ + snpe_handle_s *snpe = g_new0 (snpe_handle_s, 1); + + gst_tensors_info_init (&snpe->inputInfo); + gst_tensors_info_init (&snpe->outputInfo); + + *backend_private = snpe; + return 0; +} + +static int ml_snpe_deinit(void *backend_private) +{ + snpe_handle_s *snpe = (snpe_handle_s *) backend_private; + if (!snpe) { + g_critical ("[snpe backend] ml_snpe_deinit called with invalid backend_private"); + return HAL_ML_ERROR_INVALID_PARAMETER; + } + + if (snpe->inputMap_h) + Snpe_UserBufferMap_Delete (snpe->inputMap_h); + + if (snpe->outputMap_h) + Snpe_UserBufferMap_Delete (snpe->outputMap_h); + + for (auto &ub : snpe->user_buffers) + if (ub) + Snpe_IUserBuffer_Delete (ub); + + snpe->user_buffers.clear (); + + if (snpe->snpe_h) + Snpe_SNPE_Delete (snpe->snpe_h); + + if (snpe->model_path) + g_free (snpe->model_path); + + gst_tensors_info_free (&snpe->inputInfo); + gst_tensors_info_free (&snpe->outputInfo); + + g_free (snpe); + return 0; +} + +static int ml_snpe_configure_instance(void *backend_private, const GstTensorFilterProperties *prop) +{ + snpe_handle_s *snpe = (snpe_handle_s *) backend_private; + if (!snpe) { + g_critical ("[snpe backend] ml_snpe_configure_instance called with invalid backend_private"); + return HAL_ML_ERROR_INVALID_PARAMETER; + } + + Snpe_DlVersion_Handle_t lib_version_h = NULL; + Snpe_RuntimeList_Handle_t runtime_list_h = NULL; + Snpe_DlContainer_Handle_t container_h = NULL; + Snpe_SNPEBuilder_Handle_t snpebuilder_h = NULL; + Snpe_StringList_Handle_t inputstrListHandle = NULL; + Snpe_StringList_Handle_t outputstrListHandle = NULL; + std::vector inputTypeVec; + std::vector outputTypeVec; + + auto _clean_handles = [&] () { + if (lib_version_h) + Snpe_DlVersion_Delete (lib_version_h); + if (runtime_list_h) + Snpe_RuntimeList_Delete (runtime_list_h); + if (container_h) + Snpe_DlContainer_Delete (container_h); + if (snpebuilder_h) + Snpe_SNPEBuilder_Delete (snpebuilder_h); + if (inputstrListHandle) + Snpe_StringList_Delete (inputstrListHandle); + if (outputstrListHandle) + Snpe_StringList_Delete (outputstrListHandle); + }; + + /* default runtime is CPU */ + Snpe_Runtime_t runtime = SNPE_RUNTIME_CPU; + + /* lambda function to handle tensor */ + auto handleTensor = [&] (const char *tensorName, GstTensorInfo *info, + Snpe_UserBufferMap_Handle_t bufferMapHandle, + Snpe_UserBufferEncoding_ElementType_t type) { + Snpe_IBufferAttributes_Handle_t bufferAttributesOpt + = Snpe_SNPE_GetInputOutputBufferAttributes (snpe->snpe_h, tensorName); + if (!bufferAttributesOpt) + throw std::runtime_error ("Error obtaining buffer attributes"); + + auto default_type = Snpe_IBufferAttributes_GetEncodingType (bufferAttributesOpt); + + /* parse tensor data type with user given element type */ + switch (type) { + case SNPE_USERBUFFERENCODING_ELEMENTTYPE_UNKNOWN: + /* If the type is not provided by user, use default type */ + type = default_type; + if (default_type == SNPE_USERBUFFERENCODING_ELEMENTTYPE_FLOAT) { + info->type = _NNS_FLOAT32; + } else if (default_type == SNPE_USERBUFFERENCODING_ELEMENTTYPE_TF8) { + info->type = _NNS_UINT8; + } else { + throw std::invalid_argument ("Unsupported data type"); + } + break; + case SNPE_USERBUFFERENCODING_ELEMENTTYPE_FLOAT: + info->type = _NNS_FLOAT32; + break; + case SNPE_USERBUFFERENCODING_ELEMENTTYPE_TF8: + info->type = _NNS_UINT8; + if (default_type == SNPE_USERBUFFERENCODING_ELEMENTTYPE_FLOAT) { + throw std::invalid_argument ( + "ERROR: Quantization parameters are not present in model. Use TF8 type."); + } + break; + default: + throw std::invalid_argument ("Unsupported data type"); + } + + /* parse tensor dimension */ + auto shapeHandle = Snpe_IBufferAttributes_GetDims (bufferAttributesOpt); + auto rank = Snpe_TensorShape_Rank (shapeHandle); + const size_t *sdims = Snpe_TensorShape_GetDimensions (shapeHandle); + for (size_t j = 0; j < rank; j++) { + info->dimension[rank - 1 - j] = sdims[j]; + } + + /* calculate strides */ + std::vector strides (rank); + strides[rank - 1] = gst_tensor_get_element_size (info->type); + for (size_t j = rank - 1; j > 0; j--) { + strides[j - 1] = strides[j] * sdims[j]; + } + + auto stride_h = Snpe_TensorShape_CreateDimsSize (strides.data (), strides.size ()); + Snpe_TensorShape_Delete (shapeHandle); + Snpe_IBufferAttributes_Delete (bufferAttributesOpt); + + /* assign user_buffermap */ + size_t bufsize = gst_tensor_info_get_size (info); + Snpe_UserBufferEncoding_Handle_t ube_h = NULL; + if (type == SNPE_USERBUFFERENCODING_ELEMENTTYPE_TF8) { + Snpe_IBufferAttributes_Handle_t bufferAttributesOpt + = Snpe_SNPE_GetInputOutputBufferAttributes (snpe->snpe_h, tensorName); + Snpe_UserBufferEncoding_Handle_t ubeTfNHandle + = Snpe_IBufferAttributes_GetEncoding_Ref (bufferAttributesOpt); + uint64_t stepEquivalentTo0 = Snpe_UserBufferEncodingTfN_GetStepExactly0 (ubeTfNHandle); + float quantizedStepSize + = Snpe_UserBufferEncodingTfN_GetQuantizedStepSize (ubeTfNHandle); + ube_h = Snpe_UserBufferEncodingTfN_Create (stepEquivalentTo0, quantizedStepSize, 8); + Snpe_IBufferAttributes_Delete (bufferAttributesOpt); + Snpe_UserBufferEncodingTfN_Delete (ubeTfNHandle); + } else if (type == SNPE_USERBUFFERENCODING_ELEMENTTYPE_FLOAT) { + ube_h = Snpe_UserBufferEncodingFloat_Create (); + } + auto iub = Snpe_Util_CreateUserBuffer (NULL, bufsize, stride_h, ube_h); + snpe->user_buffers.push_back (iub); + + if (type == SNPE_USERBUFFERENCODING_ELEMENTTYPE_TF8) + Snpe_UserBufferEncodingTfN_Delete (ube_h); + else if (type == SNPE_USERBUFFERENCODING_ELEMENTTYPE_FLOAT) + Snpe_UserBufferEncodingFloat_Delete (ube_h); + Snpe_TensorShape_Delete (stride_h); + + Snpe_UserBufferMap_Add (bufferMapHandle, tensorName, iub); + }; + + auto parse_custom_prop = [&runtime, &outputstrListHandle, &inputTypeVec, + &outputTypeVec] (const char *custom_prop) { + if (!custom_prop) + return; + + gchar **options = g_strsplit (custom_prop, ",", -1); + + for (guint op = 0; op < g_strv_length (options); ++op) { + gchar **option = g_strsplit (options[op], ":", -1); + + if (g_strv_length (option) > 1) { + g_strstrip (option[0]); + g_strstrip (option[1]); + + if (g_ascii_strcasecmp (option[0], "Runtime") == 0) { + if (g_ascii_strcasecmp (option[1], "CPU") == 0) { + runtime = SNPE_RUNTIME_CPU; + } else if (g_ascii_strcasecmp (option[1], "GPU") == 0) { + runtime = SNPE_RUNTIME_GPU; + } else if (g_ascii_strcasecmp (option[1], "DSP") == 0) { + runtime = SNPE_RUNTIME_DSP; + } else if (g_ascii_strcasecmp (option[1], "NPU") == 0 + || g_ascii_strcasecmp (option[1], "AIP") == 0) { + runtime = SNPE_RUNTIME_AIP_FIXED8_TF; + } else { + g_warning ("Unknown runtime (%s), set CPU as default.", options[op]); + } + } else if (g_ascii_strcasecmp (option[0], "OutputTensor") == 0) { + /* the tensor name may contain ':' */ + gchar *_ot_str = g_strjoinv (":", &option[1]); + gchar **names = g_strsplit (_ot_str, ";", -1); + guint num_names = g_strv_length (names); + outputstrListHandle = Snpe_StringList_Create (); + for (guint i = 0; i < num_names; ++i) { + if (g_strcmp0 (names[i], "") == 0) { + throw std::invalid_argument ("Given tensor name is invalid."); + } + + g_info ("Add output tensor name of %s", names[i]); + if (Snpe_StringList_Append (outputstrListHandle, names[i]) != SNPE_SUCCESS) { + const std::string err_msg = "Failed to append output tensor name: " + + (const std::string) names[i]; + throw std::runtime_error (err_msg); + } + } + g_free (_ot_str); + g_strfreev (names); + } else if (g_ascii_strcasecmp (option[0], "OutputType") == 0) { + gchar **types = g_strsplit (option[1], ";", -1); + guint num_types = g_strv_length (types); + for (guint i = 0; i < num_types; ++i) { + if (g_ascii_strcasecmp (types[i], "FLOAT32") == 0) { + outputTypeVec.push_back (SNPE_USERBUFFERENCODING_ELEMENTTYPE_FLOAT); + } else if (g_ascii_strcasecmp (types[i], "TF8") == 0) { + outputTypeVec.push_back (SNPE_USERBUFFERENCODING_ELEMENTTYPE_TF8); + } else { + g_warning ("Ignore unknown output type (%s)", types[i]); + } + } + g_strfreev (types); + } else if (g_ascii_strcasecmp (option[0], "InputType") == 0) { + gchar **types = g_strsplit (option[1], ";", -1); + guint num_types = g_strv_length (types); + for (guint i = 0; i < num_types; ++i) { + if (g_ascii_strcasecmp (types[i], "FLOAT32") == 0) { + inputTypeVec.push_back (SNPE_USERBUFFERENCODING_ELEMENTTYPE_FLOAT); + } else if (g_ascii_strcasecmp (types[i], "TF8") == 0) { + inputTypeVec.push_back (SNPE_USERBUFFERENCODING_ELEMENTTYPE_TF8); + } else { + g_warning ("Ignore unknown input type (%s)", types[i]); + } + } + g_strfreev (types); + } else { + g_warning ("Unknown option (%s).", options[op]); + } + } + + g_strfreev (option); + } + + g_strfreev (options); + }; + + try { + /* Log SNPE version */ + lib_version_h = Snpe_Util_GetLibraryVersion (); + if (!lib_version_h) + throw std::runtime_error ("Failed to get SNPE library version"); + + g_info ("SNPE Version: %s", Snpe_DlVersion_ToString (lib_version_h)); + + int32_t ver_major = Snpe_DlVersion_GetMajor (lib_version_h); + if (ver_major < 2) { + const std::string err_msg = "Invalid SNPE version, version 2.x is supported but has " + + std::to_string (ver_major) + ".x."; + g_critical ("%s", err_msg.c_str ()); + throw std::runtime_error (err_msg); + } + + /* parse custom properties */ + parse_custom_prop (prop->custom_properties); + + /* Check the given Runtime is available */ + std::string runtime_str = std::string (Snpe_RuntimeList_RuntimeToString (runtime)); + if (Snpe_Util_IsRuntimeAvailable (runtime) == 0) + throw std::runtime_error ("Given runtime " + runtime_str + " is not available"); + + g_info ("Given runtime %s is available", runtime_str.c_str ()); + + /* set runtimelist config */ + runtime_list_h = Snpe_RuntimeList_Create (); + if (Snpe_RuntimeList_Add (runtime_list_h, runtime) != SNPE_SUCCESS) + throw std::runtime_error ("Failed to add given runtime to Snpe_RuntimeList"); + + /* Load network (dlc file) */ + if (!g_file_test (prop->model_files[0], G_FILE_TEST_IS_REGULAR)) { + const std::string err_msg + = "Given file " + (std::string) prop->model_files[0] + " is not valid"; + throw std::invalid_argument (err_msg); + } + + snpe->model_path = g_strdup (prop->model_files[0]); + container_h = Snpe_DlContainer_Open (snpe->model_path); + if (!container_h) + throw std::runtime_error ("Failed to open the model file " + std::string (snpe->model_path)); + + /* Build SNPE handle */ + snpebuilder_h = Snpe_SNPEBuilder_Create (container_h); + if (!snpebuilder_h) + throw std::runtime_error ("Failed to create SNPE builder"); + + if (Snpe_SNPEBuilder_SetRuntimeProcessorOrder (snpebuilder_h, runtime_list_h) != SNPE_SUCCESS) + throw std::runtime_error ("Failed to set runtime processor order"); + + /* set UserBuffer mode */ + if (Snpe_SNPEBuilder_SetUseUserSuppliedBuffers (snpebuilder_h, true) != SNPE_SUCCESS) + throw std::runtime_error ("Failed to set use user supplied buffers"); + + /* Set Output Tensors (if given by custom prop) */ + if (outputstrListHandle) { + if (Snpe_SNPEBuilder_SetOutputTensors (snpebuilder_h, outputstrListHandle) != SNPE_SUCCESS) { + throw std::runtime_error ("Failed to set output tensors"); + } + } + + snpe->snpe_h = Snpe_SNPEBuilder_Build (snpebuilder_h); + if (!snpe->snpe_h) + throw std::runtime_error ("Failed to build SNPE handle"); + + /* set inputTensorsInfo and inputMap */ + snpe->inputMap_h = Snpe_UserBufferMap_Create (); + inputstrListHandle = Snpe_SNPE_GetInputTensorNames (snpe->snpe_h); + if (!snpe->inputMap_h || !inputstrListHandle) + throw std::runtime_error ("Error while setting Input tensors"); + + snpe->inputInfo.num_tensors = Snpe_StringList_Size (inputstrListHandle); + for (size_t i = 0; i < snpe->inputInfo.num_tensors; i++) { + GstTensorInfo *info + = gst_tensors_info_get_nth_info (std::addressof (snpe->inputInfo), i); + const char *inputName = Snpe_StringList_At (inputstrListHandle, i); + info->name = g_strdup (inputName); + + auto inputType = SNPE_USERBUFFERENCODING_ELEMENTTYPE_UNKNOWN; + + /* set input type from custom prop if it is provided */ + if (inputTypeVec.size () > i) + inputType = inputTypeVec[i]; + handleTensor (inputName, info, snpe->inputMap_h, inputType); + } + + /* set outputTensorsInfo and outputMap */ + snpe->outputMap_h = Snpe_UserBufferMap_Create (); + + /* Get default output tensor names (if not provided by custom prop) */ + if (outputstrListHandle == NULL) + outputstrListHandle = Snpe_SNPE_GetOutputTensorNames (snpe->snpe_h); + + if (!snpe->outputMap_h || !outputstrListHandle) + throw std::runtime_error ("Error while setting Output tensors"); + + snpe->outputInfo.num_tensors = Snpe_StringList_Size (outputstrListHandle); + for (size_t i = 0; i < snpe->outputInfo.num_tensors; i++) { + GstTensorInfo *info + = gst_tensors_info_get_nth_info (std::addressof (snpe->outputInfo), i); + const char *outputName = Snpe_StringList_At (outputstrListHandle, i); + info->name = g_strdup (outputName); + + /* set output type from custom prop if it is provided */ + auto outputType = SNPE_USERBUFFERENCODING_ELEMENTTYPE_UNKNOWN; + if (outputTypeVec.size () > i) { + outputType = outputTypeVec[i]; + } + handleTensor (outputName, info, snpe->outputMap_h, outputType); + } + + _clean_handles (); + } catch (const std::exception &e) { + _clean_handles (); + g_critical ("%s", e.what ()); + return HAL_ML_ERROR_RUNTIME_ERROR; + } + + + // g_info ("configure_instance was successful"); + // gchar *inputinfostr = gst_tensors_info_to_string (&snpe->inputInfo); + // gchar *outputinfostr = gst_tensors_info_to_string (&snpe->outputInfo); + // g_info ("inputinfo: %s", inputinfostr); + // g_info ("outputinfo: %s", outputinfostr); + // g_free (inputinfostr); + // g_free (outputinfostr); + + + return HAL_ML_ERROR_NONE; +} + +static int ml_snpe_invoke(void *backend_private, const GstTensorMemory *input, GstTensorMemory *output) +{ + snpe_handle_s *snpe = (snpe_handle_s *) backend_private; + for (unsigned int i = 0; i < snpe->inputInfo.num_tensors; i++) { + GstTensorInfo *info = gst_tensors_info_get_nth_info (std::addressof (snpe->inputInfo), i); + auto iub = Snpe_UserBufferMap_GetUserBuffer_Ref (snpe->inputMap_h, info->name); + Snpe_IUserBuffer_SetBufferAddress (iub, input[i].data); + } + + for (unsigned int i = 0; i < snpe->outputInfo.num_tensors; i++) { + GstTensorInfo *info = gst_tensors_info_get_nth_info (std::addressof (snpe->outputInfo), i); + auto iub = Snpe_UserBufferMap_GetUserBuffer_Ref (snpe->outputMap_h, info->name); + Snpe_IUserBuffer_SetBufferAddress (iub, output[i].data); + } + + Snpe_SNPE_ExecuteUserBuffers (snpe->snpe_h, snpe->inputMap_h, snpe->outputMap_h); + + return HAL_ML_ERROR_NONE; +} + +static int ml_snpe_get_model_info(void *backend_private, model_info_ops ops, GstTensorsInfo *in_info, GstTensorsInfo *out_info) +{ + snpe_handle_s *snpe = (snpe_handle_s *) backend_private; + if (ops == GET_IN_OUT_INFO) { + gst_tensors_info_copy (in_info, &snpe->inputInfo); + gst_tensors_info_copy (out_info, &snpe->outputInfo); + + return HAL_ML_ERROR_NONE; + } + + return -2; +} + +static int ml_snpe_event_handler(void *backend_private, event_ops ops, GstTensorFilterFrameworkEventData *data) +{ + return HAL_ML_ERROR_NOT_SUPPORTED; +} + +static int ml_snpe_hal_backend_init(void **data) +{ + hal_backend_ml_funcs *funcs = NULL; + + if (*data) { + funcs = (hal_backend_ml_funcs *) *data; + } else { + funcs = g_new0 (hal_backend_ml_funcs, 1); + } + *data = (void *) funcs; + + funcs->init = ml_snpe_init; + funcs->deinit = ml_snpe_deinit; + funcs->configure_instance = ml_snpe_configure_instance; + funcs->invoke = ml_snpe_invoke; + funcs->get_model_info = ml_snpe_get_model_info; + funcs->event_handler = ml_snpe_event_handler; + + return 0; +} + +static int ml_snpe_hal_backend_exit(void *data) +{ + memset (data, 0x0, sizeof(hal_backend_ml_funcs)); + return 0; +} + +hal_backend hal_backend_ml_data = { + .name = "ml-snpe", + .vendor = "YONGJOO", + .init = ml_snpe_hal_backend_init, + .exit = ml_snpe_hal_backend_exit, + .major_version = 1, + .minor_version = 1, +}; diff --git a/src/hal-backend-ml-test-passthrough.cc b/src/hal-backend-ml-test-passthrough.cc new file mode 100644 index 0000000..2e37605 --- /dev/null +++ b/src/hal-backend-ml-test-passthrough.cc @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include + +#include + +#include +#include + +#include "hal-backend-ml-util.h" + + +typedef struct _pass_handle_s +{ + GstTensorsInfo inputInfo; + GstTensorsInfo outputInfo; +} pass_handle_s; + +static int ml_pass_init(void **backend_private) +{ + pass_handle_s *pass = g_new0 (pass_handle_s, 1); + gst_tensors_info_init (&pass->inputInfo); + gst_tensors_info_init (&pass->outputInfo); + *backend_private = pass; + return 0; +} + +static int ml_pass_deinit(void *backend_private) +{ + pass_handle_s *pass = (pass_handle_s *) backend_private; + if (!pass) { + g_critical ("[pass backend] ml_pass_deinit called with invalid backend_private"); + return HAL_ML_ERROR_INVALID_PARAMETER; + } + + gst_tensors_info_free (&pass->inputInfo); + gst_tensors_info_free (&pass->outputInfo); + + g_free (pass); + + return HAL_ML_ERROR_NONE; +} + +static int ml_pass_configure_instance(void *backend_private, const GstTensorFilterProperties *prop) +{ + pass_handle_s *pass = (pass_handle_s *) backend_private; + if (!pass) { + g_critical ("[pass backend] ml_pass_configure_instance called with invalid backend_private"); + return HAL_ML_ERROR_INVALID_PARAMETER; + } + + + gst_tensors_info_copy (&pass->inputInfo, &prop->input_meta); + gst_tensors_info_copy (&pass->outputInfo, &prop->output_meta); + + return 0; +} + +static int ml_pass_invoke(void *backend_private, const GstTensorMemory *input, GstTensorMemory *output) +{ + pass_handle_s *pass = (pass_handle_s *) backend_private; + if (!pass) { + g_critical ("[pass backend] ml_pass_invoke called with invalid backend_private"); + return HAL_ML_ERROR_INVALID_PARAMETER; + } + g_warning ("skip rungraph.. 됐다 치고 하하"); g_usleep (1000 * 100); + // pass->result_vsi_nn_RunGraph (pass->graph); + return 0; +} + +static int ml_pass_get_model_info(void *backend_private, model_info_ops ops, GstTensorsInfo *in_info, GstTensorsInfo *out_info) +{ + pass_handle_s *pass = (pass_handle_s *) backend_private; + if (!pass) { + g_critical ("[pass backend] ml_pass_get_model_info called with invalid backend_private"); + return HAL_ML_ERROR_INVALID_PARAMETER; + } + + gst_tensors_info_copy (in_info, &pass->inputInfo); + gst_tensors_info_copy (out_info, &pass->outputInfo); + + return 0; +} + +static int ml_pass_event_handler(void *backend_private, event_ops ops, GstTensorFilterFrameworkEventData *data) +{ + return -ENOENT; +} + +static int ml_pass_hal_backend_init(void **data) +{ + hal_backend_ml_funcs *funcs = NULL; + + if (*data) { + funcs = (hal_backend_ml_funcs *) *data; + } else { + funcs = g_new0 (hal_backend_ml_funcs, 1); + } + *data = (void *) funcs; + + funcs->init = ml_pass_init; + funcs->deinit = ml_pass_deinit; + funcs->configure_instance = ml_pass_configure_instance; + funcs->invoke = ml_pass_invoke; + funcs->get_model_info = ml_pass_get_model_info; + funcs->event_handler = ml_pass_event_handler; + + return 0; +} + +static int ml_pass_hal_backend_exit(void *data) +{ + memset (data, 0x0, sizeof(hal_backend_ml_funcs)); + return 0; +} + +hal_backend hal_backend_ml_data = { + .name = "ml-pass", + .vendor = "YONGJOO", + .init = ml_pass_hal_backend_init, + .exit = ml_pass_hal_backend_exit, + .major_version = 1, + .minor_version = 1, +}; diff --git a/src/hal-backend-ml-util.cc b/src/hal-backend-ml-util.cc new file mode 100644 index 0000000..43bcb2d --- /dev/null +++ b/src/hal-backend-ml-util.cc @@ -0,0 +1,185 @@ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include + +#include "hal-backend-ml-util.h" + +void gst_tensor_info_init (GstTensorInfo * info) +{ + guint i; + + g_return_if_fail (info != NULL); + + info->name = NULL; + info->type = _NNS_END; + + for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++) { + info->dimension[i] = 0; + } +} + +void gst_tensor_info_free (GstTensorInfo * info) +{ + g_return_if_fail (info != NULL); + + g_free (info->name); + + /* Init default */ + gst_tensor_info_init (info); +} + +void gst_tensors_info_init (GstTensorsInfo * info) +{ + guint i; + + g_return_if_fail (info != NULL); + + info->num_tensors = 0; + info->extra = NULL; + + /** @note default format is */ + info->format = _NNS_TENSOR_FORMAT_STATIC; + + for (i = 0; i < NNS_TENSOR_MEMORY_MAX; i++) { + gst_tensor_info_init (&info->info[i]); + } +} + +void gst_tensors_info_free (GstTensorsInfo * info) +{ + guint i; + + g_return_if_fail (info != NULL); + + for (i = 0; i < NNS_TENSOR_MEMORY_MAX; i++) { + gst_tensor_info_free (&info->info[i]); + } + + if (info->extra) { + for (i = 0; i < NNS_TENSOR_SIZE_EXTRA_LIMIT; ++i) + gst_tensor_info_free (&info->extra[i]); + + g_free (info->extra); + info->extra = NULL; + } + + /* Init default */ + gst_tensors_info_init (info); +} + +static const guint tensor_element_size[] = { + [_NNS_INT32] = 4, + [_NNS_UINT32] = 4, + [_NNS_INT16] = 2, + [_NNS_UINT16] = 2, + [_NNS_INT8] = 1, + [_NNS_UINT8] = 1, + [_NNS_FLOAT64] = 8, + [_NNS_FLOAT32] = 4, + [_NNS_INT64] = 8, + [_NNS_UINT64] = 8, + [_NNS_FLOAT16] = 2, + [_NNS_END] = 0, +}; + +gsize gst_tensor_get_element_size (tensor_type type) +{ + g_return_val_if_fail (type >= 0 && type <= _NNS_END, 0); + + return tensor_element_size[type]; +} + +gulong gst_tensor_get_element_count (const tensor_dim dim) +{ + gulong count = 1; + guint i; + + for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++) { + if (dim[i] == 0) + break; + + count *= dim[i]; + } + + return (i > 0) ? count : 0; +} + +gsize gst_tensor_info_get_size (const GstTensorInfo * info) +{ + gsize data_size; + + g_return_val_if_fail (info != NULL, 0); + + data_size = gst_tensor_get_element_count (info->dimension) * + gst_tensor_get_element_size (info->type); + + return data_size; +} + +GstTensorInfo * gst_tensors_info_get_nth_info (GstTensorsInfo * info, guint index) +{ + guint i; + + g_return_val_if_fail (info != NULL, NULL); + + if (index < NNS_TENSOR_MEMORY_MAX) + return &info->info[index]; + + if (!info->extra) { + info->extra = g_new0 (GstTensorInfo, NNS_TENSOR_SIZE_EXTRA_LIMIT); + + for (i = 0; i < NNS_TENSOR_SIZE_EXTRA_LIMIT; ++i) + gst_tensor_info_init (&info->extra[i]); + } + + if (index < NNS_TENSOR_SIZE_LIMIT) + return &info->extra[index - NNS_TENSOR_MEMORY_MAX]; + + g_critical ("Failed to get the information, invalid index %u (max %d).", + index, NNS_TENSOR_SIZE_LIMIT); + return NULL; +} + +void gst_tensor_info_copy_n (GstTensorInfo * dest, const GstTensorInfo * src, + const guint n) +{ + guint i; + + g_return_if_fail (dest != NULL); + g_return_if_fail (src != NULL); + + dest->name = g_strdup (src->name); + dest->type = src->type; + + for (i = 0; i < n; i++) { + dest->dimension[i] = src->dimension[i]; + } +} + +void gst_tensor_info_copy (GstTensorInfo * dest, const GstTensorInfo * src) +{ + gst_tensor_info_copy_n (dest, src, NNS_TENSOR_RANK_LIMIT); +} + +void gst_tensors_info_copy (GstTensorsInfo * dest, const GstTensorsInfo * src) +{ + guint i, num; + GstTensorInfo *_dest, *_src; + + g_return_if_fail (dest != NULL); + g_return_if_fail (src != NULL); + + gst_tensors_info_init (dest); + num = dest->num_tensors = src->num_tensors; + dest->format = src->format; + + if (src->format != _NNS_TENSOR_FORMAT_STATIC) + return; + + for (i = 0; i < num; i++) { + _dest = gst_tensors_info_get_nth_info (dest, i); + _src = gst_tensors_info_get_nth_info ((GstTensorsInfo *) src, i); + + gst_tensor_info_copy (_dest, _src); + } +} diff --git a/src/hal-backend-ml-util.h b/src/hal-backend-ml-util.h new file mode 100644 index 0000000..9d85f1f --- /dev/null +++ b/src/hal-backend-ml-util.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include + +#include "tensor_typedef.h" +#include "nnstreamer_plugin_api_filter.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void gst_tensor_info_init (GstTensorInfo * info); +void gst_tensor_info_free (GstTensorInfo * info); +void gst_tensors_info_init (GstTensorsInfo * info); +void gst_tensors_info_free (GstTensorsInfo * info); +gsize gst_tensor_get_element_size (tensor_type type); +gulong gst_tensor_get_element_count (const tensor_dim dim); +gsize gst_tensor_info_get_size (const GstTensorInfo * info); +GstTensorInfo * gst_tensors_info_get_nth_info (GstTensorsInfo * info, guint index); +void gst_tensor_info_copy_n (GstTensorInfo * dest, const GstTensorInfo * src, const guint n); +void gst_tensor_info_copy (GstTensorInfo * dest, const GstTensorInfo * src); +void gst_tensors_info_copy (GstTensorsInfo * dest, const GstTensorsInfo * src); + +#ifdef __cplusplus +} +#endif diff --git a/src/hal-backend-ml-vivante.cc b/src/hal-backend-ml-vivante.cc new file mode 100644 index 0000000..3ea095a --- /dev/null +++ b/src/hal-backend-ml-vivante.cc @@ -0,0 +1,230 @@ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include + +#include +#include + +#include +#include + +#include + +#include "hal-backend-ml-util.h" + + +static tensor_type convert_tensortype (vsi_nn_type_e tensor_type) +{ + switch (tensor_type) { + case VSI_NN_TYPE_INT8: + return _NNS_INT8; + case VSI_NN_TYPE_UINT8: + return _NNS_UINT8; + case VSI_NN_TYPE_INT16: + return _NNS_INT16; + case VSI_NN_TYPE_UINT16: + return _NNS_UINT16; + case VSI_NN_TYPE_FLOAT16: +#ifdef FLOAT16_SUPPORT + return _NNS_FLOAT16; +#else + return _NNS_UINT16; +#endif + case VSI_NN_TYPE_FLOAT32: + return _NNS_FLOAT32; + default: + break; + } + return _NNS_END; +} + +typedef struct _vivante_handle_s +{ + char *model_path; + char *so_path; + GstTensorsInfo inputInfo; + GstTensorsInfo outputInfo; + + vsi_nn_graph_t *graph; + void *handle; /* dlopened model so */ + vsi_status (*result_vsi_nn_CopyDataToTensor) (vsi_nn_graph_t *, vsi_nn_tensor_t *, uint8_t *); + void (*result_vnn_ReleaseNeuralNetwork) (vsi_nn_graph_t *); + vsi_nn_graph_t *(*result_vnn_CreateNeuralNetwork) (const char *); + vsi_status (*result_vsi_nn_RunGraph) (vsi_nn_graph_t *); + int postProcess; + vsi_status (*postProcessFunc) (vsi_nn_graph_t * graph); +} vivante_handle_s; + +static int ml_vivante_init(void **backend_private) +{ + vivante_handle_s *vivante = g_new0 (vivante_handle_s, 1); + gst_tensors_info_init (&vivante->inputInfo); + gst_tensors_info_init (&vivante->outputInfo); + *backend_private = vivante; + return 0; +} + +static int ml_vivante_deinit(void *backend_private) +{ + vivante_handle_s *vivante = (vivante_handle_s *) backend_private; + if (!vivante) { + g_error ("[vivante backend] ml_vivante_deinit called with invalid backend_private"); + return HAL_ML_ERROR_INVALID_PARAMETER; + } + + if (vivante->graph) + vivante->result_vnn_ReleaseNeuralNetwork (vivante->graph); + + if (vivante->handle) + dlclose (vivante->handle); + + gst_tensors_info_free (&vivante->inputInfo); + gst_tensors_info_free (&vivante->outputInfo); + + g_free (vivante->model_path); + g_free (vivante->so_path); + g_free (vivante); + + return HAL_ML_ERROR_NONE; +} + +static int ml_vivante_configure_instance(void *backend_private, const GstTensorFilterProperties *prop) +{ + vivante_handle_s *vivante = (vivante_handle_s *) backend_private; + if (!vivante) { + g_error ("[vivante backend] ml_vivante_configure_instance called with invalid backend_private"); + return HAL_ML_ERROR_INVALID_PARAMETER; + } + + vivante->model_path = g_strdup(prop->model_files[0]); + vivante->so_path = g_strdup(prop->model_files[1]); + + vivante->handle = dlopen (vivante->so_path, RTLD_NOW); + if (!vivante->handle) { + g_error ("Failed to load shared library: %s", vivante->so_path); + return HAL_ML_ERROR_RUNTIME_ERROR; + } + + vivante->result_vsi_nn_CopyDataToTensor = (vsi_status (*)(vsi_nn_graph_t *, vsi_nn_tensor_t *, uint8_t *)) dlsym(vivante->handle, "vsi_nn_CopyDataToTensor"); + vivante->result_vnn_ReleaseNeuralNetwork = (void (*)(vsi_nn_graph_t *)) dlsym(vivante->handle, "vnn_ReleaseNeuralNetwork"); + vivante->result_vnn_CreateNeuralNetwork = (vsi_nn_graph_t *(*)(const char *)) dlsym(vivante->handle, "vnn_CreateNeuralNetwork"); + vivante->result_vsi_nn_RunGraph = (vsi_status (*)(vsi_nn_graph_t *)) dlsym(vivante->handle, "vsi_nn_RunGraph"); + + if (vivante->postProcess) { + vivante->postProcessFunc = (vsi_status (*)(vsi_nn_graph_t *)) dlsym(vivante->handle, "vnn_PostProcessNeuralNetwork"); + } + + vivante->graph = vivante->result_vnn_CreateNeuralNetwork(vivante->model_path); + + /* setting input and output tensors info */ + gst_tensors_info_init (&vivante->inputInfo); + gst_tensors_info_init (&vivante->outputInfo); + + vivante->inputInfo.num_tensors = vivante->graph->input.num; + for (unsigned int i = 0; i < vivante->graph->input.num; i++) { + vsi_nn_tensor_t *i_tensor = vsi_nn_GetTensor (vivante->graph, vivante->graph->input.tensors[i]); + GstTensorInfo *info = gst_tensors_info_get_nth_info (&vivante->inputInfo, i); + + info->type = convert_tensortype (i_tensor->attr.dtype.vx_type); + info->name = g_strdup_printf ("%i", vivante->graph->input.tensors[i]); + for (unsigned int j = 0; j < i_tensor->attr.dim_num; ++j) { + info->dimension[j] = i_tensor->attr.size[j]; + } + } + + vivante->outputInfo.num_tensors = vivante->graph->output.num; + for (unsigned int i = 0; i < vivante->graph->output.num; i++) { + vsi_nn_tensor_t *o_tensor = vsi_nn_GetTensor (vivante->graph, vivante->graph->output.tensors[i]); + GstTensorInfo *info = gst_tensors_info_get_nth_info (&vivante->outputInfo, i); + + info->type = convert_tensortype (o_tensor->attr.dtype.vx_type); + info->name = g_strdup_printf ("%i", vivante->graph->output.tensors[i]); + for (unsigned int j = 0; j < o_tensor->attr.dim_num; ++j) { + info->dimension[j] = o_tensor->attr.size[j]; + } + } + + return 0; +} + +static int ml_vivante_invoke(void *backend_private, const GstTensorMemory *input, GstTensorMemory *output) +{ + vivante_handle_s *vivante = (vivante_handle_s *) backend_private; + if (!vivante) { + g_error ("[vivante backend] ml_vivante_invoke called with invalid backend_private"); + return HAL_ML_ERROR_INVALID_PARAMETER; + } + + for (unsigned int i = 0; i < vivante->graph->input.num; i++) { + vsi_nn_tensor_t *tensor = vsi_nn_GetTensor (vivante->graph, vivante->graph->input.tensors[i]); + vivante->result_vsi_nn_CopyDataToTensor (vivante->graph, tensor, (uint8_t *) input[i].data); + } + + g_warning ("skip rungraph.. 됐다 치고"); g_usleep (1000 * 100); + // vivante->result_vsi_nn_RunGraph (vivante->graph); + + if (vivante->postProcess) + vivante->postProcessFunc (vivante->graph); + + for (unsigned int i = 0; i < vivante->graph->output.num; i++) { + vsi_nn_tensor_t *out_tensor = vsi_nn_GetTensor (vivante->graph, vivante->graph->output.tensors[i]); + vsi_nn_CopyTensorToBuffer (vivante->graph, out_tensor, output[i].data); + } + + return 0; +} + +static int ml_vivante_get_model_info(void *backend_private, model_info_ops ops, GstTensorsInfo *in_info, GstTensorsInfo *out_info) +{ + vivante_handle_s *vivante = (vivante_handle_s *) backend_private; + if (!vivante) { + g_error ("[vivante backend] ml_vivante_get_model_info called with invalid backend_private"); + return HAL_ML_ERROR_INVALID_PARAMETER; + } + + gst_tensors_info_copy (in_info, &vivante->inputInfo); + gst_tensors_info_copy (out_info, &vivante->outputInfo); + + return 0; +} + +static int ml_vivante_event_handler(void *backend_private, event_ops ops, GstTensorFilterFrameworkEventData *data) +{ + return -ENOENT; +} + +static int ml_vivante_hal_backend_init(void **data) +{ + hal_backend_ml_funcs *funcs = NULL; + + if (*data) { + funcs = (hal_backend_ml_funcs *) *data; + } else { + funcs = g_new0 (hal_backend_ml_funcs, 1); + } + *data = (void *) funcs; + + funcs->init = ml_vivante_init; + funcs->deinit = ml_vivante_deinit; + funcs->configure_instance = ml_vivante_configure_instance; + funcs->invoke = ml_vivante_invoke; + funcs->get_model_info = ml_vivante_get_model_info; + funcs->event_handler = ml_vivante_event_handler; + + return 0; +} + +static int ml_vivante_hal_backend_exit(void *data) +{ + memset (data, 0x0, sizeof(hal_backend_ml_funcs)); + return 0; +} + +hal_backend hal_backend_ml_data = { + .name = "ml-vivante", + .vendor = "YONGJOO", + .init = ml_vivante_hal_backend_init, + .exit = ml_vivante_hal_backend_exit, + .major_version = 1, + .minor_version = 1, +}; -- 2.34.1