/* SPDX-License-Identifier: Apache-2.0 */
-#include <stdexcept>
-
#include <dlfcn.h>
#include <glib.h>
#include <json-glib/json-glib.h>
#include "hal-backend-ml-util.h"
-/* Helper to convert VSI type string from JSON to vsi_nn_type_e */
-static vsi_nn_type_e
-vivante_vsi_type_from_string (const gchar * vsi_type_str)
-{
- if (!vsi_type_str) return VSI_NN_TYPE_NONE;
-
- /** @todo Add 4-bit types when it's available */
+/**
+ * @brief Private handle for the Vivante instance.
+ */
+typedef struct _vivante_handle_s {
+ char *model_path; /* .nb file path */
+ char *so_path; /* .so file path (for .so based model loading) */
+ char *json_path; /* .json file path (for JSON based model loading) */
+ gboolean use_json_for_graph;
+ gboolean has_post_process; /** @deprecated Do not use it. */
- if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_INT8") == 0) return VSI_NN_TYPE_INT8;
- if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_INT16") == 0) return VSI_NN_TYPE_INT16;
- if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_INT32") == 0) return VSI_NN_TYPE_INT32;
- if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_INT64") == 0) return VSI_NN_TYPE_INT64;
+ GstTensorsInfo inputInfo;
+ GstTensorsInfo outputInfo;
- if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_UINT8") == 0) return VSI_NN_TYPE_UINT8;
- if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_UINT16") == 0) return VSI_NN_TYPE_UINT16;
- if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_UINT32") == 0) return VSI_NN_TYPE_UINT32;
- if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_UINT64") == 0) return VSI_NN_TYPE_UINT64;
+ vsi_nn_graph_t *graph;
- if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_FLOAT16") == 0) return VSI_NN_TYPE_FLOAT16;
- if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_FLOAT32") == 0) return VSI_NN_TYPE_FLOAT32;
- if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_FLOAT64") == 0) return VSI_NN_TYPE_FLOAT64;
- if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_BFLOAT16") == 0) return VSI_NN_TYPE_BFLOAT16;
+ /* Handles for JSON based model loading */
+ vsi_nn_context_t ctx;
- if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_BOOL8") == 0) return VSI_NN_TYPE_BOOL8;
+ /* Handles for .so based model loading */
+ void *dl_handle; /* dlopened model so */
+ vsi_nn_graph_t *(*model_specific_vnn_CreateNeuralNetwork) (const char *);
+ void (*model_specific_vnn_ReleaseNeuralNetwork) (vsi_nn_graph_t *);
+ vsi_status (*model_specific_vnn_PostProcessNeuralNetwork) (vsi_nn_graph_t *); /** @deprecated */
+} vivante_handle_s;
- g_warning ("[vivante backend] Unknown VSI tensor type string from JSON: %s", vsi_type_str);
+/* ===================================================================
+ * Forward Declarations of Static Helper Functions
+ * ===================================================================
+ */
+static void _json_release_neural_network (vivante_handle_s *self);
+static int _json_create_neural_network (vivante_handle_s *self);
+static int _so_create_neural_network (vivante_handle_s *self);
+
+/* ===================================================================
+ * Type Conversion Helpers
+ * ===================================================================
+ */
+/** @brief Converts VSI type string from JSON to vsi_nn_type_e enum. */
+static vsi_nn_type_e
+vivante_vsi_type_from_string (const gchar *vsi_type_str)
+{
+ if (!vsi_type_str)
+ return VSI_NN_TYPE_NONE;
+
+ /** @todo Add 4-bit types when available */
+ if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_INT8") == 0)
+ return VSI_NN_TYPE_INT8;
+ if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_UINT8") == 0)
+ return VSI_NN_TYPE_UINT8;
+ if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_INT16") == 0)
+ return VSI_NN_TYPE_INT16;
+ if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_UINT16") == 0)
+ return VSI_NN_TYPE_UINT16;
+ if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_INT32") == 0)
+ return VSI_NN_TYPE_INT32;
+ if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_UINT32") == 0)
+ return VSI_NN_TYPE_UINT32;
+ if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_INT64") == 0)
+ return VSI_NN_TYPE_INT64;
+ if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_UINT64") == 0)
+ return VSI_NN_TYPE_UINT64;
+ if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_FLOAT16") == 0)
+ return VSI_NN_TYPE_FLOAT16;
+ if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_FLOAT32") == 0)
+ return VSI_NN_TYPE_FLOAT32;
+ if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_FLOAT64") == 0)
+ return VSI_NN_TYPE_FLOAT64;
+ if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_BFLOAT16") == 0)
+ return VSI_NN_TYPE_BFLOAT16;
+ if (g_ascii_strcasecmp (vsi_type_str, "VSI_NN_TYPE_BOOL8") == 0)
+ return VSI_NN_TYPE_BOOL8;
+
+ g_warning ("[vivante] Unknown VSI tensor type string from JSON: %s", vsi_type_str);
return VSI_NN_TYPE_NONE;
}
-/* Helper to convert VSI quantization type string from JSON to vsi_nn_qnt_type_e */
+/** @brief Converts VSI quantization type string from JSON to vsi_nn_qnt_type_e enum. */
static vsi_nn_qnt_type_e
-vivante_qnt_type_from_string (const gchar * qnt_str)
+vivante_qnt_type_from_string (const gchar *qnt_str)
{
- if (!qnt_str) return VSI_NN_QNT_TYPE_NONE;
- if (g_ascii_strcasecmp (qnt_str, "VSI_NN_QNT_TYPE_NONE") == 0) return VSI_NN_QNT_TYPE_NONE;
- if (g_ascii_strcasecmp (qnt_str, "VSI_NN_QNT_TYPE_DFP") == 0) return VSI_NN_QNT_TYPE_DFP;
- if (g_ascii_strcasecmp (qnt_str, "VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC") == 0) return VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC;
- if (g_ascii_strcasecmp (qnt_str, "VSI_NN_QNT_TYPE_AFFINE_PERCHANNEL_SYMMETRIC") == 0) return VSI_NN_QNT_TYPE_AFFINE_PERCHANNEL_SYMMETRIC;
- if (g_ascii_strcasecmp (qnt_str, "VSI_NN_QNT_TYPE_AFFINE_SYMMETRIC") == 0) return VSI_NN_QNT_TYPE_AFFINE_SYMMETRIC;
-
- /** @todo Add VSI_NN_QNT_TYPE_AFFINE_PERCHANNEL_ASYMMETRIC when it's available */
-
- g_warning ("[vivante backend] Unknown VSI quantization type string from JSON: %s", qnt_str);
+ if (!qnt_str)
+ return VSI_NN_QNT_TYPE_NONE;
+ if (g_ascii_strcasecmp (qnt_str, "VSI_NN_QNT_TYPE_NONE") == 0)
+ return VSI_NN_QNT_TYPE_NONE;
+ if (g_ascii_strcasecmp (qnt_str, "VSI_NN_QNT_TYPE_DFP") == 0)
+ return VSI_NN_QNT_TYPE_DFP;
+ if (g_ascii_strcasecmp (qnt_str, "VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC") == 0)
+ return VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC;
+ if (g_ascii_strcasecmp (qnt_str, "VSI_NN_QNT_TYPE_AFFINE_PERCHANNEL_SYMMETRIC") == 0)
+ return VSI_NN_QNT_TYPE_AFFINE_PERCHANNEL_SYMMETRIC;
+ if (g_ascii_strcasecmp (qnt_str, "VSI_NN_QNT_TYPE_AFFINE_SYMMETRIC") == 0)
+ return VSI_NN_QNT_TYPE_AFFINE_SYMMETRIC;
+
+ /** @todo Add VSI_NN_QNT_TYPE_AFFINE_PERCHANNEL_ASYMMETRIC when available */
+ g_warning ("[vivante] Unknown VSI quantization type string from JSON: %s", qnt_str);
return VSI_NN_QNT_TYPE_NONE;
}
+/** @brief Converts vsi_nn_type_e to the framework's tensor_type. */
static tensor_type
-convert_tensortype (vsi_nn_type_e tensor_type)
+convert_to_tensor_type (vsi_nn_type_e vsi_type)
{
- switch (tensor_type) {
+ switch (vsi_type) {
case VSI_NN_TYPE_BOOL8:
case VSI_NN_TYPE_INT8:
return _NNS_INT8;
case VSI_NN_TYPE_FLOAT64:
return _NNS_FLOAT64;
default:
- break;
+ g_warning ("[vivante] Unsupported vsi_nn_type_e: %d", vsi_type);
+ return _NNS_END;
}
- return _NNS_END;
}
-typedef struct _vivante_handle_s {
- char *model_path; /* .nb file path */
- char *so_path; /* .so file path (for .so based model loading) */
- char *json_path; /* .json file path (for JSON based model loading) */
- gboolean using_json_for_graph_setup;
-
- GstTensorsInfo inputInfo;
- GstTensorsInfo outputInfo;
-
- vsi_nn_graph_t *graph;
-
- /* handles for json based model loading */
- vsi_nn_context_t ctx;
-
- /* handles for .so based model loading */
- void *handle; /* dlopened model so */
- vsi_nn_graph_t *(*result_vnn_CreateNeuralNetwork) (const char *);
- void (*result_vnn_ReleaseNeuralNetwork) (vsi_nn_graph_t *);
- vsi_status (*postProcessFunc) (vsi_nn_graph_t *);
- int postProcess; /** @todo Add postProcess custom prop */
-} vivante_handle_s;
-
+/* ===================================================================
+ * JSON Parsing and Graph Creation Helpers
+ * ===================================================================
+ */
+/** @brief Parses tensor attributes from a JSON object into a vsi_nn_tensor_attr_t struct. */
static int
-ml_vivante_init (void **backend_private)
+_helper_parse_tensor_attributes (JsonObject *tensor_obj, vsi_nn_tensor_attr_t *vsi_attr)
{
- vivante_handle_s *vivante = g_new0 (vivante_handle_s, 1);
+ memset (vsi_attr, 0, sizeof (vsi_nn_tensor_attr_t));
+ vsi_attr->vtl = FALSE;
+ vsi_attr->is_const = FALSE;
+
+ // Size and dim_num
+ JsonArray *size_array = json_object_get_array_member (tensor_obj, "size");
+ if (!size_array) {
+ g_critical ("[vivante] Tensor in JSON missing 'size' array.");
+ return HAL_ML_ERROR_INVALID_PARAMETER;
+ }
+ vsi_attr->dim_num = json_array_get_length (size_array);
+ if (vsi_attr->dim_num == 0 || vsi_attr->dim_num > VSI_NN_MAX_DIM_NUM) {
+ g_critical ("[vivante] Invalid tensor 'dim_num': %u", vsi_attr->dim_num);
+ return HAL_ML_ERROR_INVALID_PARAMETER;
+ }
+ for (guint i = 0; i < vsi_attr->dim_num; ++i) {
+ vsi_attr->size[i] = json_array_get_int_element (size_array, i);
+ }
- gst_tensors_info_init (&vivante->inputInfo);
- gst_tensors_info_init (&vivante->outputInfo);
+ // Dtype object
+ JsonObject *dtype_obj = json_object_get_object_member (tensor_obj, "dtype");
+ if (!dtype_obj) {
+ g_critical ("[vivante] Tensor in JSON missing 'dtype' object.");
+ return HAL_ML_ERROR_INVALID_PARAMETER;
+ }
+
+ // Required: vx_type
+ const gchar *vx_type_str = json_object_get_string_member (dtype_obj, "vx_type");
+ if (!vx_type_str) {
+ g_critical ("[vivante] 'dtype' missing required 'vx_type' key.");
+ return HAL_ML_ERROR_INVALID_PARAMETER;
+ }
+ vsi_attr->dtype.vx_type = vivante_vsi_type_from_string (vx_type_str);
- vivante->using_json_for_graph_setup = TRUE;
+ // Optional fields with defaults
+ vsi_attr->dtype.qnt_type
+ = vivante_qnt_type_from_string (json_object_get_string_member_with_default (
+ dtype_obj, "qnt_type", "VSI_NN_QNT_TYPE_NONE"));
+ vsi_attr->dtype.fl = json_object_get_int_member_with_default (dtype_obj, "fl", 0);
+ vsi_attr->dtype.zero_point
+ = json_object_get_int_member_with_default (dtype_obj, "zero_point", 0);
+ vsi_attr->dtype.scale
+ = json_object_get_double_member_with_default (dtype_obj, "scale", 0.0f);
+
+ /** @todo Parse scales, scale_dim, etc. for PERCHANNEL quantization */
- *backend_private = vivante;
return HAL_ML_ERROR_NONE;
}
+/** @brief Creates and sets up the neural network graph using a JSON definition file. */
static int
_json_create_neural_network (vivante_handle_s *self)
{
+ g_autofree gchar *json_string = NULL;
+ g_autoptr (GError) err = NULL;
+ g_autoptr (JsonParser) parser = NULL;
+ JsonNode *root_node = NULL;
+ JsonObject *root_obj = NULL;
+ JsonArray *input_array = NULL, *output_array = NULL;
+ guint input_tensors_num = 0, output_tensors_num = 0;
+ vsi_nn_node_t *node = NULL;
+
self->ctx = vsi_nn_CreateContext ();
- if (self->ctx == NULL) {
- g_critical ("[vivante backend] Failed to create context");
+ if (!self->ctx) {
+ g_critical ("[vivante] Failed to create VSI context.");
return HAL_ML_ERROR_RUNTIME_ERROR;
}
- // 1. Get string from the json file
- g_autofree gchar *json_string = NULL;
- if (!g_file_get_contents (self->json_path, &json_string, NULL, NULL)) {
- g_critical ("[vivante backend] Failed to read JSON file %s", self->json_path);
+ if (!g_file_get_contents (self->json_path, &json_string, NULL, &err)) {
+ g_critical ("[vivante] Failed to read JSON file '%s': %s", self->json_path,
+ err ? err->message : "Unknown error");
return HAL_ML_ERROR_IO_ERROR;
}
- // 2. Parse json string
- g_autoptr (GError) err = NULL;
- g_autoptr (JsonParser) parser = json_parser_new ();
- if (!parser) {
- g_critical ("[vivante backend] Failed to create JsonParser. OOM?");
- return HAL_ML_ERROR_OUT_OF_MEMORY;
- }
-
+ parser = json_parser_new ();
if (!json_parser_load_from_data (parser, json_string, -1, &err)) {
- g_critical ("[vivante backend] Failed to parse json. error: %s", err ? err->message : "Unknown error");
+ g_critical ("[vivante] Failed to parse JSON: %s", err ? err->message : "Unknown error");
return HAL_ML_ERROR_INVALID_PARAMETER;
}
- // 3. Get the root node and the object
- JsonNode *root;
- JsonObject *root_obj;
- root = json_parser_get_root (parser);
- if (!root) {
- g_critical ("[vivante backend] Failed to get root node");
- return HAL_ML_ERROR_RUNTIME_ERROR;
- }
-
- if (!JSON_NODE_HOLDS_OBJECT (root)) {
- g_critical ("[vivante backend] Root node is not an object");
+ root_node = json_parser_get_root (parser);
+ if (!root_node || !JSON_NODE_HOLDS_OBJECT (root_node)) {
+ g_critical ("[vivante] JSON root is not a valid object.");
return HAL_ML_ERROR_INVALID_PARAMETER;
}
+ root_obj = json_node_get_object (root_node);
- root_obj = json_node_get_object (root);
- if (!root_obj) {
- g_critical ("[vivante backend] Failed to get object from root node");
- return HAL_ML_ERROR_RUNTIME_ERROR;
- }
-
- // 4. Get the number of inputs and outputs
- guint input_tensors_num = 0U;
- guint output_tensors_num = 0U;
-
- if (!json_object_has_member (root_obj, "input_tensors")) {
- g_critical ("[vivante backend] Missing necessary key 'input_tensors' in JSON");;
- return HAL_ML_ERROR_INVALID_PARAMETER;
- }
-
- JsonArray *input_array = json_object_get_array_member (root_obj, "input_tensors");
- if (!input_array) {
- g_critical ("[vivante backend] Failed to get array from 'input_tensors'");
+ input_array = json_object_get_array_member (root_obj, "input_tensors");
+ output_array = json_object_get_array_member (root_obj, "output_tensors");
+ if (!input_array || !output_array) {
+ g_critical ("[vivante] JSON must contain 'input_tensors' and 'output_tensors' arrays.");
return HAL_ML_ERROR_INVALID_PARAMETER;
}
input_tensors_num = json_array_get_length (input_array);
-
- if (!json_object_has_member (root_obj, "output_tensors")) {
- g_critical ("[vivante backend] Missing necessary array 'output_tensors' in JSON");;
- return HAL_ML_ERROR_INVALID_PARAMETER;
- }
-
- JsonArray *output_array = json_object_get_array_member (root_obj, "output_tensors");
- if (!output_array) {
- g_critical ("[vivante backend] Failed to get array from 'output_tensors'");
- return HAL_ML_ERROR_INVALID_PARAMETER;
- }
-
output_tensors_num = json_array_get_length (output_array);
- // 5. Set up graphs
- vsi_nn_graph_t *graph = vsi_nn_CreateGraph (self->ctx, input_tensors_num + output_tensors_num + output_tensors_num, 1 /* NBG node */);
- if (graph == NULL) {
- g_critical ("[vivante backend] Failed to create graph");
- return HAL_ML_ERROR_RUNTIME_ERROR;
- }
+ const guint node_num = 1U; /* single NBG node */
+ const guint const_tensors_num = 0U; /** @todo support this */
+ const guint normal_tensors_num = input_tensors_num + output_tensors_num;
+ const guint virtual_tensors_num = output_tensors_num;
- if (!vsi_nn_SetGraphInputs (graph, NULL, input_tensors_num)) {
- g_critical ("[vivante backend] Failed to set graph inputs");
- return HAL_ML_ERROR_RUNTIME_ERROR;
+ self->graph = vsi_nn_CreateGraph (self->ctx,
+ normal_tensors_num + virtual_tensors_num + const_tensors_num, node_num);
+ if (!self->graph) {
+ g_critical ("[vivante] Failed to create VSI graph.");
+ goto cleanup;
}
- if (!vsi_nn_SetGraphOutputs (graph, NULL, output_tensors_num)) {
- g_critical ("[vivante backend] Failed to set graph outputs");
- return HAL_ML_ERROR_RUNTIME_ERROR;
+
+ if (!vsi_nn_SetGraphInputs (self->graph, NULL, input_tensors_num)
+ || !vsi_nn_SetGraphOutputs (self->graph, NULL, output_tensors_num)) {
+ g_critical ("[vivante] Failed to set graph inputs/outputs.");
+ goto cleanup;
}
- // 6. Set a single vsi_nn_node
- /** @todo Assuming there is only 1 node for each nb model. Find any exceptions. */
- vsi_nn_node_t *node = NULL;
- node = vsi_nn_AddNode(graph, VSI_NN_OP_NBG, input_tensors_num, output_tensors_num, NULL);
+ node = vsi_nn_AddNode (
+ self->graph, VSI_NN_OP_NBG, input_tensors_num, output_tensors_num, NULL);
if (!node) {
- g_critical ("[vivante backend] Failed to add node");
- return HAL_ML_ERROR_RUNTIME_ERROR;
+ g_critical ("[vivante] Failed to add NBG node to graph.");
+ goto cleanup;
}
-
node->uid = 0;
node->nn_param.nbg.type = VSI_NN_NBG_FILE;
node->nn_param.nbg.url = self->model_path;
- auto _parse_tensor_attributes_from_json = [&] (JsonObject *tensor_obj, vsi_nn_tensor_attr_t *vsi_attr) -> int {
- memset (vsi_attr, 0, sizeof (vsi_nn_tensor_attr_t));
- if (json_object_has_member (tensor_obj, "id")) {
- // do something?
- }
-
- // initialize some boolean attributes
- vsi_attr->vtl = FALSE;
- vsi_attr->is_const = FALSE;
- vsi_attr->is_created_from_handle = FALSE;
- vsi_attr->is_handle_malloc_by_ovxlib = FALSE;
-
- // parse size and dim_num
- if (!json_object_has_member (tensor_obj, "size")) {
- }
- JsonArray *size_array = json_object_get_array_member (tensor_obj, "size");
- if (!size_array) {
- g_critical ("[vivante backend] Failed to get array from 'size' in JSON");
- return HAL_ML_ERROR_INVALID_PARAMETER;
- }
-
- vsi_attr->dim_num = json_array_get_length (size_array);
- if (vsi_attr->dim_num == 0U || vsi_attr->dim_num > VSI_NN_MAX_DIM_NUM) {
- g_critical ("[vivante backend] Invalid value for 'size' in JSON");
- return HAL_ML_ERROR_INVALID_PARAMETER;
- }
-
- for (guint i = 0; i < vsi_attr->dim_num; ++i) {
- gint64 dim = json_array_get_int_element (size_array, i);
- if (dim <= 0) {
- g_critical ("[vivante backend] Invalid value for 'dim' in JSON");
- return HAL_ML_ERROR_INVALID_PARAMETER;
- }
-
- vsi_attr->size[i] = (vsi_size_t) dim;
- }
-
- // parse dtype
- if (!json_object_has_member (tensor_obj, "dtype")) {
- g_critical ("[vivante backend] Missing necessary key 'dtype' in JSON");
- return HAL_ML_ERROR_INVALID_PARAMETER;
- }
-
- JsonObject *dtype_obj = json_object_get_object_member (tensor_obj, "dtype");
-
- // parse dtype.fmt
- vsi_attr->dtype.fmt = VSI_NN_DIM_FMT_NCHW;
- if (!json_object_has_member (dtype_obj, "fmt")) {
- g_warning ("[vivante backend] Missing key 'fmt' in 'dtype' in JSON. Default: 'VSI_NN_DIM_FMT_NCHW'");
- } else {
- // @todo parse dtype.fmt
- }
-
- // parse dtype.vx_type
- if (!json_object_has_member (dtype_obj, "vx_type")) {
- g_critical ("[vivante backend] Missing necessary key 'vx_type' in 'dtype' in JSON");
- return HAL_ML_ERROR_INVALID_PARAMETER;
- } else {
- vsi_attr->dtype.vx_type = vivante_vsi_type_from_string (json_object_get_string_member (dtype_obj, "vx_type"));
- }
-
- // parse dtype.qnt_type
- vsi_attr->dtype.qnt_type = VSI_NN_QNT_TYPE_NONE;
- if (!json_object_has_member (dtype_obj, "qnt_type")) {
- g_warning ("[vivante backend] Missing key 'qnt_type' in 'dtype' in JSON. Default: 'VSI_NN_QNT_TYPE_NONE'");
- } else {
- vsi_attr->dtype.qnt_type = vivante_qnt_type_from_string (json_object_get_string_member (dtype_obj, "qnt_type"));
- }
-
- // parse dtype.fl for dynamic fixed point
- vsi_attr->dtype.fl = 0;
- if (!json_object_has_member (dtype_obj, "fl")) {
- g_warning ("[vivante backend] Missing key 'fl' in 'dtype' in JSON. Default: 0");
- } else {
- vsi_attr->dtype.fl = (int8_t) json_object_get_int_member (dtype_obj, "fl");
- }
-
- // parse dtype.zero_point for affine asymmetric
- vsi_attr->dtype.zero_point = 0;
- if (!json_object_has_member (dtype_obj, "zero_point")) {
- g_warning ("[vivante backend] Missing key 'zero_point' in 'dtype' in JSON. Default: 0");
- } else {
- vsi_attr->dtype.zero_point = (int32_t) json_object_get_int_member (dtype_obj, "zero_point");
- }
-
- // parse dtype.scale for affine asymmetric
- vsi_attr->dtype.scale = 0.0f;
- if (!json_object_has_member (dtype_obj, "scale")) {
- g_warning ("[vivante backend] Missing key 'scale' in 'dtype' in JSON. Default: 0.0f");
- } else {
- vsi_attr->dtype.scale = (float) json_object_get_double_member (dtype_obj, "scale");
- }
-
- /** @todo parse scales, scale_dim, channel_dim, zero_points, zero_points_dim for AFFINE_PERCHANNEL_SYMMETRIC */
-
- return HAL_ML_ERROR_NONE;
- };
-
- // 7. Set up input tensors
+ // Set up input tensors
for (guint i = 0; i < input_tensors_num; ++i) {
vsi_nn_tensor_attr_t tensor_attr;
// parse attr data from json
JsonObject *tensor_obj = json_array_get_object_element (input_array, i);
- if (_parse_tensor_attributes_from_json (tensor_obj, &tensor_attr) != HAL_ML_ERROR_NONE) {
- g_critical ("[vivante backend] Failed to parse tensor attributes from JSON");
- return HAL_ML_ERROR_INVALID_PARAMETER;
+ if (_helper_parse_tensor_attributes (tensor_obj, &tensor_attr) != HAL_ML_ERROR_NONE) {
+ g_critical ("[vivante] Failed to parse tensor attributes from JSON");
+ goto cleanup;
}
// Add the tensor to the graph
- vsi_nn_tensor_id_t vsi_input_id = vsi_nn_AddTensor (graph, VSI_NN_TENSOR_ID_AUTO, &tensor_attr, NULL);
+ vsi_nn_tensor_id_t vsi_input_id
+ = vsi_nn_AddTensor (self->graph, VSI_NN_TENSOR_ID_AUTO, &tensor_attr, NULL);
if (vsi_input_id == VSI_NN_TENSOR_ID_NA) {
- g_critical ("[vivante backend] Failed to add input tensor #%u", i);
- return HAL_ML_ERROR_RUNTIME_ERROR;
+ g_critical ("[vivante] Failed to add input tensor #%u", i);
+ goto cleanup;
}
- g_info ("[vivante backend] Added input tensor #%u with id %u", i, vsi_input_id);
+ g_info ("[vivante] Added input tensor #%u with id %u", i, vsi_input_id);
node->input.tensors[i] = vsi_input_id;
- graph->input.tensors[i] = vsi_input_id;
+ self->graph->input.tensors[i] = vsi_input_id;
}
for (guint i = 0; i < input_tensors_num; ++i) {
- g_info ("[vivante backend] print Input tensor #%u:", graph->input.tensors[i]);
- vsi_nn_tensor_t *tensor = vsi_nn_GetTensor (graph, graph->input.tensors[i]);
- vsi_nn_PrintTensor (tensor, graph->input.tensors[i]);
+ g_info ("[vivante] print Input tensor #%u:", self->graph->input.tensors[i]);
+ vsi_nn_tensor_t *tensor
+ = vsi_nn_GetTensor (self->graph, self->graph->input.tensors[i]);
+ vsi_nn_PrintTensor (tensor, self->graph->input.tensors[i]);
}
- // 8. Set up output tensors
+ // Set up output tensors
for (guint i = 0; i < output_tensors_num; ++i) {
vsi_nn_tensor_attr_t tensor_attr;
// parse attr data from json
JsonObject *tensor_obj = json_array_get_object_element (output_array, i);
- if (_parse_tensor_attributes_from_json (tensor_obj, &tensor_attr) != HAL_ML_ERROR_NONE) {
- g_critical ("[vivante backend] Failed to parse tensor attributes from JSON");
- return HAL_ML_ERROR_INVALID_PARAMETER;
+ if (_helper_parse_tensor_attributes (tensor_obj, &tensor_attr) != HAL_ML_ERROR_NONE) {
+ g_critical ("[vivante] Failed to parse tensor attributes from JSON");
+ goto cleanup;
}
// Add the tensor to the graph
- vsi_nn_tensor_id_t vsi_output_id = vsi_nn_AddTensor (graph, VSI_NN_TENSOR_ID_AUTO, &tensor_attr, NULL);
+ vsi_nn_tensor_id_t vsi_output_id
+ = vsi_nn_AddTensor (self->graph, VSI_NN_TENSOR_ID_AUTO, &tensor_attr, NULL);
if (vsi_output_id == VSI_NN_TENSOR_ID_NA) {
- g_critical ("[vivante backend] Failed to add output tensor #%u", i);
- return HAL_ML_ERROR_RUNTIME_ERROR;
+ g_critical ("[vivante] Failed to add output tensor #%u", i);
+ goto cleanup;
}
- g_info ("[vivante backend] Added output tensor #%u with id %u", i, vsi_output_id);
+ g_info ("[vivante] Added output tensor #%u with id %u", i, vsi_output_id);
node->output.tensors[i] = vsi_output_id;
- graph->output.tensors[i] = vsi_output_id;
+ self->graph->output.tensors[i] = vsi_output_id;
}
for (guint i = 0; i < output_tensors_num; ++i) {
- g_info ("[vivante backend] print output tensor #%u:", graph->output.tensors[i]);
- vsi_nn_tensor_t *tensor = vsi_nn_GetTensor (graph, graph->output.tensors[i]);
- vsi_nn_PrintTensor (tensor, graph->output.tensors[i]);
+ g_info ("[vivante] print output tensor #%u:", self->graph->output.tensors[i]);
+ vsi_nn_tensor_t *tensor
+ = vsi_nn_GetTensor (self->graph, self->graph->output.tensors[i]);
+ vsi_nn_PrintTensor (tensor, self->graph->output.tensors[i]);
}
- // 9. Setup graph
- if (vsi_nn_SetupGraph (graph, FALSE) != VSI_SUCCESS) {
- g_critical ("[vivante backend] Failed to setup graph");
- return HAL_ML_ERROR_RUNTIME_ERROR;
+ // setup graph
+ if (vsi_nn_SetupGraph (self->graph, FALSE) != VSI_SUCCESS) {
+ g_critical ("[vivante] Failed to setup VSI graph.");
+ goto cleanup;
}
- self->graph = graph;
-
return HAL_ML_ERROR_NONE;
+
+cleanup:
+ _json_release_neural_network (self);
+ return HAL_ML_ERROR_RUNTIME_ERROR;
}
-static int
+/** @brief Releases all resources associated with a JSON-based graph. */
+static void
_json_release_neural_network (vivante_handle_s *self)
{
+ if (self->graph) {
+ vsi_nn_ReleaseGraph (&self->graph);
+ self->graph = NULL;
+ }
if (self->ctx) {
vsi_nn_ReleaseContext (&self->ctx);
self->ctx = NULL;
}
+}
- if (self->graph) {
- vsi_nn_ReleaseGraph (&self->graph);
- self->graph = NULL;
+/* ===================================================================
+ * Shared Library (.so) Loading Helpers
+ * ===================================================================
+ */
+
+/** @brief Creates the neural network from a pre-compiled shared library. */
+static int
+_so_create_neural_network (vivante_handle_s *self)
+{
+ self->dl_handle = dlopen (self->so_path, RTLD_NOW);
+ if (!self->dl_handle) {
+ g_critical ("[vivante] Failed to load shared library '%s': %s",
+ self->so_path, dlerror ());
+ return HAL_ML_ERROR_RUNTIME_ERROR;
+ }
+
+ self->model_specific_vnn_CreateNeuralNetwork = (vsi_nn_graph_t * (*) (const char *) )
+ dlsym (self->dl_handle, "vnn_CreateNeuralNetwork");
+ self->model_specific_vnn_ReleaseNeuralNetwork
+ = (void (*) (vsi_nn_graph_t *)) dlsym (self->dl_handle, "vnn_ReleaseNeuralNetwork");
+
+ if (!self->model_specific_vnn_CreateNeuralNetwork
+ || !self->model_specific_vnn_ReleaseNeuralNetwork) {
+ g_critical ("[vivante] Could not find required symbols in '%s'", self->so_path);
+ dlclose (self->dl_handle);
+ self->dl_handle = NULL;
+ return HAL_ML_ERROR_RUNTIME_ERROR;
+ }
+
+ if (self->has_post_process) {
+ self->model_specific_vnn_PostProcessNeuralNetwork = (vsi_status (*) (
+ vsi_nn_graph_t *)) dlsym (self->dl_handle, "vnn_PostProcessNeuralNetwork");
+ if (!self->model_specific_vnn_PostProcessNeuralNetwork) {
+ g_warning ("[vivante] 'postProcess' was requested, but symbol 'vnn_PostProcessNeuralNetwork' not found.");
+ self->has_post_process = FALSE;
+ }
+ }
+
+ self->graph = self->model_specific_vnn_CreateNeuralNetwork (self->model_path);
+ if (!self->graph) {
+ g_critical ("[vivante] vnn_CreateNeuralNetwork failed for model '%s'", self->model_path);
+ return HAL_ML_ERROR_RUNTIME_ERROR;
}
+
+ return HAL_ML_ERROR_NONE;
+}
+
+/* ===================================================================
+ * Main HAL Implementation Functions
+ * ===================================================================
+ */
+static int
+ml_vivante_init (void **backend_private)
+{
+ vivante_handle_s *vivante = g_new0 (vivante_handle_s, 1);
+
+ gst_tensors_info_init (&vivante->inputInfo);
+ gst_tensors_info_init (&vivante->outputInfo);
+
+ vivante->use_json_for_graph = TRUE;
+ vivante->has_post_process = FALSE;
+
+ *backend_private = vivante;
+ return HAL_ML_ERROR_NONE;
}
static int
vivante_handle_s *vivante = (vivante_handle_s *) backend_private;
if (!vivante) {
- g_critical ("[vivante backend] invalid backend_private");
+ g_critical ("[vivante] invalid backend_private");
return HAL_ML_ERROR_INVALID_PARAMETER;
}
- if (vivante->using_json_for_graph_setup) {
+ if (vivante->use_json_for_graph) {
_json_release_neural_network (vivante);
} else {
- if (vivante->graph)
- vivante->result_vnn_ReleaseNeuralNetwork (vivante->graph);
-
- if (vivante->handle)
- dlclose (vivante->handle);
+ if (vivante->graph && vivante->model_specific_vnn_ReleaseNeuralNetwork) {
+ vivante->model_specific_vnn_ReleaseNeuralNetwork (vivante->graph);
+ }
+ if (vivante->dl_handle) {
+ dlclose (vivante->dl_handle);
+ }
}
gst_tensors_info_free (&vivante->inputInfo);
const GstTensorFilterProperties *prop = (const GstTensorFilterProperties *) prop_;
vivante_handle_s *vivante = (vivante_handle_s *) backend_private;
- if (!vivante) {
- g_critical ("[vivante backend] invalid backend_private");
+ if (!vivante || !prop) {
+ g_critical ("[vivante] invalid backend_private");
return HAL_ML_ERROR_INVALID_PARAMETER;
}
+ vivante->model_path = g_strdup (prop->model_files[0]);
+ // Default loading strategy: if more than one model file is given, assume .so loading.
+ if (prop->num_models > 1) {
+ vivante->use_json_for_graph = FALSE;
+ }
+
/* Parse custom properties */
if (prop->custom_properties) {
gchar **options = g_strsplit (prop->custom_properties, ",", -1);
g_strstrip (option[1]);
if (g_ascii_strcasecmp (option[0], "json") == 0) {
- vivante->using_json_for_graph_setup = TRUE;
+ vivante->use_json_for_graph = TRUE;
vivante->json_path = g_strdup (option[1]);
- g_info ("[vivante backend] Using JSON for graph setup with JSON file: %s", option[1]);
+ g_info ("[vivante] Using JSON for graph setup: %s", vivante->json_path);
} else {
g_warning ("Unknown option (%s).", options[op]);
}
g_strfreev (options);
}
- vivante->model_path = g_strdup (prop->model_files[0]);
- if (prop->num_models > 1) {
- vivante->using_json_for_graph_setup = FALSE;
- }
-
-
- /* do model loading */
- if (vivante->using_json_for_graph_setup) {
- /* .json based initializing */
- int error = _json_create_neural_network (vivante);
- if (error != HAL_ML_ERROR_NONE) {
- g_critical ("[vivante backend] Failed to create neural network from JSON");
- _json_release_neural_network (vivante);
- return error;
+ /* Load model based on the determined strategy JSON vs so */
+ if (vivante->use_json_for_graph) {
+ if (!vivante->json_path || !g_file_test (vivante->json_path, G_FILE_TEST_IS_REGULAR)) {
+ g_critical ("[vivante] JSON loading was selected, but no JSON path was provided via 'json:' custom property.");
+ return HAL_ML_ERROR_INVALID_PARAMETER;
+ }
+ if (_json_create_neural_network (vivante) != HAL_ML_ERROR_NONE) {
+ g_critical ("[vivante] Failed to create VSI graph.");
}
- g_info ("[vivante backend] Model loaded successfully from JSON!");
} else {
- /* setup .so */
- vivante->so_path = g_strdup (prop->model_files[1]);
-
- vivante->handle = dlopen (vivante->so_path, RTLD_NOW);
- if (!vivante->handle) {
- g_critical ("Failed to load shared library: %s", vivante->so_path);
- return HAL_ML_ERROR_RUNTIME_ERROR;
+ if (prop->num_models <= 1) {
+ g_critical ("[vivante] .so loading requires a second model file (the .so path).");
+ return HAL_ML_ERROR_INVALID_PARAMETER;
}
-
- /* get symbols */
- vivante->result_vnn_ReleaseNeuralNetwork = (void (*) (vsi_nn_graph_t *)) dlsym (
- vivante->handle, "vnn_ReleaseNeuralNetwork");
- vivante->result_vnn_CreateNeuralNetwork = (vsi_nn_graph_t * (*) (const char *) )
- dlsym (vivante->handle, "vnn_CreateNeuralNetwork");
-
- if (vivante->postProcess) {
- vivante->postProcessFunc = (vsi_status (*) (vsi_nn_graph_t *)) dlsym (
- vivante->handle, "vnn_PostProcessNeuralNetwork");
+ vivante->so_path = g_strdup (prop->model_files[1]);
+ if (_so_create_neural_network (vivante) != HAL_ML_ERROR_NONE) {
+ g_critical ("[vivante] Failed to create VSI graph.");
}
-
- /* .so based initializing */
- vivante->graph = vivante->result_vnn_CreateNeuralNetwork (vivante->model_path);
}
/* setting input and output tensors info */
- gst_tensors_info_init (&vivante->inputInfo);
- gst_tensors_info_init (&vivante->outputInfo);
-
vivante->inputInfo.num_tensors = vivante->graph->input.num;
for (unsigned int i = 0; i < vivante->graph->input.num; i++) {
vsi_nn_tensor_t *i_tensor
= vsi_nn_GetTensor (vivante->graph, vivante->graph->input.tensors[i]);
GstTensorInfo *info = gst_tensors_info_get_nth_info (&vivante->inputInfo, i);
- info->type = convert_tensortype (i_tensor->attr.dtype.vx_type);
+ info->type = convert_to_tensor_type (i_tensor->attr.dtype.vx_type);
info->name = g_strdup_printf ("%i", vivante->graph->input.tensors[i]);
for (unsigned int j = 0; j < i_tensor->attr.dim_num; ++j) {
info->dimension[j] = i_tensor->attr.size[j];
= vsi_nn_GetTensor (vivante->graph, vivante->graph->output.tensors[i]);
GstTensorInfo *info = gst_tensors_info_get_nth_info (&vivante->outputInfo, i);
- info->type = convert_tensortype (o_tensor->attr.dtype.vx_type);
+ info->type = convert_to_tensor_type (o_tensor->attr.dtype.vx_type);
info->name = g_strdup_printf ("%i", vivante->graph->output.tensors[i]);
for (unsigned int j = 0; j < o_tensor->attr.dim_num; ++j) {
info->dimension[j] = o_tensor->attr.size[j];
vivante_handle_s *vivante = (vivante_handle_s *) backend_private;
if (!vivante) {
- g_critical ("[vivante backend] invalid backend_private");
+ g_critical ("[vivante] invalid backend_private");
return HAL_ML_ERROR_INVALID_PARAMETER;
}
vsi_nn_tensor_t *tensor
= vsi_nn_GetTensor (vivante->graph, vivante->graph->input.tensors[i]);
if (vsi_nn_CopyDataToTensor (vivante->graph, tensor, (uint8_t *) input[i].data) != VSI_SUCCESS) {
- g_critical ("[vivante backend] Failed to copy data to tensor");
+ g_critical ("[vivante] Failed to copy data to tensor");
return HAL_ML_ERROR_RUNTIME_ERROR;
}
}
if (vsi_nn_RunGraph (vivante->graph) != VSI_SUCCESS) {
- g_critical ("[vivante backend] Failed to run graph");
+ g_critical ("[vivante] Failed to run graph");
return HAL_ML_ERROR_RUNTIME_ERROR;
}
- if (vivante->postProcess)
- vivante->postProcessFunc (vivante->graph);
+ if (vivante->has_post_process)
+ vivante->model_specific_vnn_PostProcessNeuralNetwork (vivante->graph);
for (unsigned int i = 0; i < vivante->graph->output.num; i++) {
vsi_nn_tensor_t *out_tensor
}
static int
-ml_vivante_get_model_info (void *backend_private, int ops_, void *in_info_, void *out_info_)
+ml_vivante_get_model_info (void *backend_private, int ops, void *in_info, void *out_info)
{
- int ops = (model_info_ops) ops_;
- GstTensorsInfo *in_info = (GstTensorsInfo *) in_info_;
- GstTensorsInfo *out_info = (GstTensorsInfo *) out_info_;
vivante_handle_s *vivante = (vivante_handle_s *) backend_private;
-
- if (!vivante) {
- g_critical ("[vivante backend] invalid backend_private");
+ if (!vivante)
return HAL_ML_ERROR_INVALID_PARAMETER;
- }
- gst_tensors_info_copy (in_info, &vivante->inputInfo);
- gst_tensors_info_copy (out_info, &vivante->outputInfo);
+ gst_tensors_info_copy ((GstTensorsInfo *) in_info, &vivante->inputInfo);
+ gst_tensors_info_copy ((GstTensorsInfo *) out_info, &vivante->outputInfo);
return HAL_ML_ERROR_NONE;
}
static int
-ml_vivante_event_handler (void *backend_private, int ops_, void *data_)
+ml_vivante_event_handler (void *backend_private, int ops, void *data)
{
return HAL_ML_ERROR_NOT_SUPPORTED;
}
projects / platform / hal / backend / ml-accelerator.git / commitdiff