This patch revises the host handler and libnpuhost APIs.
Signed-off-by: Dongju Chae <dongju.chae@samsung.com>
char reserved_compiler[2048]; /**< Reserved for NPU Compiler */
char reserved_extra[]; /**< Reserved for future; zero-length array */
-} __attribute__((packed)) npubin_meta;
+} __attribute__((packed, aligned)) npubin_meta;
/* Compile-time assert. From http://www.pixelbeat.org/programming/gcc/static_assert.html */
#define ASSERT_CONCAT_(a, b) a##b
#ifndef NPU_TYPEDEF_H__
#define NPU_TYPEDEF_H__
-#define DEVICETYPE_NPU (0x10000) /** SR-NPU 2019, RefineDet */
-#define DEVICETYPE_ASR (0x20000) /** SR-ASR-IP 2019, Based on SRP */
-#define DEVICETYPE_MASK (0xFFFF0000)
+#define DEVICETYPE_NPU (0x10000) /** SR-NPU 2019, RefineDet */
+#define DEVICETYPE_ASR (0x20000) /** SR-ASR-IP 2019, Based on SRP */
+
+/** alias */
+#define DEVICETYPE_TRIV DEVICETYPE_NPU
+#define DEVICETYPE_TRIA DEVICETYPE_ASR
+#define DEVICETYPE_TRIV2 (0x30000) /** SR-NPU 2020 */
-/* alias */
-#define DEVICETYPE_VISION DEVICETYPE_NPU
+#define DEVICETYPE_MASK (0xFFFF0000)
/**
* @brief Description of npu device types.
NPUCOND_ASR_CONN_UNKNOWN = (DEVICETYPE_ASR | 0), /**< As long as it is ASR(Audio-SRP), I don't care */
NPUCOND_ASR_CONN_SOCIP = (DEVICETYPE_ASR | 2), /**< SOCIP type ASR(Audio-SRP) */
+
+ NPUCOND_TRIV_CONN_UNKNOWN = (DEVICETYPE_TRIV | 0),
+ NPUCOND_TRIV_CONN_SOCIP = (DEVICETYPE_TRIV | 2),
+
+ NPUCOND_TRIV2_CONN_UNKNOWN = (DEVICETYPE_TRIV2 | 0),
+ NPUCOND_TRIV2_CONN_SOCIP = (DEVICETYPE_TRIV2 | 2),
+
+ NPUCOND_TRIA_CONN_UNKNOWN = (DEVICETYPE_TRIA | 0),
+ NPUCOND_TRIA_CONN_SOCIP = (DEVICETYPE_TRIA | 2),
} dev_type;
/**
/**
* @brief Operable modes of NPU when the inputs are from NPU's own hardware.
+ * @note this mode will decide which input service performs the inference of a model.
*/
typedef enum {
- NPUINPUT_STOP = 0, /**< Stop Processing */
- NPUINPUT_INTERNAL_CAM = 1, /**< Let ADSP preprocess image stream
- from MIPI and send it to NPU-core
- to be processed with the given
- model. */
- NPUINPUT_I2S_MIC = 2, /**< ASR mode with on-chip internal I2S. modelid is ignored. */
- NPUINPUT_HOST = 3, /**< Process input frames transmitted from Host. Both ASR or Vision NPUs may use this. */
- NPUINPUT_HW = 4, /**< Process input frames transmitted from third-party HW. */
+ NPUINPUT_STOP = 0, /**< Stop Processing */
+ NPUINPUT_INTERNAL_CAM = 1, /**< Let ADSP preprocess image stream from MIPI and send it to
+ NPU-core to be processed with the given model.
+ (not supported yet) */
+ NPUINPUT_I2S_MIC = 2, /**< ASR mode with on-chip internal I2S. modelid is ignored.
+ (not supported yet) */
+ NPUINPUT_HOST = 3, /**< Process input frames transmitted from Host.
+ TRIA, TRIV, and TRIV2 may use this. */
+ NPUINPUT_HW_RECURRING = 4, /**< Process input frames transmitted from third-party HW.
+ TRIV2 may use this for high-priority models. */
} npu_input_opmode;
#define IS_DEVICE (val, devname) (((val) & DEVICETYPE_MASK) == DEVICETYPE_ ## devname) /** E.g., IS_DEVICE(dev, ASR) */
* @brief Various device types to be supported for running models
*/
typedef enum {
- SMODEL_OPS_NPU = 0, /**< NPU model */
- SMODEL_OPS_ARM = 1, /**< ARM model */
- SMODEL_OPS_INTERNAL_CAM = 2, /**< Internal Camera model */
- SMODEL_OPS_I2S_MIC = 3, /**< Internal Mic model */
- SMODEL_OPS_NPU_ASR = 4, /** NPU-ASR model (used only internally) */
- SMODEL_OPS_RECURRING = 5, /** NPU Recurring model (TRIV-2) */
+ SMODEL_OPS_NPU = 0, /**< NPU model (TRIV/TRIV2) */
+ SMODEL_OPS_ARM = 1, /**< ARM model (not supported yet) */
+ SMODEL_OPS_INTERNAL_CAM = 2, /**< Internal Camera model (not supported yet) */
+ SMODEL_OPS_I2S_MIC = 3, /**< Internal Mic model (not supported yet) */
+ SMODEL_OPS_NPU_ASR = 4, /**< ASR model (TRIA) */
SMODEL_OPS_END,
} model_opmode;
int dmabuf; /**< The dma-buf fd handle of the memory allocated */
/** @todo offset is not supported yet */
uint64_t offset; /**< Offset to be applied to the base memory address */
- void *priv; /**< Private data for dma-buf mapping */
};
};
};
* The below provides the usage of buffer allocation APIs.
* {
* generic_buffer model, input;
+ * npudev_h dev;
+ *
+ * if (getNPUdevice (&dev, 0) != 0)
+ * return;
*
* model.type = BUFFER_MAPPED;
* model.size = 4096;
- * if (allocModelBuffer (&model) != 0) {
+ * if (allocNPU_modelBuffer (dev, &model) != 0) {
* ...
* }
*
* input.type = BUFFER_MAPPED;
* input.size = 4096;
- * if (allocInputBuffer (&input) != 0) {
+ * if (allocNPU_inputBuffer (dev, &input) != 0) {
* ...
* }
*
- * cleanModelBuffer (&model);
- * cleanInputBuffer (&input);
+ * cleanNPU_modelBuffer (dev, &model);
+ * cleanNPU_inputBuffer (dev, &input);
* }
*/
/**
* @brief Allocate a buffer for NPU model with the requested buffer type.
+ * @param[in] dev The NPU device handle
* @param[in/out] Buffer the buffer pointer where memory is allocated.
* @return 0 if no error, otherwise a negative errno.
*/
-int allocModelBuffer (generic_buffer *buffer);
+int allocNPU_modelBuffer (npudev_h dev, generic_buffer *buffer);
/**
* @brief Free the buffer and remove the address mapping.
+ * @param[in] dev The NPU device handle
+ * @param[in] buffer the model buffer
* @return 0 if no error, otherwise a negative errno.
*/
-int cleanModelBuffer (generic_buffer *buffer);
+int cleanNPU_modelBuffer (npudev_h dev, generic_buffer *buffer);
/**
* @brief Allocate a buffer for NPU input with the requested buffer type.
+ * @param[in] dev The NPU device handle
* @param[in/out] Buffer the buffer pointer where memory is allocated.
* @return 0 if no error, otherwise a negative errno.
* @note please utilize allocInputBuffers() for multiple input tensors because subsequent
* calls of allocInputBuffer() don't gurantee contiguous allocations between them.
*/
-int allocInputBuffer (generic_buffer *buffer);
+int allocNPU_inputBuffer (npudev_h dev, generic_buffer *buffer);
/**
* @brief Free the buffer and remove the address mapping.
+ * @param[in] dev The NPU device handle
+ * @param[in] buffer the input buffer
* @return 0 if no error, otherwise a negative errno.
-*/
-int cleanInputBuffer (generic_buffer *buffer);
+ */
+int cleanNPU_inputBuffer (npudev_h dev, generic_buffer *buffer);
/**
* @brief Allocate input buffers, which have multiple instances of generic_buffer
+ * @param[in] dev The NPU device handle
* @param[in/out] input input buffers.
* @return 0 if no error, otherwise a negative errno.
* @note it reuses allocInputBuffer().
* @details in case of BUFFER_DMABUF, this function can be used to gurantee physically-contiguous
* memory mapping for multiple tensors (in a single inference, not batch size).
*/
-int allocInputBuffers (input_buffers * input);
+int allocNPU_inputBuffers (npudev_h dev, input_buffers * input);
/**
* @brief Free input buffers allocated by allocInputBuffers().
+ * @param[in] dev The NPU device handle
* @param[in/out] input input buffers.
* @note it reuses cleanInputbuffer().
* @return 0 if no error, otherwise a negative errno.
*/
-int cleanInputBuffers (input_buffers * input);
+int cleanNPU_inputBuffers (npudev_h dev, input_buffers * input);
/** deprecated buffer APIs; please use the above APIs */
-/**
- * @deprecated
- * @brief Allocate a buffer for NPU with the requested buffer type.
- * @param[in] size the requested size of memory.
- * @param[in] type the requested buffer type.
- * @param[in] [OPTIONAL] filepath the path of file wrapped in the buffer (only for BUFFER_FILE).
- * NULL will be provided for other buffer types.
- * @param[out] buffer fill its contents if no error (it should be allocated before)
- * @return 0 if no error, otherwise a negative errno.
- */
+/** @brief deprecated */
+int allocModelBuffer (generic_buffer *buffer);
+/** @brief deprecated */
+int cleanModelBuffer (generic_buffer *buffer);
+/** @brief deprecated */
+int allocInputBuffer (generic_buffer *buffer);
+/** @brief deprecated */
+int cleanInputBuffer (generic_buffer *buffer);
+/** @brief deprecated */
+int allocInputBuffers (input_buffers * input);
+/** @brief deprecated */
+int cleanInputBuffers (input_buffers * input);
+/** @brief deprecated */
int allocNPUBuffer (uint64_t size, buffer_types type,
const char * filepath, generic_buffer *buffer);
-
-/**
- * @deprecated
- * @brief Free the buffer and remove the address mapping.
- * @param[in] buffer the generic buffer instance.
- * @return @c 0 if no error. otherwise a negative error value.
- * @note it does not free generic_buffer itself
- */
+/** @brief deprecated */
int cleanNPUBuffer (generic_buffer * buffer);
#if defined(__cplusplus)
+++ /dev/null
-/**
- * Proprietary
- * Copyright (C) 2019 Samsung Electronics
- * Copyright (C) 2019 Dongju Chae <dongju.chae@samsung.com>
- */
-/**
- * @file NE-handler.c
- * @date 25 Jun 2019
- * @brief Host (model) handler for NPU Engine (NE).
- * @see http://suprem.sec.samsung.net/confluence/display/ODLC/Software+Stack
- * @author Dongju Chae <dongju.chae@samsung.com>
- * @bug No known bugs except for NYI items
- */
-
-#include "ne-comm.h"
-#include "ne-conf.h"
-#include "ne-handler.h"
-#include "ne-mem.h"
-#include "ne-scheduler.h"
-
-#include <libnpuhost.h>
-#include <typedef.h>
-
-#include <stdio.h>
-#include <assert.h>
-#include <errno.h>
-#include <pthread.h>
-#include <string.h>
-
-#define TAG _N2
-
-/**
- * @brief private data structure for host handler
- */
-typedef struct {
- uint64_t reserved_size; /**< it can be zero if KERNEL_CMA is used */
- uint64_t model_size;
-#ifdef ENABLE_BUFFERING
- uint64_t buffer_size;
-#endif
- uint32_t num_models;
- output_ready n1_cb;
- pthread_mutex_t mutex;
- list model_priv_list;
-} handler_priv;
-
-/**
-* @brief static instance of private data structure for host handler
-*/
-static handler_priv hpriv;
-
-/**
- * @brief private data structure for model
- */
-typedef struct {
- model model;
- npubin_meta meta;
- hwmem *hwmem;
- list_node list;
-} model_priv;
-
-/**
- * @brief a global lock for host handler
- */
-#define HANDLER_LOCK() pthread_mutex_lock(&hpriv.mutex)
-#define HANDLER_UNLOCK() pthread_mutex_unlock(&hpriv.mutex)
-
-/**
- * @brief get a (sub-)model's private data
- */
-#define GET_MODEL_PRIVATE(m) CONTAINER_OF(m, model_priv, model)
-
-/**
- * @brief calculate the tensor size depending on its layout
- * @param[in] meta metadata for model
- * @param[in] idx tensor index
- * @param[in] layout tensor data layout
- * @param[in] is_input input or output
- * @return the calculated tensor size
- */
-static uint32_t
-calc_tensor_size (const npubin_meta *meta, int idx,
- data_layout layout, bool is_input)
-{
- uint32_t batch, width, height, depth;
- uint32_t elem_size, tensor_size;
- int rank_idx;
-
- switch (layout) {
- case DATA_LAYOUT_NONE:
- case DATA_LAYOUT_NHWC:
- case DATA_LAYOUT_NCHW:
- if (is_input) {
- tensor_size = meta->input_elem_size[idx];
- for (rank_idx = 0; rank_idx < MAX_RANK; rank_idx++) {
- if (meta->input_dims[idx][rank_idx] == 0)
- break;
- tensor_size *= meta->input_dims[idx][rank_idx];
- }
- } else {
- tensor_size = meta->output_elem_size[idx];
- for (rank_idx = 0; rank_idx < MAX_RANK; rank_idx++) {
- if (meta->output_dims[idx][rank_idx] == 0)
- break;
- tensor_size *= meta->output_dims[idx][rank_idx];
- }
- }
- break;
- case DATA_LAYOUT_SRNPU:
- if (is_input) {
- elem_size = meta->input_elem_size[idx];
- batch = meta->input_dims[idx][0];
- width = meta->input_dims[idx][1];
- height = meta->input_dims[idx][2];
- depth = meta->input_dims[idx][3];
- } else {
- elem_size = meta->output_elem_size[idx];
- batch = meta->output_dims[idx][0];
- width = meta->output_dims[idx][1];
- height = meta->output_dims[idx][2];
- depth = meta->output_dims[idx][3];
- }
-
- /** @todo verify the correctness of below calculation later */
- tensor_size = batch * elem_size * width * height;
- if (depth == 3 || depth % DATA_GRANULARITY == 0) {
- tensor_size *= depth;
- } else {
- tensor_size *= (1 + depth / DATA_GRANULARITY);
- }
- break;
- default:
- assert (0);
- }
-
- return tensor_size;
-}
-
-/**
- * @brief return data type size
- */
-static int get_data_size (data_type type)
-{
- switch (type) {
- case DATA_TYPE_SRNPU:
- case DATA_TYPE_INT8:
- case DATA_TYPE_UINT8:
- return sizeof(uint8_t);
- case DATA_TYPE_INT16:
- case DATA_TYPE_UINT16:
- return sizeof(uint16_t);
- case DATA_TYPE_INT32:
- case DATA_TYPE_UINT32:
- case DATA_TYPE_FLOAT32:
- return sizeof(uint32_t);
- case DATA_TYPE_INT64:
- case DATA_TYPE_UINT64:
- case DATA_TYPE_FLOAT64:
- return sizeof(uint64_t);
- default:
- logerr (TAG, "Unsupported datatype %d\n", type);
- return 0;
- }
-}
-
-#define do_quantized_memcpy(type) do {\
- idx = 0;\
- if (quant) {\
- while (idx < num_elems) {\
- val = ((type *) src)[idx];\
- val = val / _scale;\
- val += _zero_point;\
- val = (val > 255.0) ? 255.0 : 0.0;\
- ((uint8_t *) dst)[idx++] = (uint8_t) val;\
- }\
- } else {\
- while (idx < num_elems) {\
- val = *(uint8_t *) src;\
- val -= _zero_point;\
- val *= _scale;\
- ((type *) dst)[idx++] = (type) val;\
- dst = (void*)(((uint8_t *) dst) + data_size);\
- src = (void*)(((uint8_t *) src) + 1);\
- }\
- }\
- } while (0)
-/**
- * @brief memcpy during quantization
- */
-static void memcpy_with_quant (data_type type, float scale, uint32_t zero_point,
- void *dst, const void *src, uint32_t num_elems, bool quant)
-{
- double _scale = (double) scale;
- double _zero_point = (double) zero_point;
- double val;
- int data_size = get_data_size (type);
- int idx;
-
- switch (type) {
- case DATA_TYPE_INT8:
- do_quantized_memcpy (int8_t);
- break;
- case DATA_TYPE_UINT8:
- do_quantized_memcpy (uint8_t);
- break;
- case DATA_TYPE_INT16:
- do_quantized_memcpy (int16_t);
- break;
- case DATA_TYPE_UINT16:
- do_quantized_memcpy (uint16_t);
- break;
- case DATA_TYPE_INT32:
- do_quantized_memcpy (int32_t);
- break;
- case DATA_TYPE_UINT32:
- do_quantized_memcpy (uint32_t);
- break;
- case DATA_TYPE_INT64:
- do_quantized_memcpy (int64_t);
- break;
- case DATA_TYPE_UINT64:
- do_quantized_memcpy (uint64_t);
- break;
- case DATA_TYPE_FLOAT32:
- do_quantized_memcpy (float);
- break;
- case DATA_TYPE_FLOAT64:
- do_quantized_memcpy (double);
- break;
- default:
- logerr (TAG, "Unsupported datatype %d\n", type);
- }
-}
-
-/**
- * @brief perform data manipulation
- * @param[in] meta metadata for target model
- * @param[in] info the tensors info
- * @param[in] idx tensor index
- * @param[in] src_size source buffer size
- * @param[in] src source buffer (feature map)
- * @param[out] dst destination buffer
- * @param[in] is_input indicate it's input manipulation
- * @return 0 if no error, otherwise a negative errno
- *
- * @note the input data format should be NHWC
- * @todo add the extra info for data format in npubinfmt.h
- * support quantization along with data manipulation
- * @detail rules for the memory address of activations in NPU HW.
- * (https://code.sec.samsung.net/confluence/pages/viewpage.action?pageId=146491864)
- *
- * 1) Special case (depth == 3)
- * - addr(x,y,z) = addr(0,0,0) + (z) + 3 * (x + width * y)
- *
- * 2) Common case
- * - addr(x,y,z) = addr(0,0,0) + (z % MPA_L) + MPA_L * (x + width * (y + height * (z / MPA_L)))
- *
- * Thus, if depth is not a multiple of MPA_L (i.e., 64), zero padding is required
- */
-static int
-manipulate_data (const npubin_meta *meta, const tensors_data_info *info,
- uint32_t idx, uint32_t src_size, const void *src, void *dst, bool is_input)
-{
- uint32_t batch;
- uint32_t height;
- uint32_t width;
- uint32_t depth;
- uint32_t zero_point;
- float scale;
- int data_size;
- bool need_quantization = false;
-
- /** check metadata */
- if (NPUBIN_VERSION(meta->magiccode) < 2) {
- logerr (TAG, "Data manipulation requires at least npubinfmt v2\n");
- return -EINVAL;
- }
-
- if (is_input) {
- if (idx >= meta->input_num) {
- logerr (TAG, "Wrong information for input tensors in metadata\n");
- return -ERANGE;
- }
-
- batch = meta->input_dims[idx][0];
- height = meta->input_dims[idx][1];
- width = meta->input_dims[idx][2];
- depth = meta->input_dims[idx][3];
- } else {
- if (idx >= meta->output_num) {
- logerr (TAG, "Wrong information for output tensors in metadata\n");
- return -ERANGE;
- }
-
- batch = meta->output_dims[idx][0];
- height = meta->output_dims[idx][1];
- width = meta->output_dims[idx][2];
- depth = meta->output_dims[idx][3];
- }
-
-#ifdef ENABLE_MANIP
- /* check quantization parameters */
- if (is_input) {
- zero_point = meta->input_quant_z[idx];
- scale = meta->input_quant_s[idx];
- } else {
- zero_point = meta->output_quant_z[idx];
- scale = meta->output_quant_s[idx];
- }
-
- data_size = get_data_size (info->info[idx].type);
- /**
- * note that we assume DATA_TYPE_SRNPU is the smallest data type that we consider.
- * Also, DATA_TYPE_SRNPU and uint8_t may be regarded as the same in the view of apps.
- */
- if (info->info[idx].type != DATA_TYPE_SRNPU) {
- assert (data_size >= get_data_size (DATA_TYPE_SRNPU));
-
- if (data_size > get_data_size (DATA_TYPE_SRNPU) ||
- !(zero_point == 0 && scale == 1.0))
- need_quantization = true;
- }
-
- /** check data manipulation is required */
- if (depth != 3 && depth != 64 && info->info[idx].layout != DATA_LAYOUT_SRNPU) {
- uint32_t MPA_L = DATA_GRANULARITY;
- uint32_t n, h, w, d;
- uint32_t std_offset; /* standard offset in NHWC data format */
- uint32_t npu_offset; /* npu offset in NPU HW data format*/
- uint32_t src_offset;
- uint32_t dst_offset;
- uint32_t slice_size;
-
- if (idx >= info->num_info) {
- logerr (TAG, "Unmatched tensors info\n");
- return -ERANGE;
- }
-
- /* @todo we currently support only NHWC */
- if (info->info[idx].layout != DATA_LAYOUT_NHWC) {
- logerr (TAG, "data manipulation is supported for NHWC only\n");
- return -EINVAL;
- }
-
- for (n = 0; n < batch; n++) {
- for (h = 0; h < height; h++) {
- for (w = 0; w < width; w++) {
- for (d = 0; d < depth; d += MPA_L) {
- std_offset = d + depth * (w + width * (h + n * height));
- npu_offset = MPA_L * (w + width * (h + (n + d / MPA_L) * height));
- slice_size = (depth - d >= MPA_L) ? MPA_L : depth - d;
-
- if (is_input) {
- src_offset = std_offset * data_size;
- dst_offset = npu_offset;
- } else {
- src_offset = npu_offset;
- dst_offset = std_offset * data_size;
- }
-
- /* if depth is not a multiple of MPA_L, add zero paddings (not exact values) */
- if (need_quantization) {
- memcpy_with_quant (info->info[idx].type, scale, zero_point,
- dst + dst_offset, src + src_offset, slice_size, is_input);
- } else {
- memcpy (dst + dst_offset, src + src_offset, slice_size);
- }
- }
- }
- }
- }
- } else if (need_quantization) {
- /** depth == 3 || depth == 64; special cases which can directly copy input tensor data */
- if (is_input)
- src_size = src_size / data_size;
-
- memcpy_with_quant (info->info[idx].type, scale, zero_point,
- dst, src, src_size, is_input);
- } else {
-#endif
- memcpy (dst, src, src_size);
-#ifdef ENABLE_MANIP
- }
-#endif
- return 0;
-}
-
-/**
- * @brief setup data from buffer for non-BUFFER_DMABUF type
- * @param[in] in_buf the generic buffer for data (e.g., model or input)
- * @param[out] data_ptr data pointer where data is stored
- * @param[in] size number of bytes to be moved
- * @return @c 0 if no error. otherwise a negative error value
- */
-static int
-setup_data_from_buffer (const generic_buffer *in_buf, void *data_ptr, uint64_t size)
-{
- buffer_types type;
-
- if (!in_buf) {
- logerr (TAG, "Invalid input buffer\n");
- return -EINVAL;
- }
-
- type = in_buf->type;
- switch (type) {
- case BUFFER_MAPPED:
- {
- if (in_buf->addr == NULL || data_ptr == NULL) {
- logerr (TAG, "Invalid input buffer\n");
- return -EINVAL;
- }
-
- memcpy (data_ptr, in_buf->addr, size);
- }
- break;
- case BUFFER_FILE:
- {
- FILE *fp;
- size_t read_size;
-
- if (in_buf->filepath == NULL || data_ptr == NULL) {
- logerr (TAG, "Invalid input buffer\n");
- return -EINVAL;
- }
-
- fp = fopen (in_buf->filepath, "r");
- if (fp == NULL) {
- logerr (TAG, "Fail to create output file\n");
- return -errno;
- }
-
- read_size = fread (data_ptr, 1, size, fp);
- fclose (fp);
-
- /** Read the data from the file */
- if (read_size != size) {
- logerr (TAG, "Failed to read all data, only read %d out of %d\n",
- read_size, size);
- return -ENOMEM;
- }
- }
- break;
- default:
- /** in case of BUFFER_DMABUF, there is nothing to do because
- * buffers were already stored in user-provided memory */
- logerr (TAG, "Unsupported input buffer type: %d\n", type);
- return -EINVAL;
- }
-
- return 0;
-}
-
-/**
- * @brief setup buffer for data for non-BUFFER_DMABUF type
- * @param[in] data_ptr data pointer where data is located
- * @param[out] out_buf the generic buffer for output data
- * @param[in] size number of bytes to be moved
- * @return @c 0 if no error. otherwise a negative error value
- */
-static int
-setup_buffer_from_data (const void *data_ptr, generic_buffer *out_buf, uint64_t size)
-{
- buffer_types type;
-
- if (!out_buf) {
- logerr (TAG, "Invalid output buffer\n");
- return -EINVAL;
- }
-
- type = out_buf->type;
- switch (type) {
- case BUFFER_MAPPED:
- {
- if (size > out_buf->size) {
- logerr (TAG, "Buffer has no enough memory size\n");
- return -ENOMEM;
- }
-
- if (data_ptr == NULL || out_buf->addr == NULL) {
- logerr (TAG, "Invalid output buffer\n");
- return -EINVAL;
- }
-
- memcpy (out_buf->addr, data_ptr, size);
- }
- break;
- case BUFFER_FILE:
- {
- FILE *fp;
- size_t write_size;
-
- if (data_ptr == NULL || out_buf->filepath == NULL) {
- logerr (TAG, "Invalid output buffer\n");
- return -EINVAL;
- }
-
- fp = fopen (out_buf->filepath, "a");
- if (fp == NULL) {
- logerr (TAG, "Fail to create output file\n");
- return -errno;
- }
-
- /** Read the data from the file */
- write_size = fwrite (data_ptr, size, 1, fp);
- fclose (fp);
-
- if (write_size != size) {
- logerr (TAG, "Failed to write all data, only write %lu out of %lu\n",
- write_size, size);
- return -ENOMEM;
- }
- }
- break;
- default:
- /** in case of BUFFER_DMABUF, there is nothing to do because
- * buffers were already stored in user-provided memory */
- logerr (TAG, "Unsupported output buffer type: %d\n", type);
- return -EINVAL;
- }
-
- return 0;
-}
-
-/**
- * @brief retrieve the data in input buffers to hwmem
- * @param[in] meta metadata for model
- * @param[in] info the input tensors info
- * @param[in] input input buffers
- * @param[out] data_ptr data pointer for the internal buffer
- * @return 0 if no error, otherwise a negative errno
- */
-static int
-setup_input_buffers (const npubin_meta *meta, const tensors_data_info *info,
- const input_buffers *input, void *data_ptr)
-{
- int err, idx;
-
- switch (NPUBIN_VERSION(meta->magiccode)) {
- case 0: /* if not specified, it's regarded as version 1 */
- case 1:
- {
- uint64_t offset = meta->input_offset;
-
- for (idx = 0; idx < input->num_buffers; idx++) {
- const generic_buffer *buffer = &input->bufs[idx];
-
- /** assume contiguous tensor data */
- if ((err = setup_data_from_buffer (buffer, data_ptr + offset, buffer->size)) != 0) {
- logerr (TAG, "Error setting data to hwnem, errno: %d\n", err);
- return err;
- }
-
- offset += buffer->size;
- }
- }
- break;
- case 2:
- {
- for (idx = 0; idx < input->num_buffers; idx++) {
- const generic_buffer *buffer = &input->bufs[idx];
-
- /**
- * NPU Engine performs the data manipulation only for BUFFER_MAPPED type
- * because it eventually causes addtional memory copies. Thus, for other types
- * (e.g., BUFFER_FILE, BUFFER_DMABUF), users should manually manipulate feature maps.
- */
- if (buffer->type == BUFFER_MAPPED) {
- err = manipulate_data (meta, info, idx, buffer->size,
- buffer->addr, data_ptr + meta->input_offsets[idx], true);
- } else {
- err = setup_data_from_buffer (buffer, data_ptr + meta->input_offsets[idx],
- buffer->size);
- }
-
- if (err != 0) {
- logerr (TAG, "Error setting data to hwmem, errno: %d\n", err);
- return err;
- }
- }
- }
- break;
- default:
- logerr (TAG, "Unknown npubinfmt version\n");
- return -EINVAL;
- }
-
- return 0;
-}
-
-/**
- * @brief retrieve the data from hwmem to output buffers
- * @param[in] meta metadata for model
- * @param[in] info the output tensors info
- * @param[out] output output buffers
- * @param[in] data_ptr data pointer for the internal buffer
- * @return 0 if no error, otherwise a negative errno
- */
-static int
-setup_output_buffers (const npubin_meta *meta, const tensors_data_info *info,
- output_buffers *output, const void *data_ptr)
-{
- int err, idx;
- buffer_types type;
-
- /**
- * @todo currently, only support output_buffers with BUFFER_MAPPED.
- * To avoid memory copy, users may need to provide external memory,
- * but they should manually handle data manipulation.
- */
- type = BUFFER_MAPPED;
-
- /* let's fill output information using metadata */
- switch (NPUBIN_VERSION(meta->magiccode)) {
- case 0: /* if not specified, it's regarded as version 1 */
- case 1:
- {
- output->num_buffers = 1;
- output->bufs[0].type = type;
- output->bufs[0].size = meta->output_size;
- output->bufs[0].addr = malloc (meta->output_size);
- if (output->bufs[0].addr != NULL) {
- setup_buffer_from_data (data_ptr + meta->output_offset,
- &output->bufs[0], meta->output_size);
- } else {
- logerr (TAG, "Error allocating output memory\n");
- return -ENOMEM;
- }
- }
- break;
- case 2:
- {
- output->num_buffers = meta->output_num;
- for (idx = 0; idx < output->num_buffers; idx++) {
- uint32_t srnpu_tensor_size = calc_tensor_size (meta, idx,
- DATA_LAYOUT_SRNPU, false);
- uint32_t nhwc_tensor_size = calc_tensor_size (meta, idx,
- DATA_LAYOUT_NHWC, false);
-
- generic_buffer *buffer = &output->bufs[idx];
-
- assert (srnpu_tensor_size >= nhwc_tensor_size);
-
- buffer->type = type;
-
- if (info->info[idx].layout == DATA_LAYOUT_SRNPU)
- buffer->size = srnpu_tensor_size;
- else /* For other layout, only NHWC is supported */
- buffer->size = nhwc_tensor_size;
-
- buffer->addr = malloc (buffer->size);
- if (buffer->addr == NULL) {
- err = -ENOMEM;
- logerr (TAG, "Error allocating output memory\n");
- idx--;
- goto free_buffers;
- }
-
- err = manipulate_data (meta, info, idx, srnpu_tensor_size,
- data_ptr + meta->output_offsets[idx], buffer->addr, false);
- if (err != 0) {
- logerr (TAG, "Error setting data to hwmem, errno: %d\n", err);
- goto free_buffers;
- }
- }
- }
- break;
- default:
- logerr (TAG, "Unknown npubinfmt version\n");
- return -EINVAL;
- }
-
- return 0;
-
-free_buffers:
- /** free pre-allocated buffers on error */
- for (; idx >= 0; idx--)
- free (output->bufs[idx].addr);
-
- return err;
-}
-
-/**
- * @brief free resourcs of all models
- */
-static void
-free_all_models (void)
-{
- model_priv *mpriv, *tmp;
-
- list_for_each_entry_safe (mpriv, tmp, hpriv.model_priv_list, list) {
- list_del (&hpriv.model_priv_list, &mpriv->list);
- free (mpriv);
-
- hpriv.num_models--;
- }
-}
-
-/**
- * @brief get a model with the target id and version
- * @param[in] id model id
- * @param[in] version model version
- * @return model instance if exists, otherwise NULL
- */
-static model *
-get_model (uint64_t id, uint64_t version)
-{
- model_priv *mpriv;
-
- list_for_each_entry (mpriv, hpriv.model_priv_list, list) {
- if (mpriv->meta.model_id == id &&
- mpriv->meta.model_version == version) {
- return &mpriv->model;
- }
- }
-
- return NULL;
-}
-
-/**
- * @brief parse model binray data to create model instance
- * @param[in] mpriv model's private data
- * @param[in] binary model binary
- * @return 0 if no error, otherwise a negative errno
- * @note the lock is already acquired in the outside
- */
-static int
-parse_model_meta (model_priv* mpriv, void *binary)
-{
- npubin_meta *meta = &mpriv->meta;
-
- memcpy (meta, binary, NPUBIN_META_SIZE);
-
- /* check the metadata is really for SRNPU HW */
- if (!CHECK_NPUBIN(meta->magiccode)) {
- logerr (TAG, "invalid magiccode %llx\n", meta->magiccode);
- return -EINVAL;
- }
-
- return 0;
-}
-
-/**
- * @brief get the allocated memory size of models
- * @return the memory size of models
- */
-static uint64_t
-handler_get_size_models (void)
-{
- return hpriv.model_size;
-}
-
-/**
- * @brief get a number of registered models
- * @return a number of registered models
- */
-static uint32_t
-handler_get_num_registered_models (void)
-{
- return hpriv.num_models;
-}
-
-#ifdef ENABLE_BUFFERING
-/**
- * @brief get the maximum buffer size for registerd models
- * @note the lock is already acquired in the outside
- * @return the maximum buffer size
- */
-static uint64_t
-get_maximum_buffer_size (void)
-{
- uint64_t maximum = 0;
- model_priv *mpriv;
-
- list_for_each_entry (mpriv, hpriv.model_priv_list, list) {
- if (maximum < mpriv->meta.buffer_size)
- maximum = mpriv->meta.buffer_size;
- }
-
- return maximum;
-}
-
-/**
- * @brief get the allocated memory size of buffers
- * @return the memory size of buffers
- */
-static uint64_t
-handler_get_size_buffers (void)
-{
- return hpriv.buffer_size;
-}
-
-/**
- * @brief resize I/O buffers by force
- * @param[in] size the buffer size; a zero size means free
- * @return 0 if no error, otherwise a negative errno
- * @note this should be called only for ASR inferences.
- */
-static int
-handler_resize_buffers (uint64_t size)
-{
- int err = 0;
-
- HANDLER_LOCK ();
-
- if (hpriv.buffer_size < size || hpriv.num_models == 0) {
- err = GET_MEM()->resize_buffers (size);
- hpriv.buffer_size = size;
- }
-
- HANDLER_UNLOCK ();
-
- return err;
-}
-
-/**
- * @brief get the next input buffer
- * @param[in] mode How this operation when input buffers are full
- * @param[out] err Set with error-number if there is an error
- * @return the buffer for input data writing. NULL if error.
- * @detail Depending on the parameters from libNPUHost, the caller (N1)
- * may wait and retry or return error.
- * The return value is "ADDR-I1" in Activity Sequence.
- */
-static buffer*
-handler_get_current_input_buffer (npu_async_mode mode, int *err)
-{
- return n3_getNextBuffer(mode, BUFFER_ROLE_INPUT, err);
-}
-
-/**
- * @brief get the next output buffer
- * @param[out] err Set with error-number if there is an error
- * @return the buffer for output data to be transferred. NULL if error.
- * @detail Depending on the parameters from libNPUHost, the caller (N1)
- * may wait and retry or return error.
- */
-static buffer*
-handler_get_current_output_buffer (int *err)
-{
- return n3_getNextBuffer(NPUASYNC_WAIT, BUFFER_ROLE_OUTPUT, err);
-}
-#endif
-
-/**
- * @brief create and register model
- * @param[in] model_buf The generic buffer for the compiled NPU NN model.
- * @param[in] id ID of the model registered
- * @param[in] version Version of the model registered
- * @return 0 if ok. errno if error
- */
-static int
-handler_register_model (generic_buffer *model_buf, uint64_t *id, uint64_t *version)
-{
- model_priv *mpriv;
- model *new_model;
- hwmem *hwmem_ptr;
- int err;
-
- mpriv = (model_priv *) malloc (sizeof (model_priv));
- if (!mpriv) {
- return -ENOMEM;
- }
-
- new_model = &mpriv->model;
- new_model->inputSequence = 0;
- new_model->meta = &mpriv->meta;
-
- HANDLER_LOCK ();
-
- if (model_buf) {
- void *data_ptr;
- uint64_t data_size;
-
- data_size = model_buf->size;
- if (model_buf->type != BUFFER_DMABUF) {
- err = GET_MEM()->alloc (data_size, &hwmem_ptr);
- } else {
- /** This is unregistered in handler when dealloc() on model's hwmem */
- err = GET_MEM()->register_dmabuf (model_buf->dmabuf, data_size, &hwmem_ptr);
- }
-
- if (err != 0) {
- logerr (TAG, "Fail to allocate hwmem with size %lu\n", data_size);
- goto err_unlock;
- }
-
- /** TODO: add a check that the same model has not been registered before */
-
- mpriv->hwmem = hwmem_ptr;
-
- /* activate hwmem; it means that it's not compacted until unregistration */
- if ((err = hwmem_activate (hwmem_ptr)) < 0)
- goto err_dealloc;
-
- if ((err = hwmem_get_data (hwmem_ptr, &data_ptr)) < 0)
- goto err_deactivate;
-
- /* it can avoid memcpy() for BUFFER_DMABUF */
- if (model_buf->type != BUFFER_DMABUF) {
- if ((err = setup_data_from_buffer (model_buf, data_ptr, data_size)) < 0) {
- goto err_deactivate;
- }
- }
-
- if ((err = parse_model_meta (mpriv, data_ptr)) < 0)
- goto err_deactivate;
-
-#ifdef ENABLE_BUFFERING
- if (hpriv.buffer_size < mpriv->meta.buffer_size) {
- /* need to increase buffer size! */
- if (GET_MEM()->resize_buffers (mpriv->meta.buffer_size) != 0) {
- /* fail to resize... */
- err = -ENOMEM;
- goto err_deactivate;
- }
- hpriv.buffer_size = mpriv->meta.buffer_size;
- }
-#endif
-
- assert (hwmem_ptr->size >= data_size);
-
- new_model->memblock = hwmem_ptr;
- new_model->model_size = hwmem_ptr->size;
-
- hpriv.model_size += hwmem_ptr->size;
- } else {
- /* set dummy metadata */
- mpriv->meta.magiccode = NPUBIN_MAGICCODE | 0x1;
- mpriv->meta.type = SMODEL_OPS_NPU_ASR;
- mpriv->meta.model_id = 0;
- mpriv->meta.model_version = 0;
- mpriv->meta.size = 4096;
- mpriv->meta.buffer_size = 4096;
- mpriv->meta.input_offset = 0;
- mpriv->meta.input_size = 4096;
- mpriv->meta.output_offset = 0;
- mpriv->meta.output_size = 4096;
- mpriv->meta.program_size = 0;
- mpriv->meta.weight_size = 0;
-
- new_model->memblock = NULL;
- new_model->model_size = 0;
- }
-
- hpriv.num_models++;
-
- list_add (&hpriv.model_priv_list, &mpriv->list);
-
- *id = mpriv->meta.model_id;
- *version = mpriv->meta.model_version;
-
- HANDLER_UNLOCK ();
-
- return 0;
-
-/* rollback activation */
-err_deactivate:
- hwmem_deactivate (hwmem_ptr);
-err_dealloc:
- if (model_buf->type != BUFFER_DMABUF && GET_MEM()->dealloc (hwmem_ptr) < 0) {
- logwarn (TAG, "Failed to deallocate hwmem\n");
- }
-err_unlock:
- HANDLER_UNLOCK ();
-
- free (mpriv);
-
- return err;
-}
-
-/**
- * @brief unregister model
- * @param[in] id The id of model to be unregistered.
- * @param[in] version The version of model to be unregistered.
- * @return 0 if no error, otherwise a negative errno
- * @note in this function, model's hwmem is deallocated.
- */
-static int
-handler_unregister_model (uint64_t id, uint64_t version)
-{
- model *model;
- model_priv *mpriv;
- int err = 0;
- uint64_t model_size;
-
- HANDLER_LOCK ();
-
- model = get_model (id, version);
-
- if (!model) {
- logerr (TAG, "Cannot find model with id (%lu) & version (%lu)\n", id, version);
- err = -ENOENT;
- goto err_unlock;
- }
-
- mpriv = GET_MODEL_PRIVATE (model);
-
- if (mpriv->hwmem) {
- if ((err = hwmem_deactivate (mpriv->hwmem)) < 0)
- goto err_unlock;
-
- model_size = mpriv->hwmem->size;
- if ((err = GET_MEM()->dealloc (mpriv->hwmem)) < 0)
- goto err_activate;
- } else {
- model_size = 0;
- }
-
- list_del (&hpriv.model_priv_list, &mpriv->list);
- free (mpriv);
-
- assert (hpriv.num_models > 0);
-
- hpriv.num_models--;
- hpriv.model_size -= model_size;
-#ifdef ENABLE_BUFFERING
- hpriv.buffer_size = get_maximum_buffer_size ();
- /* resizing should be successful because buffer is resized to equal or smaller size */
- assert (GET_MEM()->resize_buffers (hpriv.buffer_size) == 0);
-#endif
-
- HANDLER_UNLOCK ();
-
- return 0;
-
-/* rollback deactivation */
-err_activate:
- hwmem_activate (mpriv->hwmem);
-err_unlock:
- HANDLER_UNLOCK ();
-
- return err;
-}
-
-/**
- * @brief get a model metadata with the target id and version
- * @param[in] id model id
- * @param[in] version model version
- * @param[out] meta model metadata
- * @return 0 if no error, otherwise a negative error
- */
-static int
-handler_get_model_meta (uint64_t id, uint64_t version, npubin_meta *meta)
-{
- const model *model;
- const model_priv *priv;
- int err = 0;
-
- HANDLER_LOCK ();
-
- model = get_model (id, version);
- if (!model) {
- err = -ENOENT;
- goto out;
- }
-
- priv = GET_MODEL_PRIVATE (model);
- memcpy (meta, &priv->meta, NPUBIN_META_SIZE);
-
-out:
- HANDLER_UNLOCK ();
-
- return err;
-}
-
-/** @brief N2's callback wrapper for output ready */
-static void
-n2_cb (buffer *buf, void *data)
-{
- /* TODO do pre-handler */
- hpriv.n1_cb (buf, data);
- /* TODO do post-handler */
-}
-
-/**
- * @brief Host tells us to start an operation.
- * @param[in] op The operation mode
- * @param[in] force if non-zero, try to stop/start with preemption
- * @param[in] id the model id to be activated
- * @param[in] version the model version to be activated
- * @param[in] cb The callback to be called when the output is ready
- * @param[in] cb_data The private data to be given to the callback.
- * @return 0 if ok. errno if error.
- */
-static int
-handler_set_op_mode (npu_input_opmode op, bool force, uint64_t id,
- uint64_t version, output_ready cb, void *cb_data)
-{
- model *m;
-
- HANDLER_LOCK ();
-
- m = get_model (id, version);
- if (m != NULL)
- hpriv.n1_cb = cb;
-
- HANDLER_UNLOCK ();
-
- return m ? n3_setOpMode (op, force, m, n2_cb, cb_data) : -ENOENT;
-}
-
-/**
- * @brief feed input data to the obtained buffer
- * @param[in] meta metadata for the target model
- * @param[in] info the input tensors info
- * @param[in] input_buf the input data from users
- * @param[out] buffer_ptr the buffer pointer to be filled
- * @return 0 if ok. errno if error.
- */
-static int
-handler_feed_input_buffer (const npubin_meta *meta, const tensors_data_info *info,
- const input_buffers *input_buf, buffer *buffer_ptr)
-{
- buffer_types type;
- int err = 0;
-
- if (!(meta && info && buffer_ptr))
- return -EINVAL;
-
- if (!(input_buf && input_buf->num_buffers > 0))
- return -EINVAL;
-
- /**
- * BUFFER_MAPPED/FILE and BUFFER_DMABUF have exclusive relationships, which means
- * if one of input tensors have BUFFER_DMABUF, the other tensors should also have it.
- */
- type = input_buf->bufs[0].type;
- if (type != BUFFER_DMABUF) {
- hwmem *hwmem_ptr;
- void *data_ptr;
-
- /* it activates its internal hwmem */
- if ((err = buffer_set_dmabuf (buffer_ptr, 0 /* dmabuf */)) < 0) {
- logerr (TAG, "Error disabling dmabuf sharing\n");
- return err;
- }
-
- assert (buffer_get_hwmem (buffer_ptr, &hwmem_ptr) == 0);
- assert (hwmem_get_data (hwmem_ptr, &data_ptr) == 0);
-
- /* this function may perform data manipulation if required */
- if ((err = setup_input_buffers (meta, info, input_buf, data_ptr)) < 0) {
- logerr (TAG, "Fail to setup hwmem for input buffer\n");
- assert (hwmem_deactivate (hwmem_ptr) == 0);
- }
- } else { /* BUFFER_DMABUF */
- if ((err = buffer_set_dmabuf (buffer_ptr, input_buf->bufs[0].dmabuf)) < 0) {
- logerr (TAG, "Error setting dmabuf sharing\n");
- }
- }
-
- return err;
-}
-
-/**
- * @brief extract output data from the buffer
- * @param[in] meta metadata for the target model
- * @param[in] info the output tensors info
- * @param[in] buffer_ptr the buffer pointer that contains the result data
- * @param[out] output_buf the output data to store data
- * @return 0 if ok. errno if error.
- */
-static int
-handler_extract_output_buffer (const npubin_meta *meta, const tensors_data_info *info,
- const buffer *buffer_ptr, output_buffers *output_buf)
-{
- hwmem *hwmem_ptr;
- void *data_ptr;
- bool is_external;
- int err;
-
- if (!(meta && info && buffer_ptr && output_buf))
- return -EINVAL;
-
- assert (buffer_get_hwmem (buffer_ptr, &hwmem_ptr) == 0);
- assert (hwmem_get_data (hwmem_ptr, &data_ptr) == 0);
-
- if ((err = hwmem_check_external (hwmem_ptr, &is_external)) < 0) {
- logerr (TAG, "Failed checking hwmem's type, errno: %d\n", err);
- return err;
- }
-
- if (is_external) {
- int idx;
- /** it's external memory. So, just returns output buffers by storing the required info */
- switch (NPUBIN_VERSION (meta->magiccode)) {
- case 0:
- case 1:
- output_buf->num_buffers = 1;
- output_buf->bufs[0].size = meta->output_size;
- output_buf->bufs[0].addr = (void *)((char *) data_ptr + meta->output_offset);
- output_buf->bufs[0].type = BUFFER_DMABUF;
- break;
- case 2:
- output_buf->num_buffers = meta->output_num;
- for (idx = 0; idx < meta->output_num; idx++) {
- output_buf->bufs[idx].size = calc_tensor_size (meta, idx, DATA_LAYOUT_SRNPU, false);
- output_buf->bufs[idx].addr = (void *)((char *) data_ptr + meta->output_offsets[idx]);
- output_buf->bufs[idx].type = BUFFER_DMABUF;
- }
- break;
- default:
- logerr (TAG, "Unknown npubin version\n");
- return -EINVAL;
- }
- } else {
- /** perform data manipulation if required */
- if ((err = setup_output_buffers (meta, info, output_buf, data_ptr)) < 0) {
- logerr (TAG, "Error setting output buffers, errno: %d\n", err);
- return err;
- }
- }
-
- return 0;
-}
-
-/**
- * @brief The buffer is filled and valid for inference. Start when ready.
- * @param[in] buffer The buffer with input data filled.
- * @param[in] priority The priority of this npu request
- * @param[in] timestamp The timestamp when it was requested
- * @return 0 if ok. errno if error.
- *
- * @note after validation, it's no longer accessible because it was returned.
- */
-static int
-handler_validate_buffer (buffer *buffer, npu_priority priority, uint64_t timestamp)
-{
- int err;
-
- if (!buffer) {
- logerr (TAG, "Empty buffer (NULL) provided\n");
- return -EINVAL;
- }
-
-#ifdef ENABLE_BUFFERING
- /** only input buffer can be validated */
- if (buffer_get_state (buffer) != BUFFER_STATE_INPUT_WRITING) {
- logerr (TAG, "Buffer has an invalid state\n");
- return -EINVAL;
- }
-
- if ((err = GET_MEM()->return_buffer (buffer)) < 0) {
- logerr (TAG, "Fail to return the buffer, errno: %d\n", err);
- return err;
- }
-
- err = n3_dataReady (NULL, priority, timestamp);
-#else
- err = n3_dataReady (buffer, priority, timestamp);
-#endif
- if (err < 0) {
- logerr (TAG, "Fail to notify the buffer is ready, errno: %d\n", err);
- }
-
- return err;
-}
-
-/**
- * @brief the instance of host handler. Its callbacks are called in N1.
- */
-static hostHandlerInfo handler = {
- .getSizeModels = handler_get_size_models,
- .getNumRegisteredModels = handler_get_num_registered_models,
- .getModelMeta = handler_get_model_meta,
- .setOpMode = handler_set_op_mode,
-#ifdef ENABLE_BUFFERING
- .getSizeBuffers = handler_get_size_buffers,
- .resizeBuffers = handler_resize_buffers,
- .getCurrentInputBuffer = handler_get_current_input_buffer,
- .getCurrentOutputBuffer = handler_get_current_output_buffer,
-#endif
- /* model */
- .registerModel = handler_register_model,
- .unregisterModel = handler_unregister_model,
- /* input buffers */
- .feedInputBuffer = handler_feed_input_buffer,
- .validateBuffer = handler_validate_buffer,
- /* output buffers */
- .extractOutputBuffer = handler_extract_output_buffer,
-};
-
-/**
- * @brief initilize host handler
- * @return 0 if no error, otherwise a negative errno
- * @note it should be called in main()
- */
-int
-init_ne_handler (void)
-{
- int err;
-
- memset (&hpriv, '\x00', sizeof(handler_priv));
-
- if ((err = GET_MEM()->init (conf->reserved_mem_size,
- &hpriv.reserved_size)) < 0) {
- logerr (TAG, "Fail to initialize the memory allocator\n");
- return err;
- }
-
- list_init (&hpriv.model_priv_list);
- pthread_mutex_init (&hpriv.mutex, NULL);
-
- err = initNEcomm (&handler, NPUCOND_CONN_SOCIP);
- if (err < 0) {
- pthread_mutex_destroy (&hpriv.mutex);
- GET_MEM()->fini ();
- return err;
- }
-
- init_ne_scheduler();
- return 0;
-}
-
-/**
- * @brief terminate host handler
- * @return 0 if no error, otherwise a negative errno
- * @note it should be called in main()
- */
-int
-exit_ne_handler (void)
-{
- int err;
-
- fini_ne_scheduler();
-
- free_all_models();
-
- pthread_mutex_destroy (&hpriv.mutex);
-
- if ((err = exitNEcomm ()) < 0)
- return err;
-
- return GET_MEM()->fini ();
-}
--- /dev/null
+/**
+ * Proprietary
+ * Copyright (C) 2020 Samsung Electronics
+ * Copyright (C) 2020 Dongju Chae <dongju.chae@samsung.com>
+ */
+/**
+ * @file ne-host-handler.cc
+ * @date 03 Apr 2020
+ * @brief Implementation of APIs to access NPU from Host
+ * @see http://suprem.sec.samsung.net/confluence/display/ODLC/Software+Stack
+ * @author Dongju Chae <dongju.chae@samsung.com>
+ * @bug No known bugs except for NYI items
+ */
+
+#include <npubinfmt.h>
+#include <NPUdrvAPI.h>
+#include <CommPlugin.h>
+
+#include "ne-utils.h"
+#include "ne-mem.h"
+#include "ne-scheduler.h"
+#include "ne-handler.h"
+
+#include <string.h>
+#include <assert.h>
+
+#include <condition_variable>
+#include <functional>
+#include <atomic>
+#include <map>
+
+#define TAG _N2
+
+#define INIT_HOST_HANDLER(handler, dev) \
+ Device *tdev = static_cast <Device *> (dev); \
+ if (tdev == nullptr) return -EINVAL; \
+ HostHandler *handler = tdev->getHostHandler (); \
+ if (handler == nullptr) return -EINVAL;
+
+/** @brief device class. it contains all related instances */
+class Device {
+ public:
+ /** @brief Factory method to create a trinity device dependong on dev type */
+ static Device *createInstance (dev_type device_type, int device_id);
+
+ /** @brief constructor of device */
+ Device (dev_type type, int id, bool need_model = true)
+ : comm_(CommPlugin::getCommPlugin()), type_ (type), id_ (id),
+ need_model_ (true), mode_ (NPUASYNC_WAIT), initialized_ (ATOMIC_FLAG_INIT) {}
+
+ /** @brief destructor of device */
+ virtual ~Device () {}
+
+ /** @brief initialization */
+ int init () {
+ if (!initialized_.test_and_set()) {
+ /** create the corresponding driver API */
+ api_ = DriverAPI::createDriverAPI (type_, id_);
+ if (api_.get() == nullptr) {
+ initialized_.clear();
+ logerr (TAG, "Failed to create driver API\n");
+ return -EINVAL;
+ }
+
+ handler_.reset (new HostHandler (this));
+ scheduler_.reset (new Scheduler (api_.get()));
+ mem_ = MemAllocator::createInstance (api_.get());
+ }
+
+ return 0;
+ }
+
+ HostHandler *getHostHandler () { return handler_.get(); }
+ dev_type getType () { return type_; }
+ int getID () { return id_; }
+ bool needModel () { return need_model_; }
+ void setAsyncMode (npu_async_mode mode) { mode_ = mode; }
+
+ HWmem * allocMemory () { return mem_->allocMemory (); }
+ void deallocMemory (int dmabuf_fd) { mem_->deallocMemory (dmabuf_fd); }
+
+ /** it stops all requests in this device (choose wait or force) */
+ int stop (bool force_stop) {
+ Request *req = new Request (NPUINPUT_STOP);
+ req->setForceStop (force_stop);
+ return scheduler_->submitRequest (req);
+ }
+
+ virtual Model * registerModel (const generic_buffer *model) = 0;
+ virtual int run (npu_input_opmode opmode, const Model *model,
+ const input_buffers *input, npuOutputNotify cb, void *cb_data,
+ uint64_t *sequence) = 0;
+
+ protected:
+ /** the device instance has ownership of all related components */
+ std::unique_ptr<DriverAPI> api_; /**< device api */
+ std::unique_ptr<MemAllocator> mem_; /**< memory allocator */
+ std::unique_ptr<HostHandler> handler_; /**< host handler */
+ std::unique_ptr<Scheduler> scheduler_; /**< scheduler */
+
+ CommPlugin& comm_; /**< plugin communicator */
+
+ dev_type type_; /**< device type */
+ int id_; /**< device id */
+ bool need_model_; /**< indicates whether the device needs model */
+ npu_async_mode mode_; /**< async run mode */
+
+ private:
+ std::atomic_flag initialized_;
+};
+
+/** @brief Trinity Vision (TRIV) classs */
+class TrinityVision : public Device {
+ public:
+ TrinityVision (int id) : Device (NPUCOND_TRIV_CONN_SOCIP, id) {}
+
+ static size_t manipulateData (const Model *model, uint32_t idx, bool is_input,
+ void *dst, void *src, size_t size);
+
+ Model * registerModel (const generic_buffer *model_buf) {
+ Model *model = mem_->allocModel ();
+ if (model == nullptr) {
+ logerr (TAG, "Failed to allocate model\n");
+ return nullptr;
+ }
+
+ int status;
+ if (model_buf->type == BUFFER_DMABUF) {
+ model->setDmabuf (model_buf->dmabuf);
+ model->setOffset (model_buf->offset);
+ model->setSize (model_buf->size);
+ } else {
+ status = model->alloc (model_buf->size);
+ if (status != 0) {
+ logerr (TAG, "Failed to allocate model: %d\n", status);
+ goto delete_exit;
+ }
+
+ status = comm_.extractGenericBuffer (model_buf, model->getData(), nullptr);
+ if (status != 0) {
+ logerr (TAG, "Failed to extract generic buffer: %d\n", status);
+ goto delete_exit;
+ }
+ }
+
+ status = model->setMetadata (model->getData());
+ if (status != 0)
+ goto delete_exit;
+
+ model_config_t config;
+ config.dmabuf_id = model->getDmabuf();
+ config.program_size = model->getMetadata()->getProgramSize();
+ config.program_offset_addr = model->getOffset() + model->getMetadata()->getMetaSize();
+ config.weight_offset_addr = config.program_offset_addr + config.program_size;
+
+ status = api_->setModel (&config);
+ if (status != 0)
+ goto delete_exit;
+
+ return model;
+
+delete_exit:
+ delete model;
+ return nullptr;
+ }
+
+ Buffer * prepareInputBuffers (const Model *model, const input_buffers *input) {
+ const Metadata *meta = model->getMetadata();
+ const generic_buffer *first = &input->bufs[0];
+
+ if (meta->getInputNum() != input->num_buffers)
+ return nullptr;
+
+ Buffer * buffer = mem_->allocBuffer ();
+ if (buffer != nullptr) {
+ if (first->type == BUFFER_DMABUF) {
+ buffer->setDmabuf (first->dmabuf);
+ buffer->setOffset (first->offset);
+ buffer->setSize (meta->getBufferSize());
+ } else {
+ int status = buffer->alloc (meta->getBufferSize ());
+ if (status != 0) {
+ logerr (TAG, "Failed to allocate buffer: %d\n", status);
+ delete buffer;
+ return nullptr;
+ }
+ }
+ }
+
+ buffer->createTensors (meta);
+ return buffer;
+ }
+
+ int run (npu_input_opmode opmode, const Model *model,
+ const input_buffers *input, npuOutputNotify cb, void *cb_data,
+ uint64_t *sequence) {
+ if (opmode != NPUINPUT_HOST)
+ return -EINVAL;
+
+ Buffer *buffer = prepareInputBuffers (model, input);
+ if (buffer == nullptr)
+ return -EINVAL;
+
+ for (uint32_t idx = 0; idx < input->num_buffers; idx++) {
+ auto func = std::bind (TrinityVision::manipulateData, model, idx, true,
+ std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
+ int status = comm_.extractGenericBuffer (&input->bufs[idx],
+ buffer->getInputTensor(idx)->getData(), func);
+ if (status != 0) {
+ logerr (TAG, "Failed to feed input buffer: %d\n", status);
+ return status;
+ }
+ }
+
+ /** this device uses CMA buffer */
+
+ Request *req = new Request (opmode);
+ req->setModel (model);
+ req->setBuffer (buffer);
+ req->setCallback (std::bind (&TrinityVision::callback, this, req, cb, cb_data));
+
+ if (sequence)
+ *sequence = req->getID();
+
+ return scheduler_->submitRequest (req);
+ }
+
+ void callback (Request *req, npuOutputNotify cb, void *cb_data) {
+ const Model *model = req->getModel ();
+ Buffer *buffer = req->getBuffer ();
+ output_buffers output = {
+ .num_buffers = buffer->getOutputNum ()
+ };
+
+ for (uint32_t idx = 0; idx < output.num_buffers; idx++) {
+ uint32_t output_tensor_size = model->getOutputTensorSize (idx);
+
+ output.bufs[idx].type = BUFFER_MAPPED;
+ output.bufs[idx].size = output_tensor_size;
+ /** user needs to free this */
+ output.bufs[idx].addr = malloc (output_tensor_size);
+
+ auto func = std::bind (TrinityVision::manipulateData, model, idx, false,
+ std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
+ int status = comm_.insertGenericBuffer (buffer->getOutputTensor(idx)->getData(),
+ &output.bufs[idx], func);
+ if (status != 0) {
+ logerr (TAG, "Failed to return output buffer: %d\n", status);
+ }
+ }
+
+ cb (&output, req->getID(), cb_data);
+ }
+};
+
+/** @brief Trinity Vision2 (TRIV2) classs */
+class TrinityVision2 : public Device {
+ public:
+ TrinityVision2 (int id) : Device (NPUCOND_TRIV2_CONN_SOCIP, id) {}
+
+ static size_t manipulateData (const Model *model, uint32_t idx, bool is_input,
+ void *dst, void *src, size_t size) {
+ memcpy (dst, src, size);
+ return size;
+ }
+
+ Model * registerModel (const generic_buffer *model_buf) {
+ /** TODO: model's weight values are stored in segments */
+ return nullptr;
+ }
+
+ int run (npu_input_opmode opmode, const Model *model,
+ const input_buffers *input, npuOutputNotify cb, void *cb_data,
+ uint64_t *sequence) {
+ if (opmode != NPUINPUT_HOST || opmode != NPUINPUT_HW_RECURRING)
+ return -EINVAL;
+
+ /** this device uses segment table */
+
+ Request *req = new Request (opmode);
+ req->setModel (model);
+#if 0
+ req->setSegmentTable (segt);
+#endif
+ req->setCallback (std::bind (&TrinityVision2::callback, this, req, cb, cb_data));
+
+ if (sequence)
+ *sequence = req->getID();
+
+ return scheduler_->submitRequest (req);
+ }
+
+ void callback (Request *req, npuOutputNotify cb, void *cb_data) {
+ }
+};
+
+/** @brief Trinity Asr (TRIA) classs */
+class TrinityAsr : public Device {
+ public:
+ TrinityAsr (int id) : Device (NPUCOND_TRIA_CONN_SOCIP, id, false) {}
+
+ static size_t manipulateData (const Model *model, uint32_t idx, bool is_input,
+ void *dst, void *src, size_t size) {
+ memcpy (dst, src, size);
+ return size;
+ }
+
+ Model * registerModel (const generic_buffer *model_buf) { return nullptr; }
+
+ int run (npu_input_opmode opmode, const Model *model,
+ const input_buffers *input, npuOutputNotify cb, void *cb_data,
+ uint64_t *sequence) {
+ if (opmode != NPUINPUT_HOST)
+ return -EINVAL;
+
+ /** ASR does not require model and support only a single tensor */
+ const generic_buffer *first_buf = &input->bufs[0];
+ Buffer * buffer = mem_->allocBuffer ();
+ int status;
+ if (first_buf->type == BUFFER_DMABUF) {
+ buffer->setDmabuf (first_buf->dmabuf);
+ buffer->setOffset (first_buf->offset);
+ buffer->setSize (first_buf->size);
+ } else {
+ status = buffer->alloc (first_buf->size);
+ if (status != 0) {
+ delete buffer;
+ return status;
+ }
+ }
+ buffer->createTensors ();
+
+ status = comm_.extractGenericBuffer (first_buf,
+ buffer->getInputTensor(0)->getData(), nullptr);
+ if (status != 0)
+ return status;
+
+ Request *req = new Request (opmode);
+ req->setBuffer (buffer);
+ req->setCallback (std::bind (&TrinityAsr::callback, this, req, cb, cb_data));
+
+ if (sequence)
+ *sequence = req->getID();
+
+ return scheduler_->submitRequest (req);
+ }
+
+ void callback (Request *req, npuOutputNotify cb, void *cb_data) {
+ }
+};
+
+#ifdef ENABLE_MANIP
+
+#define do_quantized_memcpy(type) do {\
+ idx = 0;\
+ if (quant) {\
+ while (idx < num_elems) {\
+ val = ((type *) src)[idx];\
+ val = val / _scale;\
+ val += _zero_point;\
+ val = (val > 255.0) ? 255.0 : 0.0;\
+ ((uint8_t *) dst)[idx++] = (uint8_t) val;\
+ }\
+ } else {\
+ while (idx < num_elems) {\
+ val = *(uint8_t *) src;\
+ val -= _zero_point;\
+ val *= _scale;\
+ ((type *) dst)[idx++] = (type) val;\
+ dst = (void*)(((uint8_t *) dst) + data_size);\
+ src = (void*)(((uint8_t *) src) + 1);\
+ }\
+ }\
+ } while (0)
+
+/**
+ * @brief memcpy during quantization
+ */
+static void memcpy_with_quant (bool quant, data_type type, float scale, uint32_t zero_point,
+ void *dst, const void *src, uint32_t num_elems)
+{
+ double _scale = (double) scale;
+ double _zero_point = (double) zero_point;
+ double val;
+ uint32_t data_size = get_data_size (type);
+ uint32_t idx;
+
+ switch (type) {
+ case DATA_TYPE_INT8:
+ do_quantized_memcpy (int8_t);
+ break;
+ case DATA_TYPE_UINT8:
+ do_quantized_memcpy (uint8_t);
+ break;
+ case DATA_TYPE_INT16:
+ do_quantized_memcpy (int16_t);
+ break;
+ case DATA_TYPE_UINT16:
+ do_quantized_memcpy (uint16_t);
+ break;
+ case DATA_TYPE_INT32:
+ do_quantized_memcpy (int32_t);
+ break;
+ case DATA_TYPE_UINT32:
+ do_quantized_memcpy (uint32_t);
+ break;
+ case DATA_TYPE_INT64:
+ do_quantized_memcpy (int64_t);
+ break;
+ case DATA_TYPE_UINT64:
+ do_quantized_memcpy (uint64_t);
+ break;
+ case DATA_TYPE_FLOAT32:
+ do_quantized_memcpy (float);
+ break;
+ case DATA_TYPE_FLOAT64:
+ do_quantized_memcpy (double);
+ break;
+ default:
+ logerr (TAG, "Unsupported datatype %d\n", type);
+ }
+}
+
+/**
+ * @brief perform data manipulation
+ * @param[in] model model instance
+ * @param[in] idx tensor index
+ * @param[in] is_input indicate it's input manipulation
+ * @param[out] dst destination buffer
+ * @param[in] src source buffer (feature map)
+ * @param[in] size size to be copied
+ * @return size of memory copy if no error, otherwise zero
+ *
+ * @note the input data format should be NHWC
+ * @detail rules for the memory address of activations in NPU HW.
+ * (https://code.sec.samsung.net/confluence/pages/viewpage.action?pageId=146491864)
+ *
+ * 1) Special case (depth == 3)
+ * - addr(x,y,z) = addr(0,0,0) + (z) + 3 * (x + width * y)
+ *
+ * 2) Common case
+ * - addr(x,y,z) = addr(0,0,0) + (z % MPA_L) + MPA_L * (x + width * (y + height * (z / MPA_L)))
+ *
+ * Thus, if depth is not a multiple of MPA_L (i.e., 64), zero padding is required
+ */
+size_t
+TrinityVision::manipulateData (const Model *model, uint32_t idx, bool is_input,
+ void *dst, void *src, size_t size)
+{
+ const Metadata *meta = model->getMetadata();
+ const tensor_data_info* info;
+ const uint32_t *dims;
+ uint32_t zero_point;
+ float scale;
+
+ /** extract required information from the metadata */
+ if (is_input) {
+ if (idx >= meta->getInputNum()) {
+ logerr (TAG, "Wrong information for input tensors in metadata\n");
+ return 0;
+ }
+
+ info = model->getInputDataInfo (idx);
+ dims = meta->getInputDims (idx);
+ zero_point = meta->getInputQuantZero (idx);
+ scale = meta->getInputQuantScale (idx);
+ } else {
+ if (idx >= meta->getOutputNum()) {
+ logerr (TAG, "Wrong information for output tensors in metadata\n");
+ return 0;
+ }
+
+ info = model->getOutputDataInfo (idx);
+ dims = meta->getOutputDims (idx);
+ zero_point = meta->getOutputQuantZero (idx);
+ scale = meta->getOutputQuantScale (idx);
+ }
+
+ if (info == nullptr) {
+ logerr (TAG, "Unmatched tensors info\n");
+ return 0;
+ }
+
+ uint32_t batch = dims[0];
+ uint32_t height = dims[1];
+ uint32_t width = dims[2];
+ uint32_t depth = dims[3];
+
+ uint32_t data_size = get_data_size (info->type);
+ if (data_size == 0) {
+ logerr (TAG, "Invalid data size\n");
+ return 0;
+ }
+
+ bool need_quantization = false;
+ /**
+ * note that we assume DATA_TYPE_SRNPU is the smallest data type that we consider.
+ * Also, DATA_TYPE_SRNPU and uint8_t may be regarded as the same in the view of apps.
+ */
+ if (info->type != DATA_TYPE_SRNPU) {
+ assert (data_size >= get_data_size (DATA_TYPE_SRNPU));
+
+ if (data_size > get_data_size (DATA_TYPE_SRNPU) ||
+ !(zero_point == DEFAULT_ZERO_POINT && scale == DEFAULT_SCALE))
+ need_quantization = true;
+ }
+
+ /** check data manipulation is required */
+ if (depth != 3 && depth != 64 && info->layout != DATA_LAYOUT_SRNPU) {
+ uint32_t MPA_L = DATA_GRANULARITY;
+ uint32_t n, h, w, d;
+ uint32_t std_offset; /* standard offset in NHWC data format */
+ uint32_t npu_offset; /* npu offset in NPU HW data format*/
+ uint32_t src_offset;
+ uint32_t dst_offset;
+ uint32_t slice_size;
+
+ /* @todo we currently support only NHWC */
+ if (info->layout != DATA_LAYOUT_NHWC) {
+ logerr (TAG, "data manipulation is supported for NHWC only\n");
+ return -EINVAL;
+ }
+
+ for (n = 0; n < batch; n++) {
+ for (h = 0; h < height; h++) {
+ for (w = 0; w < width; w++) {
+ for (d = 0; d < depth; d += MPA_L) {
+ std_offset = d + depth * (w + width * (h + n * height));
+ npu_offset = MPA_L * (w + width * (h + (n + d / MPA_L) * height));
+ slice_size = (depth - d >= MPA_L) ? MPA_L : depth - d;
+
+ if (is_input) {
+ src_offset = std_offset * data_size;
+ dst_offset = npu_offset;
+ } else {
+ src_offset = npu_offset;
+ dst_offset = std_offset * data_size;
+ }
+
+ /* if depth is not a multiple of MPA_L, add zero paddings (not exact values) */
+ if (need_quantization) {
+ memcpy_with_quant (is_input, info->type, scale, zero_point,
+ static_cast<char*>(dst) + dst_offset,
+ static_cast<char*>(src) + src_offset,
+ slice_size);
+ } else {
+ memcpy (
+ static_cast<char*>(dst) + dst_offset,
+ static_cast<char*>(src) + src_offset,
+ slice_size);
+ }
+ }
+ }
+ }
+ }
+ } else if (need_quantization) {
+ /** depth == 3 || depth == 64; special cases which can directly copy input tensor data */
+ if (is_input)
+ size = size / data_size;
+
+ memcpy_with_quant (is_input, info->type, scale, zero_point,
+ dst, src, size);
+ } else {
+ memcpy (dst, src, size);
+ }
+
+ return 0;
+}
+
+#else
+
+size_t
+TrinityVision::manipulateData (const Model *model, uint32_t idx, bool is_input,
+ void *dst, void *src, size_t size)
+{
+ memcpy (dst, src, size);
+ return size;
+}
+
+#endif
+
+/**
+ * @brief create device instance depending on device type and id
+ * @param[in] type device type
+ * @param[in] id device id
+ * @return device instance
+ */
+Device *
+Device::createInstance (dev_type type, int id)
+{
+ Device *device = nullptr;
+
+ switch (type & DEVICETYPE_MASK) {
+ case DEVICETYPE_TRIV:
+ device = new TrinityVision (id);
+ break;
+ case DEVICETYPE_TRIV2:
+ device = new TrinityVision2 (id);
+ break;
+ case DEVICETYPE_TRIA:
+ device = new TrinityAsr (id);
+ break;
+ default:
+ break;
+ }
+
+ if (device != nullptr && device->init () != 0) {
+ delete device;
+ device = nullptr;
+ }
+
+ return device;
+}
+
+/** @brief host handler constructor */
+HostHandler::HostHandler (Device *device)
+ : device_(device)
+{
+}
+
+/** @brief host handler destructor */
+HostHandler::~HostHandler ()
+{
+}
+
+/**
+ * @brief register model from generic buffer
+ * @param[in] model_buf model buffer
+ * @param[out] modelid model id
+ * @return 0 if no error. otherwise a negative errno
+ */
+int
+HostHandler::registerModel (generic_buffer *model_buf, uint32_t *modelid)
+{
+ Model *model = device_->registerModel (model_buf);
+ if (model == nullptr) {
+ logerr (TAG, "Failed to register model\n");
+ return -EINVAL;
+ }
+
+ int status = models_.insert (model->getID(), model);
+ if (status != 0) {
+ logerr (TAG, "Failed to insert model id\n");
+ delete model;
+ return status;
+ }
+
+ *modelid = model->getID();
+ return 0;
+}
+
+/**
+ * @brief remove the registered model
+ * @param[in] modelid model id
+ * @return 0 if no error. otherwise a negative errno
+ */
+int
+HostHandler::unregisterModel (uint32_t modelid)
+{
+ return models_.remove (modelid);
+}
+
+/**
+ * @brief remove all registered models
+ * @return 0
+ */
+int
+HostHandler::unregisterModels ()
+{
+ models_.clear ();
+ return 0;
+}
+
+/**
+ * @brief Set the data layout for input/output tensors
+ * @param[in] modelid The ID of model whose layouts are set
+ * @param[in] in the layout/type info for input tensors
+ * @param[in] out the layout/type info for output tensors
+ * @return @c 0 if no error. otherwise a negative error value
+ * @note if this function is not called, default layout/type will be used.
+ */
+int
+HostHandler::setDataInfo (uint32_t modelid, tensors_data_info *in,
+ tensors_data_info *out)
+{
+ Model *model = models_.find (modelid);
+ if (model == nullptr)
+ return -ENOENT;
+
+ model->setDataInfo (in, out);
+
+ return 0;
+}
+
+/**
+ * @brief Set the inference constraint for next NPU inferences
+ * @param[in] modelid The target model id
+ * @param[in] constraint inference constraint (e.g., timeout, priority)
+ * @return @c 0 if no error. otherwise a negative error value
+ * @note If this function is not called, default values are used.
+ */
+int
+HostHandler::setConstraint (uint32_t modelid, npuConstraint constraint)
+{
+ Model *model = models_.find (modelid);
+ if (model == nullptr)
+ return -ENOENT;
+
+ model->setConstraint (constraint);
+
+ return 0;
+}
+
+/**
+ * @brief find and return model instance
+ * @param[in] modelid model id
+ * @return model instance if found. otherwise nullptr
+ */
+Model *
+HostHandler::getModel (uint32_t modelid)
+{
+ return models_.find (modelid);
+}
+
+/** @brief dummay callback for runSync. */
+class callbackSync {
+ public:
+ callbackSync (output_buffers *output) : output_(output), done_(false) {}
+
+ static void callback (output_buffers *output, uint64_t sequence, void *data) {
+ callbackSync *sync = static_cast<callbackSync *>(data);
+ sync->callback (output, sequence);
+ }
+
+ void callback (output_buffers *output, uint64_t sequence) {
+ /** just copy internal variables of output buffers */
+ memcpy (output_, output, sizeof (output_buffers));
+ done_ = true;
+ cv_.notify_one ();
+ }
+
+ void wait () {
+ std::unique_lock<std::mutex> lock (m_);
+ cv_.wait (lock, [this]() { return done_; });
+ }
+
+ private:
+ std::mutex m_;
+ std::condition_variable cv_;
+ output_buffers *output_;
+ bool done_;
+};
+
+/**
+ * @brief Execute inference. Wait (block) until the output is available.
+ * @param[in] modelid The model to be inferred.
+ * @param[in] input The input data to be inferred.
+ * @param[out] output The output result.
+ * @return @c 0 if no error. otherwise a negative error value
+ */
+int
+HostHandler::runSync (uint32_t modelid, const input_buffers *input,
+ output_buffers *output)
+{
+ callbackSync sync (output);
+ int status = runAsync (modelid, input, callbackSync::callback,
+ static_cast <void*> (&sync), NPUASYNC_DROP_OLD, nullptr);
+ if (status == 0) {
+ /** sync needs to wait callback */
+ sync.wait ();
+ }
+ return status;
+}
+
+/**
+ * @brief Invoke NPU inference. Unblocking call.
+ * @param[in] modelid The model to be inferred.
+ * @param[in] input The input data to be inferred.
+ * @param[in] cb The output buffer handler.
+ * @param[in] cb_data The data given as a parameter to the runNPU_async call.
+ * @param[in] mode Configures how this operation works.
+ * @param[out] sequence The sequence number returned with runNPU_async.
+ * @return @c 0 if no error. otherwise a negative error value
+ */
+int
+HostHandler::runAsync (uint32_t modelid, const input_buffers *input,
+ npuOutputNotify cb, void *cb_data, npu_async_mode mode, uint64_t *sequence)
+{
+ Model *model = nullptr;
+
+ if (device_->needModel()) {
+ model = getModel (modelid);
+ if (model == nullptr)
+ return -ENOENT;
+ }
+
+ device_->setAsyncMode (mode);
+ return device_->run (NPUINPUT_HOST, model, input, cb, cb_data, sequence);
+}
+
+/**
+ * @brief get number of available devices
+ * @param[in] type device type
+ * @return number of devices
+ */
+int
+HostHandler::getNumDevices (dev_type type)
+{
+ return DriverAPI::getNumDevices (type);
+}
+
+/**
+ * @brief get device instance
+ * @param[out] dev device instance
+ * @param[in] type device type
+ * @param[in] id device id
+ * @return 0 if no error. otherwise a negative errno
+ */
+int
+HostHandler::getDevice (npudev_h *dev, dev_type type, uint32_t id)
+{
+ int num_devices = getNumDevices (type);
+
+ /** check the validity of device id */
+ if (!(num_devices > 0 && id < static_cast<uint32_t>(num_devices))) {
+ logerr (TAG, "Invalid arguments provided\n");
+ return -ENODEV;
+ }
+
+ Device *device = Device::createInstance (type, id);
+ if (device == nullptr) {
+ logerr (TAG, "Failed to create a device with the given type\n");
+ return -EINVAL;
+ }
+
+ *dev = device;
+ /** This is just for backward-compatility; we don't guarantee its corresness */
+ latest_dev_ = *dev;
+
+ return 0;
+}
+
+/**
+ * @brief allocate generic buffer (just for users)
+ * @param[out] buffer buffer instance
+ * @return 0 if no error. otherwise a negative errno
+ */
+int
+HostHandler::allocGenericBuffer (generic_buffer *buffer)
+{
+ if (buffer == NULL || SIZE_MAX < buffer->size)
+ return -EINVAL;
+
+ if (buffer->type == BUFFER_FILE) {
+ /* nothing to do */
+ if (buffer->filepath == nullptr)
+ return -EINVAL;
+ } else {
+ /* now, npu-engine always provides dmabuf-based allocation */
+ HWmem *hwmem = device_->allocMemory ();
+ if (hwmem == nullptr || hwmem->alloc (buffer->size) < 0)
+ return -ENOMEM;
+
+ buffer->dmabuf = hwmem->getDmabuf();
+ buffer->offset = hwmem->getOffset();
+ buffer->addr = hwmem->getData();
+ }
+ return 0;
+}
+
+/**
+ * @brief deallocate generic buffer (just for users)
+ * @param[in] buffer buffer instance
+ * @return 0 if no error. otherwise a negative errno
+ */
+int
+HostHandler::deallocGenericBuffer (generic_buffer *buffer)
+{
+ if (buffer == NULL)
+ return -EINVAL;
+
+ if (buffer->type != BUFFER_FILE)
+ device_->deallocMemory (buffer->dmabuf);
+
+ return 0;
+}
+
+/**
+ * @brief allocate multiple generic buffers (just for users)
+ * @param[out] buffers multi-buffer instance
+ * @return 0 if no error. otherwise a negative errno
+ */
+int
+HostHandler::allocGenericBuffer (generic_buffers *buffers)
+{
+ if (buffers == NULL || buffers->num_buffers < 1)
+ return -EINVAL;
+
+ buffer_types type = buffers->bufs[0].type;
+ if (type == BUFFER_FILE)
+ return 0;
+
+ uint64_t total_size = 0;
+ for (uint32_t idx = 0; idx < buffers->num_buffers; idx++)
+ total_size += buffers->bufs[idx].size;
+
+ uint64_t first_size = buffers->bufs[0].size;
+ buffers->bufs[0].size = total_size;
+ int status = allocGenericBuffer (&buffers->bufs[0]);
+ if (status != 0)
+ return status;
+
+ uint64_t offset = first_size;
+ for (uint32_t idx = 1; idx < buffers->num_buffers; idx++) {
+ buffers->bufs[idx].dmabuf = buffers->bufs[0].dmabuf;
+ buffers->bufs[idx].offset = buffers->bufs[0].offset + offset;
+ buffers->bufs[idx].type = type;
+
+ offset += buffers->bufs[idx].size;
+ }
+
+ return 0;
+}
+
+/**
+ * @brief deallocate multiple generic buffers (just for users)
+ * @param[in] buffers multi-buffer instance
+ * @return 0 if no error. otherwise a negative errno
+ */
+int
+HostHandler::deallocGenericBuffer (generic_buffers *buffers)
+{
+ if (buffers == NULL || buffers->num_buffers < 1)
+ return -EINVAL;
+
+ return deallocGenericBuffer (&buffers->bufs[0]);
+}
+
+/** just for backward-compatability */
+npudev_h HostHandler::latest_dev_ = nullptr;
+
+/** implementation of libnpuhost APIs */
+
+/**
+ * @brief Returns the number of available NPU devices.
+ * @return @c The number of NPU devices.
+ * @retval 0 if no NPU devices available. if positive (number of NPUs) if NPU devices available. otherwise, a negative error value.
+*/
+int getnumNPUdeviceByType (dev_type type)
+{
+ return HostHandler::getNumDevices (type);
+}
+
+/**
+ * @brief Returns the number of NPU devices (TRIV).
+ */
+int getnumNPUdevice (void)
+{
+ return getnumNPUdeviceByType (NPUCOND_TRIV_CONN_SOCIP);
+}
+
+/**
+ * @brief Returns the list of ASR devices (TRIA)
+ */
+int getnumASRdevice (void)
+{
+ return getnumNPUdeviceByType (NPUCOND_TRIA_CONN_SOCIP);
+}
+
+/**
+ * @brief Returns the handle of the chosen NPU devices.
+ * @param[out] dev The NPU device handle
+ * @param[in] id The NPU id to get the handle. 0 <= id < getnumNPUdeviceByType().
+ * @return @c 0 if no error. otherwise a negative error value
+ */
+int getNPUdeviceByType (npudev_h *dev, dev_type type, uint32_t id)
+{
+ return HostHandler::getDevice (dev, type, id);
+}
+
+/**
+ * @brief Returns the handle of the chosen TRIV device.
+ */
+int getNPUdevice (npudev_h *dev, uint32_t id)
+{
+ return getNPUdeviceByType (dev, NPUCOND_TRIV_CONN_SOCIP, id);
+}
+
+/**
+ * @brief Returns the handle of the chosen TRIA device.
+ */
+int getASRdevice (npudev_h *dev, uint32_t id)
+{
+ return getNPUdeviceByType (dev, NPUCOND_TRIA_CONN_SOCIP, id);
+}
+
+/**
+ * @brief Send the NN model to NPU.
+ * @param[in] dev The NPU device handle
+ * @param[in] modelfile The filepath to the compiled NPU NN model in any buffer_type
+ * @param[out] modelid The modelid allocated for this instance of NN model.
+ * @return @c 0 if no error. otherwise a negative error value
+ *
+ * @detail For ASR devices, which do not accept models, but have models
+ * embedded in devices, you do not need to call register and
+ * register calls for ASR are ignored.
+ *
+ * @todo Add a variation: in-memory model register.
+ */
+int registerNPUmodel (npudev_h dev, generic_buffer *modelfile, uint32_t *modelid)
+{
+ INIT_HOST_HANDLER (host_handler, dev);
+
+ return host_handler->registerModel (modelfile, modelid);
+}
+
+/**
+ * @brief Remove the NN model from NPU
+ * @param[in] dev The NPU device handle
+ * @param[in] modelid The model to be removed from the NPU.
+ * @return @c 0 if no error. otherwise a negative error value
+ * @detail This may incur some latency with memory compatcion.
+ */
+int unregisterNPUmodel(npudev_h dev, uint32_t modelid)
+{
+ INIT_HOST_HANDLER (host_handler, dev);
+
+ return host_handler->unregisterModel (modelid);
+}
+
+/**
+ * @brief Remove all NN models from NPU
+ * @param[in] dev The NPU device handle
+ * @return @c 0 if no error. otherwise a negative error value
+ */
+int unregisterNPUmodel_all(npudev_h dev)
+{
+ INIT_HOST_HANDLER (host_handler, dev);
+
+ return host_handler->unregisterModels ();
+}
+
+/**
+ * @brief [OPTIONAL] Set the data layout for input/output tensors
+ * @param[in] dev The NPU device handle
+ * @param[in] modelid The ID of model whose layouts are set
+ * @param[in] info_in the layout/type info for input tensors
+ * @param[in] info_out the layout/type info for output tensors
+ * @return @c 0 if no error. otherwise a negative error value
+ * @note if this function is not called, default layout/type will be used.
+ */
+int setNPU_dataInfo(npudev_h dev, uint32_t modelid,
+ tensors_data_info *info_in, tensors_data_info *info_out)
+{
+ INIT_HOST_HANDLER (host_handler, dev);
+
+ return host_handler->setDataInfo (modelid, info_in, info_out);
+}
+
+/**
+ * @brief [OPTIONAL] Set the inference constraint for next NPU inferences
+ * @param[in] dev The NPU device handle
+ * @param[in] modelid The target model id
+ * @param[in] constraint inference constraint (e.g., timeout, priority)
+ * @return @c 0 if no error. otherwise a negative error value
+ * @note If this function is not called, default values are used.
+ */
+int setNPU_constraint(npudev_h dev, uint32_t modelid, npuConstraint constraint)
+{
+ INIT_HOST_HANDLER (host_handler, dev);
+
+ return host_handler->setConstraint (modelid, constraint);
+}
+
+/**
+ * @brief Execute inference. Wait (block) until the output is available.
+ * @param[in] dev The NPU device handle
+ * @param[in] modelid The model to be inferred.
+ * @param[in] input The input data to be inferred.
+ * @param[out] output The output result. The caller MUST allocate appropriately before calling this.
+ * @return @c 0 if no error. otherwise a negative error value
+ *
+ * @detail This is a syntactic sugar of runNPU_async().
+ * CAUTION: There is a memcpy for the output buffer.
+ */
+int runNPU_sync(npudev_h dev, uint32_t modelid, const input_buffers *input,
+ output_buffers *output)
+{
+ INIT_HOST_HANDLER (host_handler, dev);
+
+ return host_handler->runSync (modelid, input, output);
+}
+
+/**
+ * @brief Invoke NPU inference. Unblocking call.
+ * @param[in] dev The NPU device handle
+ * @param[in] modelid The model to be inferred.
+ * @param[in] input The input data to be inferred.
+ * @param[in] cb The output buffer handler.
+ * @param[out] sequence The sequence number returned with runNPU_async.
+ * @param[in] data The data given as a parameter to the runNPU_async call.
+ * @param[in] mode Configures how this operation works.
+ * @return @c 0 if no error. otherwise a negative error value
+ */
+int runNPU_async(npudev_h dev, uint32_t modelid, const input_buffers *input,
+ npuOutputNotify cb, uint64_t *sequence, void *data,
+ npu_async_mode mode)
+{
+ INIT_HOST_HANDLER (host_handler, dev);
+
+ return host_handler->runAsync (modelid, input, cb, data, mode, sequence);
+}
+
+/**
+ * @brief Allocate a buffer for NPU model with the requested buffer type.
+ * @param[in] dev The NPU device handle
+ * @param[in/out] Buffer the buffer pointer where memory is allocated.
+ * @return 0 if no error, otherwise a negative errno.
+ */
+int allocNPU_modelBuffer (npudev_h dev, generic_buffer *buffer)
+{
+ INIT_HOST_HANDLER (host_handler, dev);
+
+ return host_handler->allocGenericBuffer (buffer);
+}
+
+/**
+ * @brief Free the buffer and remove the address mapping.
+ * @param[in] dev The NPU device handle
+ * @param[in] buffer the model buffer
+ * @return 0 if no error, otherwise a negative errno.
+ */
+int cleanNPU_modelBuffer (npudev_h dev, generic_buffer *buffer)
+{
+ INIT_HOST_HANDLER (host_handler, dev);
+
+ return host_handler->deallocGenericBuffer (buffer);
+}
+
+/**
+ * @brief Allocate a buffer for NPU input with the requested buffer type.
+ * @param[in] dev The NPU device handle
+ * @param[in/out] Buffer the buffer pointer where memory is allocated.
+ * @return 0 if no error, otherwise a negative errno.
+ * @note please utilize allocInputBuffers() for multiple input tensors because subsequent
+ * calls of allocInputBuffer() don't gurantee contiguous allocations between them.
+ */
+int allocNPU_inputBuffer (npudev_h dev, generic_buffer *buffer)
+{
+ INIT_HOST_HANDLER (host_handler, dev);
+
+ return host_handler->allocGenericBuffer (buffer);
+}
+
+/**
+ * @brief Free the buffer and remove the address mapping.
+ * @param[in] dev The NPU device handle
+ * @param[in] buffer the input buffer
+ * @return 0 if no error, otherwise a negative errno.
+ */
+int cleanNPU_inputBuffer (npudev_h dev, generic_buffer *buffer)
+{
+ INIT_HOST_HANDLER (host_handler, dev);
+
+ return host_handler->deallocGenericBuffer (buffer);
+}
+
+/**
+ * @brief Allocate input buffers, which have multiple instances of generic_buffer
+ * @param[in] dev The NPU device handle
+ * @param[in/out] input input buffers.
+ * @return 0 if no error, otherwise a negative errno.
+ * @note it reuses allocInputBuffer().
+ * @details in case of BUFFER_DMABUF, this function can be used to gurantee physically-contiguous
+ * memory mapping for multiple tensors (in a single inference, not batch size).
+ */
+int allocNPU_inputBuffers (npudev_h dev, input_buffers * input)
+{
+ INIT_HOST_HANDLER (host_handler, dev);
+
+ return host_handler->allocGenericBuffer (input);
+}
+
+/**
+ * @brief Free input buffers allocated by allocInputBuffers().
+ * @param[in] dev The NPU device handle
+ * @param[in/out] input input buffers.
+ * @note it reuses cleanInputbuffer().
+ * @return 0 if no error, otherwise a negative errno.
+ */
+int cleanNPU_inputBuffers (npudev_h dev, input_buffers * input)
+{
+ INIT_HOST_HANDLER (host_handler, dev);
+
+ return host_handler->deallocGenericBuffer (input);
+}
+
+/**
+ * @brief Get metadata for NPU model
+ * @param[in] model The path of model binary file
+ * @param[in] need_extra whether you want to extract the extra data in metadata
+ * @return the metadata structure to be filled if no error, otherwise nullptr
+ *
+ * @note For most npu-engine users, the extra data is not useful because it will be
+ * used for second-party users (e.g., compiler, simulator).
+ * Also, the caller needs to free the metadata.
+ *
+ * @note the caller needs to free the metadata
+ */
+npubin_meta * getNPUmodel_metadata (const char *model, bool need_extra)
+{
+ npubin_meta *meta;
+ FILE *fp;
+ size_t ret;
+
+ if (!model)
+ return nullptr;
+
+ fp = fopen (model, "rb");
+ if (!fp) {
+ logerr (TAG, "Failed to open the model binary: %d\n", -errno);
+ return nullptr;
+ }
+
+ meta = (npubin_meta *) malloc (NPUBIN_META_SIZE);
+ if (!meta) {
+ logerr (TAG, "Failed to allocate metadata\n");
+ goto exit_err;
+ }
+
+ ret = fread (meta, 1, NPUBIN_META_SIZE, fp);
+ if (ret != NPUBIN_META_SIZE) {
+ logerr (TAG, "Failed to read the metadata\n");
+ goto exit_free;
+ }
+
+ if (!CHECK_NPUBIN (meta->magiccode)) {
+ logerr (TAG, "Invalid metadata provided\n");
+ goto exit_free;
+ }
+
+ if (need_extra && NPUBIN_META_EXTRA (meta->magiccode) > 0) {
+ npubin_meta *new_meta;
+
+ new_meta = (npubin_meta *) realloc (meta, NPUBIN_META_TOTAL_SIZE(meta->magiccode));
+ if (!new_meta) {
+ logerr (TAG, "Failed to allocate extra metadata\n");
+ goto exit_free;
+ }
+
+ ret = fread (new_meta->reserved_extra, 1, NPUBIN_META_EXTRA_SIZE (meta->magiccode), fp);
+ if (ret != NPUBIN_META_EXTRA_SIZE (meta->magiccode)) {
+ logerr (TAG, "Invalid extra metadata provided\n");
+ free (new_meta);
+ goto exit_err;
+ }
+
+ meta = new_meta;
+ }
+
+ fclose (fp);
+
+ return meta;
+
+exit_free:
+ free (meta);
+exit_err:
+ fclose (fp);
+
+ return nullptr;
+}
+
+/** deprecated buffer APIs; please use the above APIs */
+
+/** @brief deprecated */
+int allocModelBuffer (generic_buffer *buffer)
+{
+ logwarn (TAG, "deprecated. Please use allocNPU_modelBuffer\n");
+ return allocNPU_modelBuffer (HostHandler::getLatestDevice(), buffer);
+}
+
+/** @brief deprecated */
+int cleanModelBuffer (generic_buffer *buffer)
+{
+ logwarn (TAG, "deprecated. Please use cleanNPU_modelBuffer\n");
+ return allocNPU_modelBuffer (HostHandler::getLatestDevice(), buffer);
+}
+
+/** @brief deprecated */
+int allocInputBuffer (generic_buffer *buffer)
+{
+ logwarn (TAG, "deprecated. Please use allocNPU_inputBuffer\n");
+ return allocNPU_inputBuffer (HostHandler::getLatestDevice(), buffer);
+}
+
+/** @brief deprecated */
+int cleanInputBuffer (generic_buffer *buffer)
+{
+ logwarn (TAG, "deprecated. Please use cleanNPU_inputBuffer\n");
+ return cleanNPU_inputBuffer (HostHandler::getLatestDevice(), buffer);
+}
+
+/** @brief deprecated */
+int allocInputBuffers (input_buffers * input)
+{
+ logwarn (TAG, "deprecated. Please use allocNPU_inputBuffers\n");
+ return allocNPU_inputBuffers (HostHandler::getLatestDevice(), input);
+}
+
+/** @brief deprecated */
+int cleanInputBuffers (input_buffers * input)
+{
+ logwarn (TAG, "deprecated. Please use cleanNPU_inputBuffers\n");
+ return cleanNPU_inputBuffers (HostHandler::getLatestDevice(), input);
+}
+
+/** @brief deprecated */
+int allocNPUBuffer (uint64_t size, buffer_types type,
+ const char * filepath, generic_buffer *buffer)
+{
+ if (buffer) {
+ buffer->size = size;
+ buffer->type = type;
+ buffer->filepath = filepath;
+ }
+
+ logwarn (TAG, "deprecated. Please use allocNPU_* APIs\n");
+ return allocModelBuffer (buffer);
+}
+
+/** @brief deprecated */
+int cleanNPUBuffer (generic_buffer * buffer)
+{
+ logwarn (TAG, "deprecated. Please use cleanNPU_* APIs\n");
+ return cleanModelBuffer (buffer);
+}
/**
* @file NE-handler.h
* @date 25 Jun 2019
- * @brief Host (model) handler for NPU Engine (NE).
+ * @brief Host handler for NPU Engine (NE).
* @see http://suprem.sec.samsung.net/confluence/display/ODLC/Software+Stack
* @author Dongju Chae <dongju.chae@samsung.com>
* @bug No known bugs except for NYI items
#ifndef __NPU_ENGINE_HANDLER_H__
#define __NPU_ENGINE_HANDLER_H__
-/**
- * @brief initilize host handler
- * @return 0 if no error, otherwise a negative errno
- * @note it should be called in main()
- */
-int init_ne_handler (void);
+#include <libnpuhost.h>
+#include <CommPlugin.h>
-/**
- * @brief terminate host handler
- * @return 0 if no error, otherwise a negative errno
- * @note it should be called in main()
- */
-int exit_ne_handler (void);
+#include "ne-scheduler.h"
+#include "ne-model.h"
+#include "ne-utils.h"
+
+class Device;
+/** @brief class def. of host handler */
+class HostHandler {
+ public:
+ HostHandler (Device *device);
+ ~HostHandler ();
+
+ int registerModel (generic_buffer *model_buf, uint32_t *modelid);
+ int unregisterModel (uint32_t modelid);
+ int unregisterModels ();
+
+ int setDataInfo (uint32_t modelid, tensors_data_info *in, tensors_data_info *out);
+ int setConstraint (uint32_t modelid, npuConstraint constraint);
+
+ Model *getModel (uint32_t modelid);
+
+ int allocGenericBuffer (generic_buffer *buffer);
+ int allocGenericBuffer (generic_buffers *buffers);
+
+ int deallocGenericBuffer (generic_buffer *buffer);
+ int deallocGenericBuffer (generic_buffers *buffers);
+
+ int runSync (uint32_t modelid, const input_buffers *input, output_buffers *output);
+ int runAsync (uint32_t modelid, const input_buffers *input, npuOutputNotify cb,
+ void *cb_data, npu_async_mode mode, uint64_t *sequence);
+
+ static int getNumDevices (dev_type type);
+ static int getDevice (npudev_h *dev, dev_type type, uint32_t id);
+
+ static npudev_h getLatestDevice () { return latest_dev_; }
+
+ private:
+ Device *device_; /**< dedicated device instance */
+ ThreadSafeMap<uint32_t, Model> models_;
+ /**< registerd models */
+ npu_async_mode async_mode_;
+ /**< async mode of runAsync */
+ static npudev_h latest_dev_;
+ /**< latest device; just for backward-compatability */
+};
#endif /* __NPU_ENGINE_HANDLER_H__ */
projects / platform / adaptation / npu / trix-engine.git / commitdiff