host: Add register model with exteranl prog,weight buffer

It adds `registerNPUModel_ext` api for using external program and weight
buffers. This buffer can be shared between threads, so application can
be divided into several threads, like a feeding program, a feeding weight,
and a inference thread. An exaple codes are included.

Signed-off-by: Jiho Chu <jiho.chu@samsung.com>

diff --git a/include/host/libnpuhost.h b/include/host/libnpuhost.h
index c00e1143d6955aba84576f3a856d41b8f2d402a8..033daffeb05c9afe53f38c687f4a65865e3e0ac6 100644 (file)
--- a/include/host/libnpuhost.h
+++ b/include/host/libnpuhost.h
@@ -161,10 +161,27 @@ int getNPUmodel_compatibleDevices(npubin_meta *meta, dev_type **dev_list, size_t
  *
  * @detail For ASR devices, which do not accept models, but have models
  *         embedded in devices, you do not need to call register and
- *         register calls for ASR are ignored.
+ *         register calls for ASR are ignored. Weight is allocated with model meta info.
  */
 int registerNPUmodel (npudev_h dev, generic_buffer *modelfile, uint32_t *model_id);
 
+/**
+ * @brief Send the NN model to NPU with external buffer
+ * @param[in] dev The NPU device handle
+ * @param[in] modelfile The filepath to the compiled NPU NN model in any buffer_type
+ * @param[in] prog The generic buffer for program data. If it's null, it uses model data.
+ * @param[in] weight The generic buffer for weight data. If it's null, it allocates with model info.
+ * @param[out] model_id The modelid allocated for this instance of NN model.
+ * @return @c 0 if no error. otherwise a negative error value
+ *
+ * @detail For ASR devices, which do not accept models, but have models
+ *         embedded in devices, you do not need to call register and
+ *         register calls for ASR are ignored.
+ *         Caution, it cannot gurantee the program and weight constistance after registering
+ */
+int registerNPUmodel_ext (npudev_h dev, generic_buffer *modelfile, generic_buffer *prog,
+                          generic_buffer *weight, uint32_t *model_id);
+
 /**
  * @brief Remove the NN model from NPU
  * @param[in] dev The NPU device handle

diff --git a/src/core/ne-handler.cc b/src/core/ne-handler.cc
index 351815a364d22870b794fd72371ba039f6aed702..5b48f04c04a45015300c8212491c14c07fb0cfdd 100644 (file)
--- a/src/core/ne-handler.cc
+++ b/src/core/ne-handler.cc
@@ -51,13 +51,27 @@ HostHandler::~HostHandler () {
  */
 int
 HostHandler::registerModel (generic_buffer *model_buf, uint32_t *modelid) {
+  return registerModel (model_buf, nullptr, nullptr, modelid);
+}
+
+/**
+ * @brief register model from generic buffer
+ * @param[in] model_buf model buffer
+ * @param[in] prog program buffer
+ * @param[in] weight weight buffer
+ * @param[out] modelid model id
+ * @return 0 if no error. otherwise a negative errno
+ */
+int
+HostHandler::registerModel (generic_buffer *model_buf, generic_buffer *prog, generic_buffer *weight,
+                            uint32_t *modelid) {
   if (model_buf == nullptr || modelid == nullptr) {
     logerr (TAG, "Invalid arguments given\n");
     return -EINVAL;
   }
 
   Model *model = nullptr;
-  int status = device_->setModel (model_buf, &model);
+  int status = device_->setModel (model_buf, prog, weight, &model);
   if (status != 0) {
     logerr (TAG, "Failed to set model: %d\n", status);
     return status;
@@ -968,6 +982,24 @@ TrinityVision2::checkVersion (uint64_t npu_version) const {
  */
 int
 TrinityVision2::setModel (const generic_buffer *model_buf, Model **model_ptr) {
+  return setModel (model_buf, nullptr, nullptr, model_ptr);
+}
+
+/**
+ * @brief implementation of TRIV2's setModel ()
+ * @param[in] model_buf the model generic buffer
+ * @param[out] model the model instance
+ * @return 0 if no error, otherwise a negative errno
+ */
+int
+TrinityVision2::setModel (const generic_buffer *model_buf, const generic_buffer *prog,
+                          const generic_buffer *weight, Model **model_ptr) {
+  HWmem *hwmem_prog = nullptr;
+  HWmem *hwmem_weight = nullptr;
+  HWmem *hwmem_extended = nullptr;
+  model_config_t config;
+  size_t extended_size;
+
   if (!initialized ()) {
     logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
     return -EPERM;
@@ -1005,7 +1037,6 @@ TrinityVision2::setModel (const generic_buffer *model_buf, Model **model_ptr) {
     default:
       return -EINVAL;
   }
-
   status = model->setMetadata (model->getData ());
   if (status != 0)
     goto delete_exit;
@@ -1019,9 +1050,18 @@ TrinityVision2::setModel (const generic_buffer *model_buf, Model **model_ptr) {
   if (status != 0)
     goto delete_exit;
 
-  /** allocate program (optional; NOP) */
-  if (model->getMetadata ()->getProgramSize () > 0) {
-    HWmem *hwmem_prog = new HWmem (new HWmemDevice);
+  if (prog) {
+    hwmem_prog = new HWmem (new HWmemDevice);
+
+    hwmem_prog->setDriverAPI (api_.get ());
+    hwmem_prog->setContiguous (true);
+    hwmem_prog->setDmabuf (prog->dmabuf);
+    hwmem_prog->setSize (prog->size);
+
+    model->setProgramData (hwmem_prog);
+  } else if (model->getMetadata ()->getProgramSize () > 0) {
+    /** allocate program (optional; NOP) */
+    hwmem_prog = new HWmem (new HWmemDevice);
     hwmem_prog->setDriverAPI (api_.get ());
     hwmem_prog->setContiguous (true);
 
@@ -1041,57 +1081,64 @@ TrinityVision2::setModel (const generic_buffer *model_buf, Model **model_ptr) {
       logerr (TAG, "Failed to extract generic buffer: %d\n", status);
       goto delete_exit;
     }
+  }
 
-    /** register this model to the driver */
-    model_config_t config;
-    config.version = model->getMetadata ()->getVersion ();
-    config.dbuf_fd = hwmem_prog->getDmabuf ();
-    config.program_size = hwmem_prog->getSize ();
-    config.program_offset_addr = 0;
-    config.metadata_dbuf_fd = model->getDmabuf ();
-    config.npu_version = model->getMetadata ()->getNPUVersion ();
+  /** register this model to the driver */
+  config.version = model->getMetadata ()->getVersion ();
+  config.dbuf_fd = hwmem_prog->getDmabuf ();
+  config.program_size = hwmem_prog->getSize ();
+  config.program_offset_addr = 0;
+  config.metadata_dbuf_fd = model->getDmabuf ();
+  config.npu_version = model->getMetadata ()->getNPUVersion ();
 
-    /** for metadata extended section */
-    size_t extended_size = model->getMetadata ()->getMetaExtendedSize ();
-    if (extended_size > 0) {
-      HWmem *hwmem_extended = new HWmem (new HWmemDevice);
-      hwmem_extended->setDriverAPI (api_.get ());
+  /** for metadata extended section */
+  extended_size = model->getMetadata ()->getMetaExtendedSize ();
+  if (extended_size > 0) {
+    hwmem_extended = new HWmem (new HWmemDevice);
+    hwmem_extended->setDriverAPI (api_.get ());
 
-      model->setExtendedMetadata (hwmem_extended);
+    model->setExtendedMetadata (hwmem_extended);
 
-      status = hwmem_extended->alloc (extended_size);
-      if (status != 0) {
-        logerr (TAG, "Failed to allocate extended metadata: %d\n", status);
-        goto delete_exit;
-      }
+    status = hwmem_extended->alloc (extended_size);
+    if (status != 0) {
+      logerr (TAG, "Failed to allocate extended metadata: %d\n", status);
+      goto delete_exit;
+    }
 
-      config.metadata_ext_dbuf_fd = hwmem_extended->getDmabuf ();
-      config.metadata_ext_size = extended_size;
+    config.metadata_ext_dbuf_fd = hwmem_extended->getDmabuf ();
+    config.metadata_ext_size = extended_size;
 
-      status = comm_.extractGenericBuffer (model_buf, hwmem_extended->getData (), nullptr,
-                                           NPUBIN_META_SIZE, extended_size);
-      hwmem_extended->putData();
-      if (status != 0) {
-        logerr (TAG, "Failed to extract generic buffer: %d\n", status);
-        goto delete_exit;
-      }
-    } else {
-      config.metadata_ext_dbuf_fd = -1;
-      config.metadata_ext_size = 0;
+    status = comm_.extractGenericBuffer (model_buf, hwmem_extended->getData (), nullptr,
+                                         NPUBIN_META_SIZE, extended_size);
+    hwmem_extended->putData ();
+    if (status != 0) {
+      logerr (TAG, "Failed to extract generic buffer: %d\n", status);
+      goto delete_exit;
     }
+  } else {
+    config.metadata_ext_dbuf_fd = -1;
+    config.metadata_ext_size = 0;
+  }
 
-    config.num_visa_insts = model->getNumInsts ();
+  config.num_visa_insts = model->getNumInsts ();
 
-    status = api_->registerModel (&config);
-    if (status != 0)
-      goto delete_exit;
+  status = api_->registerModel (&config);
+  if (status != 0)
+    goto delete_exit;
 
-    model->setInternalID (config.id);
-  }
+  model->setInternalID (config.id);
+
+  if (weight) {
+    hwmem_weight = new HWmem (new HWmemDevice);
 
-  /** allocate weight (optional) */
-  if (model->getMetadata ()->getWeightSize () > 0) {
-    HWmem *hwmem_weight = new HWmem (new HWmemDevice);
+    hwmem_weight->setDriverAPI (api_.get ());
+    hwmem_weight->setDmabuf (weight->dmabuf);
+    hwmem_weight->setSize (weight->size);
+
+    model->setWeightData (hwmem_weight);
+  } else if (model->getMetadata ()->getWeightSize () > 0) {
+    /** allocate weight (optional) */
+    hwmem_weight = new HWmem (new HWmemDevice);
     hwmem_weight->setDriverAPI (api_.get ());
 
     model->setWeightData (hwmem_weight);

diff --git a/src/core/ne-handler.h b/src/core/ne-handler.h
index b8ef0a0f7fa9340be90086a16096cbdf5b6a7ecf..71900432ecc502b3d526b8f82fba5a44271a5c37 100644 (file)
--- a/src/core/ne-handler.h
+++ b/src/core/ne-handler.h
@@ -33,6 +33,8 @@ class HostHandler {
   ~HostHandler ();
 
   int registerModel (generic_buffer *model_buf, uint32_t *modelid);
+  int registerModel (generic_buffer *model_buf, generic_buffer *prog, generic_buffer *weight,
+                     uint32_t *modelid);
   int unregisterModel (uint32_t modelid);
   int unregisterModels ();
 
@@ -131,6 +133,8 @@ class Device {
 
   /** virtual methods to implement each device's behaviors */
   virtual int setModel (const generic_buffer *model, Model **model_ptr) = 0;
+  virtual int setModel (const generic_buffer *model, const generic_buffer *prog,
+                        const generic_buffer *weight, Model **model_ptr) = 0;
   virtual int unsetModel (Model *model) = 0;
   virtual int getTensorSize (const Model *model, bool input, uint32_t index, uint32_t *size) = 0;
 
@@ -190,6 +194,8 @@ class TrinityVision2 : public Device {
                                      const output_buffers *output = nullptr);
 
   int setModel (const generic_buffer *model, Model **model_ptr);
+  int setModel (const generic_buffer *model, const generic_buffer *prog,
+                const generic_buffer *weight, Model **model_ptr);
   int unsetModel (Model *model);
 
   int getTensorSize (const Model *model, bool input, uint32_t index, uint32_t *size);

diff --git a/src/host/ne-host.cc b/src/host/ne-host.cc
index 625a74e24319de3074c596749f4e0c14a45e8de9..8eccd78e9d10f2743535ec2b9213d07b3e805451 100644 (file)
--- a/src/host/ne-host.cc
+++ b/src/host/ne-host.cc
@@ -340,6 +340,29 @@ registerNPUmodel (npudev_h dev, generic_buffer *modelfile, uint32_t *modelid) {
   return host_handler->registerModel (modelfile, modelid);
 }
 
+/**
+ * @brief Send the NN model to NPU with external buffer
+ * @param[in] dev The NPU device handle
+ * @param[in] modelfile The filepath to the compiled NPU NN model in any buffer_type
+ * @param[in] prog The generic buffer for program data. If it's null, it uses model data.
+ * @param[in] weight The generic buffer for weight data. If it's null, it allocates with model info.
+ * @param[out] modelid The modelid allocated for this instance of NN model.
+ * @return @c 0 if no error. otherwise a negative error value
+ *
+ * @detail For ASR devices, which do not accept models, but have models
+ *         embedded in devices, you do not need to call register and
+ *         register calls for ASR are ignored.
+ *
+ * @todo Add a variation: in-memory model register.
+ */
+int
+registerNPUmodel_ext (npudev_h dev, generic_buffer *modelfile, generic_buffer *prog,
+                      generic_buffer *weight, uint32_t *modelid) {
+  INIT_HOST_HANDLER (host_handler, dev);
+
+  return host_handler->registerModel (modelfile, prog, weight, modelid);
+}
+
 /**
  * @brief Remove the NN model from NPU
  * @param[in] dev The NPU device handle