3 * Copyright (C) 2020 Samsung Electronics
4 * Copyright (C) 2020 Dongju Chae <dongju.chae@samsung.com>
7 * @file ne-host-handler.cc
9 * @brief Implementation of APIs to access NPU from Host
10 * @see https://code.sec.samsung.net/confluence/display/ODLC/2020+Overall+Software+Stack
11 * @author Dongju Chae <dongju.chae@samsung.com>
12 * @bug No known bugs except for NYI items
15 #include "ne-handler.h"
18 #include <libnpuhost.h>
19 #include <npubinfmt.h>
20 #include <NPUdrvAPI.h>
21 #include <CommPlugin.h>
26 #include <condition_variable>
33 #define INIT_HOST_HANDLER(handler, dev) \
34 Device *tdev = static_cast <Device *> (dev); \
35 if (tdev == nullptr) return -EINVAL; \
36 HostHandler *handler = tdev->getHostHandler (); \
37 if (handler == nullptr) return -EINVAL;
39 /** just for backward-compatability */
40 npudev_h HostHandler::latest_dev_ = nullptr;
42 /** implement libnpuhost APIs */
45 * @brief Returns the number of available NPU devices.
46 * @return @c The number of NPU devices.
47 * @retval 0 if no NPU devices available. if positive (number of NPUs) if NPU devices available. otherwise, a negative error value.
48 * @note the caller should call putNPUdevice() to release the device handle
50 int getnumNPUdeviceByType (dev_type type)
52 return HostHandler::getNumDevices (type);
56 * @brief Returns the handle of the chosen NPU devices.
57 * @param[out] dev The NPU device handle
58 * @param[in] id The NPU id to get the handle. 0 <= id < getnumNPUdeviceByType().
59 * @return @c 0 if no error. otherwise a negative error value
60 * @note the caller should call putNPUdevice() to release the device handle
62 int getNPUdeviceByType (npudev_h *dev, dev_type type, uint32_t id)
64 return HostHandler::getDevice (dev, type, id);
68 * @brief release the NPU device instance obtained by getDevice ()
69 * @param[in] dev the NPU device handle
71 void putNPUdevice (npudev_h dev)
74 delete static_cast<Device *> (dev);
78 * @brief Get the profile information from NPU
79 * @param[in] dev The NPU device handle
80 * @param[in] task_id The identifier for each inference
81 * @param[out] profile The profile instance
82 * @return 0 if no error, otherwise a negative errno.
84 int getNPU_profile (npudev_h dev, int task_id, npu_profile *profile)
86 INIT_HOST_HANDLER (host_handler, dev);
88 return host_handler->getProfile (task_id, profile);
92 * @brief Free the profile instance obtained by getNPU_profile().
93 * @param[in] profile The profile instance
95 void putNPU_profile (npu_profile *profile)
97 if (profile != nullptr)
102 * @brief Send the NN model to NPU.
103 * @param[in] dev The NPU device handle
104 * @param[in] modelfile The filepath to the compiled NPU NN model in any buffer_type
105 * @param[out] modelid The modelid allocated for this instance of NN model.
106 * @return @c 0 if no error. otherwise a negative error value
108 * @detail For ASR devices, which do not accept models, but have models
109 * embedded in devices, you do not need to call register and
110 * register calls for ASR are ignored.
112 * @todo Add a variation: in-memory model register.
114 int registerNPUmodel (npudev_h dev, generic_buffer *modelfile, uint32_t *modelid)
116 INIT_HOST_HANDLER (host_handler, dev);
118 return host_handler->registerModel (modelfile, modelid);
122 * @brief Remove the NN model from NPU
123 * @param[in] dev The NPU device handle
124 * @param[in] modelid The model to be removed from the NPU.
125 * @return @c 0 if no error. otherwise a negative error value
126 * @detail This may incur some latency with memory compatcion.
128 int unregisterNPUmodel(npudev_h dev, uint32_t modelid)
130 INIT_HOST_HANDLER (host_handler, dev);
132 return host_handler->unregisterModel (modelid);
136 * @brief Remove all NN models from NPU
137 * @param[in] dev The NPU device handle
138 * @return @c 0 if no error. otherwise a negative error value
140 int unregisterNPUmodel_all(npudev_h dev)
142 INIT_HOST_HANDLER (host_handler, dev);
144 return host_handler->unregisterModels ();
148 * @brief [OPTIONAL] Set the data layout for input/output tensors
149 * @param[in] dev The NPU device handle
150 * @param[in] modelid The ID of model whose layouts are set
151 * @param[in] info_in the layout/type info for input tensors
152 * @param[in] info_out the layout/type info for output tensors
153 * @return @c 0 if no error. otherwise a negative error value
154 * @note if this function is not called, default layout/type will be used.
156 int setNPU_dataInfo(npudev_h dev, uint32_t modelid,
157 tensors_data_info *info_in, tensors_data_info *info_out)
159 INIT_HOST_HANDLER (host_handler, dev);
161 return host_handler->setDataInfo (modelid, info_in, info_out);
165 * @brief [OPTIONAL] Set the inference constraint for next NPU inferences
166 * @param[in] dev The NPU device handle
167 * @param[in] modelid The target model id
168 * @param[in] constraint inference constraint (e.g., timeout, priority)
169 * @return @c 0 if no error. otherwise a negative error value
170 * @note If this function is not called, default values are used.
172 int setNPU_constraint(npudev_h dev, uint32_t modelid, npuConstraint constraint)
174 INIT_HOST_HANDLER (host_handler, dev);
176 return host_handler->setConstraint (modelid, constraint);
180 * @brief Execute inference. Wait (block) until the output is available.
181 * @param[in] dev The NPU device handle
182 * @param[in] modelid The model to be inferred.
183 * @param[in] input The input data to be inferred.
184 * @param[out] output The output result. The caller MUST allocate appropriately before calling this.
185 * @return @c 0 if no error. otherwise a negative error value
187 * @detail This is a syntactic sugar of runNPU_async().
188 * CAUTION: There is a memcpy for the output buffer.
190 int runNPU_sync(npudev_h dev, uint32_t modelid, const input_buffers *input,
191 output_buffers *output)
193 INIT_HOST_HANDLER (host_handler, dev);
195 return host_handler->runSync (modelid, input, output);
199 * @brief Invoke NPU inference. Unblocking call.
200 * @param[in] dev The NPU device handle
201 * @param[in] modelid The model to be inferred.
202 * @param[in] input The input data to be inferred.
203 * @param[in] cb The output buffer handler.
204 * @param[out] sequence The sequence number returned with runNPU_async.
205 * @param[in] data The data given as a parameter to the runNPU_async call.
206 * @param[in] mode Configures how this operation works.
207 * @return @c 0 if no error. otherwise a negative error value
209 int runNPU_async(npudev_h dev, uint32_t modelid, const input_buffers *input,
210 npuOutputNotify cb, uint64_t *sequence, void *data,
213 INIT_HOST_HANDLER (host_handler, dev);
215 return host_handler->runAsync (modelid, input, cb, data, mode, sequence);
219 * @brief Let NPU accept input frames from its internal source continuously
220 * @param[in] dev The NPU device handle
221 * @param[in] modelid The model to be inferred.
222 * @param[in] opmode NPU has different opmode with auto-inputs. Choose one.
223 * @param[in] hw_dev The target device feeding input data
224 * @return @c 0 if no error. otherwise a negative error value
226 int runNPU_internalInput(npudev_h dev, uint32_t modelid, npu_input_opmode opmode,
229 INIT_HOST_HANDLER (host_handler, dev);
231 return host_handler->runInternal (modelid, opmode, hw_dev);
235 * @brief Stop the request with the given id
236 * @param[in] dev The NPU device handle
237 * @param[in] id The request id
238 * @return @c 0 if no error. otherwise a negative error value
240 int stopNPU_internalInput(npudev_h dev, int id)
242 INIT_HOST_HANDLER (host_handler, dev);
244 return host_handler->stopInternal (id);
248 * @brief Allocate a generic buffer with the requested buffer type.
249 * @param[in] dev The NPU device handle
250 * @param[in/out] Buffer the buffer pointer where memory is allocated.
251 * @return 0 if no error, otherwise a negative errno.
253 int allocNPU_genericBuffer (npudev_h dev, generic_buffer * buffer)
255 INIT_HOST_HANDLER (host_handler, dev);
257 return host_handler->allocGenericBuffer (buffer);
261 * @brief Free the generic buffer and remove the address mapping
262 * @param[in] dev The NPU device handle
263 * @param[in] buffer the model buffer
264 * @return 0 if no error, otherwise a negative errno.
266 int cleanNPU_genericBuffer (npudev_h dev, generic_buffer * buffer)
268 INIT_HOST_HANDLER (host_handler, dev);
270 return host_handler->deallocGenericBuffer (buffer);
274 * @brief Allocate generic buffers, which have multiple instances of generic_buffer
275 * @param[in] dev The NPU device handle
276 * @param[in/out] buffers generic buffers.
277 * @return 0 if no error, otherwise a negative errno.
278 * @note it reuses allocGenericBuffer().
280 int allocNPU_genericBuffers (npudev_h dev, generic_buffers * buffers)
282 INIT_HOST_HANDLER (host_handler, dev);
284 return host_handler->allocGenericBuffer (buffers);
288 * @brief Free generic buffers allocated by allocGenericBuffers().
289 * @param[in] dev The NPU device handle
290 * @param[in/out] buffers generic buffers.
291 * @note it reuses cleanGenericbuffer().
292 * @return 0 if no error, otherwise a negative errno.
294 int cleanNPU_genericBuffers (npudev_h dev, generic_buffers * buffers)
296 INIT_HOST_HANDLER (host_handler, dev);
298 return host_handler->deallocGenericBuffer (buffers);
302 * @brief alias of allocNPU_genericBuffer for model buffer
304 int allocNPU_modelBuffer (npudev_h dev, generic_buffer * model)
306 return allocNPU_genericBuffer (dev, model);
310 * @brief alias of cleanNPU_genericBuffer for model buffer
312 int cleanNPU_modelBuffer (npudev_h dev, generic_buffer * model)
314 return cleanNPU_genericBuffer (dev, model);
318 * @brief alias of allocNPU_genericBuffer for input buffer
320 int allocNPU_inputBuffer (npudev_h dev, generic_buffer * input)
322 return allocNPU_genericBuffer (dev, input);
326 * @brief alias of cleanNPU_genericBuffer for input buffer
328 int cleanNPU_inputBuffer (npudev_h dev, generic_buffer * input)
330 return cleanNPU_genericBuffer (dev, input);
334 * @brief alias of allocNPU_genericBuffers for input buffers
336 int allocNPU_inputBuffers (npudev_h dev, input_buffers * input)
338 return allocNPU_genericBuffers (dev, input);
342 * @brief alias of cleanNPU_genericBuffers for input buffers
344 int cleanNPU_inputBuffers (npudev_h dev, input_buffers * input)
346 return cleanNPU_genericBuffers (dev, input);
350 * @brief get the current memory status for the given device
351 * @param[in] dev The NPU device handle
352 * @param[out] alloc_total The size of allocated memory until now
353 * @param[out] free_total The size of freed memory until now
354 * @return @c 0 if no error. otherwise a negatice error value
356 int getNPU_memoryStatus(npudev_h dev, size_t *alloc_total, size_t *free_total)
358 INIT_HOST_HANDLER (host_handler, dev);
360 return host_handler->getMemoryStatus (alloc_total, free_total);
364 * @brief Get the current device status to be used
365 * @param[in] dev The NPU device handle
366 * @param[out] status the device status
367 * @param[out] num_requests the number of running requests (or pending)
368 * @return 0 if no error, otherwise a negative errno.
370 int getNPU_deviceStatus(npudev_h dev, npu_status *status, uint32_t *num_requests)
372 INIT_HOST_HANDLER (host_handler, dev);
374 return host_handler->getDeviceStatus (status, num_requests);
378 * @brief Get metadata for NPU model
379 * @param[in] model The path of model binary file
380 * @param[in] need_extra whether you want to extract the extra data in metadata
381 * @return the metadata structure to be filled if no error, otherwise nullptr
383 * @note For most npu-engine users, the extra data is not useful because it will be
384 * used for second-party users (e.g., compiler, simulator).
385 * Also, the caller needs to free the metadata.
387 * @note the caller needs to free the metadata
389 npubin_meta * getNPUmodel_metadata (const char *model, bool need_extra)
398 fp = fopen (model, "rb");
400 logerr (TAG, "Failed to open the model binary: %d\n", -errno);
404 meta = (npubin_meta *) malloc (NPUBIN_META_SIZE);
406 logerr (TAG, "Failed to allocate metadata\n");
410 ret = fread (meta, 1, NPUBIN_META_SIZE, fp);
411 if (ret != NPUBIN_META_SIZE) {
412 logerr (TAG, "Failed to read the metadata\n");
416 if (!CHECK_NPUBIN (meta->magiccode)) {
417 logerr (TAG, "Invalid metadata provided\n");
421 if (need_extra && NPUBIN_META_EXTRA (meta->magiccode) > 0) {
422 npubin_meta *new_meta;
424 new_meta = (npubin_meta *) malloc (NPUBIN_META_TOTAL_SIZE(meta->magiccode));
426 logerr (TAG, "Failed to allocate extra metadata\n");
430 memcpy (new_meta, meta, NPUBIN_META_TOTAL_SIZE(meta->magiccode));
433 ret = fread (new_meta->reserved_extra, 1, NPUBIN_META_EXTRA_SIZE (new_meta->magiccode), fp);
434 if (ret != NPUBIN_META_EXTRA_SIZE (new_meta->magiccode)) {
435 logerr (TAG, "Invalid extra metadata provided\n");
455 /** implement methods of HostHandler class */
457 /** @brief host handler constructor */
458 HostHandler::HostHandler (Device *device)
460 /* ignored as we don't use double buffering anymore, but for backward-compatibility */
461 async_mode_ (NPUASYNC_WAIT)
465 /** @brief host handler destructor */
466 HostHandler::~HostHandler ()
471 * @brief register model from generic buffer
472 * @param[in] model_buf model buffer
473 * @param[out] modelid model id
474 * @return 0 if no error. otherwise a negative errno
477 HostHandler::registerModel (generic_buffer *model_buf, uint32_t *modelid)
479 if (model_buf == nullptr || modelid == nullptr) {
480 logerr (TAG, "Invalid arguments given\n");
484 Model *model = nullptr;
485 int status = device_->setModel (model_buf, &model);
487 logerr (TAG, "Failed to set model: %d\n", status);
491 assert (model != nullptr);
493 status = models_.insert (model->getID(), model);
495 logerr (TAG, "Failed to insert model id\n");
500 *modelid = model->getID();
505 * @brief remove the registered model
506 * @param[in] modelid model id
507 * @return 0 if no error. otherwise a negative errno
510 HostHandler::unregisterModel (uint32_t modelid)
512 Model *model = models_.find (modelid);
513 if (model == nullptr)
516 int status = device_->unsetModel (model);
518 logerr (TAG, "Failed to unset model: %d\n", status);
522 return models_.remove (modelid);
526 * @brief remove all registered models
530 HostHandler::unregisterModels ()
532 std::function <bool (Model *)> functor =
533 [&] (Model *m) -> bool {
534 bool can_remove = true;
535 int status = device_->unsetModel (m);
537 logwarn (TAG, "Failed to unset model: %d\n", status);
543 models_.for_each (functor);
548 * @brief Get the profile information from NPU
549 * @param[in] task_id The identifier for each inference
550 * @param[out] profile The profile instance
551 * @return 0 if no error, otherwise a negative errno.
554 HostHandler::getProfile (int task_id, npu_profile *profile)
556 if (task_id < 0 || profile == nullptr) {
557 logerr (TAG, "Invalid parameter provided\n");
561 const DriverAPI * api = device_->getDriverAPI ();
562 assert (api != nullptr);
564 void *profile_buffer;
566 int status = api->getProfile (task_id, &profile_buffer, &profile_size);
568 logerr (TAG, "Failed to get profile information: %d\n", status);
572 profile->num_layers = 0;
573 profile->layers = nullptr;
574 if (profile_buffer != nullptr) {
575 // TODO: Perform parsing
582 * @brief Set the data layout for input/output tensors
583 * @param[in] modelid The ID of model whose layouts are set
584 * @param[in] in the layout/type info for input tensors
585 * @param[in] out the layout/type info for output tensors
586 * @return @c 0 if no error. otherwise a negative error value
587 * @note if this function is not called, default layout/type will be used.
590 HostHandler::setDataInfo (uint32_t modelid, tensors_data_info *in,
591 tensors_data_info *out)
593 Model *model = models_.find (modelid);
594 if (model == nullptr)
597 return model->setDataInfo (in, out);
601 * @brief Set the inference constraint for next NPU inferences
602 * @param[in] modelid The target model id
603 * @param[in] constraint inference constraint (e.g., timeout, priority)
604 * @return @c 0 if no error. otherwise a negative error value
605 * @note If this function is not called, default values are used.
608 HostHandler::setConstraint (uint32_t modelid, npuConstraint constraint)
610 Model *model = models_.find (modelid);
611 if (model == nullptr)
614 model->setConstraint (constraint);
620 * @brief find and return model instance
621 * @param[in] modelid model id
622 * @return model instance if found. otherwise nullptr
625 HostHandler::getModel (uint32_t modelid)
627 return models_.find (modelid);
630 /** @brief dummay callback for runSync. */
633 callbackSync (output_buffers *output) : output_(output), done_(false) {}
635 static void callback (output_buffers *output, uint64_t sequence, void *data) {
636 callbackSync *sync = static_cast<callbackSync *>(data);
637 sync->callback (output, sequence);
640 void callback (output_buffers *output, uint64_t sequence) {
641 if (output_ != nullptr) {
642 /** just copy internal variables of output buffers */
643 memcpy (output_, output, sizeof (output_buffers));
650 std::unique_lock<std::mutex> lock (m_);
651 cv_.wait (lock, [this]() { return done_; });
656 std::condition_variable cv_;
657 output_buffers *output_;
662 * @brief Execute inference. Wait (block) until the output is available.
663 * @param[in] modelid The model to be inferred.
664 * @param[in] input The input data to be inferred.
665 * @param[out] output The output result.
666 * @return @c 0 if no error. otherwise a negative error value
669 HostHandler::runSync (uint32_t modelid, const input_buffers *input,
670 output_buffers *output)
672 callbackSync sync (output);
673 int status = runAsync (modelid, input, callbackSync::callback,
674 static_cast <void*> (&sync), NPUASYNC_DROP_OLD, nullptr);
676 /** sync needs to wait callback */
683 * @brief Invoke NPU inference. Unblocking call.
684 * @param[in] modelid The model to be inferred.
685 * @param[in] input The input data to be inferred.
686 * @param[in] cb The output buffer handler.
687 * @param[in] cb_data The data given as a parameter to the runNPU_async call.
688 * @param[in] mode Configures how this operation works.
689 * @param[out] sequence The sequence number returned with runNPU_async.
690 * @return @c 0 if no error. otherwise a negative error value
693 HostHandler::runAsync (uint32_t modelid, const input_buffers *input,
694 npuOutputNotify cb, void *cb_data, npu_async_mode mode, uint64_t *sequence)
696 Model *model = nullptr;
698 if (device_->needModel()) {
699 model = getModel (modelid);
700 if (model == nullptr)
704 /* check the given model before running */
705 if (model != nullptr && !model->finalize ()) {
706 logerr (TAG, "Failed to finalize the model. Please see the log messages\n");
710 device_->setAsyncMode (mode);
711 return device_->run (NPUINPUT_HOST, model, input, cb, cb_data, sequence);
715 * @brief Let NPU accept input frames from its internal source continuously
716 * @param[in] modelid The model to be inferred.
717 * @param[in] opmode NPU has different opmode with auto-inputs. Choose one.
718 * @param[in] hw_dev The target device feeding input data
719 * @return @c 0 if no error. otherwise a negative error value
722 HostHandler::runInternal (uint32_t modelid, npu_input_opmode opmode,
725 Model *model = nullptr;
727 if (device_->needModel()) {
728 model = getModel (modelid);
729 if (model == nullptr)
733 /* check the given model before running */
734 if (model != nullptr && !model->finalize ()) {
735 logerr (TAG, "Failed to finalize the model. Please see the log messages\n");
739 return device_->runInternal (opmode, model, hw_dev);
743 * @brief Stop the request with the given id
744 * @param[in] dev The NPU device handle
745 * @param[in] id The request id
746 * @return @c 0 if no error. otherwise a negative error value
749 HostHandler::stopInternal (int id)
752 logerr (TAG, "Unable to stop this request with id (%d)\n", id);
756 const DriverAPI * api = device_->getDriverAPI ();
757 assert (api != nullptr);
759 return api->stop_target (id);
763 * @brief get number of available devices
764 * @param[in] type device type
765 * @return number of devices
768 HostHandler::getNumDevices (dev_type type)
770 return DriverAPI::getNumDevices (type);
774 * @brief get device instance
775 * @param[out] dev device instance
776 * @param[in] type device type
777 * @param[in] id device id
778 * @return 0 if no error. otherwise a negative errno
781 HostHandler::getDevice (npudev_h *dev, dev_type type, uint32_t id)
783 int num_devices = getNumDevices (type);
785 /** check the validity of device id */
786 if (!(num_devices > 0 && id < static_cast<uint32_t>(num_devices))) {
787 logerr (TAG, "Invalid arguments provided\n");
791 Device *device = Device::createInstance (type, id);
792 if (device == nullptr) {
793 logerr (TAG, "Failed to create a device with the given type\n");
798 /** This is just for backward-compatility; we don't guarantee its corresness */
805 * @brief allocate generic buffer (just for users)
806 * @param[out] buffer buffer instance
807 * @return 0 if no error. otherwise a negative errno
810 HostHandler::allocGenericBuffer (generic_buffer *buffer)
815 if (buffer->size == 0) {
816 logerr (TAG, "Invalid size\n");
820 if (buffer->size > UINT32_MAX) {
821 logerr (TAG, "Don't support such a large size");
825 switch (buffer->type) {
828 if (buffer->filepath == nullptr)
833 /* now, npu-engine always provides dmabuf-based allocation */
834 void *addr = nullptr;
835 int dmabuf = device_->allocMemory (buffer->size, &addr);
839 buffer->dmabuf = dmabuf;
851 * @brief deallocate generic buffer (just for users)
852 * @param[in] buffer buffer instance
853 * @return 0 if no error. otherwise a negative errno
856 HostHandler::deallocGenericBuffer (generic_buffer *buffer)
861 switch (buffer->type) {
863 /** always true cuz nothing to do */
866 return device_->deallocMemory (buffer->dmabuf, buffer->size, buffer->addr);
875 * @brief allocate multiple generic buffers (just for users)
876 * @param[out] buffers multi-buffer instance
877 * @return 0 if no error. otherwise a negative errno
880 HostHandler::allocGenericBuffer (generic_buffers *buffers)
885 if (buffers == NULL || buffers->num_buffers < 1)
888 for (idx = 0; idx < buffers->num_buffers; idx++) {
889 status = allocGenericBuffer (&buffers->bufs[idx]);
898 deallocGenericBuffer (&buffers->bufs[--idx]);
905 * @brief deallocate multiple generic buffers (just for users)
906 * @param[in] buffers multi-buffer instance
907 * @return 0 if no error. otherwise a negative errno
910 HostHandler::deallocGenericBuffer (generic_buffers *buffers)
912 if (buffers == NULL || buffers->num_buffers < 1)
915 for (uint32_t idx = 0; idx < buffers->num_buffers; idx++)
916 deallocGenericBuffer (&buffers->bufs[idx]);
917 buffers->num_buffers = 0;
923 * @brief get the current memory status
924 * @param[out] alloc_total The size of allocated memory until now
925 * @param[out] free_total The size of freed memory until now
926 * @return 0 if no error. otherwise a negatice error value
929 HostHandler::getMemoryStatus (size_t *alloc_total, size_t *free_total)
931 /** API is always set in initialize () */
932 const DriverAPI * api = device_->getDriverAPI ();
933 assert (api != nullptr);
935 return api->getMemoryStatus (alloc_total, free_total);
939 * @brief Get the current device status to be used
940 * @param[out] status the device status
941 * @param[out] num_requests the number of running requests (or pending)
942 * @return 0 if no error, otherwise a negative errno.
945 HostHandler::getDeviceStatus (npu_status *status, uint32_t *num_requests)
947 /** API is always set in initialize () */
948 const DriverAPI * api = device_->getDriverAPI ();
953 device_state_t state = api->isReady ();
954 if (state == device_state_t::STATE_READY) {
955 *num_requests = api->numRequests ();
956 if (*num_requests > 0)
968 /** implement methods of Device class */
970 /** @brief constructor of device */
971 Device::Device (dev_type type, int id, bool need_model)
972 : comm_ (CommPlugin::getCommPlugin()), type_ (type), id_ (id), need_model_ (true),
973 mode_ (NPUASYNC_WAIT), initialized_ (false), atomic_flag_ (ATOMIC_FLAG_INIT)
978 * @brief create device instance depending on device type and id
979 * @param[in] type device type
980 * @param[in] id device id
981 * @return device instance
984 Device::createInstance (dev_type type, int id)
986 Device *device = nullptr;
988 switch (type & DEVICETYPE_MASK) {
989 case DEVICETYPE_TRIV:
990 device = new TrinityVision (id);
992 case DEVICETYPE_TRIV2:
993 device = new TrinityVision2 (id);
995 case DEVICETYPE_TRIA:
996 device = new TrinityAsr (id);
997 device->setNeedModel (false);
1003 if (device != nullptr && device->init () != 0) {
1012 * @brief device initialization
1013 * @return 0 if no error, otherwise a negative errno
1014 * @note Init failures come from createDriverAPI() only.
1019 /** should be initilizaed only once */
1020 if (!atomic_flag_.test_and_set()) {
1021 /** create the corresponding driver API */
1022 api_ = DriverAPI::createDriverAPI (type_, id_);
1023 if (api_.get() == nullptr) {
1024 atomic_flag_.clear();
1025 logerr (TAG, "Failed to create driver API\n");
1029 handler_.reset (new HostHandler (this));
1030 scheduler_.reset (new Scheduler (api_.get()));
1031 mem_ = MemAllocator::createInstance (api_.get());
1033 initialized_ = true; /** c++11 does not provide test() of atomic flag */
1040 * @brief stop all requests from this device
1041 * @param[in] force_stop indicate the schedduler waits until to handle previous requests
1042 * @return 0 if no error, otherwise a negative errno
1045 Device::stop (bool force_stop)
1047 if (!initialized ()) {
1048 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1052 Request *req = new Request (NPUINPUT_STOP);
1053 req->setForceStop (force_stop);
1054 return scheduler_->submitRequest (req);
1058 * @brief allocate generic memory buffer
1059 * @param[in] size the size to allocate
1060 * @param[out] addr the mapped address
1061 * @return dmabuf fd if no error, otherwise a negative errno
1064 Device::allocMemory (size_t size, void **addr)
1066 if (!initialized ()) {
1067 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1071 if (size == 0 || addr == nullptr) {
1072 logerr (TAG, "Invalid arguments\n");
1076 return mem_->allocMemory (size, addr);
1080 * @brief deallocate generic memory buffer
1081 * @param[in] dmabuf_fd dmabuf file descriptor
1082 * @param[in] size buffer size
1083 * @param[in] addr mapped addr
1084 * @return 0 if no error, otherwise a negative errno
1087 Device::deallocMemory (int dmabuf_fd, size_t size, void * addr)
1089 if (!initialized ()) {
1090 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1094 if (dmabuf_fd < 0 || size == 0 || addr == nullptr) {
1095 logerr (TAG, "Invalid arguments\n");
1099 return mem_->deallocMemory (dmabuf_fd, size, addr);
1103 * @brief extract the buffer instance from input generic buffers
1104 * @param[in] meta the model metadata
1105 * @param[in] input the input generic buffers
1106 * @return the buffer instance
1109 TrinityVision::prepareInputBuffers (const Metadata *meta, const input_buffers *input)
1111 if (meta == nullptr || input == nullptr ||
1112 meta->getInputNum() != input->num_buffers) {
1113 logerr (TAG, "Invalid metadata info provided\n");
1118 const generic_buffer *first = &input->bufs[0];
1119 if (first->type == BUFFER_DMABUF) {
1120 buffer = mem_->allocBuffer (new HWmemExternal);
1121 if (buffer == nullptr)
1124 buffer->setDmabuf (first->dmabuf);
1125 buffer->setOffset (first->offset);
1126 buffer->setSize (meta->getBufferSize());
1128 buffer = mem_->allocBuffer (new HWmemDevice);
1129 if (buffer == nullptr)
1132 int status = buffer->alloc (meta->getBufferSize ());
1134 logerr (TAG, "Failed to allocate buffer: %d\n", status);
1140 int status = buffer->createTensors (meta);
1142 logerr (TAG, "Failed to create tensors: %d\n", status);
1151 * @brief implementation of TRIV's setModel ()
1152 * @param[in] model_buf the model generic buffer
1153 * @param[out] model the model instance
1154 * @return 0 if no error, otherwise a negative errno
1157 TrinityVision::setModel (const generic_buffer *model_buf, Model ** model_ptr)
1159 if (!initialized ()) {
1160 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1164 if (model_buf == nullptr || model_ptr == nullptr)
1167 Model *model = nullptr;
1168 HWmem * hwmem_prog = nullptr;
1169 HWmem * hwmem_weight = nullptr;
1172 /** In TRIV1, model data (including program/weight) should be contiguous */
1174 switch (model_buf->type) {
1177 model = mem_->allocModel (new HWmemDevice);
1178 if (model == nullptr) {
1179 logerr (TAG, "Failed to allocate model\n");
1183 status = model->alloc (model_buf->size);
1185 logerr (TAG, "Failed to allocate model: %d\n", status);
1189 /** extract the whole model data */
1190 status = comm_.extractGenericBuffer (model_buf, model->getData(), nullptr);
1192 logerr (TAG, "Failed to extract generic buffer: %d\n", status);
1200 status = model->setMetadata (model->getData());
1204 /** allocate program (optional; NOP) */
1205 if (model->getMetadata()->getProgramSize() > 0) {
1206 hwmem_prog = new HWmem (new HWmemChunk);
1207 model->setProgramData (hwmem_prog);
1209 hwmem_prog->setParent (model);
1210 hwmem_prog->setOffset (model->getMetadata()->getMetaSize());
1211 status = hwmem_prog->alloc (model->getMetadata()->getProgramSize());
1213 logerr (TAG, "Failed to allocate program\n");
1218 /** allocate weight (optional) */
1219 if (model->getMetadata()->getWeightSize() > 0) {
1220 hwmem_weight = new HWmem (new HWmemChunk);
1221 model->setWeightData (hwmem_weight);
1223 hwmem_weight->setParent (model);
1224 hwmem_weight->setOffset (model->getMetadata()->getMetaSize() +
1225 model->getMetadata()->getProgramSize());
1226 status = hwmem_weight->alloc (model->getMetadata()->getWeightSize());
1228 logerr (TAG, "Failed to allocate program\n");
1233 if (hwmem_prog != nullptr) {
1234 /** register this model to the driver */
1235 model_config_t config;
1236 config.dbuf_fd = hwmem_prog->getDmabuf ();
1237 config.program_size = hwmem_prog->getSize ();
1238 config.program_offset_addr = hwmem_prog->getOffset ();
1239 if (hwmem_weight != nullptr)
1240 config.weight_offset_addr = hwmem_weight->getOffset ();
1242 status = api_->registerModel (&config);
1246 model->setInternalID(config.id);
1258 * @brief implementation of TRIV's unsetModel ()
1259 * @param[in] model the model instance
1260 * @return 0 if no error, otherwise a negative errno
1263 TrinityVision::unsetModel (Model * model)
1265 if (!initialized ()) {
1266 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1270 if (model == nullptr) {
1271 logerr (TAG, "Invalid model instance\n");
1275 if (model->getMetadata()->getProgramSize() > 0)
1276 return api_->deregisterModel (model->getInternalID ());
1282 * @brief implementation of TRIV's run()
1283 * @param[in] opmode input opmode
1284 * @param[in] model the model instance
1285 * @param[in] input generic buffers of input data
1286 * @param[in] cb the output callback
1287 * @param[in] cb_data the output callback data
1288 * @param[out] sequence The sequence number returned with runNPU_async.
1291 TrinityVision::run (npu_input_opmode opmode, const Model *model,
1292 const input_buffers *input, npuOutputNotify cb, void *cb_data,
1295 if (!initialized ()) {
1296 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1300 if (opmode != NPUINPUT_HOST) {
1301 logerr (TAG, "TRIV supports only host inputservice\n");
1305 if (model == nullptr || input == nullptr) {
1306 logerr (TAG, "TRIV requires both model and input buffers\n");
1310 const_cast<Model *>(model)->updateDataInfo ();
1312 Buffer *buffer = prepareInputBuffers (model->getMetadata(), input);
1313 if (buffer == nullptr) {
1314 logerr (TAG, "Failed to extract buffer instance\n");
1318 if (!buffer->isExternal ()) {
1319 for (uint32_t idx = 0; idx < input->num_buffers; idx++) {
1320 auto func = std::bind (TrinityVision::manipulateData, model, idx, true,
1321 std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
1322 int status = comm_.extractGenericBuffer (&input->bufs[idx],
1323 buffer->getInputTensor(idx)->getData(), func);
1325 logerr (TAG, "Failed to feed input buffer: %d\n", status);
1331 /** this device uses CMA buffer */
1333 Request *req = new Request (opmode);
1334 req->setModel (model);
1335 req->setBuffer (buffer);
1338 req->setCallback (std::bind (&TrinityVision::callback, this, req, cb, cb_data));
1340 if (sequence != nullptr)
1341 *sequence = req->getID();
1343 return scheduler_->submitRequest (req);
1347 * @brief callback of TRIV2 request
1348 * @param[in] req the request instance
1349 * @param[in] cb callback for completion
1350 * @param[in] cb_data callback data
1351 * @note The callback invoke does not gurantee the request was successful
1352 * @todo Check the request failures
1355 TrinityVision::callback (Request *req, npuOutputNotify cb, void *cb_data)
1357 const Model *model = req->getModel ();
1358 Buffer *buffer = req->getBuffer ();
1359 output_buffers output = {
1360 .num_buffers = buffer->getOutputNum ()
1363 for (uint32_t idx = 0; idx < output.num_buffers; idx++) {
1364 uint32_t output_tensor_size = model->getOutputTensorSize (idx);
1366 if (buffer->isExternal ()) {
1367 output.bufs[idx].type = BUFFER_DMABUF;
1368 output.bufs[idx].size = output_tensor_size;
1369 output.bufs[idx].addr = buffer->getOutputTensor(idx)->getData();
1371 output.bufs[idx].type = BUFFER_MAPPED;
1372 output.bufs[idx].size = output_tensor_size;
1373 /** user needs to free this */
1374 output.bufs[idx].addr = malloc (output_tensor_size);
1376 auto func = std::bind (TrinityVision::manipulateData, model, idx, false,
1377 std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
1378 int status = comm_.insertGenericBuffer (buffer->getOutputTensor(idx)->getData(),
1379 &output.bufs[idx], func);
1381 logerr (TAG, "Failed to return output buffer: %d\n", status);
1386 cb (&output, req->getID(), cb_data);
1392 * @brief extract the segment table instance from input generic buffers
1393 * @param[in] model the model instance
1394 * @param[in] input the input generic buffers
1395 * @param[in] output the output generic buffers
1396 * @return the segment table instance
1399 TrinityVision2::prepareSegmentTable (const Model *model, const input_buffers *input,
1400 const output_buffers *output)
1402 const Metadata *meta = model->getMetadata ();
1403 if (meta == nullptr || (input != nullptr &&
1404 meta->getInputNum() != input->num_buffers)) {
1405 logerr (TAG, "Invalid metadata info provided\n");
1409 SegmentTable * segt = mem_->allocSegmentTable (new HWmemDevice);
1410 int status = segt->alloc ();
1412 logerr (TAG, "Failed to allocate segment table: %d\n", status);
1416 status = segt->createSegments (model, input, output);
1418 logerr (TAG, "Failed to create segments: %d\n", status);
1430 * @brief implementation of TRIV2's setModel ()
1431 * @param[in] model_buf the model generic buffer
1432 * @param[out] model the model instance
1433 * @return 0 if no error, otherwise a negative errno
1436 TrinityVision2::setModel (const generic_buffer *model_buf, Model ** model_ptr)
1438 if (!initialized ()) {
1439 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1443 if (model_buf == nullptr || model_ptr == nullptr)
1449 switch (model_buf->type) {
1452 model = mem_->allocModel (new HWmemDevice);
1453 if (model == nullptr) {
1454 logerr (TAG, "Failed to allocate model\n");
1458 status = model->alloc (NPUBIN_META_SIZE);
1460 logerr (TAG, "Failed to allocate model: %d\n", status);
1464 status = comm_.extractGenericBuffer (model_buf, model->getData(), nullptr,
1465 0, NPUBIN_META_SIZE);
1467 logerr (TAG, "Failed to extract generic buffer: %d\n", status);
1475 status = model->setMetadata (model->getData());
1479 /** allocate program (optional; NOP) */
1480 if (model->getMetadata()->getProgramSize() > 0) {
1481 HWmem * hwmem_prog = new HWmem (new HWmemDevice);
1482 hwmem_prog->setDriverAPI (api_.get());
1484 model->setProgramData (hwmem_prog);
1486 status = hwmem_prog->alloc (model->getMetadata()->getProgramSize());
1488 logerr (TAG, "Failed to allocate program\n");
1492 status = comm_.extractGenericBuffer (model_buf, hwmem_prog->getData(), nullptr,
1493 model->getMetadata()->getMetaSize(),
1494 model->getMetadata()->getProgramSize());
1496 logerr (TAG, "Failed to extract generic buffer: %d\n", status);
1500 /** register this model to the driver */
1501 model_config_t config;
1502 config.dbuf_fd = hwmem_prog->getDmabuf ();
1503 config.program_size = hwmem_prog->getSize ();
1504 config.program_offset_addr = 0;
1506 /** for metadata extra section */
1507 config.metadata_dbuf_fd = model->getDmabuf ();
1508 config.metadata_extra_addr = NPUBIN_META_SIZE;
1509 config.metadata_extra_size = model->getMetadata()->getMetaExtraSize ();
1511 status = api_->registerModel (&config);
1515 model->setInternalID(config.id);
1518 /** allocate weight (optional) */
1519 if (model->getMetadata()->getWeightSize() > 0) {
1520 HWmem * hwmem_weight = new HWmem (new HWmemDevice);
1521 hwmem_weight->setDriverAPI (api_.get());
1523 model->setWeightData (hwmem_weight);
1525 status = hwmem_weight->alloc (model->getMetadata()->getWeightSize());
1527 logerr (TAG, "Failed to allocate program\n");
1531 status = comm_.extractGenericBuffer (model_buf, hwmem_weight->getData(), nullptr,
1532 model->getMetadata()->getMetaSize() + model->getMetadata()->getProgramSize(),
1533 model->getMetadata()->getWeightSize());
1535 logerr (TAG, "Failed to extract generic buffer: %d\n", status);
1549 * @brief implementation of TRIV2's unsetModel ()
1550 * @param[in] model the model instance
1551 * @return 0 if no error, otherwise a negative errno
1554 TrinityVision2::unsetModel (Model * model)
1556 if (!initialized ()) {
1557 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1561 if (model == nullptr) {
1562 logerr (TAG, "Invalid model instance\n");
1566 if (model->getMetadata()->getProgramSize() > 0)
1567 return api_->deregisterModel (model->getInternalID ());
1572 /** @brief implementation of TRIV2's run() */
1574 TrinityVision2::run (npu_input_opmode opmode, const Model *model,
1575 const input_buffers *input, npuOutputNotify cb, void *cb_data,
1578 if (!initialized ()) {
1579 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1583 if (opmode != NPUINPUT_HOST)
1586 if (input == nullptr || input->num_buffers == 0 || model == nullptr)
1589 const_cast<Model *>(model)->updateDataInfo ();
1591 /** this device uses segment table */
1592 SegmentTable * segt = prepareSegmentTable (model, input);
1593 if (segt == nullptr) {
1594 logerr (TAG, "Failed to create segment table instance\n");
1598 /** extract input data */
1599 for (uint32_t idx = 0; idx < input->num_buffers; idx++) {
1600 if (!segt->getInputSegment(idx)->isExternal ()) {
1601 uint32_t seg_offset = segt->getInputSegmentOffset(idx);
1602 auto func = std::bind (TrinityVision2::manipulateData, model, idx, true,
1603 std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
1604 int status = comm_.extractGenericBuffer (
1606 segt->getInputSegment(idx)->getData() + seg_offset,
1609 logerr (TAG, "Failed to feed input segment: %d\n", status);
1615 Request *req = new Request (opmode);
1616 req->setModel (model);
1617 req->setSegmentTable (segt);
1618 req->setCallback (std::bind (&TrinityVision2::callback, this, req, cb, cb_data));
1621 *sequence = req->getID();
1623 return scheduler_->submitRequest (req);
1626 /** @brief implementation of TRIV2's runInternal() */
1628 TrinityVision2::runInternal (npu_input_opmode opmode, const Model *model,
1631 if (!initialized ()) {
1632 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1636 if (opmode != NPUINPUT_HW_RECURRING)
1639 /** this device uses segment table */
1640 SegmentTable * segt = prepareSegmentTable (model, nullptr, nullptr);
1641 if (segt == nullptr) {
1642 logerr (TAG, "Failed to create segment table instance\n");
1646 Request *req = new Request (opmode);
1647 req->setModel (model);
1648 req->setSegmentTable (segt);
1649 req->setHwDevice (hw_dev);
1651 return scheduler_->submitRequest (req);
1654 /** @brief callback of TRIV2 request */
1656 TrinityVision2::callback (Request *req, npuOutputNotify cb, void *cb_data)
1658 const Model *model = req->getModel ();
1659 SegmentTable *segt = req->getSegmentTable ();
1660 output_buffers output = {
1661 .num_buffers = segt->getNumOutputSegments ()
1664 for (uint32_t idx = 0; idx < output.num_buffers; idx++) {
1665 uint32_t output_tensor_size = model->getOutputTensorSize (idx);
1667 output.bufs[idx].type = BUFFER_MAPPED;
1668 output.bufs[idx].size = output_tensor_size;
1669 /** user needs to free this */
1670 output.bufs[idx].addr = calloc (1, output_tensor_size);
1672 #if defined(ENABLE_FPGA_WORKAROUND)
1674 segt->getOutputSegment(idx)->getDmabuf(),
1675 segt->getOutputSegmentOffset(idx),
1676 output.bufs[idx].addr,
1677 output.bufs[idx].size);
1679 auto func = std::bind (TrinityVision2::manipulateData, model, idx, false,
1680 std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
1681 int status = comm_.insertGenericBuffer (
1682 segt->getOutputSegment(idx)->getData() + segt->getOutputSegmentOffset(idx),
1683 &output.bufs[idx], func);
1686 logerr (TAG, "Failed to return output buffer: %d\n", status);
1691 cb (&output, req->getID(), cb_data);
1696 /** @brief implementation of TRIA's run(): WIP */
1698 TrinityAsr::run (npu_input_opmode opmode, const Model *model,
1699 const input_buffers *input, npuOutputNotify cb, void *cb_data,
1702 if (!initialized ()) {
1703 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1707 if (opmode != NPUINPUT_HOST)
1710 if (input == nullptr || input->num_buffers != 1)
1715 /** ASR does not require model and support only a single tensor */
1716 const generic_buffer *first_buf = &input->bufs[0];
1717 if (first_buf->type == BUFFER_DMABUF) {
1718 buffer = mem_->allocBuffer (new HWmemExternal);
1719 if (buffer == nullptr)
1722 buffer->setDmabuf (first_buf->dmabuf);
1723 buffer->setOffset (first_buf->offset);
1724 buffer->setSize (first_buf->size);
1726 buffer = mem_->allocBuffer (new HWmemDevice);
1727 if (buffer == nullptr)
1730 status = buffer->alloc (first_buf->size);
1737 status = buffer->createTensors ();
1739 logerr (TAG, "Failed to create tensors: %d\n", status);
1744 if (!buffer->isExternal ()) {
1745 status = comm_.extractGenericBuffer (first_buf,
1746 buffer->getInputTensor(0)->getData(), nullptr);
1751 Request *req = new Request (opmode);
1752 req->setBuffer (buffer);
1753 req->setCallback (std::bind (&TrinityAsr::callback, this, req, cb, cb_data));
1756 *sequence = req->getID();
1758 return scheduler_->submitRequest (req);
1761 /** @brief callback of TRIA request: WIP */
1763 TrinityAsr::callback (Request *req, npuOutputNotify cb, void *cb_data)
1765 Buffer *buffer = req->getBuffer ();
1766 output_buffers output = {
1770 /** TODO: finalize this impl. when the ASR's working scenario is determined */
1771 cb (&output, req->getID(), cb_data);
1776 /** Implement data manipulation (each device may have different impl.) */
1781 * @brief perform data manipulation
1782 * @param[in] model model instance
1783 * @param[in] idx tensor index
1784 * @param[in] is_input indicate it's input manipulation
1785 * @param[out] dst destination buffer
1786 * @param[in] src source buffer (feature map)
1787 * @param[in] size size to be copied
1788 * @return size of memory copy if no error, otherwise zero
1790 * @note the input data format should be NHWC
1791 * @detail rules for the memory address of activations in NPU HW.
1792 * (https://code.sec.samsung.net/confluence/pages/viewpage.action?pageId=146491864)
1794 * 1) Special case (depth == 3)
1795 * - addr(x,y,z) = addr(0,0,0) + (z) + 3 * (x + width * y)
1798 * - addr(x,y,z) = addr(0,0,0) + (z % MPA_L) + MPA_L * (x + width * (y + height * (z / MPA_L)))
1800 * Thus, if depth is not a multiple of MPA_L (i.e., 64), zero padding is required
1803 TrinityVision::manipulateData (const Model *model, uint32_t idx, bool is_input,
1804 void *dst, void *src, size_t size)
1806 const Metadata *meta = model->getMetadata();
1807 DataConverter converter (is_input);
1809 converter.setData (src, dst, size);
1812 const tensor_data_info* info = model->getInputDataInfo (idx);
1813 if (info == nullptr)
1816 converter.setDataLayout (info->layout, DATA_LAYOUT_SRNPU);
1817 converter.setDataType (info->type, DATA_TYPE_SRNPU);
1818 converter.setDataDims (meta->getInputDims (idx));
1819 converter.setQuantZero (meta->getInputQuantZero (idx));
1820 converter.setQuantScale (meta->getInputQuantScale (idx));
1822 const tensor_data_info* info = model->getOutputDataInfo (idx);
1823 if (info == nullptr)
1826 converter.setDataLayout (DATA_LAYOUT_SRNPU, info->layout);
1827 converter.setDataType (DATA_TYPE_SRNPU, info->type);
1828 converter.setDataDims (meta->getOutputDims (idx));
1829 converter.setQuantZero (meta->getOutputQuantZero (idx));
1830 converter.setQuantScale (meta->getOutputQuantScale (idx));
1833 return converter.perform ();
1837 * @brief perform data manipulation
1838 * @param[in] model model instance
1839 * @param[in] idx tensor index
1840 * @param[in] is_input indicate it's input manipulation
1841 * @param[out] dst destination buffer
1842 * @param[in] src source buffer (feature map)
1843 * @param[in] size size to be copied
1844 * @return size of memory copy if no error, otherwise zero
1846 * @note the input data format should be NHWC
1848 * @detail Feature map data in TRIV2, (x, y, z) = (width, height, depth)
1850 * 1) Image input (depth == 1 or depth == 3)
1851 * Addr(x,y,z) = Addr(0,0,0) + z + depth * x + ymod * y
1854 * Addr(x,y,z) = Addr(0,0,0) + (z % 64) + (64 * x) + ymod * y + zmod * (z / 64)
1857 TrinityVision2::manipulateData (const Model *model, uint32_t idx, bool is_input,
1858 void *dst, void *src, size_t size)
1860 const Metadata *meta = model->getMetadata();
1861 DataConverter converter (is_input);
1863 converter.setData (src, dst, size);
1866 const tensor_data_info* info = model->getInputDataInfo (idx);
1867 if (info == nullptr)
1870 converter.setDataLayout (info->layout, DATA_LAYOUT_TRIV2);
1871 converter.setDataType (info->type, meta->getInputQuantType (idx));
1872 converter.setDataDims (meta->getInputDims (idx));
1873 converter.setQuantZero (meta->getInputQuantZero (idx));
1874 converter.setQuantScale (meta->getInputQuantScale (idx));
1876 const tensor_data_info* info = model->getOutputDataInfo (idx);
1877 if (info == nullptr)
1880 converter.setDataLayout (DATA_LAYOUT_TRIV2, info->layout);
1881 converter.setDataType (meta->getOutputQuantType (idx), info->type);
1882 converter.setDataDims (meta->getOutputDims (idx));
1883 converter.setQuantZero (meta->getOutputQuantZero (idx));
1884 converter.setQuantScale (meta->getOutputQuantScale (idx));
1887 return converter.perform ();
1893 TrinityVision::manipulateData (const Model *model, uint32_t idx, bool is_input,
1894 void *dst, void *src, size_t size)
1896 memcpy (dst, src, size);
1901 TrinityVision2::manipulateData (const Model *model, uint32_t idx, bool is_input,
1902 void *dst, void *src, size_t size)
1904 memcpy (dst, src, size);
1910 /** other device types don't have data manip impl. yet */
1913 TrinityAsr::manipulateData (const Model *model, uint32_t idx, bool is_input,
1914 void *dst, void *src, size_t size)
1916 memcpy (dst, src, size);