3 * Copyright (C) 2020 Samsung Electronics
4 * Copyright (C) 2020 Dongju Chae <dongju.chae@samsung.com>
7 * @file ne-host-handler.cc
9 * @brief Implementation of APIs to access NPU from Host
10 * @see https://code.sec.samsung.net/confluence/display/ODLC/2020+Overall+Software+Stack
11 * @author Dongju Chae <dongju.chae@samsung.com>
12 * @bug No known bugs except for NYI items
15 #include "ne-handler.h"
18 #include <libnpuhost.h>
19 #include <npubinfmt.h>
20 #include <NPUdrvAPI.h>
21 #include <CommPlugin.h>
26 #include <condition_variable>
33 #define INIT_HOST_HANDLER(handler, dev) \
34 Device *tdev = static_cast <Device *> (dev); \
35 if (tdev == nullptr) return -EINVAL; \
36 HostHandler *handler = tdev->getHostHandler (); \
37 if (handler == nullptr) return -EINVAL;
39 /** just for backward-compatability */
40 npudev_h HostHandler::latest_dev_ = nullptr;
42 /** implement libnpuhost APIs */
45 * @brief Returns the number of available NPU devices.
46 * @return @c The number of NPU devices.
47 * @retval 0 if no NPU devices available. if positive (number of NPUs) if NPU devices available. otherwise, a negative error value.
48 * @note the caller should call putNPUdevice() to release the device handle
50 int getnumNPUdeviceByType (dev_type type)
52 return HostHandler::getNumDevices (type);
56 * @brief Returns the handle of the chosen NPU devices.
57 * @param[out] dev The NPU device handle
58 * @param[in] id The NPU id to get the handle. 0 <= id < getnumNPUdeviceByType().
59 * @return @c 0 if no error. otherwise a negative error value
60 * @note the caller should call putNPUdevice() to release the device handle
62 int getNPUdeviceByType (npudev_h *dev, dev_type type, uint32_t id)
64 return HostHandler::getDevice (dev, type, id);
68 * @brief release the NPU device instance obtained by getDevice ()
69 * @param[in] dev the NPU device handle
71 void putNPUdevice (npudev_h dev)
74 delete static_cast<Device *> (dev);
78 * @brief Get the profile information from NPU
79 * @param[in] dev The NPU device handle
80 * @param[in] run_id The identifier for each inference
81 * @param[out] profile The profile instance
82 * @return 0 if no error, otherwise a negative errno.
84 int getNPU_profile (npudev_h dev, int run_id, npu_profile *profile)
86 INIT_HOST_HANDLER (host_handler, dev);
88 return host_handler->getProfile (run_id, profile);
92 * @brief Free the profile instance obtained by getNPU_profile().
93 * @param[in] profile The profile instance
95 void putNPU_profile (npu_profile *profile)
97 if (profile != nullptr)
102 * @brief Send the NN model to NPU.
103 * @param[in] dev The NPU device handle
104 * @param[in] modelfile The filepath to the compiled NPU NN model in any buffer_type
105 * @param[out] modelid The modelid allocated for this instance of NN model.
106 * @return @c 0 if no error. otherwise a negative error value
108 * @detail For ASR devices, which do not accept models, but have models
109 * embedded in devices, you do not need to call register and
110 * register calls for ASR are ignored.
112 * @todo Add a variation: in-memory model register.
114 int registerNPUmodel (npudev_h dev, generic_buffer *modelfile, uint32_t *modelid)
116 INIT_HOST_HANDLER (host_handler, dev);
118 return host_handler->registerModel (modelfile, modelid);
122 * @brief Remove the NN model from NPU
123 * @param[in] dev The NPU device handle
124 * @param[in] modelid The model to be removed from the NPU.
125 * @return @c 0 if no error. otherwise a negative error value
126 * @detail This may incur some latency with memory compatcion.
128 int unregisterNPUmodel(npudev_h dev, uint32_t modelid)
130 INIT_HOST_HANDLER (host_handler, dev);
132 return host_handler->unregisterModel (modelid);
136 * @brief Remove all NN models from NPU
137 * @param[in] dev The NPU device handle
138 * @return @c 0 if no error. otherwise a negative error value
140 int unregisterNPUmodel_all(npudev_h dev)
142 INIT_HOST_HANDLER (host_handler, dev);
144 return host_handler->unregisterModels ();
148 * @brief [OPTIONAL] Set the data layout for input/output tensors
149 * @param[in] dev The NPU device handle
150 * @param[in] modelid The ID of model whose layouts are set
151 * @param[in] info_in the layout/type info for input tensors
152 * @param[in] info_out the layout/type info for output tensors
153 * @return @c 0 if no error. otherwise a negative error value
154 * @note if this function is not called, default layout/type will be used.
156 int setNPU_dataInfo(npudev_h dev, uint32_t modelid,
157 tensors_data_info *info_in, tensors_data_info *info_out)
159 INIT_HOST_HANDLER (host_handler, dev);
161 return host_handler->setDataInfo (modelid, info_in, info_out);
165 * @brief [OPTIONAL] Set the inference constraint for next NPU inferences
166 * @param[in] dev The NPU device handle
167 * @param[in] modelid The target model id
168 * @param[in] constraint inference constraint (e.g., timeout, priority)
169 * @return @c 0 if no error. otherwise a negative error value
170 * @note If this function is not called, default values are used.
172 int setNPU_constraint(npudev_h dev, uint32_t modelid, npuConstraint constraint)
174 INIT_HOST_HANDLER (host_handler, dev);
176 return host_handler->setConstraint (modelid, constraint);
180 * @brief Execute inference. Wait (block) until the output is available.
181 * @param[in] dev The NPU device handle
182 * @param[in] modelid The model to be inferred.
183 * @param[in] input The input data to be inferred.
184 * @param[out] output The output result. The caller MUST allocate appropriately before calling this.
185 * @return @c 0 if no error. otherwise a negative error value
187 * @detail This is a syntactic sugar of runNPU_async().
188 * CAUTION: There is a memcpy for the output buffer.
190 int runNPU_sync(npudev_h dev, uint32_t modelid, const input_buffers *input,
191 output_buffers *output)
193 INIT_HOST_HANDLER (host_handler, dev);
195 return host_handler->runSync (modelid, input, output);
199 * @brief Invoke NPU inference. Unblocking call.
200 * @param[in] dev The NPU device handle
201 * @param[in] modelid The model to be inferred.
202 * @param[in] input The input data to be inferred.
203 * @param[in] cb The output buffer handler.
204 * @param[out] sequence The sequence number returned with runNPU_async.
205 * @param[in] data The data given as a parameter to the runNPU_async call.
206 * @param[in] mode Configures how this operation works.
207 * @return @c 0 if no error. otherwise a negative error value
209 int runNPU_async(npudev_h dev, uint32_t modelid, const input_buffers *input,
210 npuOutputNotify cb, uint64_t *sequence, void *data,
213 INIT_HOST_HANDLER (host_handler, dev);
215 return host_handler->runAsync (modelid, input, cb, data, mode, sequence);
219 * @brief Let NPU accept input frames from its internal source continuously
220 * @param[in] dev The NPU device handle
221 * @param[in] modelid The model to be inferred.
222 * @param[in] opmode NPU has different opmode with auto-inputs. Choose one.
223 * @param[in] hw_dev The target device feeding input data
224 * @return @c 0 if no error. otherwise a negative error value
226 int runNPU_internalInput(npudev_h dev, uint32_t modelid, npu_input_opmode opmode,
229 INIT_HOST_HANDLER (host_handler, dev);
231 return host_handler->runInternal (modelid, opmode, hw_dev);
235 * @brief Stop the request with the given id
236 * @param[in] dev The NPU device handle
237 * @param[in] id The request id
238 * @return @c 0 if no error. otherwise a negative error value
240 int stopNPU_internalInput(npudev_h dev, int id)
242 INIT_HOST_HANDLER (host_handler, dev);
244 return host_handler->stopInternal (id);
248 * @brief Allocate a generic buffer with the requested buffer type.
249 * @param[in] dev The NPU device handle
250 * @param[in/out] Buffer the buffer pointer where memory is allocated.
251 * @return 0 if no error, otherwise a negative errno.
253 int allocNPU_genericBuffer (npudev_h dev, generic_buffer * buffer)
255 INIT_HOST_HANDLER (host_handler, dev);
257 return host_handler->allocGenericBuffer (buffer);
261 * @brief Free the generic buffer and remove the address mapping
262 * @param[in] dev The NPU device handle
263 * @param[in] buffer the model buffer
264 * @return 0 if no error, otherwise a negative errno.
266 int cleanNPU_genericBuffer (npudev_h dev, generic_buffer * buffer)
268 INIT_HOST_HANDLER (host_handler, dev);
270 return host_handler->deallocGenericBuffer (buffer);
274 * @brief Allocate generic buffers, which have multiple instances of generic_buffer
275 * @param[in] dev The NPU device handle
276 * @param[in/out] buffers generic buffers.
277 * @return 0 if no error, otherwise a negative errno.
278 * @note it reuses allocGenericBuffer().
280 int allocNPU_genericBuffers (npudev_h dev, generic_buffers * buffers)
282 INIT_HOST_HANDLER (host_handler, dev);
284 return host_handler->allocGenericBuffer (buffers);
288 * @brief Free generic buffers allocated by allocGenericBuffers().
289 * @param[in] dev The NPU device handle
290 * @param[in/out] buffers generic buffers.
291 * @note it reuses cleanGenericbuffer().
292 * @return 0 if no error, otherwise a negative errno.
294 int cleanNPU_genericBuffers (npudev_h dev, generic_buffers * buffers)
296 INIT_HOST_HANDLER (host_handler, dev);
298 return host_handler->deallocGenericBuffer (buffers);
302 * @brief alias of allocNPU_genericBuffer for model buffer
304 int allocNPU_modelBuffer (npudev_h dev, generic_buffer * model)
306 return allocNPU_genericBuffer (dev, model);
310 * @brief alias of cleanNPU_genericBuffer for model buffer
312 int cleanNPU_modelBuffer (npudev_h dev, generic_buffer * model)
314 return cleanNPU_genericBuffer (dev, model);
318 * @brief alias of allocNPU_genericBuffer for input buffer
320 int allocNPU_inputBuffer (npudev_h dev, generic_buffer * input)
322 return allocNPU_genericBuffer (dev, input);
326 * @brief alias of cleanNPU_genericBuffer for input buffer
328 int cleanNPU_inputBuffer (npudev_h dev, generic_buffer * input)
330 return cleanNPU_genericBuffer (dev, input);
334 * @brief alias of allocNPU_genericBuffers for input buffers
336 int allocNPU_inputBuffers (npudev_h dev, input_buffers * input)
338 return allocNPU_genericBuffers (dev, input);
342 * @brief alias of cleanNPU_genericBuffers for input buffers
344 int cleanNPU_inputBuffers (npudev_h dev, input_buffers * input)
346 return cleanNPU_genericBuffers (dev, input);
350 * @brief get the current memory status for the given device
351 * @param[in] dev The NPU device handle
352 * @param[out] alloc_total The size of allocated memory until now
353 * @param[out] free_total The size of freed memory until now
354 * @return @c 0 if no error. otherwise a negatice error value
356 int getNPU_memoryStatus(npudev_h dev, size_t *alloc_total, size_t *free_total)
358 INIT_HOST_HANDLER (host_handler, dev);
360 return host_handler->getMemoryStatus (alloc_total, free_total);
364 * @brief Get the current device status to be used
365 * @param[in] dev The NPU device handle
366 * @param[out] status the device status
367 * @param[out] num_requests the number of running requests (or pending)
368 * @return 0 if no error, otherwise a negative errno.
370 int getNPU_deviceStatus(npudev_h dev, npu_status *status, uint32_t *num_requests)
372 INIT_HOST_HANDLER (host_handler, dev);
374 return host_handler->getDeviceStatus (status, num_requests);
378 * @brief Get metadata for NPU model
379 * @param[in] model The path of model binary file
380 * @param[in] need_extra whether you want to extract the extra data in metadata
381 * @return the metadata structure to be filled if no error, otherwise nullptr
383 * @note For most npu-engine users, the extra data is not useful because it will be
384 * used for second-party users (e.g., compiler, simulator).
385 * Also, the caller needs to free the metadata.
387 * @note the caller needs to free the metadata
389 npubin_meta * getNPUmodel_metadata (const char *model, bool need_extra)
398 fp = fopen (model, "rb");
400 logerr (TAG, "Failed to open the model binary: %d\n", -errno);
404 meta = (npubin_meta *) malloc (NPUBIN_META_SIZE);
406 logerr (TAG, "Failed to allocate metadata\n");
410 ret = fread (meta, 1, NPUBIN_META_SIZE, fp);
411 if (ret != NPUBIN_META_SIZE) {
412 logerr (TAG, "Failed to read the metadata\n");
416 if (!CHECK_NPUBIN (meta->magiccode)) {
417 logerr (TAG, "Invalid metadata provided\n");
421 if (need_extra && NPUBIN_META_EXTRA (meta->magiccode) > 0) {
422 npubin_meta *new_meta;
424 new_meta = (npubin_meta *) malloc (NPUBIN_META_TOTAL_SIZE(meta->magiccode));
426 logerr (TAG, "Failed to allocate extra metadata\n");
430 memcpy (new_meta, meta, NPUBIN_META_TOTAL_SIZE(meta->magiccode));
433 ret = fread (new_meta->reserved_extra, 1, NPUBIN_META_EXTRA_SIZE (new_meta->magiccode), fp);
434 if (ret != NPUBIN_META_EXTRA_SIZE (new_meta->magiccode)) {
435 logerr (TAG, "Invalid extra metadata provided\n");
455 /** implement methods of HostHandler class */
457 /** @brief host handler constructor */
458 HostHandler::HostHandler (Device *device)
460 /* ignored as we don't use double buffering anymore, but for backward-compatibility */
461 async_mode_ (NPUASYNC_WAIT)
465 /** @brief host handler destructor */
466 HostHandler::~HostHandler ()
471 * @brief register model from generic buffer
472 * @param[in] model_buf model buffer
473 * @param[out] modelid model id
474 * @return 0 if no error. otherwise a negative errno
477 HostHandler::registerModel (generic_buffer *model_buf, uint32_t *modelid)
479 if (model_buf == nullptr || modelid == nullptr) {
480 logerr (TAG, "Invalid arguments given\n");
484 Model *model = nullptr;
485 int status = device_->setModel (model_buf, &model);
487 logerr (TAG, "Failed to set model: %d\n", status);
491 assert (model != nullptr);
493 status = models_.insert (model->getID(), model);
495 logerr (TAG, "Failed to insert model id\n");
500 *modelid = model->getID();
505 * @brief remove the registered model
506 * @param[in] modelid model id
507 * @return 0 if no error. otherwise a negative errno
510 HostHandler::unregisterModel (uint32_t modelid)
512 Model *model = models_.find (modelid);
513 if (model == nullptr)
516 int status = device_->unsetModel (model);
518 logerr (TAG, "Failed to unset model: %d\n", status);
522 return models_.remove (modelid);
526 * @brief remove all registered models
530 HostHandler::unregisterModels ()
532 std::function <bool (Model *)> functor =
533 [&] (Model *m) -> bool {
534 bool can_remove = true;
535 int status = device_->unsetModel (m);
537 logwarn (TAG, "Failed to unset model: %d\n", status);
543 models_.for_each (functor);
548 * @brief Get the profile information from NPU
549 * @param[in] run_id The identifier for each inference
550 * @param[out] profile The profile instance
551 * @return 0 if no error, otherwise a negative errno.
554 HostHandler::getProfile (int run_id, npu_profile *profile)
556 if (run_id < 0 || profile == nullptr) {
557 logerr (TAG, "Invalid parameter provided\n");
561 const DriverAPI * api = device_->getDriverAPI ();
562 assert (api != nullptr);
564 void *profile_buffer;
565 int status = api->getProfile (run_id, &profile_buffer);
567 // TODO: Perform parsing
574 * @brief Set the data layout for input/output tensors
575 * @param[in] modelid The ID of model whose layouts are set
576 * @param[in] in the layout/type info for input tensors
577 * @param[in] out the layout/type info for output tensors
578 * @return @c 0 if no error. otherwise a negative error value
579 * @note if this function is not called, default layout/type will be used.
582 HostHandler::setDataInfo (uint32_t modelid, tensors_data_info *in,
583 tensors_data_info *out)
585 Model *model = models_.find (modelid);
586 if (model == nullptr)
589 return model->setDataInfo (in, out);
593 * @brief Set the inference constraint for next NPU inferences
594 * @param[in] modelid The target model id
595 * @param[in] constraint inference constraint (e.g., timeout, priority)
596 * @return @c 0 if no error. otherwise a negative error value
597 * @note If this function is not called, default values are used.
600 HostHandler::setConstraint (uint32_t modelid, npuConstraint constraint)
602 Model *model = models_.find (modelid);
603 if (model == nullptr)
606 model->setConstraint (constraint);
612 * @brief find and return model instance
613 * @param[in] modelid model id
614 * @return model instance if found. otherwise nullptr
617 HostHandler::getModel (uint32_t modelid)
619 return models_.find (modelid);
622 /** @brief dummay callback for runSync. */
625 callbackSync (output_buffers *output) : output_(output), done_(false) {}
627 static void callback (output_buffers *output, uint64_t sequence, void *data) {
628 callbackSync *sync = static_cast<callbackSync *>(data);
629 sync->callback (output, sequence);
632 void callback (output_buffers *output, uint64_t sequence) {
633 if (output_ != nullptr) {
634 /** just copy internal variables of output buffers */
635 memcpy (output_, output, sizeof (output_buffers));
642 std::unique_lock<std::mutex> lock (m_);
643 cv_.wait (lock, [this]() { return done_; });
648 std::condition_variable cv_;
649 output_buffers *output_;
654 * @brief Execute inference. Wait (block) until the output is available.
655 * @param[in] modelid The model to be inferred.
656 * @param[in] input The input data to be inferred.
657 * @param[out] output The output result.
658 * @return @c 0 if no error. otherwise a negative error value
661 HostHandler::runSync (uint32_t modelid, const input_buffers *input,
662 output_buffers *output)
664 callbackSync sync (output);
665 int status = runAsync (modelid, input, callbackSync::callback,
666 static_cast <void*> (&sync), NPUASYNC_DROP_OLD, nullptr);
668 /** sync needs to wait callback */
675 * @brief Invoke NPU inference. Unblocking call.
676 * @param[in] modelid The model to be inferred.
677 * @param[in] input The input data to be inferred.
678 * @param[in] cb The output buffer handler.
679 * @param[in] cb_data The data given as a parameter to the runNPU_async call.
680 * @param[in] mode Configures how this operation works.
681 * @param[out] sequence The sequence number returned with runNPU_async.
682 * @return @c 0 if no error. otherwise a negative error value
685 HostHandler::runAsync (uint32_t modelid, const input_buffers *input,
686 npuOutputNotify cb, void *cb_data, npu_async_mode mode, uint64_t *sequence)
688 Model *model = nullptr;
690 if (device_->needModel()) {
691 model = getModel (modelid);
692 if (model == nullptr)
696 /* check the given model before running */
697 if (model != nullptr && !model->finalize ()) {
698 logerr (TAG, "Failed to finalize the model. Please see the log messages\n");
702 device_->setAsyncMode (mode);
703 return device_->run (NPUINPUT_HOST, model, input, cb, cb_data, sequence);
707 * @brief Let NPU accept input frames from its internal source continuously
708 * @param[in] modelid The model to be inferred.
709 * @param[in] opmode NPU has different opmode with auto-inputs. Choose one.
710 * @param[in] hw_dev The target device feeding input data
711 * @return @c 0 if no error. otherwise a negative error value
714 HostHandler::runInternal (uint32_t modelid, npu_input_opmode opmode,
717 Model *model = nullptr;
719 if (device_->needModel()) {
720 model = getModel (modelid);
721 if (model == nullptr)
725 /* check the given model before running */
726 if (model != nullptr && !model->finalize ()) {
727 logerr (TAG, "Failed to finalize the model. Please see the log messages\n");
731 return device_->runInternal (opmode, model, hw_dev);
735 * @brief Stop the request with the given id
736 * @param[in] dev The NPU device handle
737 * @param[in] id The request id
738 * @return @c 0 if no error. otherwise a negative error value
741 HostHandler::stopInternal (int id)
744 logerr (TAG, "Unable to stop this request with id (%d)\n", id);
748 const DriverAPI * api = device_->getDriverAPI ();
749 assert (api != nullptr);
751 return api->stop_target (id);
755 * @brief get number of available devices
756 * @param[in] type device type
757 * @return number of devices
760 HostHandler::getNumDevices (dev_type type)
762 return DriverAPI::getNumDevices (type);
766 * @brief get device instance
767 * @param[out] dev device instance
768 * @param[in] type device type
769 * @param[in] id device id
770 * @return 0 if no error. otherwise a negative errno
773 HostHandler::getDevice (npudev_h *dev, dev_type type, uint32_t id)
775 int num_devices = getNumDevices (type);
777 /** check the validity of device id */
778 if (!(num_devices > 0 && id < static_cast<uint32_t>(num_devices))) {
779 logerr (TAG, "Invalid arguments provided\n");
783 Device *device = Device::createInstance (type, id);
784 if (device == nullptr) {
785 logerr (TAG, "Failed to create a device with the given type\n");
790 /** This is just for backward-compatility; we don't guarantee its corresness */
797 * @brief allocate generic buffer (just for users)
798 * @param[out] buffer buffer instance
799 * @return 0 if no error. otherwise a negative errno
802 HostHandler::allocGenericBuffer (generic_buffer *buffer)
807 if (buffer->size == 0) {
808 logerr (TAG, "Invalid size\n");
812 if (buffer->size > UINT32_MAX) {
813 logerr (TAG, "Don't support such a large size");
817 switch (buffer->type) {
820 if (buffer->filepath == nullptr)
825 /* now, npu-engine always provides dmabuf-based allocation */
826 void *addr = nullptr;
827 int dmabuf = device_->allocMemory (buffer->size, &addr);
831 buffer->dmabuf = dmabuf;
843 * @brief deallocate generic buffer (just for users)
844 * @param[in] buffer buffer instance
845 * @return 0 if no error. otherwise a negative errno
848 HostHandler::deallocGenericBuffer (generic_buffer *buffer)
853 switch (buffer->type) {
855 /** always true cuz nothing to do */
858 return device_->deallocMemory (buffer->dmabuf, buffer->size, buffer->addr);
867 * @brief allocate multiple generic buffers (just for users)
868 * @param[out] buffers multi-buffer instance
869 * @return 0 if no error. otherwise a negative errno
872 HostHandler::allocGenericBuffer (generic_buffers *buffers)
877 if (buffers == NULL || buffers->num_buffers < 1)
880 for (idx = 0; idx < buffers->num_buffers; idx++) {
881 status = allocGenericBuffer (&buffers->bufs[idx]);
890 deallocGenericBuffer (&buffers->bufs[--idx]);
897 * @brief deallocate multiple generic buffers (just for users)
898 * @param[in] buffers multi-buffer instance
899 * @return 0 if no error. otherwise a negative errno
902 HostHandler::deallocGenericBuffer (generic_buffers *buffers)
904 if (buffers == NULL || buffers->num_buffers < 1)
907 for (uint32_t idx = 0; idx < buffers->num_buffers; idx++)
908 deallocGenericBuffer (&buffers->bufs[idx]);
909 buffers->num_buffers = 0;
915 * @brief get the current memory status
916 * @param[out] alloc_total The size of allocated memory until now
917 * @param[out] free_total The size of freed memory until now
918 * @return 0 if no error. otherwise a negatice error value
921 HostHandler::getMemoryStatus (size_t *alloc_total, size_t *free_total)
923 /** API is always set in initialize () */
924 const DriverAPI * api = device_->getDriverAPI ();
925 assert (api != nullptr);
927 return api->getMemoryStatus (alloc_total, free_total);
931 * @brief Get the current device status to be used
932 * @param[out] status the device status
933 * @param[out] num_requests the number of running requests (or pending)
934 * @return 0 if no error, otherwise a negative errno.
937 HostHandler::getDeviceStatus (npu_status *status, uint32_t *num_requests)
939 /** API is always set in initialize () */
940 const DriverAPI * api = device_->getDriverAPI ();
945 device_state_t state = api->isReady ();
946 if (state == device_state_t::STATE_READY) {
947 *num_requests = api->numRequests ();
948 if (*num_requests > 0)
960 /** implement methods of Device class */
962 /** @brief constructor of device */
963 Device::Device (dev_type type, int id, bool need_model)
964 : comm_ (CommPlugin::getCommPlugin()), type_ (type), id_ (id), need_model_ (true),
965 mode_ (NPUASYNC_WAIT), initialized_ (false), atomic_flag_ (ATOMIC_FLAG_INIT)
970 * @brief create device instance depending on device type and id
971 * @param[in] type device type
972 * @param[in] id device id
973 * @return device instance
976 Device::createInstance (dev_type type, int id)
978 Device *device = nullptr;
980 switch (type & DEVICETYPE_MASK) {
981 case DEVICETYPE_TRIV:
982 device = new TrinityVision (id);
984 case DEVICETYPE_TRIV2:
985 device = new TrinityVision2 (id);
987 case DEVICETYPE_TRIA:
988 device = new TrinityAsr (id);
989 device->setNeedModel (false);
995 if (device != nullptr && device->init () != 0) {
1004 * @brief device initialization
1005 * @return 0 if no error, otherwise a negative errno
1006 * @note Init failures come from createDriverAPI() only.
1011 /** should be initilizaed only once */
1012 if (!atomic_flag_.test_and_set()) {
1013 /** create the corresponding driver API */
1014 api_ = DriverAPI::createDriverAPI (type_, id_);
1015 if (api_.get() == nullptr) {
1016 atomic_flag_.clear();
1017 logerr (TAG, "Failed to create driver API\n");
1021 handler_.reset (new HostHandler (this));
1022 scheduler_.reset (new Scheduler (api_.get()));
1023 mem_ = MemAllocator::createInstance (api_.get());
1025 initialized_ = true; /** c++11 does not provide test() of atomic flag */
1032 * @brief stop all requests from this device
1033 * @param[in] force_stop indicate the schedduler waits until to handle previous requests
1034 * @return 0 if no error, otherwise a negative errno
1037 Device::stop (bool force_stop)
1039 if (!initialized ()) {
1040 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1044 Request *req = new Request (NPUINPUT_STOP);
1045 req->setForceStop (force_stop);
1046 return scheduler_->submitRequest (req);
1050 * @brief allocate generic memory buffer
1051 * @param[in] size the size to allocate
1052 * @param[out] addr the mapped address
1053 * @return dmabuf fd if no error, otherwise a negative errno
1056 Device::allocMemory (size_t size, void **addr)
1058 if (!initialized ()) {
1059 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1063 if (size == 0 || addr == nullptr) {
1064 logerr (TAG, "Invalid arguments\n");
1068 return mem_->allocMemory (size, addr);
1072 * @brief deallocate generic memory buffer
1073 * @param[in] dmabuf_fd dmabuf file descriptor
1074 * @param[in] size buffer size
1075 * @param[in] addr mapped addr
1076 * @return 0 if no error, otherwise a negative errno
1079 Device::deallocMemory (int dmabuf_fd, size_t size, void * addr)
1081 if (!initialized ()) {
1082 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1086 if (dmabuf_fd < 0 || size == 0 || addr == nullptr) {
1087 logerr (TAG, "Invalid arguments\n");
1091 return mem_->deallocMemory (dmabuf_fd, size, addr);
1095 * @brief extract the buffer instance from input generic buffers
1096 * @param[in] meta the model metadata
1097 * @param[in] input the input generic buffers
1098 * @return the buffer instance
1101 TrinityVision::prepareInputBuffers (const Metadata *meta, const input_buffers *input)
1103 if (meta == nullptr || input == nullptr ||
1104 meta->getInputNum() != input->num_buffers) {
1105 logerr (TAG, "Invalid metadata info provided\n");
1110 const generic_buffer *first = &input->bufs[0];
1111 if (first->type == BUFFER_DMABUF) {
1112 buffer = mem_->allocBuffer (new HWmemExternal);
1113 if (buffer == nullptr)
1116 buffer->setDmabuf (first->dmabuf);
1117 buffer->setOffset (first->offset);
1118 buffer->setSize (meta->getBufferSize());
1120 buffer = mem_->allocBuffer (new HWmemDevice);
1121 if (buffer == nullptr)
1124 int status = buffer->alloc (meta->getBufferSize ());
1126 logerr (TAG, "Failed to allocate buffer: %d\n", status);
1132 int status = buffer->createTensors (meta);
1134 logerr (TAG, "Failed to create tensors: %d\n", status);
1143 * @brief implementation of TRIV's setModel ()
1144 * @param[in] model_buf the model generic buffer
1145 * @param[out] model the model instance
1146 * @return 0 if no error, otherwise a negative errno
1149 TrinityVision::setModel (const generic_buffer *model_buf, Model ** model_ptr)
1151 if (!initialized ()) {
1152 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1156 if (model_buf == nullptr || model_ptr == nullptr)
1159 Model *model = nullptr;
1160 HWmem * hwmem_prog = nullptr;
1161 HWmem * hwmem_weight = nullptr;
1164 /** In TRIV1, model data (including program/weight) should be contiguous */
1166 switch (model_buf->type) {
1169 model = mem_->allocModel (new HWmemDevice);
1170 if (model == nullptr) {
1171 logerr (TAG, "Failed to allocate model\n");
1175 status = model->alloc (model_buf->size);
1177 logerr (TAG, "Failed to allocate model: %d\n", status);
1181 /** extract the whole model data */
1182 status = comm_.extractGenericBuffer (model_buf, model->getData(), nullptr);
1184 logerr (TAG, "Failed to extract generic buffer: %d\n", status);
1192 status = model->setMetadata (model->getData());
1196 /** allocate program (optional; NOP) */
1197 if (model->getMetadata()->getProgramSize() > 0) {
1198 hwmem_prog = new HWmem (new HWmemChunk);
1199 model->setProgramData (hwmem_prog);
1201 hwmem_prog->setParent (model);
1202 hwmem_prog->setOffset (model->getMetadata()->getMetaSize());
1203 status = hwmem_prog->alloc (model->getMetadata()->getProgramSize());
1205 logerr (TAG, "Failed to allocate program\n");
1210 /** allocate weight (optional) */
1211 if (model->getMetadata()->getWeightSize() > 0) {
1212 hwmem_weight = new HWmem (new HWmemChunk);
1213 model->setWeightData (hwmem_weight);
1215 hwmem_weight->setParent (model);
1216 hwmem_weight->setOffset (model->getMetadata()->getMetaSize() +
1217 model->getMetadata()->getProgramSize());
1218 status = hwmem_weight->alloc (model->getMetadata()->getWeightSize());
1220 logerr (TAG, "Failed to allocate program\n");
1225 if (hwmem_prog != nullptr) {
1226 /** register this model to the driver */
1227 model_config_t config;
1228 config.dbuf_fd = hwmem_prog->getDmabuf ();
1229 config.program_size = hwmem_prog->getSize ();
1230 config.program_offset_addr = hwmem_prog->getOffset ();
1231 if (hwmem_weight != nullptr)
1232 config.weight_offset_addr = hwmem_weight->getOffset ();
1234 status = api_->registerModel (&config);
1238 model->setInternalID(config.id);
1250 * @brief implementation of TRIV's unsetModel ()
1251 * @param[in] model the model instance
1252 * @return 0 if no error, otherwise a negative errno
1255 TrinityVision::unsetModel (Model * model)
1257 if (!initialized ()) {
1258 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1262 if (model == nullptr) {
1263 logerr (TAG, "Invalid model instance\n");
1267 if (model->getMetadata()->getProgramSize() > 0)
1268 return api_->deregisterModel (model->getInternalID ());
1274 * @brief implementation of TRIV's run()
1275 * @param[in] opmode input opmode
1276 * @param[in] model the model instance
1277 * @param[in] input generic buffers of input data
1278 * @param[in] cb the output callback
1279 * @param[in] cb_data the output callback data
1280 * @param[out] sequence The sequence number returned with runNPU_async.
1283 TrinityVision::run (npu_input_opmode opmode, const Model *model,
1284 const input_buffers *input, npuOutputNotify cb, void *cb_data,
1287 if (!initialized ()) {
1288 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1292 if (opmode != NPUINPUT_HOST) {
1293 logerr (TAG, "TRIV supports only host inputservice\n");
1297 if (model == nullptr || input == nullptr) {
1298 logerr (TAG, "TRIV requires both model and input buffers\n");
1302 const_cast<Model *>(model)->updateDataInfo ();
1304 Buffer *buffer = prepareInputBuffers (model->getMetadata(), input);
1305 if (buffer == nullptr) {
1306 logerr (TAG, "Failed to extract buffer instance\n");
1310 if (!buffer->isExternal ()) {
1311 for (uint32_t idx = 0; idx < input->num_buffers; idx++) {
1312 auto func = std::bind (TrinityVision::manipulateData, model, idx, true,
1313 std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
1314 int status = comm_.extractGenericBuffer (&input->bufs[idx],
1315 buffer->getInputTensor(idx)->getData(), func);
1317 logerr (TAG, "Failed to feed input buffer: %d\n", status);
1323 /** this device uses CMA buffer */
1325 Request *req = new Request (opmode);
1326 req->setModel (model);
1327 req->setBuffer (buffer);
1330 req->setCallback (std::bind (&TrinityVision::callback, this, req, cb, cb_data));
1332 if (sequence != nullptr)
1333 *sequence = req->getID();
1335 return scheduler_->submitRequest (req);
1339 * @brief callback of TRIV2 request
1340 * @param[in] req the request instance
1341 * @param[in] cb callback for completion
1342 * @param[in] cb_data callback data
1343 * @note The callback invoke does not gurantee the request was successful
1344 * @todo Check the request failures
1347 TrinityVision::callback (Request *req, npuOutputNotify cb, void *cb_data)
1349 const Model *model = req->getModel ();
1350 Buffer *buffer = req->getBuffer ();
1351 output_buffers output = {
1352 .num_buffers = buffer->getOutputNum ()
1355 for (uint32_t idx = 0; idx < output.num_buffers; idx++) {
1356 uint32_t output_tensor_size = model->getOutputTensorSize (idx);
1358 if (buffer->isExternal ()) {
1359 output.bufs[idx].type = BUFFER_DMABUF;
1360 output.bufs[idx].size = output_tensor_size;
1361 output.bufs[idx].addr = buffer->getOutputTensor(idx)->getData();
1363 output.bufs[idx].type = BUFFER_MAPPED;
1364 output.bufs[idx].size = output_tensor_size;
1365 /** user needs to free this */
1366 output.bufs[idx].addr = malloc (output_tensor_size);
1368 auto func = std::bind (TrinityVision::manipulateData, model, idx, false,
1369 std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
1370 int status = comm_.insertGenericBuffer (buffer->getOutputTensor(idx)->getData(),
1371 &output.bufs[idx], func);
1373 logerr (TAG, "Failed to return output buffer: %d\n", status);
1378 cb (&output, req->getID(), cb_data);
1384 * @brief extract the segment table instance from input generic buffers
1385 * @param[in] model the model instance
1386 * @param[in] input the input generic buffers
1387 * @param[in] output the output generic buffers
1388 * @return the segment table instance
1391 TrinityVision2::prepareSegmentTable (const Model *model, const input_buffers *input,
1392 const output_buffers *output)
1394 const Metadata *meta = model->getMetadata ();
1395 if (meta == nullptr || (input != nullptr &&
1396 meta->getInputNum() != input->num_buffers)) {
1397 logerr (TAG, "Invalid metadata info provided\n");
1401 SegmentTable * segt = mem_->allocSegmentTable (new HWmemDevice);
1402 int status = segt->alloc ();
1404 logerr (TAG, "Failed to allocate segment table: %d\n", status);
1408 status = segt->createSegments (model, input, output);
1410 logerr (TAG, "Failed to create segments: %d\n", status);
1422 * @brief implementation of TRIV2's setModel ()
1423 * @param[in] model_buf the model generic buffer
1424 * @param[out] model the model instance
1425 * @return 0 if no error, otherwise a negative errno
1428 TrinityVision2::setModel (const generic_buffer *model_buf, Model ** model_ptr)
1430 if (!initialized ()) {
1431 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1435 if (model_buf == nullptr || model_ptr == nullptr)
1441 switch (model_buf->type) {
1444 model = mem_->allocModel (new HWmemDevice);
1445 if (model == nullptr) {
1446 logerr (TAG, "Failed to allocate model\n");
1450 status = model->alloc (NPUBIN_META_SIZE);
1452 logerr (TAG, "Failed to allocate model: %d\n", status);
1456 status = comm_.extractGenericBuffer (model_buf, model->getData(), nullptr,
1457 0, NPUBIN_META_SIZE);
1459 logerr (TAG, "Failed to extract generic buffer: %d\n", status);
1467 status = model->setMetadata (model->getData());
1471 /** allocate program (optional; NOP) */
1472 if (model->getMetadata()->getProgramSize() > 0) {
1473 HWmem * hwmem_prog = new HWmem (new HWmemDevice);
1474 hwmem_prog->setDriverAPI (api_.get());
1476 model->setProgramData (hwmem_prog);
1478 status = hwmem_prog->alloc (model->getMetadata()->getProgramSize());
1480 logerr (TAG, "Failed to allocate program\n");
1484 status = comm_.extractGenericBuffer (model_buf, hwmem_prog->getData(), nullptr,
1485 model->getMetadata()->getMetaSize(),
1486 model->getMetadata()->getProgramSize());
1488 logerr (TAG, "Failed to extract generic buffer: %d\n", status);
1492 /** register this model to the driver */
1493 model_config_t config;
1494 config.dbuf_fd = hwmem_prog->getDmabuf ();
1495 config.program_size = hwmem_prog->getSize ();
1496 config.program_offset_addr = 0;
1498 status = api_->registerModel (&config);
1502 model->setInternalID(config.id);
1505 /** allocate weight (optional) */
1506 if (model->getMetadata()->getWeightSize() > 0) {
1507 HWmem * hwmem_weight = new HWmem (new HWmemDevice);
1508 hwmem_weight->setDriverAPI (api_.get());
1510 model->setWeightData (hwmem_weight);
1512 status = hwmem_weight->alloc (model->getMetadata()->getWeightSize());
1514 logerr (TAG, "Failed to allocate program\n");
1518 status = comm_.extractGenericBuffer (model_buf, hwmem_weight->getData(), nullptr,
1519 model->getMetadata()->getMetaSize() + model->getMetadata()->getProgramSize(),
1520 model->getMetadata()->getWeightSize());
1522 logerr (TAG, "Failed to extract generic buffer: %d\n", status);
1536 * @brief implementation of TRIV2's unsetModel ()
1537 * @param[in] model the model instance
1538 * @return 0 if no error, otherwise a negative errno
1541 TrinityVision2::unsetModel (Model * model)
1543 if (!initialized ()) {
1544 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1548 if (model == nullptr) {
1549 logerr (TAG, "Invalid model instance\n");
1553 if (model->getMetadata()->getProgramSize() > 0)
1554 return api_->deregisterModel (model->getInternalID ());
1559 /** @brief implementation of TRIV2's run() */
1561 TrinityVision2::run (npu_input_opmode opmode, const Model *model,
1562 const input_buffers *input, npuOutputNotify cb, void *cb_data,
1565 if (!initialized ()) {
1566 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1570 if (opmode != NPUINPUT_HOST)
1573 if (input == nullptr || input->num_buffers == 0 || model == nullptr)
1576 const_cast<Model *>(model)->updateDataInfo ();
1578 /** this device uses segment table */
1579 SegmentTable * segt = prepareSegmentTable (model, input);
1580 if (segt == nullptr) {
1581 logerr (TAG, "Failed to create segment table instance\n");
1585 /** extract input data */
1586 for (uint32_t idx = 0; idx < input->num_buffers; idx++) {
1587 if (!segt->getInputSegment(idx)->isExternal ()) {
1588 uint32_t seg_offset = segt->getInputSegmentOffset(idx);
1589 auto func = std::bind (TrinityVision2::manipulateData, model, idx, true,
1590 std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
1591 int status = comm_.extractGenericBuffer (
1593 segt->getInputSegment(idx)->getData() + seg_offset,
1596 logerr (TAG, "Failed to feed input segment: %d\n", status);
1602 Request *req = new Request (opmode);
1603 req->setModel (model);
1604 req->setSegmentTable (segt);
1605 req->setCallback (std::bind (&TrinityVision2::callback, this, req, cb, cb_data));
1608 *sequence = req->getID();
1610 return scheduler_->submitRequest (req);
1613 /** @brief implementation of TRIV2's runInternal() */
1615 TrinityVision2::runInternal (npu_input_opmode opmode, const Model *model,
1618 if (!initialized ()) {
1619 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1623 if (opmode != NPUINPUT_HW_RECURRING)
1626 /** this device uses segment table */
1627 SegmentTable * segt = prepareSegmentTable (model, nullptr, nullptr);
1628 if (segt == nullptr) {
1629 logerr (TAG, "Failed to create segment table instance\n");
1633 Request *req = new Request (opmode);
1634 req->setModel (model);
1635 req->setSegmentTable (segt);
1636 req->setHwDevice (hw_dev);
1638 return scheduler_->submitRequest (req);
1641 /** @brief callback of TRIV2 request */
1643 TrinityVision2::callback (Request *req, npuOutputNotify cb, void *cb_data)
1645 const Model *model = req->getModel ();
1646 SegmentTable *segt = req->getSegmentTable ();
1647 output_buffers output = {
1648 .num_buffers = segt->getNumOutputSegments ()
1651 for (uint32_t idx = 0; idx < output.num_buffers; idx++) {
1652 uint32_t output_tensor_size = model->getOutputTensorSize (idx);
1654 output.bufs[idx].type = BUFFER_MAPPED;
1655 output.bufs[idx].size = output_tensor_size;
1656 /** user needs to free this */
1657 output.bufs[idx].addr = calloc (1, output_tensor_size);
1659 #if defined(ENABLE_FPGA_WORKAROUND)
1661 segt->getOutputSegment(idx)->getDmabuf(),
1662 segt->getOutputSegmentOffset(idx),
1663 output.bufs[idx].addr,
1664 output.bufs[idx].size);
1666 auto func = std::bind (TrinityVision2::manipulateData, model, idx, false,
1667 std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
1668 int status = comm_.insertGenericBuffer (
1669 segt->getOutputSegment(idx)->getData() + segt->getOutputSegmentOffset(idx),
1670 &output.bufs[idx], func);
1673 logerr (TAG, "Failed to return output buffer: %d\n", status);
1678 cb (&output, req->getID(), cb_data);
1683 /** @brief implementation of TRIA's run(): WIP */
1685 TrinityAsr::run (npu_input_opmode opmode, const Model *model,
1686 const input_buffers *input, npuOutputNotify cb, void *cb_data,
1689 if (!initialized ()) {
1690 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1694 if (opmode != NPUINPUT_HOST)
1697 if (input == nullptr || input->num_buffers != 1)
1702 /** ASR does not require model and support only a single tensor */
1703 const generic_buffer *first_buf = &input->bufs[0];
1704 if (first_buf->type == BUFFER_DMABUF) {
1705 buffer = mem_->allocBuffer (new HWmemExternal);
1706 if (buffer == nullptr)
1709 buffer->setDmabuf (first_buf->dmabuf);
1710 buffer->setOffset (first_buf->offset);
1711 buffer->setSize (first_buf->size);
1713 buffer = mem_->allocBuffer (new HWmemDevice);
1714 if (buffer == nullptr)
1717 status = buffer->alloc (first_buf->size);
1724 status = buffer->createTensors ();
1726 logerr (TAG, "Failed to create tensors: %d\n", status);
1731 if (!buffer->isExternal ()) {
1732 status = comm_.extractGenericBuffer (first_buf,
1733 buffer->getInputTensor(0)->getData(), nullptr);
1738 Request *req = new Request (opmode);
1739 req->setBuffer (buffer);
1740 req->setCallback (std::bind (&TrinityAsr::callback, this, req, cb, cb_data));
1743 *sequence = req->getID();
1745 return scheduler_->submitRequest (req);
1748 /** @brief callback of TRIA request: WIP */
1750 TrinityAsr::callback (Request *req, npuOutputNotify cb, void *cb_data)
1752 Buffer *buffer = req->getBuffer ();
1753 output_buffers output = {
1757 /** TODO: finalize this impl. when the ASR's working scenario is determined */
1758 cb (&output, req->getID(), cb_data);
1763 /** Implement data manipulation (each device may have different impl.) */
1768 * @brief perform data manipulation
1769 * @param[in] model model instance
1770 * @param[in] idx tensor index
1771 * @param[in] is_input indicate it's input manipulation
1772 * @param[out] dst destination buffer
1773 * @param[in] src source buffer (feature map)
1774 * @param[in] size size to be copied
1775 * @return size of memory copy if no error, otherwise zero
1777 * @note the input data format should be NHWC
1778 * @detail rules for the memory address of activations in NPU HW.
1779 * (https://code.sec.samsung.net/confluence/pages/viewpage.action?pageId=146491864)
1781 * 1) Special case (depth == 3)
1782 * - addr(x,y,z) = addr(0,0,0) + (z) + 3 * (x + width * y)
1785 * - addr(x,y,z) = addr(0,0,0) + (z % MPA_L) + MPA_L * (x + width * (y + height * (z / MPA_L)))
1787 * Thus, if depth is not a multiple of MPA_L (i.e., 64), zero padding is required
1790 TrinityVision::manipulateData (const Model *model, uint32_t idx, bool is_input,
1791 void *dst, void *src, size_t size)
1793 const Metadata *meta = model->getMetadata();
1794 DataConverter converter (is_input);
1796 converter.setData (src, dst, size);
1799 const tensor_data_info* info = model->getInputDataInfo (idx);
1800 if (info == nullptr)
1803 converter.setDataLayout (info->layout, DATA_LAYOUT_SRNPU);
1804 converter.setDataType (info->type, DATA_TYPE_SRNPU);
1805 converter.setDataDims (meta->getInputDims (idx));
1806 converter.setQuantZero (meta->getInputQuantZero (idx));
1807 converter.setQuantScale (meta->getInputQuantScale (idx));
1809 const tensor_data_info* info = model->getOutputDataInfo (idx);
1810 if (info == nullptr)
1813 converter.setDataLayout (DATA_LAYOUT_SRNPU, info->layout);
1814 converter.setDataType (DATA_TYPE_SRNPU, info->type);
1815 converter.setDataDims (meta->getOutputDims (idx));
1816 converter.setQuantZero (meta->getOutputQuantZero (idx));
1817 converter.setQuantScale (meta->getOutputQuantScale (idx));
1820 return converter.perform ();
1824 * @brief perform data manipulation
1825 * @param[in] model model instance
1826 * @param[in] idx tensor index
1827 * @param[in] is_input indicate it's input manipulation
1828 * @param[out] dst destination buffer
1829 * @param[in] src source buffer (feature map)
1830 * @param[in] size size to be copied
1831 * @return size of memory copy if no error, otherwise zero
1833 * @note the input data format should be NHWC
1835 * @detail Feature map data in TRIV2, (x, y, z) = (width, height, depth)
1837 * 1) Image input (depth == 1 or depth == 3)
1838 * Addr(x,y,z) = Addr(0,0,0) + z + depth * x + ymod * y
1841 * Addr(x,y,z) = Addr(0,0,0) + (z % 64) + (64 * x) + ymod * y + zmod * (z / 64)
1844 TrinityVision2::manipulateData (const Model *model, uint32_t idx, bool is_input,
1845 void *dst, void *src, size_t size)
1847 const Metadata *meta = model->getMetadata();
1848 DataConverter converter (is_input);
1850 converter.setData (src, dst, size);
1853 const tensor_data_info* info = model->getInputDataInfo (idx);
1854 if (info == nullptr)
1857 converter.setDataLayout (info->layout, DATA_LAYOUT_TRIV2);
1858 converter.setDataType (info->type, meta->getInputQuantType (idx));
1859 converter.setDataDims (meta->getInputDims (idx));
1860 converter.setQuantZero (meta->getInputQuantZero (idx));
1861 converter.setQuantScale (meta->getInputQuantScale (idx));
1863 const tensor_data_info* info = model->getOutputDataInfo (idx);
1864 if (info == nullptr)
1867 converter.setDataLayout (DATA_LAYOUT_TRIV2, info->layout);
1868 converter.setDataType (meta->getOutputQuantType (idx), info->type);
1869 converter.setDataDims (meta->getOutputDims (idx));
1870 converter.setQuantZero (meta->getOutputQuantZero (idx));
1871 converter.setQuantScale (meta->getOutputQuantScale (idx));
1874 return converter.perform ();
1880 TrinityVision::manipulateData (const Model *model, uint32_t idx, bool is_input,
1881 void *dst, void *src, size_t size)
1883 memcpy (dst, src, size);
1888 TrinityVision2::manipulateData (const Model *model, uint32_t idx, bool is_input,
1889 void *dst, void *src, size_t size)
1891 memcpy (dst, src, size);
1897 /** other device types don't have data manip impl. yet */
1900 TrinityAsr::manipulateData (const Model *model, uint32_t idx, bool is_input,
1901 void *dst, void *src, size_t size)
1903 memcpy (dst, src, size);