3 * Copyright (C) 2020 Samsung Electronics
4 * Copyright (C) 2020 Dongju Chae <dongju.chae@samsung.com>
7 * @file ne-host-handler.cc
9 * @brief Implementation of APIs to access NPU from Host
10 * @see https://code.sec.samsung.net/confluence/display/ODLC/2020+Overall+Software+Stack
11 * @author Dongju Chae <dongju.chae@samsung.com>
12 * @bug No known bugs except for NYI items
15 #include "ne-handler.h"
17 #include <libnpuhost.h>
18 #include <npubinfmt.h>
19 #include <NPUdrvAPI.h>
20 #include <CommPlugin.h>
25 #include <condition_variable>
32 #define INIT_HOST_HANDLER(handler, dev) \
33 Device *tdev = static_cast <Device *> (dev); \
34 if (tdev == nullptr) return -EINVAL; \
35 HostHandler *handler = tdev->getHostHandler (); \
36 if (handler == nullptr) return -EINVAL;
38 /** just for backward-compatability */
39 npudev_h HostHandler::latest_dev_ = nullptr;
41 /** implement libnpuhost APIs */
44 * @brief Returns the number of available NPU devices.
45 * @return @c The number of NPU devices.
46 * @retval 0 if no NPU devices available. if positive (number of NPUs) if NPU devices available. otherwise, a negative error value.
47 * @note the caller should call putNPUdevice() to release the device handle
49 int getnumNPUdeviceByType (dev_type type)
51 return HostHandler::getNumDevices (type);
55 * @brief Returns the handle of the chosen NPU devices.
56 * @param[out] dev The NPU device handle
57 * @param[in] id The NPU id to get the handle. 0 <= id < getnumNPUdeviceByType().
58 * @return @c 0 if no error. otherwise a negative error value
59 * @note the caller should call putNPUdevice() to release the device handle
61 int getNPUdeviceByType (npudev_h *dev, dev_type type, uint32_t id)
63 return HostHandler::getDevice (dev, type, id);
67 * @brief Returns the handle of an NPU device meeting the condition
68 * @param[out] dev The NPU device handle
69 * @param[in] cond The condition for device search.
70 * @return @c 0 if no error. otherwise a negative error value
71 * @note the caller should call putNPUdevice() to release the device handle
72 * @note it's not supported yet
74 int getNPUdeviceByCondition(npudev_h *dev, const npucondition *cond)
76 /** not implmeneted yet */
77 return getNPUdeviceByType (dev, NPUCOND_TRIV_CONN_SOCIP, 0);
81 * @brief release the NPU device instance obtained by getDevice ()
82 * @param[in] dev the NPU device handle
84 void putNPUdevice (npudev_h dev)
87 delete static_cast<Device *> (dev);
91 * @brief Send the NN model to NPU.
92 * @param[in] dev The NPU device handle
93 * @param[in] modelfile The filepath to the compiled NPU NN model in any buffer_type
94 * @param[out] modelid The modelid allocated for this instance of NN model.
95 * @return @c 0 if no error. otherwise a negative error value
97 * @detail For ASR devices, which do not accept models, but have models
98 * embedded in devices, you do not need to call register and
99 * register calls for ASR are ignored.
101 * @todo Add a variation: in-memory model register.
103 int registerNPUmodel (npudev_h dev, generic_buffer *modelfile, uint32_t *modelid)
105 INIT_HOST_HANDLER (host_handler, dev);
107 return host_handler->registerModel (modelfile, modelid);
111 * @brief Remove the NN model from NPU
112 * @param[in] dev The NPU device handle
113 * @param[in] modelid The model to be removed from the NPU.
114 * @return @c 0 if no error. otherwise a negative error value
115 * @detail This may incur some latency with memory compatcion.
117 int unregisterNPUmodel(npudev_h dev, uint32_t modelid)
119 INIT_HOST_HANDLER (host_handler, dev);
121 return host_handler->unregisterModel (modelid);
125 * @brief Remove all NN models from NPU
126 * @param[in] dev The NPU device handle
127 * @return @c 0 if no error. otherwise a negative error value
129 int unregisterNPUmodel_all(npudev_h dev)
131 INIT_HOST_HANDLER (host_handler, dev);
133 return host_handler->unregisterModels ();
137 * @brief [OPTIONAL] Set the data layout for input/output tensors
138 * @param[in] dev The NPU device handle
139 * @param[in] modelid The ID of model whose layouts are set
140 * @param[in] info_in the layout/type info for input tensors
141 * @param[in] info_out the layout/type info for output tensors
142 * @return @c 0 if no error. otherwise a negative error value
143 * @note if this function is not called, default layout/type will be used.
145 int setNPU_dataInfo(npudev_h dev, uint32_t modelid,
146 tensors_data_info *info_in, tensors_data_info *info_out)
148 INIT_HOST_HANDLER (host_handler, dev);
150 return host_handler->setDataInfo (modelid, info_in, info_out);
154 * @brief [OPTIONAL] Set the inference constraint for next NPU inferences
155 * @param[in] dev The NPU device handle
156 * @param[in] modelid The target model id
157 * @param[in] constraint inference constraint (e.g., timeout, priority)
158 * @return @c 0 if no error. otherwise a negative error value
159 * @note If this function is not called, default values are used.
161 int setNPU_constraint(npudev_h dev, uint32_t modelid, npuConstraint constraint)
163 INIT_HOST_HANDLER (host_handler, dev);
165 return host_handler->setConstraint (modelid, constraint);
169 * @brief Execute inference. Wait (block) until the output is available.
170 * @param[in] dev The NPU device handle
171 * @param[in] modelid The model to be inferred.
172 * @param[in] input The input data to be inferred.
173 * @param[out] output The output result. The caller MUST allocate appropriately before calling this.
174 * @return @c 0 if no error. otherwise a negative error value
176 * @detail This is a syntactic sugar of runNPU_async().
177 * CAUTION: There is a memcpy for the output buffer.
179 int runNPU_sync(npudev_h dev, uint32_t modelid, const input_buffers *input,
180 output_buffers *output)
182 INIT_HOST_HANDLER (host_handler, dev);
184 return host_handler->runSync (modelid, input, output);
188 * @brief Invoke NPU inference. Unblocking call.
189 * @param[in] dev The NPU device handle
190 * @param[in] modelid The model to be inferred.
191 * @param[in] input The input data to be inferred.
192 * @param[in] cb The output buffer handler.
193 * @param[out] sequence The sequence number returned with runNPU_async.
194 * @param[in] data The data given as a parameter to the runNPU_async call.
195 * @param[in] mode Configures how this operation works.
196 * @return @c 0 if no error. otherwise a negative error value
198 int runNPU_async(npudev_h dev, uint32_t modelid, const input_buffers *input,
199 npuOutputNotify cb, uint64_t *sequence, void *data,
202 INIT_HOST_HANDLER (host_handler, dev);
204 return host_handler->runAsync (modelid, input, cb, data, mode, sequence);
208 * @brief Allocate a buffer for NPU model with the requested buffer type.
209 * @param[in] dev The NPU device handle
210 * @param[in/out] Buffer the buffer pointer where memory is allocated.
211 * @return 0 if no error, otherwise a negative errno.
213 int allocNPU_modelBuffer (npudev_h dev, generic_buffer *buffer)
215 INIT_HOST_HANDLER (host_handler, dev);
217 return host_handler->allocGenericBuffer (buffer);
221 * @brief Free the buffer and remove the address mapping.
222 * @param[in] dev The NPU device handle
223 * @param[in] buffer the model buffer
224 * @return 0 if no error, otherwise a negative errno.
226 int cleanNPU_modelBuffer (npudev_h dev, generic_buffer *buffer)
228 INIT_HOST_HANDLER (host_handler, dev);
230 return host_handler->deallocGenericBuffer (buffer);
234 * @brief Allocate a buffer for NPU input with the requested buffer type.
235 * @param[in] dev The NPU device handle
236 * @param[in/out] Buffer the buffer pointer where memory is allocated.
237 * @return 0 if no error, otherwise a negative errno.
238 * @note please utilize allocInputBuffers() for multiple input tensors because subsequent
239 * calls of allocInputBuffer() don't gurantee contiguous allocations between them.
241 int allocNPU_inputBuffer (npudev_h dev, generic_buffer *buffer)
243 INIT_HOST_HANDLER (host_handler, dev);
245 return host_handler->allocGenericBuffer (buffer);
249 * @brief Free the buffer and remove the address mapping.
250 * @param[in] dev The NPU device handle
251 * @param[in] buffer the input buffer
252 * @return 0 if no error, otherwise a negative errno.
254 int cleanNPU_inputBuffer (npudev_h dev, generic_buffer *buffer)
256 INIT_HOST_HANDLER (host_handler, dev);
258 return host_handler->deallocGenericBuffer (buffer);
262 * @brief Allocate input buffers, which have multiple instances of generic_buffer
263 * @param[in] dev The NPU device handle
264 * @param[in/out] input input buffers.
265 * @return 0 if no error, otherwise a negative errno.
266 * @note it reuses allocInputBuffer().
267 * @details in case of BUFFER_DMABUF, this function can be used to gurantee physically-contiguous
268 * memory mapping for multiple tensors (in a single inference, not batch size).
270 int allocNPU_inputBuffers (npudev_h dev, input_buffers * input)
272 INIT_HOST_HANDLER (host_handler, dev);
274 return host_handler->allocGenericBuffer (input);
278 * @brief Free input buffers allocated by allocInputBuffers().
279 * @param[in] dev The NPU device handle
280 * @param[in/out] input input buffers.
281 * @note it reuses cleanInputbuffer().
282 * @return 0 if no error, otherwise a negative errno.
284 int cleanNPU_inputBuffers (npudev_h dev, input_buffers * input)
286 INIT_HOST_HANDLER (host_handler, dev);
288 return host_handler->deallocGenericBuffer (input);
292 * @brief get the current memory status for the given device
293 * @param[in] dev The NPU device handle
294 * @param[out] alloc_total The size of allocated memory until now
295 * @param[out] free_total The size of freed memory until now
296 * @return @c 0 if no error. otherwise a negatice error value
298 int getNPU_memoryStatus(npudev_h dev, size_t *alloc_total, size_t *free_total)
300 INIT_HOST_HANDLER (host_handler, dev);
302 return host_handler->getMemoryStatus (alloc_total, free_total);
306 * @brief Get metadata for NPU model
307 * @param[in] model The path of model binary file
308 * @param[in] need_extra whether you want to extract the extra data in metadata
309 * @return the metadata structure to be filled if no error, otherwise nullptr
311 * @note For most npu-engine users, the extra data is not useful because it will be
312 * used for second-party users (e.g., compiler, simulator).
313 * Also, the caller needs to free the metadata.
315 * @note the caller needs to free the metadata
317 npubin_meta * getNPUmodel_metadata (const char *model, bool need_extra)
326 fp = fopen (model, "rb");
328 logerr (TAG, "Failed to open the model binary: %d\n", -errno);
332 meta = (npubin_meta *) malloc (NPUBIN_META_SIZE);
334 logerr (TAG, "Failed to allocate metadata\n");
338 ret = fread (meta, 1, NPUBIN_META_SIZE, fp);
339 if (ret != NPUBIN_META_SIZE) {
340 logerr (TAG, "Failed to read the metadata\n");
344 if (!CHECK_NPUBIN (meta->magiccode)) {
345 logerr (TAG, "Invalid metadata provided\n");
349 if (need_extra && NPUBIN_META_EXTRA (meta->magiccode) > 0) {
350 npubin_meta *new_meta;
352 new_meta = (npubin_meta *) realloc (meta, NPUBIN_META_TOTAL_SIZE(meta->magiccode));
354 logerr (TAG, "Failed to allocate extra metadata\n");
358 ret = fread (new_meta->reserved_extra, 1, NPUBIN_META_EXTRA_SIZE (meta->magiccode), fp);
359 if (ret != NPUBIN_META_EXTRA_SIZE (meta->magiccode)) {
360 logerr (TAG, "Invalid extra metadata provided\n");
380 /** deprecated buffer APIs; please use the above APIs */
383 * @brief Returns the number of NPU devices (TRIV).
385 int getnumNPUdevice (void)
387 logwarn (TAG, "deprecated. Please use getnumNPUdeviceByType ()\n");
388 return getnumNPUdeviceByType (NPUCOND_TRIV_CONN_SOCIP);
392 * @brief Returns the list of ASR devices (TRIA)
394 int getnumASRdevice (void)
396 logwarn (TAG, "deprecated. Please use getnumNPUdeviceByType ()\n");
397 return getnumNPUdeviceByType (NPUCOND_TRIA_CONN_SOCIP);
401 * @brief Returns the handle of the chosen TRIV device.
403 int getNPUdevice (npudev_h *dev, uint32_t id)
405 logwarn (TAG, "deprecated. Please use getNPUdeviceByType ()\n");
406 return getNPUdeviceByType (dev, NPUCOND_TRIV_CONN_SOCIP, id);
410 * @brief Returns the handle of the chosen TRIA device.
412 int getASRdevice (npudev_h *dev, uint32_t id)
414 logwarn (TAG, "deprecated. Please use getNPUdeviceByType ()\n");
415 return getNPUdeviceByType (dev, NPUCOND_TRIA_CONN_SOCIP, id);
418 /** @brief deprecated */
419 int allocModelBuffer (generic_buffer *buffer)
421 logwarn (TAG, "deprecated. Please use allocNPU_modelBuffer\n");
422 return allocNPU_modelBuffer (HostHandler::getLatestDevice(), buffer);
425 /** @brief deprecated */
426 int cleanModelBuffer (generic_buffer *buffer)
428 logwarn (TAG, "deprecated. Please use cleanNPU_modelBuffer\n");
429 return allocNPU_modelBuffer (HostHandler::getLatestDevice(), buffer);
432 /** @brief deprecated */
433 int allocInputBuffer (generic_buffer *buffer)
435 logwarn (TAG, "deprecated. Please use allocNPU_inputBuffer\n");
436 return allocNPU_inputBuffer (HostHandler::getLatestDevice(), buffer);
439 /** @brief deprecated */
440 int cleanInputBuffer (generic_buffer *buffer)
442 logwarn (TAG, "deprecated. Please use cleanNPU_inputBuffer\n");
443 return cleanNPU_inputBuffer (HostHandler::getLatestDevice(), buffer);
446 /** @brief deprecated */
447 int allocInputBuffers (input_buffers * input)
449 logwarn (TAG, "deprecated. Please use allocNPU_inputBuffers\n");
450 return allocNPU_inputBuffers (HostHandler::getLatestDevice(), input);
453 /** @brief deprecated */
454 int cleanInputBuffers (input_buffers * input)
456 logwarn (TAG, "deprecated. Please use cleanNPU_inputBuffers\n");
457 return cleanNPU_inputBuffers (HostHandler::getLatestDevice(), input);
460 /** @brief deprecated */
461 int allocNPUBuffer (uint64_t size, buffer_types type,
462 const char * filepath, generic_buffer *buffer)
467 buffer->filepath = filepath;
470 logwarn (TAG, "deprecated. Please use allocNPU_* APIs\n");
471 return allocModelBuffer (buffer);
474 /** @brief deprecated */
475 int cleanNPUBuffer (generic_buffer * buffer)
477 logwarn (TAG, "deprecated. Please use cleanNPU_* APIs\n");
478 return cleanModelBuffer (buffer);
481 /** implement methods of HostHandler class */
483 /** @brief host handler constructor */
484 HostHandler::HostHandler (Device *device)
486 /* ignored as we don't use double buffering anymore, but for backward-compatibility */
487 async_mode_ (NPUASYNC_WAIT)
491 /** @brief host handler destructor */
492 HostHandler::~HostHandler ()
497 * @brief register model from generic buffer
498 * @param[in] model_buf model buffer
499 * @param[out] modelid model id
500 * @return 0 if no error. otherwise a negative errno
503 HostHandler::registerModel (generic_buffer *model_buf, uint32_t *modelid)
505 if (model_buf == nullptr || modelid == nullptr) {
506 logerr (TAG, "Invalid arguments given\n");
510 Model *model = nullptr;
511 int status = device_->setModel (model_buf, &model);
513 logerr (TAG, "Failed to set model: %d\n", status);
517 assert (model != nullptr);
519 status = models_.insert (model->getID(), model);
521 logerr (TAG, "Failed to insert model id\n");
526 *modelid = model->getID();
531 * @brief remove the registered model
532 * @param[in] modelid model id
533 * @return 0 if no error. otherwise a negative errno
536 HostHandler::unregisterModel (uint32_t modelid)
538 return models_.remove (modelid);
542 * @brief remove all registered models
546 HostHandler::unregisterModels ()
553 * @brief Set the data layout for input/output tensors
554 * @param[in] modelid The ID of model whose layouts are set
555 * @param[in] in the layout/type info for input tensors
556 * @param[in] out the layout/type info for output tensors
557 * @return @c 0 if no error. otherwise a negative error value
558 * @note if this function is not called, default layout/type will be used.
561 HostHandler::setDataInfo (uint32_t modelid, tensors_data_info *in,
562 tensors_data_info *out)
564 Model *model = models_.find (modelid);
565 if (model == nullptr)
568 return model->setDataInfo (in, out);
572 * @brief Set the inference constraint for next NPU inferences
573 * @param[in] modelid The target model id
574 * @param[in] constraint inference constraint (e.g., timeout, priority)
575 * @return @c 0 if no error. otherwise a negative error value
576 * @note If this function is not called, default values are used.
579 HostHandler::setConstraint (uint32_t modelid, npuConstraint constraint)
581 Model *model = models_.find (modelid);
582 if (model == nullptr)
585 model->setConstraint (constraint);
591 * @brief find and return model instance
592 * @param[in] modelid model id
593 * @return model instance if found. otherwise nullptr
596 HostHandler::getModel (uint32_t modelid)
598 return models_.find (modelid);
601 /** @brief dummay callback for runSync. */
604 callbackSync (output_buffers *output) : output_(output), done_(false) {}
606 static void callback (output_buffers *output, uint64_t sequence, void *data) {
607 callbackSync *sync = static_cast<callbackSync *>(data);
608 sync->callback (output, sequence);
611 void callback (output_buffers *output, uint64_t sequence) {
612 if (output_ != nullptr) {
613 /** just copy internal variables of output buffers */
614 memcpy (output_, output, sizeof (output_buffers));
621 std::unique_lock<std::mutex> lock (m_);
622 cv_.wait (lock, [this]() { return done_; });
627 std::condition_variable cv_;
628 output_buffers *output_;
633 * @brief Execute inference. Wait (block) until the output is available.
634 * @param[in] modelid The model to be inferred.
635 * @param[in] input The input data to be inferred.
636 * @param[out] output The output result.
637 * @return @c 0 if no error. otherwise a negative error value
640 HostHandler::runSync (uint32_t modelid, const input_buffers *input,
641 output_buffers *output)
643 callbackSync sync (output);
644 int status = runAsync (modelid, input, callbackSync::callback,
645 static_cast <void*> (&sync), NPUASYNC_DROP_OLD, nullptr);
647 /** sync needs to wait callback */
654 * @brief Invoke NPU inference. Unblocking call.
655 * @param[in] modelid The model to be inferred.
656 * @param[in] input The input data to be inferred.
657 * @param[in] cb The output buffer handler.
658 * @param[in] cb_data The data given as a parameter to the runNPU_async call.
659 * @param[in] mode Configures how this operation works.
660 * @param[out] sequence The sequence number returned with runNPU_async.
661 * @return @c 0 if no error. otherwise a negative error value
664 HostHandler::runAsync (uint32_t modelid, const input_buffers *input,
665 npuOutputNotify cb, void *cb_data, npu_async_mode mode, uint64_t *sequence)
667 Model *model = nullptr;
669 if (device_->needModel()) {
670 model = getModel (modelid);
671 if (model == nullptr)
675 device_->setAsyncMode (mode);
676 return device_->run (NPUINPUT_HOST, model, input, cb, cb_data, sequence);
680 * @brief get number of available devices
681 * @param[in] type device type
682 * @return number of devices
685 HostHandler::getNumDevices (dev_type type)
687 return DriverAPI::getNumDevices (type);
691 * @brief get device instance
692 * @param[out] dev device instance
693 * @param[in] type device type
694 * @param[in] id device id
695 * @return 0 if no error. otherwise a negative errno
698 HostHandler::getDevice (npudev_h *dev, dev_type type, uint32_t id)
700 int num_devices = getNumDevices (type);
702 /** check the validity of device id */
703 if (!(num_devices > 0 && id < static_cast<uint32_t>(num_devices))) {
704 logerr (TAG, "Invalid arguments provided\n");
708 Device *device = Device::createInstance (type, id);
709 if (device == nullptr) {
710 logerr (TAG, "Failed to create a device with the given type\n");
715 /** This is just for backward-compatility; we don't guarantee its corresness */
722 * @brief allocate generic buffer (just for users)
723 * @param[out] buffer buffer instance
724 * @return 0 if no error. otherwise a negative errno
727 HostHandler::allocGenericBuffer (generic_buffer *buffer)
732 if (buffer->size == 0) {
733 logerr (TAG, "Invalid size\n");
737 if (buffer->size > UINT32_MAX) {
738 logerr (TAG, "Don't support such a large size");
742 switch (buffer->type) {
745 if (buffer->filepath == nullptr)
751 /* now, npu-engine always provides dmabuf-based allocation */
752 void *addr = nullptr;
753 int dmabuf = device_->allocMemory (buffer->size, &addr);
757 buffer->dmabuf = dmabuf;
769 * @brief deallocate generic buffer (just for users)
770 * @param[in] buffer buffer instance
771 * @return 0 if no error. otherwise a negative errno
774 HostHandler::deallocGenericBuffer (generic_buffer *buffer)
780 switch (buffer->type) {
782 status = 0; /** always true cuz nothing to do */
786 status = device_->deallocMemory (buffer->dmabuf, buffer->size, buffer->addr);
797 * @brief allocate multiple generic buffers (just for users)
798 * @param[out] buffers multi-buffer instance
799 * @return 0 if no error. otherwise a negative errno
802 HostHandler::allocGenericBuffer (generic_buffers *buffers)
804 if (buffers == NULL || buffers->num_buffers < 1)
807 buffer_types type = buffers->bufs[0].type;
808 if (type == BUFFER_FILE)
811 uint64_t total_size = 0;
812 for (uint32_t idx = 0; idx < buffers->num_buffers; idx++)
813 total_size += buffers->bufs[idx].size;
815 uint64_t first_size = buffers->bufs[0].size;
816 buffers->bufs[0].size = total_size;
817 int status = allocGenericBuffer (&buffers->bufs[0]);
821 uint64_t offset = first_size;
822 for (uint32_t idx = 1; idx < buffers->num_buffers; idx++) {
823 buffers->bufs[idx].dmabuf = buffers->bufs[0].dmabuf;
824 buffers->bufs[idx].offset = buffers->bufs[0].offset + offset;
825 buffers->bufs[idx].addr = static_cast<char*>(buffers->bufs[0].addr) + offset;
826 buffers->bufs[idx].type = type;
828 offset += buffers->bufs[idx].size;
831 buffers->bufs[0].size = first_size;
837 * @brief deallocate multiple generic buffers (just for users)
838 * @param[in] buffers multi-buffer instance
839 * @return 0 if no error. otherwise a negative errno
842 HostHandler::deallocGenericBuffer (generic_buffers *buffers)
844 if (buffers == NULL || buffers->num_buffers < 1)
847 return deallocGenericBuffer (&buffers->bufs[0]);
851 * @brief get the current memory status
852 * @param[out] alloc_total The size of allocated memory until now
853 * @param[out] free_total The size of freed memory until now
854 * @return 0 if no error. otherwise a negatice error value
857 HostHandler::getMemoryStatus (size_t *alloc_total, size_t *free_total)
859 /** API is always set in initialize () */
860 const DriverAPI * api = device_->getDriverAPI ();
861 assert (api != nullptr);
863 return api->getMemoryStatus (alloc_total, free_total);
866 /** implement methods of Device class */
868 /** @brief constructor of device */
869 Device::Device (dev_type type, int id, bool need_model)
870 : comm_ (CommPlugin::getCommPlugin()), type_ (type), id_ (id), need_model_ (true),
871 mode_ (NPUASYNC_WAIT), initialized_ (false), atomic_flag_ (ATOMIC_FLAG_INIT)
876 * @brief create device instance depending on device type and id
877 * @param[in] type device type
878 * @param[in] id device id
879 * @return device instance
882 Device::createInstance (dev_type type, int id)
884 Device *device = nullptr;
886 switch (type & DEVICETYPE_MASK) {
887 case DEVICETYPE_TRIV:
888 device = new TrinityVision (id);
890 case DEVICETYPE_TRIV2:
891 device = new TrinityVision2 (id);
893 case DEVICETYPE_TRIA:
894 device = new TrinityAsr (id);
900 if (device != nullptr && device->init () != 0) {
909 * @brief device initialization
910 * @return 0 if no error, otherwise a negative errno
911 * @note Init failures come from createDriverAPI() only.
916 /** should be initilizaed only once */
917 if (!atomic_flag_.test_and_set()) {
918 /** create the corresponding driver API */
919 api_ = DriverAPI::createDriverAPI (type_, id_);
920 if (api_.get() == nullptr) {
921 atomic_flag_.clear();
922 logerr (TAG, "Failed to create driver API\n");
926 handler_.reset (new HostHandler (this));
927 scheduler_.reset (new Scheduler (api_.get()));
928 mem_ = MemAllocator::createInstance (api_.get());
930 initialized_ = true; /** c++11 does not provide test() of atomic flag */
937 * @brief stop all requests from this device
938 * @param[in] force_stop indicate the schedduler waits until to handle previous requests
939 * @return 0 if no error, otherwise a negative errno
942 Device::stop (bool force_stop)
944 if (!initialized ()) {
945 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
949 Request *req = new Request (NPUINPUT_STOP);
950 req->setForceStop (force_stop);
951 return scheduler_->submitRequest (req);
955 * @brief allocate generic memory buffer
956 * @param[in] size the size to allocate
957 * @param[out] addr the mapped address
958 * @return dmabuf fd if no error, otherwise a negative errno
961 Device::allocMemory (size_t size, void **addr)
963 if (!initialized ()) {
964 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
968 if (size == 0 || addr == nullptr) {
969 logerr (TAG, "Invalid arguments\n");
973 return mem_->allocMemory (size, addr);
977 * @brief deallocate generic memory buffer
978 * @param[in] dmabuf_fd dmabuf file descriptor
979 * @param[in] size buffer size
980 * @param[in] addr mapped addr
981 * @return 0 if no error, otherwise a negative errno
984 Device::deallocMemory (int dmabuf_fd, size_t size, void * addr)
986 if (!initialized ()) {
987 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
991 if (dmabuf_fd < 0 || size == 0 || addr == nullptr) {
992 logerr (TAG, "Invalid arguments\n");
996 return mem_->deallocMemory (dmabuf_fd, size, addr);
1000 * @brief extract the buffer instance from input generic buffers
1001 * @param[in] meta the model metadata
1002 * @param[in] input the input generic buffers
1003 * @return the buffer instance
1006 TrinityVision::prepareInputBuffers (const Metadata *meta, const input_buffers *input)
1008 if (meta == nullptr || input == nullptr ||
1009 meta->getInputNum() != input->num_buffers) {
1010 logerr (TAG, "Invalid metadata info provided\n");
1015 const generic_buffer *first = &input->bufs[0];
1016 if (first->type == BUFFER_DMABUF) {
1017 buffer = mem_->allocBuffer (new HWmemExternal);
1018 if (buffer == nullptr)
1021 buffer->setDmabuf (first->dmabuf);
1022 buffer->setOffset (first->offset);
1023 buffer->setSize (meta->getBufferSize());
1025 buffer = mem_->allocBuffer (new HWmemDevice);
1026 if (buffer == nullptr)
1029 int status = buffer->alloc (meta->getBufferSize ());
1031 logerr (TAG, "Failed to allocate buffer: %d\n", status);
1037 int status = buffer->createTensors (meta);
1039 logerr (TAG, "Failed to create tensors: %d\n", status);
1048 * @brief implementation of TRIV's setModel ()
1049 * @param[in] model_buf the model generic buffer
1050 * @param[out] model the model instance
1051 * @return 0 if no error, otherwise a negative errno
1054 TrinityVision::setModel (const generic_buffer *model_buf, Model ** model_ptr)
1056 if (!initialized ()) {
1057 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1061 if (model_buf == nullptr || model_ptr == nullptr)
1067 /** In TRIV1, model data (including program/weight) should be contiguous */
1069 switch (model_buf->type) {
1072 model = mem_->allocModel (new HWmemDevice);
1073 if (model == nullptr) {
1074 logerr (TAG, "Failed to allocate model\n");
1078 status = model->alloc (model_buf->size);
1080 logerr (TAG, "Failed to allocate model: %d\n", status);
1084 /** extract the whole model data */
1085 status = comm_.extractGenericBuffer (model_buf, model->getData(), nullptr);
1087 logerr (TAG, "Failed to extract generic buffer: %d\n", status);
1095 status = model->setMetadata (model->getData());
1099 /** allocate program (optional; NOP) */
1100 if (model->getMetadata()->getProgramSize() > 0) {
1101 HWmem * hwmem_prog = new HWmem (new HWmemChunk);
1102 model->setProgramData (hwmem_prog);
1104 hwmem_prog->setParent (model);
1105 hwmem_prog->setOffset (model->getMetadata()->getMetaSize());
1106 status = hwmem_prog->alloc (model->getMetadata()->getProgramSize());
1108 logerr (TAG, "Failed to allocate program\n");
1112 /** register this model to the driver */
1113 model_config_t config;
1114 config.dbuf_fd = hwmem_prog->getDmabuf ();
1115 config.program_size = hwmem_prog->getSize ();
1116 config.program_offset_addr = hwmem_prog->getOffset ();
1118 status = api_->registerModel (&config);
1122 model->setInternalID(config.id);
1125 /** allocate weight (optional) */
1126 if (model->getMetadata()->getWeightSize() > 0) {
1127 HWmem * hwmem_weight = new HWmem (new HWmemChunk);
1128 model->setWeightData (hwmem_weight);
1130 hwmem_weight->setParent (model);
1131 hwmem_weight->setOffset (model->getMetadata()->getMetaSize() +
1132 model->getMetadata()->getProgramSize());
1133 status = hwmem_weight->alloc (model->getMetadata()->getWeightSize());
1135 logerr (TAG, "Failed to allocate program\n");
1149 * @brief implementation of TRIV's run()
1150 * @param[in] opmode input opmode
1151 * @param[in] model the model instance
1152 * @param[in] input generic buffers of input data
1153 * @param[in] cb the output callback
1154 * @param[in] cb_data the output callback data
1155 * @param[out] sequence The sequence number returned with runNPU_async.
1158 TrinityVision::run (npu_input_opmode opmode, const Model *model,
1159 const input_buffers *input, npuOutputNotify cb, void *cb_data,
1162 if (!initialized ()) {
1163 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1167 if (opmode != NPUINPUT_HOST) {
1168 logerr (TAG, "TRIV supports only host inputservice\n");
1172 if (model == nullptr || input == nullptr) {
1173 logerr (TAG, "TRIV requires both model and input buffers\n");
1177 Buffer *buffer = prepareInputBuffers (model->getMetadata(), input);
1178 if (buffer == nullptr) {
1179 logerr (TAG, "Failed to extract buffer instance\n");
1183 if (!buffer->isExternal ()) {
1184 for (uint32_t idx = 0; idx < input->num_buffers; idx++) {
1185 auto func = std::bind (TrinityVision::manipulateData, model, idx, true,
1186 std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
1187 int status = comm_.extractGenericBuffer (&input->bufs[idx],
1188 buffer->getInputTensor(idx)->getData(), func);
1190 logerr (TAG, "Failed to feed input buffer: %d\n", status);
1196 /** this device uses CMA buffer */
1198 Request *req = new Request (opmode);
1199 req->setModel (model);
1200 req->setBuffer (buffer);
1203 req->setCallback (std::bind (&TrinityVision::callback, this, req, cb, cb_data));
1205 if (sequence != nullptr)
1206 *sequence = req->getID();
1208 return scheduler_->submitRequest (req);
1212 * @brief callback of TRIV2 request
1213 * @param[in] req the request instance
1214 * @param[in] cb callback for completion
1215 * @param[in] cb_data callback data
1216 * @note The callback invoke does not gurantee the request was successful
1217 * @todo Check the request failures
1220 TrinityVision::callback (Request *req, npuOutputNotify cb, void *cb_data)
1222 const Model *model = req->getModel ();
1223 Buffer *buffer = req->getBuffer ();
1224 output_buffers output = {
1225 .num_buffers = buffer->getOutputNum ()
1228 for (uint32_t idx = 0; idx < output.num_buffers; idx++) {
1229 uint32_t output_tensor_size = model->getOutputTensorSize (idx);
1231 if (buffer->isExternal ()) {
1232 output.bufs[idx].type = BUFFER_DMABUF;
1233 output.bufs[idx].size = output_tensor_size;
1234 output.bufs[idx].addr = buffer->getOutputTensor(idx)->getData();
1236 output.bufs[idx].type = BUFFER_MAPPED;
1237 output.bufs[idx].size = output_tensor_size;
1238 /** user needs to free this */
1239 output.bufs[idx].addr = malloc (output_tensor_size);
1241 auto func = std::bind (TrinityVision::manipulateData, model, idx, false,
1242 std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
1243 int status = comm_.insertGenericBuffer (buffer->getOutputTensor(idx)->getData(),
1244 &output.bufs[idx], func);
1246 logerr (TAG, "Failed to return output buffer: %d\n", status);
1251 cb (&output, req->getID(), cb_data);
1257 * @brief extract the segment table instance from input generic buffers
1258 * @param[in] model the model instance
1259 * @param[in] input the input generic buffers
1260 * @return the segment table instance
1263 TrinityVision2::prepareSegmentTable (const Model *model, const input_buffers *input)
1265 if (model == nullptr || input == nullptr) {
1266 logerr (TAG, "Invalid arguments provided\n");
1270 const Metadata *meta = model->getMetadata ();
1271 if (meta == nullptr ||
1272 meta->getInputNum() != input->num_buffers) {
1273 logerr (TAG, "Invalid metadata info provided\n");
1277 SegmentTable * segt = mem_->allocSegmentTable (new HWmemDevice);
1278 int status = segt->alloc ();
1280 logerr (TAG, "Failed to allocate segment table: %d\n", status);
1284 status = segt->createSegments (model, input);
1286 logerr (TAG, "Failed to create segments: %d\n", status);
1298 * @brief implementation of TRIV2's setModel ()
1299 * @param[in] model_buf the model generic buffer
1300 * @param[out] model the model instance
1301 * @return 0 if no error, otherwise a negative errno
1304 TrinityVision2::setModel (const generic_buffer *model_buf, Model ** model_ptr)
1306 if (!initialized ()) {
1307 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1311 if (model_buf == nullptr || model_ptr == nullptr)
1317 switch (model_buf->type) {
1320 model = mem_->allocModel (new HWmemDevice);
1321 if (model == nullptr) {
1322 logerr (TAG, "Failed to allocate model\n");
1326 status = model->alloc (NPUBIN_META_SIZE);
1328 logerr (TAG, "Failed to allocate model: %d\n", status);
1332 status = comm_.extractGenericBuffer (model_buf, model->getData(), nullptr,
1333 0, NPUBIN_META_SIZE);
1335 logerr (TAG, "Failed to extract generic buffer: %d\n", status);
1343 status = model->setMetadata (model->getData());
1347 /** allocate program (optional; NOP) */
1348 if (model->getMetadata()->getProgramSize() > 0) {
1349 HWmem * hwmem_prog = new HWmem (new HWmemDevice);
1350 hwmem_prog->setDriverAPI (api_.get());
1352 model->setProgramData (hwmem_prog);
1354 status = hwmem_prog->alloc (model->getMetadata()->getProgramSize());
1356 logerr (TAG, "Failed to allocate program\n");
1360 status = comm_.extractGenericBuffer (model_buf, hwmem_prog->getData(), nullptr,
1361 model->getMetadata()->getMetaSize(),
1362 model->getMetadata()->getProgramSize());
1364 logerr (TAG, "Failed to extract generic buffer: %d\n", status);
1368 /** register this model to the driver */
1369 model_config_t config;
1370 config.dbuf_fd = hwmem_prog->getDmabuf ();
1371 config.program_size = hwmem_prog->getSize ();
1372 config.program_offset_addr = 0;
1374 status = api_->registerModel (&config);
1378 model->setInternalID(config.id);
1381 /** allocate weight (optional) */
1382 if (model->getMetadata()->getWeightSize() > 0) {
1383 HWmem * hwmem_weight = new HWmem (new HWmemDevice);
1384 hwmem_weight->setDriverAPI (api_.get());
1386 model->setWeightData (hwmem_weight);
1388 status = hwmem_weight->alloc (model->getMetadata()->getWeightSize());
1390 logerr (TAG, "Failed to allocate program\n");
1394 status = comm_.extractGenericBuffer (model_buf, hwmem_weight->getData(), nullptr,
1395 model->getMetadata()->getMetaSize() + model->getMetadata()->getProgramSize(),
1396 model->getMetadata()->getWeightSize());
1398 logerr (TAG, "Failed to extract generic buffer: %d\n", status);
1411 /** @brief implementation of TRIV2's run() */
1413 TrinityVision2::run (npu_input_opmode opmode, const Model *model,
1414 const input_buffers *input, npuOutputNotify cb, void *cb_data,
1417 if (!initialized ()) {
1418 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1422 if (opmode != NPUINPUT_HOST && opmode != NPUINPUT_HW_RECURRING)
1425 /** this device uses segment table */
1426 SegmentTable * segt = prepareSegmentTable (model, input);
1427 if (segt == nullptr) {
1428 logerr (TAG, "Failed to create segment table instance\n");
1432 /** extract input data */
1433 for (uint32_t idx = 0; idx < input->num_buffers; idx++) {
1434 if (!segt->getInputSegment(idx)->isExternal ()) {
1435 auto func = std::bind (TrinityVision2::manipulateData, model, idx, true,
1436 std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
1437 int status = comm_.extractGenericBuffer (
1439 segt->getInputSegment(idx)->getData() + segt->getInputSegmentOffset(idx),
1442 logerr (TAG, "Failed to feed input segment: %d\n", status);
1448 Request *req = new Request (opmode);
1449 req->setModel (model);
1450 req->setSegmentTable (segt);
1451 req->setCallback (std::bind (&TrinityVision2::callback, this, req, cb, cb_data));
1454 *sequence = req->getID();
1456 return scheduler_->submitRequest (req);
1459 /** @brief callback of TRIV2 request */
1461 TrinityVision2::callback (Request *req, npuOutputNotify cb, void *cb_data)
1463 const Model *model = req->getModel ();
1464 SegmentTable *segt = req->getSegmentTable ();
1465 output_buffers output = {
1466 .num_buffers = segt->getNumOutputSegments ()
1469 for (uint32_t idx = 0; idx < output.num_buffers; idx++) {
1470 uint32_t output_tensor_size = model->getOutputTensorSize (idx);
1472 output.bufs[idx].type = BUFFER_MAPPED;
1473 output.bufs[idx].size = output_tensor_size;
1474 /** user needs to free this */
1475 output.bufs[idx].addr = malloc (output_tensor_size);
1477 auto func = std::bind (TrinityVision2::manipulateData, model, idx, false,
1478 std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
1479 int status = comm_.insertGenericBuffer (
1480 segt->getOutputSegment(idx)->getData() + segt->getOutputSegmentOffset(idx),
1481 &output.bufs[idx], func);
1483 logerr (TAG, "Failed to return output buffer: %d\n", status);
1487 cb (&output, req->getID(), cb_data);
1492 /** @brief implementation of TRIA's run(): WIP */
1494 TrinityAsr::run (npu_input_opmode opmode, const Model *model,
1495 const input_buffers *input, npuOutputNotify cb, void *cb_data,
1498 if (!initialized ()) {
1499 logerr (TAG, "Uninitialized device; should use libnpuhost APIs\n");
1503 if (opmode != NPUINPUT_HOST)
1508 /** ASR does not require model and support only a single tensor */
1509 const generic_buffer *first_buf = &input->bufs[0];
1510 if (first_buf->type == BUFFER_DMABUF) {
1511 buffer = mem_->allocBuffer (new HWmemExternal);
1512 if (buffer == nullptr)
1515 buffer->setDmabuf (first_buf->dmabuf);
1516 buffer->setOffset (first_buf->offset);
1517 buffer->setSize (first_buf->size);
1519 buffer = mem_->allocBuffer (new HWmemDevice);
1520 if (buffer == nullptr)
1523 status = buffer->alloc (first_buf->size);
1530 status = buffer->createTensors ();
1532 logerr (TAG, "Failed to create tensors: %d\n", status);
1537 if (!buffer->isExternal ()) {
1538 status = comm_.extractGenericBuffer (first_buf,
1539 buffer->getInputTensor(0)->getData(), nullptr);
1544 Request *req = new Request (opmode);
1545 req->setBuffer (buffer);
1546 req->setCallback (std::bind (&TrinityAsr::callback, this, req, cb, cb_data));
1549 *sequence = req->getID();
1551 return scheduler_->submitRequest (req);
1554 /** @brief callback of TRIA request: WIP */
1556 TrinityAsr::callback (Request *req, npuOutputNotify cb, void *cb_data)
1560 /** Implement data manipulation (each device may have different impl.) */
1564 #define do_quantized_memcpy(type) do {\
1567 while (idx < num_elems) {\
1568 val = ((type *) src)[idx];\
1569 val = val / _scale;\
1570 val += _zero_point;\
1571 val = (val > 255.0) ? 255.0 : 0.0;\
1572 ((uint8_t *) dst)[idx++] = (uint8_t) val;\
1575 while (idx < num_elems) {\
1576 val = *(uint8_t *) src;\
1577 val -= _zero_point;\
1579 ((type *) dst)[idx++] = (type) val;\
1580 dst = (void*)(((uint8_t *) dst) + data_size);\
1581 src = (void*)(((uint8_t *) src) + 1);\
1587 * @brief memcpy during quantization
1589 static void memcpy_with_quant (bool quant, data_type type, float scale, uint32_t zero_point,
1590 void *dst, const void *src, uint32_t num_elems)
1592 double _scale = (double) scale;
1593 double _zero_point = (double) zero_point;
1595 uint32_t data_size = get_data_size (type);
1599 case DATA_TYPE_INT8:
1600 do_quantized_memcpy (int8_t);
1602 case DATA_TYPE_UINT8:
1603 do_quantized_memcpy (uint8_t);
1605 case DATA_TYPE_INT16:
1606 do_quantized_memcpy (int16_t);
1608 case DATA_TYPE_UINT16:
1609 do_quantized_memcpy (uint16_t);
1611 case DATA_TYPE_INT32:
1612 do_quantized_memcpy (int32_t);
1614 case DATA_TYPE_UINT32:
1615 do_quantized_memcpy (uint32_t);
1617 case DATA_TYPE_INT64:
1618 do_quantized_memcpy (int64_t);
1620 case DATA_TYPE_UINT64:
1621 do_quantized_memcpy (uint64_t);
1623 case DATA_TYPE_FLOAT32:
1624 do_quantized_memcpy (float);
1626 case DATA_TYPE_FLOAT64:
1627 do_quantized_memcpy (double);
1630 logerr (TAG, "Unsupported datatype %d\n", type);
1635 * @brief perform data manipulation
1636 * @param[in] model model instance
1637 * @param[in] idx tensor index
1638 * @param[in] is_input indicate it's input manipulation
1639 * @param[out] dst destination buffer
1640 * @param[in] src source buffer (feature map)
1641 * @param[in] size size to be copied
1642 * @return size of memory copy if no error, otherwise zero
1644 * @note the input data format should be NHWC
1645 * @detail rules for the memory address of activations in NPU HW.
1646 * (https://code.sec.samsung.net/confluence/pages/viewpage.action?pageId=146491864)
1648 * 1) Special case (depth == 3)
1649 * - addr(x,y,z) = addr(0,0,0) + (z) + 3 * (x + width * y)
1652 * - addr(x,y,z) = addr(0,0,0) + (z % MPA_L) + MPA_L * (x + width * (y + height * (z / MPA_L)))
1654 * Thus, if depth is not a multiple of MPA_L (i.e., 64), zero padding is required
1657 TrinityVision::manipulateData (const Model *model, uint32_t idx, bool is_input,
1658 void *dst, void *src, size_t size)
1660 const Metadata *meta = model->getMetadata();
1661 const tensor_data_info* info;
1662 const uint32_t *dims;
1663 uint32_t zero_point;
1666 /** extract required information from the metadata */
1668 if (idx >= meta->getInputNum()) {
1669 logerr (TAG, "Wrong information for input tensors in metadata\n");
1673 info = model->getInputDataInfo (idx);
1674 dims = meta->getInputDims (idx);
1675 zero_point = meta->getInputQuantZero (idx);
1676 scale = meta->getInputQuantScale (idx);
1678 if (idx >= meta->getOutputNum()) {
1679 logerr (TAG, "Wrong information for output tensors in metadata\n");
1683 info = model->getOutputDataInfo (idx);
1684 dims = meta->getOutputDims (idx);
1685 zero_point = meta->getOutputQuantZero (idx);
1686 scale = meta->getOutputQuantScale (idx);
1689 if (info == nullptr) {
1690 logerr (TAG, "Unmatched tensors info\n");
1694 uint32_t batch = dims[0];
1695 uint32_t height = dims[1];
1696 uint32_t width = dims[2];
1697 uint32_t depth = dims[3];
1699 uint32_t data_size = get_data_size (info->type);
1700 if (data_size == 0) {
1701 logerr (TAG, "Invalid data size\n");
1705 bool need_quantization = false;
1707 * note that we assume DATA_TYPE_SRNPU is the smallest data type that we consider.
1708 * Also, DATA_TYPE_SRNPU and uint8_t may be regarded as the same in the view of apps.
1710 if (info->type != DATA_TYPE_SRNPU) {
1711 assert (data_size >= get_data_size (DATA_TYPE_SRNPU));
1713 if (data_size > get_data_size (DATA_TYPE_SRNPU) ||
1714 !(zero_point == default_quant_zero && scale == default_quant_scale))
1715 need_quantization = true;
1718 /** check data manipulation is required */
1719 if (depth != 3 && depth != 64 && info->layout != DATA_LAYOUT_SRNPU) {
1720 uint32_t MPA_L = DATA_GRANULARITY;
1721 uint32_t n, h, w, d;
1722 uint32_t std_offset; /* standard offset in NHWC data format */
1723 uint32_t npu_offset; /* npu offset in NPU HW data format*/
1724 uint32_t src_offset;
1725 uint32_t dst_offset;
1726 uint32_t slice_size;
1728 /* @todo we currently support only NHWC */
1729 if (info->layout != DATA_LAYOUT_NHWC) {
1730 logerr (TAG, "data manipulation is supported for NHWC only\n");
1734 for (n = 0; n < batch; n++) {
1735 for (h = 0; h < height; h++) {
1736 for (w = 0; w < width; w++) {
1737 for (d = 0; d < depth; d += MPA_L) {
1738 std_offset = d + depth * (w + width * (h + n * height));
1739 npu_offset = MPA_L * (w + width * (h + (n + d / MPA_L) * height));
1740 slice_size = (depth - d >= MPA_L) ? MPA_L : depth - d;
1743 src_offset = std_offset * data_size;
1744 dst_offset = npu_offset;
1746 src_offset = npu_offset;
1747 dst_offset = std_offset * data_size;
1750 /* if depth is not a multiple of MPA_L, add zero paddings (not exact values) */
1751 if (need_quantization) {
1752 memcpy_with_quant (is_input, info->type, scale, zero_point,
1753 static_cast<char*>(dst) + dst_offset,
1754 static_cast<char*>(src) + src_offset,
1758 static_cast<char*>(dst) + dst_offset,
1759 static_cast<char*>(src) + src_offset,
1766 } else if (need_quantization) {
1767 /** depth == 3 || depth == 64; special cases which can directly copy input tensor data */
1768 memcpy_with_quant (is_input, info->type, scale, zero_point,
1769 dst, src, is_input ? size / data_size : size);
1771 memcpy (dst, src, size);
1780 TrinityVision::manipulateData (const Model *model, uint32_t idx, bool is_input,
1781 void *dst, void *src, size_t size)
1783 memcpy (dst, src, size);
1789 /** other device types don't have data manip impl. yet */
1792 TrinityVision2::manipulateData (const Model *model, uint32_t idx, bool is_input,
1793 void *dst, void *src, size_t size)
1795 memcpy (dst, src, size);
1800 TrinityAsr::manipulateData (const Model *model, uint32_t idx, bool is_input,
1801 void *dst, void *src, size_t size)
1803 memcpy (dst, src, size);