src/inference_engine_mlapi.cpp

   1 /**
   2  * Copyright (c) 2020 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include <inference_engine_error.h>
  18 #include "inference_engine_private_type.h"
  19 #include "inference_engine_mlapi_private.h"
  20
  21 #include <fstream>
  22 #include <iostream>
  23 #include <unistd.h>
  24 #include <time.h>
  25 #include <queue>
  26
  27 // TODO. Below is test code. DO NOT use ML internal function.
  28 #define ENABLE_NO_ALLOC
  29 #if defined(ENABLE_NO_ALLOC)
  30 extern "C" int ml_single_invoke_no_alloc(ml_single_h single, const ml_tensors_data_h input, ml_tensors_data_h output);
  31 #endif
  32
  33 namespace InferenceEngineImpl
  34 {
  35 namespace MLAPIImpl
  36 {
  37         InferenceMLAPI::InferenceMLAPI(void) :
  38                         mPluginType(),
  39                         mTargetDevice(),
  40                         mSingle(),
  41                         mInputInfoHandle(),
  42                         mOutputInfoHandle(),
  43                         mInputDataHandle(),
  44                         mOutputDataHandle(),
  45                         mDesignated_inputs(),
  46                         mDesignated_outputs(),
  47                         mInputProperty(),
  48                         mOutputProperty()
  49         {
  50                 LOGI("ENTER");
  51
  52                 LOGI("LEAVE");
  53         }
  54
  55         InferenceMLAPI::~InferenceMLAPI()
  56         {
  57                 mDesignated_inputs.clear();
  58                 std::vector<std::string>().swap(mDesignated_inputs);
  59
  60                 mDesignated_outputs.clear();
  61                 std::vector<std::string>().swap(mDesignated_outputs);
  62
  63                 ml_single_close(mSingle);
  64
  65                 if (mInputInfoHandle)
  66                         ml_tensors_info_destroy(mInputInfoHandle);
  67
  68                 if (mOutputInfoHandle)
  69                         ml_tensors_info_destroy(mOutputInfoHandle);
  70
  71                 if (mInputDataHandle)
  72                         ml_tensors_data_destroy(mInputDataHandle);
  73
  74                 if (mOutputDataHandle)
  75                         ml_tensors_data_destroy(mOutputDataHandle);
  76
  77                 mInputInfoHandle = NULL;
  78                 mOutputInfoHandle = NULL;
  79                 mInputDataHandle = NULL;
  80                 mOutputDataHandle = NULL;
  81         }
  82
  83         int InferenceMLAPI::SetPrivateData(void *data)
  84         {
  85                 LOGI("ENTER");
  86
  87                 inference_backend_type_e type =
  88                                 *(static_cast<inference_backend_type_e *>(data));
  89
  90                 if (INFERENCE_BACKEND_NONE >= type || INFERENCE_BACKEND_MAX <= type ||
  91                                 INFERENCE_BACKEND_OPENCV == type) {
  92                         LOGE("Invalid backend type.");
  93                         return INFERENCE_ENGINE_ERROR_NOT_SUPPORTED;
  94                 }
  95
  96                 mPluginType = type;
  97
  98                 LOGI("LEAVE");
  99
 100                 return INFERENCE_ENGINE_ERROR_NONE;
 101         }
 102
 103         int InferenceMLAPI::SetTargetDevices(int types)
 104         {
 105                 LOGI("ENTER");
 106
 107                 LOGI("Inference targets are, ");
 108                 if (types & INFERENCE_TARGET_CPU) {
 109                         mTargetDevice |= INFERENCE_TARGET_CPU;
 110                         LOGI("CPU");
 111                 }
 112
 113                 if (types & INFERENCE_TARGET_GPU) {
 114                         mTargetDevice |= INFERENCE_TARGET_GPU;
 115                         LOGI("GPU");
 116                 }
 117
 118                 if (types & INFERENCE_TARGET_CUSTOM) {
 119                         mTargetDevice |= INFERENCE_TARGET_CUSTOM;
 120                         LOGI("NPU");
 121                 }
 122
 123                 LOGI("LEAVE");
 124
 125                 return INFERENCE_ENGINE_ERROR_NONE;
 126         }
 127
 128         int InferenceMLAPI::Load(std::vector<std::string> model_paths,
 129                                                          inference_model_format_e model_format)
 130         {
 131                 LOGI("ENTER");
 132
 133                 std::string model_str(model_paths[0]);
 134
 135                 // TODO. Set NNFW backend type and HW type properly.
 136
 137                 ml_nnfw_type_e nnfw_type;
 138                 ml_nnfw_hw_e nnfw_hw;
 139
 140                 switch (mPluginType) {
 141                 case INFERENCE_BACKEND_NPU_VIVANTE:
 142                         nnfw_type = ML_NNFW_TYPE_VIVANTE;
 143                         nnfw_hw = ML_NNFW_HW_ANY;
 144                         LOGI("Vivante tensor filter will be used.");
 145
 146                         if (access(model_str.c_str(), R_OK) ||
 147                                         access(model_paths[1].c_str(), R_OK)) {
 148                                 LOGE("model file path in [%s,%s]", model_str.c_str(),
 149                                                                                                    model_paths[1].c_str());
 150                                 return INFERENCE_ENGINE_ERROR_INVALID_PATH;
 151                         }
 152
 153                         // ML Single API of MLAPI requires model_paths rule like below,
 154                         // "so library file path,nb model file path" or vise versa.
 155                         model_str += "," + model_paths[1];
 156                         break;
 157                 case INFERENCE_BACKEND_ONE:
 158                 case INFERENCE_BACKEND_ARMNN:
 159                 case INFERENCE_BACKEND_TFLITE:
 160                         if (mPluginType == INFERENCE_BACKEND_ONE) {
 161                                 nnfw_type = ML_NNFW_TYPE_NNFW;
 162
 163                                 if (mTargetDevice == INFERENCE_TARGET_CPU) {
 164                                         nnfw_hw = ML_NNFW_HW_CPU_NEON;
 165                                         LOGI("Target device is NEON.");
 166                                 } else if (mTargetDevice == INFERENCE_TARGET_GPU) {
 167                                         nnfw_hw = ML_NNFW_HW_GPU;
 168                                         LOGI("Target device is GPU");
 169                                 } else {
 170                                         LOGE("Invalid inference target device type.");
 171                                         return INFERENCE_ENGINE_ERROR_INVALID_PARAMETER;
 172                                 }
 173
 174                                 LOGI("NNFW tensor filter will be used.");
 175                         }
 176
 177                         if (mPluginType == INFERENCE_BACKEND_ARMNN) {
 178                                 nnfw_type = ML_NNFW_TYPE_ARMNN;
 179                                 LOGI("ARMNN tensor filter will be used.");
 180                         }
 181
 182                         if (mPluginType == INFERENCE_BACKEND_TFLITE) {
 183                                 nnfw_type = ML_NNFW_TYPE_TENSORFLOW_LITE;
 184                                 LOGI("TFLITE tensor filter will be used.");
 185                         }
 186
 187                         if (access(model_str.c_str(), R_OK)) {
 188                                 LOGE("model file path in [%s]", model_str.c_str());
 189                                 return INFERENCE_ENGINE_ERROR_INVALID_PATH;
 190                         }
 191
 192                         break;
 193                 // TODO.
 194                 default:
 195                         LOGE("Invalid plugin type.");
 196                         return INFERENCE_ENGINE_ERROR_INVALID_PARAMETER;
 197                 }
 198
 199                 LOGI("Model name = %s", model_str.c_str());
 200
 201                 // TODO. create ml_tensor_info for input and output tensor and pass
 202                 //               them as parameters of ml_single_open function.
 203
 204                 int err = ml_single_open(&mSingle, model_str.c_str(), NULL, NULL,
 205                                                                  nnfw_type, nnfw_hw);
 206                 if (err != ML_ERROR_NONE) {
 207                         LOGE("Failed to request ml_single_open(%d).", err);
 208                         return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 209                 }
 210
 211                 err = UpdateTensorsInfo();
 212                 if (err != INFERENCE_ENGINE_ERROR_NONE) {
 213                         ml_single_close(mSingle);
 214                         mSingle = NULL;
 215                 }
 216
 217                 LOGI("LEAVE");
 218
 219                 return err;
 220         }
 221
 222         int InferenceMLAPI::GetInputTensorBuffers(
 223                         std::vector<inference_engine_tensor_buffer> &buffers)
 224         {
 225                 LOGI("ENTER");
 226
 227                 // TODO. Implement this function according to a given ML Single API backend properly.
 228
 229                 // ML Single API will always provide internal tensor buffers so
 230                 // get the tensor buffers back to Mediavision framework so that
 231                 // Mediavision framework doesn't allocate the tensor buffers internally.
 232
 233                 buffers.clear();
 234
 235                 int ret;
 236                 unsigned int cnt;
 237
 238                 ret = ml_tensors_info_get_count(mInputInfoHandle, &cnt);
 239                 if (ret != ML_ERROR_NONE) {
 240                         LOGE("Failed to request ml_tensors_info_get_count(%d).", ret);
 241                         return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 242                 }
 243
 244                 LOGI("input tensor count = %u", cnt);
 245
 246                 // TODO. Below is test code, should we allocate new buffer for every inference?
 247                 if (mInputDataHandle == NULL) {
 248                         ret = ml_tensors_data_create(mInputInfoHandle, &mInputDataHandle);
 249                         if (ret != ML_ERROR_NONE) {
 250                                 LOGE("Failed to request ml_tensors_data_create(%d).", ret);
 251                                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 252                         }
 253                 }
 254
 255                 // TODO. Cache tensor info and reduce function call in UpdateTensorsInfo()
 256                 for (unsigned int i = 0; i < cnt; ++i) {
 257                         inference_engine_tensor_buffer in_buffer;
 258                         ml_tensor_type_e in_type;
 259
 260                         ret = ml_tensors_data_get_tensor_data(mInputDataHandle, i, &in_buffer.buffer, &in_buffer.size);
 261                         if (ret != ML_ERROR_NONE) {
 262                                 LOGE("Failed to request ml_tensors_data_get_tensor_data(%d).", ret);
 263                                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 264                         }
 265
 266                         LOGE("buffer = %p, size = %d\n", in_buffer.buffer, in_buffer.size);
 267
 268                         ret = ml_tensors_info_get_tensor_type(mInputInfoHandle, i, &in_type);
 269                         if (ret != ML_ERROR_NONE) {
 270                                 LOGE("Failed to request ml_tensors_info_get_tensor_type(%d).", ret);
 271                                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 272                         }
 273
 274                         LOGI("input tensor type = %d", in_type);
 275
 276                         int type = ConvertTensorType(in_type);
 277                         if (type == -1) {
 278                                 return INFERENCE_ENGINE_ERROR_NOT_SUPPORTED;
 279                         }
 280
 281                         in_buffer.data_type = static_cast<inference_tensor_data_type_e>(type);
 282                         in_buffer.owner_is_backend = 1;
 283
 284                         buffers.push_back(in_buffer);
 285                 }
 286
 287                 LOGI("LEAVE");
 288
 289                 return INFERENCE_ENGINE_ERROR_NONE;
 290         }
 291
 292         int InferenceMLAPI::GetOutputTensorBuffers(
 293                         std::vector<inference_engine_tensor_buffer> &buffers)
 294         {
 295                 LOGI("ENTER");
 296
 297                 // TODO. Need to check if model file loading is done.
 298
 299                 // ML Single API will always provide internal tensor buffers so
 300                 // get the tensor buffers back to Mediavision framework so that
 301                 // Mediavision framework doesn't allocate the tensor buffers internally.
 302
 303                 buffers.clear();
 304
 305                 int ret;
 306                 unsigned int cnt;
 307
 308                 ret = ml_tensors_info_get_count(mOutputInfoHandle, &cnt);
 309                 if (ret != ML_ERROR_NONE) {
 310                         LOGE("Failed to request ml_tensors_info_get_count(%d).", ret);
 311                         return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 312                 }
 313
 314                 LOGI("output tensor count = %u", cnt);
 315
 316                 // TODO. Below is test code, should we allocate new buffer for every inference?
 317                 if (mOutputDataHandle == NULL) {
 318                         ret = ml_tensors_data_create(mOutputInfoHandle, &mOutputDataHandle);
 319                         if (ret != ML_ERROR_NONE) {
 320                                 LOGE("Failed to request ml_tensors_data_create(%d).", ret);
 321                                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 322                         }
 323                 }
 324
 325                 // TODO. Cache tensor info and reduce function call in UpdateTensorsInfo()
 326                 for (unsigned int i = 0; i < cnt; ++i) {
 327                         inference_engine_tensor_buffer out_buffer;
 328                         ml_tensor_type_e out_type;
 329
 330                         ret = ml_tensors_data_get_tensor_data(mOutputDataHandle, i, &out_buffer.buffer, &out_buffer.size);
 331                         if (ret != ML_ERROR_NONE) {
 332                                 LOGE("Failed to request ml_tensors_data_get_tensor_data(%d).", ret);
 333                                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 334                         }
 335
 336                         LOGE("buffer = %p, size = %d\n", out_buffer.buffer, out_buffer.size);
 337
 338                         ret = ml_tensors_info_get_tensor_type(mOutputInfoHandle, i, &out_type);
 339                         if (ret != ML_ERROR_NONE) {
 340                                 LOGE("Failed to request ml_tensors_info_get_tensor_type(%d).", ret);
 341                                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 342                         }
 343
 344                         LOGI("output tensor type = %d", out_type);
 345
 346                         int type = ConvertTensorType(out_type);
 347                         if (type == -1) {
 348                                 return INFERENCE_ENGINE_ERROR_NOT_SUPPORTED;
 349                         }
 350
 351                         out_buffer.data_type = static_cast<inference_tensor_data_type_e>(type);
 352                         out_buffer.owner_is_backend = 1;
 353
 354                         buffers.push_back(out_buffer);
 355                 }
 356
 357                 LOGI("LEAVE");
 358
 359                 return INFERENCE_ENGINE_ERROR_NONE;
 360         }
 361
 362         int InferenceMLAPI::GetInputLayerProperty(
 363                         inference_engine_layer_property &property)
 364         {
 365                 LOGI("ENTER");
 366
 367                 // TODO. Need to check if model file loading is done.
 368                 int ret;
 369                 unsigned int cnt;
 370
 371                 ret = ml_tensors_info_get_count(mInputInfoHandle, &cnt);
 372                 if (ret != ML_ERROR_NONE) {
 373                         LOGE("Failed to request ml_tensors_info_get_count(%d).", ret);
 374                         return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 375                 }
 376
 377                 LOGI("input tensor count = %u", cnt);
 378
 379                 for (unsigned int i = 0; i < cnt; ++i) {
 380                         inference_engine_tensor_info tensor_info;
 381                         ml_tensor_type_e in_type;
 382                         ml_tensor_dimension in_dim;
 383                         char *in_name = NULL;
 384                         size_t in_size = 1;
 385
 386                         ret = ml_tensors_info_get_tensor_type(mInputInfoHandle, i, &in_type);
 387                         if (ret != ML_ERROR_NONE) {
 388                                 LOGE("Failed to request ml_tensors_info_get_tensor_type(%d).",
 389                                          ret);
 390                                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 391                         }
 392
 393                         LOGI("input tensor type = %d", in_type);
 394
 395                         int type = ConvertTensorType(in_type);
 396                         if (type == -1) {
 397                                 return INFERENCE_ENGINE_ERROR_NOT_SUPPORTED;
 398                         }
 399
 400                         ret = ml_tensors_info_get_tensor_dimension(mInputInfoHandle, i, in_dim);
 401                         if (ret != ML_ERROR_NONE) {
 402                                 LOGE("Failed to request ml_tensors_info_get_tensor_dimension(%d).",
 403                                          ret);
 404                                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 405                         }
 406
 407                         LOGI("Input tensor dimension:");
 408                         for (unsigned int shape_idx = 0; shape_idx < ML_TENSOR_RANK_LIMIT; ++shape_idx) {
 409                                 tensor_info.shape.push_back(in_dim[shape_idx]);
 410                                 in_size *= static_cast<size_t>(in_dim[shape_idx]);
 411                                 LOGI("%u", in_dim[shape_idx]);
 412                         }
 413
 414                         LOGI("input tensor size = %zu", in_size);
 415
 416                         ret = ml_tensors_info_get_tensor_name(mInputInfoHandle, i, &in_name);
 417                         if (ret != ML_ERROR_NONE) {
 418                                 LOGE("Failed to request ml_tensors_info_get_tensor_name(%d).",
 419                                          ret);
 420                                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 421                         }
 422
 423                         LOGI("input tensor name = %s", in_name);
 424
 425                         tensor_info.data_type = static_cast<inference_tensor_data_type_e>(type);
 426                         tensor_info.size = in_size;
 427
 428                         property.tensor_infos.push_back(tensor_info);
 429
 430                         // TODO. Compare tensor info from engine to one from a given property.
 431                 }
 432
 433                 property.layer_names = mInputProperty.layer_names;
 434
 435                 LOGI("LEAVE");
 436
 437                 return INFERENCE_ENGINE_ERROR_NONE;
 438         }
 439
 440         int InferenceMLAPI::GetOutputLayerProperty(
 441                         inference_engine_layer_property &property)
 442         {
 443                 LOGI("ENTER");
 444
 445                 // TODO. Need to check if model file loading is done.
 446                 int ret;
 447                 unsigned int cnt;
 448
 449                 ret = ml_tensors_info_get_count(mOutputInfoHandle, &cnt);
 450                 if (ret != ML_ERROR_NONE) {
 451                         LOGE("Failed to request ml_tensors_info_get_count(%d).", ret);
 452                         return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 453                 }
 454
 455                 LOGI("output tensor count = %u", cnt);
 456
 457                 for (unsigned int i = 0; i < cnt; ++i) {
 458                         inference_engine_tensor_info tensor_info;
 459                         ml_tensor_type_e out_type;
 460                         unsigned int out_dim[ML_TENSOR_RANK_LIMIT];
 461                         char *out_name = NULL;
 462                         size_t out_size = 1;
 463
 464                         ret = ml_tensors_info_get_tensor_type(mOutputInfoHandle, i, &out_type);
 465                         if (ret != ML_ERROR_NONE) {
 466                                 LOGE("Failed to request ml_tensors_info_get_tensor_type(%d).",
 467                                          ret);
 468                                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 469                         }
 470
 471                         LOGI("output tensor type = %d", out_type);
 472
 473                         int type = ConvertTensorType(out_type);
 474                         if (type == -1) {
 475                                 return INFERENCE_ENGINE_ERROR_NOT_SUPPORTED;
 476                         }
 477
 478                         ret = ml_tensors_info_get_tensor_dimension(mOutputInfoHandle, i, out_dim);
 479                         if (ret != ML_ERROR_NONE) {
 480                                 LOGE("Failed to request ml_tensors_info_get_tensor_dimension(%d).",
 481                                          ret);
 482                                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 483                         }
 484
 485                         int shape_size = 0;
 486
 487                         LOGI("Output tensor dimension:");
 488
 489                         for (unsigned int shape_idx = 0; shape_idx < ML_TENSOR_RANK_LIMIT; ++shape_idx) {
 490                                 out_size *= static_cast<size_t>(out_dim[shape_idx]);
 491
 492                                 if (out_dim[shape_idx] == 1 && shape_size == 0)
 493                                         shape_size = shape_idx;
 494
 495                                 LOGI("%d", out_dim[shape_idx]);
 496                         }
 497
 498                         LOGI("Shape size of output tensor : %d", shape_size);
 499                         LOGI("Reversed output tensor dimension:");
 500
 501                         // Reverse shape order.
 502                         for (int idx = shape_size; idx >= 0; --idx) {
 503                                 tensor_info.shape.push_back(out_dim[idx]);
 504                                 LOGI("%u", out_dim[idx]);
 505                         }
 506
 507                         LOGI("output tensor size = %zu", out_size);
 508
 509                         ret = ml_tensors_info_get_tensor_name(mOutputInfoHandle, i, &out_name);
 510                         if (ret != ML_ERROR_NONE) {
 511                                 LOGE("Failed to request ml_tensors_info_get_tensor_name(%d).",
 512                                          ret);
 513                                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 514                         }
 515
 516                         LOGI("output tensor name = %s", out_name);
 517
 518                         tensor_info.data_type = static_cast<inference_tensor_data_type_e>(type);
 519                         tensor_info.size = out_size;
 520
 521                         property.tensor_infos.push_back(tensor_info);
 522
 523                         // TODO. Compare tensor info from engine to one from a given property.
 524                 }
 525
 526                 property.layer_names = mOutputProperty.layer_names;
 527
 528                 LOGI("LEAVE");
 529
 530                 return INFERENCE_ENGINE_ERROR_NONE;
 531         }
 532
 533         int InferenceMLAPI::SetInputLayerProperty(
 534                         inference_engine_layer_property &property)
 535         {
 536                 LOGI("ENTER");
 537
 538                 std::vector<std::string>::iterator iter;
 539                 for (iter = property.layer_names.begin();
 540                          iter != property.layer_names.end(); iter++) {
 541                         std::string name = *iter;
 542                         LOGI("input layer name = %s", name.c_str());
 543                 }
 544
 545                 mDesignated_inputs.clear();
 546                 std::vector<std::string>().swap(mDesignated_inputs);
 547
 548                 // TODO. Request input property information to a given ML Single API of nnstreamer backend,
 549                 // and set it instead of user-given one,
 550                 // Call UpdateTensorsInfo() after requesting input info.
 551                 mDesignated_inputs = property.layer_names;
 552                 mInputProperty = property;
 553
 554                 LOGI("LEAVE");
 555
 556                 return INFERENCE_ENGINE_ERROR_NONE;
 557         }
 558
 559         int InferenceMLAPI::SetOutputLayerProperty(
 560                         inference_engine_layer_property &property)
 561         {
 562                 LOGI("ENTER");
 563
 564                 std::vector<std::string>::iterator iter;
 565                 for (iter = property.layer_names.begin();
 566                          iter != property.layer_names.end(); iter++) {
 567                         std::string name = *iter;
 568                         LOGI("output layer name = %s", name.c_str());
 569                 }
 570
 571                 mDesignated_outputs.clear();
 572                 std::vector<std::string>().swap(mDesignated_outputs);
 573
 574                 // TODO. Request output property information to a given ML Single API of nnstreamer backend,
 575                 // and set it instead of user-given one,
 576                 // Call UpdateTensorsInfo() after requesting output info.
 577                 mDesignated_outputs = property.layer_names;
 578                 mOutputProperty = property;
 579
 580                 LOGI("LEAVE");
 581
 582                 return INFERENCE_ENGINE_ERROR_NONE;
 583         }
 584
 585         int InferenceMLAPI::GetBackendCapacity(inference_engine_capacity *capacity)
 586         {
 587                 LOGI("ENTER");
 588
 589                 if (capacity == NULL) {
 590                         LOGE("Bad pointer.");
 591                         return INFERENCE_ENGINE_ERROR_INVALID_PARAMETER;
 592                 }
 593
 594                 // TODO. flag supported accel device types according to a given ML Single API of nnstreamer backend.
 595                 if (mPluginType == INFERENCE_BACKEND_NPU_VIVANTE) {
 596                         capacity->supported_accel_devices = INFERENCE_TARGET_CUSTOM;
 597                 } else {
 598                         capacity->supported_accel_devices = INFERENCE_TARGET_GPU |
 599                                                                                                 INFERENCE_TARGET_CPU;
 600                 }
 601
 602                 LOGI("LEAVE");
 603
 604                 return INFERENCE_ENGINE_ERROR_NONE;
 605         }
 606
 607         int InferenceMLAPI::CheckTensorBuffers(
 608                         std::vector<inference_engine_tensor_buffer> &input_buffers,
 609                         std::vector<inference_engine_tensor_buffer> &output_buffers)
 610         {
 611                 LOGI("ENTER");
 612
 613                 LOGI("LEAVE");
 614
 615                 return INFERENCE_ENGINE_ERROR_NONE;
 616         }
 617
 618         int InferenceMLAPI::ConvertTensorType(int tensor_type)
 619         {
 620                 LOGI("ENTER");
 621
 622                 switch (tensor_type) {
 623                 case ML_TENSOR_TYPE_FLOAT32:
 624                         return INFERENCE_TENSOR_DATA_TYPE_FLOAT32;
 625                 case ML_TENSOR_TYPE_UINT8:
 626                         return INFERENCE_TENSOR_DATA_TYPE_UINT8;
 627                 case ML_TENSOR_TYPE_UINT16:
 628                         return INFERENCE_TENSOR_DATA_TYPE_UINT16;
 629                 case ML_TENSOR_TYPE_INT64:
 630                         return INFERENCE_TENSOR_DATA_TYPE_INT64;
 631                 case ML_TENSOR_TYPE_UINT64:
 632                         return INFERENCE_TENSOR_DATA_TYPE_UINT64;
 633                 default:
 634                         LOGE("Tensor type(%d) is invalid.", tensor_type);
 635                         return INFERENCE_ENGINE_ERROR_INVALID_PARAMETER;
 636                 }
 637
 638                 LOGI("LEAVE");
 639
 640                 return -1;
 641         }
 642
 643         int InferenceMLAPI::UpdateTensorsInfo()
 644         {
 645                 LOGI("ENTER");
 646
 647                 if (!mSingle) {
 648                         LOGE("Invalid state, single-shot handle is not initialized.");
 649                         return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 650                 }
 651
 652                 if (mInputInfoHandle) {
 653                         ml_tensors_info_destroy(mInputInfoHandle);
 654                         mInputInfoHandle = NULL;
 655                 }
 656
 657                 if (mOutputInfoHandle) {
 658                         ml_tensors_info_destroy(mOutputInfoHandle);
 659                         mOutputInfoHandle = NULL;
 660                 }
 661
 662                 int ret = ml_single_get_input_info(mSingle, &mInputInfoHandle);
 663                 if (ret != ML_ERROR_NONE) {
 664                         LOGE("Failed to request ml_single_get_input_info(%d).", ret);
 665                         return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 666                 }
 667
 668                 ret = ml_single_get_output_info(mSingle, &mOutputInfoHandle);
 669                 if (ret != ML_ERROR_NONE) {
 670                         LOGE("Failed to request ml_single_get_output_info(%d).", ret);
 671                         return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 672                 }
 673
 674                 LOGI("LEAVE");
 675                 return INFERENCE_ENGINE_ERROR_NONE;
 676         }
 677
 678         int InferenceMLAPI::Run(
 679                         std::vector<inference_engine_tensor_buffer> &input_buffers,
 680                         std::vector<inference_engine_tensor_buffer> &output_buffers)
 681         {
 682                 LOGI("ENTER");
 683
 684                 // Make sure to check if tensor buffer count and binding info one are same.
 685                 int err = CheckTensorBuffers(input_buffers, output_buffers);
 686                 if (err != INFERENCE_ENGINE_ERROR_NONE) {
 687                         return err;
 688                 }
 689
 690 #if defined(ENABLE_NO_ALLOC)
 691                 err = ml_single_invoke_no_alloc(mSingle, mInputDataHandle, mOutputDataHandle);
 692                 if (err != ML_ERROR_NONE) {
 693                         LOGE("Failed to request ml_single_invoke_no_alloc(%d).", err);
 694                         return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 695                 }
 696 #else
 697                 ml_tensors_data_h out_data = NULL;
 698                 void *data_ptr;
 699                 size_t data_size;
 700                 unsigned int out_cnt;
 701
 702                 err = ml_tensors_info_get_count(mOutputInfoHandle, &out_cnt);
 703                 if (err != ML_ERROR_NONE) {
 704                         LOGE("Failed to request ml_tensors_info_get_count(%d).", err);
 705                         return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 706                 }
 707
 708                 // Be carefull, ml_single_invoke() returns newly allocated output handle.
 709                 err = ml_single_invoke(mSingle, mInputDataHandle, &out_data);
 710                 if (err != ML_ERROR_NONE) {
 711                         LOGE("Failed to request ml_single_invoke(%d).", err);
 712                         return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 713                 }
 714
 715                 for (unsigned int i = 0; i < out_cnt; ++i) {
 716                         err = ml_tensors_data_get_tensor_data(out_data, i, &data_ptr, &data_size);
 717                         if (err != ML_ERROR_NONE) {
 718                                 LOGE("Failed to request ml_tensors_data_get_tensor_data(%d).", err);
 719                                 ml_tensors_data_destroy(out_data);
 720                                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 721                         }
 722
 723                         // TODO. Remove memcpy() using ml_single_invoke_fill() later.
 724                         memcpy(output_buffers[i].buffer, data_ptr, output_buffers[i].size);
 725                         LOGI("Output tensor[%u] = %zu", i, output_buffers[i].size);
 726                 }
 727
 728                 ml_tensors_data_destroy(out_data);
 729 #endif
 730                 LOGI("LEAVE");
 731
 732                 return INFERENCE_ENGINE_ERROR_NONE;
 733         }
 734
 735         extern "C"
 736         {
 737                 class IInferenceEngineCommon *EngineCommonInit(void)
 738                 {
 739                         LOGI("ENTER");
 740
 741                         InferenceMLAPI *engine = new InferenceMLAPI();
 742
 743                         LOGI("LEAVE");
 744
 745                         return engine;
 746                 }
 747
 748                 void EngineCommonDestroy(class IInferenceEngineCommon *engine)
 749                 {
 750                         LOGI("ENTER");
 751
 752                         delete engine;
 753
 754                         LOGI("LEAVE");
 755                 }
 756         }
 757 } /* MLAPIImpl */
 758 } /* InferenceEngineImpl */