tools/src/inference_engine_cltuner.cpp

   1 /**
   2  * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include <glib.h>
  18 #include <glib/gprintf.h>
  19 #include <iostream>
  20 #include <json-glib/json-glib.h>
  21 #include <random>
  22
  23 #include <algorithm>
  24 #include <chrono>
  25 #include <fcntl.h>
  26 #include <fstream>
  27 #include <map>
  28 #include <queue>
  29 #include <string.h>
  30 #include <tuple>
  31 #include <unistd.h>
  32
  33 #include "inference_engine_cltuner.h"
  34
  35 extern "C"
  36 {
  37 #ifdef LOG_TAG
  38 #undef LOG_TAG
  39 #endif
  40 #define MAX_STR 256
  41 #define LOG_TAG "INFERENCE_ENGINE_CLTUNER"
  42 }
  43
  44 #define MAX_INFERENCE_COUNT 10
  45
  46 using namespace InferenceEngineInterface::Common;
  47 using namespace InferenceEngineInterface::Cltuner;
  48
  49 int ConfigureInputInfo(InferenceEngineCommon* backend, Metadata& metadata,
  50                        InferenceConfig& tensorConfig)
  51 {
  52         LOGI("ENTER");
  53
  54         const InputMetadata& inputMeta = metadata.GetInputMeta();
  55
  56         if (!inputMeta.parsed) {
  57                 LOGE("No meta data parsed.");
  58                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
  59         }
  60
  61         auto& layerInfo = inputMeta.layer.begin()->second;
  62
  63         if (layerInfo.shapeType == INFERENCE_TENSOR_SHAPE_NCHW) {
  64                 tensorConfig.mTensorInfo.ch = layerInfo.dims[1];
  65                 tensorConfig.mTensorInfo.dim = layerInfo.dims[0];
  66                 tensorConfig.mTensorInfo.width = layerInfo.dims[3];
  67                 tensorConfig.mTensorInfo.height = layerInfo.dims[2];
  68         } else if (layerInfo.shapeType == INFERENCE_TENSOR_SHAPE_NHWC) {
  69                 tensorConfig.mTensorInfo.ch = layerInfo.dims[3];
  70                 tensorConfig.mTensorInfo.dim = layerInfo.dims[0];
  71                 tensorConfig.mTensorInfo.width = layerInfo.dims[2];
  72                 tensorConfig.mTensorInfo.height = layerInfo.dims[1];
  73         } else {
  74                 LOGE("Invalid shape type[%d]", layerInfo.shapeType);
  75                 return INFERENCE_ENGINE_ERROR_INVALID_PARAMETER;
  76         }
  77
  78         if (!inputMeta.option.empty()) {
  79                 auto& option = inputMeta.option.begin()->second;
  80                 if (option.normalization.use) {
  81                         tensorConfig.mMeanValue = option.normalization.mean[0];
  82                         tensorConfig.mStdValue = option.normalization.std[0];
  83                 }
  84         }
  85
  86         if (layerInfo.dataType == 0)
  87                 tensorConfig.mDataType = INFERENCE_TENSOR_DATA_TYPE_FLOAT32;
  88         else
  89                 tensorConfig.mDataType = INFERENCE_TENSOR_DATA_TYPE_UINT8;
  90
  91         tensorConfig.mInputLayerNames.clear();
  92
  93         for (auto& layer : inputMeta.layer)
  94                 tensorConfig.mInputLayerNames.push_back(layer.first);
  95
  96         inference_engine_layer_property property;
  97
  98         for (auto& name : tensorConfig.mInputLayerNames) {
  99                 inference_engine_tensor_info tensor_info;
 100
 101                 tensor_info.data_type = tensorConfig.mDataType;
 102                 tensor_info.shape_type = INFERENCE_TENSOR_SHAPE_NCHW;
 103                 tensor_info.shape.push_back(tensorConfig.mTensorInfo.dim);
 104                 tensor_info.shape.push_back(tensorConfig.mTensorInfo.ch);
 105                 tensor_info.shape.push_back(tensorConfig.mTensorInfo.height);
 106                 tensor_info.shape.push_back(tensorConfig.mTensorInfo.width);
 107                 tensor_info.size = 1;
 108
 109                 for (auto& dim : tensor_info.shape)
 110                         tensor_info.size *= dim;
 111
 112                 property.layers.insert(std::make_pair(name, tensor_info));
 113         }
 114
 115         int ret = backend->SetInputLayerProperty(property);
 116
 117         if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 118                 LOGE("Fail to set input layer property");
 119                 return ret;
 120         }
 121
 122         LOGI("LEAVE");
 123         return INFERENCE_ENGINE_ERROR_NONE;
 124 }
 125
 126 int ConfigureOutputInfo(InferenceEngineCommon* backend, Metadata& metadata,
 127                         InferenceConfig& tensorConfig)
 128 {
 129         LOGI("ENTER");
 130
 131         OutputMetadata& outputMeta = metadata.GetOutputMeta();
 132
 133         if (!outputMeta.IsParsed()) {
 134                 LOGE("No meta data parsed.");
 135                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 136         }
 137
 138         tensorConfig.mOutputLayerNames.clear();
 139         if (!outputMeta.GetScore().GetName().empty())
 140                 tensorConfig.mOutputLayerNames.push_back(
 141                         outputMeta.GetScore().GetName());
 142
 143         if (!outputMeta.GetBox().GetName().empty())
 144                 tensorConfig.mOutputLayerNames.push_back(
 145                         outputMeta.GetBox().GetName());
 146
 147         if (!outputMeta.GetLabel().GetName().empty())
 148                 tensorConfig.mOutputLayerNames.push_back(
 149                         outputMeta.GetLabel().GetName());
 150
 151         if (!outputMeta.GetNumber().GetName().empty())
 152                 tensorConfig.mOutputLayerNames.push_back(
 153                         outputMeta.GetNumber().GetName());
 154
 155         if (!outputMeta.GetLandmark().GetName().empty())
 156                 tensorConfig.mOutputLayerNames.push_back(
 157                         outputMeta.GetLandmark().GetName());
 158
 159         if (!outputMeta.GetOffset().GetName().empty())
 160                 tensorConfig.mOutputLayerNames.push_back(
 161                         outputMeta.GetOffset().GetName());
 162
 163         for (auto& dispVec : outputMeta.GetDispVecAll())
 164                 tensorConfig.mOutputLayerNames.push_back(dispVec.GetName());
 165
 166         inference_engine_layer_property property;
 167         inference_engine_tensor_info tensor_info = {
 168             std::vector<size_t>{1}, INFERENCE_TENSOR_SHAPE_NCHW,
 169             INFERENCE_TENSOR_DATA_TYPE_FLOAT32, 1};
 170
 171         for (auto& name : tensorConfig.mOutputLayerNames) {
 172                 LOGI("Configure %s layer as output", name.c_str());
 173                 property.layers.insert(std::make_pair(name, tensor_info));
 174         }
 175
 176         int ret = backend->SetOutputLayerProperty(property);
 177
 178         if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 179                 LOGE("Fail to set output layer property");
 180                 return ret;
 181         }
 182
 183         LOGI("LEAVE");
 184         return INFERENCE_ENGINE_ERROR_NONE;
 185 }
 186
 187 int ParseMetadata(Metadata& metadata, std::string filePath)
 188 {
 189         LOGI("ENTER");
 190         LOGI("filePath : %s", filePath.c_str());
 191
 192         int ret = metadata.Init(filePath);
 193
 194         if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 195                 LOGE("Fail to init metadata[%d]", ret);
 196                 return ret;
 197         }
 198
 199         ret = metadata.Parse();
 200         if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 201                 LOGE("Fail to parse metadata[%d]", ret);
 202                 return ret;
 203         }
 204
 205         LOGI("LEAVE");
 206         return INFERENCE_ENGINE_ERROR_NONE;
 207 }
 208
 209 void _FillOutputResult(InferenceEngineCommon* engine, IETensorBuffer& outputs,
 210                        tensor_t& outputData)
 211 {
 212         inference_engine_layer_property property;
 213
 214         engine->GetOutputLayerProperty(property);
 215
 216         for (auto& layer : property.layers) {
 217                 const inference_engine_tensor_info& tensor_info = layer.second;
 218                 std::vector<int> tmpDimInfo;
 219
 220                 for (auto& dim : tensor_info.shape) {
 221                         LOGI("dim size %zu", dim);
 222                         tmpDimInfo.push_back(dim);
 223                 }
 224
 225                 outputData.dimInfo.push_back(tmpDimInfo);
 226
 227                 if (tensor_info.data_type == INFERENCE_TENSOR_DATA_TYPE_UINT8) {
 228                         auto* ori_buf = static_cast<unsigned char*>(
 229                             outputs[layer.first].buffer);
 230                         float* new_buf = new float[tensor_info.size];
 231
 232                         for (int j = 0; j < (int)tensor_info.size; j++) {
 233                                 new_buf[j] = (float)ori_buf[j] / 255.0f;
 234                         }
 235
 236                         // replace original buffer with new one, and release
 237                         // origin one.
 238                         outputs[layer.first].buffer = new_buf;
 239                         if (!outputs[layer.first].owner_is_backend) {
 240                                 delete[] ori_buf;
 241                         }
 242                 }
 243
 244                 LOGI("tensor_info.data_type  %d", tensor_info.data_type);
 245                 outputData.data.push_back(
 246                     static_cast<void*>(outputs[layer.first].buffer));
 247         }
 248 }
 249
 250 static void printTensor(tensor_t& outputData)
 251 {
 252         std::vector<std::vector<int>> inferDimInfo(outputData.dimInfo);
 253         std::vector<void*> inferResults(outputData.data.begin(),
 254                                         outputData.data.end());
 255         int count = inferDimInfo[0][1];
 256         int idx = -1;
 257         float value = 0.0f;
 258         float* prediction = reinterpret_cast<float*>(inferResults[0]);
 259
 260         for (int i = 0; i < count; ++i) {
 261                 LOGI(" prediction[%d] %f", i, prediction[i]);
 262                 if (value < prediction[i]) {
 263                         value = prediction[i];
 264                         idx = i;
 265                 }
 266         }
 267
 268         LOGI("Best Prediction  : prediction[%d] : %f ", idx, value);
 269 }
 270
 271 static void show_menu(const char* title)
 272 {
 273         g_print("*******************************************\n");
 274         g_print("*  %-38s *\n", title);
 275         g_print("*-----------------------------------------*\n");
 276         g_print("*  %-38s *\n", "Input Tuning mode and Model file");
 277         g_print("*  %-38s *\n", "ex)1 "
 278                                 "/usr/share/capi-media-vision/models/IC/tflite/"
 279                                 "ic_tflite_model.tflite");
 280         g_print("*  %-38s *\n", "**caution**");
 281         g_print("*  %-38s *\n", "'READ' mode should be executed");
 282         g_print("*  %-38s *\n", "after generating tune file.");
 283         g_print("*-----------------------------------------*\n");
 284         g_print("*  %-38s *\n", "[MODE LIST]");
 285         g_print("* %2i. %-34s *\n", 0, "INFERENCE_ENGINE_CLTUNER_READ ");
 286         g_print("* %2i. %-34s *\n", 1, "INFERENCE_ENGINE_CLTUNER_EXHAUSTIVE");
 287         g_print("* %2i. %-34s *\n", 2, "INFERENCE_ENGINE_CLTUNER_NORMAL    ");
 288         g_print("* %2i. %-34s *\n", 3, "INFERENCE_ENGINE_CLTUNER_RAPID     ");
 289         g_print("*-----------------------------------------*\n");
 290         g_print("* %2c. %34s *\n", 'q', "Exit  ");
 291         g_print("*******************************************\n\n");
 292 }
 293
 294 int CheckTuneFile(std::vector<std::string>& model_paths)
 295 {
 296         std::string tune_file = model_paths[0];
 297
 298         tune_file.append(".tune");
 299
 300         int fd = open(tune_file.c_str(), O_RDONLY);
 301
 302         if (fd == -1) {
 303                 g_print("Tune file open failed!! (It could be genereation "
 304                         "failure.)\n");
 305                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 306         }
 307
 308         off_t fsize;
 309
 310         fsize = lseek(fd, 0, SEEK_END);
 311         g_print("************TUNE FILE GENERATED**************\n");
 312         g_print("Location \n[%s] \nSize \n[%lld]\n", tune_file.c_str(),
 313                 (long long)fsize);
 314         g_print("*-------------------------------------------*\n\n\n");
 315         close(fd);
 316
 317         return INFERENCE_ENGINE_ERROR_NONE;
 318 }
 319
 320 int CopyRandomMatrixToMemory(inference_engine_tensor_buffer& buffer, InferenceConfig tensorConfig)
 321 {
 322         if (tensorConfig.mDataType <= INFERENCE_TENSOR_DATA_TYPE_NONE ||
 323                 tensorConfig.mDataType >= INFERENCE_TENSOR_DATA_TYPE_MAX) {
 324                 LOGE("tensorConfig.mDataType [%d] is not supported", tensorConfig.mDataType);
 325                 return INFERENCE_ENGINE_ERROR_INVALID_OPERATION;
 326         }
 327
 328         std::random_device rd;
 329         std::mt19937 generator(rd());
 330         std::uniform_real_distribution<> distribution(1.0, 255.0);
 331
 332         int height = tensorConfig.mTensorInfo.height;
 333         int width = tensorConfig.mTensorInfo.width;
 334         int ch = tensorConfig.mTensorInfo.ch;
 335         for (int h_offset = 0; h_offset < height; h_offset++)
 336                 for (int w_offset = 0; w_offset < width; w_offset++)
 337                         for (int ch_offset = 0; ch_offset < ch; ch_offset++) {
 338                                 int offset = h_offset * width * ch + w_offset * ch + ch_offset;
 339                                 if (tensorConfig.mDataType == INFERENCE_TENSOR_DATA_TYPE_FLOAT32)
 340                                         static_cast<float*>(buffer.buffer)[offset] = distribution(generator);
 341                                 else
 342                                         static_cast<char*>(buffer.buffer)[offset] = distribution(generator);
 343                         }
 344         return INFERENCE_ENGINE_ERROR_NONE;
 345 }
 346
 347 static gboolean process(std::vector<std::string>& model_paths,
 348                         tensor_t& result_tensor, Metadata& metadata,
 349                         bool is_supported, bool is_actived, bool is_updated,
 350                         inference_engine_cltuner_mode_e mode)
 351 {
 352         InferenceEngineCommon* backend;
 353         std::vector<std::string> models;
 354         inference_engine_cltuner cltuner;
 355         InferenceConfig tensorConfig;
 356         inference_engine_config engineConfig = {
 357             .backend_name = "armnn",
 358             .backend_type = INFERENCE_BACKEND_ARMNN,
 359             .target_devices = INFERENCE_TARGET_GPU};
 360
 361         backend = new InferenceEngineCommon();
 362
 363         int ret = backend->EnableProfiler(true);
 364         if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 365                 LOGE("EnableProfiler(); failed");
 366                 return FALSE;
 367         }
 368
 369         ret = backend->LoadConfigFile();
 370         if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 371                 LOGE("LoadConfigFile(); failed");
 372                 return FALSE;
 373         }
 374
 375         ret = backend->BindBackend(&engineConfig);
 376         if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 377                 LOGE("BindBackend failed");
 378                 return FALSE;
 379         }
 380
 381         inference_engine_capacity capacity;
 382
 383         ret = backend->GetBackendCapacity(&capacity);
 384         if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 385                 LOGE("GetBackendCapacity failed");
 386                 return FALSE;
 387         }
 388
 389         if (capacity.cltuner_supported && is_supported) {
 390                 LOGI("cltuner is set");
 391                 cltuner.active = is_actived;
 392                 cltuner.update = is_updated;
 393                 cltuner.tuning_mode = mode;
 394
 395                 ret = backend->SetCLTuner(&cltuner);
 396                 if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 397                         LOGE("SetCLTuner failed");
 398                         return FALSE;
 399                 }
 400         }
 401
 402         ret = backend->SetTargetDevices(engineConfig.target_devices);
 403         if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 404                 LOGE("SetTargetDevices failed");
 405                 return FALSE;
 406         }
 407
 408         int model_type = GetModelInfo(model_paths, models);
 409
 410         if (model_type <= INFERENCE_MODEL_NONE) {
 411                 LOGE("GetModelInfo failed");
 412                 return FALSE;
 413         }
 414
 415         ret = ConfigureInputInfo(backend, metadata, tensorConfig);
 416         if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 417                 LOGE("ConfigureInputInfo failed");
 418                 return FALSE;
 419         }
 420
 421         ret = ConfigureOutputInfo(backend, metadata, tensorConfig);
 422         if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 423                 LOGE("ConfigureOutputInfo failed");
 424                 return FALSE;
 425         }
 426
 427         ret = backend->Load(models, (inference_model_format_e)model_type);
 428         if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 429                 LOGE("Load failed");
 430                 return FALSE;
 431         }
 432
 433         IETensorBuffer inputs, outputs;
 434
 435         ret = PrepareTensorBuffers(backend, inputs, outputs);
 436         if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 437                 LOGE("PrepareTensorBuffers failed");
 438                 return FALSE;
 439         }
 440
 441         for (auto& input : inputs) {
 442                 LOGI("input.second.size :[%zu]", input.second.size);
 443                 ret = CopyRandomMatrixToMemory(input.second, tensorConfig);
 444                 if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 445                         LOGE("CopyRandomMatrixToMemory failed");
 446                         return FALSE;
 447                 }
 448         }
 449
 450         std::chrono::system_clock::time_point StartTime =
 451             std::chrono::system_clock::now();
 452
 453         for (int i = 0; i < MAX_INFERENCE_COUNT; i++) {
 454                 ret = backend->Run(inputs, outputs);
 455                 if (ret != INFERENCE_ENGINE_ERROR_NONE) {
 456                         LOGE("Run failed");
 457                         return FALSE;
 458                 }
 459         }
 460
 461         std::chrono::milliseconds ms =
 462             std::chrono::duration_cast<std::chrono::milliseconds>(
 463                 std::chrono::system_clock::now() - StartTime);
 464
 465         _FillOutputResult(backend, outputs, result_tensor);
 466         CleanupTensorBuffers(inputs, outputs);
 467         backend->UnbindBackend();
 468
 469         if (mode == INFERENCE_ENGINE_CLTUNER_READ) {
 470                 std::cout << "*****************************" << std::endl;
 471
 472                 if (is_actived == false)
 473                         std::cout << "Inference Time " << std::endl;
 474                 else
 475                         std::cout << "Average Inference Time with tune file"
 476                                   << std::endl;
 477
 478                 std::cout << ms.count() / 10 << " ms (10 times average)"
 479                           << std::endl;
 480                 std::cout << "*****************************" << std::endl;
 481         }
 482
 483         return TRUE;
 484 }
 485
 486 static gboolean __interpret(char* cmd, char* cmd2)
 487 {
 488         std::vector<std::string> model_paths;
 489         Metadata metadata;
 490         inference_engine_cltuner_mode_e tuning_mode;
 491         int res = 0;
 492         char* model_path;
 493         char* json_path;
 494
 495         if (strncmp(cmd, "", 1) != 0) {
 496                 if (strncmp(cmd, "q", 1) == 0)
 497                         return FALSE;
 498
 499                 char** value;
 500
 501                 tuning_mode = (inference_engine_cltuner_mode_e)atoi(cmd);
 502                 model_path = g_strdup(cmd2);
 503                 value = g_strsplit(cmd2, ".", 0);
 504                 json_path = g_strdup_printf("%s.json", value[0]);
 505                 model_paths.push_back(model_path);
 506
 507                 LOGI("model_path : [%s]\n", model_path);
 508                 LOGI("jsonfile path [%s] \n", json_path);
 509                 g_free(model_path);
 510                 g_strfreev(value);
 511
 512                 res = ParseMetadata(metadata, std::string(json_path));
 513                 g_free(json_path);
 514
 515                 if (res != INFERENCE_ENGINE_ERROR_NONE) {
 516                         LOGE("ParseMetadata failed");
 517                         return FALSE;
 518                 }
 519
 520                 if (tuning_mode == INFERENCE_ENGINE_CLTUNER_READ) {
 521                         tensor_t orig_tensor;
 522                         if (!process(model_paths, orig_tensor, metadata, false,
 523                                      false, false, tuning_mode)) {
 524                                 LOGE("Error is occurred while doing process. "
 525                                      "\n ");
 526                                 return FALSE;
 527                         }
 528
 529                         printTensor(orig_tensor);
 530
 531                         tensor_t tuned_tensor;
 532                         if (!process(model_paths, tuned_tensor, metadata, true,
 533                                      true, false, tuning_mode)) {
 534                                 LOGE("Error is occurred while doing process "
 535                                      "with tune file. "
 536                                      "\n ");
 537                                 return FALSE;
 538                         }
 539
 540                         printTensor(tuned_tensor);
 541                 } else {
 542                         tensor_t tuned_tensor;
 543                         if (!process(model_paths, tuned_tensor, metadata, true,
 544                                      true, true, tuning_mode)) {
 545                                 LOGE("Error is occurred while generating tune "
 546                                      "file. \n ");
 547                                 return FALSE;
 548                         }
 549
 550                         res = CheckTuneFile(model_paths);
 551                         if (res != INFERENCE_ENGINE_ERROR_NONE) {
 552                                 LOGE("CheckTuneFile failed");
 553                                 return FALSE;
 554                         }
 555                 }
 556         }
 557
 558         return TRUE;
 559 }
 560
 561 int main()
 562 {
 563         show_menu("CLtuner Generator");
 564
 565         char mode[MAX_STR];
 566         int ret = scanf("%s", mode);
 567
 568         if (strncmp(mode, "q", 1) == 0) {
 569                 g_print("exit!\n");
 570                 return 0;
 571         }
 572
 573         char file_path[MAX_STR];
 574
 575         ret = scanf("%s", file_path);
 576         if (ret == 0) {
 577                 g_print("wrong input.\n");
 578                 return -1;
 579         }
 580
 581         int _mode = atoi(mode);
 582
 583         if (_mode < 0 || _mode > 3) {
 584                 g_print(
 585                     "Check tuning mode. It could be out of between RAPID and "
 586                     "EXHAUST mode.(1~3)\n");
 587                 return -1;
 588         }
 589
 590         char** value = g_strsplit(file_path, ".", 0);
 591
 592         if (value[0] == NULL || value[1] == NULL) {
 593                 g_print("Check filepath. Please write full path. i.g "
 594                         "/root/model.tflite\n");
 595                 return -1;
 596         }
 597
 598         __interpret(mode, file_path);
 599
 600         return 0;
 601 }