mv_machine_learning/face_recognition/src/simple_shot.cpp

   1 /**
   2  * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include <string.h>
  18 #include <fstream>
  19 #include <istream>
  20 #include <tuple>
  21 #include <map>
  22 #include <algorithm>
  23
  24 #include <sys/stat.h>
  25
  26 #include "machine_learning_exception.h"
  27 #include "simple_shot.h"
  28 #include "data_set_manager.h"
  29 #include "feature_vector_manager.h"
  30 #include "file_util.h"
  31
  32 using namespace std;
  33 using namespace TrainingEngineInterface::Common;
  34 using namespace mediavision::machine_learning::exception;
  35
  36 SimpleShot::SimpleShot(const training_backend_type_e backend_type, const training_target_type_e target_type,
  37                                            vector<size_t> input_tensor_shape, const string internal_model_file)
  38                 : TrainingModel(backend_type, target_type, input_tensor_shape, internal_model_file)
  39 {
  40         // Make sure to initialize input_tesnor_info.shape with { 1, 1, 1, 1}
  41         // because 4 channels are used as N C H W.
  42         inference_engine_tensor_info input_tensor_info = {
  43                 { 1, 1, 1, 1 }, INFERENCE_TENSOR_SHAPE_NCHW, INFERENCE_TENSOR_DATA_TYPE_FLOAT32, 1
  44         };
  45
  46         for (size_t idx = 0; idx < input_tensor_shape.size(); ++idx) {
  47                 input_tensor_info.size *= input_tensor_shape[idx];
  48                 input_tensor_info.shape[idx] = input_tensor_shape[idx];
  49         }
  50
  51         _engine_info.input_layer_names.push_back("preprocess_l2norm0");
  52         _engine_info.input_tensor_info.push_back(input_tensor_info);
  53
  54         // size of output tensor will be updated by RecognizeFace function
  55         // because the size should be changed according to maximum label count
  56         // so it has 1 in default.
  57         inference_engine_tensor_info output_tensor_info = { vector<size_t> { 1, 1, 1, 1 }, INFERENCE_TENSOR_SHAPE_NCHW,
  58                                                                                                                 INFERENCE_TENSOR_DATA_TYPE_FLOAT32, 1 };
  59
  60         _engine_info.output_layer_names.push_back("centroid_knn1");
  61         _engine_info.output_tensor_info.push_back(output_tensor_info);
  62
  63         _engine_info.optimizer_property = { .options = { "learning_rate=0.1" } };
  64         _engine_info.compile_property = { .options = { "batch_size=1" } };
  65 }
  66
  67 SimpleShot::~SimpleShot()
  68 {
  69         // If a model exists then destroy the model.
  70         if (_model)
  71                 _training->DestroyModel(_model.get());
  72 }
  73
  74 void SimpleShot::configureModel(int num_of_class)
  75 {
  76         // TODO. Check the capacity.
  77
  78         _model = _training->CreateModel();
  79         if (!_model)
  80                 throw InvalidOperation("Fail to create a model.");
  81
  82         auto l2norm = _training->CreateLayer(TRAINING_LAYER_TYPE_L2NORM);
  83         if (!l2norm)
  84                 throw InvalidOperation("Fail to create l2norm layer.");
  85
  86         auto knn = _training->CreateLayer(TRAINING_LAYER_TYPE_CENTROID_KNN);
  87         if (!knn)
  88                 throw InvalidOperation("Fail to create knn layer.");
  89
  90         inference_engine_tensor_info input_tensor_info = _engine_info.input_tensor_info[0];
  91         string input_shape_str = "input_shape=";
  92
  93         // NNTrainer needs revered tensor order so make sure to reverse the tensor order.
  94         reverse(input_tensor_info.shape.begin(), input_tensor_info.shape.end());
  95
  96         for (size_t shape_idx = 0; shape_idx < input_tensor_info.shape.size(); ++shape_idx) {
  97                 input_shape_str += to_string(input_tensor_info.shape[shape_idx]);
  98                 if (shape_idx < input_tensor_info.shape.size() - 1)
  99                         input_shape_str += ":";
 100         }
 101
 102         // Ps. In case of the first layer, input_shape property is mandatorily required.
 103         training_engine_layer_property l2norm_property = { .options = { input_shape_str.c_str(), "trainable=false" } };
 104
 105         int ret = _training->SetLayerProperty(l2norm.get(), l2norm_property);
 106         if (ret != TRAINING_ENGINE_ERROR_NONE)
 107                 throw InvalidOperation("Fail to set layer property.");
 108
 109         const string num_class_prop = "num_class=" + to_string(num_of_class);
 110         training_engine_layer_property knn_property = { .options = { num_class_prop, "trainable=false" } };
 111
 112         ret = _training->SetLayerProperty(knn.get(), knn_property);
 113         if (ret != TRAINING_ENGINE_ERROR_NONE)
 114                 throw InvalidOperation("Fail to set layer property.");
 115
 116         ret = _training->AddLayer(_model.get(), l2norm.get());
 117         if (ret != TRAINING_ENGINE_ERROR_NONE)
 118                 throw InvalidOperation("Fail to add l2norm layer.");
 119
 120         ret = _training->AddLayer(_model.get(), knn.get());
 121         if (ret != TRAINING_ENGINE_ERROR_NONE)
 122                 throw InvalidOperation("Fail to add knn layer.");
 123 }
 124
 125 TrainingEngineBackendInfo &SimpleShot::getTrainingEngineInfo()
 126 {
 127         return _engine_info;
 128 }
 129
 130 void SimpleShot::saveModel(const string file_path)
 131 {
 132         string bin_file_path = file_path.substr(0, file_path.find('.')) + ".bin";
 133         int ret = 0;
 134
 135         // NNStreamer returns an error if internal model file(ini and bin files) exists before generating it.
 136         // So remove existing files.
 137         if (FaceRecogUtil::isFileExist(bin_file_path)) {
 138                 ret = ::remove(bin_file_path.c_str());
 139                 if (ret)
 140                         throw InvalidOperation("Fail to remove internal model file.");
 141         }
 142
 143         if (FaceRecogUtil::isFileExist(file_path)) {
 144                 ret = ::remove(file_path.c_str());
 145                 if (ret)
 146                         throw InvalidOperation("Fail to remove internal model file.");
 147         }
 148
 149         ret = _training->SaveModel(_model.get(), file_path);
 150         if (ret != TRAINING_ENGINE_ERROR_NONE)
 151                 throw InvalidOperation("Fail to save a model.");
 152 }
 153
 154 void SimpleShot::removeModel(const string file_path)
 155 {
 156         string bin_file_path = file_path.substr(0, file_path.find('.')) + ".bin";
 157         int ret = 0;
 158
 159         // Remove existing files forcely.
 160         if (FaceRecogUtil::isFileExist(bin_file_path)) {
 161                 ret = ::remove(bin_file_path.c_str());
 162                 if (ret)
 163                         throw InvalidOperation("Fail to remove internal model file.");
 164         }
 165
 166         if (FaceRecogUtil::isFileExist(file_path)) {
 167                 ret = ::remove(file_path.c_str());
 168                 if (ret)
 169                         throw InvalidOperation("Fail to remove internal model file.");
 170         }
 171 }