mv_face/face/src/FaceRecognitionModel.cpp

   1 /**
   2  * Copyright (c) 2015 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include "FaceRecognitionModel.h"
  18
  19 #include "mv_private.h"
  20 #include "mv_common.h"
  21
  22 #include <map>
  23
  24 #include <stdio.h>
  25 #include <unistd.h>
  26
  27 namespace MediaVision {
  28 namespace Face {
  29 namespace {
  30
  31 unsigned int DefaultUnisizeWidth = 200;
  32 unsigned int DefaultUnisizeHeight = 200;
  33
  34 int CopyOpenCVAlgorithmParameters(const cv::Ptr<cv::FaceRecognizer>& srcAlg,
  35                 cv::Ptr<cv::FaceRecognizer>& dstAlg)
  36 {
  37         char tempPath[1024];
  38
  39         snprintf(tempPath, 1024, "/tmp/alg_copy_%p_%p", srcAlg.obj, dstAlg.obj);
  40
  41         srcAlg->save(tempPath);
  42         dstAlg->load(tempPath);
  43
  44         if (0 != remove(tempPath)) {
  45                 LOGW("Error removing serialized FaceRecognizer in %s", tempPath);
  46         }
  47
  48         /* todo: consider to uncomment this lines if OpenCV will support deep
  49         / copy of AlgorithmInfo objects: */
  50
  51         /*std::vector<std::string> paramNames;
  52         srcAlg->getParams(paramNames);
  53         size_t paramSize = paramNames.size();
  54         for (size_t i = 0; i < paramSize; ++i) {
  55                 int pType = srcAlg->paramType(paramNames[i]);
  56
  57                 switch(pType) {
  58                 case cv::Param::INT:
  59                 case cv::Param::UNSIGNED_INT:
  60                 case cv::Param::UINT64:
  61                 case cv::Param::SHORT:
  62                 case cv::Param::UCHAR:
  63                         dstAlg->set(paramNames[i], srcAlg->getInt(paramNames[i]));
  64                         break;
  65                 case cv::Param::BOOLEAN:
  66                         dstAlg->set(paramNames[i], srcAlg->getBool(paramNames[i]));
  67                         break;
  68                 case cv::Param::REAL:
  69                 case cv::Param::FLOAT:
  70                         dstAlg->set(paramNames[i], srcAlg->getDouble(paramNames[i]));
  71                         break;
  72                 case cv::Param::STRING:
  73                         dstAlg->set(paramNames[i], srcAlg->getString(paramNames[i]));
  74                         break;
  75                 case cv::Param::MAT:
  76                         dstAlg->set(paramNames[i], srcAlg->getMat(paramNames[i]));
  77                         break;
  78                 case cv::Param::MAT_VECTOR:
  79                 {
  80                         //std::vector<cv::Mat> value = srcAlg->getMatVector(paramNames[i]);
  81                         //dstAlg->info()->addParam(*(dstAlg.obj), paramNames[i].c_str(), value);
  82                         dstAlg->set(paramNames[i], srcAlg->getMatVector(paramNames[i]));
  83                         break;
  84                 }
  85                 case cv::Param::ALGORITHM:
  86                         dstAlg->set(paramNames[i], srcAlg->getAlgorithm(paramNames[i]));
  87                         break;
  88                 default:
  89                         LOGE("While copying algorothm parameters unsupported parameter "
  90                                 "%s was found.", paramNames[i].c_str());
  91
  92                         return MEDIA_VISION_ERROR_NOT_SUPPORTED;
  93                 }
  94         }*/
  95
  96         return MEDIA_VISION_ERROR_NONE;
  97 }
  98
  99 void ParseOpenCVLabels(
 100                 const cv::Ptr<cv::FaceRecognizer>& recognizer,
 101                 std::set<int>& outLabels)
 102 {
 103         if (!recognizer.empty()) {
 104                 cv::Mat labels = recognizer->getMat("labels");
 105                 for (int i = 0; i < labels.rows; ++i) {
 106                         outLabels.insert(labels.at<int>(i, 0));
 107                 }
 108         }
 109 }
 110
 111 } /* anonymous namespace */
 112
 113 FaceRecognitionModelConfig::FaceRecognitionModelConfig() :
 114                 mModelType(MEDIA_VISION_FACE_MODEL_TYPE_UNKNOWN),
 115                 mNumComponents(0),
 116                 mThreshold(DBL_MAX),
 117                 mRadius(1),
 118                 mNeighbors(8),
 119                 mGridX(8),
 120                 mGridY(8),
 121                 mImgWidth(DefaultUnisizeWidth),
 122                 mImgHeight(DefaultUnisizeHeight)
 123 {
 124         ; /* NULL */
 125 }
 126
 127 FaceRecognitionResults::FaceRecognitionResults() :
 128                 mIsRecognized(false),
 129                 mFaceLabel(-1),
 130                 mConfidence(0.0)
 131 {
 132         ; /* NULL */
 133 }
 134
 135 bool FaceRecognitionModelConfig::operator!=(
 136                 const FaceRecognitionModelConfig& other) const
 137 {
 138         return mModelType     != other.mModelType     ||
 139                         mNumComponents != other.mNumComponents ||
 140                         mThreshold     != other.mThreshold     ||
 141                         mRadius        != other.mRadius        ||
 142                         mNeighbors     != other.mNeighbors     ||
 143                         mGridX         != other.mGridX         ||
 144                         mGridY         != other.mGridY         ||
 145                         mImgWidth      != other.mImgWidth      ||
 146                         mImgHeight     != other.mImgHeight;
 147 }
 148
 149 FaceRecognitionModel::FaceRecognitionModel() :
 150                 m_canRecognize(false),
 151                 m_recognizer(NULL)
 152 {
 153         ; /* NULL */
 154 }
 155
 156 FaceRecognitionModel::FaceRecognitionModel(const FaceRecognitionModel& origin) :
 157                 m_canRecognize(origin.m_canRecognize),
 158                 m_faceSamples(origin.m_faceSamples),
 159                 m_learnAlgorithmConfig(origin.m_learnAlgorithmConfig),
 160                 m_recognizer(CreateRecognitionAlgorithm(origin.m_learnAlgorithmConfig)),
 161                 m_learnedLabels(origin.m_learnedLabels)
 162 {
 163         if (!m_recognizer.empty()) {
 164                 CopyOpenCVAlgorithmParameters(origin.m_recognizer, m_recognizer);
 165         }
 166 }
 167
 168 FaceRecognitionModel& FaceRecognitionModel::operator=(
 169                 const FaceRecognitionModel& copy)
 170 {
 171         if (this != &copy) {
 172                 m_canRecognize = copy.m_canRecognize;
 173                 m_faceSamples = copy.m_faceSamples;
 174                 m_learnAlgorithmConfig = copy.m_learnAlgorithmConfig;
 175                 m_recognizer = CreateRecognitionAlgorithm(m_learnAlgorithmConfig);
 176                 m_learnedLabels = copy.m_learnedLabels;
 177
 178                 if (!m_recognizer.empty()) {
 179                         CopyOpenCVAlgorithmParameters(copy.m_recognizer, m_recognizer);
 180                 }
 181         }
 182
 183         return *this;
 184 }
 185
 186 FaceRecognitionModel::~FaceRecognitionModel()
 187 {
 188         ; /* NULL */
 189 }
 190
 191 int FaceRecognitionModel::save(const std::string& fileName)
 192 {
 193         if (!m_recognizer.empty()) {
 194                 std::string filePath;
 195
 196                 filePath = fileName;
 197
 198                 std::string prefixPath = filePath.substr(0, filePath.find_last_of('/'));
 199                 LOGD("prefixPath: %s", prefixPath.c_str());
 200
 201                 /* check the directory is available */
 202                 if (access(prefixPath.c_str(), F_OK)) {
 203                         LOGE("Can't save recognition model. Path[%s] doesn't existed.", prefixPath.c_str());
 204
 205                         return MEDIA_VISION_ERROR_INVALID_PATH;
 206                 }
 207
 208                 cv::FileStorage storage(filePath, cv::FileStorage::WRITE);
 209                 if (!storage.isOpened()) {
 210                         LOGE("Can't save recognition model. Write to file permission denied.");
 211                         return MEDIA_VISION_ERROR_PERMISSION_DENIED;
 212                 }
 213
 214                 switch (m_learnAlgorithmConfig.mModelType) {
 215                 case MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES:
 216                         storage << "algorithm" << "Eigenfaces";
 217                         storage << "resizeW" << m_learnAlgorithmConfig.mImgWidth;
 218                         storage << "resizeH" << m_learnAlgorithmConfig.mImgHeight;
 219                         break;
 220                 case MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES:
 221                         storage << "algorithm" << "Fisherfaces";
 222                         storage << "resizeW" << m_learnAlgorithmConfig.mImgWidth;
 223                         storage << "resizeH" << m_learnAlgorithmConfig.mImgHeight;
 224                         break;
 225                 case MEDIA_VISION_FACE_MODEL_TYPE_LBPH:
 226                         storage << "algorithm" << "LBPH";
 227                         break;
 228                 default:
 229                         storage.release();
 230                         return MEDIA_VISION_ERROR_NOT_SUPPORTED_FORMAT;
 231                 }
 232
 233                 storage << "can_recognize" << m_canRecognize;
 234                 m_recognizer->save(storage);
 235
 236                 storage.release();
 237         } else {
 238                 LOGE("Attempt to save recognition model before learn");
 239                 return MEDIA_VISION_ERROR_INVALID_OPERATION;
 240         }
 241
 242         return MEDIA_VISION_ERROR_NONE;
 243 }
 244
 245 int FaceRecognitionModel::load(const std::string& fileName)
 246 {
 247         std::string filePath;
 248
 249         filePath = fileName;
 250
 251         if (access(filePath.c_str(), F_OK)) {
 252                 LOGE("Can't load face recognition model. File[%s] doesn't existed.", filePath.c_str());
 253
 254                 return MEDIA_VISION_ERROR_INVALID_PATH;
 255         }
 256
 257         cv::FileStorage storage(filePath, cv::FileStorage::READ);
 258         if (!storage.isOpened()) {
 259                 LOGE("Can't load recognition model. Read from file permission denied.");
 260
 261                 return MEDIA_VISION_ERROR_PERMISSION_DENIED;
 262         }
 263
 264         LOGD("Loading recognition model from file.");
 265
 266         std::string algName;
 267         int canRecognize = 0;
 268         storage["algorithm"] >> algName;
 269         storage["can_recognize"] >> canRecognize;
 270
 271         cv::Ptr<cv::FaceRecognizer> tempRecognizer;
 272         FaceRecognitionModelConfig tempConfig;
 273         std::set<int> tempLearnedLabels;
 274
 275         if (algName == "Eigenfaces") {
 276                 tempRecognizer = cv::createEigenFaceRecognizer();
 277                 storage["resizeW"] >> tempConfig.mImgWidth;
 278                 storage["resizeH"] >> tempConfig.mImgHeight;
 279                 tempRecognizer->load(storage);
 280                 tempConfig.mModelType =
 281                                 MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES;
 282                 tempConfig.mNumComponents =
 283                                 tempRecognizer->getInt("ncomponents");
 284                 ParseOpenCVLabels(tempRecognizer, tempLearnedLabels);
 285         } else if (algName == "Fisherfaces") {
 286                 tempRecognizer = cv::createFisherFaceRecognizer();
 287                 storage["resizeW"] >> tempConfig.mImgWidth;
 288                 storage["resizeH"] >> tempConfig.mImgHeight;
 289                 tempRecognizer->load(storage);
 290                 tempConfig.mModelType =
 291                                 MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES;
 292                 tempConfig.mNumComponents =
 293                                 tempRecognizer->getInt("ncomponents");
 294                 ParseOpenCVLabels(tempRecognizer, tempLearnedLabels);
 295         } else if (algName == "LBPH") {
 296                 tempRecognizer = cv::createLBPHFaceRecognizer();
 297                 tempRecognizer->load(storage);
 298                 tempConfig.mModelType =
 299                                 MEDIA_VISION_FACE_MODEL_TYPE_LBPH;
 300                 tempConfig.mGridX = tempRecognizer->getInt("grid_x");
 301                 tempConfig.mGridY = tempRecognizer->getInt("grid_y");
 302                 tempConfig.mNeighbors = tempRecognizer->getInt("neighbors");
 303                 tempConfig.mRadius = tempRecognizer->getInt("radius");
 304                 ParseOpenCVLabels(tempRecognizer, tempLearnedLabels);
 305         } else {
 306                 tempConfig = FaceRecognitionModelConfig();
 307                 LOGE("Failed to load face recognition model from file. File is in "
 308                         "unsupported format");
 309
 310                 storage.release();
 311
 312                 return MEDIA_VISION_ERROR_NOT_SUPPORTED_FORMAT;
 313         }
 314
 315         tempConfig.mThreshold = tempRecognizer->getDouble("threshold");
 316
 317         LOGD("Recognition model of [%s] type has been loaded from file",
 318                         algName.c_str());
 319
 320         storage.release();
 321
 322         m_recognizer = tempRecognizer;
 323         m_learnAlgorithmConfig = tempConfig;
 324         m_canRecognize = (canRecognize == 1);
 325         m_learnedLabels.clear();
 326         m_learnedLabels = tempLearnedLabels;
 327
 328         return MEDIA_VISION_ERROR_NONE;
 329 }
 330
 331 int FaceRecognitionModel::addFaceExample(
 332                 const cv::Mat& faceImage,
 333                 int faceLabel)
 334 {
 335         m_faceSamples[faceLabel].push_back(faceImage);
 336
 337         LOGD("Added face image example for label %i for recognition model",
 338                         faceLabel);
 339
 340         return MEDIA_VISION_ERROR_NONE;
 341 }
 342
 343 int FaceRecognitionModel::resetFaceExamples(void)
 344 {
 345         m_faceSamples.clear();
 346
 347         LOGD("All face image examples have been removed from recognition model");
 348
 349         return MEDIA_VISION_ERROR_NONE;
 350 }
 351
 352 int FaceRecognitionModel::resetFaceExamples(int faceLabel)
 353 {
 354         if (1 > m_faceSamples.erase(faceLabel)) {
 355                 LOGD("Failed to remove face image examples for label %i. "
 356                         "No such examples", faceLabel);
 357
 358                 return MEDIA_VISION_ERROR_KEY_NOT_AVAILABLE;
 359         }
 360
 361         LOGD("Face image examples for label %i have been removed from "
 362                         "recognition model", faceLabel);
 363
 364         return MEDIA_VISION_ERROR_NONE;
 365 }
 366
 367 const std::set<int>& FaceRecognitionModel::getFaceLabels(void) const
 368 {
 369         return m_learnedLabels;
 370 }
 371
 372 int FaceRecognitionModel::learn(const FaceRecognitionModelConfig& config)
 373 {
 374         /* Check number of classes collected for learning, some algorithms
 375          * require specific class number constraints. For example, Fisherfaces
 376          * requires more that 1 class in training set */
 377         if (MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES == config.mModelType &&
 378                         m_faceSamples.size() < 2) {
 379                 LOGE("Can't apply Fisherfaces learning algorithm. It requires at "
 380                                 "least two classes (face labes) to learn on.");
 381
 382                 return MEDIA_VISION_ERROR_INVALID_OPERATION;
 383         }
 384
 385         bool isIncremental = false;
 386         bool isUnisize = false;
 387
 388         if (MEDIA_VISION_FACE_MODEL_TYPE_LBPH == config.mModelType) {
 389                 isIncremental = true;
 390         }
 391
 392         if (MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES == config.mModelType ||
 393                 MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES == config.mModelType) {
 394                 isUnisize = true;
 395         }
 396
 397         std::vector<cv::Mat> samples;
 398         std::vector<int> labels;
 399         std::set<int> learnedLabels;
 400
 401         if (isIncremental) {
 402                 learnedLabels.insert(m_learnedLabels.begin(), m_learnedLabels.end());
 403         }
 404
 405         std::map<int, std::vector<cv::Mat> >::const_iterator it =
 406                         m_faceSamples.begin();
 407         for (; it != m_faceSamples.end(); ++it) {
 408                 const size_t faceClassSamplesSize = it->second.size();
 409                 labels.insert(labels.end(), faceClassSamplesSize, it->first);
 410                 learnedLabels.insert(it->first);
 411
 412                 if (!isUnisize) {
 413                         LOGD("%u examples has been added with label %i",
 414                                         it->second.size(), it->first);
 415                         samples.insert(samples.end(), it->second.begin(), it->second.end());
 416                 } else {
 417                         for (size_t sampleInd = 0; sampleInd < faceClassSamplesSize; ++sampleInd) {
 418                                 cv::Mat resizedSample;
 419                                 cv::resize(it->second[sampleInd],
 420                                 resizedSample,
 421                                 cv::Size(config.mImgWidth, config.mImgHeight),
 422                                 0.0, 0.0, cv::INTER_CUBIC);
 423                                 samples.push_back(resizedSample);
 424                         }
 425                 }
 426         }
 427
 428         const size_t samplesSize = samples.size();
 429         const size_t labelsSize = labels.size();
 430
 431         if (0 != samplesSize && samplesSize == labelsSize) {
 432                 LOGD("Start to learn the model for %u samples and %u labels",
 433                         samplesSize, labelsSize);
 434
 435                 if (m_learnAlgorithmConfig != config || m_recognizer.empty()) {
 436                         m_recognizer = CreateRecognitionAlgorithm(config);
 437                 }
 438
 439                 if (m_recognizer.empty()) {
 440                         LOGE("Can't create recognition algorithm for recognition model. "
 441                                 "Configuration is not supported by any of known algorithms.");
 442
 443                         return MEDIA_VISION_ERROR_NOT_SUPPORTED;
 444                 }
 445
 446                 isIncremental ? m_recognizer->update(samples, labels) :
 447                                                 m_recognizer->train(samples, labels);
 448                 m_canRecognize = true;
 449                 m_learnedLabels.clear();
 450                 m_learnedLabels = learnedLabels;
 451         } else {
 452                 LOGE("Can't create recognition algorithm for no examples. Try to add "
 453                                 "some face examples before learning");
 454
 455                 return MEDIA_VISION_ERROR_NO_DATA;
 456         }
 457
 458         m_learnAlgorithmConfig = config;
 459
 460         LOGD("Recognition model has been learned");
 461
 462         return MEDIA_VISION_ERROR_NONE;
 463 }
 464
 465 int FaceRecognitionModel::recognize(const cv::Mat& image, FaceRecognitionResults& results)
 466 {
 467         if (!m_recognizer.empty() && m_canRecognize) {
 468                 double absConf = 0.0;
 469                 cv::Mat predictionImg(m_learnAlgorithmConfig.mImgWidth,
 470                                 m_learnAlgorithmConfig.mImgHeight, CV_8UC1);
 471
 472                 if ((MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES == m_learnAlgorithmConfig.mModelType ||
 473                                 MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES == m_learnAlgorithmConfig.mModelType) &&
 474                                 (image.cols != m_learnAlgorithmConfig.mImgWidth ||
 475                                 image.rows != m_learnAlgorithmConfig.mImgHeight))
 476                         cv::resize(image, predictionImg, predictionImg.size());
 477                 else
 478                         predictionImg = image;
 479
 480                 m_recognizer->predict(predictionImg, results.mFaceLabel, absConf);
 481
 482                 if (-1 != results.mFaceLabel) {
 483                         double normShift = 7.5;
 484                         double normSmooth = 0.05;
 485
 486                         if (MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES == m_learnAlgorithmConfig.mModelType) {
 487                                 normShift = 5.0;
 488                                 normSmooth = 0.0015;
 489                         } else if (MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES == m_learnAlgorithmConfig.mModelType) {
 490                                 normShift = 5.0;
 491                                 normSmooth = 0.01;
 492                         }
 493
 494                         /* Normalize the absolute value of the confidence */
 495                         absConf = exp(normShift - (normSmooth * absConf));
 496                         results.mConfidence = absConf / (1 + absConf);
 497                         results.mIsRecognized = true;
 498                 } else {
 499                         results.mConfidence = 0.0;
 500                         results.mIsRecognized = false;
 501                 }
 502
 503                 results.mFaceLocation = cv::Rect(0, 0, image.cols, image.rows);
 504         } else {
 505                 LOGE("Attempt to recognize faces with untrained model");
 506                 return MEDIA_VISION_ERROR_INVALID_OPERATION;
 507         }
 508
 509         return MEDIA_VISION_ERROR_NONE;
 510 }
 511
 512 cv::Ptr<cv::FaceRecognizer> FaceRecognitionModel::CreateRecognitionAlgorithm(
 513                 const FaceRecognitionModelConfig& config)
 514 {
 515         cv::Ptr<cv::FaceRecognizer> tempRecognizer;
 516         switch (config.mModelType) {
 517         case MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES:
 518                 tempRecognizer = cv::createEigenFaceRecognizer(
 519                                                                         config.mNumComponents,
 520                                                                         config.mThreshold);
 521                 break;
 522         case MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES:
 523                 tempRecognizer = cv::createFisherFaceRecognizer(
 524                                                                         config.mNumComponents,
 525                                                                         config.mThreshold);
 526                 break;
 527         case MEDIA_VISION_FACE_MODEL_TYPE_LBPH:
 528                 tempRecognizer = cv::createLBPHFaceRecognizer(
 529                                                                         config.mRadius,
 530                                                                         config.mNeighbors,
 531                                                                         config.mGridX,
 532                                                                         config.mGridY,
 533                                                                         config.mThreshold);
 534                 break;
 535         default:
 536                 return NULL;
 537         }
 538
 539         return tempRecognizer;
 540 }
 541
 542 } /* Face */
 543 } /* MediaVision */