mv_face/face/src/FaceRecognitionModel.cpp

   1 /**
   2  * Copyright (c) 2015 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include "FaceRecognitionModel.h"
  18
  19 #include "mv_private.h"
  20 #include "mv_common.h"
  21
  22 #include <map>
  23
  24 #include <stdio.h>
  25 #include <unistd.h>
  26 #include <fstream>
  27
  28 namespace MediaVision {
  29 namespace Face {
  30 namespace {
  31
  32 unsigned int DefaultUnisizeWidth = 200;
  33 unsigned int DefaultUnisizeHeight = 200;
  34
  35 bool isEmptyAlgorithmParam(const std::string& path)
  36 {
  37         char valid[256] = "";
  38         std::ifstream ifs;
  39
  40         ifs.open(path.c_str());
  41
  42         if (!ifs.is_open()) {
  43                 LOGE("[%s] Can't open file.", path.c_str());
  44                 return true;
  45         }
  46
  47         ifs.getline(valid, 256);
  48         ifs.close();
  49
  50         LOGD("Validation string: %s", valid);
  51         if (strlen(valid) <= 0) {
  52                 LOGE("algorithm params is empty.");
  53                 return true;
  54         }
  55
  56         return false;
  57 }
  58
  59 int CopyOpenCVAlgorithmParameters(const cv::Ptr<cv::face::FaceRecognizer>& srcAlg,
  60                 cv::Ptr<cv::face::FaceRecognizer>& dstAlg)
  61 {
  62         char tempPath[1024] = "";
  63
  64         snprintf(tempPath, 1024, "/tmp/alg_copy_%p_%p", srcAlg.get(), dstAlg.get());
  65
  66         srcAlg->write(tempPath);
  67
  68         if (!isEmptyAlgorithmParam(tempPath))
  69                 dstAlg->read(tempPath);
  70
  71         if (0 != remove(tempPath))
  72                 LOGW("Error removing serialized FaceRecognizer in %s", tempPath);
  73
  74         /* todo: consider to uncomment this lines if OpenCV will support deep
  75                         copy of AlgorithmInfo objects: */
  76
  77         /*std::vector<std::string> paramNames;
  78         srcAlg->getParams(paramNames);
  79         size_t paramSize = paramNames.size();
  80         for (size_t i = 0; i < paramSize; ++i) {
  81                 int pType = srcAlg->paramType(paramNames[i]);
  82
  83                 switch(pType) {
  84                 case cv::Param::INT:
  85                 case cv::Param::UNSIGNED_INT:
  86                 case cv::Param::UINT64:
  87                 case cv::Param::SHORT:
  88                 case cv::Param::UCHAR:
  89                         dstAlg->set(paramNames[i], srcAlg->getInt(paramNames[i]));
  90                         break;
  91                 case cv::Param::BOOLEAN:
  92                         dstAlg->set(paramNames[i], srcAlg->getBool(paramNames[i]));
  93                         break;
  94                 case cv::Param::REAL:
  95                 case cv::Param::FLOAT:
  96                         dstAlg->set(paramNames[i], srcAlg->getDouble(paramNames[i]));
  97                         break;
  98                 case cv::Param::STRING:
  99                         dstAlg->set(paramNames[i], srcAlg->getString(paramNames[i]));
 100                         break;
 101                 case cv::Param::MAT:
 102                         dstAlg->set(paramNames[i], srcAlg->getMat(paramNames[i]));
 103                         break;
 104                 case cv::Param::MAT_VECTOR:
 105                 {
 106                         //std::vector<cv::Mat> value = srcAlg->getMatVector(paramNames[i]);
 107                         //dstAlg->info()->addParam(*(dstAlg.obj), paramNames[i].c_str(), value);
 108                         dstAlg->set(paramNames[i], srcAlg->getMatVector(paramNames[i]));
 109                         break;
 110                 }
 111                 case cv::Param::ALGORITHM:
 112                         dstAlg->set(paramNames[i], srcAlg->getAlgorithm(paramNames[i]));
 113                         break;
 114                 default:
 115                         LOGE("While copying algorothm parameters unsupported parameter "
 116                                 "%s was found.", paramNames[i].c_str());
 117
 118                         return MEDIA_VISION_ERROR_NOT_SUPPORTED;
 119                 }
 120         }*/
 121
 122         return MEDIA_VISION_ERROR_NONE;
 123 }
 124
 125 void ParseOpenCVLabels(
 126                 const cv::Mat labels,
 127                 std::set<int>& outLabels)
 128 {
 129         if (!labels.empty()) {
 130                 for (int i = 0; i < labels.rows; ++i)
 131                         outLabels.insert(labels.at<int>(i, 0));
 132         }
 133 }
 134
 135 } /* anonymous namespace */
 136
 137 FaceRecognitionModelConfig::FaceRecognitionModelConfig() :
 138                 mModelType(MEDIA_VISION_FACE_MODEL_TYPE_UNKNOWN),
 139                 mNumComponents(0),
 140                 mThreshold(DBL_MAX),
 141                 mRadius(1),
 142                 mNeighbors(8),
 143                 mGridX(8),
 144                 mGridY(8),
 145                 mImgWidth(DefaultUnisizeWidth),
 146                 mImgHeight(DefaultUnisizeHeight)
 147 {
 148         ; /* NULL */
 149 }
 150
 151 FaceRecognitionResults::FaceRecognitionResults() :
 152                 mIsRecognized(false),
 153                 mFaceLabel(-1),
 154                 mConfidence(0.0)
 155 {
 156         ; /* NULL */
 157 }
 158
 159 bool FaceRecognitionModelConfig::operator!=(
 160                 const FaceRecognitionModelConfig& other) const
 161 {
 162         return mModelType      != other.mModelType     ||
 163                         mNumComponents != other.mNumComponents ||
 164                         mThreshold     != other.mThreshold     ||
 165                         mRadius        != other.mRadius        ||
 166                         mNeighbors     != other.mNeighbors     ||
 167                         mGridX         != other.mGridX         ||
 168                         mGridY         != other.mGridY         ||
 169                         mImgWidth      != other.mImgWidth      ||
 170                         mImgHeight     != other.mImgHeight;
 171 }
 172
 173 FaceRecognitionModel::FaceRecognitionModel() :
 174                 m_canRecognize(false),
 175                 m_recognizer() // The default constructor creates a null Ptr
 176 {
 177         ; /* NULL */
 178 }
 179
 180 FaceRecognitionModel::FaceRecognitionModel(const FaceRecognitionModel& origin) :
 181                 m_canRecognize(origin.m_canRecognize),
 182                 m_faceSamples(origin.m_faceSamples),
 183                 m_learnAlgorithmConfig(origin.m_learnAlgorithmConfig),
 184                 m_recognizer(CreateRecognitionAlgorithm(origin.m_learnAlgorithmConfig)),
 185                 m_learnedLabels(origin.m_learnedLabels)
 186 {
 187         if (!m_recognizer.empty())
 188                 CopyOpenCVAlgorithmParameters(origin.m_recognizer, m_recognizer);
 189 }
 190
 191 FaceRecognitionModel& FaceRecognitionModel::operator=(
 192                 const FaceRecognitionModel& copy)
 193 {
 194         if (this != &copy) {
 195                 m_canRecognize = copy.m_canRecognize;
 196                 m_faceSamples = copy.m_faceSamples;
 197                 m_learnAlgorithmConfig = copy.m_learnAlgorithmConfig;
 198                 m_recognizer = CreateRecognitionAlgorithm(m_learnAlgorithmConfig);
 199                 m_learnedLabels = copy.m_learnedLabels;
 200
 201                 if (!m_recognizer.empty())
 202                         CopyOpenCVAlgorithmParameters(copy.m_recognizer, m_recognizer);
 203         }
 204
 205         return *this;
 206 }
 207
 208 FaceRecognitionModel::~FaceRecognitionModel()
 209 {
 210         ; /* NULL */
 211 }
 212
 213 int FaceRecognitionModel::save(const std::string& fileName)
 214 {
 215         if (!m_recognizer.empty()) {
 216                 std::string filePath;
 217
 218                 filePath = fileName;
 219
 220                 std::string prefixPath = filePath.substr(0, filePath.find_last_of('/'));
 221                 LOGD("prefixPath: %s", prefixPath.c_str());
 222
 223                 /* check the directory is available */
 224                 if (access(prefixPath.c_str(), F_OK)) {
 225                         LOGE("Can't save recognition model. Path[%s] doesn't existed.", prefixPath.c_str());
 226
 227                         return MEDIA_VISION_ERROR_INVALID_PATH;
 228                 }
 229
 230                 cv::FileStorage storage(filePath, cv::FileStorage::WRITE);
 231                 if (!storage.isOpened()) {
 232                         LOGE("Can't save recognition model. Write to file permission denied.");
 233                         return MEDIA_VISION_ERROR_PERMISSION_DENIED;
 234                 }
 235
 236                 switch (m_learnAlgorithmConfig.mModelType) {
 237                 case MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES:
 238                         storage << "algorithm" << "Eigenfaces";
 239                         storage << "resizeW" << m_learnAlgorithmConfig.mImgWidth;
 240                         storage << "resizeH" << m_learnAlgorithmConfig.mImgHeight;
 241                         break;
 242                 case MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES:
 243                         storage << "algorithm" << "Fisherfaces";
 244                         storage << "resizeW" << m_learnAlgorithmConfig.mImgWidth;
 245                         storage << "resizeH" << m_learnAlgorithmConfig.mImgHeight;
 246                         break;
 247                 case MEDIA_VISION_FACE_MODEL_TYPE_LBPH:
 248                         storage << "algorithm" << "LBPH";
 249                         break;
 250                 default:
 251                         storage.release();
 252                         return MEDIA_VISION_ERROR_NOT_SUPPORTED_FORMAT;
 253                 }
 254
 255                 storage << "can_recognize" << m_canRecognize;
 256                 m_recognizer->write(storage);
 257
 258                 storage.release();
 259         } else {
 260                 LOGE("Attempt to save recognition model before learn");
 261                 return MEDIA_VISION_ERROR_INVALID_OPERATION;
 262         }
 263
 264         return MEDIA_VISION_ERROR_NONE;
 265 }
 266
 267 int FaceRecognitionModel::load(const std::string& fileName)
 268 {
 269         std::string filePath;
 270
 271         filePath = fileName;
 272
 273         if (access(filePath.c_str(), F_OK)) {
 274                 LOGE("Can't load face recognition model. File[%s] doesn't existed.", filePath.c_str());
 275
 276                 return MEDIA_VISION_ERROR_INVALID_PATH;
 277         }
 278
 279         cv::FileStorage storage(filePath, cv::FileStorage::READ);
 280         if (!storage.isOpened()) {
 281                 LOGE("Can't load recognition model. Read from file permission denied.");
 282
 283                 return MEDIA_VISION_ERROR_PERMISSION_DENIED;
 284         }
 285
 286         LOGD("Loading recognition model from file.");
 287
 288         std::string algName;
 289         int canRecognize = 0;
 290         storage["algorithm"] >> algName;
 291         storage["can_recognize"] >> canRecognize;
 292
 293         cv::Ptr<cv::face::FaceRecognizer> tempRecognizer;
 294         FaceRecognitionModelConfig tempConfig;
 295         std::set<int> tempLearnedLabels;
 296         cv::Mat labels;
 297
 298         if (algName == "Eigenfaces") {
 299                 tempRecognizer = cv::face::EigenFaceRecognizer::create();
 300                 storage["resizeW"] >> tempConfig.mImgWidth;
 301                 storage["resizeH"] >> tempConfig.mImgHeight;
 302                 tempRecognizer->read(storage.root());
 303                 tempConfig.mModelType =
 304                                 MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES;
 305                 cv::face::EigenFaceRecognizer* recognizer = dynamic_cast<cv::face::EigenFaceRecognizer*>(tempRecognizer.get());
 306                 if (recognizer != NULL) {
 307                         tempConfig.mNumComponents = recognizer->getNumComponents();
 308                         labels = recognizer->getLabels();
 309                 } else {
 310                         return MEDIA_VISION_ERROR_NOT_SUPPORTED_FORMAT;
 311                 }
 312         } else if (algName == "Fisherfaces") {
 313                 tempRecognizer = cv::face::FisherFaceRecognizer::create();
 314                 storage["resizeW"] >> tempConfig.mImgWidth;
 315                 storage["resizeH"] >> tempConfig.mImgHeight;
 316                 tempRecognizer->read(storage.root());
 317                 tempConfig.mModelType =
 318                                 MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES;
 319                 cv::face::FisherFaceRecognizer* recognizer = dynamic_cast<cv::face::FisherFaceRecognizer*>(tempRecognizer.get());
 320                 if (recognizer != NULL) {
 321                         tempConfig.mNumComponents = recognizer->getNumComponents();
 322                         labels = recognizer->getLabels();
 323                 } else {
 324                         return MEDIA_VISION_ERROR_NOT_SUPPORTED_FORMAT;
 325                 }
 326         } else if (algName == "LBPH") {
 327                 tempRecognizer = cv::face::LBPHFaceRecognizer::create();
 328                 tempRecognizer->read(storage.root());
 329                 cv::face::LBPHFaceRecognizer* recognizer = dynamic_cast<cv::face::LBPHFaceRecognizer*>(tempRecognizer.get());
 330                 if (recognizer != NULL) {
 331                         tempConfig.mModelType =
 332                                         MEDIA_VISION_FACE_MODEL_TYPE_LBPH;
 333                         tempConfig.mGridX = recognizer->getGridX();
 334                         tempConfig.mGridY = recognizer->getGridY();
 335                         tempConfig.mNeighbors = recognizer->getNeighbors();
 336                         tempConfig.mRadius = recognizer->getRadius();
 337                         labels = recognizer->getLabels();
 338                 } else {
 339                         return MEDIA_VISION_ERROR_NOT_SUPPORTED_FORMAT;
 340                 }
 341         } else {
 342                 tempConfig = FaceRecognitionModelConfig();
 343                 LOGE("Failed to load face recognition model from file. File is in "
 344                                 "unsupported format");
 345
 346                 storage.release();
 347
 348                 return MEDIA_VISION_ERROR_NOT_SUPPORTED_FORMAT;
 349         }
 350         ParseOpenCVLabels(labels, tempLearnedLabels);
 351         tempConfig.mThreshold = tempRecognizer->getThreshold();
 352
 353         LOGD("Recognition model of [%s] type has been loaded from file",
 354                         algName.c_str());
 355
 356         storage.release();
 357
 358         m_recognizer = tempRecognizer;
 359         m_learnAlgorithmConfig = tempConfig;
 360         m_canRecognize = (canRecognize == 1);
 361         m_learnedLabels.clear();
 362         m_learnedLabels = tempLearnedLabels;
 363
 364         return MEDIA_VISION_ERROR_NONE;
 365 }
 366
 367 int FaceRecognitionModel::addFaceExample(
 368                 const cv::Mat& faceImage,
 369                 int faceLabel)
 370 {
 371         m_faceSamples[faceLabel].push_back(faceImage);
 372
 373         LOGD("Added face image example for label %i for recognition model",
 374                         faceLabel);
 375
 376         return MEDIA_VISION_ERROR_NONE;
 377 }
 378
 379 int FaceRecognitionModel::resetFaceExamples(void)
 380 {
 381         m_faceSamples.clear();
 382
 383         LOGD("All face image examples have been removed from recognition model");
 384
 385         return MEDIA_VISION_ERROR_NONE;
 386 }
 387
 388 int FaceRecognitionModel::resetFaceExamples(int faceLabel)
 389 {
 390         if (1 > m_faceSamples.erase(faceLabel)) {
 391                 LOGD("Failed to remove face image examples for label %i. "
 392                                 "No such examples", faceLabel);
 393
 394                 return MEDIA_VISION_ERROR_KEY_NOT_AVAILABLE;
 395         }
 396
 397         LOGD("Face image examples for label %i have been removed from "
 398                         "recognition model", faceLabel);
 399
 400         return MEDIA_VISION_ERROR_NONE;
 401 }
 402
 403 const std::set<int>& FaceRecognitionModel::getFaceLabels(void) const
 404 {
 405         return m_learnedLabels;
 406 }
 407
 408 int FaceRecognitionModel::learn(const FaceRecognitionModelConfig& config)
 409 {
 410         /* Check number of classes collected for learning, some algorithms
 411          * require specific class number constraints. For example, Fisherfaces
 412          * requires more that 1 class in training set */
 413         if (MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES == config.mModelType &&
 414                         m_faceSamples.size() < 2) {
 415                 LOGE("Can't apply Fisherfaces learning algorithm. It requires at "
 416                                 "least two classes (face labes) to learn on.");
 417
 418                 return MEDIA_VISION_ERROR_INVALID_OPERATION;
 419         }
 420
 421         bool isIncremental = false;
 422         bool isUnisize = false;
 423
 424         if (MEDIA_VISION_FACE_MODEL_TYPE_LBPH == config.mModelType)
 425                 isIncremental = true;
 426
 427         if (MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES == config.mModelType ||
 428                         MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES == config.mModelType)
 429                 isUnisize = true;
 430
 431         std::vector<cv::Mat> samples;
 432         std::vector<int> labels;
 433         std::set<int> learnedLabels;
 434
 435         if (isIncremental)
 436                 learnedLabels.insert(m_learnedLabels.begin(), m_learnedLabels.end());
 437
 438         std::map<int, std::vector<cv::Mat> >::const_iterator it = m_faceSamples.begin();
 439
 440         for (; it != m_faceSamples.end(); ++it) {
 441                 const size_t faceClassSamplesSize = it->second.size();
 442                 labels.insert(labels.end(), faceClassSamplesSize, it->first);
 443                 learnedLabels.insert(it->first);
 444
 445                 if (!isUnisize) {
 446                         LOGD("%zu examples has been added with label %i",
 447                                         it->second.size(), it->first);
 448                         samples.insert(samples.end(), it->second.begin(), it->second.end());
 449                 } else {
 450                         for (size_t sampleInd = 0; sampleInd < faceClassSamplesSize; ++sampleInd) {
 451                                 cv::Mat resizedSample;
 452                                 cv::resize(it->second[sampleInd],
 453                                                 resizedSample,
 454                                                 cv::Size(config.mImgWidth, config.mImgHeight),
 455                                                 0.0, 0.0, cv::INTER_CUBIC);
 456                                 samples.push_back(resizedSample);
 457                         }
 458                 }
 459         }
 460
 461         const size_t samplesSize = samples.size();
 462         const size_t labelsSize = labels.size();
 463
 464         if (0 != samplesSize && samplesSize == labelsSize) {
 465                 LOGD("Start to learn the model for %zu samples and %zu labels",
 466                                 samplesSize, labelsSize);
 467
 468                 if (m_learnAlgorithmConfig != config || m_recognizer.empty())
 469                         m_recognizer = CreateRecognitionAlgorithm(config);
 470
 471                 if (m_recognizer.empty()) {
 472                         LOGE("Can't create recognition algorithm for recognition model. "
 473                                         "Configuration is not supported by any of known algorithms.");
 474
 475                         return MEDIA_VISION_ERROR_NOT_SUPPORTED;
 476                 }
 477
 478                 isIncremental ? m_recognizer->update(samples, labels) :
 479                                                 m_recognizer->train(samples, labels);
 480                 m_canRecognize = true;
 481                 m_learnedLabels.clear();
 482                 m_learnedLabels = learnedLabels;
 483         } else {
 484                 LOGE("Can't create recognition algorithm for no examples. Try to add "
 485                                 "some face examples before learning");
 486
 487                 return MEDIA_VISION_ERROR_NO_DATA;
 488         }
 489
 490         m_learnAlgorithmConfig = config;
 491
 492         LOGD("Recognition model has been learned");
 493
 494         return MEDIA_VISION_ERROR_NONE;
 495 }
 496
 497 int FaceRecognitionModel::recognize(const cv::Mat& image, FaceRecognitionResults& results)
 498 {
 499         if (!m_recognizer.empty() && m_canRecognize) {
 500                 double absConf = 0.0;
 501                 cv::Mat predictionImg(m_learnAlgorithmConfig.mImgWidth,
 502                                 m_learnAlgorithmConfig.mImgHeight, CV_8UC1);
 503
 504                 if ((MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES == m_learnAlgorithmConfig.mModelType ||
 505                                 MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES == m_learnAlgorithmConfig.mModelType) &&
 506                                 (image.cols != m_learnAlgorithmConfig.mImgWidth ||
 507                                 image.rows != m_learnAlgorithmConfig.mImgHeight))
 508                         cv::resize(image, predictionImg, predictionImg.size());
 509                 else
 510                         predictionImg = image;
 511
 512                 m_recognizer->predict(predictionImg, results.mFaceLabel, absConf);
 513
 514                 if (-1 != results.mFaceLabel) {
 515                         double normShift = 7.5;
 516                         double normSmooth = 0.05;
 517
 518                         if (MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES == m_learnAlgorithmConfig.mModelType) {
 519                                 normShift = 5.0;
 520                                 normSmooth = 0.0015;
 521                         } else if (MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES == m_learnAlgorithmConfig.mModelType) {
 522                                 normShift = 5.0;
 523                                 normSmooth = 0.01;
 524                         }
 525
 526                         /* Normalize the absolute value of the confidence */
 527                         absConf = exp(normShift - (normSmooth * absConf));
 528                         results.mConfidence = absConf / (1 + absConf);
 529                         results.mIsRecognized = true;
 530                 } else {
 531                         results.mConfidence = 0.0;
 532                         results.mIsRecognized = false;
 533                 }
 534
 535                 results.mFaceLocation = cv::Rect(0, 0, image.cols, image.rows);
 536         } else {
 537                 LOGE("Attempt to recognize faces with untrained model");
 538                 return MEDIA_VISION_ERROR_INVALID_OPERATION;
 539         }
 540
 541         return MEDIA_VISION_ERROR_NONE;
 542 }
 543
 544 cv::Ptr<cv::face::FaceRecognizer> FaceRecognitionModel::CreateRecognitionAlgorithm(
 545                 const FaceRecognitionModelConfig& config)
 546 {
 547         cv::Ptr<cv::face::FaceRecognizer> tempRecognizer;
 548         switch (config.mModelType) {
 549         case MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES:
 550                 tempRecognizer = cv::face::EigenFaceRecognizer::create(
 551                                                                         config.mNumComponents,
 552                                                                         config.mThreshold);
 553                 break;
 554         case MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES:
 555                 tempRecognizer = cv::face::FisherFaceRecognizer::create(
 556                                                                         config.mNumComponents,
 557                                                                         config.mThreshold);
 558                 break;
 559         case MEDIA_VISION_FACE_MODEL_TYPE_LBPH:
 560                 tempRecognizer = cv::face::LBPHFaceRecognizer::create(
 561                                                                         config.mRadius,
 562                                                                         config.mNeighbors,
 563                                                                         config.mGridX,
 564                                                                         config.mGridY,
 565                                                                         config.mThreshold);
 566                 break;
 567         default:
 568                 LOGE("Unknown FaceRecognition model");
 569         }
 570
 571         return tempRecognizer;
 572 }
 573
 574 } /* Face */
 575 } /* MediaVision */