Add INFERENCE_TARGET_CUSTOM, Support multiple output tensors

Change-Id: Iab1965c027b4207ae699f5998f789b85020f7a8b
Signed-off-by: Tae-Young Chung <ty83.chung@samsung.com>

diff --git a/packaging/inference-engine-opencv.spec b/packaging/inference-engine-opencv.spec
index c4cd8c1..a60e0c9 100644 (file)
--- a/packaging/inference-engine-opencv.spec
+++ b/packaging/inference-engine-opencv.spec
@@ -1,7 +1,7 @@
 Name:       inference-engine-opencv
 Summary:    OpenCV based implementation of inference-engine-interface
 Version:    0.0.1
-Release:    1
+Release:    2
 Group:      Multimedia/Libraries
 License:    Apache-2.0
 Source0:    %{name}-%{version}.tar.gz

diff --git a/src/inference_engine_opencv.cpp b/src/inference_engine_opencv.cpp
index 119e9d1..3c33a80 100644 (file)
--- a/src/inference_engine_opencv.cpp
+++ b/src/inference_engine_opencv.cpp
@@ -89,6 +89,7 @@ int InferenceOpenCV::SetInputTensorParamNorm(double deviation, double mean)
 
 int InferenceOpenCV::SetInputTensorParamNode(std::string node)
 {
+    mInputLayer = cv::String(node);
     return INFERENCE_ENGINE_ERROR_NONE;
 }
 
@@ -116,8 +117,14 @@ int InferenceOpenCV::SetOutputTensorParamType(int type)
     return INFERENCE_ENGINE_ERROR_NONE;
 }
 
-int InferenceOpenCV::SetOutPutTensorParamNodes(std::string node)
+int InferenceOpenCV::SetOutputTensorParamNodes(std::vector<std::string> nodes)
 {
+    mOutputLayer.clear();
+    for (std::vector<std::string>::iterator iter = nodes.begin();
+        iter != nodes.end(); ++iter) {
+        mOutputLayer.push_back(cv::String(*iter));
+    }
+
     return INFERENCE_ENGINE_ERROR_NONE;
 }
 
@@ -130,6 +137,7 @@ int InferenceOpenCV::SetTargetDevice(inference_target_type_e type)
     case INFERENCE_TARGET_GPU :
         mNet.setPreferableTarget(cv::dnn::DNN_TARGET_OPENCL);
         break;
+    case INFERENCE_TARGET_CUSTOM:
     case INFERENCE_TARGET_NONE:
     default:
         LOGE("Not supported device type [%d], Set CPU mode", (int)type);
@@ -157,7 +165,16 @@ int InferenceOpenCV::Load()
        }
 
     // This call may be changed if OpenCV version would be upgraded
-    mNet = cv::dnn::readNetFromCaffe(mConfigFile, mWeightFile);
+    int nPos = mWeightFile.find_last_of(".");
+    std::string weightFileExt = mWeightFile.substr(nPos+1);
+    LOGI("%s", weightFileExt.c_str());
+    if (weightFileExt.compare("caffemodel") == 0) {
+        mNet = cv::dnn::readNetFromCaffe(mConfigFile, mWeightFile);
+    } else if (weightFileExt.compare("pb") == 0) {
+        mNet = cv::dnn::readNetFromTensorflow(mWeightFile, mConfigFile);
+    } else {
+        LOGE("Not supported model file!");
+    }
 
     if (mNet.empty()) {
         LOGE("Net is empty");
@@ -173,23 +190,40 @@ int InferenceOpenCV::Load()
 
 int InferenceOpenCV::CreateInputLayerPassage()
 {
-    return INFERENCE_ENGINE_ERROR_NOT_SUPPORTED;
+    /* Do Nothing
+     * In OpenCV, don't need to call this CreateInputLayerPassage()
+     * because that it can uses cv::Mat directly
+     */
+
+    return INFERENCE_ENGINE_ERROR_NONE;
 }
 
 int InferenceOpenCV::PrepareInputLayerPassage()
 {
-    return INFERENCE_ENGINE_ERROR_NOT_SUPPORTED;
+    /* Do Nothing
+     * In OpenCV, don't need to call this PrepareInputLayerPassage()
+     * because that it can uses cv::Mat directly
+     */
+    return INFERENCE_ENGINE_ERROR_NONE;
 }
 
 int InferenceOpenCV::PrepareInputLayerPassage(inference_input_type_e type)
 {
-    return INFERENCE_ENGINE_ERROR_NOT_SUPPORTED;
+    /* Do Nothing
+     * In OpenCV, don't need to call this PrepareInputLayerPassage()
+     * because that it can uses cv::Mat directly
+     */
+    return INFERENCE_ENGINE_ERROR_NONE;
 }
 
 int InferenceOpenCV::Run(cv::Mat tensor)
 {
     double scaleVal = 1.0/mDeviation;
     mSourceSize = tensor.size();
+    if (mInputSize == cv::Size(-1,-1)) {
+        mInputSize = mSourceSize;
+    }
+
     cv::Scalar meanScalar = cv::Scalar((float)mMean,(float)mMean, (float)mMean);
 
     mInputBlob = cv::dnn::blobFromImage(tensor, scaleVal, mInputSize, meanScalar, false, false);
@@ -199,14 +233,10 @@ int InferenceOpenCV::Run(cv::Mat tensor)
         return INFERENCE_ENGINE_ERROR_INTERNAL;
     }
 
-    mNet.setInput(mInputBlob);
+    mNet.setInput(mInputBlob, mInputLayer);
 
     /* foward */
-    mOutputProb = mNet.forward();
-    /* TODO */
-    // std::vector<cv::Mat> mOutputProb, 
-    // std::vector<std::string> mOuputLayer;
-    // mNet.forward(mOutputProb, mOutputLayer);
+    mNet.forward(mOutputProb, mOutputLayer);
 
     if (mOutputProb.empty()) {
         LOGE("OutputProb is empty");
@@ -223,12 +253,12 @@ int InferenceOpenCV::Run(std::vector<float> tensor)
 
 int InferenceOpenCV::GetInferenceResult(ImageClassificationResults& results)
 {
-    int dims = mOutputProb.dims;
+    int dims = mOutputProb[0].dims;
     for (int k = 0; k < dims; ++k) {
-        LOGE("%d: %d", k, mOutputProb.size[k]);
+        LOGE("%d: %d", k, mOutputProb[0].size[k]);
     }
     LOGI("dims: %d", dims);
-    cv::Mat reShapedProb = mOutputProb.reshape(1,1);
+    cv::Mat reShapedProb = mOutputProb[0].reshape(1,1);
 
 
     int classIdx = -1;
@@ -261,12 +291,12 @@ int InferenceOpenCV::GetInferenceResult(ImageClassificationResults& results)
 
 int InferenceOpenCV::GetInferenceResult(ObjectDetectionResults& results)
 {
-    int dims = mOutputProb.dims;
+    int dims = mOutputProb[0].dims;
     for (int k = 0; k < dims; ++k) {
-        LOGE("%d: %d", k, mOutputProb.size[k]);
+        LOGE("%d: %d", k, mOutputProb[0].size[k]);
     }
     LOGI("dims: %d", dims);
-    cv::Mat detectionMat(mOutputProb.size[2], mOutputProb.size[3], CV_32F, mOutputProb.ptr<float>());
+    cv::Mat detectionMat(mOutputProb[0].size[2], mOutputProb[0].size[3], CV_32F, mOutputProb[0].ptr<float>());
 
     float confidence;
     size_t objectClass;
@@ -307,13 +337,13 @@ int InferenceOpenCV::GetInferenceResult(ObjectDetectionResults& results)
 
 int InferenceOpenCV::GetInferenceResult(FaceDetectionResults& results)
 {
-    int dims = mOutputProb.dims;
+    int dims = mOutputProb[0].dims;
     for (int k = 0; k < dims; ++k) {
-        LOGE("%d: %d", k, mOutputProb.size[k]);
+        LOGE("%d: %d", k, mOutputProb[0].size[k]);
     }
     LOGI("dims: %d", dims);
 
-    cv::Mat detectionMat(mOutputProb.size[2], mOutputProb.size[3], CV_32F, mOutputProb.ptr<float>());
+    cv::Mat detectionMat(mOutputProb[0].size[2], mOutputProb[0].size[3], CV_32F, mOutputProb[0].ptr<float>());
 
     float confidence;
     int left, top, right, bottom;
@@ -348,13 +378,13 @@ int InferenceOpenCV::GetInferenceResult(FaceDetectionResults& results)
 
 int InferenceOpenCV::GetInferenceResult(FacialLandMarkDetectionResults& results)
 {
-    int dims = mOutputProb.dims;
+    int dims = mOutputProb[0].dims;
     for (int k = 0; k < dims; ++k) {
-        LOGE("%d: %d", k, mOutputProb.size[k]);
+        LOGE("%d: %d", k, mOutputProb[0].size[k]);
     }
     LOGI("dims: %d", dims);
 
-    cv::Mat outputs = mOutputProb.reshape(1,1);
+    cv::Mat outputs = mOutputProb[0].reshape(1,1);
 
     int number_of_landmarks = (outputs.cols >> 1);
 
@@ -378,15 +408,18 @@ int InferenceOpenCV::GetInferenceResult(std::vector<std::vector<int>>& dimInfo,
 {
     dimInfo.clear();
     results.clear();
-
-    int dims = mOutputProb.dims;
     std::vector<int> tmpDimInfo;
-    for (int d = 0; d < dims; ++d) {
-        tmpDimInfo.push_back(mOutputProb.size[d]); 
-    }
+    LOGE("outputProb size: %d", mOutputProb.size());
+    for (std::vector<cv::Mat>::iterator iter = mOutputProb.begin();
+        iter != mOutputProb.end(); ++iter) {
+        tmpDimInfo.clear();
+        for (int d = 0; d < (*iter).dims; ++d) {
+            tmpDimInfo.push_back((*iter).size[d]);
+        }
 
-    dimInfo.push_back(tmpDimInfo);
-    results.push_back(mOutputProb.ptr<float>());
+        dimInfo.push_back(tmpDimInfo);
+        results.push_back((*iter).ptr<float>());
+    }
 
     return INFERENCE_ENGINE_ERROR_NONE;
 }
@@ -428,4 +461,4 @@ void EngineCommonDestroy(class IInferenceEngineCommon *engine)
 }
 }
 } /* OpenCVImpl */
-} /* InferenceEngineImpl */
\ No newline at end of file
+} /* InferenceEngineImpl */

diff --git a/src/inference_engine_opencv_private.h b/src/inference_engine_opencv_private.h
index 3e7acc8..f824dd6 100644 (file)
--- a/src/inference_engine_opencv_private.h
+++ b/src/inference_engine_opencv_private.h
@@ -69,7 +69,7 @@ public:
 
     int SetOutputTensorParamType(int type) override;
 
-    int SetOutPutTensorParamNodes(std::string node) override;
+    int SetOutputTensorParamNodes(std::vector<std::string> nodes) override;
 
     int SetTargetDevice(inference_target_type_e type) override;
 
@@ -105,9 +105,11 @@ public:
     
 private:
     cv::Mat mInputBlob;
-    cv::Mat mOutputProb;
+    std::vector<cv::Mat> mOutputProb;
     cv::dnn::Net mNet; /**< Network associated with a network model */
 
+    cv::String mInputLayer;
+    std::vector<cv::String> mOutputLayer;
     int mCh;
     int mDim;
     cv::Size mInputSize; /**< input tensor size */
@@ -127,4 +129,4 @@ private:
 } /* InferenceEngineImpl */
 } /* TFImpl */
 
-#endif /* __MEDIA_VISION_INFERENCE_OPENCV_H__ */
\ No newline at end of file
+#endif /* __MEDIA_VISION_INFERENCE_OPENCV_H__ */