Update EfficientDet and its test

[Issue type] new feature and update

Change-Id: I6011facc16444e9d2e9e75f7deb6dd5eaece1863
Signed-off-by: Tae-Young Chung <ty83.chung@samsung.com>

diff --git a/mv_machine_learning/inference/include/BoxInfo.h b/mv_machine_learning/inference/include/BoxInfo.h
index dedc7e3..673c97b 100644 (file)
--- a/mv_machine_learning/inference/include/BoxInfo.h
+++ b/mv_machine_learning/inference/include/BoxInfo.h
@@ -40,12 +40,14 @@ struct Label
 {
        std::string name;
        DimInfo dimInfo;
+       int offset = 0;
 };
 
 struct Number
 {
        std::string name;
        DimInfo dimInfo;
+       int offset = 0;
 };
 
 struct BoxInfo
@@ -57,6 +59,7 @@ struct BoxInfo
        inference_box_coordinate_type_e coordinate = INFERENCE_BOX_COORDINATE_TYPE_RATIO; // 0: ratio, 1: pixel
        inference_box_decoding_type_e decodingType = INFERENCE_BOX_DECODING_TYPE_BYPASS; // 0: bypass , 1:ssd with anchor
        DecodeInfo decodingInfo;
+       int offset = 0;
        Label label;
        Number number;
 
@@ -102,10 +105,18 @@ struct BoxInfo
        {
                return decodingInfo;
        }
+       int GetOffset()
+       {
+               return offset;
+       }
        std::string GetLabelName()
        {
                return label.name;
        }
+       int GetLabelOffset()
+       {
+               return label.offset;
+       }
        std::string GetNumberName()
        {
                return number.name;
@@ -114,6 +125,10 @@ struct BoxInfo
        {
                return number.dimInfo;
        }
+       int GetNumberOffset()
+       {
+               return number.offset;
+       }
 
        int ParseBox(JsonObject *root);
 
@@ -127,4 +142,4 @@ struct BoxInfo
 } /* Inference */
 } /* MediaVision */
 
-#endif
\ No newline at end of file
+#endif

diff --git a/mv_machine_learning/inference/include/OutputMetadata.h b/mv_machine_learning/inference/include/OutputMetadata.h
index f2d514a..cdc1eb9 100644 (file)
--- a/mv_machine_learning/inference/include/OutputMetadata.h
+++ b/mv_machine_learning/inference/include/OutputMetadata.h
@@ -114,6 +114,10 @@ struct OutputMetadata
        {
                return score.GetDeQuantZeroPoint();
        }
+       int GetScoreOffset()
+       {
+               return score.GetOffset();
+       }
        std::string GetBoxName()
        {
                return box.GetName();
@@ -146,6 +150,18 @@ struct OutputMetadata
        {
                return box.GetNumberDimInfo();
        }
+       int GetBoxOffset()
+       {
+               return box.GetOffset();
+       }
+       int GetLabelOffset()
+       {
+               return box.GetLabelOffset();
+       }
+       int GetNumberOffset()
+       {
+               return box.GetNumberOffset();
+       }
 
        int GetScoreCoordinate()
        {

diff --git a/mv_machine_learning/inference/include/ScoreInfo.h b/mv_machine_learning/inference/include/ScoreInfo.h
index 4592f7f..237d37c 100644 (file)
--- a/mv_machine_learning/inference/include/ScoreInfo.h
+++ b/mv_machine_learning/inference/include/ScoreInfo.h
@@ -47,6 +47,7 @@ struct ScoreInfo
        DimInfo dimInfo;
        double threshold = 0.0;
        int topNumber = 1;
+       int offset = 0;
        inference_score_type_e type = INFERENCE_SCORE_TYPE_NORMAL;
        std::shared_ptr<DeQuantization> deQuantization;
        std::map<std::string, inference_score_type_e> supportedScoreTypes = { { "NORMAL", INFERENCE_SCORE_TYPE_NORMAL },
@@ -73,6 +74,10 @@ struct ScoreInfo
        {
                return topNumber;
        }
+       int GetOffset()
+       {
+               return offset;
+       }
        std::shared_ptr<DeQuantization> GetDeQuant()
        {
                return deQuantization;
@@ -91,4 +96,4 @@ struct ScoreInfo
 } /* Inference */
 } /* MediaVision */
 
-#endif
\ No newline at end of file
+#endif

diff --git a/mv_machine_learning/inference/src/BoxInfo.cpp b/mv_machine_learning/inference/src/BoxInfo.cpp
index 43f055d..56dfc83 100644 (file)
--- a/mv_machine_learning/inference/src/BoxInfo.cpp
+++ b/mv_machine_learning/inference/src/BoxInfo.cpp
@@ -68,6 +68,15 @@ int BoxInfo::ParseBox(JsonObject *root)
                return MEDIA_VISION_ERROR_INVALID_OPERATION;
        }
 
+       if (json_object_has_member(pObject, "offset")) {
+               offset = static_cast<int>(json_object_get_int_member(pObject, "offset"));
+               if (offset < 0) {
+                       LOGW("Invalid offset. Reset to 0");
+                       offset = 0;
+               }
+       }
+       LOGI("offset: %d", offset);
+
        array = json_object_get_array_member(pObject, "box_order");
        elements2 = json_array_get_length(array);
        LOGI("box order should have 4 elements and it has [%u]", elements2);
@@ -110,6 +119,15 @@ int BoxInfo::ParseLabel(JsonObject *root)
                        if (static_cast<int>(json_array_get_int_element(array, elem2)) == 1)
                                label.dimInfo.SetValidIndex(elem2);
                }
+
+               if (json_object_has_member(pObject, "offset")) {
+                       label.offset = static_cast<int>(json_object_get_int_member(pObject, "offset"));
+                       if (label.offset < 0) {
+                               LOGW("Invalid offset. Reset to 0");
+                               label.offset = 0;
+                       }
+               }
+               LOGI("offset: %d", label.offset);
        }
 
        LOGI("LEAVE");
@@ -147,6 +165,15 @@ int BoxInfo::ParseNumber(JsonObject *root)
                for (unsigned int elem2 = 0; elem2 < elements2; ++elem2)
                        if (static_cast<int>(json_array_get_int_element(array, elem2)) == 1)
                                number.dimInfo.SetValidIndex(elem2);
+
+               if (json_object_has_member(pObject, "offset")) {
+                       number.offset = static_cast<int>(json_object_get_int_member(pObject, "offset"));
+                       if (number.offset < 0) {
+                               LOGW("Invalid offset. Reset to 0");
+                               number.offset = 0;
+                       }
+               }
+               LOGI("offset: %d", number.offset);
        }
 
        LOGI("LEAVE");
@@ -206,4 +233,4 @@ int BoxInfo::ParseDecodeInfo(JsonObject *root)
 
        LOGI("LEAVE");
        return MEDIA_VISION_ERROR_NONE;
-}
\ No newline at end of file
+}

diff --git a/mv_machine_learning/inference/src/Inference.cpp b/mv_machine_learning/inference/src/Inference.cpp
index 7c4c465..58369ef 100644 (file)
--- a/mv_machine_learning/inference/src/Inference.cpp
+++ b/mv_machine_learning/inference/src/Inference.cpp
@@ -1074,6 +1074,15 @@ int Inference::GetObjectDetectionResults(ObjectDetectionResults *results)
                int boxOffset = mOutputLayerProperty.layers[outputMeta.GetBoxName()].shape[boxIndexes[0]];
                int numberOfObjects = 0;
 
+               if (mOutputTensorBuffers.exist(outputMeta.GetBoxNumberName())) {
+                       std::vector<int> numberIndexes = outputMeta.GetBoxNumberDimInfo().GetValidIndexAll();
+                       if (numberIndexes.size() != 1) {
+                               LOGE("Invalid dim size. It should be 1");
+                               return MEDIA_VISION_ERROR_INVALID_OPERATION;
+                       }
+                       numberOfObjects = mOutputLayerProperty.layers[outputMeta.GetBoxNumberName()].shape[numberIndexes[0]];
+               }
+
                if (outputMeta.GetBoxDecodingType() == INFERENCE_BOX_DECODING_TYPE_SSD_ANCHOR) {
                        std::vector<int> scoreIndexes = outputMeta.GetScoreDimInfo().GetValidIndexAll();
                        if (scoreIndexes.size() != 1) {

diff --git a/mv_machine_learning/inference/src/ObjectDecoder.cpp b/mv_machine_learning/inference/src/ObjectDecoder.cpp
index e14fea9..08c2a3b 100644 (file)
--- a/mv_machine_learning/inference/src/ObjectDecoder.cpp
+++ b/mv_machine_learning/inference/src/ObjectDecoder.cpp
@@ -44,13 +44,16 @@ int ObjectDecoder::init()
 
                // mNumberOfObjects is set again if INFERENCE_BOX_DECODING_TYPE_BYPASS.
                // Otherwise it is set already within ctor.
-               mNumberOfOjects = mTensorBuffer.getValue<int>(mMeta.GetBoxNumberName(), indexes[0]);
+               int numberOfObjects = mTensorBuffer.getValue<int>(mMeta.GetBoxNumberName(), indexes[0]);
+               if (numberOfObjects > 0)
+                       mNumberOfOjects = numberOfObjects;
        } else if (mMeta.GetBoxDecodingType() == INFERENCE_BOX_DECODING_TYPE_SSD_ANCHOR) {
                if (mMeta.GetBoxDecodeInfo().IsAnchorBoxEmpty()) {
                        LOGE("Anchor boxes are required but empty.");
                        return MEDIA_VISION_ERROR_INVALID_OPERATION;
                }
        } else {
+               mNumberOfOjects = 0;
                LOGI("YOLO_ANCHOR does nothing");
        }
 
@@ -100,6 +103,9 @@ Box ObjectDecoder::decodeBox(int idx, float score, int label, int offset)
                cHeight /= mScaleH;
        }
 
+       if (!mMeta.GetBoxLabelName().empty() && mMeta.GetLabelOffset())
+               idx =  idx * mBoxOffset + mMeta.GetLabelOffset();
+
        Box box = { .index = mMeta.GetBoxLabelName().empty() ? label :
                                                                                                                   mTensorBuffer.getValue<int>(mMeta.GetBoxLabelName(), idx),
                                .score = score,
@@ -247,14 +253,16 @@ int ObjectDecoder::decode()
        Boxes boxes;
        int ret = MEDIA_VISION_ERROR_NONE;
        int totalIdx = mNumberOfOjects;
-
+       int scoreOffset = mMeta.GetScoreOffset();
+       int boxOffset = mMeta.GetBoxOffset();
        for (int idx = 0; idx < totalIdx; ++idx) {
                if (mMeta.GetBoxDecodingType() == INFERENCE_BOX_DECODING_TYPE_BYPASS) {
-                       float score = decodeScore(idx);
+                       float score = scoreOffset > 0 ? decodeScore(idx * mBoxOffset + scoreOffset) :
+                                                                                       decodeScore(idx);
                        if (score <= 0.0f)
                                continue;
 
-                       Box box = decodeBox(idx, score);
+                       Box box = decodeBox(idx, score, idx, boxOffset);
                        mResultBoxes.push_back(box);
                } else if (mMeta.GetBoxDecodingType() == INFERENCE_BOX_DECODING_TYPE_SSD_ANCHOR) {
                        int anchorIdx = -1;

diff --git a/mv_machine_learning/inference/src/ScoreInfo.cpp b/mv_machine_learning/inference/src/ScoreInfo.cpp
index 0560218..3e851c5 100644 (file)
--- a/mv_machine_learning/inference/src/ScoreInfo.cpp
+++ b/mv_machine_learning/inference/src/ScoreInfo.cpp
@@ -59,6 +59,15 @@ int ScoreInfo::ParseScore(JsonObject *root)
                return MEDIA_VISION_ERROR_INVALID_OPERATION;
        }
 
+       if (json_object_has_member(pObject, "offset")) {
+               offset = static_cast<int>(json_object_get_int_member(pObject, "offset"));
+               if (offset < 0) {
+                       LOGW("Invalid offset. Reset to 0");
+                       offset = 0;
+               }
+       }
+       LOGI("offset: %d", offset);
+
        if (json_object_has_member(pObject, "dequantization")) {
                array = json_object_get_array_member(pObject, "dequantization");
                JsonNode *node = json_array_get_element(array, 0);
@@ -70,4 +79,4 @@ int ScoreInfo::ParseScore(JsonObject *root)
 
        LOGI("LEAVE");
        return MEDIA_VISION_ERROR_NONE;
-}
\ No newline at end of file
+}

diff --git a/test/testsuites/machine_learning/inference/test_object_detection.cpp b/test/testsuites/machine_learning/inference/test_object_detection.cpp
index 6ae988b..13bbb29 100644 (file)
--- a/test/testsuites/machine_learning/inference/test_object_detection.cpp
+++ b/test/testsuites/machine_learning/inference/test_object_detection.cpp
@@ -28,6 +28,12 @@
 #define IMG_DOG    \
        MV_CONFIG_PATH \
        "/res/inference/images/dog2.jpg"
+#define OD_LABEL_EFFICIENTDET_448_PATH \
+       MV_CONFIG_PATH                           \
+       "models/OD/snpe/label_coco_90.txt"
+#define OD_SNPE_WEIGHT_QC_EFFCIENTDET_448_PATH \
+       MV_CONFIG_PATH                                   \
+       "/models/OD/snpe/fp32.dlc"
 #define OD_LABEL_EFFICIENTDET_LITE2_448_PATH \
        MV_CONFIG_PATH                           \
        "models/OD/snpe/label_coco_90.txt"
@@ -141,6 +147,16 @@ public:
        }
 };
 
+
+TEST_P(TestObjectDetectionSnpe, DISABLED_EFD2QC)
+{
+       ASSERT_TRUE(_use_json_parser);
+       engine_config_hosted_snpe_model(engine_cfg, OD_SNPE_WEIGHT_QC_EFFCIENTDET_448_PATH,
+                                                                       OD_LABEL_EFFICIENTDET_448_PATH, _use_json_parser, _target_device_type);
+
+       inference("person", IMG_BASEBALL);
+}
+
 TEST_P(TestObjectDetectionSnpe, DISABLED_EFDLite2QC)
 {
        ASSERT_TRUE(_use_json_parser);