From ade8239371eb41feb901cced7cd1a51e5e514e60 Mon Sep 17 00:00:00 2001 From: Kwanghoon Son Date: Thu, 15 Sep 2022 01:08:26 -0400 Subject: [PATCH 01/16] Change Info class to struct [Issue type] refactoring ObjectDecoder class needs access all of Info member variables. So all of Info class variables need to get method because it is just struct of data. Since Info classes not have any special member function, struct is more suitable for data access. Change-Id: Ifa470fbb17344451584e0b93ad6faf859c021eba Signed-off-by: Kwanghoon Son --- mv_machine_learning/inference/include/BoxInfo.h | 4 +--- mv_machine_learning/inference/include/DecodeInfo.h | 4 +--- mv_machine_learning/inference/include/DimInfo.h | 4 +--- mv_machine_learning/inference/include/ScoreInfo.h | 5 +---- 4 files changed, 4 insertions(+), 13 deletions(-) diff --git a/mv_machine_learning/inference/include/BoxInfo.h b/mv_machine_learning/inference/include/BoxInfo.h index 131e607..dedc7e3 100644 --- a/mv_machine_learning/inference/include/BoxInfo.h +++ b/mv_machine_learning/inference/include/BoxInfo.h @@ -48,9 +48,8 @@ struct Number DimInfo dimInfo; }; -class BoxInfo +struct BoxInfo { -private: std::vector names; DimInfo dimInfo; inference_box_type_e type = INFERENCE_BOX_TYPE_ORIGIN_LEFTTOP; // 0:L-T-R-B, 1: Cx-Cy-W-H @@ -75,7 +74,6 @@ private: { "YOLO_ANCHOR", INFERENCE_BOX_DECODING_TYPE_YOLO_ANCHOR } }; -public: ~BoxInfo() = default; std::string GetName(); diff --git a/mv_machine_learning/inference/include/DecodeInfo.h b/mv_machine_learning/inference/include/DecodeInfo.h index 3a5fef8..67c7e32 100644 --- a/mv_machine_learning/inference/include/DecodeInfo.h +++ b/mv_machine_learning/inference/include/DecodeInfo.h @@ -101,9 +101,8 @@ struct RoiOptionParam int mode; }; -class DecodeInfo +struct DecodeInfo { -private: AnchorParam anchorParam; std::vector anchorBoxes; CellParam cellParam; @@ -111,7 +110,6 @@ private: RotateParam rotParam; RoiOptionParam roiOptParam; -public: DecodeInfo() { cellParam.type = INFERENCE_SCORE_TYPE_NORMAL; diff --git a/mv_machine_learning/inference/include/DimInfo.h b/mv_machine_learning/inference/include/DimInfo.h index 3c0782a..e32c43f 100644 --- a/mv_machine_learning/inference/include/DimInfo.h +++ b/mv_machine_learning/inference/include/DimInfo.h @@ -24,12 +24,10 @@ namespace mediavision { namespace inference { -class DimInfo +struct DimInfo { -private: std::vector dims; -public: std::vector GetValidIndexAll() const { LOGI("ENTER"); diff --git a/mv_machine_learning/inference/include/ScoreInfo.h b/mv_machine_learning/inference/include/ScoreInfo.h index 69a0d45..4592f7f 100644 --- a/mv_machine_learning/inference/include/ScoreInfo.h +++ b/mv_machine_learning/inference/include/ScoreInfo.h @@ -41,9 +41,8 @@ struct DeQuantization {} }; -class ScoreInfo +struct ScoreInfo { -private: std::vector names; DimInfo dimInfo; double threshold = 0.0; @@ -52,8 +51,6 @@ private: std::shared_ptr deQuantization; std::map supportedScoreTypes = { { "NORMAL", INFERENCE_SCORE_TYPE_NORMAL }, { "SIGMOID", INFERENCE_SCORE_TYPE_SIGMOID } }; - -public: ~ScoreInfo() = default; std::string GetName() -- 2.7.4 From 70529c98bef276f4e42206db8b125e3c1c62f74a Mon Sep 17 00:00:00 2001 From: Kwanghoon Son Date: Tue, 20 Sep 2022 22:37:37 -0400 Subject: [PATCH 02/16] Drop Yolo CellParm Yolo object detection not use CellParm anymore Change-Id: Id9842754634b6f134b02f417c96ea24f5fc69096 Signed-off-by: Kwanghoon Son --- mv_machine_learning/inference/include/DecodeInfo.h | 22 ---------- mv_machine_learning/inference/src/BoxInfo.cpp | 6 --- mv_machine_learning/inference/src/DecodeInfo.cpp | 47 ---------------------- mv_machine_learning/inference/src/Inference.cpp | 2 - .../inference/src/ObjectDecoder.cpp | 39 ------------------ 5 files changed, 116 deletions(-) diff --git a/mv_machine_learning/inference/include/DecodeInfo.h b/mv_machine_learning/inference/include/DecodeInfo.h index 67c7e32..6a38e54 100644 --- a/mv_machine_learning/inference/include/DecodeInfo.h +++ b/mv_machine_learning/inference/include/DecodeInfo.h @@ -60,16 +60,6 @@ struct AnchorParam std::map supportedCellType; std::vector > vxScales; std::vector > vyScales; - int numAnchorsPerCell; -}; - -struct CellParam -{ - int numScales; - std::vector scales; - int offsetScales; - inference_score_type_e type; - std::map supportedCellType; }; struct NMSParam @@ -105,17 +95,12 @@ struct DecodeInfo { AnchorParam anchorParam; std::vector anchorBoxes; - CellParam cellParam; NMSParam nmsParam; RotateParam rotParam; RoiOptionParam roiOptParam; DecodeInfo() { - cellParam.type = INFERENCE_SCORE_TYPE_NORMAL; - cellParam.supportedCellType.insert({ "NORMAL", INFERENCE_SCORE_TYPE_NORMAL }); - cellParam.supportedCellType.insert({ "SIGMOID", INFERENCE_SCORE_TYPE_SIGMOID }); - nmsParam.mode = INFERENCE_BOX_NMS_TYPE_NONE; nmsParam.iouThreshold = 0.2f; nmsParam.supportedBoxNmsTypes.insert({ "STANDARD", INFERENCE_BOX_NMS_TYPE_STANDARD }); @@ -155,13 +140,6 @@ struct DecodeInfo float GetAnchorHscale(); float CalculateScale(float min, float max, int index, int maxStride); - // Cell param - int ParseCellParam(JsonObject *root); - std::vector &GetCellScalesAll(); - int GetCellNumScales(); - int GetCellOffsetScales(); - inference_score_type_e GetCellType(); - // Nms param int ParseNms(JsonObject *root); int GetNmsMode(); diff --git a/mv_machine_learning/inference/src/BoxInfo.cpp b/mv_machine_learning/inference/src/BoxInfo.cpp index 5091db2..43f055d 100644 --- a/mv_machine_learning/inference/src/BoxInfo.cpp +++ b/mv_machine_learning/inference/src/BoxInfo.cpp @@ -179,12 +179,6 @@ int BoxInfo::ParseDecodeInfo(JsonObject *root) LOGE("Fail to ParseAnchorParam[%d]", ret); return ret; } - } else if (json_object_has_member(cObject, "cell")) { - ret = GetDecodeInfo().ParseCellParam(cObject); - if (ret != MEDIA_VISION_ERROR_NONE) { - LOGE("Fail to ParseCellParam[%d]", ret); - return ret; - } } else { LOGE("anchor is mandatory. Invalid metadata"); LOGI("LEAVE"); diff --git a/mv_machine_learning/inference/src/DecodeInfo.cpp b/mv_machine_learning/inference/src/DecodeInfo.cpp index e49245a..f33f062 100644 --- a/mv_machine_learning/inference/src/DecodeInfo.cpp +++ b/mv_machine_learning/inference/src/DecodeInfo.cpp @@ -73,8 +73,6 @@ int DecodeInfo::ParseAnchorParam(JsonObject *root) LOGI("aspectRatio: %.4f", aspectRatio); } } else if (anchorParam.mode == 1) { // Yolo - anchorParam.numAnchorsPerCell = static_cast(json_object_get_int_member(object, "num_anchors")); - anchorParam.offsetAnchors = static_cast(json_object_get_int_member(object, "offset_anchors")); JsonArray *xScales = json_object_get_array_member(object, "x_scales"); JsonArray *yScales = json_object_get_array_member(object, "y_scales"); @@ -127,51 +125,6 @@ int DecodeInfo::ParseAnchorParam(JsonObject *root) return MEDIA_VISION_ERROR_NONE; } -int DecodeInfo::ParseCellParam(JsonObject *root) -{ - JsonObject *object = json_object_get_object_member(root, "cell"); - - cellParam.numScales = static_cast(json_object_get_int_member(object, "num_scales")); - - JsonArray *array = json_object_get_array_member(object, "scales"); - unsigned int elements2 = json_array_get_length(array); - for (unsigned int elem2 = 0; elem2 < elements2; ++elem2) { - auto scale = static_cast(json_array_get_int_element(array, elem2)); - cellParam.scales.push_back(scale); - LOGI("scale: %d", scale); - } - - cellParam.offsetScales = static_cast(json_object_get_int_member(object, "offset_scales")); - try { - cellParam.type = GetSupportedType(object, "type", cellParam.supportedCellType); - } catch (const std::exception &e) { - LOGE("Invalid %s", e.what()); - return MEDIA_VISION_ERROR_INVALID_OPERATION; - } - - return MEDIA_VISION_ERROR_NONE; -} - -std::vector &DecodeInfo::GetCellScalesAll() -{ - return cellParam.scales; -} - -int DecodeInfo::GetCellNumScales() -{ - return cellParam.numScales; -} - -int DecodeInfo::GetCellOffsetScales() -{ - return cellParam.offsetScales; -} - -inference_score_type_e DecodeInfo::GetCellType() -{ - return cellParam.type; -} - float DecodeInfo::CalculateScale(float min, float max, int index, int maxStride) { return min + (max - min) * 1.0 * index / (maxStride - 1.0f); diff --git a/mv_machine_learning/inference/src/Inference.cpp b/mv_machine_learning/inference/src/Inference.cpp index 41a446d..9f3b25c 100644 --- a/mv_machine_learning/inference/src/Inference.cpp +++ b/mv_machine_learning/inference/src/Inference.cpp @@ -1073,8 +1073,6 @@ int Inference::GetObjectDetectionResults(ObjectDetectionResults *results) return MEDIA_VISION_ERROR_INVALID_OPERATION; } numberOfObjects = mOutputLayerProperty.layers[outputMeta.GetScoreName()].shape[scoreIndexes[0]]; - } else if (outputMeta.GetBoxDecodingType() == INFERENCE_BOX_DECODING_TYPE_YOLO_ANCHOR) { - numberOfObjects = boxOffset / outputMeta.GetBoxDecodeInfo().GetCellNumScales() - 5; } ObjectDecoder objDecoder(mOutputTensorBuffers, outputMeta, boxOffset, diff --git a/mv_machine_learning/inference/src/ObjectDecoder.cpp b/mv_machine_learning/inference/src/ObjectDecoder.cpp index c617f2d..06e8653 100644 --- a/mv_machine_learning/inference/src/ObjectDecoder.cpp +++ b/mv_machine_learning/inference/src/ObjectDecoder.cpp @@ -80,13 +80,6 @@ Box ObjectDecoder::decodeBox(int idx, float score, int label, int offset) float cHeight = mTensorBuffer.getValue(mMeta.GetBoxName(), idx * mBoxOffset + offset + mMeta.GetBoxOrder()[3]); - if (mMeta.GetBoxDecodeInfo().GetCellType() == INFERENCE_SCORE_TYPE_SIGMOID) { - cx = PostProcess::sigmoid(cx); - cy = PostProcess::sigmoid(cy); - cWidth = PostProcess::sigmoid(cWidth); - cHeight = PostProcess::sigmoid(cHeight); - } - LOGI("cx:%.2f, cy:%.2f, cW:%.2f, cH:%.2f", cx, cy, cWidth, cHeight); // convert type to ORIGIN_CENTER if ORIGIN_LEFTTOP if (mMeta.GetBoxType() == INFERENCE_BOX_TYPE_ORIGIN_LEFTTOP) { @@ -148,15 +141,6 @@ int ObjectDecoder::decode() int ret = MEDIA_VISION_ERROR_NONE; int totalIdx = mNumberOfOjects; - if (mMeta.GetBoxDecodingType() == INFERENCE_BOX_DECODING_TYPE_YOLO_ANCHOR) { - totalIdx = 0; - for (auto &scale : mMeta.GetBoxDecodeInfo().GetCellScalesAll()) { - totalIdx += (static_cast(mScaleW) / scale * static_cast(mScaleH) / scale) * - mMeta.GetBoxDecodeInfo().GetCellNumScales() / mMeta.GetBoxDecodeInfo().GetCellOffsetScales(); - } - boxList.reserve(mNumberOfOjects); - } - for (int idx = 0; idx < totalIdx; ++idx) { if (mMeta.GetBoxDecodingType() == INFERENCE_BOX_DECODING_TYPE_BYPASS) { float score = decodeScore(idx); @@ -183,29 +167,6 @@ int ObjectDecoder::decode() boxList.push_back(boxes); } else { // INFERENCE_BOX_DECODING_TYPE_YOLO_ANCHOR int cellIdx = idx * mBoxOffset; - for (int j = 0; j < mMeta.GetBoxDecodeInfo().GetCellOffsetScales(); ++j) { - float score = decodeScore(cellIdx + (mNumberOfOjects + 5) * j + 4); - if (score <= 0.0f) { - continue; - } - LOGI("score[%d]: %.2f", j, score); - // need to check the score - float topObjScore = 0.0f; - int topObjIdx = 0; - for (int objIdx_ = 0; objIdx_ < mNumberOfOjects; ++objIdx_) { - float objScore_ = decodeScore(cellIdx + (mNumberOfOjects + 5) * j + 5 + objIdx_); - if (objScore_ > topObjScore) { - topObjScore = objScore_; - topObjIdx = objIdx_; - } - } - - if (topObjScore < mMeta.GetScoreThreshold()) - continue; - - Box box = decodeBox(idx, topObjScore, topObjIdx, (mNumberOfOjects + 5) * j); - boxes.push_back(box); - } } } -- 2.7.4 From a2a2a25253ed0ef8aeef89ae8e12e7a1380bda7c Mon Sep 17 00:00:00 2001 From: Kwanghoon Son Date: Wed, 21 Sep 2022 01:05:23 -0400 Subject: [PATCH 03/16] Change metadata class to struct [Issue type] refactoring OutputMetadata class needs access all of Info member variables. So all of Info class variables need to get method because it is just struct of data. Since Info classes not have any special member function, struct is more suitable for data access Change-Id: I33594b225173314a42585738ac8a32d49efc8493 Signed-off-by: Kwanghoon Son --- mv_machine_learning/inference/include/OutputMetadata.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mv_machine_learning/inference/include/OutputMetadata.h b/mv_machine_learning/inference/include/OutputMetadata.h index 72f51fe..f2d514a 100644 --- a/mv_machine_learning/inference/include/OutputMetadata.h +++ b/mv_machine_learning/inference/include/OutputMetadata.h @@ -46,9 +46,8 @@ namespace mediavision { namespace inference { -class OutputMetadata +struct OutputMetadata { -private: bool parsed = false; ScoreInfo score; box::BoxInfo box; @@ -64,7 +63,6 @@ private: int ParseLandmark(JsonObject *root); int ParseOffset(JsonObject *root); -public: /** * @brief Destroys an OutputMetadata class instance including * its all resources. -- 2.7.4 From bc5261eeca8187dfc06355387efbbf8870a471e9 Mon Sep 17 00:00:00 2001 From: Tae-Young Chung Date: Mon, 22 Aug 2022 20:22:12 +0900 Subject: [PATCH 04/16] Add mv_3d apis supporting depth and pointcloud data [Issue type] new feature Change-Id: I04539a0ab6c393302a895a5c1402983e7c20d0ed Signed-off-by: Tae-Young Chung --- CMakeLists.txt | 26 +- doc/mediavision_doc.h | 41 +- include/mv_3d.h | 432 +++++ include/mv_3d_private.h | 51 + include/mv_3d_type.h | 75 + media-vision-config.json | 45 + mv_3d/3d/CMakeLists.txt | 21 + mv_3d/3d/include/Depth.h | 93 + mv_3d/3d/include/Mv3d.h | 47 + mv_3d/3d/include/PointCloud.h | 42 + mv_3d/3d/include/mv_3d_open.h | 80 + mv_3d/3d/src/Depth.cpp | 299 ++++ mv_3d/3d/src/PointCloud.cpp | 28 + mv_3d/3d/src/mv_3d.c | 169 ++ mv_3d/3d/src/mv_3d_open.cpp | 254 +++ mv_3d/CMakeLists.txt | 5 + packaging/capi-media-vision.spec | 44 + test/testsuites/CMakeLists.txt | 5 + test/testsuites/common/CMakeLists.txt | 4 + .../common/testsuite_common/mv_testsuite_common.h | 17 +- test/testsuites/common/visualizer/CMakeLists.txt | 37 + .../common/visualizer/include/mv_util_matrix.h | 47 + .../common/visualizer/include/mv_util_render_2d.h | 61 + .../common/visualizer/include/mv_util_render_3d.h | 41 + .../common/visualizer/include/mv_util_shader.h | 66 + .../visualizer/include/mv_util_visualizer_2d.h | 38 + .../visualizer/include/mv_util_visualizer_3d.h | 47 + .../common/visualizer/include/mv_util_winsys.h | 57 + .../common/visualizer/mv_util_visualizer_rd.proto | 18 + .../common/visualizer/src/mv_util_matrix.cpp | 841 +++++++++ .../common/visualizer/src/mv_util_render_2d.cpp | 618 +++++++ .../common/visualizer/src/mv_util_render_3d.cpp | 277 +++ .../common/visualizer/src/mv_util_shader.cpp | 114 ++ .../visualizer/src/mv_util_visualizer_2d.cpp | 145 ++ .../visualizer/src/mv_util_visualizer_3d.cpp | 186 ++ .../common/visualizer/src/mv_util_winsys.cpp | 148 ++ test/testsuites/mv3d/CMakeLists.txt | 71 + test/testsuites/mv3d/depth_test_suite.cpp | 607 +++++++ test/testsuites/mv3d/depthstream_test_suite.cpp | 622 +++++++ test/testsuites/visualizer/CMakeLists.txt | 16 + test/testsuites/visualizer/bunny.h | 1892 ++++++++++++++++++++ .../visualizer/remote_display_server/build.sh | 3 + .../visualizer/remote_display_server/img/intro.png | Bin 0 -> 6598 bytes .../remote_display_server/proto/build.sh | 4 + .../proto/mv_util_visualizer_rd.proto | 18 + .../visualizer/remote_display_server/server.cpp | 79 + .../visualizer/visualizer_test_suite.cpp | 235 +++ 47 files changed, 8055 insertions(+), 11 deletions(-) create mode 100644 include/mv_3d.h create mode 100644 include/mv_3d_private.h create mode 100644 include/mv_3d_type.h create mode 100644 mv_3d/3d/CMakeLists.txt create mode 100644 mv_3d/3d/include/Depth.h create mode 100644 mv_3d/3d/include/Mv3d.h create mode 100644 mv_3d/3d/include/PointCloud.h create mode 100644 mv_3d/3d/include/mv_3d_open.h create mode 100644 mv_3d/3d/src/Depth.cpp create mode 100644 mv_3d/3d/src/PointCloud.cpp create mode 100644 mv_3d/3d/src/mv_3d.c create mode 100644 mv_3d/3d/src/mv_3d_open.cpp create mode 100644 mv_3d/CMakeLists.txt create mode 100644 test/testsuites/common/visualizer/CMakeLists.txt create mode 100644 test/testsuites/common/visualizer/include/mv_util_matrix.h create mode 100644 test/testsuites/common/visualizer/include/mv_util_render_2d.h create mode 100644 test/testsuites/common/visualizer/include/mv_util_render_3d.h create mode 100644 test/testsuites/common/visualizer/include/mv_util_shader.h create mode 100644 test/testsuites/common/visualizer/include/mv_util_visualizer_2d.h create mode 100644 test/testsuites/common/visualizer/include/mv_util_visualizer_3d.h create mode 100644 test/testsuites/common/visualizer/include/mv_util_winsys.h create mode 100644 test/testsuites/common/visualizer/mv_util_visualizer_rd.proto create mode 100644 test/testsuites/common/visualizer/src/mv_util_matrix.cpp create mode 100644 test/testsuites/common/visualizer/src/mv_util_render_2d.cpp create mode 100644 test/testsuites/common/visualizer/src/mv_util_render_3d.cpp create mode 100644 test/testsuites/common/visualizer/src/mv_util_shader.cpp create mode 100644 test/testsuites/common/visualizer/src/mv_util_visualizer_2d.cpp create mode 100644 test/testsuites/common/visualizer/src/mv_util_visualizer_3d.cpp create mode 100644 test/testsuites/common/visualizer/src/mv_util_winsys.cpp create mode 100644 test/testsuites/mv3d/CMakeLists.txt create mode 100644 test/testsuites/mv3d/depth_test_suite.cpp create mode 100644 test/testsuites/mv3d/depthstream_test_suite.cpp create mode 100644 test/testsuites/visualizer/CMakeLists.txt create mode 100644 test/testsuites/visualizer/bunny.h create mode 100755 test/testsuites/visualizer/remote_display_server/build.sh create mode 100644 test/testsuites/visualizer/remote_display_server/img/intro.png create mode 100755 test/testsuites/visualizer/remote_display_server/proto/build.sh create mode 100644 test/testsuites/visualizer/remote_display_server/proto/mv_util_visualizer_rd.proto create mode 100644 test/testsuites/visualizer/remote_display_server/server.cpp create mode 100755 test/testsuites/visualizer/visualizer_test_suite.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0dbc0e0..75065a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,12 @@ project(${fw_name}) cmake_minimum_required(VERSION 2.6...3.13) option(BUILD_ML_ONLY "Build mv_machine_learning only" OFF) +option(BUILD_VISUALIZER "Build visualizer for testing 2D/3D output" OFF) + +if(BUILD_VISUALIZER) + add_compile_definitions(BUILD_VISUALIZER) +endif() + set(MV_COMMON_LIB_NAME "mv_common") set(MV_BARCODE_DETECTOR_LIB_NAME "mv_barcode_detector" CACHE STRING @@ -21,6 +27,8 @@ set(MV_TRAINING_LIB_NAME "mv_training" CACHE STRING "Name of the library will be built for training module (without extension).") set(MV_FACE_RECOG_LIB_NAME "mv_face_recognition" CACHE STRING "Name of the library will be built for face recognition module (without extension).") +set(MV_3D_LIB_NAME "mv_3d" CACHE STRING + "Name of the library will be built for 3d module (without extension).") set(MV_ROI_TRACKER_LIB_NAME "mv_roi_tracker" CACHE STRING "Name of the library will be built for tracker module (without extension).") @@ -34,6 +42,7 @@ add_subdirectory(mv_barcode) add_subdirectory(mv_image) add_subdirectory(mv_face) add_subdirectory(mv_surveillance) +add_subdirectory(mv_3d) add_subdirectory(mv_roi_tracker) add_subdirectory(test) endif() @@ -57,13 +66,13 @@ endif() set(PC_NAME ${fw_name}) if (${ENABLE_ML_FACE_RECOGNITION}) - set(PC_REQUIRED "${fw_name}-barcode ${fw_name}-face ${fw_name}-image ${fw_name}-surveillance ${fw_name}-inference ${fw_name}-training ${fw_name}-face-recognition ${fw_name}-tracker") + set(PC_REQUIRED "${fw_name}-barcode ${fw_name}-face ${fw_name}-image ${fw_name}-surveillance ${fw_name}-inference ${fw_name}-training ${fw_name}-face-recognition ${fw_name}-tracker ${fw_name}-3d") set(PC_LDFLAGS "-l${MV_COMMON_LIB_NAME} -l${MV_BARCODE_DETECTOR_LIB_NAME} -l${MV_BARCODE_GENERATOR_LIB_NAME} \ --l${MV_IMAGE_LIB_NAME} -l${MV_FACE_LIB_NAME} -l${MV_SURVEILLANCE_LIB_NAME} -l${MV_INFERENCE_LIB_NAME} -l${MV_TRAINING_LIB_NAME} -l${MV_FACE_RECOG_LIB_NAME} -l${MV_ROI_TRACKER_LIB_NAME}") +-l${MV_IMAGE_LIB_NAME} -l${MV_FACE_LIB_NAME} -l${MV_SURVEILLANCE_LIB_NAME} -l${MV_INFERENCE_LIB_NAME} -l${MV_TRAINING_LIB_NAME} -l${MV_FACE_RECOG_LIB_NAME} -l${MV_ROI_TRACKER_LIB_NAME} -l${MV_3D_LIB_NAME}") else() - set(PC_REQUIRED "${fw_name}-barcode ${fw_name}-face ${fw_name}-image ${fw_name}-surveillance ${fw_name}-inference ${fw_name}-tracker") + set(PC_REQUIRED "${fw_name}-barcode ${fw_name}-face ${fw_name}-image ${fw_name}-surveillance ${fw_name}-inference ${fw_name}-tracker ${fw_name}-3d") set(PC_LDFLAGS "-l${MV_COMMON_LIB_NAME} -l${MV_BARCODE_DETECTOR_LIB_NAME} -l${MV_BARCODE_GENERATOR_LIB_NAME} \ --l${MV_IMAGE_LIB_NAME} -l${MV_FACE_LIB_NAME} -l${MV_SURVEILLANCE_LIB_NAME} -l${MV_INFERENCE_LIB_NAME} -l${MV_ROI_TRACKER_LIB_NAME}") +-l${MV_IMAGE_LIB_NAME} -l${MV_FACE_LIB_NAME} -l${MV_SURVEILLANCE_LIB_NAME} -l${MV_INFERENCE_LIB_NAME} -l${MV_ROI_TRACKER_LIB_NAME} -l${MV_3D_LIB_NAME}") endif() configure_file( @@ -126,6 +135,15 @@ configure_file( @ONLY ) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/${fw_name}-surveillance.pc DESTINATION ${LIB_INSTALL_DIR}/pkgconfig) + +set(PC_NAME ${fw_name}-3d) +set(PC_LDFLAGS "-l${MV_DEPTH_LIB_NAME} -l${MV_COMMON_LIB_NAME}") +configure_file( + ${fw_name}.pc.in + ${CMAKE_CURRENT_SOURCE_DIR}/${fw_name}-3d.pc + @ONLY +) +install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/${fw_name}-3d.pc DESTINATION ${LIB_INSTALL_DIR}/pkgconfig) endif() set(PC_NAME ${fw_name}-inference) diff --git a/doc/mediavision_doc.h b/doc/mediavision_doc.h index bf43202..3ce5c4b 100644 --- a/doc/mediavision_doc.h +++ b/doc/mediavision_doc.h @@ -31,6 +31,7 @@ * face detection, facial landmark detection and face recognition;\n * * Training: Face recognition;\n * * Roi Tracker: Tracking Region of interest inside image;\n + * * 3D: Depth and PointCloud process * * @defgroup CAPI_MEDIA_VISION_COMMON_MODULE Media Vision Common * @ingroup CAPI_MEDIA_VISION_MODULE @@ -498,6 +499,42 @@ * It is set to MV_ROI_TRACKER_TYPE_BALANCE as default. * After setting tracker type, mv_roi_tracker_perform() has to be called to tracking roi of images on #mv_source_h, * and it sets newly generated coordinates to result parameter. + * + * @defgroup CAPI_MEDIA_VISION_3D_MODULE Media Vision 3D + * @ingroup CAPI_MEDIA_VISION_MODULE + * @brief 3D Depth and PointCloud Process. + * @section CAPI_MEDIA_VISION_3D_MODULE_HEADER Required Header + * \#include + * + * @section CAPI_MEDIA_VISION_3D_MODULE_FEATURE Related Features + * This API is related with the following features:\n + * - %http://tizen.org/feature/vision.3d\n + * - %http://tizen.org/feature/vision.3d.depth\n + * - %http://tizen.org/feature/vision.3d.pointcloud\n + * + * It is recommended to design feature related codes in your application for + * reliability.\n + * You can check if a device supports the related features for this API by using + * @ref CAPI_SYSTEM_SYSTEM_INFO_MODULE, thereby controlling the procedure of + * your application.\n + * To ensure your application is only running on the device with specific + * features, please define the features in your manifest file using the manifest + * editor in the SDK.\n + * More details on featuring your application can be found from + * + * Feature Element. + * + * + * @section CAPI_MEDIA_VISION_3D_MODULE_OVERVIEW Overview + * @ref CAPI_MEDIA_VISION_3D_MODULE contains #mv_3d_h handle to perform Depth and PointCloud process. + * 3D handle should be created with mv_3d_create() function and destroyed with mv_3d_destroy() function. + * #mv_3d_h should be configured by calling mv_3d_configure() function. After configuration, callback functions + * for depth and pointcloud should be set to #mv_3d_h. To get depth, callback mv_3d_depth_cb() should be set + * by mv_3d_set_depth_cb() function. To get pointcloud, callback mv_3d_pointcloud_cb() should be set + * by mv_3d_set_pointcloud_cb() function. After setting callbacks, mv_3d_h should be prepared by + * calling mv_3d_prepare() function which initializes depth and pointcloud process. + * After preparation, mv_3d_run() has to be called to process synchronously depth or pointcloud from #mv_source_h, + * and callback functions will be invoked with processed depth or pointcloud. + * Module also contains mv_3d_run_async() functions to process depth or pointcloud asynchronously. */ - -#endif /* __TIZEN_MEDIAVISION_DOC_H__ */ +#endif /* __TIZEN_MEDIAVISION_DOC_H__ */ diff --git a/include/mv_3d.h b/include/mv_3d.h new file mode 100644 index 0000000..583307f --- /dev/null +++ b/include/mv_3d.h @@ -0,0 +1,432 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __TIZEN_MEDIAVISION_3D_H__ +#define __TIZEN_MEDIAVISION_3D_H__ + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** + * @file mv_3d.h + * @brief This file contains the 3D API + * supporting depth and point cloud data process. + */ + +/** + * @addtogroup CAPI_MEDIA_VISION_3D_MODULE + * @{ + */ + +/** + * @brief Defines #MV_3D_DEPTH_MODE to set the engine configuration. + * Use #mv_3d_depth_mode_e for a value. + * + * @since_tizen 7.0 + */ +#define MV_3D_DEPTH_MODE "MV_3D_DEPTH_MODE" + +/** + * @brief Defines #MV_3D_DEPTH_WIDTH to set the engine configuration. + * + * @since_tizen 7.0 + */ +#define MV_3D_DEPTH_WIDTH "MV_3D_DEPTH_WIDTH" + +/** + * @brief Defines #MV_3D_DEPTH_HEIGHT to set the engine configuration. + * + * @since_tizen 7.0 + */ +#define MV_3D_DEPTH_HEIGHT "MV_3D_DEPTH_HEIGHT" + +/** + * @brief Defines #MV_3D_DEPTH_MIN_DISPARITY to set the engine configuration. + * + * @since_tizen 7.0 + */ +#define MV_3D_DEPTH_MIN_DISPARITY "MV_3D_DEPTH_MIN_DISPARITY" + +/** + * @brief Defines #MV_3D_DEPTH_MAX_DISPARITY to set the engine configuration. + * + * @since_tizen 7.0 + */ +#define MV_3D_DEPTH_MAX_DISPARITY "MV_3D_DEPTH_MAX_DISPARITY" + +/** + * @brief Defines #MV_3D_DEPTH_STEREO_CONFIG_FILE_PATH to set the stereo configuration + * file path to the engine configuration. + * @since_tizen 7.0 + */ +#define MV_3D_DEPTH_STEREO_CONFIG_FILE_PATH "MV_3D_DEPTH_STEREO_CONFIG_FILE_PATH" + +/** + * @brief Defines #MV_3D_POINTCLOUD_OUTPUT_FILE_PATH to set the output file path + * to the engine configuration. + * @since_tizen 7.0 + */ +#define MV_3D_POINTCLOUD_OUTPUT_FILE_PATH "MV_3D_POINTCLOUD_OUTPUT_FILE_PATH" + +/** + * @brief Defines #MV_3D_POINTCLOUD_SAMPLING_RATIO to set the downsampling ratio + * to the engine configuration. + * @since_tizen 7.0 + */ +#define MV_3D_POINTCLOUD_SAMPLING_RATIO "MV_3D_POINTCLOUD_SAMPLING_RATIO" + +/** + * @brief Defines #MV_3D_POINTCLOUD_OUTLIER_REMOVAL_POINTS to set the criteria number of outlier + * removal points to the engine configuration. + * @since_tizen 7.0 + */ +#define MV_3D_POINTCLOUD_OUTLIER_REMOVAL_POINTS "MV_3D_POINTCLOUD_OUTLIER_REMOVAL_POINTS" + +/** + * @brief Defines #MV_3D_POINTCLOUD_OUTLIER_REMOVAL_RADIUS to set the outlier + * removal radius to the engine configuration. + * @since_tizen 7.0 + */ +#define MV_3D_POINTCLOUD_OUTLIER_REMOVAL_RADIUS "MV_3D_POINTCLOUD_OUTLIER_REMOVAL_RADIUS" + +/** + * @brief Creates mv3d handle. + * + * @since_tizen 7.0 + * @remarks The @a mv3d should be released using mv_3d_destroy() + * if there is no more usage of @a mv3d. + * @param[out] mv3d The created mv3d handle + * + * @return @c 0 on success, otherwise a negative error value + * @retval #MEDIA_VISION_ERROR_NONE Successful + * @retval #MEDIA_VISION_ERROR_NOT_SUPPORTED Not supported + * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter + * + * @see mv_3d_destroy() + */ +int mv_3d_create(mv_3d_h *mv3d); + +/** + * @brief Destroys mv3d handle and release all its resources. + * + * @since_tizen 7.0 + * + * @param[in] mv3d The handle to the mv3d to be destroyed + * + * @return @c 0 on success, otherwise a negative error value + * @retval #MEDIA_VISION_ERROR_NONE Successful + * @retval #MEDIA_VISION_ERROR_NOT_SUPPORTED Not supported + * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter + * + * @pre Create mv3d handle by using mv_3d_create() + * + * @see mv_3d_create() + */ +int mv_3d_destroy(mv_3d_h mv3d); + +/** + * @brief Configures handle. + * @details Use this function to configure parameters of the mv3d + * which is set to @a engine_config. + * + * @since_tizen 7.0 + * + * @param[in] mv3d The handle to the mv3d + * @param[in] engine_config The handle to the configuration of engine + * + * @return @c 0 on success, otherwise a negative error value + * @retval #MEDIA_VISION_ERROR_NONE Successful + * @retval #MEDIA_VISION_ERROR_NOT_SUPPORTED Not supported + * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter + * + * @pre Create mv3d handle by using mv_3d_create() + * @pre Create engine_config handle by using mv_create_engine_config() + * + * @see mv_3d_create() + * @see mv_create_engine_config() + */ +int mv_3d_configure(mv_3d_h mv3d, + mv_engine_config_h engine_config); + +/** + * @brief Prepares handle. + * + * @since_tizen 7.0 + * + * @param[in] mv3d The handle to the mv3d + * + * @return @c 0 on success, otherwise a negative error value + * @retval #MEDIA_VISION_ERROR_NONE Successful + * @retval #MEDIA_VISION_ERROR_NOT_SUPPORTED Not supported + * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter + * @retval #MEDIA_VISION_ERROR_INVALID_OPERATION Invalid operation + * + * @pre Configure mv3d handle by using mv_3d_configure() + * + * @see mv_3d_configure() + */ +int mv_3d_prepare(mv_3d_h mv3d); + +/** + * @brief Invoked when @a depth is ready. + * @details This callback is invoked each time when + * mv_3d_run() or mv3d_run_async() is called + * to report estimated depth from given source(s). + * + * @since_tizen 7.0 + * @remarks The @a source is available until it is released by mv_destroy_source(). + * The @a depth can be used only in the callback. + * To use outside, make a copy. The @a depth is managed by the platform + * and should not be freed. + * @param[in] source The handle to the source of the media where + * the depth data comes from + * @param[in] depth The pointer of the depth data + * @param[in] width The width of @a depth + * @param[in] height The height of @a depth + * @param[in] user_data The user data passed from callback invoking code + * + * @pre Call mv_3d_run() or mv_3d_run_async() function to get depth data + * and to invoke this callback as a result + * + * @see mv_3d_run() + * @see mv_3d_run_async() + */ +typedef void (*mv_3d_depth_cb)( + mv_source_h source, + unsigned short *depth, + unsigned int width, + unsigned int height, + void *user_data); + +/** + * @brief Sets mv_3d_depth_cb() callback. + * @details Use this function to set mv_3d_depth_cb() callback. + * + * @since_tizen 7.0 + * + * @param[in] mv3d The handle to the mv3d + * @param[in] depth_cb The callback which will be invoked for + * getting depth data + * @param[in] user_data The user data passed from the code where + * mv_3d_run() or mv_3d_run_async() is invoked. This data will be + * accessible in @a depth_cb callback + * + * @return @c 0 on success, otherwise a negative error value + * @retval #MEDIA_VISION_ERROR_NONE Successful + * @retval #MEDIA_VISION_ERROR_NOT_SUPPORTED Not supported + * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter + * + * @pre Create mv3d handle by using mv_3d_create() + * @pre Create engine_config handle by using mv_create_engine_config() + * @pre Configure mv3d handle by using mv_3d_configure() + * @see mv_3d_create() + * @see mv_create_engine_config() + * @see mv_3d_configure() + */ +int mv_3d_set_depth_cb(mv_3d_h mv3d, + mv_3d_depth_cb depth_cb, + void *user_data); + +/** + * @brief Invoked when @a pointcloud is ready. + * @details This callback is invoked each time when + * mv_3d_run() or mv3d_run_async() is called + * to report estimated pointcloud from given source(s). + * + * @since_tizen 7.0 + * @remarks The @a source is available until it is released by mv_destroy_source(). + * The @a pointcloud can be used only in the callback. + * To use outside, make a copy. The @a pointcloud is managed by the platform + * and should not be freed. + * @param[in] source The handle to the source of the media where + * the pointcloud data comes from + * @param[in] pointcloud The pointer of the pointcloud result + * @param[in] user_data The user data passed from callback invoking code + * + * @pre Call mv_3d_run() or mv_3d_run_async() function to get pointcloud data + * and to invoke this callback as a result + * + * @see mv_3d_run() + * @see mv_3d_run_async() + */ +typedef void (*mv_3d_pointcloud_cb)( + mv_source_h source, + mv_3d_pointcloud_h pointcloud, + void *user_data); + +/** + * @brief Sets mv_3d_pointcloud_cb() callback. + * @details Use this function to set mv_3d_pointcloud_cb() callback. + * + * @since_tizen 7.0 + * + * @param[in] mv3d The handle to the mv3d + * @param[in] pointcloud_cb The callback which will be invoked for + * getting pointcloud data + * @param[in] user_data The user data passed from the code where + * mv_3d_run() or mv_3d_run_async() is invoked. This data will be + * accessible in @a pointcloud_cb callback + * + * @return @c 0 on success, otherwise a negative error value + * @retval #MEDIA_VISION_ERROR_NONE Successful + * @retval #MEDIA_VISION_ERROR_NOT_SUPPORTED Not supported + * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter + * + * @pre Create mv3d handle by using mv_3d_create() + * @pre Create engine_config handle by using mv_create_engine_config() + * @pre Configure mv3d handle by using mv_3d_configure() + * @see mv_3d_create() + * @see mv_create_engine_config() + * @see mv_3d_configure() + */ +int mv_3d_set_pointcloud_cb(mv_3d_h mv3d, + mv_3d_pointcloud_cb pointcloud_cb, + void *user_data); + +/** + * @brief Gets depth or pointcloud synchronously from given @a source + * or @a source_extra. + * @details Use this function to get depth data. + * @a source_extra can be null if @a source is a stereoscopic format + * media, for example a left and a right media are concatenated + * as a side-by-side format and then it should be given to @a source + * but @a source_extra should be null. + * @a source_extra should not be null if @a source is a mono format media or + * a single side/channel of a stereoscopic format, for example a left + * and a right media are separated and then they should be given to @a source + * and @a source_extra, respectively. + * @a color may not be null if pointcloud data includes color. + * + * @since_tizen 7.0 + * + * @param[in] mv3d The handle to the mv3d + * @param[in] source The handle to the source of the media + * @param[in] source_extra The handle to the extra source of the media + * @param[in] color The handle to the color of the media + * + * @return @c 0 on success, otherwise a negative error value + * @retval #MEDIA_VISION_ERROR_NONE Successful + * @retval #MEDIA_VISION_ERROR_NOT_SUPPORTED Not supported + * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter + * + * @pre Create a source handle by using mv_create_source() + * @pre Create a source_extra handle by using mv_create_source() + * @pre Create a mv3d handle by using mv_3d_create() + * @pre Configure a mv3d handle by using mv_3d_configure() + * @pre Set depth callback to a depth handle by using mv_3d_set_depth_cb() + * @pre Prepare a mv3d handle by using mv_3d_prepare() + * @post Callback which is set by mv_3d_set_depth_cb() will be invoked + * to provide depth data + * + * @see mv_3d_set_depth_cb() + * @see mv_3d_depth_cb() + */ +int mv_3d_run(mv_3d_h mv3d, + mv_source_h source, + mv_source_h source_extra, + mv_source_h color); + +/** + * @brief Gets depth or pointcloud asynchronously from given @a source + * or @a source_extra. + * @details Use this function to get depth data. + * @a source_extra can be null if @a source is a stereoscopic format + * media, for example a left and a right media are concatenated + * as a side-by-side format and then it should be given to @a source + * but @a source_extra should be null. + * @a source_extra should not be null if @a source is a mono format media or + * a single side/channel of a stereoscopic format, for example a left + * and a right media are separated and then they should be given to @a source + * and @a source_extra, respectively. + * @a color may not be null if pointcloud data includes color. + * + * @since_tizen 7.0 + * + * @param[in] mv3d The handle to the mv3d + * @param[in] source The handle to the source of the media + * @param[in] source_extra The handle to the extra source of the media + * @param[in] color The handle to the color of the media + * + * @return @c 0 on success, otherwise a negative error value + * @retval #MEDIA_VISION_ERROR_NONE Successful + * @retval #MEDIA_VISION_ERROR_NOT_SUPPORTED Not supported + * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter + * + * @pre Create a source handle by using mv_create_source() + * @pre Create a source_extra handle by using mv_create_source() + * @pre Create a mv3d handle by using mv_3d_create() + * @pre Configure a mv3d handle by using mv_3d_configure() + * @pre Set depth callback to a depth handle by using mv_3d_set_depth_cb() + * @pre Prepare a mv3d handle by using mv_3d_prepare() + * @post Callback which is set by mv_3d_set_depth_cb() will be invoked + * to provide depth data + * + * @see mv_3d_set_depth_cb() + * @see mv_3d_depth_cb() + */ +int mv_3d_run_async(mv_3d_h mv3d, + mv_source_h source, + mv_source_h source_extra, + mv_source_h color); + +/** + * @brief Writes pointcloud data to a file. + * @details Use this function to write pointcloud data to a file. + * + * @since_tizen 7.0 + * @remarks The mediastorage privilege %http://tizen.org/privilege/mediastorage + * is needed if @a filename is relevant to media storage.\n + * The externalstorage privilege %http://tizen.org/privilege/externalstorage + * is needed if @a filename is relevant to external storage. + * @param[in] mv3d The handle to the mv3d + * @param[in] pointcloud The handle to the pointcloud + * @param[in] type The file format type to be written + * @param[in] filename The filename to save @a pointcloud + * + * @return @c 0 on success, otherwise a negative error value + * @retval #MEDIA_VISION_ERROR_NONE Successful + * @retval #MEDIA_VISION_ERROR_NOT_SUPPORTED Not supported + * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter + * @retval #MEDIA_VISION_ERROR_PERMISSION_DENIED Permission denied + * @retval #MEDIA_VISION_ERROR_INVALID_PATH Invalid path + * + * @pre Create mv3d handle by using mv_3d_create() + * @pre Create engine_config handle by using mv_create_engine_config() + * @pre Configure mv3d handle by using mv_3d_configure() + * @see mv_3d_create() + * @see mv_create_engine_config() + * @see mv_3d_configure() + */ +int mv_3d_pointcloud_write_file(mv_3d_h mv3d, + mv_3d_pointcloud_h pointcloud, + mv_3d_pointcloud_type_e type, + char *filename); + +/** + * @} + */ +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __TIZEN_MEDIAVISION_3D_H__ */ + + diff --git a/include/mv_3d_private.h b/include/mv_3d_private.h new file mode 100644 index 0000000..84dcc98 --- /dev/null +++ b/include/mv_3d_private.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __TIZEN_MEDIAVISION_3D_PRIVATE_H__ +#define __TIZEN_MEDIAVISION_3D_PRIVATE_H__ + +#include +#include "mv_3d_type.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** + * @file mv_3d_private.h + * @brief This file contains the internal structure for depth or pointcloud. + * @since_tizen 7.0 + */ + +/** + * @brief Landmarks of a pose. + * + * @since_tizen 6.0 + * + */ +typedef struct { + mv_pointcloud_type_e type; + FILE *pointcloud; +} mv_pointcloud_s; + +/** + * @} + */ +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __TIZEN_MEDIAVISION_3D_TYPE_H__ */ diff --git a/include/mv_3d_type.h b/include/mv_3d_type.h new file mode 100644 index 0000000..3bee94f --- /dev/null +++ b/include/mv_3d_type.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __TIZEN_MEDIAVISION_3D_TYPE_H__ +#define __TIZEN_MEDIAVISION_3D_TYPE_H__ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** + * @file mv_3d_type.h + * @brief This file contains the mv3d type used. + * @since_tizen 7.0 + */ + +/** + * @addtogroup CAPI_MEDIA_VISION_3D_MODULE + * @{ + */ + +/** + * @brief Enumeration for mv3d depth mode. + * @since_tizen 7.0 + */ +typedef enum { + MV_3D_DEPTH_MODE_NONE, /**< NONE */ + MV_3D_DEPTH_MODE_STEREO /**< Stereo images are used */ +} mv_3d_depth_mode_e; + +/** + * @brief Enumeration for mv3d pointcloud file format type. + * @since_tizen 7.0 + */ +typedef enum { + MV_3D_POINTCLOUD_TYPE_PCD_TXT, /**< PointCloudData format as text */ + MV_3D_POINTCLOUD_TYPE_PCD_BIN, /**< PointCloudData format as binary */ + MV_3D_POINTCLOUD_TYPE_PLY_TXT, /**< Polygon format as text */ + MV_3D_POINTCLOUD_TYPE_PLY_BIN /**< Polygon format as binary */ +} mv_3d_pointcloud_type_e; + +/** + * @brief The mv3d handle. + * @since_tizen 7.0 + */ +typedef void *mv_3d_h; + +/** + * @brief The pointcloud result handle. + * @since_tizen 7.0 + */ +typedef void *mv_3d_pointcloud_h; + + +/** + * @} + */ +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __TIZEN_MEDIAVISION_3D_TYPE_H__ */ diff --git a/media-vision-config.json b/media-vision-config.json index 0c20bc6..d48ff9e 100644 --- a/media-vision-config.json +++ b/media-vision-config.json @@ -252,6 +252,51 @@ "name" : "MV_ROI_TRACKER_TYPE", "type" : "integer", "value" : "2" + }, + { + "name" : "MV_3D_DEPTH_MODE", + "type" : "integer", + "value" : 0 + }, + { + "name" : "MV_3D_DEPTH_WIDTH", + "type" : "integer", + "value" : 640 + }, + { + "name" : "MV_3D_DEPTH_HEIGHT", + "type" : "integer", + "value" : 480 + }, + { + "name" : "MV_3D_DEPTH_MIN_DISPARITY", + "type" : "integer", + "value" : 0 + }, + { + "name" : "MV_3D_DEPTH_MAX_DISPARITY", + "type" : "integer", + "value" : 64 + }, + { + "name" : "MV_3D_DEPTH_STEREO_CONFIG_FILE_PATH", + "type" : "string", + "value" : "" + }, + { + "name" : "MV_3D_POINTCLOUD_SAMPLING_RATIO", + "type" : "double", + "value" : 1.0 + }, + { + "name" : "MV_3D_POINTCLOUD_OUTLIER_REMOVAL_POINTS", + "type" : "integer", + "value" : 10 + }, + { + "name" : "MV_3D_POINTCLOUD_OUTLIER_REMOVAL_RADIUS", + "type" : "double", + "value" : 1.0 } ] } diff --git a/mv_3d/3d/CMakeLists.txt b/mv_3d/3d/CMakeLists.txt new file mode 100644 index 0000000..d2eb9af --- /dev/null +++ b/mv_3d/3d/CMakeLists.txt @@ -0,0 +1,21 @@ +project(${MV_3D_LIB_NAME}) +cmake_minimum_required(VERSION 2.6...3.13) + +pkg_check_modules(${PROJECT_NAME}_DEP REQUIRED dfs-adaptation) +file(GLOB MV_DFS_SOURCE_LIST "${PROJECT_SOURCE_DIR}/src/*.c" "${PROJECT_SOURCE_DIR}/src/*.cpp") + +find_package(OpenCV REQUIRED calib3d imgproc) +if(NOT OpenCV_FOUND) + message(SEND_ERROR "OpenCV NOT FOUND") + return() +endif() + +if(FORCED_STATIC_BUILD) + add_library(${PROJECT_NAME} STATIC ${MV_DFS_SOURCE_LIST}) +else() + add_library(${PROJECT_NAME} SHARED ${MV_DFS_SOURCE_LIST}) +endif() + +target_link_libraries(${PROJECT_NAME} ${MV_COMMON_LIB_NAME} ${OpenCV_LIBS} ${${PROJECT_NAME}_DEP_LIBRARIES}) +target_include_directories(${PROJECT_NAME} PRIVATE include) +install(TARGETS ${PROJECT_NAME} DESTINATION ${LIB_INSTALL_DIR}) diff --git a/mv_3d/3d/include/Depth.h b/mv_3d/3d/include/Depth.h new file mode 100644 index 0000000..b3236c4 --- /dev/null +++ b/mv_3d/3d/include/Depth.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MEDIA_VISION_DEPTH_H__ +#define __MEDIA_VISION_DEPTH_H__ + +#include +#include +#include "dfs_parameter.h" +#include "dfs_adaptation_impl.h" +#include "mv_3d.h" +#include "mv_3d_type.h" + +/** + * @file Depth.h + * @brief This file contains the depth class definition + * which supports depth-from-stereo (DFS). + */ +using namespace DfsAdaptation; +using DepthType = uint16_t; +using DepthTypePtr = DepthType*; +namespace mediavision +{ +namespace depth +{ + class Depth + { + private: + DfsParameter mDfsParameter; + DfsAdaptor *mDfsAdaptor; + int mMode; + + size_t mWidth; + size_t mHeight; + size_t mMinDisp; + size_t mMaxDisp; + std::string mStereoConfigPath; + + GThread *mThread; + void *mUserData; + mv_3d_depth_cb mDepthCallback; + bool mIsLive; + + GAsyncQueue *mAsyncQueue; + + void GetBufferFromSource(mv_source_h source, + unsigned char*& buffer, + unsigned int& width, + unsigned int& height, + int& type, + size_t& stride); + + void GetDfsDataFromSources(mv_source_h baseSource, + mv_source_h extraSource, + DfsInputData& input); + + + static gpointer ThreadLoop(gpointer data); + public: + Depth(); + ~Depth(); + void SetParameters(double threshold, + size_t windowWidth, + size_t windowHeight, + size_t speckleSize); + + int Configure(int mode, int width, int height, int minDisp, int maxDisp, + std::string stereoConfigPath); + + void SetCallback(mv_3d_depth_cb depthCallback, void *userData); + + int Prepare(); + + int Run(mv_source_h baseSource, mv_source_h extraSource); + + int RunAsync(mv_source_h baseSource, mv_source_h extraSource); + }; +} +} +#endif /* __MEDIA_VISION_DEPTH_H__ */ diff --git a/mv_3d/3d/include/Mv3d.h b/mv_3d/3d/include/Mv3d.h new file mode 100644 index 0000000..dc8b36c --- /dev/null +++ b/mv_3d/3d/include/Mv3d.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MEDIA_VISION_3D_H__ +#define __MEDIA_VISION_3D_H__ + +#include +#include "Depth.h" +#include "PointCloud.h" + +/** + * @file Mv3d.h + * @brief This file contains the mv3d class definition + * which supports depth and point cloud data. + */ +using namespace mediavision::depth; +using namespace mediavision::pointcloud; +namespace mediavision +{ +namespace mv3d +{ + class Mv3d + { + public: + Depth mDepth; + PointCloud mPointCloud; + + public: + Mv3d() = default; + ~Mv3d() = default; + }; +} +} +#endif /* __MEDIA_VISION_3D_H__ */ diff --git a/mv_3d/3d/include/PointCloud.h b/mv_3d/3d/include/PointCloud.h new file mode 100644 index 0000000..dcc91cb --- /dev/null +++ b/mv_3d/3d/include/PointCloud.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MEDIA_VISION_POINT_CLOUD_H__ +#define __MEDIA_VISION_POINT_CLOUD_H__ + +#include +#include +#include "mv_3d.h" +#include "mv_3d_type.h" + +/** + * @file PointCloud.h + * @brief This file contains the PointCloud class definition + * which supports Point Cloud. + */ +namespace mediavision +{ +namespace pointcloud +{ + class PointCloud + { + public: + PointCloud() = default; + ~PointCloud() = default; + }; +} +} +#endif /* __MEDIA_VISION_DEPTH_H__ */ diff --git a/mv_3d/3d/include/mv_3d_open.h b/mv_3d/3d/include/mv_3d_open.h new file mode 100644 index 0000000..b2c2582 --- /dev/null +++ b/mv_3d/3d/include/mv_3d_open.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MEDIA_VISION_3D_OPEN_H__ +#define __MEDIA_VISION_3D_OPEN_H__ + +#include +#include "mv_3d_type.h" +#include "mv_3d.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** + * @brief Creates mv3d handle. + * @since_tizen 7.0 + */ +int mv3dCreate(mv_3d_h *mv3d); + +/** + * @brief Destroys mv3d handle and release all its resources. + * @since_tizen 7.0 + */ +int mv3dDestroy(mv_3d_h mv3d); + +/** + * @brief Configure mv3d handle. + * @sicne_tizen 7.0 + */ +int mv3dConfigure(mv_3d_h mv3d, mv_engine_config_h engine_config); + +/** + * @brief Set depth callback to mv3d handle. + * @sicne_tizen 7.0 + */ +int mv3dSetDepthCallback(mv_3d_h mv3d, mv_3d_depth_cb depth_cb, void *user_data); + +/** + * @brief Prepares mv3d handle. + * @since_tizen 7.0 + */ +int mv3dPrepare(mv_3d_h mv3d); + +/** + * @brief Gets depth data from source(s). + * @since_tizen 7.0 + */ +int mv3dRun(mv_source_h source, + mv_source_h source_extra, + mv_3d_h mv3d); + +/** + * @brief Run depth estimation asynchronousely with source(s). + * @since_tizen 7.0 + */ +int mv3dRunAsync(mv_source_h source, + mv_source_h source_extra, + mv_3d_h mv3d); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __MEDIA_VISION_3D_OPEN_H__ */ + + diff --git a/mv_3d/3d/src/Depth.cpp b/mv_3d/3d/src/Depth.cpp new file mode 100644 index 0000000..01e3197 --- /dev/null +++ b/mv_3d/3d/src/Depth.cpp @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mv_private.h" +#include +#include +#include "Depth.h" + +namespace mediavision +{ +namespace depth +{ + Depth::Depth() : + mDfsParameter(), + mDfsAdaptor(nullptr), + mMode(MV_3D_DEPTH_MODE_NONE), + mWidth(0), + mHeight(0), + mMinDisp(0), + mMaxDisp(0), + mThread(nullptr), + mUserData(nullptr), + mDepthCallback(nullptr), + mIsLive(false), + mAsyncQueue(nullptr) + { + LOGI("ENTER"); + LOGI("LEAVE"); + } + + Depth::~Depth() + { + LOGI("ENTER"); + + if (mThread) { + mIsLive = false; + g_thread_join(mThread); + } + + if (mAsyncQueue) { + g_async_queue_unref(mAsyncQueue); + } + + if (mDfsAdaptor) { + mDfsAdaptor->unBind(); + delete mDfsAdaptor; + } + + LOGI("LEAVE"); + } + + void Depth::SetParameters(double threshold, + size_t windowWidth, + size_t windowHeight, + size_t speckleSize) + { + mDfsParameter.textureThreshold = threshold; + mDfsParameter.aggregationWindowWidth = windowWidth; + mDfsParameter.aggregationWindowHeight = windowHeight; + mDfsParameter.maxSpeckleSize = speckleSize; + } + + int Depth::Configure(int mode, int width, int height, + int minDisp, int maxDisp, std::string stereoConfigPath) + { + mMode = mode; + mWidth = width; + mHeight = height; + mMinDisp = minDisp; + mMaxDisp = maxDisp; + mStereoConfigPath = stereoConfigPath; + + try { + mDfsAdaptor = new DfsAdaptor(); + mDfsAdaptor->bind(); + } catch (const std::bad_alloc &e) { + LOGE("Failed to create dfs adaptation : %s", e.what()); + return MEDIA_VISION_ERROR_OUT_OF_MEMORY; + } catch (const std::runtime_error &e) { + LOGE("Failed to bind %s adpator", e.what()); + return MEDIA_VISION_ERROR_INVALID_OPERATION; + } + + return MEDIA_VISION_ERROR_NONE; + } + + void Depth::SetCallback(mv_3d_depth_cb depthCallback, void *userData) + { + mDepthCallback = depthCallback; + mUserData = userData; + } + + int Depth::Prepare() + { + if (!mDfsAdaptor) { + LOGE("Invalid Opertation. Do Configure first."); + return MEDIA_VISION_ERROR_INVALID_OPERATION; + } + + try { + mDfsAdaptor->initialize(mDfsParameter, mWidth, mHeight, + mMinDisp, mMaxDisp, mStereoConfigPath); + } catch (const std::exception& e) { + LOGE("Failed to initialize"); + return MEDIA_VISION_ERROR_INVALID_OPERATION; + } + + return MEDIA_VISION_ERROR_NONE; + } + + + void Depth::GetBufferFromSource(mv_source_h source, + unsigned char*& buffer, + unsigned int& width, + unsigned int& height, + int& type, + size_t& stride) + { + unsigned char* _buffer = nullptr; + unsigned int _bufferSize = 0; + unsigned int _width = 0; + unsigned int _height = 0; + mv_colorspace_e _colorSpace = MEDIA_VISION_COLORSPACE_INVALID; + + int ret = mv_source_get_buffer(source, &_buffer, &_bufferSize); + if (ret != MEDIA_VISION_ERROR_NONE) + throw std::runtime_error("invalid buffer pointer"); + + ret = mv_source_get_width(source, &_width); + if (ret != MEDIA_VISION_ERROR_NONE) + throw std::runtime_error("invalid width"); + + ret = mv_source_get_height(source, &_height); + if (ret != MEDIA_VISION_ERROR_NONE) + throw std::runtime_error("invalid height"); + + ret = mv_source_get_colorspace(source, &_colorSpace); + if (ret != MEDIA_VISION_ERROR_NONE) + throw std::runtime_error("invalid color space"); + + buffer = new unsigned char [_bufferSize]; + memcpy(buffer, _buffer, _bufferSize); + width = _width; + height = _height; + type = _colorSpace == MEDIA_VISION_COLORSPACE_RGB888 ? + DFS_DATA_TYPE_UINT8C3 : + DFS_DATA_TYPE_UINT8C1; + stride = _bufferSize / _height; + } + + void Depth::GetDfsDataFromSources(mv_source_h baseSource, + mv_source_h extraSource, + DfsInputData& input) + { + unsigned char* baseBuffer = nullptr; + unsigned char* extraBuffer = nullptr; + unsigned int width = 0; + unsigned int height = 0; + int type = 0; + size_t stride = 0; + + GetBufferFromSource(baseSource, baseBuffer, width, height, type, stride); + input.data = static_cast(baseBuffer); + input.type = type; + input.width = width; + input.height = height; + input.stride = stride; + input.format = DFS_DATA_INPUT_FORMAT_COUPLED_SBS; + + if (extraSource) + { + extraBuffer = nullptr; + GetBufferFromSource(extraSource, extraBuffer, width, height, type, stride); + + if (input.type != type || input.width != width || + input.height != height || input.stride != stride) { + throw std::runtime_error("left and right image's properties are different"); + } + + input.extraData = static_cast(extraBuffer); + input.format = DFS_DATA_INPUT_FORMAT_DECOUPLED_SBS; + } + } + + int Depth::Run(mv_source_h baseSource, mv_source_h extraSource) + { + DfsInputData input; + try { + if (mThread) { + mIsLive = false; + g_thread_join(mThread); + mThread = nullptr; + } + + if (mAsyncQueue) { + g_async_queue_unref(mAsyncQueue); + mAsyncQueue = nullptr; + } + + GetDfsDataFromSources(baseSource, extraSource, input); + + mDfsAdaptor->run(input); + + auto depthData = mDfsAdaptor->getOutputData(); + + mDepthCallback( + baseSource, + static_cast(depthData.data), + depthData.width, depthData.height, + mUserData); + + delete [] input.data; + delete [] input.extraData; + } catch (const std::exception &e) { + LOGE("Failed to Run with %s", e.what()); + return MEDIA_VISION_ERROR_INVALID_OPERATION; + } + + return MEDIA_VISION_ERROR_NONE; + } + + int Depth::RunAsync(mv_source_h baseSource, mv_source_h extraSource) + { + try { + if (!mAsyncQueue) { + mAsyncQueue = g_async_queue_new(); + if (!mAsyncQueue) { + LOGE("Fail to g_async_queue_new()"); + return MEDIA_VISION_ERROR_INTERNAL; + } + } + + if (!mThread) { + mThread = g_thread_new("depth_thread", &Depth::ThreadLoop, static_cast(this)); + + if (!mThread) { + g_async_queue_unref(mAsyncQueue); + mAsyncQueue = nullptr; + LOGE("Fail to g_thread_new()"); + return MEDIA_VISION_ERROR_INTERNAL; + } + mIsLive = true; + } + + std::shared_ptr input(new DfsInputData); + GetDfsDataFromSources(baseSource, extraSource, *input); + g_async_queue_push(mAsyncQueue, static_cast( + new std::shared_ptr( + std::move(input)))); + } catch (const std::exception &e) { + LOGE("Failed to Run with %s", e.what()); + return MEDIA_VISION_ERROR_INVALID_OPERATION; + } + + return MEDIA_VISION_ERROR_NONE; + } + + gpointer Depth::ThreadLoop(gpointer data) + { + Depth *handle = static_cast(data); + while(handle->mIsLive) { + gpointer base = g_async_queue_try_pop(handle->mAsyncQueue); + if (!base) { + continue; + } + + auto pInput = static_cast*>(base); + auto input = std::move(*pInput); + delete pInput; + handle->mDfsAdaptor->run(*input); + delete [] static_cast(input->data); + delete [] static_cast(input->extraData); + input.reset(); + + auto depthData = handle->mDfsAdaptor->getOutputData(); + handle->mDepthCallback( + static_cast(base), + static_cast(depthData.data), + depthData.width, depthData.height, + handle->mUserData); + } + + return nullptr; + } +} +} diff --git a/mv_3d/3d/src/PointCloud.cpp b/mv_3d/3d/src/PointCloud.cpp new file mode 100644 index 0000000..083764d --- /dev/null +++ b/mv_3d/3d/src/PointCloud.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mv_private.h" +#include +#include +#include "PointCloud.h" + +namespace mediavision +{ +namespace pointCloud +{ +// +} +} diff --git a/mv_3d/3d/src/mv_3d.c b/mv_3d/3d/src/mv_3d.c new file mode 100644 index 0000000..d766ed0 --- /dev/null +++ b/mv_3d/3d/src/mv_3d.c @@ -0,0 +1,169 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mv_private.h" +#include "mv_3d.h" +#include "mv_3d_type.h" +#include "mv_3d_open.h" + +/** + * @file mv_3d.c + * @brief This file contains Media Vision 3D module. + */ + +int mv_3d_create(mv_3d_h *mv3d) +{ + MEDIA_VISION_NULL_ARG_CHECK(mv3d); + + MEDIA_VISION_FUNCTION_ENTER(); + + int ret = mv3dCreate(mv3d); + + MEDIA_VISION_FUNCTION_LEAVE(); + + return ret; +} + +int mv_3d_destroy(mv_3d_h mv3d) +{ + MEDIA_VISION_INSTANCE_CHECK(mv3d); + + MEDIA_VISION_FUNCTION_ENTER(); + + int ret = mv3dDestroy(mv3d); + + MEDIA_VISION_FUNCTION_LEAVE(); + + return ret; +} + +int mv_3d_configure(mv_3d_h mv3d, + mv_engine_config_h engine_config) +{ + MEDIA_VISION_INSTANCE_CHECK(mv3d); + MEDIA_VISION_INSTANCE_CHECK(engine_config); + + MEDIA_VISION_FUNCTION_ENTER(); + + int ret = mv3dConfigure(mv3d, engine_config); + + MEDIA_VISION_FUNCTION_LEAVE(); + + return ret; +} + +int mv_3d_set_depth_cb(mv_3d_h mv3d, + mv_3d_depth_cb depth_cb, + void *user_data) +{ + MEDIA_VISION_INSTANCE_CHECK(mv3d); + + MEDIA_VISION_NULL_ARG_CHECK(depth_cb); + + MEDIA_VISION_FUNCTION_ENTER(); + + int ret = mv3dSetDepthCallback(mv3d, depth_cb, user_data); + + MEDIA_VISION_FUNCTION_LEAVE(); + + return ret; +} + +int mv_3d_set_pointcloud_cb(mv_3d_h mv3d, + mv_3d_pointcloud_cb pointcloud_cb, + void *user_data) +{ + MEDIA_VISION_INSTANCE_CHECK(mv3d); + + MEDIA_VISION_NULL_ARG_CHECK(pointcloud_cb); + + MEDIA_VISION_FUNCTION_ENTER(); + + int ret = MEDIA_VISION_ERROR_NONE; + + MEDIA_VISION_FUNCTION_LEAVE(); + + return ret; +} + +int mv_3d_prepare(mv_3d_h mv3d) +{ + MEDIA_VISION_INSTANCE_CHECK(mv3d); + + MEDIA_VISION_FUNCTION_ENTER(); + + int ret = mv3dPrepare(mv3d); + + MEDIA_VISION_FUNCTION_LEAVE(); + + return ret; +} + +int mv_3d_run(mv_3d_h mv3d, + mv_source_h source, + mv_source_h source_extra, + mv_source_h color) +{ + MEDIA_VISION_INSTANCE_CHECK(mv3d); + MEDIA_VISION_INSTANCE_CHECK(source); + + MEDIA_VISION_FUNCTION_ENTER(); + + int ret = mv3dRun(mv3d, + source, + source_extra); + + MEDIA_VISION_FUNCTION_LEAVE(); + + return ret; +} + +int mv_3d_run_async(mv_3d_h mv3d, + mv_source_h source, + mv_source_h source_extra, + mv_source_h color) +{ + MEDIA_VISION_INSTANCE_CHECK(mv3d); + MEDIA_VISION_INSTANCE_CHECK(source); + + MEDIA_VISION_FUNCTION_ENTER(); + + int ret = mv3dRunAsync(source, + source_extra, + mv3d); + + MEDIA_VISION_FUNCTION_LEAVE(); + + return ret; +} + +int mv_3d_pointcloud_write_file(mv_3d_h mv3d, + mv_3d_pointcloud_h pointcloud, + mv_3d_pointcloud_type_e type, + char *filename) +{ + MEDIA_VISION_INSTANCE_CHECK(mv3d); + MEDIA_VISION_INSTANCE_CHECK(pointcloud); + + MEDIA_VISION_FUNCTION_ENTER(); + + int ret = MEDIA_VISION_ERROR_NONE; + + MEDIA_VISION_FUNCTION_LEAVE(); + + return ret; +} + diff --git a/mv_3d/3d/src/mv_3d_open.cpp b/mv_3d/3d/src/mv_3d_open.cpp new file mode 100644 index 0000000..7fd9026 --- /dev/null +++ b/mv_3d/3d/src/mv_3d_open.cpp @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include "mv_private.h" +#include "mv_common.h" +#include "mv_3d.h" +#include "mv_3d_open.h" +#include "Mv3d.h" + +//using namespace mediavision::depth; +using namespace mediavision::mv3d; + +int mv3dCreate(mv_3d_h *mv3d) +{ + if (!mv3d) { + LOGE("Handle pointer is NULL"); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + + try { + (*mv3d) = static_cast(new Mv3d()); + } catch (const std::exception &e) { + LOGE("Failed to create mv3d handle with %s", e.what()); + return MEDIA_VISION_ERROR_OUT_OF_MEMORY; + } + + return MEDIA_VISION_ERROR_NONE; +} + +int mv3dDestroy(mv_3d_h mv3d) +{ + if (!mv3d) { + LOGE("Handle is NULL"); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + + delete static_cast(mv3d); + + return MEDIA_VISION_ERROR_NONE; +} + +int mv3dSetDepthParameters(mv_3d_h mv3d, mv_engine_config_h engine_config) +{ + LOGI("ENTER"); + + if (!mv3d || !engine_config) { + LOGE("Handle is NULL"); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + + auto pMv3d = static_cast(mv3d); + + pMv3d->mDepth.SetParameters(127.5, // threshold + 3, // aggregation window width + 3, // aggregation window height + 0 // speckleSize + ); + LOGI("LEAVE"); + + return MEDIA_VISION_ERROR_NONE; +} + +int mv3dConfigure(mv_3d_h mv3d, mv_engine_config_h engine_config) +{ + LOGI("ENTER"); + + if (!mv3d || !engine_config) { + LOGE("Handle is NULL"); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + + int mode = MV_3D_DEPTH_MODE_NONE; + + int ret = mv_engine_config_get_int_attribute( + engine_config, MV_3D_DEPTH_MODE, &mode); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to get depth mode"); + return ret; + } + + ret = mv3dSetDepthParameters(mv3d, engine_config); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to set dfs parameters"); + return ret; + } + + int depthWidth, depthHeight, minDisp, maxDisp; + ret = mv_engine_config_get_int_attribute( + engine_config, MV_3D_DEPTH_WIDTH, &depthWidth); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to get depth width"); + return ret; + } + + ret = mv_engine_config_get_int_attribute( + engine_config, MV_3D_DEPTH_HEIGHT, &depthHeight); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to get depth height"); + return ret; + } + + ret = mv_engine_config_get_int_attribute( + engine_config, MV_3D_DEPTH_MIN_DISPARITY, &minDisp); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to get min disparity"); + return ret; + } + + ret = mv_engine_config_get_int_attribute( + engine_config, MV_3D_DEPTH_MAX_DISPARITY, &maxDisp); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to get max disparity"); + return ret; + } + + char *stereoConfigFilePath = NULL; + ret = mv_engine_config_get_string_attribute( + engine_config, MV_3D_DEPTH_STEREO_CONFIG_FILE_PATH, &stereoConfigFilePath); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to get stereo configuration file path"); + if (stereoConfigFilePath) { + free(stereoConfigFilePath); + stereoConfigFilePath = NULL; + } + return ret; + } + + auto pMv3d = static_cast(mv3d); + ret = pMv3d->mDepth.Configure(mode, depthWidth, depthHeight, minDisp, maxDisp, + stereoConfigFilePath); + + if (stereoConfigFilePath) { + free(stereoConfigFilePath); + stereoConfigFilePath = NULL; + } + + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to configure Depth"); + return ret; + } + + LOGI("LEAVE"); + + return ret; +} + +int mv3dSetDepthCallback(mv_3d_h mv3d, mv_3d_depth_cb depth_cb, void *user_data) +{ + LOGI("ENTER"); + + if (!mv3d) { + LOGE("Handle is NULL"); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + + if (!depth_cb) { + LOGE("Callbakc is NULL"); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + + auto pMv3d = static_cast(mv3d); + pMv3d->mDepth.SetCallback(depth_cb, user_data); + + LOGI("LEAVE"); + + return MEDIA_VISION_ERROR_NONE; +} + +int mv3dPrepare(mv_3d_h mv3d) +{ + LOGI("ENTER"); + + if (!mv3d) { + LOGE("Handle is NULL"); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + + auto pMv3d = static_cast(mv3d); + int ret = pMv3d->mDepth.Prepare(); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to prepare depth"); + return ret; + } + + LOGI("LEAVE"); + + return ret; +} + +int mv3dRun(mv_source_h source, + mv_source_h source_extra, + mv_3d_h mv3d) +{ + LOGI("ENTER"); + + if (!source || !mv3d) { + LOGE("source[%p], mv3d[%p]", + source, mv3d); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + + auto pMv3d = static_cast(mv3d); + + int ret = pMv3d->mDepth.Run(source, source_extra); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to run depth"); + return ret; + } + + LOGI("LEAVE"); + + return ret; +} + +int mv3dRunAsync(mv_source_h source, + mv_source_h source_extra, + mv_3d_h mv3d) +{ + LOGI("ENTER"); + + if (!source || !mv3d) { + LOGE("source[%p], mv3d[%p]", + source, mv3d); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + + auto pMv3d = static_cast(mv3d); + int ret = pMv3d->mDepth.RunAsync(source, source_extra); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to run depth"); + return ret; + } + + LOGI("LEAVE"); + + return ret; +} \ No newline at end of file diff --git a/mv_3d/CMakeLists.txt b/mv_3d/CMakeLists.txt new file mode 100644 index 0000000..4be9838 --- /dev/null +++ b/mv_3d/CMakeLists.txt @@ -0,0 +1,5 @@ +project(mv_3d) +cmake_minimum_required(VERSION 2.6...3.13) + +add_subdirectory(${PROJECT_SOURCE_DIR}/3d +) diff --git a/packaging/capi-media-vision.spec b/packaging/capi-media-vision.spec index ba8863e..64f6bb8 100644 --- a/packaging/capi-media-vision.spec +++ b/packaging/capi-media-vision.spec @@ -28,7 +28,16 @@ BuildRequires: pkgconfig(gstreamer-base-1.0) BuildRequires: pkgconfig(gstreamer-app-1.0) BuildRequires: pkgconfig(libtzplatform-config) BuildRequires: pkgconfig(ncurses) +BuildRequires: pkgconfig(libdrm) +BuildRequires: pkgconfig(wayland-egl) +BuildRequires: pkgconfig(glesv2) BuildRequires: gtest-devel +BuildRequires: pkgconfig(dfs-adaptation) +BuildRequires: Open3D-devel +%if 0%{?visualizer:1} +BuildRequires: pkgconfig(protobuf) +BuildRequires: pkgconfig(grpc++) +%endif %endif # Build options @@ -50,6 +59,7 @@ Requires: %{name}-barcode Requires: %{name}-face Requires: %{name}-image Requires: %{name}-surveillance +Requires: %{name}-3d Requires: %{name}-roi_tracker %endif @@ -66,6 +76,7 @@ Requires: %{name}-face-devel Requires: %{name}-image-devel Requires: %{name}-machine_learning-devel Requires: %{name}-surveillance-devel +Requires: %{name}-3d-devel Requires: %{name}-roi_tracker-devel %description devel @@ -164,6 +175,23 @@ Requires: %{name}-common-devel %description surveillance-devel Media Vision Surveillance library for Tizen Native API (DEV). + +%package 3d +Summary: Multimedia Vision 3d Library +Group: Multimedia/Framework +Requires: %{name}-common + +%description 3d +Media Vision 3d library for Tizen Native API. + +%package 3d-devel +Summary: Multimedia Vision 3d Library +Group: Multimedia/Framework +Requires: %{name}-3d +Requires: %{name}-common-devel + +%description 3d-devel +Media Vision 3d library for Tizen Native API (DEV). %endif %package machine_learning @@ -211,6 +239,7 @@ Requires: %{name}-face Requires: %{name}-image Requires: %{name}-surveillance Requires: %{name}-machine_learning +Requires: %{name}-3d Requires: %{name}-roi_tracker Requires: check @@ -245,6 +274,9 @@ export LDFLAGS+=" -lgcov" %if 0%{?ml_only:1} -DBUILD_ML_ONLY=ON \ %endif +%if 0%{?visualizer:1} + -DBUILD_VISUALIZER=ON \ +%endif %ifarch %arm aarch64 -DENABLE_NEON=ON %endif @@ -319,6 +351,15 @@ find . -name '*.gcno' -exec cp --parents '{}' "$gcno_obj_dir" ';' %files surveillance-devel %{_includedir}/media/mv_surveillance*.h %{_libdir}/pkgconfig/*surveillance.pc + +%files 3d +%manifest %{name}.manifest +%license LICENSE.APLv2 +%{_libdir}/libmv_3d*.so + +%files 3d-devel +%{_includedir}/media/mv_3d*.h +%{_libdir}/pkgconfig/*3d.pc %endif %files machine_learning @@ -355,6 +396,9 @@ find . -name '*.gcno' -exec cp --parents '{}' "$gcno_obj_dir" ';' %manifest %{name}.manifest %license LICENSE.APLv2 %{_libdir}/libmv_*helper.so +%if 0%{?visualizer:1} +%{_libdir}/libmv_visualizer.so +%endif %{_libdir}/libmv_testsuite*.so %{_bindir}/mv_* %if "%{enable_ml_face_recognition}" == "1" diff --git a/test/testsuites/CMakeLists.txt b/test/testsuites/CMakeLists.txt index a8ec29b..115273d 100644 --- a/test/testsuites/CMakeLists.txt +++ b/test/testsuites/CMakeLists.txt @@ -11,3 +11,8 @@ add_subdirectory(${PROJECT_SOURCE_DIR}/image) add_subdirectory(${PROJECT_SOURCE_DIR}/surveillance) add_subdirectory(${PROJECT_SOURCE_DIR}/machine_learning) add_subdirectory(${PROJECT_SOURCE_DIR}/tracker) +add_subdirectory(${PROJECT_SOURCE_DIR}/mv3d) + +if(BUILD_VISUALIZER) + add_subdirectory(${PROJECT_SOURCE_DIR}/visualizer) +endif() diff --git a/test/testsuites/common/CMakeLists.txt b/test/testsuites/common/CMakeLists.txt index 7f37580..280042a 100644 --- a/test/testsuites/common/CMakeLists.txt +++ b/test/testsuites/common/CMakeLists.txt @@ -1,3 +1,7 @@ add_subdirectory(image_helper) add_subdirectory(video_helper) add_subdirectory(testsuite_common) + +if(BUILD_VISUALIZER) + add_subdirectory(visualizer) +endif() diff --git a/test/testsuites/common/testsuite_common/mv_testsuite_common.h b/test/testsuites/common/testsuite_common/mv_testsuite_common.h index 5b0bc7c..af0a3a3 100644 --- a/test/testsuites/common/testsuite_common/mv_testsuite_common.h +++ b/test/testsuites/common/testsuite_common/mv_testsuite_common.h @@ -21,10 +21,13 @@ #include -typedef enum -{ - FAIL_OR_SUCCESSS, - FAIL_OR_DONE +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +typedef enum { + FAIL_OR_SUCCESSS, + FAIL_OR_DONE } notification_type_e; /** @@ -148,4 +151,8 @@ int show_menu_yes_or_no(const char *title); */ int load_mv_source_from_file(const char *path_to_image, mv_source_h source); -#endif /* __MEDIA_VISION_MV_TESTSUITE_COMMON_H__ */ +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __MEDIA_VISION_MV_TESTSUITE_COMMON_H__ */ \ No newline at end of file diff --git a/test/testsuites/common/visualizer/CMakeLists.txt b/test/testsuites/common/visualizer/CMakeLists.txt new file mode 100644 index 0000000..86cc7b4 --- /dev/null +++ b/test/testsuites/common/visualizer/CMakeLists.txt @@ -0,0 +1,37 @@ +project(mv_visualizer) +cmake_minimum_required(VERSION 2.6...3.13) + +execute_process(COMMAND protoc -I=${CMAKE_CURRENT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/mv_util_visualizer_rd.proto + --cpp_out=${PROJECT_SOURCE_DIR}/src) + +execute_process(COMMAND protoc -I=${CMAKE_CURRENT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/mv_util_visualizer_rd.proto + --grpc_out=${PROJECT_SOURCE_DIR}/src + --plugin=protoc-gen-grpc=/usr/bin/grpc_cpp_plugin) + +file(GLOB MV_VISUALIZER_SRC_LIST "${PROJECT_SOURCE_DIR}/src/*.cpp" + "${PROJECT_SOURCE_DIR}/src/*.cc") + +pkg_check_modules(${PROJECT_NAME}_DEP REQUIRED capi-media-tool + libdrm wayland-egl glesv2 dlog protobuf grpc++) + +find_package(OpenCV REQUIRED imgproc) +if(NOT OpenCV_FOUND) + message(SEND_ERROR "Failed to find OpenCV") + return() +endif() + +if(FORCED_STATIC_BUILD) + add_library(${PROJECT_NAME} STATIC ${MV_VISUALIZER_SRC_LIST}) +else() + add_library(${PROJECT_NAME} SHARED ${MV_VISUALIZER_SRC_LIST}) +endif() + +target_include_directories(${PROJECT_NAME} PUBLIC + ${${PROJECT_NAME}_DEP_INCLUDE_DIRS} + ${OpenCV_INCLUDE_DIRS} + ${PROJECT_SOURCE_DIR}/include) +target_link_libraries(${PROJECT_NAME} ${${PROJECT_NAME}_DEP_LIBRARIES} + ${OpenCV_LIBS} mv_common) +install(TARGETS ${PROJECT_NAME} DESTINATION ${LIB_INSTALL_DIR}) diff --git a/test/testsuites/common/visualizer/include/mv_util_matrix.h b/test/testsuites/common/visualizer/include/mv_util_matrix.h new file mode 100644 index 0000000..62fa9d7 --- /dev/null +++ b/test/testsuites/common/visualizer/include/mv_util_matrix.h @@ -0,0 +1,47 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MEDIA_VISION_UTIL_MATRIX_H__ +#define __MEDIA_VISION_UTIL_MATRIX_H__ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#include +#include + +int matrix_translate(float *m, float x, float y, float z); +int matrix_rotate(float *m, float angle, float x, float y, float z); +int matrix_scale(float *m, float x, float y, float z); +int matrix_mult(float *m, float *m1, float *m2); +int matrix_identity(float *m); +void matrix_copy(float *d, float *s); +void matrix_invert(float *m); +void matrix_transpose(float *m); +void matrix_proj_perspective(float *mat, float fovy, float aspect, float znear, float zfar); +void quaternion_mult(float *lpR, float *lpP, float *lpQ); +void quaternion_to_matrix(float *lpM, float *lpQ); +void quaternion_rotate(float *lpQ, float rad, float ax, float ay, float az); +void quaternion_identity(float *lpQ); +void quaternion_copy (float *lpTo, float *lpFrom); +float vec3_normalize (float *v); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __MEDIA_VISION_UTIL_MATRIX_H__ */ diff --git a/test/testsuites/common/visualizer/include/mv_util_render_2d.h b/test/testsuites/common/visualizer/include/mv_util_render_2d.h new file mode 100644 index 0000000..4918ba1 --- /dev/null +++ b/test/testsuites/common/visualizer/include/mv_util_render_2d.h @@ -0,0 +1,61 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MEDIA_VISION_UTIL_RENDER_2D_H__ +#define __MEDIA_VISION_UTIL_RENDER_2D_H__ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#include +#include +#include +#include +#include +#include +#include "mv_util_matrix.h" +#include "mv_util_shader.h" +#include "mv_util_winsys.h" + +#ifdef __cplusplus +#include +#else +#include +#endif + +#define pixfmt_fourcc(a, b, c, d)\ + ((uint32_t)(a) | ((uint32_t)(b) << 8) | ((uint32_t)(c) << 16) | ((uint32_t)(d) << 24)) + +typedef struct _texture_2d_t { + uint32_t texid; + int width; + int height; + uint32_t format; +} texture_2d_t; + +uint32_t create_2d_texture(void *imgbuf, int width, int height); +int init_2d_renderer(int w, int h); +int load_texture(mv_source_h source, int *lpTexID, int *lpWidth, int *lpHeight); +int draw_2d_texture(texture_2d_t *tex, int x, int y, int w, int h, int upsidedown); +int egl_init_with_platform_window_surface(int gles_version, int depth_size, int stencil_size, int sample_num, int win_w, int win_h); +int egl_swap(); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __MEDIA_VISION_UTIL_RENDER_2D_H__ */ diff --git a/test/testsuites/common/visualizer/include/mv_util_render_3d.h b/test/testsuites/common/visualizer/include/mv_util_render_3d.h new file mode 100644 index 0000000..dd4b361 --- /dev/null +++ b/test/testsuites/common/visualizer/include/mv_util_render_3d.h @@ -0,0 +1,41 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MEDIA_VISION_UTIL_RENDER_3D_H__ +#define __MEDIA_VISION_UTIL_RENDER_3D_H__ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#include +#include +#include +#include +#include +#include +#include "mv_util_shader.h" + +int init_cube(float aspect); +int draw_line(float *mtxGlobal, float *p0, float *p1, float *color); +int draw_point_arrays(float *mtxGlobal, float *vtx, float *uv, int num, int texid, float *color); +int create_mesh(mesh_obj_t *mesh, int num_tile_w, int num_tile_h); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __MEDIA_VISION_UTIL_RENDER_3D_H__*/ \ No newline at end of file diff --git a/test/testsuites/common/visualizer/include/mv_util_shader.h b/test/testsuites/common/visualizer/include/mv_util_shader.h new file mode 100644 index 0000000..bd482bb --- /dev/null +++ b/test/testsuites/common/visualizer/include/mv_util_shader.h @@ -0,0 +1,66 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MEDIA_VISION_UTIL_SHADER_H__ +#define __MEDIA_VISION_UTIL_SHADER_H__ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#include +#include +#include +#include +#include +#include +#include + +typedef struct shader_obj_t { + GLuint program; + GLint loc_vtx; + GLint loc_nrm; + GLint loc_clr; + GLint loc_uv; + GLint loc_tex; + GLint loc_mtx; + GLint loc_mtx_nrm; +} shader_obj_t; + +typedef struct _mesh_obj_t +{ + float *vtx_array; + float *uv_array; + unsigned short *idx_array; + + GLuint vbo_vtx; + GLuint vbo_uv; + GLuint vbo_idx; + + int num_tile_w; + int num_tile_h; + int num_idx; +} mesh_obj_t; + +GLuint compile_shader_text(GLenum shaderType, const char *text); +GLuint link_shaders(GLuint vertShader, GLuint fragShader); +int generate_shader(shader_obj_t *sobj, char *str_vs, char *str_fs); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __MEDIA_VISION_UTIL_SHADER_H__*/ \ No newline at end of file diff --git a/test/testsuites/common/visualizer/include/mv_util_visualizer_2d.h b/test/testsuites/common/visualizer/include/mv_util_visualizer_2d.h new file mode 100644 index 0000000..e7c3a83 --- /dev/null +++ b/test/testsuites/common/visualizer/include/mv_util_visualizer_2d.h @@ -0,0 +1,38 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MEDIA_VISION_UTIL_VISUALIZER_2D_H__ +#define __MEDIA_VISION_UTIL_VISUALIZER_2D_H__ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#include +#include +#include + +#include "mv_util_matrix.h" +#include "mv_util_render_2d.h" +#include "mv_common.h" + +int mv_util_visualizer_2d(mv_source_h source, const char *url); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __MEDIA_VISION_UTIL_VISUALIZER_2D_H__*/ diff --git a/test/testsuites/common/visualizer/include/mv_util_visualizer_3d.h b/test/testsuites/common/visualizer/include/mv_util_visualizer_3d.h new file mode 100644 index 0000000..539fa2a --- /dev/null +++ b/test/testsuites/common/visualizer/include/mv_util_visualizer_3d.h @@ -0,0 +1,47 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MEDIA_VISION_UTIL_VISUALIZER_3D_H__ +#define __MEDIA_VISION_UTIL_VISUALIZER_3D_H__ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + + +#include +#include +#include + +#include "mv_util_matrix.h" +#include "mv_util_render_2d.h" +#include "mv_util_render_3d.h" + +#ifdef __cplusplus +#include +#include +#else +#include +#include +#endif + +int mv_util_visualizer_3d(mv_source_h source, float *depth, int xpos, int ypos); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __MEDIA_VISION_UTIL_VISUALIZER_3D_H__*/ diff --git a/test/testsuites/common/visualizer/include/mv_util_winsys.h b/test/testsuites/common/visualizer/include/mv_util_winsys.h new file mode 100644 index 0000000..42e4415 --- /dev/null +++ b/test/testsuites/common/visualizer/include/mv_util_winsys.h @@ -0,0 +1,57 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MEDIA_VISION_UTIL_WINSYS_H_ +#define __MEDIA_VISION_UTIL_WINSYS_H_ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#include + +struct Window; +struct Display { + struct wl_display *wlDisplay; + struct wl_registry *wlRegistry; + struct wl_compositor *wlCompositor; + struct wl_shell *wlShell; + struct wl_pointer *pointer; + uint32_t serial; + struct Window *window; +}; + +struct Geometry { + int width, height; +}; + +struct Window { + struct Display *display; + struct wl_egl_window *wlEGLNativeWindow; + struct wl_surface *wlSurface; + struct wl_shell_surface *wlShellSurface; + struct wl_callback *callback; + struct Geometry geometry,window_size; +}; + +void *winsys_init_native_display(void); +void *winsys_init_native_window(void *dpy, int win_w, int win_h); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __MEDIA_VISION_UTIL_WINSYS_H_ */ diff --git a/test/testsuites/common/visualizer/mv_util_visualizer_rd.proto b/test/testsuites/common/visualizer/mv_util_visualizer_rd.proto new file mode 100644 index 0000000..ada3f63 --- /dev/null +++ b/test/testsuites/common/visualizer/mv_util_visualizer_rd.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +message Empty { +} + +message NLImage { + int32 length = 1; + bytes data = 2; +} + +message NLImageDrawRequest { + NLImage image = 1; +} + +service NLImageService { + rpc DrawImage(NLImageDrawRequest) returns (Empty); +} + diff --git a/test/testsuites/common/visualizer/src/mv_util_matrix.cpp b/test/testsuites/common/visualizer/src/mv_util_matrix.cpp new file mode 100644 index 0000000..d9fd913 --- /dev/null +++ b/test/testsuites/common/visualizer/src/mv_util_matrix.cpp @@ -0,0 +1,841 @@ +/** + * Copyright(c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0(the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _USE_MATH_DEFINES +#include + +#include "mv_util_matrix.h" + +static float vec3_length(float *v) +{ + return (float) sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]); +} + +static void turn_x(float *m, float cosA, float sinA) +{ + float mx01, mx02; + float mx11, mx12; + float mx21, mx22; + float mx31, mx32; + + mx01 = cosA * m[4]; + mx02 = sinA * m[4]; + + mx11 = cosA * m[5]; + mx12 = sinA * m[5]; + + mx21 = cosA * m[6]; + mx22 = sinA * m[6]; + + mx31 = cosA * m[7]; + mx32 = sinA * m[7]; + + mx01 = sinA * m[8] + mx01; + mx02 = cosA * m[8] - mx02; + + mx11 = sinA * m[9] + mx11; + mx12 = cosA * m[9] - mx12; + + mx21 = sinA * m[10] + mx21; + mx22 = cosA * m[10] - mx22; + + mx31 = sinA * m[11] + mx31; + mx32 = cosA * m[11] - mx32; + + m[4] = mx01; + m[8] = mx02; + + m[5] = mx11; + m[9] = mx12; + + m[6] = mx21; + m[10] = mx22; + + m[7] = mx31; + m[11] = mx32; +} + +static void turn_y(float *m, float cosA, float sinA) +{ + float mx00, mx02; + float mx10, mx12; + float mx20, mx22; + float mx30, mx32; + + mx00 = cosA * m[0]; + mx02 = sinA * m[0]; + + mx10 = cosA * m[1]; + mx12 = sinA * m[1]; + + mx20 = cosA * m[2]; + mx22 = sinA * m[2]; + + mx30 = cosA * m[3]; + mx32 = sinA * m[3]; + + mx00 = -sinA * m[8] + mx00; + mx02 = cosA * m[8] + mx02; + + mx10 = -sinA * m[9] + mx10; + mx12 = cosA * m[9] + mx12; + + mx20 = -sinA * m[10] + mx20; + mx22 = cosA * m[10] + mx22; + + mx30 = -sinA * m[11] + mx30; + mx32 = cosA * m[11] + mx32; + + m[0] = mx00; + m[8] = mx02; + + m[1] = mx10; + m[9] = mx12; + + m[2] = mx20; + m[10] = mx22; + + m[3] = mx30; + m[11] = mx32; +} + +static void turn_z(float *m, float cosA, float sinA) +{ + float mx00, mx01; + float mx10, mx11; + float mx20, mx21; + float mx30, mx31; + + mx00 = cosA * m[0]; + mx01 = sinA * m[0]; + + mx10 = cosA * m[1]; + mx11 = sinA * m[1]; + + mx20 = cosA * m[2]; + mx21 = sinA * m[2]; + + mx30 = cosA * m[3]; + mx31 = sinA * m[3]; + + mx00 = sinA * m[4] + mx00; + mx01 = cosA * m[4] - mx01; + + mx10 = sinA * m[5] + mx10; + mx11 = cosA * m[5] - mx11; + + mx20 = sinA * m[6] + mx20; + mx21 = cosA * m[6] - mx21; + + mx30 = sinA * m[7] + mx30; + mx31 = cosA * m[7] - mx31; + + m[0] = mx00; + m[4] = mx01; + m[1] = mx10; + m[5] = mx11; + m[2] = mx20; + m[6] = mx21; + m[3] = mx30; + m[7] = mx31; +} + +int matrix_translate(float *m, float x, float y, float z) +{ + if (m == NULL) { + LOGE("m is NULL"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + m[12] += m[8] * z; + m[13] += m[9] * z; + m[14] += m[10] * z; + m[15] += m[11] * z; + + m[12] += m[4] * y; + m[13] += m[5] * y; + m[14] += m[6] * y; + m[15] += m[7] * y; + + m[12] += m[0] * x; + m[13] += m[1] * x; + m[14] += m[2] * x; + m[15] += m[3] * x; + + return MEDIA_VISION_ERROR_NONE; +} + +int matrix_rotate(float *m, float angle, float x, float y, float z) +{ + float v[3], angleRadian; + float sinA, cosA, cosA2; + float xcosA2, ycosA2, zcosA2; + float xsinA, ysinA, zsinA; + + if (m == NULL) { + LOGE("m is NULL"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + angleRadian = angle * (M_PI/180.0f); + sinA = (float)sin(angleRadian); + cosA = (float)cos(angleRadian); + + if (x == 0.0f && y == 0.0f && z != 0.0f) { + if (z < 0.0f) { + sinA = -sinA; + } + turn_z(m, cosA, sinA); + return MEDIA_VISION_ERROR_NONE; + } else if (x == 0.0f && y != 0.0f && z == 0.0f) { + if (y < 0.0f) { + sinA = -sinA; + } + turn_y(m, cosA, sinA); + return MEDIA_VISION_ERROR_NONE; + } else if (x != 0.0f && y == 0.0f && z == 0.0f) { + if (x < 0.0f) { + sinA = -sinA; + } + turn_x(m, cosA, sinA); + return MEDIA_VISION_ERROR_NONE; + } + + { + float r00, r01, r02; + float r10, r11, r12; + float r20, r21, r22; + + v[0] = x; + v[1] = y; + v[2] = z; + vec3_normalize(v); + + x = v[0]; + y = v[1]; + z = v[2]; + + cosA2 = 1.0f - cosA; + xsinA = x * sinA; + ysinA = y * sinA; + zsinA = z * sinA; + xcosA2 = x * cosA2; + ycosA2 = y * cosA2; + zcosA2 = z * cosA2; + + r00 = x * xcosA2 + cosA; + r10 = y * xcosA2 + zsinA; + r20 = z * xcosA2 - ysinA; + + r01 = x * ycosA2 - zsinA; + r11 = y * ycosA2 + cosA; + r21 = z * ycosA2 + xsinA; + + r02 = x * zcosA2 + ysinA; + r12 = y * zcosA2 - xsinA; + r22 = z * zcosA2 + cosA; + + { + float fm0, fm1, fm2; + float mx, my, mz; + + fm0 = m[0]; fm1 = m[4]; fm2 = m[8]; + + mx = fm0 * r00; + my = fm0 * r01; + mz = fm0 * r02; + + mx += fm1 * r10; + my += fm1 * r11; + mz += fm1 * r12; + + mx += fm2 * r20; + my += fm2 * r21; + mz += fm2 * r22; + + fm0 = m[1]; fm1 = m[5]; fm2 = m[9]; + + m[0] = mx; m[4] = my; m[8] = mz; + + mx = fm0 * r00; + my = fm0 * r01; + mz = fm0 * r02; + + mx += fm1 * r10; + my += fm1 * r11; + mz += fm1 * r12; + + mx += fm2 * r20; + my += fm2 * r21; + mz += fm2 * r22; + + fm0 = m[2]; fm1 = m[6]; fm2 = m[10]; + + m[1] = mx; m[5] = my; m[9] = mz; + + mx = fm0 * r00; + my = fm0 * r01; + mz = fm0 * r02; + + mx += fm1 * r10; + my += fm1 * r11; + mz += fm1 * r12; + + mx += fm2 * r20; + my += fm2 * r21; + mz += fm2 * r22; + + fm0 = m[3]; fm1 = m[7]; fm2 = m[11]; + + m[2] = mx; m[6] = my; m[10] = mz; + + mx = fm0 * r00; + my = fm0 * r01; + mz = fm0 * r02; + + mx += fm1 * r10; + my += fm1 * r11; + mz += fm1 * r12; + + mx += fm2 * r20; + my += fm2 * r21; + mz += fm2 * r22; + + m[3] = mx; m[7] = my; m[11] = mz; + } + } + + return MEDIA_VISION_ERROR_NONE; +} + +int matrix_scale(float *m, float x, float y, float z) +{ + float m00, m01, m02, m03; + float m04, m05, m06, m07; + float m08, m09, m10, m11; + + if (m == NULL) { + LOGE("m is NULL"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + m00 = m[0]; m04 = m[4]; m08 = m[8]; + m01 = m[1]; m05 = m[5]; m09 = m[9]; + m02 = m[2]; m06 = m[6]; m10 = m[10]; + m03 = m[3]; m07 = m[7]; m11 = m[11]; + + m00 = m00 * x; + m04 = m04 * y; + m08 = m08 * z; + + m01 = m01 * x; + m05 = m05 * y; + m09 = m09 * z; + + m02 = m02 * x; + m06 = m06 * y; + m10 = m10 * z; + + m03 = m03 * x; + m07 = m07 * y; + m11 = m11 * z; + + m[0] = m00; + m[4] = m04; + m[8] = m08; + + m[1] = m01; + m[5] = m05; + m[9] = m09; + + m[2] = m02; + m[6] = m06; + m[10] = m10; + + m[3] = m03; + m[7] = m07; + m[11] = m11; + + return MEDIA_VISION_ERROR_NONE; +} + +int matrix_mult(float *m, float *m1, float *m2) +{ + float fm0, fm1, fm2, fm3; + float fpm00, fpm01, fpm02, fpm03; + float fpm10, fpm11, fpm12, fpm13; + float fpm20, fpm21, fpm22, fpm23; + float fpm30, fpm31, fpm32, fpm33; + float x, y, z, w; + + if (m == NULL) { + LOGE("m is NULL"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + /* load pMb */ + fpm00 = m2[0]; + fpm01 = m2[4]; + fpm02 = m2[8]; + fpm03 = m2[12]; + + fpm10 = m2[1]; + fpm11 = m2[5]; + fpm12 = m2[9]; + fpm13 = m2[13]; + + fpm20 = m2[2]; + fpm21 = m2[6]; + fpm22 = m2[10]; + fpm23 = m2[14]; + + fpm30 = m2[3]; + fpm31 = m2[7]; + fpm32 = m2[11]; + fpm33 = m2[15]; + + fm0 = m1[0]; + fm1 = m1[4]; + fm2 = m1[8]; + fm3 = m1[12]; + + x = fm0 * fpm00; + y = fm0 * fpm01; + z = fm0 * fpm02; + w = fm0 * fpm03; + + x += fm1 * fpm10; + y += fm1 * fpm11; + z += fm1 * fpm12; + w += fm1 * fpm13; + + x += fm2 * fpm20; + y += fm2 * fpm21; + z += fm2 * fpm22; + w += fm2 * fpm23; + + x += fm3 * fpm30; + y += fm3 * fpm31; + z += fm3 * fpm32; + w += fm3 * fpm33; + + fm0 = m1[1]; + fm1 = m1[5]; + fm2 = m1[9]; + fm3 = m1[13]; + + m[0] = x; + m[4] = y; + m[8] = z; + m[12] = w; + + x = fm0 * fpm00; + y = fm0 * fpm01; + z = fm0 * fpm02; + w = fm0 * fpm03; + + x += fm1 * fpm10; + y += fm1 * fpm11; + z += fm1 * fpm12; + w += fm1 * fpm13; + + x += fm2 * fpm20; + y += fm2 * fpm21; + z += fm2 * fpm22; + w += fm2 * fpm23; + + x += fm3 * fpm30; + y += fm3 * fpm31; + z += fm3 * fpm32; + w += fm3 * fpm33; + + fm0 = m1[2]; + fm1 = m1[6]; + fm2 = m1[10]; + fm3 = m1[14]; + + m[1] = x; + m[5] = y; + m[9] = z; + m[13] = w; + + x = fm0 * fpm00; + y = fm0 * fpm01; + z = fm0 * fpm02; + w = fm0 * fpm03; + + x += fm1 * fpm10; + y += fm1 * fpm11; + z += fm1 * fpm12; + w += fm1 * fpm13; + + x += fm2 * fpm20; + y += fm2 * fpm21; + z += fm2 * fpm22; + w += fm2 * fpm23; + + x += fm3 * fpm30; + y += fm3 * fpm31; + z += fm3 * fpm32; + w += fm3 * fpm33; + + fm0 = m1[3]; + fm1 = m1[7]; + fm2 = m1[11]; + fm3 = m1[15]; + + m[2] = x; + m[6] = y; + m[10] = z; + m[14] = w; + + x = fm0 * fpm00; + y = fm0 * fpm01; + z = fm0 * fpm02; + w = fm0 * fpm03; + + x += fm1 * fpm10; + y += fm1 * fpm11; + z += fm1 * fpm12; + w += fm1 * fpm13; + + x += fm2 * fpm20; + y += fm2 * fpm21; + z += fm2 * fpm22; + w += fm2 * fpm23; + + x += fm3 * fpm30; + y += fm3 * fpm31; + z += fm3 * fpm32; + w += fm3 * fpm33; + + m[3] = x; + m[7] = y; + m[11] = z; + m[15] = w; + + return MEDIA_VISION_ERROR_NONE; +} + +int matrix_identity(float *m) +{ + if (m == NULL) { + LOGE("m is NULL"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + m[0] = 1.0f; m[4] = 0.0f; m[8] = 0.0f; m[12] = 0.0f; + m[1] = 0.0f; m[5] = 1.0f; m[9] = 0.0f; m[13] = 0.0f; + m[2] = 0.0f; m[6] = 0.0f; m[10] = 1.0f; m[14] = 0.0f; + m[3] = 0.0f; m[7] = 0.0f; m[11] = 0.0f; m[15] = 1.0f; + + return MEDIA_VISION_ERROR_NONE; +} + +void matrix_copy(float *d, float *s) +{ + memcpy(d, s, sizeof(float) * 16); +} + +void matrix_transpose(float *m) +{ + float m01, m02, m03; + float m04, m06, m07; + float m08, m09, m11; + float m12, m13, m14; + + ; m04 = m[ 4]; m08 = m[ 8]; m12 = m[12]; + m01 = m[ 1]; m09 = m[ 9]; m13 = m[13]; + m02 = m[ 2]; m06 = m[ 6]; m14 = m[14]; + m03 = m[ 3]; m07 = m[ 7]; m11 = m[11]; + + /*m[ 0] = m00;*/ + m[ 1] = m04; + m[ 2] = m08; + m[ 3] = m12; + m[ 4] = m01; + /*m[ 5] = m05;*/ + m[ 6] = m09; + m[ 7] = m13; + m[ 8] = m02; + m[ 9] = m06; + /*m[10] = m10;*/ + m[11] = m14; + m[12] = m03; + m[13] = m07; + m[14] = m11; + /*m[15] = m15;*/ +} + +void matrix_invert(float *m) +{ + float m00, m01, m02, m03; + float m04, m05, m06, m07; + float m08, m09, m10, m11; + float m12, m13, m14, m15; + float W00, W04, W08, W12; + float W01, W05, W09, W13; + float W02, W06, W10, W14; + float W03, W07, W11, W15; + float det, invdet; + + m00 = m[ 0]; m04 = m[ 4]; m08 = m[ 8]; m12 = m[12]; + m01 = m[ 1]; m05 = m[ 5]; m09 = m[ 9]; m13 = m[13]; + m02 = m[ 2]; m06 = m[ 6]; m10 = m[10]; m14 = m[14]; + m03 = m[ 3]; m07 = m[ 7]; m11 = m[11]; m15 = m[15]; + + if (m03 == 0.0f && m07 == 0.0f && m11 == 0.0f && m15 == 1.0f) { + W00 = m05 * m10 - m09 * m06; + W04 = -(m01 * m10 - m09 * m02); + W08 = m01 * m06 - m05 * m02; + + det = m00 * W00 + m04 * W04 + m08 * W08; + + if ( det == 0.0f ) + { + return; + } + invdet = 1.0f / det; + + W01 = -(m04 * m10 - m08 * m06); + W05 = m00 * m10 - m08 * m02; + W09 = -(m00 * m06 - m04 * m02); + + W02 = m04 * m09 - m08 * m05; + W06 = -(m00 * m09 - m08 * m01); + W10 = m00 * m05 - m04 * m01; + + W03 = -(W00 * m12 + W01 * m13 + W02 * m14); + W07 = -(W04 * m12 + W05 * m13 + W06 * m14); + W11 = -(W08 * m12 + W09 * m13 + W10 * m14); + + /* M^-1[ij] = invdet * M[ji] */ + m[ 0] = W00 * invdet; + m[ 4] = W01 * invdet; + m[ 8] = W02 * invdet; + m[12] = W03 * invdet; + + m[ 1] = W04 * invdet; + m[ 5] = W05 * invdet; + m[ 9] = W06 * invdet; + m[13] = W07 * invdet; + + m[ 2] = W08 * invdet; + m[ 6] = W09 * invdet; + m[10] = W10 * invdet; + m[14] = W11 * invdet; + + m[ 3] = 0.0f; + m[ 7] = 0.0f; + m[11] = 0.0f; + m[15] = 1.0f; + } else { + W00 = (m05 * (m10 * m15 - m14 * m11)) + + (m09 * (m14 * m07 - m06 * m15)) + + (m13 * (m06 * m11 - m10 * m07)); + W01 = (m09 * (m14 * m03 - m02 * m15)) + + (m13 * (m02 * m11 - m10 * m03)) + + (m01 * (m10 * m15 - m14 * m11)); + W02 = (m13 * (m02 * m07 - m06 * m03)) + + (m01 * (m06 * m15 - m14 * m07)) + + (m05 * (m14 * m03 - m02 * m15)); + W03 = (m01 * (m06 * m11 - m10 * m07)) + + (m05 * (m10 * m03 - m02 * m11)) + + (m09 * (m02 * m07 - m06 * m03)); + W04 = (m06 * (m11 * m12 - m15 * m08)) + + (m10 * (m15 * m04 - m07 * m12)) + + (m14 * (m07 * m08 - m11 * m04)); + W05 = (m10 * (m15 * m00 - m03 * m12)) + + (m14 * (m03 * m08 - m11 * m00)) + + (m02 * (m11 * m12 - m15 * m08)); + W06 = (m14 * (m03 * m04 - m07 * m00)) + + (m02 * (m07 * m12 - m15 * m04)) + + (m06 * (m15 * m00 - m03 * m12)); + W07 = (m02 * (m07 * m08 - m11 * m04)) + + (m06 * (m11 * m00 - m03 * m08)) + + (m10 * (m03 * m04 - m07 * m00)); + W08 = (m07 * (m08 * m13 - m12 * m09)) + + (m11 * (m12 * m05 - m04 * m13)) + + (m15 * (m04 * m09 - m08 * m05)); + W09 = (m11 * (m12 * m01 - m00 * m13)) + + (m15 * (m00 * m09 - m08 * m01)) + + (m03 * (m08 * m13 - m12 * m09)); + W10 = (m15 * (m00 * m05 - m04 * m01)) + + (m03 * (m04 * m13 - m12 * m05)) + + (m07 * (m12 * m01 - m00 * m13)); + W11 = (m03 * (m04 * m09 - m08 * m05)) + + (m07 * (m08 * m01 - m00 * m09)) + + (m11 * (m00 * m05 - m04 * m01)); + W12 = (m04 * (m09 * m14 - m13 * m10)) + + (m08 * (m13 * m06 - m05 * m14)) + + (m12 * (m05 * m10 - m09 * m06)); + W13 = (m08 * (m13 * m02 - m01 * m14)) + + (m12 * (m01 * m10 - m09 * m02)) + + (m00 * (m09 * m14 - m13 * m10)); + W14 = (m12 * (m01 * m06 - m05 * m02)) + + (m00 * (m05 * m14 - m13 * m06)) + + (m04 * (m13 * m02 - m01 * m14)); + W15 = (m00 * (m05 * m10 - m09 * m06)) + + (m04 * (m09 * m02 - m01 * m10)) + + (m08 * (m01 * m06 - m05 * m02)); + + det = W00 * m00 - W01 * m04 + W02 * m08 - W03 * m12; + + if (det == 0.0f) { + return; + } + invdet = 1.0f / det; + + m[ 0] = W00 * invdet; + m[ 1] = -W01 * invdet; + m[ 2] = W02 * invdet; + m[ 3] = -W03 * invdet; + m[ 4] = -W04 * invdet; + m[ 5] = W05 * invdet; + m[ 6] = -W06 * invdet; + m[ 7] = W07 * invdet; + m[ 8] = W08 * invdet; + m[ 9] = -W09 * invdet; + m[10] = W10 * invdet; + m[11] = -W11 * invdet; + m[12] = -W12 * invdet; + m[13] = W13 * invdet; + m[14] = -W14 * invdet; + m[15] = W15 * invdet; + } +} + +void matrix_proj_perspective (float *mat, float fovy, float aspect, float znear, float zfar) +{ + float a,b,c,d,f; + + memset( mat, 0x0, sizeof(float) * 16 ); + + f = (float)(cos(2.0f*3.14f*(fovy/2.0f)/360.0f)/sin(2.0f*3.14f*(fovy/2.0f)/360.0f)); + a = f/aspect; /* a = f/aspect */ + b = f; /* b = f */ + c = (zfar+znear)/(znear-zfar); /* c = (far+near)/(near-far) */ + d = (2.0f*zfar*znear)/(znear-zfar); /* d = (2*far*near)/(near-far) */ + + mat[ 0] = a; + mat[ 5] = b; + mat[10] = c; + mat[11] = -1.0f; + mat[14] = d; +} + + +/* lpR = lpP * lpQ */ +void quaternion_mult (float *lpR, float *lpP, float *lpQ) +{ + float pw, px, py, pz; + float qw, qx, qy, qz; + + pw = lpP[0]; px = lpP[1]; py = lpP[2]; pz = lpP[3]; + qw = lpQ[0]; qx = lpQ[1]; qy = lpQ[2]; qz = lpQ[3]; + + lpR[0] = pw * qw - px * qx - py * qy - pz * qz; + lpR[1] = pw * qx + px * qw + py * qz - pz * qy; + lpR[2] = pw * qy - px * qz + py * qw + pz * qx; + lpR[3] = pw * qz + px * qy - py * qx + pz * qw; +} + + +void quaternion_to_matrix (float *lpM, float *lpQ) +{ + float qw, qx, qy, qz; + float x2, y2, z2; + float xy, yz, zx; + float wx, wy, wz; + + qw = lpQ[0]; qx = lpQ[1]; qy = lpQ[2]; qz = lpQ[3]; + + x2 = 2.0f * qx * qx; + y2 = 2.0f * qy * qy; + z2 = 2.0f * qz * qz; + + xy = 2.0f * qx * qy; + yz = 2.0f * qy * qz; + zx = 2.0f * qz * qx; + + wx = 2.0f * qw * qx; + wy = 2.0f * qw * qy; + wz = 2.0f * qw * qz; + + lpM[ 0] = 1.0f - y2 - z2; + lpM[ 4] = xy - wz; + lpM[ 8] = zx + wy; + lpM[12] = 0.0f; + + lpM[ 1] = xy + wz; + lpM[ 5] = 1.0f - z2 - x2; + lpM[ 9] = yz - wx; + lpM[13] = 0.0f; + + lpM[ 2] = zx - wy; + lpM[ 6] = yz + wx; + lpM[10] = 1.0f - x2 - y2; + lpM[14] = 0.0f; + + lpM[ 3] = lpM[ 7] = lpM[11] = 0.0f; + lpM[15] = 1.0f; + } + + +void quaternion_rotate (float *lpQ, float rad, float ax, float ay, float az) +{ + float hrad = 0.5f * rad; + float s = sinf(hrad); + + lpQ[0] = cosf(hrad); /* w */ + lpQ[1] = s * ax; /* x */ + lpQ[2] = s * ay; /* y */ + lpQ[3] = s * az; /* z */ +} + +void quaternion_identity (float *lpQ) +{ + lpQ[0] = 1.0f; /* w */ + lpQ[1] = 0.0f; /* x */ + lpQ[2] = 0.0f; /* y */ + lpQ[3] = 0.0f; /* z */ +} + +void quaternion_copy (float *lpTo, float *lpFrom) +{ + lpTo[0] = lpFrom[0]; /* w */ + lpTo[1] = lpFrom[1]; /* x */ + lpTo[2] = lpFrom[2]; /* y */ + lpTo[3] = lpFrom[3]; /* z */ +} + +float vec3_normalize(float *v) +{ + float len, invLen; + + len = vec3_length(v); + if (len == 0.0f) + return 0.0f; + + invLen = 1.0f / len; + + v[0] *= invLen; + v[1] *= invLen; + v[2] *= invLen; + + return len; +} + diff --git a/test/testsuites/common/visualizer/src/mv_util_render_2d.cpp b/test/testsuites/common/visualizer/src/mv_util_render_2d.cpp new file mode 100644 index 0000000..4559bd6 --- /dev/null +++ b/test/testsuites/common/visualizer/src/mv_util_render_2d.cpp @@ -0,0 +1,618 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mv_util_render_2d.h" + +static char vs_fill[] = " \n\ + \n\ +attribute vec4 a_Vertex; \n\ +uniform mat4 u_PMVMatrix; \n\ +void main(void) \n\ +{ \n\ + gl_Position = u_PMVMatrix * a_Vertex; \n\ +} "; + +static char fs_fill[] = " \n\ + \n\ +precision mediump float; \n\ +uniform vec4 u_Color; \n\ + \n\ +void main(void) \n\ +{ \n\ + gl_FragColor = u_Color; \n\ +} "; + +static char vs_tex[] = " \n\ +attribute vec4 a_Vertex; \n\ +attribute vec2 a_TexCoord; \n\ +varying vec2 v_TexCoord; \n\ +uniform mat4 u_PMVMatrix; \n\ + \n\ +void main(void) \n\ +{ \n\ + gl_Position = u_PMVMatrix * a_Vertex; \n\ + v_TexCoord = a_TexCoord; \n\ +} \n"; + +static char fs_tex[] = " \n\ +precision mediump float; \n\ +varying vec2 v_TexCoord; \n\ +uniform sampler2D u_sampler; \n\ +uniform vec4 u_Color; \n\ + \n\ +void main(void) \n\ +{ \n\ + gl_FragColor = texture2D(u_sampler, v_TexCoord); \n\ + gl_FragColor *= u_Color; \n\ +} \n"; + +static char fs_extex[] = " \n\ +#extension GL_NV_EGL_stream_consumer_external: enable \n\ +#extension GL_OES_EGL_image_external : enable \n\ +precision mediump float; \n\ +varying vec2 v_TexCoord; \n\ +uniform samplerExternalOES u_sampler; \n\ +uniform vec4 u_Color; \n\ + \n\ +void main(void) \n\ +{ \n\ + gl_FragColor = texture2D(u_sampler, v_TexCoord); \n\ + gl_FragColor *= u_Color; \n\ +} \n"; + +static char fs_cmap_jet[] =" \n\ +precision mediump float; \n\ +varying vec2 v_TexCoord; \n\ +uniform sampler2D u_sampler; \n\ +uniform vec4 u_Color; \n\ + \n\ +float cmap_jet_red(float x) \n\ +{ \n\ + if (x < 0.7) { \n\ + return 4.0 * x - 1.5; \n\ + } else { \n\ + return -4.0 * x + 4.5; \n\ + } \n\ +} \n\ + \n\ +float cmap_jet_green(float x) \n\ +{ \n\ + if (x < 0.5) { \n\ + return 4.0 * x - 0.5; \n\ + } else { \n\ + return -4.0 * x + 3.5; \n\ + } \n\ +} \n\ + \n\ +float cmap_jet_blue(float x) \n\ +{ \n\ + if (x < 0.3) { \n\ + return 4.0 * x + 0.5; \n\ + } else { \n\ + return -4.0 * x + 2.5; \n\ + } \n\ +} \n\ + \n\ +vec4 colormap_jet(float x) \n\ +{ \n\ + float r = clamp(cmap_jet_red(x), 0.0, 1.0); \n\ + float g = clamp(cmap_jet_green(x), 0.0, 1.0); \n\ + float b = clamp(cmap_jet_blue(x), 0.0, 1.0); \n\ + return vec4(r, g, b, 1.0); \n\ +} \n\ + \n\ +void main(void) \n\ +{ \n\ + vec4 src_col = texture2D(u_sampler, v_TexCoord); \n\ + gl_FragColor = colormap_jet(src_col.r); \n\ + gl_FragColor *= u_Color; \n\ +} \n"; + +static char vs_tex_yuyv[] = " \n\ +attribute vec4 a_Vertex; \n\ +attribute vec2 a_TexCoord; \n\ +varying vec2 v_TexCoord; \n\ +varying vec2 v_TexCoordPix; \n\ +uniform mat4 u_PMVMatrix; \n\ +uniform vec2 u_TexDim; \n\ + \n\ +void main(void) \n\ +{ \n\ + gl_Position = u_PMVMatrix * a_Vertex; \n\ + v_TexCoord = a_TexCoord; \n\ + v_TexCoordPix = a_TexCoord * u_TexDim; \n\ +} \n"; + +static char fs_tex_yuyv[] = " \n\ +precision mediump float; \n\ +varying vec2 v_TexCoord; \n\ +varying vec2 v_TexCoordPix; \n\ +uniform sampler2D u_sampler; \n\ +uniform vec4 u_Color; \n\ + \n\ +void main(void) \n\ +{ \n\ + vec2 evenodd = mod(v_TexCoordPix, 2.0); \n\ + vec3 yuv, rgb; \n\ + vec4 texcol = texture2D(u_sampler, v_TexCoord); \n\ + if (evenodd.x < 1.0) { \n\ + yuv.r = texcol.r; /* Y */ \n\ + yuv.g = texcol.g - 0.5; /* U */ \n\ + yuv.b = texcol.a - 0.5; /* V */ \n\ + } else { \n\ + yuv.r = texcol.b; /* Y */ \n\ + yuv.g = texcol.g - 0.5; /* U */ \n\ + yuv.b = texcol.a - 0.5; /* V */ \n\ + } \n\ + \n\ + rgb = mat3 ( 1, 1, 1, \n\ + 0, -0.34413, 1.772, \n\ + 1.402, -0.71414, 0) * yuv; \n\ + gl_FragColor = vec4(rgb, 1.0); \n\ + gl_FragColor *= u_Color; \n\ +} \n"; + +#define SHADER_NUM 5 +static char *s_shader[SHADER_NUM * 2] = { + vs_fill, fs_fill, + vs_tex, fs_tex, + vs_tex, fs_extex, + vs_tex, fs_cmap_jet, + vs_tex_yuyv, fs_tex_yuyv +}; + +static shader_obj_t s_sobj[SHADER_NUM]; +static int s_loc_mtx[SHADER_NUM]; +static int s_loc_color[SHADER_NUM]; +static int s_loc_texdim[SHADER_NUM]; + +static EGLDisplay s_dpy; +static EGLSurface s_sfc; +static EGLContext s_ctx; + +GLuint create_2d_texture(void *imgbuf, int width, int height) +{ + GLuint texid; + + glGenTextures(1, &texid ); + glBindTexture(GL_TEXTURE_2D, texid); + + glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + + glPixelStorei(GL_UNPACK_ALIGNMENT, 4); + + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, + width, height, 0, GL_RGB, + GL_UNSIGNED_BYTE, imgbuf); + + return texid; +} + +static EGLConfig find_egl_config(int r, int g, int b, int a, int d, + int s, int ms, int sfc_type, int ver) +{ + EGLint num_conf, i; + EGLBoolean ret; + EGLConfig conf = 0, *conf_array = NULL; + + EGLint config_attribs[] = { + EGL_RED_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_BLUE_SIZE, 8, + EGL_ALPHA_SIZE, 8, + EGL_DEPTH_SIZE, EGL_DONT_CARE, + EGL_STENCIL_SIZE, EGL_DONT_CARE, + EGL_SAMPLES, EGL_DONT_CARE, + EGL_SURFACE_TYPE, EGL_WINDOW_BIT, + EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL_NONE + }; + + config_attribs[1] = r; + config_attribs[3] = g; + config_attribs[5] = b; + config_attribs[7] = a; + config_attribs[9] = d; + config_attribs[11] = s; + config_attribs[13] = ms; + config_attribs[15] = sfc_type; + + switch (ver) { + case 1: + case 2: config_attribs[17] = EGL_OPENGL_ES2_BIT; + break; + default: + LOGE("Invalid version"); + goto exit; + } + + ret = eglChooseConfig(s_dpy, config_attribs, NULL, 0, &num_conf); + if (ret != EGL_TRUE || num_conf == 0) { + LOGE("Failed to call eglChooseConfig"); + goto exit; + } + + conf_array = (EGLConfig *)calloc(num_conf, sizeof(EGLConfig)); + if (conf_array == NULL) { + LOGE("EGLConfig is NULL"); + goto exit; + } + + ret = eglChooseConfig(s_dpy, config_attribs, conf_array, num_conf, &num_conf); + if (ret != EGL_TRUE) { + LOGE("Failed to call eglChooseConfig"); + goto exit; + } + + for (i = 0; i < num_conf; i ++) { + EGLint id, rsize, gsize, bsize, asize; + + eglGetConfigAttrib(s_dpy, conf_array[i], EGL_CONFIG_ID, &id); + eglGetConfigAttrib(s_dpy, conf_array[i], EGL_RED_SIZE, &rsize); + eglGetConfigAttrib(s_dpy, conf_array[i], EGL_GREEN_SIZE, &gsize); + eglGetConfigAttrib(s_dpy, conf_array[i], EGL_BLUE_SIZE, &bsize); + eglGetConfigAttrib(s_dpy, conf_array[i], EGL_ALPHA_SIZE, &asize); + + if (rsize == r && gsize == g && bsize == b && asize == a) { + conf = conf_array[i]; + break; + } + } + + if (i == num_conf) { + LOGE("Index is out of range"); + goto exit; + } + +exit: + if (conf_array) + free(conf_array); + + return conf; +} + +int egl_init_with_platform_window_surface(int gles_version, int depth_size, int stencil_size, int sample_num, + int win_w, int win_h) +{ + void *native_dpy, *native_win; + EGLint major, minor; + EGLConfig config; + EGLBoolean ret; + EGLint context_attribs[] = { + EGL_CONTEXT_CLIENT_VERSION, 2, + EGL_NONE + }; + EGLint sfc_attr[] = { EGL_NONE }; + + native_dpy = winsys_init_native_display(); + if ((native_dpy != EGL_DEFAULT_DISPLAY) &&(native_dpy == NULL)) { + LOGE("native_dpy is not valid"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + s_dpy = eglGetDisplay(native_dpy); + if (s_dpy == EGL_NO_DISPLAY) { + LOGE("s_dpy is EGL_NO_DISPLAY"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + ret = eglInitialize(s_dpy, &major, &minor); + if (ret != EGL_TRUE) { + LOGE("Failed to initialize egl"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + eglBindAPI(EGL_OPENGL_ES_API); + + config = find_egl_config(8, 8, 8, 8, depth_size, stencil_size, sample_num, EGL_WINDOW_BIT, gles_version); + if (config == NULL) { + LOGE("Failed to find egl configuration"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + native_win = winsys_init_native_window(s_dpy, win_w, win_h); + if (native_win == NULL) { + LOGE("window is NULL"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + s_sfc = eglCreateWindowSurface(s_dpy, config, (NativeWindowType)native_win, sfc_attr); + if (s_sfc== EGL_NO_SURFACE) { + LOGE("s_sfc is EGL_NO_SURFACE"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + switch (gles_version) { + case 1: context_attribs[1] = 1; + break; + case 2: context_attribs[1] = 2; + break; + case 3: context_attribs[1] = 3; + break; + default: + LOGE("Invalid gles version"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + s_ctx = eglCreateContext(s_dpy, config, EGL_NO_CONTEXT, context_attribs); + if (s_ctx == EGL_NO_CONTEXT) { + LOGE("s_ctx is EGL_NO_CONTEXT"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + ret = eglMakeCurrent(s_dpy, s_sfc, s_sfc, s_ctx); + if (ret != EGL_TRUE) { + LOGE("Falied to call eglMakeCurrent()"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + return MEDIA_VISION_ERROR_NONE; +} + +int egl_swap() +{ + EGLBoolean ret; + + ret = eglSwapBuffers(s_dpy, s_sfc); + if (ret != EGL_TRUE) { + LOGE("Failed to call eglSwapBuffers()"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + return MEDIA_VISION_ERROR_NONE; +} + +static float varray[] = { + 0.0, 0.0, + 0.0, 1.0, + 1.0, 0.0, + 1.0, 1.0 +}; + +static float tarray[] = { + 0.0, 0.0, + 0.0, 1.0, + 1.0, 0.0, + 1.0, 1.0 +}; + +static float tarray2[] = { + 0.0, 1.0, + 0.0, 0.0, + 1.0, 1.0, + 1.0, 0.0 +}; + +static float s_matprj[16]; +static void set_projection_matrix(int w, int h) +{ + float mat_proj[] = { + 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, + -1.0f, 1.0f, 0.0f, 1.0f}; + + mat_proj[0] = 2.0f / (float)w; + mat_proj[5] = -2.0f / (float)h; + + memcpy(s_matprj, mat_proj, 16*sizeof(float)); +} + +int init_2d_renderer(int w, int h) +{ + int i; + + for (i = 0; i < SHADER_NUM; i ++) { + if (generate_shader(&s_sobj[i], s_shader[2*i], s_shader[2*i + 1]) < 0) { + LOGE("%s(%d)", __FILE__, __LINE__); + return MEDIA_VISION_ERROR_INTERNAL; + } + + s_loc_mtx[i] = glGetUniformLocation(s_sobj[i].program, "u_PMVMatrix"); + s_loc_color[i] = glGetUniformLocation(s_sobj[i].program, "u_Color"); + s_loc_texdim[i] = glGetUniformLocation(s_sobj[i].program, "u_TexDim"); + } + + set_projection_matrix(w, h); + + return MEDIA_VISION_ERROR_NONE; +} + +typedef struct _texparam { + int textype; + int texid; + int x, y, w, h; + int texw, texh; + int upsidedown; + float color[4]; + float rot; + float px, py; + int blendfunc_en; + unsigned int blendfunc[4]; + float *user_texcoord; +} texparam_t; + +static void draw_2d_texture_in(texparam_t *tparam) +{ + int ttype = tparam->textype; + int texid = tparam->texid; + float x = tparam->x; + float y = tparam->y; + float w = tparam->w; + float h = tparam->h; + float rot = tparam->rot; + shader_obj_t *sobj = &s_sobj[ttype]; + float matrix[16]; + float *uv = tarray; + + glBindBuffer(GL_ARRAY_BUFFER, 0); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); + + glUseProgram(sobj->program); + glUniform1i(sobj->loc_tex, 0); + + switch (ttype) { + case 0: /* fill */ + break; + case 1: /* tex */ + case 4: /* tex_yuyv */ + glBindTexture(GL_TEXTURE_2D, texid); + uv = tparam->upsidedown ? tarray2 : tarray; + break; + case 2: /* tex_extex */ + glBindTexture(GL_TEXTURE_EXTERNAL_OES, texid); + uv = tparam->upsidedown ? tarray : tarray2; + break; + default: + break; + } + + if (tparam->user_texcoord) + uv = tparam->user_texcoord; + + if (sobj->loc_uv >= 0) { + glEnableVertexAttribArray(sobj->loc_uv); + glVertexAttribPointer(sobj->loc_uv, 2, GL_FLOAT, GL_FALSE, 0, uv); + } + + glEnable(GL_BLEND); + + if (tparam->blendfunc_en) { + glBlendFuncSeparate(tparam->blendfunc[0], tparam->blendfunc[1], + tparam->blendfunc[2], tparam->blendfunc[3]); + } else { + glBlendFuncSeparate(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA, + GL_ONE, GL_ONE_MINUS_SRC_ALPHA); + } + + if (matrix_identity(matrix) != MEDIA_VISION_ERROR_NONE) + LOGE("Failed to call matrix_identity"); + if (matrix_translate(matrix, x, y, 0.0f) != MEDIA_VISION_ERROR_NONE) + LOGE("Failed to call matrix_translate"); + + if (rot != 0) { + float px = tparam->px; + float py = tparam->py; + if (matrix_translate(matrix, px, py, 0.0f) != MEDIA_VISION_ERROR_NONE) + LOGE("Failed to call matrix_translate"); + + if (matrix_rotate(matrix, rot, 0.0f, 0.0f, 1.0f) != MEDIA_VISION_ERROR_NONE) + LOGE("Failed to call matrix_rotate"); + if (matrix_translate(matrix, -px, -py, 0.0f) != MEDIA_VISION_ERROR_NONE) + LOGE("Failed to call matrix_translate"); + } + if (matrix_scale(matrix, w, h, 1.0f) != MEDIA_VISION_ERROR_NONE) + LOGE("Failed to call matrix_scale"); + if (matrix_mult(matrix, s_matprj, matrix) != MEDIA_VISION_ERROR_NONE) + LOGE("Failed to call matrix_mult"); + + glUniformMatrix4fv(s_loc_mtx[ttype], 1, GL_FALSE, matrix); + glUniform4fv(s_loc_color[ttype], 1, tparam->color); + + if (s_loc_texdim[ttype] >= 0) { + float texdim[2]; + texdim[0] = tparam->texw; + texdim[1] = tparam->texh; + glUniform2fv(s_loc_texdim[ttype], 1, texdim); + } + + if (sobj->loc_vtx >= 0) { + glEnableVertexAttribArray(sobj->loc_vtx); + glVertexAttribPointer(sobj->loc_vtx, 2, GL_FLOAT, GL_FALSE, 0, varray); + } + + glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + + glDisable(GL_BLEND); +} + +int load_texture(mv_source_h source, int *lpTexID, int *lpWidth, int *lpHeight) +{ + GLuint texid; + unsigned char *data_buffer = NULL; + unsigned int width, height; + unsigned int buffer_size = 0; + int err = mv_source_get_buffer(source, &data_buffer, &buffer_size); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE( + "ERROR: Errors were occurred during getting buffer from the " + "source; code %i\n", + err); + return err; + } + + err = mv_source_get_width(source, &width); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE( + "ERROR: Errors were occurred during getting width from the " + "source; code %i\n", + err); + return err; + } + + err = mv_source_get_height(source, &height); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE( + "ERROR: Errors were occurred during getting height from the " + "source; code %i\n", + err); + return err; + } + texid = create_2d_texture(data_buffer, width, height); + + if (lpTexID) + *lpTexID = texid; + if (lpWidth) + *lpWidth = width; + if (lpHeight) + *lpHeight = height; + + return MEDIA_VISION_ERROR_NONE; +} + +int draw_2d_texture(texture_2d_t *tex, int x, int y, int w, int h, int upsidedown) +{ + texparam_t tparam = {0}; + + if (tex == NULL) { + LOGE("tex is NULL"); + return MEDIA_VISION_ERROR_INTERNAL; + } + + tparam.x = x; + tparam.y = y; + tparam.w = w; + tparam.h = h; + tparam.texid = tex->texid; + tparam.textype = 1; + tparam.texw = tex->width; + tparam.texh = tex->height; + tparam.color[0]= 1.0f; + tparam.color[1]= 1.0f; + tparam.color[2]= 1.0f; + tparam.color[3]= 1.0f; + tparam.upsidedown = upsidedown; + + if (tex->format == pixfmt_fourcc('Y', 'U', 'Y', 'V')) + tparam.textype = 4; + + draw_2d_texture_in(&tparam); + + return MEDIA_VISION_ERROR_NONE; +} diff --git a/test/testsuites/common/visualizer/src/mv_util_render_3d.cpp b/test/testsuites/common/visualizer/src/mv_util_render_3d.cpp new file mode 100644 index 0000000..7ecc43c --- /dev/null +++ b/test/testsuites/common/visualizer/src/mv_util_render_3d.cpp @@ -0,0 +1,277 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "mv_util_render_2d.h" +#include "mv_util_render_3d.h" + +static int s_texid_dummy = 0; + +static shader_obj_t s_sobj; +static float s_matPrj[16]; +static GLint s_loc_mtx_mv; +static GLint s_loc_mtx_pmv; +static GLint s_loc_mtx_nrm; +static GLint s_loc_color; +static GLint s_loc_alpha; +static GLint s_loc_lightpos; + +static GLfloat s_nrm[] = { + 0.0f, 0.0f, 1.0f, + 0.0f, 0.0f, -1.0f, + 1.0f, 0.0f, 0.0f, + -1.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, + 0.0f, -1.0f, 0.0f, +}; + + +static GLfloat s_uv[] = { + 0.0f, 0.0f, + 0.0f, 1.0f, + 1.0f, 0.0f, + 1.0f, 1.0f, +}; + +static char s_strVS[] = " \n\ + \n\ +attribute vec4 a_Vertex; \n\ +attribute vec3 a_Normal; \n\ +attribute vec2 a_TexCoord; \n\ +uniform mat4 u_PMVMatrix; \n\ +uniform mat4 u_MVMatrix; \n\ +uniform mat3 u_ModelViewIT; \n\ +varying vec3 v_diffuse; \n\ +varying vec3 v_specular; \n\ +varying vec2 v_texcoord; \n\ +const float shiness = 16.0; \n\ +uniform vec3 u_LightPos; \n\ +const vec3 LightCol = vec3(1.0, 1.0, 1.0); \n\ + \n\ +void DirectionalLight(vec3 normal, vec3 eyePos) \n\ +{ \n\ + vec3 lightDir = normalize(u_LightPos); \n\ + vec3 halfV = normalize(u_LightPos - eyePos); \n\ + float dVP = max(dot(normal, lightDir), 0.0); \n\ + float dHV = max(dot(normal, halfV), 0.0); \n\ + \n\ + float pf = 0.0; \n\ + if(dVP > 0.0) \n\ + pf = pow(dHV, shiness); \n\ + \n\ + v_diffuse += dVP * LightCol; \n\ + v_specular+= pf * LightCol * 0.5; \n\ +} \n\ + \n\ +void main(void) \n\ +{ \n\ + gl_Position = u_PMVMatrix * a_Vertex; \n\ + vec3 normal = normalize(u_ModelViewIT * a_Normal); \n\ + vec3 eyePos = vec3(u_MVMatrix * a_Vertex); \n\ + \n\ + v_diffuse = vec3(0.5); \n\ + v_specular = vec3(0.0); \n\ + DirectionalLight(normal, eyePos); \n\ + \n\ + v_diffuse = clamp(v_diffuse, 0.0, 1.0); \n\ + v_texcoord = a_TexCoord; \n\ +} "; + +static char s_strFS[] = " \n\ +precision mediump float; \n\ + \n\ +uniform vec3 u_color; \n\ +uniform float u_alpha; \n\ +varying vec3 v_diffuse; \n\ +varying vec3 v_specular; \n\ +varying vec2 v_texcoord; \n\ +uniform sampler2D u_sampler; \n\ + \n\ +void main(void) \n\ +{ \n\ + vec3 color; \n\ + color = vec3(texture2D(u_sampler, v_texcoord)); \n\ + color *= (u_color * v_diffuse); \n\ + gl_FragColor = vec4(color, u_alpha); \n\ +} "; + +static void compute_invmat3x3(float *matMVI3x3, float *matMV) +{ + float matMVI4x4[16]; + + matrix_copy(matMVI4x4, matMV); + matrix_invert(matMVI4x4); + matrix_transpose(matMVI4x4); + matMVI3x3[0] = matMVI4x4[0]; + matMVI3x3[1] = matMVI4x4[1]; + matMVI3x3[2] = matMVI4x4[2]; + matMVI3x3[3] = matMVI4x4[4]; + matMVI3x3[4] = matMVI4x4[5]; + matMVI3x3[5] = matMVI4x4[6]; + matMVI3x3[6] = matMVI4x4[8]; + matMVI3x3[7] = matMVI4x4[9]; + matMVI3x3[8] = matMVI4x4[10]; +} + +int init_cube(float aspect) +{ + generate_shader(&s_sobj, s_strVS, s_strFS); + s_loc_mtx_mv = glGetUniformLocation(s_sobj.program, "u_MVMatrix" ); + s_loc_mtx_pmv = glGetUniformLocation(s_sobj.program, "u_PMVMatrix" ); + s_loc_mtx_nrm = glGetUniformLocation(s_sobj.program, "u_ModelViewIT" ); + s_loc_color = glGetUniformLocation(s_sobj.program, "u_color" ); + s_loc_alpha = glGetUniformLocation(s_sobj.program, "u_alpha" ); + s_loc_lightpos= glGetUniformLocation(s_sobj.program, "u_LightPos" ); + + matrix_proj_perspective(s_matPrj, 72.0f, aspect, 1.f, 10000.f); + + unsigned char imgbuf[] = {255, 255, 255, 255}; + s_texid_dummy = create_2d_texture(imgbuf, 1, 1); + + return 0; +} + +int draw_line(float *mtxGlobal, float *p0, float *p1, float *color) +{ + float matMV[16], matPMV[16], matMVI3x3[9]; + GLfloat floor_vtx [9]; + + for (int i = 0; i < 3; i ++) { + floor_vtx[0 + i] = p0[i]; + floor_vtx[3 + i] = p1[i]; + } + + glEnable(GL_DEPTH_TEST); + glDisable(GL_CULL_FACE); + + glUseProgram( s_sobj.program ); + + glEnableVertexAttribArray(s_sobj.loc_vtx); + glEnableVertexAttribArray(s_sobj.loc_uv ); + glDisableVertexAttribArray(s_sobj.loc_nrm); + glVertexAttribPointer(s_sobj.loc_vtx, 3, GL_FLOAT, GL_FALSE, 0, floor_vtx); + glVertexAttribPointer(s_sobj.loc_uv , 2, GL_FLOAT, GL_FALSE, 0, s_uv ); + glVertexAttrib4fv(s_sobj.loc_nrm, s_nrm); + + matrix_identity(matMV); + compute_invmat3x3(matMVI3x3, matMV); + + matrix_mult(matMV, mtxGlobal, matMV); + matrix_mult(matPMV, s_matPrj, matMV); + + glUniformMatrix4fv(s_loc_mtx_mv, 1, GL_FALSE, matMV ); + glUniformMatrix4fv(s_loc_mtx_pmv, 1, GL_FALSE, matPMV); + glUniformMatrix3fv(s_loc_mtx_nrm, 1, GL_FALSE, matMVI3x3); + glUniform3f(s_loc_lightpos, 1.0f, 1.0f, 1.0f); + glUniform3f(s_loc_color, color[0], color[1], color[2]); + glUniform1f(s_loc_alpha, color[3]); + + glEnable(GL_BLEND); + + glBindTexture(GL_TEXTURE_2D, s_texid_dummy); + glDrawArrays(GL_LINES, 0, 2); + + glDisable(GL_BLEND); + + return 0; +} + +int draw_point_arrays(float *mtxGlobal, float *vtx, float *uv, int num, int texid, float *color) +{ + float matMV[16], matPMV[16], matMVI3x3[9]; + + glEnable(GL_DEPTH_TEST); + glDisable(GL_CULL_FACE); + + glUseProgram( s_sobj.program ); + + glEnableVertexAttribArray(s_sobj.loc_vtx); + glEnableVertexAttribArray(s_sobj.loc_uv ); + glDisableVertexAttribArray(s_sobj.loc_nrm); + glVertexAttribPointer(s_sobj.loc_vtx, 3, GL_FLOAT, GL_FALSE, 0, vtx); + glVertexAttribPointer(s_sobj.loc_uv , 2, GL_FLOAT, GL_FALSE, 0, uv ); + glVertexAttrib4fv(s_sobj.loc_nrm, s_nrm); + + matrix_identity(matMV); + compute_invmat3x3(matMVI3x3, matMV); + + matrix_mult(matMV, mtxGlobal, matMV); + matrix_mult(matPMV, s_matPrj, matMV); + + glUniformMatrix4fv(s_loc_mtx_mv, 1, GL_FALSE, matMV ); + glUniformMatrix4fv(s_loc_mtx_pmv, 1, GL_FALSE, matPMV); + glUniformMatrix3fv(s_loc_mtx_nrm, 1, GL_FALSE, matMVI3x3); + glUniform3f(s_loc_lightpos, 1.0f, 1.0f, 1.0f); + glUniform3f(s_loc_color, color[0], color[1], color[2]); + glUniform1f(s_loc_alpha, color[3]); + + glLineWidth(3); + glEnable(GL_BLEND); + + glBindTexture(GL_TEXTURE_2D, texid); + glDrawArrays(GL_POINTS, 0, num); + + glDisable(GL_BLEND); + + return 0; +} + +int create_mesh(mesh_obj_t *mesh, int num_tile_w, int num_tile_h) +{ + int num_vtx_u = num_tile_w + 1; + int num_vtx_v = num_tile_h + 1; + int num_vtx = num_vtx_u * num_vtx_v; + + mesh->vtx_array = (float *)malloc (num_vtx * 3 * sizeof(float)); + mesh->uv_array = (float *)malloc (num_vtx * 2 * sizeof(float)); + + GLuint vbos[3]; + glGenBuffers (3, vbos); + mesh->vbo_vtx = vbos[0]; + mesh->vbo_uv = vbos[1]; + mesh->vbo_idx = vbos[2]; + + int num_tri = num_tile_w * num_tile_h * 2; + int num_idx = num_tri * 3; + int idx_buf_size = num_idx * sizeof (unsigned short); + unsigned short *idx_array = (unsigned short *)malloc (idx_buf_size); + + for (int tile_y = 0; tile_y < num_tile_h; tile_y ++) { + for (int tile_x = 0; tile_x < num_tile_w; tile_x ++) { + int idx = tile_y * num_tile_w + tile_x; + + idx_array[6 * idx + 0] = (tile_y ) * num_vtx_u + (tile_x); + idx_array[6 * idx + 1] = (tile_y+1) * num_vtx_u + (tile_x); + idx_array[6 * idx + 2] = (tile_y ) * num_vtx_u + (tile_x+1); + idx_array[6 * idx + 3] = (tile_y ) * num_vtx_u + (tile_x+1); + idx_array[6 * idx + 4] = (tile_y+1) * num_vtx_u + (tile_x); + idx_array[6 * idx + 5] = (tile_y+1) * num_vtx_u + (tile_x+1); + } + } + mesh->idx_array = idx_array; + mesh->num_tile_w = num_tile_w; + mesh->num_tile_h = num_tile_h; + mesh->num_idx = num_idx; + + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, mesh->vbo_idx); + glBufferData(GL_ELEMENT_ARRAY_BUFFER, idx_buf_size, idx_array, GL_STATIC_DRAW); + + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); + + return 0; +} diff --git a/test/testsuites/common/visualizer/src/mv_util_shader.cpp b/test/testsuites/common/visualizer/src/mv_util_shader.cpp new file mode 100644 index 0000000..44b42b6 --- /dev/null +++ b/test/testsuites/common/visualizer/src/mv_util_shader.cpp @@ -0,0 +1,114 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mv_util_shader.h" + +GLuint compile_shader_text(GLenum shaderType, const char *text) +{ + GLuint shader; + GLint stat; + + shader = glCreateShader(shaderType); + glShaderSource(shader, 1, (const char **)&text, NULL); + glCompileShader(shader); + + glGetShaderiv(shader, GL_COMPILE_STATUS, &stat); + if (!stat) { + GLsizei len; + char *lpBuf; + + glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &len); + lpBuf = (char *)malloc(len); + + glGetShaderInfoLog(shader, len, &len, lpBuf); + LOGE("Error: problem compiling shader."); + LOGE("-----------------------------------"); + LOGE("%s", lpBuf); + LOGE("-----------------------------------"); + + free(lpBuf); + + return 0; + } + + return shader; +} + +GLuint link_shaders(GLuint vertShader, GLuint fragShader) +{ + GLuint program = glCreateProgram(); + + if (fragShader) glAttachShader(program, fragShader); + if (vertShader) glAttachShader(program, vertShader); + + glLinkProgram(program); + + { + GLint stat; + glGetProgramiv(program, GL_LINK_STATUS, &stat); + if (!stat) { + GLsizei len; + char *lpBuf; + + glGetProgramiv(program, GL_INFO_LOG_LENGTH, &len); + lpBuf = (char *)malloc(len); + + glGetProgramInfoLog(program, len, &len, lpBuf); + LOGE("Error: problem linking shader."); + LOGE("-----------------------------------"); + LOGE("%s", lpBuf); + LOGE("-----------------------------------"); + + free(lpBuf); + + return 0; + } + } + + return program; +} + +int generate_shader(shader_obj_t *sobj, char *str_vs, char *str_fs) +{ + GLuint fs, vs, program; + + vs = compile_shader_text(GL_VERTEX_SHADER, str_vs); + fs = compile_shader_text(GL_FRAGMENT_SHADER, str_fs); + if (vs == 0 || fs == 0) { + LOGE("Failed to compile shader."); + return -1; + } + + program = link_shaders(vs, fs); + if (program == 0) { + LOGE("Failed to link shaders."); + return -1; + } + + glDeleteShader(vs); + glDeleteShader(fs); + + sobj->program = program; + sobj->loc_vtx = glGetAttribLocation(program, "a_Vertex"); + sobj->loc_nrm = glGetAttribLocation(program, "a_Normal"); + sobj->loc_clr = glGetAttribLocation(program, "a_Color"); + sobj->loc_uv = glGetAttribLocation(program, "a_TexCoord"); + sobj->loc_tex = glGetUniformLocation(program, "u_sampler"); + sobj->loc_mtx = glGetUniformLocation(program, "u_PMVMatrix"); + sobj->loc_mtx_nrm = glGetUniformLocation(program, "u_NrmMatrix"); + + return 0; +} diff --git a/test/testsuites/common/visualizer/src/mv_util_visualizer_2d.cpp b/test/testsuites/common/visualizer/src/mv_util_visualizer_2d.cpp new file mode 100644 index 0000000..0b69bf8 --- /dev/null +++ b/test/testsuites/common/visualizer/src/mv_util_visualizer_2d.cpp @@ -0,0 +1,145 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include +#include +#include +#include + +#include "mv_util_visualizer_rd.grpc.pb.h" +#include "mv_util_visualizer_2d.h" + +static int win_w = 1920; +static int win_h = 1080; +static bool initialized = false; + +class ImageClientImpl { +public: + ImageClientImpl(std::shared_ptr channel) : stub_(NLImageService::NewStub(channel)) {} + + void DrawImage(mv_source_h source) { + unsigned int width = 0u, height = 0u; + unsigned int bufferSize = 0u; + unsigned char *buffer = NULL; + + mv_source_get_width(source, &width); + mv_source_get_height(source, &height); + mv_source_get_buffer(source, &buffer, &bufferSize); + + cv::Mat _frame, frame; + + _frame = cv::Mat(cv::Size(width, height), CV_8UC4, buffer); + cv::cvtColor(_frame, frame, cv::COLOR_RGBA2BGR); + + std::vector qualityType; + std::vector buff; + qualityType.push_back(cv::IMWRITE_JPEG_QUALITY); + qualityType.push_back(90); + cv::imencode(".jpg", frame, buff, qualityType); + + NLImageDrawRequest *request = new NLImageDrawRequest; + NLImage tmp; + unsigned long length = buff.size(); + + std::string str(buff.begin(), buff.end()); + std::string NLstring(str); + + tmp.set_length(length); + tmp.set_data(NLstring); + + request->mutable_image()->set_data(tmp.data()); + request->mutable_image()->set_length(tmp.length()); + + ::Empty reply; + + grpc::ClientContext context; + grpc::Status status = stub_->DrawImage(&context, *request, &reply); + } + +private: + std::unique_ptr stub_; + +}; + +static int create_window_surface() +{ + int err = MEDIA_VISION_ERROR_NONE; + err = egl_init_with_platform_window_surface(2, 8, 0, 0, win_w, win_h); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("egl_init_with_platform_window_surface: %i", err); + return MEDIA_VISION_ERROR_INTERNAL; + } + err = init_2d_renderer(win_w, win_h); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("init_2d_renderer: %i", err); + return MEDIA_VISION_ERROR_INTERNAL; + } + glClearColor(0.f, 0.f, 0.f, 1.0f); + glClear(GL_COLOR_BUFFER_BIT); + glViewport(0, 0, win_w, win_h); + + return MEDIA_VISION_ERROR_NONE; +} + +int mv_util_visualizer_2d(mv_source_h source, const char *url) +{ + int texid; + int texw, texh; + int err = MEDIA_VISION_ERROR_NONE; + texture_2d_t captex = {0}; + + if (url == NULL) { // target display + if (!initialized) { + if (create_window_surface() != MEDIA_VISION_ERROR_NONE) { + return MEDIA_VISION_ERROR_INTERNAL; + } + initialized = true; + } + + + err = load_texture(source, &texid, &texw, &texh); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("load_texture: %i", err); + return MEDIA_VISION_ERROR_INTERNAL; + } + + captex.texid = texid; + captex.width = texw; + captex.height = texh; + captex.format = pixfmt_fourcc('R', 'G', 'B', 'A'); + + float scale = (float)win_h / (float)texh; + + err = draw_2d_texture(&captex, (win_w - scale * texw) * 0.5f, + 0, scale * texw, scale * texh, 0); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("draw_2d_texture_ex: %i", err); + return MEDIA_VISION_ERROR_INTERNAL; + } + + err = egl_swap(); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("egl_swap: %i", err); + return MEDIA_VISION_ERROR_INTERNAL; + } + } else { // remote display + ImageClientImpl imageChannel(grpc::CreateChannel(url, grpc::InsecureChannelCredentials())); + imageChannel.DrawImage(source); + } + + return MEDIA_VISION_ERROR_NONE; +} diff --git a/test/testsuites/common/visualizer/src/mv_util_visualizer_3d.cpp b/test/testsuites/common/visualizer/src/mv_util_visualizer_3d.cpp new file mode 100644 index 0000000..74572dc --- /dev/null +++ b/test/testsuites/common/visualizer/src/mv_util_visualizer_3d.cpp @@ -0,0 +1,186 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mv_util_visualizer_3d.h" + +#define MAX_DEPTH_WIDTH 640 +#define MAX_DEPTH_HEIGHT 480 + +static int win_w = 1920; +static int win_h = 1080; +static bool initialized = false; + +static int create_window_surface() +{ + int err = MEDIA_VISION_ERROR_NONE; + err = egl_init_with_platform_window_surface(2, 8, 0, 0, win_w, win_h); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("egl_init_with_platform_window_surface: %i", err); + return MEDIA_VISION_ERROR_INTERNAL; + } + err = init_2d_renderer(win_w, win_h); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("init_2d_renderer: %i", err); + return MEDIA_VISION_ERROR_INTERNAL; + } + init_cube((float)win_w / (float)win_h); + glClearColor(0.f, 0.f, 0.f, 1.0f); + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + glViewport(0, 0, win_w, win_h); + + return MEDIA_VISION_ERROR_NONE; +} + +//static int xpos = 100; +static float s_mdl_qtn0[4]; +static float s_mdl_mtx[16]; +static float s_mdl_qtn [4]; + +int mv_util_visualizer_3d(mv_source_h source, float *depth, int xpos, int ypos) +{ + int texid; + int texw, texh; + int err = MEDIA_VISION_ERROR_NONE; + static int is_first_render3d = 1; + static mesh_obj_t depth_mesh; + static texture_2d_t captex = {0}; + + if (!initialized) { + initialized = true; + if (create_window_surface() != MEDIA_VISION_ERROR_NONE) { + return MEDIA_VISION_ERROR_INTERNAL; + } + } + err = load_texture(source, &texid, &texw, &texh); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("load_texture: %i", err); + return MEDIA_VISION_ERROR_INTERNAL; + } + captex.texid = texid; + captex.width = texw; + captex.height = texh; + captex.format = pixfmt_fourcc('R', 'G', 'B', 'A'); + + float mtxGlobal[16]; + + quaternion_identity(s_mdl_qtn); + quaternion_to_matrix(s_mdl_mtx, s_mdl_qtn); + + matrix_identity(mtxGlobal); + matrix_translate(mtxGlobal, 0, 0, -300.0f); + + int height = texh; + int width = texw; + + //xpos += 10; + //xpos = xpos % 1900; + + float dx = xpos; + float dy = ypos; + + float axis[3]; + axis[0] = 2 * M_PI * dy / 1080; + axis[1] = 2 * M_PI * dx / 1920; + axis[2] = 0; + + float rot = vec3_normalize(axis); + + quaternion_copy(s_mdl_qtn0, s_mdl_qtn); + + float dqtn[4]; + quaternion_rotate(dqtn, rot, axis[0], axis[1], axis[2]); + quaternion_mult(s_mdl_qtn, dqtn, s_mdl_qtn); + + quaternion_to_matrix(s_mdl_mtx, s_mdl_qtn); + matrix_mult(mtxGlobal, mtxGlobal, s_mdl_mtx); + + quaternion_identity(s_mdl_qtn); + quaternion_to_matrix(s_mdl_mtx, s_mdl_qtn); + + /* create mesh object */ + if (is_first_render3d) + { + create_mesh(&depth_mesh, MAX_DEPTH_WIDTH - 1, MAX_DEPTH_HEIGHT - 1); + is_first_render3d = 0; + } + float *vtx = depth_mesh.vtx_array; + float *uv = depth_mesh.uv_array; + int x, y; + + /* create 3D vertex coordinate */ + for (y = 0; y < height; y ++) { + for (x = 0; x < width; x ++) { + int idx = (y * width + x); + float d = depth[idx]; + + vtx[3 * idx + 0] = ((x / (float)height) * 2.0f - 1.0f) * 100.0f; + vtx[3 * idx + 1] = ((y / (float)height) * 2.0f - 1.0f) * 100.0f; + if (d > 0.0) + vtx[3 * idx + 2] = (d * 2.0f - 1.0f) * 100.0f - 50.0f; + else + vtx[3 * idx + 2] = 10000000.0; + + uv [2 * idx + 0] = x / (float)width; + uv [2 * idx + 1] = y / (float)height; + } + } + float colb[] = {1.0, 1.0, 1.0, 1.0}; + draw_point_arrays(mtxGlobal, vtx, uv, height * width, captex.texid, colb); + + { + /* (xyz)-AXIS */ + for (int i = -1; i <= 1; i ++) + { + for (int j = -1; j <= 1; j ++) + { + float col_base[] = {0.1, 0.5, 0.5, 0.5}; + float dx = 300.0; + float dy = 300.0; + float dz = 300.0; + + { + float v0[3] = {-dx, i * dy, j * dz}; + float v1[3] = { dx, i * dy, j * dz}; + float col_red[] = {1.0, 0.0, 0.0, 1.0}; + float *col = (i == 0 && j == 0) ? col_red : col_base; + draw_line(mtxGlobal, v0, v1, col); + } + { + float v0[3] = {i * dx, -dy, j * dz}; + float v1[3] = {i * dx, dy, j * dz}; + float col_green[] = {0.0, 1.0, 0.0, 1.0}; + float *col = (i == 0 && j == 0) ? col_green : col_base; + draw_line(mtxGlobal, v0, v1, col); + } + { + float v0[3] = {i * dx, j * dy, -dz}; + float v1[3] = {i * dx, j * dy, dz}; + float col_blue[] = {0.0, 0.0, 1.0, 1.0}; + float *col = (i == 0 && j == 0) ? col_blue : col_base; + draw_line(mtxGlobal, v0, v1, col); + } + } + } + } + + err = egl_swap(); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("egl_swap: %i", err); + return MEDIA_VISION_ERROR_INTERNAL; + } + + return MEDIA_VISION_ERROR_NONE; +} diff --git a/test/testsuites/common/visualizer/src/mv_util_winsys.cpp b/test/testsuites/common/visualizer/src/mv_util_winsys.cpp new file mode 100644 index 0000000..e31b733 --- /dev/null +++ b/test/testsuites/common/visualizer/src/mv_util_winsys.cpp @@ -0,0 +1,148 @@ +/** + * Copyright(c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0(the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "wayland-client.h" +#include "wayland-egl.h" +#include "mv_util_winsys.h" + +#define UNUSED(x)(void)(x) + +struct Display s_display; +struct Window s_window; + +static void handle_ping(void *data, struct wl_shell_surface *wlShellSurface, + uint32_t serial) +{ + UNUSED(data); + + wl_shell_surface_pong(wlShellSurface, serial); +} + +static void handle_configure(void *data, struct wl_shell_surface *shell_surface, + uint32_t edges, int32_t width, int32_t height) +{ + struct Window *window = (struct Window *)data; + UNUSED(data); + UNUSED(shell_surface); + UNUSED(edges); + + if (window->wlEGLNativeWindow) { + wl_egl_window_resize(window->wlEGLNativeWindow, width, height, 0, 0); + } + + window->geometry.width = width; + window->geometry.height = height; +} + +static const struct wl_shell_surface_listener shell_surface_listener = +{ + handle_ping, + handle_configure, + NULL +}; + +static void registry_handle_global(void *data, struct wl_registry *registry, + uint32_t name, const char *interface, uint32_t version) +{ + struct Display *d = (struct Display *)data; + UNUSED(version); + + if (strcmp(interface, "wl_compositor") == 0) { + d->wlCompositor = (wl_compositor *)wl_registry_bind(registry, name, + &wl_compositor_interface, 1); + } else if (strcmp(interface, "wl_shell") == 0) { + d->wlShell = (wl_shell *)wl_registry_bind(registry, name, + &wl_shell_interface, 1); + } +} + +static void registry_handle_global_remove(void *data, struct wl_registry *registry, + uint32_t name) +{ + UNUSED(data); + UNUSED(registry); + UNUSED(name); +} + +static const struct wl_registry_listener registry_listener = { + registry_handle_global, + registry_handle_global_remove +}; + +void * winsys_init_native_display(void) +{ + memset(&s_display, 0, sizeof(s_display)); + + + s_display.wlDisplay = wl_display_connect(NULL); + if (s_display.wlDisplay == NULL) { + LOGE("%s(%d)", __FILE__, __LINE__); + return NULL; + } + + s_display.wlRegistry = wl_display_get_registry(s_display.wlDisplay); + if (s_display.wlRegistry == NULL) { + LOGE("%s(%d)", __FILE__, __LINE__); + return NULL; + } + + wl_registry_add_listener(s_display.wlRegistry, ®istry_listener, &s_display); + + wl_display_dispatch(s_display.wlDisplay); + + return s_display.wlDisplay; +} + + +void * winsys_init_native_window(void *dpy, int win_w, int win_h) +{ + UNUSED(dpy); + memset(&s_window, 0, sizeof(s_window)); + + if (!s_display.wlCompositor || !s_display.wlShell) { + LOGE("window compositor or shell is NULL"); + return NULL; + } + + s_window.wlSurface = wl_compositor_create_surface(s_display.wlCompositor); + if (s_window.wlSurface == NULL) { + LOGE("window surface is NULL"); + return NULL; + } + + s_window.wlShellSurface = wl_shell_get_shell_surface(s_display.wlShell, s_window.wlSurface); + if (s_window.wlShellSurface == NULL) { + LOGE("window shell surface is NULL"); + return NULL; + } + + wl_shell_surface_add_listener(s_window.wlShellSurface, &shell_surface_listener, &s_window); + + s_window.window_size.width = win_w; + s_window.window_size.height = win_h; + s_window.display = &s_display; + + s_window.wlEGLNativeWindow = wl_egl_window_create(s_window.wlSurface, win_w, win_h); + if (s_window.wlEGLNativeWindow == NULL) { + LOGE("window is NULL"); + return NULL; + } + + return s_window.wlEGLNativeWindow; +} diff --git a/test/testsuites/mv3d/CMakeLists.txt b/test/testsuites/mv3d/CMakeLists.txt new file mode 100644 index 0000000..38abf66 --- /dev/null +++ b/test/testsuites/mv3d/CMakeLists.txt @@ -0,0 +1,71 @@ +project(mv_depth_test_suite) +cmake_minimum_required(VERSION 2.6...3.13) + +find_package(OpenCV REQUIRED core videoio imgcodecs) +if(NOT OpenCV_FOUND) + message(SEND_ERROR "OpenCV NOT FOUND") + return() +endif() + +find_package(Open3D REQUIRED NO_POLICY_SCOPE) +if(NOT Open3D_FOUND) + message(SEND_ERROR "Open3D NOT FOUND") + return() +else() + include_directories(${Open3D_INCLUDE_DIRS}) +endif() + +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format-truncation") +add_executable(${PROJECT_NAME} depth_test_suite.cpp) +target_link_libraries(${PROJECT_NAME} ${MV_3D_LIB_NAME} + ${Open3D_LIBRARIES} + ${OpenCV_LIBS} + mv_image_helper + mv_video_helper + mv_testsuite_common) +if(BUILD_VISUALIZER) + target_link_libraries(${PROJECT_NAME} mv_visualizer) +endif() +target_include_directories(${PROJECT_NAME} PUBLIC ${Open3D_INCLUDE_DIRS}) +install(TARGETS ${PROJECT_NAME} DESTINATION ${CMAKE_INSTALL_BINDIR}) + +## mv_depthstream +pkg_check_modules(GLIB_PKG glib-2.0) + +if (NOT GLIB_PKG_FOUND) + message(SEND_ERROR "Failed to find glib") + return() +else() + include_directories(${GLIB_PKG_INCLUDE_DIRS}) +endif() + +SET(dependents "gstreamer-1.0 gstreamer-app-1.0 gstreamer-video-1.0") + +INCLUDE(FindPkgConfig) +pkg_check_modules(${PROJECT_NAME} REQUIRED ${dependents}) +FOREACH(flag ${${PROJECT_NAME}_CFLAGS}) + SET(EXTRA_CFLAGS "${EXTRA_CFLAGS} ${flag}") + SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} ${flag}") +ENDFOREACH(flag) + +SET(CMAKE_CXX_FLAGS "-I./include -I./include/headers ${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fPIC -Wall") +SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") + +add_executable(mv_depthstream_test_suite depthstream_test_suite.cpp) + +target_link_libraries(mv_depthstream_test_suite ${MV_3D_LIB_NAME} + ${Open3D_LIBRARIES} + ${OpenCV_LIBS} + gstreamer-1.0 + glib-2.0 + capi-system-info + dlog + mv_image_helper + mv_video_helper + mv_testsuite_common) +if(BUILD_VISUALIZER) + target_link_libraries(mv_depthstream_test_suite mv_visualizer) +endif() + +target_include_directories(mv_depthstream_test_suite PUBLIC ${Open3D_INCLUDE_DIRS}) +install(TARGETS mv_depthstream_test_suite DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/test/testsuites/mv3d/depth_test_suite.cpp b/test/testsuites/mv3d/depth_test_suite.cpp new file mode 100644 index 0000000..71bbe3d --- /dev/null +++ b/test/testsuites/mv3d/depth_test_suite.cpp @@ -0,0 +1,607 @@ +/** + * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "mv_3d.h" + +#ifdef BUILD_VISUALIZER +#include "mv_util_visualizer_3d.h" +#endif + +#define MAX_STRING_LENGTH 1024 +#define ARRAY_SIZE(x) (sizeof((x)) / sizeof((x)[0])) + +#define __max(a,b) (((a) > (b)) ? (a) : (b)) +#define __min(a,b) (((a) < (b)) ? (a) : (b)) + +using namespace open3d; + +/* There are calib.txt, im0.png, and im1.png in each dataset directories.*/ +static const char* dataset[] = { + "Adirondack", + "Jadeplant", + "Motorcycle", + "Piano", + "Pipes", + "Playroom", + "Playtable", + "Recycle", + "Shelves", + "Vintage", + "Backpack", + "Bicycle1", + "Cable", + "Classroom1", + "Couch", + "Flowers", + "Mask", + "Shopvac", + "Sticks", + "Storage", + "Sword1", + "Sword2", + "Umbrella" +}; + +class StopWatch +{ +public: + StopWatch() { + start = std::chrono::steady_clock::now(); + } + ~StopWatch() = default; + + std::chrono::milliseconds elapsedTime() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + } +private: + std::chrono::steady_clock::time_point start; +}; + +typedef struct _appdata { + std::chrono::milliseconds diffMs; + std::string dataPath; + std::string intrinsicName; + std::string rgbName; + std::string datasetName; + float minDisp; + float maxDisp; + int fmt; +} appdata; + +enum { + FMT_PFM = 1, + FMT_PNG +}; + +int littleendian() +{ + int intval = 1; + uchar *uval = (uchar *)&intval; + return uval[0] == 1; +} + +void WriteFilePFM(float *data, int width, int height, const char* filename, + float minDisp, float maxDisp, float scalefactor = 1 / 255.0) +{ + FILE *stream = fopen(filename, "wb"); + if (stream == 0) { + fprintf(stderr, "WriteFilePFM: could not open %s\n", filename); + exit(1); + } + + if (littleendian()) + scalefactor = -scalefactor; + + fprintf(stream, "Pf\n%d %d\n%f\n", width, height, scalefactor); + + int n = width; + for (int y = height - 1; y >= 0; y--) { + float* ptr = data + y * width; + for (int x = 0; x < width; x++) { + if (ptr[x] <= 0 || ptr[x] > maxDisp) { //xsh modified + ptr[x] = INFINITY; + } + } + if ((int)fwrite(ptr, sizeof(float), n, stream) != n) { + fprintf(stderr, "WriteFilePFM: problem writing data\n"); + exit(1); + } + } + fclose(stream); +} + +// translate value x in [0..1] into color triplet using "jet" color map +// if out of range, use darker colors +// variation of an idea by http://www.metastine.com/?p=7 +void jet(float x, int& r, int& g, int& b) +{ + if (x < 0) x = -0.05; + if (x > 1) x = 1.05; + x = x / 1.15 + 0.1; // use slightly asymmetric range to avoid darkest shades of blue. + r = __max(0, __min(255, (int)(round(255 * (1.5 - 4*fabs(x - .75)))))); + g = __max(0, __min(255, (int)(round(255 * (1.5 - 4*fabs(x - .5)))))); + b = __max(0, __min(255, (int)(round(255 * (1.5 - 4*fabs(x - .25)))))); +} + +void WriteFilePNG(unsigned short *data, int width, int height, const char* filename, + float minDisp, float maxDisp) +{ + // Open the file + FILE *stream = fopen(filename, "wb"); + if (stream == 0) { + fprintf(stderr, "WriteFilePNG: could not open %s\n", filename); + exit(1); + } + + cv::Mat dump(cv::Size(width, height), CV_16U); + + for (int y = 0; y < height; y++) { + unsigned short* ptr = data + y * width; + for (int x = 0; x < width; x++) { + unsigned short f = ptr[x]; + + dump.at(y,x) = f; + } + } + + printf("%d x %d", width, height); + cv::imwrite(filename, dump); + + fclose(stream); +} + +void WritePLY(double *data, int size, const char* filename) +{ + std::ofstream stream(filename, std::ofstream::out); + stream << "ply" << std::endl; + stream << "format ascii 1.0" << std::endl; + stream << "element vertex " << size << std::endl; + stream << "property float x" << std::endl; + stream << "property float y" << std::endl; + stream << "property float z" << std::endl; + stream << "end_header" << std::endl; + + for (int pc = 0; pc < size; ++pc) { + stream << std::fixed << std::setprecision(2) << + data[pc*3] << " " << data[pc*3 + 1] << " " << data[pc*3 + 2] << std::endl; + } + + return; +} + +void WritePointCloud(const char* data_path, + const char* color_filename, + const char* depth_filename, + unsigned int width, + unsigned int height, + const camera::PinholeCameraIntrinsic& intrinsic) +{ + geometry::Image img_color, img_depth; + + io::ReadImage(color_filename, img_color); + io::ReadImage(depth_filename, img_depth); + + utility::LogInfo("depth filename : {}", depth_filename); + utility::LogInfo("Reading RGBD image : "); + utility::LogInfo(" Color : {:d} x {:d} x {:d} ({:d} bits per channel)", + img_color.width_, img_color.height_, img_color.num_of_channels_, + img_color.bytes_per_channel_ * 8); + utility::LogInfo(" Depth : {:d} x {:d} x {:d} ({:d} bits per channel)", + img_depth.width_, img_depth.height_, img_depth.num_of_channels_, + img_depth.bytes_per_channel_ * 8); + double depth_scale = 1000.0, depth_trunc = 200.0; + bool convert_rgb_to_intensity = false; + std::shared_ptr rgbd_image = + geometry::RGBDImage::CreateFromColorAndDepth( + img_color, img_depth, depth_scale, depth_trunc, + convert_rgb_to_intensity); + + auto pcd = geometry::PointCloud::CreateFromRGBDImage(*rgbd_image, intrinsic); + std::string dataPath = std::string(data_path); + const std::string filename_ply(std::string(data_path) + std::string("/") + + dataPath.substr(dataPath.find_last_of("/\\") + 1) + + std::string(".ply")); + + utility::LogInfo("pcd counts : {}", pcd->points_.size()); +#ifndef BUILD_VISUALIZER + if (io::WritePointCloud(filename_ply, *pcd)) { + utility::LogInfo("Successfully wrote {}", filename_ply); + } else { + utility::LogError("Failed to write {}", filename_ply); + } +#else + unsigned int i, x, y, idx; + mv_source_h source = NULL; + int err = MEDIA_VISION_ERROR_NONE; + + cv::Mat rgb; + float *depth = NULL; + + Eigen::Vector3d min_xyz = pcd->GetMinBound(); + Eigen::Vector3d max_xyz = pcd->GetMaxBound(); + + int depth_width = (int)((max_xyz[0] - min_xyz[0]) * 100.0) + 1; + int depth_height = (int)((max_xyz[1] - min_xyz[1]) * 100.0) + 1; + + utility::LogInfo("min {}, max {}", min_xyz, max_xyz); + utility::LogInfo("depth W:H {}:{}", depth_width, depth_height); + + err = mv_create_source(&source); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("Errors were occurred during creating the source!!! code : %i", err); + goto out; + } + depth = (float *)malloc(depth_width * depth_height * 4); + + rgb = cv::imread(color_filename, cv::IMREAD_COLOR); + cv::resize(rgb, rgb, cv::Size(depth_width, depth_height)); + cv::flip(rgb, rgb, 0); + cv::cvtColor(rgb, rgb, cv::COLOR_BGR2RGBA); + err = mv_source_fill_by_buffer(source, + rgb.ptr(), + rgb.elemSize() * rgb.size().width + * rgb.size().height, + rgb.size().width, + rgb.size().height, + MEDIA_VISION_COLORSPACE_RGB888); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("Errors were occurred during filling the source!!! code : %i", err); + goto out; + } + + for (i = 0; i < pcd->points_.size(); i++) { + x = (int)((pcd->points_.data()[i][0] - min_xyz[0]) * 100.0); + y = (int)((max_xyz[1] - pcd->points_.data()[i][1]) * 100.0); + idx = y * depth_width + x; + depth[idx] = max_xyz[2] - pcd->points_.data()[i][2]; + } + if (source != NULL && depth != NULL) + while(1) + mv_util_visualizer_3d(source, depth, 100, 25); + +out: + if (depth != NULL) { + delete [] depth; + depth = NULL; + } + + if (source != NULL) { + err = mv_destroy_source(source); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("Errors were occurred during destroying the source!!! code : %i", err); + } + } +#endif +} + +void _depth_middlebury_cb(mv_source_h source, + unsigned short* depth, + unsigned int width, + unsigned int height, + void* user_data) +{ + if (!user_data) { + printf("user_data is null. Skip..\n"); + return; + } + + auto udata = static_cast(user_data); + + StopWatch stopWatch; + + if (udata->fmt == FMT_PFM) { + WriteFilePFM(reinterpret_cast(depth), width, height, udata->datasetName.c_str(), + udata->minDisp, udata->maxDisp); + } else { + WriteFilePNG(depth, width, height, udata->datasetName.c_str(), + udata->minDisp, udata->maxDisp); + } + + std::string intrinsic_path; + intrinsic_path = udata->intrinsicName; + utility::LogInfo("Camera intrinsic path {}", intrinsic_path); + + camera::PinholeCameraIntrinsic intrinsic; + if (intrinsic_path.empty() || + !io::ReadIJsonConvertible(intrinsic_path, intrinsic)) { + utility::LogWarning( + "Failed to read intrinsic parameters for depth image."); + utility::LogWarning("Using default value for Primesense camera."); + intrinsic = camera::PinholeCameraIntrinsic( + camera::PinholeCameraIntrinsicParameters::PrimeSenseDefault); + } + utility::LogInfo("focal_x = {}", intrinsic.GetFocalLength().first); + utility::LogInfo("focal_y = {}", intrinsic.GetFocalLength().second); + utility::LogInfo("cx = {}", intrinsic.GetPrincipalPoint().first); + utility::LogInfo("cy = {}", intrinsic.GetPrincipalPoint().second); + + WritePointCloud(udata->dataPath.c_str(), udata->rgbName.c_str(), + udata->datasetName.c_str(), width, height, intrinsic); + + udata->diffMs = stopWatch.elapsedTime(); +} + +int perform_middlebury_test() +{ + char* path_to_dataset = NULL; + char* suffix_for_algo = NULL; + int err = MEDIA_VISION_ERROR_NONE; + mv_engine_config_h engine_config = NULL; + mv_source_h left_source = NULL; + mv_source_h right_source = NULL; + mv_3d_h mv3d_handle = NULL; + size_t datasize = ARRAY_SIZE(dataset); + cv::Mat left_frame, right_frame; + + while (input_string("root path including dataset:", + 1024, &path_to_dataset) == -1) { + printf("Incorrect! Try again.\n"); + } + + while (input_string("suffix for algorithm:", + 1024, &suffix_for_algo) == -1) { + printf("Incorrect! Try again.\n"); + } + + const char* formats[] = {"pfm", "png"}; + int sel_fmt = -1; + while(sel_fmt <= 0 || sel_fmt > ARRAY_SIZE(formats)) { + sel_fmt = show_menu_linear("Select Action", formats, ARRAY_SIZE(formats)); + } + + err = mv_create_engine_config(&engine_config); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to create engine config\n"); + goto _err; + } + + err = mv_create_source(&left_source); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to create left source\n"); + goto _err; + } + + err = mv_create_source(&right_source); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to create right source\n"); + goto _err; + } + + for (size_t data = 0; data < datasize; ++data) { + char dataPath[MAX_STRING_LENGTH]; + char calibFilename[MAX_STRING_LENGTH]; + char leftFilename[MAX_STRING_LENGTH]; + char rightFilename[MAX_STRING_LENGTH]; + char stereoConfigFileName[MAX_STRING_LENGTH]; + snprintf(dataPath, MAX_STRING_LENGTH, "%s/%s", path_to_dataset, dataset[data]); + snprintf(calibFilename, MAX_STRING_LENGTH, "%s/calib.txt", dataPath); + snprintf(stereoConfigFileName, MAX_STRING_LENGTH, "%s/calibOcv.xml", dataPath); + snprintf(leftFilename, MAX_STRING_LENGTH, "%s/im0.png", dataPath); + snprintf(rightFilename, MAX_STRING_LENGTH, "%s/im1.png", dataPath); + + if ((access(calibFilename, F_OK) < 0) || + (access(leftFilename, F_OK) < 0) || + (access(rightFilename, F_OK) < 0) ) { + printf("Invalid dataset path:\n"); + goto _err; + } + + printf("Run dataset %s\n", dataset[data]); + FILE* fp = fopen(calibFilename, "r"); + char line[MAX_STRING_LENGTH]; + float fVal; + float dmin, dmax; + int iVal; + int width, height; + if (fp != NULL) { + while (fgets(line, MAX_STRING_LENGTH, fp) != NULL) { + if (sscanf(line, "vmin= %f", &fVal) == 1) dmin = fVal; + if (sscanf(line, "vmax= %f", &fVal) == 1) dmax = fVal; + if (sscanf(line, "width= %d", &iVal) == 1) width = iVal; + if (sscanf(line, "height= %d", &iVal) == 1) height = iVal; + } + } + fclose(fp); + + int minDisp = static_cast(dmin); + int maxDisp = static_cast(dmax); + + // engine_config + err = mv_engine_config_set_int_attribute(engine_config, + MV_3D_DEPTH_MODE, MV_3D_DEPTH_MODE_STEREO); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to set depth mode\n"); + goto _err; + } + + err = mv_engine_config_set_int_attribute(engine_config, + MV_3D_DEPTH_WIDTH, width); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to set width\n"); + goto _err; + } + + err = mv_engine_config_set_int_attribute(engine_config, + MV_3D_DEPTH_HEIGHT, height); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to set height\n"); + goto _err; + } + + err = mv_engine_config_set_int_attribute(engine_config, + MV_3D_DEPTH_MIN_DISPARITY, minDisp); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to set min disparity\n"); + goto _err; + } + + err = mv_engine_config_set_int_attribute(engine_config, + MV_3D_DEPTH_MAX_DISPARITY, maxDisp); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to set max disparity\n"); + goto _err; + } + + err = mv_engine_config_set_string_attribute(engine_config, + MV_3D_DEPTH_STEREO_CONFIG_FILE_PATH, stereoConfigFileName); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to set stereo config file path\n"); + goto _err; + } + + // left_source, right_source + left_frame = cv::imread(leftFilename, cv::IMREAD_GRAYSCALE); + right_frame = cv::imread(rightFilename, cv::IMREAD_GRAYSCALE); + + err = mv_source_fill_by_buffer(left_source, + left_frame.ptr(), + left_frame.elemSize() * left_frame.size().width + * left_frame.size().height, + left_frame.size().width, + left_frame.size().height, + MEDIA_VISION_COLORSPACE_Y800); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to fill left_source\n"); + goto _err; + } + + err = mv_source_fill_by_buffer(right_source, + right_frame.ptr(), + right_frame.elemSize() * right_frame.size().width + * right_frame.size().height, + right_frame.size().width, + right_frame.size().height, + MEDIA_VISION_COLORSPACE_Y800); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to fill right_source\n"); + goto _err; + } + + // depth_handle + err = mv_3d_create(&mv3d_handle); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to create depth handle\n"); + goto _err; + } + + err = mv_3d_configure(mv3d_handle, engine_config); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to configure depth handle\n"); + goto _err; + } + + // get depth + appdata dump { + std::chrono::milliseconds::zero(), + std::string(dataPath), + std::string(dataPath) + std::string("/intrinsics.json"), + std::string(leftFilename), + std::string(dataPath) + std::string("/disp0") + std::string(suffix_for_algo), + static_cast(minDisp), + static_cast(maxDisp), + sel_fmt + }; + dump.datasetName += sel_fmt == FMT_PFM ? std::string(".pfm") : std::string(".png"); + + err = mv_3d_set_depth_cb(mv3d_handle, _depth_middlebury_cb, static_cast(&dump)); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to set callback to depth handle\n"); + goto _err; + } + + err = mv_3d_prepare(mv3d_handle); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to prepare depth handle\n"); + goto _err; + } + StopWatch stopWatch; + err = mv_3d_run(mv3d_handle, + left_source, + right_source, + NULL); + + /* time to save a file is excluded */ + std::chrono::milliseconds diffMs = stopWatch.elapsedTime() - dump.diffMs; + printf("time: %lld ms\n\n", static_cast(diffMs.count())); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to get depth\n"); + break; + } + } + +_err: + if (path_to_dataset) + free(path_to_dataset); + + if (engine_config) + mv_destroy_engine_config(engine_config); + + if (left_source) + mv_destroy_source(left_source); + + if (right_source) + mv_destroy_source(right_source); + + if (mv3d_handle) + mv_3d_destroy(mv3d_handle); + + return err; +} + +int main() +{ + int err = MEDIA_VISION_ERROR_NONE; + const char* names[] = {"Middlebury - TrainingQ(Imperf)"}; + + utility::SetVerbosityLevel(utility::VerbosityLevel::Debug); + + int sel_opt = show_menu_linear("Select Action", names, ARRAY_SIZE(names)); + if (sel_opt <= 0 || sel_opt > ARRAY_SIZE(names)) { + printf("Invalid option"); + return -1; + } + + switch (sel_opt) { + case 1: + err = perform_middlebury_test(); + break; + default: + err = MEDIA_VISION_ERROR_INVALID_OPERATION; + } + if (err != MEDIA_VISION_ERROR_NONE) + printf("Failed to perform test"); + + return err; +} diff --git a/test/testsuites/mv3d/depthstream_test_suite.cpp b/test/testsuites/mv3d/depthstream_test_suite.cpp new file mode 100644 index 0000000..c2441bc --- /dev/null +++ b/test/testsuites/mv3d/depthstream_test_suite.cpp @@ -0,0 +1,622 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "mv_3d.h" +#include + +#include +#include +#include +#include + +#ifdef BUILD_VISUALIZER +#include "mv_util_visualizer_3d.h" +#endif + +#define MAX_STRING_LENGTH 1024 +#define ARRAY_SIZE(x) (sizeof((x)) / sizeof((x)[0])) +#define MAX_FRAMES 1800 // 30 fps * 60s + +#define INTRINSIC_FILE_PATH "/usr/share/capi-media-vision/stereoCalibZedVGA.json" +using namespace open3d; + +class StopWatch +{ +public: + StopWatch() + { + start = std::chrono::steady_clock::now(); + } + ~StopWatch() = default; + + std::chrono::milliseconds elapsedTime() + { + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + } + +private: + std::chrono::steady_clock::time_point start; +}; + +typedef struct _appdata +{ + std::chrono::milliseconds diffMs; + std::string dataPath; + std::string intrinsicName; + std::string rgbName; + std::string datasetName; + float minDisp; + float maxDisp; + int fmt; +} appdata; + +static float *vertex3d = NULL; +static unsigned char *data_buffer_3d = NULL; +mv_source_h mv_source3d; + +gulong handler; + +// Gstreamer +GstElement *pipeline; + +// Gstreamer - camera src +GstElement *pipecam, *source, *filter, *queue1; +GstElement *vrate, *vrfilter; +GstElement *vconv, *vcfilter, *fsink; + +// Gstreamer - depth handling +GstElement *pipedepth, *appsource; +GstElement *asfilter; +GstElement *queue2; +GstElement *asvconv, *mjpegenc; +GstElement *flsink; +GstElement *fsink2; +GstElement *sink; + +typedef struct _SrcData +{ + GstElement *src; + int numFrame; +} SrcData; + +// MediaVision +mv_engine_config_h engine_config; +mv_source_h mv_source; +mv_3d_h mv3d_handle; + +/* internal functions for testsuite */ +GCond g_thread_cond; +GMutex g_thread_mutex; + +void __wait() +{ + g_mutex_lock(&g_thread_mutex); + g_print("\t[__wait] waiting... until finishing\n"); + g_cond_wait(&g_thread_cond, &g_thread_mutex); + g_print("\t[__wait] get signal from callback\n"); + g_mutex_unlock(&g_thread_mutex); +} + +void __signal() +{ + g_mutex_lock(&g_thread_mutex); + g_cond_signal(&g_thread_cond); + g_print("\t[__signal] send signal to test proc\n"); + g_mutex_unlock(&g_thread_mutex); +} + +static bool isLive; +static bool isAsync; +static double depth_scale; +static double depth_trunc; +static int camWidth; +static int camHeight; +static int minDisp; +static int maxDisp; +static int display_xpos; +static int display_ypos; +static int fidx = 0; +void int_handler(int sig) +{ + char c; + + signal(sig, SIG_IGN); + while ((getchar()) != '\n'); + + c = getchar(); + if (c == 'y' || c == 'Y') { + g_signal_handler_disconnect(fsink, handler); + printf("delete buffer, used by visualize 3d\n"); + //if (data_buffer_3d != NULL) { + // delete [] data_buffer_3d; + // data_buffer_3d = NULL; + //} + + gst_element_send_event(pipecam, gst_event_new_eos()); + gst_element_send_event(pipedepth, gst_event_new_eos()); + + if (vertex3d != NULL) { + delete [] vertex3d; + vertex3d = NULL; + } + + if (mv_source3d != NULL) { + mv_destroy_source(mv_source3d); + mv_source3d = NULL; + } + + sleep(1); + + } else { + printf("no"); + signal(SIGINT, int_handler); + } + + getchar(); // Get new line character +} + +static gboolean bus_call(GstBus *bus, GstMessage *msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *) data; + + switch (GST_MESSAGE_TYPE(msg)) { + case GST_MESSAGE_EOS: + printf("End of stream\n"); + g_main_loop_quit(loop); + break; + + case GST_MESSAGE_ERROR: { + gchar *debug; + GError *error; + + gst_message_parse_error(msg, &error, &debug); + g_free(debug); + + printf("Error: %s\n", error->message); + g_error_free(error); + + g_main_loop_quit(loop); + break; + } + default: + break; + } + + return TRUE; +} + +void WritePointCloud(const char* data_path, + const char* color_filename, + const char* depth_filename, + unsigned int width, + unsigned int height, + const camera::PinholeCameraIntrinsic& intrinsic) +{ + geometry::Image img_color, img_depth; + + io::ReadImage(color_filename, img_color); + io::ReadImage(depth_filename, img_depth); + + utility::LogInfo("depth filename : {}", depth_filename); + utility::LogInfo("Reading RGBD image : "); + utility::LogInfo(" Color : {:d} x {:d} x {:d} ({:d} bits per channel)", + img_color.width_, img_color.height_, img_color.num_of_channels_, + img_color.bytes_per_channel_ * 8); + utility::LogInfo(" Depth : {:d} x {:d} x {:d} ({:d} bits per channel)", + img_depth.width_, img_depth.height_, img_depth.num_of_channels_, + img_depth.bytes_per_channel_ * 8); + double depth_scale = 1000.0, depth_trunc = 200.0; + bool convert_rgb_to_intensity = true; + std::shared_ptr rgbd_image = + geometry::RGBDImage::CreateFromColorAndDepth( + img_color, img_depth, depth_scale, depth_trunc, + convert_rgb_to_intensity); + + auto pcd = geometry::PointCloud::CreateFromRGBDImage(*rgbd_image, intrinsic); + std::string dataPath = std::string(data_path); + const std::string filename_ply(std::string(data_path) + std::string("/") + + dataPath.substr(dataPath.find_last_of("/\\") + 1) + + std::string(".ply")); + + if (io::WritePointCloud(filename_ply, *pcd)) { + utility::LogInfo("Successfully wrote {}", filename_ply); + } else { + utility::LogError("Failed to write {}", filename_ply); + } +} + +void _depth_stereo_cb(mv_source_h source, unsigned short *depth, unsigned int width, + unsigned int height, void *user_data) +{ +#if 0 + if (!user_data) { + LOGW("user_data is null, Skip..."); + return; + } + + GstMapInfo *buffer = static_cast(user_data); + + + for (int j = 0; j < height; j++) { + for (int i = 0; i < width; ++i) { + buffer->data[j * width + i] = + static_cast(depth[j * width + i]); + } + } +#endif + double maxDepth = 2000.0; + double minDepth = 175.0; + double depthDenom = static_cast(maxDepth - minDepth); + + camera::PinholeCameraIntrinsic intrinsic; + io::ReadIJsonConvertible(INTRINSIC_FILE_PATH, intrinsic); + + vertex3d = (float *)malloc(width * height * 4); + memset(vertex3d, 0, width * height * 4); + utility::LogInfo("focal_x = {}", intrinsic.GetFocalLength().first); + utility::LogInfo("focal_y = {}", intrinsic.GetFocalLength().second); + utility::LogInfo("cx = {}", intrinsic.GetPrincipalPoint().first); + utility::LogInfo("cy = {}", intrinsic.GetPrincipalPoint().second); + + // depth (16bit) + geometry::Image img_depth; + img_depth.Prepare(width, height, 1, sizeof(unsigned short)); + uint16_t *pImg_depth_data = (unsigned short*)img_depth.data_.data(); + memcpy(pImg_depth_data, depth, width*height*sizeof(unsigned short)); + auto pcd = geometry::PointCloud::CreateFromDepthImage( + img_depth, intrinsic, + Eigen::Matrix4d::Identity(), + 1000.0, 2.0, 1, + false); + + cv::Mat colorMap = cv::Mat(cv::Size(width,height), CV_8UC1); + for (int j = 0; j < height; ++j) { + for (int i = 0; i < width; ++i) { + int idx = j * width + i; + int idxInv = width * height - idx - 1; + auto depth_raw = pcd->points_.data()[idx][2] * 1000.0; + auto depthCut = std::min(std::max(depth_raw, minDepth), maxDepth); + colorMap.at(height -j - 1, width -i - 1) = + static_cast( + (depthCut - minDepth) * 255.0 / depthDenom); + + // 175 ~ 1900 + auto inverseDepthCut = maxDepth - depthCut; + if (inverseDepthCut <= (maxDepth - 1900)) { + vertex3d[idxInv] = 0.0f; + } else if (inverseDepthCut > (maxDepth - 1900) && inverseDepthCut < 1825) { + vertex3d[idxInv] = inverseDepthCut * 0.001f; + } else { + vertex3d[idxInv] = 0.0f; + } + } + } + + cv::Mat colorized_depth; + cv::applyColorMap(colorMap, colorized_depth, cv::COLORMAP_JET); + + int err; + if (mv_source3d == NULL) { + err = mv_create_source(&mv_source3d); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("Errors were occurred during creating the source!!! code : %i", err); + } + } + + mv_source_fill_by_buffer(mv_source3d, + colorized_depth.ptr(), + colorized_depth.elemSize() * colorized_depth.size().width + * colorized_depth.size().height, + colorized_depth.size().width, + colorized_depth.size().height, + MEDIA_VISION_COLORSPACE_RGB888); +#ifdef BUILD_VISUALIZER + mv_util_visualizer_3d(mv_source3d, vertex3d, display_xpos, display_ypos); +#endif + mv_destroy_source(mv_source3d); + mv_source3d = NULL; + delete [] vertex3d; + vertex3d = NULL; + __signal(); +} + +static void stereo_handoff(GstElement *object, GstBuffer *buffer, GstPad *pad, + gpointer userData) +{ + GstMapInfo map; + SrcData *srcData = static_cast(userData); + int ret = MEDIA_VISION_ERROR_NONE; + if (srcData->numFrame >= MAX_FRAMES) { + GstFlowReturn ret; + g_signal_emit_by_name(srcData->src, "end-of-stream", &ret); + return; + } + + if (buffer != NULL) { + gst_buffer_map(buffer, &map, GST_MAP_READ); + + mv_source_fill_by_buffer(mv_source, map.data, 1344 * 376 * 1, 1344, 376, + MEDIA_VISION_COLORSPACE_Y800); + + gst_buffer_unmap(buffer, &map); + + if (isAsync) { + ret = mv_3d_run_async(mv3d_handle, mv_source, NULL, NULL); + __wait(); + } else { + ret = mv_3d_run(mv3d_handle, mv_source, NULL, NULL); + } + + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Fail to mv_3d_run_async()"); + } + + mv_source_clear(mv_source); + } +} + +int createPipelineCam(SrcData &srcData) +{ + /* + v4l2src device=/dev/video0 ! + video/x-raw, width=1344, height=376, framerate=30/1, format=YUY2 ! + queue ! videorate ! video/x-raw, framerate=30/1 ! + videoconvert ! video/x-raw, format=GRAY8 ! fakesink -v -e + */ + printf("enter - createPipelineCam\n"); + source = gst_element_factory_make("v4l2src", "src"); + filter = gst_element_factory_make("capsfilter", "filter"); + + vconv = gst_element_factory_make("videoconvert", "convert"); + vcfilter = gst_element_factory_make("capsfilter", "vcfilter"); + fsink = gst_element_factory_make("fakesink", "fsink"); + + if (!pipecam || !source || !filter || !vconv || !vcfilter || !fsink) { + printf("pipecam[%p]\n", pipecam); + printf("source [%p], filter [%p]\n", source, filter); + printf("vconv [%p], vcfilter [%p] \n", vconv, vcfilter); + printf("fsink [%p] \n ", fsink); + return -1; + } + + /* set up the pipeline */ + g_object_set(G_OBJECT(source), "device", "/dev/video0", NULL); + g_object_set( + G_OBJECT(filter), "caps", + gst_caps_from_string( + "video/x-raw, width=1344, height=376, framerate=30/1, format=YUY2"), + NULL); + g_object_set(G_OBJECT(vcfilter), "caps", + gst_caps_from_string("video/x-raw, format=GRAY8"), NULL); + g_object_set(G_OBJECT(fsink), "sync", FALSE, NULL); + + g_object_set(G_OBJECT(fsink), "signal-handoffs", TRUE, NULL); + handler = g_signal_connect(fsink, "handoff", G_CALLBACK(stereo_handoff), + &srcData); + + /* add elements into the pipeline */ + gst_bin_add_many(GST_BIN(pipecam), source, filter, vconv, vcfilter, fsink, + NULL); + + /* link elements */ + gst_element_link_many(source, filter, /*queue1, vrate, vrfilter,*/ + vconv, vcfilter, fsink, NULL); + return 0; +} + +int createPipelineDepth() +{ + /* + appsrc ! video/x-raw, width=672, height=376, framerate=30/1, format=GRAY8 ! + videoconvert ! fakesink -v -e + */ + appsource = gst_element_factory_make("appsrc", "appsrc"); + asfilter = gst_element_factory_make("capsfilter", "asfilter"); + asvconv = gst_element_factory_make("videoconvert", "asvconv"); + //sink = gst_element_factory_make("tizenwlsink", "sink"); + sink = gst_element_factory_make("fakesink", "sink"); + + if (!pipedepth || !appsource || !asfilter || !asvconv || !sink) { + printf("pipeline[%p]\n", pipedepth); + printf("appsrc [%p], asfilter[%p] \n", appsource, asfilter); + printf("asvconv [%p], sink [%p] \n", asvconv, sink); + return -1; + } + + /* set up the pipeline */ + GstCaps *caps = gst_caps_from_string( + "video/x-raw, width=672, height=376, framerate=30/1, format=GRAY16_LE"); + gst_app_src_set_caps((GstAppSrc *) appsource, caps); + g_object_set( + G_OBJECT(asfilter), "caps", + gst_caps_from_string( + "video/x-raw, width=672, height=376, framerate=30/1, format=GRAY16_LE"), + NULL); + g_object_set(G_OBJECT(sink), "use-tbm", FALSE, NULL); + g_object_set(G_OBJECT(sink), "sync", FALSE, NULL); + + /* add elements into the pipeline */ + gst_bin_add_many(GST_BIN(pipedepth), appsource, asfilter, asvconv, sink, + NULL); + + /* link elements */ + gst_element_link_many(appsource, asfilter, asvconv, sink, NULL); + + return 0; +} + +int main(int argc, char *argv[]) +{ + signal(SIGINT, int_handler); + + gst_init(&argc, &argv); + GMainLoop *loop; + GstBus *bus; + guint busWatchIdCam; + guint busWatchIdDepth; + camWidth = 672; + camHeight = 376; + + isLive = true; + isAsync = true; + depth_scale = 1000.0; + depth_trunc = 200.0; + if (argc > 1) { + isLive = atoi(argv[1]) !=0 ? true : false; + depth_scale = static_cast(atof(argv[2])); + depth_trunc = static_cast(atof(argv[3])); + + isAsync = atoi(argv[4]) !=0 ? true : false; + minDisp = atoi(argv[5]); + maxDisp = atoi(argv[6]); + display_xpos = atoi(argv[7]); + display_ypos = atoi(argv[8]); + } + + utility::SetVerbosityLevel(utility::VerbosityLevel::Debug); + + SrcData srcData; + int ret = MEDIA_VISION_ERROR_NONE; + + printf("enter main\n"); + + // mediavision + // engine_config + ret = mv_create_engine_config(&engine_config); + if (ret != MEDIA_VISION_ERROR_NONE) { + return -1; + } + + ret = mv_engine_config_set_int_attribute(engine_config, MV_3D_DEPTH_WIDTH, + camWidth); + if (ret != MEDIA_VISION_ERROR_NONE) { + return -1; + } + + ret = mv_engine_config_set_int_attribute(engine_config, MV_3D_DEPTH_HEIGHT, + camHeight); + if (ret != MEDIA_VISION_ERROR_NONE) { + return -1; + } + + ret = mv_engine_config_set_int_attribute(engine_config, + MV_3D_DEPTH_MIN_DISPARITY, minDisp); + if (ret != MEDIA_VISION_ERROR_NONE) { + return -1; + } + + ret = mv_engine_config_set_int_attribute(engine_config, + MV_3D_DEPTH_MAX_DISPARITY, maxDisp); + if (ret != MEDIA_VISION_ERROR_NONE) { + return -1; + } + + ret = mv_engine_config_set_string_attribute( + engine_config, MV_3D_DEPTH_STEREO_CONFIG_FILE_PATH, + "/usr/share/capi-media-vision/stereoCalibZedVGA.yaml"); + + if (ret != MEDIA_VISION_ERROR_NONE) { + return -1; + } + + // mv3d handle + ret = mv_3d_create(&mv3d_handle); + if (ret != MEDIA_VISION_ERROR_NONE) { + return -1; + } + + ret = mv_3d_configure(mv3d_handle, engine_config); + if (ret != MEDIA_VISION_ERROR_NONE) { + return -1; + } + + ret = mv_3d_set_depth_cb(mv3d_handle, _depth_stereo_cb, nullptr); + if (ret != MEDIA_VISION_ERROR_NONE) { + return -1; + } + + ret = mv_3d_prepare(mv3d_handle); + if (ret != MEDIA_VISION_ERROR_NONE) { + return -1; + } + + // source + ret = mv_create_source(&mv_source); + if (ret != MEDIA_VISION_ERROR_NONE) { + return -1; + } + + // gstreamer + loop = g_main_loop_new(NULL, FALSE); + pipedepth = gst_pipeline_new("pipedepth"); + pipecam = gst_pipeline_new("pipecam"); + + // create depth and prepare + ret = createPipelineDepth(); + if (ret != 0) + return -1; + + srcData.src = appsource; + srcData.numFrame = 0; + // create camera + ret = createPipelineCam(srcData); + if (ret != 0) + return -1; + + bus = gst_pipeline_get_bus(GST_PIPELINE(pipecam)); + busWatchIdCam = gst_bus_add_watch(bus, bus_call, loop); + gst_object_unref(bus); + + bus = gst_pipeline_get_bus(GST_PIPELINE(pipedepth)); + busWatchIdDepth = gst_bus_add_watch(bus, bus_call, loop); + gst_object_unref(bus); + + gst_element_set_state(pipecam, GST_STATE_PLAYING); + gst_element_set_state(pipedepth, GST_STATE_PLAYING); + + g_main_loop_run(loop); + + gst_element_set_state(pipecam, GST_STATE_NULL); + gst_element_set_state(pipedepth, GST_STATE_NULL); + + gst_object_unref(GST_OBJECT(pipecam)); + gst_object_unref(GST_OBJECT(pipedepth)); + g_source_remove(busWatchIdCam); + g_source_remove(busWatchIdDepth); + g_main_loop_unref(loop); + + mv_3d_destroy(mv3d_handle); + mv_destroy_engine_config(engine_config); + mv_destroy_source(mv_source); + + return 0; +} diff --git a/test/testsuites/visualizer/CMakeLists.txt b/test/testsuites/visualizer/CMakeLists.txt new file mode 100644 index 0000000..d27cacb --- /dev/null +++ b/test/testsuites/visualizer/CMakeLists.txt @@ -0,0 +1,16 @@ +project(mv_visualizer_test_suite) +cmake_minimum_required(VERSION 2.6...3.13) + +pkg_check_modules(${PROJECT_NAME}_DEP REQUIRED) + +add_executable(${PROJECT_NAME} visualizer_test_suite.cpp) +target_include_directories(${PROJECT_NAME} PUBLIC + ${${PROJECT_NAME}_DEP_INCLUDE_DIR}) +target_link_libraries(${PROJECT_NAME} ${MV_IMAGE_LIB_NAME} + ${${PROJECT_NAME}_DEP_LIBRARIES} + ${MV_BARCODE_DETECTOR_LIB_NAME} + capi-system-info + mv_testsuite_common + mv_visualizer) + +install(TARGETS ${PROJECT_NAME} DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/test/testsuites/visualizer/bunny.h b/test/testsuites/visualizer/bunny.h new file mode 100644 index 0000000..71c7de0 --- /dev/null +++ b/test/testsuites/visualizer/bunny.h @@ -0,0 +1,1892 @@ +#define POINTS_BUNNY 1889 +float bunny[][3] = { +{-0.0369122, 0.127512, 0.00276757}, +{-0.0457707, 0.130327, 0.00306785}, +{-0.0708847, 0.149834, 0.0388672}, +{-0.00331557, 0.130403, 0.0212208}, +{-0.0211979, 0.1272, 0.00915278}, +{-0.0265255, 0.12592, 0.00874866}, +{0.0339261, 0.112038, 0.0269672}, +{0.0376485, 0.110455, 0.0145481}, +{-0.0259368, 0.111118, 0.0379115}, +{0.027952, 0.120939, 0.0215377}, +{-0.0628308, 0.155987, -0.0150105}, +{0.0390029, 0.106711, 0.0215202}, +{0.0447976, 0.0950477, 0.00866471}, +{-0.0330636, 0.173619, -0.0031267}, +{-0.0808069, 0.136243, 0.0495014}, +{-0.0705086, 0.12445, 0.0526685}, +{0.00874873, 0.131225, 0.0145345}, +{0.0401015, 0.106711, 0.00874166}, +{0.0379483, 0.100145, -0.00827134}, +{-0.0906538, 0.137201, 0.0207305}, +{-0.0841655, 0.110667, 0.0275273}, +{-0.0705214, 0.156214, 0.0144536}, +{-0.083872, 0.15212, 0.0282652}, +{0.00305028, 0.12432, 0.0332425}, +{0.00870828, 0.124165, 0.0330348}, +{-0.0328896, 0.12613, 0.00300653}, +{-0.0506302, 0.143065, 0.0150611}, +{-0.0757863, 0.13637, 0.050172}, +{-0.0027191, 0.128962, 0.0264678}, +{-0.0460961, 0.125118, 0.0263142}, +{-0.0785104, 0.0942728, -0.0109192}, +{0.0216915, 0.125373, 0.0211452}, +{-0.0888469, 0.124305, 0.00237041}, +{0.040386, 0.100825, -0.00303043}, +{-0.0884145, 0.117791, 0.00268555}, +{-0.0319074, 0.177421, -0.00491879}, +{-0.0765825, 0.143224, 0.0455148}, +{-0.0209748, 0.112544, 0.0388613}, +{-0.020836, 0.179425, -0.0221622}, +{-0.0377039, 0.167987, -0.0130391}, +{-0.0331765, 0.12681, 0.00839958}, +{0.00893926, 0.127114, 0.0292916}, +{-0.044944, 0.131083, 0.0147963}, +{-0.0266041, 0.12515, 0.00282384}, +{0.0144285, 0.12328, 0.0319185}, +{0.019244, 0.122284, 0.0308314}, +{-0.0390225, 0.167317, 0.00215527}, +{-0.08808, 0.129976, 0.00206377}, +{-0.0537203, 0.142608, 0.0266058}, +{0.043095, 0.0980072, 0.0191617}, +{0.0432138, 0.100117, 0.00866473}, +{0.0415448, 0.0944954, 0.0275695}, +{-0.0578726, 0.155337, 0.0149245}, +{-0.0231577, 0.157375, -0.0046304}, +{-0.0683123, 0.145735, 0.0420568}, +{-0.0708351, 0.142847, 0.0451248}, +{-0.070664, 0.0642894, 0.0209789}, +{-0.0761519, 0.130581, 0.0525324}, +{-0.0640036, 0.161784, -0.0208118}, +{-0.0706461, 0.155711, 0.00252406}, +{-0.0924366, 0.118434, 0.0399838}, +{-0.0635349, 0.156052, 0.0148814}, +{0.0282675, 0.118192, 0.0274382}, +{0.0392736, 0.0938857, -0.00915453}, +{-0.0695973, 0.164844, -0.0174846}, +{-0.00892354, 0.123904, 0.0330319}, +{0.0269099, 0.0942476, 0.0444911}, +{-0.0146258, 0.162377, -0.0144398}, +{-0.0450983, 0.167072, 0.00289327}, +{-0.0761536, 0.172742, -0.0384391}, +{-0.0858274, 0.105458, 0.00472318}, +{0.0370431, 0.110443, 0.0207229}, +{0.0321593, 0.0994027, 0.0380657}, +{-0.075287, 0.146433, 0.0428582}, +{-0.0395145, 0.171107, 0.000531747}, +{-0.0839586, 0.11215, 0.00148754}, +{0.0446848, 0.0883378, 0.0216285}, +{0.0161783, 0.127819, 0.0220535}, +{-0.00251635, 0.0397232, 0.0474087}, +{0.00303163, 0.0406968, 0.0460422}, +{-0.0143059, 0.128197, 0.00333856}, +{-0.0526117, 0.155596, 0.0109972}, +{-0.0332043, 0.17776, -0.00906223}, +{0.0394391, 0.106654, 0.00306577}, +{-0.0923799, 0.1249, 0.0327641}, +{0.0454681, 0.0882959, 0.0146642}, +{-0.0274495, 0.179802, -0.00925837}, +{-0.072504, 0.146297, 0.0429682}, +{-0.0579959, 0.129793, 0.0383118}, +{0.043117, 0.0923689, 0.0251649}, +{-0.00865718, 0.130323, 0.0149721}, +{-0.0141304, 0.129188, 0.0147431}, +{-0.0707877, 0.15583, 0.00921954}, +{-0.00952731, 0.127041, 0.0281475}, +{-0.0646289, 0.153404, 0.0329146}, +{-0.0706939, 0.15347, 0.0328596}, +{0.0208126, 0.118434, 0.0336393}, +{-0.0396566, 0.173008, -0.00299705}, +{-0.0442176, 0.170815, -0.00391429}, +{-0.0582565, 0.0395149, 0.0457796}, +{-0.0523033, 0.0401501, 0.04623}, +{-0.0760211, 0.161274, -0.0145891}, +{-0.0693983, 0.163016, -0.0140293}, +{0.0399663, 0.106491, 0.014952}, +{0.041536, 0.0950084, -0.00475737}, +{-0.0470079, 0.163779, 0.00528295}, +{-0.0402546, 0.161678, 0.00298655}, +{-0.0386569, 0.0389805, 0.0441153}, +{-0.0704175, 0.166991, -0.0216976}, +{-0.0254201, 0.0886622, 0.0503827}, +{-0.0886334, 0.137429, 0.00876953}, +{-0.014179, 0.12627, 0.0266417}, +{-0.0360017, 0.17408, -0.0118959}, +{-0.0886251, 0.0937834, 0.00823534}, +{-0.0648672, 0.155874, -0.00891497}, +{-0.0704508, 0.137752, -0.00774011}, +{-0.0750154, 0.166247, -0.0219558}, +{0.0299465, 0.114869, 0.0300239}, +{0.0398138, 0.0998788, 0.0273101}, +{-0.015242, 0.111698, 0.0407424}, +{-0.0700387, 0.118219, 0.0524379}, +{0.0149973, 0.112399, 0.0386082}, +{-0.036487, 0.171225, 0.000545037}, +{-0.0641664, 0.118551, -0.00968333}, +{-0.071817, 0.166979, -0.0463822}, +{-0.0913559, 0.14534, 0.0246937}, +{0.00903703, 0.112569, 0.0396571}, +{0.0324674, 0.0997396, -0.0141603}, +{0.0417911, 0.101845, 0.00188609}, +{0.00302992, 0.112517, 0.0415434}, +{-0.0650368, 0.148485, 0.0382561}, +{-0.0706519, 0.13063, 0.0502497}, +{-0.0144471, 0.128935, 0.00903509}, +{0.00292575, 0.131541, 0.00912318}, +{-0.0625682, 0.151125, 0.035875}, +{0.0349829, 0.113328, 0.0214487}, +{0.021327, 0.0385664, 0.0392992}, +{0.0145125, 0.093771, 0.0501571}, +{-0.00923752, 0.112849, 0.0413907}, +{0.0415329, 0.100906, 0.0210277}, +{0.0422859, 0.101486, 0.0146614}, +{-0.0773783, 0.112839, -0.00448759}, +{-0.078035, 0.137641, -0.00517379}, +{0.00873437, 0.106347, -0.0202193}, +{0.0090324, 0.13035, 0.0211569}, +{0.00301322, 0.130902, 0.0206741}, +{-0.00286342, 0.13115, 0.0147367}, +{-0.0391578, 0.12569, 0.0207996}, +{-0.0205725, 0.123523, 0.0265579}, +{-0.0644194, 0.155634, 0.00928477}, +{-0.0463385, 0.131411, 0.0207671}, +{-0.0532034, 0.0439067, 0.044658}, +{-0.00297651, 0.131046, 0.00884967}, +{-0.089664, 0.137755, 0.0263925}, +{-0.00888731, 0.124273, -0.00880284}, +{-0.0460971, 0.0385107, 0.0446891}, +{-0.0649255, 0.178874, -0.0579325}, +{-0.0329347, 0.124601, 0.0211235}, +{-0.0831301, 0.149901, 0.0334123}, +{-0.0895652, 0.093948, 0.0149303}, +{-0.0328901, 0.124518, -0.00282055}, +{-0.0845271, 0.106161, 0.00204328}, +{-0.0469341, 0.155816, 0.00872921}, +{0.0206202, 0.123943, 0.0267275}, +{-0.026256, 0.117499, 0.0321672}, +{-0.021392, 0.118632, 0.0336445}, +{-0.0195069, 0.116132, 0.0368525}, +{-0.0761618, 0.118382, 0.0520923}, +{0.00889281, 0.0395765, 0.0451727}, +{-0.0534736, 0.159548, 0.00753828}, +{-0.0469464, 0.161226, 0.00680216}, +{-0.0574886, 0.154862, 0.0204748}, +{0.0317199, 0.117635, 0.0202007}, +{0.0378683, 0.105514, -0.00259159}, +{-0.0811847, 0.137693, -0.00253994}, +{-0.0764348, 0.124515, 0.0528345}, +{0.0343816, 0.106104, -0.00900254}, +{0.0457922, 0.088316, 0.00867097}, +{-0.0703288, 0.0944195, -0.0159143}, +{-0.0756048, 0.0937947, -0.0135536}, +{-0.058657, 0.156369, 0.0093256}, +{-0.0637335, 0.153848, 0.00222718}, +{-0.0777278, 0.0960024, 0.0363437}, +{-0.0868519, 0.136556, 0.00309926}, +{-0.0455299, 0.0432404, 0.0432162}, +{-0.0402011, 0.045749, 0.0408051}, +{-0.0654123, 0.160403, -0.0149066}, +{-0.0318898, 0.0387174, 0.0510004}, +{-0.0267997, 0.0453977, 0.0509311}, +{-0.0271043, 0.0396972, 0.0535379}, +{-0.0215575, 0.0460868, 0.0517209}, +{-0.0143078, 0.0445295, 0.0504368}, +{-0.00981594, 0.043264, 0.0493162}, +{-0.00348436, 0.044054, 0.0472086}, +{0.009577, 0.0458139, 0.0465877}, +{0.02048, 0.111086, 0.0379569}, +{-0.0141831, 0.128547, 0.0200007}, +{-0.0526702, 0.144108, 0.0210347}, +{-0.0634838, 0.17384, -0.0527131}, +{-0.0366553, 0.171999, -0.0125745}, +{-0.0525548, 0.131228, 0.0328277}, +{-0.0659567, 0.132023, 0.0442925}, +{-0.0921726, 0.11832, 0.0267606}, +{0.0452792, 0.0882737, 0.00268175}, +{-0.00305651, 0.112889, 0.0417789}, +{-0.0451955, 0.161396, -0.00871567}, +{-0.0402914, 0.160933, -0.0115368}, +{-0.0521414, 0.0701165, 0.0389584}, +{-0.0383315, 0.093604, -0.0232581}, +{-0.0690556, 0.137374, 0.046352}, +{-0.0695996, 0.167401, -0.0516299}, +{-0.00246047, 0.124102, 0.0337609}, +{-0.0398624, 0.128204, 0.00299348}, +{-0.0753331, 0.149032, 0.0395625}, +{-0.0701432, 0.160618, -0.00917801}, +{-0.0589378, 0.0440425, 0.0434222}, +{-0.0207164, 0.126445, 0.00312493}, +{-0.00850666, 0.0467286, 0.0481052}, +{0.00300323, 0.0450308, 0.0469911}, +{-0.0802174, 0.148665, 0.0379438}, +{-0.0819961, 0.130698, 0.0513437}, +{0.00273088, 0.106333, -0.0209927}, +{-0.0757273, 0.0885687, -0.0138399}, +{-0.0698477, 0.0882875, -0.0167823}, +{-0.0668508, 0.159243, -0.0102161}, +{-0.0226988, 0.0885773, 0.0536309}, +{-0.00281419, 0.0990077, 0.0505614}, +{0.0452902, 0.0696213, 0.0253974}, +{-0.0525629, 0.0472823, 0.040482}, +{-0.046959, 0.0466581, 0.0408127}, +{-0.0691348, 0.156682, -0.00276369}, +{-0.0897599, 0.150073, 0.0220744}, +{-0.0702883, 0.155637, 0.0263654}, +{-0.0765031, 0.154893, 0.0266005}, +{-0.00804843, 0.0987379, 0.0505998}, +{0.0300791, 0.11567, -0.00430465}, +{-0.0923054, 0.117757, 0.0334441}, +{-0.0331192, 0.0449511, 0.0462474}, +{-0.0337794, 0.113308, 0.034612}, +{-0.0521291, 0.113769, 0.0349566}, +{0.0437636, 0.0825382, -0.0027974}, +{-0.0202819, 0.126016, 0.0210507}, +{0.0327872, 0.043925, 0.0295904}, +{-0.0453372, 0.155266, -0.0075525}, +{-0.0284609, 0.173987, -0.0175958}, +{0.0268448, 0.0881755, -0.0223077}, +{-0.0308231, 0.0923023, -0.0246377}, +{-0.0899732, 0.149975, 0.0141115}, +{0.0381804, 0.105121, 0.0266947}, +{0.00842001, 0.12907, 0.0258154}, +{-0.0266549, 0.0942999, -0.0265555}, +{-0.0279896, 0.0475815, 0.0485532}, +{-0.0150037, 0.048073, 0.0483203}, +{-0.00298993, 0.0473817, 0.0491102}, +{0.00376754, 0.0477551, 0.0502037}, +{0.00748504, 0.0473851, 0.0493363}, +{-0.0581651, 0.149751, 0.032858}, +{-0.0720688, 0.136456, 0.0490662}, +{-0.0810638, 0.0939541, -0.0082617}, +{0.0380863, 0.0458646, 0.0307423}, +{-0.0253234, 0.182998, -0.0108168}, +{-0.0230508, 0.183235, -0.0110157}, +{0.00323317, 0.129146, 0.0263855}, +{-0.0626125, 0.149788, -0.00343342}, +{-0.0591471, 0.0466998, 0.0395843}, +{-0.0353862, 0.0471292, 0.0414241}, +{-0.0194948, 0.0486404, 0.0485565}, +{-0.00849455, 0.0521633, 0.0517688}, +{-0.00296485, 0.051429, 0.0527827}, +{0.00279019, 0.0517664, 0.0528352}, +{0.00904034, 0.0517165, 0.051222}, +{0.0443839, 0.0943042, 0.00268377}, +{-0.0886145, 0.111113, 0.0148415}, +{-0.0885219, 0.144027, 0.0329221}, +{0.0440719, 0.0937787, 0.0206165}, +{0.0436531, 0.0980341, 0.0146596}, +{-0.0650976, 0.153799, -0.00285808}, +{-0.0517297, 0.0490759, 0.0371355}, +{-0.0331222, 0.0518259, 0.0385377}, +{-0.0377352, 0.127448, 0.0152358}, +{-0.00906608, 0.100701, 0.0460122}, +{-0.0410683, 0.128416, 0.0134054}, +{-0.0712056, 0.158724, -0.00521868}, +{-0.0266313, 0.0501544, 0.044695}, +{-0.0211065, 0.0519946, 0.0455753}, +{-0.0168667, 0.0505241, 0.0476889}, +{-0.0147601, 0.0527687, 0.050103}, +{-0.0626395, 0.149972, -0.00897733}, +{-0.090861, 0.124732, 0.00627835}, +{-0.0255786, 0.0923499, -0.0315595}, +{-0.0709738, 0.172947, -0.052768}, +{-0.0588974, 0.143232, -0.00327646}, +{-0.0943643, 0.12436, 0.0216467}, +{0.0337044, 0.112449, -0.00269877}, +{-0.0515051, 0.136557, 0.0263185}, +{-0.00886593, 0.121199, 0.0360577}, +{-0.061729, 0.155665, -0.0259512}, +{-0.0862637, 0.10567, 0.0206042}, +{-0.0895584, 0.138606, 0.032689}, +{-0.0268168, 0.123904, 0.0208113}, +{0.0341937, 0.0515433, 0.033081}, +{0.0401268, 0.0512743, 0.0322702}, +{0.0449306, 0.0526595, 0.0319582}, +{-0.0405348, 0.117168, 0.0319438}, +{-0.0636902, 0.155546, -0.0390642}, +{0.0278663, 0.100401, 0.0410064}, +{-0.0275828, 0.179275, -0.0157605}, +{-0.0758871, 0.0942302, 0.0383961}, +{0.0138371, 0.129201, 0.0203961}, +{-0.0152723, 0.0998429, 0.0451638}, +{-0.00916763, 0.129718, 0.0206646}, +{-0.0512444, 0.0516901, 0.0334801}, +{-0.0461563, 0.0523184, 0.0379981}, +{-0.0410001, 0.05272, 0.0393793}, +{-0.0270993, 0.0526642, 0.0393104}, +{0.0434924, 0.0931097, -0.00154028}, +{-0.0823819, 0.112683, 0.045427}, +{-0.092066, 0.118055, 0.00909937}, +{-0.00448884, 0.121713, 0.0362976}, +{0.0147346, 0.129423, 0.0143146}, +{-0.0158113, 0.161888, -0.00973584}, +{-0.0778838, 0.149704, -0.00337488}, +{-0.0865357, 0.12477, -0.00166991}, +{0.0153656, 0.126058, 0.0275381}, +{-0.0388913, 0.123761, 0.0249778}, +{-0.0390351, 0.121238, 0.0283673}, +{-0.0324963, 0.120237, 0.0283344}, +{-0.0149052, 0.12311, 0.0316417}, +{-0.0582873, 0.117688, 0.0386719}, +{-0.0626536, 0.161861, -0.0264031}, +{-0.0818147, 0.141639, 0.0444825}, +{0.0350734, 0.100071, 0.0345975}, +{0.0311856, 0.11215, 0.0310216}, +{-0.0335778, 0.11743, 0.031458}, +{-0.059637, 0.153475, 0.031348}, +{-0.0481256, 0.0536625, 0.0362191}, +{-0.059026, 0.156388, 0.00269852}, +{-0.0211187, 0.0578754, 0.0461125}, +{-0.082738, 0.124721, 0.050554}, +{-0.0466997, 0.11363, 0.0348133}, +{-0.0107262, 0.179662, -0.0277472}, +{0.0347725, 0.0894441, -0.0170339}, +{-0.0891825, 0.100351, 0.0148945}, +{0.0257275, 0.122894, 0.0207337}, +{-0.0883949, 0.100277, 0.00841226}, +{-0.0649858, 0.155518, 0.0263367}, +{-0.0768402, 0.154073, 0.00257877}, +{-0.0576877, 0.154146, 0.0262123}, +{-0.0266966, 0.125729, 0.0145923}, +{-0.076376, 0.155782, 0.0208875}, +{-0.0763295, 0.167188, -0.039594}, +{-0.0771877, 0.100229, -0.0103313}, +{-0.0153681, 0.0590839, 0.0519909}, +{-0.010206, 0.0576345, 0.0535443}, +{-0.00350044, 0.0578672, 0.0543757}, +{0.00300818, 0.0568916, 0.0538692}, +{0.0088308, 0.0580497, 0.0529859}, +{0.0410915, 0.0820775, -0.00893411}, +{0.0395449, 0.0576373, 0.0318985}, +{-0.0762443, 0.139336, 0.0484763}, +{-0.0324306, 0.120379, -0.00955344}, +{-0.0194451, 0.0881559, 0.0557639}, +{-0.074787, 0.159471, -0.00898201}, +{-0.0639935, 0.15611, 0.0210031}, +{-0.0762438, 0.153101, 0.0322442}, +{-0.00876679, 0.128727, 0.025102}, +{0.0282216, 0.112237, -0.00983067}, +{-0.0451341, 0.0593225, 0.0387559}, +{-0.0405005, 0.0579499, 0.040202}, +{-0.033993, 0.0584028, 0.038704}, +{-0.0272756, 0.0585468, 0.0382285}, +{-0.0248608, 0.122913, 0.0245429}, +{-0.0825276, 0.154355, 0.0206132}, +{-0.00884271, 0.129403, 0.00305159}, +{0.0207587, 0.126654, 0.0147646}, +{-0.0394868, 0.173351, -0.00839443}, +{-0.028421, 0.114019, 0.0347746}, +{-0.0193575, 0.122009, 0.0306737}, +{-0.0691626, 0.161675, -0.0514614}, +{-0.0516736, 0.15006, 0.0148119}, +{-0.0156325, 0.120151, 0.0349054}, +{0.0467454, 0.0582319, 0.0314404}, +{-0.0770165, 0.0685425, 0.0147863}, +{-0.00967101, 0.173225, -0.0264945}, +{-0.0213141, 0.184813, -0.0151112}, +{-0.0766524, 0.0882188, 0.0382876}, +{-0.0540219, 0.0521463, 0.0110698}, +{-0.0219451, 0.126821, 0.0155536}, +{-0.0820391, 0.153392, 0.0264506}, +{-0.0213183, 0.124468, -0.00290836}, +{-0.0268364, 0.123465, -0.00321538}, +{-0.0312035, 0.177796, -0.0133521}, +{-0.00749945, 0.0598042, 0.0553302}, +{-0.00108951, 0.0601245, 0.0554892}, +{0.00280202, 0.0599746, 0.0555283}, +{-0.051797, 0.118119, 0.033678}, +{0.00302464, 0.131618, 0.0149353}, +{0.0446005, 0.0942619, 0.0151198}, +{-0.0880636, 0.111855, 0.00852285}, +{-0.0704321, 0.144096, -0.0148369}, +{-0.0820967, 0.0943634, 0.0322765}, +{-0.0269642, 0.120812, 0.0275676}, +{-0.0540164, 0.149968, 0.0253393}, +{-0.0800337, 0.0995053, -0.00770139}, +{0.00922138, 0.12038, 0.0360924}, +{0.00286056, 0.117968, 0.0387331}, +{-0.0936229, 0.118494, 0.0206524}, +{-0.0409923, 0.113229, 0.035109}, +{-0.0822185, 0.154488, 0.0146661}, +{-0.0625956, 0.155202, -0.0329876}, +{-0.0462511, 0.124621, -0.00898124}, +{-0.0220336, 0.160676, -0.00426008}, +{-0.065621, 0.172767, -0.0466049}, +{-0.0762614, 0.155884, 0.0148687}, +{-0.0644988, 0.149044, -0.0265159}, +{-0.0581979, 0.0593456, 0.0210895}, +{-0.0335439, 0.122618, 0.0254024}, +{-0.0826578, 0.153434, 0.00921403}, +{-0.049999, 0.132417, 0.0286961}, +{0.0088217, 0.131096, 0.00864908}, +{-0.0154842, 0.0644282, 0.0533754}, +{-0.00871951, 0.065015, 0.0556827}, +{-0.00324815, 0.0640003, 0.0562816}, +{0.00292601, 0.0643094, 0.0563956}, +{0.00738462, 0.0651614, 0.0553402}, +{-0.0143174, 0.116971, 0.037836}, +{-0.00299223, 0.118083, 0.0390751}, +{-0.00864301, 0.117816, 0.0385662}, +{-0.0532884, 0.0571719, 0.0206631}, +{-0.0882588, 0.100387, 0.0210097}, +{-0.0324377, 0.099703, -0.0227313}, +{0.0425072, 0.0603725, 0.0302275}, +{0.0523383, 0.0580401, 0.0290457}, +{0.0413612, 0.0877503, -0.00929235}, +{-0.0581547, 0.0620148, 0.0270981}, +{-0.0530328, 0.0590503, 0.0266933}, +{-0.0477227, 0.135526, 0.0148654}, +{0.00323512, 0.0983053, 0.0504424}, +{0.0150627, 0.119642, 0.034806}, +{-0.0453373, 0.0643061, 0.0391142}, +{-0.0394097, 0.0644278, 0.0414133}, +{-0.033068, 0.0642666, 0.0396407}, +{-0.0270237, 0.0644489, 0.0395335}, +{-0.0881604, 0.149479, 0.0268507}, +{-0.0640727, 0.143434, -0.00894036}, +{0.00286033, 0.121151, 0.036139}, +{-0.0827306, 0.138152, 0.0466993}, +{-0.00261511, 0.127006, 0.030132}, +{0.0355841, 0.108498, -0.00452523}, +{0.0219203, 0.114136, 0.0356941}, +{-0.0379555, 0.161954, -0.0128021}, +{-0.0526362, 0.0643632, 0.0340621}, +{0.025874, 0.123374, 0.0143811}, +{-0.0451406, 0.131184, 0.00901599}, +{-0.075778, 0.155361, -0.00310678}, +{-0.0739145, 0.156437, -0.0274945}, +{-0.0833056, 0.100778, -0.00354288}, +{-0.0767099, 0.173942, -0.0452732}, +{0.00846106, 0.116985, 0.038033}, +{-0.0200899, 0.184788, -0.020546}, +{-0.046571, 0.120413, 0.0285524}, +{-0.0515313, 0.123718, -0.0088569}, +{0.0212116, 0.105804, -0.0171101}, +{-0.0938613, 0.124487, 0.0151416}, +{0.0414591, 0.064577, 0.0290352}, +{0.0466725, 0.0643471, 0.0285539}, +{0.0526423, 0.0634018, 0.0283831}, +{-0.0468141, 0.168322, -0.00285433}, +{-0.0869152, 0.0944156, 0.00293118}, +{-0.0773713, 0.161559, -0.0267238}, +{-0.0396095, 0.126677, -0.00334699}, +{-0.0271315, 0.0764239, 0.0455715}, +{-0.0587953, 0.107012, -0.0177177}, +{-0.0748314, 0.11156, -0.00720996}, +{-0.0642623, 0.0888181, -0.018733}, +{-0.0325172, 0.0881157, -0.0255424}, +{0.00325654, 0.0700086, 0.0561047}, +{0.0103151, 0.0636713, 0.0537558}, +{0.0432701, 0.0979967, 0.00267804}, +{-0.0708223, 0.156244, 0.021207}, +{-0.0584176, 0.0702277, 0.0384322}, +{-0.0703207, 0.112305, -0.00963846}, +{-0.0581653, 0.0881983, -0.0208369}, +{-0.0443038, 0.0877156, -0.0218942}, +{-0.0488091, 0.0660127, 0.0373959}, +{0.00269411, 0.126911, 0.030114}, +{0.0239692, 0.12105, 0.0288706}, +{-0.0469203, 0.117468, 0.0314407}, +{-0.091552, 0.143361, 0.0201623}, +{-0.0907563, 0.143859, 0.0263089}, +{-0.0495713, 0.144022, 0.00976642}, +{-0.0770934, 0.15583, -0.0147903}, +{-0.0868322, 0.105634, 0.00887573}, +{-0.082848, 0.131648, -0.00299747}, +{-0.0384249, 0.106407, -0.0201393}, +{-0.0823953, 0.118841, -0.00336022}, +{-0.0102333, 0.0876697, -0.0375101}, +{-0.00789361, 0.089842, -0.0363492}, +{-0.0579097, 0.111769, -0.0161856}, +{0.0140074, 0.105793, -0.0193841}, +{-0.00328561, 0.105435, -0.0225198}, +{-0.0409613, 0.070972, 0.0419904}, +{-0.033501, 0.0710512, 0.0409793}, +{-0.0272732, 0.0701361, 0.0410332}, +{-0.0161963, 0.127121, 0.0228897}, +{-0.0190644, 0.127936, 0.0133818}, +{-0.0149926, 0.0694778, 0.0545159}, +{-0.00932719, 0.0707313, 0.0562936}, +{-0.002994, 0.0710941, 0.0575426}, +{0.00838831, 0.0714267, 0.0556585}, +{0.0102531, 0.0693533, 0.0547665}, +{-0.0323939, 0.153399, -0.00240332}, +{0.0435981, 0.0881514, 0.0254203}, +{-0.0586529, 0.124882, -0.00781093}, +{-0.0204287, 0.107045, -0.022046}, +{-0.0382961, 0.0879422, -0.0229335}, +{-0.081573, 0.113394, -0.00173083}, +{-0.0380811, 0.154778, -0.00889149}, +{-0.00212588, 0.0889926, -0.0354677}, +{0.00904065, 0.100193, -0.0222794}, +{-0.0467068, 0.0700493, 0.0405769}, +{-0.0779974, 0.151244, 0.0352264}, +{0.0149019, 0.116126, 0.0367849}, +{-0.07603, 0.106301, -0.0087688}, +{-0.0885261, 0.137839, 0.0393964}, +{-0.0703112, 0.131278, -0.00857724}, +{0.0419377, 0.0703605, 0.0288832}, +{0.0514194, 0.0684326, 0.0256968}, +{-0.0922548, 0.124813, 0.0393757}, +{0.0135035, 0.128105, 0.0250558}, +{-0.0704618, 0.125421, -0.00881334}, +{-0.0703931, 0.118731, -0.00840961}, +{-0.0719685, 0.106305, -0.0114493}, +{-0.0646972, 0.161498, -0.0573125}, +{0.0463693, 0.0715128, 0.0216754}, +{-0.0538246, 0.153497, 0.0152346}, +{-0.0142869, 0.0724666, 0.0554243}, +{-0.0394057, 0.118512, -0.01336}, +{-0.0280509, 0.0880065, -0.0330858}, +{-0.00957701, 0.168254, -0.0212321}, +{-0.0445856, 0.167324, -0.00782662}, +{-0.0513101, 0.161594, -0.00355965}, +{-0.0702356, 0.179304, -0.0569867}, +{-0.0644695, 0.168402, -0.0398946}, +{-0.0089459, 0.130139, 0.00911776}, +{0.00219503, 0.0880369, -0.0342201}, +{-0.0268891, 0.16726, -0.0174204}, +{-0.0525985, 0.155054, -0.00368706}, +{-0.0761618, 0.131736, -0.00696723}, +{-0.0759576, 0.07099, 0.0265672}, +{-0.00875341, 0.10588, -0.02285}, +{-0.0519242, 0.1493, -0.00277595}, +{-0.016371, 0.18465, -0.0214272}, +{-0.020548, 0.0705632, 0.0520411}, +{-0.0813371, 0.120073, 0.049533}, +{-0.0625087, 0.149934, -0.0150319}, +{-0.0831098, 0.10651, 0.0273461}, +{-0.011119, 0.163582, -0.018751}, +{-0.00291057, 0.101147, 0.0456419}, +{-0.0635467, 0.0660523, 0.0318653}, +{-0.0511979, 0.0873878, -0.0217212}, +{-0.0530335, 0.0740367, 0.0417219}, +{-0.0465007, 0.0756701, 0.0421325}, +{-0.022314, 0.0760359, 0.0530306}, +{-0.0151351, 0.0764056, 0.0563566}, +{-0.00900601, 0.0766621, 0.0575852}, +{-0.00299732, 0.0767339, 0.0584651}, +{0.00347424, 0.0769755, 0.0565905}, +{0.00860763, 0.0767538, 0.0557293}, +{-0.0271239, 0.156216, -0.00302734}, +{-0.0633091, 0.16738, -0.0580906}, +{-0.0873943, 0.144225, 0.00902371}, +{-0.0626891, 0.162297, -0.0470925}, +{0.0370111, 0.110397, 0.00265294}, +{-0.0744006, 0.144062, -0.00864565}, +{-0.0244124, 0.183841, -0.0135068}, +{-0.0803381, 0.0715473, 0.0150483}, +{-0.0644528, 0.0761561, 0.040638}, +{-0.0588413, 0.0753794, 0.0421022}, +{-0.0524294, 0.077372, 0.0433357}, +{-0.0484981, 0.0769334, 0.043281}, +{-0.0414954, 0.0773856, 0.0429005}, +{-0.0395008, 0.0754808, 0.0425134}, +{-0.033488, 0.0764759, 0.0414605}, +{-0.0627838, 0.162163, -0.0530538}, +{0.0381456, 0.0881056, -0.0138675}, +{-0.0642837, 0.0396418, 0.039624}, +{-0.0526672, 0.121335, -0.010917}, +{-0.0738104, 0.162942, -0.037093}, +{-0.0490869, 0.13938, 0.00889895}, +{-0.0495771, 0.166027, -0.00171113}, +{-0.0709736, 0.161609, -0.0450808}, +{0.0251847, 0.12195, 0.0254854}, +{-0.0193615, 0.0781018, 0.0558163}, +{-0.0265458, 0.120645, -0.00911332}, +{-0.061796, 0.155741, -0.0207923}, +{-0.082476, 0.110295, 0.0324103}, +{-0.0691674, 0.156314, -0.050857}, +{-0.0622848, 0.16236, -0.0396288}, +{-0.088248, 0.113803, 0.0264606}, +{-0.0575392, 0.0787026, 0.0436363}, +{-0.0298439, 0.0782596, 0.0421168}, +{-0.0677617, 0.0876701, 0.0434928}, +{-0.0921939, 0.131884, 0.015227}, +{-0.0878987, 0.111742, 0.0209206}, +{-0.049353, 0.139298, 0.0147955}, +{-0.0327071, 0.173321, -0.0149209}, +{-0.0866298, 0.152851, 0.0149144}, +{-0.0779646, 0.100025, 0.035185}, +{-0.0935537, 0.118404, 0.0151524}, +{-0.084908, 0.10801, 0.0228537}, +{-0.0210677, 0.0821213, 0.0562096}, +{-0.0149957, 0.082187, 0.0572635}, +{-0.00899671, 0.0822178, 0.0576875}, +{-0.00299966, 0.0822055, 0.0574653}, +{0.0034748, 0.0817533, 0.0567544}, +{0.00824833, 0.082992, 0.0556315}, +{0.0102414, 0.0812949, 0.0546523}, +{-0.0398496, 0.123966, -0.00878898}, +{-0.092257, 0.124769, 0.00902091}, +{-0.0436728, 0.126191, 0.0209533}, +{-0.0820425, 0.105873, -0.00271871}, +{-0.0663016, 0.0807623, 0.0424437}, +{-0.0639939, 0.0836688, 0.0439754}, +{-0.058539, 0.0825906, 0.0439671}, +{-0.0521209, 0.0822523, 0.0446262}, +{-0.0467559, 0.0828569, 0.0439458}, +{-0.0424962, 0.0810729, 0.0423266}, +{-0.0404903, 0.0830123, 0.0430984}, +{-0.0365108, 0.0825773, 0.0434355}, +{-0.032204, 0.0824171, 0.0421121}, +{-0.0864005, 0.152981, 0.0204492}, +{-0.0235661, 0.115415, 0.0353667}, +{-0.0764871, 0.111685, 0.0461598}, +{-0.0763895, 0.14977, -0.00829972}, +{-0.0754801, 0.161855, -0.0327796}, +{-0.0285733, 0.0828247, 0.0462702}, +{-0.0862819, 0.100797, 0.0028483}, +{0.021088, 0.08242, 0.0504086}, +{-0.0801892, 0.143128, -0.00230055}, +{0.00844098, 0.124407, -0.00878569}, +{0.0147552, 0.0825883, 0.0529115}, +{-0.061995, 0.161169, -0.032654}, +{-0.0807571, 0.1525, 0.0307996}, +{-0.00295953, 0.130272, 0.00279699}, +{-0.0153619, 0.0884791, 0.0565599}, +{-0.00899729, 0.0878977, 0.0570287}, +{-0.00299611, 0.0880658, 0.0568489}, +{0.00301457, 0.0885291, 0.0562756}, +{0.00834267, 0.0873808, 0.0555541}, +{-0.00897481, 0.0941651, -0.0338408}, +{0.0314278, 0.11673, 0.0250113}, +{-0.0760602, 0.155337, 0.0093949}, +{0.0257808, 0.116776, -0.00728909}, +{-0.0646577, 0.0882843, 0.0447113}, +{-0.058996, 0.0882997, 0.0449149}, +{-0.0529958, 0.0883132, 0.0451395}, +{-0.0465421, 0.0881579, 0.0443187}, +{-0.0404961, 0.0876863, 0.0430941}, +{-0.0331792, 0.0885648, 0.04366}, +{-0.0280482, 0.0879652, 0.046363}, +{0.0150626, 0.0881784, 0.0517745}, +{0.0205955, 0.087113, 0.0492325}, +{-0.0702712, 0.0823874, 0.0409431}, +{-0.0296926, 0.0896882, -0.0286839}, +{-0.0236137, 0.179242, -0.0115629}, +{-0.0809391, 0.100029, 0.0323433}, +{-0.0928336, 0.130683, 0.0207107}, +{-0.0761771, 0.156201, -0.0204165}, +{0.0146577, 0.129396, 0.00843576}, +{0.0104845, 0.089766, 0.0542005}, +{-0.072579, 0.161253, -0.0389447}, +{-0.0322741, 0.110391, -0.0184574}, +{-0.0550172, 0.150108, 0.027792}, +{-0.071635, 0.0883254, 0.0414652}, +{-0.0424904, 0.0895336, 0.0426086}, +{0.0207945, 0.0897491, 0.0484315}, +{0.0273189, 0.118845, -0.00265658}, +{0.0285218, 0.121112, 0.0162366}, +{-0.00899735, 0.0930598, 0.0559298}, +{-0.00291176, 0.118727, -0.0144021}, +{-0.0885191, 0.113233, 0.0327948}, +{-0.0713744, 0.0938304, 0.0415269}, +{-0.0641029, 0.0935514, 0.0439488}, +{-0.0584965, 0.0944146, 0.0446213}, +{-0.0515853, 0.0939836, 0.0442383}, +{-0.0465591, 0.0937901, 0.0436103}, +{-0.0414914, 0.0942416, 0.0425268}, +{-0.0377723, 0.0933327, 0.0434889}, +{-0.0332864, 0.0945766, 0.0443868}, +{-0.0263807, 0.094318, 0.0450568}, +{-0.0141606, 0.0929618, 0.0553898}, +{-0.00319641, 0.0930898, 0.0557853}, +{0.00150357, 0.0931879, 0.0551544}, +{0.00367616, 0.0950752, 0.0535295}, +{0.00915739, 0.0941794, 0.0519212}, +{0.0216553, 0.0937794, 0.0473202}, +{-0.0702968, 0.174481, -0.045888}, +{-0.0305889, 0.168899, -0.00702359}, +{-0.0528191, 0.162649, 0.00296711}, +{-0.0890968, 0.0940104, 0.0208024}, +{-0.0626249, 0.173112, -0.0586131}, +{-0.0443835, 0.105923, -0.0201903}, +{-0.0664958, 0.0951776, 0.0424531}, +{-0.0324384, 0.126415, 0.0146752}, +{-0.0152469, 0.0961657, 0.0518098}, +{-0.0097537, 0.0960506, 0.0535818}, +{-0.00304601, 0.0963367, 0.0537791}, +{0.01642, 0.0957081, 0.0480381}, +{-0.0876548, 0.105191, 0.0148253}, +{-0.0699417, 0.0763232, 0.0381496}, +{0.0358078, 0.0958594, -0.0120328}, +{0.0374966, 0.100154, 0.031249}, +{-0.0530195, 0.150059, 0.0207323}, +{-0.0905911, 0.131765, 0.0328667}, +{-0.0709717, 0.147309, -0.0268389}, +{-0.0443321, 0.0935075, -0.0222668}, +{-0.0400911, 0.128618, 0.00909496}, +{-0.0710054, 0.100275, 0.0398128}, +{-0.0653063, 0.100124, 0.0417262}, +{-0.0589969, 0.0980495, 0.0430328}, +{-0.0529938, 0.0980631, 0.0432952}, +{-0.0469951, 0.0980659, 0.043235}, +{-0.0408476, 0.100401, 0.0414668}, +{-0.0323344, 0.0988071, 0.0435216}, +{-0.0259464, 0.0998425, 0.0438947}, +{-0.0212066, 0.0999849, 0.0444194}, +{0.00749586, 0.09835, 0.0488255}, +{0.0090271, 0.101109, 0.0469975}, +{0.0153076, 0.100008, 0.0472449}, +{0.0208175, 0.100067, 0.0453866}, +{-0.0648326, 0.131509, -0.00838673}, +{-0.0740297, 0.150832, -0.0323367}, +{-0.0932444, 0.124885, 0.026841}, +{-0.0633239, 0.169093, -0.0610358}, +{-0.0771158, 0.162488, -0.0202679}, +{-0.0585669, 0.0647555, 0.0323611}, +{0.0377689, 0.110383, 0.00969065}, +{-0.0503559, 0.0935892, -0.0218956}, +{-0.0589961, 0.101543, 0.042437}, +{-0.0516647, 0.101981, 0.0417488}, +{-0.0469248, 0.101325, 0.0421166}, +{-0.0352173, 0.101965, 0.0413638}, +{0.00285015, 0.100935, 0.0464433}, +{-0.075479, 0.150312, -0.0143808}, +{-0.078936, 0.108126, -0.00525459}, +{-0.0251472, 0.168981, -0.0187156}, +{-0.071457, 0.113692, 0.0499983}, +{-0.0747771, 0.0997536, 0.0377868}, +{-0.0902919, 0.137212, 0.0146286}, +{-0.0264568, 0.105883, 0.0411765}, +{-0.0209966, 0.1044, 0.0429589}, +{-0.0145208, 0.105597, 0.0430511}, +{-0.00899316, 0.10622, 0.0435541}, +{-0.00289533, 0.105882, 0.0438861}, +{0.00245231, 0.105621, 0.0429868}, +{0.00945613, 0.104903, 0.0439002}, +{0.0149913, 0.104769, 0.0443348}, +{-0.0772186, 0.106139, 0.0350601}, +{-0.0708601, 0.106945, 0.0381598}, +{-0.0652985, 0.106577, 0.0390805}, +{-0.0583896, 0.105623, 0.0405326}, +{-0.0529341, 0.106445, 0.0398435}, +{-0.0461638, 0.105797, 0.0404843}, +{-0.0400204, 0.106789, 0.0388993}, +{-0.03311, 0.106322, 0.0394461}, +{0.0193026, 0.10477, 0.0431964}, +{-0.0501412, 0.13774, 0.00286739}, +{0.0266104, 0.105911, 0.0384052}, +{0.0438719, 0.088439, -0.0031027}, +{-0.0590381, 0.113203, 0.0362299}, +{-0.021499, 0.107851, 0.0414162}, +{-0.0164951, 0.107881, 0.0420289}, +{0.00450524, 0.107918, 0.0419336}, +{0.00856234, 0.108229, 0.0410531}, +{0.0149994, 0.10779, 0.0412845}, +{0.0213049, 0.106041, 0.0409433}, +{-0.0336665, 0.167843, -0.00338268}, +{-0.0587789, 0.131705, -0.00671001}, +{-0.0443517, 0.100306, -0.0215281}, +{-0.0147306, 0.179604, -0.0266222}, +{0.0159582, 0.108177, -0.0177822}, +{-0.0638447, 0.138119, -0.00733006}, +{-0.0330953, 0.167861, -0.0155539}, +{-0.0643684, 0.125359, -0.00876153}, +{-0.032583, 0.161992, -0.0142418}, +{-0.068568, 0.110392, 0.0392194}, +{-0.0643494, 0.112195, 0.0388907}, +{-0.0593722, 0.112082, 0.0373875}, +{-0.0529986, 0.110472, 0.0373551}, +{-0.0468613, 0.11028, 0.0378862}, +{-0.040984, 0.110496, 0.0370883}, +{-0.0320055, 0.110468, 0.0370438}, +{-0.0074871, 0.110717, 0.042649}, +{-0.00449218, 0.110714, 0.0426582}, +{0.0250033, 0.110611, 0.0368459}, +{0.025919, 0.0995286, -0.0189206}, +{-0.06973, 0.112153, 0.0457184}, +{-0.045604, 0.148834, -0.00329924}, +{-0.0653006, 0.0947889, -0.0177657}, +{-0.0906677, 0.13318, 0.0277848}, +{-0.0331508, 0.094474, -0.0237799}, +{-0.0575764, 0.0941613, -0.0208023}, +{-0.0200586, 0.0397198, 0.0532237}, +{-0.0203685, 0.0352888, 0.051184}, +{-0.0764163, 0.125947, -0.00745144}, +{-0.0205906, 0.167551, -0.0139677}, +{0.025858, 0.116851, 0.0315289}, +{-0.0139279, 0.167191, -0.021044}, +{-0.0587481, 0.149802, -0.00133886}, +{0.0144191, 0.0395247, 0.0443396}, +{0.0332953, 0.105473, 0.0329627}, +{-0.0647461, 0.114313, -0.0115219}, +{-0.0520818, 0.0353771, 0.0449331}, +{-0.015004, 0.0392095, 0.0513548}, +{-0.0094925, 0.0384962, 0.049554}, +{-0.0638496, 0.117631, 0.0454477}, +{-0.0573025, 0.136864, 0.033162}, +{0.0189101, 0.0400942, 0.0428502}, +{-0.0508192, 0.124393, 0.0332635}, +{-0.0182623, 0.180885, -0.017743}, +{-0.0651271, 0.150343, -0.0325707}, +{0.0332966, 0.0936886, 0.0400216}, +{-0.0463011, 0.149493, 0.00833001}, +{0.00260773, 0.0354887, 0.0450013}, +{-0.0780807, 0.10971, 0.0423535}, +{-0.0542262, 0.124756, 0.0369858}, +{-0.0402584, 0.0361447, 0.0436625}, +{-0.00317483, 0.0942874, -0.0331049}, +{-0.0151032, 0.179716, -0.0207621}, +{0.026141, 0.0403246, 0.0327265}, +{-0.0640247, 0.111376, -0.0136272}, +{0.027817, 0.112309, 0.0339118}, +{-0.0586332, 0.142774, 0.0334953}, +{-0.0146622, 0.167501, -0.0154455}, +{-0.0270893, 0.167298, -0.00866399}, +{0.0152056, 0.045813, 0.0442638}, +{0.0190988, 0.0442996, 0.0429}, +{0.0215694, 0.0456112, 0.041209}, +{0.0257452, 0.0459137, 0.0381185}, +{0.0387365, 0.0944447, 0.0327088}, +{0.0287308, 0.0456722, 0.0347466}, +{-0.0151805, 0.173809, -0.0213305}, +{-0.0658903, 0.118253, 0.0498126}, +{-0.0628345, 0.093206, -0.0188544}, +{-0.0643065, 0.142451, 0.0394123}, +{-0.040079, 0.150283, 0.00280951}, +{-0.026851, 0.173268, -0.00983852}, +{-0.0207913, 0.173767, -0.0147826}, +{-0.0582334, 0.124238, 0.0403406}, +{-0.0683337, 0.131545, 0.0479709}, +{-0.0693547, 0.10637, -0.012803}, +{-0.0428668, 0.157627, 0.0050419}, +{-0.0476449, 0.130368, 0.0258834}, +{0.0379451, 0.0817167, -0.0141547}, +{0.0312298, 0.0470286, 0.0324465}, +{-0.0662284, 0.138149, 0.042896}, +{-0.0644094, 0.105575, -0.0158634}, +{0.0411271, 0.0443713, 0.0285474}, +{-0.0830031, 0.0762361, 0.0150296}, +{-0.0660167, 0.123488, 0.0501643}, +{-0.0416352, 0.155329, 0.00636435}, +{-0.0388456, 0.155994, 0.00477206}, +{-0.0551732, 0.116538, 0.0359195}, +{-0.0600069, 0.134082, 0.0369434}, +{0.0452816, 0.0453284, 0.0263124}, +{0.0513161, 0.0463154, 0.0204963}, +{-0.0106687, 0.172847, -0.0215627}, +{-0.0147735, 0.18419, -0.0259341}, +{0.0301064, 0.106776, 0.0358091}, +{-0.063709, 0.125122, 0.0457451}, +{0.0473431, 0.0499217, 0.0295077}, +{0.0497106, 0.0482066, 0.0259506}, +{0.0518484, 0.0518415, 0.0267161}, +{-0.0162732, 0.172938, -0.0174582}, +{0.0355097, 0.107304, 0.0291151}, +{-0.0552656, 0.143077, 0.0300537}, +{-0.0637191, 0.136482, 0.0388176}, +{-0.0199086, 0.161072, -0.00863325}, +{-0.0209172, 0.179282, -0.0148523}, +{0.014511, 0.0513519, 0.0474271}, +{-0.0610259, 0.126912, 0.0416133}, +{0.0539905, 0.0494141, 0.0219114}, +{0.00925922, 0.118865, -0.0148674}, +{-0.0268384, 0.162091, -0.00836699}, +{-0.0367024, 0.163198, -0.00107067}, +{-0.0336432, 0.155948, 0.00188963}, +{-0.0280966, 0.159587, 0.000483069}, +{-0.026491, 0.16163, -0.00321758}, +{0.0206613, 0.0528733, 0.0451655}, +{0.0231576, 0.0513069, 0.0414753}, +{0.0266044, 0.0526516, 0.039853}, +{0.0309772, 0.0527823, 0.0371348}, +{0.0239371, 0.103424, 0.0418106}, +{0.0568895, 0.0527484, 0.0209204}, +{-0.0664209, 0.11329, 0.0441331}, +{-0.0326789, 0.162384, -0.00243762}, +{0.0145199, 0.0932586, -0.026363}, +{-0.0543983, 0.119186, 0.0365781}, +{-0.0564272, 0.132376, 0.0357966}, +{-0.0501636, 0.142911, 0.00230897}, +{-0.043714, 0.147707, 0.0038501}, +{-0.0291346, 0.177171, -0.00534178}, +{0.0357304, 0.100363, -0.0111604}, +{0.0133943, 0.0541536, 0.0499521}, +{0.0551089, 0.0545007, 0.0253961}, +{0.0291033, 0.0572886, 0.0407089}, +{0.0585723, 0.0583402, 0.0214893}, +{-0.0740322, 0.0656952, 0.0144875}, +{-0.0749844, 0.179305, -0.0518221}, +{0.0145778, 0.0585769, 0.0501691}, +{0.0214876, 0.058332, 0.0470549}, +{0.0259507, 0.0590004, 0.0437762}, +{0.032833, 0.0585633, 0.0387158}, +{0.0358218, 0.0578374, 0.0350365}, +{0.0360585, 0.0951301, 0.0364902}, +{-0.0886806, 0.118283, 0.0459142}, +{0.0562736, 0.0586365, 0.0253398}, +{0.0303311, 0.0951295, 0.0419589}, +{-0.0222315, 0.167389, -0.0110472}, +{-0.0543257, 0.136577, 0.0307959}, +{-0.0500074, 0.150447, 0.0117579}, +{-0.0616289, 0.137406, 0.0354744}, +{-0.0319367, 0.159507, 0.00191749}, +{-0.0634458, 0.132148, 0.0406867}, +{0.0368678, 0.0921989, 0.0367449}, +{-0.0728433, 0.156137, -0.0339112}, +{0.0389872, 0.0640689, 0.0330299}, +{-0.0636611, 0.1488, -0.0205996}, +{0.0153938, 0.0648444, 0.0513036}, +{0.0213958, 0.0645506, 0.0473078}, +{0.0265105, 0.0649235, 0.0439721}, +{0.0302364, 0.0650657, 0.0415975}, +{0.0331295, 0.0642221, 0.0397381}, +{0.0367885, 0.065027, 0.0366867}, +{0.0563131, 0.0650782, 0.0252208}, +{0.0591364, 0.0644742, 0.0211357}, +{-0.0110683, 0.167098, -0.0167807}, +{-0.0605202, 0.146205, 0.0366666}, +{0.0194528, 0.0665736, 0.0491642}, +{-0.0286777, 0.158132, 0.000508817}, +{0.0253025, 0.0989569, 0.0434277}, +{-0.0349979, 0.152158, 8.20736e-05}, +{0.014665, 0.070627, 0.0528306}, +{0.0202908, 0.071041, 0.0498828}, +{0.0230702, 0.0702991, 0.0473835}, +{0.0263693, 0.0706238, 0.0441789}, +{0.0328306, 0.0707606, 0.0401362}, +{0.0368832, 0.070672, 0.0365953}, +{0.0398878, 0.0705632, 0.0325808}, +{0.0579544, 0.0694794, 0.0198345}, +{-0.0641704, 0.063724, 0.0268682}, +{-0.0919499, 0.114216, 0.0149265}, +{0.0351624, 0.0819076, -0.0172502}, +{-0.0862408, 0.119271, -0.00117534}, +{-0.0294401, 0.174958, -0.00579982}, +{-0.0175288, 0.165418, -0.0114925}, +{-0.0617869, 0.117789, 0.0409144}, +{0.0301891, 0.0723658, 0.0418804}, +{-0.0822099, 0.149486, 0.00288044}, +{-0.0760271, 0.175704, -0.0506937}, +{-0.0652343, 0.0614738, 0.0211346}, +{-0.0266574, 0.110394, -0.019007}, +{-0.0813538, 0.0779161, 0.0268055}, +{0.021417, 0.118723, -0.00893569}, +{0.0149346, 0.0759297, 0.0536191}, +{0.0209886, 0.0761609, 0.0506055}, +{0.0268396, 0.0762089, 0.0459193}, +{0.0336785, 0.0760737, 0.0405166}, +{0.0373422, 0.0760306, 0.0366776}, +{0.0400324, 0.0763062, 0.0328345}, +{0.0419048, 0.076876, 0.0296092}, +{0.0438094, 0.0763805, 0.0258638}, +{-0.0452412, 0.118472, -0.0142046}, +{0.0456773, 0.0768089, 0.0208187}, +{-0.050165, 0.137714, 0.0207618}, +{-0.00327054, 0.111563, -0.0203549}, +{-0.0483236, 0.145111, 0.00757835}, +{0.0310833, 0.0775315, 0.0432282}, +{-0.046855, 0.145222, 0.00288431}, +{-0.0141716, 0.10541, -0.0225802}, +{0.0470348, 0.0753979, 0.0148736}, +{-0.0611433, 0.140542, 0.0356184}, +{0.0272779, 0.0823714, 0.0459243}, +{0.0309212, 0.08255, 0.0430252}, +{0.0343037, 0.0825412, 0.0402907}, +{0.0370354, 0.0824663, 0.0369099}, +{-0.0799946, 0.147989, -0.000835337}, +{-0.0774435, 0.0690153, 0.00961977}, +{0.0404363, 0.0826995, 0.0326021}, +{0.0417479, 0.0827335, 0.0302524}, +{0.0436887, 0.0825508, 0.0263844}, +{0.0454407, 0.0825465, 0.0207137}, +{-0.0822812, 0.116295, 0.0482855}, +{-0.0844726, 0.0947391, -0.00345192}, +{-0.020271, 0.168003, -0.0193935}, +{-0.0742716, 0.0668501, 0.0190414}, +{0.026747, 0.0882417, 0.0458314}, +{0.0308722, 0.0882572, 0.0430146}, +{0.0344922, 0.0883047, 0.0403697}, +{0.0372481, 0.0881263, 0.0366393}, +{0.039927, 0.088094, 0.0326668}, +{0.0419027, 0.0877782, 0.0290815}, +{0.00264738, 0.112302, -0.019871}, +{-0.0703315, 0.1455, -0.0205576}, +{-0.0749446, 0.137879, -0.00653312}, +{-0.0266967, 0.114299, -0.0159903}, +{-0.0869924, 0.113518, 0.00410409}, +{-0.0142186, 0.174013, -0.0259807}, +{-0.0221564, 0.157852, -0.00861651}, +{-0.011587, 0.164129, -0.0163045}, +{-0.00997381, 0.169338, -0.0247765}, +{-0.082875, 0.143405, 0.00186692}, +{0.0203757, 0.0354405, -0.00287175}, +{0.0191274, 0.0363337, -0.00917714}, +{0.0184456, 0.036388, -0.013479}, +{0.0149535, 0.0347732, -0.0154937}, +{0.0221204, 0.0372026, 0.0342324}, +{0.039271, 0.0382866, 0.00854708}, +{0.0397549, 0.0398545, 0.002614}, +{0.0221892, 0.0380614, -0.00446361}, +{0.0179901, 0.0369066, -0.0161835}, +{0.0154148, 0.0392444, -0.0212861}, +{0.0208023, 0.100118, -0.0213392}, +{0.0446004, 0.0409064, 0.00927401}, +{0.0435625, 0.0411355, 0.00427044}, +{0.0381381, 0.0411139, -0.00147908}, +{-0.0478807, 0.135207, 0.00885778}, +{0.0217274, 0.0404287, -0.00964433}, +{0.0206744, 0.0405956, -0.0144437}, +{0.0192578, 0.0411681, -0.0195074}, +{-0.0885736, 0.112913, 0.0395856}, +{-0.026793, 0.106457, -0.0218501}, +{0.0481487, 0.0428585, 0.0145594}, +{0.0521212, 0.0461655, 0.0089655}, +{0.0480438, 0.0430647, 0.00724585}, +{0.0460936, 0.0434131, 0.00284357}, +{0.0285003, 0.100485, -0.0168103}, +{0.0269462, 0.0395833, -0.00334578}, +{-0.0907856, 0.117838, 0.00647331}, +{-0.062721, 0.167567, -0.0470628}, +{-0.0799532, 0.106813, 0.0316838}, +{0.0527437, 0.0462125, 0.0139554}, +{0.0504533, 0.0466263, 0.00264513}, +{-0.0322581, 0.117324, -0.0133273}, +{0.0272475, 0.0455966, -0.00927071}, +{-0.0146455, 0.0942084, -0.0337341}, +{-0.0411545, 0.16722, -0.010818}, +{-0.0721385, 0.156112, -0.0384102}, +{0.0456803, 0.0474217, -0.00311192}, +{0.0239407, 0.0433254, -0.00969837}, +{0.021084, 0.0462585, -0.0205303}, +{-0.0348527, 0.0351549, -0.0307351}, +{-0.0699867, 0.0663066, 0.0259153}, +{-0.0747071, 0.149891, -0.0201453}, +{-0.0845448, 0.13725, 0.000743181}, +{0.0549514, 0.0484178, 0.0163982}, +{0.0264565, 0.0466261, -0.0141039}, +{0.0225276, 0.0444655, -0.0157683}, +{0.0330538, 0.0938135, -0.0160538}, +{0.0526476, 0.0694992, 0.00297306}, +{0.0528544, 0.0581339, -0.00277966}, +{-0.0571464, 0.0671799, 0.0361705}, +{-0.0651544, 0.157167, -0.0515491}, +{-0.0493189, 0.133682, 0.00119868}, +{-0.032962, 0.10595, -0.0206729}, +{-0.0649538, 0.155656, -0.045631}, +{-0.0390456, 0.150445, -0.00354536}, +{0.0574365, 0.051618, 0.0145183}, +{0.0574129, 0.0522531, 0.00903377}, +{0.0536112, 0.0500965, 0.00204174}, +{0.0512204, 0.0520121, -0.00218354}, +{0.0471226, 0.0515811, -0.00481298}, +{0.033443, 0.047576, -0.0063817}, +{0.00280933, 0.118297, -0.0158208}, +{-0.0147841, 0.10125, -0.0238408}, +{-0.0620037, 0.167422, -0.0527165}, +{0.0559147, 0.0528382, 0.00339683}, +{0.0334801, 0.0518506, -0.00825293}, +{0.0287814, 0.0501171, -0.0157926}, +{0.0256197, 0.0485542, -0.0190548}, +{-0.00863537, 0.118406, -0.0146114}, +{-0.0148322, 0.117675, -0.014701}, +{-0.0615138, 0.145712, -0.00481276}, +{0.0232531, 0.12083, -0.00456186}, +{-0.0401535, 0.0342718, -0.0275149}, +{0.0302657, 0.0496868, -0.0107289}, +{0.0320066, 0.111334, -0.00737407}, +{-0.0211003, 0.120417, -0.0102482}, +{-0.0204991, 0.117125, -0.0140803}, +{-0.00910263, 0.0383602, -0.025776}, +{-0.0525144, 0.11229, -0.0171034}, +{0.0202353, 0.123713, -0.00247094}, +{-0.0701749, 0.0347541, -0.0017891}, +{-0.00340266, 0.114844, -0.0176928}, +{0.0310248, 0.053713, -0.0140522}, +{0.0268191, 0.0528482, -0.020339}, +{-0.0147458, 0.120673, -0.0105853}, +{0.0270905, 0.106214, -0.0146756}, +{0.0465541, 0.0697991, 0.00228503}, +{-0.00300122, 0.100676, -0.0235814}, +{-0.0755874, 0.076212, 0.033468}, +{0.059738, 0.0572998, 0.0151736}, +{0.0595394, 0.0578717, 0.00861672}, +{0.0572091, 0.0580526, 0.00253507}, +{-0.0142907, 0.123147, -0.00746744}, +{0.0211831, 0.112303, -0.0140834}, +{0.0347455, 0.0565046, -0.010714}, +{0.0249138, 0.0825163, -0.0245877}, +{-0.0382227, 0.114521, -0.016178}, +{-0.0819485, 0.0761672, 0.0208322}, +{-0.0269557, 0.0392251, -0.0293943}, +{0.0377037, 0.0593401, -0.00852013}, +{0.0330295, 0.0586306, -0.014729}, +{0.0218121, 0.0515865, -0.0236492}, +{-0.0204953, 0.0935908, -0.0331675}, +{-0.0872217, 0.113521, 0.0440666}, +{-0.0271537, 0.0351608, 0.0509267}, +{-0.0503825, 0.106302, -0.0194598}, +{0.0266611, 0.0585067, -0.0219134}, +{0.00975018, 0.0945932, -0.0280451}, +{-0.0205524, 0.122391, -0.00754739}, +{-0.0668021, 0.0909191, -0.0174744}, +{-0.0856155, 0.0942099, -0.00109094}, +{-0.0915274, 0.11444, 0.0204492}, +{-0.0909048, 0.131701, 0.00809159}, +{0.0404851, 0.0578886, -0.0051698}, +{0.0295964, 0.0580473, -0.0178274}, +{0.0266986, 0.0941359, -0.0205949}, +{-0.0677104, 0.172869, -0.0572602}, +{0.0142001, 0.118043, -0.013917}, +{-0.0698171, 0.0699687, 0.0326375}, +{0.0607097, 0.0648802, 0.0151632}, +{0.0609346, 0.0630505, 0.0131585}, +{0.0602205, 0.0643718, 0.00864139}, +{0.0574055, 0.0638877, 0.00271573}, +{-0.0797793, 0.103858, -0.00660016}, +{-0.0563867, 0.137359, -0.00421998}, +{0.0344512, 0.0638263, -0.0152012}, +{0.0307139, 0.0605317, -0.0184589}, +{0.0185684, 0.121789, -0.00725624}, +{-0.0456617, 0.112414, -0.0169658}, +{0.0456177, 0.0644845, -0.00162168}, +{-0.0584268, 0.0349015, 0.0441202}, +{-0.0747982, 0.0723674, 0.0308514}, +{-0.0699373, 0.0621854, 0.0151778}, +{-0.052889, 0.136519, -0.00170821}, +{0.0410205, 0.0644886, -0.00476733}, +{0.0388712, 0.0646166, -0.00976797}, +{0.0514871, 0.0637279, -0.00174794}, +{-0.0787297, 0.0744551, 0.0267421}, +{-0.0850281, 0.144269, 0.00618082}, +{0.0313094, 0.064487, -0.0188936}, +{0.0267274, 0.0646171, -0.0220842}, +{0.0318737, 0.0877439, -0.0192705}, +{-0.0772455, 0.143995, -0.00470939}, +{0.0132576, 0.110443, -0.0183541}, +{-0.00289343, 0.124723, -0.00863032}, +{-0.0342868, 0.038582, 0.0485461}, +{0.0200397, 0.0876233, -0.0261205}, +{0.0585453, 0.0705354, 0.0146976}, +{0.0581405, 0.0699819, 0.00856199}, +{0.056099, 0.069436, 0.00424359}, +{0.0370479, 0.0665186, -0.0132637}, +{-0.062561, 0.172971, -0.0616721}, +{-0.0702718, 0.15494, -0.0455472}, +{-0.0916259, 0.130499, 0.00930481}, +{-0.070021, 0.148229, -0.0328231}, +{-0.0721274, 0.0680183, 0.0267753}, +{-0.0745337, 0.15067, -0.0264303}, +{0.0431087, 0.0713461, -0.002764}, +{0.0421659, 0.0692525, -0.00466106}, +{0.0345404, 0.0699378, -0.0160391}, +{-0.0342368, 0.122912, -0.00708584}, +{0.0401518, 0.070932, -0.00951127}, +{0.0370706, 0.0707408, -0.013301}, +{0.0310856, 0.0702175, -0.0192905}, +{0.0283004, 0.0705453, -0.0222447}, +{-0.00859023, 0.101699, -0.0237897}, +{-0.0328234, 0.0400139, -0.029875}, +{-0.0830588, 0.11047, 0.0397334}, +{0.0142724, 0.123237, -0.00806485}, +{-0.0760443, 0.108637, 0.0389078}, +{-0.0732762, 0.154939, -0.0321392}, +{0.0160324, 0.0889232, -0.0282477}, +{-0.0901677, 0.131361, 0.0394374}, +{0.0455828, 0.0768365, 0.00270178}, +{-0.0516717, 0.0553965, 0.014906}, +{-0.0376545, 0.121002, -0.0109724}, +{0.0466318, 0.0762885, 0.00910629}, +{0.0437303, 0.0769241, -0.00295564}, +{0.0405043, 0.0766784, -0.0084913}, +{0.0369463, 0.0762836, -0.0128837}, +{0.0349351, 0.0766648, -0.0155944}, +{0.0319237, 0.0763904, -0.0194186}, +{0.0285208, 0.0758075, -0.0225233}, +{-0.0646857, 0.068809, 0.0348219}, +{-0.00335573, 0.0986136, -0.0269283}, +{-0.0383606, 0.100112, -0.0217661}, +{-0.0705433, 0.149897, -0.0387319}, +{-0.0247871, 0.179215, -0.0188356}, +{0.00339058, 0.0937023, -0.0318365}, +{-0.09099, 0.142689, 0.0226645}, +{-0.0851088, 0.102115, 0.000391121}, +{0.00299202, 0.124707, -0.00864775}, +{-0.0649459, 0.167336, -0.0329944}, +{0.045975, 0.0827243, 0.0146716}, +{0.0461931, 0.0827376, 0.00867911}, +{0.0453461, 0.0826602, 0.00269811}, +{0.032594, 0.082231, -0.0190597}, +{-0.0707752, 0.142011, -0.00901143}, +{-0.0396694, 0.045239, -0.0210351}, +{-0.0736488, 0.145787, -0.0131048}, +{-0.0661855, 0.1779, -0.0529018}, +{-0.0698006, 0.179227, -0.0517285}, +{-0.0719677, 0.177848, -0.0474604}, +{-0.0131817, 0.0974247, 0.0509808}, +{-0.0760529, 0.177651, -0.0471457}, +{-0.0875274, 0.149451, 0.00937476}, +{-0.0847504, 0.149536, 0.00652369}, +{-0.0853843, 0.0980412, -0.000554198}, +{-0.070162, 0.172945, -0.0393132}, +{-0.0669053, 0.17136, -0.0404187}, +{-0.0915765, 0.114644, 0.0108349}, +{0.0311175, 0.116345, -0.00142056}, +{-0.09039, 0.144074, 0.0142555}, +{0.0533752, 0.0724173, 0.00805773}, +{0.0348115, 0.113636, 0.00289967}, +{0.0321047, 0.117128, 0.00373672}, +{-0.0558554, 0.16013, 0.00226313}, +{0.0284127, 0.12005, 0.00266093}, +{-0.0693417, 0.151526, -0.0443255}, +{0.0509143, 0.0733396, 0.0112131}, +{0.0485286, 0.0726358, 0.00856732}, +{0.0251471, 0.122517, 0.00254898}, +{-0.0684168, 0.170157, -0.0319531}, +{-0.071028, 0.171274, -0.0325886}, +{-0.0765634, 0.155757, -0.00874762}, +{0.0525206, 0.0734678, 0.0148876}, +{0.035521, 0.113454, 0.00908801}, +{0.0208324, 0.125627, 0.00327965}, +{-0.0476722, 0.134348, 0.0194434}, +{-0.0746083, 0.171229, -0.0326516}, +{0.0322027, 0.117616, 0.0093642}, +{0.0162523, 0.127588, 0.00132734}, +{-0.0914669, 0.142805, 0.0167223}, +{0.0290775, 0.120474, 0.00686894}, +{0.0135909, 0.12914, 0.00336546}, +{-0.0861635, 0.100458, 0.025719}, +{-0.0653051, 0.165945, -0.0269849}, +{-0.0698492, 0.16889, -0.0268648}, +{-0.07827, 0.167473, -0.032496}, +{0.0215557, 0.0945234, -0.0226594}, +{0.0260612, 0.123082, 0.00873766}, +{0.00920342, 0.130081, 0.00248247}, +{-0.0709934, 0.170517, -0.0295248}, +{-0.0760202, 0.167938, -0.0272636}, +{0.0525229, 0.0716654, 0.0211203}, +{0.0207167, 0.126566, 0.00922145}, +{-0.0746025, 0.0998033, -0.0126456}, +{-0.0864333, 0.0890874, 0.0257055}, +{0.0354941, 0.113435, 0.0150848}, +{0.0320737, 0.117698, 0.0146262}, +{0.00294754, 0.130714, 0.00292443}, +{-0.0256391, 0.0823957, 0.0519489}, +{-0.0666258, 0.165416, -0.0221631}, +{-0.0804177, 0.153092, 0.00488677}, +{-0.0645623, 0.0350017, 0.0151892}, +{-0.0627936, 0.0352479, 0.02012}, +{-0.0642932, 0.0349381, 0.0264604}, +{-0.0642421, 0.0397497, 0.0267659}, +{-0.0652419, 0.0352202, 0.0324357}, +{-0.06432, 0.0352261, 0.0387914}, +{-0.0869014, 0.0944088, 0.0260869}, +{-0.026376, 0.100403, -0.0237519}, +{-0.0704394, 0.0348288, 0.00888692}, +{-0.0696375, 0.039673, 0.0091864}, +{-0.0678064, 0.035728, 0.013362}, +{-0.0778433, 0.0819732, 0.0354617}, +{-0.0809318, 0.0827942, 0.0325}, +{-0.0712316, 0.038974, 0.00275642}, +{-0.0616101, 0.0379618, 0.0219344}, +{-0.0653778, 0.0407054, 0.0323415}, +{-0.0612949, 0.040108, 0.0438783}, +{-0.0748891, 0.0826916, 0.0381154}, +{-0.0841641, 0.133769, 0.0486564}, +{-0.0849106, 0.0945271, 0.0290479}, +{-0.082994, 0.144712, 0.0404065}, +{-0.0265479, 0.117619, -0.0132781}, +{-0.0679678, 0.0383221, 0.0123903}, +{-0.0639259, 0.0401146, 0.0151101}, +{-0.0588527, 0.0407802, 0.0202136}, +{-0.0869621, 0.135589, 0.0440584}, +{-0.038827, 0.0398484, 0.042564}, +{-0.0253238, 0.0773437, 0.0501603}, +{0.00864855, 0.111878, -0.0192252}, +{-0.0625014, 0.04424, 0.0388616}, +{-0.088493, 0.125258, 0.0461673}, +{0.0150785, 0.10107, -0.0220372}, +{-0.0810533, 0.0876325, 0.0334622}, +{-0.0636602, 0.0439221, 0.0322355}, +{-0.0823757, 0.12585, -0.00459555}, +{-0.0374554, 0.042873, 0.0429512}, +{-0.031328, 0.0432863, 0.0501185}, +{-0.0841802, 0.0875016, 0.0285815}, +{-0.0690099, 0.0427216, 0.00298087}, +{-0.0690323, 0.0427133, 0.00739115}, +{-0.0642007, 0.0449178, 0.00895163}, +{-0.0630005, 0.0427497, 0.0133004}, +{-0.0580777, 0.0444032, 0.0143596}, +{-0.087476, 0.130712, 0.0458544}, +{-0.0837712, 0.0999337, 0.029339}, +{-0.083719, 0.0822846, 0.0270932}, +{-0.0209183, 0.0934772, 0.0512134}, +{-0.0868983, 0.142651, 0.0383505}, +{-0.0588984, 0.0467651, 0.00989959}, +{-0.0529144, 0.0464475, 0.0158024}, +{-0.0881654, 0.0882094, 0.0209192}, +{-0.0494075, 0.165901, 0.000731671}, +{-0.0586114, 0.0473978, 0.0337061}, +{-0.05614, 0.0517476, 0.00835186}, +{-0.0865231, 0.148073, 0.0321271}, +{-0.0308497, 0.0493297, 0.0429654}, +{-0.0769102, 0.114994, 0.0501188}, +{-0.0209065, 0.0959579, 0.0474195}, +{-0.0509947, 0.0509637, 0.0150799}, +{0.00842415, 0.0889657, -0.0320537}, +{-0.0240561, 0.0544386, 0.0416973}, +{-0.0510392, 0.0524223, 0.0203213}, +{-0.0526208, 0.0518271, 0.027021}, +{-0.0504022, 0.0591186, 0.0326891}, +{-0.0478821, 0.0590694, 0.0363134}, +{-0.0239128, 0.0586553, 0.0421308}, +{-0.0759314, 0.119228, -0.00697007}, +{-0.0183181, 0.0604564, 0.0506182}, +{-0.0298441, 0.0972531, -0.0235715}, +{-0.0241926, 0.0628773, 0.0422936}, +{-0.0223998, 0.06467, 0.045979}, +{-0.0192899, 0.0641483, 0.0503928}, +{-0.0260109, 0.172925, -0.0191453}, +{-0.0265331, 0.161574, -0.0144318}, +{-0.0558556, 0.15572, -0.00121016}, +{-0.0599028, 0.136466, -0.0064456}, +{-0.063538, 0.071665, 0.0379463}, +{-0.0200417, 0.0869862, -0.0378876}, +{-0.0557176, 0.105745, -0.0186241}, +{-0.0530691, 0.143914, -0.00100898}, +{-0.0256688, 0.0704637, 0.0438935}, +{-0.0235577, 0.0693774, 0.0470203}, +{-0.0525759, 0.127247, -0.00521525}, +{-0.0787859, 0.131858, -0.00545913}, +{-0.0580212, 0.120088, -0.0102747}, +{-0.0396294, 0.110441, -0.0186258}, +{-0.0210282, 0.173113, -0.0214922}, +{-0.0547593, 0.0563289, 0.0107147}, +{-0.0435534, 0.0345758, -0.024752}, +{-0.0449833, 0.0346921, -0.0207483}, +{-0.0443576, 0.0390403, -0.0217491}, +{-0.0462855, 0.0345037, -0.0153112}, +{-0.046619, 0.0396457, -0.0141457}, +{-0.00904923, 0.0343826, -0.0246429}, +{0.00311748, 0.100303, -0.0227929}, +{-0.0690809, 0.0392217, -0.00181724}, +{-0.0920289, 0.131041, 0.0262349}, +{-0.043414, 0.0372487, -0.0253064}, +{0.0280974, 0.0818294, -0.0220931}, +{-0.067702, 0.169446, -0.0560134}, +{-0.0915377, 0.129674, 0.0312365}, +{-0.0663086, 0.0411162, -0.00443149}, +{-0.0731255, 0.151935, -0.0368879}, +{-0.0390145, 0.0394889, -0.027598}, +{-0.0637372, 0.0437827, -0.00264533}, +{-0.0605427, 0.0425565, 0.0246975}, +{-0.0857603, 0.130763, -0.000714461}, +{-0.0520472, 0.0403573, -0.0107411}, +{-0.0568522, 0.0434504, 0.0224413}, +{-0.043239, 0.0429342, -0.0193166}, +{-0.0438787, 0.0441322, -0.0144222}, +{-0.0457505, 0.046486, -0.0105694}, +{-0.0645938, 0.0456897, 0.00313082}, +{-0.0525978, 0.0464843, 0.0207116}, +{-0.0572578, 0.0459489, 0.026887}, +{-0.0618962, 0.0443648, 0.0286813}, +{-0.0331467, 0.0453179, -0.0267282}, +{-0.0377669, 0.0443547, -0.0252099}, +{-0.0320922, 0.114425, -0.0162304}, +{-0.0578027, 0.0470669, -0.0032674}, +{-0.0914954, 0.147994, 0.0205137}, +{-0.0400067, 0.0471536, -0.0151042}, +{0.00454895, 0.121869, -0.0124797}, +{0.0151282, 0.112708, -0.0165496}, +{-0.0525787, 0.0463291, -0.00775444}, +{-0.0599276, 0.0475112, 0.00267117}, +{-0.0726458, 0.147126, -0.0218625}, +{-0.0740924, 0.168686, -0.0440312}, +{-0.057494, 0.0515426, 0.00319413}, +{-0.0536918, 0.0483048, 0.0264945}, +{-0.0147156, 0.114453, -0.0172255}, +{-0.0335191, 0.0480424, -0.021246}, +{0.019461, 0.0924333, -0.0244344}, +{0.0169402, 0.0952065, -0.0238278}, +{0.0201047, 0.104156, -0.0188197}, +{-0.0319642, 0.0516657, -0.0152509}, +{-0.0368448, 0.0488256, -0.0131071}, +{-0.0391265, 0.0518909, -0.0109467}, +{-0.00892221, 0.111576, -0.0202733}, +{-0.0515659, 0.0515158, -0.00751393}, +{-0.0557028, 0.05294, -0.00268598}, +{-0.0293421, 0.0526398, -0.0213991}, +{-0.0314453, 0.0496351, -0.0193539}, +{0.0322381, 0.10409, -0.0128482}, +{-0.0261025, 0.0525801, -0.0264669}, +{-0.0583031, 0.116733, -0.0130038}, +{-0.014851, 0.111599, -0.0191484}, +{-0.0521348, 0.118189, -0.0137451}, +{-0.0517493, 0.0582798, -0.00896954}, +{-0.0561982, 0.0582462, -0.00310645}, +{-0.0587989, 0.0586119, 0.00276734}, +{-0.0585564, 0.0578416, 0.00857596}, +{0.019026, 0.11614, -0.0131686}, +{-0.0211893, 0.111662, -0.0190883}, +{-0.0239176, 0.0561149, -0.030057}, +{-0.0272603, 0.058548, -0.027478}, +{-0.0295766, 0.0582799, -0.0217551}, +{-0.0320928, 0.0589382, -0.0147618}, +{0.0073938, 0.121789, -0.0126555}, +{-0.0251946, 0.0595227, -0.0308632}, +{-0.0307167, 0.06013, -0.0194181}, +{-0.0650113, 0.0632174, -0.00293095}, +{-0.0696479, 0.065751, -0.00198101}, +{-0.0699926, 0.0635013, 0.00374106}, +{-0.0799435, 0.0724812, 0.0191514}, +{-0.0676844, 0.160922, -0.0559942}, +{-0.0215435, 0.0636559, -0.0350431}, +{-0.0258325, 0.0648252, -0.0322087}, +{-0.028982, 0.0636438, -0.0274997}, +{-0.0304226, 0.0629368, -0.0224261}, +{-0.0319042, 0.0651819, -0.0201942}, +{-0.0332741, 0.0636337, -0.0160032}, +{-0.0205547, 0.034111, -0.026401}, +{-0.0743367, 0.0658286, 0.00833126}, +{0.016103, 0.120745, -0.0103843}, +{-0.0770212, 0.0700544, 0.00316631}, +{-0.0748219, 0.06693, 0.00451345}, +{-0.0306317, 0.0657524, -0.025453}, +{-0.0711433, 0.0687078, -0.00390291}, +{-0.0762625, 0.0716316, -0.00295918}, +{-0.0802204, 0.0713935, 0.00991267}, +{-0.0913413, 0.148143, 0.0161458}, +{-0.0273736, 0.0700052, -0.0335323}, +{-0.0300274, 0.0692073, -0.0289677}, +{-0.0316277, 0.0711218, -0.0266514}, +{-0.0330629, 0.0699765, -0.0212743}, +{-0.0353642, 0.0705896, -0.0177097}, +{-0.0587004, 0.0391044, -0.0090027}, +{-0.0697696, 0.0703857, -0.00808666}, +{-0.0804832, 0.0726462, 0.00472466}, +{0.0151616, 0.126104, -0.00266395}, +{-0.0745721, 0.072883, -0.00757069}, +{-0.0823908, 0.076277, 0.00270117}, +{-0.0912831, 0.133698, 0.0142161}, +{0.00371049, 0.0968817, -0.0280931}, +{-0.0761392, 0.0766258, -0.00859487}, +{-0.0784749, 0.0748827, -0.00523624}, +{-0.0806781, 0.0771902, -0.00290803}, +{-0.0834622, 0.0765209, 0.00927112}, +{0.00983826, 0.11402, -0.0178612}, +{0.00210649, 0.0981565, -0.0261244}, +{-0.0285085, 0.0757575, -0.0348118}, +{-0.0330874, 0.0761249, -0.0270661}, +{-0.0346568, 0.0757906, -0.0215029}, +{0.0231104, 0.0892807, -0.0240236}, +{-0.0312132, 0.0771357, -0.0320416}, +{-0.0700425, 0.0763633, -0.0141464}, +{-0.0861137, 0.0814707, 0.00908143}, +{-0.086319, 0.08152, 0.0149936}, +{-0.0208042, 0.0963182, -0.0270563}, +{-0.0211078, 0.114391, -0.0171285}, +{-0.0746162, 0.0828529, -0.0139325}, +{-0.077295, 0.081216, -0.0100568}, +{-0.0800127, 0.0821605, -0.00722237}, +{-0.0826334, 0.0820868, -0.00324616}, +{-0.0844667, 0.0817669, 0.00249573}, +{-0.0860445, 0.0832591, 0.0203255}, +{-0.084816, 0.0816746, 0.0219849}, +{0.0545549, 0.0661692, 0.000765649}, +{-0.0331604, 0.0828369, -0.0270493}, +{-0.0358028, 0.0829047, -0.0227723}, +{-0.0861942, 0.0842505, 0.00298565}, +{-0.0287072, 0.0827267, -0.0349537}, +{-0.0311601, 0.0822387, -0.0315627}, +{-0.085403, 0.141865, 0.00516647}, +{-0.0785169, 0.0885628, -0.0107607}, +{-0.0807046, 0.0887676, -0.00826584}, +{-0.0843972, 0.0878743, -0.00349923}, +{-0.0855708, 0.0882073, -0.00109946}, +{-0.0876157, 0.0881286, 0.00369184}, +{-0.0885339, 0.0876942, 0.00897158}, +{-0.0885791, 0.0877213, 0.0149616}, +{-0.0643854, 0.0348576, -0.00775085}, +{-0.0512932, 0.034227, -0.0129013}, +{-0.0266839, 0.0458556, -0.027274}, +{-0.0146368, 0.0981541, -0.0264318}, +{-0.0213468, 0.10077, -0.0239588}, +{0.020932, 0.0825954, -0.0267347}, +{0.00759225, 0.0928541, -0.0309237}, +{-0.0144478, 0.0879274, -0.0380297}, +{-0.00859724, 0.11451, -0.0173132}, +{0.0264818, 0.109935, -0.0126182}, +{-0.0145855, 0.0385179, -0.0267991}, +{-0.0330054, 0.0337044, -0.0272991}, +{-0.0267872, 0.0340475, -0.0271901}, +{-0.00849157, 0.0985859, -0.0270535}, +{-0.0110954, 0.120824, -0.0120135}, +{0.0367379, 0.0925992, -0.0129888}, +{-0.0571635, 0.0435755, -0.00717607}, +{-0.0193328, 0.0979251, -0.024792}, +{-0.0203798, 0.0385467, -0.0283088}, +{-0.0587681, 0.0337133, -0.00871891}, +{-0.0517919, 0.100655, -0.0213258}, +{0.00702627, 0.0978418, -0.0246055}, +{-0.0148892, 0.126068, -0.00252126}, +{0.0307578, 0.092446, -0.0188519}, +{0.0211049, 0.0578126, -0.0266116}, +{-0.0169237, 0.0970481, -0.0278718}, +{0.0460004, 0.0581866, -0.00508589}, +{-0.00944331, 0.0822271, -0.0381067}, +{-0.0635881, 0.0392124, -0.00717766}, +{0.00864227, 0.0386371, -0.0233053}, +{0.0252935, 0.0769557, -0.0248407}, +{-0.0229653, 0.0895159, -0.036199}, +{-0.0523791, 0.0341193, -0.00994653}, +{0.0211693, 0.0643935, -0.0268578}, +{-0.0515867, 0.13164, -0.0028092}, +{-0.0149669, 0.0345529, -0.0254273}, +{-0.0161167, 0.127288, 0.00169291}, +{-0.0469232, 0.128515, -0.00163965}, +{-0.00961381, 0.127158, -0.00378809}, +{-0.0074566, 0.128562, -0.00130751}, +{-0.00304493, 0.128909, -0.00174857}, +{0.0028379, 0.129022, -0.00194723}, +{0.00903363, 0.128674, -0.00165013}, +{-0.0561607, 0.131588, -0.00571429}, +{-0.0457551, 0.127167, -0.00484962}, +{-0.00304746, 0.127678, -0.00456004}, +{0.00303811, 0.12768, -0.00442}, +{0.0101526, 0.126812, -0.00466464}, +{-0.0553259, 0.126836, -0.00601308}, +{0.00799473, 0.034846, -0.0206913}, +{0.0027179, 0.0342191, -0.0204737}, +{-0.00295804, 0.0342418, -0.0216222}, +{0.0134674, 0.0353221, -0.0196961}, +{0.00440963, 0.0383063, -0.0240776}, +{0.00140752, 0.0383474, -0.0246147}, +{-0.00309177, 0.0383877, -0.0251866}, +{-0.0575023, 0.100661, -0.0195211}, +{-0.0485739, 0.15316, -0.00547278}, +{-0.0646573, 0.0334831, -0.00296009}, +{-0.0640796, 0.100426, -0.0173936}, +{-0.0704415, 0.100139, -0.0146037}, +{-0.0326376, 0.155806, -0.00949884}, +{0.0336094, 0.0373624, 0.00273412}, +{0.0320943, 0.0397885, -0.00195136}, +{0.0158502, 0.0449602, -0.0237212}, +{0.00889467, 0.0426449, -0.0242659}, +{0.00312499, 0.0452721, -0.026588}, +{-0.00298345, 0.044686, -0.0272222}, +{-0.00912346, 0.0448524, -0.0280671}, +{-0.0145351, 0.0443266, -0.0277771}, +{-0.0209223, 0.0460913, -0.0281918}, +{0.034052, 0.0448434, -0.00540113}, +{-0.0312646, 0.158257, -0.01223}, +{0.0401509, 0.0448981, -0.00354586}, +{0.0143253, 0.0473484, -0.0251513}, +{0.00937888, 0.0466526, -0.0261685}, +{-0.0766531, 0.0695423, 0.0207982}, +{0.0087246, 0.0517916, -0.0291615}, +{0.00299372, 0.0506927, -0.0298557}, +{-0.00164566, 0.0489436, -0.0304144}, +{-0.00321397, 0.0522596, -0.0314075}, +{-0.00915341, 0.0509217, -0.0318681}, +{-0.0146018, 0.0513752, -0.0319045}, +{-0.0161558, 0.0488543, -0.0303763}, +{-0.0205843, 0.0508011, -0.0296435}, +{0.0405252, 0.0518855, -0.00654453}, +{0.0149309, 0.0520772, -0.0273859}, +{0.041884, 0.0490868, -0.00604367}, +{0.019962, 0.0529908, -0.0261219}, +{-0.0198501, 0.0534234, -0.0312267}, +{-0.0336273, 0.0527187, -0.0106243}, +{-0.0461112, 0.0529158, -0.0101664}, +{-0.0204, 0.161875, -0.014658}, +{0.0449924, 0.0530898, -0.00614891}, +{0.00733679, 0.0546532, -0.0305038}, +{0.00283568, 0.0546532, -0.0307468}, +{-0.00302245, 0.0577, -0.0331477}, +{-0.00914668, 0.0576676, -0.0341165}, +{-0.01517, 0.058199, -0.0349877}, +{-0.0202707, 0.0581031, -0.0333681}, +{0.0140844, 0.057965, -0.028983}, +{0.0103301, 0.0588553, -0.0299472}, +{0.00732823, 0.0588898, -0.0306117}, +{0.0027369, 0.0590151, -0.0321928}, +{-0.0337187, 0.0579742, -0.0115824}, +{-0.0390711, 0.0582467, -0.0115033}, +{-0.0460474, 0.0579124, -0.0115174}, +{-0.00961439, 0.0642168, -0.0358564}, +{-0.044157, 0.0599825, -0.0123877}, +{0.015251, 0.0645803, -0.029567}, +{0.00839294, 0.0649214, -0.0316957}, +{0.00325858, 0.0643529, -0.0332439}, +{-0.00361257, 0.0645861, -0.034907}, +{-0.0144709, 0.065006, -0.0371603}, +{-0.0366623, 0.060679, -0.0122791}, +{-0.0526404, 0.0636402, -0.0101297}, +{-0.0381866, 0.0648919, -0.0142073}, +{-0.0452495, 0.0647856, -0.0139819}, +{-0.0599262, 0.0622966, -0.00429285}, +{-0.0778641, 0.117463, -0.00576778}, +{-0.0187447, 0.0664151, -0.0374779}, +{-0.0577616, 0.0644884, -0.00779097}, +{-0.0625778, 0.0655353, -0.00741131}, +{0.0251088, 0.0710945, -0.0248604}, +{0.021457, 0.0702729, -0.0273415}, +{0.0166747, 0.0701586, -0.0297203}, +{0.0132745, 0.0702643, -0.0312074}, +{0.00867525, 0.0703509, -0.0324278}, +{0.00229643, 0.0708694, -0.0343123}, +{-0.0030646, 0.070381, -0.0353565}, +{-0.00773679, 0.0691749, -0.0362051}, +{-0.0101988, 0.0715122, -0.0373778}, +{-0.0147454, 0.0704429, -0.0382943}, +{-0.0203984, 0.0706516, -0.038158}, +{-0.0240967, 0.0693418, -0.0362521}, +{-0.0605175, 0.0673597, -0.0108259}, +{-0.0387293, 0.0706355, -0.0168457}, +{-0.0451347, 0.0705064, -0.0164504}, +{-0.0523435, 0.0697862, -0.0145984}, +{-0.0591515, 0.0702891, -0.0147203}, +{-0.0652515, 0.0688492, -0.00993982}, +{-0.0247614, 0.0719777, -0.0368317}, +{-0.0637884, 0.0712697, -0.0138535}, +{0.0211454, 0.0769268, -0.0268772}, +{0.0162128, 0.0765268, -0.0293784}, +{0.0133247, 0.0760196, -0.0306715}, +{0.00907695, 0.076038, -0.0330382}, +{0.00245085, 0.0760857, -0.0351615}, +{-0.00176321, 0.0762288, -0.0360688}, +{-0.00476487, 0.076286, -0.0369742}, +{-0.00962992, 0.0765936, -0.0378651}, +{-0.0144481, 0.0764118, -0.0385775}, +{-0.021453, 0.0763574, -0.038668}, +{-0.024977, 0.0762484, -0.0374518}, +{-0.0377453, 0.0766164, -0.0189124}, +{-0.0397395, 0.0746623, -0.0180255}, +{-0.0437423, 0.0765905, -0.0187922}, +{-0.0466377, 0.0744845, -0.0173668}, +{-0.0518623, 0.0745812, -0.0175084}, +{-0.0589866, 0.0745368, -0.01766}, +{-0.0644081, 0.0756279, -0.0167529}, +{-0.0721295, 0.0740256, -0.0105719}, +{-0.0615233, 0.0354132, 0.043881}, +{-0.0524971, 0.0769872, -0.0189536}, +{-0.0587482, 0.0767445, -0.0187462}, +{0.013102, 0.0809953, -0.0307917}, +{0.00892296, 0.0820652, -0.0325478}, +{0.0022917, 0.0820297, -0.0349279}, +{-0.00177837, 0.0804805, -0.0364471}, +{-0.00379684, 0.0824193, -0.037328}, +{-0.0142988, 0.0820384, -0.0390211}, +{-0.0207708, 0.0823862, -0.0387335}, +{-0.0248089, 0.0818968, -0.0377031}, +{-0.0735819, 0.0777026, -0.0122023}, +{0.015425, 0.0831288, -0.0295207}, +{-0.0383994, 0.0817919, -0.0209596}, +{-0.0451184, 0.0815526, -0.020434}, +{-0.051814, 0.0818472, -0.0211348}, +{-0.0583689, 0.0812724, -0.0202975}, +{-0.063949, 0.082768, -0.0188935}, +{-0.0662709, 0.080065, -0.0177832}, +{-0.0695594, 0.0830593, -0.0170582}, +{-0.00481814, 0.086841, -0.0367951}, +{-0.0248206, 0.0867524, -0.0367639}, +{0.0132046, 0.0871602, -0.0305473}, +{-0.0360837, 0.0867076, -0.023791}, +{-0.00877843, 0.0340556, -0.0204927}, +{-0.0207128, 0.0342382, -0.0208728}, +{-0.0147915, 0.0341096, -0.0207616}, +{-0.0265767, 0.0342963, -0.0210989}, +{0.00282685, 0.0351053, -0.0158136}, +{0.00885967, 0.034471, -0.0147487}, +{-0.0390848, 0.0337228, -0.0202617}, +{-0.0326656, 0.0345334, -0.0201874}, +{-0.00224535, 0.0351539, -0.0166234}, +{-0.0149096, 0.0357313, -0.0180956}, +{-0.0114808, 0.0353662, -0.0177045}, +{-0.00921575, 0.0380183, -0.0149732}, +{-0.00282494, 0.0382292, -0.0140636}, +{0.00285919, 0.0377324, -0.0134715}, +{0.0159109, 0.0347098, -0.00882204}, +{-0.0306839, 0.036693, -0.0184598}, +{-0.0265216, 0.0367471, -0.0188177}, +{-0.0218341, 0.0369718, -0.0184303}, +{-0.0203027, 0.0382765, -0.0152577}, +{-0.0152596, 0.0382328, -0.0156428}, +{0.00738356, 0.0366172, -0.0125003}, +{0.00992361, 0.0351979, -0.00924624}, +{0.00702596, 0.0378387, -0.00879015}, +{-0.0396958, 0.0342843, -0.014578}, +{-0.0329517, 0.0382154, -0.014678}, +{-0.0263862, 0.0385778, -0.0153644}, +{0.00320835, 0.0389424, -0.00953857}, +{-0.0364387, 0.0357946, -0.0155844}, +{-0.00301526, 0.0391061, -0.00886496}, +{0.00831664, 0.0348156, -0.00321961}, +{0.0145039, 0.0343685, -0.0028433}, +{-0.0365752, 0.0370276, -0.0136534}, +{-0.0146234, 0.0388055, -0.00887465}, +{-0.00886749, 0.0389394, -0.00890173}, +{-0.0451032, 0.0336721, -0.00848668}, +{-0.040313, 0.0350801, -0.00861758}, +{-0.0206235, 0.0386, -0.00878063}, +{0.00267879, 0.038424, -0.00319748}, +{0.015044, 0.0350517, 0.00289039}, +{0.0201479, 0.0347806, 0.00348327}, +{0.027119, 0.0353514, 0.00366834}, +{0.0280785, 0.0365531, 0.000826759}, +{-0.0376066, 0.0375692, -0.00942418}, +{-0.0332748, 0.0384549, -0.00855692}, +{-0.0264541, 0.0384497, -0.00886193}, +{-0.00299262, 0.0389582, -0.00292437}, +{0.00451408, 0.0356078, -0.00103635}, +{0.00881079, 0.0350428, 0.00356828}, +{0.0314184, 0.0360255, 0.00457907}, +{-0.00888202, 0.0387884, -0.00299409}, +{0.00271787, 0.0349091, 0.00339755}, +{-0.041199, 0.0341471, -0.00327644}, +{-0.0205479, 0.0384259, -0.00283766}, +{-0.0146618, 0.0385908, -0.00288739}, +{0.00103528, 0.0375917, 0.000952222}, +{0.0215747, 0.0354906, 0.0086194}, +{0.0264794, 0.0346514, 0.00870654}, +{0.0322391, 0.0355412, 0.00882378}, +{-0.0521057, 0.0334794, -0.00318207}, +{-0.0455078, 0.0336572, -0.00225818}, +{-0.0334104, 0.0383259, -0.00292317}, +{-0.0265122, 0.0383343, -0.00296504}, +{-0.00224847, 0.0383354, 0.00320971}, +{-0.0589386, 0.0334143, -0.00291301}, +{-0.00874044, 0.0385976, 0.00291227}, +{0.00273457, 0.0342734, 0.0088248}, +{0.00621941, 0.0351341, 0.00654928}, +{-0.080018, 0.109279, 0.0373655}, +{-0.0393178, 0.0336443, 0.00354096}, +{-0.0213111, 0.0382973, 0.00334866}, +{-0.0146196, 0.0384265, 0.00290922}, +{-0.00353554, 0.0379644, 0.00874752}, +{0.0276681, 0.0349662, 0.0149532}, +{0.03282, 0.0359255, 0.0147037}, +{0.0389763, 0.0383079, 0.0145025}, +{-0.0523961, 0.0335249, 0.00326874}, +{-0.0462346, 0.0335696, 0.00267776}, +{-0.0277984, 0.0382296, 0.00286126}, +{-0.000947006, 0.0357374, 0.0103469}, +{0.0222276, 0.0358262, 0.0160256}, +{0.0448051, 0.0411192, 0.0150961}, +{-0.0581064, 0.033504, 0.00272997}, +{-0.0352323, 0.0337248, 0.00491425}, +{-0.0312985, 0.0381858, 0.00167702}, +{-0.0088641, 0.03847, 0.00876261}, +{0.0028919, 0.0342894, 0.0147059}, +{-0.0703332, 0.0340583, 0.00286723}, +{-0.0648245, 0.0334924, 0.00301734}, +{-0.0387963, 0.034763, 0.00935652}, +{-0.0332327, 0.0337932, 0.00943608}, +{-0.0203456, 0.0382265, 0.00836296}, +{-0.0152156, 0.0383161, 0.00935801}, +{-0.000385714, 0.0351459, 0.0134171}, +{0.00663645, 0.0342324, 0.0159688}, +{0.0268074, 0.0356469, 0.0204126}, +{0.0309391, 0.0362152, 0.0189937}, +{0.0334119, 0.0376179, 0.0210082}, +{-0.0515734, 0.0338904, 0.00817232}, +{-0.0454999, 0.0352808, 0.00804865}, +{-0.0263229, 0.0380313, 0.00871732}, +{-0.0031858, 0.0377098, 0.014513}, +{0.0211051, 0.0351552, 0.0207004}, +{0.0391983, 0.0395969, 0.0205879}, +{0.0441778, 0.0418755, 0.0204802}, +{-0.0580282, 0.0335624, 0.00918162}, +{-0.00922404, 0.0383488, 0.0150261}, +{0.00313746, 0.0352426, 0.0204176}, +{0.00877508, 0.0346179, 0.020856}, +{0.0468489, 0.0434226, 0.0210936}, +{-0.0648031, 0.0337402, 0.00884817}, +{-0.0338156, 0.0345063, 0.0150293}, +{-0.0149173, 0.0382498, 0.0147214}, +{0.0146344, 0.0345628, 0.0222588}, +{-0.0365655, 0.0357926, 0.0130139}, +{-0.0262153, 0.0376693, 0.0148666}, +{-0.0205165, 0.0381248, 0.0146779}, +{-0.00229335, 0.0382456, 0.020565}, +{0.014723, 0.0347707, 0.0263935}, +{0.0210245, 0.0353476, 0.0265418}, +{0.0250756, 0.0364517, 0.0246847}, +{0.0273584, 0.0381522, 0.0267127}, +{0.0321164, 0.0401984, 0.026762}, +{-0.053829, 0.0335431, 0.0139547}, +{0.00114114, 0.037661, 0.0223414}, +{0.00915473, 0.0353589, 0.0262457}, +{0.0380552, 0.0412819, 0.02589}, +{-0.0588034, 0.0336951, 0.0146283}, +{-0.0339319, 0.0346253, 0.0202274}, +{-0.0152545, 0.0382629, 0.0204704}, +{-0.00888844, 0.0384087, 0.0207206}, +{0.00307272, 0.0384964, 0.0264151}, +{-0.0261643, 0.0378491, 0.0205422}, +{-0.0205429, 0.0381473, 0.0213758}, +{-0.0538188, 0.0335608, 0.0210581}, +{-0.00301594, 0.03875, 0.0263901}, +{0.00756209, 0.0380712, 0.0285007}, +{0.0143741, 0.0348327, 0.0331833}, +{0.0198279, 0.03555, 0.0321213}, +{0.0236875, 0.0373106, 0.0299772}, +{-0.0588476, 0.033906, 0.020465}, +{-0.00882687, 0.0386047, 0.0265705}, +{0.00847025, 0.0383344, 0.0315598}, +{0.0108958, 0.035647, 0.0330663}, +{-0.0366651, 0.0353042, 0.023032}, +{-0.0340084, 0.0344659, 0.0266224}, +{-0.0270447, 0.0379104, 0.0270529}, +{-0.0210471, 0.0383013, 0.026282}, +{-0.0147317, 0.0384888, 0.0265233}, +{-0.0712786, 0.0733348, 0.0355839}, +{-0.0388887, 0.0346255, 0.0265538}, +{0.00290004, 0.0393205, 0.032168}, +{0.0155389, 0.0350901, 0.0393977}, +{0.0195159, 0.0358111, 0.0367948}, +{-0.0589139, 0.0341314, 0.0264586}, +{-0.052234, 0.0340737, 0.0268887}, +{-0.0447866, 0.0339274, 0.0274346}, +{-0.0310127, 0.0369382, 0.02848}, +{-0.00908756, 0.0390146, 0.0330901}, +{-0.00293287, 0.039209, 0.03365}, +{0.00861952, 0.0346654, 0.0391536}, +{-0.0149144, 0.0388312, 0.0324344}, +{0.00392423, 0.0347398, 0.0399064}, +{-0.0657827, 0.0618455, 0.00187562}, +{-0.0640051, 0.0606097, 0.00361345}, +{-0.0455164, 0.0345095, 0.0326748}, +{-0.0385699, 0.0344168, 0.033204}, +{-0.0342024, 0.0351611, 0.0325685}, +{-0.0270303, 0.0384799, 0.0326469}, +{-0.0209433, 0.0387397, 0.0332273}, +{-0.0520994, 0.0344582, 0.0326775}, +{-0.0313489, 0.0377268, 0.0321213}, +{-0.00219023, 0.0348305, 0.0410082}, +{0.00818206, 0.0355366, 0.0443043}, +{0.014947, 0.0361331, 0.0431407}, +{-0.0642564, 0.0597236, 0.0092932}, +{-0.0584732, 0.0343588, 0.0331559}, +{-0.0145859, 0.0393004, 0.0380317}, +{-0.00937548, 0.0394517, 0.037871}, +{-0.0588297, 0.0579582, 0.0145443}, +{-0.038732, 0.0346956, 0.0400227}, +{-0.0331487, 0.034492, 0.0390527}, +{-0.0201914, 0.0391628, 0.0381696}, +{-0.00878985, 0.0348233, 0.0452949}, +{-0.0031441, 0.0351515, 0.045825}, +{-0.0701619, 0.0622789, 0.00863964}, +{-0.0451191, 0.034688, 0.0396457}, +{-0.0256628, 0.0389081, 0.0373249}, +{-0.0146115, 0.0348173, 0.0458198}, +{-0.0636462, 0.0593677, 0.014889}, +{-0.0531671, 0.0345191, 0.0391729}, +{-0.0595372, 0.034497, 0.0397515}, +{-0.0329555, 0.0349777, 0.045552}, +{-0.0262436, 0.034809, 0.0452831}, +{-0.0215554, 0.0348112, 0.0459347}, +{-0.0633407, 0.0601272, 0.0190813}, +{-0.0471, 0.0351015, 0.0434178}, +{-0.0120723, 0.0353434, 0.0494553}, +{-0.016313, 0.0351836, 0.0504037}, +{-0.0483699, 0.146034, -0.00115148}, +{-0.0264335, 0.156562, -0.00835956}, +{-0.065003, 0.144791, -0.0142909}, +{-0.066228, 0.151547, -0.0394609}, +{-0.0663323, 0.145309, -0.018858}, +{-0.0412403, 0.152108, -0.00674014} +}; diff --git a/test/testsuites/visualizer/remote_display_server/build.sh b/test/testsuites/visualizer/remote_display_server/build.sh new file mode 100755 index 0000000..2827cca --- /dev/null +++ b/test/testsuites/visualizer/remote_display_server/build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +g++ -g -std=c++11 server.cpp proto/mv_util_visualizer_rd.grpc.pb.cc proto/mv_util_visualizer_rd.pb.cc `pkg-config --cflags --libs opencv4 protobuf grpc++` -o server diff --git a/test/testsuites/visualizer/remote_display_server/img/intro.png b/test/testsuites/visualizer/remote_display_server/img/intro.png new file mode 100644 index 0000000000000000000000000000000000000000..e8583edb820f2bb044f1a00010e4b6b9e86efad8 GIT binary patch literal 6598 zcmeAS@N?(olHy`uVBq!ia0y~yU~gbxV6os}0*a(>3=?5sU~I{Bb`J1#c2+1T%1_J8 zNo8Qrm{>c}*5j~)%+dH@Z&$6Rw+fF0mRLm!h!$Sa5?Sc9LNuz>VCNU^Stcr)dVNjV z2M_iiUDe#Zc^%)nCWRl2k6t`k@CtFvV;l-%}F zeUzlOTZ=7T&_#lgXK{SmG(&Yy&k~vV?;k#X)jPJ?YX0xyFRLuO8SN%kOp{#ZyT{RH zZ>IE|@14qPG*%yZek55fcEXQ|ue*-t&*9eNwlPjx7X0~$VA945$GoIBFIp++FX$fX zySzs6@y1h?-J8SKZF*+=liS&{Hg)D}llBP*l%^>-wlryZX&iRz37adzt$C}z`~&~4 zy3QD`on8q~A{!-=WaX}hG+k*#`Bk4kBa)j@!~d{-n!^no7iLCF);hL<(yjgWVz;(&j)@>)&`4l+-I`Qj>|t= zJI9V8^OaU5vy2SWV%Ae}??1eI_wC+4+skj>v)gSen$*vzyayO9oCO|{#S9F3${@^G zvDCf{D9B#o>Fdh=j9r*j$&|a_Y#UHWvcxr_#5q4VH#M(>!MP|ku_QG`p**uBL&4qC zHz2%`Pn>~4e1oTpV@SoVHwPIRfm#|S{9FHy1C$y@!DtAKhQMeDjE2By2#kinXb6mk zz-S1JhQMeDjE2By2#kinunvKS+s({EARjvZkN>V5GOYbN>bcPn7!85Z5Eu=C(GVC4 bA+Uq<>n;XHK}AbfkZV0%{an^LB{Ts5>?-1V literal 0 HcmV?d00001 diff --git a/test/testsuites/visualizer/remote_display_server/proto/build.sh b/test/testsuites/visualizer/remote_display_server/proto/build.sh new file mode 100755 index 0000000..4b6b99d --- /dev/null +++ b/test/testsuites/visualizer/remote_display_server/proto/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +protoc mv_util_visualizer_rd.proto --cpp_out=. +protoc --grpc_out=. --plugin=protoc-gen-grpc=`which grpc_cpp_plugin` mv_util_visualizer_rd.proto diff --git a/test/testsuites/visualizer/remote_display_server/proto/mv_util_visualizer_rd.proto b/test/testsuites/visualizer/remote_display_server/proto/mv_util_visualizer_rd.proto new file mode 100644 index 0000000..ada3f63 --- /dev/null +++ b/test/testsuites/visualizer/remote_display_server/proto/mv_util_visualizer_rd.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +message Empty { +} + +message NLImage { + int32 length = 1; + bytes data = 2; +} + +message NLImageDrawRequest { + NLImage image = 1; +} + +service NLImageService { + rpc DrawImage(NLImageDrawRequest) returns (Empty); +} + diff --git a/test/testsuites/visualizer/remote_display_server/server.cpp b/test/testsuites/visualizer/remote_display_server/server.cpp new file mode 100644 index 0000000..0c3ed43 --- /dev/null +++ b/test/testsuites/visualizer/remote_display_server/server.cpp @@ -0,0 +1,79 @@ + +#include +#include +#include "proto/mv_util_visualizer_rd.grpc.pb.h" + +#include +#include + +#include +#include +#include +#include + +typedef unsigned char byte; + +using grpc::Server; +using grpc::ServerBuilder; +using grpc::ServerContext; +using grpc::Status; + +using std::cout; +using std::cin; +using std::endl; + +#define WIN_NAME "draw_image" + +static cv::Mat image; + +class ImageServiceImpl final : public NLImageService::Service { + +public: + + Status DrawImage(ServerContext* context, const NLImageDrawRequest* request, ::Empty* reply) { + NLImage image_message = (*request).image(); + + unsigned long length = image_message.data().length(); + byte *imgData = (byte*)malloc(sizeof(image_message.data()[0])*image_message.data().length()); + memcpy(imgData, image_message.data().c_str(), image_message.data().size()); + std::vector jpeg(imgData, imgData+length); + image = cv::imdecode(jpeg, cv::IMREAD_COLOR); + + return Status::OK; + } +}; + + +void runServer() { + + printf("Runserver function\n"); + + std::string server_address("0.0.0.0:50051"); + ImageServiceImpl service; + + grpc::ServerBuilder builder; + builder.SetMaxReceiveMessageSize(INT_MAX); + builder.SetMaxSendMessageSize(INT_MAX); + builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); + builder.RegisterService(&service); + std::unique_ptr server(builder.BuildAndStart()); + std::cout << "Server listening on " << server_address << std::endl; + + image = cv::imread("img/intro.png"); + cv::namedWindow(WIN_NAME, cv::WINDOW_NORMAL); + cv::moveWindow(WIN_NAME, 0,0); + cv::resizeWindow(WIN_NAME, 1280,720); + while(1) { + cv::imshow(WIN_NAME, image); + cv::waitKey(1); + } +} + + +int main() { + printf("server.cpp starting.. \n"); + + runServer(); + + return 0; +} diff --git a/test/testsuites/visualizer/visualizer_test_suite.cpp b/test/testsuites/visualizer/visualizer_test_suite.cpp new file mode 100755 index 0000000..2cbd1cf --- /dev/null +++ b/test/testsuites/visualizer/visualizer_test_suite.cpp @@ -0,0 +1,235 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include + +#include "mv_util_visualizer_2d.h" +#include "mv_util_visualizer_3d.h" +#include "bunny.h" + +#define MAX_ARGS 2 + +static const char doc[] = "[visualizer]\vmediavision visualizer test\n\ + e.g) mv_visualizer_testsuite [0|1] \n\ + 0 : 2D Target Display \n\ + 1 : 3D Target Display \n\ + 2 : 2D Remote Display"; + +#define IMAGE_WIDTH 1920 +#define IMAGE_HEIGHT 1080 +#define RGB888 4 + +#define DEPTH_WIDTH 320 +#define DEPTH_HEIGHT 240 + +#define HOST_URL "192.168.0.5:50051" + +static const char args_doc[] = "2D|3D SOURCE"; + +struct arguments +{ + int type; +}; + +static bool is_raw_initialized = false; +static float *depth = NULL; +static unsigned char *data_buffer_3d = NULL; + +void fill_buffer_2d(unsigned char *data, unsigned int r, unsigned int g, unsigned int b) +{ + int i = 0; + + for (i = 0; i < IMAGE_WIDTH * IMAGE_HEIGHT; i++) { + data[i*4] = r; + data[i*4+1] = g; + data[i*4+2] = b; + data[i*4+3] = 255; + } +} + +void perform_visualize_2d(struct arguments *arguments, unsigned int r, unsigned int g, unsigned int b, const char *url) +{ + MEDIA_VISION_FUNCTION_ENTER(); + + mv_source_h source = NULL; + unsigned char *data_buffer = NULL; + unsigned long buffer_size = IMAGE_WIDTH * IMAGE_WIDTH * RGB888; + int err = MEDIA_VISION_ERROR_NONE; + + data_buffer = new(std::nothrow)unsigned char[buffer_size]; + fill_buffer_2d(data_buffer, r, g, b); + + err = mv_create_source(&source); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("Errors were occurred during creating the source!!! code : %i", err); + goto out; + } + + err = mv_source_fill_by_buffer(source, data_buffer, buffer_size, + IMAGE_WIDTH, IMAGE_HEIGHT, MEDIA_VISION_COLORSPACE_RGBA); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("Errors were occurred during filling the source!!! code : %i", err); + goto out; + } + + mv_util_visualizer_2d(source, url); + +out: + if (data_buffer != NULL) { + delete [] data_buffer; + data_buffer = NULL; + } + + if (source != NULL) { + err = mv_destroy_source(source); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("Errors were occurred during destroying the source!!! code : %i", err); + } + } + + MEDIA_VISION_FUNCTION_LEAVE(); +} + +void perform_visualize_3d(struct arguments *arguments) +{ + MEDIA_VISION_FUNCTION_ENTER(); + + mv_source_h source = NULL; + int err = MEDIA_VISION_ERROR_NONE; + + if (!is_raw_initialized) { + is_raw_initialized = true; + int i = 0; + int x, y; + int idx; + + depth = (float *)malloc(DEPTH_WIDTH * DEPTH_HEIGHT * 4); + for (i = 0; i < POINTS_BUNNY; i++) { + x = (int)((bunny[i][0] + 0.0944) * 1000.0); + y = (int)((0.1848 - bunny[i][1]) * 1000.0); + idx = y * DEPTH_WIDTH + x; + depth[idx] = (0.0585 - bunny[i][2]) * 5.0; + } + err = mv_create_source(&source); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("Errors were occurred during creating the source!!! code : %i", err); + goto out; + } + unsigned long buffer_size = DEPTH_WIDTH * DEPTH_HEIGHT * 4; + data_buffer_3d = (unsigned char*)malloc(buffer_size); + memset(data_buffer_3d, 255, buffer_size); + err = mv_source_fill_by_buffer(source, data_buffer_3d, buffer_size, + DEPTH_WIDTH, DEPTH_HEIGHT, MEDIA_VISION_COLORSPACE_RGB888); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("Errors were occurred during filling the source!!! code : %i", err); + goto out; + } + } + + if (source != NULL && depth != NULL) + while(1) + mv_util_visualizer_3d(source, depth, 140, 25); + +out: + if (data_buffer_3d != NULL) { + delete [] data_buffer_3d; + data_buffer_3d = NULL; + } + + if (source != NULL) { + err = mv_destroy_source(source); + if (MEDIA_VISION_ERROR_NONE != err) { + LOGE("Errors were occurred during destroying the source!!! code : %i", err); + } + } + + MEDIA_VISION_FUNCTION_LEAVE(); +} + +static error_t parse_opt(int key, char *arg, struct argp_state *state) +{ + struct arguments *arguments = (struct arguments *)state->input; + + switch (key) { + case ARGP_KEY_NO_ARGS: + argp_usage(state); + break; + case ARGP_KEY_ARG: + if (state->argc > MAX_ARGS) + argp_failure(state, 1, 0, "too many arguments"); + else if (state->argc < MAX_ARGS) + argp_failure(state, 1, 0, "too few arguments"); + arguments->type = atoi(arg); + + state->next = state->argc; + if (arguments->type < 0 || arguments->type > 2) { + argp_failure(state, 1, 0, "Invalid type"); + } + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +static struct argp argp = { NULL, parse_opt, args_doc, doc }; + +int main(int argc, char *argv[]) +{ + LOGI("Media Vision visualizer-Testsuite is launched."); + int i = 0; + + struct arguments arguments; + argp_parse (&argp, argc, argv, 0, 0, &arguments); + + std::chrono::system_clock::time_point StartTime = std::chrono::system_clock::now(); + if (arguments.type == 0) { + std::cout << "visualize_2d localhost" << std::endl; + for (i = 0; i < 6; i++) + perform_visualize_2d(&arguments, 0, 0 , 0, NULL); // dummy black screen + for (i = 0; i < 6; i++) + perform_visualize_2d(&arguments, 255, 0 , 0, NULL); // red + for (i = 0; i < 6; i++) + perform_visualize_2d(&arguments, 0, 255, 0, NULL); // green + for (i = 0; i < 6; i++) + perform_visualize_2d(&arguments, 0, 0, 255, NULL); // blue + } else if (arguments.type == 1) { + std::cout << "visualize_3d localhost" << std::endl; + perform_visualize_3d(&arguments); + } else if (arguments.type == 2) { + std::cout << "visualize_2d remote display" << std::endl; + for (i = 0; i < 6; i++) + perform_visualize_2d(&arguments, 0, 0 , 0, HOST_URL); // dummy black screen + for (i = 0; i < 6; i++) + perform_visualize_2d(&arguments, 255, 0 , 0, HOST_URL); // red + for (i = 0; i < 6; i++) + perform_visualize_2d(&arguments, 0, 255, 0, HOST_URL); // green + for (i = 0; i < 6; i++) + perform_visualize_2d(&arguments, 0, 0, 255, HOST_URL); // blue + } + std::chrono::milliseconds ms = std::chrono::duration_cast(std::chrono::system_clock::now() - StartTime); + std::cout << "total : " << ms.count() << "ms" << std::endl; + + LOGI("Media Vision visualizer-Testsuite is closed."); + + return 0; +} -- 2.7.4 From 849a2e27ef5018fe648df9a101c3b389826a7da7 Mon Sep 17 00:00:00 2001 From: sangho park Date: Wed, 21 Sep 2022 16:46:01 +0900 Subject: [PATCH 05/16] Add pointcloud implementation for mv_3d APIs [Issue type] new feature Change-Id: I7571bb1d67607e7d0c423296ce7d4c5c6fb7de6e Signed-off-by: sangho park --- include/mv_3d_internal.h | 103 ++++++ include/mv_3d_private.h | 12 +- media-vision-config.json | 15 +- mv_3d/3d/CMakeLists.txt | 10 +- mv_3d/3d/include/Depth.h | 93 ----- mv_3d/3d/include/Mv3d.h | 87 ++++- mv_3d/3d/include/PointCloud.h | 42 --- mv_3d/3d/include/mv_3d_open.h | 27 +- mv_3d/3d/src/Depth.cpp | 299 ---------------- mv_3d/3d/src/Mv3d.cpp | 443 ++++++++++++++++++++++++ mv_3d/3d/src/PointCloud.cpp | 28 -- mv_3d/3d/src/mv_3d internal.cpp | 154 ++++++++ mv_3d/3d/src/mv_3d.c | 4 +- mv_3d/3d/src/mv_3d_open.cpp | 114 +++++- test/testsuites/mv3d/CMakeLists.txt | 4 - test/testsuites/mv3d/depth_test_suite.cpp | 208 ++++------- test/testsuites/mv3d/depthstream_test_suite.cpp | 4 +- 17 files changed, 998 insertions(+), 649 deletions(-) create mode 100644 include/mv_3d_internal.h delete mode 100644 mv_3d/3d/include/Depth.h delete mode 100644 mv_3d/3d/include/PointCloud.h delete mode 100644 mv_3d/3d/src/Depth.cpp create mode 100644 mv_3d/3d/src/Mv3d.cpp delete mode 100644 mv_3d/3d/src/PointCloud.cpp create mode 100644 mv_3d/3d/src/mv_3d internal.cpp diff --git a/include/mv_3d_internal.h b/include/mv_3d_internal.h new file mode 100644 index 0000000..e499715 --- /dev/null +++ b/include/mv_3d_internal.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __TIZEN_MEDIAVISION_3D_INTERNAL_H__ +#define __TIZEN_MEDIAVISION_3D_INTERNAL_H__ + +#include "mv_3d.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** + * @file mv_3d_internal.h + * @brief This file contains the mv3d internal structure and function for plane segmentation. + * @since_tizen 7.0 + */ + +/** + * @brief The plane model handle. + * @since_tizen 7.0 + */ +typedef void *mv_3d_pointcloud_plane_model_h; + +/** + * @brief The plane inlier handle. + * @since_tizen 7.0 + */ +typedef void *mv_3d_pointcloud_plane_inlier_h; + +/** + * @brief Creates plane model handle. + * @details Use this function to create a plane model handle. + * + * @since_tizen 7.0 + */ +int mv_3d_pointcloud_plane_model_create(mv_3d_pointcloud_plane_model_h *handle); + +/** + * @brief Destroys plane model handle and release all its resources. + * + * @since_tizen 7.0 + */ +int mv_3d_pointcloud_plane_model_destroy(mv_3d_pointcloud_plane_model_h handle); + +/** + * @brief Creates plane inlier handle. + * @details Use this function to create a plane model handle. + * + * @since_tizen 7.0 + */ +int mv_3d_pointcloud_plane_inlier_create(mv_3d_pointcloud_plane_inlier_h *handle); + +/** + * @brief Destroys plane inlier handle and release all its resources. + * + * @since_tizen 7.0 + */ +int mv_3d_pointcloud_plane_inlier_destroy(mv_3d_pointcloud_plane_inlier_h handle); + +/** + * @brief Segment PointCloud plane. + * @details Use this function to segment pointcloud plane using the RANSAC algorithm. + * + * @since_tizen 7.0 + */ +int mv_3d_pointcloud_segment_plane(mv_3d_h mv3d, + mv_3d_pointcloud_h pointcloud, + mv_3d_pointcloud_plane_model_h *plane_model, + mv_3d_pointcloud_plane_inlier_h *plane_inlier); + +/** + * @brief Writes pointcloud plane data to a file. + * @details Use this function to write pointcloud plane data to a file. + * + * @since_tizen 7.0 + */ + +int mv_3d_pointcloud_plane_write_file(mv_3d_h mv3d, + mv_3d_pointcloud_plane_model_h model, + mv_3d_pointcloud_plane_inlier_h inlier, + mv_3d_pointcloud_h pointcloud, + mv_3d_pointcloud_type_e type, + char *filename); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __TIZEN_MEDIAVISION_3D_INTERNAL_H__ */ diff --git a/include/mv_3d_private.h b/include/mv_3d_private.h index 84dcc98..37bcc0b 100644 --- a/include/mv_3d_private.h +++ b/include/mv_3d_private.h @@ -31,15 +31,15 @@ extern "C" { */ /** - * @brief Landmarks of a pose. + * @brief structure of pointcloud. * - * @since_tizen 6.0 + * @since_tizen 7.0 * */ typedef struct { - mv_pointcloud_type_e type; - FILE *pointcloud; -} mv_pointcloud_s; + mv_3d_pointcloud_type_e type; + void *pointcloud; +} mv_3d_pointcloud_s; /** * @} @@ -48,4 +48,4 @@ typedef struct { } #endif /* __cplusplus */ -#endif /* __TIZEN_MEDIAVISION_3D_TYPE_H__ */ +#endif /* __TIZEN_MEDIAVISION_3D_PRIVATE_H__ */ diff --git a/media-vision-config.json b/media-vision-config.json index d48ff9e..6703092 100644 --- a/media-vision-config.json +++ b/media-vision-config.json @@ -284,19 +284,24 @@ "value" : "" }, { - "name" : "MV_3D_POINTCLOUD_SAMPLING_RATIO", + "name" : "MV_3D_POINTCLOUD_SAMPLING_RATIO", "type" : "double", "value" : 1.0 }, { - "name" : "MV_3D_POINTCLOUD_OUTLIER_REMOVAL_POINTS", + "name" : "MV_3D_POINTCLOUD_OUTLIER_REMOVAL_POINTS", "type" : "integer", - "value" : 10 + "value" : 0 }, { - "name" : "MV_3D_POINTCLOUD_OUTLIER_REMOVAL_RADIUS", + "name" : "MV_3D_POINTCLOUD_OUTLIER_REMOVAL_RADIUS", "type" : "double", - "value" : 1.0 + "value" : 0.0 + }, + { + "name" : "MV_3D_POINTCLOUD_OUTPUT_FILE_PATH", + "type" : "string", + "value" : "/opt/usr/home/owner/media/Images" } ] } diff --git a/mv_3d/3d/CMakeLists.txt b/mv_3d/3d/CMakeLists.txt index d2eb9af..2265cbd 100644 --- a/mv_3d/3d/CMakeLists.txt +++ b/mv_3d/3d/CMakeLists.txt @@ -10,12 +10,20 @@ if(NOT OpenCV_FOUND) return() endif() +find_package(Open3D REQUIRED NO_POLICY_SCOPE) +if(NOT Open3D_FOUND) + message(SEND_ERROR "Open3D NOT FOUND") + return() +else() + include_directories(${Open3D_INCLUDE_DIRS}) +endif() + if(FORCED_STATIC_BUILD) add_library(${PROJECT_NAME} STATIC ${MV_DFS_SOURCE_LIST}) else() add_library(${PROJECT_NAME} SHARED ${MV_DFS_SOURCE_LIST}) endif() -target_link_libraries(${PROJECT_NAME} ${MV_COMMON_LIB_NAME} ${OpenCV_LIBS} ${${PROJECT_NAME}_DEP_LIBRARIES}) +target_link_libraries(${PROJECT_NAME} ${MV_COMMON_LIB_NAME} ${OpenCV_LIBS} ${Open3D_LIBRARIES} ${${PROJECT_NAME}_DEP_LIBRARIES}) target_include_directories(${PROJECT_NAME} PRIVATE include) install(TARGETS ${PROJECT_NAME} DESTINATION ${LIB_INSTALL_DIR}) diff --git a/mv_3d/3d/include/Depth.h b/mv_3d/3d/include/Depth.h deleted file mode 100644 index b3236c4..0000000 --- a/mv_3d/3d/include/Depth.h +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __MEDIA_VISION_DEPTH_H__ -#define __MEDIA_VISION_DEPTH_H__ - -#include -#include -#include "dfs_parameter.h" -#include "dfs_adaptation_impl.h" -#include "mv_3d.h" -#include "mv_3d_type.h" - -/** - * @file Depth.h - * @brief This file contains the depth class definition - * which supports depth-from-stereo (DFS). - */ -using namespace DfsAdaptation; -using DepthType = uint16_t; -using DepthTypePtr = DepthType*; -namespace mediavision -{ -namespace depth -{ - class Depth - { - private: - DfsParameter mDfsParameter; - DfsAdaptor *mDfsAdaptor; - int mMode; - - size_t mWidth; - size_t mHeight; - size_t mMinDisp; - size_t mMaxDisp; - std::string mStereoConfigPath; - - GThread *mThread; - void *mUserData; - mv_3d_depth_cb mDepthCallback; - bool mIsLive; - - GAsyncQueue *mAsyncQueue; - - void GetBufferFromSource(mv_source_h source, - unsigned char*& buffer, - unsigned int& width, - unsigned int& height, - int& type, - size_t& stride); - - void GetDfsDataFromSources(mv_source_h baseSource, - mv_source_h extraSource, - DfsInputData& input); - - - static gpointer ThreadLoop(gpointer data); - public: - Depth(); - ~Depth(); - void SetParameters(double threshold, - size_t windowWidth, - size_t windowHeight, - size_t speckleSize); - - int Configure(int mode, int width, int height, int minDisp, int maxDisp, - std::string stereoConfigPath); - - void SetCallback(mv_3d_depth_cb depthCallback, void *userData); - - int Prepare(); - - int Run(mv_source_h baseSource, mv_source_h extraSource); - - int RunAsync(mv_source_h baseSource, mv_source_h extraSource); - }; -} -} -#endif /* __MEDIA_VISION_DEPTH_H__ */ diff --git a/mv_3d/3d/include/Mv3d.h b/mv_3d/3d/include/Mv3d.h index dc8b36c..8e345a8 100644 --- a/mv_3d/3d/include/Mv3d.h +++ b/mv_3d/3d/include/Mv3d.h @@ -17,30 +17,93 @@ #ifndef __MEDIA_VISION_3D_H__ #define __MEDIA_VISION_3D_H__ + #include -#include "Depth.h" -#include "PointCloud.h" +#include +#include "dfs_parameter.h" +#include "dfs_adaptation_impl.h" +#include "mv_3d.h" +#include "mv_3d_type.h" +#include "mv_3d_private.h" /** - * @file Mv3d.h - * @brief This file contains the mv3d class definition - * which supports depth and point cloud data. + * @file Depth.h + * @brief This file contains the depth class definition + * which supports depth-from-stereo (DFS). */ -using namespace mediavision::depth; -using namespace mediavision::pointcloud; +using namespace DfsAdaptation; +using DepthType = uint16_t; +using DepthTypePtr = DepthType*; namespace mediavision { namespace mv3d { class Mv3d { - public: - Depth mDepth; - PointCloud mPointCloud; + private: + DfsParameter mDfsParameter; + DfsAdaptor *mDfsAdaptor; + int mMode; + + size_t mWidth; + size_t mHeight; + size_t mMinDisp; + size_t mMaxDisp; + double mSamplingRatio; + int mOutlierRemovalPoints; + double mOutlierRemovalRadius; + std::string mStereoConfigPath; + std::string mIntrinsicPath; + std::string mPointcloudOutputPath; + + GThread *mDfsThread; + void *mDepthUserData; + void *mPointcloudUserData; + mv_3d_depth_cb mDepthCallback; + mv_3d_pointcloud_cb mPointcloudCallback; + bool mDfsIsLive; + + GAsyncQueue *mDfsAsyncQueue; + + void GetBufferFromSource(mv_source_h source, + unsigned char*& buffer, + unsigned int& width, + unsigned int& height, + int& type, + size_t& stride); + + void GetDfsDataFromSources(mv_source_h baseSource, + mv_source_h extraSource, + DfsInputData& input); + + static gpointer DfsThreadLoop(gpointer data); + void GetPointcloudFromSource(DfsInputData &intput, + DfsOutputData &depthData, + mv_3d_pointcloud_s &pointcloud); public: - Mv3d() = default; - ~Mv3d() = default; + Mv3d(); + ~Mv3d(); + void SetParameters(double threshold, + size_t windowWidth, + size_t windowHeight, + size_t speckleSize); + + int Configure(int mode, int width, int height, int minDisp, int maxDisp, + double samplingRatio, int outlierRemovalPoints, double outlierRemovalRadius, + std::string stereoConfigPath, std::string pointcloudOutputPath); + + void SetDepthCallback(mv_3d_depth_cb depthCallback, void *depthUserData); + + void SetPointcloudCallback(mv_3d_pointcloud_cb pointcloudCallback, void *pointcloudUserData); + + int WritePointcloudFile(mv_3d_pointcloud_h pointcloud, mv_3d_pointcloud_type_e type, char *fileName); + + int Prepare(); + + int Run(mv_source_h baseSource, mv_source_h extraSource); + + int RunAsync(mv_source_h baseSource, mv_source_h extraSource); }; } } diff --git a/mv_3d/3d/include/PointCloud.h b/mv_3d/3d/include/PointCloud.h deleted file mode 100644 index dcc91cb..0000000 --- a/mv_3d/3d/include/PointCloud.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __MEDIA_VISION_POINT_CLOUD_H__ -#define __MEDIA_VISION_POINT_CLOUD_H__ - -#include -#include -#include "mv_3d.h" -#include "mv_3d_type.h" - -/** - * @file PointCloud.h - * @brief This file contains the PointCloud class definition - * which supports Point Cloud. - */ -namespace mediavision -{ -namespace pointcloud -{ - class PointCloud - { - public: - PointCloud() = default; - ~PointCloud() = default; - }; -} -} -#endif /* __MEDIA_VISION_DEPTH_H__ */ diff --git a/mv_3d/3d/include/mv_3d_open.h b/mv_3d/3d/include/mv_3d_open.h index b2c2582..00e8dd5 100644 --- a/mv_3d/3d/include/mv_3d_open.h +++ b/mv_3d/3d/include/mv_3d_open.h @@ -50,6 +50,12 @@ int mv3dConfigure(mv_3d_h mv3d, mv_engine_config_h engine_config); int mv3dSetDepthCallback(mv_3d_h mv3d, mv_3d_depth_cb depth_cb, void *user_data); /** + * @brief Set pointcloud callback to mv3d handle. + * @sicne_tizen 7.0 + */ +int mv3dSetPointcloudCallback(mv_3d_h mv3d, mv_3d_pointcloud_cb pointcloud_cb, void *user_data); + +/** * @brief Prepares mv3d handle. * @since_tizen 7.0 */ @@ -59,17 +65,26 @@ int mv3dPrepare(mv_3d_h mv3d); * @brief Gets depth data from source(s). * @since_tizen 7.0 */ -int mv3dRun(mv_source_h source, - mv_source_h source_extra, - mv_3d_h mv3d); +int mv3dRun(mv_3d_h mv3d, + mv_source_h source, + mv_source_h source_extra); /** * @brief Run depth estimation asynchronousely with source(s). * @since_tizen 7.0 */ -int mv3dRunAsync(mv_source_h source, - mv_source_h source_extra, - mv_3d_h mv3d); +int mv3dRunAsync(mv_3d_h mv3d, + mv_source_h source, + mv_source_h source_extra); + +/** + * @brief Write Pointcloud file. + * @since_tizen 7.0 + */ +int mv3dWritePointcloudFile(mv_3d_h mv3d, + mv_3d_pointcloud_h pointcloud, + mv_3d_pointcloud_type_e type, + char *fileName); #ifdef __cplusplus } diff --git a/mv_3d/3d/src/Depth.cpp b/mv_3d/3d/src/Depth.cpp deleted file mode 100644 index 01e3197..0000000 --- a/mv_3d/3d/src/Depth.cpp +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mv_private.h" -#include -#include -#include "Depth.h" - -namespace mediavision -{ -namespace depth -{ - Depth::Depth() : - mDfsParameter(), - mDfsAdaptor(nullptr), - mMode(MV_3D_DEPTH_MODE_NONE), - mWidth(0), - mHeight(0), - mMinDisp(0), - mMaxDisp(0), - mThread(nullptr), - mUserData(nullptr), - mDepthCallback(nullptr), - mIsLive(false), - mAsyncQueue(nullptr) - { - LOGI("ENTER"); - LOGI("LEAVE"); - } - - Depth::~Depth() - { - LOGI("ENTER"); - - if (mThread) { - mIsLive = false; - g_thread_join(mThread); - } - - if (mAsyncQueue) { - g_async_queue_unref(mAsyncQueue); - } - - if (mDfsAdaptor) { - mDfsAdaptor->unBind(); - delete mDfsAdaptor; - } - - LOGI("LEAVE"); - } - - void Depth::SetParameters(double threshold, - size_t windowWidth, - size_t windowHeight, - size_t speckleSize) - { - mDfsParameter.textureThreshold = threshold; - mDfsParameter.aggregationWindowWidth = windowWidth; - mDfsParameter.aggregationWindowHeight = windowHeight; - mDfsParameter.maxSpeckleSize = speckleSize; - } - - int Depth::Configure(int mode, int width, int height, - int minDisp, int maxDisp, std::string stereoConfigPath) - { - mMode = mode; - mWidth = width; - mHeight = height; - mMinDisp = minDisp; - mMaxDisp = maxDisp; - mStereoConfigPath = stereoConfigPath; - - try { - mDfsAdaptor = new DfsAdaptor(); - mDfsAdaptor->bind(); - } catch (const std::bad_alloc &e) { - LOGE("Failed to create dfs adaptation : %s", e.what()); - return MEDIA_VISION_ERROR_OUT_OF_MEMORY; - } catch (const std::runtime_error &e) { - LOGE("Failed to bind %s adpator", e.what()); - return MEDIA_VISION_ERROR_INVALID_OPERATION; - } - - return MEDIA_VISION_ERROR_NONE; - } - - void Depth::SetCallback(mv_3d_depth_cb depthCallback, void *userData) - { - mDepthCallback = depthCallback; - mUserData = userData; - } - - int Depth::Prepare() - { - if (!mDfsAdaptor) { - LOGE("Invalid Opertation. Do Configure first."); - return MEDIA_VISION_ERROR_INVALID_OPERATION; - } - - try { - mDfsAdaptor->initialize(mDfsParameter, mWidth, mHeight, - mMinDisp, mMaxDisp, mStereoConfigPath); - } catch (const std::exception& e) { - LOGE("Failed to initialize"); - return MEDIA_VISION_ERROR_INVALID_OPERATION; - } - - return MEDIA_VISION_ERROR_NONE; - } - - - void Depth::GetBufferFromSource(mv_source_h source, - unsigned char*& buffer, - unsigned int& width, - unsigned int& height, - int& type, - size_t& stride) - { - unsigned char* _buffer = nullptr; - unsigned int _bufferSize = 0; - unsigned int _width = 0; - unsigned int _height = 0; - mv_colorspace_e _colorSpace = MEDIA_VISION_COLORSPACE_INVALID; - - int ret = mv_source_get_buffer(source, &_buffer, &_bufferSize); - if (ret != MEDIA_VISION_ERROR_NONE) - throw std::runtime_error("invalid buffer pointer"); - - ret = mv_source_get_width(source, &_width); - if (ret != MEDIA_VISION_ERROR_NONE) - throw std::runtime_error("invalid width"); - - ret = mv_source_get_height(source, &_height); - if (ret != MEDIA_VISION_ERROR_NONE) - throw std::runtime_error("invalid height"); - - ret = mv_source_get_colorspace(source, &_colorSpace); - if (ret != MEDIA_VISION_ERROR_NONE) - throw std::runtime_error("invalid color space"); - - buffer = new unsigned char [_bufferSize]; - memcpy(buffer, _buffer, _bufferSize); - width = _width; - height = _height; - type = _colorSpace == MEDIA_VISION_COLORSPACE_RGB888 ? - DFS_DATA_TYPE_UINT8C3 : - DFS_DATA_TYPE_UINT8C1; - stride = _bufferSize / _height; - } - - void Depth::GetDfsDataFromSources(mv_source_h baseSource, - mv_source_h extraSource, - DfsInputData& input) - { - unsigned char* baseBuffer = nullptr; - unsigned char* extraBuffer = nullptr; - unsigned int width = 0; - unsigned int height = 0; - int type = 0; - size_t stride = 0; - - GetBufferFromSource(baseSource, baseBuffer, width, height, type, stride); - input.data = static_cast(baseBuffer); - input.type = type; - input.width = width; - input.height = height; - input.stride = stride; - input.format = DFS_DATA_INPUT_FORMAT_COUPLED_SBS; - - if (extraSource) - { - extraBuffer = nullptr; - GetBufferFromSource(extraSource, extraBuffer, width, height, type, stride); - - if (input.type != type || input.width != width || - input.height != height || input.stride != stride) { - throw std::runtime_error("left and right image's properties are different"); - } - - input.extraData = static_cast(extraBuffer); - input.format = DFS_DATA_INPUT_FORMAT_DECOUPLED_SBS; - } - } - - int Depth::Run(mv_source_h baseSource, mv_source_h extraSource) - { - DfsInputData input; - try { - if (mThread) { - mIsLive = false; - g_thread_join(mThread); - mThread = nullptr; - } - - if (mAsyncQueue) { - g_async_queue_unref(mAsyncQueue); - mAsyncQueue = nullptr; - } - - GetDfsDataFromSources(baseSource, extraSource, input); - - mDfsAdaptor->run(input); - - auto depthData = mDfsAdaptor->getOutputData(); - - mDepthCallback( - baseSource, - static_cast(depthData.data), - depthData.width, depthData.height, - mUserData); - - delete [] input.data; - delete [] input.extraData; - } catch (const std::exception &e) { - LOGE("Failed to Run with %s", e.what()); - return MEDIA_VISION_ERROR_INVALID_OPERATION; - } - - return MEDIA_VISION_ERROR_NONE; - } - - int Depth::RunAsync(mv_source_h baseSource, mv_source_h extraSource) - { - try { - if (!mAsyncQueue) { - mAsyncQueue = g_async_queue_new(); - if (!mAsyncQueue) { - LOGE("Fail to g_async_queue_new()"); - return MEDIA_VISION_ERROR_INTERNAL; - } - } - - if (!mThread) { - mThread = g_thread_new("depth_thread", &Depth::ThreadLoop, static_cast(this)); - - if (!mThread) { - g_async_queue_unref(mAsyncQueue); - mAsyncQueue = nullptr; - LOGE("Fail to g_thread_new()"); - return MEDIA_VISION_ERROR_INTERNAL; - } - mIsLive = true; - } - - std::shared_ptr input(new DfsInputData); - GetDfsDataFromSources(baseSource, extraSource, *input); - g_async_queue_push(mAsyncQueue, static_cast( - new std::shared_ptr( - std::move(input)))); - } catch (const std::exception &e) { - LOGE("Failed to Run with %s", e.what()); - return MEDIA_VISION_ERROR_INVALID_OPERATION; - } - - return MEDIA_VISION_ERROR_NONE; - } - - gpointer Depth::ThreadLoop(gpointer data) - { - Depth *handle = static_cast(data); - while(handle->mIsLive) { - gpointer base = g_async_queue_try_pop(handle->mAsyncQueue); - if (!base) { - continue; - } - - auto pInput = static_cast*>(base); - auto input = std::move(*pInput); - delete pInput; - handle->mDfsAdaptor->run(*input); - delete [] static_cast(input->data); - delete [] static_cast(input->extraData); - input.reset(); - - auto depthData = handle->mDfsAdaptor->getOutputData(); - handle->mDepthCallback( - static_cast(base), - static_cast(depthData.data), - depthData.width, depthData.height, - handle->mUserData); - } - - return nullptr; - } -} -} diff --git a/mv_3d/3d/src/Mv3d.cpp b/mv_3d/3d/src/Mv3d.cpp new file mode 100644 index 0000000..747a786 --- /dev/null +++ b/mv_3d/3d/src/Mv3d.cpp @@ -0,0 +1,443 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mv_private.h" +#include +#include +#include +#include +#include +#include +#include + +#include "Mv3d.h" + +using namespace open3d; + +namespace mediavision +{ +namespace mv3d +{ + Mv3d::Mv3d() : + mDfsParameter(), + mDfsAdaptor(nullptr), + mMode(MV_3D_DEPTH_MODE_NONE), + mWidth(0), + mHeight(0), + mMinDisp(0), + mMaxDisp(0), + mSamplingRatio(1.0), + mOutlierRemovalPoints(0), + mOutlierRemovalRadius(0.0), + mDfsThread(nullptr), + mDepthUserData(nullptr), + mPointcloudUserData(nullptr), + mDepthCallback(nullptr), + mPointcloudCallback(nullptr), + mDfsIsLive(false), + mDfsAsyncQueue(nullptr) + { + LOGI("ENTER"); + utility::SetVerbosityLevel(utility::VerbosityLevel::Debug); + LOGI("LEAVE"); + } + + Mv3d::~Mv3d() + { + LOGI("ENTER"); + + if (mDfsThread) { + mDfsIsLive = false; + g_thread_join(mDfsThread); + } + + if (mDfsAsyncQueue) { + g_async_queue_unref(mDfsAsyncQueue); + } + + if (mDfsAdaptor) { + mDfsAdaptor->unBind(); + delete mDfsAdaptor; + } + + LOGI("LEAVE"); + } + + void Mv3d::SetParameters(double threshold, + size_t windowWidth, + size_t windowHeight, + size_t speckleSize) + { + mDfsParameter.textureThreshold = threshold; + mDfsParameter.aggregationWindowWidth = windowWidth; + mDfsParameter.aggregationWindowHeight = windowHeight; + mDfsParameter.maxSpeckleSize = speckleSize; + } + + int Mv3d::Configure(int mode, int width, int height, + int minDisp, int maxDisp, double samplingRatio, + int outlierRemovalPoints, double outlierRemovalRadius, + std::string stereoConfigPath, + std::string pointcloudOutputPath) + { + mMode = mode; + mWidth = width; + mHeight = height; + mMinDisp = minDisp; + mMaxDisp = maxDisp; + mSamplingRatio = samplingRatio; + mOutlierRemovalPoints = outlierRemovalPoints; + mOutlierRemovalRadius = outlierRemovalRadius; + mStereoConfigPath = stereoConfigPath; + size_t found = stereoConfigPath.rfind("."); + mIntrinsicPath = stereoConfigPath.substr(0, found) + std::string(".json"); + mPointcloudOutputPath = pointcloudOutputPath; + + try { + mDfsAdaptor = new DfsAdaptor(); + mDfsAdaptor->bind(); + } catch (const std::bad_alloc &e) { + LOGE("Failed to create dfs adaptation : %s", e.what()); + return MEDIA_VISION_ERROR_OUT_OF_MEMORY; + } catch (const std::runtime_error &e) { + LOGE("Failed to bind %s adpator", e.what()); + return MEDIA_VISION_ERROR_INVALID_OPERATION; + } + + return MEDIA_VISION_ERROR_NONE; + } + + void Mv3d::SetDepthCallback(mv_3d_depth_cb depthCallback, void *depthUserData) + { + mDepthCallback = depthCallback; + mDepthUserData = depthUserData; + } + + void Mv3d::SetPointcloudCallback(mv_3d_pointcloud_cb pointcloudCallback, void *pointcloudUserData) + { + mPointcloudCallback = pointcloudCallback; + mPointcloudUserData = pointcloudUserData; + } + + int Mv3d::Prepare() + { + if (!mDfsAdaptor) { + LOGE("Invalid Opertation. Do Configure first."); + return MEDIA_VISION_ERROR_INVALID_OPERATION; + } + + try { + mDfsAdaptor->initialize(mDfsParameter, mWidth, mHeight, + mMinDisp, mMaxDisp, mStereoConfigPath); + } catch (const std::exception& e) { + LOGE("Failed to initialize"); + return MEDIA_VISION_ERROR_INVALID_OPERATION; + } + + return MEDIA_VISION_ERROR_NONE; + } + + + void Mv3d::GetBufferFromSource(mv_source_h source, + unsigned char*& buffer, + unsigned int& width, + unsigned int& height, + int& type, + size_t& stride) + { + unsigned char* _buffer = nullptr; + unsigned int _bufferSize = 0; + unsigned int _width = 0; + unsigned int _height = 0; + mv_colorspace_e _colorSpace = MEDIA_VISION_COLORSPACE_INVALID; + + int ret = mv_source_get_buffer(source, &_buffer, &_bufferSize); + if (ret != MEDIA_VISION_ERROR_NONE) + throw std::runtime_error("invalid buffer pointer"); + + ret = mv_source_get_width(source, &_width); + if (ret != MEDIA_VISION_ERROR_NONE) + throw std::runtime_error("invalid width"); + + ret = mv_source_get_height(source, &_height); + if (ret != MEDIA_VISION_ERROR_NONE) + throw std::runtime_error("invalid height"); + + ret = mv_source_get_colorspace(source, &_colorSpace); + if (ret != MEDIA_VISION_ERROR_NONE) + throw std::runtime_error("invalid color space"); + + buffer = new unsigned char [_bufferSize]; + memcpy(buffer, _buffer, _bufferSize); + width = _width; + height = _height; + type = _colorSpace == MEDIA_VISION_COLORSPACE_RGB888 ? + DFS_DATA_TYPE_UINT8C3 : + DFS_DATA_TYPE_UINT8C1; + stride = _bufferSize / _height; + } + + void Mv3d::GetDfsDataFromSources(mv_source_h baseSource, + mv_source_h extraSource, + DfsInputData& input) + { + unsigned char* baseBuffer = nullptr; + unsigned char* extraBuffer = nullptr; + unsigned int width = 0; + unsigned int height = 0; + int type = 0; + size_t stride = 0; + + GetBufferFromSource(baseSource, baseBuffer, width, height, type, stride); + input.data = static_cast(baseBuffer); + input.type = type; + input.width = width; + input.height = height; + input.stride = stride; + input.format = DFS_DATA_INPUT_FORMAT_COUPLED_SBS; + + if (extraSource) + { + extraBuffer = nullptr; + GetBufferFromSource(extraSource, extraBuffer, width, height, type, stride); + + if (input.type != type || input.width != width || + input.height != height || input.stride != stride) { + throw std::runtime_error("left and right image's properties are different"); + } + + input.extraData = static_cast(extraBuffer); + input.format = DFS_DATA_INPUT_FORMAT_DECOUPLED_SBS; + } + } + + void Mv3d::GetPointcloudFromSource(DfsInputData &input, + DfsOutputData &depthData, + mv_3d_pointcloud_s &pointcloud) + { + camera::PinholeCameraIntrinsic intrinsic; + io::ReadIJsonConvertible(mIntrinsicPath, intrinsic); + cv::Mat img; + double depth_scale = 1000.0, depth_trunc = 200.0; + if (input.type == DFS_DATA_TYPE_UINT8C1) { + img = cv::Mat(cv::Size(input.width, input.height), CV_8UC1, input.data); + cv::cvtColor(img, img, cv::COLOR_GRAY2RGB); + } else if (input.type == DFS_DATA_TYPE_UINT8C3) { + img = cv::Mat(cv::Size(input.width, input.height), CV_8UC3, input.data); + cv::cvtColor(img, img, cv::COLOR_BGR2RGB); + } + + geometry::Image img_color, img_depth; + img_color.Prepare(input.width, input.height, 3, 1); + uint8_t *pImg_color_data = img_color.data_.data(); + memcpy(pImg_color_data, img.data, input.width * input.height * 3 * sizeof(unsigned char)); + + img_depth.Prepare(input.width, input.height, 1, 2); + uint16_t *pImg_depth_data = (unsigned short*)img_depth.data_.data(); + memcpy(pImg_depth_data, static_cast(depthData.data), depthData.width * depthData.height * 2); + + std::shared_ptr rgbd_image = + geometry::RGBDImage::CreateFromColorAndDepth( + img_color, img_depth, depth_scale, depth_trunc, false); + + auto pcd = geometry::PointCloud::CreateFromRGBDImage(*rgbd_image, intrinsic); + if (mSamplingRatio < 1.0) { + utility::LogInfo("Downsampling... {}", mSamplingRatio); + pcd = pcd->RandomDownSample(mSamplingRatio); + } + + if (mOutlierRemovalPoints > 0 && mOutlierRemovalRadius > 0.0) { + utility::LogInfo("RemoveRadiusOutliers... {} {}", mOutlierRemovalPoints, mOutlierRemovalRadius); + std::vector pt_map; + std::tie(pcd, pt_map) = pcd->RemoveRadiusOutliers(mOutlierRemovalPoints, mOutlierRemovalRadius, true); + } + + pointcloud.pointcloud = static_cast( + new std::shared_ptr( + std::move(pcd))); + + } + + int Mv3d::Run(mv_source_h baseSource, mv_source_h extraSource) + { + DfsInputData input; + try { + if (mDfsThread) { + mDfsIsLive = false; + g_thread_join(mDfsThread); + mDfsThread = nullptr; + } + + if (mDfsAsyncQueue) { + g_async_queue_unref(mDfsAsyncQueue); + mDfsAsyncQueue = nullptr; + } + + GetDfsDataFromSources(baseSource, extraSource, input); + + mDfsAdaptor->run(input); + auto depthData = mDfsAdaptor->getOutputData(); + + mDepthCallback( + baseSource, + static_cast(depthData.data), + depthData.width, depthData.height, + mDepthUserData); + + if (mPointcloudCallback) { + mv_3d_pointcloud_s p = {.type = MV_3D_POINTCLOUD_TYPE_PCD_BIN, .pointcloud = NULL}; + GetPointcloudFromSource(input, depthData, p); + + mv_3d_pointcloud_h pcd = &p; + mPointcloudCallback(baseSource, + pcd, + mPointcloudUserData); + auto pPcd = static_cast*>(p.pointcloud); + auto _pcd = std::move(pPcd); + delete _pcd; + } + delete [] static_cast(input.data); + delete [] static_cast(input.extraData); + } catch (const std::exception &e) { + LOGE("Failed to Run with %s", e.what()); + return MEDIA_VISION_ERROR_INVALID_OPERATION; + } + + return MEDIA_VISION_ERROR_NONE; + } + + int Mv3d::RunAsync(mv_source_h baseSource, mv_source_h extraSource) + { + try { + if (!mDfsAsyncQueue) { + mDfsAsyncQueue = g_async_queue_new(); + if (!mDfsAsyncQueue) { + LOGE("Fail to g_async_queue_new()"); + return MEDIA_VISION_ERROR_INTERNAL; + } + } + + if (!mDfsThread) { + mDfsThread = g_thread_new("depth_thread", &Mv3d::DfsThreadLoop, static_cast(this)); + + if (!mDfsThread) { + g_async_queue_unref(mDfsAsyncQueue); + mDfsAsyncQueue = nullptr; + LOGE("Fail to g_thread_new()"); + return MEDIA_VISION_ERROR_INTERNAL; + } + mDfsIsLive = true; + } + + std::shared_ptr input(new DfsInputData); + GetDfsDataFromSources(baseSource, extraSource, *input); + g_async_queue_push(mDfsAsyncQueue, static_cast( + new std::shared_ptr( + std::move(input)))); + } catch (const std::exception &e) { + LOGE("Failed to Run with %s", e.what()); + return MEDIA_VISION_ERROR_INVALID_OPERATION; + } + + return MEDIA_VISION_ERROR_NONE; + } + + gpointer Mv3d::DfsThreadLoop(gpointer data) + { + Mv3d *handle = static_cast(data); + while(handle->mDfsIsLive) { + gpointer base = g_async_queue_try_pop(handle->mDfsAsyncQueue); + if (!base) { + continue; + } + + auto pInput = static_cast*>(base); + auto input = std::move(*pInput); + delete pInput; + handle->mDfsAdaptor->run(*input); + + auto depthData = handle->mDfsAdaptor->getOutputData(); + handle->mDepthCallback( + static_cast(base), + static_cast(depthData.data), + depthData.width, depthData.height, + handle->mDepthUserData); + + if (handle->mPointcloudCallback) { + mv_3d_pointcloud_s p = {.type = MV_3D_POINTCLOUD_TYPE_PCD_BIN, .pointcloud = NULL}; + //mPointcloudThread = g_thread_new("pointcloud_thread", + // &Mv3d::PointcloudThreadLoop, + // static_cast(this)); + handle->GetPointcloudFromSource(*input, depthData, p); + + mv_3d_pointcloud_h pcd = &p; + handle->mPointcloudCallback(static_cast(base), + pcd, + handle->mPointcloudUserData); + auto pPcd = static_cast*>(p.pointcloud); + auto _pcd = std::move(pPcd); + delete _pcd; + } + + delete [] static_cast(input->data); + delete [] static_cast(input->extraData); + input.reset(); + } + + return nullptr; + } + + int Mv3d::WritePointcloudFile(mv_3d_pointcloud_h pointcloud, mv_3d_pointcloud_type_e type, char *fileName) + { + mv_3d_pointcloud_s *s = (mv_3d_pointcloud_s*) pointcloud; + if (s == NULL) { + LOGE("Pointcloud data is NULL"); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + + if (access(mPointcloudOutputPath.c_str(), W_OK) != 0) { + if (errno == EACCES || errno == EPERM) { + utility::LogInfo("Fail to access path[%s]: Permission Denied", mPointcloudOutputPath.c_str()); + return MEDIA_VISION_ERROR_PERMISSION_DENIED;; + } else { + utility::LogInfo("Fail to access path[%s]: Invalid Path", mPointcloudOutputPath.c_str()); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + } + + bool bText = false; + if (type == MV_3D_POINTCLOUD_TYPE_PCD_TXT || type == MV_3D_POINTCLOUD_TYPE_PLY_TXT) + bText = true; + else + bText = false; + + auto pPcd = static_cast*>(s->pointcloud); + open3d::geometry::PointCloud p; + p += **pPcd; + + std::string fullPath = mPointcloudOutputPath + std::string("/") + std::string(fileName); + + if (io::WritePointCloud(fullPath.c_str(), p, {bText, false, false, {}})) { + utility::LogInfo("Successfully wrote {}", fullPath.c_str()); + } else { + utility::LogError("Failed to write {}", fullPath.c_str()); + return MEDIA_VISION_ERROR_INTERNAL; + } + + return MEDIA_VISION_ERROR_NONE; + } +} +} diff --git a/mv_3d/3d/src/PointCloud.cpp b/mv_3d/3d/src/PointCloud.cpp deleted file mode 100644 index 083764d..0000000 --- a/mv_3d/3d/src/PointCloud.cpp +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mv_private.h" -#include -#include -#include "PointCloud.h" - -namespace mediavision -{ -namespace pointCloud -{ -// -} -} diff --git a/mv_3d/3d/src/mv_3d internal.cpp b/mv_3d/3d/src/mv_3d internal.cpp new file mode 100644 index 0000000..4c52156 --- /dev/null +++ b/mv_3d/3d/src/mv_3d internal.cpp @@ -0,0 +1,154 @@ +/** + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "mv_3d_internal.h" +#include "mv_private.h" +#include "mv_3d_private.h" +#include "Mv3d.h" + +#define DISTANCE_THRESHOLD 0.01 +#define RANSAC_NUMBER 3 +#define NUM_ITERATIONS 1000 + +using namespace open3d; +using namespace mediavision::mv3d; + +int mv_3d_pointcloud_plane_model_create(mv_3d_pointcloud_plane_model_h *handle) +{ + MEDIA_VISION_NULL_ARG_CHECK(handle); + + MEDIA_VISION_FUNCTION_ENTER(); + + /* do nothing */ + + MEDIA_VISION_FUNCTION_LEAVE(); + + return MEDIA_VISION_ERROR_NONE; +} + +int mv_3d_pointcloud_plane_model_destroy(mv_3d_pointcloud_plane_model_h handle) +{ + MEDIA_VISION_INSTANCE_CHECK(handle); + + MEDIA_VISION_FUNCTION_ENTER(); + + Eigen::Vector4d *model = (Eigen::Vector4d *) handle; + delete model; + + MEDIA_VISION_FUNCTION_LEAVE(); + + return MEDIA_VISION_ERROR_NONE; +} + +int mv_3d_pointcloud_plane_inlier_create(mv_3d_pointcloud_plane_inlier_h *handle) +{ + MEDIA_VISION_NULL_ARG_CHECK(handle); + + MEDIA_VISION_FUNCTION_ENTER(); + + /* do nothing */ + + MEDIA_VISION_FUNCTION_LEAVE(); + + return MEDIA_VISION_ERROR_NONE; +} + +int mv_3d_pointcloud_plane_inlier_destroy(mv_3d_pointcloud_plane_inlier_h handle) +{ + MEDIA_VISION_INSTANCE_CHECK(handle); + + MEDIA_VISION_FUNCTION_ENTER(); + + std::vector *inlier = (std::vector *) handle; + delete inlier; + + MEDIA_VISION_FUNCTION_LEAVE(); + + return MEDIA_VISION_ERROR_NONE; +} + +/** + * @file mv_3d_internal.c + * @brief This file contains Media Vision 3D internal module. + */ + +int mv_3d_pointcloud_segment_plane(mv_3d_h mv3d, + mv_3d_pointcloud_h pointcloud, + mv_3d_pointcloud_plane_model_h *model, + mv_3d_pointcloud_plane_inlier_h *inlier) +{ + MEDIA_VISION_NULL_ARG_CHECK(mv3d); + MEDIA_VISION_NULL_ARG_CHECK(pointcloud); + MEDIA_VISION_NULL_ARG_CHECK(model); + MEDIA_VISION_NULL_ARG_CHECK(inlier); + MEDIA_VISION_FUNCTION_ENTER(); + + mv_3d_pointcloud_s *s = (mv_3d_pointcloud_s*) pointcloud; + Eigen::Vector4d *best_plane_model = new Eigen::Vector4d; + std::vector *plane_inlier = new std::vector; + auto pPcd = static_cast*>(s->pointcloud); + open3d::geometry::PointCloud p; + p += **pPcd; + + std::tie(*best_plane_model, *plane_inlier) = p.SegmentPlane(DISTANCE_THRESHOLD, RANSAC_NUMBER, NUM_ITERATIONS); + *model = (mv_3d_pointcloud_plane_model_h) best_plane_model; + *inlier = (mv_3d_pointcloud_plane_inlier_h) plane_inlier; + + MEDIA_VISION_FUNCTION_LEAVE(); + + return MEDIA_VISION_ERROR_NONE; +} + +int mv_3d_pointcloud_plane_write_file(mv_3d_h mv3d, + mv_3d_pointcloud_plane_model_h model, + mv_3d_pointcloud_plane_inlier_h inlier, + mv_3d_pointcloud_h pointcloud, + mv_3d_pointcloud_type_e type, + char *filename) +{ + MEDIA_VISION_NULL_ARG_CHECK(mv3d); + MEDIA_VISION_NULL_ARG_CHECK(model); + MEDIA_VISION_NULL_ARG_CHECK(inlier); + MEDIA_VISION_FUNCTION_ENTER(); + + mv_3d_pointcloud_s *s = (mv_3d_pointcloud_s*) pointcloud; + Eigen::Vector4d *best_plane_model = (Eigen::Vector4d*) model; + std::vector *plane_inlier = (std::vector*) inlier; + auto pPcd = static_cast*>(s->pointcloud); + open3d::geometry::PointCloud p; + p += **pPcd; + + std::shared_ptr plane = p.SelectByIndex(*plane_inlier); + mv_3d_pointcloud_s _pcd = {.type = MV_3D_POINTCLOUD_TYPE_PCD_BIN, .pointcloud = NULL}; + _pcd.pointcloud = static_cast( + new std::shared_ptr( + std::move(plane))); + mv_3d_pointcloud_h pcd = &_pcd; + + auto pMv3d = static_cast(mv3d); + int ret = pMv3d->WritePointcloudFile(pcd, type, filename); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to write pointcloud plane file"); + ret = MEDIA_VISION_ERROR_INTERNAL; + } + + MEDIA_VISION_FUNCTION_LEAVE(); + + return ret; +} \ No newline at end of file diff --git a/mv_3d/3d/src/mv_3d.c b/mv_3d/3d/src/mv_3d.c index d766ed0..c640e13 100644 --- a/mv_3d/3d/src/mv_3d.c +++ b/mv_3d/3d/src/mv_3d.c @@ -92,7 +92,7 @@ int mv_3d_set_pointcloud_cb(mv_3d_h mv3d, MEDIA_VISION_FUNCTION_ENTER(); - int ret = MEDIA_VISION_ERROR_NONE; + int ret = mv3dSetPointcloudCallback(mv3d, pointcloud_cb, user_data); MEDIA_VISION_FUNCTION_LEAVE(); @@ -160,7 +160,7 @@ int mv_3d_pointcloud_write_file(mv_3d_h mv3d, MEDIA_VISION_FUNCTION_ENTER(); - int ret = MEDIA_VISION_ERROR_NONE; + int ret = mv3dWritePointcloudFile(mv3d, pointcloud, type, filename); MEDIA_VISION_FUNCTION_LEAVE(); diff --git a/mv_3d/3d/src/mv_3d_open.cpp b/mv_3d/3d/src/mv_3d_open.cpp index 7fd9026..9f914d2 100644 --- a/mv_3d/3d/src/mv_3d_open.cpp +++ b/mv_3d/3d/src/mv_3d_open.cpp @@ -24,7 +24,6 @@ #include "mv_3d_open.h" #include "Mv3d.h" -//using namespace mediavision::depth; using namespace mediavision::mv3d; int mv3dCreate(mv_3d_h *mv3d) @@ -67,7 +66,7 @@ int mv3dSetDepthParameters(mv_3d_h mv3d, mv_engine_config_h engine_config) auto pMv3d = static_cast(mv3d); - pMv3d->mDepth.SetParameters(127.5, // threshold + pMv3d->SetParameters(127.5, // threshold 3, // aggregation window width 3, // aggregation window height 0 // speckleSize @@ -142,15 +141,57 @@ int mv3dConfigure(mv_3d_h mv3d, mv_engine_config_h engine_config) return ret; } + double samplingRatio; + ret = mv_engine_config_get_double_attribute( + engine_config, MV_3D_POINTCLOUD_SAMPLING_RATIO,&samplingRatio); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to get stereo configuration file path"); + return ret; + } + + int outlierRemovalPoints; + ret = mv_engine_config_get_int_attribute( + engine_config, MV_3D_POINTCLOUD_OUTLIER_REMOVAL_POINTS, &outlierRemovalPoints); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to get stereo configuration file path"); + return ret; + } + + double outlierRemovalRadius; + ret = mv_engine_config_get_double_attribute( + engine_config, MV_3D_POINTCLOUD_OUTLIER_REMOVAL_RADIUS, &outlierRemovalRadius); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to get stereo configuration file path"); + return ret; + } + + char *pointcloudOutputFilePath = NULL; + ret = mv_engine_config_get_string_attribute( + engine_config, MV_3D_POINTCLOUD_OUTPUT_FILE_PATH, &pointcloudOutputFilePath); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to get stereo configuration file path"); + if (pointcloudOutputFilePath) { + free(pointcloudOutputFilePath); + pointcloudOutputFilePath = NULL; + } + return ret; + } + auto pMv3d = static_cast(mv3d); - ret = pMv3d->mDepth.Configure(mode, depthWidth, depthHeight, minDisp, maxDisp, - stereoConfigFilePath); + ret = pMv3d->Configure(mode, depthWidth, depthHeight, minDisp, maxDisp, + samplingRatio, outlierRemovalPoints, outlierRemovalRadius, + stereoConfigFilePath, pointcloudOutputFilePath); if (stereoConfigFilePath) { free(stereoConfigFilePath); stereoConfigFilePath = NULL; } + if (pointcloudOutputFilePath) { + free(pointcloudOutputFilePath); + pointcloudOutputFilePath = NULL; + } + if (ret != MEDIA_VISION_ERROR_NONE) { LOGE("Failed to configure Depth"); return ret; @@ -176,13 +217,35 @@ int mv3dSetDepthCallback(mv_3d_h mv3d, mv_3d_depth_cb depth_cb, void *user_data) } auto pMv3d = static_cast(mv3d); - pMv3d->mDepth.SetCallback(depth_cb, user_data); + pMv3d->SetDepthCallback(depth_cb, user_data); LOGI("LEAVE"); return MEDIA_VISION_ERROR_NONE; } + +int mv3dSetPointcloudCallback(mv_3d_h mv3d, mv_3d_pointcloud_cb pointcloud_cb, void *user_data) +{ + LOGI("ENTER"); + + if (!mv3d) { + LOGE("Handle is NULL"); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + + if (!pointcloud_cb) { + LOGE("Callbakc is NULL"); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + + auto pMv3d = static_cast(mv3d); + pMv3d->SetPointcloudCallback(pointcloud_cb, user_data); + + LOGI("LEAVE"); + return MEDIA_VISION_ERROR_NONE; +} + int mv3dPrepare(mv_3d_h mv3d) { LOGI("ENTER"); @@ -193,7 +256,7 @@ int mv3dPrepare(mv_3d_h mv3d) } auto pMv3d = static_cast(mv3d); - int ret = pMv3d->mDepth.Prepare(); + int ret = pMv3d->Prepare(); if (ret != MEDIA_VISION_ERROR_NONE) { LOGE("Failed to prepare depth"); return ret; @@ -204,9 +267,9 @@ int mv3dPrepare(mv_3d_h mv3d) return ret; } -int mv3dRun(mv_source_h source, - mv_source_h source_extra, - mv_3d_h mv3d) +int mv3dRun(mv_3d_h mv3d, + mv_source_h source, + mv_source_h source_extra) { LOGI("ENTER"); @@ -218,7 +281,7 @@ int mv3dRun(mv_source_h source, auto pMv3d = static_cast(mv3d); - int ret = pMv3d->mDepth.Run(source, source_extra); + int ret = pMv3d->Run(source, source_extra); if (ret != MEDIA_VISION_ERROR_NONE) { LOGE("Failed to run depth"); return ret; @@ -229,9 +292,9 @@ int mv3dRun(mv_source_h source, return ret; } -int mv3dRunAsync(mv_source_h source, - mv_source_h source_extra, - mv_3d_h mv3d) +int mv3dRunAsync(mv_3d_h mv3d, + mv_source_h source, + mv_source_h source_extra) { LOGI("ENTER"); @@ -242,7 +305,7 @@ int mv3dRunAsync(mv_source_h source, } auto pMv3d = static_cast(mv3d); - int ret = pMv3d->mDepth.RunAsync(source, source_extra); + int ret = pMv3d->RunAsync(source, source_extra); if (ret != MEDIA_VISION_ERROR_NONE) { LOGE("Failed to run depth"); return ret; @@ -251,4 +314,25 @@ int mv3dRunAsync(mv_source_h source, LOGI("LEAVE"); return ret; -} \ No newline at end of file +} + +int mv3dWritePointcloudFile(mv_3d_h mv3d, mv_3d_pointcloud_h pointcloud, mv_3d_pointcloud_type_e type, char *fileName) +{ + LOGI("ENTER"); + + if (!mv3d) { + LOGE("Handle is NULL"); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + + auto pMv3d = static_cast(mv3d); + int ret = pMv3d->WritePointcloudFile(pointcloud, type, fileName); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to write pointcloud file"); + return ret; + } + + LOGI("LEAVE"); + + return ret; +} diff --git a/test/testsuites/mv3d/CMakeLists.txt b/test/testsuites/mv3d/CMakeLists.txt index 38abf66..8509a3d 100644 --- a/test/testsuites/mv3d/CMakeLists.txt +++ b/test/testsuites/mv3d/CMakeLists.txt @@ -18,7 +18,6 @@ endif() SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format-truncation") add_executable(${PROJECT_NAME} depth_test_suite.cpp) target_link_libraries(${PROJECT_NAME} ${MV_3D_LIB_NAME} - ${Open3D_LIBRARIES} ${OpenCV_LIBS} mv_image_helper mv_video_helper @@ -26,7 +25,6 @@ target_link_libraries(${PROJECT_NAME} ${MV_3D_LIB_NAME} if(BUILD_VISUALIZER) target_link_libraries(${PROJECT_NAME} mv_visualizer) endif() -target_include_directories(${PROJECT_NAME} PUBLIC ${Open3D_INCLUDE_DIRS}) install(TARGETS ${PROJECT_NAME} DESTINATION ${CMAKE_INSTALL_BINDIR}) ## mv_depthstream @@ -54,7 +52,6 @@ SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") add_executable(mv_depthstream_test_suite depthstream_test_suite.cpp) target_link_libraries(mv_depthstream_test_suite ${MV_3D_LIB_NAME} - ${Open3D_LIBRARIES} ${OpenCV_LIBS} gstreamer-1.0 glib-2.0 @@ -67,5 +64,4 @@ if(BUILD_VISUALIZER) target_link_libraries(mv_depthstream_test_suite mv_visualizer) endif() -target_include_directories(mv_depthstream_test_suite PUBLIC ${Open3D_INCLUDE_DIRS}) install(TARGETS mv_depthstream_test_suite DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/test/testsuites/mv3d/depth_test_suite.cpp b/test/testsuites/mv3d/depth_test_suite.cpp index 71bbe3d..e062039 100644 --- a/test/testsuites/mv3d/depth_test_suite.cpp +++ b/test/testsuites/mv3d/depth_test_suite.cpp @@ -26,14 +26,15 @@ #include #include #include -#include #include #include #include #include "mv_3d.h" +#include "mv_3d_internal.h" +#define MEDIA_FILE_PATH "/opt/usr/home/owner/media/Images" #ifdef BUILD_VISUALIZER #include "mv_util_visualizer_3d.h" #endif @@ -44,8 +45,6 @@ #define __max(a,b) (((a) > (b)) ? (a) : (b)) #define __min(a,b) (((a) < (b)) ? (a) : (b)) -using namespace open3d; - /* There are calib.txt, im0.png, and im1.png in each dataset directories.*/ static const char* dataset[] = { "Adirondack", @@ -90,6 +89,7 @@ private: }; typedef struct _appdata { + mv_3d_h mv3d; std::chrono::milliseconds diffMs; std::string dataPath; std::string intrinsicName; @@ -98,6 +98,7 @@ typedef struct _appdata { float minDisp; float maxDisp; int fmt; + std::string dataset; } appdata; enum { @@ -201,111 +202,6 @@ void WritePLY(double *data, int size, const char* filename) return; } -void WritePointCloud(const char* data_path, - const char* color_filename, - const char* depth_filename, - unsigned int width, - unsigned int height, - const camera::PinholeCameraIntrinsic& intrinsic) -{ - geometry::Image img_color, img_depth; - - io::ReadImage(color_filename, img_color); - io::ReadImage(depth_filename, img_depth); - - utility::LogInfo("depth filename : {}", depth_filename); - utility::LogInfo("Reading RGBD image : "); - utility::LogInfo(" Color : {:d} x {:d} x {:d} ({:d} bits per channel)", - img_color.width_, img_color.height_, img_color.num_of_channels_, - img_color.bytes_per_channel_ * 8); - utility::LogInfo(" Depth : {:d} x {:d} x {:d} ({:d} bits per channel)", - img_depth.width_, img_depth.height_, img_depth.num_of_channels_, - img_depth.bytes_per_channel_ * 8); - double depth_scale = 1000.0, depth_trunc = 200.0; - bool convert_rgb_to_intensity = false; - std::shared_ptr rgbd_image = - geometry::RGBDImage::CreateFromColorAndDepth( - img_color, img_depth, depth_scale, depth_trunc, - convert_rgb_to_intensity); - - auto pcd = geometry::PointCloud::CreateFromRGBDImage(*rgbd_image, intrinsic); - std::string dataPath = std::string(data_path); - const std::string filename_ply(std::string(data_path) + std::string("/") + - dataPath.substr(dataPath.find_last_of("/\\") + 1) + - std::string(".ply")); - - utility::LogInfo("pcd counts : {}", pcd->points_.size()); -#ifndef BUILD_VISUALIZER - if (io::WritePointCloud(filename_ply, *pcd)) { - utility::LogInfo("Successfully wrote {}", filename_ply); - } else { - utility::LogError("Failed to write {}", filename_ply); - } -#else - unsigned int i, x, y, idx; - mv_source_h source = NULL; - int err = MEDIA_VISION_ERROR_NONE; - - cv::Mat rgb; - float *depth = NULL; - - Eigen::Vector3d min_xyz = pcd->GetMinBound(); - Eigen::Vector3d max_xyz = pcd->GetMaxBound(); - - int depth_width = (int)((max_xyz[0] - min_xyz[0]) * 100.0) + 1; - int depth_height = (int)((max_xyz[1] - min_xyz[1]) * 100.0) + 1; - - utility::LogInfo("min {}, max {}", min_xyz, max_xyz); - utility::LogInfo("depth W:H {}:{}", depth_width, depth_height); - - err = mv_create_source(&source); - if (MEDIA_VISION_ERROR_NONE != err) { - LOGE("Errors were occurred during creating the source!!! code : %i", err); - goto out; - } - depth = (float *)malloc(depth_width * depth_height * 4); - - rgb = cv::imread(color_filename, cv::IMREAD_COLOR); - cv::resize(rgb, rgb, cv::Size(depth_width, depth_height)); - cv::flip(rgb, rgb, 0); - cv::cvtColor(rgb, rgb, cv::COLOR_BGR2RGBA); - err = mv_source_fill_by_buffer(source, - rgb.ptr(), - rgb.elemSize() * rgb.size().width - * rgb.size().height, - rgb.size().width, - rgb.size().height, - MEDIA_VISION_COLORSPACE_RGB888); - if (MEDIA_VISION_ERROR_NONE != err) { - LOGE("Errors were occurred during filling the source!!! code : %i", err); - goto out; - } - - for (i = 0; i < pcd->points_.size(); i++) { - x = (int)((pcd->points_.data()[i][0] - min_xyz[0]) * 100.0); - y = (int)((max_xyz[1] - pcd->points_.data()[i][1]) * 100.0); - idx = y * depth_width + x; - depth[idx] = max_xyz[2] - pcd->points_.data()[i][2]; - } - if (source != NULL && depth != NULL) - while(1) - mv_util_visualizer_3d(source, depth, 100, 25); - -out: - if (depth != NULL) { - delete [] depth; - depth = NULL; - } - - if (source != NULL) { - err = mv_destroy_source(source); - if (MEDIA_VISION_ERROR_NONE != err) { - LOGE("Errors were occurred during destroying the source!!! code : %i", err); - } - } -#endif -} - void _depth_middlebury_cb(mv_source_h source, unsigned short* depth, unsigned int width, @@ -329,28 +225,34 @@ void _depth_middlebury_cb(mv_source_h source, udata->minDisp, udata->maxDisp); } - std::string intrinsic_path; - intrinsic_path = udata->intrinsicName; - utility::LogInfo("Camera intrinsic path {}", intrinsic_path); - - camera::PinholeCameraIntrinsic intrinsic; - if (intrinsic_path.empty() || - !io::ReadIJsonConvertible(intrinsic_path, intrinsic)) { - utility::LogWarning( - "Failed to read intrinsic parameters for depth image."); - utility::LogWarning("Using default value for Primesense camera."); - intrinsic = camera::PinholeCameraIntrinsic( - camera::PinholeCameraIntrinsicParameters::PrimeSenseDefault); + udata->diffMs = stopWatch.elapsedTime(); +} + +void _pointcloud_middlebury_cb(mv_source_h source, + mv_3d_pointcloud_h pointcloud, + void *user_data) +{ + if (!user_data) { + printf("user_data is null. Skip..\n"); + return; } - utility::LogInfo("focal_x = {}", intrinsic.GetFocalLength().first); - utility::LogInfo("focal_y = {}", intrinsic.GetFocalLength().second); - utility::LogInfo("cx = {}", intrinsic.GetPrincipalPoint().first); - utility::LogInfo("cy = {}", intrinsic.GetPrincipalPoint().second); - WritePointCloud(udata->dataPath.c_str(), udata->rgbName.c_str(), - udata->datasetName.c_str(), width, height, intrinsic); + auto udata = static_cast(user_data); + std::string filename = udata->dataset + std::string(".pcd"); + mv_3d_pointcloud_write_file(udata->mv3d, pointcloud, MV_3D_POINTCLOUD_TYPE_PCD_BIN, (char*)filename.c_str()); - udata->diffMs = stopWatch.elapsedTime(); + mv_3d_pointcloud_plane_model_h model; + mv_3d_pointcloud_plane_inlier_h inlier; + + mv_3d_pointcloud_plane_model_create(&model); + mv_3d_pointcloud_plane_inlier_create(&inlier); + mv_3d_pointcloud_segment_plane(udata->mv3d, pointcloud, &model, &inlier); + + filename = udata->dataset + std::string("-plane.pcd"); + mv_3d_pointcloud_plane_write_file(udata->mv3d, model, inlier, pointcloud, MV_3D_POINTCLOUD_TYPE_PCD_BIN, (char*)filename.c_str()); + + mv_3d_pointcloud_plane_model_destroy(model); + mv_3d_pointcloud_plane_inlier_destroy(inlier); } int perform_middlebury_test() @@ -405,6 +307,10 @@ int perform_middlebury_test() char leftFilename[MAX_STRING_LENGTH]; char rightFilename[MAX_STRING_LENGTH]; char stereoConfigFileName[MAX_STRING_LENGTH]; + double samplingRatio = 0.01; + int outlierRemovalPoints = 3; + double outlierRemovalRadius = 0.5; + snprintf(dataPath, MAX_STRING_LENGTH, "%s/%s", path_to_dataset, dataset[data]); snprintf(calibFilename, MAX_STRING_LENGTH, "%s/calib.txt", dataPath); snprintf(stereoConfigFileName, MAX_STRING_LENGTH, "%s/calibOcv.xml", dataPath); @@ -481,9 +387,37 @@ int perform_middlebury_test() goto _err; } + err = mv_engine_config_set_double_attribute(engine_config, + MV_3D_POINTCLOUD_SAMPLING_RATIO, samplingRatio); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to set stereo config file path\n"); + goto _err; + } + + err = mv_engine_config_set_int_attribute(engine_config, + MV_3D_POINTCLOUD_OUTLIER_REMOVAL_POINTS, outlierRemovalPoints); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to set stereo config file path\n"); + goto _err; + } + + err = mv_engine_config_set_double_attribute(engine_config, + MV_3D_POINTCLOUD_OUTLIER_REMOVAL_RADIUS, outlierRemovalRadius); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to set stereo config file path\n"); + goto _err; + } + + err = mv_engine_config_set_string_attribute(engine_config, + MV_3D_POINTCLOUD_OUTPUT_FILE_PATH, MEDIA_FILE_PATH); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to set stereo config file path\n"); + goto _err; + } + // left_source, right_source - left_frame = cv::imread(leftFilename, cv::IMREAD_GRAYSCALE); - right_frame = cv::imread(rightFilename, cv::IMREAD_GRAYSCALE); + left_frame = cv::imread(leftFilename, cv::IMREAD_COLOR); + right_frame = cv::imread(rightFilename, cv::IMREAD_COLOR); err = mv_source_fill_by_buffer(left_source, left_frame.ptr(), @@ -491,7 +425,7 @@ int perform_middlebury_test() * left_frame.size().height, left_frame.size().width, left_frame.size().height, - MEDIA_VISION_COLORSPACE_Y800); + MEDIA_VISION_COLORSPACE_RGB888); if (err != MEDIA_VISION_ERROR_NONE) { printf("Failed to fill left_source\n"); goto _err; @@ -503,7 +437,7 @@ int perform_middlebury_test() * right_frame.size().height, right_frame.size().width, right_frame.size().height, - MEDIA_VISION_COLORSPACE_Y800); + MEDIA_VISION_COLORSPACE_RGB888); if (err != MEDIA_VISION_ERROR_NONE) { printf("Failed to fill right_source\n"); goto _err; @@ -524,6 +458,7 @@ int perform_middlebury_test() // get depth appdata dump { + mv3d_handle, std::chrono::milliseconds::zero(), std::string(dataPath), std::string(dataPath) + std::string("/intrinsics.json"), @@ -534,10 +469,17 @@ int perform_middlebury_test() sel_fmt }; dump.datasetName += sel_fmt == FMT_PFM ? std::string(".pfm") : std::string(".png"); + dump.dataset = std::string(dataset[data]); err = mv_3d_set_depth_cb(mv3d_handle, _depth_middlebury_cb, static_cast(&dump)); if (err != MEDIA_VISION_ERROR_NONE) { - printf("Failed to set callback to depth handle\n"); + printf("Failed to set depth callback to handle\n"); + goto _err; + } + + err = mv_3d_set_pointcloud_cb(mv3d_handle, _pointcloud_middlebury_cb, static_cast(&dump)); + if (err != MEDIA_VISION_ERROR_NONE) { + printf("Failed to set pointcloud callback to handle\n"); goto _err; } @@ -585,8 +527,6 @@ int main() int err = MEDIA_VISION_ERROR_NONE; const char* names[] = {"Middlebury - TrainingQ(Imperf)"}; - utility::SetVerbosityLevel(utility::VerbosityLevel::Debug); - int sel_opt = show_menu_linear("Select Action", names, ARRAY_SIZE(names)); if (sel_opt <= 0 || sel_opt > ARRAY_SIZE(names)) { printf("Invalid option"); diff --git a/test/testsuites/mv3d/depthstream_test_suite.cpp b/test/testsuites/mv3d/depthstream_test_suite.cpp index c2441bc..4051ef5 100644 --- a/test/testsuites/mv3d/depthstream_test_suite.cpp +++ b/test/testsuites/mv3d/depthstream_test_suite.cpp @@ -370,10 +370,10 @@ static void stereo_handoff(GstElement *object, GstBuffer *buffer, GstPad *pad, gst_buffer_unmap(buffer, &map); if (isAsync) { - ret = mv_3d_run_async(mv3d_handle, mv_source, NULL, NULL); + ret = mv_3d_run_async(mv_source, NULL, NULL, mv3d_handle); __wait(); } else { - ret = mv_3d_run(mv3d_handle, mv_source, NULL, NULL); + ret = mv_3d_run(mv_source, NULL, NULL, mv3d_handle); } if (ret != MEDIA_VISION_ERROR_NONE) { -- 2.7.4 From 181c880d30031dd16a6a2a736cb1724baba73d75 Mon Sep 17 00:00:00 2001 From: Inki Dae Date: Wed, 21 Sep 2022 18:00:07 +0900 Subject: [PATCH 06/16] mv3d: fix build warning and cleanup [Issue type] : code cleanup Change-Id: I86daac442b5aadb0b40ead94f28b10fc4944ffed Signed-off-by: Inki Dae --- .../src/{mv_3d internal.cpp => mv_3d_internal.cpp} | 1 - .../inference/src/ObjectDecoder.cpp | 2 -- test/testsuites/mv3d/depth_test_suite.cpp | 8 +++---- test/testsuites/mv3d/depthstream_test_suite.cpp | 26 ++-------------------- 4 files changed, 6 insertions(+), 31 deletions(-) rename mv_3d/3d/src/{mv_3d internal.cpp => mv_3d_internal.cpp} (98%) diff --git a/mv_3d/3d/src/mv_3d internal.cpp b/mv_3d/3d/src/mv_3d_internal.cpp similarity index 98% rename from mv_3d/3d/src/mv_3d internal.cpp rename to mv_3d/3d/src/mv_3d_internal.cpp index 4c52156..40b6237 100644 --- a/mv_3d/3d/src/mv_3d internal.cpp +++ b/mv_3d/3d/src/mv_3d_internal.cpp @@ -128,7 +128,6 @@ int mv_3d_pointcloud_plane_write_file(mv_3d_h mv3d, MEDIA_VISION_FUNCTION_ENTER(); mv_3d_pointcloud_s *s = (mv_3d_pointcloud_s*) pointcloud; - Eigen::Vector4d *best_plane_model = (Eigen::Vector4d*) model; std::vector *plane_inlier = (std::vector*) inlier; auto pPcd = static_cast*>(s->pointcloud); open3d::geometry::PointCloud p; diff --git a/mv_machine_learning/inference/src/ObjectDecoder.cpp b/mv_machine_learning/inference/src/ObjectDecoder.cpp index 06e8653..82939f2 100644 --- a/mv_machine_learning/inference/src/ObjectDecoder.cpp +++ b/mv_machine_learning/inference/src/ObjectDecoder.cpp @@ -165,8 +165,6 @@ int ObjectDecoder::decode() boxes.push_back(box); } boxList.push_back(boxes); - } else { // INFERENCE_BOX_DECODING_TYPE_YOLO_ANCHOR - int cellIdx = idx * mBoxOffset; } } diff --git a/test/testsuites/mv3d/depth_test_suite.cpp b/test/testsuites/mv3d/depth_test_suite.cpp index e062039..26997e5 100644 --- a/test/testsuites/mv3d/depth_test_suite.cpp +++ b/test/testsuites/mv3d/depth_test_suite.cpp @@ -279,7 +279,7 @@ int perform_middlebury_test() const char* formats[] = {"pfm", "png"}; int sel_fmt = -1; - while(sel_fmt <= 0 || sel_fmt > ARRAY_SIZE(formats)) { + while(sel_fmt <= 0 || sel_fmt > static_cast(ARRAY_SIZE(formats))) { sel_fmt = show_menu_linear("Select Action", formats, ARRAY_SIZE(formats)); } @@ -328,9 +328,9 @@ int perform_middlebury_test() FILE* fp = fopen(calibFilename, "r"); char line[MAX_STRING_LENGTH]; float fVal; - float dmin, dmax; + float dmin = 0, dmax = 0; int iVal; - int width, height; + int width = 0, height = 0; if (fp != NULL) { while (fgets(line, MAX_STRING_LENGTH, fp) != NULL) { if (sscanf(line, "vmin= %f", &fVal) == 1) dmin = fVal; @@ -528,7 +528,7 @@ int main() const char* names[] = {"Middlebury - TrainingQ(Imperf)"}; int sel_opt = show_menu_linear("Select Action", names, ARRAY_SIZE(names)); - if (sel_opt <= 0 || sel_opt > ARRAY_SIZE(names)) { + if (sel_opt <= 0 || sel_opt > static_cast(ARRAY_SIZE(names))) { printf("Invalid option"); return -1; } diff --git a/test/testsuites/mv3d/depthstream_test_suite.cpp b/test/testsuites/mv3d/depthstream_test_suite.cpp index 4051ef5..1f43e46 100644 --- a/test/testsuites/mv3d/depthstream_test_suite.cpp +++ b/test/testsuites/mv3d/depthstream_test_suite.cpp @@ -80,7 +80,6 @@ typedef struct _appdata } appdata; static float *vertex3d = NULL; -static unsigned char *data_buffer_3d = NULL; mv_source_h mv_source3d; gulong handler; @@ -144,7 +143,6 @@ static int minDisp; static int maxDisp; static int display_xpos; static int display_ypos; -static int fidx = 0; void int_handler(int sig) { char c; @@ -156,10 +154,6 @@ void int_handler(int sig) if (c == 'y' || c == 'Y') { g_signal_handler_disconnect(fsink, handler); printf("delete buffer, used by visualize 3d\n"); - //if (data_buffer_3d != NULL) { - // delete [] data_buffer_3d; - // data_buffer_3d = NULL; - //} gst_element_send_event(pipecam, gst_event_new_eos()); gst_element_send_event(pipedepth, gst_event_new_eos()); @@ -257,22 +251,6 @@ void WritePointCloud(const char* data_path, void _depth_stereo_cb(mv_source_h source, unsigned short *depth, unsigned int width, unsigned int height, void *user_data) { -#if 0 - if (!user_data) { - LOGW("user_data is null, Skip..."); - return; - } - - GstMapInfo *buffer = static_cast(user_data); - - - for (int j = 0; j < height; j++) { - for (int i = 0; i < width; ++i) { - buffer->data[j * width + i] = - static_cast(depth[j * width + i]); - } - } -#endif double maxDepth = 2000.0; double minDepth = 175.0; double depthDenom = static_cast(maxDepth - minDepth); @@ -299,8 +277,8 @@ void _depth_stereo_cb(mv_source_h source, unsigned short *depth, unsigned int wi false); cv::Mat colorMap = cv::Mat(cv::Size(width,height), CV_8UC1); - for (int j = 0; j < height; ++j) { - for (int i = 0; i < width; ++i) { + for (unsigned int j = 0; j < height; ++j) { + for (unsigned int i = 0; i < width; ++i) { int idx = j * width + i; int idxInv = width * height - idx - 1; auto depth_raw = pcd->points_.data()[idx][2] * 1000.0; -- 2.7.4 From 79c1c15ac88bbd2e3c5c5105520b90cd81cf3b13 Mon Sep 17 00:00:00 2001 From: sangho park Date: Wed, 21 Sep 2022 20:10:49 +0900 Subject: [PATCH 07/16] mv3d: add MV_3D_POINTCLOUD_AVAILABLE macro [Issue type] : supports pointcloud feature for aarch64 only Change-Id: I32323b3e587f17b74d06fe1d2058f20084ebdaae Signed-off-by: sangho park --- mv_3d/3d/CMakeLists.txt | 20 +++++--- mv_3d/3d/include/Mv3d.h | 4 +- mv_3d/3d/src/Mv3d.cpp | 37 +++++++++++---- mv_3d/3d/src/mv_3d_internal.cpp | 62 ++++++++++++++++++++++--- mv_3d/3d/src/mv_3d_open.cpp | 6 ++- packaging/capi-media-vision.spec | 5 ++ test/testsuites/mv3d/CMakeLists.txt | 18 +++---- test/testsuites/mv3d/depthstream_test_suite.cpp | 11 ++++- 8 files changed, 131 insertions(+), 32 deletions(-) diff --git a/mv_3d/3d/CMakeLists.txt b/mv_3d/3d/CMakeLists.txt index 2265cbd..5d2e81a 100644 --- a/mv_3d/3d/CMakeLists.txt +++ b/mv_3d/3d/CMakeLists.txt @@ -10,12 +10,14 @@ if(NOT OpenCV_FOUND) return() endif() -find_package(Open3D REQUIRED NO_POLICY_SCOPE) -if(NOT Open3D_FOUND) - message(SEND_ERROR "Open3D NOT FOUND") - return() -else() - include_directories(${Open3D_INCLUDE_DIRS}) +if(MV_3D_POINTCLOUD_IS_AVAILABLE) + find_package(Open3D REQUIRED NO_POLICY_SCOPE) + if(NOT Open3D_FOUND) + message(SEND_ERROR "Open3D NOT FOUND") + return() + else() + include_directories(${Open3D_INCLUDE_DIRS}) + endif() endif() if(FORCED_STATIC_BUILD) @@ -24,6 +26,10 @@ else() add_library(${PROJECT_NAME} SHARED ${MV_DFS_SOURCE_LIST}) endif() -target_link_libraries(${PROJECT_NAME} ${MV_COMMON_LIB_NAME} ${OpenCV_LIBS} ${Open3D_LIBRARIES} ${${PROJECT_NAME}_DEP_LIBRARIES}) +target_link_libraries(${PROJECT_NAME} ${MV_COMMON_LIB_NAME} ${OpenCV_LIBS} ${${PROJECT_NAME}_DEP_LIBRARIES}) + +if(MV_3D_POINTCLOUD_IS_AVAILABLE) + target_link_libraries(${PROJECT_NAME} ${Open3D_LIBRARIES}) +endif() target_include_directories(${PROJECT_NAME} PRIVATE include) install(TARGETS ${PROJECT_NAME} DESTINATION ${LIB_INSTALL_DIR}) diff --git a/mv_3d/3d/include/Mv3d.h b/mv_3d/3d/include/Mv3d.h index 8e345a8..df04d92 100644 --- a/mv_3d/3d/include/Mv3d.h +++ b/mv_3d/3d/include/Mv3d.h @@ -78,9 +78,11 @@ namespace mv3d static gpointer DfsThreadLoop(gpointer data); +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE void GetPointcloudFromSource(DfsInputData &intput, DfsOutputData &depthData, mv_3d_pointcloud_s &pointcloud); +#endif public: Mv3d(); ~Mv3d(); @@ -95,7 +97,7 @@ namespace mv3d void SetDepthCallback(mv_3d_depth_cb depthCallback, void *depthUserData); - void SetPointcloudCallback(mv_3d_pointcloud_cb pointcloudCallback, void *pointcloudUserData); + int SetPointcloudCallback(mv_3d_pointcloud_cb pointcloudCallback, void *pointcloudUserData); int WritePointcloudFile(mv_3d_pointcloud_h pointcloud, mv_3d_pointcloud_type_e type, char *fileName); diff --git a/mv_3d/3d/src/Mv3d.cpp b/mv_3d/3d/src/Mv3d.cpp index 747a786..521c947 100644 --- a/mv_3d/3d/src/Mv3d.cpp +++ b/mv_3d/3d/src/Mv3d.cpp @@ -21,12 +21,14 @@ #include #include #include + +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE #include +using namespace open3d; +#endif #include "Mv3d.h" -using namespace open3d; - namespace mediavision { namespace mv3d @@ -51,7 +53,9 @@ namespace mv3d mDfsAsyncQueue(nullptr) { LOGI("ENTER"); +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE utility::SetVerbosityLevel(utility::VerbosityLevel::Debug); +#endif LOGI("LEAVE"); } @@ -126,11 +130,23 @@ namespace mv3d mDepthUserData = depthUserData; } - void Mv3d::SetPointcloudCallback(mv_3d_pointcloud_cb pointcloudCallback, void *pointcloudUserData) +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE + int Mv3d::SetPointcloudCallback(mv_3d_pointcloud_cb pointcloudCallback, void *pointcloudUserData) { + if (pointcloudCallback == NULL || pointcloudUserData == NULL) + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + mPointcloudCallback = pointcloudCallback; mPointcloudUserData = pointcloudUserData; + + return MEDIA_VISION_ERROR_NONE; } +#else + int Mv3d::SetPointcloudCallback(mv_3d_pointcloud_cb pointcloudCallback, void *pointcloudUserData) + { + return MEDIA_VISION_ERROR_NOT_SUPPORTED; + } +#endif int Mv3d::Prepare() { @@ -224,6 +240,7 @@ namespace mv3d } } +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE void Mv3d::GetPointcloudFromSource(DfsInputData &input, DfsOutputData &depthData, mv_3d_pointcloud_s &pointcloud) @@ -268,8 +285,8 @@ namespace mv3d pointcloud.pointcloud = static_cast( new std::shared_ptr( std::move(pcd))); - } +#endif int Mv3d::Run(mv_source_h baseSource, mv_source_h extraSource) { @@ -296,7 +313,7 @@ namespace mv3d static_cast(depthData.data), depthData.width, depthData.height, mDepthUserData); - +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE if (mPointcloudCallback) { mv_3d_pointcloud_s p = {.type = MV_3D_POINTCLOUD_TYPE_PCD_BIN, .pointcloud = NULL}; GetPointcloudFromSource(input, depthData, p); @@ -309,6 +326,7 @@ namespace mv3d auto _pcd = std::move(pPcd); delete _pcd; } +#endif delete [] static_cast(input.data); delete [] static_cast(input.extraData); } catch (const std::exception &e) { @@ -375,7 +393,7 @@ namespace mv3d static_cast(depthData.data), depthData.width, depthData.height, handle->mDepthUserData); - +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE if (handle->mPointcloudCallback) { mv_3d_pointcloud_s p = {.type = MV_3D_POINTCLOUD_TYPE_PCD_BIN, .pointcloud = NULL}; //mPointcloudThread = g_thread_new("pointcloud_thread", @@ -391,7 +409,7 @@ namespace mv3d auto _pcd = std::move(pPcd); delete _pcd; } - +#endif delete [] static_cast(input->data); delete [] static_cast(input->extraData); input.reset(); @@ -402,6 +420,7 @@ namespace mv3d int Mv3d::WritePointcloudFile(mv_3d_pointcloud_h pointcloud, mv_3d_pointcloud_type_e type, char *fileName) { +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE mv_3d_pointcloud_s *s = (mv_3d_pointcloud_s*) pointcloud; if (s == NULL) { LOGE("Pointcloud data is NULL"); @@ -436,7 +455,9 @@ namespace mv3d utility::LogError("Failed to write {}", fullPath.c_str()); return MEDIA_VISION_ERROR_INTERNAL; } - +#else + return MEDIA_VISION_ERROR_NOT_SUPPORTED; +#endif return MEDIA_VISION_ERROR_NONE; } } diff --git a/mv_3d/3d/src/mv_3d_internal.cpp b/mv_3d/3d/src/mv_3d_internal.cpp index 40b6237..95870f8 100644 --- a/mv_3d/3d/src/mv_3d_internal.cpp +++ b/mv_3d/3d/src/mv_3d_internal.cpp @@ -15,7 +15,9 @@ */ #include +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE #include +#endif #include "mv_3d_internal.h" #include "mv_private.h" @@ -26,9 +28,12 @@ #define RANSAC_NUMBER 3 #define NUM_ITERATIONS 1000 +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE using namespace open3d; using namespace mediavision::mv3d; +#endif +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE int mv_3d_pointcloud_plane_model_create(mv_3d_pointcloud_plane_model_h *handle) { MEDIA_VISION_NULL_ARG_CHECK(handle); @@ -41,7 +46,14 @@ int mv_3d_pointcloud_plane_model_create(mv_3d_pointcloud_plane_model_h *handle) return MEDIA_VISION_ERROR_NONE; } +#else +int mv_3d_pointcloud_plane_model_create(mv_3d_pointcloud_plane_model_h *handle) +{ + return MEDIA_VISION_ERROR_NOT_SUPPORTED; +} +#endif +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE int mv_3d_pointcloud_plane_model_destroy(mv_3d_pointcloud_plane_model_h handle) { MEDIA_VISION_INSTANCE_CHECK(handle); @@ -55,7 +67,14 @@ int mv_3d_pointcloud_plane_model_destroy(mv_3d_pointcloud_plane_model_h handle) return MEDIA_VISION_ERROR_NONE; } +#else +int mv_3d_pointcloud_plane_model_destroy(mv_3d_pointcloud_plane_model_h handle) +{ + return MEDIA_VISION_ERROR_NOT_SUPPORTED; +} +#endif +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE int mv_3d_pointcloud_plane_inlier_create(mv_3d_pointcloud_plane_inlier_h *handle) { MEDIA_VISION_NULL_ARG_CHECK(handle); @@ -68,7 +87,14 @@ int mv_3d_pointcloud_plane_inlier_create(mv_3d_pointcloud_plane_inlier_h *handle return MEDIA_VISION_ERROR_NONE; } +#else +int mv_3d_pointcloud_plane_inlier_create(mv_3d_pointcloud_plane_inlier_h *handle) +{ + return MEDIA_VISION_ERROR_NOT_SUPPORTED; +} +#endif +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE int mv_3d_pointcloud_plane_inlier_destroy(mv_3d_pointcloud_plane_inlier_h handle) { MEDIA_VISION_INSTANCE_CHECK(handle); @@ -82,17 +108,20 @@ int mv_3d_pointcloud_plane_inlier_destroy(mv_3d_pointcloud_plane_inlier_h handle return MEDIA_VISION_ERROR_NONE; } +#else +int mv_3d_pointcloud_plane_inlier_destroy(mv_3d_pointcloud_plane_inlier_h handle) +{ + return MEDIA_VISION_ERROR_NOT_SUPPORTED; +} +#endif -/** - * @file mv_3d_internal.c - * @brief This file contains Media Vision 3D internal module. - */ - +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE int mv_3d_pointcloud_segment_plane(mv_3d_h mv3d, mv_3d_pointcloud_h pointcloud, mv_3d_pointcloud_plane_model_h *model, mv_3d_pointcloud_plane_inlier_h *inlier) { + MEDIA_VISION_NULL_ARG_CHECK(mv3d); MEDIA_VISION_NULL_ARG_CHECK(pointcloud); MEDIA_VISION_NULL_ARG_CHECK(model); @@ -114,7 +143,17 @@ int mv_3d_pointcloud_segment_plane(mv_3d_h mv3d, return MEDIA_VISION_ERROR_NONE; } +#else +int mv_3d_pointcloud_segment_plane(mv_3d_h mv3d, + mv_3d_pointcloud_h pointcloud, + mv_3d_pointcloud_plane_model_h *model, + mv_3d_pointcloud_plane_inlier_h *inlier) +{ + return MEDIA_VISION_ERROR_NOT_SUPPORTED; +} +#endif +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE int mv_3d_pointcloud_plane_write_file(mv_3d_h mv3d, mv_3d_pointcloud_plane_model_h model, mv_3d_pointcloud_plane_inlier_h inlier, @@ -150,4 +189,15 @@ int mv_3d_pointcloud_plane_write_file(mv_3d_h mv3d, MEDIA_VISION_FUNCTION_LEAVE(); return ret; -} \ No newline at end of file +} +#else +int mv_3d_pointcloud_plane_write_file(mv_3d_h mv3d, + mv_3d_pointcloud_plane_model_h model, + mv_3d_pointcloud_plane_inlier_h inlier, + mv_3d_pointcloud_h pointcloud, + mv_3d_pointcloud_type_e type, + char *filename) +{ + return MEDIA_VISION_ERROR_NOT_SUPPORTED; +} +#endif diff --git a/mv_3d/3d/src/mv_3d_open.cpp b/mv_3d/3d/src/mv_3d_open.cpp index 9f914d2..72021a2 100644 --- a/mv_3d/3d/src/mv_3d_open.cpp +++ b/mv_3d/3d/src/mv_3d_open.cpp @@ -240,7 +240,11 @@ int mv3dSetPointcloudCallback(mv_3d_h mv3d, mv_3d_pointcloud_cb pointcloud_cb, v } auto pMv3d = static_cast(mv3d); - pMv3d->SetPointcloudCallback(pointcloud_cb, user_data); + int ret = pMv3d->SetPointcloudCallback(pointcloud_cb, user_data); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to run depth"); + return ret; + } LOGI("LEAVE"); return MEDIA_VISION_ERROR_NONE; diff --git a/packaging/capi-media-vision.spec b/packaging/capi-media-vision.spec index 5853507..36761cc 100644 --- a/packaging/capi-media-vision.spec +++ b/packaging/capi-media-vision.spec @@ -33,7 +33,9 @@ BuildRequires: pkgconfig(wayland-egl) BuildRequires: pkgconfig(glesv2) BuildRequires: gtest-devel BuildRequires: pkgconfig(dfs-adaptation) +%ifarch aarch64 BuildRequires: Open3D-devel +%endif %if 0%{?visualizer:1} BuildRequires: pkgconfig(protobuf) BuildRequires: pkgconfig(grpc++) @@ -271,6 +273,9 @@ export LDFLAGS+=" -lgcov" %endif %cmake . -DCMAKE_BUILD_TYPE=RelWithDebInfo %{build_options} \ +%ifarch aarch64 +-DMMV_3D_POINTCLOUD_IS_AVAILABLE=TRUE \ +%endif %if 0%{?ml_only:1} -DBUILD_ML_ONLY=ON \ %endif diff --git a/test/testsuites/mv3d/CMakeLists.txt b/test/testsuites/mv3d/CMakeLists.txt index 8509a3d..47a1378 100644 --- a/test/testsuites/mv3d/CMakeLists.txt +++ b/test/testsuites/mv3d/CMakeLists.txt @@ -3,16 +3,18 @@ cmake_minimum_required(VERSION 2.6...3.13) find_package(OpenCV REQUIRED core videoio imgcodecs) if(NOT OpenCV_FOUND) - message(SEND_ERROR "OpenCV NOT FOUND") - return() + message(SEND_ERROR "OpenCV NOT FOUND") + return() endif() -find_package(Open3D REQUIRED NO_POLICY_SCOPE) -if(NOT Open3D_FOUND) - message(SEND_ERROR "Open3D NOT FOUND") - return() -else() - include_directories(${Open3D_INCLUDE_DIRS}) +if(MV_3D_POINTCLOUD_IS_AVAILABLE) + find_package(Open3D REQUIRED NO_POLICY_SCOPE) + if(NOT Open3D_FOUND) + message(SEND_ERROR "Open3D NOT FOUND") + return() + else() + include_directories(${Open3D_INCLUDE_DIRS}) + endif() endif() SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format-truncation") diff --git a/test/testsuites/mv3d/depthstream_test_suite.cpp b/test/testsuites/mv3d/depthstream_test_suite.cpp index 1f43e46..c8241b9 100644 --- a/test/testsuites/mv3d/depthstream_test_suite.cpp +++ b/test/testsuites/mv3d/depthstream_test_suite.cpp @@ -25,7 +25,9 @@ #include #include #include +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE #include +#endif #include #include @@ -46,7 +48,9 @@ #define MAX_FRAMES 1800 // 30 fps * 60s #define INTRINSIC_FILE_PATH "/usr/share/capi-media-vision/stereoCalibZedVGA.json" +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE using namespace open3d; +#endif class StopWatch { @@ -208,6 +212,7 @@ static gboolean bus_call(GstBus *bus, GstMessage *msg, gpointer data) return TRUE; } +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE void WritePointCloud(const char* data_path, const char* color_filename, const char* depth_filename, @@ -247,10 +252,12 @@ void WritePointCloud(const char* data_path, utility::LogError("Failed to write {}", filename_ply); } } +#endif void _depth_stereo_cb(mv_source_h source, unsigned short *depth, unsigned int width, unsigned int height, void *user_data) { +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE double maxDepth = 2000.0; double minDepth = 175.0; double depthDenom = static_cast(maxDepth - minDepth); @@ -325,6 +332,7 @@ void _depth_stereo_cb(mv_source_h source, unsigned short *depth, unsigned int wi delete [] vertex3d; vertex3d = NULL; __signal(); +#endif } static void stereo_handoff(GstElement *object, GstBuffer *buffer, GstPad *pad, @@ -479,8 +487,9 @@ int main(int argc, char *argv[]) display_xpos = atoi(argv[7]); display_ypos = atoi(argv[8]); } - +#ifdef MV_3D_POINTCLOUD_IS_AVAILABLE utility::SetVerbosityLevel(utility::VerbosityLevel::Debug); +#endif SrcData srcData; int ret = MEDIA_VISION_ERROR_NONE; -- 2.7.4 From 45c929981cd3a4c02d9f4b2765dda362a9cf5e31 Mon Sep 17 00:00:00 2001 From: Tae-Young Chung Date: Thu, 22 Sep 2022 13:21:45 +0900 Subject: [PATCH 08/16] mv3d: fix a bug using nullptr [Issue type] : bug fix (SVACE: WGID 499264) mDepthCallback uses deallocated memory of the variable 'base'. It should be mv_source_h of a left image. With dfs-adaptation change, ``` commit 1607630a5887d961cd71d0e9663e9945d70fc3ae Author: Tae-Young Chung Date: Thu Sep 22 13:19:04 2022 +0900 Add getLeftData() and getRigthData() [Version] 1.0.9-0 [Issue type] update ``` getLeftData() returns the pointer of a left image and the returned pointer is used to fill mv_source_h. Change-Id: I89af7686e485a102bc444720583bf26739e08809 --- mv_3d/3d/include/Mv3d.h | 2 ++ mv_3d/3d/src/Mv3d.cpp | 36 +++++++++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/mv_3d/3d/include/Mv3d.h b/mv_3d/3d/include/Mv3d.h index df04d92..8f3558c 100644 --- a/mv_3d/3d/include/Mv3d.h +++ b/mv_3d/3d/include/Mv3d.h @@ -65,6 +65,8 @@ namespace mv3d GAsyncQueue *mDfsAsyncQueue; + mv_source_h mInternalSource; + void GetBufferFromSource(mv_source_h source, unsigned char*& buffer, unsigned int& width, diff --git a/mv_3d/3d/src/Mv3d.cpp b/mv_3d/3d/src/Mv3d.cpp index 521c947..dc4c05c 100644 --- a/mv_3d/3d/src/Mv3d.cpp +++ b/mv_3d/3d/src/Mv3d.cpp @@ -50,7 +50,8 @@ namespace mv3d mDepthCallback(nullptr), mPointcloudCallback(nullptr), mDfsIsLive(false), - mDfsAsyncQueue(nullptr) + mDfsAsyncQueue(nullptr), + mInternalSource(nullptr) { LOGI("ENTER"); #ifdef MV_3D_POINTCLOUD_IS_AVAILABLE @@ -77,6 +78,11 @@ namespace mv3d delete mDfsAdaptor; } + if (mInternalSource) { + mv_destroy_source(mInternalSource); + mInternalSource = nullptr; + } + LOGI("LEAVE"); } @@ -303,6 +309,14 @@ namespace mv3d mDfsAsyncQueue = nullptr; } + if (mInternalSource) { + int ret = mv_destroy_source(mInternalSource); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Fail to destroy intern source. But keep going.."); + } + mInternalSource = nullptr; + } + GetDfsDataFromSources(baseSource, extraSource, input); mDfsAdaptor->run(input); @@ -362,6 +376,13 @@ namespace mv3d std::shared_ptr input(new DfsInputData); GetDfsDataFromSources(baseSource, extraSource, *input); + if (!mInternalSource) { + int ret = mv_create_source(&mInternalSource); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Fail to create internal source"); + return MEDIA_VISION_ERROR_INTERNAL; + } + } g_async_queue_push(mDfsAsyncQueue, static_cast( new std::shared_ptr( std::move(input)))); @@ -388,8 +409,16 @@ namespace mv3d handle->mDfsAdaptor->run(*input); auto depthData = handle->mDfsAdaptor->getOutputData(); + auto leftData = handle->mDfsAdaptor->getLeftData(); + mv_source_fill_by_buffer(handle->mInternalSource, + static_cast(leftData.data), + leftData.width, + leftData.height, + leftData.stride * leftData.height, + leftData.type == DFS_DATA_TYPE_UINT8C3 ? MEDIA_VISION_COLORSPACE_RGB888 : + MEDIA_VISION_COLORSPACE_Y800); handle->mDepthCallback( - static_cast(base), + static_cast(handle->mInternalSource), static_cast(depthData.data), depthData.width, depthData.height, handle->mDepthUserData); @@ -402,7 +431,7 @@ namespace mv3d handle->GetPointcloudFromSource(*input, depthData, p); mv_3d_pointcloud_h pcd = &p; - handle->mPointcloudCallback(static_cast(base), + handle->mPointcloudCallback(static_cast(mInternalSource), pcd, handle->mPointcloudUserData); auto pPcd = static_cast*>(p.pointcloud); @@ -412,6 +441,7 @@ namespace mv3d #endif delete [] static_cast(input->data); delete [] static_cast(input->extraData); + mv_source_clear(handle->mInternalSource); input.reset(); } -- 2.7.4 From 8cf7ba9a6489688f4502ba77b047e5fe86188b0e Mon Sep 17 00:00:00 2001 From: Tae-Young Chung Date: Thu, 22 Sep 2022 14:24:48 +0900 Subject: [PATCH 09/16] mv3d: fix bugs [Issue type] : bug fix (SVACE: WGID 499255,499256,499263,499267) 499255: fix flcose() with nullptr 499256: fix delete operation with malloc() 499263: fix assignment of a signed integer value to a bigger size_t 499267: fix memory leak of suffix_for_algo Change-Id: I415baa95286527007e3b8aff8befd218ebe0a09b Signed-off-by: Tae-Young Chung --- mv_3d/3d/include/Mv3d.h | 4 ++-- mv_3d/3d/src/Mv3d.cpp | 4 ++-- mv_3d/3d/src/mv_3d_open.cpp | 5 +++++ test/testsuites/mv3d/depth_test_suite.cpp | 8 +++++++- test/testsuites/mv3d/depthstream_test_suite.cpp | 2 +- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/mv_3d/3d/include/Mv3d.h b/mv_3d/3d/include/Mv3d.h index 8f3558c..3614cea 100644 --- a/mv_3d/3d/include/Mv3d.h +++ b/mv_3d/3d/include/Mv3d.h @@ -47,8 +47,8 @@ namespace mv3d size_t mWidth; size_t mHeight; - size_t mMinDisp; - size_t mMaxDisp; + int mMinDisp; + int mMaxDisp; double mSamplingRatio; int mOutlierRemovalPoints; double mOutlierRemovalRadius; diff --git a/mv_3d/3d/src/Mv3d.cpp b/mv_3d/3d/src/Mv3d.cpp index dc4c05c..258307a 100644 --- a/mv_3d/3d/src/Mv3d.cpp +++ b/mv_3d/3d/src/Mv3d.cpp @@ -104,8 +104,8 @@ namespace mv3d std::string pointcloudOutputPath) { mMode = mode; - mWidth = width; - mHeight = height; + mWidth = static_cast(width); + mHeight = static_cast(height); mMinDisp = minDisp; mMaxDisp = maxDisp; mSamplingRatio = samplingRatio; diff --git a/mv_3d/3d/src/mv_3d_open.cpp b/mv_3d/3d/src/mv_3d_open.cpp index 72021a2..24d2189 100644 --- a/mv_3d/3d/src/mv_3d_open.cpp +++ b/mv_3d/3d/src/mv_3d_open.cpp @@ -115,6 +115,11 @@ int mv3dConfigure(mv_3d_h mv3d, mv_engine_config_h engine_config) return ret; } + if (depthWidth <= 0 || depthHeight <= 0) { + LOGE("Invalid depth resolution %d x %d", depthWidth, depthHeight); + return MEDIA_VISION_ERROR_INVALID_PARAMETER; + } + ret = mv_engine_config_get_int_attribute( engine_config, MV_3D_DEPTH_MIN_DISPARITY, &minDisp); if (ret != MEDIA_VISION_ERROR_NONE) { diff --git a/test/testsuites/mv3d/depth_test_suite.cpp b/test/testsuites/mv3d/depth_test_suite.cpp index 26997e5..bee2201 100644 --- a/test/testsuites/mv3d/depth_test_suite.cpp +++ b/test/testsuites/mv3d/depth_test_suite.cpp @@ -338,8 +338,11 @@ int perform_middlebury_test() if (sscanf(line, "width= %d", &iVal) == 1) width = iVal; if (sscanf(line, "height= %d", &iVal) == 1) height = iVal; } + fclose(fp); + } else { + printf("Fail to open %s\n", calibFilename); + goto _err; } - fclose(fp); int minDisp = static_cast(dmin); int maxDisp = static_cast(dmax); @@ -507,6 +510,9 @@ _err: if (path_to_dataset) free(path_to_dataset); + if (suffix_for_algo) + free(suffix_for_algo); + if (engine_config) mv_destroy_engine_config(engine_config); diff --git a/test/testsuites/mv3d/depthstream_test_suite.cpp b/test/testsuites/mv3d/depthstream_test_suite.cpp index c8241b9..16a4d4e 100644 --- a/test/testsuites/mv3d/depthstream_test_suite.cpp +++ b/test/testsuites/mv3d/depthstream_test_suite.cpp @@ -265,7 +265,7 @@ void _depth_stereo_cb(mv_source_h source, unsigned short *depth, unsigned int wi camera::PinholeCameraIntrinsic intrinsic; io::ReadIJsonConvertible(INTRINSIC_FILE_PATH, intrinsic); - vertex3d = (float *)malloc(width * height * 4); + vertex3d = new float[width * height * 4]; memset(vertex3d, 0, width * height * 4); utility::LogInfo("focal_x = {}", intrinsic.GetFocalLength().first); utility::LogInfo("focal_y = {}", intrinsic.GetFocalLength().second); -- 2.7.4 From 13ecaa7e6b95fda4c5c9fe4223120560cc7c6669 Mon Sep 17 00:00:00 2001 From: Tae-Young Chung Date: Thu, 22 Sep 2022 16:49:08 +0900 Subject: [PATCH 10/16] mv3d: fix bugs wrong lib name in CMakeLists.txt [Issue type] : bug fix Change-Id: Ica7aa7c43f0c49bb685c5a4e93989f9ea28ea5fd Signed-off-by: Tae-Young Chung --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 75065a2..9bedda5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -137,7 +137,7 @@ configure_file( install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/${fw_name}-surveillance.pc DESTINATION ${LIB_INSTALL_DIR}/pkgconfig) set(PC_NAME ${fw_name}-3d) -set(PC_LDFLAGS "-l${MV_DEPTH_LIB_NAME} -l${MV_COMMON_LIB_NAME}") +set(PC_LDFLAGS "-l${MV_3D_LIB_NAME} -l${MV_COMMON_LIB_NAME}") configure_file( ${fw_name}.pc.in ${CMAKE_CURRENT_SOURCE_DIR}/${fw_name}-3d.pc -- 2.7.4 From 530f5ba836a60cf9d4dfb224790b2f13670c87ca Mon Sep 17 00:00:00 2001 From: Inki Dae Date: Tue, 27 Sep 2022 11:19:02 +0900 Subject: [PATCH 11/16] mv_machine_learning: fix coverity issue [Version] : 0.23.26-0 [Issue type] : bug fix Fixed a coverity issue by handling an error value correctly. Change-Id: Icce85a27e2be29f38799a4f312d40978d60e59eb Signed-off-by: Inki Dae --- mv_machine_learning/inference/src/Inference.cpp | 8 ++++++-- packaging/capi-media-vision.spec | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mv_machine_learning/inference/src/Inference.cpp b/mv_machine_learning/inference/src/Inference.cpp index 9f3b25c..18f6544 100644 --- a/mv_machine_learning/inference/src/Inference.cpp +++ b/mv_machine_learning/inference/src/Inference.cpp @@ -810,7 +810,11 @@ int Inference::Preprocess(std::vector &mv_sources, std::vector &mv_sources, std::vector Date: Tue, 27 Sep 2022 12:42:43 +0900 Subject: [PATCH 12/16] mv_3d: fix coverity issue [Version] : 0.23.27-0 [Issue type] : bug fix fix coverity DefectId 1612133 (resource leak) Change-Id: I8a8b9a0783db79f3695dc637445dfe62d0b2ea92 Signed-off-by: sangho park --- mv_3d/3d/src/mv_3d_open.cpp | 42 +++++++++++++++++----------------------- packaging/capi-media-vision.spec | 2 +- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/mv_3d/3d/src/mv_3d_open.cpp b/mv_3d/3d/src/mv_3d_open.cpp index 24d2189..0319351 100644 --- a/mv_3d/3d/src/mv_3d_open.cpp +++ b/mv_3d/3d/src/mv_3d_open.cpp @@ -80,6 +80,7 @@ int mv3dConfigure(mv_3d_h mv3d, mv_engine_config_h engine_config) { LOGI("ENTER"); + Mv3d *pMv3d = NULL; if (!mv3d || !engine_config) { LOGE("Handle is NULL"); return MEDIA_VISION_ERROR_INVALID_PARAMETER; @@ -134,18 +135,6 @@ int mv3dConfigure(mv_3d_h mv3d, mv_engine_config_h engine_config) return ret; } - char *stereoConfigFilePath = NULL; - ret = mv_engine_config_get_string_attribute( - engine_config, MV_3D_DEPTH_STEREO_CONFIG_FILE_PATH, &stereoConfigFilePath); - if (ret != MEDIA_VISION_ERROR_NONE) { - LOGE("Failed to get stereo configuration file path"); - if (stereoConfigFilePath) { - free(stereoConfigFilePath); - stereoConfigFilePath = NULL; - } - return ret; - } - double samplingRatio; ret = mv_engine_config_get_double_attribute( engine_config, MV_3D_POINTCLOUD_SAMPLING_RATIO,&samplingRatio); @@ -170,36 +159,41 @@ int mv3dConfigure(mv_3d_h mv3d, mv_engine_config_h engine_config) return ret; } + char *stereoConfigFilePath = NULL; + ret = mv_engine_config_get_string_attribute( + engine_config, MV_3D_DEPTH_STEREO_CONFIG_FILE_PATH, &stereoConfigFilePath); + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to get stereo configuration file path"); + return ret; + } + char *pointcloudOutputFilePath = NULL; ret = mv_engine_config_get_string_attribute( engine_config, MV_3D_POINTCLOUD_OUTPUT_FILE_PATH, &pointcloudOutputFilePath); if (ret != MEDIA_VISION_ERROR_NONE) { LOGE("Failed to get stereo configuration file path"); - if (pointcloudOutputFilePath) { - free(pointcloudOutputFilePath); - pointcloudOutputFilePath = NULL; - } - return ret; + goto out; } - auto pMv3d = static_cast(mv3d); + pMv3d = static_cast(mv3d); ret = pMv3d->Configure(mode, depthWidth, depthHeight, minDisp, maxDisp, samplingRatio, outlierRemovalPoints, outlierRemovalRadius, stereoConfigFilePath, pointcloudOutputFilePath); - if (stereoConfigFilePath) { - free(stereoConfigFilePath); - stereoConfigFilePath = NULL; + if (ret != MEDIA_VISION_ERROR_NONE) { + LOGE("Failed to configure Depth"); + goto out; } +out: if (pointcloudOutputFilePath) { free(pointcloudOutputFilePath); pointcloudOutputFilePath = NULL; } - if (ret != MEDIA_VISION_ERROR_NONE) { - LOGE("Failed to configure Depth"); - return ret; + if (stereoConfigFilePath) { + free(stereoConfigFilePath); + stereoConfigFilePath = NULL; } LOGI("LEAVE"); diff --git a/packaging/capi-media-vision.spec b/packaging/capi-media-vision.spec index 3e53cfe..fdb9d02 100644 --- a/packaging/capi-media-vision.spec +++ b/packaging/capi-media-vision.spec @@ -1,6 +1,6 @@ Name: capi-media-vision Summary: Media Vision library for Tizen Native API -Version: 0.23.26 +Version: 0.23.27 Release: 0 Group: Multimedia/Framework License: Apache-2.0 and BSD-3-Clause -- 2.7.4 From b3b01a043ca3897180319cea107e6ba101ae0570 Mon Sep 17 00:00:00 2001 From: sangho park Date: Tue, 27 Sep 2022 12:49:37 +0900 Subject: [PATCH 13/16] mv_3d: fix coverity issue [Version] : 0.23.28-0 [Issue type] : bug fix fix coverity DefectId 1612132, 1612134, 1612135 (Unchecked return value) Change-Id: Ib618ab460996f40572e8084b743954eb7fad20c1 Signed-off-by: sangho park --- mv_3d/3d/src/Mv3d.cpp | 8 ++++++-- packaging/capi-media-vision.spec | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mv_3d/3d/src/Mv3d.cpp b/mv_3d/3d/src/Mv3d.cpp index 258307a..b63ad9a 100644 --- a/mv_3d/3d/src/Mv3d.cpp +++ b/mv_3d/3d/src/Mv3d.cpp @@ -410,13 +410,17 @@ namespace mv3d auto depthData = handle->mDfsAdaptor->getOutputData(); auto leftData = handle->mDfsAdaptor->getLeftData(); - mv_source_fill_by_buffer(handle->mInternalSource, + int ret = mv_source_fill_by_buffer(handle->mInternalSource, static_cast(leftData.data), leftData.width, leftData.height, leftData.stride * leftData.height, leftData.type == DFS_DATA_TYPE_UINT8C3 ? MEDIA_VISION_COLORSPACE_RGB888 : MEDIA_VISION_COLORSPACE_Y800); + if (MEDIA_VISION_ERROR_NONE != ret) { + LOGW("Errors were occurred during source filling %i", ret); + continue; + } handle->mDepthCallback( static_cast(handle->mInternalSource), static_cast(depthData.data), @@ -431,7 +435,7 @@ namespace mv3d handle->GetPointcloudFromSource(*input, depthData, p); mv_3d_pointcloud_h pcd = &p; - handle->mPointcloudCallback(static_cast(mInternalSource), + handle->mPointcloudCallback(static_cast(handle->mInternalSource), pcd, handle->mPointcloudUserData); auto pPcd = static_cast*>(p.pointcloud); diff --git a/packaging/capi-media-vision.spec b/packaging/capi-media-vision.spec index fdb9d02..bcc2f5d 100644 --- a/packaging/capi-media-vision.spec +++ b/packaging/capi-media-vision.spec @@ -1,6 +1,6 @@ Name: capi-media-vision Summary: Media Vision library for Tizen Native API -Version: 0.23.27 +Version: 0.23.28 Release: 0 Group: Multimedia/Framework License: Apache-2.0 and BSD-3-Clause -- 2.7.4 From 17462e7ea83766a479e5f12e4d295fb631eb38a6 Mon Sep 17 00:00:00 2001 From: Seungbae Shin Date: Tue, 27 Sep 2022 13:57:53 +0900 Subject: [PATCH 14/16] mv_3d: fix svace issue (SIGNED_TO_BIGGER_UNSIGNED) [Version] : 0.23.29-0 [Issue type] : bug fix fix svace wgid 499263 Change-Id: Iea3cf1342f2b1b090d3fef626e26898e65a0c631 --- mv_3d/3d/include/Mv3d.h | 2 +- mv_3d/3d/src/Mv3d.cpp | 2 +- mv_3d/3d/src/mv_3d_open.cpp | 12 +++++++++--- packaging/capi-media-vision.spec | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/mv_3d/3d/include/Mv3d.h b/mv_3d/3d/include/Mv3d.h index 3614cea..eea895b 100644 --- a/mv_3d/3d/include/Mv3d.h +++ b/mv_3d/3d/include/Mv3d.h @@ -93,7 +93,7 @@ namespace mv3d size_t windowHeight, size_t speckleSize); - int Configure(int mode, int width, int height, int minDisp, int maxDisp, + int Configure(int mode, unsigned int width, unsigned int height, int minDisp, int maxDisp, double samplingRatio, int outlierRemovalPoints, double outlierRemovalRadius, std::string stereoConfigPath, std::string pointcloudOutputPath); diff --git a/mv_3d/3d/src/Mv3d.cpp b/mv_3d/3d/src/Mv3d.cpp index b63ad9a..9982635 100644 --- a/mv_3d/3d/src/Mv3d.cpp +++ b/mv_3d/3d/src/Mv3d.cpp @@ -97,7 +97,7 @@ namespace mv3d mDfsParameter.maxSpeckleSize = speckleSize; } - int Mv3d::Configure(int mode, int width, int height, + int Mv3d::Configure(int mode, unsigned int width, unsigned int height, int minDisp, int maxDisp, double samplingRatio, int outlierRemovalPoints, double outlierRemovalRadius, std::string stereoConfigPath, diff --git a/mv_3d/3d/src/mv_3d_open.cpp b/mv_3d/3d/src/mv_3d_open.cpp index 0319351..4c09f60 100644 --- a/mv_3d/3d/src/mv_3d_open.cpp +++ b/mv_3d/3d/src/mv_3d_open.cpp @@ -176,9 +176,15 @@ int mv3dConfigure(mv_3d_h mv3d, mv_engine_config_h engine_config) } pMv3d = static_cast(mv3d); - ret = pMv3d->Configure(mode, depthWidth, depthHeight, minDisp, maxDisp, - samplingRatio, outlierRemovalPoints, outlierRemovalRadius, - stereoConfigFilePath, pointcloudOutputFilePath); + ret = pMv3d->Configure(mode, + static_cast(depthWidth), + static_cast(depthHeight), + minDisp, maxDisp, + samplingRatio, + outlierRemovalPoints, + outlierRemovalRadius, + stereoConfigFilePath, + pointcloudOutputFilePath); if (ret != MEDIA_VISION_ERROR_NONE) { LOGE("Failed to configure Depth"); diff --git a/packaging/capi-media-vision.spec b/packaging/capi-media-vision.spec index bcc2f5d..4b02bd8 100644 --- a/packaging/capi-media-vision.spec +++ b/packaging/capi-media-vision.spec @@ -1,6 +1,6 @@ Name: capi-media-vision Summary: Media Vision library for Tizen Native API -Version: 0.23.28 +Version: 0.23.29 Release: 0 Group: Multimedia/Framework License: Apache-2.0 and BSD-3-Clause -- 2.7.4 From 74436d78c4f228cc7bc05de6e583a5bf65003406 Mon Sep 17 00:00:00 2001 From: Tae-Young Chung Date: Tue, 27 Sep 2022 17:14:38 +0900 Subject: [PATCH 15/16] Check new vision.3d features [Version] : 0.23.29-0 [Issue type] : update According to ACR-1720, new features, vision.3d, vision.3d.depth, vision.3d.pointcloud, are added. Change-Id: I02ff894588cd20ff11039a80cf9fcd91cada602f Signed-off-by: Tae-Young Chung --- include/mv_private.h | 4 +++ mv_3d/3d/src/mv_3d.c | 12 +++++++++ mv_common/src/mv_private.c | 67 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) diff --git a/include/mv_private.h b/include/mv_private.h index 49eb3aa..94c5a7b 100644 --- a/include/mv_private.h +++ b/include/mv_private.h @@ -69,6 +69,10 @@ bool _mv_inference_check_system_info_feature_supported(void); bool _mv_inference_image_check_system_info_feature_supported(void); bool _mv_inference_face_check_system_info_feature_supported(void); bool _mv_roi_tracking_check_system_info_feature_supported(void); +bool _mv_3d_all_check_system_info_feature_supported(void); +bool _mv_3d_check_system_info_feature_supported(void); +bool _mv_3d_depth_check_system_info_feature_supported(void); +bool _mv_3d_pointcloud_check_system_info_feature_supported(void); #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/mv_3d/3d/src/mv_3d.c b/mv_3d/3d/src/mv_3d.c index c640e13..27ea242 100644 --- a/mv_3d/3d/src/mv_3d.c +++ b/mv_3d/3d/src/mv_3d.c @@ -26,6 +26,7 @@ int mv_3d_create(mv_3d_h *mv3d) { + MEDIA_VISION_SUPPORT_CHECK(_mv_3d_all_check_system_info_feature_supported()); MEDIA_VISION_NULL_ARG_CHECK(mv3d); MEDIA_VISION_FUNCTION_ENTER(); @@ -39,6 +40,7 @@ int mv_3d_create(mv_3d_h *mv3d) int mv_3d_destroy(mv_3d_h mv3d) { + MEDIA_VISION_SUPPORT_CHECK(_mv_3d_all_check_system_info_feature_supported()); MEDIA_VISION_INSTANCE_CHECK(mv3d); MEDIA_VISION_FUNCTION_ENTER(); @@ -53,6 +55,7 @@ int mv_3d_destroy(mv_3d_h mv3d) int mv_3d_configure(mv_3d_h mv3d, mv_engine_config_h engine_config) { + MEDIA_VISION_SUPPORT_CHECK(_mv_3d_all_check_system_info_feature_supported()); MEDIA_VISION_INSTANCE_CHECK(mv3d); MEDIA_VISION_INSTANCE_CHECK(engine_config); @@ -69,6 +72,8 @@ int mv_3d_set_depth_cb(mv_3d_h mv3d, mv_3d_depth_cb depth_cb, void *user_data) { + MEDIA_VISION_SUPPORT_CHECK(_mv_3d_depth_check_system_info_feature_supported() || + _mv_3d_check_system_info_feature_supported()); MEDIA_VISION_INSTANCE_CHECK(mv3d); MEDIA_VISION_NULL_ARG_CHECK(depth_cb); @@ -86,6 +91,8 @@ int mv_3d_set_pointcloud_cb(mv_3d_h mv3d, mv_3d_pointcloud_cb pointcloud_cb, void *user_data) { + MEDIA_VISION_SUPPORT_CHECK(_mv_3d_pointcloud_check_system_info_feature_supported() || + _mv_3d_check_system_info_feature_supported()); MEDIA_VISION_INSTANCE_CHECK(mv3d); MEDIA_VISION_NULL_ARG_CHECK(pointcloud_cb); @@ -101,6 +108,7 @@ int mv_3d_set_pointcloud_cb(mv_3d_h mv3d, int mv_3d_prepare(mv_3d_h mv3d) { + MEDIA_VISION_SUPPORT_CHECK(_mv_3d_all_check_system_info_feature_supported()); MEDIA_VISION_INSTANCE_CHECK(mv3d); MEDIA_VISION_FUNCTION_ENTER(); @@ -117,6 +125,7 @@ int mv_3d_run(mv_3d_h mv3d, mv_source_h source_extra, mv_source_h color) { + MEDIA_VISION_SUPPORT_CHECK(_mv_3d_all_check_system_info_feature_supported()); MEDIA_VISION_INSTANCE_CHECK(mv3d); MEDIA_VISION_INSTANCE_CHECK(source); @@ -136,6 +145,7 @@ int mv_3d_run_async(mv_3d_h mv3d, mv_source_h source_extra, mv_source_h color) { + MEDIA_VISION_SUPPORT_CHECK(_mv_3d_all_check_system_info_feature_supported()); MEDIA_VISION_INSTANCE_CHECK(mv3d); MEDIA_VISION_INSTANCE_CHECK(source); @@ -155,6 +165,8 @@ int mv_3d_pointcloud_write_file(mv_3d_h mv3d, mv_3d_pointcloud_type_e type, char *filename) { + MEDIA_VISION_SUPPORT_CHECK(_mv_3d_pointcloud_check_system_info_feature_supported() || + _mv_3d_check_system_info_feature_supported()); MEDIA_VISION_INSTANCE_CHECK(mv3d); MEDIA_VISION_INSTANCE_CHECK(pointcloud); diff --git a/mv_common/src/mv_private.c b/mv_common/src/mv_private.c index 84be33a..f6dc75a 100644 --- a/mv_common/src/mv_private.c +++ b/mv_common/src/mv_private.c @@ -259,3 +259,70 @@ bool _mv_roi_tracking_check_system_info_feature_supported(void) return supported; } + +bool _mv_3d_all_check_system_info_feature_supported(void) +{ + return _mv_3d_check_system_info_feature_supported() || + _mv_3d_depth_check_system_info_feature_supported() || + _mv_3d_pointcloud_check_system_info_feature_supported(); +} + +bool _mv_3d_check_system_info_feature_supported(void) +{ + bool is3dSupported = false; + + const int nRetVal1 = system_info_get_platform_bool("http://tizen.org/feature/vision.3d", + &is3dSupported); + + if (nRetVal1 != SYSTEM_INFO_ERROR_NONE) { + LOGE("SYSTEM_INFO_ERROR: vision.3d"); + return false; + } + + is3dSupported ? LOGI("system_info_get_platform_bool returned " + "Supported 3d feature capability\n") : + LOGE("system_info_get_platform_bool returned " + "Unsupported 3d feature capability\n"); + + return is3dSupported; +} + +bool _mv_3d_depth_check_system_info_feature_supported(void) +{ + bool is3dDepthSupported = false; + + const int nRetVal1 = system_info_get_platform_bool("http://tizen.org/feature/vision.3d.depth", + &is3dDepthSupported); + + if (nRetVal1 != SYSTEM_INFO_ERROR_NONE) { + LOGE("SYSTEM_INFO_ERROR: vision.3d.depth"); + return false; + } + + is3dDepthSupported ? LOGI("system_info_get_platform_bool returned " + "Supported 3d depth feature capability\n") : + LOGE("system_info_get_platform_bool returned " + "Unsupported 3d depth feature capability\n"); + + return is3dDepthSupported; +} + +bool _mv_3d_pointcloud_check_system_info_feature_supported(void) +{ + bool is3dPointCloudSupported = false; + + const int nRetVal1 = system_info_get_platform_bool("http://tizen.org/feature/vision.3d.pointcloud", + &is3dPointCloudSupported); + + if (nRetVal1 != SYSTEM_INFO_ERROR_NONE) { + LOGE("SYSTEM_INFO_ERROR: vision.3d.pointcloud"); + return false; + } + + is3dPointCloudSupported ? LOGI("system_info_get_platform_bool returned " + "Supported 3d pointcloud feature capability\n") : + LOGE("system_info_get_platform_bool returned " + "Unsupported 3d pointcloud feature capability\n"); + + return is3dPointCloudSupported; +} \ No newline at end of file -- 2.7.4 From f0c6d72be23fc5cc89f38ec16e6717ebd825552e Mon Sep 17 00:00:00 2001 From: sangho park Date: Fri, 23 Sep 2022 15:42:31 +0900 Subject: [PATCH 16/16] mv3d: fix bugs [Issue type] : bug fix for building mv_3d (add compile_definitions) Change-Id: Ic325fc0bb82122ae014538fb60c7cd2096e71e6b Signed-off-by: sangho park --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9bedda5..fab7f73 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,9 @@ if(BUILD_VISUALIZER) add_compile_definitions(BUILD_VISUALIZER) endif() +if(MV_3D_POINTCLOUD_IS_AVAILABLE) + add_compile_definitions(MV_3D_POINTCLOUD_IS_AVAILABLE) +endif() set(MV_COMMON_LIB_NAME "mv_common") set(MV_BARCODE_DETECTOR_LIB_NAME "mv_barcode_detector" CACHE STRING -- 2.7.4