From: Vladimir Gavrilov Date: Tue, 10 Nov 2020 04:18:40 +0000 (+0300) Subject: Apply rest of the comments for NMS-5 review (#3037) X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=13b2ef18cc8598066c04d8f22f1e45b796e91773;p=platform%2Fupstream%2Fdldt.git Apply rest of the comments for NMS-5 review (#3037) * Commit. * Moved the preprocessing functions for NMS-5 to Interpreter backend. * Code style fix. --- diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/non_max_suppression.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/non_max_suppression.hpp index db343ab..1ef9e73 100644 --- a/ngraph/core/reference/include/ngraph/runtime/reference/non_max_suppression.hpp +++ b/ngraph/core/reference/include/ngraph/runtime/reference/non_max_suppression.hpp @@ -36,25 +36,6 @@ namespace ngraph { namespace reference { - struct InfoForNMS5 - { - int64_t max_output_boxes_per_class; - float iou_threshold; - float score_threshold; - float soft_nms_sigma; - Shape out_shape; - Shape boxes_shape; - Shape scores_shape; - std::vector boxes_data; - std::vector scores_data; - size_t out_shape_size; - bool sort_result_descending; - ngraph::element::Type output_type; - }; - - InfoForNMS5 get_info_for_nms5_evaluation(const op::v5::NonMaxSuppression* nms5, - const HostTensorVector& inputs); - void non_max_suppression(const float* boxes_data, const Shape& boxes_data_shape, const float* scores_data, diff --git a/ngraph/core/reference/src/runtime/reference/non_max_suppression.cpp b/ngraph/core/reference/src/runtime/reference/non_max_suppression.cpp index 44fafd7..a68eb42 100644 --- a/ngraph/core/reference/src/runtime/reference/non_max_suppression.cpp +++ b/ngraph/core/reference/src/runtime/reference/non_max_suppression.cpp @@ -128,196 +128,12 @@ struct BoxInfo float score = 0.0f; }; -using V5BoxEncoding = op::v5::NonMaxSuppression::BoxEncodingType; - -namespace -{ - constexpr size_t boxes_port = 0; - constexpr size_t scores_port = 1; - constexpr size_t max_output_boxes_port = 2; - constexpr size_t iou_threshold_port = 3; - constexpr size_t score_threshold_port = 4; - constexpr size_t soft_nms_sigma_port = 5; - - PartialShape - infer_selected_indices_shape(const std::vector>& inputs, - int64_t max_output_boxes_per_class) - { - const auto boxes_ps = inputs[boxes_port]->get_partial_shape(); - const auto scores_ps = inputs[scores_port]->get_partial_shape(); - - // NonMaxSuppression produces triplets - // that have the following format: [batch_index, class_index, box_index] - PartialShape result = {Dimension::dynamic(), 3}; - - if (boxes_ps.rank().is_static() && scores_ps.rank().is_static()) - { - const auto num_boxes_boxes = boxes_ps[1]; - if (num_boxes_boxes.is_static() && scores_ps[0].is_static() && scores_ps[1].is_static()) - { - const auto num_boxes = num_boxes_boxes.get_length(); - const auto num_classes = scores_ps[1].get_length(); - - result[0] = std::min(num_boxes, max_output_boxes_per_class) * num_classes * - scores_ps[0].get_length(); - } - } - - return result; - } - - void normalize_corner(float* boxes, const Shape& boxes_shape) - { - size_t total_num_of_boxes = shape_size(boxes_shape) / 4; - for (size_t i = 0; i < total_num_of_boxes; ++i) - { - float* current_box = boxes + 4 * i; - - float y1 = current_box[0]; - float x1 = current_box[1]; - float y2 = current_box[2]; - float x2 = current_box[3]; - - float ymin = std::min(y1, y2); - float ymax = std::max(y1, y2); - float xmin = std::min(x1, x2); - float xmax = std::max(x1, x2); - - current_box[0] = ymin; - current_box[1] = xmin; - current_box[2] = ymax; - current_box[3] = xmax; - } - } - - void normalize_center(float* boxes, const Shape& boxes_shape) - { - size_t total_num_of_boxes = shape_size(boxes_shape) / 4; - for (size_t i = 0; i < total_num_of_boxes; ++i) - { - float* current_box = boxes + 4 * i; - - float x_center = current_box[0]; - float y_center = current_box[1]; - float width = current_box[2]; - float height = current_box[3]; - - float y1 = y_center - height / 2.0; - float x1 = x_center - width / 2.0; - float y2 = y_center + height / 2.0; - float x2 = x_center + width / 2.0; - - current_box[0] = y1; - current_box[1] = x1; - current_box[2] = y2; - current_box[3] = x2; - } - } - - void normalize_box_encoding(float* boxes, - const Shape& boxes_shape, - const V5BoxEncoding box_encoding) - { - if (box_encoding == V5BoxEncoding::CORNER) - { - normalize_corner(boxes, boxes_shape); - } - else - { - normalize_center(boxes, boxes_shape); - } - } - - std::vector get_floats(const std::shared_ptr& input, const Shape& shape) - { - size_t input_size = shape_size(shape); - std::vector result(input_size); - - switch (input->get_element_type()) - { - case element::Type_t::bf16: - { - bfloat16* p = input->get_data_ptr(); - for (size_t i = 0; i < input_size; ++i) - { - result[i] = float(p[i]); - } - } - break; - case element::Type_t::f16: - { - float16* p = input->get_data_ptr(); - for (size_t i = 0; i < input_size; ++i) - { - result[i] = float(p[i]); - } - } - break; - case element::Type_t::f32: - { - float* p = input->get_data_ptr(); - memcpy(result.data(), p, input_size * sizeof(float)); - } - break; - default: throw std::runtime_error("Unsupported data type in op NonMaxSuppression-5"); break; - } - - return result; - } - - std::vector prepare_boxes_data(const std::shared_ptr& boxes, - const Shape& boxes_shape, - const V5BoxEncoding box_encoding) - { - auto result = get_floats(boxes, boxes_shape); - normalize_box_encoding(result.data(), boxes_shape, box_encoding); - return result; - } - - std::vector prepare_scores_data(const std::shared_ptr& scores, - const Shape& scores_shape) - { - auto result = get_floats(scores, scores_shape); - return result; - } -} - namespace ngraph { namespace runtime { namespace reference { - InfoForNMS5 get_info_for_nms5_evaluation(const op::v5::NonMaxSuppression* nms5, - const HostTensorVector& inputs) - { - InfoForNMS5 result; - - result.max_output_boxes_per_class = nms5->max_boxes_output_from_input(); - result.iou_threshold = nms5->iou_threshold_from_input(); - result.score_threshold = nms5->score_threshold_from_input(); - result.soft_nms_sigma = nms5->soft_nms_sigma_from_input(); - - auto selected_indices_shape = - infer_selected_indices_shape(inputs, result.max_output_boxes_per_class); - result.out_shape = selected_indices_shape.to_shape(); - - result.boxes_shape = inputs[boxes_port]->get_shape(); - result.scores_shape = inputs[scores_port]->get_shape(); - - result.boxes_data = prepare_boxes_data( - inputs[boxes_port], result.boxes_shape, nms5->get_box_encoding()); - result.scores_data = prepare_scores_data(inputs[scores_port], result.scores_shape); - - result.out_shape_size = shape_size(result.out_shape); - - result.sort_result_descending = nms5->get_sort_result_descending(); - - result.output_type = nms5->get_output_type(); - - return result; - } - void non_max_suppression(const float* boxes_data, const Shape& boxes_data_shape, const float* scores_data, diff --git a/ngraph/test/runtime/interpreter/int_executable.cpp b/ngraph/test/runtime/interpreter/int_executable.cpp index 4efdb29..5e6a991 100644 --- a/ngraph/test/runtime/interpreter/int_executable.cpp +++ b/ngraph/test/runtime/interpreter/int_executable.cpp @@ -35,6 +35,192 @@ using namespace ngraph; NGRAPH_SUPPRESS_DEPRECATED_START +using V5BoxEncoding = op::v5::NonMaxSuppression::BoxEncodingType; + +namespace +{ + constexpr size_t boxes_port = 0; + constexpr size_t scores_port = 1; + constexpr size_t max_output_boxes_port = 2; + constexpr size_t iou_threshold_port = 3; + constexpr size_t score_threshold_port = 4; + constexpr size_t soft_nms_sigma_port = 5; + + PartialShape + infer_selected_indices_shape(const std::vector>& inputs, + int64_t max_output_boxes_per_class) + { + const auto boxes_ps = inputs[boxes_port]->get_partial_shape(); + const auto scores_ps = inputs[scores_port]->get_partial_shape(); + + // NonMaxSuppression produces triplets + // that have the following format: [batch_index, class_index, box_index] + PartialShape result = {Dimension::dynamic(), 3}; + + if (boxes_ps.rank().is_static() && scores_ps.rank().is_static()) + { + const auto num_boxes_boxes = boxes_ps[1]; + if (num_boxes_boxes.is_static() && scores_ps[0].is_static() && scores_ps[1].is_static()) + { + const auto num_boxes = num_boxes_boxes.get_length(); + const auto num_classes = scores_ps[1].get_length(); + + result[0] = std::min(num_boxes, max_output_boxes_per_class) * num_classes * + scores_ps[0].get_length(); + } + } + + return result; + } + + void normalize_corner(float* boxes, const Shape& boxes_shape) + { + size_t total_num_of_boxes = shape_size(boxes_shape) / 4; + for (size_t i = 0; i < total_num_of_boxes; ++i) + { + float* current_box = boxes + 4 * i; + + float y1 = current_box[0]; + float x1 = current_box[1]; + float y2 = current_box[2]; + float x2 = current_box[3]; + + float ymin = std::min(y1, y2); + float ymax = std::max(y1, y2); + float xmin = std::min(x1, x2); + float xmax = std::max(x1, x2); + + current_box[0] = ymin; + current_box[1] = xmin; + current_box[2] = ymax; + current_box[3] = xmax; + } + } + + void normalize_center(float* boxes, const Shape& boxes_shape) + { + size_t total_num_of_boxes = shape_size(boxes_shape) / 4; + for (size_t i = 0; i < total_num_of_boxes; ++i) + { + float* current_box = boxes + 4 * i; + + float x_center = current_box[0]; + float y_center = current_box[1]; + float width = current_box[2]; + float height = current_box[3]; + + float y1 = y_center - height / 2.0; + float x1 = x_center - width / 2.0; + float y2 = y_center + height / 2.0; + float x2 = x_center + width / 2.0; + + current_box[0] = y1; + current_box[1] = x1; + current_box[2] = y2; + current_box[3] = x2; + } + } + + void normalize_box_encoding(float* boxes, + const Shape& boxes_shape, + const V5BoxEncoding box_encoding) + { + if (box_encoding == V5BoxEncoding::CORNER) + { + normalize_corner(boxes, boxes_shape); + } + else + { + normalize_center(boxes, boxes_shape); + } + } + + std::vector get_floats(const std::shared_ptr& input, const Shape& shape) + { + size_t input_size = shape_size(shape); + std::vector result(input_size); + + switch (input->get_element_type()) + { + case element::Type_t::bf16: + { + bfloat16* p = input->get_data_ptr(); + for (size_t i = 0; i < input_size; ++i) + { + result[i] = float(p[i]); + } + } + break; + case element::Type_t::f16: + { + float16* p = input->get_data_ptr(); + for (size_t i = 0; i < input_size; ++i) + { + result[i] = float(p[i]); + } + } + break; + case element::Type_t::f32: + { + float* p = input->get_data_ptr(); + memcpy(result.data(), p, input_size * sizeof(float)); + } + break; + default: throw std::runtime_error("Unsupported data type in op NonMaxSuppression-5"); break; + } + + return result; + } + + std::vector prepare_boxes_data(const std::shared_ptr& boxes, + const Shape& boxes_shape, + const V5BoxEncoding box_encoding) + { + auto result = get_floats(boxes, boxes_shape); + normalize_box_encoding(result.data(), boxes_shape, box_encoding); + return result; + } + + std::vector prepare_scores_data(const std::shared_ptr& scores, + const Shape& scores_shape) + { + auto result = get_floats(scores, scores_shape); + return result; + } +} + +runtime::interpreter::INTExecutable::InfoForNMS5 + runtime::interpreter::INTExecutable::get_info_for_nms5_eval( + const op::v5::NonMaxSuppression* nms5, + const std::vector>& inputs) +{ + InfoForNMS5 result; + + result.max_output_boxes_per_class = nms5->max_boxes_output_from_input(); + result.iou_threshold = nms5->iou_threshold_from_input(); + result.score_threshold = nms5->score_threshold_from_input(); + result.soft_nms_sigma = nms5->soft_nms_sigma_from_input(); + + auto selected_indices_shape = + infer_selected_indices_shape(inputs, result.max_output_boxes_per_class); + result.out_shape = selected_indices_shape.to_shape(); + + result.boxes_shape = inputs[boxes_port]->get_shape(); + result.scores_shape = inputs[scores_port]->get_shape(); + + result.boxes_data = + prepare_boxes_data(inputs[boxes_port], result.boxes_shape, nms5->get_box_encoding()); + result.scores_data = prepare_scores_data(inputs[scores_port], result.scores_shape); + + result.out_shape_size = shape_size(result.out_shape); + + result.sort_result_descending = nms5->get_sort_result_descending(); + + result.output_type = nms5->get_output_type(); + + return result; +} + runtime::interpreter::OP_TYPEID runtime::interpreter::INTExecutable::get_typeid(const Node& node) { const NodeTypeInfo& type_info = node.get_type_info(); diff --git a/ngraph/test/runtime/interpreter/int_executable.hpp b/ngraph/test/runtime/interpreter/int_executable.hpp index 0ece39b..ad909e6 100644 --- a/ngraph/test/runtime/interpreter/int_executable.hpp +++ b/ngraph/test/runtime/interpreter/int_executable.hpp @@ -178,6 +178,25 @@ protected: const std::vector>& outputs, const std::vector>& inputs); + struct InfoForNMS5 + { + int64_t max_output_boxes_per_class; + float iou_threshold; + float score_threshold; + float soft_nms_sigma; + Shape out_shape; + Shape boxes_shape; + Shape scores_shape; + std::vector boxes_data; + std::vector scores_data; + size_t out_shape_size; + bool sort_result_descending; + ngraph::element::Type output_type; + }; + + InfoForNMS5 get_info_for_nms5_eval(const op::v5::NonMaxSuppression* nms5, + const std::vector>& inputs); + template void op_engine(const Node& node, const std::vector>& out, @@ -1309,7 +1328,7 @@ protected: const op::v5::NonMaxSuppression* nms = static_cast(&node); - auto info = reference::get_info_for_nms5_evaluation(nms, args); + auto info = get_info_for_nms5_eval(nms, args); std::vector selected_indices(info.out_shape_size); std::vector selected_scores(info.out_shape_size);