From 1350a52f0989f2c44c4a9f05eb28f9e1a20333ff Mon Sep 17 00:00:00 2001 From: Yelin Jeong Date: Thu, 3 Nov 2022 15:53:18 +0900 Subject: [PATCH] [Caps] update caps for negotiation This patch adds update caps dimension properties. `update_caps_dimension` uses peer_caps for negotiation. `gst_tensor_caps_can_intersect` tries intersecting two tensor caps. Both functions treat `a:b:c` and `a:b:c:1` equal. Related issue #3937 Signed-off-by: Yelin Jeong --- gst/nnstreamer/elements/gsttensor_reposrc.c | 2 +- gst/nnstreamer/elements/gsttensor_transform.c | 14 ++ gst/nnstreamer/include/nnstreamer_plugin_api.h | 17 +++ gst/nnstreamer/nnstreamer_plugin_api_impl.c | 188 +++++++++++++++++++++++++ gst/nnstreamer/tensor_filter/tensor_filter.c | 9 ++ 5 files changed, 229 insertions(+), 1 deletion(-) diff --git a/gst/nnstreamer/elements/gsttensor_reposrc.c b/gst/nnstreamer/elements/gsttensor_reposrc.c index 0bfc76f..1f4add1 100644 --- a/gst/nnstreamer/elements/gsttensor_reposrc.c +++ b/gst/nnstreamer/elements/gsttensor_reposrc.c @@ -344,7 +344,7 @@ gst_tensor_reposrc_create (GstPushSrc * src, GstBuffer ** buffer) goto handle_eos; if (!self->negotiation && buf != NULL) { - if (!gst_caps_can_intersect (self->caps, caps)) { + if (!gst_tensor_caps_can_intersect (self->caps, caps)) { GST_ELEMENT_ERROR (GST_ELEMENT (self), CORE, NEGOTIATION, ("Negotiation Failed! : repo_sink & repos_src"), (NULL)); diff --git a/gst/nnstreamer/elements/gsttensor_transform.c b/gst/nnstreamer/elements/gsttensor_transform.c index b36c3b5..968f140 100644 --- a/gst/nnstreamer/elements/gsttensor_transform.c +++ b/gst/nnstreamer/elements/gsttensor_transform.c @@ -1989,12 +1989,26 @@ gst_tensor_transform_transform_caps (GstBaseTransform * trans, if (filtercap && gst_caps_get_size (filtercap) > 0) { GstCaps *intersection; + GstPad *pad; + GstCaps *peer_caps; + + gst_tensor_caps_update_dimension (result, filtercap); intersection = gst_caps_intersect_full (result, filtercap, GST_CAPS_INTERSECT_FIRST); gst_caps_unref (result); result = intersection; + + if (direction == GST_PAD_SINK) + pad = GST_BASE_TRANSFORM_SRC_PAD (filter); + else + pad = GST_BASE_TRANSFORM_SINK_PAD (filter); + + if ((peer_caps = gst_pad_peer_query_caps (pad, NULL))) { + gst_tensor_caps_update_dimension (result, peer_caps); + gst_caps_unref (peer_caps); + } } silent_debug_caps (filter, result, "to"); diff --git a/gst/nnstreamer/include/nnstreamer_plugin_api.h b/gst/nnstreamer/include/nnstreamer_plugin_api.h index c4e3e36..9a32a55 100644 --- a/gst/nnstreamer/include/nnstreamer_plugin_api.h +++ b/gst/nnstreamer/include/nnstreamer_plugin_api.h @@ -100,5 +100,22 @@ gst_tensor_meta_info_parse_memory (GstTensorMetaInfo * meta, GstMemory * mem); extern GstMemory * gst_tensor_meta_info_append_header (GstTensorMetaInfo * meta, GstMemory * mem); +/** + * @brief Update caps dimension for negotiation + * @param caps caps to compare and update + * @param peer_caps caps to compare + */ +extern void +gst_tensor_caps_update_dimension (GstCaps *caps, GstCaps *peer_caps); + +/** + * @brief Try intersecting @caps1 and @caps2 for tensor stream + * @param caps1 a GstCaps to intersect + * @param caps2 a GstCaps to intersect + * @return TRUE if intersection would be not empty. + */ +extern gboolean +gst_tensor_caps_can_intersect (GstCaps *caps1, GstCaps *caps2); + G_END_DECLS #endif /* __NNS_PLUGIN_API_H__ */ diff --git a/gst/nnstreamer/nnstreamer_plugin_api_impl.c b/gst/nnstreamer/nnstreamer_plugin_api_impl.c index ea550ca..d78d67f 100644 --- a/gst/nnstreamer/nnstreamer_plugin_api_impl.c +++ b/gst/nnstreamer/nnstreamer_plugin_api_impl.c @@ -924,6 +924,185 @@ _peer_is_flexible_tensor_caps (GstPad * pad) } /** + * @brief Check whether the tensor dimensions are same + */ +static gboolean +_is_tensor_dim_same (tensor_dim dim1, tensor_dim dim2) +{ + guint i; + for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++) { + if (dim1[i] != dim2[i]) + return FALSE; + } + return TRUE; +} + +/** + * @brief Check whether two structures have the same dimension + */ +static gboolean +_is_structure_dimension_same (GstStructure * structure1, + GstStructure * structure2) +{ + tensor_dim dim1, dim2; + const char *dim_str1; + const char *dim_str2; + + g_return_val_if_fail (gst_structure_has_field (structure1, "dimension"), + FALSE); + g_return_val_if_fail (gst_structure_has_field (structure2, "dimension"), + FALSE); + + dim_str1 = gst_structure_get_string (structure1, "dimension"); + gst_tensor_parse_dimension (dim_str1, dim1); + + dim_str2 = gst_structure_get_string (structure2, "dimension"); + gst_tensor_parse_dimension (dim_str2, dim2); + + return _is_tensor_dim_same (dim1, dim2); +} + +/** + * @brief Check whether two structures have the same dimensions + */ +static gboolean +_is_structure_dimensions_same (GstStructure * structure1, + GstStructure * structure2) +{ + GstTensorsInfo info1, info2; + const char *dim_str1; + const char *dim_str2; + guint num_dim1, num_dim2, i; + + g_return_val_if_fail (gst_structure_has_field (structure1, "dimensions"), + FALSE); + g_return_val_if_fail (gst_structure_has_field (structure2, "dimensions"), + FALSE); + + dim_str1 = gst_structure_get_string (structure1, "dimensions"); + num_dim1 = gst_tensors_info_parse_dimensions_string (&info1, dim_str1); + + dim_str2 = gst_structure_get_string (structure2, "dimensions"); + num_dim2 = gst_tensors_info_parse_dimensions_string (&info2, dim_str2); + + if (num_dim1 != num_dim2) + return FALSE; + + for (i = 0; i < num_dim1; i++) { + if (!_is_tensor_dim_same (info1.info[i].dimension, info2.info[i].dimension)) + return FALSE; + } + return TRUE; +} + +/** + * @brief Update caps dimensions for negotiation + * @param caps caps to compare and update + * @param peer_caps caps to compare + */ +void +gst_tensor_caps_update_dimension (GstCaps * caps, GstCaps * peer_caps) +{ + GstStructure *structure; + GstStructure *structure_peer; + guint i, j; + + g_return_if_fail (caps != NULL); + g_return_if_fail (peer_caps != NULL); + + for (i = 0; i < gst_caps_get_size (caps); i++) { + for (j = 0; j < gst_caps_get_size (peer_caps); j++) { + structure = gst_caps_get_structure (caps, i); + structure_peer = gst_caps_get_structure (peer_caps, j); + + /* other/tensor */ + if (gst_structure_has_field (structure, "dimension") + && gst_structure_has_field (structure_peer, "dimension")) { + /* update dimensions for negotiation */ + if (_is_structure_dimension_same (structure, structure_peer)) + gst_caps_set_simple (caps, "dimension", G_TYPE_STRING, + gst_structure_get_string (structure_peer, "dimension"), NULL); + } + /* other/tensors */ + else if (gst_structure_has_field (structure, "dimensions") + && gst_structure_has_field (structure_peer, "dimensions")) { + /* update dimensions for negotiation */ + if (_is_structure_dimensions_same (structure, structure_peer)) { + gst_structure_set (structure, "dimensions", G_TYPE_STRING, + gst_structure_get_string (structure_peer, "dimensions"), NULL); + } + } + } + } +} + +/** + * @brief Try intersecting @caps1 and @caps2 for tensor stream + * @param caps1 a GstCaps to intersect + * @param caps2 a GstCaps to intersect + * @return TRUE if intersection would be not empty. + */ +gboolean +gst_tensor_caps_can_intersect (GstCaps * caps1, GstCaps * caps2) +{ + GstStructure *structure1; + GstStructure *structure2; + GstStructure *structure_copy1; + GstStructure *structure_copy2; + + const gchar *name1; + const gchar *name2; + + gboolean intersectable; + + if (gst_caps_can_intersect (caps1, caps2)) + return TRUE; + + structure1 = gst_caps_get_structure (caps1, 0); + structure2 = gst_caps_get_structure (caps2, 0); + + name1 = gst_structure_get_name (structure1); + name2 = gst_structure_get_name (structure2); + + if (!gst_structure_is_tensor_stream (structure1) + || !gst_structure_is_tensor_stream (structure2)) + return FALSE; + if (!g_str_equal (name1, name2)) + return FALSE; + + /* other/tensor */ + if (g_str_equal (name1, NNS_MIMETYPE_TENSOR)) { + if (gst_structure_has_field (structure1, "dimension") + && gst_structure_has_field (structure2, "dimension")) { + if (!_is_structure_dimension_same (structure1, structure2)) + return FALSE; + } + } + /* other/tensors */ + else if (gst_structure_has_field (structure1, "dimensions") + && gst_structure_has_field (structure2, "dimensions")) { + if (!_is_structure_dimensions_same (structure1, structure2)) + return FALSE; + } + + structure_copy1 = gst_structure_copy (structure1); + structure_copy2 = gst_structure_copy (structure2); + + gst_structure_remove_field (structure_copy1, "dimension"); + gst_structure_remove_field (structure_copy1, "dimensions"); + gst_structure_remove_field (structure_copy2, "dimension"); + gst_structure_remove_field (structure_copy2, "dimensions"); + + intersectable = + gst_structure_can_intersect (structure_copy1, structure_copy2); + + gst_structure_free (structure_copy1); + gst_structure_free (structure_copy2); + + return intersectable; +} + +/** * @brief Get pad caps from tensors config and caps of the peer connected to the pad. * @param pad GstPad to get possible caps * @param config tensors config structure @@ -935,6 +1114,7 @@ gst_tensor_pad_caps_from_config (GstPad * pad, const GstTensorsConfig * config) GstCaps *caps = NULL; GstCaps *templ; gboolean is_flexible; + GstCaps *peer_caps; g_return_val_if_fail (GST_IS_PAD (pad), NULL); g_return_val_if_fail (config != NULL, NULL); @@ -956,6 +1136,10 @@ gst_tensor_pad_caps_from_config (GstPad * pad, const GstTensorsConfig * config) /* other/tensor */ if (config->info.num_tensors == 1 && _peer_has_tensor_caps (pad)) { caps = _get_tensor_caps (config); + if ((peer_caps = gst_pad_peer_query_caps (pad, NULL))) { + gst_tensor_caps_update_dimension (caps, peer_caps); + gst_caps_unref (peer_caps); + } if (gst_caps_can_intersect (caps, templ)) goto done; @@ -964,6 +1148,10 @@ gst_tensor_pad_caps_from_config (GstPad * pad, const GstTensorsConfig * config) /* other/tensors (static) */ caps = _get_tensors_caps (config); + if ((peer_caps = gst_pad_peer_query_caps (pad, NULL))) { + gst_tensor_caps_update_dimension (caps, peer_caps); + gst_caps_unref (peer_caps); + } intersectable: if (!gst_caps_can_intersect (caps, templ)) { diff --git a/gst/nnstreamer/tensor_filter/tensor_filter.c b/gst/nnstreamer/tensor_filter/tensor_filter.c index 6aed06a..776c152 100644 --- a/gst/nnstreamer/tensor_filter/tensor_filter.c +++ b/gst/nnstreamer/tensor_filter/tensor_filter.c @@ -1105,6 +1105,7 @@ gst_tensor_filter_transform_caps (GstBaseTransform * trans, GstTensorsConfig in_config, out_config; GstPad *pad; GstCaps *result; + GstCaps *peer_caps; GstStructure *structure; gboolean configured = FALSE; @@ -1183,6 +1184,14 @@ gst_tensor_filter_transform_caps (GstBaseTransform * trans, result = gst_caps_from_string (CAPS_STRING); } + /* Update caps dimension for src pad cap */ + if (direction == GST_PAD_SINK) { + if ((peer_caps = gst_pad_peer_query_caps (pad, NULL))) { + gst_tensor_caps_update_dimension (result, peer_caps); + gst_caps_unref (peer_caps); + } + } + if (filter && gst_caps_get_size (filter) > 0) { GstCaps *intersection; -- 2.7.4