From f3a938747f2d657f5ce092c7ae41fa4f5e01205f Mon Sep 17 00:00:00 2001 From: Yongjoo Ahn Date: Mon, 29 Mar 2021 16:47:17 +0900 Subject: [PATCH] [tensor_transform] Add "per-channel" option to stand - "channel" is the first dimension. - Support channel-wise default and dc-average transform Signed-off-by: Yongjoo Ahn --- gst/nnstreamer/tensor_data.c | 129 +++++++++++++++++ gst/nnstreamer/tensor_data.h | 43 +++++- gst/nnstreamer/tensor_transform/tensor_transform.c | 154 +++++++++++++++------ gst/nnstreamer/tensor_transform/tensor_transform.h | 1 + 4 files changed, 285 insertions(+), 42 deletions(-) diff --git a/gst/nnstreamer/tensor_data.c b/gst/nnstreamer/tensor_data.c index ab4a0b5..a2a84b7 100644 --- a/gst/nnstreamer/tensor_data.c +++ b/gst/nnstreamer/tensor_data.c @@ -10,6 +10,7 @@ * @bug No known bugs except for NYI items */ +#include #include "tensor_data.h" #include "nnstreamer_log.h" #include "nnstreamer_plugin_api.h" @@ -293,3 +294,131 @@ gst_tensor_data_raw_average (gpointer raw, gsize length, tensor_type type) return average; } + +/** + * @brief Calculate average value of the tensor per channel (the first dim). + * @param raw pointer of raw tensor data + * @param length byte size of raw tensor data + * @param type tensor type + * @param tensor_dim tensor dimension + * @param results double array contains average values of each channel + * @return TRUE if no error + */ +gboolean +gst_tensor_data_raw_average_per_channel (gpointer raw, gsize length, + tensor_type type, tensor_dim dim, gdouble * results) +{ + gdouble value, average; + gulong ch, i, num, offset; + gsize element_size; + guint8 *data; + + g_return_val_if_fail (raw != NULL, FALSE); + g_return_val_if_fail (length > 0, FALSE); + g_return_val_if_fail (dim[0] > 0, FALSE); + g_return_val_if_fail (type != _NNS_END, FALSE); + + element_size = gst_tensor_get_element_size (type); + num = length / element_size; + + offset = dim[0]; + num = num / offset; + + for (ch = 0; ch < offset; ++ch) { + average = 0.0; + for (i = 0; i < num; ++i) { + /* extract value and typecast to double */ + data = (guint8 *) raw + element_size * ((i * offset) + ch); + gst_tensor_data_raw_typecast (data, type, &value, _NNS_FLOAT64); + average = (value - average) / (i + 1) + average; + } + + results[ch] = average; + } + + return TRUE; +} + +/** + * @brief Calculate standard deviation of the tensor. + * @param raw pointer of raw tensor data + * @param length byte size of raw tensor data + * @param type tensor type + * @param average average value of given tensor + * @return standard deviation + */ +gdouble +gst_tensor_data_raw_std (gpointer raw, gsize length, tensor_type type, + gdouble average) +{ + gdouble value, std; + gulong i, num; + gsize element_size; + guint8 *data; + + g_return_val_if_fail (raw != NULL, 0.0); + g_return_val_if_fail (length > 0, 0.0); + g_return_val_if_fail (type != _NNS_END, 0.0); + + element_size = gst_tensor_get_element_size (type); + num = length / element_size; + + std = 0.0; + for (i = 0; i < num; ++i) { + /* extract value and typecast to double */ + data = (guint8 *) raw + element_size * i; + gst_tensor_data_raw_typecast (data, type, &value, _NNS_FLOAT64); + + std += pow (value - average, 2) / (num - 1); + } + + std = (std != 0.0) ? sqrt (std) : (1e-10); + + return std; +} + +/** + * @brief Calculate standard deviation of the tensor per channel (the first dim). + * @param raw pointer of raw tensor data + * @param length byte size of raw tensor data + * @param type tensor type + * @param tensor_dim tensor dimension + * @param averages average values of given tensor per-channel + * @param results double array contains standard deviation of each channel + * @return TRUE if no error + */ +gboolean +gst_tensor_data_raw_std_per_channel (gpointer raw, gsize length, + tensor_type type, tensor_dim dim, gdouble * averages, gdouble * results) +{ + gdouble value, std; + gulong ch, i, num, offset; + gsize element_size; + guint8 *data; + + g_return_val_if_fail (raw != NULL, FALSE); + g_return_val_if_fail (length > 0, FALSE); + g_return_val_if_fail (dim[0] > 0, FALSE); + g_return_val_if_fail (type != _NNS_END, FALSE); + + element_size = gst_tensor_get_element_size (type); + num = length / element_size; + + offset = dim[0]; + num = num / offset; + + for (ch = 0; ch < offset; ++ch) { + std = 0.0; + for (i = 0; i < num; ++i) { + /* extract value and typecast to double */ + data = (guint8 *) raw + element_size * ((i * offset) + ch); + gst_tensor_data_raw_typecast (data, type, &value, _NNS_FLOAT64); + std += pow (value - averages[ch], 2) / (num - 1); + } + + std = (std != 0.0) ? sqrt (std) : (1e-10); + results[ch] = std; + } + + return TRUE; +} diff --git a/gst/nnstreamer/tensor_data.h b/gst/nnstreamer/tensor_data.h index a1f2996..5383709 100644 --- a/gst/nnstreamer/tensor_data.h +++ b/gst/nnstreamer/tensor_data.h @@ -17,7 +17,6 @@ #include G_BEGIN_DECLS - /** * @brief Structure for tensor data. */ @@ -64,7 +63,8 @@ gst_tensor_data_typecast (tensor_data_s * td, tensor_type type); * @return TRUE if no error */ extern gboolean -gst_tensor_data_raw_typecast (gpointer input, tensor_type in_type, gpointer output, tensor_type out_type); +gst_tensor_data_raw_typecast (gpointer input, tensor_type in_type, + gpointer output, tensor_type out_type); /** * @brief Calculate average value of the tensor. @@ -76,5 +76,44 @@ gst_tensor_data_raw_typecast (gpointer input, tensor_type in_type, gpointer outp extern gdouble gst_tensor_data_raw_average (gpointer raw, gsize length, tensor_type type); +/** + * @brief Calculate average value of the tensor per channel (the first dim). + * @param raw pointer of raw tensor data + * @param length byte size of raw tensor data + * @param type tensor type + * @param tensor_dim tensor dimension + * @param results double array contains average values of each channel + * @return TRUE if no error + */ +extern gboolean +gst_tensor_data_raw_average_per_channel (gpointer raw, gsize length, + tensor_type type, tensor_dim dim, gdouble * results); + +/** + * @brief Calculate standard deviation of the tensor. + * @param raw pointer of raw tensor data + * @param length byte size of raw tensor data + * @param type tensor type + * @param average average value of given tensor + * @return standard deviation + */ +extern gdouble +gst_tensor_data_raw_std (gpointer raw, gsize length, tensor_type type, + gdouble average); + +/** + * @brief Calculate standard deviation of the tensor per channel (the first dim). + * @param raw pointer of raw tensor data + * @param length byte size of raw tensor data + * @param type tensor type + * @param tensor_dim tensor dimension + * @param averages average values of given tensor per-channel + * @param results double array contains standard deviation of each channel + * @return TRUE if no error + */ +extern gboolean +gst_tensor_data_raw_std_per_channel (gpointer raw, gsize length, + tensor_type type, tensor_dim dim, gdouble * averages, gdouble * results); + G_END_DECLS #endif /* __NNS_TENSOR_DATA_H__ */ diff --git a/gst/nnstreamer/tensor_transform/tensor_transform.c b/gst/nnstreamer/tensor_transform/tensor_transform.c index 6f8ac42..8f689ef 100644 --- a/gst/nnstreamer/tensor_transform/tensor_transform.c +++ b/gst/nnstreamer/tensor_transform/tensor_transform.c @@ -202,7 +202,7 @@ gst_tensor_transform_mode_get_type (void) "option=D1\':D2\':D3\':D4 (fixed to 3)", "transpose"}, {GTT_STAND, "Mode for statistical standardization of tensor, " - "option=(default|dc-average)[:TYPE]", + "option=(default|dc-average)[:TYPE][,per-channel:(false|true)]", "stand"}, {GTT_CLAMP, "Mode for clamping all elements of tensor into the range, " "option=CLAMP_MIN:CLAMP_MAX", @@ -694,20 +694,41 @@ gst_tensor_transform_set_option_data (GstTensorTransform * filter) } case GTT_STAND: { - gchar **strv = NULL; - - strv = g_strsplit (filter->option, ":", -1); + gchar **options = NULL; + guint i, num_options; - filter->data_stand.mode = gst_tensor_transform_get_stand_mode (strv[0]); filter->data_stand.out_type = _NNS_END; - if (g_strv_length (strv) > 1) - filter->data_stand.out_type = gst_tensor_get_type (strv[1]); + filter->data_stand.per_channel = FALSE; + + options = g_strsplit (filter->option, ",", -1); + num_options = g_strv_length (options); + + for (i = 0; i < num_options; i++) { + gchar **strv = g_strsplit (options[i], ":", -1); + + if (g_ascii_strcasecmp (strv[0], "default") == 0 || + g_ascii_strcasecmp (strv[0], "dc-average") == 0) { + filter->data_stand.mode = + gst_tensor_transform_get_stand_mode (strv[0]); + if (g_strv_length (strv) > 1) + filter->data_stand.out_type = gst_tensor_get_type (strv[1]); + } else if (g_ascii_strcasecmp (strv[0], "per-channel") == 0) { + if (g_strv_length (strv) > 1 && + g_ascii_strcasecmp (strv[1], "true") == 0) + filter->data_stand.per_channel = TRUE; + } else { + filter->data_stand.mode = STAND_END; + ml_logw ("Unknown option for stand mode: %s", strv[0]); + } - g_strfreev (strv); + g_strfreev (strv); + } + + g_strfreev (options); if (filter->data_stand.mode == STAND_END) { ml_loge - ("%s: stand: \'%s\' is not valid option string: it should be in the form of (default|dc-average)[:TYPE]\n", + ("%s: stand: \'%s\' is not a valid option string: it should be in the form of (default|dc-average)[:TYPE][,per-channel:(false|true)]\n", filter_name, filter->option); break; } @@ -1221,60 +1242,113 @@ gst_tensor_transform_stand (GstTensorTransform * filter, { tensor_type in_tensor_type = filter->in_config.info.info[idx].type; tensor_type out_tensor_type = filter->out_config.info.info[idx].type; - gsize in_element_size, out_element_size, data_size; - gulong i, num, data_idx; - gdouble tmp, average, stand; + gsize in_element_size, out_element_size, data_size, ch_size; + gulong i, num, data_idx, ch; + gdouble tmp, average, std; + gdouble *avg_per_ch, *std_per_ch; in_element_size = gst_tensor_get_element_size (in_tensor_type); out_element_size = gst_tensor_get_element_size (out_tensor_type); num = gst_tensor_get_element_count (filter->in_config.info.info[idx].dimension); - /* calc average */ + ch_size = filter->in_config.info.info[idx].dimension[0]; + + /* calc average and std */ data_size = gst_tensor_info_get_size (&filter->in_config.info.info[idx]); average = gst_tensor_data_raw_average ((gpointer) inptr, data_size, in_tensor_type); + std = + gst_tensor_data_raw_std ((gpointer) inptr, data_size, in_tensor_type, + average); switch (filter->data_stand.mode) { case STAND_DEFAULT: { - stand = 0.0; - - for (i = 0; i < num; i++) { - data_idx = in_element_size * i; - gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx), - in_tensor_type, &tmp, _NNS_FLOAT64); - - stand += pow (tmp - average, 2) / (num - 1); - } + if (filter->data_stand.per_channel == FALSE) { + for (i = 0; i < num; i++) { + data_idx = in_element_size * i; + gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx), + in_tensor_type, &tmp, _NNS_FLOAT64); - stand = (stand != 0.0) ? sqrt (stand) : (1e-10); - for (i = 0; i < num; i++) { - data_idx = in_element_size * i; - gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx), - in_tensor_type, &tmp, _NNS_FLOAT64); + tmp = fabs ((tmp - average) / std); - tmp = fabs ((tmp - average) / stand); + data_idx = out_element_size * i; + gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64, + (gpointer) (outptr + data_idx), out_tensor_type); + } + } else { + avg_per_ch = + (gdouble *) g_malloc0 (sizeof (gdouble) * + filter->in_config.info.info[idx].dimension[0]); + + gst_tensor_data_raw_average_per_channel ((gpointer) inptr, data_size, + in_tensor_type, filter->in_config.info.info[idx].dimension, + avg_per_ch); + + std_per_ch = + (gdouble *) g_malloc0 (sizeof (gdouble) * + filter->in_config.info.info[idx].dimension[0]); + gst_tensor_data_raw_std_per_channel ((gpointer) inptr, data_size, + in_tensor_type, filter->in_config.info.info[idx].dimension, + avg_per_ch, std_per_ch); + + for (ch = 0; ch < ch_size; ++ch) { + for (i = 0; i < num / ch_size; i++) { + data_idx = in_element_size * ((i * ch_size) + ch); + gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx), + in_tensor_type, &tmp, _NNS_FLOAT64); + + tmp = fabs ((tmp - avg_per_ch[ch]) / std_per_ch[ch]); + + data_idx = out_element_size * ((i * ch_size) + ch); + gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64, + (gpointer) (outptr + data_idx), out_tensor_type); + } + } - data_idx = out_element_size * i; - gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64, - (gpointer) (outptr + data_idx), out_tensor_type); + g_free (avg_per_ch); + g_free (std_per_ch); } - break; } case STAND_DC_AVERAGE: { - for (i = 0; i < num; i++) { - data_idx = in_element_size * i; - gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx), - in_tensor_type, &tmp, _NNS_FLOAT64); + if (filter->data_stand.per_channel == FALSE) { + for (i = 0; i < num; i++) { + data_idx = in_element_size * i; + gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx), + in_tensor_type, &tmp, _NNS_FLOAT64); + + tmp -= average; + data_idx = out_element_size * i; + gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64, + (gpointer) (outptr + data_idx), out_tensor_type); + } + } else { + avg_per_ch = + (gdouble *) g_malloc0 (sizeof (gdouble) * + filter->in_config.info.info[idx].dimension[0]); + + gst_tensor_data_raw_average_per_channel ((gpointer) inptr, data_size, + in_tensor_type, filter->in_config.info.info[idx].dimension, + avg_per_ch); + + for (ch = 0; ch < ch_size; ++ch) { + for (i = 0; i < num / ch_size; i++) { + data_idx = in_element_size * ((i * ch_size) + ch); + gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx), + in_tensor_type, &tmp, _NNS_FLOAT64); - tmp -= average; + tmp -= avg_per_ch[ch]; + + data_idx = out_element_size * ((i * ch_size) + ch); + gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64, + (gpointer) (outptr + data_idx), out_tensor_type); + } + } - data_idx = out_element_size * i; - gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64, - (gpointer) (outptr + data_idx), out_tensor_type); + g_free (avg_per_ch); } break; } diff --git a/gst/nnstreamer/tensor_transform/tensor_transform.h b/gst/nnstreamer/tensor_transform/tensor_transform.h index 812c6ea..92df411 100644 --- a/gst/nnstreamer/tensor_transform/tensor_transform.h +++ b/gst/nnstreamer/tensor_transform/tensor_transform.h @@ -127,6 +127,7 @@ typedef struct _tensor_transform_transpose { typedef struct _tensor_transform_stand { tensor_transform_stand_mode mode; tensor_type out_type; + gboolean per_channel; } tensor_transform_stand; /** -- 2.7.4