tools/accuracy_checker/accuracy_checker/metrics/regression.py

   1 """
   2 Copyright (c) 2019 Intel Corporation
   3
   4 Licensed under the Apache License, Version 2.0 (the "License");
   5 you may not use this file except in compliance with the License.
   6 You may obtain a copy of the License at
   7
   8       http://www.apache.org/licenses/LICENSE-2.0
   9
  10 Unless required by applicable law or agreed to in writing, software
  11 distributed under the License is distributed on an "AS IS" BASIS,
  12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 See the License for the specific language governing permissions and
  14 limitations under the License.
  15 """
  16
  17 import warnings
  18 import math
  19 import numpy as np
  20
  21 from ..representation import (
  22     RegressionAnnotation,
  23     RegressionPrediction,
  24     FacialLandmarksAnnotation,
  25     FacialLandmarksPrediction,
  26     SuperResolutionAnnotation,
  27     SuperResolutionPrediction,
  28     GazeVectorAnnotation,
  29     GazeVectorPrediction
  30 )
  31
  32 from .metric import PerImageEvaluationMetric, BaseMetricConfig
  33 from ..config import BaseField, NumberField, BoolField, ConfigError, StringField
  34 from ..utils import string_to_tuple, finalize_metric_result
  35
  36
  37 class BaseIntervalRegressionMetricConfig(BaseMetricConfig):
  38     intervals = BaseField(optional=True)
  39     start = NumberField(optional=True)
  40     end = NumberField(optional=True)
  41     step = NumberField(optional=True)
  42     ignore_values_not_in_interval = BoolField(optional=True)
  43
  44
  45 class BaseRegressionMetric(PerImageEvaluationMetric):
  46     annotation_types = (RegressionAnnotation, )
  47     prediction_types = (RegressionPrediction, )
  48
  49     def __init__(self, value_differ, *args, **kwargs):
  50         super().__init__(*args, **kwargs)
  51         self.value_differ = value_differ
  52
  53     def configure(self):
  54         self.meta.update({'names': ['mean', 'std'], 'scale': 1, 'postfix': ' ', 'calculate_mean': False})
  55         self.magnitude = []
  56
  57     def update(self, annotation, prediction):
  58         self.magnitude.append(self.value_differ(annotation.value, prediction.value))
  59
  60     def evaluate(self, annotations, predictions):
  61         return np.mean(self.magnitude), np.std(self.magnitude)
  62
  63
  64 class BaseRegressionOnIntervals(PerImageEvaluationMetric):
  65     annotation_types = (RegressionAnnotation, )
  66     prediction_types = (RegressionPrediction, )
  67     _config_validator_type = BaseIntervalRegressionMetricConfig
  68
  69     def __init__(self, value_differ, *args, **kwargs):
  70         super().__init__(*args, **kwargs)
  71         self.value_differ = value_differ
  72
  73     def configure(self):
  74         self.meta.update({'scale': 1, 'postfix': ' ', 'calculate_mean': False})
  75         self.ignore_out_of_range = self.config.get('ignore_values_not_in_interval', True)
  76
  77         self.intervals = self.config.get('intervals')
  78         if not self.intervals:
  79             stop = self.config.get('end')
  80             if not stop:
  81                 raise ConfigError('intervals or start-step-end of interval should be specified for metric')
  82
  83             start = self.config.get('start', 0.0)
  84             step = self.config.get('step', 1.0)
  85             self.intervals = np.arange(start, stop + step, step)
  86
  87         if not isinstance(self.intervals, (list, np.ndarray)):
  88             self.intervals = string_to_tuple(self.intervals)
  89
  90         self.intervals = np.unique(self.intervals)
  91         self.magnitude = [[] for _ in range(len(self.intervals) + 1)]
  92
  93         self.meta['names'] = ([])
  94         if not self.ignore_out_of_range:
  95             self.meta['names'] = (['mean: < ' + str(self.intervals[0]), 'std: < ' + str(self.intervals[0])])
  96
  97         for index in range(len(self.intervals) - 1):
  98             self.meta['names'].append('mean: <= ' + str(self.intervals[index]) + ' < ' + str(self.intervals[index + 1]))
  99             self.meta['names'].append('std: <= ' + str(self.intervals[index]) + ' < ' + str(self.intervals[index + 1]))
 100
 101         if not self.ignore_out_of_range:
 102             self.meta['names'].append('mean: > ' + str(self.intervals[-1]))
 103             self.meta['names'].append('std: > ' + str(self.intervals[-1]))
 104
 105     def update(self, annotation, prediction):
 106         index = find_interval(annotation.value, self.intervals)
 107         self.magnitude[index].append(self.value_differ(annotation.value, prediction.value))
 108
 109     def evaluate(self, annotations, predictions):
 110         if self.ignore_out_of_range:
 111             self.magnitude = self.magnitude[1:-1]
 112
 113         result = [[np.mean(values), np.std(values)] if values else [np.nan, np.nan] for values in self.magnitude]
 114         result, self.meta['names'] = finalize_metric_result(np.reshape(result, -1), self.meta['names'])
 115
 116         if not result:
 117             warnings.warn("No values in given interval")
 118             result.append(0)
 119
 120         return result
 121
 122
 123 class MeanAbsoluteError(BaseRegressionMetric):
 124     __provider__ = 'mae'
 125
 126     def __init__(self, *args, **kwargs):
 127         super().__init__(mae_differ, *args, **kwargs)
 128
 129
 130 class MeanSquaredError(BaseRegressionMetric):
 131     __provider__ = 'mse'
 132
 133     def __init__(self, *args, **kwargs):
 134         super().__init__(mse_differ, *args, **kwargs)
 135
 136
 137 class RootMeanSquaredError(BaseRegressionMetric):
 138     __provider__ = 'rmse'
 139
 140     def __init__(self, *args, **kwargs):
 141         super().__init__(mse_differ, *args, **kwargs)
 142
 143     def evaluate(self, annotations, predictions):
 144         return np.sqrt(np.mean(self.magnitude)), np.sqrt(np.std(self.magnitude))
 145
 146
 147 class MeanAbsoluteErrorOnInterval(BaseRegressionOnIntervals):
 148     __provider__ = 'mae_on_interval'
 149
 150     def __init__(self, *args, **kwargs):
 151         super().__init__(mae_differ, *args, **kwargs)
 152
 153
 154 class MeanSquaredErrorOnInterval(BaseRegressionOnIntervals):
 155     __provider__ = 'mse_on_interval'
 156
 157     def __init__(self, *args, **kwargs):
 158         super().__init__(mse_differ, *args, **kwargs)
 159
 160
 161 class RootMeanSquaredErrorOnInterval(BaseRegressionOnIntervals):
 162     __provider__ = 'rmse_on_interval'
 163
 164     def __init__(self, *args, **kwargs):
 165         super().__init__(mse_differ, *args, **kwargs)
 166
 167     def evaluate(self, annotations, predictions):
 168         if self.ignore_out_of_range:
 169             self.magnitude = self.magnitude[1:-1]
 170
 171         result = []
 172         for values in self.magnitude:
 173             error = [np.sqrt(np.mean(values)), np.sqrt(np.std(values))] if values else [np.nan, np.nan]
 174             result.append(error)
 175
 176         result, self.meta['names'] = finalize_metric_result(np.reshape(result, -1), self.meta['names'])
 177
 178         if not result:
 179             warnings.warn("No values in given interval")
 180             result.append(0)
 181
 182         return result
 183
 184
 185 class FacialLandmarksPerPointNormedError(PerImageEvaluationMetric):
 186     __provider__ = 'per_point_normed_error'
 187
 188     annotation_types = (FacialLandmarksAnnotation, )
 189     prediction_types = (FacialLandmarksPrediction, )
 190
 191     def configure(self):
 192         self.meta.update({'scale': 1, 'postfix': ' ', 'calculate_mean': True, 'data_format': '{:.4f}'})
 193         self.magnitude = []
 194
 195     def update(self, annotation, prediction):
 196         result = point_regression_differ(
 197             annotation.x_values, annotation.y_values, prediction.x_values, prediction.y_values
 198         )
 199         result /= np.maximum(annotation.interocular_distance, np.finfo(np.float64).eps)
 200         self.magnitude.append(result)
 201
 202     def evaluate(self, annotations, predictions):
 203         num_points = np.shape(self.magnitude)[1]
 204         point_result_name_pattern = 'point_{}_normed_error'
 205         self.meta['names'] = [point_result_name_pattern.format(point_id) for point_id in range(num_points)]
 206         per_point_rmse = np.mean(self.magnitude, axis=1)
 207         per_point_rmse, self.meta['names'] = finalize_metric_result(per_point_rmse, self.meta['names'])
 208
 209         return per_point_rmse
 210
 211
 212 class NormedErrorMetricConfig(BaseMetricConfig):
 213     calculate_std = BoolField(optional=True)
 214     percentile = NumberField(optional=True, floats=False, min_value=0, max_value=100)
 215
 216
 217 class FacialLandmarksNormedError(PerImageEvaluationMetric):
 218     __provider__ = 'normed_error'
 219
 220     annotation_types = (FacialLandmarksAnnotation, )
 221     prediction_types = (FacialLandmarksPrediction, )
 222     _config_validator_type = NormedErrorMetricConfig
 223
 224     def configure(self):
 225         self.calculate_std = self.config.get('calculate_std', False)
 226         self.percentile = self.config.get('percentile')
 227         self.meta.update({
 228             'scale': 1,
 229             'postfix': ' ',
 230             'calculate_mean': not self.calculate_std or not self.percentile,
 231             'data_format': '{:.4f}',
 232             'names': ['mean']
 233         })
 234         self.magnitude = []
 235
 236     def update(self, annotation, prediction):
 237         per_point_result = point_regression_differ(
 238             annotation.x_values, annotation.y_values, prediction.x_values, prediction.y_values
 239         )
 240         avg_result = np.sum(per_point_result) / len(per_point_result)
 241         avg_result /= np.maximum(annotation.interocular_distance, np.finfo(np.float64).eps)
 242         self.magnitude.append(avg_result)
 243
 244     def evaluate(self, annotations, predictions):
 245         result = [np.mean(self.magnitude)]
 246
 247         if self.calculate_std:
 248             result.append(np.std(self.magnitude))
 249             self.meta['names'].append('std')
 250
 251         if self.percentile:
 252             sorted_magnitude = np.sort(self.magnitude)
 253             index = len(self.magnitude) / 100 * self.percentile
 254             result.append(sorted_magnitude[int(index)])
 255             self.meta['names'].append('{}th percentile'.format(self.percentile))
 256
 257         return result
 258
 259
 260 def calculate_distance(x_coords, y_coords, selected_points):
 261     first_point = [x_coords[selected_points[0]], y_coords[selected_points[0]]]
 262     second_point = [x_coords[selected_points[1]], y_coords[selected_points[1]]]
 263     return np.linalg.norm(np.subtract(first_point, second_point))
 264
 265
 266 def mae_differ(annotation_val, prediction_val):
 267     return np.abs(annotation_val - prediction_val)
 268
 269
 270 def mse_differ(annotation_val, prediction_val):
 271     return (annotation_val - prediction_val)**2
 272
 273
 274 def find_interval(value, intervals):
 275     for index, point in enumerate(intervals):
 276         if value < point:
 277             return index
 278
 279     return len(intervals)
 280
 281
 282 def point_regression_differ(annotation_val_x, annotation_val_y, prediction_val_x, prediction_val_y):
 283     loss = np.subtract(list(zip(annotation_val_x, annotation_val_y)), list(zip(prediction_val_x, prediction_val_y)))
 284     return np.linalg.norm(loss, 2, axis=1)
 285
 286
 287 class PeakSignalToNoiseRatio(BaseRegressionMetric):
 288     __provider__ = 'psnr'
 289
 290     annotation_types = (SuperResolutionAnnotation, )
 291     prediction_types = (SuperResolutionPrediction, )
 292
 293     def __init__(self, *args, **kwargs):
 294         super().__init__(self._psnr_differ, *args, **kwargs)
 295
 296     def validate_config(self):
 297         class _PSNRConfig(BaseMetricConfig):
 298             scale_border = NumberField(optional=True, min_value=0)
 299             color_order = StringField(optional=True, choices=['BGR', 'RGB'])
 300
 301         config_validator = _PSNRConfig('psnr', on_extra_argument=_PSNRConfig.ERROR_ON_EXTRA_ARGUMENT)
 302         config_validator.validate(self.config)
 303
 304     def configure(self):
 305         super().configure()
 306         self.scale_border = self.config.get('scale_border', 4)
 307         color_order = self.config.get('color_order', 'RGB')
 308         channel_order = {
 309             'BGR': [2, 1, 0],
 310             'RGB': [0, 1, 2]
 311         }
 312         self.meta['postfix'] = 'Db'
 313         self.channel_order = channel_order[color_order]
 314
 315     def _psnr_differ(self, annotation_image, prediction_image):
 316         prediction = np.asarray(prediction_image).astype(np.float)
 317         ground_truth = np.asarray(annotation_image).astype(np.float)
 318
 319         height, width = prediction.shape[:2]
 320         prediction = prediction[
 321             self.scale_border:height - self.scale_border,
 322             self.scale_border:width - self.scale_border
 323         ]
 324         ground_truth = ground_truth[
 325             self.scale_border:height - self.scale_border,
 326             self.scale_border:width - self.scale_border
 327         ]
 328         image_difference = (prediction - ground_truth) / 255.  # rgb color space
 329
 330         r_channel_diff = image_difference[:, :, self.channel_order[0]]
 331         g_channel_diff = image_difference[:, :, self.channel_order[1]]
 332         b_channel_diff = image_difference[:, :, self.channel_order[2]]
 333
 334         channels_diff = (r_channel_diff * 65.738 + g_channel_diff * 129.057 + b_channel_diff * 25.064) / 256
 335
 336         mse = np.mean(channels_diff ** 2)
 337         if mse == 0:
 338             return np.Infinity
 339
 340         return -10 * math.log10(mse)
 341
 342
 343 def angle_differ(gt_gaze_vector, predicted_gaze_vector):
 344     return np.arccos(
 345         gt_gaze_vector.dot(predicted_gaze_vector) / np.linalg.norm(gt_gaze_vector)
 346         / np.linalg.norm(predicted_gaze_vector)
 347     ) * 180 / np.pi
 348
 349
 350 class AngleError(BaseRegressionMetric):
 351     __provider__ = 'angle_error'
 352
 353     annotation_types = (GazeVectorAnnotation, )
 354     prediction_types = (GazeVectorPrediction, )
 355
 356     def __init__(self, *args, **kwargs):
 357         super().__init__(angle_differ, *args, **kwargs)