tools/accuracy_checker/tests/test_presenter.py

   1 """
   2 Copyright (c) 2019 Intel Corporation
   3
   4 Licensed under the Apache License, Version 2.0 (the "License");
   5 you may not use this file except in compliance with the License.
   6 You may obtain a copy of the License at
   7
   8       http://www.apache.org/licenses/LICENSE-2.0
   9
  10 Unless required by applicable law or agreed to in writing, software
  11 distributed under the License is distributed on an "AS IS" BASIS,
  12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 See the License for the specific language governing permissions and
  14 limitations under the License.
  15 """
  16
  17 import numpy as np
  18 import pytest
  19 from unittest.mock import MagicMock, call
  20 from accuracy_checker.metrics import MetricsExecutor
  21 from accuracy_checker.presenters import ScalarPrintPresenter, VectorPrintPresenter, EvaluationResult
  22 from accuracy_checker.representation import ClassificationAnnotation, ClassificationPrediction
  23
  24
  25 class TestPresenter:
  26     def test_config_default_presenter(self):
  27         annotations = [ClassificationAnnotation('identifier', 3)]
  28         predictions = [ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])]
  29         config = {'annotation': 'mocked', 'metrics': [{'type': 'accuracy', 'top_k': 1}]}
  30         dispatcher = MetricsExecutor(config, None)
  31         dispatcher.update_metrics_on_batch(annotations, predictions)
  32
  33         for presenter, _ in dispatcher.iterate_metrics(annotations, predictions):
  34             assert isinstance(presenter, ScalarPrintPresenter)
  35
  36     def test_config_scalar_presenter(self):
  37         annotations = [ClassificationAnnotation('identifier', 3)]
  38         predictions = [ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])]
  39         config = {'annotation': 'mocked', 'metrics': [{'type': 'accuracy', 'top_k': 1, 'presenter': 'print_scalar'}]}
  40         dispatcher = MetricsExecutor(config, None)
  41         dispatcher.update_metrics_on_batch(annotations, predictions)
  42
  43         for presenter, _ in dispatcher.iterate_metrics(annotations, predictions):
  44             assert isinstance(presenter, ScalarPrintPresenter)
  45
  46     def test_config_vector_presenter(self):
  47         annotations = [ClassificationAnnotation('identifier', 3)]
  48         predictions = [ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])]
  49         config = {'annotation': 'mocked', 'metrics': [{'type': 'accuracy', 'top_k': 1, 'presenter': 'print_vector'}]}
  50         dispatcher = MetricsExecutor(config, None)
  51         dispatcher.update_metrics_on_batch(annotations, predictions)
  52
  53         for presenter, _ in dispatcher.iterate_metrics(annotations, predictions):
  54             assert isinstance(presenter, VectorPrintPresenter)
  55
  56     def test_config_unknown_presenter(self):
  57         config = {'annotation': 'mocked', 'metrics': [{'type': 'accuracy', 'top_k': 1, 'presenter': 'print_somehow'}]}
  58         with pytest.raises(ValueError):
  59             MetricsExecutor(config, None)
  60
  61     def test_scalar_presenter_with_scalar_data(self, mocker):
  62         mock_write_scalar_result = mocker.patch('accuracy_checker.presenters.write_scalar_result')  # type: MagicMock
  63         result = EvaluationResult(
  64             name='scalar_metric',
  65             evaluated_value=0.1,
  66             reference_value=None,
  67             threshold=None,
  68             meta={},
  69         )
  70         presenter = ScalarPrintPresenter()
  71         presenter.write_result(result)
  72         mock_write_scalar_result.assert_called_once_with(
  73             result.evaluated_value,
  74             result.name,
  75             result.reference_value,
  76             result.threshold,
  77             postfix='%',
  78             scale=100,
  79             result_format='{:.2f}'
  80         )
  81
  82     def test_scalar_presenter_with_vector_data(self, mocker):
  83         mock_write_scalar_res = mocker.patch('accuracy_checker.presenters.write_scalar_result')  # type: MagicMock
  84         result = EvaluationResult(
  85             name='vector_metric',
  86             evaluated_value=[0.4, 0.6],
  87             reference_value=None,
  88             threshold=None,
  89             meta={},
  90         )
  91         presenter = ScalarPrintPresenter()
  92         presenter.write_result(result)
  93         mock_write_scalar_res.assert_called_once_with(
  94             np.mean(result.evaluated_value),
  95             result.name,
  96             result.reference_value,
  97             result.threshold,
  98             postfix='%',
  99             scale=100,
 100             result_format='{:.2f}'
 101         )
 102
 103     def test_default_format_for_scalar_presenter_with_ignore_formatting(self, mocker):
 104         mock_write_scalar_res = mocker.patch('accuracy_checker.presenters.write_scalar_result')  # type: MagicMock
 105         result = EvaluationResult(
 106             name='vector_metric',
 107             evaluated_value=[0.456],
 108             reference_value=None,
 109             threshold=None,
 110             meta={},
 111         )
 112         presenter = ScalarPrintPresenter()
 113         presenter.write_result(result, ignore_results_formatting=True)
 114         mock_write_scalar_res.assert_called_once_with(
 115             np.mean(result.evaluated_value),
 116             result.name,
 117             result.reference_value,
 118             result.threshold,
 119             postfix=' ',
 120             scale=1,
 121             result_format='{}'
 122         )
 123
 124     def test_specific_format_for_scalar_presenter_with_ignore_formatting(self, mocker):
 125         mock_write_scalar_res = mocker.patch('accuracy_checker.presenters.write_scalar_result')  # type: MagicMock
 126         result = EvaluationResult(
 127             name='vector_metric',
 128             evaluated_value=[0.456],
 129             reference_value=None,
 130             threshold=None,
 131             meta={'scale': 0.5, 'postfix': 'km/h', 'data_format': '{:.4f}'},
 132         )
 133         presenter = ScalarPrintPresenter()
 134         presenter.write_result(result, ignore_results_formatting=True)
 135         mock_write_scalar_res.assert_called_once_with(
 136             np.mean(result.evaluated_value),
 137             result.name,
 138             result.reference_value,
 139             result.threshold,
 140             postfix=' ',
 141             scale=1,
 142             result_format='{}'
 143         )
 144
 145     def test_vector_presenter_with_scaler_data(self, mocker):
 146         mock_write_scalar_res = mocker.patch('accuracy_checker.presenters.write_scalar_result')  # type: MagicMock
 147         result = EvaluationResult(
 148             name='scalar_metric',
 149             evaluated_value=0.4,
 150             reference_value=None,
 151             threshold=None,
 152             meta={},
 153         )
 154         presenter = VectorPrintPresenter()
 155         presenter.write_result(result)
 156         mock_write_scalar_res.assert_called_once_with(
 157             result.evaluated_value,
 158             result.name,
 159             result.reference_value,
 160             result.threshold,
 161             postfix='%',
 162             scale=100,
 163             value_name=None,
 164             result_format='{:.2f}'
 165         )
 166
 167     def test_vector_presenter_with_vector_data_contain_one_element(self, mocker):
 168         mock_write_scalar_res = mocker.patch('accuracy_checker.presenters.write_scalar_result')  # type: MagicMock
 169         result = EvaluationResult(
 170             name='scalar_metric',
 171             evaluated_value=[0.4],
 172             reference_value=None,
 173             threshold=None,
 174             meta={'names': ['prediction']}
 175         )
 176         presenter = VectorPrintPresenter()
 177         presenter.write_result(result)
 178         mock_write_scalar_res.assert_called_once_with(
 179             result.evaluated_value,
 180             result.name,
 181             result.reference_value,
 182             result.threshold,
 183             postfix='%',
 184             scale=100,
 185             value_name=result.meta['names'][0],
 186             result_format='{:.2f}'
 187         )
 188
 189     def test_vector_presenter_with_vector_data_with_default_postfix_and_scale(self, mocker):
 190         mock_write_scalar_res = mocker.patch('accuracy_checker.presenters.write_scalar_result')  # type: MagicMock
 191         result = EvaluationResult(
 192             name='scalar_metric',
 193             evaluated_value=[0.4, 0.6],
 194             reference_value=None,
 195             threshold=None,
 196             meta={'names': ['class1', 'class2']}
 197         )
 198         presenter = VectorPrintPresenter()
 199         presenter.write_result(result)
 200         calls = [
 201             call(
 202                 result.evaluated_value[0], result.name, result.reference_value, result.threshold,
 203                 postfix='%', scale=100, value_name=result.meta['names'][0], result_format='{:.2f}'
 204             ),
 205             call(
 206                 result.evaluated_value[1], result.name, result.reference_value, result.threshold,
 207                 postfix='%', scale=100, value_name=result.meta['names'][1],  result_format='{:.2f}'
 208             ),
 209             call(
 210                 np.mean(np.multiply(result.evaluated_value, 100)), result.name, result.reference_value,
 211                 result.threshold, value_name='mean', postfix='%', scale=1, result_format='{:.2f}'
 212             )
 213         ]
 214         mock_write_scalar_res.assert_has_calls(calls)
 215
 216     def test_vector_presenter_with_vector_data_has_default_format_with_ignore_formatting(self, mocker):
 217         mock_write_scalar_res = mocker.patch('accuracy_checker.presenters.write_scalar_result')  # type: MagicMock
 218         result = EvaluationResult(
 219             name='scalar_metric',
 220             evaluated_value=[0.4, 0.6],
 221             reference_value=None,
 222             threshold=None,
 223             meta={'names': ['class1', 'class2']}
 224         )
 225         presenter = VectorPrintPresenter()
 226         presenter.write_result(result, ignore_results_formatting=True)
 227         calls = [
 228             call(
 229                 result.evaluated_value[0], result.name, result.reference_value, result.threshold,
 230                 postfix=' ', scale=1, value_name=result.meta['names'][0], result_format='{}'
 231             ),
 232             call(
 233                 result.evaluated_value[1], result.name, result.reference_value, result.threshold,
 234                 postfix=' ', scale=1, value_name=result.meta['names'][1], result_format='{}'
 235             ),
 236             call(
 237                 np.mean(np.multiply(result.evaluated_value, 1)), result.name, result.reference_value, result.threshold,
 238                 value_name='mean', postfix=' ', scale=1, result_format='{}'
 239             )
 240         ]
 241         mock_write_scalar_res.assert_has_calls(calls)
 242
 243     def test_vector_presenter_with_vector_data_has_specific_format_with_ignore_formatting(self, mocker):
 244         mock_write_scalar_res = mocker.patch('accuracy_checker.presenters.write_scalar_result')  # type: MagicMock
 245         result = EvaluationResult(
 246             name='scalar_metric',
 247             evaluated_value=[0.4, 0.6],
 248             reference_value=None,
 249             threshold=None,
 250             meta={'names': ['class1', 'class2'], 'scale': 0.5, 'postfix': 'km/h', 'data_format': '{:.4f}'}
 251         )
 252         presenter = VectorPrintPresenter()
 253         presenter.write_result(result, ignore_results_formatting=True)
 254         calls = [
 255             call(
 256                 result.evaluated_value[0], result.name, result.reference_value, result.threshold,
 257                 postfix=' ', scale=1, value_name=result.meta['names'][0], result_format='{}'
 258             ),
 259             call(
 260                 result.evaluated_value[1], result.name, result.reference_value, result.threshold,
 261                 postfix=' ', scale=1, value_name=result.meta['names'][1], result_format='{}'
 262             ),
 263             call(
 264                 np.mean(np.multiply(result.evaluated_value, 1)), result.name, result.reference_value, result.threshold,
 265                 value_name='mean', postfix=' ', scale=1, result_format='{}'
 266             )
 267         ]
 268         mock_write_scalar_res.assert_has_calls(calls)
 269
 270     def test_vector_presenter_with_vector_data_with_scalar_postfix(self, mocker):
 271         mock_write_scalar_res = mocker.patch('accuracy_checker.presenters.write_scalar_result')  # type: MagicMock
 272         result = EvaluationResult(
 273             name='scalar_metric',
 274             evaluated_value=[0.4, 0.6],
 275             reference_value=None,
 276             threshold=None,
 277             meta={'names': ['class1', 'class2'], 'postfix': '_'}
 278         )
 279         presenter = VectorPrintPresenter()
 280         presenter.write_result(result)
 281         calls = [
 282             call(result.evaluated_value[0], result.name, result.reference_value, result.threshold,
 283                  postfix=result.meta['postfix'], scale=100, value_name=result.meta['names'][0], result_format='{:.2f}'
 284                  ),
 285             call(
 286                 result.evaluated_value[1], result.name, result.reference_value, result.threshold,
 287                 postfix=result.meta['postfix'], scale=100, value_name=result.meta['names'][1], result_format='{:.2f}'
 288             ),
 289             call(
 290                 np.mean(np.multiply(result.evaluated_value, 100)), result.name, result.reference_value,
 291                 result.threshold, value_name='mean', postfix=result.meta['postfix'], scale=1,  result_format='{:.2f}'
 292             )
 293         ]
 294         mock_write_scalar_res.assert_has_calls(calls)
 295
 296     def test_vector_presenter_with_vector_data_with_scalar_scale(self, mocker):
 297         mock_write_scalar_res = mocker.patch('accuracy_checker.presenters.write_scalar_result')  # type: MagicMock
 298         result = EvaluationResult(
 299             name='scalar_metric',
 300             evaluated_value=[0.4, 0.6],
 301             reference_value=None,
 302             threshold=None,
 303             meta={'names': ['class1', 'class2'], 'scale': 10}
 304         )
 305         presenter = VectorPrintPresenter()
 306         presenter.write_result(result)
 307         calls = [
 308             call(
 309                 result.evaluated_value[0], result.name, result.reference_value, result.threshold,
 310                 postfix='%', scale=result.meta['scale'], value_name=result.meta['names'][0], result_format='{:.2f}'
 311             ),
 312             call(
 313                 result.evaluated_value[1], result.name, result.reference_value, result.threshold,
 314                 postfix='%', scale=result.meta['scale'], value_name=result.meta['names'][1], result_format='{:.2f}'
 315             ),
 316             call(
 317                 np.mean(np.multiply(result.evaluated_value, result.meta['scale'])), result.name, result.reference_value,
 318                 result.threshold, value_name='mean', postfix='%', scale=1, result_format='{:.2f}'
 319             )
 320         ]
 321         mock_write_scalar_res.assert_has_calls(calls)
 322
 323     def test_vector_presenter_with_vector_data_with_vector_scale(self, mocker):
 324         mock_write_scalar_res = mocker.patch('accuracy_checker.presenters.write_scalar_result')  # type: MagicMock
 325         result = EvaluationResult(
 326             name='scalar_metric',
 327             evaluated_value=[0.4, 0.6],
 328             reference_value=None,
 329             threshold=None,
 330             meta={'names': ['class1', 'class2'], 'scale': [1, 2]}
 331         )
 332         presenter = VectorPrintPresenter()
 333         presenter.write_result(result)
 334         calls = [
 335             call(
 336                 result.evaluated_value[0], result.name, result.reference_value, result.threshold,
 337                 postfix='%', scale=result.meta['scale'][0], result_format='{:.2f}', value_name=result.meta['names'][0]
 338             ),
 339             call(
 340                 result.evaluated_value[1], result.name, result.reference_value, result.threshold, postfix='%',
 341                 scale=result.meta['scale'][1], result_format='{:.2f}', value_name=result.meta['names'][1]
 342             ),
 343             call(
 344                 np.mean(np.multiply(result.evaluated_value, result.meta['scale'])), result.name, result.reference_value,
 345                 result.threshold, result_format='{:.2f}', value_name='mean', postfix='%', scale=1
 346             )
 347         ]
 348         mock_write_scalar_res.assert_has_calls(calls)