tools/accuracy_checker/tests/test_config_validator.py

   1 """
   2 Copyright (c) 2019 Intel Corporation
   3
   4 Licensed under the Apache License, Version 2.0 (the "License");
   5 you may not use this file except in compliance with the License.
   6 You may obtain a copy of the License at
   7
   8       http://www.apache.org/licenses/LICENSE-2.0
   9
  10 Unless required by applicable law or agreed to in writing, software
  11 distributed under the License is distributed on an "AS IS" BASIS,
  12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 See the License for the specific language governing permissions and
  14 limitations under the License.
  15 """
  16
  17 from math import inf, nan
  18 from pathlib import Path
  19 from unittest.mock import ANY
  20
  21 import pytest
  22 from accuracy_checker.config.config_validator import (
  23     ConfigError,
  24     ConfigValidator,
  25     DictField,
  26     ListField,
  27     NumberField,
  28     PathField,
  29     StringField
  30 )
  31 from tests.common import mock_filesystem
  32
  33
  34 class TestStringField:
  35     def test_expects_string(self):
  36         string_field = StringField()
  37
  38         with pytest.raises(ConfigError):
  39             string_field.validate(b"foo")
  40         with pytest.raises(ConfigError):
  41             string_field.validate({})
  42         with pytest.raises(ConfigError):
  43             string_field.validate(42)
  44
  45         string_field.validate("foo")
  46
  47     def test_choices(self):
  48         string_field = StringField(choices=['foo', 'bar'])
  49
  50         with pytest.raises(ConfigError):
  51             string_field.validate('baz')
  52
  53         string_field.validate('bar')
  54
  55     def test_case_sensitive(self):
  56         string_field = StringField(choices=['foo', 'bar'], case_sensitive=False)
  57
  58         string_field.validate('foo')
  59         string_field.validate('FOO')
  60
  61         string_field = StringField(choices=['foo', 'bar'], case_sensitive=True)
  62
  63         string_field.validate('foo')
  64         with pytest.raises(ConfigError):
  65             string_field.validate('FOO')
  66
  67     def test_regex(self):
  68         string_field = StringField(regex=r'foo\d*')
  69
  70         string_field.validate('foo')
  71         string_field.validate('foo42')
  72
  73         with pytest.raises(ConfigError):
  74             string_field.validate('baz')
  75
  76     def test_custom_exception(self, mocker):
  77         stub = mocker.stub(name='custom_on_error')
  78         string_field = StringField(choices=['foo'], on_error=stub)
  79
  80         with pytest.raises(ConfigError):
  81             string_field.validate('bar', 'foo')
  82         stub.assert_called_once_with('bar', 'foo', ANY)
  83
  84     def test_custom_validator(self, mocker):
  85         stub = mocker.stub(name='custom_validator')
  86         string_field = StringField(choices=['foo'], additional_validator=stub)
  87
  88         string_field.validate('foo', 'baz')
  89         stub.assert_called_once_with('foo', 'baz')
  90
  91
  92 class TestNumberField:
  93     def test_expects_number(self):
  94         number_field = NumberField(floats=True)
  95
  96         number_field.validate(1.0)
  97         with pytest.raises(ConfigError):
  98             number_field.validate("foo")
  99         with pytest.raises(ConfigError):
 100             number_field.validate({})
 101         with pytest.raises(ConfigError):
 102             number_field.validate([])
 103
 104         number_field = NumberField(floats=False)
 105         number_field.validate(1)
 106         with pytest.raises(ConfigError):
 107             number_field.validate(1.0)
 108
 109     def test_nans(self):
 110         number_field = NumberField(allow_nan=True)
 111         number_field.validate(nan)
 112
 113         number_field = NumberField(allow_nan=False)
 114         with pytest.raises(ConfigError):
 115             number_field.validate(nan)
 116
 117     def test_infinity(self):
 118         number_field = NumberField(allow_inf=True)
 119         number_field.validate(inf)
 120
 121         number_field = NumberField(allow_inf=False)
 122         with pytest.raises(ConfigError):
 123             number_field.validate(inf)
 124
 125     def test_ranges(self):
 126         number_field = NumberField(min_value=0, max_value=5)
 127
 128         number_field.validate(0)
 129         number_field.validate(1)
 130         number_field.validate(2)
 131
 132         with pytest.raises(ConfigError):
 133             number_field.validate(-1)
 134         with pytest.raises(ConfigError):
 135             number_field.validate(7)
 136
 137
 138 class TestDictField:
 139     def test_expects_dict(self):
 140         dict_field = DictField()
 141
 142         dict_field.validate({})
 143         with pytest.raises(ConfigError):
 144             dict_field.validate("foo")
 145         with pytest.raises(ConfigError):
 146             dict_field.validate(42)
 147         with pytest.raises(ConfigError):
 148             dict_field.validate([])
 149
 150     def test_validates_keys(self):
 151         dict_field = DictField()
 152         dict_field.validate({'foo': 42, 1: 'bar'})
 153
 154         dict_field = DictField(key_type=str)
 155         dict_field.validate({'foo': 42, 'bar': 'bar'})
 156         with pytest.raises(ConfigError):
 157             dict_field.validate({'foo': 42, 1: 'bar'})
 158
 159         dict_field = DictField(key_type=StringField(choices=['foo', 'bar']))
 160         dict_field.validate({'foo': 42, 'bar': 42})
 161         with pytest.raises(ConfigError):
 162             dict_field.validate({'foo': 42, 1: 'bar'})
 163         with pytest.raises(ConfigError):
 164             dict_field.validate({'foo': 42, 'baz': 42})
 165
 166     def test_validates_values(self):
 167         dict_field = DictField()
 168         dict_field.validate({'foo': 42, 1: 'bar'})
 169
 170         dict_field = DictField(value_type=str)
 171         dict_field.validate({'foo': 'foo', 1: 'bar'})
 172         with pytest.raises(ConfigError):
 173             dict_field.validate({'foo': 42, 1: 2})
 174
 175         dict_field = DictField(value_type=StringField(choices=['foo', 'bar']))
 176         dict_field.validate({1: 'foo', 'bar': 'bar'})
 177         with pytest.raises(ConfigError):
 178             dict_field.validate({1: 'foo', 2: 3})
 179         with pytest.raises(ConfigError):
 180             dict_field.validate({1: 'foo', 2: 'baz'})
 181
 182     def test_converts_basic_types(self):
 183         dict_field = DictField(value_type=str)
 184         assert isinstance(dict_field.value_type, StringField)
 185
 186         dict_field = DictField(value_type=int)
 187         assert isinstance(dict_field.value_type, NumberField)
 188         assert dict_field.value_type.floats is False
 189
 190         dict_field = DictField(value_type=float)
 191         assert isinstance(dict_field.value_type, NumberField)
 192         assert dict_field.value_type.floats is True
 193
 194         dict_field = DictField(value_type=list)
 195         assert isinstance(dict_field.value_type, ListField)
 196
 197         dict_field = DictField(value_type=dict)
 198         assert isinstance(dict_field.value_type, DictField)
 199
 200         dict_field = DictField(value_type=Path)
 201         assert isinstance(dict_field.value_type, PathField)
 202
 203     def test_empty(self):
 204         dict_field = DictField()
 205         dict_field.validate({})
 206
 207         dict_field = DictField(allow_empty=False)
 208         with pytest.raises(ConfigError):
 209             dict_field.validate({})
 210
 211
 212 class TestListField:
 213     def test_expects_list(self):
 214         list_field = ListField()
 215
 216         list_field.validate([])
 217         with pytest.raises(ConfigError):
 218             list_field.validate("foo")
 219         with pytest.raises(ConfigError):
 220             list_field.validate(42)
 221         with pytest.raises(ConfigError):
 222             list_field.validate({})
 223
 224     def test_validates_values(self):
 225         list_field = ListField()
 226         list_field.validate(['foo', 42])
 227
 228         list_field = ListField(value_type=str)
 229         list_field.validate(['foo', 'bar'])
 230         with pytest.raises(ConfigError):
 231             list_field.validate(['foo', 42])
 232
 233         list_field = ListField(value_type=StringField(choices=['foo', 'bar']))
 234         list_field.validate(['foo', 'bar'])
 235         with pytest.raises(ConfigError):
 236             list_field.validate(['foo', 42])
 237         with pytest.raises(ConfigError):
 238             list_field.validate(['foo', 'bar', 'baz'])
 239
 240     def test_empty(self):
 241         list_field = ListField()
 242         list_field.validate([])
 243
 244         list_field = ListField(allow_empty=False)
 245         with pytest.raises(ConfigError):
 246             list_field.validate([])
 247
 248
 249 class TestPathField:
 250     @pytest.mark.usefixtures('mock_path_exists')
 251     def test_expects_path_like(self):
 252         path_field = PathField()
 253         path_field.validate('foo/bar')
 254         path_field.validate('/home/user')
 255         path_field.validate(Path('foo/bar'))
 256
 257         with pytest.raises(ConfigError):
 258             path_field.validate(42)
 259         with pytest.raises(ConfigError):
 260             path_field.validate({})
 261         with pytest.raises(ConfigError):
 262             path_field.validate([])
 263
 264     def test_path_is_checked(self):
 265         with mock_filesystem(['foo/bar']) as prefix:
 266             prefix_path = Path(prefix)
 267             file_field = PathField(is_directory=False)
 268             with pytest.raises(ConfigError):
 269                 file_field.validate(prefix_path / 'foo')
 270             file_field.validate(prefix_path / 'foo' / 'bar')
 271
 272             dir_field = PathField(is_directory=True)
 273             dir_field.validate(prefix_path / 'foo')
 274
 275             with pytest.raises(ConfigError):
 276                 dir_field.validate(prefix_path / 'foo' / 'bar')
 277
 278     def test_path_not_checked(self):
 279         with mock_filesystem(['foo/bar']) as prefix:
 280             prefix_path = Path(prefix)
 281             file_field = PathField(is_directory=False, check_exists=False)
 282             file_field.validate(prefix_path / 'foo' / 'bar')
 283
 284
 285 class TestConfigValidator:
 286     def test_compound(self):
 287         class SampleValidator(ConfigValidator):
 288             foo = StringField(choices=['foo'])
 289             bar = NumberField()
 290
 291         sample_validator = SampleValidator('Sample')
 292         sample_validator.validate({'foo': 'foo', 'bar': 1})
 293
 294         with pytest.raises(ConfigError):
 295             sample_validator.validate({'foo': 'foo'})
 296         with pytest.raises(ConfigError):
 297             sample_validator.validate({'foo': 'bar', 'bar': 1})
 298
 299     def test_optional_fields(self):
 300         class SampleValidatorNoOptionals(ConfigValidator):
 301             foo = StringField(choices=['foo'])
 302             bar = NumberField(optional=False)
 303
 304         sample_validator = SampleValidatorNoOptionals('Sample')
 305         sample_validator.validate({'foo': 'foo', 'bar': 1})
 306         with pytest.raises(ConfigError):
 307             sample_validator.validate({'foo': 'bar'})
 308
 309         class SampleValidatorWithOptionals(ConfigValidator):
 310             foo = StringField(choices=['foo'])
 311             bar = NumberField(optional=True)
 312
 313         sample_validator = SampleValidatorWithOptionals('Sample')
 314         sample_validator.validate({'foo': 'foo', 'bar': 1})
 315         sample_validator.validate({'foo': 'foo'})
 316
 317     def test_extra_fields__warn_on_extra(self):
 318         class SampleValidatorWarnOnExtra(ConfigValidator):
 319             foo = StringField(choices=['foo'])
 320
 321         sample_validator = SampleValidatorWarnOnExtra(
 322             'Sample', on_extra_argument=ConfigValidator.WARN_ON_EXTRA_ARGUMENT
 323         )
 324
 325         with pytest.warns(UserWarning):
 326             sample_validator.validate({'foo': 'foo', 'bar': 'bar'})
 327
 328     def test_extra_fields__error_on_extra(self):
 329         class SampleValidatorErrorOnExtra(ConfigValidator):
 330             foo = StringField(choices=['foo'])
 331
 332         sample_validator = SampleValidatorErrorOnExtra(
 333             'Sample', on_extra_argument=ConfigValidator.ERROR_ON_EXTRA_ARGUMENT)
 334
 335         with pytest.raises(ConfigError):
 336             sample_validator.validate({'foo': 'bar', 'bar': 'bar'})
 337
 338     def test_extra_fields__ignore_extra(self):
 339         class SampleValidatorIgnoresExtra(ConfigValidator):
 340             foo = StringField(choices=['foo'])
 341
 342         sample_validator = SampleValidatorIgnoresExtra(
 343             'Sample', on_extra_argument=ConfigValidator.IGNORE_ON_EXTRA_ARGUMENT)
 344
 345         sample_validator.validate({'foo': 'foo', 'bar': 'bar'})
 346
 347     def test_custom_exception(self, mocker):
 348         class SampleValidator(ConfigValidator):
 349             foo = StringField(choices=['foo'])
 350
 351         stub = mocker.stub(name='custom_on_error')
 352         sample_validator = SampleValidator('Sample', on_error=stub)
 353         sample_validator.validate({})
 354         stub.assert_called_once_with(ANY, 'Sample', ANY)
 355
 356     def test_custom_validator(self, mocker):
 357         class SampleValidator(ConfigValidator):
 358             foo = StringField(choices=['foo'])
 359
 360         stub = mocker.stub(name='custom_validator')
 361         sample_validator = SampleValidator('Sample', additional_validator=stub)
 362         entry = {'foo': 'foo'}
 363         sample_validator.validate(entry)
 364         stub.assert_called_once_with(entry, 'Sample')
 365
 366     def test_nested(self):
 367         class InnerValidator(ConfigValidator):
 368             foo = StringField(choices=['foo'])
 369
 370         class OuterValidator(ConfigValidator):
 371             bar = ListField(InnerValidator('Inner'))
 372
 373         outer_validator = OuterValidator('Outer', on_extra_argument=ConfigValidator.ERROR_ON_EXTRA_ARGUMENT)
 374
 375         outer_validator.validate({'bar': [{'foo': 'foo'}, {'foo': 'foo'}]})
 376
 377     def test_inheritance(self):
 378         class ParentValidator(ConfigValidator):
 379             foo = StringField(choices=['foo'])
 380
 381         class DerivedValidator(ParentValidator):
 382             bar = StringField(choices=['bar'])
 383
 384         derived_validator = DerivedValidator('Derived', on_extra_argument=ConfigValidator.ERROR_ON_EXTRA_ARGUMENT)
 385         derived_validator.validate({'foo': 'foo', 'bar': 'bar'})