tools/accuracy_checker/tests/test_config_validator.py

   1 """
   2 Copyright (c) 2019 Intel Corporation
   3
   4 Licensed under the Apache License, Version 2.0 (the "License");
   5 you may not use this file except in compliance with the License.
   6 You may obtain a copy of the License at
   7
   8       http://www.apache.org/licenses/LICENSE-2.0
   9
  10 Unless required by applicable law or agreed to in writing, software
  11 distributed under the License is distributed on an "AS IS" BASIS,
  12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 See the License for the specific language governing permissions and
  14 limitations under the License.
  15 """
  16
  17 from math import inf, nan
  18 from pathlib import Path
  19 from unittest.mock import ANY
  20
  21 import pytest
  22 from accuracy_checker.config.config_validator import (
  23     ConfigError,
  24     ConfigValidator,
  25     DictField,
  26     ListField,
  27     NumberField,
  28     PathField,
  29     StringField
  30 )
  31 from tests.common import mock_filesystem
  32
  33
  34 class TestStringField:
  35     def test_expects_string(self):
  36         string_field = StringField()
  37
  38         with pytest.raises(ConfigError):
  39             string_field.validate(b"foo")
  40         with pytest.raises(ConfigError):
  41             string_field.validate({})
  42         with pytest.raises(ConfigError):
  43             string_field.validate(42)
  44
  45         string_field.validate("foo")
  46
  47     def test_choices(self):
  48         string_field = StringField(choices=['foo', 'bar'])
  49
  50         with pytest.raises(ConfigError):
  51             string_field.validate('baz')
  52
  53         string_field.validate('bar')
  54
  55     def test_case_sensitive(self):
  56         string_field = StringField(choices=['foo', 'bar'], case_sensitive=False)
  57
  58         string_field.validate('foo')
  59         string_field.validate('FOO')
  60
  61         string_field = StringField(choices=['foo', 'bar'], case_sensitive=True)
  62
  63         string_field.validate('foo')
  64         with pytest.raises(ConfigError):
  65             string_field.validate('FOO')
  66
  67     def test_regex(self):
  68         string_field = StringField(regex=r'foo\d*')
  69
  70         string_field.validate('foo')
  71         string_field.validate('foo42')
  72
  73         with pytest.raises(ConfigError):
  74             string_field.validate('baz')
  75
  76     def test_custom_exception(self, mocker):
  77         stub = mocker.stub(name='custom_on_error')
  78         string_field = StringField(choices=['foo'], on_error=stub)
  79
  80         with pytest.raises(ConfigError):
  81             string_field.validate('bar', 'foo')
  82         stub.assert_called_once_with('bar', 'foo', ANY)
  83
  84     def test_custom_validator(self, mocker):
  85         stub = mocker.stub(name='custom_validator')
  86         string_field = StringField(choices=['foo'], additional_validator=stub)
  87
  88         string_field.validate('foo', 'baz')
  89         stub.assert_called_once_with('foo', 'baz')
  90
  91
  92 class TestNumberField:
  93     def test_expects_number(self):
  94         number_field = NumberField(floats=True)
  95
  96         number_field.validate(1.0)
  97         with pytest.raises(ConfigError):
  98             number_field.validate("foo")
  99         with pytest.raises(ConfigError):
 100             number_field.validate({})
 101         with pytest.raises(ConfigError):
 102             number_field.validate([])
 103
 104         number_field = NumberField(floats=False)
 105         number_field.validate(1)
 106         with pytest.raises(ConfigError):
 107             number_field.validate(1.0)
 108
 109     def test_nans(self):
 110         number_field = NumberField(allow_nan=True)
 111         number_field.validate(nan)
 112
 113         number_field = NumberField(allow_nan=False)
 114         with pytest.raises(ConfigError):
 115             number_field.validate(nan)
 116
 117     def test_infinity(self):
 118         number_field = NumberField(allow_inf=True)
 119         number_field.validate(inf)
 120
 121         number_field = NumberField(allow_inf=False)
 122         with pytest.raises(ConfigError):
 123             number_field.validate(inf)
 124
 125     def test_ranges(self):
 126         number_field = NumberField(min_value=0, max_value=5)
 127
 128         number_field.validate(0)
 129         number_field.validate(1)
 130         number_field.validate(2)
 131
 132         with pytest.raises(ConfigError):
 133             number_field.validate(-1)
 134         with pytest.raises(ConfigError):
 135             number_field.validate(7)
 136
 137
 138 class TestDictField:
 139     def test_expects_dict(self):
 140         dict_field = DictField()
 141
 142         dict_field.validate({})
 143         with pytest.raises(ConfigError):
 144             dict_field.validate("foo")
 145         with pytest.raises(ConfigError):
 146             dict_field.validate(42)
 147         with pytest.raises(ConfigError):
 148             dict_field.validate([])
 149
 150     def test_validates_keys(self):
 151         dict_field = DictField()
 152         dict_field.validate({'foo': 42, 1: 'bar'})
 153
 154         dict_field = DictField(key_type=str)
 155         dict_field.validate({'foo': 42, 'bar': 'bar'})
 156         with pytest.raises(ConfigError):
 157             dict_field.validate({'foo': 42, 1: 'bar'})
 158
 159         dict_field = DictField(key_type=StringField(choices=['foo', 'bar']))
 160         dict_field.validate({'foo': 42, 'bar': 42})
 161         with pytest.raises(ConfigError):
 162             dict_field.validate({'foo': 42, 1: 'bar'})
 163         with pytest.raises(ConfigError):
 164             dict_field.validate({'foo': 42, 'baz': 42})
 165
 166     def test_validates_values(self):
 167         dict_field = DictField()
 168         dict_field.validate({'foo': 42, 1: 'bar'})
 169
 170         dict_field = DictField(value_type=str)
 171         dict_field.validate({'foo': 'foo', 1: 'bar'})
 172         with pytest.raises(ConfigError):
 173             dict_field.validate({'foo': 42, 1: 2})
 174
 175         dict_field = DictField(value_type=StringField(choices=['foo', 'bar']))
 176         dict_field.validate({1: 'foo', 'bar': 'bar'})
 177         with pytest.raises(ConfigError):
 178             dict_field.validate({1: 'foo', 2: 3})
 179         with pytest.raises(ConfigError):
 180             dict_field.validate({1: 'foo', 2: 'baz'})
 181
 182     def test_converts_basic_types(self):
 183         dict_field = DictField(value_type=str)
 184         assert isinstance(dict_field.value_type, StringField)
 185
 186         dict_field = DictField(value_type=int)
 187         assert isinstance(dict_field.value_type, NumberField)
 188         assert dict_field.value_type.floats is False
 189
 190         dict_field = DictField(value_type=float)
 191         assert isinstance(dict_field.value_type, NumberField)
 192         assert dict_field.value_type.floats is True
 193
 194         dict_field = DictField(value_type=list)
 195         assert isinstance(dict_field.value_type, ListField)
 196
 197         dict_field = DictField(value_type=dict)
 198         assert isinstance(dict_field.value_type, DictField)
 199
 200         dict_field = DictField(value_type=Path)
 201         assert isinstance(dict_field.value_type, PathField)
 202
 203     def test_empty(self):
 204         dict_field = DictField()
 205         dict_field.validate({})
 206
 207         dict_field = DictField(allow_empty=False)
 208         with pytest.raises(ConfigError):
 209             dict_field.validate({})
 210
 211
 212 class TestListField:
 213     def test_expects_list(self):
 214         list_field = ListField()
 215
 216         list_field.validate([])
 217         with pytest.raises(ConfigError):
 218             list_field.validate("foo")
 219         with pytest.raises(ConfigError):
 220             list_field.validate(42)
 221         with pytest.raises(ConfigError):
 222             list_field.validate({})
 223
 224     def test_validates_values(self):
 225         list_field = ListField()
 226         list_field.validate(['foo', 42])
 227
 228         list_field = ListField(value_type=str)
 229         list_field.validate(['foo', 'bar'])
 230         with pytest.raises(ConfigError):
 231             list_field.validate(['foo', 42])
 232
 233         list_field = ListField(value_type=StringField(choices=['foo', 'bar']))
 234         list_field.validate(['foo', 'bar'])
 235         with pytest.raises(ConfigError):
 236             list_field.validate(['foo', 42])
 237         with pytest.raises(ConfigError):
 238             list_field.validate(['foo', 'bar', 'baz'])
 239
 240     def test_empty(self):
 241         list_field = ListField()
 242         list_field.validate([])
 243
 244         list_field = ListField(allow_empty=False)
 245         with pytest.raises(ConfigError):
 246             list_field.validate([])
 247
 248
 249 class TestPathField:
 250     @pytest.mark.usefixtures('mock_path_exists')
 251     def test_expects_path_like(self):
 252         path_field = PathField()
 253         path_field.validate('foo/bar')
 254         path_field.validate('/home/user')
 255         path_field.validate(Path('foo/bar'))
 256
 257         with pytest.raises(ConfigError):
 258             path_field.validate(42)
 259         with pytest.raises(ConfigError):
 260             path_field.validate({})
 261         with pytest.raises(ConfigError):
 262             path_field.validate([])
 263
 264     def test_path_is_checked(self):
 265         with mock_filesystem(['foo/bar']) as prefix:
 266             prefix_path = Path(prefix)
 267             file_field = PathField(is_directory=False)
 268             with pytest.raises(ConfigError):
 269                 file_field.validate(prefix_path / 'foo')
 270             file_field.validate(prefix_path / 'foo' / 'bar')
 271
 272             dir_field = PathField(is_directory=True)
 273             dir_field.validate(prefix_path / 'foo')
 274
 275             with pytest.raises(ConfigError):
 276                 dir_field.validate(prefix_path / 'foo' / 'bar')
 277
 278
 279 class TestConfigValidator:
 280     def test_compound(self):
 281         class SampleValidator(ConfigValidator):
 282             foo = StringField(choices=['foo'])
 283             bar = NumberField()
 284
 285         sample_validator = SampleValidator('Sample')
 286         sample_validator.validate({'foo': 'foo', 'bar': 1})
 287
 288         with pytest.raises(ConfigError):
 289             sample_validator.validate({'foo': 'foo'})
 290         with pytest.raises(ConfigError):
 291             sample_validator.validate({'foo': 'bar', 'bar': 1})
 292
 293     def test_optional_fields(self):
 294         class SampleValidatorNoOptionals(ConfigValidator):
 295             foo = StringField(choices=['foo'])
 296             bar = NumberField(optional=False)
 297
 298         sample_validator = SampleValidatorNoOptionals('Sample')
 299         sample_validator.validate({'foo': 'foo', 'bar': 1})
 300         with pytest.raises(ConfigError):
 301             sample_validator.validate({'foo': 'bar'})
 302
 303         class SampleValidatorWithOptionals(ConfigValidator):
 304             foo = StringField(choices=['foo'])
 305             bar = NumberField(optional=True)
 306
 307         sample_validator = SampleValidatorWithOptionals('Sample')
 308         sample_validator.validate({'foo': 'foo', 'bar': 1})
 309         sample_validator.validate({'foo': 'foo'})
 310
 311     def test_extra_fields__warn_on_extra(self):
 312         class SampleValidatorWarnOnExtra(ConfigValidator):
 313             foo = StringField(choices=['foo'])
 314
 315         sample_validator = SampleValidatorWarnOnExtra(
 316             'Sample', on_extra_argument=ConfigValidator.WARN_ON_EXTRA_ARGUMENT
 317         )
 318
 319         with pytest.warns(UserWarning):
 320             sample_validator.validate({'foo': 'foo', 'bar': 'bar'})
 321
 322     def test_extra_fields__error_on_extra(self):
 323         class SampleValidatorErrorOnExtra(ConfigValidator):
 324             foo = StringField(choices=['foo'])
 325
 326         sample_validator = SampleValidatorErrorOnExtra(
 327             'Sample', on_extra_argument=ConfigValidator.ERROR_ON_EXTRA_ARGUMENT)
 328
 329         with pytest.raises(ConfigError):
 330             sample_validator.validate({'foo': 'bar', 'bar': 'bar'})
 331
 332     def test_extra_fields__ignore_extra(self):
 333         class SampleValidatorIgnoresExtra(ConfigValidator):
 334             foo = StringField(choices=['foo'])
 335
 336         sample_validator = SampleValidatorIgnoresExtra(
 337             'Sample', on_extra_argument=ConfigValidator.IGNORE_ON_EXTRA_ARGUMENT)
 338
 339         sample_validator.validate({'foo': 'foo', 'bar': 'bar'})
 340
 341     def test_custom_exception(self, mocker):
 342         class SampleValidator(ConfigValidator):
 343             foo = StringField(choices=['foo'])
 344
 345         stub = mocker.stub(name='custom_on_error')
 346         sample_validator = SampleValidator('Sample', on_error=stub)
 347         sample_validator.validate({})
 348         stub.assert_called_once_with(ANY, 'Sample', ANY)
 349
 350     def test_custom_validator(self, mocker):
 351         class SampleValidator(ConfigValidator):
 352             foo = StringField(choices=['foo'])
 353
 354         stub = mocker.stub(name='custom_validator')
 355         sample_validator = SampleValidator('Sample', additional_validator=stub)
 356         entry = {'foo': 'foo'}
 357         sample_validator.validate(entry)
 358         stub.assert_called_once_with(entry, 'Sample')
 359
 360     def test_nested(self):
 361         class InnerValidator(ConfigValidator):
 362             foo = StringField(choices=['foo'])
 363
 364         class OuterValidator(ConfigValidator):
 365             bar = ListField(InnerValidator('Inner'))
 366
 367         outer_validator = OuterValidator('Outer', on_extra_argument=ConfigValidator.ERROR_ON_EXTRA_ARGUMENT)
 368
 369         outer_validator.validate({'bar': [{'foo': 'foo'}, {'foo': 'foo'}]})
 370
 371     def test_inheritance(self):
 372         class ParentValidator(ConfigValidator):
 373             foo = StringField(choices=['foo'])
 374
 375         class DerivedValidator(ParentValidator):
 376             bar = StringField(choices=['bar'])
 377
 378         derived_validator = DerivedValidator('Derived', on_extra_argument=ConfigValidator.ERROR_ON_EXTRA_ARGUMENT)
 379         derived_validator.validate({'foo': 'foo', 'bar': 'bar'})