name: CPU
model:
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml
+ name: resnet-50-pytorch
precision: FP16
framework: caffe2
- device:
name: GPU
model:
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml
+ name: resnet-50-pytorch
precision: FP16
framework: caffe2
- device:
name: CPU
model:
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml
+ name: resnet-50-pytorch
precision: FP16-INT8
framework: caffe2
- device:
name: GPU
model:
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml
+ name: resnet-50-pytorch
precision: FP16-INT8
framework: caffe2
- device:
name: CPU
model:
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml
+ name: mobilenet-v2
precision: FP16
framework: caffe2
- device:
name: GPU
model:
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml
+ name: mobilenet-v2
precision: FP16
framework: caffe2
- device:
name: CPU
model:
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml
+ name: mobilenet-v2
precision: FP16-INT8
framework: caffe2
- device:
name: GPU
model:
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml
+ name: mobilenet-v2
precision: FP16-INT8
framework: caffe2
\ No newline at end of file
import shutil
import logging
import tempfile
+from jsonschema import validate, ValidationError
from test_runner.utils import upload_timetest_data, \
DATABASE, DB_COLLECTIONS
pytestconfig.session_info.append(request.node._request.test_info)
+@pytest.fixture(scope="function")
+def validate_test_case(request, test_info):
+ """Fixture for validating test case on correctness.
+
+ Fixture checks current test case contains all fields required for
+ a correct work. To submit results to a database test case have
+ contain several additional properties.
+ """
+ schema = {
+ "type": "object",
+ "properties": {
+ "device": {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"}
+ }},
+ "model": {
+ "type": "object",
+ "properties": {
+ "path": {"type": "string"}
+ }},
+ },
+ }
+ if request.config.getoption("db_submit"):
+ # For submission data to a database some additional fields are required
+ schema["properties"]["model"]["properties"].update({
+ "name": {"type": "string"},
+ "precision": {"type": "string"},
+ "framework": {"type": "string"}
+ })
+ test_info["submit_to_db"] = True
+ try:
+ validate(instance=request.node.funcargs["instance"], schema=schema)
+ except ValidationError:
+ test_info["submit_to_db"] = False
+ raise
+ yield
+
+
@pytest.fixture(scope="session", autouse=True)
def prepare_tconf_with_refs(pytestconfig):
"""Fixture for preparing test config based on original test config
if not (run_id and db_url and db_collection):
yield
return
+ if not item._request.test_info["submit_to_db"]:
+ logging.error("Data won't be uploaded to a database on '{}' step".format(call.when))
+ yield
+ return
data = item.funcargs.copy()
data["timetest"] = data.pop("executable").stem
pytest==4.0.1
attrs==19.1.0 # required for pytest==4.0.1 to resolve compatibility issues
-PyYAML==5.3.1
\ No newline at end of file
+PyYAML==5.3.1
+jsonschema==3.2.0
\ No newline at end of file
name: CPU
model:
path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml # TODO: add link to `test_data` repo model
+ name: alexnet
+ precision: FP32
+ framework: caffe
- device:
name: GPU
model:
- path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml
\ No newline at end of file
+ path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml
+ name: alexnet
+ precision: FP32
+ framework: caffe
REFS_FACTOR = 1.2 # 120%
-def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir):
+def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir, validate_test_case):
"""Parameterized test.
:param instance: test instance. Should not be changed during test run
:param cl_cache_dir: directory to store OpenCL cache
:param test_info: custom `test_info` field of built-in `request` pytest fixture
:param temp_dir: path to a temporary directory. Will be cleaned up after test run
+ :param validate_test_case: custom pytest fixture. Should be declared as test argument to be enabled
"""
# Prepare model to get model_path
model_path = instance["model"].get("path")