From 255c4e69684b2b026de961b0642f9f05e299ffa0 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EC=9D=B4=EC=B6=98=EC=84=9D/=EB=8F=99=EC=9E=91=EC=A0=9C?= =?utf8?q?=EC=96=B4Lab=28SR=29/Staff=20Engineer/=EC=82=BC=EC=84=B1?= =?utf8?q?=EC=A0=84=EC=9E=90?= Date: Wed, 12 Dec 2018 16:38:50 +0900 Subject: [PATCH] Make TestGenerated file for each version (#3975) * Make TestGenerated file for each version On coverage build, build have failed since all_generated_tests.cpp is too large. This patch will generate a test file for each NNAPI version. Signed-off-by: Chunseok Lee * Extract common code from TestGenerated_XX.cpp - Extract common code from TestGenerated_XX.cpp --- runtimes/tests/neural_networks_test/CMakeLists.txt | 6 +- .../neural_networks_test/TestGenerated_Ex.cpp | 24 ++++ .../neural_networks_test/TestGenerated_V1_0.cpp | 24 ++++ .../neural_networks_test/TestGenerated_V1_1.cpp | 24 ++++ .../neural_networks_test/TestGenerated_common.cpp | 134 +++++++++++++++++++++ .../neural_networks_test/specs/generate_test.sh | 7 +- 6 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 runtimes/tests/neural_networks_test/TestGenerated_Ex.cpp create mode 100644 runtimes/tests/neural_networks_test/TestGenerated_V1_0.cpp create mode 100644 runtimes/tests/neural_networks_test/TestGenerated_V1_1.cpp create mode 100644 runtimes/tests/neural_networks_test/TestGenerated_common.cpp diff --git a/runtimes/tests/neural_networks_test/CMakeLists.txt b/runtimes/tests/neural_networks_test/CMakeLists.txt index 75efb51..677fc67 100644 --- a/runtimes/tests/neural_networks_test/CMakeLists.txt +++ b/runtimes/tests/neural_networks_test/CMakeLists.txt @@ -6,12 +6,14 @@ set(RUNTIME_ANDROID_NN_TEST_SRC TestMain.cpp # g++-6.3 can build TestGenerated.cpp. if (CMAKE_COMPILER_IS_GNUCC AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.3) - set(RUNTIME_ANDROID_NN_TEST_SRC ${RUNTIME_ANDROID_NN_TEST_SRC} TestGenerated.cpp) + set(RUNTIME_ANDROID_NN_TEST_SRC ${RUNTIME_ANDROID_NN_TEST_SRC} + TestGenerated_V1_0.cpp TestGenerated_V1_1.cpp TestGenerated_Ex.cpp) endif() # Both gbs and cross for tizen support building generated test if (OBS_BUILD) - set(RUNTIME_ANDROID_NN_TEST_SRC ${RUNTIME_ANDROID_NN_TEST_SRC} TestGenerated.cpp) + set(RUNTIME_ANDROID_NN_TEST_SRC ${RUNTIME_ANDROID_NN_TEST_SRC} + TestGenerated_V1_0.cpp TestGenerated_V1_1.cpp TestGenerated_Ex.cpp) endif() add_executable(${RUNTIME_ANDROID_NN_TEST} ${RUNTIME_ANDROID_NN_TEST_SRC}) diff --git a/runtimes/tests/neural_networks_test/TestGenerated_Ex.cpp b/runtimes/tests/neural_networks_test/TestGenerated_Ex.cpp new file mode 100644 index 0000000..2953c34 --- /dev/null +++ b/runtimes/tests/neural_networks_test/TestGenerated_Ex.cpp @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved + * + * generated/all_generated_tests.cpp that is included in the bottom line + * in file is also modified. + * + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "TestGenerated_common.cpp" +using namespace generated_tests; +#include "generated/all_generated_Ex_cts_tests.cpp" diff --git a/runtimes/tests/neural_networks_test/TestGenerated_V1_0.cpp b/runtimes/tests/neural_networks_test/TestGenerated_V1_0.cpp new file mode 100644 index 0000000..7266e77 --- /dev/null +++ b/runtimes/tests/neural_networks_test/TestGenerated_V1_0.cpp @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved + * + * generated/all_generated_tests.cpp that is included in the bottom line + * in file is also modified. + * + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "TestGenerated_common.cpp" +using namespace generated_tests; +#include "generated/all_generated_V1_0_cts_tests.cpp" diff --git a/runtimes/tests/neural_networks_test/TestGenerated_V1_1.cpp b/runtimes/tests/neural_networks_test/TestGenerated_V1_1.cpp new file mode 100644 index 0000000..e3251a0 --- /dev/null +++ b/runtimes/tests/neural_networks_test/TestGenerated_V1_1.cpp @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved + * + * generated/all_generated_tests.cpp that is included in the bottom line + * in file is also modified. + * + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "TestGenerated_common.cpp" +using namespace generated_tests; +#include "generated/all_generated_V1_1_cts_tests.cpp" diff --git a/runtimes/tests/neural_networks_test/TestGenerated_common.cpp b/runtimes/tests/neural_networks_test/TestGenerated_common.cpp new file mode 100644 index 0000000..2d17efe --- /dev/null +++ b/runtimes/tests/neural_networks_test/TestGenerated_common.cpp @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved + * + * generated/all_generated_tests.cpp that is included in the bottom line + * in file is also modified. + * + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Top level driver for models and examples generated by test_generator.py + +#include "NeuralNetworksWrapper.h" +#include "TestHarness.h" + +#include +#include +#include +#include +#include +#include + +// Uncomment the following line to generate DOT graphs. +// +// #define GRAPH GRAPH + +namespace generated_tests { +using namespace nnfw::rt::wrapper; + +template +static void print(std::ostream& os, const MixedTyped& test) { + // dump T-typed inputs + for_each(test, [&os](int idx, const std::vector& f) { + os << " aliased_output" << idx << ": ["; + for (size_t i = 0; i < f.size(); ++i) { + os << (i == 0 ? "" : ", ") << +f[i]; + } + os << "],\n"; + }); +} + +static void printAll(std::ostream& os, const MixedTyped& test) { + print(os, test); + print(os, test); + print(os, test); +} + +// Test driver for those generated from ml/nn/runtime/test/spec +static void execute(std::function createModel, + std::function isIgnored, + std::vector& examples, + std::string dumpFile = "") { + Model model; + createModel(&model); + model.finish(); + bool dumpToFile = !dumpFile.empty(); + + std::ofstream s; + if (dumpToFile) { + s.open(dumpFile, std::ofstream::trunc); + ASSERT_TRUE(s.is_open()); + } + + int exampleNo = 0; + Compilation compilation(&model); + compilation.finish(); + + const float fpRange = 1e-5f; + for (auto& example : examples) { + SCOPED_TRACE(exampleNo); + // TODO: We leave it as a copy here. + // Should verify if the input gets modified by the test later. + MixedTyped inputs = example.first; + const MixedTyped& golden = example.second; + + Execution execution(&compilation); + + // Set all inputs + for_all(inputs, [&execution](int idx, const void* p, size_t s) { + const void* buffer = s == 0 ? nullptr : p; + ASSERT_EQ(Result::NO_ERROR, execution.setInput(idx, buffer, s)); + }); + + MixedTyped test; + // Go through all typed outputs + resize_accordingly(golden, test); + for_all(test, [&execution](int idx, void* p, size_t s) { + void* buffer = s == 0 ? nullptr : p; + ASSERT_EQ(Result::NO_ERROR, execution.setOutput(idx, buffer, s)); + }); + + Result r = execution.compute(); + ASSERT_EQ(Result::NO_ERROR, r); + + // Dump all outputs for the slicing tool + if (dumpToFile) { + s << "output" << exampleNo << " = {\n"; + printAll(s, test); + // all outputs are done + s << "}\n"; + } + + // Filter out don't cares + MixedTyped filteredGolden = filter(golden, isIgnored); + MixedTyped filteredTest = filter(test, isIgnored); + // We want "close-enough" results for float + + compare(filteredGolden, filteredTest, fpRange); + exampleNo++; + } +} + +}; // namespace generated_tests + +using namespace nnfw::rt::wrapper; + +// Mixed-typed examples +typedef generated_tests::MixedTypedExampleType MixedTypedExample; + +class GeneratedTests : public ::testing::Test { + protected: + virtual void SetUp() {} +}; diff --git a/runtimes/tests/neural_networks_test/specs/generate_test.sh b/runtimes/tests/neural_networks_test/specs/generate_test.sh index 906f987..411f8e8 100755 --- a/runtimes/tests/neural_networks_test/specs/generate_test.sh +++ b/runtimes/tests/neural_networks_test/specs/generate_test.sh @@ -87,12 +87,13 @@ function generate_wrapper { mkdir -vp $NNAPI_BASE/$TEST_DIR/generated/examples cd $NNAPI_BASE/$TEST_DIR/specs - OUTFILE=$NNAPI_BASE/$TEST_DIR/generated/all_generated_tests.cpp - echo "// DO NOT EDIT;" > $OUTFILE - echo "// Generated by ${TEST_DIR}/specs/generate_test.sh" >> $OUTFILE for ver in $NNAPI_VERSION; do + OUTFILE=$NNAPI_BASE/$TEST_DIR/generated/all_generated_${ver}_cts_tests.cpp + echo "// DO NOT EDIT;" > $OUTFILE + echo "// Generated by ${TEST_DIR}/specs/generate_test.sh" >> $OUTFILE + VER_DIR=$NNAPI_BASE/$TEST_DIR/specs/$ver [ ! -d $VER_DIR ] && continue cd $VER_DIR -- 2.7.4