2 // Copyright © 2017 Arm Ltd. All rights reserved.
3 // See LICENSE file in the project root for full license information.
5 #include "InferenceTest.hpp"
7 #include <boost/algorithm/string.hpp>
8 #include <boost/numeric/conversion/cast.hpp>
9 #include <boost/log/trivial.hpp>
10 #include <boost/filesystem/path.hpp>
11 #include <boost/assert.hpp>
12 #include <boost/format.hpp>
13 #include <boost/program_options.hpp>
14 #include <boost/filesystem/operations.hpp>
23 using namespace std::chrono;
24 using namespace armnn::test;
32 template <typename TTestCaseDatabase, typename TModel>
33 ClassifierTestCase<TTestCaseDatabase, TModel>::ClassifierTestCase(
34 int& numInferencesRef,
35 int& numCorrectInferencesRef,
36 const std::vector<unsigned int>& validationPredictions,
37 std::vector<unsigned int>* validationPredictionsOut,
39 unsigned int testCaseId,
41 std::vector<typename TModel::DataType> modelInput)
42 : InferenceModelTestCase<TModel>(model, testCaseId, std::move(modelInput), model.GetOutputSize())
44 , m_QuantizationParams(model.GetQuantizationParams())
45 , m_NumInferencesRef(numInferencesRef)
46 , m_NumCorrectInferencesRef(numCorrectInferencesRef)
47 , m_ValidationPredictions(validationPredictions)
48 , m_ValidationPredictionsOut(validationPredictionsOut)
52 template <typename TTestCaseDatabase, typename TModel>
53 TestCaseResult ClassifierTestCase<TTestCaseDatabase, TModel>::ProcessResult(const InferenceTestOptions& params)
55 auto& output = this->GetOutput();
56 const auto testCaseId = this->GetTestCaseId();
58 std::map<float,int> resultMap;
61 for (const auto & o : output)
63 resultMap[ToFloat<typename TModel::DataType>::Convert(o, m_QuantizationParams)] = index++;
68 BOOST_LOG_TRIVIAL(info) << "= Prediction values for test #" << testCaseId;
69 auto it = resultMap.rbegin();
70 for (int i=0; i<5 && it != resultMap.rend(); ++i)
72 BOOST_LOG_TRIVIAL(info) << "Top(" << (i+1) << ") prediction is " << it->second <<
73 " with confidence: " << 100.0*(it->first) << "%";
78 const unsigned int prediction = boost::numeric_cast<unsigned int>(
79 std::distance(output.begin(), std::max_element(output.begin(), output.end())));
81 // If we're just running the defaultTestCaseIds, each one must be classified correctly.
82 if (params.m_IterationCount == 0 && prediction != m_Label)
84 BOOST_LOG_TRIVIAL(error) << "Prediction for test case " << testCaseId << " (" << prediction << ")" <<
85 " is incorrect (should be " << m_Label << ")";
86 return TestCaseResult::Failed;
89 // If a validation file was provided as input, it checks that the prediction matches.
90 if (!m_ValidationPredictions.empty() && prediction != m_ValidationPredictions[testCaseId])
92 BOOST_LOG_TRIVIAL(error) << "Prediction for test case " << testCaseId << " (" << prediction << ")" <<
93 " doesn't match the prediction in the validation file (" << m_ValidationPredictions[testCaseId] << ")";
94 return TestCaseResult::Failed;
97 // If a validation file was requested as output, it stores the predictions.
98 if (m_ValidationPredictionsOut)
100 m_ValidationPredictionsOut->push_back(prediction);
103 // Updates accuracy stats.
104 m_NumInferencesRef++;
105 if (prediction == m_Label)
107 m_NumCorrectInferencesRef++;
110 return TestCaseResult::Ok;
113 template <typename TDatabase, typename InferenceModel>
114 template <typename TConstructDatabaseCallable, typename TConstructModelCallable>
115 ClassifierTestCaseProvider<TDatabase, InferenceModel>::ClassifierTestCaseProvider(
116 TConstructDatabaseCallable constructDatabase, TConstructModelCallable constructModel)
117 : m_ConstructModel(constructModel)
118 , m_ConstructDatabase(constructDatabase)
120 , m_NumCorrectInferences(0)
124 template <typename TDatabase, typename InferenceModel>
125 void ClassifierTestCaseProvider<TDatabase, InferenceModel>::AddCommandLineOptions(
126 boost::program_options::options_description& options)
128 namespace po = boost::program_options;
130 options.add_options()
131 ("validation-file-in", po::value<std::string>(&m_ValidationFileIn)->default_value(""),
132 "Reads expected predictions from the given file and confirms they match the actual predictions.")
133 ("validation-file-out", po::value<std::string>(&m_ValidationFileOut)->default_value(""),
134 "Predictions are saved to the given file for later use via --validation-file-in.")
135 ("data-dir,d", po::value<std::string>(&m_DataDir)->required(),
136 "Path to directory containing test data");
138 InferenceModel::AddCommandLineOptions(options, m_ModelCommandLineOptions);
141 template <typename TDatabase, typename InferenceModel>
142 bool ClassifierTestCaseProvider<TDatabase, InferenceModel>::ProcessCommandLineOptions()
144 if (!ValidateDirectory(m_DataDir))
151 m_Model = m_ConstructModel(m_ModelCommandLineOptions);
157 m_Database = std::make_unique<TDatabase>(m_ConstructDatabase(m_DataDir.c_str(), *m_Model));
166 template <typename TDatabase, typename InferenceModel>
167 std::unique_ptr<IInferenceTestCase>
168 ClassifierTestCaseProvider<TDatabase, InferenceModel>::GetTestCase(unsigned int testCaseId)
170 std::unique_ptr<typename TDatabase::TTestCaseData> testCaseData = m_Database->GetTestCaseData(testCaseId);
171 if (testCaseData == nullptr)
176 return std::make_unique<ClassifierTestCase<TDatabase, InferenceModel>>(
178 m_NumCorrectInferences,
179 m_ValidationPredictions,
180 m_ValidationFileOut.empty() ? nullptr : &m_ValidationPredictionsOut,
183 testCaseData->m_Label,
184 std::move(testCaseData->m_InputImage));
187 template <typename TDatabase, typename InferenceModel>
188 bool ClassifierTestCaseProvider<TDatabase, InferenceModel>::OnInferenceTestFinished()
190 const double accuracy = boost::numeric_cast<double>(m_NumCorrectInferences) /
191 boost::numeric_cast<double>(m_NumInferences);
192 BOOST_LOG_TRIVIAL(info) << std::fixed << std::setprecision(3) << "Overall accuracy: " << accuracy;
194 // If a validation file was requested as output, the predictions are saved to it.
195 if (!m_ValidationFileOut.empty())
197 std::ofstream validationFileOut(m_ValidationFileOut.c_str(), std::ios_base::trunc | std::ios_base::out);
198 if (validationFileOut.good())
200 for (const unsigned int prediction : m_ValidationPredictionsOut)
202 validationFileOut << prediction << std::endl;
207 BOOST_LOG_TRIVIAL(error) << "Failed to open output validation file: " << m_ValidationFileOut;
215 template <typename TDatabase, typename InferenceModel>
216 void ClassifierTestCaseProvider<TDatabase, InferenceModel>::ReadPredictions()
218 // Reads the expected predictions from the input validation file (if provided).
219 if (!m_ValidationFileIn.empty())
221 std::ifstream validationFileIn(m_ValidationFileIn.c_str(), std::ios_base::in);
222 if (validationFileIn.good())
224 while (!validationFileIn.eof())
227 validationFileIn >> i;
228 m_ValidationPredictions.emplace_back(i);
233 throw armnn::Exception(boost::str(boost::format("Failed to open input validation file: %1%")
234 % m_ValidationFileIn));
239 template<typename TConstructTestCaseProvider>
240 int InferenceTestMain(int argc,
242 const std::vector<unsigned int>& defaultTestCaseIds,
243 TConstructTestCaseProvider constructTestCaseProvider)
245 // Configures logging for both the ARMNN library and this test program.
247 armnn::LogSeverity level = armnn::LogSeverity::Info;
249 armnn::LogSeverity level = armnn::LogSeverity::Debug;
251 armnn::ConfigureLogging(true, true, level);
252 armnnUtils::ConfigureLogging(boost::log::core::get().get(), true, true, level);
256 std::unique_ptr<IInferenceTestCaseProvider> testCaseProvider = constructTestCaseProvider();
257 if (!testCaseProvider)
262 InferenceTestOptions inferenceTestOptions;
263 if (!ParseCommandLine(argc, argv, *testCaseProvider, inferenceTestOptions))
268 const bool success = InferenceTest(inferenceTestOptions, defaultTestCaseIds, *testCaseProvider);
269 return success ? 0 : 1;
271 catch (armnn::Exception const& e)
273 BOOST_LOG_TRIVIAL(fatal) << "Armnn Error: " << e.what();
279 // This function allows us to create a classifier inference test based on:
280 // - a model file name
281 // - which can be a binary or a text file for protobuf formats
282 // - an input tensor name
283 // - an output tensor name
284 // - a set of test case ids
285 // - a callback method which creates an object that can return images
286 // called 'Database' in these tests
287 // - and an input tensor shape
289 template<typename TDatabase,
291 typename TConstructDatabaseCallable>
292 int ClassifierInferenceTestMain(int argc,
294 const char* modelFilename,
296 const char* inputBindingName,
297 const char* outputBindingName,
298 const std::vector<unsigned int>& defaultTestCaseIds,
299 TConstructDatabaseCallable constructDatabase,
300 const armnn::TensorShape* inputTensorShape)
302 return InferenceTestMain(argc, argv, defaultTestCaseIds,
306 using InferenceModel = InferenceModel<TParser, typename TDatabase::DataType>;
307 using TestCaseProvider = ClassifierTestCaseProvider<TDatabase, InferenceModel>;
309 return make_unique<TestCaseProvider>(constructDatabase,
311 (typename InferenceModel::CommandLineOptions modelOptions)
313 if (!ValidateDirectory(modelOptions.m_ModelDir))
315 return std::unique_ptr<InferenceModel>();
318 typename InferenceModel::Params modelParams;
319 modelParams.m_ModelPath = modelOptions.m_ModelDir + modelFilename;
320 modelParams.m_InputBinding = inputBindingName;
321 modelParams.m_OutputBinding = outputBindingName;
322 modelParams.m_InputTensorShape = inputTensorShape;
323 modelParams.m_IsModelBinary = isModelBinary;
324 modelParams.m_ComputeDevice = modelOptions.m_ComputeDevice;
325 modelParams.m_VisualizePostOptimizationModel = modelOptions.m_VisualizePostOptimizationModel;
326 modelParams.m_EnableFp16TurboMode = modelOptions.m_EnableFp16TurboMode;
328 return std::make_unique<InferenceModel>(modelParams);