2 // Copyright © 2017 Arm Ltd. All rights reserved.
3 // SPDX-License-Identifier: MIT
7 #include "InferenceTest.hpp"
8 #include "YoloDatabase.hpp"
14 #include <boost/assert.hpp>
15 #include <boost/multi_array.hpp>
16 #include <boost/test/tools/floating_point_comparison.hpp>
18 constexpr size_t YoloOutputSize = 1470;
20 template <typename Model>
21 class YoloTestCase : public InferenceModelTestCase<Model>
24 YoloTestCase(Model& model,
25 unsigned int testCaseId,
26 YoloTestCaseData& testCaseData)
27 : InferenceModelTestCase<Model>(model, testCaseId, { std::move(testCaseData.m_InputImage) }, { YoloOutputSize })
28 , m_FloatComparer(boost::math::fpc::percent_tolerance(1.0f))
29 , m_TopObjectDetections(std::move(testCaseData.m_TopObjectDetections))
33 virtual TestCaseResult ProcessResult(const InferenceTestOptions& options) override
35 using Boost3dArray = boost::multi_array<float, 3>;
37 const std::vector<float>& output = boost::get<std::vector<float>>(this->GetOutputs()[0]);
38 BOOST_ASSERT(output.size() == YoloOutputSize);
40 constexpr Boost3dArray::index gridSize = 7;
41 constexpr Boost3dArray::index numClasses = 20;
42 constexpr Boost3dArray::index numScales = 2;
44 const float* outputPtr = output.data();
46 // Range 0-980. Class probabilities. 7x7x20
47 Boost3dArray classProbabilities(boost::extents[gridSize][gridSize][numClasses]);
48 for (Boost3dArray::index y = 0; y < gridSize; ++y)
50 for (Boost3dArray::index x = 0; x < gridSize; ++x)
52 for (Boost3dArray::index c = 0; c < numClasses; ++c)
54 classProbabilities[y][x][c] = *outputPtr++;
59 // Range 980-1078. Scales. 7x7x2
60 Boost3dArray scales(boost::extents[gridSize][gridSize][numScales]);
61 for (Boost3dArray::index y = 0; y < gridSize; ++y)
63 for (Boost3dArray::index x = 0; x < gridSize; ++x)
65 for (Boost3dArray::index s = 0; s < numScales; ++s)
67 scales[y][x][s] = *outputPtr++;
72 // Range 1078-1469. Bounding boxes. 7x7x2x4
73 constexpr float imageWidthAsFloat = static_cast<float>(YoloImageWidth);
74 constexpr float imageHeightAsFloat = static_cast<float>(YoloImageHeight);
76 boost::multi_array<float, 4> boxes(boost::extents[gridSize][gridSize][numScales][4]);
77 for (Boost3dArray::index y = 0; y < gridSize; ++y)
79 for (Boost3dArray::index x = 0; x < gridSize; ++x)
81 for (Boost3dArray::index s = 0; s < numScales; ++s)
83 float bx = *outputPtr++;
84 float by = *outputPtr++;
85 float bw = *outputPtr++;
86 float bh = *outputPtr++;
88 boxes[y][x][s][0] = ((bx + static_cast<float>(x)) / 7.0f) * imageWidthAsFloat;
89 boxes[y][x][s][1] = ((by + static_cast<float>(y)) / 7.0f) * imageHeightAsFloat;
90 boxes[y][x][s][2] = bw * bw * static_cast<float>(imageWidthAsFloat);
91 boxes[y][x][s][3] = bh * bh * static_cast<float>(imageHeightAsFloat);
95 BOOST_ASSERT(output.data() + YoloOutputSize == outputPtr);
97 std::vector<YoloDetectedObject> detectedObjects;
98 detectedObjects.reserve(gridSize * gridSize * numScales * numClasses);
100 for (Boost3dArray::index y = 0; y < gridSize; ++y)
102 for (Boost3dArray::index x = 0; x < gridSize; ++x)
104 for (Boost3dArray::index s = 0; s < numScales; ++s)
106 for (Boost3dArray::index c = 0; c < numClasses; ++c)
108 // Resolved confidence: class probabilities * scales.
109 const float confidence = classProbabilities[y][x][c] * scales[y][x][s];
111 // Resolves bounding box and stores.
113 box.m_X = boxes[y][x][s][0];
114 box.m_Y = boxes[y][x][s][1];
115 box.m_W = boxes[y][x][s][2];
116 box.m_H = boxes[y][x][s][3];
118 detectedObjects.emplace_back(c, box, confidence);
124 // Sorts detected objects by confidence.
125 std::sort(detectedObjects.begin(), detectedObjects.end(),
126 [](const YoloDetectedObject& a, const YoloDetectedObject& b)
128 // Sorts by largest confidence first, then by class.
129 return a.m_Confidence > b.m_Confidence
130 || (a.m_Confidence == b.m_Confidence && a.m_Class > b.m_Class);
133 // Checks the top N detections.
134 auto outputIt = detectedObjects.begin();
135 auto outputEnd = detectedObjects.end();
137 for (const YoloDetectedObject& expectedDetection : m_TopObjectDetections)
139 if (outputIt == outputEnd)
141 // Somehow expected more things to check than detections found by the model.
142 return TestCaseResult::Abort;
145 const YoloDetectedObject& detectedObject = *outputIt;
146 if (detectedObject.m_Class != expectedDetection.m_Class)
148 BOOST_LOG_TRIVIAL(error) << "Prediction for test case " << this->GetTestCaseId() <<
149 " is incorrect: Expected (" << expectedDetection.m_Class << ")" <<
150 " but predicted (" << detectedObject.m_Class << ")";
151 return TestCaseResult::Failed;
154 if (!m_FloatComparer(detectedObject.m_Box.m_X, expectedDetection.m_Box.m_X) ||
155 !m_FloatComparer(detectedObject.m_Box.m_Y, expectedDetection.m_Box.m_Y) ||
156 !m_FloatComparer(detectedObject.m_Box.m_W, expectedDetection.m_Box.m_W) ||
157 !m_FloatComparer(detectedObject.m_Box.m_H, expectedDetection.m_Box.m_H) ||
158 !m_FloatComparer(detectedObject.m_Confidence, expectedDetection.m_Confidence))
160 BOOST_LOG_TRIVIAL(error) << "Detected bounding box for test case " << this->GetTestCaseId() <<
162 return TestCaseResult::Failed;
168 return TestCaseResult::Ok;
172 boost::math::fpc::close_at_tolerance<float> m_FloatComparer;
173 std::vector<YoloDetectedObject> m_TopObjectDetections;
176 template <typename Model>
177 class YoloTestCaseProvider : public IInferenceTestCaseProvider
180 template <typename TConstructModelCallable>
181 explicit YoloTestCaseProvider(TConstructModelCallable constructModel)
182 : m_ConstructModel(constructModel)
186 virtual void AddCommandLineOptions(boost::program_options::options_description& options) override
188 namespace po = boost::program_options;
190 options.add_options()
191 ("data-dir,d", po::value<std::string>(&m_DataDir)->required(),
192 "Path to directory containing test data");
194 Model::AddCommandLineOptions(options, m_ModelCommandLineOptions);
197 virtual bool ProcessCommandLineOptions(const InferenceTestOptions &commonOptions) override
199 if (!ValidateDirectory(m_DataDir))
204 m_Model = m_ConstructModel(commonOptions, m_ModelCommandLineOptions);
210 m_Database = std::make_unique<YoloDatabase>(m_DataDir.c_str());
219 virtual std::unique_ptr<IInferenceTestCase> GetTestCase(unsigned int testCaseId) override
221 std::unique_ptr<YoloTestCaseData> testCaseData = m_Database->GetTestCaseData(testCaseId);
227 return std::make_unique<YoloTestCase<Model>>(*m_Model, testCaseId, *testCaseData);
231 typename Model::CommandLineOptions m_ModelCommandLineOptions;
232 std::function<std::unique_ptr<Model>(const InferenceTestOptions&,
233 typename Model::CommandLineOptions)> m_ConstructModel;
234 std::unique_ptr<Model> m_Model;
236 std::string m_DataDir;
237 std::unique_ptr<YoloDatabase> m_Database;