2 // Copyright © 2017 Arm Ltd. All rights reserved.
3 // SPDX-License-Identifier: MIT
7 #include "InferenceTest.hpp"
8 #include "YoloDatabase.hpp"
10 #include <armnn/utility/Assert.hpp>
11 #include <armnn/utility/IgnoreUnused.hpp>
12 #include <armnnUtils/FloatingPointComparison.hpp>
18 constexpr size_t YoloOutputSize = 1470;
20 template <typename Model>
21 class YoloTestCase : public InferenceModelTestCase<Model>
24 YoloTestCase(Model& model,
25 unsigned int testCaseId,
26 YoloTestCaseData& testCaseData)
27 : InferenceModelTestCase<Model>(model, testCaseId, { std::move(testCaseData.m_InputImage) }, { YoloOutputSize })
28 , m_TopObjectDetections(std::move(testCaseData.m_TopObjectDetections))
32 virtual TestCaseResult ProcessResult(const InferenceTestOptions& options) override
34 armnn::IgnoreUnused(options);
36 const std::vector<float>& output = mapbox::util::get<std::vector<float>>(this->GetOutputs()[0]);
37 ARMNN_ASSERT(output.size() == YoloOutputSize);
39 constexpr unsigned int gridSize = 7;
40 constexpr unsigned int numClasses = 20;
41 constexpr unsigned int numScales = 2;
43 const float* outputPtr = output.data();
45 // Range 0-980. Class probabilities. 7x7x20
46 vector<vector<vector<float>>> classProbabilities(gridSize, vector<vector<float>>(gridSize,
47 vector<float>(numClasses)));
48 for (unsigned int y = 0; y < gridSize; ++y)
50 for (unsigned int x = 0; x < gridSize; ++x)
52 for (unsigned int c = 0; c < numClasses; ++c)
54 classProbabilities[y][x][c] = *outputPtr++;
59 // Range 980-1078. Scales. 7x7x2
60 vector<vector<vector<float>>> scales(gridSize, vector<vector<float>>(gridSize, vector<float>(numScales)));
61 for (unsigned int y = 0; y < gridSize; ++y)
63 for (unsigned int x = 0; x < gridSize; ++x)
65 for (unsigned int s = 0; s < numScales; ++s)
67 scales[y][x][s] = *outputPtr++;
72 // Range 1078-1469. Bounding boxes. 7x7x2x4
73 constexpr float imageWidthAsFloat = static_cast<float>(YoloImageWidth);
74 constexpr float imageHeightAsFloat = static_cast<float>(YoloImageHeight);
76 vector<vector<vector<vector<float>>>> boxes(gridSize, vector<vector<vector<float>>>
77 (gridSize, vector<vector<float>>(numScales, vector<float>(4))));
78 for (unsigned int y = 0; y < gridSize; ++y)
80 for (unsigned int x = 0; x < gridSize; ++x)
82 for (unsigned int s = 0; s < numScales; ++s)
84 float bx = *outputPtr++;
85 float by = *outputPtr++;
86 float bw = *outputPtr++;
87 float bh = *outputPtr++;
89 boxes[y][x][s][0] = ((bx + static_cast<float>(x)) / 7.0f) * imageWidthAsFloat;
90 boxes[y][x][s][1] = ((by + static_cast<float>(y)) / 7.0f) * imageHeightAsFloat;
91 boxes[y][x][s][2] = bw * bw * static_cast<float>(imageWidthAsFloat);
92 boxes[y][x][s][3] = bh * bh * static_cast<float>(imageHeightAsFloat);
96 ARMNN_ASSERT(output.data() + YoloOutputSize == outputPtr);
98 std::vector<YoloDetectedObject> detectedObjects;
99 detectedObjects.reserve(gridSize * gridSize * numScales * numClasses);
101 for (unsigned int y = 0; y < gridSize; ++y)
103 for (unsigned int x = 0; x < gridSize; ++x)
105 for (unsigned int s = 0; s < numScales; ++s)
107 for (unsigned int c = 0; c < numClasses; ++c)
109 // Resolved confidence: class probabilities * scales.
110 const float confidence = classProbabilities[y][x][c] * scales[y][x][s];
112 // Resolves bounding box and stores.
114 box.m_X = boxes[y][x][s][0];
115 box.m_Y = boxes[y][x][s][1];
116 box.m_W = boxes[y][x][s][2];
117 box.m_H = boxes[y][x][s][3];
119 detectedObjects.emplace_back(c, box, confidence);
125 // Sorts detected objects by confidence.
126 std::sort(detectedObjects.begin(), detectedObjects.end(),
127 [](const YoloDetectedObject& a, const YoloDetectedObject& b)
129 // Sorts by largest confidence first, then by class.
130 return a.m_Confidence > b.m_Confidence
131 || (a.m_Confidence == b.m_Confidence && a.m_Class > b.m_Class);
134 // Checks the top N detections.
135 auto outputIt = detectedObjects.begin();
136 auto outputEnd = detectedObjects.end();
138 for (const YoloDetectedObject& expectedDetection : m_TopObjectDetections)
140 if (outputIt == outputEnd)
142 // Somehow expected more things to check than detections found by the model.
143 return TestCaseResult::Abort;
146 const YoloDetectedObject& detectedObject = *outputIt;
147 if (detectedObject.m_Class != expectedDetection.m_Class)
149 ARMNN_LOG(error) << "Prediction for test case " << this->GetTestCaseId() <<
150 " is incorrect: Expected (" << expectedDetection.m_Class << ")" <<
151 " but predicted (" << detectedObject.m_Class << ")";
152 return TestCaseResult::Failed;
155 if (!armnnUtils::within_percentage_tolerance(detectedObject.m_Box.m_X, expectedDetection.m_Box.m_X) ||
156 !armnnUtils::within_percentage_tolerance(detectedObject.m_Box.m_Y, expectedDetection.m_Box.m_Y) ||
157 !armnnUtils::within_percentage_tolerance(detectedObject.m_Box.m_W, expectedDetection.m_Box.m_W) ||
158 !armnnUtils::within_percentage_tolerance(detectedObject.m_Box.m_H, expectedDetection.m_Box.m_H) ||
159 !armnnUtils::within_percentage_tolerance(detectedObject.m_Confidence, expectedDetection.m_Confidence))
161 ARMNN_LOG(error) << "Detected bounding box for test case " << this->GetTestCaseId() <<
163 return TestCaseResult::Failed;
169 return TestCaseResult::Ok;
173 std::vector<YoloDetectedObject> m_TopObjectDetections;
176 template <typename Model>
177 class YoloTestCaseProvider : public IInferenceTestCaseProvider
180 template <typename TConstructModelCallable>
181 explicit YoloTestCaseProvider(TConstructModelCallable constructModel)
182 : m_ConstructModel(constructModel)
186 virtual void AddCommandLineOptions(cxxopts::Options& options, std::vector<std::string>& required) override
189 .allow_unrecognised_options()
191 ("d,data-dir", "Path to directory containing test data", cxxopts::value<std::string>(m_DataDir));
193 Model::AddCommandLineOptions(options, m_ModelCommandLineOptions, required);
196 virtual bool ProcessCommandLineOptions(const InferenceTestOptions& commonOptions) override
198 if (!ValidateDirectory(m_DataDir))
203 m_Model = m_ConstructModel(commonOptions, m_ModelCommandLineOptions);
209 m_Database = std::make_unique<YoloDatabase>(m_DataDir.c_str());
218 virtual std::unique_ptr<IInferenceTestCase> GetTestCase(unsigned int testCaseId) override
220 std::unique_ptr<YoloTestCaseData> testCaseData = m_Database->GetTestCaseData(testCaseId);
226 return std::make_unique<YoloTestCase<Model>>(*m_Model, testCaseId, *testCaseData);
230 typename Model::CommandLineOptions m_ModelCommandLineOptions;
231 std::function<std::unique_ptr<Model>(const InferenceTestOptions&,
232 typename Model::CommandLineOptions)> m_ConstructModel;
233 std::unique_ptr<Model> m_Model;
235 std::string m_DataDir;
236 std::unique_ptr<YoloDatabase> m_Database;