2 // Copyright © 2020 Arm Ltd. All rights reserved.
3 // SPDX-License-Identifier: MIT
6 #include "CounterDirectory.hpp"
7 #include "CounterIdMap.hpp"
9 #include "MockBackend.hpp"
10 #include "MockBackendId.hpp"
11 #include "PeriodicCounterCapture.hpp"
12 #include "PeriodicCounterSelectionCommandHandler.hpp"
13 #include "ProfilingStateMachine.hpp"
14 #include "ProfilingUtils.hpp"
15 #include "RequestCounterDirectoryCommandHandler.hpp"
17 #include <Runtime.hpp>
19 #include <armnn/utility/IgnoreUnused.hpp>
20 #include <armnn/BackendId.hpp>
21 #include <armnn/Logging.hpp>
22 #include <armnn/profiling/ISendTimelinePacket.hpp>
24 #include <boost/algorithm/string.hpp>
25 #include <boost/numeric/conversion/cast.hpp>
26 #include <boost/test/unit_test.hpp>
31 #include <backends/BackendProfiling.hpp>
33 using namespace armnn::profiling;
35 class ReadCounterVals : public IReadCounterValues
37 virtual bool IsCounterRegistered(uint16_t counterUid) const override
39 return (counterUid > 4 && counterUid < 11);
41 virtual uint16_t GetCounterCount() const override
45 virtual uint32_t GetCounterValue(uint16_t counterUid) const override
51 class MockBackendSendCounterPacket : public ISendCounterPacket
54 using IndexValuePairsVector = std::vector<CounterValue>;
56 /// Create and write a StreamMetaDataPacket in the buffer
57 virtual void SendStreamMetaDataPacket() {}
59 /// Create and write a CounterDirectoryPacket from the parameters to the buffer.
60 virtual void SendCounterDirectoryPacket(const ICounterDirectory& counterDirectory)
62 armnn::IgnoreUnused(counterDirectory);
65 /// Create and write a PeriodicCounterCapturePacket from the parameters to the buffer.
66 virtual void SendPeriodicCounterCapturePacket(uint64_t timestamp, const IndexValuePairsVector& values)
68 m_timestamps.emplace_back(Timestamp{timestamp, values});
71 /// Create and write a PeriodicCounterSelectionPacket from the parameters to the buffer.
72 virtual void SendPeriodicCounterSelectionPacket(uint32_t capturePeriod,
73 const std::vector<uint16_t>& selectedCounterIds)
75 armnn::IgnoreUnused(capturePeriod);
76 armnn::IgnoreUnused(selectedCounterIds);
79 std::vector<Timestamp> GetTimestamps()
84 void ClearTimestamps()
90 std::vector<Timestamp> m_timestamps;
93 Packet PacketWriter(uint32_t period, std::vector<uint16_t> countervalues)
95 const uint32_t packetId = 0x40000;
97 uint32_t dataLength = static_cast<uint32_t>(4 + countervalues.size() * 2);
98 std::unique_ptr<unsigned char[]> uniqueData = std::make_unique<unsigned char[]>(dataLength);
99 unsigned char* data1 = reinterpret_cast<unsigned char*>(uniqueData.get());
101 WriteUint32(data1, offset, period);
103 for (auto countervalue : countervalues)
105 WriteUint16(data1, offset, countervalue);
109 return {packetId, dataLength, uniqueData};
112 BOOST_AUTO_TEST_SUITE(BackendProfilingTestSuite)
114 BOOST_AUTO_TEST_CASE(BackendProfilingCounterRegisterMockBackendTest)
116 // Reset the profiling service to the uninitialized state
117 armnn::IRuntime::CreationOptions options;
118 options.m_ProfilingOptions.m_EnableProfiling = true;;
120 armnn::MockBackendInitialiser initialiser;
122 // armnn::IRuntimePtr runtime(armnn::IRuntime::Create(options));
123 armnn::Runtime runtime(options);
125 // Check if the MockBackends 3 dummy counters {0, 1, 2-5 (four cores)} are registered
126 armnn::BackendId mockId = armnn::MockBackendId();
127 const armnn::profiling::ICounterMappings& counterMap = GetProfilingService(&runtime).GetCounterMappings();
128 BOOST_CHECK(counterMap.GetGlobalId(0, mockId) == 5);
129 BOOST_CHECK(counterMap.GetGlobalId(1, mockId) == 6);
130 BOOST_CHECK(counterMap.GetGlobalId(2, mockId) == 7);
131 BOOST_CHECK(counterMap.GetGlobalId(3, mockId) == 8);
132 BOOST_CHECK(counterMap.GetGlobalId(4, mockId) == 9);
133 BOOST_CHECK(counterMap.GetGlobalId(5, mockId) == 10);
134 options.m_ProfilingOptions.m_EnableProfiling = false;
135 GetProfilingService(&runtime).ResetExternalProfilingOptions(options.m_ProfilingOptions, true);
138 BOOST_AUTO_TEST_CASE(TestBackendCounters)
141 PacketVersionResolver packetVersionResolver;
142 ProfilingStateMachine stateMachine;
143 ReadCounterVals readCounterVals;
144 CounterIdMap counterIdMap;
145 MockBackendSendCounterPacket sendCounterPacket;
147 const armnn::BackendId cpuAccId(armnn::Compute::CpuAcc);
148 const armnn::BackendId gpuAccId(armnn::Compute::GpuAcc);
150 armnn::IRuntime::CreationOptions options;
151 options.m_ProfilingOptions.m_EnableProfiling = true;
153 armnn::profiling::ProfilingService profilingService;
155 std::unique_ptr<armnn::profiling::IBackendProfiling> cpuBackendProfilingPtr =
156 std::make_unique<BackendProfiling>(options, profilingService, cpuAccId);
157 std::unique_ptr<armnn::profiling::IBackendProfiling> gpuBackendProfilingPtr =
158 std::make_unique<BackendProfiling>(options, profilingService, gpuAccId);
160 std::shared_ptr<armnn::profiling::IBackendProfilingContext> cpuProfilingContextPtr =
161 std::make_shared<armnn::MockBackendProfilingContext>(cpuBackendProfilingPtr);
162 std::shared_ptr<armnn::profiling::IBackendProfilingContext> gpuProfilingContextPtr =
163 std::make_shared<armnn::MockBackendProfilingContext>(gpuBackendProfilingPtr);
165 std::unordered_map<armnn::BackendId,
166 std::shared_ptr<armnn::profiling::IBackendProfilingContext>> backendProfilingContexts;
168 backendProfilingContexts[cpuAccId] = cpuProfilingContextPtr;
169 backendProfilingContexts[gpuAccId] = gpuProfilingContextPtr;
171 uint16_t globalId = 5;
173 counterIdMap.RegisterMapping(globalId++, 0, cpuAccId);
174 counterIdMap.RegisterMapping(globalId++, 1, cpuAccId);
175 counterIdMap.RegisterMapping(globalId++, 2, cpuAccId);
177 counterIdMap.RegisterMapping(globalId++, 0, gpuAccId);
178 counterIdMap.RegisterMapping(globalId++, 1, gpuAccId);
179 counterIdMap.RegisterMapping(globalId++, 2, gpuAccId);
181 backendProfilingContexts[cpuAccId] = cpuProfilingContextPtr;
182 backendProfilingContexts[gpuAccId] = gpuProfilingContextPtr;
184 PeriodicCounterCapture periodicCounterCapture(holder, sendCounterPacket, readCounterVals,
185 counterIdMap, backendProfilingContexts);
187 uint16_t maxArmnnCounterId = 4;
189 PeriodicCounterSelectionCommandHandler periodicCounterSelectionCommandHandler(0,
191 packetVersionResolver.ResolvePacketVersion(0, 4).GetEncodedValue(),
192 backendProfilingContexts,
196 periodicCounterCapture,
201 stateMachine.TransitionToState(ProfilingState::NotConnected);
202 stateMachine.TransitionToState(ProfilingState::WaitingForAck);
203 stateMachine.TransitionToState(ProfilingState::Active);
205 uint32_t period = 12345u;
207 std::vector<uint16_t> cpuCounters{5, 6, 7};
208 std::vector<uint16_t> gpuCounters{8, 9, 10};
210 // Request only gpu counters
211 periodicCounterSelectionCommandHandler(PacketWriter(period, gpuCounters));
212 periodicCounterCapture.Stop();
214 std::set<armnn::BackendId> activeIds = holder.GetCaptureData().GetActiveBackends();
215 BOOST_CHECK(activeIds.size() == 1);
216 BOOST_CHECK(activeIds.find(gpuAccId) != activeIds.end());
218 std::vector<Timestamp> recievedTimestamp = sendCounterPacket.GetTimestamps();
220 BOOST_CHECK(recievedTimestamp[0].timestamp == period);
221 BOOST_CHECK(recievedTimestamp.size() == 1);
222 BOOST_CHECK(recievedTimestamp[0].counterValues.size() == gpuCounters.size());
223 for (unsigned long i=0; i< gpuCounters.size(); ++i)
225 BOOST_CHECK(recievedTimestamp[0].counterValues[i].counterId == gpuCounters[i]);
226 BOOST_CHECK(recievedTimestamp[0].counterValues[i].counterValue == i + 1u);
228 sendCounterPacket.ClearTimestamps();
230 // Request only cpu counters
231 periodicCounterSelectionCommandHandler(PacketWriter(period, cpuCounters));
232 periodicCounterCapture.Stop();
234 activeIds = holder.GetCaptureData().GetActiveBackends();
235 BOOST_CHECK(activeIds.size() == 1);
236 BOOST_CHECK(activeIds.find(cpuAccId) != activeIds.end());
238 recievedTimestamp = sendCounterPacket.GetTimestamps();
240 BOOST_CHECK(recievedTimestamp[0].timestamp == period);
241 BOOST_CHECK(recievedTimestamp.size() == 1);
242 BOOST_CHECK(recievedTimestamp[0].counterValues.size() == cpuCounters.size());
243 for (unsigned long i=0; i< cpuCounters.size(); ++i)
245 BOOST_CHECK(recievedTimestamp[0].counterValues[i].counterId == cpuCounters[i]);
246 BOOST_CHECK(recievedTimestamp[0].counterValues[i].counterValue == i + 1u);
248 sendCounterPacket.ClearTimestamps();
250 // Request combination of cpu & gpu counters with new period
252 periodicCounterSelectionCommandHandler(PacketWriter(period, {cpuCounters[0], gpuCounters[2],
253 gpuCounters[1], cpuCounters[1], gpuCounters[0]}));
254 periodicCounterCapture.Stop();
256 activeIds = holder.GetCaptureData().GetActiveBackends();
257 BOOST_CHECK(activeIds.size() == 2);
258 BOOST_CHECK(activeIds.find(cpuAccId) != activeIds.end());
259 BOOST_CHECK(activeIds.find(gpuAccId) != activeIds.end());
261 recievedTimestamp = sendCounterPacket.GetTimestamps();
263 BOOST_CHECK(recievedTimestamp[0].timestamp == period);
264 BOOST_CHECK(recievedTimestamp[1].timestamp == period);
266 BOOST_CHECK(recievedTimestamp.size() == 2);
267 BOOST_CHECK(recievedTimestamp[0].counterValues.size() == 2);
268 BOOST_CHECK(recievedTimestamp[1].counterValues.size() == gpuCounters.size());
270 BOOST_CHECK(recievedTimestamp[0].counterValues[0].counterId == cpuCounters[0]);
271 BOOST_CHECK(recievedTimestamp[0].counterValues[0].counterValue == 1u);
272 BOOST_CHECK(recievedTimestamp[0].counterValues[1].counterId == cpuCounters[1]);
273 BOOST_CHECK(recievedTimestamp[0].counterValues[1].counterValue == 2u);
275 for (unsigned long i=0; i< gpuCounters.size(); ++i)
277 BOOST_CHECK(recievedTimestamp[1].counterValues[i].counterId == gpuCounters[i]);
278 BOOST_CHECK(recievedTimestamp[1].counterValues[i].counterValue == i + 1u);
281 sendCounterPacket.ClearTimestamps();
283 // Request all counters
284 std::vector<uint16_t> counterValues;
285 counterValues.insert(counterValues.begin(), cpuCounters.begin(), cpuCounters.end());
286 counterValues.insert(counterValues.begin(), gpuCounters.begin(), gpuCounters.end());
288 periodicCounterSelectionCommandHandler(PacketWriter(period, counterValues));
289 periodicCounterCapture.Stop();
291 activeIds = holder.GetCaptureData().GetActiveBackends();
292 BOOST_CHECK(activeIds.size() == 2);
293 BOOST_CHECK(activeIds.find(cpuAccId) != activeIds.end());
294 BOOST_CHECK(activeIds.find(gpuAccId) != activeIds.end());
296 recievedTimestamp = sendCounterPacket.GetTimestamps();
298 BOOST_CHECK(recievedTimestamp[0].counterValues.size() == cpuCounters.size());
299 for (unsigned long i=0; i< cpuCounters.size(); ++i)
301 BOOST_CHECK(recievedTimestamp[0].counterValues[i].counterId == cpuCounters[i]);
302 BOOST_CHECK(recievedTimestamp[0].counterValues[i].counterValue == i + 1u);
305 BOOST_CHECK(recievedTimestamp[1].counterValues.size() == gpuCounters.size());
306 for (unsigned long i=0; i< gpuCounters.size(); ++i)
308 BOOST_CHECK(recievedTimestamp[1].counterValues[i].counterId == gpuCounters[i]);
309 BOOST_CHECK(recievedTimestamp[1].counterValues[i].counterValue == i + 1u);
311 sendCounterPacket.ClearTimestamps();
313 // Request random counters with duplicates and invalid counters
314 counterValues = {0, 0, 200, cpuCounters[2], gpuCounters[0],3 ,30, cpuCounters[0],cpuCounters[2], gpuCounters[1], 3,
315 90, 0, 30, gpuCounters[0], gpuCounters[0]};
317 periodicCounterSelectionCommandHandler(PacketWriter(period, counterValues));
318 periodicCounterCapture.Stop();
320 activeIds = holder.GetCaptureData().GetActiveBackends();
321 BOOST_CHECK(activeIds.size() == 2);
322 BOOST_CHECK(activeIds.find(cpuAccId) != activeIds.end());
323 BOOST_CHECK(activeIds.find(gpuAccId) != activeIds.end());
325 recievedTimestamp = sendCounterPacket.GetTimestamps();
327 BOOST_CHECK(recievedTimestamp.size() == 2);
329 BOOST_CHECK(recievedTimestamp[0].counterValues.size() == 2);
331 BOOST_CHECK(recievedTimestamp[0].counterValues[0].counterId == cpuCounters[0]);
332 BOOST_CHECK(recievedTimestamp[0].counterValues[0].counterValue == 1u);
333 BOOST_CHECK(recievedTimestamp[0].counterValues[1].counterId == cpuCounters[2]);
334 BOOST_CHECK(recievedTimestamp[0].counterValues[1].counterValue == 3u);
336 BOOST_CHECK(recievedTimestamp[1].counterValues.size() == 2);
338 BOOST_CHECK(recievedTimestamp[1].counterValues[0].counterId == gpuCounters[0]);
339 BOOST_CHECK(recievedTimestamp[1].counterValues[0].counterValue == 1u);
340 BOOST_CHECK(recievedTimestamp[1].counterValues[1].counterId == gpuCounters[1]);
341 BOOST_CHECK(recievedTimestamp[1].counterValues[1].counterValue == 2u);
343 sendCounterPacket.ClearTimestamps();
345 // Request no counters
346 periodicCounterSelectionCommandHandler(PacketWriter(period, {}));
347 periodicCounterCapture.Stop();
349 activeIds = holder.GetCaptureData().GetActiveBackends();
350 BOOST_CHECK(activeIds.size() == 0);
352 recievedTimestamp = sendCounterPacket.GetTimestamps();
353 BOOST_CHECK(recievedTimestamp.size() == 0);
355 sendCounterPacket.ClearTimestamps();
357 // Request period of zero
358 periodicCounterSelectionCommandHandler(PacketWriter(0, counterValues));
359 periodicCounterCapture.Stop();
361 activeIds = holder.GetCaptureData().GetActiveBackends();
362 BOOST_CHECK(activeIds.size() == 0);
364 recievedTimestamp = sendCounterPacket.GetTimestamps();
365 BOOST_CHECK(recievedTimestamp.size() == 0);
368 BOOST_AUTO_TEST_CASE(TestBackendCounterLogging)
370 std::stringstream ss;
372 struct StreamRedirector
375 StreamRedirector(std::ostream &stream, std::streambuf *newStreamBuffer)
376 : m_Stream(stream), m_BackupBuffer(m_Stream.rdbuf(newStreamBuffer))
380 { m_Stream.rdbuf(m_BackupBuffer); }
383 std::ostream &m_Stream;
384 std::streambuf *m_BackupBuffer;
388 PacketVersionResolver packetVersionResolver;
389 ProfilingStateMachine stateMachine;
390 ReadCounterVals readCounterVals;
391 StreamRedirector redirect(std::cout, ss.rdbuf());
392 CounterIdMap counterIdMap;
393 MockBackendSendCounterPacket sendCounterPacket;
395 const armnn::BackendId cpuAccId(armnn::Compute::CpuAcc);
396 const armnn::BackendId gpuAccId(armnn::Compute::GpuAcc);
398 armnn::IRuntime::CreationOptions options;
399 options.m_ProfilingOptions.m_EnableProfiling = true;
401 armnn::profiling::ProfilingService profilingService;
403 std::unique_ptr<armnn::profiling::IBackendProfiling> cpuBackendProfilingPtr =
404 std::make_unique<BackendProfiling>(options, profilingService, cpuAccId);
406 std::shared_ptr<armnn::profiling::IBackendProfilingContext> cpuProfilingContextPtr =
407 std::make_shared<armnn::MockBackendProfilingContext>(cpuBackendProfilingPtr);
409 std::unordered_map<armnn::BackendId,
410 std::shared_ptr<armnn::profiling::IBackendProfilingContext>> backendProfilingContexts;
412 uint16_t globalId = 5;
413 counterIdMap.RegisterMapping(globalId, 0, cpuAccId);
414 backendProfilingContexts[cpuAccId] = cpuProfilingContextPtr;
416 PeriodicCounterCapture periodicCounterCapture(holder, sendCounterPacket, readCounterVals,
417 counterIdMap, backendProfilingContexts);
419 uint16_t maxArmnnCounterId = 4;
421 PeriodicCounterSelectionCommandHandler periodicCounterSelectionCommandHandler(0,
423 packetVersionResolver.ResolvePacketVersion(0, 4).GetEncodedValue(),
424 backendProfilingContexts,
428 periodicCounterCapture,
433 stateMachine.TransitionToState(ProfilingState::NotConnected);
434 stateMachine.TransitionToState(ProfilingState::WaitingForAck);
435 stateMachine.TransitionToState(ProfilingState::Active);
437 uint32_t period = 15939u;
439 armnn::SetAllLoggingSinks(true, false, false);
440 SetLogFilter(armnn::LogSeverity::Warning);
441 periodicCounterSelectionCommandHandler(PacketWriter(period, {5}));
442 periodicCounterCapture.Stop();
443 SetLogFilter(armnn::LogSeverity::Fatal);
445 BOOST_CHECK(boost::contains(ss.str(), "ActivateCounters example test error"));
448 BOOST_AUTO_TEST_CASE(BackendProfilingContextGetSendTimelinePacket)
450 // Reset the profiling service to the uninitialized state
451 armnn::IRuntime::CreationOptions options;
452 options.m_ProfilingOptions.m_EnableProfiling = true;
453 armnn::profiling::ProfilingService profilingService;
454 profilingService.ConfigureProfilingService(options.m_ProfilingOptions, true);
456 armnn::MockBackendInitialiser initialiser;
457 // Create a runtime. During this the mock backend will be registered and context returned.
458 armnn::IRuntimePtr runtime(armnn::IRuntime::Create(options));
459 armnn::MockBackendProfilingService mockProfilingService = armnn::MockBackendProfilingService::Instance();
460 armnn::MockBackendProfilingContext *mockBackEndProfilingContext = mockProfilingService.GetContext();
461 // Check that there is a valid context set.
462 BOOST_CHECK(mockBackEndProfilingContext);
463 armnn::IBackendInternal::IBackendProfilingPtr& backendProfilingIface =
464 mockBackEndProfilingContext->GetBackendProfiling();
465 BOOST_CHECK(backendProfilingIface);
467 // Now for the meat of the test. We're just going to send a random packet and make sure there
468 // are no exceptions or errors. The sending of packets is already tested in SendTimelinePacketTests.
469 std::unique_ptr<armnn::profiling::ISendTimelinePacket> timelinePacket =
470 backendProfilingIface->GetSendTimelinePacket();
471 // Send TimelineEntityClassBinaryPacket
472 const uint64_t entityBinaryPacketProfilingGuid = 123456u;
473 timelinePacket->SendTimelineEntityBinaryPacket(entityBinaryPacketProfilingGuid);
474 timelinePacket->Commit();
476 // Reset the profiling servie after the test.
477 options.m_ProfilingOptions.m_EnableProfiling = false;
478 profilingService.ResetExternalProfilingOptions(options.m_ProfilingOptions, true);
481 BOOST_AUTO_TEST_CASE(GetProfilingGuidGenerator)
483 // Reset the profiling service to the uninitialized state
484 armnn::IRuntime::CreationOptions options;
485 options.m_ProfilingOptions.m_EnableProfiling = true;
487 armnn::MockBackendInitialiser initialiser;
488 // Create a runtime. During this the mock backend will be registered and context returned.
489 armnn::IRuntimePtr runtime(armnn::IRuntime::Create(options));
490 armnn::MockBackendProfilingService mockProfilingService = armnn::MockBackendProfilingService::Instance();
491 armnn::MockBackendProfilingContext *mockBackEndProfilingContext = mockProfilingService.GetContext();
492 // Check that there is a valid context set.
493 BOOST_CHECK(mockBackEndProfilingContext);
494 armnn::IBackendInternal::IBackendProfilingPtr& backendProfilingIface =
495 mockBackEndProfilingContext->GetBackendProfiling();
496 BOOST_CHECK(backendProfilingIface);
498 // Get the Guid generator and check the getting two Guid's results in the second being greater than the first.
499 armnn::profiling::IProfilingGuidGenerator& guidGenerator = backendProfilingIface->GetProfilingGuidGenerator();
500 BOOST_CHECK(backendProfilingIface);
501 const armnn::profiling::ProfilingDynamicGuid& firstGuid = guidGenerator.NextGuid();
502 BOOST_CHECK(firstGuid);
503 const armnn::profiling::ProfilingDynamicGuid& secondGuid = guidGenerator.NextGuid();
504 BOOST_CHECK(secondGuid > firstGuid);
506 // Reset the profiling servie after the test.
507 options.m_ProfilingOptions.m_EnableProfiling = false;
510 BOOST_AUTO_TEST_SUITE_END()