2 * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "benchmark/Result.h"
18 #include "benchmark/Phases.h"
19 #include "benchmark/CsvWriter.h"
32 template <class T, class R> R average(const std::vector<T> &v)
34 T sum = std::accumulate(v.begin(), v.end(), 0);
35 R avg = sum / static_cast<R>(v.size());
39 double averageTimeMs(const benchmark::Phase &phase)
41 double avg_us = average<uint64_t, double>(phase.time);
45 double maxTimeMs(const benchmark::Phase &phase)
47 auto it = max_element(std::begin(phase.time), std::end(phase.time));
51 double minTimeMs(const benchmark::Phase &phase)
53 auto it = min_element(std::begin(phase.time), std::end(phase.time));
57 double geomeanTimeMs(const benchmark::Phase &phase)
60 for (auto &&t_us : phase.time)
62 log_sum += std::log(t_us / 1e3);
65 // Calculating geometric mean with logs
66 // "Geometric Mean of (V1, V2, ... Vn)"
67 // = (V1*V2*...*Vn)^(1/n)
68 // = exp(log((V1*V2*...*Vn)^(1/n)))
69 // = exp(log((V1*V2*...*Vn)/n)))
70 // = exp((log(V1) + log(V2) + ... + log(Vn))/n)
71 // = exp(_log_sum/num)
72 return std::exp(log_sum / static_cast<double>(phase.time.size()));
75 uint32_t averageMemoryKb(const benchmark::Phase &phase, int type)
77 return average<uint32_t, uint32_t>(phase.memory[type]);
81 const uint32_t memory[benchmark::PhaseEnum::END_OF_PHASE][benchmark::MemoryType::END_OF_MEM_TYPE],
84 using namespace benchmark;
85 // tricky. handle WARMUP as EXECUTE
86 return std::max({memory[PhaseEnum::MODEL_LOAD][type], memory[PhaseEnum::PREPARE][type],
87 memory[PhaseEnum::WARMUP][type]});
91 const double time[benchmark::PhaseEnum::END_OF_PHASE][benchmark::FigureType::END_OF_FIG_TYPE])
93 using namespace benchmark;
95 std::cout << "===================================" << std::endl;
97 std::streamsize ss_precision = std::cout.precision();
98 std::cout << std::setprecision(3);
99 std::cout << std::fixed;
101 for (int i = PhaseEnum::MODEL_LOAD; i <= PhaseEnum::EXECUTE; ++i)
103 // Note. Tricky. Ignore WARMUP
104 if (i == PhaseEnum::WARMUP)
106 std::cout << std::setw(12) << std::left << getPhaseString(i) << " takes "
107 << time[i][FigureType::MEAN] << " ms" << std::endl;
110 for (int j = FigureType::MEAN; j <= FigureType::GEOMEAN; ++j)
112 std::cout << "- " << std::setw(9) << std::left << getFigureTypeString(j) << ": "
113 << time[PhaseEnum::EXECUTE][j] << " ms" << std::endl;
116 std::cout << std::setprecision(ss_precision);
117 std::cout << std::defaultfloat;
119 std::cout << "===================================" << std::endl;
122 void printResultMemory(
123 const uint32_t memory[benchmark::PhaseEnum::END_OF_PHASE][benchmark::MemoryType::END_OF_MEM_TYPE])
125 using namespace benchmark;
127 for (int j = MemoryType::RSS; j <= MemoryType::PSS; ++j)
129 std::cout << getMemoryTypeString(j) << std::endl;
130 for (int i = PhaseEnum::MODEL_LOAD; i <= PhaseEnum::PREPARE; ++i)
132 std::cout << "- " << std::setw(12) << std::left << getPhaseString(i) << " takes "
133 << memory[i][j] << " kb" << std::endl;
135 // Tricky. Handle WARMUP as EXECUTE
136 std::cout << "- " << std::setw(12) << std::left << getPhaseString(PhaseEnum::EXECUTE)
137 << " takes " << memory[PhaseEnum::WARMUP][j] << " kb" << std::endl;
138 std::cout << "- " << std::setw(12) << std::left << "PEAK"
139 << " takes " << peakMemory(memory, j) << " kb" << std::endl;
140 std::cout << "===================================" << std::endl;
144 void printUsedPeakMemory(uint32_t init_memory, uint32_t peak_memory)
146 uint32_t used_peak_memory = peak_memory - init_memory;
147 std::cout << "Used Peak Memory : " << used_peak_memory << " kb" << std::endl;
148 std::cout << "- HWM after run : " << peak_memory << " kb" << std::endl;
149 std::cout << "- HWM before init: " << init_memory << " kb" << std::endl;
150 std::cout << "===================================" << std::endl;
158 Result::Result(const Phases &phases)
160 const auto option = phases.option();
162 for (int i = PhaseEnum::MODEL_LOAD; i <= PhaseEnum::PREPARE; ++i)
164 auto phase = phases.at(gPhaseStrings[i]);
165 time[i][FigureType::MEAN] = averageTimeMs(phase);
168 int i = PhaseEnum::EXECUTE;
169 auto exec_phase = phases.at(gPhaseStrings[i]);
170 time[i][FigureType::MEAN] = averageTimeMs(exec_phase);
171 time[i][FigureType::MAX] = maxTimeMs(exec_phase);
172 time[i][FigureType::MIN] = minTimeMs(exec_phase);
173 time[i][FigureType::GEOMEAN] = geomeanTimeMs(exec_phase);
178 for (int i = PhaseEnum::MODEL_LOAD; i < PhaseEnum::EXECUTE; ++i)
180 auto phase = phases.at(gPhaseStrings[i]);
181 for (int j = MemoryType::RSS; j <= MemoryType::PSS; ++j)
183 memory[i][j] = averageMemoryKb(phase, j);
187 init_memory = phases.mem_before_init();
188 peak_memory = phases.mem_after_run();
191 void printResult(const Result &result)
193 printResultTime(result.time);
195 if (result.print_memory == false)
198 printResultMemory(result.memory);
199 printUsedPeakMemory(result.init_memory, result.peak_memory);
202 // TODO There are necessary for a kind of output data file so that it doesn't have to be csv file
204 void writeResult(const Result &result, const std::string &exec, const std::string &model,
205 const std::string &backend)
207 std::string csv_filename = exec + "-" + model + "-" + backend + ".csv";
210 CsvWriter writer(csv_filename);
211 writer << model << backend;
215 auto time = result.time;
216 writer << time[PhaseEnum::MODEL_LOAD][FigureType::MEAN]
217 << time[PhaseEnum::PREPARE][FigureType::MEAN] << time[PhaseEnum::EXECUTE][FigureType::MIN]
218 << time[PhaseEnum::EXECUTE][FigureType::MAX] << time[PhaseEnum::EXECUTE][FigureType::MEAN];
221 auto memory = result.memory;
222 for (int j = MemoryType::RSS; j <= MemoryType::PSS; ++j)
224 // Tricky. Handle WARMUP as EXECUTE
225 for (int i = PhaseEnum::MODEL_LOAD; i <= PhaseEnum::WARMUP; ++i)
227 writer << memory[i][j];
229 writer << peakMemory(memory, j);
232 bool done = writer.done();
236 std::cerr << "Writing to " << csv_filename << " is failed" << std::endl;
240 } // namespace benchmark