From dc24c02cd2b8829a38da4404f9ff85e3535f5b46 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EB=B0=95=EC=A2=85=ED=98=84/On-Device=20Lab=28SR=29/Staff?= =?utf8?q?=20Engineer/=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?= Date: Wed, 7 Aug 2019 10:16:30 +0900 Subject: [PATCH] Introduce "xtrace" (#6158) * Introduce "xtrace" This commit introduces "xtract" which collects trace/benchmark events via xray pipe and records it as Chrome Trace Event file format. Signed-off-by: Jonghyun Park * Add missing copyright --- runtimes/contrib/xtrace/CMakeLists.txt | 16 +++ runtimes/contrib/xtrace/src/benchmark_event.cc | 36 ++++++ runtimes/contrib/xtrace/src/benchmark_event.h | 77 ++++++++++++ runtimes/contrib/xtrace/src/benchmark_runner.cc | 123 +++++++++++++++++++ runtimes/contrib/xtrace/src/benchmark_runner.h | 37 ++++++ runtimes/contrib/xtrace/src/event_collector.cc | 157 ++++++++++++++++++++++++ runtimes/contrib/xtrace/src/event_collector.h | 39 ++++++ runtimes/contrib/xtrace/src/event_recorder.cc | 130 ++++++++++++++++++++ runtimes/contrib/xtrace/src/event_recorder.h | 69 +++++++++++ runtimes/contrib/xtrace/src/str.h | 38 ++++++ runtimes/contrib/xtrace/src/xtrace.cc | 64 ++++++++++ 11 files changed, 786 insertions(+) create mode 100644 runtimes/contrib/xtrace/CMakeLists.txt create mode 100644 runtimes/contrib/xtrace/src/benchmark_event.cc create mode 100644 runtimes/contrib/xtrace/src/benchmark_event.h create mode 100644 runtimes/contrib/xtrace/src/benchmark_runner.cc create mode 100644 runtimes/contrib/xtrace/src/benchmark_runner.h create mode 100644 runtimes/contrib/xtrace/src/event_collector.cc create mode 100644 runtimes/contrib/xtrace/src/event_collector.h create mode 100644 runtimes/contrib/xtrace/src/event_recorder.cc create mode 100644 runtimes/contrib/xtrace/src/event_recorder.h create mode 100644 runtimes/contrib/xtrace/src/str.h create mode 100644 runtimes/contrib/xtrace/src/xtrace.cc diff --git a/runtimes/contrib/xtrace/CMakeLists.txt b/runtimes/contrib/xtrace/CMakeLists.txt new file mode 100644 index 0000000..0c27483 --- /dev/null +++ b/runtimes/contrib/xtrace/CMakeLists.txt @@ -0,0 +1,16 @@ +# Enable xtrace build only when there is an explicit user request +option(BUILD_CONTRIB_XTRACE "Build xtrace" OFF) + +file(GLOB_RECURSE SOURCES "src/*.cc") + +add_executable(xtrace ${SOURCES}) +target_link_libraries(xtrace nnfw_lib_tflite) +target_link_libraries(xtrace nnfw_lib_misc) +target_link_libraries(xtrace nnfw_lib_cpp14) +target_link_libraries(xtrace nnfw_lib_xray_pipe) +target_link_libraries(xtrace nnfw_lib_xray_mux) +target_link_libraries(xtrace nnfw_lib_xdata) +target_link_libraries(xtrace ${LIB_PTHREAD}) +target_link_libraries(xtrace dl) + +install(TARGETS xtrace DESTINATION bin) diff --git a/runtimes/contrib/xtrace/src/benchmark_event.cc b/runtimes/contrib/xtrace/src/benchmark_event.cc new file mode 100644 index 0000000..54727d6 --- /dev/null +++ b/runtimes/contrib/xtrace/src/benchmark_event.cc @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "benchmark_event.h" + +#include + +xray::event_code BMCategory::set(std::unique_ptr &&event) +{ + _event = std::move(event); + return xray::event_code{0}; +} + +void BMCategory::reset(void) { _event.reset(); } + +void BMCategory::post(std::unique_ptr &&evt_info) +{ + auto evt_cat = this; + auto evt_idx = set(std::move(evt_info)); + const xray::event evt{evt_cat, evt_idx}; + xray::pipe::post(&evt); + reset(); +} diff --git a/runtimes/contrib/xtrace/src/benchmark_event.h b/runtimes/contrib/xtrace/src/benchmark_event.h new file mode 100644 index 0000000..d544cb0 --- /dev/null +++ b/runtimes/contrib/xtrace/src/benchmark_event.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __BENCHMARK_EVENT_H__ +#define __BENCHMARK_EVENT_H__ + +#include +#include +#include + +#include +#include + +enum BMPhase +{ + Warmup, + Stable +}; + +struct BMEvent +{ + virtual ~BMEvent() = default; +}; + +struct BMBegin : public BMEvent +{ + BMPhase phase; + uint32_t cur_iter; +}; + +struct BMEnd : public BMEvent +{ + BMPhase phase; + uint32_t cur_iter; + std::chrono::milliseconds elapsed; +}; + +class BMCategory final : public xray::event_category +{ +private: + BMCategory() = default; + +public: + xray::event_code set(std::unique_ptr &&event); + void reset(void); + +public: + const BMEvent *event(void) const { return _event.get(); } + +private: + std::unique_ptr _event; + +public: + static BMCategory *get(void) + { + static BMCategory cat; + return &cat; + } + +public: + void post(std::unique_ptr &&event); +}; + +#endif // __BENCHMARK_EVENT_H__ diff --git a/runtimes/contrib/xtrace/src/benchmark_runner.cc b/runtimes/contrib/xtrace/src/benchmark_runner.cc new file mode 100644 index 0000000..72cbef0 --- /dev/null +++ b/runtimes/contrib/xtrace/src/benchmark_runner.cc @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "benchmark_runner.h" +#include "benchmark_event.h" + +#include + +#include +#include +#include + +#include +#include + +#include + +#include + +using namespace tflite; +using namespace nnfw::tflite; +using namespace std::chrono; + +namespace +{ + +void notify(const BMBegin &event) +{ + BMCategory::get()->post(nnfw::cpp14::make_unique(event)); +} + +void notify(const BMEnd &event) { BMCategory::get()->post(nnfw::cpp14::make_unique(event)); } + +} // namespace + +void BMRunner::run(const std::string &filename) const +{ + BuiltinOpResolver op_resolver; + StderrReporter error_reporter; + + auto model = FlatBufferModel::BuildFromFile(filename.c_str(), &error_reporter); + + if (model == nullptr) + { + throw std::runtime_error{"Cannot create model"}; + } + + InterpreterBuilder builder(*model, op_resolver); + + std::unique_ptr interp; + TFLITE_ENSURE(builder(&interp)); + + auto sess = std::make_shared(interp.release()); + + // Iteration! + for (auto phase : {Warmup, Stable}) + { + auto get_iteration_count = [](const BMPhase &phase) { + switch (phase) + { + case Warmup: + return 1; // Allow configuration + case Stable: + return 3; + default: + break; + } + + throw std::runtime_error{"Error!"}; + }; + + uint32_t iteration_count = get_iteration_count(phase); + + for (uint32_t n = 0; n < iteration_count; ++n) + { + // Notify event + { + BMBegin event; + + event.phase = phase; + event.cur_iter = n; + + notify(event); + } + + sess->prepare(); + + std::chrono::milliseconds elapsed(0); + nnfw::misc::benchmark::measure(elapsed) << [&](void) { + if (!sess->run()) + { + throw std::runtime_error{"run failed"}; + } + }; + + sess->teardown(); + + // Notify + { + BMEnd event; + + event.phase = phase; + event.cur_iter = n; + event.elapsed = elapsed; + + notify(event); + } + } + } +} diff --git a/runtimes/contrib/xtrace/src/benchmark_runner.h b/runtimes/contrib/xtrace/src/benchmark_runner.h new file mode 100644 index 0000000..f76555d --- /dev/null +++ b/runtimes/contrib/xtrace/src/benchmark_runner.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __BENCHMARK_RUNNER_H__ +#define __BENCHMAKR_RUNNER_H__ + +#include + +using TFLModelPath = std::string; + +enum BMRunnerType +{ + // Use T/F Lite interpreter with Android NN API Delegate + TFL_NNAPI_DELEGATE +}; + +template struct BMRunner; + +template <> struct BMRunner +{ + void run(const TFLModelPath &filename) const; +}; + +#endif // __BENCHMARK_RUNNER_H__ diff --git a/runtimes/contrib/xtrace/src/event_collector.cc b/runtimes/contrib/xtrace/src/event_collector.cc new file mode 100644 index 0000000..2b37bf4 --- /dev/null +++ b/runtimes/contrib/xtrace/src/event_collector.cc @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "event_collector.h" +#include "benchmark_event.h" + +// xtrace-internal libraries +#include "str.h" + +// NNFW-internal libraries +#include +#include + +// C++ standard libraries +#include +#include + +// POSIX standard libraries +#include +#include + +using nnfw::cpp14::make_unique; + +namespace +{ + +std::string timestamp(void) +{ + auto now = std::chrono::steady_clock::now(); + return std::to_string( + std::chrono::duration_cast(now.time_since_epoch()).count()); +} + +class DurationEventBuilder +{ +public: + DurationEventBuilder(const std::string &ts) : _ts{ts} {} + + DurationEvent build(const std::string &name, const std::string &ph) const + { + DurationEvent evt; + + evt.name = name; + evt.ph = ph; + evt.ts = _ts; + + return evt; + } + +private: + std::string _ts; +}; + +void emit_rusage(EventRecorder *rec, const std::string &ts) +{ + struct rusage ru; + + getrusage(RUSAGE_SELF, &ru); + { + CounterEvent evt; + + evt.name = "maxrss"; + evt.ph = "C"; + evt.ts = ts; + evt.values["value"] = std::to_string(ru.ru_maxrss); + + rec->emit(evt); + } + + { + CounterEvent evt; + + evt.name = "minflt"; + evt.ph = "C"; + evt.ts = ts; + evt.values["value"] = std::to_string(ru.ru_minflt); + + rec->emit(evt); + } +} + +std::ostream &operator<<(std::ostream &os, const BMPhase &phase) +{ + os << ((phase == Warmup) ? "Warmup" : "Iteration"); + return os; +} + +std::ostream &operator<<(std::ostream &os, const std::chrono::milliseconds &dur) +{ + os << dur.count(); + return os; +} + +} // namespace + +void EventCollector::notify(const xray::event *e) +{ + auto ts = timestamp(); + + // Record trace events (region enter/leave) + if (e->cat() == xdata::trace::category::get()) + { + auto info = xdata::trace::category::get()->info(); + + switch (info->action()) + { + case xdata::trace::enter: + _rec->emit(DurationEventBuilder(ts).build(info->region()->name(), "B")); + break; + + case xdata::trace::leave: + _rec->emit(DurationEventBuilder(ts).build(info->region()->name(), "E")); + break; + } + } + + // Record benchmark events + if (e->cat() == BMCategory::get()) + { + auto make_head = [](const BMPhase &phase, uint32_t iter) { return str(phase, " ", iter); }; + + if (auto info = dynamic_cast(BMCategory::get()->event())) + { + auto name = str(info->phase, info->cur_iter); + _rec->emit(DurationEventBuilder(ts).build(name, "B")); + + auto head = make_head(info->phase, info->cur_iter); + std::cout << head << std::endl; + } + + if (auto info = dynamic_cast(BMCategory::get()->event())) + { + auto name = str(info->phase, info->cur_iter); + _rec->emit(DurationEventBuilder(ts).build(name, "E")); + + auto head = make_head(info->phase, info->cur_iter); + std::cout << head << " - done " << std::endl; + std::cout << head << " takes " << info->elapsed << "ms" << std::endl; + } + } + + // Trace resource usage per each event notification + emit_rusage(_rec, ts); +} diff --git a/runtimes/contrib/xtrace/src/event_collector.h b/runtimes/contrib/xtrace/src/event_collector.h new file mode 100644 index 0000000..f088ecd --- /dev/null +++ b/runtimes/contrib/xtrace/src/event_collector.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __EVENT_COLLECTOR_H__ +#define __EVENT_COLLECTOR_H__ + +#include "event_recorder.h" + +#include + +class EventCollector final : public xray::listener +{ +public: + EventCollector(EventRecorder *rec) : _rec{rec} + { + // DO NOTHING + } + +public: + void notify(const xray::event *e) final; + +private: + EventRecorder *_rec = nullptr; +}; + +#endif // __EVENT_COLLECTOR_H__ diff --git a/runtimes/contrib/xtrace/src/event_recorder.cc b/runtimes/contrib/xtrace/src/event_recorder.cc new file mode 100644 index 0000000..780eae1 --- /dev/null +++ b/runtimes/contrib/xtrace/src/event_recorder.cc @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "event_recorder.h" +#include "str.h" + +#include + +namespace +{ + +std::string quote(const std::string &value) +{ + std::stringstream ss; + ss << '"' << value << '"'; + return ss.str(); +} + +std::string field(const std::string &k, const std::string &v) +{ + std::stringstream ss; + ss << quote(k) << " : " << quote(v); + return ss.str(); +} + +struct Content // One Entry in Chrome Event Trace +{ + std::vector> flds; + std::vector> args; +}; + +std::string object(const Content &content) +{ + std::stringstream ss; + + ss << "{ "; + + ss << field(content.flds[0].first, content.flds[0].second); + + for (uint32_t n = 1; n < content.flds.size(); ++n) + { + ss << ", " << field(content.flds.at(n).first, content.flds.at(n).second); + } + + if (content.args.size() > 0) + { + ss << ", " << quote("args") << " : { "; + ss << field(content.args.at(0).first, content.args.at(0).second); + + for (uint32_t n = 1; n < content.args.size(); ++n) + { + ss << ", " << field(content.args.at(n).first, content.args.at(n).second); + } + + ss << "}"; + } + + ss << " }"; + + return ss.str(); +} + +void fill(Content &content, const Event &evt) +{ + content.flds.emplace_back("name", evt.name); + content.flds.emplace_back("pid", "0"); + content.flds.emplace_back("tid", "0"); + content.flds.emplace_back("ph", evt.ph); + content.flds.emplace_back("ts", evt.ts); +} + +std::string object(const DurationEvent &evt) +{ + Content content; + + fill(content, evt); + + return ::object(content); +} + +std::string object(const CounterEvent &evt) +{ + Content content; + + fill(content, evt); + + for (auto it = evt.values.begin(); it != evt.values.end(); ++it) + { + content.args.emplace_back(it->first, it->second); + } + + return ::object(content); +} + +} // namespace + +void EventRecorder::init() +{ + _os << "{" << std::endl; + _os << " " << quote("traceEvents") << ": [" << std::endl; +} + +void EventRecorder::emit(const DurationEvent &evt) +{ + _os << " " << object(evt) << "," << std::endl; +} +void EventRecorder::emit(const CounterEvent &evt) +{ + _os << " " << object(evt) << "," << std::endl; +} + +void EventRecorder::fini() +{ + _os << " { }" << std::endl; + _os << " ]" << std::endl; + _os << "}" << std::endl; +} diff --git a/runtimes/contrib/xtrace/src/event_recorder.h b/runtimes/contrib/xtrace/src/event_recorder.h new file mode 100644 index 0000000..9cc9921 --- /dev/null +++ b/runtimes/contrib/xtrace/src/event_recorder.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __EVENT_RECORDER_H__ +#define __EVENT_RECORDER_H__ + +#include +#include + +#include + +struct Event +{ + std::string name; + std::string ph; /* REQUIRED */ + std::string ts; /* REQUIRED */ +}; + +struct DurationEvent : public Event +{ + // TO BE FILLED +}; + +struct CounterEvent : public Event +{ + std::map values; +}; + +// +// Record Event as Chrome Trace Event File Format +// +// Refrence: https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/edit +// +class EventRecorder +{ +public: + EventRecorder(std::ostream &os) : _os(os) + { + // DO NOTHING + } + +public: + void init(); + +public: + void emit(const DurationEvent &evt); + void emit(const CounterEvent &evt); + +public: + void fini(); + +private: + std::ostream &_os; +}; + +#endif // __EVENT_RECORDER_H__ diff --git a/runtimes/contrib/xtrace/src/str.h b/runtimes/contrib/xtrace/src/str.h new file mode 100644 index 0000000..a6d53a5 --- /dev/null +++ b/runtimes/contrib/xtrace/src/str.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __STR_H__ +#define __STR_H__ + +#include +#include + +template void _str(std::ostream &os, Arg &&arg) { os << std::forward(arg); } + +template void _str(std::ostream &os, Arg &&arg, Args &&... args) +{ + _str(os, std::forward(arg)); + _str(os, std::forward(args)...); +} + +template std::string str(Args &&... args) +{ + std::stringstream ss; + _str(ss, std::forward(args)...); + return ss.str(); +} + +#endif // __STR_H__ diff --git a/runtimes/contrib/xtrace/src/xtrace.cc b/runtimes/contrib/xtrace/src/xtrace.cc new file mode 100644 index 0000000..117a2e6 --- /dev/null +++ b/runtimes/contrib/xtrace/src/xtrace.cc @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "event_recorder.h" +#include "event_collector.h" +#include "benchmark_runner.h" + +#include +#include +#include +#include + +// xtrace --out ().run(argv[3]); + + xray::mux::get().detach(&event_collector); + + recorder.fini(); + + return 0; +} + +int main(int argc, char **argv) +{ + try + { + return entry(argc, argv); + } + catch (const std::exception &e) + { + std::cerr << e.what() << std::endl; + return 255; + } +} -- 2.7.4