tests/tools/nnpackage_run/src/nnpackage_run.cc

   1 /*
   2  * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *    http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include "allocation.h"
  18 #include "args.h"
  19 #include "benchmark.h"
  20 #if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1
  21 #include "h5formatter.h"
  22 #endif
  23 #include "nnfw.h"
  24 #include "nnfw_util.h"
  25 #include "nnfw_internal.h"
  26 #include "randomgen.h"
  27 #ifdef RUY_PROFILER
  28 #include "ruy/profiler/profiler.h"
  29 #endif
  30
  31 #include <cassert>
  32 #include <chrono>
  33 #include <cstdlib>
  34 #include <iostream>
  35 #include <libgen.h>
  36 #include <stdexcept>
  37 #include <unordered_map>
  38 #include <vector>
  39
  40 static const char *default_backend_cand = "acl_cl";
  41
  42 NNFW_STATUS resolve_op_backend(nnfw_session *session)
  43 {
  44   static std::unordered_map<std::string, std::string> operation_map = {
  45       {"TRANSPOSE_CONV", "OP_BACKEND_TransposeConv"},      {"CONV_2D", "OP_BACKEND_Conv2D"},
  46       {"DEPTHWISE_CONV_2D", "OP_BACKEND_DepthwiseConv2D"}, {"MEAN", "OP_BACKEND_Mean"},
  47       {"AVERAGE_POOL_2D", "OP_BACKEND_AvgPool2D"},         {"MAX_POOL_2D", "OP_BACKEND_MaxPool2D"},
  48       {"INSTANCE_NORM", "OP_BACKEND_InstanceNorm"},        {"ADD", "OP_BACKEND_Add"}};
  49
  50   for (auto i : operation_map)
  51   {
  52     char *default_backend = std::getenv(i.second.c_str());
  53     if (default_backend)
  54     {
  55       NNFW_STATUS return_result = nnfw_set_op_backend(session, i.first.c_str(), default_backend);
  56       if (return_result == NNFW_STATUS_ERROR)
  57         return return_result;
  58     }
  59   }
  60
  61   return NNFW_STATUS_NO_ERROR;
  62 }
  63
  64 int main(const int argc, char **argv)
  65 {
  66   using namespace nnpkg_run;
  67
  68   try
  69   {
  70     Args args(argc, argv);
  71     auto nnpackage_path = args.getPackageFilename();
  72     if (args.printVersion())
  73     {
  74       uint32_t version;
  75       NNPR_ENSURE_STATUS(nnfw_query_info_u32(NULL, NNFW_INFO_ID_VERSION, &version));
  76       std::cout << "nnpkg_run (nnfw runtime: v" << (version >> 24) << "."
  77                 << ((version & 0x0000FF00) >> 8) << "." << (version & 0xFF) << ")" << std::endl;
  78       exit(0);
  79     }
  80
  81 #ifdef RUY_PROFILER
  82     ruy::profiler::ScopeProfile ruy_profile;
  83 #endif
  84
  85     // TODO Apply verbose level to phases
  86     const int verbose = args.getVerboseLevel();
  87     benchmark::Phases phases(
  88         benchmark::PhaseOption{args.getMemoryPoll(), args.getGpuMemoryPoll(), args.getRunDelay()});
  89
  90     nnfw_session *session = nullptr;
  91     NNPR_ENSURE_STATUS(nnfw_create_session(&session));
  92
  93     // ModelLoad
  94     phases.run("MODEL_LOAD", [&](const benchmark::Phase &, uint32_t) {
  95       NNPR_ENSURE_STATUS(nnfw_load_model_from_file(session, nnpackage_path.c_str()));
  96     });
  97
  98     char *available_backends = std::getenv("BACKENDS");
  99     if (available_backends)
 100       NNPR_ENSURE_STATUS(nnfw_set_available_backends(session, available_backends));
 101     NNPR_ENSURE_STATUS(resolve_op_backend(session));
 102
 103     uint32_t num_inputs;
 104     NNPR_ENSURE_STATUS(nnfw_input_size(session, &num_inputs));
 105
 106     // verify input and output
 107
 108     auto verifyInputTypes = [session]() {
 109       uint32_t sz;
 110       NNPR_ENSURE_STATUS(nnfw_input_size(session, &sz));
 111       for (uint32_t i = 0; i < sz; ++i)
 112       {
 113         nnfw_tensorinfo ti;
 114         NNPR_ENSURE_STATUS(nnfw_input_tensorinfo(session, i, &ti));
 115
 116         if (ti.dtype < NNFW_TYPE_TENSOR_FLOAT32 || ti.dtype > NNFW_TYPE_TENSOR_INT64)
 117         {
 118           std::cerr << "E: not supported input type" << std::endl;
 119           exit(-1);
 120         }
 121       }
 122     };
 123
 124     auto verifyOutputTypes = [session]() {
 125       uint32_t sz;
 126       NNPR_ENSURE_STATUS(nnfw_output_size(session, &sz));
 127
 128       for (uint32_t i = 0; i < sz; ++i)
 129       {
 130         nnfw_tensorinfo ti;
 131         NNPR_ENSURE_STATUS(nnfw_output_tensorinfo(session, i, &ti));
 132
 133         if (ti.dtype < NNFW_TYPE_TENSOR_FLOAT32 || ti.dtype > NNFW_TYPE_TENSOR_INT64)
 134         {
 135           std::cerr << "E: not supported output type" << std::endl;
 136           exit(-1);
 137         }
 138       }
 139     };
 140
 141     auto setTensorInfo = [session](const TensorShapeMap &tensor_shape_map) {
 142       for (auto tensor_shape : tensor_shape_map)
 143       {
 144         auto ind = tensor_shape.first;
 145         auto &shape = tensor_shape.second;
 146         nnfw_tensorinfo ti;
 147         // to fill dtype
 148         NNPR_ENSURE_STATUS(nnfw_input_tensorinfo(session, ind, &ti));
 149
 150         ti.rank = shape.size();
 151         for (int i = 0; i < ti.rank; i++)
 152           ti.dims[i] = shape.at(i);
 153         NNPR_ENSURE_STATUS(nnfw_set_input_tensorinfo(session, ind, &ti));
 154       }
 155     };
 156
 157     verifyInputTypes();
 158     verifyOutputTypes();
 159
 160     // set input shape before compilation
 161     setTensorInfo(args.getShapeMapForPrepare());
 162
 163     // prepare execution
 164
 165     // TODO When nnfw_{prepare|run} are failed, can't catch the time
 166     phases.run("PREPARE", [&](const benchmark::Phase &, uint32_t) {
 167       NNPR_ENSURE_STATUS(nnfw_prepare(session));
 168     });
 169
 170     // set input shape after compilation and before execution
 171     setTensorInfo(args.getShapeMapForRun());
 172
 173     // prepare input
 174     std::vector<Allocation> inputs(num_inputs);
 175 #if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1
 176     if (!args.getLoadFilename().empty())
 177       H5Formatter(session).loadInputs(args.getLoadFilename(), inputs);
 178     else
 179       RandomGenerator(session).generate(inputs);
 180 #else
 181     RandomGenerator(session).generate(inputs);
 182 #endif
 183
 184     // prepare output
 185     uint32_t num_outputs = 0;
 186     NNPR_ENSURE_STATUS(nnfw_output_size(session, &num_outputs));
 187     std::vector<Allocation> outputs(num_outputs);
 188     auto output_sizes = args.getOutputSizes();
 189     for (uint32_t i = 0; i < num_outputs; i++)
 190     {
 191       nnfw_tensorinfo ti;
 192       uint64_t output_size_in_bytes = 0;
 193       {
 194         auto found = output_sizes.find(i);
 195         if (found == output_sizes.end())
 196         {
 197           NNPR_ENSURE_STATUS(nnfw_output_tensorinfo(session, i, &ti));
 198           output_size_in_bytes = bufsize_for(&ti);
 199         }
 200         else
 201         {
 202           output_size_in_bytes = found->second;
 203         }
 204       }
 205       outputs[i].alloc(output_size_in_bytes);
 206       NNPR_ENSURE_STATUS(
 207           nnfw_set_output(session, i, ti.dtype, outputs[i].data(), output_size_in_bytes));
 208       NNPR_ENSURE_STATUS(nnfw_set_output_layout(session, i, NNFW_LAYOUT_CHANNELS_LAST));
 209     }
 210
 211     // NOTE: Measuring memory can't avoid taking overhead. Therefore, memory will be measured on the
 212     // only warmup.
 213     if (verbose == 0)
 214     {
 215       phases.run("WARMUP",
 216                  [&](const benchmark::Phase &, uint32_t) { NNPR_ENSURE_STATUS(nnfw_run(session)); },
 217                  args.getWarmupRuns());
 218       phases.run("EXECUTE",
 219                  [&](const benchmark::Phase &, uint32_t) { NNPR_ENSURE_STATUS(nnfw_run(session)); },
 220                  args.getNumRuns(), true);
 221     }
 222     else
 223     {
 224       phases.run("WARMUP",
 225                  [&](const benchmark::Phase &, uint32_t) { NNPR_ENSURE_STATUS(nnfw_run(session)); },
 226                  [&](const benchmark::Phase &phase, uint32_t nth) {
 227                    std::cout << "... "
 228                              << "warmup " << nth + 1 << " takes " << phase.time[nth] / 1e3 << " ms"
 229                              << std::endl;
 230                  },
 231                  args.getWarmupRuns());
 232       phases.run("EXECUTE",
 233                  [&](const benchmark::Phase &, uint32_t) { NNPR_ENSURE_STATUS(nnfw_run(session)); },
 234                  [&](const benchmark::Phase &phase, uint32_t nth) {
 235                    std::cout << "... "
 236                              << "run " << nth + 1 << " takes " << phase.time[nth] / 1e3 << " ms"
 237                              << std::endl;
 238                  },
 239                  args.getNumRuns(), true);
 240     }
 241
 242 #if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1
 243     // dump output tensors
 244     if (!args.getDumpFilename().empty())
 245       H5Formatter(session).dumpOutputs(args.getDumpFilename(), outputs);
 246 #endif
 247
 248     NNPR_ENSURE_STATUS(nnfw_close_session(session));
 249
 250     // TODO Apply verbose level to result
 251
 252     // prepare result
 253     benchmark::Result result(phases);
 254
 255     // to stdout
 256     benchmark::printResult(result);
 257
 258     // to csv
 259     if (args.getWriteReport() == false)
 260       return 0;
 261
 262     // prepare csv task
 263     std::string exec_basename;
 264     std::string nnpkg_basename;
 265     std::string backend_name = (available_backends) ? available_backends : default_backend_cand;
 266     {
 267       char buf[PATH_MAX];
 268       char *res = realpath(nnpackage_path.c_str(), buf);
 269       if (res)
 270       {
 271         nnpkg_basename = basename(buf);
 272       }
 273       else
 274       {
 275         std::cerr << "E: during getting realpath from nnpackage_path." << std::endl;
 276         exit(-1);
 277       }
 278       exec_basename = basename(argv[0]);
 279     }
 280
 281     benchmark::writeResult(result, exec_basename, nnpkg_basename, backend_name);
 282
 283     return 0;
 284   }
 285   catch (std::runtime_error &e)
 286   {
 287     std::cerr << "E: Fail to run by runtime error:" << e.what() << std::endl;
 288     exit(-1);
 289   }
 290 }