tests/InferenceModel.hpp

   1 //
   2 // Copyright © 2017 Arm Ltd. All rights reserved.
   3 // SPDX-License-Identifier: MIT
   4 //
   5 #pragma once
   6 #include <armnn/ArmNN.hpp>
   7
   8 #if defined(ARMNN_SERIALIZER)
   9 #include "armnnDeserializer/IDeserializer.hpp"
  10 #endif
  11 #if defined(ARMNN_TF_LITE_PARSER)
  12 #include <armnnTfLiteParser/ITfLiteParser.hpp>
  13 #endif
  14 #if defined(ARMNN_ONNX_PARSER)
  15 #include <armnnOnnxParser/IOnnxParser.hpp>
  16 #endif
  17
  18 #include <HeapProfiling.hpp>
  19 #include <TensorIOUtils.hpp>
  20
  21 #include <backendsCommon/BackendRegistry.hpp>
  22
  23 #include <boost/algorithm/string/join.hpp>
  24 #include <boost/exception/exception.hpp>
  25 #include <boost/exception/diagnostic_information.hpp>
  26 #include <boost/log/trivial.hpp>
  27 #include <boost/format.hpp>
  28 #include <boost/program_options.hpp>
  29 #include <boost/filesystem.hpp>
  30 #include <boost/lexical_cast.hpp>
  31 #include <boost/variant.hpp>
  32
  33 #include <algorithm>
  34 #include <chrono>
  35 #include <iterator>
  36 #include <fstream>
  37 #include <map>
  38 #include <string>
  39 #include <vector>
  40 #include <type_traits>
  41
  42 namespace
  43 {
  44
  45 inline bool CheckRequestedBackendsAreValid(const std::vector<armnn::BackendId>& backendIds,
  46                                            armnn::Optional<std::string&> invalidBackendIds = armnn::EmptyOptional())
  47 {
  48     if (backendIds.empty())
  49     {
  50         return false;
  51     }
  52
  53     armnn::BackendIdSet validBackendIds = armnn::BackendRegistryInstance().GetBackendIds();
  54
  55     bool allValid = true;
  56     for (const auto& backendId : backendIds)
  57     {
  58         if (std::find(validBackendIds.begin(), validBackendIds.end(), backendId) == validBackendIds.end())
  59         {
  60             allValid = false;
  61             if (invalidBackendIds)
  62             {
  63                 if (!invalidBackendIds.value().empty())
  64                 {
  65                     invalidBackendIds.value() += ", ";
  66                 }
  67                 invalidBackendIds.value() += backendId;
  68             }
  69         }
  70     }
  71     return allValid;
  72 }
  73
  74 } // anonymous namespace
  75
  76 namespace InferenceModelInternal
  77 {
  78 using BindingPointInfo = armnn::BindingPointInfo;
  79
  80 using QuantizationParams = std::pair<float,int32_t>;
  81
  82 struct Params
  83 {
  84     std::string                     m_ModelPath;
  85     std::vector<std::string>        m_InputBindings;
  86     std::vector<armnn::TensorShape> m_InputShapes;
  87     std::vector<std::string>        m_OutputBindings;
  88     std::vector<armnn::BackendId>   m_ComputeDevices;
  89     std::string                     m_DynamicBackendsPath;
  90     size_t                          m_SubgraphId;
  91     bool                            m_IsModelBinary;
  92     bool                            m_VisualizePostOptimizationModel;
  93     bool                            m_EnableFp16TurboMode;
  94     bool                            m_PrintIntermediateLayers;
  95
  96     Params()
  97         : m_ComputeDevices{}
  98         , m_SubgraphId(0)
  99         , m_IsModelBinary(true)
 100         , m_VisualizePostOptimizationModel(false)
 101         , m_EnableFp16TurboMode(false)
 102         , m_PrintIntermediateLayers(false)
 103     {}
 104 };
 105
 106 } // namespace InferenceModelInternal
 107
 108 template <typename IParser>
 109 struct CreateNetworkImpl
 110 {
 111 public:
 112     using Params = InferenceModelInternal::Params;
 113
 114     static armnn::INetworkPtr Create(const Params& params,
 115                                      std::vector<armnn::BindingPointInfo>& inputBindings,
 116                                      std::vector<armnn::BindingPointInfo>& outputBindings)
 117     {
 118         const std::string& modelPath = params.m_ModelPath;
 119
 120         // Create a network from a file on disk
 121         auto parser(IParser::Create());
 122
 123         std::map<std::string, armnn::TensorShape> inputShapes;
 124         if (!params.m_InputShapes.empty())
 125         {
 126             const size_t numInputShapes   = params.m_InputShapes.size();
 127             const size_t numInputBindings = params.m_InputBindings.size();
 128             if (numInputShapes < numInputBindings)
 129             {
 130                 throw armnn::Exception(boost::str(boost::format(
 131                     "Not every input has its tensor shape specified: expected=%1%, got=%2%")
 132                     % numInputBindings % numInputShapes));
 133             }
 134
 135             for (size_t i = 0; i < numInputShapes; i++)
 136             {
 137                 inputShapes[params.m_InputBindings[i]] = params.m_InputShapes[i];
 138             }
 139         }
 140
 141         std::vector<std::string> requestedOutputs = params.m_OutputBindings;
 142         armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}};
 143
 144         {
 145             ARMNN_SCOPED_HEAP_PROFILING("Parsing");
 146             // Handle text and binary input differently by calling the corresponding parser function
 147             network = (params.m_IsModelBinary ?
 148                 parser->CreateNetworkFromBinaryFile(modelPath.c_str(), inputShapes, requestedOutputs) :
 149                 parser->CreateNetworkFromTextFile(modelPath.c_str(), inputShapes, requestedOutputs));
 150         }
 151
 152         for (const std::string& inputLayerName : params.m_InputBindings)
 153         {
 154             inputBindings.push_back(parser->GetNetworkInputBindingInfo(inputLayerName));
 155         }
 156
 157         for (const std::string& outputLayerName : params.m_OutputBindings)
 158         {
 159             outputBindings.push_back(parser->GetNetworkOutputBindingInfo(outputLayerName));
 160         }
 161
 162         return network;
 163     }
 164 };
 165
 166 #if defined(ARMNN_SERIALIZER)
 167 template <>
 168 struct CreateNetworkImpl<armnnDeserializer::IDeserializer>
 169 {
 170 public:
 171     using IParser          = armnnDeserializer::IDeserializer;
 172     using Params           = InferenceModelInternal::Params;
 173
 174     static armnn::INetworkPtr Create(const Params& params,
 175                                      std::vector<armnn::BindingPointInfo>& inputBindings,
 176                                      std::vector<armnn::BindingPointInfo>& outputBindings)
 177     {
 178         auto parser(IParser::Create());
 179         BOOST_ASSERT(parser);
 180
 181         armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}};
 182
 183         {
 184             ARMNN_SCOPED_HEAP_PROFILING("Parsing");
 185
 186             boost::system::error_code errorCode;
 187             boost::filesystem::path pathToFile(params.m_ModelPath);
 188             if (!boost::filesystem::exists(pathToFile, errorCode))
 189             {
 190                 throw armnn::FileNotFoundException(boost::str(
 191                                                    boost::format("Cannot find the file (%1%) errorCode: %2% %3%") %
 192                                                    params.m_ModelPath %
 193                                                    errorCode %
 194                                                    CHECK_LOCATION().AsString()));
 195             }
 196             std::ifstream file(params.m_ModelPath, std::ios::binary);
 197
 198             network = parser->CreateNetworkFromBinary(file);
 199         }
 200
 201         unsigned int subgraphId = boost::numeric_cast<unsigned int>(params.m_SubgraphId);
 202
 203         for (const std::string& inputLayerName : params.m_InputBindings)
 204         {
 205             armnnDeserializer::BindingPointInfo inputBinding =
 206                 parser->GetNetworkInputBindingInfo(subgraphId, inputLayerName);
 207             inputBindings.push_back(std::make_pair(inputBinding.m_BindingId, inputBinding.m_TensorInfo));
 208         }
 209
 210         for (const std::string& outputLayerName : params.m_OutputBindings)
 211         {
 212             armnnDeserializer::BindingPointInfo outputBinding =
 213                 parser->GetNetworkOutputBindingInfo(subgraphId, outputLayerName);
 214             outputBindings.push_back(std::make_pair(outputBinding.m_BindingId, outputBinding.m_TensorInfo));
 215         }
 216
 217         return network;
 218     }
 219 };
 220 #endif
 221
 222 #if defined(ARMNN_TF_LITE_PARSER)
 223 template <>
 224 struct CreateNetworkImpl<armnnTfLiteParser::ITfLiteParser>
 225 {
 226 public:
 227     using IParser = armnnTfLiteParser::ITfLiteParser;
 228     using Params = InferenceModelInternal::Params;
 229
 230     static armnn::INetworkPtr Create(const Params& params,
 231                                      std::vector<armnn::BindingPointInfo>& inputBindings,
 232                                      std::vector<armnn::BindingPointInfo>& outputBindings)
 233     {
 234         const std::string& modelPath = params.m_ModelPath;
 235
 236         // Create a network from a file on disk
 237         auto parser(IParser::Create());
 238
 239         armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}};
 240
 241         {
 242             ARMNN_SCOPED_HEAP_PROFILING("Parsing");
 243             network = parser->CreateNetworkFromBinaryFile(modelPath.c_str());
 244         }
 245
 246         for (const std::string& inputLayerName : params.m_InputBindings)
 247         {
 248             armnn::BindingPointInfo inputBinding =
 249                 parser->GetNetworkInputBindingInfo(params.m_SubgraphId, inputLayerName);
 250             inputBindings.push_back(inputBinding);
 251         }
 252
 253         for (const std::string& outputLayerName : params.m_OutputBindings)
 254         {
 255             armnn::BindingPointInfo outputBinding =
 256                 parser->GetNetworkOutputBindingInfo(params.m_SubgraphId, outputLayerName);
 257             outputBindings.push_back(outputBinding);
 258         }
 259
 260         return network;
 261     }
 262 };
 263 #endif
 264
 265 #if defined(ARMNN_ONNX_PARSER)
 266 template <>
 267 struct CreateNetworkImpl<armnnOnnxParser::IOnnxParser>
 268 {
 269 public:
 270     using IParser = armnnOnnxParser::IOnnxParser;
 271     using Params = InferenceModelInternal::Params;
 272     using BindingPointInfo = InferenceModelInternal::BindingPointInfo;
 273
 274     static armnn::INetworkPtr Create(const Params& params,
 275                                      std::vector<BindingPointInfo>& inputBindings,
 276                                      std::vector<BindingPointInfo>& outputBindings)
 277     {
 278         const std::string& modelPath = params.m_ModelPath;
 279
 280         // Create a network from a file on disk
 281         auto parser(IParser::Create());
 282
 283         armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}};
 284
 285         {
 286             ARMNN_SCOPED_HEAP_PROFILING("Parsing");
 287             network = (params.m_IsModelBinary ?
 288                 parser->CreateNetworkFromBinaryFile(modelPath.c_str()) :
 289                 parser->CreateNetworkFromTextFile(modelPath.c_str()));
 290         }
 291
 292         for (const std::string& inputLayerName : params.m_InputBindings)
 293         {
 294             BindingPointInfo inputBinding = parser->GetNetworkInputBindingInfo(inputLayerName);
 295             inputBindings.push_back(inputBinding);
 296         }
 297
 298         for (const std::string& outputLayerName : params.m_OutputBindings)
 299         {
 300             BindingPointInfo outputBinding = parser->GetNetworkOutputBindingInfo(outputLayerName);
 301             outputBindings.push_back(outputBinding);
 302         }
 303
 304         return network;
 305     }
 306 };
 307 #endif
 308
 309
 310
 311 template <typename IParser, typename TDataType>
 312 class InferenceModel
 313 {
 314 public:
 315     using DataType           = TDataType;
 316     using Params             = InferenceModelInternal::Params;
 317     using QuantizationParams = InferenceModelInternal::QuantizationParams;
 318     using TContainer         = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;
 319
 320     struct CommandLineOptions
 321     {
 322         std::string m_ModelDir;
 323         std::vector<std::string> m_ComputeDevices;
 324         std::string m_DynamicBackendsPath;
 325         bool m_VisualizePostOptimizationModel;
 326         bool m_EnableFp16TurboMode;
 327         std::string m_Labels;
 328
 329         std::vector<armnn::BackendId> GetComputeDevicesAsBackendIds()
 330         {
 331             std::vector<armnn::BackendId> backendIds;
 332             std::copy(m_ComputeDevices.begin(), m_ComputeDevices.end(), std::back_inserter(backendIds));
 333             return backendIds;
 334         }
 335     };
 336
 337     static void AddCommandLineOptions(boost::program_options::options_description& desc, CommandLineOptions& options)
 338     {
 339         namespace po = boost::program_options;
 340
 341         const std::vector<std::string> defaultComputes = { "CpuAcc", "CpuRef" };
 342
 343         const std::string backendsMessage = "Which device to run layers on by default. Possible choices: "
 344                                           + armnn::BackendRegistryInstance().GetBackendIdsAsString();
 345
 346         desc.add_options()
 347             ("model-dir,m", po::value<std::string>(&options.m_ModelDir)->required(),
 348                 "Path to directory containing model files (.caffemodel/.prototxt/.tflite)")
 349             ("compute,c", po::value<std::vector<std::string>>(&options.m_ComputeDevices)->
 350                 default_value(defaultComputes, boost::algorithm::join(defaultComputes, ", "))->
 351                 multitoken(), backendsMessage.c_str())
 352             ("dynamic-backends-path,b", po::value(&options.m_DynamicBackendsPath),
 353                 "Path where to load any available dynamic backend from. "
 354                 "If left empty (the default), dynamic backends will not be used.")
 355             ("labels,l", po::value<std::string>(&options.m_Labels),
 356                 "Text file containing one image filename - correct label pair per line, "
 357                 "used to test the accuracy of the network.")
 358             ("visualize-optimized-model,v",
 359                 po::value<bool>(&options.m_VisualizePostOptimizationModel)->default_value(false),
 360              "Produce a dot file useful for visualizing the graph post optimization."
 361                 "The file will have the same name as the model with the .dot extention.")
 362             ("fp16-turbo-mode", po::value<bool>(&options.m_EnableFp16TurboMode)->default_value(false),
 363                 "If this option is enabled FP32 layers, weights and biases will be converted "
 364                 "to FP16 where the backend supports it.");
 365     }
 366
 367     InferenceModel(const Params& params,
 368                    bool enableProfiling,
 369                    const std::string& dynamicBackendsPath,
 370                    const std::shared_ptr<armnn::IRuntime>& runtime = nullptr)
 371         : m_EnableProfiling(enableProfiling)
 372         , m_DynamicBackendsPath(dynamicBackendsPath)
 373     {
 374         if (runtime)
 375         {
 376             m_Runtime = runtime;
 377         }
 378         else
 379         {
 380             armnn::IRuntime::CreationOptions options;
 381             options.m_EnableGpuProfiling = m_EnableProfiling;
 382             options.m_DynamicBackendsPath = m_DynamicBackendsPath;
 383             m_Runtime = std::move(armnn::IRuntime::Create(options));
 384         }
 385
 386         std::string invalidBackends;
 387         if (!CheckRequestedBackendsAreValid(params.m_ComputeDevices, armnn::Optional<std::string&>(invalidBackends)))
 388         {
 389             throw armnn::Exception("Some backend IDs are invalid: " + invalidBackends);
 390         }
 391
 392         armnn::INetworkPtr network = CreateNetworkImpl<IParser>::Create(params, m_InputBindings, m_OutputBindings);
 393
 394         armnn::IOptimizedNetworkPtr optNet{nullptr, [](armnn::IOptimizedNetwork*){}};
 395         {
 396             ARMNN_SCOPED_HEAP_PROFILING("Optimizing");
 397
 398             armnn::OptimizerOptions options;
 399             options.m_ReduceFp32ToFp16 = params.m_EnableFp16TurboMode;
 400             options.m_Debug = params.m_PrintIntermediateLayers;
 401
 402             optNet = armnn::Optimize(*network, params.m_ComputeDevices, m_Runtime->GetDeviceSpec(), options);
 403             if (!optNet)
 404             {
 405                 throw armnn::Exception("Optimize returned nullptr");
 406             }
 407         }
 408
 409         if (params.m_VisualizePostOptimizationModel)
 410         {
 411             boost::filesystem::path filename = params.m_ModelPath;
 412             filename.replace_extension("dot");
 413             std::fstream file(filename.c_str(), std::ios_base::out);
 414             optNet->SerializeToDot(file);
 415         }
 416
 417         armnn::Status ret;
 418         {
 419             ARMNN_SCOPED_HEAP_PROFILING("LoadNetwork");
 420             ret = m_Runtime->LoadNetwork(m_NetworkIdentifier, std::move(optNet));
 421         }
 422
 423         if (ret == armnn::Status::Failure)
 424         {
 425             throw armnn::Exception("IRuntime::LoadNetwork failed");
 426         }
 427     }
 428
 429     void CheckInputIndexIsValid(unsigned int inputIndex) const
 430     {
 431         if (m_InputBindings.size() < inputIndex + 1)
 432         {
 433             throw armnn::Exception(boost::str(boost::format("Input index out of range: %1%") % inputIndex));
 434         }
 435     }
 436
 437     void CheckOutputIndexIsValid(unsigned int outputIndex) const
 438     {
 439         if (m_OutputBindings.size() < outputIndex + 1)
 440         {
 441             throw armnn::Exception(boost::str(boost::format("Output index out of range: %1%") % outputIndex));
 442         }
 443     }
 444
 445     unsigned int GetOutputSize(unsigned int outputIndex = 0u) const
 446     {
 447         CheckOutputIndexIsValid(outputIndex);
 448         return m_OutputBindings[outputIndex].second.GetNumElements();
 449     }
 450
 451     std::chrono::duration<double, std::milli> Run(
 452             const std::vector<TContainer>& inputContainers,
 453             std::vector<TContainer>& outputContainers)
 454     {
 455         for (unsigned int i = 0; i < outputContainers.size(); ++i)
 456         {
 457             const unsigned int expectedOutputDataSize = GetOutputSize(i);
 458
 459             boost::apply_visitor([expectedOutputDataSize, i](auto&& value)
 460             {
 461                 const unsigned int actualOutputDataSize   = boost::numeric_cast<unsigned int>(value.size());
 462                 if (actualOutputDataSize < expectedOutputDataSize)
 463                 {
 464                     unsigned int outputIndex = boost::numeric_cast<unsigned int>(i);
 465                     throw armnn::Exception(
 466                             boost::str(boost::format("Not enough data for output #%1%: expected "
 467                             "%2% elements, got %3%") % outputIndex % expectedOutputDataSize % actualOutputDataSize));
 468                 }
 469             },
 470             outputContainers[i]);
 471         }
 472
 473         std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);
 474         if (profiler)
 475         {
 476             profiler->EnableProfiling(m_EnableProfiling);
 477         }
 478
 479         // Start timer to record inference time in EnqueueWorkload (in milliseconds)
 480         const auto start_time = GetCurrentTime();
 481
 482         armnn::Status ret = m_Runtime->EnqueueWorkload(m_NetworkIdentifier,
 483                                                        MakeInputTensors(inputContainers),
 484                                                        MakeOutputTensors(outputContainers));
 485
 486         const auto end_time = GetCurrentTime();
 487
 488         // if profiling is enabled print out the results
 489         if (profiler && profiler->IsProfilingEnabled())
 490         {
 491             profiler->Print(std::cout);
 492         }
 493
 494         if (ret == armnn::Status::Failure)
 495         {
 496             throw armnn::Exception("IRuntime::EnqueueWorkload failed");
 497         }
 498         else
 499         {
 500             return std::chrono::duration<double, std::milli>(end_time - start_time);
 501         }
 502     }
 503
 504     const armnn::BindingPointInfo& GetInputBindingInfo(unsigned int inputIndex = 0u) const
 505     {
 506         CheckInputIndexIsValid(inputIndex);
 507         return m_InputBindings[inputIndex];
 508     }
 509
 510     const std::vector<armnn::BindingPointInfo>& GetInputBindingInfos() const
 511     {
 512         return m_InputBindings;
 513     }
 514
 515     const armnn::BindingPointInfo& GetOutputBindingInfo(unsigned int outputIndex = 0u) const
 516     {
 517         CheckOutputIndexIsValid(outputIndex);
 518         return m_OutputBindings[outputIndex];
 519     }
 520
 521     const std::vector<armnn::BindingPointInfo>& GetOutputBindingInfos() const
 522     {
 523         return m_OutputBindings;
 524     }
 525
 526     QuantizationParams GetQuantizationParams(unsigned int outputIndex = 0u) const
 527     {
 528         CheckOutputIndexIsValid(outputIndex);
 529         return std::make_pair(m_OutputBindings[outputIndex].second.GetQuantizationScale(),
 530                               m_OutputBindings[outputIndex].second.GetQuantizationOffset());
 531     }
 532
 533     QuantizationParams GetInputQuantizationParams(unsigned int inputIndex = 0u) const
 534     {
 535         CheckInputIndexIsValid(inputIndex);
 536         return std::make_pair(m_InputBindings[inputIndex].second.GetQuantizationScale(),
 537                               m_InputBindings[inputIndex].second.GetQuantizationOffset());
 538     }
 539
 540     std::vector<QuantizationParams> GetAllQuantizationParams() const
 541     {
 542         std::vector<QuantizationParams> quantizationParams;
 543         for (unsigned int i = 0u; i < m_OutputBindings.size(); i++)
 544         {
 545             quantizationParams.push_back(GetQuantizationParams(i));
 546         }
 547         return quantizationParams;
 548     }
 549
 550 private:
 551     armnn::NetworkId m_NetworkIdentifier;
 552     std::shared_ptr<armnn::IRuntime> m_Runtime;
 553
 554     std::vector<armnn::BindingPointInfo> m_InputBindings;
 555     std::vector<armnn::BindingPointInfo> m_OutputBindings;
 556     bool m_EnableProfiling;
 557     std::string m_DynamicBackendsPath;
 558
 559     template<typename TContainer>
 560     armnn::InputTensors MakeInputTensors(const std::vector<TContainer>& inputDataContainers)
 561     {
 562         return armnnUtils::MakeInputTensors(m_InputBindings, inputDataContainers);
 563     }
 564
 565     template<typename TContainer>
 566     armnn::OutputTensors MakeOutputTensors(std::vector<TContainer>& outputDataContainers)
 567     {
 568         return armnnUtils::MakeOutputTensors(m_OutputBindings, outputDataContainers);
 569     }
 570
 571     std::chrono::high_resolution_clock::time_point GetCurrentTime()
 572     {
 573         return std::chrono::high_resolution_clock::now();
 574     }
 575
 576     std::chrono::duration<double, std::milli> GetTimeDuration(
 577             std::chrono::high_resolution_clock::time_point& start_time,
 578             std::chrono::high_resolution_clock::time_point& end_time)
 579     {
 580         return std::chrono::duration<double, std::milli>(end_time - start_time);
 581     }
 582
 583 };