src/armnnTfLiteParser/test/ParserFlatbuffersFixture.hpp

   1 //
   2 // Copyright © 2017 Arm Ltd. All rights reserved.
   3 // SPDX-License-Identifier: MIT
   4 //
   5
   6 #pragma once
   7
   8 #include <boost/filesystem.hpp>
   9 #include <boost/assert.hpp>
  10 #include <boost/format.hpp>
  11 #include <experimental/filesystem>
  12 #include <armnn/IRuntime.hpp>
  13 #include <armnn/TypesUtils.hpp>
  14 #include "test/TensorHelpers.hpp"
  15
  16 #include "armnnTfLiteParser/ITfLiteParser.hpp"
  17
  18 #include <backendsCommon/BackendRegistry.hpp>
  19
  20 #include "flatbuffers/idl.h"
  21 #include "flatbuffers/util.h"
  22
  23 #include <schema_generated.h>
  24 #include <iostream>
  25
  26 using armnnTfLiteParser::ITfLiteParser;
  27 using TensorRawPtr = const tflite::TensorT *;
  28
  29 struct ParserFlatbuffersFixture
  30 {
  31     ParserFlatbuffersFixture()
  32             : m_Parser(ITfLiteParser::Create()), m_NetworkIdentifier(-1)
  33     {
  34         armnn::IRuntime::CreationOptions options;
  35
  36         const armnn::BackendIdSet availableBackendIds = armnn::BackendRegistryInstance().GetBackendIds();
  37         for (auto& backendId : availableBackendIds)
  38         {
  39             m_Runtimes.push_back(std::make_pair(armnn::IRuntime::Create(options), backendId));
  40         }
  41     }
  42
  43     std::vector<uint8_t> m_GraphBinary;
  44     std::string m_JsonString;
  45     std::unique_ptr<ITfLiteParser, void (*)(ITfLiteParser *parser)> m_Parser;
  46     std::vector<std::pair<armnn::IRuntimePtr, armnn::BackendId>> m_Runtimes;
  47     armnn::NetworkId m_NetworkIdentifier;
  48
  49     /// If the single-input-single-output overload of Setup() is called, these will store the input and output name
  50     /// so they don't need to be passed to the single-input-single-output overload of RunTest().
  51     std::string m_SingleInputName;
  52     std::string m_SingleOutputName;
  53
  54     void Setup()
  55     {
  56         bool ok = ReadStringToBinary();
  57         if (!ok) {
  58             throw armnn::Exception("LoadNetwork failed while reading binary input");
  59         }
  60
  61         for (auto&& runtime : m_Runtimes)
  62         {
  63             armnn::INetworkPtr network =
  64                     m_Parser->CreateNetworkFromBinary(m_GraphBinary);
  65
  66             if (!network) {
  67                 throw armnn::Exception("The parser failed to create an ArmNN network");
  68             }
  69
  70             auto optimized = Optimize(*network,
  71                                       { runtime.second, armnn::Compute::CpuRef },
  72                                       runtime.first->GetDeviceSpec());
  73             std::string errorMessage;
  74
  75             armnn::Status ret = runtime.first->LoadNetwork(m_NetworkIdentifier,
  76                                                      move(optimized),
  77                                                      errorMessage);
  78
  79             if (ret != armnn::Status::Success)
  80             {
  81                 throw armnn::Exception(
  82                     boost::str(
  83                         boost::format("The runtime failed to load the network. "
  84                                       "Error was: %1%. in %2% [%3%:%4%]") %
  85                         errorMessage %
  86                         __func__ %
  87                         __FILE__ %
  88                         __LINE__));
  89             }
  90         }
  91     }
  92
  93     void SetupSingleInputSingleOutput(const std::string& inputName, const std::string& outputName)
  94     {
  95         // Store the input and output name so they don't need to be passed to the single-input-single-output RunTest().
  96         m_SingleInputName = inputName;
  97         m_SingleOutputName = outputName;
  98         Setup();
  99     }
 100
 101     bool ReadStringToBinary()
 102     {
 103         const char* schemafileName = getenv("ARMNN_TF_LITE_SCHEMA_PATH");
 104         if (schemafileName == nullptr)
 105         {
 106             schemafileName = ARMNN_TF_LITE_SCHEMA_PATH;
 107         }
 108         std::string schemafile;
 109
 110         bool ok = flatbuffers::LoadFile(schemafileName, false, &schemafile);
 111         BOOST_ASSERT_MSG(ok, "Couldn't load schema file " ARMNN_TF_LITE_SCHEMA_PATH);
 112         if (!ok)
 113         {
 114             return false;
 115         }
 116
 117         // parse schema first, so we can use it to parse the data after
 118         flatbuffers::Parser parser;
 119
 120         ok &= parser.Parse(schemafile.c_str());
 121         BOOST_ASSERT_MSG(ok, "Failed to parse schema file");
 122
 123         ok &= parser.Parse(m_JsonString.c_str());
 124         BOOST_ASSERT_MSG(ok, "Failed to parse json input");
 125
 126         if (!ok)
 127         {
 128             return false;
 129         }
 130
 131         {
 132             const uint8_t * bufferPtr = parser.builder_.GetBufferPointer();
 133             size_t size = static_cast<size_t>(parser.builder_.GetSize());
 134             m_GraphBinary.assign(bufferPtr, bufferPtr+size);
 135         }
 136         return ok;
 137     }
 138
 139     /// Executes the network with the given input tensor and checks the result against the given output tensor.
 140     /// This overload assumes the network has a single input and a single output.
 141     template <std::size_t NumOutputDimensions, typename DataType>
 142     void RunTest(size_t subgraphId,
 143          const std::vector<DataType>& inputData,
 144          const std::vector<DataType>& expectedOutputData);
 145
 146     /// Executes the network with the given input tensors and checks the results against the given output tensors.
 147     /// This overload supports multiple inputs and multiple outputs, identified by name.
 148     template <std::size_t NumOutputDimensions, typename DataType>
 149     void RunTest(size_t subgraphId,
 150                  const std::map<std::string, std::vector<DataType>>& inputData,
 151                  const std::map<std::string, std::vector<DataType>>& expectedOutputData);
 152
 153     void CheckTensors(const TensorRawPtr& tensors, size_t shapeSize, const std::vector<int32_t>& shape,
 154                       tflite::TensorType tensorType, uint32_t buffer, const std::string& name,
 155                       const std::vector<float>& min, const std::vector<float>& max,
 156                       const std::vector<float>& scale, const std::vector<int64_t>& zeroPoint)
 157     {
 158         BOOST_CHECK(tensors);
 159         BOOST_CHECK_EQUAL(shapeSize, tensors->shape.size());
 160         BOOST_CHECK_EQUAL_COLLECTIONS(shape.begin(), shape.end(), tensors->shape.begin(), tensors->shape.end());
 161         BOOST_CHECK_EQUAL(tensorType, tensors->type);
 162         BOOST_CHECK_EQUAL(buffer, tensors->buffer);
 163         BOOST_CHECK_EQUAL(name, tensors->name);
 164         BOOST_CHECK(tensors->quantization);
 165         BOOST_CHECK_EQUAL_COLLECTIONS(min.begin(), min.end(), tensors->quantization.get()->min.begin(),
 166                                       tensors->quantization.get()->min.end());
 167         BOOST_CHECK_EQUAL_COLLECTIONS(max.begin(), max.end(), tensors->quantization.get()->max.begin(),
 168                                       tensors->quantization.get()->max.end());
 169         BOOST_CHECK_EQUAL_COLLECTIONS(scale.begin(), scale.end(), tensors->quantization.get()->scale.begin(),
 170                                       tensors->quantization.get()->scale.end());
 171         BOOST_CHECK_EQUAL_COLLECTIONS(zeroPoint.begin(), zeroPoint.end(),
 172                                       tensors->quantization.get()->zero_point.begin(),
 173                                       tensors->quantization.get()->zero_point.end());
 174     }
 175 };
 176
 177 template <std::size_t NumOutputDimensions, typename DataType>
 178 void ParserFlatbuffersFixture::RunTest(size_t subgraphId,
 179                                        const std::vector<DataType>& inputData,
 180                                        const std::vector<DataType>& expectedOutputData)
 181 {
 182     RunTest<NumOutputDimensions, DataType>(subgraphId,
 183                                            { { m_SingleInputName, inputData } },
 184                                            { { m_SingleOutputName, expectedOutputData } });
 185 }
 186
 187 template <std::size_t NumOutputDimensions, typename DataType>
 188 void
 189 ParserFlatbuffersFixture::RunTest(size_t subgraphId,
 190                                   const std::map<std::string, std::vector<DataType>>& inputData,
 191                                   const std::map<std::string, std::vector<DataType>>& expectedOutputData)
 192 {
 193     for (auto&& runtime : m_Runtimes)
 194     {
 195         using BindingPointInfo = std::pair<armnn::LayerBindingId, armnn::TensorInfo>;
 196
 197         // Setup the armnn input tensors from the given vectors.
 198         armnn::InputTensors inputTensors;
 199         for (auto&& it : inputData)
 200         {
 201             BindingPointInfo bindingInfo = m_Parser->GetNetworkInputBindingInfo(subgraphId, it.first);
 202             armnn::VerifyTensorInfoDataType<DataType>(bindingInfo.second);
 203             inputTensors.push_back({ bindingInfo.first, armnn::ConstTensor(bindingInfo.second, it.second.data()) });
 204         }
 205
 206         // Allocate storage for the output tensors to be written to and setup the armnn output tensors.
 207         std::map<std::string, boost::multi_array<DataType, NumOutputDimensions>> outputStorage;
 208         armnn::OutputTensors outputTensors;
 209         for (auto&& it : expectedOutputData)
 210         {
 211             BindingPointInfo bindingInfo = m_Parser->GetNetworkOutputBindingInfo(subgraphId, it.first);
 212             armnn::VerifyTensorInfoDataType<DataType>(bindingInfo.second);
 213             outputStorage.emplace(it.first, MakeTensor<DataType, NumOutputDimensions>(bindingInfo.second));
 214             outputTensors.push_back(
 215                     { bindingInfo.first, armnn::Tensor(bindingInfo.second, outputStorage.at(it.first).data()) });
 216         }
 217
 218         runtime.first->EnqueueWorkload(m_NetworkIdentifier, inputTensors, outputTensors);
 219
 220         // Compare each output tensor to the expected values
 221         for (auto&& it : expectedOutputData)
 222         {
 223             BindingPointInfo bindingInfo = m_Parser->GetNetworkOutputBindingInfo(subgraphId, it.first);
 224             auto outputExpected = MakeTensor<DataType, NumOutputDimensions>(bindingInfo.second, it.second);
 225             BOOST_TEST(CompareTensors(outputExpected, outputStorage[it.first]));
 226         }
 227     }
 228 }