runtime/libs/tflite/src/Diff.cpp

   1 /*
   2  * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *    http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include "tflite/Diff.h"
  18
  19 #include "misc/fp32.h"
  20
  21 #include "misc/tensor/IndexFormatter.h"
  22 #include "misc/tensor/Zipper.h"
  23 #include "misc/tensor/Comparator.h"
  24
  25 #include <iostream>
  26 #include <cassert>
  27
  28 class DiffSummary : public nnfw::misc::tensor::Comparator::Observer
  29 {
  30 public:
  31   DiffSummary()
  32       : max_abs_diff_index(0), max_abs_diff_expected{0.0f}, max_abs_diff_obtained{0.0f},
  33         max_abs_diff_value{0.0f}, max_rel_diff_index(0), max_rel_diff_expected{0.0f},
  34         max_rel_diff_obtained{0.0f}, max_rel_diff_value{0.0f}
  35   {
  36     // DO NOTHING
  37   }
  38
  39 public:
  40   void notify(const nnfw::misc::tensor::Index &index, float expected, float obtained) override;
  41
  42 public:
  43   nnfw::misc::tensor::Index max_abs_diff_index;
  44   float max_abs_diff_expected;
  45   float max_abs_diff_obtained;
  46   float max_abs_diff_value;
  47
  48   nnfw::misc::tensor::Index max_rel_diff_index;
  49   float max_rel_diff_expected;
  50   float max_rel_diff_obtained;
  51   float max_rel_diff_value;
  52 };
  53
  54 void DiffSummary::notify(const nnfw::misc::tensor::Index &index, float expected, float obtained)
  55 {
  56   const auto abs_diff_value = std::fabs(expected - obtained);
  57
  58   if (max_abs_diff_value < abs_diff_value)
  59   {
  60     max_abs_diff_index = index;
  61     max_abs_diff_value = abs_diff_value;
  62     max_abs_diff_expected = expected;
  63     max_abs_diff_obtained = obtained;
  64   }
  65
  66   const auto rel_diff_value = nnfw::misc::fp32::relative_diff(expected, obtained);
  67
  68   if (max_rel_diff_value < rel_diff_value)
  69   {
  70     max_rel_diff_index = index;
  71     max_rel_diff_value = rel_diff_value;
  72     max_rel_diff_expected = expected;
  73     max_rel_diff_obtained = obtained;
  74   }
  75 }
  76
  77 template <typename T>
  78 bool TfLiteInterpMatchApp::compareSingleTensorView(const nnfw::tflite::TensorView<T> &expected,
  79                                                    const nnfw::tflite::TensorView<T> &obtained,
  80                                                    int id) const
  81 {
  82   std::vector<nnfw::misc::tensor::Diff<T>> diffs;
  83   assert(expected.shape() == obtained.shape());
  84
  85   using nnfw::misc::tensor::Index;
  86   using nnfw::misc::tensor::zip;
  87
  88   zip(expected.shape(), expected, obtained)
  89       << [&](const Index &index, T expected_value, T obtained_value) {
  90            if (expected_value != obtained_value)
  91            {
  92              diffs.emplace_back(index, expected_value, obtained_value);
  93            }
  94          };
  95
  96   // TODO Unify summary generation code
  97   if (diffs.size() == 0)
  98   {
  99     std::cout << "  Tensor #" << id << ": MATCHED" << std::endl;
 100   }
 101   else
 102   {
 103     std::cout << "  Tensor #" << id << ": UNMATCHED" << std::endl;
 104     std::cout << "    " << diffs.size() << " diffs are detected" << std::endl;
 105   }
 106
 107   if (diffs.size() > 0 && _verbose != 0)
 108   {
 109     std::cout << "    ---- Details ---" << std::endl;
 110     for (const auto &diff : diffs)
 111     {
 112       std::cout << "    Diff at [" << nnfw::misc::tensor::IndexFormatter(diff.index) << "]"
 113                 << std::endl;
 114       std::cout << "      expected: " << diff.expected << std::endl;
 115       std::cout << "      obtained: " << diff.obtained << std::endl;
 116     }
 117   }
 118
 119   return diffs.size() == 0;
 120 }
 121
 122 template <>
 123 bool TfLiteInterpMatchApp::compareSingleTensorView<float>(
 124     const nnfw::tflite::TensorView<float> &expected,
 125     const nnfw::tflite::TensorView<float> &obtained, int id) const
 126 {
 127   DiffSummary summary;
 128
 129   assert(expected.shape() == obtained.shape());
 130   auto diffs = _comparator.compare(expected.shape(), expected, obtained, &summary);
 131
 132   // TODO Unify summary generation code
 133   if (diffs.size() == 0)
 134   {
 135     std::cout << "  Tensor #" << id << ": MATCHED" << std::endl;
 136   }
 137   else
 138   {
 139     std::cout << "  Tensor #" << id << ": UNMATCHED" << std::endl;
 140     std::cout << "    " << diffs.size() << " diffs are detected" << std::endl;
 141   }
 142
 143   // Print out max_diff
 144   if (summary.max_abs_diff_value > 0)
 145   {
 146     std::cout << "    Max absolute diff at ["
 147               << nnfw::misc::tensor::IndexFormatter(summary.max_abs_diff_index) << "]" << std::endl;
 148     std::cout << "       expected: " << summary.max_abs_diff_expected << std::endl;
 149     std::cout << "       obtained: " << summary.max_abs_diff_obtained << std::endl;
 150     std::cout << "       absolute diff: " << summary.max_abs_diff_value << std::endl;
 151   }
 152
 153   if (summary.max_rel_diff_value > 0)
 154   {
 155     const auto tolerance_level = summary.max_rel_diff_value / FLT_EPSILON;
 156
 157     std::cout << "    Max relative diff at ["
 158               << nnfw::misc::tensor::IndexFormatter(summary.max_rel_diff_index) << "]" << std::endl;
 159     std::cout << "       expected: " << summary.max_rel_diff_expected << std::endl;
 160     std::cout << "       obtained: " << summary.max_rel_diff_obtained << std::endl;
 161     std::cout << "       relative diff: " << summary.max_rel_diff_value << std::endl;
 162     std::cout << "         (tolerance level = " << tolerance_level << ")" << std::endl;
 163   }
 164
 165   if (diffs.size() > 0)
 166   {
 167     if (_verbose != 0)
 168     {
 169       std::cout << "    ---- Details ---" << std::endl;
 170       for (const auto &diff : diffs)
 171       {
 172         const auto absolute_diff = std::fabs(diff.expected - diff.obtained);
 173         const auto relative_diff = nnfw::misc::fp32::relative_diff(diff.expected, diff.obtained);
 174         const auto tolerance_level = relative_diff / FLT_EPSILON;
 175
 176         std::cout << "    Diff at [" << nnfw::misc::tensor::IndexFormatter(diff.index) << "]"
 177                   << std::endl;
 178         std::cout << "      expected: " << diff.expected << std::endl;
 179         std::cout << "      obtained: " << diff.obtained << std::endl;
 180         std::cout << "      absolute diff: " << absolute_diff << std::endl;
 181         std::cout << "      relative diff: " << relative_diff << std::endl;
 182         std::cout << "         (tolerance level = " << tolerance_level << ")" << std::endl;
 183       }
 184     }
 185
 186     return false;
 187   }
 188   return true;
 189 }
 190
 191 #include <map>
 192
 193 bool TfLiteInterpMatchApp::run(::tflite::Interpreter &interp, ::tflite::Interpreter &nnapi) const
 194 {
 195   assert(interp.outputs() == nnapi.outputs());
 196
 197   bool all_matched = true;
 198
 199   using Comparator = std::function<bool(int id, ::tflite::Interpreter &, ::tflite::Interpreter &)>;
 200
 201   std::map<TfLiteType, Comparator> comparators;
 202
 203   comparators[kTfLiteUInt8] = [this](int id, ::tflite::Interpreter &interp,
 204                                      ::tflite::Interpreter &nnapi) {
 205     const auto expected = nnfw::tflite::TensorView<uint8_t>::make(interp, id);
 206     const auto obtained = nnfw::tflite::TensorView<uint8_t>::make(nnapi, id);
 207
 208     return compareSingleTensorView(expected, obtained, id);
 209   };
 210
 211   comparators[kTfLiteInt32] = [this](int id, ::tflite::Interpreter &interp,
 212                                      ::tflite::Interpreter &nnapi) {
 213     const auto expected = nnfw::tflite::TensorView<int32_t>::make(interp, id);
 214     const auto obtained = nnfw::tflite::TensorView<int32_t>::make(nnapi, id);
 215
 216     return compareSingleTensorView(expected, obtained, id);
 217   };
 218
 219   comparators[kTfLiteFloat32] = [this](int id, ::tflite::Interpreter &interp,
 220                                        ::tflite::Interpreter &nnapi) {
 221     const auto expected = nnfw::tflite::TensorView<float>::make(interp, id);
 222     const auto obtained = nnfw::tflite::TensorView<float>::make(nnapi, id);
 223
 224     return compareSingleTensorView(expected, obtained, id);
 225   };
 226
 227   comparators[kTfLiteBool] = [this](int id, ::tflite::Interpreter &interp,
 228                                     ::tflite::Interpreter &nnapi) {
 229     const auto expected = nnfw::tflite::TensorView<bool>::make(interp, id);
 230     const auto obtained = nnfw::tflite::TensorView<bool>::make(nnapi, id);
 231
 232     return compareSingleTensorView(expected, obtained, id);
 233   };
 234
 235   for (const auto &id : interp.outputs())
 236   {
 237     assert(interp.tensor(id)->type == nnapi.tensor(id)->type);
 238
 239     auto it = comparators.find(interp.tensor(id)->type);
 240
 241     if (it == comparators.end())
 242     {
 243       throw std::runtime_error{"Not supported output type"};
 244     }
 245
 246     const auto &comparator = it->second;
 247
 248     if (!comparator(id, interp, nnapi))
 249     {
 250       all_matched = false;
 251     }
 252   }
 253
 254   return all_matched;
 255 }