compiler/tflite2circle/src/CircleModel.cpp

   1 /*
   2  * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *    http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include <cassert>
  18 #include <iostream>
  19 #include <map>
  20 #include <memory>
  21
  22 #include "CircleModel.h"
  23 #include "DataLookup.h"
  24
  25 #include <mio_tflite2121/Helper.h>
  26
  27 namespace tflite2circle
  28 {
  29
  30 template <> void Offset<MetaDataBufferLink>::build(const TFLFlatBufVec *tflite_flatbuffer_vec)
  31 {
  32   if (tflite_flatbuffer_vec == nullptr)
  33     return;
  34   std::vector<int32_t> metadata_buffer_vec{tflite_flatbuffer_vec->begin(),
  35                                            tflite_flatbuffer_vec->end()};
  36   _circle_flatbuffer_vec_offset = _fb->CreateVector(metadata_buffer_vec);
  37 }
  38
  39 template <> void Offset<BufferLink>::build(const TFLFlatBufVec *tflite_flatbuffer_vec)
  40 {
  41   std::vector<flatbuffers::Offset<circle::Buffer>> buffers_vec;
  42
  43   for (auto it : *tflite_flatbuffer_vec)
  44   {
  45     flatbuffers::Offset<flatbuffers::Vector<uint8_t>> buffer_data;
  46     if (it->data())
  47     {
  48       std::vector<uint8_t> data_vec{it->data()->begin(), it->data()->end()};
  49       buffer_data = _fb->CreateVector(data_vec);
  50     }
  51     circle::BufferBuilder circle_buffer_builder{*_fb};
  52     circle_buffer_builder.add_data(buffer_data);
  53     auto circle_buffers = circle_buffer_builder.Finish();
  54     buffers_vec.emplace_back(circle_buffers);
  55   }
  56   _circle_flatbuffer_vec_offset = _fb->CreateVector(buffers_vec);
  57 }
  58
  59 template <> void Offset<SubGraphLink>::build(const TFLFlatBufVec *tflite_flatbuffer_vec)
  60 {
  61   std::vector<flatbuffers::Offset<circle::SubGraph>> subgprahs_vec;
  62
  63   int32_t subgraph_index = 0;
  64
  65   for (auto it_sg : *tflite_flatbuffer_vec)
  66   {
  67     // tensors of subgraph
  68     std::vector<flatbuffers::Offset<circle::Tensor>> tensor_vec;
  69
  70     auto tflite_tensors = it_sg->tensors();
  71     for (auto it : *tflite_tensors)
  72     {
  73       // shape
  74       flatbuffers::Offset<flatbuffers::Vector<int32_t>> shape;
  75       if (it->shape())
  76       {
  77         auto shape_vec = std::vector<int32_t>({it->shape()->begin(), it->shape()->end()});
  78         shape = _fb->CreateVector(shape_vec);
  79       }
  80       // name
  81       flatbuffers::Offset<flatbuffers::String> name;
  82       if (it->name())
  83         name = _fb->CreateString(it->name()->str());
  84       // quantization
  85       flatbuffers::Offset<circle::QuantizationParameters> quantization;
  86       if (it->quantization())
  87       {
  88         std::vector<float> tfmin;
  89         std::vector<float> tfmax;
  90         std::vector<float> tfscale;
  91         std::vector<int64_t> tfzerop;
  92         flatbuffers::Offset<flatbuffers::Vector<float>> min;
  93         flatbuffers::Offset<flatbuffers::Vector<float>> max;
  94         flatbuffers::Offset<flatbuffers::Vector<float>> scale;
  95         flatbuffers::Offset<flatbuffers::Vector<int64_t>> zero_point;
  96         int32_t quantized_dimension = it->quantization()->quantized_dimension();
  97
  98         if (it->quantization()->min() && it->quantization()->max())
  99         {
 100           auto rmin = it->quantization()->min();
 101           auto rmax = it->quantization()->max();
 102           tfmin = std::vector<float>{rmin->begin(), rmin->end()};
 103           tfmax = std::vector<float>{rmax->begin(), rmax->end()};
 104           min = _fb->CreateVector(tfmin);
 105           max = _fb->CreateVector(tfmax);
 106         }
 107
 108         if (it->quantization()->scale() && it->quantization()->zero_point())
 109         {
 110           auto rs = it->quantization()->scale();
 111           auto rz = it->quantization()->zero_point();
 112           tfscale = std::vector<float>{rs->begin(), rs->end()};
 113           tfzerop = std::vector<int64_t>{rz->begin(), rz->end()};
 114           scale = _fb->CreateVector(tfscale);
 115           zero_point = _fb->CreateVector(tfzerop);
 116         }
 117
 118         quantization = circle::CreateQuantizationParameters(*_fb, min, max, scale, zero_point,
 119                                                             circle::QuantizationDetails_NONE, 0,
 120                                                             quantized_dimension);
 121       }
 122       // is_variable
 123       bool is_variable = it->is_variable();
 124
 125       flatbuffers::Offset<circle::SparsityParameters> sparsity;
 126       // sparsity
 127       if (it->sparsity())
 128       {
 129         flatbuffers::Offset<flatbuffers::Vector<int32_t>> traversal_order;
 130         flatbuffers::Offset<flatbuffers::Vector<int32_t>> block_map;
 131         flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<circle::DimensionMetadata>>>
 132           dim_metadata;
 133
 134         // traversal_order
 135         if (it->sparsity()->traversal_order())
 136         {
 137           auto traversal_order_vec = std::vector<int32_t>{
 138             it->sparsity()->traversal_order()->begin(), it->sparsity()->traversal_order()->end()};
 139           traversal_order = _fb->CreateVector(traversal_order_vec);
 140         }
 141
 142         // block_map
 143         if (it->sparsity()->block_map())
 144         {
 145           auto block_map_vec = std::vector<int32_t>{it->sparsity()->block_map()->begin(),
 146                                                     it->sparsity()->block_map()->end()};
 147           block_map = _fb->CreateVector(block_map_vec);
 148         }
 149
 150         // dim_metadata
 151         std::vector<flatbuffers::Offset<circle::DimensionMetadata>> dim_metadata_vec;
 152         auto tflite_dim_metadata = it->sparsity()->dim_metadata();
 153         for (auto it : *tflite_dim_metadata)
 154         {
 155           // array_segments
 156           auto tflite_array_segments_type = it->array_segments_type();
 157           auto circle_array_segments =
 158             get_circle_sparse_index_vector(*_fb, it->array_segments(), tflite_array_segments_type);
 159           auto circle_array_segments_type =
 160             get_circle_sparse_index_vector_type(tflite_array_segments_type);
 161
 162           // array_indices
 163           auto tflite_array_indices_type = it->array_indices_type();
 164           auto circle_array_indices =
 165             get_circle_sparse_index_vector(*_fb, it->array_indices(), tflite_array_indices_type);
 166           auto circle_array_indices_type =
 167             get_circle_sparse_index_vector_type(tflite_array_indices_type);
 168
 169           auto circle_dim_metadata_builder = circle::DimensionMetadataBuilder{*_fb};
 170
 171           circle_dim_metadata_builder.add_format(get_circle_dimension_type(it->format()));
 172           circle_dim_metadata_builder.add_dense_size(it->dense_size());
 173           circle_dim_metadata_builder.add_array_segments(circle_array_segments);
 174           circle_dim_metadata_builder.add_array_segments_type(circle_array_segments_type);
 175           circle_dim_metadata_builder.add_array_indices(circle_array_indices);
 176           circle_dim_metadata_builder.add_array_indices_type(circle_array_indices_type);
 177           auto dim_metadata = circle_dim_metadata_builder.Finish();
 178           dim_metadata_vec.emplace_back(dim_metadata);
 179         }
 180         dim_metadata = _fb->CreateVector(dim_metadata_vec);
 181
 182         sparsity = circle::CreateSparsityParameters(*_fb, traversal_order, block_map, dim_metadata);
 183       }
 184
 185       // shape signature
 186       flatbuffers::Offset<flatbuffers::Vector<int32_t>> shape_signature;
 187       if (it->shape_signature())
 188       {
 189         auto shape_signature_vec =
 190           std::vector<int32_t>({it->shape_signature()->begin(), it->shape_signature()->end()});
 191         shape_signature = _fb->CreateVector(shape_signature_vec);
 192       }
 193
 194       circle::TensorBuilder tensor_builder{*_fb};
 195       tensor_builder.add_shape(shape);
 196       tensor_builder.add_type(get_circle_tensortype(it->type()));
 197       tensor_builder.add_buffer(it->buffer());
 198       tensor_builder.add_name(name);
 199       tensor_builder.add_quantization(quantization);
 200       tensor_builder.add_is_variable(is_variable);
 201       tensor_builder.add_sparsity(sparsity);
 202       tensor_builder.add_shape_signature(shape_signature);
 203       auto tensor = tensor_builder.Finish();
 204       tensor_vec.emplace_back(tensor);
 205     }
 206     auto circle_tensors = _fb->CreateVector(tensor_vec);
 207
 208     // inputs of subgraph
 209     auto tflite_inputs = it_sg->inputs();
 210     std::vector<int32_t> input_vec{tflite_inputs->begin(), tflite_inputs->end()};
 211
 212     // apply signature_def to input tensor index so that input orders follow like tensorflow lite
 213     // interpreter._get_full_signature_list() method, which is ordered(sorted) in name
 214     // NOTE we do not need this when circle format supports signature_def
 215     if (_tfl_signature_def_offsets != nullptr)
 216     {
 217       for (auto it_signdef : *_tfl_signature_def_offsets)
 218       {
 219         if (it_signdef->subgraph_index() == subgraph_index)
 220         {
 221           auto inputs = it_signdef->inputs();
 222           assert(inputs->size() == input_vec.size());
 223
 224           std::map<std::string, uint32_t> map_name_index;
 225           for (auto it_tm : *inputs)
 226           {
 227             map_name_index[it_tm->name()->str()] = it_tm->tensor_index();
 228           }
 229           uint32_t input_vec_idx = 0;
 230           for (auto &item : map_name_index)
 231           {
 232             input_vec[input_vec_idx++] = item.second;
 233           }
 234         }
 235       }
 236     }
 237
 238     auto circle_inputs = _fb->CreateVector(input_vec);
 239
 240     // outputs of subgraph
 241     auto tflite_outputs = it_sg->outputs();
 242     std::vector<int32_t> output_vec{tflite_outputs->begin(), tflite_outputs->end()};
 243
 244     if (_tfl_signature_def_offsets != nullptr)
 245     {
 246       // apply SignatureDef
 247       for (auto it_signdef : *_tfl_signature_def_offsets)
 248       {
 249         if (it_signdef->subgraph_index() == subgraph_index)
 250         {
 251           auto outputs = it_signdef->outputs();
 252           assert(outputs->size() == output_vec.size());
 253
 254           std::map<std::string, uint32_t> map_name_index;
 255           for (auto it_tm : *outputs)
 256           {
 257             map_name_index[it_tm->name()->str()] = it_tm->tensor_index();
 258           }
 259           uint32_t output_vec_idx = 0;
 260           for (auto &item : map_name_index)
 261           {
 262             output_vec[output_vec_idx++] = item.second;
 263           }
 264         }
 265       }
 266     }
 267
 268     auto circle_outputs = _fb->CreateVector(output_vec);
 269
 270     // operators of subgraph
 271     std::vector<flatbuffers::Offset<circle::Operator>> operator_vec;
 272
 273     auto tflite_operators = it_sg->operators();
 274     if (tflite_operators != nullptr)
 275     {
 276       for (auto it : *tflite_operators)
 277       {
 278         // inputs
 279         std::vector<int32_t> input_vec{it->inputs()->begin(), it->inputs()->end()};
 280         auto circle_inputs = _fb->CreateVector(input_vec);
 281         // outputs
 282         std::vector<int32_t> output_vec{it->outputs()->begin(), it->outputs()->end()};
 283         auto circle_outputs = _fb->CreateVector(output_vec);
 284         // builtin options
 285         auto circle_builtin_options = get_circle_builtin_options(*_fb, it);
 286         auto circle_builtin_options_type = get_circle_builtin_options_type(it);
 287         // custom options
 288         flatbuffers::Offset<flatbuffers::Vector<uint8_t>> circle_custom_options;
 289         if (it->custom_options())
 290         {
 291           std::vector<uint8_t> custom_options_vec{it->custom_options()->begin(),
 292                                                   it->custom_options()->end()};
 293           circle_custom_options = _fb->CreateVector(custom_options_vec);
 294         }
 295         // custom options format
 296         // TODO Make get_circle_custom_options_format
 297         assert(it->custom_options_format() == tflite::CustomOptionsFormat_FLEXBUFFERS);
 298         auto circle_custom_options_format = circle::CustomOptionsFormat_FLEXBUFFERS;
 299
 300         circle::OperatorBuilder operator_builder{*_fb};
 301         operator_builder.add_opcode_index(it->opcode_index());
 302         operator_builder.add_inputs(circle_inputs);
 303         operator_builder.add_outputs(circle_outputs);
 304         operator_builder.add_builtin_options(circle_builtin_options);
 305         operator_builder.add_builtin_options_type(circle_builtin_options_type);
 306         operator_builder.add_custom_options(circle_custom_options);
 307         operator_builder.add_custom_options_format(circle_custom_options_format);
 308         // TODO mutating_variable_inputs
 309         auto opeartor = operator_builder.Finish();
 310         operator_vec.emplace_back(opeartor);
 311       }
 312     }
 313     auto circle_operators = _fb->CreateVector(operator_vec);
 314
 315     // name of subgraph
 316     auto subgraphs_name = _fb->CreateString(it_sg->name());
 317
 318     // subgraphs
 319     auto circle_subgraph_builder = circle::SubGraphBuilder{*_fb};
 320
 321     circle_subgraph_builder.add_tensors(circle_tensors);
 322     circle_subgraph_builder.add_inputs(circle_inputs);
 323     circle_subgraph_builder.add_outputs(circle_outputs);
 324     circle_subgraph_builder.add_operators(circle_operators);
 325     circle_subgraph_builder.add_name(subgraphs_name);
 326     circle_subgraph_builder.add_data_format(circle::DataFormat_CHANNELS_LAST);
 327
 328     auto circle_subgraph = circle_subgraph_builder.Finish();
 329     subgprahs_vec.emplace_back(circle_subgraph);
 330
 331     // next subgraph
 332     subgraph_index = subgraph_index + 1;
 333   }
 334   _circle_flatbuffer_vec_offset = _fb->CreateVector(subgprahs_vec);
 335 }
 336
 337 template <> void Offset<OperatorCodeLink>::build(const TFLFlatBufVec *tflite_flatbuffer_vec)
 338 {
 339   std::vector<flatbuffers::Offset<circle::OperatorCode>> operator_code_vec;
 340
 341   for (auto it : *tflite_flatbuffer_vec)
 342   {
 343     auto custom_code = _fb->CreateString(it->custom_code());
 344     circle::OperatorCodeBuilder operator_code_builder{*_fb};
 345     auto de_code = it->deprecated_builtin_code();
 346     auto bt_code = it->builtin_code();
 347
 348     // There are two builtin codes (deprecated_builtin, (extended) builtin)
 349     // deprecated builtin code uses 0~126
 350     // extended builtin code uses 127~
 351     // NOTE 127 = BuiltinOperator_PLACEHOLDER_FOR_GREATER_OP_CODES
 352     if (de_code >= 0 and de_code < 127)
 353     {
 354       // Use deprecated builtin opcode.
 355       auto cir_de_code = get_circle_builtin_code(de_code);
 356       auto cir_bt_code = get_circle_builtin_code(bt_code);
 357       // correct bt_code where bt_code == 0 for old tflite format
 358       if (cir_bt_code == 0)
 359         cir_bt_code = static_cast<circle::BuiltinOperator>(cir_de_code);
 360       operator_code_builder.add_deprecated_builtin_code(cir_de_code);
 361       operator_code_builder.add_builtin_code(cir_bt_code);
 362     }
 363     else
 364     {
 365       // Use extended builtin opcode
 366       // Set 127 (PLACEHOLDER_FOR_GREATER_OP_CODES) for deprecated builtin code
 367       auto cir_bt_code = get_circle_builtin_code(bt_code);
 368       operator_code_builder.add_deprecated_builtin_code(
 369         tflite::BuiltinOperator_PLACEHOLDER_FOR_GREATER_OP_CODES);
 370       operator_code_builder.add_builtin_code(cir_bt_code);
 371     }
 372     operator_code_builder.add_custom_code(custom_code);
 373     operator_code_builder.add_version(it->version());
 374     auto code = operator_code_builder.Finish();
 375     operator_code_vec.emplace_back(code);
 376   }
 377   _circle_flatbuffer_vec_offset = _fb->CreateVector(operator_code_vec);
 378 }
 379
 380 CircleModel::CircleModel(FlatBufBuilder &fb)
 381   : _version{0}, _description{fb->CreateString("ONE-tflite2circle")}, _fb{fb}
 382 {
 383   // NOTHING TODO
 384 }
 385
 386 void CircleModel::load_offsets(const tflite::Model *tfl_model)
 387 {
 388   _operator_codes_offset = std::make_unique<Offset<OperatorCodeLink>>(_fb);
 389   _subGraphs_offset = std::make_unique<Offset<SubGraphLink>>(_fb);
 390   _buffers_offset = std::make_unique<Offset<BufferLink>>(_fb);
 391   _metadata_buffer_offset = std::make_unique<Offset<MetaDataBufferLink>>(_fb);
 392
 393   _subGraphs_offset->set_signature_defs(tfl_model->signature_defs());
 394
 395   _operator_codes_offset->build(tfl_model->operator_codes());
 396   _subGraphs_offset->build(tfl_model->subgraphs());
 397   _buffers_offset->build(tfl_model->buffers());
 398   _metadata_buffer_offset->build(tfl_model->metadata_buffer());
 399 }
 400
 401 void CircleModel::model_build(void) const
 402 {
 403   circle::ModelBuilder model_builder{*_fb};
 404
 405   model_builder.add_version(_version);
 406   model_builder.add_description(_description);
 407   model_builder.add_operator_codes(_operator_codes_offset->offset());
 408   model_builder.add_subgraphs(_subGraphs_offset->offset());
 409   model_builder.add_buffers(_buffers_offset->offset());
 410   model_builder.add_metadata_buffer(_metadata_buffer_offset->offset());
 411
 412   auto model = model_builder.Finish();
 413   circle::FinishModelBuffer(*_fb, model);
 414 }
 415
 416 const char *CircleModel::base(void) const
 417 {
 418   return reinterpret_cast<const char *>(_fb->GetBufferPointer());
 419 }
 420
 421 size_t CircleModel::size(void) const { return _fb->GetSize(); }
 422
 423 } // namespace tflite2circle