2 * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "exec/DynamicShapeInference.h"
18 #include "util/ShapeInference.h"
26 inline backend::IDynamicTensorManager *
27 dynamicTensorManagerOf(const std::shared_ptr<backend::ITensor> &tensor)
29 if (!tensor->dynamic_tensor_manager())
30 throw std::runtime_error{"Dynamic Tensor Manager is not available for this tensor."};
31 return tensor->dynamic_tensor_manager();
34 void DynamicShapeInferer::handleBinaryArithmeticOp(const ir::Operation &op,
35 const ir::OperandIndex lhs_idx,
36 const ir::OperandIndex rhs_idx)
38 auto lhs = _tensor_registry->getITensor(lhs_idx);
39 auto lhs_shape = lhs->getShape();
41 auto rhs = _tensor_registry->getITensor(rhs_idx);
42 auto rhs_shape = rhs->getShape();
45 Here, the state after compilation (satic shape inference) could be one of the following:
47 lhs rhs output execution-time shape inf required
48 ------------------------------------------ ---------------------------------
49 case 1) static static static X
50 case 2) one or both are dynamic dynamic O
52 Then nnfw_apply_tensorinf() could change one or both inputs dynamic.
53 So, in this method, we have one more state and we have to re-calculate shape for this shape.
55 case 3) one or both are dynamic static O
57 So, only when all inputs are static, we can skip dynamic shape inference.
59 if ((!lhs->is_dynamic()) && (!rhs->is_dynamic()))
62 auto output_idx = op.getOutputs().at(0);
63 auto output = _tensor_registry->getITensor(output_idx);
65 ir::Shape new_shape = shape_inference::inferEltwiseShape(lhs_shape, rhs_shape);
67 dynamicTensorManagerOf(output)->applyShape(output_idx, new_shape);
68 assert(output->buffer() != nullptr);
71 void DynamicShapeInferer::handleSimpleUnaryOp(const ir::Operation &op,
72 const ir::OperandIndex input_ind)
74 // check if input is not dynamic
75 auto input = _tensor_registry->getITensor(input_ind);
76 auto output_shape = input->getShape();
79 Here, the state after compilation (satic shape inference) could be one of the following:
81 input output execution-time shape inf required
82 ------------------------- ---------------------------------
83 case 1) static static X
84 case 2) dynamic dynamic O
86 Then nnfw_apply_tensorinf() could change input dynamic.
87 So, in this method, we have one more state and we have to re-calculate shape for this shape.
89 case 3) dynamic static O
91 So, only when input is static, we can skip dynamic shape inference.
93 if (!input->is_dynamic())
96 auto output_ind = op.getOutputs().at(0);
97 auto output = _tensor_registry->getITensor(output_ind);
99 dynamicTensorManagerOf(output)->applyShape(output_ind, output_shape);
100 assert(output->buffer() != nullptr);
103 void DynamicShapeInferer::visit(const ir::operation::Abs &op)
105 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::Abs::INPUT));
108 void DynamicShapeInferer::visit(const ir::operation::Add &op)
110 handleBinaryArithmeticOp(op, op.getInputs().at(ir::operation::Add::Input::LHS),
111 op.getInputs().at(ir::operation::Add::Input::RHS));
114 void DynamicShapeInferer::visit(const ir::operation::ArgMax &op)
116 const auto input_idx{op.getInputs().at(ir::operation::ArgMax::Input::INPUT)};
117 const auto &input = _tensor_registry->getITensor(input_idx);
118 auto input_shape = input->getShape();
120 if (!input->is_dynamic())
123 const auto rank = input_shape.rank();
124 const auto axis = ((op.param().axis < 0) ? rank + op.param().axis : op.param().axis);
126 assert(0 <= axis && axis < rank);
128 auto output_ind = op.getOutputs().at(0);
129 auto output = _tensor_registry->getITensor(output_ind);
131 ir::Shape new_shape = shape_inference::inferArgMaxShape(input_shape, axis, rank);
133 dynamicTensorManagerOf(output)->applyShape(output_ind, new_shape);
134 assert(output->buffer() != nullptr);
137 void DynamicShapeInferer::visit(const ir::operation::BatchMatMul &op)
139 const auto lhs_index = op.getInputs().at(ir::operation::BatchMatMul::Input::LHS);
140 const auto rhs_index = op.getInputs().at(ir::operation::BatchMatMul::Input::RHS);
141 auto lhs = _tensor_registry->getITensor(lhs_index);
142 auto rhs = _tensor_registry->getITensor(rhs_index);
144 if (!lhs->is_dynamic() && !rhs->is_dynamic())
147 const auto output_index = op.getOutputs().at(0);
148 auto output = _tensor_registry->getITensor(output_index);
150 auto lhs_shape = lhs->getShape();
151 auto rhs_shape = rhs->getShape();
154 auto new_shape = shape_inference::inferBatchMatMulShape(lhs_shape, rhs_shape, op.param());
155 dynamicTensorManagerOf(output)->applyShape(output_index, new_shape);
158 void DynamicShapeInferer::visit(const ir::operation::BroadcastTo &op)
160 auto output_ind = op.getOutputs().at(0);
161 auto output = _tensor_registry->getITensor(output_ind);
163 auto input_idx = op.getInputs().at(ir::operation::BroadcastTo::INPUT);
164 auto input = _tensor_registry->getITensor(input_idx);
166 if ((!input->is_dynamic()) && (!output->is_dynamic()))
169 auto shape_idx = op.getInputs().at(ir::operation::Tile::Input::MULTIPLES);
170 const auto &shape = _tensor_registry->getITensor(shape_idx);
172 assert(shape); // It shouldn't be 0.
174 auto output_shape = shape_inference::inferBroadcastToShape(
175 shape->getShape(), reinterpret_cast<const int32_t *>(shape->buffer()));
177 // set output shape and output buffer
178 dynamicTensorManagerOf(output)->applyShape(output_ind, output_shape);
179 assert(output->buffer() != nullptr);
182 void DynamicShapeInferer::visit(const ir::operation::Cast &op)
184 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::Cast::INPUT));
187 void DynamicShapeInferer::visit(const ir::operation::Comparison &op)
189 handleBinaryArithmeticOp(op, op.getInputs().at(ir::operation::Comparison::Input::INPUT0),
190 op.getInputs().at(ir::operation::Comparison::Input::INPUT1));
193 void DynamicShapeInferer::visit(const ir::operation::Concat &op)
196 The state after compilation (satic shape inference) could be one of the following:
198 inputs output execution-time shape inf required
199 ------------------------------------------ ---------------------------------
200 case 1) all static static X
201 case 2) at least on is dynamic dynamic O
203 Then nnfw_apply_tensorinf() could change one or both inputs dynamic.
204 So, in this method, we have one more state and we have to re-calculate shape for this shape.
206 case 3) at least on is dynamic static O
208 So, only when all inputs are static, we can skip dynamic shape inference.
210 bool all_static = true;
211 for (auto input_ind : op.getInputs())
213 auto input = _tensor_registry->getITensor(input_ind);
214 if (input->is_dynamic())
226 auto isConcatible = [](const backend::ITensor *input1, const backend::ITensor *input2,
228 if (input1->num_dimensions() != input2->num_dimensions())
231 for (size_t i = 0; i < input1->num_dimensions(); i++)
233 auto positive_axis = (axis >= 0) ? axis : axis + input1->num_dimensions();
235 if (i != positive_axis)
236 if (input1->dimension(i) != input2->dimension(i))
243 auto first_input_ind = op.getInputs().at(0);
244 auto first_input = _tensor_registry->getITensor(first_input_ind);
246 for (auto input_ind : op.getInputs())
248 auto input = _tensor_registry->getITensor(input_ind);
249 if (input != first_input && !isConcatible(first_input.get(), input.get(), op.param().axis))
250 throw std::runtime_error("input shapes does not matched for concat");
254 // getting output shape
255 onert::shape_inference::Shapes in_shapes;
256 for (auto input_ind : op.getInputs())
258 auto input = _tensor_registry->getITensor(input_ind);
259 ir::Shape shape = input->getShape();
261 in_shapes.emplace_back(shape);
264 auto output_ind = op.getOutputs().at(0);
265 auto output = _tensor_registry->getITensor(output_ind);
266 auto output_shape = shape_inference::inferConcatShape(in_shapes, op.param());
268 dynamicTensorManagerOf(output)->applyShape(output_ind, output_shape);
271 void DynamicShapeInferer::visit(const ir::operation::Conv2D &op)
273 // check if input is not dynamic
274 auto input_ind = op.getInputs().at(ir::operation::Conv2D::INPUT);
275 auto input = _tensor_registry->getITensor(input_ind);
277 auto ker_ind = op.getInputs().at(ir::operation::Conv2D::KERNEL);
278 auto ker = _tensor_registry->getITensor(ker_ind);
280 if ((!input->is_dynamic()) && (!ker->is_dynamic()))
283 ir::Shape input_shape = input->getShape();
284 ir::Shape ker_shape = ker->getShape();
286 auto output_ind = op.getOutputs().at(0);
287 auto output = _tensor_registry->getITensor(output_ind);
289 ir::Shape output_shape = shape_inference::inferConv2DShape(input_shape, ker_shape, op.param());
291 dynamicTensorManagerOf(output)->applyShape(output_ind, output_shape);
292 assert(output->buffer() != nullptr);
295 void DynamicShapeInferer::visit(const ir::operation::Cos &op)
297 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::Cos::Input::INPUT));
300 void DynamicShapeInferer::visit(const ir::operation::Div &op)
302 handleBinaryArithmeticOp(op, op.getInputs().at(ir::operation::Div::Input::LHS),
303 op.getInputs().at(ir::operation::Div::Input::RHS));
306 void DynamicShapeInferer::visit(const ir::operation::Exp &op)
308 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::Exp::Input::INPUT));
311 void DynamicShapeInferer::visit(const ir::operation::ExpandDims &op)
313 // check if input is not dynamic
314 auto input_ind = op.getInputs().at(ir::operation::ExpandDims::INPUT);
315 auto input = _tensor_registry->getITensor(input_ind);
317 // check if output is not dynamic, meaning when 1st input is static and 2nd input is const
318 auto output_ind = op.getOutputs().at(0);
319 auto output = _tensor_registry->getITensor(output_ind);
322 Here, the state after compilation (satic shape inference) could be one of the following:
324 input1 input2 output execution-time shape inf required
325 ----------------------------- --------------------------------
326 case 1) static const static X
327 case 2) static placeholder dynamic O
328 case 3) dynamic const dynamic O
329 case 4) dynamic placeholder dynamic O
331 Then nnfw_apply_tensorinf() could change input dynamic.
332 So, in this method, we could have one more state and we have to re-calculate shape
335 case 5) dynamic const static O
337 So, only when input1 and ouput are static, we can skip dynamic shape inference.
339 if ((!input->is_dynamic()) && (!output->is_dynamic()))
342 ir::Shape input_shape = input->getShape();
344 auto axis_ind = op.getInputs().at(ir::operation::ExpandDims::AXIS);
345 auto axis = _tensor_registry->getITensor(axis_ind);
346 auto axis_buf = reinterpret_cast<const int32_t *>(axis->buffer());
349 auto output_shape = shape_inference::inferExpandDimsShape(input_shape, axis_buf[0]);
351 dynamicTensorManagerOf(output)->applyShape(output_ind, output_shape);
352 assert(output->buffer() != nullptr);
355 void DynamicShapeInferer::visit(const ir::operation::Fill &op)
357 // check if output is not dynamic
358 auto output_ind = op.getOutputs().at(0);
359 auto output = _tensor_registry->getITensor(output_ind);
360 auto input_ind = op.getInputs().at(ir::operation::Fill::Input::INPUT);
361 auto input = _tensor_registry->getITensor(input_ind);
362 ir::Shape input_shape = input->getShape();
364 if ((!input->is_dynamic()) && (!output->is_dynamic()))
367 assert(input.get()->data_type() == ir::DataType::INT32);
369 auto input_buf = reinterpret_cast<const int32_t *>(input->buffer());
372 auto output_shape = shape_inference::inferFillShape(input_shape, input_buf);
374 dynamicTensorManagerOf(output)->applyShape(output_ind, output_shape);
375 assert(output->buffer() != nullptr);
378 void DynamicShapeInferer::visit(const ir::operation::FullyConnected &op)
380 const auto input_idx{op.getInputs().at(ir::operation::FullyConnected::Input::INPUT)};
381 const auto &input = _tensor_registry->getITensor(input_idx);
383 const auto ker_idx{op.getInputs().at(ir::operation::FullyConnected::Input::WEIGHT)};
384 const auto &ker = _tensor_registry->getITensor(ker_idx);
386 if (!input->is_dynamic() && !ker->is_dynamic())
389 auto input_shape = input->getShape();
390 auto ker_shape = ker->getShape();
392 ir::Shape new_shape = shape_inference::inferFullyConnectedShape(input_shape, ker_shape);
394 auto output_ind = op.getOutputs().at(0);
395 auto output = _tensor_registry->getITensor(output_ind);
397 dynamicTensorManagerOf(output)->applyShape(output_ind, new_shape);
398 assert(output->buffer() != nullptr);
401 void DynamicShapeInferer::visit(const ir::operation::FusedBatchNorm &op)
403 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::FusedBatchNorm::Input::INPUT));
406 void DynamicShapeInferer::visit(const ir::operation::Gather &op)
408 const auto input_idx{op.getInputs().at(ir::operation::Gather::Input::INPUT)};
409 const auto &input = _tensor_registry->getITensor(input_idx);
410 auto input_shape = input->getShape();
412 const auto indices_idx{op.getInputs().at(ir::operation::Gather::Input::INDICES)};
413 const auto &indices = _tensor_registry->getITensor(indices_idx);
414 auto indices_shape = indices->getShape();
416 if (!(input->is_dynamic()) && !(indices->is_dynamic()))
419 const auto rank = input_shape.rank();
420 const auto axis = ((op.param().axis < 0) ? rank + op.param().axis : op.param().axis);
422 assert(0 <= axis && axis < rank);
424 ir::Shape new_shape = shape_inference::inferGatherShape(input_shape, indices_shape, axis, rank);
426 auto output_ind = op.getOutputs().at(0);
427 auto output = _tensor_registry->getITensor(output_ind);
429 dynamicTensorManagerOf(output)->applyShape(output_ind, new_shape);
430 assert(output->buffer() != nullptr);
433 void DynamicShapeInferer::visit(const ir::operation::Log &op)
435 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::Log::Input::INPUT));
438 void DynamicShapeInferer::visit(const ir::operation::LogicalNot &op)
440 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::LogicalNot::Input::INPUT));
443 void DynamicShapeInferer::visit(const ir::operation::LogicalOr &op)
445 handleBinaryArithmeticOp(op, op.getInputs().at(ir::operation::LogicalOr::Input::INPUT0),
446 op.getInputs().at(ir::operation::LogicalOr::Input::INPUT1));
449 void DynamicShapeInferer::visit(const ir::operation::Logistic &op)
451 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::Logistic::INPUT));
454 void DynamicShapeInferer::visit(const ir::operation::L2Normalization &op)
456 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::L2Normalization::INPUT));
459 void DynamicShapeInferer::visit(const ir::operation::MatrixBandPart &op)
461 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::MatrixBandPart::INPUT));
464 void DynamicShapeInferer::visit(const ir::operation::Max &op)
466 handleBinaryArithmeticOp(op, op.getInputs().at(ir::operation::Max::Input::LHS),
467 op.getInputs().at(ir::operation::Max::Input::RHS));
470 void DynamicShapeInferer::visit(const ir::operation::Min &op)
472 handleBinaryArithmeticOp(op, op.getInputs().at(ir::operation::Min::Input::LHS),
473 op.getInputs().at(ir::operation::Min::Input::RHS));
476 void DynamicShapeInferer::visit(const ir::operation::Mul &op)
478 handleBinaryArithmeticOp(op, op.getInputs().at(ir::operation::Mul::Input::LHS),
479 op.getInputs().at(ir::operation::Mul::Input::RHS));
482 void DynamicShapeInferer::visit(const ir::operation::Neg &op)
484 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::Neg::Input::INPUT));
487 void DynamicShapeInferer::visit(const ir::operation::OneHot &op)
489 auto output_ind = op.getOutputs().at(0);
490 auto output = _tensor_registry->getITensor(output_ind);
492 auto indices_ind = op.getInputs().at(ir::operation::OneHot::INDICES);
493 const auto &indices = _tensor_registry->getITensor(indices_ind);
494 auto indices_shape = indices->getShape();
496 auto depth_ind = op.getInputs().at(ir::operation::OneHot::DEPTH);
497 const auto &depth = _tensor_registry->getITensor(depth_ind);
499 if (!indices->is_dynamic() && !depth->is_dynamic())
504 int32_t *depth_buf = reinterpret_cast<int32_t *>(depth->buffer());
506 const auto axis_val = op.param().axis;
508 ir::Shape new_shape = shape_inference::inferOnehotShape(indices_shape, *depth_buf, axis_val);
509 dynamicTensorManagerOf(output)->applyShape(output_ind, new_shape);
510 assert(output->buffer() != nullptr);
513 void DynamicShapeInferer::visit(const ir::operation::Pack &op)
515 bool is_any_of_inputs_dynamic = [&]() -> bool {
516 for (uint32_t i = 0; i < op.getInputs().size(); ++i)
518 const auto &input = _tensor_registry->getITensor(op.getInputs().at(i));
519 if (input->is_dynamic())
527 const auto input_idx{op.getInputs().at(0)};
528 const auto &input = _tensor_registry->getITensor(input_idx);
529 auto input_shape = input->getShape();
531 auto output_ind = op.getOutputs().at(0);
532 auto output = _tensor_registry->getITensor(output_ind);
534 if (!is_any_of_inputs_dynamic && !output->is_dynamic())
537 const auto rank = input_shape.rank() + 1;
538 const auto axis = ((op.param().axis < 0) ? rank + op.param().axis : op.param().axis);
539 const auto num = op.param().num;
541 assert(0 <= axis && axis < rank);
543 ir::Shape new_shape = shape_inference::inferPackShape(input_shape, axis, rank, num);
545 dynamicTensorManagerOf(output)->applyShape(output_ind, new_shape);
546 assert(output->buffer() != nullptr);
549 void DynamicShapeInferer::visit(const ir::operation::Pad &op)
551 // check if output is not dynamic
552 auto output_ind = op.getOutputs().at(0);
553 auto output = _tensor_registry->getITensor(output_ind);
555 auto input_ind = op.getInputs().at(ir::operation::Pad::Input::INPUT);
556 auto input = _tensor_registry->getITensor(input_ind);
558 auto pad_ind = op.getInputs().at(ir::operation::Pad::Input::PAD);
559 auto pad = _tensor_registry->getITensor(pad_ind);
561 // check if input and output are not dynamic
562 if ((!input->is_dynamic()) && (!output->is_dynamic()))
565 int32_t *pad_buf = reinterpret_cast<int32_t *>(pad->buffer());
569 shape_inference::inferPadShape(input->getShape(), pad_buf, pad->getShape().num_elements());
571 // change output shape and reallocate output tensor memory
572 dynamicTensorManagerOf(output)->applyShape(output_ind, output_shape);
573 assert(output->buffer() != nullptr);
576 void DynamicShapeInferer::visit(const ir::operation::Permute & /* op */)
578 // NOTE Permute is a special operation which does not do shape inference before the actual
579 // function(kernel) execution. Shape inference and output allocation will be done in the kernel
580 // on-the-fly, as it must support inter-backend inference/allocation.
583 void DynamicShapeInferer::visit(const ir::operation::Pow &op)
585 handleBinaryArithmeticOp(op, op.getInputs().at(ir::operation::Pow::Input::LHS),
586 op.getInputs().at(ir::operation::Pow::Input::RHS));
589 void DynamicShapeInferer::visit(const ir::operation::Range &op)
591 // check if output is not dynamic
592 auto output_ind = op.getOutputs().at(0);
593 auto output = _tensor_registry->getITensor(output_ind);
595 // from op, access the buffer of second input to read new shape
596 auto start_idx = op.getInputs().at(ir::operation::Range::Input::START);
597 auto start_tensor = _tensor_registry->getITensor(start_idx);
599 auto limit_idx = op.getInputs().at(ir::operation::Range::Input::LIMIT);
600 auto limit_tensor = _tensor_registry->getITensor(limit_idx);
602 auto delta_idx = op.getInputs().at(ir::operation::Range::Input::DELTA);
603 auto delta_tensor = _tensor_registry->getITensor(delta_idx);
605 if (!start_tensor->is_dynamic() && !limit_tensor->is_dynamic() && !delta_tensor->is_dynamic() &&
606 !output->is_dynamic())
610 if (output->data_type() == ir::DataType::FLOAT32)
613 shape_inference::inferRangeShape<float>(*reinterpret_cast<float *>(start_tensor->buffer()),
614 *reinterpret_cast<float *>(limit_tensor->buffer()),
615 *reinterpret_cast<float *>(delta_tensor->buffer()));
617 else if (output->data_type() == ir::DataType::INT32)
619 new_shape = shape_inference::inferRangeShape<int32_t>(
620 *reinterpret_cast<int32_t *>(start_tensor->buffer()),
621 *reinterpret_cast<int32_t *>(limit_tensor->buffer()),
622 *reinterpret_cast<int32_t *>(delta_tensor->buffer()));
624 dynamicTensorManagerOf(output)->applyShape(output_ind, new_shape);
625 assert(output->buffer() != nullptr);
628 void DynamicShapeInferer::visit(const ir::operation::Reduce &op)
630 const auto input_idx{op.getInputs().at(ir::operation::Reduce::Input::INPUT)};
631 const auto &input = _tensor_registry->getITensor(input_idx);
632 auto input_shape = input->getShape();
634 const auto axes_idx{op.getInputs().at(ir::operation::Reduce::Input::AXES)};
635 const auto &axes = _tensor_registry->getITensor(axes_idx);
637 if (!input->is_dynamic())
640 std::vector<int32_t> axes_vec;
641 for (uint32_t i = 0; i < axes->getShape().num_elements(); ++i)
643 const auto buffer = axes->buffer() + axes->calcOffset({i});
644 switch (axes->data_type())
646 case ir::DataType::INT32:
648 axes_vec.emplace_back(*reinterpret_cast<const int32_t *>(buffer));
651 case ir::DataType::INT64:
653 axes_vec.emplace_back(*reinterpret_cast<const int64_t *>(buffer));
657 throw std::runtime_error("DynamicShapeInferer " + op.name() + ": Not supported data type");
661 const auto keep_dims = op.param().keep_dims;
663 auto output_ind = op.getOutputs().at(0);
664 auto output = _tensor_registry->getITensor(output_ind);
666 ir::Shape new_shape = shape_inference::inferReduceShape(input_shape, axes_vec, keep_dims);
668 dynamicTensorManagerOf(output)->applyShape(output_ind, new_shape);
669 assert(output->buffer() != nullptr);
672 void DynamicShapeInferer::visit(const ir::operation::Reshape &op)
674 // check if output is not dynamic
675 auto output_ind = op.getOutputs().at(0);
676 auto output = _tensor_registry->getITensor(output_ind);
678 auto input_ind = op.getInputs().at(ir::operation::Reshape::Input::INPUT);
679 auto input = _tensor_registry->getITensor(input_ind);
682 Here, the state after compilation (satic shape inference) could be one of the following:
684 input1 input2 (or option) output execution-time shape inf required
685 ------------------------------------ --------------------------------
686 case 1) static const static X
687 case 2) static placeholder dynamic O
688 case 3) dynamic const dynamic O
689 case 4) dynamic placeholder dynamic O
691 Then nnfw_apply_tensorinf() could change input dynamic.
692 So, in this method, we could have one more state and we have to re-calculate shape
695 case 5) dynamic const static O
697 So, only when both input1 and ouput are static, we can skip dynamic shape inference.
699 if ((!input->is_dynamic()) && (!output->is_dynamic()))
702 // New shape is given by second input tensor
703 if (op.getInputs().size() == 2)
705 // from op, access the buffer of second input to read new shape
706 auto new_shape_ind = op.getInputs().at(ir::operation::Reshape::Input::SHAPE);
708 // getting output shape by reading new_shape tensor buffer
709 auto new_shape = _tensor_registry->getITensor(new_shape_ind);
712 int32_t *new_shape_buf = reinterpret_cast<int32_t *>(new_shape->buffer());
713 assert(new_shape_buf);
715 auto output_shape = shape_inference::inferReshapeShape(
716 new_shape_buf, new_shape->getShape().num_elements(), input->getShape().num_elements());
718 // if shape is changed, change output shape and reallocate output tensor memory
719 if (output_shape != output->getShape() || output->buffer() == nullptr)
721 // change on output shape
722 dynamicTensorManagerOf(output)->applyShape(output_ind, output_shape);
724 assert(output->buffer() != nullptr);
726 // New shape is given by option
727 else if (op.param().new_shape.size() != 0)
729 // Let's check the new_shape option
730 auto shape = op.param().new_shape;
731 auto output_shape = shape_inference::inferReshapeShape(shape.data(), shape.size(),
732 input->getShape().num_elements());
734 // if shape is changed, change output shape and reallocate output tensor memory
735 if (output_shape != output->getShape() || output->buffer() == nullptr)
737 // change on output shape
738 dynamicTensorManagerOf(output)->applyShape(output_ind, output_shape);
740 assert(output->buffer() != nullptr);
744 throw std::runtime_error("Reshape: new shape is missing");
749 void DynamicShapeInferer::visit(const ir::operation::ResizeBilinear &op)
751 // check if output is not dynamic
752 auto output_ind = op.getOutputs().at(0);
753 auto output = _tensor_registry->getITensor(output_ind);
755 auto input_ind = op.getInputs().at(ir::operation::Reshape::Input::INPUT);
756 auto input = _tensor_registry->getITensor(input_ind);
758 if ((!input->is_dynamic()) && (!output->is_dynamic()))
761 // getting output shape from input shape and Params
762 auto output_shape = shape_inference::inferResizeBilinearShape(
763 input->getShape(), op.param().height_out, op.param().width_out);
765 // if shape is changed, change output shape and reallocate output tensor memory
766 if (output_shape != output->getShape() || output->buffer() == nullptr)
768 // change on output shape
769 _dynamic_tensor_manager->applyShape(output_ind, output_shape);
771 assert(output->buffer() != nullptr);
774 void DynamicShapeInferer::visit(const ir::operation::Reverse &op)
776 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::Reverse::INPUT));
779 void DynamicShapeInferer::visit(const ir::operation::Round &op)
781 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::Round::Input::INPUT));
784 void DynamicShapeInferer::visit(const ir::operation::RSQRT &op)
786 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::RSQRT::INPUT));
789 void DynamicShapeInferer::visit(const ir::operation::Select &op)
791 const auto input_cond_idx = op.getInputs().at(ir::operation::Select::Input::CONDITION);
792 const auto &input_cond = _tensor_registry->getITensor(input_cond_idx);
794 const auto input_true_idx = op.getInputs().at(ir::operation::Select::Input::INPUT_TRUE);
795 const auto &input_true = _tensor_registry->getITensor(input_true_idx);
797 const auto input_false_idx = op.getInputs().at(ir::operation::Select::Input::INPUT_FALSE);
798 const auto &input_false = _tensor_registry->getITensor(input_false_idx);
800 if ((!input_cond->is_dynamic()) && (!input_true->is_dynamic()) && (!input_false->is_dynamic()))
805 auto input_cond_shape = input_cond->getShape();
806 auto input_true_shape = input_true->getShape();
807 auto input_false_shape = input_false->getShape();
809 // Select output shpae
810 ir::Shape new_shape =
811 shape_inference::inferSelectShape(input_cond_shape, input_true_shape, input_false_shape);
813 auto output_ind = op.getOutputs().at(0);
814 auto output = _tensor_registry->getITensor(output_ind);
816 dynamicTensorManagerOf(output)->applyShape(output_ind, new_shape);
817 assert(output->buffer() != nullptr);
820 void DynamicShapeInferer::visit(const ir::operation::Shape &op)
822 const auto input_idx{op.getInputs().at(0)};
823 const auto &input = _tensor_registry->getITensor(input_idx);
824 auto input_shape = input->getShape();
826 if (!input->is_dynamic())
829 auto output_ind = op.getOutputs().at(0);
830 auto output = _tensor_registry->getITensor(output_ind);
832 ir::Shape output_shape;
833 output_shape.append(input_shape.rank());
835 dynamicTensorManagerOf(output)->applyShape(output_ind, output_shape);
836 assert(output->buffer() != nullptr);
839 void DynamicShapeInferer::visit(const ir::operation::Sin &op)
841 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::Sin::Input::INPUT));
844 void DynamicShapeInferer::visit(const ir::operation::Slice &op)
846 const auto input_index{op.getInputs().at(ir::operation::Slice::Input::INPUT)};
847 const auto input = _tensor_registry->getITensor(input_index);
848 const auto begins_index{op.getInputs().at(ir::operation::Slice::Input::BEGINS)};
849 const auto begins = _tensor_registry->getITensor(begins_index);
850 const auto sizes_index{op.getInputs().at(ir::operation::Slice::Input::SIZES)};
851 const auto sizes = _tensor_registry->getITensor(sizes_index);
852 auto output_index = op.getOutputs().at(0);
853 auto output = _tensor_registry->getITensor(output_index);
855 if (!(input->is_dynamic() || begins->is_dynamic() || sizes->is_dynamic() || output->is_dynamic()))
860 ir::Shape input_shape = input->getShape();
861 auto begins_buf = reinterpret_cast<const int32_t *>(begins->buffer());
862 auto sizes_buf = reinterpret_cast<const int32_t *>(sizes->buffer());
864 ir::Shape new_shape = shape_inference::inferSliceShape(input_shape, begins_buf, sizes_buf);
866 dynamicTensorManagerOf(output)->applyShape(output_index, new_shape);
867 assert(output->buffer() != nullptr);
870 void DynamicShapeInferer::visit(const ir::operation::Softmax &op)
872 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::Softmax::INPUT));
875 void DynamicShapeInferer::visit(const ir::operation::SpaceToBatchND &op)
877 const auto input_idx{op.getInputs().at(ir::operation::SpaceToBatchND::Input::INPUT)};
878 const auto block_shape_idx{op.getInputs().at(ir::operation::SpaceToBatchND::Input::BLOCK_SIZE)};
879 const auto padding_idx{op.getInputs().at(ir::operation::SpaceToBatchND::Input::PADDINGS)};
880 auto output_idx{op.getOutputs().at(0)};
882 const auto &input = _tensor_registry->getITensor(input_idx);
883 const auto &block_shape = _tensor_registry->getITensor(block_shape_idx);
884 const auto &padding = _tensor_registry->getITensor(padding_idx);
885 auto output = _tensor_registry->getITensor(output_idx);
887 if (!(input->is_dynamic() || block_shape->is_dynamic() || padding->is_dynamic() ||
888 output->is_dynamic()))
893 auto input_shape = input->getShape();
894 auto block_shape_shape = block_shape->getShape();
895 auto padding_shape = padding->getShape();
897 auto block_shape_data = reinterpret_cast<int32_t *>(block_shape->buffer());
898 auto padding_data = reinterpret_cast<int32_t *>(padding->buffer());
900 ir::Shape new_shape = shape_inference::inferSpaceToBatchNDShape(
901 input_shape, block_shape_shape, padding_shape, block_shape_data, padding_data);
903 dynamicTensorManagerOf(output)->applyShape(output_idx, new_shape);
904 assert(output->buffer() != nullptr);
907 void DynamicShapeInferer::visit(const ir::operation::Split &op)
909 const auto input_idx{op.getInputs().at(ir::operation::Split::Input::INPUT)};
910 const auto &input = _tensor_registry->getITensor(input_idx);
912 if (!input->is_dynamic())
917 auto input_shape = input->getShape();
919 const auto axis = op.param().axis;
920 const auto num_splits = op.param().num_splits;
921 const auto rank = input_shape.rank();
922 auto axis_resolved = axis < 0 ? axis + rank : axis;
924 assert(0 <= axis_resolved && axis_resolved < rank);
926 ir::Shape new_shape = shape_inference::inferSplitShape(input_shape, axis_resolved, num_splits);
927 for (int out_tensor_idx = 0; out_tensor_idx < num_splits; out_tensor_idx++)
929 auto output_ind = op.getOutputs().at(out_tensor_idx);
930 auto output = _tensor_registry->getITensor(output_ind);
932 dynamicTensorManagerOf(output)->applyShape(output_ind, new_shape);
933 assert(output->buffer() != nullptr);
937 void DynamicShapeInferer::visit(const ir::operation::SquaredDifference &op)
939 handleBinaryArithmeticOp(op, op.getInputs().at(ir::operation::SquaredDifference::Input::LHS),
940 op.getInputs().at(ir::operation::SquaredDifference::Input::RHS));
943 void DynamicShapeInferer::visit(const ir::operation::Squeeze &op)
945 const auto input_idx{op.getInputs().at(ir::operation::Squeeze::Input::INPUT)};
946 const auto &input = _tensor_registry->getITensor(input_idx);
948 if (!input->is_dynamic())
953 auto input_shape = input->getShape();
955 // Squeeze output shpae
956 ir::Shape new_shape = shape_inference::inferSqueezeShape(input_shape, op.param());
958 auto output_ind = op.getOutputs().at(0);
959 auto output = _tensor_registry->getITensor(output_ind);
961 dynamicTensorManagerOf(output)->applyShape(output_ind, new_shape);
962 assert(output->buffer() != nullptr);
965 void DynamicShapeInferer::visit(const ir::operation::StridedSlice &op)
968 const auto input_index{op.getInputs().at(ir::operation::StridedSlice::Input::INPUT)};
969 auto input = _tensor_registry->getITensor(input_index);
970 ir::Shape input_shape = input->getShape();
972 const auto starts_index{op.getInputs().at(ir::operation::StridedSlice::Input::STARTS)};
973 auto starts = _tensor_registry->getITensor(starts_index);
975 const auto ends_index{op.getInputs().at(ir::operation::StridedSlice::Input::ENDS)};
976 auto ends = _tensor_registry->getITensor(ends_index);
978 const auto strides_index{op.getInputs().at(ir::operation::StridedSlice::Input::STRIDES)};
979 auto strides = _tensor_registry->getITensor(strides_index);
981 if (!(input->is_dynamic() || starts->is_dynamic() || ends->is_dynamic() || strides->is_dynamic()))
986 const auto begin_mask = op.param().begin_mask;
987 const auto end_mask = op.param().end_mask;
988 const auto shrink_axis_mask = op.param().shrink_axis_mask;
989 const auto rank = input_shape.rank();
991 auto op_params = shape_inference::buildStridedSliceParams(
992 reinterpret_cast<uint32_t *>(starts->buffer()), reinterpret_cast<uint32_t *>(ends->buffer()),
993 reinterpret_cast<uint32_t *>(strides->buffer()), begin_mask, end_mask, shrink_axis_mask,
996 auto output_index = op.getOutputs().at(0);
997 auto output = _tensor_registry->getITensor(output_index);
999 ir::Shape output_shape =
1000 onert::shape_inference::inferStridedSliceShape(input_shape, op_params, rank);
1002 dynamicTensorManagerOf(output)->applyShape(output_index, output_shape);
1003 assert(output->buffer() != nullptr);
1006 void DynamicShapeInferer::visit(const ir::operation::Sub &op)
1008 handleBinaryArithmeticOp(op, op.getInputs().at(ir::operation::Sub::Input::LHS),
1009 op.getInputs().at(ir::operation::Sub::Input::RHS));
1012 void DynamicShapeInferer::visit(const ir::operation::Tanh &op)
1014 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::Tanh::INPUT));
1017 void DynamicShapeInferer::visit(const ir::operation::Tile &op)
1019 auto output_ind = op.getOutputs().at(0);
1020 auto output = _tensor_registry->getITensor(output_ind);
1022 auto input_idx = op.getInputs().at(ir::operation::Tile::Input::INPUT);
1023 auto input = _tensor_registry->getITensor(input_idx);
1025 auto multiplier_idx = op.getInputs().at(ir::operation::Tile::Input::MULTIPLES);
1026 auto multiplier = _tensor_registry->getITensor(multiplier_idx);
1028 if ((!input->is_dynamic()) && (!output->is_dynamic()))
1031 auto input_shape = input->getShape();
1032 auto multiplier_buffer = reinterpret_cast<const int32_t *>(multiplier->buffer());
1033 assert(multiplier_buffer);
1035 auto output_shape = shape_inference::inferTileShape(input_shape, multiplier_buffer);
1037 // set output shape and output buffer
1038 dynamicTensorManagerOf(output)->applyShape(output_ind, output_shape);
1039 assert(output->buffer() != nullptr);
1042 void DynamicShapeInferer::visit(const ir::operation::Transpose &op)
1044 // check if output is not dynamic
1045 auto output_ind = op.getOutputs().at(0);
1046 auto output = _tensor_registry->getITensor(output_ind);
1048 // from op, access the buffer of second input to read new shape
1049 auto input_ind = op.getInputs().at(ir::operation::Transpose::Input::INPUT);
1050 auto input_tensor = _tensor_registry->getITensor(input_ind);
1051 auto input_shape = input_tensor->getShape();
1053 if (!input_tensor->is_dynamic())
1056 const auto perm{op.param().perm};
1057 // set output shape, based on input and params
1058 ir::Shape new_shape = shape_inference::inferTransposeShape(input_shape, perm);
1060 dynamicTensorManagerOf(output)->applyShape(output_ind, new_shape);
1061 assert(output->buffer() != nullptr);
1064 void DynamicShapeInferer::visit(const ir::operation::Unpack &op)
1066 // check if output is not dynamic
1067 const auto input_idx{op.getInputs().at(0)};
1068 const auto &input = _tensor_registry->getITensor(input_idx);
1070 if (!input->is_dynamic())
1073 auto input_shape = input->getShape();
1075 const auto rank = input_shape.rank();
1076 const auto axis = ((op.param().axis < 0) ? rank + op.param().axis : op.param().axis);
1077 const auto num = op.param().num;
1079 assert(0 <= axis && axis < rank);
1081 ir::Shape new_shape = shape_inference::inferUnpackShape(input_shape, axis, rank);
1083 for (int out_tensor_idx = 0; out_tensor_idx < num; out_tensor_idx++)
1085 auto output_ind = op.getOutputs().at(out_tensor_idx);
1086 auto output = _tensor_registry->getITensor(output_ind);
1088 dynamicTensorManagerOf(output)->applyShape(output_ind, new_shape);
1090 assert(output->buffer() != nullptr);
1094 void DynamicShapeInferer::visit(const ir::operation::ZerosLike &op)
1096 handleSimpleUnaryOp(op, op.getInputs().at(ir::operation::ZerosLike::INPUT));
1100 } // namespace onert