2 * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved
3 * Copyright 2017 The TensorFlow Authors. All Rights Reserved.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
18 #ifndef __NNFW_CKER_MAX_POOL_H__
19 #define __NNFW_CKER_MAX_POOL_H__
21 #include "cker/Shape.h"
22 #include "cker/Types.h"
23 #include "cker/Utils.h"
24 #include "cker/neon/neon_check.h"
25 #include "cker/eigen/Utils.h"
34 inline void MaxPool(const PoolParams ¶ms, const Shape &input_shape, const float *input_data,
35 const Shape &output_shape, float *output_data)
37 assert(input_shape.DimensionsCount() == 4);
38 assert(output_shape.DimensionsCount() == 4);
39 const int batches = MatchingDim(input_shape, 0, output_shape, 0);
40 const int input_height = input_shape.Dims(1);
41 const int input_width = input_shape.Dims(2);
42 const int output_height = output_shape.Dims(1);
43 const int output_width = output_shape.Dims(2);
44 const int stride_height = params.stride_height;
45 const int stride_width = params.stride_width;
47 const auto in_mat = MapAsMatrixWithLastDimAsRows(input_data, input_shape);
48 auto out_mat = MapAsMatrixWithLastDimAsRows(output_data, output_shape);
49 // Prefill the output to minimum representable float value
50 out_mat.setConstant(std::numeric_limits<float>::lowest());
51 for (int b = 0; b < batches; ++b)
53 for (int h = 0; h < input_height; ++h)
55 for (int w = 0; w < input_width; ++w)
57 // (h_start, h_end) * (w_start, w_end) is the range that the input
58 // vector projects to.
59 int hpad = h + params.padding_values.height;
60 int wpad = w + params.padding_values.width;
62 (hpad < params.filter_height) ? 0 : (hpad - params.filter_height) / stride_height + 1;
63 int h_end = std::min(hpad / stride_height + 1, output_height);
65 (wpad < params.filter_width) ? 0 : (wpad - params.filter_width) / stride_width + 1;
66 int w_end = std::min(wpad / stride_width + 1, output_width);
67 // compute elementwise sum
68 for (int ph = h_start; ph < h_end; ++ph)
70 for (int pw = w_start; pw < w_end; ++pw)
72 int out_offset = NodeOffset(b, ph, pw, output_height, output_width);
73 out_mat.col(out_offset) =
74 out_mat.col(out_offset)
75 .cwiseMax(in_mat.col(NodeOffset(b, h, w, input_height, input_width)));
81 const int flat_size = output_shape.FlatSize();
82 for (int i = 0; i < flat_size; ++i)
84 output_data[i] = ActivationFunctionWithMinMax(output_data[i], params.float_activation_min,
85 params.float_activation_max);
89 inline void MaxPool(const PoolParams ¶ms, const Shape &input_shape, const uint8_t *input_data,
90 const Shape &output_shape, uint8_t *output_data)
93 // Here, and in other pooling ops, in order to maintain locality of reference,
94 // to minimize some recalculations, and to load into NEON vector registers, we
95 // use an inner loop down the depth. Since depths can be large and hence we
96 // would need arbitrarily large temporary storage, we divide the work up into
97 // depth tranches just within the batch loop.
98 static constexpr int kPoolingAccTrancheSize = 256;
100 assert(params.quantized_activation_min <= params.quantized_activation_max);
101 assert(input_shape.DimensionsCount() == 4);
102 assert(output_shape.DimensionsCount() == 4);
103 const int batches = MatchingDim(input_shape, 0, output_shape, 0);
104 const int depth = MatchingDim(input_shape, 3, output_shape, 3);
105 const int input_height = input_shape.Dims(1);
106 const int input_width = input_shape.Dims(2);
107 const int output_height = output_shape.Dims(1);
108 const int output_width = output_shape.Dims(2);
109 const int stride_height = params.stride_height;
110 const int stride_width = params.stride_width;
112 uint8_t acc[kPoolingAccTrancheSize];
113 for (int batch = 0; batch < batches; ++batch)
115 // We proceed through the depth in tranches (see comment above). The
116 // depth_base is the depth at the beginning of the tranche. The
117 // tranche_depth is the depth dimension of the tranche.
118 for (int depth_base = 0; depth_base < depth; depth_base += kPoolingAccTrancheSize)
120 const int tranche_depth = std::min(depth - depth_base, kPoolingAccTrancheSize);
121 for (int out_y = 0; out_y < output_height; ++out_y)
123 for (int out_x = 0; out_x < output_width; ++out_x)
125 const int in_x_origin = (out_x * stride_width) - params.padding_values.width;
126 const int in_y_origin = (out_y * stride_height) - params.padding_values.height;
127 const int filter_x_start = std::max(0, -in_x_origin);
128 const int filter_x_end = std::min(params.filter_width, input_width - in_x_origin);
129 const int filter_y_start = std::max(0, -in_y_origin);
130 const int filter_y_end = std::min(params.filter_height, input_height - in_y_origin);
131 memset(acc, 0, tranche_depth * sizeof(acc[0]));
132 const uint8_t *input_ptr =
133 input_data + depth_base +
134 depth * (in_x_origin + input_width * (in_y_origin + input_height * batch));
135 for (int fy = filter_y_start; fy < filter_y_end; fy++)
137 const uint8_t *input_row_ptr = input_ptr + depth * (fy * input_width + filter_x_start);
138 for (int fx = filter_x_start; fx < filter_x_end; fx++)
140 const uint8_t *input_channel_ptr = input_row_ptr;
143 for (; channel <= tranche_depth - 16; channel += 16)
145 uint8x16_t acc_reg = vld1q_u8(acc + channel);
146 uint8x16_t input_reg = vld1q_u8(input_channel_ptr);
147 input_channel_ptr += 16;
148 acc_reg = vmaxq_u8(acc_reg, input_reg);
149 vst1q_u8(acc + channel, acc_reg);
152 for (; channel <= tranche_depth - 8; channel += 8)
154 uint8x8_t acc_reg = vld1_u8(acc + channel);
155 uint8x8_t input_reg = vld1_u8(input_channel_ptr);
156 input_channel_ptr += 8;
157 acc_reg = vmax_u8(acc_reg, input_reg);
158 vst1_u8(acc + channel, acc_reg);
161 for (; channel < tranche_depth; ++channel)
163 acc[channel] = std::max(acc[channel], *input_channel_ptr++);
165 input_row_ptr += depth;
168 uint8_t *output_ptr = output_data + Offset(output_shape, batch, out_y, out_x, depth_base);
171 for (; channel <= tranche_depth - 16; channel += 16)
173 uint8x16_t a = vld1q_u8(acc + channel);
174 a = vminq_u8(a, vdupq_n_u8(params.quantized_activation_max));
175 a = vmaxq_u8(a, vdupq_n_u8(params.quantized_activation_min));
176 vst1q_u8(output_ptr + channel, a);
178 for (; channel <= tranche_depth - 8; channel += 8)
180 uint8x8_t a = vld1_u8(acc + channel);
181 a = vmin_u8(a, vdup_n_u8(params.quantized_activation_max));
182 a = vmax_u8(a, vdup_n_u8(params.quantized_activation_min));
183 vst1_u8(output_ptr + channel, a);
186 for (; channel < tranche_depth; ++channel)
188 uint8_t a = acc[channel];
189 a = std::max<uint8_t>(a, params.quantized_activation_min);
190 a = std::min<uint8_t>(a, params.quantized_activation_max);
191 output_ptr[channel] = static_cast<uint8_t>(a);
202 #endif // __NNFW_CKER_MAX_POOL_H__