#include <omp.h>
#endif
+#include "caffe2/core/logging.h"
#include "dnnlowp_op.h"
#include "dnnlowp_partition.h"
#include "fbgemm_pack_op.h"
C10_DECLARE_int32(caffe2_dnnlowp_nbits_in_non_outlier);
C10_DECLARE_int32(caffe2_dnnlowp_copy_to_32bit_frequency);
C10_DECLARE_bool(caffe2_dnnlowp_shared_int32_buffer);
-
// Thresholds to fallback to 32-bit accumulation when 16-bit accumulation
// doesn't provide performance benefits.
C10_DEFINE_double(
template <bool ReluFused>
bool ConvDNNLowPAcc16Op<ReluFused>::GetQuantizationParameters_() {
if (fallback_to_32_bit_accumulation_) {
- return true;
- }
-
- if (!BaseType::GetQuantizationParameters_()) {
- return false;
- }
-
- if (!Wq_acc16_packed_ &&
- this->template InputIsType<Int8ConvDNNLowPPackedWeightBlob>(FILTER)) {
- CAFFE_ENFORCE_EQ(
- this->order_,
- StorageOrder::NHWC,
- "Pre-packed weight only works with NHWC layout");
- // If the input is already packed
- const auto& packed_filter =
- this->template Input<Int8ConvDNNLowPPackedWeightBlob>(FILTER);
- Wq_outlier_ = packed_filter.W_outlier;
- Wq_acc16_packed_ = packed_filter.W_acc16;
-
- if (nbits_in_non_outlier_ != packed_filter.nbits_in_non_outlier) {
- LOG(WARNING)
- << "nbits_in_non_outlier in packed weight "
- << packed_filter.nbits_in_non_outlier
- << " doesn't match with nbits_in_non_outlier specified in operator "
- << nbits_in_non_outlier_;
- }
-
- first_invocation_ = false;
- return true;
+ // Short cut if we already know we are falling back to acc32
+ return BaseType::GetQuantizationParameters_();
}
int kernel_dim = this->KernelDim_();
int num_out_channels = filter.dim32(0);
// Check if we should fallback to 32-bit accumulation
- if (this->order_ == StorageOrder::NHWC) {
+ // We should do this before GetQuantizationParameters_ to make sure
+ // GetQuantizationParameters_ initialize things like Wq_packed_ for acc32
+ // properly.
+
+ // We can't fallback if layout is not NHWC or
+ // if weight is prepacked and the prepacked weight doesn't have acc32.
+ bool can_fallback_to_32_bit_accumulation =
+ this->order_ == StorageOrder::NHWC &&
+ (!this->template InputIsType<Int8ConvDNNLowPPackedWeightBlob>(FILTER) ||
+ this->template Input<Int8ConvDNNLowPPackedWeightBlob>(FILTER).W);
+ if (can_fallback_to_32_bit_accumulation) {
const Tensor& X = InputTensorCPU_(INPUT);
int N = X.dim32(0);
}
if (N * output_image_size < FLAGS_caffe2_dnnlowp_acc16_m_threshold) {
- LOG(INFO) << "M " << N * output_image_size
- << " of Conv layer with weight blob "
- << this->debug_def().input(1) << " is smaller than threshold "
- << FLAGS_caffe2_dnnlowp_acc16_m_threshold
- << " . Falling back to acc32";
+ C10_LOG_FIRST_N(INFO, 10)
+ << "M " << N * output_image_size << " of Conv layer with weight blob "
+ << this->debug_def().input(FILTER) << " is smaller than threshold "
+ << FLAGS_caffe2_dnnlowp_acc16_m_threshold
+ << " . Falling back to acc32";
+ fallback_to_32_bit_accumulation_ = true;
+ }
+ if (!fallback_to_32_bit_accumulation_ &&
+ num_out_channels / group_ < acc16_n_threshold) {
+ C10_LOG_FIRST_N(INFO, 10)
+ << "N " << num_out_channels / group_
+ << " of Conv layer with weight blob "
+ << this->debug_def().input(FILTER) << " is smaller than threshold "
+ << acc16_n_threshold << " . Falling back to acc32";
fallback_to_32_bit_accumulation_ = true;
- return true;
}
- if (num_out_channels / group_ < acc16_n_threshold) {
- LOG(INFO) << "N " << num_out_channels / group_
- << " of Conv layer with weight blob "
- << this->debug_def().input(1) << " is smaller than threshold "
- << acc16_n_threshold << " . Falling back to acc32";
+ if (!fallback_to_32_bit_accumulation_ && kernel_dim < acc16_k_threshold) {
+ C10_LOG_FIRST_N(INFO, 10)
+ << "K " << kernel_dim << " of Conv layer with weight blob "
+ << this->debug_def().input(FILTER) << " is smaller than threshold "
+ << acc16_k_threshold << " . Falling back to acc32";
fallback_to_32_bit_accumulation_ = true;
- return true;
}
- if (kernel_dim < acc16_k_threshold) {
- LOG(INFO) << "K " << kernel_dim << " of Conv layer with weight blob "
- << this->debug_def().input(1) << " is smaller than threshold "
- << acc16_k_threshold << " . Falling back to acc32";
+ if (!fallback_to_32_bit_accumulation_ &&
+ this->template InputIsType<Int8ConvDNNLowPPackedWeightBlob>(FILTER) &&
+ !this->template Input<Int8ConvDNNLowPPackedWeightBlob>(FILTER)
+ .W_acc16) {
+ C10_LOG_FIRST_N(INFO, 10)
+ << "Falling back to acc32 because packed weight for acc16 is not "
+ "available";
fallback_to_32_bit_accumulation_ = true;
- return true;
}
}
+ if (!BaseType::GetQuantizationParameters_()) {
+ return false;
+ }
+
+ if (fallback_to_32_bit_accumulation_) {
+ return true;
+ }
+
+ if (!Wq_acc16_packed_ &&
+ this->template InputIsType<Int8ConvDNNLowPPackedWeightBlob>(FILTER)) {
+ CAFFE_ENFORCE_EQ(
+ this->order_,
+ StorageOrder::NHWC,
+ "Pre-packed weight only works with NHWC layout");
+ // If the input is already packed
+ const auto& packed_filter =
+ this->template Input<Int8ConvDNNLowPPackedWeightBlob>(FILTER);
+ Wq_outlier_ = packed_filter.W_outlier;
+ Wq_acc16_packed_ = packed_filter.W_acc16;
+
+ if (nbits_in_non_outlier_ != packed_filter.nbits_in_non_outlier) {
+ C10_LOG_FIRST_N(WARNING, 10)
+ << "nbits_in_non_outlier in packed weight "
+ << packed_filter.nbits_in_non_outlier
+ << " doesn't match with nbits_in_non_outlier specified in operator "
+ << nbits_in_non_outlier_;
+ }
+ first_invocation_ = false;
+ return true;
+ }
+
// Separate out outliers
if (!Wq_outlier_ && this->order_ == StorageOrder::NHWC &&
nbits_in_non_outlier_ < 8) {
W_quantized_));
int outlier_cnt = Wq_outlier_->ColPtr()[num_out_channels];
- LOG(INFO) << "Proportion of outlier for Conv layer with weight blob "
- << this->debug_def().input(1) << " is "
- << static_cast<float>(outlier_cnt) / W_quantized_.size();
- LOG(INFO) << "nbits_in_non_outlier " << nbits_in_non_outlier_
- << " copy_to_32bit_frequency " << copy_to_32bit_frequency_;
-
- if (static_cast<float>(outlier_cnt) / W_quantized_.size() >
- FLAGS_caffe2_dnnlowp_acc16_density_threshold) {
- LOG(INFO) << "Density of outliers is higher than threshold "
- << FLAGS_caffe2_dnnlowp_acc16_density_threshold
- << " . Falling back to acc32";
+ C10_LOG_FIRST_N(INFO, 10)
+ << "Proportion of outlier for Conv layer with weight blob "
+ << this->debug_def().input(FILTER) << " is "
+ << static_cast<float>(outlier_cnt) / W_quantized_.size();
+ C10_LOG_FIRST_N(INFO, 10)
+ << "nbits_in_non_outlier " << nbits_in_non_outlier_
+ << " copy_to_32bit_frequency " << copy_to_32bit_frequency_;
+
+ if (can_fallback_to_32_bit_accumulation &&
+ static_cast<float>(outlier_cnt) / W_quantized_.size() >
+ FLAGS_caffe2_dnnlowp_acc16_density_threshold) {
+ C10_LOG_FIRST_N(INFO, 10)
+ << "Density of outliers is higher than threshold "
+ << FLAGS_caffe2_dnnlowp_acc16_density_threshold
+ << " . Falling back to acc32";
fallback_to_32_bit_accumulation_ = true;
Wq_outlier_.reset();
- return true;
+ // We need to call GetQuantizationParameters_ again to pack for acc32
+ return BaseType::GetQuantizationParameters_();
}
}
static int log_occurences = 0;
if (log_occurences < 32) {
++log_occurences;
- LOG(WARNING) << "Conv with weight " << this->debug_def().input(FILTER)
- << " falls back to slow path because " << reason;
+ C10_LOG_FIRST_N(WARNING, 10)
+ << "Conv with weight " << this->debug_def().input(FILTER)
+ << " falls back to slow path because " << reason;
}
}
}
static int log_occurences = 0;
if (log_occurences < 32) {
++log_occurences;
- LOG(WARNING) << "Outlier-aware quantization only supports "
- "NHWC layout";
+ C10_LOG_FIRST_N(WARNING, 10)
+ << "Outlier-aware quantization only supports "
+ "NHWC layout";
}
}
first_invocation_ = false;
static int log_occurences = 0;
if (log_occurences < 32) {
++log_occurences;
- LOG(WARNING)
+ C10_LOG_FIRST_N(WARNING, 10)
<< "Consider using DNNLOWP instead of DNNLOWP_ACC16 engine since "
"we're falling back to a slow path because of NCHW layout";
}
#include "caffe2_dnnlowp_utils.h"
C10_DECLARE_int32(caffe2_dnnlowp_nbits_in_non_outlier);
+C10_DECLARE_double(caffe2_dnnlowp_acc16_density_threshold);
+C10_DECLARE_int32(caffe2_dnnlowp_acc16_n_threshold);
+C10_DECLARE_int32(caffe2_dnnlowp_acc16_k_threshold);
namespace caffe2 {
ComputeColumnOffsets(
kernel_dim, M, W_quantized.data(), Y->qparams, *Y->column_offsets);
+ // Check if we should fallback to 32-bit accumulation.
+ // This check is only meaningful when engine is DNNLOWP_ACC16.
+ bool fallback_to_32_bit_accumulation = false;
+ if (nbits_in_non_outlier_ == 0) {
+ LOG(INFO) << "nbits_in_non_outlier == 0 means everything is outlier so we "
+ "fallback to acc32";
+ fallback_to_32_bit_accumulation = true;
+ }
+ // In Skylake, acc16 is not faster when N or K is smaller than 128
+ // FIXME : code duplication with conv_dnnlowp_acc16_op.cc
+ constexpr int SKYLAKE_ACC16_N_THRESHOLD_MIN = 128,
+ SKYLAKE_ACC16_K_THRESHOLD_MIN = 128;
+ int acc16_n_threshold = FLAGS_caffe2_dnnlowp_acc16_n_threshold;
+ if (caffe2::GetCpuId().avx512f() &&
+ acc16_n_threshold < SKYLAKE_ACC16_N_THRESHOLD_MIN) {
+ acc16_n_threshold = SKYLAKE_ACC16_N_THRESHOLD_MIN;
+ }
+ int acc16_k_threshold = FLAGS_caffe2_dnnlowp_acc16_k_threshold;
+ if (caffe2::GetCpuId().avx512f() &&
+ acc16_k_threshold < SKYLAKE_ACC16_K_THRESHOLD_MIN) {
+ acc16_k_threshold = SKYLAKE_ACC16_K_THRESHOLD_MIN;
+ }
+ if (!fallback_to_32_bit_accumulation && M / group_ < acc16_n_threshold) {
+ LOG(INFO) << "N " << M / group_ << " of weight blob "
+ << this->debug_def().input(0) << " is smaller than threshold "
+ << acc16_n_threshold << " . Falling back to acc32";
+ fallback_to_32_bit_accumulation = true;
+ }
+ if (!fallback_to_32_bit_accumulation && kernel_dim < acc16_k_threshold) {
+ LOG(INFO) << "K " << kernel_dim << " of weight blob "
+ << this->debug_def().input(0) << " is smaller than threshold "
+ << acc16_k_threshold << " . Falling back to acc32";
+ fallback_to_32_bit_accumulation = true;
+ }
+
// When nbits_in_non_outlier == 0, we fall back to acc32
if (this->debug_def().engine() == "DNNLOWP_ACC16" &&
- nbits_in_non_outlier_ > 0) {
+ !fallback_to_32_bit_accumulation) {
if (nbits_in_non_outlier_ < 8) {
Y->W_outlier.reset(ExtractOutlierMatrix(
group_, kernel_dim, M, nbits_in_non_outlier_, W_quantized));
<< this->debug_def().input(0) << " is "
<< static_cast<float>(outlier_cnt) / W_quantized.size();
LOG(INFO) << "nbits_in_non_outlier " << nbits_in_non_outlier_;
+
+ if (static_cast<float>(outlier_cnt) / W_quantized.size() >
+ FLAGS_caffe2_dnnlowp_acc16_density_threshold) {
+ LOG(INFO) << "Density of outliers is higher than threshold "
+ << FLAGS_caffe2_dnnlowp_acc16_density_threshold
+ << " . Falling back to acc32";
+ fallback_to_32_bit_accumulation = true;
+ }
}
- Y->nbits_in_non_outlier = nbits_in_non_outlier_;
- Y->W_acc16.reset(new fbgemm::PackBMatrix<int8_t, int16_t>(
- fbgemm::matrix_op_t::Transpose,
- group_ * kernel_dim,
- M / group_,
- W_quantized.data(),
- kernel_dim,
- nullptr, // pmat
- group_));
- } else if (TakeDepthWise3x3FastPath_()) {
- Y->W_depthwise_3x3.reset(
- new fbgemm::Packed3x3ConvMatrix(group_, W_quantized.data()));
- } else if (TakeDepthWise3x3x3FastPath_()) {
- Y->W_depthwise_3x3x3.reset(
- new fbgemm::Packed3x3x3ConvMatrix(group_, W_quantized.data()));
- } else if (TakeGConvFastPath_()) {
- fbgemm::conv_param_t<> conv_p(
- 1,
- group_ * C_per_group,
- M,
- {1, 1},
- group_,
- {this->kernel_[0], this->kernel_[1]},
- {this->stride_[0], this->stride_[1]},
- {this->pads_[0], this->pads_[1], this->pads_[2], this->pads_[3]});
-
- Y->W_gconv.reset(new fbgemm::PackWeightMatrixForGConv<int8_t>(
- fbgemm::matrix_op_t::Transpose, conv_p, W_quantized.data()));
- } else {
- Y->W.reset(new fbgemm::PackBMatrix<int8_t>(
- fbgemm::matrix_op_t::Transpose,
- group_ * kernel_dim,
- M / group_,
- W_quantized.data(),
- kernel_dim,
- nullptr, // pmat
- group_));
+ if (!fallback_to_32_bit_accumulation) {
+ Y->nbits_in_non_outlier = nbits_in_non_outlier_;
+ Y->W_acc16.reset(new fbgemm::PackBMatrix<int8_t, int16_t>(
+ fbgemm::matrix_op_t::Transpose,
+ group_ * kernel_dim,
+ M / group_,
+ W_quantized.data(),
+ kernel_dim,
+ nullptr, // pmat
+ group_));
+ }
+ }
+
+ if (fallback_to_32_bit_accumulation) {
+ Y->W_acc16.reset();
+ Y->W_outlier.reset();
+ }
+
+ if (this->debug_def().engine() != "DNNLOWP_ACC16" ||
+ fallback_to_32_bit_accumulation) {
+ // acc32
+ if (TakeDepthWise3x3FastPath_()) {
+ Y->W_depthwise_3x3.reset(
+ new fbgemm::Packed3x3ConvMatrix(group_, W_quantized.data()));
+ } else if (TakeDepthWise3x3x3FastPath_()) {
+ Y->W_depthwise_3x3x3.reset(
+ new fbgemm::Packed3x3x3ConvMatrix(group_, W_quantized.data()));
+ } else if (TakeGConvFastPath_()) {
+ fbgemm::conv_param_t<> conv_p(
+ 1,
+ group_ * C_per_group,
+ M,
+ {1, 1},
+ group_,
+ {this->kernel_[0], this->kernel_[1]},
+ {this->stride_[0], this->stride_[1]},
+ {this->pads_[0], this->pads_[1], this->pads_[2], this->pads_[3]});
+
+ Y->W_gconv.reset(new fbgemm::PackWeightMatrixForGConv<int8_t>(
+ fbgemm::matrix_op_t::Transpose, conv_p, W_quantized.data()));
+ } else {
+ Y->W.reset(new fbgemm::PackBMatrix<int8_t>(
+ fbgemm::matrix_op_t::Transpose,
+ group_ * kernel_dim,
+ M / group_,
+ W_quantized.data(),
+ kernel_dim,
+ nullptr, // pmat
+ group_));
+ }
}
if (InputSize() >= 2) {
projects / platform / upstream / pytorch.git / commitdiff