8 #include "caffe/blob.hpp"
9 #include "caffe/common.hpp"
10 #include "caffe/layer_factory.hpp"
11 #include "caffe/proto/caffe.pb.h"
12 #include "caffe/util/math_functions.hpp"
15 #define NANO_SEC_CONV 1000000
16 extern unsigned int acl_log_flags;
17 #endif //USE_PROFILING
19 Forward declare boost::thread instead of including boost/thread.hpp
20 to avoid a boost/NVCC issues (#1009, #1010) on OSX.
22 namespace boost { class mutex; }
29 logtime_util(int mask_, const char* information_){
31 if(acl_log_flags & mask){
32 strncpy(information, information_, 255);
33 gettimeofday(&tv[0], NULL);
37 if(acl_log_flags & mask){
39 gettimeofday(&tv[1], NULL);
40 time[0] = tv[0].tv_sec * NANO_SEC_CONV + tv[0].tv_usec;
41 time[1] = tv[1].tv_sec * NANO_SEC_CONV + tv[1].tv_usec;
42 printf("%s %.6lf\n", information, (((double)time[1] - time[0]) / NANO_SEC_CONV));
45 void log_time(bool start)
47 if(acl_log_flags & mask){
49 gettimeofday(&tv[0], NULL);
53 gettimeofday(&tv[1], NULL);
54 time[0] = tv[0].tv_sec * NANO_SEC_CONV + tv[0].tv_usec;
55 time[1] = tv[1].tv_sec * NANO_SEC_CONV + tv[1].tv_usec;
56 printf("%s %.6lf\n", information, (((double)time[1] - time[0]) / NANO_SEC_CONV));
63 char information[256];
66 #ifdef LAYER_PERF_STAT
76 perf_stat(): total(0),start(0),end(0),count(0){};
82 #endif //USE_PROFILING
85 * @brief An interface for the units of computation which can be composed into a
88 * Layer%s must implement a Forward function, in which they take their input
89 * (bottom) Blob%s (if any) and compute their output Blob%s (if any).
90 * They may also implement a Backward function, in which they compute the error
91 * gradients with respect to their input Blob%s, given the error gradients with
92 * their output Blob%s.
94 template <typename Dtype>
98 * You should not implement your own constructor. Any set up code should go
99 * to SetUp(), where the dimensions of the bottom blobs are provided to the
102 explicit Layer(const LayerParameter& param)
103 : layer_param_(param) {
104 // Set phase and copy blobs (if there are any).
105 phase_ = param.phase();
106 if (layer_param_.blobs_size() > 0) {
107 blobs_.resize(layer_param_.blobs_size());
108 for (int i = 0; i < layer_param_.blobs_size(); ++i) {
109 blobs_[i].reset(new Blob<Dtype>());
110 blobs_[i]->FromProto(layer_param_.blobs(i));
117 * @brief Implements common layer setup functionality.
119 * @param bottom the preshaped input blobs
121 * the allocated but unshaped output blobs, to be shaped by Reshape
123 * Checks that the number of bottom and top blobs is correct.
124 * Calls LayerSetUp to do special layer setup for individual layer types,
125 * followed by Reshape to set up sizes of top blobs and internal buffers.
126 * Sets up the loss weight multiplier blobs for any non-zero loss weights.
127 * This method may not be overridden.
129 void SetUp(const vector<Blob<Dtype>*>& bottom,
130 const vector<Blob<Dtype>*>& top) {
131 CheckBlobCounts(bottom, top);
132 LayerSetUp(bottom, top);
133 Reshape(bottom, top);
138 * @brief Does layer-specific setup: your layer should implement this function
139 * as well as Reshape.
142 * the preshaped input blobs, whose data fields store the input data for
145 * the allocated but unshaped output blobs
147 * This method should do one-time layer specific setup. This includes reading
148 * and processing relevent parameters from the <code>layer_param_</code>.
149 * Setting up the shapes of top blobs and internal buffers should be done in
150 * <code>Reshape</code>, which will be called before the forward pass to
151 * adjust the top blob sizes.
153 virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
154 const vector<Blob<Dtype>*>& top) {}
157 * @brief Adjust the shapes of top blobs and internal buffers to accommodate
158 * the shapes of the bottom blobs.
160 * @param bottom the input blobs, with the requested input shapes
161 * @param top the top blobs, which should be reshaped as needed
163 * This method should reshape top blobs as needed according to the shapes
164 * of the bottom (input) blobs, as well as reshaping any internal buffers
165 * and making any other necessary adjustments so that the layer can
166 * accommodate the bottom blobs.
168 virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
169 const vector<Blob<Dtype>*>& top) = 0;
172 * @brief Given the bottom blobs, compute the top blobs and the loss.
175 * the input blobs, whose data fields store the input data for this layer
177 * the preshaped output blobs, whose data fields will store this layers'
179 * \return The total loss from the layer.
181 * The Forward wrapper calls the relevant device wrapper function
182 * (Forward_cpu or Forward_gpu) to compute the top blob values given the
183 * bottom blobs. If the layer has any non-zero loss_weights, the wrapper
184 * then computes and returns the loss.
186 * Your layer should implement Forward_cpu and (optionally) Forward_gpu.
189 Dtype Forward(const vector<Blob<Dtype>*>& bottom,
190 const vector<Blob<Dtype>*>& top);
192 inline Dtype Forward(const vector<Blob<Dtype>*>& bottom,
193 const vector<Blob<Dtype>*>& top);
194 #endif //USE_PROFILING
197 * @brief Given the top blob error gradients, compute the bottom blob error
201 * the output blobs, whose diff fields store the gradient of the error
202 * with respect to themselves
203 * @param propagate_down
204 * a vector with equal length to bottom, with each index indicating
205 * whether to propagate the error gradients down to the bottom blob at
206 * the corresponding index
208 * the input blobs, whose diff fields will store the gradient of the error
209 * with respect to themselves after Backward is run
211 * The Backward wrapper calls the relevant device wrapper function
212 * (Backward_cpu or Backward_gpu) to compute the bottom blob diffs given the
215 * Your layer should implement Backward_cpu and (optionally) Backward_gpu.
217 inline void Backward(const vector<Blob<Dtype>*>& top,
218 const vector<bool>& propagate_down,
219 const vector<Blob<Dtype>*>& bottom);
222 * @brief Returns the vector of learnable parameter blobs.
224 vector<shared_ptr<Blob<Dtype> > >& blobs() {
229 * @brief Returns the layer parameter.
231 const LayerParameter& layer_param() const { return layer_param_; }
234 * @brief Writes the layer parameter to a protocol buffer
236 virtual void ToProto(LayerParameter* param, bool write_diff = false);
239 * @brief Returns the scalar loss associated with a top blob at a given index.
241 inline Dtype loss(const int top_index) const {
242 return (loss_.size() > top_index) ? loss_[top_index] : Dtype(0);
246 * @brief Sets the loss associated with a top blob at a given index.
248 inline void set_loss(const int top_index, const Dtype value) {
249 if (loss_.size() <= top_index) {
250 loss_.resize(top_index + 1, Dtype(0));
252 loss_[top_index] = value;
256 * @brief Returns the layer type.
258 virtual inline const char* type() const { return ""; }
261 * @brief Returns the exact number of bottom blobs required by the layer,
262 * or -1 if no exact number is required.
264 * This method should be overridden to return a non-negative value if your
265 * layer expects some exact number of bottom blobs.
267 virtual inline int ExactNumBottomBlobs() const { return -1; }
269 * @brief Returns the minimum number of bottom blobs required by the layer,
270 * or -1 if no minimum number is required.
272 * This method should be overridden to return a non-negative value if your
273 * layer expects some minimum number of bottom blobs.
275 virtual inline int MinBottomBlobs() const { return -1; }
277 * @brief Returns the maximum number of bottom blobs required by the layer,
278 * or -1 if no maximum number is required.
280 * This method should be overridden to return a non-negative value if your
281 * layer expects some maximum number of bottom blobs.
283 virtual inline int MaxBottomBlobs() const { return -1; }
285 * @brief Returns the exact number of top blobs required by the layer,
286 * or -1 if no exact number is required.
288 * This method should be overridden to return a non-negative value if your
289 * layer expects some exact number of top blobs.
291 virtual inline int ExactNumTopBlobs() const { return -1; }
293 * @brief Returns the minimum number of top blobs required by the layer,
294 * or -1 if no minimum number is required.
296 * This method should be overridden to return a non-negative value if your
297 * layer expects some minimum number of top blobs.
299 virtual inline int MinTopBlobs() const { return -1; }
301 * @brief Returns the maximum number of top blobs required by the layer,
302 * or -1 if no maximum number is required.
304 * This method should be overridden to return a non-negative value if your
305 * layer expects some maximum number of top blobs.
307 virtual inline int MaxTopBlobs() const { return -1; }
309 * @brief Returns true if the layer requires an equal number of bottom and
312 * This method should be overridden to return true if your layer expects an
313 * equal number of bottom and top blobs.
315 virtual inline bool EqualNumBottomTopBlobs() const { return false; }
318 * @brief Return whether "anonymous" top blobs are created automatically
321 * If this method returns true, Net::Init will create enough "anonymous" top
322 * blobs to fulfill the requirement specified by ExactNumTopBlobs() or
325 virtual inline bool AutoTopBlobs() const { return false; }
328 * @brief Return whether to allow force_backward for a given bottom blob
331 * If AllowForceBackward(i) == false, we will ignore the force_backward
332 * setting and backpropagate to blob i only if it needs gradient information
333 * (as is done when force_backward == false).
335 virtual inline bool AllowForceBackward(const int bottom_index) const {
340 * @brief Specifies whether the layer should compute gradients w.r.t. a
341 * parameter at a particular index given by param_id.
343 * You can safely ignore false values and always compute gradients
344 * for all parameters, but possibly with wasteful computation.
346 inline bool param_propagate_down(const int param_id) {
347 return (param_propagate_down_.size() > param_id) ?
348 param_propagate_down_[param_id] : false;
351 * @brief Sets whether the layer should compute gradients w.r.t. a
352 * parameter at a particular index given by param_id.
354 inline void set_param_propagate_down(const int param_id, const bool value) {
355 if (param_propagate_down_.size() <= param_id) {
356 param_propagate_down_.resize(param_id + 1, true);
358 param_propagate_down_[param_id] = value;
363 #ifdef LAYER_PERF_STAT
365 const vector<Blob<Dtype>*> * saved_top;
366 const vector<Blob<Dtype>*> * saved_bottom;
367 perf_stat * get_time_stat(void) { return &time_stat_;}
368 perf_stat * get_pmu_stat(int index) { return &pmu_stat_[index];}
372 #endif //USE_PROFILING
375 /** The protobuf that stores the layer parameters */
376 LayerParameter layer_param_;
377 /** The phase: TRAIN or TEST */
379 /** The vector that stores the learnable parameters as a set of blobs. */
380 vector<shared_ptr<Blob<Dtype> > > blobs_;
381 /** Vector indicating whether to compute the diff of each param blob. */
382 vector<bool> param_propagate_down_;
384 /** The vector that indicates whether each top blob has a non-zero weight in
385 * the objective function. */
388 /** @brief Using the CPU device, compute the layer output. */
389 virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
390 const vector<Blob<Dtype>*>& top) = 0;
392 * @brief Using the GPU device, compute the layer output.
393 * Fall back to Forward_cpu() if unavailable.
395 virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
396 const vector<Blob<Dtype>*>& top) {
397 // LOG(WARNING) << "Using CPU code as backup.";
398 return Forward_cpu(bottom, top);
402 * @brief Using the CPU device, compute the gradients for any parameters and
403 * for the bottom blobs if propagate_down is true.
405 virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
406 const vector<bool>& propagate_down,
407 const vector<Blob<Dtype>*>& bottom) = 0;
409 * @brief Using the GPU device, compute the gradients for any parameters and
410 * for the bottom blobs if propagate_down is true.
411 * Fall back to Backward_cpu() if unavailable.
413 virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
414 const vector<bool>& propagate_down,
415 const vector<Blob<Dtype>*>& bottom) {
416 // LOG(WARNING) << "Using CPU code as backup.";
417 Backward_cpu(top, propagate_down, bottom);
421 * Called by the parent Layer's SetUp to check that the number of bottom
422 * and top Blobs provided as input match the expected numbers specified by
423 * the {ExactNum,Min,Max}{Bottom,Top}Blobs() functions.
425 virtual void CheckBlobCounts(const vector<Blob<Dtype>*>& bottom,
426 const vector<Blob<Dtype>*>& top) {
427 if (ExactNumBottomBlobs() >= 0) {
428 CHECK_EQ(ExactNumBottomBlobs(), bottom.size())
429 << type() << " Layer takes " << ExactNumBottomBlobs()
430 << " bottom blob(s) as input.";
432 if (MinBottomBlobs() >= 0) {
433 CHECK_LE(MinBottomBlobs(), bottom.size())
434 << type() << " Layer takes at least " << MinBottomBlobs()
435 << " bottom blob(s) as input.";
437 if (MaxBottomBlobs() >= 0) {
438 CHECK_GE(MaxBottomBlobs(), bottom.size())
439 << type() << " Layer takes at most " << MaxBottomBlobs()
440 << " bottom blob(s) as input.";
442 if (ExactNumTopBlobs() >= 0) {
443 CHECK_EQ(ExactNumTopBlobs(), top.size())
444 << type() << " Layer produces " << ExactNumTopBlobs()
445 << " top blob(s) as output.";
447 if (MinTopBlobs() >= 0) {
448 CHECK_LE(MinTopBlobs(), top.size())
449 << type() << " Layer produces at least " << MinTopBlobs()
450 << " top blob(s) as output.";
452 if (MaxTopBlobs() >= 0) {
453 CHECK_GE(MaxTopBlobs(), top.size())
454 << type() << " Layer produces at most " << MaxTopBlobs()
455 << " top blob(s) as output.";
457 if (EqualNumBottomTopBlobs()) {
458 CHECK_EQ(bottom.size(), top.size())
459 << type() << " Layer produces one top blob as output for each "
460 << "bottom blob input.";
465 * Called by SetUp to initialize the weights associated with any top blobs in
466 * the loss function. Store non-zero loss weights in the diff blob.
468 inline void SetLossWeights(const vector<Blob<Dtype>*>& top) {
469 const int num_loss_weights = layer_param_.loss_weight_size();
470 if (num_loss_weights) {
471 CHECK_EQ(top.size(), num_loss_weights) << "loss_weight must be "
472 "unspecified or specified once per top blob.";
473 for (int top_id = 0; top_id < top.size(); ++top_id) {
474 const Dtype loss_weight = layer_param_.loss_weight(top_id);
475 if (loss_weight == Dtype(0)) { continue; }
476 this->set_loss(top_id, loss_weight);
477 const int count = top[top_id]->count();
478 Dtype* loss_multiplier = top[top_id]->mutable_cpu_diff();
479 caffe_set(count, loss_weight, loss_multiplier);
485 DISABLE_COPY_AND_ASSIGN(Layer);
488 #ifdef LAYER_PERF_STAT
489 perf_stat time_stat_;
490 perf_stat pmu_stat_[16];
492 #endif //USE_PROFILING
496 #ifndef LAYER_PERF_STAT
497 // Forward and backward wrappers. You should implement the cpu and
498 // gpu specific implementations instead, and should not change these
500 template <typename Dtype>
501 inline Dtype Layer<Dtype>::Forward(const vector<Blob<Dtype>*>& bottom,
502 const vector<Blob<Dtype>*>& top) {
504 Reshape(bottom, top);
505 switch (Caffe::mode()) {
507 Forward_cpu(bottom, top);
508 for (int top_id = 0; top_id < top.size(); ++top_id) {
509 if (!this->loss(top_id)) { continue; }
510 const int count = top[top_id]->count();
511 const Dtype* data = top[top_id]->cpu_data();
512 const Dtype* loss_weights = top[top_id]->cpu_diff();
513 loss += caffe_cpu_dot(count, data, loss_weights);
517 Forward_gpu(bottom, top);
519 for (int top_id = 0; top_id < top.size(); ++top_id) {
520 if (!this->loss(top_id)) { continue; }
521 const int count = top[top_id]->count();
522 const Dtype* data = top[top_id]->gpu_data();
523 const Dtype* loss_weights = top[top_id]->gpu_diff();
525 caffe_gpu_dot(count, data, loss_weights, &blob_loss);
531 LOG(FATAL) << "Unknown caffe mode.";
538 template <typename Dtype>
539 inline void Layer<Dtype>::Backward(const vector<Blob<Dtype>*>& top,
540 const vector<bool>& propagate_down,
541 const vector<Blob<Dtype>*>& bottom) {
542 switch (Caffe::mode()) {
544 Backward_cpu(top, propagate_down, bottom);
547 Backward_gpu(top, propagate_down, bottom);
550 LOG(FATAL) << "Unknown caffe mode.";
554 // Serialize LayerParameter to protocol buffer
555 template <typename Dtype>
556 void Layer<Dtype>::ToProto(LayerParameter* param, bool write_diff) {
558 param->CopyFrom(layer_param_);
559 param->clear_blobs();
560 for (int i = 0; i < blobs_.size(); ++i) {
561 blobs_[i]->ToProto(param->add_blobs(), write_diff);
567 #endif // CAFFE_LAYER_H_