}
message BatchNormParameter {
- // If false, accumulate global mean/variance values via a moving average. If
- // true, use those accumulated values instead of computing mean/variance
- // across the batch.
+ // If false, normalization is performed over the current mini-batch
+ // and global statistics are accumulated (but not yet used) by a moving
+ // average.
+ // If true, those accumulated mean and variance values are used for the
+ // normalization.
+ // By default, it is set to false when the network is in the training
+ // phase and true when the network is in the testing phase.
optional bool use_global_stats = 1;
- // How much does the moving average decay each iteration?
+ // What fraction of the moving average remains each iteration?
+ // Smaller values make the moving average decay faster, giving more
+ // weight to the recent values.
+ // Each iteration updates the moving average @f$S_{t-1}@f$ with the
+ // current mean @f$ Y_t @f$ by
+ // @f$ S_t = (1-\beta)Y_t + \beta \cdot S_{t-1} @f$, where @f$ \beta @f$
+ // is the moving_average_fraction parameter.
optional float moving_average_fraction = 2 [default = .999];
// Small value to add to the variance estimate so that we don't divide by
// zero.