}
}
+template <typename Dtype>
+void AdamSolver<Dtype>::AdamPreSolve() {
+ // Add the extra history entries for Adam after those from
+ // SGDSolver::PreSolve
+ const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
+ for (int i = 0; i < net_params.size(); ++i) {
+ const vector<int>& shape = net_params[i]->shape();
+ this->history_.push_back(
+ shared_ptr<Blob<Dtype> >(new Blob<Dtype>(shape)));
+ }
+}
+
+template <typename Dtype>
+void AdamSolver<Dtype>::ComputeUpdateValue(int param_id, Dtype rate) {
+ const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
+ const vector<float>& net_params_lr = this->net_->params_lr();
+ Dtype local_rate = rate * net_params_lr[param_id];
+ const Dtype beta1 = this->param_.momentum();
+ const Dtype beta2 = this->param_.momentum2();
+
+ // we create aliases for convenience
+ size_t update_history_offset = net_params.size();
+ Blob<Dtype>* val_m = this->history_[param_id].get();
+ Blob<Dtype>* val_v = this->history_[param_id + update_history_offset].get();
+ Blob<Dtype>* val_t = this->temp_[param_id].get();
+
+ const int t = this->iter_ + 1;
+ const Dtype correction = std::sqrt(Dtype(1) - pow(beta2, t)) /
+ (Dtype(1.) - pow(beta1, t));
+ const int N = net_params[param_id]->count();
+ const Dtype eps_hat = this->param_.delta();
+
+ switch (Caffe::mode()) {
+ case Caffe::CPU: {
+ // update m <- \beta_1 m_{t-1} + (1-\beta_1)g_t
+ caffe_cpu_axpby(N, Dtype(1)-beta1,
+ net_params[param_id]->cpu_diff(), beta1,
+ val_m->mutable_cpu_data());
+
+ // update v <- \beta_2 m_{t-1} + (1-\beta_2)g_t^2
+ caffe_mul(N,
+ net_params[param_id]->cpu_diff(),
+ net_params[param_id]->cpu_diff(),
+ val_t->mutable_cpu_data());
+ caffe_cpu_axpby(N, Dtype(1)-beta2,
+ val_t->cpu_data(), beta2,
+ val_v->mutable_cpu_data());
+
+ // set update
+ caffe_powx(N,
+ val_v->cpu_data(), Dtype(0.5),
+ val_t->mutable_cpu_data());
+ caffe_add_scalar(N, eps_hat, val_t->mutable_cpu_data());
+ caffe_div(N,
+ val_m->cpu_data(),
+ val_t->cpu_data(),
+ val_t->mutable_cpu_data());
+
+ caffe_cpu_scale(N, local_rate*correction,
+ val_t->cpu_data(),
+ net_params[param_id]->mutable_cpu_diff());
+ break;
+ }
+ case Caffe::GPU: {
+#ifndef CPU_ONLY
+ // update m <- \beta_1 m_{t-1} + (1-\beta_1)g_t
+ caffe_gpu_axpby(N, Dtype(1)-beta1,
+ net_params[param_id]->gpu_diff(), beta1,
+ val_m->mutable_gpu_data());
+
+ // update v <- \beta_2 m_{t-1} + (1-\beta_2)g_t^2
+ caffe_gpu_mul(N,
+ net_params[param_id]->gpu_diff(),
+ net_params[param_id]->gpu_diff(),
+ val_t->mutable_gpu_data());
+ caffe_gpu_axpby(N, Dtype(1)-beta2,
+ val_t->gpu_data(), beta2,
+ val_v->mutable_gpu_data());
+
+ // set update
+ caffe_gpu_powx(N,
+ val_v->gpu_data(), Dtype(0.5),
+ val_t->mutable_gpu_data());
+ caffe_gpu_add_scalar(N, eps_hat,
+ val_t->mutable_gpu_data());
+ caffe_gpu_div(N,
+ val_m->gpu_data(),
+ val_t->gpu_data(),
+ val_t->mutable_gpu_data());
+
+ caffe_gpu_scale(N, local_rate*correction,
+ val_t->gpu_data(),
+ net_params[param_id]->mutable_gpu_diff());
+#else
+ NO_GPU;
+#endif
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
+ }
+}
+
INSTANTIATE_CLASS(Solver);
INSTANTIATE_CLASS(SGDSolver);
INSTANTIATE_CLASS(NesterovSolver);
INSTANTIATE_CLASS(AdaGradSolver);
INSTANTIATE_CLASS(RMSPropSolver);
INSTANTIATE_CLASS(AdaDeltaSolver);
+INSTANTIATE_CLASS(AdamSolver);
} // namespace caffe
// TODO this is brittle and the hdf5 file should be checked instead.
int num_, channels_, height_, width_;
bool share_;
- Dtype delta_; // Stability constant for AdaGrad.
+ Dtype delta_; // Stability constant for RMSProp, AdaGrad, AdaDelta and Adam
// Test data: check out generate_sample_data.py in the same directory.
string* input_file_;
LOG(FATAL) << "Unknown Caffe mode: " << Caffe::mode();
}
InitSolver(param);
- delta_ = (solver_type() == SolverParameter_SolverType_ADAGRAD ||
- solver_type() == SolverParameter_SolverType_RMSPROP ||
- solver_type() == SolverParameter_SolverType_ADADELTA) ?
- param.delta() : 0;
+ delta_ = param.delta();
}
string RunLeastSquaresSolver(const Dtype learning_rate,
// updated_params will store the updated weight and bias results,
// using the blobs' diffs to hold the update values themselves.
void ComputeLeastSquaresUpdate(const Dtype learning_rate,
- const Dtype weight_decay, const Dtype momentum,
+ const Dtype weight_decay, const Dtype momentum, const int num_iters,
vector<shared_ptr<Blob<Dtype> > >* updated_params) {
const int N = num_;
const int D = channels_ * height_ * width_;
((i == D) ? bias.cpu_data()[0] : weights.cpu_data()[i]);
// Finally, compute update.
const vector<shared_ptr<Blob<Dtype> > >& history = solver_->history();
- if (solver_type() != SolverParameter_SolverType_ADADELTA) {
+ if (solver_type() != SolverParameter_SolverType_ADADELTA
+ && solver_type() != SolverParameter_SolverType_ADAM) {
ASSERT_EQ(2, history.size()); // 1 blob for weights, 1 for bias
} else {
ASSERT_EQ(4, history.size()); // additional blobs for update history
case SolverParameter_SolverType_ADADELTA:
{
const Dtype update_history_value = (i == D) ?
- history[3]->cpu_data()[0] : history[2]->cpu_data()[i];
+ history[1 + num_param_blobs]->cpu_data()[0] :
+ history[0 + num_param_blobs]->cpu_data()[i];
const Dtype weighted_gradient_average =
momentum * history_value + (1 - momentum) * (grad * grad);
update_value = grad * std::sqrt((update_history_value + delta_) /
- (weighted_gradient_average + delta_));
+ (weighted_gradient_average + delta_)) * learning_rate;
// not actually needed, just here for illustrative purposes
// const Dtype weighted_update_average =
// momentum * update_history_value + (1 - momentum) * (update_value);
break;
}
+ case SolverParameter_SolverType_ADAM: {
+ const Dtype momentum2 = 0.999;
+ const Dtype m = history_value;
+ const Dtype v = (i == D) ?
+ history[1 + num_param_blobs]->cpu_data()[0] :
+ history[0 + num_param_blobs]->cpu_data()[i];
+ const Dtype val_m = (1 - momentum) * grad + momentum * m;
+ const Dtype val_v = (1 - momentum2) * grad * grad + momentum2 * v;
+ Dtype alpha_t = learning_rate *
+ std::sqrt(Dtype(1) - pow(momentum2, num_iters)) /
+ (Dtype(1.) - pow(momentum, num_iters));
+ update_value = alpha_t * val_m / (std::sqrt(val_v) + delta_);
+ break;
+ }
default:
LOG(FATAL) << "Unknown solver type: " << solver_type();
}
// Compute the (K+1)th update using the analytic least squares gradient.
vector<shared_ptr<Blob<Dtype> > > updated_params;
ComputeLeastSquaresUpdate(learning_rate, weight_decay, momentum,
- &updated_params);
+ iter_to_check + 1, &updated_params);
// Reinitialize the solver and run K+1 solver iterations.
num_ = kNum;
TYPED_TEST(AdaDeltaSolverTest, TestAdaDeltaLeastSquaresUpdate) {
typedef typename TypeParam::Dtype Dtype;
- const Dtype kLearningRate = 1.0;
+ const Dtype kLearningRate = 0.1;
this->TestLeastSquaresUpdate(kLearningRate);
}
TYPED_TEST(AdaDeltaSolverTest, TestAdaDeltaLeastSquaresUpdateWithWeightDecay) {
typedef typename TypeParam::Dtype Dtype;
- const Dtype kLearningRate = 1.0;
+ const Dtype kLearningRate = 0.1;
const Dtype kWeightDecay = 0.5;
const Dtype kMomentum = 0.95;
this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum);
TYPED_TEST(AdaDeltaSolverTest, TestAdaDeltaLeastSquaresUpdateWithHalfMomentum) {
typedef typename TypeParam::Dtype Dtype;
- const Dtype kLearningRate = 1.0;
+ const Dtype kLearningRate = 0.1;
const Dtype kWeightDecay = 0.0;
const Dtype kMomentum = 0.5;
const int kNumIters = 1;
for (int i = 0; i <= kNumIters; ++i) {
- this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum);
+ this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum);
}
}
TYPED_TEST(AdaDeltaSolverTest, TestAdaDeltaLeastSquaresUpdateWithMomentum) {
typedef typename TypeParam::Dtype Dtype;
- const Dtype kLearningRate = 1.0;
+ const Dtype kLearningRate = 0.1;
const Dtype kWeightDecay = 0.0;
const Dtype kMomentum = 0.95;
const int kNumIters = 1;
for (int i = 0; i <= kNumIters; ++i) {
- this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum);
+ this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum);
}
}
TYPED_TEST(AdaDeltaSolverTest, TestLeastSquaresUpdateWithMomentumMultiIter) {
typedef typename TypeParam::Dtype Dtype;
- const Dtype kLearningRate = 1.0;
+ const Dtype kLearningRate = 0.1;
const Dtype kWeightDecay = 0.0;
const Dtype kMomentum = 0.95;
const int kNumIters = 4;
for (int i = 0; i <= kNumIters; ++i) {
- this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum, i);
+ this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum, i);
}
}
TYPED_TEST(AdaDeltaSolverTest, TestAdaDeltaLeastSquaresUpdateWithEverything) {
typedef typename TypeParam::Dtype Dtype;
- const Dtype kLearningRate = 1.0;
+ const Dtype kLearningRate = 0.1;
const Dtype kWeightDecay = 0.1;
const Dtype kMomentum = 0.95;
const int kNumIters = 4;
for (int i = 0; i <= kNumIters; ++i) {
- this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum, i);
+ this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum, i);
}
}
TYPED_TEST(AdaDeltaSolverTest,
TestAdaDeltaLeastSquaresUpdateWithEverythingShare) {
typedef typename TypeParam::Dtype Dtype;
- const Dtype kLearningRate = 1.0;
+ const Dtype kLearningRate = 0.1;
const Dtype kWeightDecay = 0.1;
const Dtype kMomentum = 0.95;
const int kNumIters = 4;
this->share_ = true;
for (int i = 0; i <= kNumIters; ++i) {
- this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum, i);
+ this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum, i);
}
}
TYPED_TEST(AdaDeltaSolverTest, TestLeastSquaresUpdateWithEverythingAccum) {
typedef typename TypeParam::Dtype Dtype;
- const Dtype kLearningRate = 1.0;
+ const Dtype kLearningRate = 0.1;
const Dtype kWeightDecay = 0.1;
const Dtype kMomentum = 0.95;
const int kNumIters = 4;
TYPED_TEST(AdaDeltaSolverTest, TestLeastSquaresUpdateWithEverythingAccumShare) {
typedef typename TypeParam::Dtype Dtype;
- const Dtype kLearningRate = 1.0;
+ const Dtype kLearningRate = 0.1;
const Dtype kWeightDecay = 0.1;
const Dtype kMomentum = 0.95;
const int kNumIters = 4;
TYPED_TEST(AdaDeltaSolverTest, TestSnapshot) {
typedef typename TypeParam::Dtype Dtype;
- const Dtype kLearningRate = 1.0;
+ const Dtype kLearningRate = 0.1;
const Dtype kWeightDecay = 0.1;
const Dtype kMomentum = 0.95;
const int kNumIters = 4;
TYPED_TEST(AdaDeltaSolverTest, TestSnapshotShare) {
typedef typename TypeParam::Dtype Dtype;
- const Dtype kLearningRate = 1.0;
+ const Dtype kLearningRate = 0.1;
const Dtype kWeightDecay = 0.1;
const Dtype kMomentum = 0.95;
const int kNumIters = 4;
}
template <typename TypeParam>
+class AdamSolverTest : public GradientBasedSolverTest<TypeParam> {
+ typedef typename TypeParam::Dtype Dtype;
+
+ protected:
+ virtual void InitSolver(const SolverParameter& param) {
+ SolverParameter new_param = param;
+ const Dtype momentum = 0.9;
+ new_param.set_momentum(momentum);
+ const Dtype momentum2 = 0.999;
+ new_param.set_momentum2(momentum2);
+ this->solver_.reset(new AdamSolver<Dtype>(new_param));
+ }
+ virtual SolverParameter_SolverType solver_type() {
+ return SolverParameter_SolverType_ADAM;
+ }
+};
+
+TYPED_TEST_CASE(AdamSolverTest, TestDtypesAndDevices);
+
+TYPED_TEST(AdamSolverTest, TestAdamLeastSquaresUpdate) {
+ typedef typename TypeParam::Dtype Dtype;
+ const Dtype kLearningRate = 0.01;
+ const Dtype kWeightDecay = 0;
+ const Dtype kMomentum = 0.9;
+ this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum);
+}
+
+TYPED_TEST(AdamSolverTest, TestAdamLeastSquaresUpdateWithWeightDecay) {
+ typedef typename TypeParam::Dtype Dtype;
+ const Dtype kLearningRate = 0.01;
+ const Dtype kWeightDecay = 0.5;
+ const Dtype kMomentum = 0.9;
+ this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum);
+}
+
+TYPED_TEST(AdamSolverTest, TestAdamLeastSquaresUpdateWithEverything) {
+ typedef typename TypeParam::Dtype Dtype;
+ const Dtype kLearningRate = 0.01;
+ const Dtype kWeightDecay = 0.5;
+ const Dtype kMomentum = 0.9;
+ const int kNumIters = 4;
+ for (int i = 0; i <= kNumIters; ++i) {
+ this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum, i);
+ }
+}
+
+TYPED_TEST(AdamSolverTest, TestAdamLeastSquaresUpdateWithEverythingShare) {
+ typedef typename TypeParam::Dtype Dtype;
+ const Dtype kLearningRate = 0.01;
+ const Dtype kWeightDecay = 0.5;
+ const Dtype kMomentum = 0.9;
+ const int kNumIters = 4;
+ this->share_ = true;
+ for (int i = 0; i <= kNumIters; ++i) {
+ this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum, i);
+ }
+}
+
+TYPED_TEST(AdamSolverTest, TestLeastSquaresUpdateWithEverythingAccum) {
+ typedef typename TypeParam::Dtype Dtype;
+ const Dtype kLearningRate = 0.01;
+ const Dtype kWeightDecay = 0.5;
+ const Dtype kMomentum = 0.9;
+ const int kNumIters = 4;
+ const int kIterSize = 2;
+ this->CheckAccumulation(kLearningRate, kWeightDecay, kMomentum, kNumIters,
+ kIterSize);
+}
+
+TYPED_TEST(AdamSolverTest, TestLeastSquaresUpdateWithEverythingAccumShare) {
+ typedef typename TypeParam::Dtype Dtype;
+ const Dtype kLearningRate = 0.01;
+ const Dtype kWeightDecay = 0.5;
+ const Dtype kMomentum = 0.9;
+ const int kNumIters = 4;
+ const int kIterSize = 2;
+ this->share_ = true;
+ this->CheckAccumulation(kLearningRate, kWeightDecay, kMomentum, kNumIters,
+ kIterSize);
+}
+
+TYPED_TEST(AdamSolverTest, TestSnapshot) {
+ typedef typename TypeParam::Dtype Dtype;
+ const Dtype kLearningRate = 0.01;
+ const Dtype kWeightDecay = 0.5;
+ const Dtype kMomentum = 0.9;
+ const int kNumIters = 4;
+ for (int i = 1; i <= kNumIters; ++i) {
+ this->TestSnapshot(kLearningRate, kWeightDecay, kMomentum, i);
+ }
+}
+
+TYPED_TEST(AdamSolverTest, TestSnapshotShare) {
+ typedef typename TypeParam::Dtype Dtype;
+ const Dtype kLearningRate = 0.01;
+ const Dtype kWeightDecay = 0.5;
+ const Dtype kMomentum = 0.9;
+ const int kNumIters = 4;
+ this->share_ = true;
+ for (int i = 1; i <= kNumIters; ++i) {
+ this->TestSnapshot(kLearningRate, kWeightDecay, kMomentum, i);
+ }
+}
+
+template <typename TypeParam>
class RMSPropSolverTest : public GradientBasedSolverTest<TypeParam> {
typedef typename TypeParam::Dtype Dtype;