const vector<shared_ptr<Blob<Dtype> > >& history() { return history_; }
protected:
- void PreSolve();
Dtype GetLearningRate();
virtual void ApplyUpdate();
virtual void Normalize(int param_id);
virtual void Regularize(int param_id);
virtual void ComputeUpdateValue(int param_id, Dtype rate);
+ virtual void PreSolve();
virtual void ClipGradients();
virtual void SnapshotSolverState(const string& model_filename);
virtual void SnapshotSolverStateToBinaryProto(const string& model_filename);
class AdaDeltaSolver : public SGDSolver<Dtype> {
public:
explicit AdaDeltaSolver(const SolverParameter& param)
- : SGDSolver<Dtype>(param) { constructor_sanity_check(); }
+ : SGDSolver<Dtype>(param) { PreSolve(); constructor_sanity_check(); }
explicit AdaDeltaSolver(const string& param_file)
- : SGDSolver<Dtype>(param_file) { constructor_sanity_check(); }
+ : SGDSolver<Dtype>(param_file) { PreSolve(); constructor_sanity_check(); }
protected:
virtual void PreSolve();
template <typename Dtype>
void AdaDeltaSolver<Dtype>::PreSolve() {
- // Initialize the history
- vector<shared_ptr<Blob<Dtype> > >& net_params = this->net_->params();
- this->history_.clear();
- this->update_.clear();
- this->temp_.clear();
- for (int i = 0; i < net_params.size(); ++i) {
- const Blob<Dtype>* net_param = net_params[i].get();
- this->history_.push_back(shared_ptr<Blob<Dtype> >(new Blob<Dtype>(
- net_param->num(), net_param->channels(), net_param->height(),
- net_param->width())));
- this->update_.push_back(shared_ptr<Blob<Dtype> >(new Blob<Dtype>(
- net_param->num(), net_param->channels(), net_param->height(),
- net_param->width())));
- this->temp_.push_back(shared_ptr<Blob<Dtype> >(new Blob<Dtype>(
- net_param->num(), net_param->channels(), net_param->height(),
- net_param->width())));
- }
+ // Add the extra history entries for AdaDelta after those from
+ // SGDSolver::PreSolve
+ const vector<shared_ptr<Blob<Dtype> > >& net_params = this->net_->params();
for (int i = 0; i < net_params.size(); ++i) {
- const Blob<Dtype>* net_param = net_params[i].get();
- this->history_.push_back(shared_ptr<Blob<Dtype> >(new Blob<Dtype>(
- net_param->num(), net_param->channels(), net_param->height(),
- net_param->width())));
+ const vector<int>& shape = net_params[i]->shape();
+ this->history_.push_back(
+ shared_ptr<Blob<Dtype> >(new Blob<Dtype>(shape)));
}
}
template <typename Dtype>
void AdaDeltaSolver<Dtype>::ComputeUpdateValue() {
- vector<shared_ptr<Blob<Dtype> > >& net_params = this->net_->params();
- vector<float>& net_params_weight_decay = this->net_->params_weight_decay();
+ const vector<shared_ptr<Blob<Dtype> > >& net_params = this->net_->params();
+ const vector<float>& net_params_weight_decay =
+ this->net_->params_weight_decay();
Dtype delta = this->param_.delta();
Dtype momentum = this->param_.momentum();
Dtype weight_decay = this->param_.weight_decay();
const Dtype kLearningRate = 0.0;
const Dtype kWeightDecay = 0.0;
const Dtype kMomentum = 0.95;
- const int kNumIters = 500;
+ const int kNumIters = 4;
for (int i = 0; i <= kNumIters; ++i) {
this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum, i);
}
const Dtype kLearningRate = 0.0;
const Dtype kWeightDecay = 0.1;
const Dtype kMomentum = 0.95;
- const int kNumIters = 500;
+ const int kNumIters = 4;
for (int i = 0; i <= kNumIters; ++i) {
this->TestLeastSquaresUpdate(kLearningRate, kWeightDecay, kMomentum, i);
}