Make SGD match python (#15840)

Summary:
Fixes https://github.com/pytorch/pytorch/issues/15530
Pull Request resolved: https://github.com/pytorch/pytorch/pull/15840

Differential Revision: D13608503

Pulled By: goldsborough

fbshipit-source-id: aad17c110d64cbe2c126bccd36d228e4108ffa9a

diff --git a/torch/csrc/api/src/optim/sgd.cpp b/torch/csrc/api/src/optim/sgd.cpp
index 2a407a8..d7f43e8 100644 (file)
--- a/torch/csrc/api/src/optim/sgd.cpp
+++ b/torch/csrc/api/src/optim/sgd.cpp
@@ -23,10 +23,10 @@ void SGD::step() {
       continue;
     }
 
-    auto update = options.learning_rate_ * p.grad();
+    auto update = p.grad();
 
     if (options.weight_decay_ > 0) {
-      update += options.learning_rate_ * options.weight_decay_ * p;
+      update += options.weight_decay_ * p;
     }
 
     if (options.momentum_ != 0) {
@@ -43,7 +43,7 @@ void SGD::step() {
     }
 
     NoGradGuard guard;
-    p.add_(-update);
+    p.add_(-options.learning_rate_ * update);
   }
   iteration_ += 1;
 }