#include <layer_internal.h>
#include <nntrainer_error.h>
#include <nntrainer_log.h>
-#include <optimizer_factory.h>
+#include <optimizer_internal.h>
#include <parse_util.h>
#include <util_func.h>
activation_type = acti;
}
-int Layer::setOptimizer(std::shared_ptr<Optimizer> opt) {
- this->opt = opt;
- this->opt->addOptimizerVariable(weights);
- return ML_ERROR_NONE;
-}
-
int Layer::checkValidation() {
int status = ML_ERROR_NONE;
for (auto const &w : weights)
weights.push_back(w.clone());
- // TODO: fix this #630
- this->opt = l->opt;
this->input_dim = l->input_dim;
this->output_dim = l->output_dim;
this->input.copy(l->input);
return out;
}
-sharedConstTensors Layer::backwarding_with_val(int iteration,
- sharedConstTensors deriv,
- sharedConstTensors in) {
+sharedConstTensors
+Layer::backwarding_with_val(int iteration, sharedConstTensors deriv,
+ sharedConstTensors in,
+ std::shared_ptr<Optimizer> optimizer) {
for (unsigned int i = 0; i < num_outputs; ++i) {
net_hidden[i]->var = deriv[i]->clone();
// TODO Need to fix to use LossLayer::type instead of "loss". But cyclic
// includes!
if (istrequal(getType(), "loss")) {
- backwarding(iteration, in);
+ backwarding(in);
} else {
- backwarding(iteration, deriv);
+ backwarding(deriv);
}
+ applyGradient(iteration, optimizer);
nntrainer::sharedConstTensors out;
for (unsigned int i = 0; i < num_inputs; ++i) {
for (auto &weight : weights) {
weight.getVariableRef().read(file);
}
- if (opt)
- opt->read(file);
}
void Layer::save(std::ofstream &file) {
for (auto &weight : weights) {
weight.getVariableRef().save(file);
}
- if (opt)
- opt->save(file);
-}
-
-void Layer::applyGradient(unsigned int iteration) {
- if (trainable && !weights.empty()) {
- opt->apply_gradients(weights, iteration);
- }
}
int Layer::setProperty(std::vector<std::string> values) {
* @brief Apply the gradient for the layer
* @param[in] iteration Iteration value for the Optimizer
*/
- virtual void applyGradient(unsigned int iteration);
+ virtual void applyGradient(unsigned int iteration,
+ std::shared_ptr<Optimizer> optimizer) {
+ if (optimizer)
+ optimizer->apply_gradients(weights, iteration);
+ }
/**
* @brief Back Propagate the derivative to the previous layer
* @param[in] in List of Derivative Tensor from the next layer
* @retval Derivative List of Tensor for the previous layer
*/
- virtual void backwarding(int iteration, sharedConstTensors in = {}) {
+ virtual void backwarding(sharedConstTensors in = {}) {
calcGradient(in);
calcDerivative(in);
- applyGradient(iteration);
}
- virtual sharedConstTensors backwarding_with_val(int iteration,
- sharedConstTensors deriv,
- sharedConstTensors in = {});
+ virtual sharedConstTensors
+ backwarding_with_val(int iteration, sharedConstTensors deriv,
+ sharedConstTensors in = {},
+ std::shared_ptr<Optimizer> optimizer = nullptr);
/**
* @brief read layer Weight & Bias data from file
const std::string &value = "");
/**
- * @brief Optimizer Setter
- * @param[in] opt Optimizer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setOptimizer(std::shared_ptr<Optimizer> opt);
-
- /**
- * @brief Get the Optimizer object
- *
- * @return std::shared_ptr<Optimizer> optimizer
- */
- std::shared_ptr<Optimizer> getOptimizer() { return opt; }
-
- /**
* @brief Activation Type Getter
* @retval Activation Type.
*/
std::vector<Tensor> getDerivatives();
+ /**
+ * @brief Get reference to the weights
+ * @retval Reference of the list of weights in the layer
+ */
+ std::vector<Weight> &getWeightsRef() { return weights; }
+
#ifdef ENABLE_TEST
void resizeNetInput(unsigned int size) { net_input.resize(size); }
std::vector<TensorDim> output_dim;
/**
- * @brief Optimizer for this layer
- */
- // TODO: fix with #630
- std::shared_ptr<Optimizer> opt;
-
- /**
* @brief Loss value added by this layer
*/
float loss;
input_str.push_back((*i).str());
}
- std::shared_ptr<nntrainer::Optimizer> op;
- EXPECT_NO_THROW(op = nntrainer::createOptimizer(type));
+ EXPECT_NO_THROW(opt = nntrainer::createOptimizer(type));
- status = op->setProperty(input_str);
+ status = opt->setProperty(input_str);
EXPECT_EQ(status, ML_ERROR_NONE);
- status = layer.setOptimizer(op);
+
+ status = opt->initialize();
EXPECT_EQ(status, ML_ERROR_NONE);
+ EXPECT_NO_THROW(opt->addOptimizerVariable(layer.getWeightsRef()));
+ // status = layer.setOptimizer(op);
+ // EXPECT_EQ(status, ML_ERROR_NONE);
+
return status;
}
nntrainer::Tensor out;
float local_tolerance = tolerance;
nntrainer::Manager manager;
+ std::shared_ptr<nntrainer::Optimizer> opt;
};
class nntrainer_InputLayer
if (layers.size() && nntrainer::istrequal(layers.back()->getType(),
nntrainer::LossLayer::type)) {
if (with_loss) {
- EXPECT_NO_THROW(layers.back()->backwarding(1, {label}));
+ EXPECT_NO_THROW(layers.back()->backwarding({label}));
back_out = MAKE_SHARED_TENSOR(layers.back()->getDerivatives()[0]);
} else {
back_out = def_derivative;
}
for (; idx >= 0; --idx)
- EXPECT_NO_THROW(back_out =
- layers[idx]->backwarding_with_val(1, {back_out})[0]);
+ EXPECT_NO_THROW(back_out = layers[idx]->backwarding_with_val(
+ 1, {back_out}, {}, opt)[0]);
- EXPECT_NO_THROW(back_out = layer.backwarding_with_val(1, {back_out})[0]);
+ EXPECT_NO_THROW(back_out =
+ layer.backwarding_with_val(1, {back_out}, {}, opt)[0]);
matchOutput(*back_out.get(), file_dx);
loadUpdatedWeightsGradients(file_uw, file_g);
nntrainer::Tensor result;
EXPECT_NO_THROW(result = *layer.backwarding_with_val(
- 1, {MAKE_SHARED_TENSOR(derivatives)})[0]);
+ 1, {MAKE_SHARED_TENSOR(derivatives)}, {}, opt)[0]);
matchOutput(result, "tc_fc_1_goldenFCGradientAdam.out");
nntrainer::Tensor backward_in(layer.getOutputDimension()[0]);
loadFile("tc_bn_fc_1_goldenBNLayerBackwardDxIn.out", backward_in);
- nntrainer::Tensor backward_result =
- *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(backward_in)})[0];
+ nntrainer::Tensor backward_result = *layer.backwarding_with_val(
+ 1, {MAKE_SHARED_TENSOR(backward_in)}, {}, opt)[0];
matchOutput(backward_result, "tc_bn_fc_1_goldenBNLayerBackwardDx.out");
}
nntrainer::Tensor backward_in(layer.getOutputDimension()[0]);
loadFile("tc_bn_conv_1_goldenBNLayerBackwardDxIn.out", backward_in);
- nntrainer::Tensor backward_result =
- *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(backward_in)})[0];
+ nntrainer::Tensor backward_result = *layer.backwarding_with_val(
+ 1, {MAKE_SHARED_TENSOR(backward_in)}, {}, opt)[0];
matchOutput(backward_result, "tc_bn_conv_1_goldenBNLayerBackwardDx.out");
}
nntrainer::Tensor backward_in(layer.getOutputDimension()[0]);
loadFile("tc_bn_conv_2_goldenBNLayerBackwardDxIn.out", backward_in);
- nntrainer::Tensor backward_result =
- *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(backward_in)})[0];
+ nntrainer::Tensor backward_result = *layer.backwarding_with_val(
+ 1, {MAKE_SHARED_TENSOR(backward_in)}, {}, opt)[0];
matchOutput(backward_result, "tc_bn_conv_2_goldenBNLayerBackwardDx.out");
}
}
EXPECT_NO_THROW(result = *layer.backwarding_with_val(
- 1, {MAKE_SHARED_TENSOR(derivatives)})[0]);
+ 1, {MAKE_SHARED_TENSOR(derivatives)}, {}, opt)[0]);
auto param_data = layer.getWeights();
const float *weight_grad = param_data[0].getGradient().getData();
derivatives.getData()[i] = 1.0;
}
EXPECT_NO_THROW(result = *layer.backwarding_with_val(
- 1, {MAKE_SHARED_TENSOR(derivatives)})[0]);
+ 1, {MAKE_SHARED_TENSOR(derivatives)}, {}, opt)[0]);
auto param_data = layer.getWeights();
const float *weight_grad = param_data[0].getGradient().getData();
EXPECT_NO_THROW(out =
*layer.forwarding_with_val({MAKE_SHARED_TENSOR(in)})[0]);
EXPECT_NO_THROW(result = *layer.backwarding_with_val(
- 0, {MAKE_SHARED_TENSOR(derivatives)})[0]);
+ 0, {MAKE_SHARED_TENSOR(derivatives)}, {}, opt)[0]);
}
/// @fixme: the output value of this test is around +/- 1.0e+07 which can't
EXPECT_NO_THROW(op = nntrainer::createOptimizer(nntrainer::OptType::SGD));
status = op->setProperty({"learning_rate=1.0"});
EXPECT_EQ(status, ML_ERROR_NONE);
- status = layer1.setOptimizer(op);
- EXPECT_EQ(status, ML_ERROR_NONE);
nntrainer::Conv2DLayer layer2;
status = layer2.setProperty(
EXPECT_NO_THROW(op2 = nntrainer::createOptimizer(nntrainer::OptType::SGD));
status = op2->setProperty({"learning_rate=1.0"});
EXPECT_EQ(status, ML_ERROR_NONE);
- status = layer2.setOptimizer(op2);
- EXPECT_EQ(status, ML_ERROR_NONE);
setOptimizer(nntrainer::OptType::SGD, "learning_rate=1.0");
nntrainer::Tensor result2;
EXPECT_NO_THROW(result2 = *layer2.backwarding_with_val(
- 1, {MAKE_SHARED_TENSOR(derivatives)})[0]);
+ 1, {MAKE_SHARED_TENSOR(derivatives)}, {}, opt)[0]);
- EXPECT_NO_THROW(
- result = *layer1.backwarding_with_val(1, {MAKE_SHARED_TENSOR(result2)})[0]);
+ EXPECT_NO_THROW(result = *layer1.backwarding_with_val(
+ 1, {MAKE_SHARED_TENSOR(result2)}, {}, opt)[0]);
/** Compare second conv */
auto param_data = layer2.getWeights();
derivatives.getData()[i] = 1.0;
}
EXPECT_NO_THROW(result = *layer.backwarding_with_val(
- 1, {MAKE_SHARED_TENSOR(derivatives)})[0]);
+ 1, {MAKE_SHARED_TENSOR(derivatives)}, {}, opt)[0]);
auto param_data = layer.getWeights();
const float *weight_grad = param_data[0].getGradient().getData();
grad.getData()[i] = 1.0;
}
- EXPECT_NO_THROW(
- in = *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(grad)})[0]);
+ EXPECT_NO_THROW(in = *layer.backwarding_with_val(
+ 1, {MAKE_SHARED_TENSOR(grad)}, {}, opt)[0]);
matchOutput(in, "tc_pooling2d_1_goldenPooling2DmaxGrad.out");
}
grad->getData()[i] = 1.0;
}
- EXPECT_NO_THROW(in = *layer.backwarding_with_val(1, {grad})[0]);
+ EXPECT_NO_THROW(in = *layer.backwarding_with_val(1, {grad}, {}, opt)[0]);
matchOutput(in, "tc_pooling2d_1_goldenPooling2DaverageGrad.out");
}
grad.getData()[i] = 1.0;
}
- EXPECT_NO_THROW(
- in = *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(grad)})[0]);
+ EXPECT_NO_THROW(in = *layer.backwarding_with_val(
+ 1, {MAKE_SHARED_TENSOR(grad)}, {}, opt)[0]);
matchOutput(in, "tc_pooling2d_1_goldenPooling2Dglobal_maxGrad.out");
}
grad.getData()[i] = 1.0;
}
- EXPECT_NO_THROW(
- in = *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(grad)})[0]);
+ EXPECT_NO_THROW(in = *layer.backwarding_with_val(
+ 1, {MAKE_SHARED_TENSOR(grad)}, {}, opt)[0]);
matchOutput(in, "tc_pooling2d_1_goldenPooling2Dglobal_averageGrad.out");
}
loadFile("tc_pooling2d_1_goldenPooling2Dmax.out", out);
EXPECT_NO_THROW(
- in = *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(out)})[0]);
+ in = *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(out)}, {}, opt)[0]);
EXPECT_EQ(in.getDim(), nntrainer::TensorDim(1, 2, 4, 4));
matchOutput(in, "tc_pooling2d_1_goldenPooling2Dmax.out");
loadFile("tc_pooling2d_2_goldenPooling2Dmax.out", out);
EXPECT_NO_THROW(
- in = *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(out)})[0]);
+ in = *layer.backwarding_with_val(1, {MAKE_SHARED_TENSOR(out)}, {}, opt)[0]);
EXPECT_EQ(in.getDim(), nntrainer::TensorDim(2, 2, 4, 4));
matchOutput(in, "tc_pooling2d_2_goldenPooling2Dmax.out");
std::make_unique<nntrainer::NetBuffers>();
layer.setInputBuffer(0, in_buffer);
layer.setOutputBuffer(0, out_buffer);
- EXPECT_THROW(layer.backwarding(1, {MAKE_SHARED_TENSOR(a)}),
- std::runtime_error);
+ EXPECT_THROW(layer.backwarding({MAKE_SHARED_TENSOR(a)}), std::runtime_error);
}
TEST(nntrainer_LossLayer, forward_loss_forward_entropy_n) {
std::make_unique<nntrainer::NetBuffers>();
layer.setInputBuffer(0, in_buffer);
layer.setOutputBuffer(0, out_buffer);
- EXPECT_THROW(layer.backwarding(1, {MAKE_SHARED_TENSOR(a)}),
- std::runtime_error);
+ EXPECT_THROW(layer.backwarding({MAKE_SHARED_TENSOR(a)}), std::runtime_error);
}
/**