layer->exportTo(exporter, method);
}
-void LayerNode::read(std::ifstream &file, bool opt_var, bool load_opt_var) {
+void LayerNode::read(std::ifstream &file, bool opt_var) {
NNTR_THROW_IF(!run_context, std::runtime_error)
<< __func__ << " layer needs to be finalized first!";
if (opt_var) {
if (run_context->weightHasGradient(i)) {
for (unsigned int j = 0; j < run_context->getNumWeightOptVar(i);
++j) {
- if (load_opt_var) {
- run_context->getWeightOptVar(i, j).read(file);
- } else {
- file.seekg(run_context->getWeightOptVar(i, j).bytes(),
- std::ios::cur);
- }
+ run_context->getWeightOptVar(i, j).read(file);
}
}
}
* @brief read layer Weight & Bias data from file
* @param file input file stream
* @param bool read optimizer variables
- * @param bool load optimizer variables
*/
- void read(std::ifstream &file, bool opt_var = false,
- bool load_opt_var = true);
+ void read(std::ifstream &file, bool opt_var = false);
/**
* @brief save layer Weight & Bias data from file
(*iter)->save(model_file);
}
- model_file.write((char *)&epoch_idx, sizeof(epoch_idx));
- model_file.write((char *)&iter, sizeof(iter));
-
opt->save(model_file);
- unsigned int write_op_var = 0;
- if (istrequal(opt->getType(), "adam")) {
- write_op_var = 1;
- }
-
- model_file.write((char *)&write_op_var, sizeof(write_op_var));
if (istrequal(opt->getType(), "adam")) {
for (auto iter = model_graph.cbegin(); iter != model_graph.cend();
}
}
+ model_file.write((char *)&epoch_idx, sizeof(epoch_idx));
+ model_file.write((char *)&iter, sizeof(iter));
+
model_file.close();
break;
}
-
case ml::train::ModelFormat::MODEL_FORMAT_INI:
saveModelIni(file_path);
break;
/// read. so, after this line, additional read shouldn't be called
model_file.seekg(bin_file_pos);
+ if (istrequal(opt->getType(), "adam")) {
+ char opt_type[4];
+ model_file.read(opt_type, 4);
+ if (istrequal(opt_type, "adam")) {
+ for (auto iter = model_graph.cbegin(); iter != model_graph.cend();
+ iter++) {
+ (*iter)->read(model_file, true);
+ }
+ }
+ }
+
checkedRead(model_file, (char *)&epoch_idx, sizeof(epoch_idx),
"[NeuralNetwork::readModel] failed to read epoch_idx");
checkedRead(model_file, (char *)&iter, sizeof(iter),
"[NeuralNetwork::readModel] failed to read iteration");
-
- opt->read(model_file);
- unsigned int exist_op_var = 0;
-
- checkedRead(model_file, (char *)&exist_op_var, sizeof(exist_op_var));
-
- if (istrequal(opt->getType(), "adam") && exist_op_var) {
- bool load_opt_var = opt->is_load_var();
- for (auto iter = model_graph.cbegin(); iter != model_graph.cend();
- iter++) {
- (*iter)->read(model_file, true, load_opt_var);
- }
- }
-
} catch (...) {
std::cerr << "failed to read epoch idx, proceeding with default index\n";
}
namespace nntrainer {
-Adam::Adam() :
- adam_props(PropsB1(), PropsB2(), PropsEpsilon(), TorchRef(), LoadVar()) {
+Adam::Adam() : adam_props(PropsB1(), PropsB2(), PropsEpsilon(), TorchRef()) {
/** default properties */
setProperty({"learning_rate=0.001"});
- auto &[b1, b2, eps, torch_ref, load_mv] = adam_props;
+ auto &[b1, b2, eps, torch_ref] = adam_props;
b1.set(0.9f);
b2.set(0.999f);
eps.set(1.0e-7f);
torch_ref.set(false);
- load_mv.set(true);
}
Adam::~Adam() {}
// This is implementation of adam from original paper.
// This is not deleted intentionally.
unsigned int iteration = context.getIteration();
- float biasCorrection1 = 1.0f - pow(beta1, iteration + 1);
- float biasCorrection2 = 1.0f - pow(beta2, iteration + 1);
+ float biasCorrection1 = 1 - pow(beta1, iteration + 1);
+ float biasCorrection2 = 1 - pow(beta2, iteration + 1);
Tensor &wm = context.getOptimizerVariable(AdamParams::wm);
Tensor &wv = context.getOptimizerVariable(AdamParams::wv);
if (torch_ref) {
Tensor denom = wv.apply(sqrtFloat);
- denom.divide_i(sqrtDouble(biasCorrection2));
+ denom.divide_i(sqrtFloat(biasCorrection2));
denom.add_i(epsilon);
wm.divide(denom, x_grad);
};
/**
- * @brief load momentum
- *
- */
-class LoadVar : public Property<bool> {
-public:
- static constexpr const char *key = "load_var"; /**< unique key to access */
- using prop_tag = bool_prop_tag; /**< property type */
-};
-
-/**
* @class Adam optimizer class
* @brief Adam optimizer
*/
*/
void setProperty(const std::vector<std::string> &values) override;
- /**
- * @copydoc Optimizer::is_load_var()
- */
- bool is_load_var() { return std::get<LoadVar>(adam_props).get(); }
-
private:
- std::tuple<PropsB1, PropsB2, PropsEpsilon, TorchRef, LoadVar> adam_props;
+ std::tuple<PropsB1, PropsB2, PropsEpsilon, TorchRef> adam_props;
};
} /* namespace nntrainer */
}
void Optimizer::read(std::ifstream &file) {
- std::string loaded_type;
- unsigned int opt_type = ml::train::OptimizerType::UNKNOWN;
- checkedRead(file, (char *)&opt_type, sizeof(opt_type));
- switch (opt_type) {
- case ml::train::OptimizerType::ADAM:
- loaded_type = "adam";
- break;
- case ml::train::OptimizerType::SGD:
- loaded_type = "sgd";
- break;
- default:
- break;
- }
+ std::string loaded_type = readString(file);
if (loaded_type != getType()) {
throw std::runtime_error(
}
}
-void Optimizer::save(std::ofstream &file) {
- unsigned int opt_type = ml::train::OptimizerType::UNKNOWN;
- if (istrequal(getType(), "adam"))
- opt_type = ml::train::OptimizerType::ADAM;
- if (istrequal(getType(), "sgd"))
- opt_type = ml::train::OptimizerType::SGD;
- file.write((char *)&opt_type, sizeof(opt_type));
-}
+void Optimizer::save(std::ofstream &file) { writeString(file, getType()); }
} // namespace nntrainer
virtual void save(std::ofstream &file);
/**
- * @brief get the option of loading optimizer variables
- */
- virtual bool is_load_var() { return false; };
-
- /**
* @brief Get dimension of extra variables if the optimizer needs any.
* @param dim Dimension of tensor to be added as a optimizer variable
* @return Vector of dimensions