From: Jihoon Lee Date: Fri, 17 Dec 2021 04:26:39 +0000 (+0900) Subject: [RecurrentRealizer] Modify to have connection X-Git-Tag: accepted/tizen/unified/20220323.062643~66 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=02417aa1d7833e5721f6beb9d56d241f3e3378d6;p=platform%2Fcore%2Fml%2Fnntrainer.git [RecurrentRealizer] Modify to have connection This patch modifies recurrent realizer to have connetion **Self evaluation:** 1. Build test: [X]Passed [ ]Failed [ ]Skipped 2. Run test: [X]Passed [ ]Failed [ ]Skipped Signed-off-by: Jihoon Lee --- diff --git a/nntrainer/compiler/recurrent_realizer.cpp b/nntrainer/compiler/recurrent_realizer.cpp index 46095e5..a0af694 100644 --- a/nntrainer/compiler/recurrent_realizer.cpp +++ b/nntrainer/compiler/recurrent_realizer.cpp @@ -97,23 +97,21 @@ RecurrentOutput::RecurrentOutput() {} RecurrentOutput::RecurrentOutput(const Connection &con) { set(con); }; } // namespace props -RecurrentRealizer::RecurrentRealizer( - const std::vector &properties, - const std::vector &input_layers, - const std::vector &end_layers) : - input_layers(input_layers.begin(), input_layers.end()), - end_layers(end_layers), - sequenced_return_layers(), +RecurrentRealizer::RecurrentRealizer(const std::vector &properties, + const std::vector &input_conns, + const std::vector &end_conns) : + input_layers(), + end_conns(end_conns), + sequenced_return_conns(), recurrent_props(new PropTypes( std::vector(), std::vector(), std::vector(), props::UnrollFor(1))) { auto left = loadProperties(properties, *recurrent_props); - /// @note AsSequence must be identifier based (not connection based) for now - /// consider A(layer) outputs a0, a1 connection and a0 needs return seq - /// Then it is impossible to locate a0 and a1 with the same name unless we - /// have some kind of multi,multiout identity layer. Until this is supported, - /// AsSequenced stays as identifier based + /// @todo support AsSequence with index with identity layer + std::transform(input_conns.begin(), input_conns.end(), + std::inserter(this->input_layers, this->input_layers.begin()), + [](const Connection &c) { return c.getName(); }); auto &[inputs, outputs, as_sequence, unroll_for] = *recurrent_props; @@ -123,18 +121,19 @@ RecurrentRealizer::RecurrentRealizer( "different size. input: " << inputs.size() << " output: " << outputs.size(); + /// @todo Deal as sequence as proper connection with identity layer NNTR_THROW_IF(!std::all_of(as_sequence.begin(), as_sequence.end(), - [&end_layers](const std::string &seq) { - return std::find(end_layers.begin(), - end_layers.end(), - seq) != end_layers.end(); + [&end_conns](const Connection &seq) { + return std::find(end_conns.begin(), + end_conns.end(), + seq) != end_conns.end(); }), std::invalid_argument) << "as_sequence property must be subset of end_layers"; std::unordered_set check_seqs; for (auto &name : as_sequence) { - sequenced_return_layers.emplace(name.get()); + sequenced_return_conns.emplace(name.get()); }; NNTR_THROW_IF(!left.empty(), std::invalid_argument) @@ -281,11 +280,11 @@ RecurrentRealizer::realize(const GraphRepresentation &reference) { * @todo support connection using node->remapConnection */ auto naive_output = [](const GraphRepresentation &reference_, - const std::string &con, unsigned unroll_for) { - auto target = con + "/" + std::to_string(unroll_for - 1); + const Connection &con, unsigned unroll_for) { + auto target = con.getName() + "/" + std::to_string(unroll_for - 1); RemapRealizer r([target, con](std::string &name) { if (name == target) { - name = con; + name = con.getName(); } }); @@ -298,16 +297,20 @@ RecurrentRealizer::realize(const GraphRepresentation &reference) { * */ auto concat_output = [this](const GraphRepresentation &reference_, - const std::string &con, unsigned unroll_for) { + const Connection &con, unsigned unroll_for) { GraphRepresentation processed(reference_.begin(), reference_.end()); - std::vector names; + std::vector conns; for (unsigned int i = 0; i < unroll_for; ++i) { - names.push_back(con + "/" + std::to_string(i)); + conns.emplace_back(Connection{ + con.getName() + "/" + std::to_string(i), + con.getIndex(), + }); } /// @todo have axis in concat layer + /// @todo this has to be wrapped with identity layer as #1793 auto node = createLayerNode( - "concat", {"name=" + con, "input_layers=" + to_string(names)}); + "concat", {"name=" + con.getName(), "input_layers=" + to_string(conns)}); processed.push_back(std::move(node)); return processed; @@ -323,10 +326,10 @@ RecurrentRealizer::realize(const GraphRepresentation &reference) { /// @note below is inefficient way of processing nodes. consider optimize /// below as needed by calling remap realizer only once auto processed = reference_; - for (auto &name : end_layers) { - processed = sequenced_return_layers.count(name) - ? concat_output(processed, name, unroll_for) - : naive_output(processed, name, unroll_for); + for (auto &conn : end_conns) { + processed = sequenced_return_conns.count(conn) + ? concat_output(processed, conn, unroll_for) + : naive_output(processed, conn, unroll_for); } return processed; diff --git a/nntrainer/compiler/recurrent_realizer.h b/nntrainer/compiler/recurrent_realizer.h index 5315f2e..de84efc 100644 --- a/nntrainer/compiler/recurrent_realizer.h +++ b/nntrainer/compiler/recurrent_realizer.h @@ -50,16 +50,16 @@ public: * steps, where steps > 0 * * @param properties - * unroll_for = // define timestep of unrolloing + * unroll_for = // define timestep of unrolling * return_sequences = // return sequences * recurrent_inputs = > // start of the loop * recurrent_ouptuts = > // end of the loop - * @param input_layers input layer from outer side - * @param end_layers end layers (output of the internal graph) + * @param input_conns input conns from outer side + * @param end_conns end connections (output of the internal graph) */ RecurrentRealizer(const std::vector &properties, - const std::vector &input_layers, - const std::vector &end_layers); + const std::vector &input_conns, + const std::vector &end_conns); /** * @brief Construct a new Recurrent Realizer object @@ -91,10 +91,10 @@ private: std::vector, props::UnrollFor>; std::unordered_set input_layers; /**< external input layers */ - std::vector end_layers; /**< final output layers id */ - std::unordered_set - sequenced_return_layers; /**< sequenced return layers, subset of end_layers - */ + std::vector end_conns; /**< final output layers id */ + std::unordered_set + sequenced_return_conns; /**< sequenced return conns, subset of end_conns + */ std::unordered_map recurrent_info; /**< final output layers id */ std::unique_ptr recurrent_props; /**< recurrent properties */ diff --git a/nntrainer/graph/connection.h b/nntrainer/graph/connection.h index e3caaf2..87e4413 100644 --- a/nntrainer/graph/connection.h +++ b/nntrainer/graph/connection.h @@ -35,9 +35,9 @@ public: * @brief Construct a new Connection object from string representation * string representation is format of {layer_name, idx}; * - * @param string_representation string format of {layer_name}({idx}) + * @param str_repr string format of {layer_name}({idx}) */ - explicit Connection(const std::string &string_representation); + explicit Connection(const std::string &str_repr); /** * @brief Construct a new Connection object diff --git a/nntrainer/layers/common_properties.h b/nntrainer/layers/common_properties.h index bcfced0..38a3f67 100644 --- a/nntrainer/layers/common_properties.h +++ b/nntrainer/layers/common_properties.h @@ -578,10 +578,10 @@ public: * used in recurrent realizer * */ -class AsSequence : public Name { +class AsSequence : public Property { public: static constexpr const char *key = "as_sequence"; - using prop_tag = str_prop_tag; + using prop_tag = connection_prop_tag; }; /** diff --git a/nntrainer/models/neuralnet.cpp b/nntrainer/models/neuralnet.cpp index 36b9fcc..b16b013 100644 --- a/nntrainer/models/neuralnet.cpp +++ b/nntrainer/models/neuralnet.cpp @@ -934,15 +934,6 @@ void NeuralNetwork::addWithReferenceLayers( nodes.push_back(node->cloneConfiguration()); } - auto normalize = [](const std::vector &names) { - std::vector prop_names(names.begin(), names.end()); - return std::vector(prop_names.begin(), prop_names.end()); - }; - - auto input_layers_ = normalize(input_layers); - auto start_layers_ = normalize(start_layers); - auto end_layers_ = normalize(end_layers); - auto start_conns = std::vector(start_layers.begin(), start_layers.end()); auto input_conns = @@ -955,20 +946,20 @@ void NeuralNetwork::addWithReferenceLayers( realizers.emplace_back(new PreviousInputRealizer(start_conns)); realizers.emplace_back(new SliceRealizer(start_conns, end_conns)); - if (!input_layers_.empty()) { + if (!input_conns.empty()) { realizers.emplace_back(new InputRealizer(start_conns, input_conns)); } if (type == ml::train::ReferenceLayersType::RECURRENT) { realizers.emplace_back( - new RecurrentRealizer(type_properties, input_layers_, end_layers_)); + new RecurrentRealizer(type_properties, input_conns, end_conns)); } if (!scope.empty()) { realizers.emplace_back( - new RemapRealizer([&scope, &input_layers_](std::string &name) { - for (auto &i : input_layers_) { - if (i == name) { + new RemapRealizer([&scope, &input_conns](std::string &name) { + for (auto &i : input_conns) { + if (i.getName() == name) { return; } } diff --git a/test/unittest/compiler/unittest_realizer.cpp b/test/unittest/compiler/unittest_realizer.cpp index 4fa973f..4e31e3c 100644 --- a/test/unittest/compiler/unittest_realizer.cpp +++ b/test/unittest/compiler/unittest_realizer.cpp @@ -58,10 +58,11 @@ TEST(FlattenRealizer, flatten_p) { } TEST(RecurrentRealizer, recurrent_no_return_sequence_p) { + using C = Connection; RecurrentRealizer r( {"unroll_for=3", "recurrent_input=fc_in", "recurrent_output=fc_out"}, - {"source"}, {"fc_out"}); + {C("source")}, {C("fc_out")}); std::vector before = { {"fully_connected", {"name=fc_in", "input_layers=source"}}, @@ -84,10 +85,10 @@ TEST(RecurrentRealizer, recurrent_no_return_sequence_p) { } TEST(RecurrentRealizer, recurrent_return_sequence_single_p) { - + using C = Connection; RecurrentRealizer r({"unroll_for=3", "as_sequence=fc_out", "recurrent_input=lstm", "recurrent_output=fc_out"}, - {"source"}, {"fc_out"}); + {C("source")}, {C("fc_out")}); std::vector before = { {"lstm", {"name=lstm", "input_layers=source"}}, @@ -114,13 +115,14 @@ TEST(RecurrentRealizer, recurrent_return_sequence_single_p) { } TEST(RecurrentRealizer, recurrent_multi_inout_p) { + using C = Connection; RecurrentRealizer r( { "unroll_for=3", "recurrent_input=lstm,source3_dummy", "recurrent_output=fc_out,output_dummy", }, - {"source", "source2", "source3"}, {"fc_out"}); + {C("source"), C("source2"), C("source3")}, {C("fc_out")}); /// @note for below graph, /// 1. fc_out feds back to lstm @@ -196,6 +198,7 @@ TEST(RecurrentRealizer, recurrent_multi_inout_p) { } TEST(RecurrentRealizer, recurrent_multi_inout_return_seq_p) { + using C = Connection; RecurrentRealizer r( { "unroll_for=3", @@ -203,7 +206,7 @@ TEST(RecurrentRealizer, recurrent_multi_inout_return_seq_p) { "as_sequence=fc_out", "recurrent_output=fc_out,output_dummy", }, - {"source", "source2", "source3"}, {"fc_out"}); + {C("source"), C("source2"), C("source3")}, {C("fc_out")}); /// @note for below graph, /// 1. fc_out feds back to lstm @@ -282,7 +285,7 @@ TEST(RecurrentRealizer, recurrent_multi_inout_return_seq_p) { } TEST(RecurrentRealizer, recurrent_multi_inout_using_connection_return_seq_p) { - + using C = Connection; RecurrentRealizer r( { "unroll_for=3", @@ -290,7 +293,7 @@ TEST(RecurrentRealizer, recurrent_multi_inout_using_connection_return_seq_p) { "recurrent_input=lstm,add(2)", "recurrent_output=fc_out,split(1)", }, - {"source", "source2", "source3"}, {"fc_out"}); + {C("source"), C("source2"), C("source3")}, {C("fc_out")}); /// @note for below graph, /// 1. fc_out feds back to lstm @@ -341,13 +344,14 @@ TEST(RecurrentRealizer, recurrent_multi_inout_using_connection_return_seq_p) { } TEST(RecurrentRealizer, recurrent_multi_inout_using_connection_p) { + using C = Connection; RecurrentRealizer r( { "unroll_for=3", "recurrent_input=lstm,add(2)", "recurrent_output=fc_out,split(1)", }, - {"source", "source2", "source3"}, {"fc_out"}); + {C("source"), C("source2"), C("source3")}, {C("fc_out")}); /// @note for below graph, /// 1. fc_out feds back to lstm