From: Sebastian Messmer Date: Thu, 18 Apr 2019 07:57:44 +0000 (-0700) Subject: Move function schema parser to ATen/core build target (#19282) X-Git-Tag: accepted/tizen/6.5/unified/20211028.231830~173 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=41dc54e291934d1fccd8dbd9fd7ad5293c9397ff;p=platform%2Fupstream%2Fpytorch.git Move function schema parser to ATen/core build target (#19282) Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/19282 This is largely a hack because we need to use the function schema parser from ATen/core but aren't clear yet on how the final software architecture should look like. - Add function schema parser files from jit to ATen/core build target. - Also move ATen/core build target one directory up to allow this. We only change the build targets and don't move the files yet because this is likely not the final build set up and we want to avoid repeated interruptions for other developers. cc zdevito Reviewed By: dzhulgakov Differential Revision: D14931922 fbshipit-source-id: 26462e2e7aec9e0964706138edd3d87a83b964e3 --- diff --git a/aten/src/ATen/core/CMakeLists.txt b/aten/src/ATen/core/CMakeLists.txt index 02160a7..0f5d86f 100644 --- a/aten/src/ATen/core/CMakeLists.txt +++ b/aten/src/ATen/core/CMakeLists.txt @@ -6,6 +6,25 @@ FILE(GLOB_RECURSE ATen_CORE_SRCS "*.cpp") FILE(GLOB_RECURSE ATen_CORE_TEST_SRCS "*_test.cpp") EXCLUDE(ATen_CORE_SRCS "${ATen_CORE_SRCS}" ${ATen_CORE_TEST_SRCS}) +# Add files needed from jit folders +LIST(APPEND ATen_CORE_HEADERS + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/source_range.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/source_location.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/function_schema_parser.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/lexer.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/strtod.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/parse_string_literal.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/schema_type_parser.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/error_report.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/tree.h +) +LIST(APPEND ATen_CORE_SRCS + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/function_schema_parser.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/lexer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/strtod.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/schema_type_parser.cpp +) + # Pass to parent set(ATen_CORE_HEADERS ${ATen_CORE_HEADERS} PARENT_SCOPE) set(ATen_CORE_SRCS ${ATen_CORE_SRCS} PARENT_SCOPE) diff --git a/tools/build_variables.py b/tools/build_variables.py index db521a0..ff2b5f2 100644 --- a/tools/build_variables.py +++ b/tools/build_variables.py @@ -101,8 +101,6 @@ libtorch_sources = [ "torch/csrc/jit/script/edit_distance.cpp", "torch/csrc/jit/script/logging.cpp", "torch/csrc/jit/script/final_returns.cpp", - "torch/csrc/jit/script/function_schema_parser.cpp", - "torch/csrc/jit/script/schema_type_parser.cpp", "torch/csrc/jit/script/script_type_parser.cpp", "torch/csrc/jit/script/sugared_value.cpp", "torch/csrc/jit/script/schema_matching.cpp", @@ -112,8 +110,6 @@ libtorch_sources = [ "torch/csrc/jit/import_source.cpp", "torch/csrc/jit/hooks_for_testing.cpp", "torch/csrc/jit/script/builtin_functions.cpp", - "torch/csrc/jit/script/lexer.cpp", - "torch/csrc/jit/script/strtod.cpp", "torch/csrc/jit/script/module.cpp", "torch/csrc/jit/tracer.cpp", "torch/csrc/utils/tensor_flatten.cpp", @@ -156,6 +152,11 @@ def add_torch_libs(): # to avoid redefinitions of symbols defined in # dynamic_library_unix.cpp "torch/csrc/jit/fuser/cpu/dynamic_library_win.cpp", + # exclude files that are already included in ATen/core + "torch/csrc/jit/script/function_schema_parser.cpp", + "torch/csrc/jit/script/schema_type_parser.cpp", + "torch/csrc/jit/script/lexer.cpp", + "torch/csrc/jit/script/strtod.cpp", ], ) + [ "torch/csrc/distributed/Module.cpp", diff --git a/torch/CMakeLists.txt b/torch/CMakeLists.txt index af64a00..4dc8841 100644 --- a/torch/CMakeLists.txt +++ b/torch/CMakeLists.txt @@ -180,17 +180,13 @@ set(TORCH_SRCS ${TORCH_SRC_DIR}/csrc/api/src/jit.cpp ${TORCH_SRC_DIR}/csrc/jit/testing/file_check.cpp ${TORCH_SRC_DIR}/csrc/jit/script/final_returns.cpp - ${TORCH_SRC_DIR}/csrc/jit/script/function_schema_parser.cpp ${TORCH_SRC_DIR}/csrc/jit/script/schema_matching.cpp - ${TORCH_SRC_DIR}/csrc/jit/script/schema_type_parser.cpp ${TORCH_SRC_DIR}/csrc/jit/script/script_type_parser.cpp ${TORCH_SRC_DIR}/csrc/jit/script/sugared_value.cpp ${TORCH_SRC_DIR}/csrc/jit/script/class_type.cpp ${TORCH_SRC_DIR}/csrc/jit/script/parser.cpp ${TORCH_SRC_DIR}/csrc/jit/script/builtin_functions.cpp ${TORCH_SRC_DIR}/csrc/jit/script/edit_distance.cpp - ${TORCH_SRC_DIR}/csrc/jit/script/lexer.cpp - ${TORCH_SRC_DIR}/csrc/jit/script/strtod.cpp ${TORCH_SRC_DIR}/csrc/jit/script/logging.cpp ${TORCH_SRC_DIR}/csrc/jit/script/module.cpp ${TORCH_SRC_DIR}/csrc/jit/tracer.cpp @@ -532,8 +528,6 @@ if (BUILD_PYTHON) ${TORCH_SRC_DIR}/csrc/jit/python_ir.cpp ${TORCH_SRC_DIR}/csrc/jit/python_tracer.cpp ${TORCH_SRC_DIR}/csrc/jit/script/init.cpp - ${TORCH_SRC_DIR}/csrc/jit/script/lexer.cpp - ${TORCH_SRC_DIR}/csrc/jit/script/strtod.cpp ${TORCH_SRC_DIR}/csrc/jit/script/python_tree_views.cpp ${TORCH_SRC_DIR}/csrc/multiprocessing/init.cpp ${TORCH_SRC_DIR}/csrc/nn/THNN.cpp diff --git a/torch/csrc/jit/script/function_schema_parser.cpp b/torch/csrc/jit/script/function_schema_parser.cpp index f6b751c..639a848 100644 --- a/torch/csrc/jit/script/function_schema_parser.cpp +++ b/torch/csrc/jit/script/function_schema_parser.cpp @@ -256,7 +256,7 @@ struct SchemaParser { } // namespace } // namespace script -FunctionSchema parseSchema(const std::string& schema) { +C10_EXPORT FunctionSchema parseSchema(const std::string& schema) { return script::SchemaParser(schema).parseDeclarations().at(0); } diff --git a/torch/csrc/jit/script/function_schema_parser.h b/torch/csrc/jit/script/function_schema_parser.h index 41dba25..71e9f8c 100644 --- a/torch/csrc/jit/script/function_schema_parser.h +++ b/torch/csrc/jit/script/function_schema_parser.h @@ -1,13 +1,13 @@ #pragma once #include -#include +#include #include namespace torch { namespace jit { -TORCH_API ::c10::FunctionSchema parseSchema(const std::string& schema); +CAFFE2_API ::c10::FunctionSchema parseSchema(const std::string& schema); } // namespace jit } // namespace torch diff --git a/torch/csrc/jit/script/lexer.cpp b/torch/csrc/jit/script/lexer.cpp index 653607b..3e060cb 100644 --- a/torch/csrc/jit/script/lexer.cpp +++ b/torch/csrc/jit/script/lexer.cpp @@ -60,7 +60,7 @@ bool SharedParserData::isBinary(int kind, int* prec) { return false; } -int stringToKind(const std::string& str) { +C10_EXPORT int stringToKind(const std::string& str) { static std::once_flag init_flag; static std::unordered_map str_to_kind; std::call_once(init_flag, []() { @@ -79,7 +79,7 @@ int stringToKind(const std::string& str) { } } -std::string kindToString(int kind) { +C10_EXPORT std::string kindToString(int kind) { if (kind < 256) return std::string(1, kind); switch (kind) { @@ -93,7 +93,7 @@ std::string kindToString(int kind) { } } -SharedParserData& sharedParserData() { +C10_EXPORT SharedParserData& sharedParserData() { static SharedParserData data; // safely handles multi-threaded init return data; } diff --git a/torch/csrc/jit/script/lexer.h b/torch/csrc/jit/script/lexer.h index e21a2c6..7e46c60 100644 --- a/torch/csrc/jit/script/lexer.h +++ b/torch/csrc/jit/script/lexer.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -114,8 +115,8 @@ enum TokenKind { #undef DEFINE_TOKEN }; -std::string kindToString(int kind); -int stringToKind(const std::string& str); +CAFFE2_API std::string kindToString(int kind); +CAFFE2_API int stringToKind(const std::string& str); // nested hash tables that indicate char-by-char what is a valid token. struct TokenTrie; @@ -148,7 +149,7 @@ struct TokenTrie { // stuff that is shared against all TC lexers/parsers and is initialized only // once. -struct SharedParserData { +struct CAFFE2_API SharedParserData { SharedParserData() : head(new TokenTrie()) { std::stringstream ss; for (const char* c = valid_single_char_tokens; *c; c++) { @@ -352,7 +353,7 @@ struct SharedParserData { TokenTrieRef head; }; -SharedParserData& sharedParserData(); +CAFFE2_API SharedParserData& sharedParserData(); struct Token { int kind; diff --git a/torch/csrc/jit/script/schema_type_parser.h b/torch/csrc/jit/script/schema_type_parser.h index 266990a..fc53c80 100644 --- a/torch/csrc/jit/script/schema_type_parser.h +++ b/torch/csrc/jit/script/schema_type_parser.h @@ -1,6 +1,7 @@ #include #include #include +#include namespace torch { namespace jit { @@ -9,7 +10,7 @@ namespace script { using TypePtr = c10::TypePtr; using TypeAndAlias = std::pair>; -struct SchemaTypeParser { +struct CAFFE2_API SchemaTypeParser { TypeAndAlias parseBaseType(); c10::optional parseAliasAnnotation(); std::pair> parseType();