Move function schema parser to ATen/core build target (#19282)

Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/19282

This is largely a hack because we need to use the function schema parser from ATen/core
but aren't clear yet on how the final software architecture should look like.

- Add function schema parser files from jit to ATen/core build target.
- Also move ATen/core build target one directory up to allow this.

We only change the build targets and don't move the files yet because this is likely
not the final build set up and we want to avoid repeated interruptions
for other developers. cc zdevito

Reviewed By: dzhulgakov

Differential Revision: D14931922

fbshipit-source-id: 26462e2e7aec9e0964706138edd3d87a83b964e3

diff --git a/aten/src/ATen/core/CMakeLists.txt b/aten/src/ATen/core/CMakeLists.txt
index 02160a7..0f5d86f 100644 (file)
--- a/aten/src/ATen/core/CMakeLists.txt
+++ b/aten/src/ATen/core/CMakeLists.txt
@@ -6,6 +6,25 @@ FILE(GLOB_RECURSE ATen_CORE_SRCS "*.cpp")
 FILE(GLOB_RECURSE ATen_CORE_TEST_SRCS "*_test.cpp")
 EXCLUDE(ATen_CORE_SRCS "${ATen_CORE_SRCS}" ${ATen_CORE_TEST_SRCS})
 
+# Add files needed from jit folders
+LIST(APPEND ATen_CORE_HEADERS
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/source_range.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/source_location.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/function_schema_parser.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/lexer.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/strtod.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/parse_string_literal.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/schema_type_parser.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/error_report.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/tree.h
+)
+LIST(APPEND ATen_CORE_SRCS
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/function_schema_parser.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/lexer.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/strtod.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../torch/csrc/jit/script/schema_type_parser.cpp
+)
+
 # Pass to parent
 set(ATen_CORE_HEADERS ${ATen_CORE_HEADERS} PARENT_SCOPE)
 set(ATen_CORE_SRCS ${ATen_CORE_SRCS} PARENT_SCOPE)

diff --git a/tools/build_variables.py b/tools/build_variables.py
index db521a0..ff2b5f2 100644 (file)
--- a/tools/build_variables.py
+++ b/tools/build_variables.py
@@ -101,8 +101,6 @@ libtorch_sources = [
     "torch/csrc/jit/script/edit_distance.cpp",
     "torch/csrc/jit/script/logging.cpp",
     "torch/csrc/jit/script/final_returns.cpp",
-    "torch/csrc/jit/script/function_schema_parser.cpp",
-    "torch/csrc/jit/script/schema_type_parser.cpp",
     "torch/csrc/jit/script/script_type_parser.cpp",
     "torch/csrc/jit/script/sugared_value.cpp",
     "torch/csrc/jit/script/schema_matching.cpp",
@@ -112,8 +110,6 @@ libtorch_sources = [
     "torch/csrc/jit/import_source.cpp",
     "torch/csrc/jit/hooks_for_testing.cpp",
     "torch/csrc/jit/script/builtin_functions.cpp",
-    "torch/csrc/jit/script/lexer.cpp",
-    "torch/csrc/jit/script/strtod.cpp",
     "torch/csrc/jit/script/module.cpp",
     "torch/csrc/jit/tracer.cpp",
     "torch/csrc/utils/tensor_flatten.cpp",
@@ -156,6 +152,11 @@ def add_torch_libs():
             # to avoid redefinitions of symbols defined in
             # dynamic_library_unix.cpp
             "torch/csrc/jit/fuser/cpu/dynamic_library_win.cpp",
+            # exclude files that are already included in ATen/core
+            "torch/csrc/jit/script/function_schema_parser.cpp",
+            "torch/csrc/jit/script/schema_type_parser.cpp",
+            "torch/csrc/jit/script/lexer.cpp",
+            "torch/csrc/jit/script/strtod.cpp",
         ],
     ) + [
         "torch/csrc/distributed/Module.cpp",

diff --git a/torch/CMakeLists.txt b/torch/CMakeLists.txt
index af64a00..4dc8841 100644 (file)
--- a/torch/CMakeLists.txt
+++ b/torch/CMakeLists.txt
@@ -180,17 +180,13 @@ set(TORCH_SRCS
   ${TORCH_SRC_DIR}/csrc/api/src/jit.cpp
   ${TORCH_SRC_DIR}/csrc/jit/testing/file_check.cpp
   ${TORCH_SRC_DIR}/csrc/jit/script/final_returns.cpp
-  ${TORCH_SRC_DIR}/csrc/jit/script/function_schema_parser.cpp
   ${TORCH_SRC_DIR}/csrc/jit/script/schema_matching.cpp
-  ${TORCH_SRC_DIR}/csrc/jit/script/schema_type_parser.cpp
   ${TORCH_SRC_DIR}/csrc/jit/script/script_type_parser.cpp
   ${TORCH_SRC_DIR}/csrc/jit/script/sugared_value.cpp
   ${TORCH_SRC_DIR}/csrc/jit/script/class_type.cpp
   ${TORCH_SRC_DIR}/csrc/jit/script/parser.cpp
   ${TORCH_SRC_DIR}/csrc/jit/script/builtin_functions.cpp
   ${TORCH_SRC_DIR}/csrc/jit/script/edit_distance.cpp
-  ${TORCH_SRC_DIR}/csrc/jit/script/lexer.cpp
-  ${TORCH_SRC_DIR}/csrc/jit/script/strtod.cpp
   ${TORCH_SRC_DIR}/csrc/jit/script/logging.cpp
   ${TORCH_SRC_DIR}/csrc/jit/script/module.cpp
   ${TORCH_SRC_DIR}/csrc/jit/tracer.cpp
@@ -532,8 +528,6 @@ if (BUILD_PYTHON)
     ${TORCH_SRC_DIR}/csrc/jit/python_ir.cpp
     ${TORCH_SRC_DIR}/csrc/jit/python_tracer.cpp
     ${TORCH_SRC_DIR}/csrc/jit/script/init.cpp
-    ${TORCH_SRC_DIR}/csrc/jit/script/lexer.cpp
-    ${TORCH_SRC_DIR}/csrc/jit/script/strtod.cpp
     ${TORCH_SRC_DIR}/csrc/jit/script/python_tree_views.cpp
     ${TORCH_SRC_DIR}/csrc/multiprocessing/init.cpp
     ${TORCH_SRC_DIR}/csrc/nn/THNN.cpp

diff --git a/torch/csrc/jit/script/function_schema_parser.cpp b/torch/csrc/jit/script/function_schema_parser.cpp
index f6b751c..639a848 100644 (file)
--- a/torch/csrc/jit/script/function_schema_parser.cpp
+++ b/torch/csrc/jit/script/function_schema_parser.cpp
@@ -256,7 +256,7 @@ struct SchemaParser {
 } // namespace
 } // namespace script
 
-FunctionSchema parseSchema(const std::string& schema) {
+C10_EXPORT FunctionSchema parseSchema(const std::string& schema) {
   return script::SchemaParser(schema).parseDeclarations().at(0);
 }
 

diff --git a/torch/csrc/jit/script/function_schema_parser.h b/torch/csrc/jit/script/function_schema_parser.h
index 41dba25..71e9f8c 100644 (file)
--- a/torch/csrc/jit/script/function_schema_parser.h
+++ b/torch/csrc/jit/script/function_schema_parser.h
@@ -1,13 +1,13 @@
 #pragma once
 
 #include <ATen/core/function_schema.h>
-#include <torch/csrc/WindowsTorchApiMacro.h>
+#include <ATen/core/Macros.h>
 #include <string>
 
 namespace torch {
 namespace jit {
 
-TORCH_API ::c10::FunctionSchema parseSchema(const std::string& schema);
+CAFFE2_API ::c10::FunctionSchema parseSchema(const std::string& schema);
 
 } // namespace jit
 } // namespace torch

diff --git a/torch/csrc/jit/script/lexer.cpp b/torch/csrc/jit/script/lexer.cpp
index 653607b..3e060cb 100644 (file)
--- a/torch/csrc/jit/script/lexer.cpp
+++ b/torch/csrc/jit/script/lexer.cpp
@@ -60,7 +60,7 @@ bool SharedParserData::isBinary(int kind, int* prec) {
   return false;
 }
 
-int stringToKind(const std::string& str) {
+C10_EXPORT int stringToKind(const std::string& str) {
   static std::once_flag init_flag;
   static std::unordered_map<std::string, int> str_to_kind;
   std::call_once(init_flag, []() {
@@ -79,7 +79,7 @@ int stringToKind(const std::string& str) {
   }
 }
 
-std::string kindToString(int kind) {
+C10_EXPORT std::string kindToString(int kind) {
   if (kind < 256)
     return std::string(1, kind);
   switch (kind) {
@@ -93,7 +93,7 @@ std::string kindToString(int kind) {
   }
 }
 
-SharedParserData& sharedParserData() {
+C10_EXPORT SharedParserData& sharedParserData() {
   static SharedParserData data; // safely handles multi-threaded init
   return data;
 }

diff --git a/torch/csrc/jit/script/lexer.h b/torch/csrc/jit/script/lexer.h
index e21a2c6..7e46c60 100644 (file)
--- a/torch/csrc/jit/script/lexer.h
+++ b/torch/csrc/jit/script/lexer.h
@@ -3,6 +3,7 @@
 #include <c10/util/C++17.h>
 #include <torch/csrc/jit/source_range.h>
 #include <torch/csrc/jit/script/strtod.h>
+#include <ATen/core/Macros.h>
 #include <algorithm>
 #include <clocale>
 #include <cstring>
@@ -114,8 +115,8 @@ enum TokenKind {
 #undef DEFINE_TOKEN
 };
 
-std::string kindToString(int kind);
-int stringToKind(const std::string& str);
+CAFFE2_API std::string kindToString(int kind);
+CAFFE2_API int stringToKind(const std::string& str);
 
 // nested hash tables that indicate char-by-char what is a valid token.
 struct TokenTrie;
@@ -148,7 +149,7 @@ struct TokenTrie {
 
 // stuff that is shared against all TC lexers/parsers and is initialized only
 // once.
-struct SharedParserData {
+struct CAFFE2_API SharedParserData {
   SharedParserData() : head(new TokenTrie()) {
     std::stringstream ss;
     for (const char* c = valid_single_char_tokens; *c; c++) {
@@ -352,7 +353,7 @@ struct SharedParserData {
   TokenTrieRef head;
 };
 
-SharedParserData& sharedParserData();
+CAFFE2_API SharedParserData& sharedParserData();
 
 struct Token {
   int kind;

diff --git a/torch/csrc/jit/script/schema_type_parser.h b/torch/csrc/jit/script/schema_type_parser.h
index 266990a..fc53c80 100644 (file)
--- a/torch/csrc/jit/script/schema_type_parser.h
+++ b/torch/csrc/jit/script/schema_type_parser.h
@@ -1,6 +1,7 @@
 #include <ATen/core/jit_type.h>
 #include <ATen/core/alias_info.h>
 #include <torch/csrc/jit/script/lexer.h>
+#include <ATen/core/Macros.h>
 
 namespace torch {
 namespace jit {
@@ -9,7 +10,7 @@ namespace script {
 using TypePtr = c10::TypePtr;
 using TypeAndAlias = std::pair<TypePtr, c10::optional<c10::AliasInfo>>;
 
-struct SchemaTypeParser {
+struct CAFFE2_API SchemaTypeParser {
   TypeAndAlias parseBaseType();
   c10::optional<c10::AliasInfo> parseAliasAnnotation();
   std::pair<TypePtr, c10::optional<c10::AliasInfo>> parseType();