Commit explicit libtorch_python sources (#19607)

Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/19607

Explicit is better than implicit - it's pretty hard to debug where particular file is if it's not greppable.

As a follow up step - we should look whether we can just include build_variables.py in CMake directly to share setups of two build systems

Reviewed By: ezyang

Differential Revision: D15023348

fbshipit-source-id: 600ef2d1871bc28530c6a02681b284f7499904df

diff --git a/tools/build_variables.py b/tools/build_variables.py
index dd2a041ad09179ddf1b6ee7f9a5ae2657b7cea7d..646b9e1c6d3229e56d8aa6f753f4bea2d08783a2 100644 (file)
--- a/tools/build_variables.py
+++ b/tools/build_variables.py
@@ -138,37 +138,109 @@ libtorch_cuda_sources = [
 
 def add_torch_libs():
     r = {}
-    # We start torch_python_sources with all cpp files, and exclude some
-    # including the files already contained in the torch and cuda bindings
-    globbed_sources = (native.glob(
-        ["torch/csrc/**/*.cpp"],
-        exclude=[
-            # remove anything that has "generic" in it"s path
-            "torch/csrc/**/generic/**/*.cpp",
-            # distributed only uses Module.cpp
-            # so remove all other files and just include that
-            "torch/csrc/distributed/**/*.cpp",
-            # top-level hook of extension registration lives in a separate file
-            "torch/csrc/stub.cpp",
-            # to avoid redefinitions of symbols defined in
-            # dynamic_library_unix.cpp
-            "torch/csrc/jit/fuser/cpu/dynamic_library_win.cpp",
-            # exclude files that are already included in ATen/core
-            "torch/csrc/jit/script/function_schema_parser.cpp",
-            "torch/csrc/jit/script/schema_type_parser.cpp",
-            "torch/csrc/jit/script/lexer.cpp",
-            "torch/csrc/jit/script/strtod.cpp",
-        ],
-    ) + [
+    libtorch_python_sources = [
+        ":generate-code=THCUNN.cpp",
+        ":generate-code=THNN.cpp",
+        ":generate-code=python_functions.cpp",
+        ":generate-code=python_nn_functions.cpp",
+        ":generate-code=python_torch_functions.cpp",
+        ":generate-code=python_variable_methods.cpp",
+        "torch/csrc/CudaIPCTypes.cpp",
+        "torch/csrc/DataLoader.cpp",
+        "torch/csrc/Device.cpp",
+        "torch/csrc/Dtype.cpp",
+        "torch/csrc/DynamicTypes.cpp",
+        "torch/csrc/Generator.cpp",
+        "torch/csrc/Layout.cpp",
+        "torch/csrc/Module.cpp",
+        "torch/csrc/PtrWrapper.cpp",
+        "torch/csrc/Size.cpp",
+        "torch/csrc/Storage.cpp",
+        "torch/csrc/TypeInfo.cpp",
+        "torch/csrc/api/src/cuda.cpp",
+        "torch/csrc/api/src/data/datasets/mnist.cpp",
+        "torch/csrc/api/src/data/samplers/distributed.cpp",
+        "torch/csrc/api/src/data/samplers/random.cpp",
+        "torch/csrc/api/src/data/samplers/sequential.cpp",
+        "torch/csrc/api/src/data/samplers/stream.cpp",
+        "torch/csrc/api/src/nn/init.cpp",
+        "torch/csrc/api/src/nn/module.cpp",
+        "torch/csrc/api/src/nn/modules/batchnorm.cpp",
+        "torch/csrc/api/src/nn/modules/conv.cpp",
+        "torch/csrc/api/src/nn/modules/dropout.cpp",
+        "torch/csrc/api/src/nn/modules/embedding.cpp",
+        "torch/csrc/api/src/nn/modules/functional.cpp",
+        "torch/csrc/api/src/nn/modules/linear.cpp",
+        "torch/csrc/api/src/nn/modules/named_any.cpp",
+        "torch/csrc/api/src/nn/modules/rnn.cpp",
+        "torch/csrc/api/src/optim/adagrad.cpp",
+        "torch/csrc/api/src/optim/adam.cpp",
+        "torch/csrc/api/src/optim/lbfgs.cpp",
+        "torch/csrc/api/src/optim/optimizer.cpp",
+        "torch/csrc/api/src/optim/rmsprop.cpp",
+        "torch/csrc/api/src/optim/serialize.cpp",
+        "torch/csrc/api/src/optim/sgd.cpp",
+        "torch/csrc/api/src/python/init.cpp",
+        "torch/csrc/api/src/serialize/input-archive.cpp",
+        "torch/csrc/api/src/serialize/output-archive.cpp",
+        "torch/csrc/autograd/functions/init.cpp",
+        "torch/csrc/autograd/init.cpp",
+        "torch/csrc/autograd/python_anomaly_mode.cpp",
+        "torch/csrc/autograd/python_cpp_function.cpp",
+        "torch/csrc/autograd/python_engine.cpp",
+        "torch/csrc/autograd/python_function.cpp",
+        "torch/csrc/autograd/python_hook.cpp",
+        "torch/csrc/autograd/python_legacy_variable.cpp",
+        "torch/csrc/autograd/python_variable.cpp",
+        "torch/csrc/autograd/python_variable_indexing.cpp",
+        "torch/csrc/byte_order.cpp",
+        "torch/csrc/cuda/Event.cpp",
+        "torch/csrc/cuda/Module.cpp",
+        "torch/csrc/cuda/Storage.cpp",
+        "torch/csrc/cuda/Stream.cpp",
+        "torch/csrc/cuda/Tensor.cpp",
+        "torch/csrc/cuda/python_comm.cpp",
+        "torch/csrc/cuda/python_nccl.cpp",
+        "torch/csrc/cuda/serialization.cpp",
+        "torch/csrc/cuda/utils.cpp",
         "torch/csrc/distributed/Module.cpp",
-        "torch/csrc/distributed/c10d/init.cpp",
         "torch/csrc/distributed/c10d/ddp.cpp",
+        "torch/csrc/distributed/c10d/init.cpp",
         "torch/csrc/distributed/c10d/reducer.cpp",
-    ] + [":generate-code=" + x for x in GENERATED_CPP])
-    libtorch_python_sources = sets.to_list(sets.difference(
-        sets.make(globbed_sources),
-        sets.make(libtorch_sources + libtorch_cuda_sources),
-    ))
+        "torch/csrc/jit/init.cpp",
+        "torch/csrc/jit/passes/inline_fork_wait.cpp",
+        "torch/csrc/jit/passes/onnx.cpp",
+        "torch/csrc/jit/passes/onnx/constant_fold.cpp",
+        "torch/csrc/jit/passes/onnx/fixup_onnx_loop.cpp",
+        "torch/csrc/jit/passes/onnx/peephole.cpp",
+        "torch/csrc/jit/passes/onnx/prepare_division_for_onnx.cpp",
+        "torch/csrc/jit/passes/remove_inplace_ops.cpp",
+        "torch/csrc/jit/passes/utils/check_alias_annotation.cpp",
+        "torch/csrc/jit/python_arg_flatten.cpp",
+        "torch/csrc/jit/python_interpreter.cpp",
+        "torch/csrc/jit/python_ir.cpp",
+        "torch/csrc/jit/python_tracer.cpp",
+        "torch/csrc/jit/script/init.cpp",
+        "torch/csrc/jit/script/python_tree_views.cpp",
+        "torch/csrc/multiprocessing/init.cpp",
+        "torch/csrc/onnx/init.cpp",
+        "torch/csrc/serialization.cpp",
+        "torch/csrc/tensor/python_tensor.cpp",
+        "torch/csrc/utils.cpp",
+        "torch/csrc/utils/cuda_lazy_init.cpp",
+        "torch/csrc/utils/invalid_arguments.cpp",
+        "torch/csrc/utils/object_ptr.cpp",
+        "torch/csrc/utils/python_arg_parser.cpp",
+        "torch/csrc/utils/structseq.cpp",
+        "torch/csrc/utils/tensor_apply.cpp",
+        "torch/csrc/utils/tensor_dtypes.cpp",
+        "torch/csrc/utils/tensor_layouts.cpp",
+        "torch/csrc/utils/tensor_list.cpp",
+        "torch/csrc/utils/tensor_new.cpp",
+        "torch/csrc/utils/tensor_numpy.cpp",
+        "torch/csrc/utils/tensor_types.cpp",
+        "torch/csrc/utils/tuple_parser.cpp",
+    ]
 
     common_flags = {
         "compiler_flags": [