_(aten, _cast_Int) \
_(aten, _cast_Long) \
_(aten, _cast_Short) \
-_(aten, _cat) \
_(aten, _ceil) \
_(aten, _clamp_max) \
_(aten, _clamp_min) \
_(aten, broadcast_tensors) \
_(aten, broadcast_to) \
_(aten, cartesian_prod) \
-_(aten, cat) \
_(aten, cauchy) \
_(aten, ceil) \
_(aten, celu) \
_(aten, bin) \
_(aten, pop) \
_(aten, insert) \
+ _(aten, _cat) \
+ _(aten, cat) \
+ _(aten, concat) \
_(aten, vstack) \
_(aten, row_stack) \
_(prim, unchecked_unwrap_optional) \
// Returns true if resize is necessary
bool resize_output_check(const Tensor& output, IntArrayRef shape) {
- // Tests for resizing of tensors with one more elements
+ // Tests for resizing of tensors with one or more elements
if (output.sizes().equals(shape)) {
return false;
}
namespace at { namespace native {
// TODO: make all operations that resize given outputs use this function
-// for consistency and maintainability
+// for consistency and maintainability.
+// Some operations like `cat` might not be able to make the use of
+// resize_output directly. For more details to understand how it works in `cat`,
+// see https://github.com/pytorch/pytorch/pull/62560#discussion_r687363362
// Resizes outputs
// Functions accepting output tensors, like with the "out" kwarg, should
// call this function to handle resizing their output tensor.
// Returns a bool saying whether or not the resize actually happened or not
TORCH_API bool resize_output(const Tensor& output, IntArrayRef shape);
+// Utility for resize_output
+// Returns a bool saying resize should happen or not and
+// raises a warning if resizing for one or more elements
TORCH_API bool resize_output_check(const Tensor& output, IntArrayRef shape);
TORCH_API void resize_bytes_cpu(StorageImpl* storage, size_t size_bytes);
#include <ATen/NamedTensorUtils.h>
#include <ATen/core/DimVector.h>
#include <ATen/native/Copy.h>
-#include <ATen/native/cpu/CatKernel.h>
#include <ATen/native/Resize.h>
#include <ATen/native/TensorIterator.h>
#include <ATen/native/TypeProperties.h>
result_size[dim] = cat_dim_size;
// skip resizing if size of result is same as expected
- if (result.sizes() != result_size) {
+ // raise a warning while resizing if output has one or more elements
+ // See https://github.com/pytorch/pytorch/pull/62560#discussion_r687363362
+ // for understanding why at::native::resize_output is not called directly.
+ if (at::native::resize_output_check(result, result_size)) {
result.resize_(result_size, first_tensor_mem_format);
}
return at::cat(tensors, dimname_to_position(tensors[0], dim));
}
+// torch.concat, alias for torch.cat
+Tensor& concat_out(TensorList tensors, Dimname dim, Tensor& result) {
+ return at::cat_out(result, tensors, dimname_to_position(tensors[0], dim));
+}
+
+Tensor concat(TensorList tensors, Dimname dim) {
+ return at::cat(tensors, dimname_to_position(tensors[0], dim));
+}
+
+Tensor & concat_out(TensorList tensors, int64_t dim, Tensor & result) {
+ return at::cat_out(result, tensors, dim);
+}
+
+Tensor concat(TensorList tensors, int64_t dim) {
+ return at::cat(tensors, dim);
+}
+
static bool sizes_match_except(IntArrayRef s1, IntArrayRef s2, int64_t dim_except /* should already be wrapped */) {
if (s1.size() != s2.size()) {
return false;
result_sizes.insert(result_sizes.begin() + dim, tensors.size());
// skip resizing if size of result is same as expected
- if (result.sizes() != result_sizes) {
- result.resize_(result_sizes);
- }
+ // raise a warning while resizing if output has one or more elements
+ at::native::resize_output(result, result_sizes);
stack_serial_stub(kCPU, result, tensors, dim);
return true;
}
#include <ATen/cuda/CUDAContext.h>
#include <ATen/MemoryOverlap.h>
#include <ATen/cuda/detail/IndexUtils.cuh>
+#include <ATen/native/Resize.h>
#include <ATen/native/TypeProperties.h>
#include <ATen/Dispatch.h>
#include <c10/core/MemoryFormat.h>
size[dimension] = cat_dim_size;
// skip resizing if size of result is same as expected
- if (out.sizes() != size) {
+ // raise a warning while resizing if output has one or more elements
+ // See https://github.com/pytorch/pytorch/pull/62560#discussion_r687363362
+ // for understanding why at::native::resize_output is not called directly.
+ if (at::native::resize_output_check(out, size)) {
out.resize_(size, memory_format);
}
- func: cat.names_out(Tensor[] tensors, Dimname dim, *, Tensor(a!) out) -> Tensor(a!)
+# alias for torch.cat
+- func: concat(Tensor[] tensors, int dim=0) -> Tensor
+
+- func: concat.out(Tensor[] tensors, int dim=0, *, Tensor(a!) out) -> Tensor(a!)
+
+- func: concat.names(Tensor[] tensors, Dimname dim) -> Tensor
+
+- func: concat.names_out(Tensor[] tensors, Dimname dim, *, Tensor(a!) out) -> Tensor(a!)
+
- func: block_diag(Tensor[] tensors) -> Tensor
variants: function
:nosignatures:
cat
+ concat
conj
chunk
dsplit
lambda a, b, c: torch.block_diag(a, b, c),
True, f_args_variable, f_args_tensor)
- def test_cat(self):
- f_args_variable = (torch.randn(1, S, S, dtype=torch.double, requires_grad=True),
- torch.randn(2, S, S, dtype=torch.double, requires_grad=True),
- torch.randn(3, S, S, dtype=torch.double, requires_grad=True),
- 0)
- f_args_tensor = deepcopy(unpack_variables(f_args_variable))
- run_functional_checks(self, "test_cat", "cat",
- lambda a, b, c, dim: torch.cat((a, b, c), dim),
- True, f_args_variable, f_args_tensor, check_forward_ad=True)
-
- def test_cat_negdim_1(self):
- f_args_variable = (torch.randn(S, S, 1, dtype=torch.double, requires_grad=True),
- torch.randn(S, S, 2, dtype=torch.double, requires_grad=True),
- torch.randn(S, S, 3, dtype=torch.double, requires_grad=True),
- -1)
- f_args_tensor = deepcopy(unpack_variables(f_args_variable))
- run_functional_checks(self, "test_cat_negdim_1", "cat",
- lambda a, b, c, dim: torch.cat((a, b, c), dim),
- True, f_args_variable, f_args_tensor, check_forward_ad=True)
-
- def test_cat_negdim_2(self):
- f_args_variable = (torch.randn(S, 1, S, dtype=torch.double, requires_grad=True),
- torch.randn(S, 2, S, dtype=torch.double, requires_grad=True),
- torch.randn(S, 3, S, dtype=torch.double, requires_grad=True),
- -2)
- f_args_tensor = deepcopy(unpack_variables(f_args_variable))
- run_functional_checks(self, "test_cat_negdim_2", "cat",
- lambda a, b, c, dim: torch.cat((a, b, c), dim),
- True, f_args_variable, f_args_tensor, check_forward_ad=True)
-
def test_cat_empty_legacy(self):
f_args_variable = (torch.randn(0, dtype=torch.double, requires_grad=True),
torch.randn(S, S, dtype=torch.double, requires_grad=True))
False, f_args_variable, f_args_tensor, check_forward_ad=True)
self.assertTrue(gradcheck(lambda a, b: torch.cat((a, b)), f_args_variable, eps=1e-6, atol=PRECISION))
- def test_cat_empty(self):
- f_args_variable = (torch.randn(0, S, dtype=torch.double, requires_grad=True),
- torch.randn(S, S, dtype=torch.double, requires_grad=True))
- f_args_tensor = deepcopy(unpack_variables(f_args_variable))
- run_functional_checks(self, "test_cat_empty", "cat",
- lambda a, b: torch.cat((a, b)),
- True, f_args_variable, f_args_tensor, check_forward_ad=True)
-
def test_var_mean_differentiable(self):
dim = [2, 4]
keepdim = False
return
# These ops currently don't trace in FX for various reasons (i.e. they take a list of tensors)
- fx_fail = {"stack", "hstack", "vstack", "dstack", "linalg.multi_dot"}
+ fx_fail = {"cat", "stack", "hstack", "vstack", "dstack", "linalg.multi_dot"}
sample_inputs_itr = op.sample_inputs(device, dtype, requires_grad=False)
for sample_input in sample_inputs_itr:
unsupported_arg_type = False
self.assertEqual(res1, res2)
self.assertTrue(res1.is_contiguous(memory_format=torch.channels_last))
+ @onlyCUDA
+ def test_cat_out_memory_format(self, device):
+ inp_size = (4, 4, 4, 4)
+ expected_size = (8, 4, 4, 4)
+ a_cuda = torch.randn(inp_size, device=device).contiguous(memory_format=torch.channels_last)
+ a_cpu = torch.randn(inp_size, device='cpu').contiguous(memory_format=torch.channels_last)
+ b_cuda = torch.randn(inp_size, device=device).contiguous(memory_format=torch.contiguous_format)
+ b_cpu = torch.randn(inp_size, device='cpu').contiguous(memory_format=torch.contiguous_format)
+ c_cuda = torch.randn(inp_size, device=device).contiguous(memory_format=torch.channels_last)
+
+ # Case 1: if out= is the correct shape then the memory format of out= is respected
+
+ out_cuda = torch.empty(expected_size, device=device).contiguous(memory_format=torch.contiguous_format)
+ res1_cuda = torch.cat((a_cuda, b_cuda), out=out_cuda)
+
+ out_cpu = torch.empty(expected_size, device='cpu').contiguous(memory_format=torch.contiguous_format)
+ res1_cpu = torch.cat((a_cpu, b_cpu), out=out_cpu)
+
+ self.assertTrue(res1_cuda.is_contiguous(memory_format=torch.contiguous_format))
+ self.assertTrue(res1_cpu.is_contiguous(memory_format=torch.contiguous_format))
+
+ # Case 2: if out= is not the correct shape then the output it is resized internally
+ # - For the CPU variant the memory format is that of the first tensor
+ # - For the CUDA variant it only propagates memory format if all the tensors have
+ # the same memory format, otherwise it just uses contiguous_format as a default
+
+ out_cuda = torch.empty((0), device=device).contiguous(memory_format=torch.contiguous_format)
+ # a_cuda and b_cuda have different memory_format
+ res2_cuda = torch.cat((a_cuda, b_cuda), out=out_cuda)
+
+ out_cpu = torch.empty((0), device='cpu').contiguous(memory_format=torch.contiguous_format)
+ res2_cpu = torch.cat((a_cpu, b_cpu), out=out_cpu)
+
+ self.assertTrue(res2_cuda.is_contiguous(memory_format=torch.contiguous_format))
+ self.assertTrue(res2_cpu.is_contiguous(memory_format=torch.channels_last))
+
+ out_cuda = torch.empty((0), device=device).contiguous(memory_format=torch.contiguous_format)
+ # a_cuda and c_cuda have same memory_format
+ res3_cuda = torch.cat((a_cuda, c_cuda), out=out_cuda)
+
+ self.assertTrue(res3_cuda.is_contiguous(memory_format=torch.channels_last))
@onlyCUDA
@deviceCountAtLeast(2)
def test_cat_stack_cross_devices(self, device):
cuda = torch.randn((3, 3), device=device)
cpu = torch.randn((3, 3), device='cpu')
- out_cpu = cpu.clone()
- out_cuda = cuda.clone()
+
+ # cat
with self.assertRaisesRegex(RuntimeError,
"Expected all tensors to be on the same device"):
torch.cat((cuda, cpu))
"Expected all tensors to be on the same device"):
torch.cat((cpu, cuda))
- with self.assertRaisesRegex(RuntimeError,
- "Expected all tensors to be on the same device"):
- torch.cat((cpu, cuda), out=out_cuda)
-
- with self.assertRaisesRegex(RuntimeError,
- "Expected all tensors to be on the same device"):
- torch.cat((cpu, cpu), out=out_cuda)
-
- with self.assertRaisesRegex(RuntimeError,
- "Expected all tensors to be on the same device"):
- torch.cat((cuda, cuda), out=out_cpu)
-
# Stack
with self.assertRaisesRegex(RuntimeError,
"Expected all tensors to be on the same device"):
"Expected all tensors to be on the same device"):
torch.stack((cpu, cuda))
- with self.assertRaisesRegex(RuntimeError,
- "Expected all tensors to be on the same device"):
- torch.stack((cpu, cuda), out=out_cuda)
-
- with self.assertRaisesRegex(RuntimeError,
- "Expected all tensors to be on the same device"):
- torch.stack((cpu, cpu), out=out_cuda)
-
- with self.assertRaisesRegex(RuntimeError,
- "Expected all tensors to be on the same device"):
- torch.stack((cuda, cuda), out=out_cpu)
-
# TODO: reconcile with other cat tests
# TODO: Compare with a NumPy reference instead of CPU
@onlyCUDA
-0.5790, 0.1497]])
""".format(**common_args))
+add_docstr(torch.concat,
+ r"""
+concat(tensors, dim=0, *, out=None) -> Tensor
+
+Alias of :func:`torch.cat`.
+""")
+
add_docstr(torch.ceil,
r"""
ceil(input, *, out=None) -> Tensor
{aten::multiply_, aten::mul_},
{aten::true_divide, aten::div},
{aten::true_divide_, aten::div_},
+ {aten::concat, aten::cat},
{aten::row_stack, aten::vstack},
{aten::swapdims, aten::transpose},
{aten::swapdims_, aten::transpose_},
torch.bucketize: lambda input, boundaries, out_int32=False, right=False, out=None: -1,
torch.cartesian_prod: lambda *tensors: -1,
torch.cat: lambda tensors, dim=0, out=None: -1,
+ torch.concat: lambda tensors, dim=0, out=None: -1, # alias for torch.cat
torch.cdist: lambda x1, x2, p=2.0, compute_mode='use_mm_for_euclid_dist_if_necessary': -1,
torch.ceil: lambda input, out=None: -1,
torch.celu: lambda input, alhpa=1., inplace=False: -1,
return (SampleInput(tensors, args=(0,)),)
+def sample_inputs_cat_concat(op_info, device, dtype, requires_grad, **kwargs):
+ make_arg = partial(make_tensor, device=device, dtype=dtype, requires_grad=requires_grad)
+
+ cases: Tuple[tuple, tuple, dict] = ( # type: ignore[assignment]
+ ((S, S), (S, S), {'dim': -1}),
+ ((S, S), (S, S), {'dim': 1}),
+ ((M, S), (S, S), {'dim': 0}), # different shapes
+ ((1, 2, 3), (1, 2, 3), {'dim': -2}),
+ ((0,), (0,), {'dim': 0}), # empty tensor
+ ((0, S), (S, S), {'dim': 0}),
+ ((1,), (1,), {}) # dim not passed, fallback to default
+ )
+
+ def generator():
+ for input_shape1, input_shape2, kwargs in cases:
+ yield SampleInput([make_arg(input_shape1), make_arg(input_shape2)], kwargs=kwargs)
+
+ return list(generator())
+
def sample_inputs_hstack_dstack_vstack(op_info, device, dtype, requires_grad, **kwargs):
tensors = [
make_tensor((S, S), device, dtype, requires_grad=requires_grad),
OpInfo('stack',
dtypes=all_types_and_complex_and(torch.bool, torch.float16, torch.bfloat16),
sample_inputs_func=sample_inputs_stack,
- assert_autodiffed=True,
- skips=(
- # stack does not correctly warn when resizing out= inputs
- SkipInfo('TestCommon', 'test_out'),),),
+ assert_autodiffed=True),
OpInfo('hstack',
dtypes=all_types_and_complex_and(torch.bool, torch.float16, torch.bfloat16),
sample_inputs_func=sample_inputs_hstack_dstack_vstack,
- supports_forward_ad=True,
- skips=(
- # hstack does not correctly warn when resizing out= inputs
- SkipInfo('TestCommon', 'test_out'),),),
+ supports_forward_ad=True),
OpInfo('hypot',
dtypes=floating_types(),
dtypesIfCPU=floating_types_and(torch.bfloat16),
# JIT tests don't work with Tensor keyword arguments
# https://github.com/pytorch/pytorch/issues/58507
SkipInfo('TestJit', 'test_variant_consistency_jit'),),),
+ OpInfo('cat',
+ ref=lambda input_seq, dim=0, **kwargs: np.concatenate(input_seq, axis=dim, **kwargs),
+ aliases=('concat',),
+ dtypes=all_types_and_complex_and(torch.bool, torch.float16, torch.bfloat16),
+ sample_inputs_func=sample_inputs_cat_concat,
+ supports_forward_ad=True,
+ assert_autodiffed=True,
+ skips=(
+ # RuntimeError: Arguments for call not valid.
+ # Expected a value of type 'List[Tensor]' for argument
+ # 'tensors' but instead found type 'Tensor (inferred)'.
+ SkipInfo('TestJit', 'test_jit_alias_remapping'),)),
OpInfo('vstack',
aliases=('row_stack',),
dtypes=all_types_and_complex_and(torch.bool, torch.float16, torch.bfloat16),
sample_inputs_func=sample_inputs_hstack_dstack_vstack,
supports_forward_ad=True,
skips=(
- # vstack does not correctly warn when resizing out= inputs
- SkipInfo('TestCommon', 'test_out'),
# RuntimeError: _fn() Expected a value of type
# 'Tensor (inferred)' for argument 't0' but instead found type 'tuple'.
- SkipInfo('TestJit', 'test_jit_alias_remapping'))),
+ SkipInfo('TestJit', 'test_jit_alias_remapping'),)),
OpInfo('dstack',
dtypes=all_types_and_complex_and(torch.bool, torch.float16, torch.bfloat16),
sample_inputs_func=sample_inputs_hstack_dstack_vstack,
- supports_forward_ad=True,
- skips=(
- # dstack does not correctly warn when resizing out= inputs
- SkipInfo('TestCommon', 'test_out'),)),
+ supports_forward_ad=True),
OpInfo('unfold',
op=lambda x, *args: x.unfold(*args),
dtypes=all_types_and_complex_and(torch.bool, torch.float16, torch.bfloat16),
projects / platform / upstream / pytorch.git / commitdiff