test_foreach.py

# Owner(s): ["module: mta"]

from contextlib import nullcontext
from numbers import Number
import random
import re
import torch
import unittest
import itertools

from torch.testing import make_tensor
from torch.testing._comparison import default_tolerances
from torch.testing._internal.common_utils import \
    TestCase, run_tests, TEST_WITH_ROCM, skipIfTorchDynamo, parametrize, gradcheck
from torch.testing._internal.common_device_type import \
    (instantiate_device_type_tests, dtypes, onlyCUDA, ops, OpDTypes)
from torch.testing._internal.common_methods_invocations import (
    foreach_unary_op_db, foreach_binary_op_db, foreach_pointwise_op_db,
    foreach_reduce_op_db, foreach_lerp_op_db)
from torch.testing._internal.common_dtype import (
    all_types_and_complex_and, integral_types, complex_types,
    floating_types_and, floating_types, integral_types_and,
)


_BOOL_SUB_ERR_MSG = "Subtraction, the `-` operator"


class RegularFuncWrapper:
    def __init__(self, func):
        self.func = func

    def __call__(self, inputs, values=None, **kwargs):
        if values is not None:
            assert len(inputs) == 3
            if isinstance(values, Number):
                values = [values for _ in range(len(inputs[0]))]
            return [self.func(*i, value=values[idx], **kwargs) for idx, i in enumerate(zip(*inputs))]
        if len(inputs) == 2 and isinstance(inputs[1], Number):
            # binary op with tensorlist and scalar.
            inputs[1] = [inputs[1] for _ in range(len(inputs[0]))]
        return [self.func(*i, **kwargs) for i in zip(*inputs)]


class ForeachFuncWrapper:
    def __init__(self, func):
        self.func = func
        # Some foreach functions don't have in-place implementations.
        self.is_inplace = False if func is None else func.__name__.endswith('_')

    def __call__(self, inputs, is_cuda, is_fastpath, **kwargs):
        actual = None
        zero_size = kwargs.pop("zero_size")
        if (
            is_cuda and
            torch.autograd.kineto_available() and
            torch.profiler.ProfilerActivity.CUDA in torch.profiler.supported_activities()
        ):
            with torch.profiler.profile() as p:
                actual = self.func(*inputs, **kwargs)
            keys = tuple([e.key for e in p.key_averages()])
            mta_called = any("multi_tensor_apply_kernel" in k for k in keys)
            assert mta_called == (is_fastpath and (not zero_size))
        else:
            actual = self.func(*inputs, **kwargs)
        # note(mkozuki): inplace foreach functions are void functions.
        return inputs[0] if self.is_inplace else actual


class InplaceForeachVersionBumpCheck:

    def __init__(self, testcase: TestCase, tensorlist: "List[torch.Tensor]") -> None:
        self._testcase = testcase
        self._tensorlist = tensorlist
        self._orig_version_counts = [t._version for t in tensorlist]

    def __enter__(self):
        pass

    def __exit__(self, exc_type, exc_value, traceback):
        # note(crcrpar): some methods e.g. `_binary_test` could call the given inplace function multiple times
        self._testcase.assertGreaterEqual([t._version for t in self._tensorlist], self._orig_version_counts)


def get_transform_func(num_tensors, dtype, device, is_fastpath):
    def transform(t):
        if not torch.is_tensor(t):
            return t
        return make_tensor(
            (num_tensors, num_tensors), dtype=dtype, device=device,
            requires_grad=True, noncontiguous=not is_fastpath,
        )

    return transform


def assert_multiple_grad_fns(tensors, test_case):
    test_case.assertEqual(len({t.grad_fn for t in tensors}), len(tensors), msg=f"{[t.grad_fn for t in tensors]}")


def clone(arg):
    if isinstance(arg, (list, tuple)):
        return [clone(a) for a in arg]
    if torch.is_tensor(arg):
        return arg.clone().detach().requires_grad_()
    else:
        return arg


# note(crcrpar): `zero_size` is `False` unless (dtype, device) == (torch.float32, "cuda")
# as the pair would go through `multi_tensor_apply_kernel` if inputs are not zero size.
class TestForeach(TestCase):
    @property
    def is_cuda(self):
        return self.device_type == 'cuda'

    def _get_funcs(self, op):
        return (
            ForeachFuncWrapper(op.method_variant),
            RegularFuncWrapper(op.ref),
            ForeachFuncWrapper(op.inplace_variant),
            RegularFuncWrapper(op.ref_inplace),
        )

    def _binary_test(
        self,
        dtype, op, ref, inputs, is_fastpath, is_inplace,
        *,
        alpha, scalar_self_arg: bool, zero_size: bool,
    ):
        if zero_size:
            with InplaceForeachVersionBumpCheck(self, inputs[0]) if op.is_inplace else nullcontext():
                op(inputs, self.is_cuda, is_fastpath, zero_size=zero_size)
                return

        ref_inputs = [[t.clone().detach() for t in inputs[0]], inputs[1]] if is_inplace else inputs
        try:
            with InplaceForeachVersionBumpCheck(self, inputs[0]) if op.is_inplace else nullcontext():
                actual = op(inputs, self.is_cuda, is_fastpath, zero_size=zero_size)
        except RuntimeError as e:
            with self.assertRaisesRegex(type(e), re.escape(str(e))):
                if not scalar_self_arg:
                    ref(ref_inputs)
                else:
                    [ref.func(ref_inputs[0], t) for t in ref_inputs[1]]
        else:
            expected = ref(ref_inputs) if not scalar_self_arg else [ref.func(ref_inputs[0], t) for t in ref_inputs[1]]
            self.assertEqual(actual, expected)
        if alpha is not None and not scalar_self_arg:
            kwargs = {'alpha': alpha}
            ref_inputs = inputs
            try:
                op_kwargs = {}
                op_kwargs.update(kwargs)
                op_kwargs['zero_size'] = zero_size
                with InplaceForeachVersionBumpCheck(self, inputs[0]) if op.is_inplace else nullcontext():
                    actual = op(inputs, self.is_cuda, is_fastpath, **op_kwargs)
            except RuntimeError as e:
                with self.assertRaisesRegex(type(e), re.escape(str(e))):
                    ref(ref_inputs, **kwargs)
            else:
                expected = ref(ref_inputs, **kwargs)
                if dtype in (torch.float16, torch.bfloat16) and TEST_WITH_ROCM:
                    self.assertEqual(expected, actual, atol=1.e-3, rtol=default_tolerances(dtype)[0])
                else:
                    self.assertEqual(expected, actual)

    @ops(foreach_binary_op_db)
    @parametrize("is_fastpath", (True, False))
    def test_binary_op(self, device, dtype, op, is_fastpath):
        scalar_self_arg_test_complete = False
        for i, sample in enumerate(op.sample_inputs(device, dtype, noncontiguous=not is_fastpath)):
            (rhs_arg,) = sample.args
            zero_size = sample.kwargs.pop("zero_size")
            kwargs = {} or sample.kwargs
            alpha = kwargs.pop("alpha", None)
            disable_fastpath = kwargs.pop("disable_fastpath") if is_fastpath else False
            wrapped_op, ref, inplace_op, inplace_ref = self._get_funcs(op)
            self._binary_test(
                dtype, wrapped_op, ref, [sample.input, rhs_arg],
                is_fastpath and not disable_fastpath, False,
                alpha=alpha, zero_size=zero_size, scalar_self_arg=False,
            )
            self._binary_test(
                dtype, inplace_op, inplace_ref, [sample.input, rhs_arg],
                is_fastpath and not disable_fastpath, True,
                alpha=alpha, zero_size=zero_size, scalar_self_arg=False,
            )

            if op.supports_autograd and dtype in floating_types() and not zero_size:
                transformed_sample = sample.transform(get_transform_func(len(sample.input), dtype, device, is_fastpath))
                tensors = transformed_sample.input
                (rhs_arg,) = transformed_sample.args
                ref_tensors, ref_rhs_arg = clone(tensors), clone(rhs_arg)
                try:
                    sum(
                        wrapped_op([tensors, rhs_arg], is_cuda=False, is_fastpath=False, zero_size=zero_size)
                    ).mean().backward()
                except RuntimeError:
                    with self.assertRaises(RuntimeError):
                        sum(ref([ref_tensors, ref_rhs_arg])).mean().backward()
                else:
                    sum(ref([ref_tensors, ref_rhs_arg])).mean().backward()
                    self.assertEqual([t.grad for t in tensors], [t.grad for t in ref_tensors])
                    if isinstance(rhs_arg, list) and isinstance(rhs_arg[0], torch.Tensor):
                        self.assertEqual([t.grad for t in rhs_arg], [t.grad for t in ref_rhs_arg])
                    tensors = [t.clone().detach().requires_grad_().clone() for t in tensors]
                    ref_tensors = [t.clone().detach().requires_grad_().clone() for t in tensors]
                    inplace_op([tensors, rhs_arg], is_cuda=False, is_fastpath=False, zero_size=zero_size)
                    assert_multiple_grad_fns(tensors, self)

                    # note(crcrpar): the following ops' reference torch functions don't have the overload with Scalar/ScalarList.
                    is_foreach_max_min_imum_with_scalar_or_scalarlist = (
                        inplace_op.func in (torch._foreach_minimum_, torch._foreach_maximum_)
                        and (
                            isinstance(rhs_arg, Number) or (isinstance(rhs_arg, list) and isinstance(rhs_arg[0], Number))
                        )
                    )
                    if not is_foreach_max_min_imum_with_scalar_or_scalarlist:
                        inplace_ref([ref_tensors, rhs_arg])
                        torch.autograd.backward(sum([t.clone() for t in tensors]).sum(), inputs=tensors)
                        torch.autograd.backward(sum([t.clone() for t in ref_tensors]).sum(), inputs=ref_tensors)
                        self.assertEqual([t.grad for t in tensors], [t.grad for t in ref_tensors])
            if (
                op.supports_scalar_self_arg
                and isinstance(rhs_arg, Number)
                and not scalar_self_arg_test_complete
                and not zero_size
            ):
                scalar_self_arg_test_complete = True
                self._binary_test(
                    dtype, wrapped_op, ref, [rhs_arg, sample.input], is_fastpath, False,
                    alpha=alpha, scalar_self_arg=True, zero_size=False,
                )
                if op.supports_autograd and dtype == torch.float32 and not zero_size:
                    transformed_sample = sample.transform(
                        get_transform_func(len(sample.input), dtype, device, is_fastpath))
                    tensors = transformed_sample.input
                    (rhs_arg,) = transformed_sample.args
                    ref_tensors, ref_rhs_arg = clone(tensors), clone(rhs_arg)
                    sum(wrapped_op(
                        [rhs_arg, tensors], is_cuda=False, is_fastpath=False, zero_size=False
                    )).mean().backward()
                    sum([ref.func(ref_rhs_arg, t) for t in ref_tensors]).mean().backward()
                    self.assertEqual([t.grad for t in tensors], [t.grad for t in ref_tensors])

    @ops(foreach_pointwise_op_db)
    @parametrize("is_fastpath", (True, False))
    def test_pointwise_op(self, device, dtype, op, is_fastpath):
        for sample in op.sample_inputs(device, dtype, noncontiguous=not is_fastpath):
            assert isinstance(sample.args, tuple)
            assert len(sample.args) == 2
            inputs = [sample.input, *sample.args]
            zero_size = sample.kwargs.pop("zero_size")
            kwargs = sample.kwargs
            disable_fastpath = kwargs.pop("disable_fastpath") if is_fastpath else False
            wrapped_op, ref, inplace_op, inplace_ref = self._get_funcs(op)
            values = kwargs.pop("values")
            self._pointwise_test(
                wrapped_op, ref, inputs, is_fastpath and not disable_fastpath, False, values=values, zero_size=zero_size
            )
            self._pointwise_test(
                inplace_op, inplace_ref, inputs, is_fastpath and not disable_fastpath,
                True, values=values, zero_size=zero_size)

            if op.supports_autograd and dtype in floating_types() and not zero_size:
                transformed_sample = sample.transform(get_transform_func(len(sample.input), dtype, device, is_fastpath))
                tensors = transformed_sample.input
                rhs_arg = transformed_sample.args
                ref_tensors, ref_rhs_arg = clone(tensors), clone(rhs_arg)
                try:
                    sum(
                        wrapped_op([tensors, *rhs_arg], is_cuda=False, is_fastpath=False, zero_size=zero_size)
                    ).mean().backward()
                except RuntimeError:
                    with self.assertRaises(RuntimeError):
                        sum(ref([ref_tensors, *ref_rhs_arg])).mean().backward()
                else:
                    sum(ref([ref_tensors, *ref_rhs_arg])).mean().backward()
                    self.assertEqual([t.grad for t in tensors], [t.grad for t in ref_tensors])
                    for op_list, ref_list in zip(rhs_arg, ref_rhs_arg):
                        if isinstance(op_list, list) and isinstance(op_list[0], torch.Tensor):
                            self.assertEqual([t.grad for t in op_list], [t.grad for t in ref_list])
                    tensors = [t.clone().detach().requires_grad_().clone() for t in tensors]
                    ref_tensors = [t.clone().detach().requires_grad_().clone() for t in tensors]
                    inplace_op([tensors, *rhs_arg], is_cuda=False, is_fastpath=False, zero_size=zero_size)
                    assert_multiple_grad_fns(tensors, self)
                    inplace_ref([ref_tensors, *rhs_arg])
                    torch.autograd.backward(sum([t.clone() for t in tensors]).sum(), inputs=tensors)
                    torch.autograd.backward(sum([t.clone() for t in ref_tensors]).sum(), inputs=ref_tensors)
                    self.assertEqual([t.grad for t in tensors], [t.grad for t in ref_tensors])

            if is_fastpath and isinstance(values, list) and not zero_size:
                sample = sample.transform(lambda t: t.clone().detach() if torch.is_tensor(t) else t)
                inputs = [sample.input, *sample.args]
                tensor_values = torch.tensor(values)
                # 1D Tensor of scalars
                for is_inplace, op_, ref_ in ((False, wrapped_op, ref), (True, inplace_op, inplace_ref)):
                    self._pointwise_test(
                        op_, ref_, inputs, is_fastpath and not disable_fastpath, is_inplace,
                        values=tensor_values, zero_size=False)
                    self._pointwise_test(
                        op_, ref_, inputs, is_fastpath and not disable_fastpath, is_inplace,
                        values=tensor_values[0],
                        custom_values_err="Expected packed scalar Tensor to be of dimension 1. Got 0 instead.",
                        zero_size=False,
                    )
                    if self.is_cuda:
                        self._pointwise_test(
                            op_, ref_, inputs, is_fastpath and not disable_fastpath, is_inplace,
                            values=tensor_values.cuda(),
                            custom_values_err="Expected scalars to be on CPU, got cuda:0 instead.",
                            zero_size=False,
                        )
                    self._pointwise_test(
                        op_, ref_, inputs, is_fastpath and not disable_fastpath, is_inplace,
                        values=tensor_values[:2],
                        custom_values_err=f"Expected length of scalars to match input of length {len(values)} but got 2 instead.",
                        zero_size=False,
                    )
                    self._pointwise_test(
                        op_, ref_, inputs, is_fastpath and not disable_fastpath, is_inplace,
                        values=torch.tensor([[0, 1], [2, 3]])[:, 1],
                        custom_values_err="Expected scalars to be contiguous.",
                        zero_size=False,
                    )

            if not zero_size:
                # Tests of implicit broadcasting
                N = len(sample.input)
                inputs = [
                    [make_tensor((N, N), device=device, dtype=dtype, noncontiguous=not is_fastpath) for _ in range(N)],
                    [
                        make_tensor((N - i, 1), device=device, dtype=dtype, noncontiguous=not is_fastpath)
                        for i in range(N)
                    ],
                    [
                        make_tensor((1, N - i), device=device, dtype=dtype, noncontiguous=not is_fastpath)
                        for i in range(N)
                    ],
                ]
                self._pointwise_test(
                    wrapped_op, ref, inputs, is_fastpath and disable_fastpath, is_inplace=False,
                    values=values, zero_size=zero_size)
                self._pointwise_test(
                    inplace_op, inplace_ref, inputs, is_fastpath and disable_fastpath,
                    is_inplace=True, values=values, zero_size=zero_size)

    def _pointwise_test(
        self,
        op, ref, inputs, is_fastpath, is_inplace,
        *,
        values=None, custom_values_err=None, zero_size,
    ):
        kwargs = {'zero_size': zero_size}
        if zero_size:
            op(inputs, self.is_cuda, is_fastpath, **kwargs)
            return
        ref_inputs = [[t.clone().detach() for t in inputs[0]], inputs[1], inputs[2]] if is_inplace else inputs
        try:
            with (InplaceForeachVersionBumpCheck(self, inputs[0]) if is_inplace else nullcontext()):
                actual = op(inputs, self.is_cuda, is_fastpath, **kwargs)
        except RuntimeError as e:
            with self.assertRaisesRegex(type(e), re.escape(str(e))):
                ref(ref_inputs)
        else:
            expected = ref(ref_inputs)
            self.assertEqual(expected, actual)
        if values is not None:
            try:
                actual = op(inputs + [values], self.is_cuda, is_fastpath, **kwargs)
            except RuntimeError as e:
                # Match with error messages from regular non-foreach reference if no
                # custom error message was provided.
                if custom_values_err is None:
                    with self.assertRaisesRegex(type(e), re.escape(str(e))):
                        ref(ref_inputs, values=values)
                else:
                    self.assertEqual(re.escape(str(e)), re.escape(custom_values_err))
            else:
                expected = ref(ref_inputs, values=values)
                self.assertEqual(expected, actual)

    # note(mkozuki): why `try-except` for both fastpath?
    # - inputs for fastpath can be integer tensors.
    #    - this is because opinfo dtypes are configured for out-place implementation
    # - for integer inputs, trigonometric functions and exponential function returns float outputs,
    #   which causes "result type Float can't be case to the desired type" error.
    # Thus, `try-except` is used even if `is_fastpath` is `True`.
    def _inplace_unary_test(self, inplace, inplace_ref, inputs, is_fastpath, **kwargs):
        copied_inputs = [[t.clone().detach() for t in tensors] for tensors in inputs]
        try:
            with InplaceForeachVersionBumpCheck(self, inputs[0]):
                inplace(inputs, self.is_cuda, is_fastpath, **kwargs)
        except RuntimeError as e:
            with self.assertRaisesRegex(type(e), re.escape(str(e))):
                inplace_ref(copied_inputs)
        else:
            inplace_ref(copied_inputs)
            self.assertEqual(copied_inputs, inputs)

    @ops(foreach_unary_op_db)
    @parametrize("is_fastpath", (True, False))
    def test_unary_op(self, device, dtype, op, is_fastpath):
        out_place_defined = op.name != "_foreach_zero"
        wrapped_op, ref, inplace_op, inplace_ref = self._get_funcs(op)
        samples = op.sample_inputs(device, dtype, noncontiguous=not is_fastpath)
        disable_fastpath = op.name == "_foreach_abs" and dtype in complex_types()
        for sample in samples:
            zero_size = sample.kwargs.pop('zero_size')
            inputs = [sample.input]
            if zero_size:
                if out_place_defined:
                    wrapped_op(inputs, self.is_cuda, is_fastpath and not disable_fastpath, zero_size=zero_size)
                inplace_op(inputs, self.is_cuda, is_fastpath and not disable_fastpath, zero_size=zero_size)
                continue
            inputs = [sample.input]
            disable_fastpath = (op.name == "_foreach_abs" and dtype in complex_types()) or sample.kwargs.pop(
                "disable_fastpath"
            )
            if out_place_defined:
                self.assertEqual(
                    ref(inputs),
                    wrapped_op(inputs, self.is_cuda, is_fastpath and not disable_fastpath, zero_size=zero_size),
                )
            self._inplace_unary_test(
                inplace_op, inplace_ref, [sample.input], is_fastpath and not disable_fastpath, zero_size=zero_size
            )
            if op.supports_autograd and dtype in floating_types() and not zero_size:
                tensors = [t.clone().detach().requires_grad_() for t in sample.input]
                ref_tensors = [t.clone().detach().requires_grad_() for t in tensors]
                if out_place_defined:
                    out = wrapped_op.func(tensors)
                    # tensors have different shapes
                    torch.cat([t.view(-1) for t in out]).mean().backward()
                    torch.cat([ref.func(t).view(-1) for t in ref_tensors]).mean().backward()
                    self.assertEqual([t.grad for t in tensors], [t.grad for t in ref_tensors])
                    self.assertEqual(len({t.grad_fn for t in out}), 1)

                inplace_input_tensors = [t.clone().detach().requires_grad_() for t in tensors]
                inplace_inputs = [t.clone() for t in inplace_input_tensors]
                # set both to False to skip multi_tensor_apply_kernel check
                inplace_op([inplace_inputs], False, False, zero_size=zero_size)
                assert_multiple_grad_fns(inplace_inputs, self)

                # per-tensor `grad_fn` check.
                hook_buffer = []

                def get_grad_fn_hook(i):

                    def hook(grad_inputs, grad_outputs) -> None:
                        hook_buffer.append(i)

                    return hook

                for i, t in enumerate(inplace_inputs):
                    t.grad_fn.register_hook(get_grad_fn_hook(i))

                _ = torch.autograd.grad(
                    inplace_inputs[0],
                    inputs=(inplace_input_tensors[0],),
                    grad_outputs=(torch.rand_like(inplace_inputs[0]),),
                    retain_graph=True,
                )
                self.assertEqual(hook_buffer, [0])
                hook_buffer.clear()

                # tensors have different shapes.
                sum_of_cloned_tensors = torch.cat([t.view(-1) for t in inplace_inputs]).sum()
                grad_output = torch.rand_like(sum_of_cloned_tensors)
                grad_inputs = torch.autograd.grad(
                    sum_of_cloned_tensors,
                    inputs=tuple(inplace_input_tensors),
                    grad_outputs=(grad_output,),
                    retain_graph=False,
                )
                self.assertEqual(hook_buffer, list(reversed(range(len(inplace_inputs)))))

                ref_inplace_input_tensors = [t.clone().detach().requires_grad_() for t in inplace_input_tensors]
                ref_inplace_inputs = [t.clone() for t in ref_inplace_input_tensors]
                ref_output = inplace_ref([ref_inplace_inputs])
                ref_grad_inputs = torch.autograd.grad(
                    torch.cat([t.view(-1) for t in ref_output]).sum(),
                    inputs=tuple(ref_inplace_input_tensors),
                    grad_outputs=(grad_output,),
                )
                self.assertEqual(grad_inputs, ref_grad_inputs)

    @ops(foreach_reduce_op_db)
    @parametrize("is_fastpath", (True, False))
    def test_reduce_op(self, device, dtype, op, is_fastpath):
        for sample in op.sample_inputs(device, dtype, noncontiguous=not is_fastpath):
            ord = sample.kwargs.pop("ord")
            zero_size = sample.kwargs.pop("zero_size")
            disable_fastpath = sample.kwargs.pop("disable_fastpath", False)

            inputs = (sample.input,)
            wrapped_op, ref, _, _ = self._get_funcs(op)

            self.assertEqual(
                ref(inputs, ord=ord),
                wrapped_op(
                    inputs, self.is_cuda, is_fastpath and not disable_fastpath, ord=ord,
                    zero_size=zero_size,
                ),
            )
            if op.supports_autograd and dtype in floating_types() and not zero_size:
                transformed_sample = sample.transform(get_transform_func(len(sample.input), dtype, device, is_fastpath))
                tensors = transformed_sample.input
                ref_tensors = clone(tensors)
                sum(wrapped_op((tensors,), False, False, ord=ord, zero_size=zero_size)).backward()
                sum(ref((ref_tensors,), ord=ord)).backward()
                self.assertEqual([t.grad for t in tensors], [t.grad for t in ref_tensors])

    @dtypes(*all_types_and_complex_and(torch.half, torch.bfloat16, torch.bool))
    def test_add_scalar_with_empty_list_and_empty_tensor(self, device, dtype):
        # TODO: enable empty list case
        for tensors in [[torch.randn([0])]]:
            res = torch._foreach_add(tensors, 1)
            self.assertEqual(res, tensors)

            torch._foreach_add_(tensors, 1)
            self.assertEqual(res, tensors)

    @ops(foreach_binary_op_db, dtypes=OpDTypes.supported)
    def test_binary_op_scalar_with_overlapping_tensors(self, device, dtype, op):
        foreach_op, ref = op.method_variant, op.ref
        tensors = [torch.ones(1, 1, device=device, dtype=dtype).expand(2, 1, 3)]

        if ref == torch.sub and dtype == torch.bool:
            with self.assertRaisesRegex(RuntimeError, re.escape(_BOOL_SUB_ERR_MSG)):
                [ref(t, 1) for t in tensors]
            with self.assertRaisesRegex(RuntimeError, re.escape(_BOOL_SUB_ERR_MSG)):
                foreach_op(tensors, 1)
            return

        expected = [ref(t, 1) for t in tensors]
        res = foreach_op(tensors, 1)
        self.assertEqual(res, expected)

    @ops(foreach_binary_op_db, allowed_dtypes=[torch.float])
    def test_binary_op_scalar_with_different_tensor_dtypes(self, device, dtype, op):
        foreach_op = op.method_variant
        tensors = [
            torch.tensor([1.1], dtype=torch.float, device=device),
            torch.tensor([1], dtype=torch.long, device=device),
        ]
        runtime_error = None
        try:
            foreach_op(tensors, 1)
        except RuntimeError as e:
            runtime_error = e
        self.assertIsNone(runtime_error)

    @skipIfTorchDynamo("Different error msgs, TODO")
    @ops(foreach_binary_op_db, dtypes=OpDTypes.supported)
    def test_binary_op_list_error_cases(self, device, dtype, op):
        foreach_op, foreach_op_, ref, ref_ = op.method_variant, op.inplace_variant, op.ref, op.ref_inplace
        tensors1 = []
        tensors2 = []

        # Empty lists
        with self.assertRaisesRegex(RuntimeError, "There were no tensor arguments to this function"):
            foreach_op(tensors1, tensors2)
        with self.assertRaisesRegex(RuntimeError, "There were no tensor arguments to this function"):
            foreach_op_(tensors1, tensors2)

        # One empty list
        tensors1.append(torch.tensor([1], device=device, dtype=dtype))
        with self.assertRaisesRegex(RuntimeError, "Tensor list must have same number of elements as scalar list."):
            foreach_op(tensors1, tensors2)
        with self.assertRaisesRegex(RuntimeError, "Tensor list must have same number of elements as scalar list."):
            foreach_op_(tensors1, tensors2)

        # Lists have different amount of tensors
        tensors2.append(torch.tensor([1], device=device))
        tensors2.append(torch.tensor([1], device=device))
        with self.assertRaisesRegex(RuntimeError, "Tensor lists must have the same number of tensors, got 1 and 2"):
            foreach_op(tensors1, tensors2)
        with self.assertRaisesRegex(RuntimeError, "Tensor lists must have the same number of tensors, got 1 and 2"):
            foreach_op_(tensors1, tensors2)

        # Corresponding tensors with different sizes that aren't compatible with broadcast
        # If sizes are different then foreach chooses slow path, thus error messages are expected
        # to be the same as torch regular function.
        tensors1 = [torch.zeros(10, 10, device=device, dtype=dtype) for _ in range(10)]
        tensors2 = [torch.ones(11, 11, device=device, dtype=dtype) for _ in range(10)]
        try:
            foreach_op(tensors1, tensors2)
        except RuntimeError as e:
            with self.assertRaisesRegex(type(e), re.escape(str(e))):
                [ref(t1, t2) for t1, t2 in zip(tensors1, tensors2)]
        try:
            foreach_op_(tensors1, tensors2)
        except RuntimeError as e:
            with self.assertRaisesRegex(type(e), re.escape(str(e))):
                [ref_(t1, t2) for t1, t2 in zip(tensors1, tensors2)]

        # different devices
        if self.device_type == "cuda" and torch.cuda.device_count() > 1:
            tensor1 = torch.zeros(10, 10, device="cuda:0", dtype=dtype)
            tensor2 = torch.ones(10, 10, device="cuda:1", dtype=dtype)
            if dtype == torch.bool and foreach_op == torch._foreach_sub:
                with self.assertRaisesRegex(RuntimeError, re.escape(_BOOL_SUB_ERR_MSG)):
                    foreach_op([tensor1], [tensor2])
                with self.assertRaisesRegex(RuntimeError, re.escape(_BOOL_SUB_ERR_MSG)):
                    foreach_op_([tensor1], [tensor2])
                return
            with self.assertRaisesRegex(RuntimeError, "Expected all tensors to be on the same device"):
                foreach_op([tensor1], [tensor2])
            if dtype in integral_types_and(torch.bool) and foreach_op == torch._foreach_div:
                with self.assertRaisesRegex(RuntimeError, "result type"):
                    foreach_op_([tensor1], [tensor2])
            else:
                with self.assertRaisesRegex(RuntimeError, "Expected all tensors to be on the same device"):
                    foreach_op_([tensor1], [tensor2])

    @unittest.skipIf(not torch.cuda.is_available(), "CUDA not found")
    @ops(foreach_binary_op_db, dtypes=OpDTypes.supported)
    def test_binary_op_list_slow_path(self, device, dtype, op):
        foreach_op, native_op, foreach_op_, native_op_ = self._get_funcs(op)
        # 0-strides
        tensor1 = make_tensor((10, 10), dtype=dtype, device=device)
        tensor2 = make_tensor((1,), device=device, dtype=dtype).expand_as(tensor1)
        inputs = ([tensor1], [tensor2])
        self._binary_test(
            dtype, foreach_op, native_op, inputs, is_fastpath=False, is_inplace=False,
            zero_size=False, alpha=None, scalar_self_arg=False)
        self._binary_test(
            dtype, foreach_op_, native_op_, inputs, is_fastpath=False, is_inplace=True,
            zero_size=False, alpha=None, scalar_self_arg=False)

        # different strides
        tensor1 = torch.zeros(10, 10, device=device, dtype=dtype)
        tensor2 = torch.ones(10, 10, device=device, dtype=dtype)
        inputs = ([tensor1], [tensor2.t()])
        self._binary_test(
            dtype, foreach_op, native_op, inputs, is_fastpath=False, is_inplace=False,
            zero_size=False, alpha=None, scalar_self_arg=False)
        self._binary_test(
            dtype, foreach_op_, native_op_, inputs, is_fastpath=False, is_inplace=True,
            zero_size=False, alpha=None, scalar_self_arg=False)

        # non contiguous
        tensor1 = make_tensor((5, 2, 1, 3), device=device, dtype=dtype, noncontiguous=True)
        tensor2 = make_tensor((5, 2, 1, 3), device=device, dtype=dtype, noncontiguous=True)
        self.assertFalse(tensor1.is_contiguous())
        self.assertFalse(tensor2.is_contiguous())
        inputs = ([tensor1], [tensor2])
        self._binary_test(
            dtype, foreach_op, native_op, inputs, is_fastpath=False, is_inplace=False,
            zero_size=False, alpha=None, scalar_self_arg=False)
        self._binary_test(
            dtype, foreach_op_, native_op_, inputs, is_fastpath=False, is_inplace=True,
            zero_size=False, alpha=None, scalar_self_arg=False)

        # sliced tensor
        tensor1 = make_tensor((5, 2, 1, 3), device=device, dtype=dtype)
        tensor2 = make_tensor((5, 2, 1, 3 * 7), device=device, dtype=dtype)[:, :, :, ::7]
        inputs = ([tensor1], [tensor2])
        self._binary_test(
            dtype, foreach_op, native_op, inputs, is_fastpath=False, is_inplace=False,
            zero_size=False, alpha=None, scalar_self_arg=False)
        self._binary_test(
            dtype, foreach_op_, native_op_, inputs, is_fastpath=False, is_inplace=True,
            zero_size=False, alpha=None, scalar_self_arg=False)

    @ops(foreach_binary_op_db, dtypes=floating_types_and(torch.half, torch.bfloat16))
    def test_binary_op_float_inf_nan(self, device, dtype, op):
        inputs = (
            [
                torch.tensor([float("inf")], device=device, dtype=dtype),
                torch.tensor([-float("inf")], device=device, dtype=dtype),
                torch.tensor([float("nan")], device=device, dtype=dtype),
                torch.tensor([float("nan")], device=device, dtype=dtype),
            ],
            [
                torch.tensor([-float("inf")], device=device, dtype=dtype),
                torch.tensor([float("inf")], device=device, dtype=dtype),
                torch.tensor([float("inf")], device=device, dtype=dtype),
                torch.tensor([float("nan")], device=device, dtype=dtype),
            ],
        )
        op, ref, inplace_op, inplace_ref = self._get_funcs(op)
        self._binary_test(dtype, op, ref, inputs, True, False, zero_size=False, alpha=None, scalar_self_arg=False)
        self._binary_test(
            dtype, inplace_op, inplace_ref, inputs, True, True, zero_size=False, alpha=None, scalar_self_arg=False
        )

    # note: Below three tests (postfixed with `_tensors_on_different_devices`)
    # checks whether foreach works with lists of tensors on different devices
    # but tensors of the same index are on the same device, e.g., ['cuda', 'cpu].
    @onlyCUDA
    @ops(foreach_unary_op_db)
    def test_unary_op_tensors_on_different_devices(self, device, dtype, op):
        out_place_defined = op.name != "_foreach_zero"
        method, ref, inplace_method, ref_inplace = self._get_funcs(op)
        # tensors: ['cuda', 'cpu]
        tensors = list(op.sample_inputs(device, dtype, num_input_tensors=[2]))[0].input
        tensors[1] = tensors[1].to("cpu")
        if out_place_defined:
            try:
                actual = method((tensors,), False, False, zero_size=False)
            except RuntimeError as e:
                with self.assertRaisesRegex(type(e), str(e)):
                    ref((tensors,))
            else:
                expected = ref((tensors,))
                self.assertEqual(expected, actual)

        try:
            inplace_method((tensors,), False, False, zero_size=False)
        except RuntimeError as e:
            with self.assertRaisesRegex(type(e), str(e)):
                ref_inplace((tensors,))
        else:
            if out_place_defined:
                self.assertEqual(expected, tensors)
            else:
                self.assertEqual([torch.zeros_like(t) for t in tensors], tensors)

    @onlyCUDA
    @ops(foreach_binary_op_db)
    def test_binary_op_tensors_on_different_devices(self, device, dtype, op):
        # `tensors1`: ['cuda', 'cpu']
        # `tensors2`: ['cuda', 'cpu']
        _cuda_tensors = list(op.sample_inputs(device, dtype, num_input_tensors=[2], same_size=True))[0].input
        _cpu_tensors = list(op.sample_inputs("cpu", dtype, num_input_tensors=[2], same_size=True))[0].input
        tensors1, tensors2 = list(zip(_cuda_tensors, _cpu_tensors))

        foreach_op, foreach_op_ = op.method_variant, op.inplace_variant
        native_op, native_op_ = op.ref, op.ref_inplace
        try:
            actual = foreach_op(tensors1, tensors2)
        except RuntimeError as e:
            with self.assertRaisesRegex(type(e), re.escape(str(e))):
                [native_op(t1, t2) for t1, t2 in zip(tensors1, tensors2)]
        else:
            expected = [native_op(t1, t2) for t1, t2 in zip(tensors1, tensors2)]
            self.assertEqual(expected, actual)
        try:
            foreach_op_(tensors1, tensors2)
        except RuntimeError as e:
            with self.assertRaisesRegex(type(e), re.escape(str(e))):
                [native_op_(t1, t2) for t1, t2 in zip(tensors1, tensors2)]
        else:
            self.assertEqual(actual, tensors1)

    @onlyCUDA
    @ops(foreach_pointwise_op_db, allowed_dtypes=floating_types())
    def test_pointwise_op_tensors_on_different_devices(self, device, dtype, op):
        # tensors1: ['cuda', 'cpu]
        # tensors2: ['cuda', 'cpu]
        # tensors3: ['cuda', 'cpu]
        # first tensorlist is zero-size when float32
        _cuda_tensors = list(
            op.sample_inputs(device, dtype, num_input_tensors=[3], same_size=True)
        )[int(dtype == torch.float32)].input
        _cpu_tensors = list(op.sample_inputs("cpu", dtype, num_input_tensors=[3], same_size=True))[0].input
        tensors1, tensors2, tensors3 = list(zip(_cuda_tensors, _cpu_tensors))

        foreach_op, foreach_op_, native_op = op.method_variant, op.inplace_variant, op.ref
        actual = foreach_op(tensors1, tensors2, tensors3)
        expected = [native_op(*_cuda_tensors), native_op(*_cpu_tensors)]
        self.assertEqual(expected, actual)

        # note(mkozuki): Limiting dtypes to FP32&FP64, we can safely run inplace ops.
        foreach_op_(tensors1, tensors2, tensors3)
        self.assertEqual(expected, tensors1)

    # note: BFloat16 has the same number of exponent bits as FP32
    # so if squared L2 norm overflows in BF16, then it also overflows in FP32.
    @onlyCUDA
    @ops(foreach_reduce_op_db, allowed_dtypes=(torch.half, torch.bfloat16))
    def test_foreach_l2_large_value_input(self, device, dtype, op):
        ord, N = 2, 10
        max_value = torch.finfo(dtype).max
        scaler = torch.tensor([max_value]).sqrt().to(device=device, dtype=dtype)
        inputs = ([
            t * scaler for t in list(
                op.sample_inputs(device, dtype, requries_grad=True, num_input_tensors=[N], low=1)
            )[0].input
        ],)
        # make sure that the min. of squared L2 norm value per tensor is greater than the max value of `dtype`.
        self.assertTrue(scaler * scaler * N > max_value)
        fn, ref_fn, *_ = self._get_funcs(op)
        actual = fn(inputs, is_cuda=True, is_fastpath=True, ord=ord, zero_size=False)
        expect = ref_fn(inputs, ord=ord)
        if dtype == torch.float16:
            # making sure the reference L2 norm values are in the range of FP16.
            self.assertFalse(any(torch.isinf(e) for e in expect))
        else:
            self.assertTrue(all(torch.isinf(e) for e in expect))
        self.assertEqual(expect, actual, equal_nan=False)

    @parametrize("is_fastpath", (True, False))
    @ops(foreach_lerp_op_db)
    def test_lerp(self, device, dtype, op, is_fastpath):
        for sample in op.sample_inputs(device, dtype, noncontiguous=not is_fastpath):
            wrapped_op, ref, inplace_op, inplace_ref = self._get_funcs(op)
            args = [*sample.args]
            inputs = [sample.input, args[0]]
            zero_size = sample.kwargs.pop("zero_size")

            kwargs, ref_kwargs = {"zero_size": zero_size}, {}
            if isinstance(args[1], list):
                inputs.append(args[1])
            else:
                kwargs["weight"] = args[1]
                ref_kwargs["weight"] = args[1]

            if dtype in integral_types() or dtype == torch.bool or (not self.is_cuda and dtype == torch.half):
                with self.assertRaises(RuntimeError):
                    wrapped_op(inputs, self.is_cuda, is_fastpath, **kwargs)
                return
            actual = wrapped_op(inputs, self.is_cuda, is_fastpath, **kwargs)
            expected = ref(inputs, **ref_kwargs)
            self.assertEqual(actual, expected)

            inplace_inputs = [[t.clone() for t in inputs[0]]] + inputs[1:]
            with InplaceForeachVersionBumpCheck(self, inplace_inputs[0]):
                inplace_actual = inplace_op(inplace_inputs, self.is_cuda, is_fastpath, **kwargs)
            self.assertEqual(inplace_actual, expected)

            if op.supports_autograd and dtype in floating_types() and not zero_size:
                transformed_sample = sample.transform(get_transform_func(len(sample.input), dtype, device, is_fastpath))
                args = [*transformed_sample.args]
                inputs = [transformed_sample.input, args[0]]

                kwargs, ref_kwargs = {}, {}
                if isinstance(args[1], list):
                    inputs.append(args[1])
                else:
                    kwargs = ref_kwargs = {"weight": args[1]}
                ref_tensors = clone(transformed_sample.input)
                sum(
                    wrapped_op((transformed_sample.input, *inputs[1:]), False, False, **kwargs, zero_size=zero_size)
                ).mean().backward()
                sum(ref((ref_tensors, *inputs[1:]), **ref_kwargs)).mean().backward()
                self.assertEqual(
                    [t.grad for t in transformed_sample.input],
                    [t.grad for t in ref_tensors],
                )
                _tensors = [t.clone().detach().requires_grad_() for t in transformed_sample.input]
                _ref_tensors = [t.clone().detach().requires_grad_() for t in _tensors]
                tensors = [t.clone() for t in _tensors]
                inplace_op((tensors, *inputs[1:]), False, False, **kwargs, zero_size=False)
                ref_tensors = [t.clone() for t in _ref_tensors]
                inplace_ref((ref_tensors, *inputs[1:]), **ref_kwargs)
                assert_multiple_grad_fns(tensors, self)

                # tensors have different shapes.
                torch.autograd.backward(torch.cat([t.clone().view(-1) for t in tensors]).sum(), inputs=tensors)
                torch.autograd.backward(torch.cat([t.clone().view(-1) for t in ref_tensors]).sum(), inputs=ref_tensors)
                self.assertEqual([t.grad for t in tensors], [t.grad for t in ref_tensors])

    @onlyCUDA
    @ops(foreach_reduce_op_db)
    def test_foreach_reduce_large_input(self, device, dtype, op):
        # test inputs larger than kChunkSize = 65536
        ord, N = 2, 65536 * 2
        disable_fastpath = True
        if ord in (1, 2) and dtype in floating_types_and(torch.half, torch.bfloat16):
            disable_fastpath = False
        inputs = ([make_tensor((N,), dtype=dtype, device=device, noncontiguous=False)],)
        wrapped_op, ref, _, _ = self._get_funcs(op)
        self.assertEqual(
            ref(inputs, ord=ord),
            wrapped_op(inputs, self.is_cuda, not disable_fastpath, ord=ord, zero_size=False),
        )

    @onlyCUDA
    @ops(
        foreach_unary_op_db + foreach_binary_op_db + foreach_pointwise_op_db + foreach_lerp_op_db,
        dtypes=(torch.float,),
    )
    def test_inplace_foreach_leaf_check_and_grad_fn(self, device, dtype, op):
        inplace_op = op.inplace_variant
        if inplace_op is None:
            self.skipTest("no in-place op available")

        sample = list(op.sample_inputs(dtype=dtype, device=device, num_input_tensors=[2], same_size=True))[0]
        sample.input[0].requires_grad_(True)
        with self.assertRaisesRegex(RuntimeError, "a leaf Variable that requires grad"):
            inplace_op(sample.input, *sample.args)
        sample.input[1].requires_grad_(True)
        with self.assertRaisesRegex(RuntimeError, "a leaf Variable that requires grad"):
            inplace_op(sample.input, *sample.args)

        _tensors = [t.clone().detach().requires_grad_(i == 0) for i, t in enumerate(sample.input)]
        tensors = [t.clone() for t in _tensors]
        inplace_op(tensors, *sample.args)
        self.assertIsNotNone(tensors[0].grad_fn)
        self.assertIsNone(tensors[1].grad_fn)

    @onlyCUDA
    @ops(
        foreach_unary_op_db + foreach_binary_op_db + foreach_pointwise_op_db + foreach_lerp_op_db,
        dtypes=(torch.float,),
    )
    def test_outplace_with_invalid_grads(self, device, dtype, op):
        if op.name in {"_foreach_zero"}:
            self.skipTest(f"{op.name} does not have out-place implementation")
        func, *_ = self._get_funcs(op)
        sample = list(op.sample_inputs(dtype=dtype, device=device, requires_grad=True, num_input_tensors=[2], same_size=True))[0]
        self.assertTrue(all(t.requires_grad for t in sample.input))
        sample.kwargs.pop("disable_fastpath")
        if func.func in (torch._foreach_addcmul, torch._foreach_addcdiv):
            if sample.kwargs.get("values") is None:
                sample.kwargs.pop("values")
        (out1, out2) = func([sample.input, *sample.args], is_cuda=False, is_fastpath=False, **sample.kwargs)
        out1.backward(torch.ones_like(out1))
        self.assertIsNotNone(sample.input[0].grad)
        self.assertIsNone(sample.input[1].grad)

    @ops(
        foreach_unary_op_db + foreach_binary_op_db + foreach_pointwise_op_db + foreach_lerp_op_db,
        dtypes=OpDTypes.supported,
        allowed_dtypes=(torch.float64, torch.complex128),
    )
    def test_inplace_forward_mode_AD(self, device, dtype, op):
        if not op.supports_forward_ad:
            self.skipTest("forward AD not supported")

        # note(crcrpar): The combinations below are failing in its forward path,
        # which is before forward-mode AD happens. This function gates the combinations where
        # - subtraction with Scalar/ScalarList of boolean value:
        # - combinations where the in-place op in questions tries to write out complex result
        #   into float storage (= `self`)
        def check_sample_eligibility(op, sample, dtype):
            if (
                op.name == "_foreach_sub"
                and (
                    (isinstance(sample.args[0], list) and any(isinstance(a, bool) for a in sample.args[0]))
                    or isinstance(sample.args[0], bool)
                )
            ):
                return False, _BOOL_SUB_ERR_MSG
            rhs_arg_has_complex_number = sample.args and ((
                isinstance(sample.args[0], list)
                and any(isinstance(a, complex) for a in sample.args[0])
            ) or (
                isinstance(sample.args[0], complex)
            ))
            if dtype == torch.float64 and rhs_arg_has_complex_number:
                if op.name in ("_foreach_add", "_foreach_sub", "_foreach_mul", "_foreach_div"):
                    return False, "result type ComplexDouble can't be cast to the desired output type Double"
                if op.name in ("_foreach_clamp_max", "_foreach_clamp_min"):
                    return False, "clamp is not supported for complex types"
                if op.name == "_foreach_pow":
                    return False, "Found dtype Double but expected ComplexDouble"

            return True, ""

        for sample in op.sample_inputs(
            device, dtype, requires_grad=True, num_input_tensors=[5], same_size=True,
        ):
            # Call `clone` to avoid inplace modifications likewise
            # `torch.testing._internal.common_utils.TestGradients._get_safe_inplace`
            def inplace_func(*tensorlist):
                kwargs = {"alpha": sample.kwargs["alpha"]} if "alpha" in sample.kwargs else {}
                op.inplace_variant(tuple(t.clone() for t in tensorlist), *sample.args, **kwargs)
                return tensorlist

            working_sample, err_msg_pattern = check_sample_eligibility(op, sample, dtype)
            if not working_sample:
                with self.assertRaisesRegex(RuntimeError, re.escape(err_msg_pattern)):
                    gradcheck(
                        inplace_func,
                        sample.input,
                        raise_exception=True,
                        check_forward_ad=True,
                        check_backward_ad=False,
                        check_batched_grad=False,
                    )
            else:
                gradcheck(
                    inplace_func,
                    sample.input,
                    raise_exception=True,
                    check_forward_ad=True,
                    check_backward_ad=False,
                    check_batched_grad=False,
                )

    @unittest.skipIf(not (torch.cuda.is_available() and torch.cuda.device_count() > 1), "requires multiple GPUs")
    def test_tensors_grouping(self):
        num_tensors_per_list = 10
        num_devices = torch.cuda.device_count()
        dtypes = (torch.float16, torch.float32, torch.float64)
        list1 = [
            torch.tensor(
                i,
                device=torch.device("cuda", random.randint(0, num_devices - 1)),
                dtype=dtypes[random.randint(0, 2)],
            ) for i in range(num_tensors_per_list)
        ]
        list2 = [None for _ in list1]
        list3 = [torch.rand_like(t) for t in list1]
        nested_tensorlists = [list1, list2, list3]
        grouped_tensors = torch.utils._foreach_utils._group_tensors_by_device_and_dtype(nested_tensorlists, with_indices=True)
        num_tensors_seen = 0
        for (device, dtype), ([l1, l2, l3], indices) in grouped_tensors.items():
            for t in itertools.chain(l1, l3):
                self.assertEquals(t.device, device)
                self.assertEquals(t.dtype, dtype)
                num_tensors_seen += 1
            self.assertEqual(len(l1), len(l2))
            self.assertTrue(all(p is None for p in l2))
            for i, index in enumerate(indices):
                self.assertEquals(l1[i], list1[index])
                self.assertEquals(l2[i], list2[index])
                self.assertEquals(l3[i], list3[index])
        self.assertEquals(num_tensors_seen, 2 * num_tensors_per_list)


instantiate_device_type_tests(TestForeach, globals())


if __name__ == "__main__":
    run_tests()