diff --git a/pyg_lib/csrc/ops/autograd/scatter_kernel.cpp b/pyg_lib/csrc/ops/autograd/scatter_kernel.cpp index 455535929..b527c05db 100644 --- a/pyg_lib/csrc/ops/autograd/scatter_kernel.cpp +++ b/pyg_lib/csrc/ops/autograd/scatter_kernel.cpp @@ -139,6 +139,105 @@ at::Tensor scatter_mul_autograd(const at::Tensor& src, return ScatterMul::apply(src, index, dim, optional_out, dim_size)[0]; } +// Autograd Function for `pyg::scatter_mean`. +// +// Composite op: forward dispatches to `scatter_sum` twice (numerator over +// `src` and denominator over a `ones`-tensor sized like `index`) and then +// divides. Because there is no dedicated CPU/CUDA kernel, the wrapper is +// registered to `CompositeExplicitAutograd` rather than `Autograd`: that +// key routes the call here for *all* backends and regardless of whether +// any input requires grad. `Function::apply` itself decides whether to +// record on the autograd tape based on input grad state, so this single +// registration handles both `requires_grad=True` and `requires_grad=False` +// callers correctly. +// +// Backward formula (upstream pytorch_scatter `scatter.cpp:149-160`): +// +// grad_src = (grad_out.gather(dim, index_b)) / count.gather(dim, index_b) +// +// where `count` is the broadcast per-bucket count tensor saved from forward. +class ScatterMean : public torch::autograd::Function { + public: + static variable_list forward(torch::autograd::AutogradContext* ctx, + const at::Tensor& src, + const at::Tensor& index, + int64_t dim, + const std::optional& optional_out, + std::optional dim_size) { + at::AutoDispatchBelowADInplaceOrView g; + + const int64_t dim_norm = dim < 0 ? src.dim() + dim : dim; + + // Broadcast `index` up to `src.shape` for the sum dispatch and so + // backward's `gather` can use it directly. + auto index_b = broadcast(index, src, dim_norm); + + // Numerator: per-bucket sum of `src` values. + auto out_tensor = + scatter_sum(src, index_b, dim_norm, optional_out, dim_size); + + // Denominator: per-bucket count. Use the *original* (non-broadcast) + // `index` against a 1-D `ones` of length `index.size(...)`. Upstream + // mirrors the same trick to avoid double-counting when `index` is + // broadcast across multiple non-`dim` axes. The reduction `dim` for + // this auxiliary call is the last axis of `index` if `index` has + // fewer dims than `dim_norm` (i.e. `index` is 1-D and the broadcast + // adds leading singletons), else `dim_norm`. + const int64_t count_dim = + index.dim() <= dim_norm ? index.dim() - 1 : dim_norm; + auto ones = at::ones(index.sizes(), src.options()); + auto count = scatter_sum(ones, index, count_dim, std::nullopt, + out_tensor.size(dim_norm)); + count.masked_fill_(count < 1, 1); + + // Broadcast `count` (1-D along `dim_norm`) up to `out_tensor.shape` + // so it aligns for in-place division along the reduction axis. + auto count_b = broadcast(count, out_tensor, dim_norm); + + if (out_tensor.is_floating_point()) { + out_tensor.true_divide_(count_b); + } else { + out_tensor.div_(count_b, "floor"); + } + + // Save the *broadcast* count: backward's `gather(count_b, dim, index_b)` + // expects `count` to be shaped like `out_tensor` so gather aligns. + ctx->save_for_backward({index_b, count_b}); + ctx->saved_data["dim"] = dim_norm; + + if (optional_out.has_value()) { + ctx->mark_dirty({optional_out.value()}); + } + + return {out_tensor}; + } + + static variable_list backward(torch::autograd::AutogradContext* ctx, + variable_list grad_outs) { + const auto grad_out = grad_outs[0]; + const auto saved = ctx->get_saved_variables(); + const auto index_b = saved[0]; + const auto count_b = saved[1]; + const auto dim = ctx->saved_data["dim"].toInt(); + + // `grad_src[i] = grad_out[index[i]] / count[index[i]]`. Gather both + // along `dim` using the broadcast index so per-position counts line up. + auto count_gathered = count_b.gather(dim, index_b); + auto grad_src = grad_out.gather(dim, index_b).true_divide_(count_gathered); + + // 5 input slots: (src, index, dim, out, dim_size). + return {grad_src, at::Tensor(), at::Tensor(), at::Tensor(), at::Tensor()}; + } +}; + +at::Tensor scatter_mean_autograd(const at::Tensor& src, + const at::Tensor& index, + int64_t dim, + const std::optional& optional_out, + std::optional dim_size) { + return ScatterMean::apply(src, index, dim, optional_out, dim_size)[0]; +} + } // namespace TORCH_LIBRARY_IMPL(pyg, Autograd, m) { @@ -146,6 +245,22 @@ TORCH_LIBRARY_IMPL(pyg, Autograd, m) { TORCH_FN(scatter_sum_autograd)); m.impl(TORCH_SELECTIVE_NAME("pyg::scatter_mul"), TORCH_FN(scatter_mul_autograd)); + // `scatter_mean` is also registered to `Autograd` (in addition to + // `CompositeExplicitAutograd` below) to silence the runtime warning + // emitted when PyTorch finds no Autograd-key kernel and falls back to + // the not-implemented sentinel; the same wrapper handles both paths. + m.impl(TORCH_SELECTIVE_NAME("pyg::scatter_mean"), + TORCH_FN(scatter_mean_autograd)); +} + +// `scatter_mean` has no dedicated CPU/CUDA kernel; it is fully composed of +// other dispatcher calls. Registering to `CompositeExplicitAutograd` makes +// the dispatcher route the call to our wrapper for *every* backend (and +// regardless of `requires_grad`), so non-grad callers do not fall through +// to an empty backend kernel slot. +TORCH_LIBRARY_IMPL(pyg, CompositeExplicitAutograd, m) { + m.impl(TORCH_SELECTIVE_NAME("pyg::scatter_mean"), + TORCH_FN(scatter_mean_autograd)); } } // namespace ops diff --git a/pyg_lib/csrc/ops/scatter.cpp b/pyg_lib/csrc/ops/scatter.cpp index cf9d45f79..3f3c39526 100644 --- a/pyg_lib/csrc/ops/scatter.cpp +++ b/pyg_lib/csrc/ops/scatter.cpp @@ -64,6 +64,35 @@ PYG_API at::Tensor scatter_mul(const at::Tensor& src, return op.call(src, index, dim, out, dim_size); } +PYG_API at::Tensor scatter_mean(const at::Tensor& src, + const at::Tensor& index, + int64_t dim, + const std::optional& out, + std::optional dim_size) { + at::TensorArg src_arg{src, "src", 0}; + at::TensorArg index_arg{index, "index", 1}; + at::CheckedFrom c{"scatter_mean"}; + + at::checkAllDefined(c, {src_arg, index_arg}); + TORCH_CHECK(src.device() == index.device(), + "scatter_mean: src and index must be on the same device " + "(got src=", + src.device(), ", index=", index.device(), ")"); + if (out.has_value()) { + at::TensorArg out_arg{out.value(), "out", 3}; + at::checkAllDefined(c, {out_arg}); + TORCH_CHECK(src.device() == out.value().device(), + "scatter_mean: src and out must be on the same device " + "(got src=", + src.device(), ", out=", out.value().device(), ")"); + } + + static auto op = c10::Dispatcher::singleton() + .findSchemaOrThrow("pyg::scatter_mean", "") + .typed(); + return op.call(src, index, dim, out, dim_size); +} + TORCH_LIBRARY_FRAGMENT(pyg, m) { m.def(TORCH_SELECTIVE_SCHEMA( "pyg::scatter_sum(Tensor src, Tensor index, int dim=-1, " @@ -71,6 +100,9 @@ TORCH_LIBRARY_FRAGMENT(pyg, m) { m.def(TORCH_SELECTIVE_SCHEMA( "pyg::scatter_mul(Tensor src, Tensor index, int dim=-1, " "Tensor? out=None, int? dim_size=None) -> Tensor")); + m.def(TORCH_SELECTIVE_SCHEMA( + "pyg::scatter_mean(Tensor src, Tensor index, int dim=-1, " + "Tensor? out=None, int? dim_size=None) -> Tensor")); } } // namespace ops diff --git a/pyg_lib/csrc/ops/scatter.h b/pyg_lib/csrc/ops/scatter.h index c941e6725..3f8380543 100644 --- a/pyg_lib/csrc/ops/scatter.h +++ b/pyg_lib/csrc/ops/scatter.h @@ -36,5 +36,24 @@ PYG_API at::Tensor scatter_mul( const std::optional& out = std::nullopt, std::optional dim_size = std::nullopt); +// Averages all values from `src` into `out` at the indices specified in +// `index` along a given axis `dim`. +// +// Implemented as two `scatter_sum` calls: one to accumulate the per-bucket +// sum, and one (over a ones tensor sized to match `index`) to compute the +// per-bucket count. The final result is the sum divided by the count, with +// empty buckets producing `0` (the count is `masked_fill`-ed up to `1` so +// the division does not generate NaNs). Integer dtypes use floor division +// to match the upstream contract; floating dtypes use true division. +// +// When `out` is provided, the accumulation happens in the caller's buffer +// (no zero-init), matching the upstream `scatter_sum` contract. +PYG_API at::Tensor scatter_mean( + const at::Tensor& src, + const at::Tensor& index, + int64_t dim = -1, + const std::optional& out = std::nullopt, + std::optional dim_size = std::nullopt); + } // namespace ops } // namespace pyg diff --git a/pyg_lib/ops/__init__.py b/pyg_lib/ops/__init__.py index 120a2de47..7d58f0f57 100644 --- a/pyg_lib/ops/__init__.py +++ b/pyg_lib/ops/__init__.py @@ -411,6 +411,34 @@ def scatter_mul( return torch.ops.pyg.scatter_mul(src, index, dim, out, dim_size) +def scatter_mean( + src: Tensor, + index: Tensor, + dim: int = -1, + out: Optional[Tensor] = None, + dim_size: Optional[int] = None, +) -> Tensor: + r"""Reduces all values from the :obj:`src` tensor into :obj:`out` at the + indices specified in the :obj:`index` tensor along a given axis + :obj:`dim`, using ``mean`` as the reduction. Empty buckets yield zero. + + For floating-point inputs, division is exact. For integer inputs, + floor-division is used (matching upstream ``pytorch_scatter``). + + Args: + src: The source tensor. + index: The indices of elements to scatter. + dim: The axis along which to index. + out: The destination tensor. + dim_size: If :obj:`out` is not given, automatically create output with + size :obj:`dim_size` at dimension :obj:`dim`. + + Returns: + The reduced tensor. + """ + return torch.ops.pyg.scatter_mean(src, index, dim, out, dim_size) + + def spline_basis( pseudo: Tensor, kernel_size: Tensor, @@ -652,6 +680,7 @@ def edge_sample( 'scatter_sum', 'scatter_add', 'scatter_mul', + 'scatter_mean', 'spline_basis', 'spline_weighting', 'grid_cluster', diff --git a/test/ops/test_scatter.py b/test/ops/test_scatter.py index e6ddca14e..1b6f4aa18 100644 --- a/test/ops/test_scatter.py +++ b/test/ops/test_scatter.py @@ -69,10 +69,8 @@ def test_scatter_add_is_scatter_sum_alias(): ], ) def test_scatter_sum_forward_dtypes(dtype, device): - if device.type == 'cpu' and dtype == torch.float16: - # CPU half-precision arithmetic is supported but tolerances are loose; - # we still run it to check parity. - pass + # Seed for deterministic random inputs across dtypes. + torch.manual_seed(0) if dtype in (torch.int32, torch.int64): src = torch.randint(-10, 10, (8, 4), dtype=dtype, device=device) else: @@ -81,7 +79,11 @@ def test_scatter_sum_forward_dtypes(dtype, device): out = pyg_lib.ops.scatter_sum(src, index, dim=0) expected = _scatter_sum_ref(src, index, dim=0) - torch.testing.assert_close(out, expected) + # Half/bfloat16 accumulation drifts further than the default rtol allows. + if dtype in (torch.float16, torch.bfloat16): + torch.testing.assert_close(out, expected, atol=1e-2, rtol=1e-2) + else: + torch.testing.assert_close(out, expected) @withCUDA @@ -568,3 +570,270 @@ def test_scatter_mul_gradient_at_src_zero(device): grad2[5], torch.tensor(5.0, dtype=torch.double, device=device), ) + + +# --------------------------------------------------------------------------- +# scatter_mean +# --------------------------------------------------------------------------- + + +def _scatter_mean_ref( + src: torch.Tensor, + index: torch.Tensor, + dim: int = -1, + out: torch.Tensor = None, + dim_size: int = None, +) -> torch.Tensor: + """Pure-PyTorch reference implementation of :func:`scatter_mean`. + + Mirrors the contract of ``pyg_lib.ops.scatter_mean``: + * Index is broadcast to ``src`` along ``dim``. + * Computes scatter_sum of ``src`` and divides by per-bucket counts. + * Empty buckets (count == 0) are masked-filled to 1 to avoid div-by-0; + the numerator is also zero there, so the result is zero. + * Floats use true-division; integers use floor-division (matches the + upstream ``div(_, rounding_mode="floor")`` semantics). + * If ``out`` is :obj:`None`, an output of the appropriate shape is + zero-initialised before the scatter_sum. + * If ``out`` is provided, the sum accumulates into it *before* division. + * If ``dim_size`` is :obj:`None`, infer from ``index.max() + 1``. + """ + if dim < 0: + dim = dim + src.dim() + bcast_index = _broadcast_index(index, src, dim) + + # Numerator: scatter_sum into (a clone of) ``out``. + if out is None: + if dim_size is None: + dim_size = int(index.max().item()) + 1 if index.numel() > 0 else 0 + size = list(src.size()) + size[dim] = dim_size + numer = torch.zeros(size, dtype=src.dtype, device=src.device) + else: + # Don't mutate the caller's tensor in the reference. + numer = out.clone() + numer.scatter_add_(dim, bcast_index, src) + + # Denominator: per-bucket counts. Use a 1-D scatter_sum-of-ones along + # ``dim``; then broadcast to ``numer.shape`` for the division. + ones = torch.ones(src.size(dim), dtype=src.dtype, device=src.device) + count = torch.zeros(numer.size(dim), dtype=src.dtype, device=src.device) + count.scatter_add_(0, index, ones) + count.masked_fill_(count < 1, 1) + # Broadcast count along ``dim`` to ``numer.shape``. + view = [1] * numer.dim() + view[dim] = -1 + count = count.view(view).expand_as(numer) + + if numer.is_floating_point(): + return numer / count + else: + return torch.div(numer, count, rounding_mode='floor') + + +@withCUDA +@pytest.mark.parametrize( + 'dtype', + [ + torch.float16, + torch.float32, + torch.float64, + torch.bfloat16, + ], +) +def test_scatter_mean_forward_dtypes(dtype, device): + src = torch.randn(8, 4, dtype=dtype, device=device) + index = torch.tensor([0, 1, 0, 1, 1, 3, 2, 0], device=device) + + out = pyg_lib.ops.scatter_mean(src, index, dim=0) + expected = _scatter_mean_ref(src, index, dim=0) + if dtype in (torch.float16, torch.bfloat16): + torch.testing.assert_close(out, expected, atol=1e-2, rtol=1e-2) + else: + torch.testing.assert_close(out, expected) + + +@withCUDA +@pytest.mark.parametrize('dtype', [torch.int32, torch.int64]) +def test_scatter_mean_forward_integer_floor_div(dtype, device): + """Integer dtype: per-bucket result is the floor of sum/count.""" + src = torch.randint(-10, 10, (8, 4), dtype=dtype, device=device) + index = torch.tensor([0, 1, 0, 1, 1, 3, 2, 0], device=device) + + out = pyg_lib.ops.scatter_mean(src, index, dim=0) + expected = _scatter_mean_ref(src, index, dim=0) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_scatter_mean_forward_dim_neg1(device): + src = torch.randn(3, 8, device=device) + index = torch.tensor([0, 1, 0, 1, 1, 3, 2, 0], device=device) + + out = pyg_lib.ops.scatter_mean(src, index, dim=-1) + expected = _scatter_mean_ref(src, index, dim=-1) + assert out.size() == (3, 4) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_scatter_mean_forward_dim_nonneg(device): + src = torch.randn(8, 4, device=device) + index = torch.tensor([0, 1, 0, 1, 1, 3, 2, 0], device=device) + + out = pyg_lib.ops.scatter_mean(src, index, dim=0) + expected = _scatter_mean_ref(src, index, dim=0) + assert out.size() == (4, 4) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_scatter_mean_forward_dim_middle(device): + src = torch.randn(3, 6, 5, device=device) + index = torch.tensor([0, 2, 1, 0, 2, 1], device=device) + + out = pyg_lib.ops.scatter_mean(src, index, dim=1) + expected = _scatter_mean_ref(src, index, dim=1) + assert out.size() == (3, 3, 5) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_scatter_mean_broadcasting_1d_index(device): + """1-D ``index`` broadcasts to 2-D ``src`` along ``dim``.""" + src = torch.randn(6, 4, device=device) + index = torch.tensor([0, 1, 0, 2, 1, 2], device=device) + + out = pyg_lib.ops.scatter_mean(src, index, dim=0) + expected = _scatter_mean_ref(src, index, dim=0) + assert out.size() == (3, 4) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_scatter_mean_dim_size_auto_infer(device): + src = torch.randn(6, device=device) + index = torch.tensor([0, 1, 0, 1, 1, 3], device=device) + + out = pyg_lib.ops.scatter_mean(src, index, dim=-1) + # Auto-inferred dim_size = index.max() + 1 = 4 + assert out.size(-1) == 4 + expected = _scatter_mean_ref(src, index, dim=-1) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_scatter_mean_dim_size_explicit_empty_buckets(device): + """Explicit ``dim_size`` larger than implicit — trailing buckets are empty + and must yield exactly zero (count masked-filled to 1, then 0/1 = 0). + This is the key contract that distinguishes scatter_mean from a naive + sum/count which would produce NaN. + """ + src = torch.randn(6, device=device) + index = torch.tensor([0, 1, 0, 1, 1, 3], device=device) + + out = pyg_lib.ops.scatter_mean(src, index, dim=-1, dim_size=6) + assert out.size(-1) == 6 + expected = _scatter_mean_ref(src, index, dim=-1, dim_size=6) + torch.testing.assert_close(out, expected) + # Empty trailing buckets at positions 4, 5 must be exactly zero, not NaN. + torch.testing.assert_close(out[4:], torch.zeros(2, device=device)) + assert torch.isfinite(out).all(), ( + f'scatter_mean produced non-finite values: {out}' + ) + + +@withCUDA +def test_scatter_mean_empty_bucket_in_middle(device): + """Empty buckets anywhere (not just trailing) must yield zero, not NaN.""" + # index skips bucket 2 entirely. + src = torch.randn(6, 3, device=device) + index = torch.tensor([0, 1, 0, 1, 3, 3], device=device) + + out = pyg_lib.ops.scatter_mean(src, index, dim=0, dim_size=4) + assert out.size() == (4, 3) + expected = _scatter_mean_ref(src, index, dim=0, dim_size=4) + torch.testing.assert_close(out, expected) + # Bucket 2 has no contributors — expect a zero row. + torch.testing.assert_close(out[2], torch.zeros(3, device=device)) + assert torch.isfinite(out).all() + + +@withCUDA +def test_scatter_mean_out_accumulates_then_divides(device): + """When ``out`` is provided, the numerator accumulates into it before the + final divide by per-bucket counts. + """ + src = torch.randn(6, 4, device=device) + index = torch.tensor([0, 1, 0, 1, 1, 3], device=device) + + out_init = torch.randn(4, 4, device=device) + out = out_init.clone() + result = pyg_lib.ops.scatter_mean(src, index, dim=0, out=out) + + expected = _scatter_mean_ref(src, index, dim=0, out=out_init) + torch.testing.assert_close(result, expected) + # The op should also have updated ``out`` in-place. + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_scatter_mean_empty_input(device): + """An empty source with an empty index should yield an all-zero output + (every bucket is empty -> count masked to 1 -> 0/1 = 0). + """ + src = torch.empty(0, 4, device=device) + index = torch.empty(0, dtype=torch.long, device=device) + + out = pyg_lib.ops.scatter_mean(src, index, dim=0, dim_size=3) + assert out.size() == (3, 4) + torch.testing.assert_close(out, torch.zeros(3, 4, device=device)) + + +@withCUDA +def test_scatter_mean_backward_gradcheck(device): + src = torch.randn( + 6, + 3, + dtype=torch.double, + device=device, + requires_grad=True, + ) + index = torch.tensor([0, 1, 0, 1, 1, 3], device=device) + + def fn(s): + return pyg_lib.ops.scatter_mean(s, index, dim=0) + + assert torch.autograd.gradcheck(fn, (src,)) + + +@withCUDA +def test_scatter_mean_backward_gradcheck_dim_middle(device): + src = torch.randn( + 2, + 6, + 3, + dtype=torch.double, + device=device, + requires_grad=True, + ) + index = torch.tensor([0, 1, 0, 1, 2, 1], device=device) + + def fn(s): + return pyg_lib.ops.scatter_mean(s, index, dim=1) + + assert torch.autograd.gradcheck(fn, (src,)) + + +@withCUDA +def test_scatter_mean_backward_gradcheck_with_dim_size(device): + """Gradcheck with an explicit (larger) ``dim_size`` exercising empty + buckets in the output. Empty buckets have zero gradient w.r.t. ``src``. + """ + src = torch.randn(6, dtype=torch.double, device=device, requires_grad=True) + index = torch.tensor([0, 1, 0, 1, 1, 3], device=device) + + def fn(s): + return pyg_lib.ops.scatter_mean(s, index, dim=-1, dim_size=6) + + assert torch.autograd.gradcheck(fn, (src,))