diff --git a/pyg_lib/csrc/ops/autograd/segment_coo_kernel.cpp b/pyg_lib/csrc/ops/autograd/segment_coo_kernel.cpp new file mode 100644 index 000000000..aba114777 --- /dev/null +++ b/pyg_lib/csrc/ops/autograd/segment_coo_kernel.cpp @@ -0,0 +1,136 @@ +#include "../segment_coo.h" + +#include + +namespace pyg { +namespace ops { + +namespace { + +using torch::autograd::variable_list; + +// Autograd Function for `pyg::segment_sum_coo`. +// +// COO ops fix the reduction axis at `dim = index.dim() - 1`. Forward saves +// `index` (the kernel handles its own `expand`); backward is exactly +// `gather_coo(grad_out, index)` — the symmetric-pair inverse of forward. +// +// `out=` contract: the C++ dispatcher accumulates into the caller's buffer +// when `out=` is supplied. `mark_dirty` lets gradcheck pick up the in-place +// mutation; the backward gradient itself is computed the same way regardless. +class SegmentSumCOO : public torch::autograd::Function { + public: + static variable_list forward(torch::autograd::AutogradContext* ctx, + const at::Tensor& src, + const at::Tensor& index, + const std::optional& optional_out, + std::optional dim_size) { + at::AutoDispatchBelowADInplaceOrView g; + + auto out = segment_sum_coo(src, index, optional_out, dim_size); + + // Save `index` for backward's `gather_coo` call. The gather kernel + // applies the same `expand` convention, so we save the *original* + // (possibly pre-broadcast) `index`. + ctx->save_for_backward({index}); + + if (optional_out.has_value()) { + ctx->mark_dirty({optional_out.value()}); + } + + return {out}; + } + + static variable_list backward(torch::autograd::AutogradContext* ctx, + variable_list grad_outs) { + const auto grad_out = grad_outs[0]; + const auto saved = ctx->get_saved_variables(); + const auto index = saved[0]; + + // `grad_src[..., e, ...] = grad_out[..., index[..., e], ...]` — exactly + // what `gather_coo` computes along `dim = index.dim() - 1`. + auto grad_src = gather_coo(grad_out, index, std::nullopt); + + // 4 input slots: (src, index, out, dim_size). Only `src` is a real + // differentiable tensor; the rest get undefined-Tensor grads. + return {grad_src, at::Tensor(), at::Tensor(), at::Tensor()}; + } +}; + +at::Tensor segment_sum_coo_autograd( + const at::Tensor& src, + const at::Tensor& index, + const std::optional& optional_out, + std::optional dim_size) { + return SegmentSumCOO::apply(src, index, optional_out, dim_size)[0]; +} + +// Autograd Function for `pyg::gather_coo`. +// +// COO ops fix the gather axis at `dim = index.dim() - 1`. Forward saves +// `index` and `src.size(dim)`; backward is exactly `segment_sum_coo` with +// the saved `dim_size`, deposited into a freshly allocated `zeros(src_shape)` +// buffer via the `out=` accumulate contract. +class GatherCOO : public torch::autograd::Function { + public: + static variable_list forward(torch::autograd::AutogradContext* ctx, + const at::Tensor& src, + const at::Tensor& index, + const std::optional& optional_out) { + at::AutoDispatchBelowADInplaceOrView g; + + auto out = gather_coo(src, index, optional_out); + + // Backward needs `index` (where each output position pulled from), the + // original `src.sizes()` (to allocate the right-shaped `grad_in`), and + // `src.size(dim)` (passed as `dim_size` to `segment_sum_coo`). + const int64_t dim = index.dim() - 1; + ctx->save_for_backward({index}); + ctx->saved_data["src_shape"] = src.sizes(); + ctx->saved_data["src_size_dim"] = src.size(dim); + + if (optional_out.has_value()) { + ctx->mark_dirty({optional_out.value()}); + } + + return {out}; + } + + static variable_list backward(torch::autograd::AutogradContext* ctx, + variable_list grad_outs) { + const auto grad_out = grad_outs[0]; + const auto saved = ctx->get_saved_variables(); + const auto index = saved[0]; + auto src_shape = ctx->saved_data["src_shape"].toIntList().vec(); + const int64_t src_size_dim = ctx->saved_data["src_size_dim"].toInt(); + + // Allocate the gradient buffer at `src.shape` and use the `out=` + // accumulate contract of `segment_sum_coo` to deposit each `grad_out` + // entry at its source position. This is the inverse of forward. + auto grad_in = at::zeros(src_shape, grad_out.options()); + segment_sum_coo(grad_out, index, /*out=*/grad_in, + /*dim_size=*/src_size_dim); + + // 3 input slots: (src, index, out). Only `src` is a real differentiable + // tensor; the rest get undefined-Tensor grads. + return {grad_in, at::Tensor(), at::Tensor()}; + } +}; + +at::Tensor gather_coo_autograd(const at::Tensor& src, + const at::Tensor& index, + const std::optional& optional_out) { + return GatherCOO::apply(src, index, optional_out)[0]; +} + +} // namespace + +TORCH_LIBRARY_IMPL(pyg, Autograd, m) { + m.impl(TORCH_SELECTIVE_NAME("pyg::segment_sum_coo"), + TORCH_FN(segment_sum_coo_autograd)); + m.impl(TORCH_SELECTIVE_NAME("pyg::gather_coo"), + TORCH_FN(gather_coo_autograd)); +} + +} // namespace ops +} // namespace pyg diff --git a/pyg_lib/csrc/ops/cpu/segment_coo_kernel.cpp b/pyg_lib/csrc/ops/cpu/segment_coo_kernel.cpp new file mode 100644 index 000000000..e450e6846 --- /dev/null +++ b/pyg_lib/csrc/ops/cpu/segment_coo_kernel.cpp @@ -0,0 +1,276 @@ +#include "../segment_coo.h" + +#include +#include +#include +#include + +namespace pyg { +namespace ops { + +namespace { + +// CPU implementation of `pyg::segment_sum_coo`. +// +// COO ops fix the reduction axis at `dim = index.dim() - 1` (upstream +// `segment_coo_cpu.cpp:24`). The `index` tensor is sorted-ascending along +// that axis; equal-index runs collapse into a single output bucket. +// +// Layout: `index` is expanded to `src.shape[:index.dim()]` (each leading +// dim of `index` matches `src`). With `dim = index.dim() - 1`: +// * B = product of leading dims before `dim` (= index.numel() / +// src.size(dim)), +// * E = src.size(dim) = index.size(dim), +// * K = product of trailing dims of `src` past `index.dim()` +// (= src.numel() / index.numel()), +// * N = out.size(dim). +// +// `out=` contract: when the caller supplies `optional_out`, we **accumulate** +// into it (no zero-init). Matches upstream and is what makes +// `GatherCOO::backward` efficient. +at::Tensor segment_sum_coo_kernel(const at::Tensor& src, + const at::Tensor& index, + const std::optional& optional_out, + std::optional dim_size) { + TORCH_CHECK(src.dim() >= index.dim(), + "segment_sum_coo: src.dim() must be >= index.dim() " + "(got src.dim()=", + src.dim(), ", index.dim()=", index.dim(), ")"); + + // COO ops fix the reduction axis at the last index dim. + const int64_t dim = index.dim() - 1; + TORCH_CHECK(dim >= 0, + "segment_sum_coo: index must have at least 1 dimension"); + + // Broadcast `index` up to `src.shape[:index.dim()]` (upstream + // `segment_coo_cpu.cpp:19-22`). Then force contiguous for the linear scan. + auto sizes = index.sizes().vec(); + for (int64_t i = 0; i < index.dim(); ++i) + sizes[i] = src.size(i); + auto index_b = index.expand(sizes).contiguous(); + + auto src_c = src.contiguous(); + + at::Tensor out; + if (optional_out.has_value()) { + out = optional_out.value().contiguous(); + for (int64_t i = 0; i < out.dim(); ++i) { + if (i != dim) + TORCH_CHECK(src_c.size(i) == out.size(i), "segment_sum_coo: out.size(", + i, ") must match src.size(", i, ")"); + } + } else { + auto out_sizes = src_c.sizes().vec(); + if (dim_size.has_value()) { + out_sizes[dim] = dim_size.value(); + } else if (index_b.numel() == 0) { + out_sizes[dim] = 0; + } else { + // Last index in the (sorted-ascending) last row gives `dim_size - 1`. + auto tmp = index_b.select(dim, index_b.size(dim) - 1); + tmp = tmp.numel() > 1 ? tmp.max() : tmp; + out_sizes[dim] = 1 + *tmp.data_ptr(); + } + // Zero-init the freshly allocated buffer so the accumulate loop below + // produces the correct result. + out = at::zeros(out_sizes, src_c.options()); + } + + if (src_c.numel() == 0) { + return out; + } + + const int64_t E = src_c.size(dim); + // `B` is the product of leading dims of `index` before `dim`. We compute + // it from `index_b` (post-expand) so it stays well-defined even when the + // original `index` is 1-D. + int64_t B = 1; + for (int64_t i = 0; i < dim; ++i) + B *= index_b.size(i); + // `K` is the product of trailing dims of `src` past `index.dim()`. + const int64_t K = src_c.numel() / index_b.numel(); + const int64_t N = out.size(dim); + + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::Half, at::ScalarType::BFloat16, src_c.scalar_type(), + "segment_sum_coo_cpu", [&] { + using opmath_t = at::opmath_type; + const auto* src_data = src_c.data_ptr(); + auto* out_data = out.data_ptr(); + const auto* index_data = index_b.data_ptr(); + + // Parallelize over the leading (B) dim only. Each `b` slice writes + // to a disjoint sub-region of `out`, so no atomics are needed. The + // inner E loop must remain sequential to correctly accumulate runs + // of equal indices. + at::parallel_for( + 0, B, at::internal::GRAIN_SIZE, [&](int64_t b_beg, int64_t b_end) { + for (int64_t b = b_beg; b < b_end; ++b) { + const int64_t src_off = b * E * K; + const int64_t out_off = b * N * K; + const int64_t idx_off = b * E; + + if (E == 0) + continue; + + // Sequential pass over the sorted-ascending `index` row. + // Maintain a per-K accumulator that flushes to `out` when + // the index changes (or at the end of the row). + int64_t cur_idx = index_data[idx_off]; + + if (K == 1) { + // Initialize accumulator from current `out` slot so the + // `out=` accumulate contract is honored. + opmath_t acc = + static_cast(out_data[out_off + cur_idx]); + for (int64_t e = 0; e < E; ++e) { + acc += static_cast(src_data[src_off + e]); + const bool last = (e == E - 1); + const int64_t next_idx = + last ? cur_idx : index_data[idx_off + e + 1]; + if (last || next_idx != cur_idx) { + out_data[out_off + cur_idx] = static_cast(acc); + if (!last) { + cur_idx = next_idx; + acc = + static_cast(out_data[out_off + cur_idx]); + } + } + } + } else { + std::vector acc(K); + for (int64_t k = 0; k < K; ++k) + acc[k] = static_cast( + out_data[out_off + cur_idx * K + k]); + for (int64_t e = 0; e < E; ++e) { + for (int64_t k = 0; k < K; ++k) + acc[k] += + static_cast(src_data[src_off + e * K + k]); + const bool last = (e == E - 1); + const int64_t next_idx = + last ? cur_idx : index_data[idx_off + e + 1]; + if (last || next_idx != cur_idx) { + for (int64_t k = 0; k < K; ++k) + out_data[out_off + cur_idx * K + k] = + static_cast(acc[k]); + if (!last) { + cur_idx = next_idx; + for (int64_t k = 0; k < K; ++k) + acc[k] = static_cast( + out_data[out_off + cur_idx * K + k]); + } + } + } + } + } + }); + }); + + return out; +} + +// CPU implementation of `pyg::gather_coo`. +// +// COO ops fix the gather axis at `dim = index.dim() - 1`. Per element: +// +// out[..., e, ...] = src[..., index[..., e], ...] +// +// Layout (mirrors `segment_sum_coo_kernel`): +// * B = product of leading dims of `index` before `dim`, +// * E = index.size(dim) (= out.size(dim)), +// * N = src.size(dim), +// * K = product of trailing dims of `src` past `index.dim()`. +// +// `out=` contract: **overwrite** the caller's buffer per element. +at::Tensor gather_coo_kernel(const at::Tensor& src, + const at::Tensor& index, + const std::optional& optional_out) { + TORCH_CHECK(src.dim() >= index.dim(), + "gather_coo: src.dim() must be >= index.dim() (got src.dim()=", + src.dim(), ", index.dim()=", index.dim(), ")"); + + const int64_t dim = index.dim() - 1; + TORCH_CHECK(dim >= 0, "gather_coo: index must have at least 1 dimension"); + + // For the leading dims before `dim`, upstream requires src.size(i) == + // index.size(i) (no broadcasting). Match that. + for (int64_t i = 0; i < dim; ++i) { + TORCH_CHECK(src.size(i) == index.size(i), "gather_coo: src.size(", i, + ") must match index.size(", i, ")"); + } + + auto src_c = src.contiguous(); + auto index_c = index.contiguous(); + + at::Tensor out; + if (optional_out.has_value()) { + out = optional_out.value().contiguous(); + for (int64_t i = 0; i < src_c.dim(); ++i) { + if (i != dim) + TORCH_CHECK(src_c.size(i) == out.size(i), "gather_coo: out.size(", i, + ") must match src.size(", i, ")"); + } + } else { + auto out_sizes = src_c.sizes().vec(); + out_sizes[dim] = index_c.size(dim); + out = at::empty(out_sizes, src_c.options()); + } + + if (src_c.numel() == 0 || index_c.numel() == 0) { + if (!optional_out.has_value()) { + out.fill_(0); + } + return out; + } + + const int64_t E = index_c.size(dim); + int64_t B = 1; + for (int64_t i = 0; i < dim; ++i) + B *= index_c.size(i); + const int64_t K = out.numel() / index_c.numel(); + const int64_t N = src_c.size(dim); + + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::Half, at::ScalarType::BFloat16, src_c.scalar_type(), + "gather_coo_cpu", [&] { + const auto* src_data = src_c.data_ptr(); + auto* out_data = out.data_ptr(); + const auto* index_data = index_c.data_ptr(); + + // Parallelize over the leading (B) dim only. Each `b` slice reads + // from a disjoint sub-region of `src` and writes to a disjoint + // sub-region of `out`. + at::parallel_for( + 0, B, at::internal::GRAIN_SIZE, [&](int64_t b_beg, int64_t b_end) { + for (int64_t b = b_beg; b < b_end; ++b) { + const int64_t src_off = b * N * K; + const int64_t out_off = b * E * K; + const int64_t idx_off = b * E; + for (int64_t e = 0; e < E; ++e) { + const int64_t idx = index_data[idx_off + e]; + if (K == 1) { + out_data[out_off + e] = src_data[src_off + idx]; + } else { + for (int64_t k = 0; k < K; ++k) { + out_data[out_off + e * K + k] = + src_data[src_off + idx * K + k]; + } + } + } + } + }); + }); + + return out; +} + +} // namespace + +TORCH_LIBRARY_IMPL(pyg, CPU, m) { + m.impl(TORCH_SELECTIVE_NAME("pyg::segment_sum_coo"), + TORCH_FN(segment_sum_coo_kernel)); + m.impl(TORCH_SELECTIVE_NAME("pyg::gather_coo"), TORCH_FN(gather_coo_kernel)); +} + +} // namespace ops +} // namespace pyg diff --git a/pyg_lib/csrc/ops/cuda/segment_coo_kernel.cu b/pyg_lib/csrc/ops/cuda/segment_coo_kernel.cu new file mode 100644 index 000000000..5fe5c73e4 --- /dev/null +++ b/pyg_lib/csrc/ops/cuda/segment_coo_kernel.cu @@ -0,0 +1,480 @@ +#include "../segment_coo.h" + +#include +#include +#include +#include +#include +#include + +#include "atomics.cuh" +#include "shuffle.cuh" + +namespace pyg { +namespace ops { + +namespace { + +// Convention 13: WARP_SIZE is fixed at 32 on every NVIDIA architecture pyg-lib +// targets (sm_60+). Already defined by `scatter_kernel.cu` in its own TU; we +// redefine guarded here so the two TUs stay independent. +#ifndef WARP_SIZE +#define WARP_SIZE 32 +#endif + +// Full-warp mask for `__shfl_*_sync` — every lane participates. +#define FULL_MASK 0xffffffff + +// Convention 12: dynamic block sizing. Replicates the inline `threads()` / +// `blocks()` pattern from `scatter_kernel.cu` (which itself is a copy of +// `pyg_lib/csrc/sampler/cuda/random_walk_kernel.cu`). Each new kernel file in +// `pyg_lib/csrc/ops/cuda/` carries its own copy until a shared helper lands. +int threads() { + const auto props = at::cuda::getCurrentDeviceProperties(); + return std::min(props->maxThreadsPerBlock, 1024); +} + +int blocks(int numel) { + const auto props = at::cuda::getCurrentDeviceProperties(); + const auto blocks_per_sm = props->maxThreadsPerMultiProcessor / 256; + const auto max_blocks = props->multiProcessorCount * blocks_per_sm; + const auto max_threads = threads(); + return std::max( + 1, std::min(max_blocks, (numel + max_threads - 1) / max_threads)); +} + +// ============================================================================ +// `segment_sum_coo` — Commit 6. +// +// Two CUDA kernel variants: +// +// 1. `segment_sum_coo_cuda_kernel` (K==1, no trailing feature dims). One +// thread +// per `src` entry. `SHFL_UP_SYNC` performs a warp-level tree reduction +// across equal indices within a single warp. Lanes that hold the tail of +// a run (next-lane index differs or warp boundary) atomically write the +// accumulated value into `out[index]`. Port of upstream +// `pytorch_scatter/csrc/cuda/segment_coo_cuda.cu:14-53` — see also the +// design note in `PLAN_PYTORCH_SCATTER.md:363-366`. +// +// 2. `segment_sum_coo_broadcast_cuda_kernel` (K>1). Each thread +// processes a single column and `TB` consecutive index entries. The +// host-side dispatcher chooses `TB ∈ {4, 8, 16, 32}` based on the +// average run length `avg_len = index.size(dim) / dim_size`. Port of +// upstream `segment_coo_cuda.cu:76-127, 225-248`. + +// K==1 kernel — one thread per `src` entry, warp-reduce equal-index runs. +// +// `index_info` is `at::cuda::detail::TensorInfo` so we can +// resolve a possibly-strided `index` view through `IndexToOffset` without +// materializing a contiguous copy. The dispatcher passes the linear-strided +// `.contiguous()` index, but the TensorInfo handle is the cheapest way to +// reuse upstream's offset arithmetic verbatim. +template +__global__ void segment_sum_coo_cuda_kernel( + const scalar_t* __restrict__ src_data, + const at::cuda::detail::TensorInfo index_info, + scalar_t* __restrict__ out_data, + size_t E, + size_t N) { + int row_idx = blockIdx.x * blockDim.x + threadIdx.x; + int lane_idx = row_idx & (WARP_SIZE - 1); + int D = index_info.sizes[index_info.dims - 1]; + + if (row_idx < E) { + int offset = at::cuda::detail::IndexToOffset::get( + row_idx, index_info); + int64_t idx = index_info.data[offset]; + int64_t next_idx; + int out_idx = (row_idx / D) * N + idx; + + scalar_t val = src_data[row_idx]; + scalar_t tmp; + +#pragma unroll + for (int i = 1; i < WARP_SIZE; i *= 2) { + // Parallel reduction inside a single warp. `SHFL_UP_SYNC` pulls the + // value/index from lane `lane_idx - i`; if that lane belongs to the + // same outer-batch and shares the same `idx`, we fold it into `val`. + tmp = SHFL_UP_SYNC(FULL_MASK, val, i); + next_idx = SHFL_UP_SYNC(FULL_MASK, idx, i); + if (lane_idx >= i && row_idx / D == (row_idx - i) / D) { + // Upstream asserts `idx >= next_idx` (sorted-index UB contract). + if (idx == next_idx) { + val = val + tmp; + } + } + } + + // The lane that holds the *tail* of a run writes the accumulated value + // to `out`. Conditions for being a tail: + // * last lane in the warp (`lane_idx == WARP_SIZE - 1`), or + // * boundary in the outer-batch dimension (`row_idx / D != (row_idx + + // 1) / D`), or + // * next-row's index differs from this row's. + next_idx = SHFL_DOWN_SYNC(FULL_MASK, idx, 1); + if (lane_idx == WARP_SIZE - 1 || row_idx / D != (row_idx + 1) / D || + idx != next_idx) { + atomicAdd(out_data + out_idx, val); + } + } +} + +// K>1 broadcast kernel. Each thread owns one column `(col_idx)` and `TB` +// consecutive index entries starting at `(dim_start, row_start)`. We sweep +// the `TB` entries in registers, accumulating equal-index runs and writing +// out to `out` via `atomicAdd` whenever the run ends (next index differs or +// we hit the `TB` boundary). Port of upstream +// `segment_coo_broadcast_kernel`. +// +// `TB` is a compile-time literal (4, 8, 16, or 32) so the `#pragma unroll` +// over the inner loop bites and the per-thread register pressure stays +// predictable. +template +__global__ void segment_sum_coo_broadcast_cuda_kernel( + const scalar_t* __restrict__ src_data, + const at::cuda::detail::TensorInfo index_info, + scalar_t* __restrict__ out_data, + size_t E, + size_t K, + size_t N) { + int D = index_info.sizes[index_info.dims - 1]; + int E_1 = E / D; + // Round `D-1` up to a multiple of `TB`. The thread that lands on + // `dim_start * D + row_start + i` for `i ∈ [0, TB)` only does work while + // `row_start + i < D`. + int E_2 = (D - 1) + TB - ((D - 1) % TB); + + int row_idx = blockIdx.x * blockDim.y + threadIdx.y; + int col_idx = blockIdx.y * blockDim.x + threadIdx.x; + + int dim_start = (row_idx * TB) / E_2; + int row_start = (row_idx * TB) % E_2; + + if (dim_start < E_1 && col_idx < K) { + int offset = at::cuda::detail::IndexToOffset::get( + dim_start * D + row_start, index_info); + int64_t idx1 = index_info.data[offset]; + int64_t idx2; + + scalar_t val = src_data[K * (dim_start * D + row_start) + col_idx]; + +#pragma unroll + for (int i = 1; i < TB; ++i) { + if (row_start + i >= D) + break; + + idx2 = + index_info.data[offset + i * index_info.strides[index_info.dims - 1]]; + // Upstream asserts `idx1 <= idx2` (sorted-index UB contract). + if (idx1 == idx2) { + val = val + src_data[K * (dim_start * D + row_start + i) + col_idx]; + } else { + atomicAdd(out_data + (dim_start * N + idx1) * K + col_idx, val); + val = src_data[K * (dim_start * D + row_start + i) + col_idx]; + } + idx1 = idx2; + } + + atomicAdd(out_data + (dim_start * N + idx1) * K + col_idx, val); + } +} + +at::Tensor segment_sum_coo_kernel(const at::Tensor& src, + const at::Tensor& index, + const std::optional& optional_out, + std::optional dim_size) { + TORCH_CHECK(src.is_cuda(), + "segment_sum_coo (CUDA): src must be a CUDA tensor"); + TORCH_CHECK(index.is_cuda(), + "segment_sum_coo (CUDA): index must be a CUDA tensor"); + TORCH_CHECK(src.device() == index.device(), + "segment_sum_coo (CUDA): src and index must be on the same " + "device"); + TORCH_CHECK(src.dim() >= index.dim(), + "segment_sum_coo (CUDA): src.dim() must be >= index.dim() (got ", + src.dim(), " vs ", index.dim(), ")"); + + const c10::cuda::OptionalCUDAGuard device_guard(device_of(src)); + + // Broadcast `index` up to `src.shape[:index.dim()]` (upstream convention; + // matches `segment_coo_cpu.cpp` and `segment_coo_cuda.cu:164-168`). + auto sizes = index.sizes().vec(); + for (int64_t i = 0; i < index.dim(); ++i) { + sizes[i] = src.size(i); + } + auto index_b = index.expand(sizes); + + // COO ops do not take `dim` from Python — it is always `index.dim() - 1`. + const auto dim = index_b.dim() - 1; + + auto src_c = src.contiguous(); + auto index_c = index_b.contiguous(); + + at::Tensor out; + const bool out_was_provided = optional_out.has_value(); + if (out_was_provided) { + // Accumulate-into contract: caller owns the buffer, no zero-init. + out = optional_out.value().contiguous(); + for (int64_t i = 0; i < out.dim(); ++i) { + if (i != dim) { + TORCH_CHECK(src_c.size(i) == out.size(i), + "segment_sum_coo (CUDA): out.size(", i, + ") must match src.size(", i, ")"); + } + } + } else { + auto out_sizes = src_c.sizes().vec(); + if (dim_size.has_value()) { + out_sizes[dim] = dim_size.value(); + } else if (index_c.numel() == 0) { + out_sizes[dim] = 0; + } else { + // Auto-infer from the last index along `dim` — sorted-index contract + // means this is the maximum index. Mirrors upstream + // `segment_coo_cuda.cu:187-189`. + auto tail = index_c.select(dim, index_c.size(dim) - 1); + tail = tail.numel() > 1 ? tail.max() : tail; + out_sizes[dim] = 1 + tail.cpu().data_ptr()[0]; + } + // Zero-init so the `atomicAdd` accumulation starts from a clean slate. + out = at::zeros(out_sizes, src_c.options()); + } + + if (index_c.numel() == 0) { + return out; + } + + const auto E = index_c.numel(); + const auto E_2 = index_c.size(dim); + const auto E_1 = E / E_2; + const auto K = src_c.numel() / E; + const auto N = out.size(dim); + const float avg_len = static_cast(E_2) / static_cast(N); + + auto index_info = at::cuda::detail::getTensorInfo(index_c); + const auto stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::Half, at::ScalarType::BFloat16, src_c.scalar_type(), + "segment_sum_coo_cuda", [&] { + const auto* src_data = src_c.data_ptr(); + auto* out_data = out.data_ptr(); + + if (K == 1) { + // One thread per `src` entry; warp-reduce equal-index runs in + // shuffle, then a single `atomicAdd` per run's tail. The launch + // grid uses the dynamic `threads()` / `blocks(E)` helpers; the + // shuffle reduction depends only on warp lanes (warp-relative + // indices via `row_idx & 31`), not block geometry, so any block + // size that is a multiple of WARP_SIZE works. `threads()` always + // returns a multiple of 32 on every supported arch. + const int T = threads(); + const int B = std::max(1, static_cast((E + T - 1) / T)); + segment_sum_coo_cuda_kernel + <<>>(src_data, index_info, out_data, E, N); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + } else { + // Broadcast path. Pick `TB ∈ {4, 8, 16, 32}` from `avg_len` — the + // larger the average run, the more entries each thread should + // sweep before writing out. Mirrors upstream + // `segment_coo_cuda.cu:225-248`. + // + // The grid geometry is `(rows_blocks, cols_blocks)` with block + // `(32, 8)`: 32 threads along `K` (one warp per column tile) and + // 8 threads along the row axis. We launch enough row blocks to + // cover `E_1 * ceil(E_2 / TB)` row-positions in groups of 8. + const dim3 block_dim(32, 8); + const dim3 grid_cols((K + 31) / 32); + if (avg_len <= 8.0f) { + constexpr int TB = 4; + const dim3 grid_dim((E_1 * ((E_2 + TB - 1) / TB) + 7) / 8, + grid_cols.x); + segment_sum_coo_broadcast_cuda_kernel + <<>>(src_data, index_info, + out_data, E, K, N); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + } else if (avg_len <= 16.0f) { + constexpr int TB = 8; + const dim3 grid_dim((E_1 * ((E_2 + TB - 1) / TB) + 7) / 8, + grid_cols.x); + segment_sum_coo_broadcast_cuda_kernel + <<>>(src_data, index_info, + out_data, E, K, N); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + } else if (avg_len <= 32.0f) { + constexpr int TB = 16; + const dim3 grid_dim((E_1 * ((E_2 + TB - 1) / TB) + 7) / 8, + grid_cols.x); + segment_sum_coo_broadcast_cuda_kernel + <<>>(src_data, index_info, + out_data, E, K, N); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + } else { + constexpr int TB = 32; + const dim3 grid_dim((E_1 * ((E_2 + TB - 1) / TB) + 7) / 8, + grid_cols.x); + segment_sum_coo_broadcast_cuda_kernel + <<>>(src_data, index_info, + out_data, E, K, N); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + } + } + }); + + return out; +} + +// ============================================================================ +// `gather_coo` — Commit 6. +// +// Trivial gather: `out[i] = src[index[i]]` along `dim = index.dim() - 1`, +// with broadcasting over the trailing feature dims. Two kernel variants: +// +// * K==1: one thread per output element, straight `__ldg`-load gather. +// * K>1: one thread per `(row, col)` pair, same load pattern. +// +// `out=` contract: overwrite (not accumulate). When `out=` is supplied we +// write into the caller's buffer; otherwise we allocate `at::empty(...)`. +// No zero-init in either case — every output element is written exactly once. + +template +__global__ void gather_coo_cuda_kernel( + const scalar_t* __restrict__ src_data, + const at::cuda::detail::TensorInfo index_info, + scalar_t* __restrict__ out_data, + size_t E, + size_t N) { + int row_idx = blockIdx.x * blockDim.x + threadIdx.x; + + if (row_idx < E) { + int offset = at::cuda::detail::IndexToOffset::get( + row_idx, index_info); + int row = static_cast(index_info.data[offset]); + + // `(row_idx / D) * N` picks the outer-batch slice of `src` and `row` + // is the index within that slice. `D = index.size(dim)`. + offset = + (row_idx / index_info.sizes[index_info.dims - 1]) * static_cast(N); + out_data[row_idx] = src_data[offset + row]; + } +} + +template +__global__ void gather_coo_broadcast_cuda_kernel( + const scalar_t* __restrict__ src_data, + const at::cuda::detail::TensorInfo index_info, + scalar_t* __restrict__ out_data, + size_t E, + size_t K, + size_t N) { + int thread_idx = blockIdx.x * blockDim.x + threadIdx.x; + int row_idx = thread_idx / static_cast(K); + int col_idx = thread_idx % static_cast(K); + + if (thread_idx < static_cast(E * K)) { + int offset = at::cuda::detail::IndexToOffset::get( + row_idx, index_info); + int row = static_cast(index_info.data[offset]); + + offset = (row_idx / index_info.sizes[index_info.dims - 1]) * + static_cast(N) * static_cast(K); + out_data[thread_idx] = + src_data[offset + static_cast(K) * row + col_idx]; + } +} + +at::Tensor gather_coo_kernel(const at::Tensor& src, + const at::Tensor& index, + const std::optional& optional_out) { + TORCH_CHECK(src.is_cuda(), "gather_coo (CUDA): src must be a CUDA tensor"); + TORCH_CHECK(index.is_cuda(), + "gather_coo (CUDA): index must be a CUDA tensor"); + TORCH_CHECK(src.device() == index.device(), + "gather_coo (CUDA): src and index must be on the same device"); + TORCH_CHECK(src.dim() >= index.dim(), + "gather_coo (CUDA): src.dim() must be >= index.dim() (got ", + src.dim(), " vs ", index.dim(), ")"); + + const c10::cuda::OptionalCUDAGuard device_guard(device_of(src)); + + // Broadcast `index`'s leading dims up to `src.shape[:index.dim() - 1]`. + // Upstream uses `index.dim() - 1` as the broadcast bound (not `index.dim()`) + // because `index.size(dim)` is the output's *reduction-axis* extent, not a + // batch dim. Matches `segment_coo_cuda.cu:337-340`. + auto sizes = index.sizes().vec(); + for (int64_t i = 0; i < index.dim() - 1; ++i) { + sizes[i] = src.size(i); + } + auto index_b = index.expand(sizes); + + const auto dim = index_b.dim() - 1; + + auto src_c = src.contiguous(); + auto index_c = index_b.contiguous(); + + at::Tensor out; + if (optional_out.has_value()) { + // Overwrite contract: every output element is written exactly once. + out = optional_out.value().contiguous(); + for (int64_t i = 0; i < src_c.dim(); ++i) { + if (i != dim) { + TORCH_CHECK(src_c.size(i) == out.size(i), + "gather_coo (CUDA): out.size(", i, ") must match src.size(", + i, ")"); + } + } + TORCH_CHECK(index_c.size(dim) == out.size(dim), + "gather_coo (CUDA): out.size(", dim, ") must match index.size(", + dim, ")"); + } else { + auto out_sizes = src_c.sizes().vec(); + out_sizes[dim] = index_c.size(dim); + out = at::empty(out_sizes, src_c.options()); + } + + if (index_c.numel() == 0) { + return out; + } + + const auto E = index_c.numel(); + const auto K = out.numel() / E; + const auto N = src_c.size(dim); + + auto index_info = at::cuda::detail::getTensorInfo(index_c); + const auto stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::Half, at::ScalarType::BFloat16, src_c.scalar_type(), + "gather_coo_cuda", [&] { + const auto* src_data = src_c.data_ptr(); + auto* out_data = out.data_ptr(); + + if (K == 1) { + const int T = threads(); + const int B = blocks(static_cast(E)); + gather_coo_cuda_kernel + <<>>(src_data, index_info, out_data, E, N); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + } else { + const int T = threads(); + const int B = blocks(static_cast(E * K)); + gather_coo_broadcast_cuda_kernel + <<>>(src_data, index_info, out_data, E, K, N); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + } + }); + + return out; +} + +} // namespace + +TORCH_LIBRARY_IMPL(pyg, CUDA, m) { + m.impl(TORCH_SELECTIVE_NAME("pyg::segment_sum_coo"), + TORCH_FN(segment_sum_coo_kernel)); + m.impl(TORCH_SELECTIVE_NAME("pyg::gather_coo"), TORCH_FN(gather_coo_kernel)); +} + +} // namespace ops +} // namespace pyg diff --git a/pyg_lib/csrc/ops/cuda/shuffle.cuh b/pyg_lib/csrc/ops/cuda/shuffle.cuh new file mode 100644 index 000000000..5c0e5342a --- /dev/null +++ b/pyg_lib/csrc/ops/cuda/shuffle.cuh @@ -0,0 +1,147 @@ +#pragma once + +// Warp-shuffle helpers for `pyg::segment_*` CUDA kernels. +// +// Port of upstream `pytorch_scatter/csrc/cuda/utils.cuh:16-46`. CUDA's built-in +// `__shfl_up_sync` / `__shfl_down_sync` intrinsics are templated over the +// arithmetic dtypes (int, float, double, ...) but do **not** natively support +// `at::Half` / `at::BFloat16`. We provide thin overloads that bit-cast the +// half-precision value through the underlying `unsigned short` storage so the +// compiler can resolve the shuffle to the native 32-bit `__shfl_*_sync` +// variant. +// +// The `SHFL_UP_SYNC` / `SHFL_DOWN_SYNC` macros expand to the `_sync` variants +// on CUDA and to the deprecated unsync `__shfl_*` builtins on ROCm (where the +// `_sync` family is not available). Mirrors the upstream `#ifdef USE_ROCM` +// split. + +#include +#include + +// `warp_mask_t`: 32-bit on CUDA, 64-bit on ROCm. The mask is the first +// argument of `__shfl_*_sync` and selects which lanes of the warp participate. +// On CUDA we always use the full mask `0xffffffff`; on ROCm the wavefront is +// 64-wide so the mask is 64-bit. +#ifdef USE_ROCM +using warp_mask_t = unsigned long long; +#else +using warp_mask_t = unsigned int; +#endif + +// `at::Half` shuffle overloads. CUDA's `__shfl_*_sync` does not have a native +// overload for `__half` on older toolkits (the 32-bit-wide `__shfl_xor_sync` +// for `__half` was added in CUDA 9, but the `_up`/`_down` variants still want +// an integral or floating-point lane value). We bit-cast through `unsigned +// short` — `at::Half::x` is the IEEE 754 binary16 bit pattern stored as +// `uint16_t` — which lets the compiler resolve to the 32-bit `__shfl_sync` +// instruction (widened/narrowed automatically). +// +// The four-argument overload (mask, value, delta, width) is what the +// `SHFL_UP_SYNC` / `SHFL_DOWN_SYNC` macros expand to from `__shfl_*_sync` +// (which itself has an optional fourth `width` parameter defaulting to +// `warpSize = 32`). We provide a 3-arg overload that forwards to the 4-arg one +// with the default width. +__device__ __inline__ at::Half __shfl_up_sync(const warp_mask_t mask, + const at::Half var, + const unsigned int delta, + const int width) { + // `at::Half::x` is `unsigned short`; pass it through `__shfl_up_sync(..., + // unsigned int, ...)`. The implicit widening to `unsigned int` is lossless. + at::Half ret; + ret.x = static_cast( + __shfl_up_sync(mask, static_cast(var.x), delta, width)); + return ret; +} + +__device__ __inline__ at::Half __shfl_up_sync(const warp_mask_t mask, + const at::Half var, + const unsigned int delta) { + return __shfl_up_sync(mask, var, delta, warpSize); +} + +__device__ __inline__ at::Half __shfl_down_sync(const warp_mask_t mask, + const at::Half var, + const unsigned int delta, + const int width) { + at::Half ret; + ret.x = static_cast( + __shfl_down_sync(mask, static_cast(var.x), delta, width)); + return ret; +} + +__device__ __inline__ at::Half __shfl_down_sync(const warp_mask_t mask, + const at::Half var, + const unsigned int delta) { + return __shfl_down_sync(mask, var, delta, warpSize); +} + +// `at::BFloat16` shuffle overloads. Mirror the `at::Half` overloads above. +// `at::BFloat16::x` is also `uint16_t`, holding the BF16 bit pattern. +__device__ __inline__ at::BFloat16 __shfl_up_sync(const warp_mask_t mask, + const at::BFloat16 var, + const unsigned int delta, + const int width) { + at::BFloat16 ret; + ret.x = static_cast( + __shfl_up_sync(mask, static_cast(var.x), delta, width)); + return ret; +} + +__device__ __inline__ at::BFloat16 __shfl_up_sync(const warp_mask_t mask, + const at::BFloat16 var, + const unsigned int delta) { + return __shfl_up_sync(mask, var, delta, warpSize); +} + +__device__ __inline__ at::BFloat16 __shfl_down_sync(const warp_mask_t mask, + const at::BFloat16 var, + const unsigned int delta, + const int width) { + at::BFloat16 ret; + ret.x = static_cast( + __shfl_down_sync(mask, static_cast(var.x), delta, width)); + return ret; +} + +__device__ __inline__ at::BFloat16 __shfl_down_sync(const warp_mask_t mask, + const at::BFloat16 var, + const unsigned int delta) { + return __shfl_down_sync(mask, var, delta, warpSize); +} + +// ROCm fallback uses the unsync `__shfl_up` / `__shfl_down` instead. +#ifdef USE_ROCM +__device__ __inline__ at::Half __shfl_up(const at::Half var, + const unsigned int delta) { + at::Half ret; + ret.x = static_cast( + __shfl_up(static_cast(var.x), delta)); + return ret; +} +__device__ __inline__ at::Half __shfl_down(const at::Half var, + const unsigned int delta) { + at::Half ret; + ret.x = static_cast( + __shfl_down(static_cast(var.x), delta)); + return ret; +} +__device__ __inline__ at::BFloat16 __shfl_up(const at::BFloat16 var, + const unsigned int delta) { + at::BFloat16 ret; + ret.x = static_cast( + __shfl_up(static_cast(var.x), delta)); + return ret; +} +__device__ __inline__ at::BFloat16 __shfl_down(const at::BFloat16 var, + const unsigned int delta) { + at::BFloat16 ret; + ret.x = static_cast( + __shfl_down(static_cast(var.x), delta)); + return ret; +} +#define SHFL_UP_SYNC(mask, var, delta) __shfl_up(var, delta) +#define SHFL_DOWN_SYNC(mask, var, delta) __shfl_down(var, delta) +#else +#define SHFL_UP_SYNC __shfl_up_sync +#define SHFL_DOWN_SYNC __shfl_down_sync +#endif diff --git a/pyg_lib/csrc/ops/segment_coo.cpp b/pyg_lib/csrc/ops/segment_coo.cpp new file mode 100644 index 000000000..2956116f5 --- /dev/null +++ b/pyg_lib/csrc/ops/segment_coo.cpp @@ -0,0 +1,73 @@ +#include "segment_coo.h" + +#include +#include + +namespace pyg { +namespace ops { + +PYG_API at::Tensor segment_sum_coo(const at::Tensor& src, + const at::Tensor& index, + const std::optional& out, + std::optional dim_size) { + at::TensorArg src_arg{src, "src", 0}; + at::TensorArg index_arg{index, "index", 1}; + at::CheckedFrom c{"segment_sum_coo"}; + + at::checkAllDefined(c, {src_arg, index_arg}); + TORCH_CHECK(src.device() == index.device(), + "segment_sum_coo: src and index must be on the same device " + "(got src=", + src.device(), ", index=", index.device(), ")"); + if (out.has_value()) { + at::TensorArg out_arg{out.value(), "out", 2}; + at::checkAllDefined(c, {out_arg}); + TORCH_CHECK(src.device() == out.value().device(), + "segment_sum_coo: src and out must be on the same device " + "(got src=", + src.device(), ", out=", out.value().device(), ")"); + } + + static auto op = c10::Dispatcher::singleton() + .findSchemaOrThrow("pyg::segment_sum_coo", "") + .typed(); + return op.call(src, index, out, dim_size); +} + +PYG_API at::Tensor gather_coo(const at::Tensor& src, + const at::Tensor& index, + const std::optional& out) { + at::TensorArg src_arg{src, "src", 0}; + at::TensorArg index_arg{index, "index", 1}; + at::CheckedFrom c{"gather_coo"}; + + at::checkAllDefined(c, {src_arg, index_arg}); + TORCH_CHECK(src.device() == index.device(), + "gather_coo: src and index must be on the same device " + "(got src=", + src.device(), ", index=", index.device(), ")"); + if (out.has_value()) { + at::TensorArg out_arg{out.value(), "out", 2}; + at::checkAllDefined(c, {out_arg}); + TORCH_CHECK(src.device() == out.value().device(), + "gather_coo: src and out must be on the same device " + "(got src=", + src.device(), ", out=", out.value().device(), ")"); + } + + static auto op = c10::Dispatcher::singleton() + .findSchemaOrThrow("pyg::gather_coo", "") + .typed(); + return op.call(src, index, out); +} + +TORCH_LIBRARY_FRAGMENT(pyg, m) { + m.def(TORCH_SELECTIVE_SCHEMA( + "pyg::segment_sum_coo(Tensor src, Tensor index, " + "Tensor? out=None, int? dim_size=None) -> Tensor")); + m.def(TORCH_SELECTIVE_SCHEMA( + "pyg::gather_coo(Tensor src, Tensor index, Tensor? out=None) -> Tensor")); +} + +} // namespace ops +} // namespace pyg diff --git a/pyg_lib/csrc/ops/segment_coo.h b/pyg_lib/csrc/ops/segment_coo.h new file mode 100644 index 000000000..b3cac7272 --- /dev/null +++ b/pyg_lib/csrc/ops/segment_coo.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include "pyg_lib/csrc/macros.h" + +namespace pyg { +namespace ops { + +// Sums all values from `src` into `out` at the segment positions specified in +// `index` along the implicit axis `dim = index.dim() - 1`. +// +// COO ops do **not** take a `dim` argument: upstream `pytorch_scatter` +// fixes the reduction axis at `index.dim() - 1` (the last index dim). The +// `index` tensor must be **sorted ascending** along that axis; unsorted input +// is undefined behavior. +// +// When `out` is not provided, a fresh zero-initialized buffer is allocated. +// When `out` *is* provided, the operation **accumulates** into the caller's +// buffer (no zero-init); this matches the upstream contract and is what makes +// the symmetric-pair backward of `gather_coo` efficient (the backward +// allocates `at::zeros(src_shape)` and calls `segment_sum_coo` with it as +// `out=` to deposit the gradient in-place). +PYG_API at::Tensor segment_sum_coo( + const at::Tensor& src, + const at::Tensor& index, + const std::optional& out = std::nullopt, + std::optional dim_size = std::nullopt); + +// Gathers values from `src` at positions specified in `index`, along the +// implicit axis `dim = index.dim() - 1`. Concretely: +// +// out[..., i, ...] = src[..., index[..., i], ...] along `dim` +// +// COO ops do **not** take a `dim` argument: upstream `pytorch_scatter` fixes +// the gather axis at `index.dim() - 1`. +// +// When `out` is not provided, a fresh buffer is allocated. When `out` *is* +// provided, the operation **overwrites** the caller's buffer per element +// (matches upstream `pytorch_scatter`). +PYG_API at::Tensor gather_coo( + const at::Tensor& src, + const at::Tensor& index, + const std::optional& out = std::nullopt); + +} // namespace ops +} // namespace pyg diff --git a/pyg_lib/ops/__init__.py b/pyg_lib/ops/__init__.py index f664f6386..f1f66490a 100644 --- a/pyg_lib/ops/__init__.py +++ b/pyg_lib/ops/__init__.py @@ -499,6 +499,59 @@ def scatter_max( return torch.ops.pyg.scatter_max(src, index, dim, out, dim_size) +def segment_sum_coo( + src: Tensor, + index: Tensor, + out: Optional[Tensor] = None, + dim_size: Optional[int] = None, +) -> Tensor: + r"""Reduces all values from the :obj:`src` tensor into :obj:`out` according + to a sorted COO :obj:`index`, using ``sum`` as the reduction. + + Unlike :func:`scatter_sum`, the reduction axis is always + ``index.dim() - 1`` (so no :obj:`dim` argument is exposed). The + :obj:`index` must be sorted in ascending order along that axis. + + If :obj:`out` is not given, a new zero-initialized tensor is allocated. + If :obj:`out` is given, values are **accumulated** into it (no zero-init). + + Args: + src: The source tensor. + index: Sorted COO indices. + out: The destination tensor. + dim_size: If :obj:`out` is not given, the output size along the + reduction axis. Auto-inferred from :obj:`index.max() + 1` if + :obj:`None`. + + Returns: + The reduced tensor. + """ + return torch.ops.pyg.segment_sum_coo(src, index, out, dim_size) + + +segment_add_coo = segment_sum_coo + + +def gather_coo( + src: Tensor, + index: Tensor, + out: Optional[Tensor] = None, +) -> Tensor: + r"""Gathers values from :obj:`src` at positions specified by :obj:`index` + along axis ``index.dim() - 1``. This is the symmetric inverse of + :func:`segment_sum_coo`. + + Args: + src: The source tensor. + index: The COO indices to gather. + out: The destination tensor (overwritten if given). + + Returns: + ``out[i] = src[index[i]]`` along the gather axis. + """ + return torch.ops.pyg.gather_coo(src, index, out) + + def spline_basis( pseudo: Tensor, kernel_size: Tensor, @@ -743,6 +796,9 @@ def edge_sample( 'scatter_mean', 'scatter_min', 'scatter_max', + 'segment_sum_coo', + 'segment_add_coo', + 'gather_coo', 'spline_basis', 'spline_weighting', 'grid_cluster', diff --git a/test/ops/test_segment_coo.py b/test/ops/test_segment_coo.py new file mode 100644 index 000000000..efa0f1282 --- /dev/null +++ b/test/ops/test_segment_coo.py @@ -0,0 +1,466 @@ +import pytest +import torch + +import pyg_lib +from pyg_lib.testing import withCUDA + +# --------------------------------------------------------------------------- +# Reference implementations +# --------------------------------------------------------------------------- + + +def _broadcast_index( + index: torch.Tensor, + src: torch.Tensor, +) -> torch.Tensor: + """Broadcasts ``index`` up to the shape of ``src`` along the COO dim + (``index.dim() - 1``). Mirrors the upstream / C++ ``broadcast`` util. + """ + dim = index.dim() - 1 + if index.dim() == src.dim(): + return index + size = [1] * src.dim() + size[dim] = -1 + # ``index`` is 1-D in the broadcast case; reshape then expand. + return index.view(size).expand_as(src) + + +def _segment_sum_coo_ref( + src: torch.Tensor, + index: torch.Tensor, + out: torch.Tensor = None, + dim_size: int = None, +) -> torch.Tensor: + """Pure-PyTorch reference for :func:`segment_sum_coo`. + + COO ops always reduce along ``dim = index.dim() - 1``. With a sorted + ``index``, segment-sum is equivalent to scatter-sum, so the reference is + a plain ``zeros + scatter_add_``. + """ + dim = index.dim() - 1 + bcast_index = _broadcast_index(index, src) + if out is None: + if dim_size is None: + if index.numel() > 0: + dim_size = int(index.max().item()) + 1 + else: + dim_size = 0 + size = list(src.size()) + size[dim] = dim_size + out = torch.zeros(size, dtype=src.dtype, device=src.device) + out.scatter_add_(dim, bcast_index, src) + return out + + +def _gather_coo_ref( + src: torch.Tensor, + index: torch.Tensor, + out: torch.Tensor = None, +) -> torch.Tensor: + """Pure-PyTorch reference for :func:`gather_coo`. + + ``out[..., i, ...] = src[..., index[i], ...]`` along ``dim = index.dim() + - 1``. With a 1-D ``index`` and multi-D ``src``, the index is broadcast + along the last dim, equivalent to ``src.index_select(dim, index)``. + """ + dim = index.dim() - 1 + if index.dim() == src.dim(): + result = torch.gather(src, dim, index) + else: + # 1-D index path: pick rows / cols along ``dim``. + result = src.index_select(dim, index) + if out is not None: + out.copy_(result) + return out + return result + + +# --------------------------------------------------------------------------- +# Aliases +# --------------------------------------------------------------------------- + + +def test_segment_add_coo_is_segment_sum_coo_alias(): + assert pyg_lib.ops.segment_add_coo is pyg_lib.ops.segment_sum_coo + + +# --------------------------------------------------------------------------- +# segment_sum_coo — forward +# --------------------------------------------------------------------------- + + +@withCUDA +@pytest.mark.parametrize('dtype', [torch.float32, torch.float64]) +def test_segment_sum_coo_forward_dtypes(dtype, device): + torch.manual_seed(0) + src = torch.randn(8, 4, dtype=dtype, device=device) + # Sorted ascending — required by segment_*_coo. + index = torch.tensor([0, 0, 0, 1, 1, 2, 3, 3], device=device) + + out = pyg_lib.ops.segment_sum_coo(src, index) + expected = _segment_sum_coo_ref(src, index) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_segment_sum_coo_forward_1d(device): + src = torch.randn(8, device=device) + index = torch.tensor([0, 0, 0, 1, 1, 2, 3, 3], device=device) + + out = pyg_lib.ops.segment_sum_coo(src, index) + expected = _segment_sum_coo_ref(src, index) + assert out.size() == (4,) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_segment_sum_coo_forward_2d_broadcast_index(device): + """1-D ``index`` broadcasts over a 2-D ``src`` along ``dim = 0``.""" + src = torch.randn(6, 4, device=device) + index = torch.tensor([0, 0, 1, 1, 1, 2], device=device) + + out = pyg_lib.ops.segment_sum_coo(src, index) + expected = _segment_sum_coo_ref(src, index) + assert out.size() == (3, 4) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_segment_sum_coo_forward_k1_small_trailing(device): + """K==1 path: no trailing feature dims (purely 1-D).""" + src = torch.randn(10, device=device) + index = torch.tensor([0, 0, 1, 1, 2, 2, 3, 3, 4, 4], device=device) + + out = pyg_lib.ops.segment_sum_coo(src, index) + expected = _segment_sum_coo_ref(src, index) + assert out.size() == (5,) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_segment_sum_coo_forward_k_large_broadcast(device): + """K>1 path: large trailing feature dim exercises the broadcast kernel.""" + src = torch.randn(12, 64, device=device) + index = torch.tensor([0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4], device=device) + + out = pyg_lib.ops.segment_sum_coo(src, index) + expected = _segment_sum_coo_ref(src, index) + assert out.size() == (5, 64) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_segment_sum_coo_forward_matches_scatter_sum(device): + """Sorted ``index`` -> segment_sum_coo is equivalent to scatter_sum.""" + src = torch.randn(8, 4, device=device) + index = torch.tensor([0, 0, 0, 1, 1, 2, 3, 3], device=device) + + out_coo = pyg_lib.ops.segment_sum_coo(src, index) + out_scatter = pyg_lib.ops.scatter_sum(src, index, dim=0) + torch.testing.assert_close(out_coo, out_scatter) + + +@withCUDA +def test_segment_sum_coo_dim_size_auto_infer(device): + src = torch.randn(6, device=device) + index = torch.tensor([0, 1, 1, 2, 2, 3], device=device) + + out = pyg_lib.ops.segment_sum_coo(src, index) + # Auto-inferred dim_size = index.max() + 1 = 4. + assert out.size(-1) == 4 + expected = _segment_sum_coo_ref(src, index) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_segment_sum_coo_dim_size_explicit(device): + src = torch.randn(6, device=device) + index = torch.tensor([0, 1, 1, 2, 2, 3], device=device) + + out = pyg_lib.ops.segment_sum_coo(src, index, dim_size=6) + assert out.size(-1) == 6 + expected = _segment_sum_coo_ref(src, index, dim_size=6) + torch.testing.assert_close(out, expected) + # Trailing empty buckets should be exactly zero. + torch.testing.assert_close(out[4:], torch.zeros(2, device=device)) + + +@withCUDA +def test_segment_sum_coo_empty_rows_in_middle(device): + """Row 1 has no entries — its output must be zero.""" + src = torch.randn(5, 4, device=device) + index = torch.tensor([0, 0, 2, 2, 2], device=device) + + out = pyg_lib.ops.segment_sum_coo(src, index) + expected = _segment_sum_coo_ref(src, index) + assert out.size() == (3, 4) + torch.testing.assert_close(out, expected) + # Row 1 is empty -> zeros. + torch.testing.assert_close(out[1], torch.zeros(4, device=device)) + + +@withCUDA +def test_segment_sum_coo_empty_input(device): + """Empty source and index — output is all-zero of the requested shape.""" + src = torch.empty(0, 4, device=device) + index = torch.empty(0, dtype=torch.long, device=device) + + out = pyg_lib.ops.segment_sum_coo(src, index, dim_size=3) + assert out.size() == (3, 4) + torch.testing.assert_close(out, torch.zeros(3, 4, device=device)) + + +@withCUDA +def test_segment_sum_coo_sorted_smoke(device): + """Sorted-ascending index is the supported contract — smoke test that + a long, well-formed sorted index produces the reference result. + """ + torch.manual_seed(1) + src = torch.randn(32, 3, device=device) + # Long sorted run with varying segment sizes. + index = torch.tensor( + [0] * 5 + [1] * 1 + [2] * 8 + [3] * 4 + [4] * 7 + [5] * 7, + device=device, + ) + assert index.numel() == 32 + + out = pyg_lib.ops.segment_sum_coo(src, index) + expected = _segment_sum_coo_ref(src, index) + torch.testing.assert_close(out, expected) + + +# --------------------------------------------------------------------------- +# segment_sum_coo — backward +# --------------------------------------------------------------------------- + + +@withCUDA +def test_segment_sum_coo_backward_gradcheck(device): + src = torch.randn( + 6, + 3, + dtype=torch.double, + device=device, + requires_grad=True, + ) + index = torch.tensor([0, 0, 1, 1, 1, 2], device=device) + + def fn(s): + return pyg_lib.ops.segment_sum_coo(s, index) + + assert torch.autograd.gradcheck(fn, (src,)) + + +@withCUDA +def test_segment_sum_coo_backward_gradcheck_with_dim_size(device): + """Gradcheck with explicit (larger) ``dim_size`` -> empty trailing rows.""" + src = torch.randn( + 5, + dtype=torch.double, + device=device, + requires_grad=True, + ) + index = torch.tensor([0, 0, 2, 2, 2], device=device) + + def fn(s): + return pyg_lib.ops.segment_sum_coo(s, index, dim_size=5) + + assert torch.autograd.gradcheck(fn, (src,)) + + +@withCUDA +def test_segment_sum_coo_backward_gradcheck_empty_rows(device): + """Empty-row fixture under gradcheck.""" + src = torch.randn( + 5, + 4, + dtype=torch.double, + device=device, + requires_grad=True, + ) + index = torch.tensor([0, 0, 2, 2, 2], device=device) + + def fn(s): + return pyg_lib.ops.segment_sum_coo(s, index) + + assert torch.autograd.gradcheck(fn, (src,)) + + +# --------------------------------------------------------------------------- +# gather_coo — forward +# --------------------------------------------------------------------------- + + +@withCUDA +@pytest.mark.parametrize('dtype', [torch.float32, torch.float64]) +def test_gather_coo_forward_dtypes(dtype, device): + torch.manual_seed(0) + src = torch.randn(4, 3, dtype=dtype, device=device) + index = torch.tensor([0, 1, 1, 2, 3, 3, 0, 2], device=device) + + out = pyg_lib.ops.gather_coo(src, index) + expected = _gather_coo_ref(src, index) + assert out.size() == (8, 3) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_gather_coo_forward_1d(device): + src = torch.randn(5, device=device) + index = torch.tensor([0, 0, 1, 2, 2, 2, 3, 4, 4], device=device) + + out = pyg_lib.ops.gather_coo(src, index) + expected = _gather_coo_ref(src, index) + assert out.size() == (9,) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_gather_coo_forward_2d_broadcast_index(device): + """1-D ``index`` broadcasts over a 2-D ``src``; output trailing dim + matches ``src``'s trailing dim. + """ + src = torch.randn(4, 8, device=device) + index = torch.tensor([0, 1, 1, 2, 3, 3, 0, 2], device=device) + + out = pyg_lib.ops.gather_coo(src, index) + expected = _gather_coo_ref(src, index) + assert out.size() == (8, 8) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_gather_coo_forward_k1_small_trailing(device): + """K==1 path: 1-D src, 1-D index.""" + src = torch.tensor([10.0, 20.0, 30.0, 40.0], device=device) + index = torch.tensor([0, 1, 1, 2, 3, 3, 0, 2], device=device) + + out = pyg_lib.ops.gather_coo(src, index) + expected = torch.tensor( + [10.0, 20.0, 20.0, 30.0, 40.0, 40.0, 10.0, 30.0], + device=device, + ) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_gather_coo_forward_k_large_broadcast(device): + """K>1 path: large trailing feature dim exercises broadcast kernel.""" + src = torch.randn(5, 64, device=device) + index = torch.tensor([0, 0, 1, 2, 2, 2, 3, 4, 4, 4, 4], device=device) + + out = pyg_lib.ops.gather_coo(src, index) + expected = _gather_coo_ref(src, index) + assert out.size() == (11, 64) + torch.testing.assert_close(out, expected) + + +@withCUDA +def test_gather_coo_forward_inverse_of_segment_sum_signature(device): + """``segment_sum_coo`` output rows have shape ``[N, *trailing]``; passing + it through ``gather_coo`` with the same ``index`` recovers a tensor of + shape ``[E, *trailing]`` (the backward of segment_sum_coo). + """ + src = torch.randn(8, 3, device=device) + index = torch.tensor([0, 0, 1, 1, 2, 2, 2, 3], device=device) + + summed = pyg_lib.ops.segment_sum_coo(src, index) # [4, 3] + scattered = pyg_lib.ops.gather_coo(summed, index) # [8, 3] + assert scattered.size() == (8, 3) + expected = _gather_coo_ref(summed, index) + torch.testing.assert_close(scattered, expected) + + +@withCUDA +def test_gather_coo_empty_index(device): + """Empty index -> empty output (shape ``[0, *trailing]``).""" + src = torch.randn(3, 4, device=device) + index = torch.empty(0, dtype=torch.long, device=device) + + out = pyg_lib.ops.gather_coo(src, index) + assert out.size() == (0, 4) + + +# --------------------------------------------------------------------------- +# gather_coo — backward +# --------------------------------------------------------------------------- + + +@withCUDA +def test_gather_coo_backward_gradcheck(device): + src = torch.randn( + 4, + 3, + dtype=torch.double, + device=device, + requires_grad=True, + ) + index = torch.tensor([0, 1, 1, 2, 3, 3, 0, 2], device=device) + + def fn(s): + return pyg_lib.ops.gather_coo(s, index) + + assert torch.autograd.gradcheck(fn, (src,)) + + +@withCUDA +def test_gather_coo_backward_gradcheck_1d(device): + src = torch.randn( + 5, + dtype=torch.double, + device=device, + requires_grad=True, + ) + index = torch.tensor([0, 0, 1, 2, 2, 2, 3, 4, 4], device=device) + + def fn(s): + return pyg_lib.ops.gather_coo(s, index) + + assert torch.autograd.gradcheck(fn, (src,)) + + +# --------------------------------------------------------------------------- +# gradgradcheck — symmetric pair (each op's backward calls the other) +# --------------------------------------------------------------------------- + + +@withCUDA +def test_segment_sum_coo_gradgradcheck(device): + """``SegmentSumCOO.backward`` invokes ``gather_coo``; gradgradcheck + exercises ``gather_coo``'s backward (which in turn invokes + ``segment_sum_coo``). + """ + src = torch.randn( + 6, + 3, + dtype=torch.double, + device=device, + requires_grad=True, + ) + index = torch.tensor([0, 0, 1, 1, 1, 2], device=device) + + def fn(s): + return pyg_lib.ops.segment_sum_coo(s, index) + + assert torch.autograd.gradgradcheck(fn, (src,)) + + +@withCUDA +def test_gather_coo_gradgradcheck(device): + """``GatherCOO.backward`` invokes ``segment_sum_coo``; gradgradcheck + exercises ``segment_sum_coo``'s backward (which in turn invokes + ``gather_coo``). + """ + src = torch.randn( + 4, + 3, + dtype=torch.double, + device=device, + requires_grad=True, + ) + index = torch.tensor([0, 1, 1, 2, 3, 3, 0, 2], device=device) + + def fn(s): + return pyg_lib.ops.gather_coo(s, index) + + assert torch.autograd.gradgradcheck(fn, (src,))