From ac836ec85e8b6c69b7ddf43b3183b5c22463e3e7 Mon Sep 17 00:00:00 2001 From: Anshuman Goswami Date: Mon, 20 Apr 2026 16:58:32 -0700 Subject: [PATCH] gin: add GDAKI plugin with stub implementations Add a GDAKI ncclGin_v13_t plugin with stub implementations, following the pattern where GDAKI and Proxy have separate plugins. New files: include/rdma/gin/nccl_ofi_gin_gdaki.h - GDAKI plugin + enabled check src/rdma/gin/nccl_ofi_gin_gdaki.cpp - GDAKI-specific function stubs The GDAKI plugin has: - get_properties: reports NCCL_NET_DEVICE_GIN_GDAKI via ncclNetProperties_v12_t - createContext: takes ncclGinConfig_v13_t, returns ncclInternalError - regMrSym/regMrSymDmaBuf: returns ncclInternalError (not yet implemented) - destroyContext/deregMrSym: no-op stubs - ginProgress/queryLastError: no-op stubs - closeColl/closeListen/finalize: no-op stubs - iput/iputSignal/iget/iflush/test: nullptr (no CPU involvement in GDAKI mode) - init/devices/listen/connect: copied from proxy plugin at init time When OFI_NCCL_GIN_GDAKI=1 is set, init() morphs the exported ncclGinPlugin_v13 by copying shared functions from proxy into the GDAKI plugin, then overwriting the exported symbol. Tested on p5en cluster (1 and 2 node): alltoall_perf -R 2 -D 3 (pure GIN) alltoall_perf -R 2 -D 4 (hybrid GIN) All configs: createContext called on every rank, NCCL WARN 'createContext not yet implemented', clean exit code 3. --- include/Makefile.am | 1 + include/nccl_ofi_param.h | 6 + include/rdma/gin/nccl_ofi_gin_gdaki.h | 21 ++++ src/Makefile.am | 3 +- src/rdma/gin/nccl_ofi_gin_api.cpp | 22 ++++ src/rdma/gin/nccl_ofi_gin_gdaki.cpp | 158 ++++++++++++++++++++++++++ 6 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 include/rdma/gin/nccl_ofi_gin_gdaki.h create mode 100644 src/rdma/gin/nccl_ofi_gin_gdaki.cpp diff --git a/include/Makefile.am b/include/Makefile.am index 8e6ce9d92..ddeaeddd8 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -14,6 +14,7 @@ noinst_HEADERS = \ rdma/gin/nccl_ofi_gin_reqs.h \ rdma/gin/nccl_ofi_gin_resources.h \ rdma/gin/nccl_ofi_gin_types.h \ + rdma/gin/nccl_ofi_gin_gdaki.h \ nccl_ofi_gin_base.h \ nccl_ofi.h \ nccl_ofi_api.h \ diff --git a/include/nccl_ofi_param.h b/include/nccl_ofi_param.h index 1af8ef95e..6e5ca34ad 100644 --- a/include/nccl_ofi_param.h +++ b/include/nccl_ofi_param.h @@ -374,4 +374,10 @@ OFI_NCCL_PARAM(std::string, platform, "PLATFORM", ""); OFI_NCCL_PARAM_VALUE_SET(NVTX_TRACE_DIMENSION, (PER_COMM)(PER_DEV)) OFI_NCCL_PARAM(NVTX_TRACE_DIMENSION, nvtx_trace_dimension, "NVTX_TRACE_DIMENSION", NVTX_TRACE_DIMENSION::PER_COMM) +/* + * Enable GDAKI (GPUDirect Async) mode for the GIN plugin. + * Not yet supported — stub implementation only. + */ +OFI_NCCL_PARAM(bool, gin_gdaki, "GIN_GDAKI", false); + #endif // End NCCL_OFI_PARAM_H_ diff --git a/include/rdma/gin/nccl_ofi_gin_gdaki.h b/include/rdma/gin/nccl_ofi_gin_gdaki.h new file mode 100644 index 000000000..c65762d23 --- /dev/null +++ b/include/rdma/gin/nccl_ofi_gin_gdaki.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2026 Amazon.com, Inc. or its affiliates. All rights reserved. + */ + +#ifndef NCCL_OFI_GIN_GDAKI_H_ +#define NCCL_OFI_GIN_GDAKI_H_ + +#include "nccl_ofi.h" + +/* + * Return true if GDAKI mode is requested via OFI_NCCL_GIN_GDAKI=1 env var. + */ +bool nccl_ofi_gin_gdaki_enabled(); + +/* + * The GDAKI plugin. Shared functions (init, devices, listen, connect) + * are nullptr and get copied from the proxy plugin at init time. + */ +extern ncclGin_v13_t nccl_ofi_gin_gdaki_plugin; + +#endif /* NCCL_OFI_GIN_GDAKI_H_ */ diff --git a/src/Makefile.am b/src/Makefile.am index f0dca34bd..80254b661 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -56,7 +56,8 @@ sources += \ rdma/gin/nccl_ofi_gin_allgather.cpp \ rdma/gin/nccl_ofi_gin_api.cpp \ rdma/gin/nccl_ofi_gin_reqs.cpp \ - rdma/gin/nccl_ofi_gin_resources.cpp + rdma/gin/nccl_ofi_gin_resources.cpp \ + rdma/gin/nccl_ofi_gin_gdaki.cpp endif # add the tuner sources into the library diff --git a/src/rdma/gin/nccl_ofi_gin_api.cpp b/src/rdma/gin/nccl_ofi_gin_api.cpp index d26e3b22a..c725773f7 100644 --- a/src/rdma/gin/nccl_ofi_gin_api.cpp +++ b/src/rdma/gin/nccl_ofi_gin_api.cpp @@ -4,12 +4,18 @@ #include "config.h" +#include + #include "rdma/gin/nccl_ofi_gin.h" #include "rdma/gin/nccl_ofi_gin_types.h" +#include "rdma/gin/nccl_ofi_gin_gdaki.h" #include "nccl_ofi.h" #include "nccl_ofi_api.h" #include "nccl_ofi_param.h" +/* Forward declaration — defined at bottom of this file */ +extern ncclGin_v13_t ncclGinPlugin_v13; + /** * Structure to hold GIN context data. * This is created once per NCCL communicator and passed to all listen() calls @@ -69,6 +75,22 @@ static ncclResult_t nccl_ofi_gin_init(void **ctx, uint64_t commId, ncclDebugLogg return ncclSystemError; } + /* + * Morph the exported plugin to GDAKI if requested. + * + * Copy shared functions (init, devices, listen, connect) from the + * proxy plugin into the GDAKI plugin, then overwrite the exported + * symbol with the GDAKI plugin. + */ + if (nccl_ofi_gin_gdaki_enabled()) { + NCCL_OFI_INFO(NCCL_NET | NCCL_INIT, "gin: GDAKI mode enabled (OFI_NCCL_GIN_GDAKI=1)"); + nccl_ofi_gin_gdaki_plugin.init = ncclGinPlugin_v13.init; + nccl_ofi_gin_gdaki_plugin.devices = ncclGinPlugin_v13.devices; + nccl_ofi_gin_gdaki_plugin.listen = ncclGinPlugin_v13.listen; + nccl_ofi_gin_gdaki_plugin.connect = ncclGinPlugin_v13.connect; + memcpy(&ncclGinPlugin_v13, &nccl_ofi_gin_gdaki_plugin, sizeof(ncclGinPlugin_v13)); + } + return ncclSuccess; } diff --git a/src/rdma/gin/nccl_ofi_gin_gdaki.cpp b/src/rdma/gin/nccl_ofi_gin_gdaki.cpp new file mode 100644 index 000000000..e731a40d9 --- /dev/null +++ b/src/rdma/gin/nccl_ofi_gin_gdaki.cpp @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2026 Amazon.com, Inc. or its affiliates. All rights reserved. + * + * GDAKI stub implementations for the GIN plugin API. + * These provide the full ncclGin_v13_t plugin for GDAKI mode. + * + * Task: GDAKI stub implementation + */ + +#include "config.h" + +#include "rdma/gin/nccl_ofi_gin_gdaki.h" +#include "nccl_ofi.h" +#include "nccl_ofi_api.h" +#include "nccl_ofi_param.h" + +bool nccl_ofi_gin_gdaki_enabled() +{ + return ofi_nccl_gin_gdaki.get(); +} + +static ncclResult_t nccl_ofi_gin_gdaki_get_properties(int dev, ncclNetProperties_v12_t *props) +{ + nccl_ofi_properties_t ofi_properties; + ncclResult_t ret = nccl_net_ofi_get_properties(dev, &ofi_properties); + if (ret != ncclSuccess) { + return ret; + } + + props->name = ofi_properties.name; + props->pciPath = ofi_properties.pci_path; + props->guid = ofi_properties.guid; + props->ptrSupport = NCCL_PTR_HOST; + if (ofi_properties.hmem_support) { + props->ptrSupport |= NCCL_PTR_CUDA; + } + if (ofi_properties.dmabuf_support) { + props->ptrSupport |= NCCL_PTR_DMABUF; + } + + props->regIsGlobal = ofi_properties.regIsGlobal; + props->forceFlush = 0; + props->speed = ofi_properties.port_speed; + props->port = ofi_properties.port_number; + props->latency = ofi_properties.latency; + props->maxComms = ofi_properties.max_communicators; + props->maxRecvs = ofi_properties.max_group_receives; + props->netDeviceType = NCCL_NET_DEVICE_GIN_GDAKI; + props->netDeviceVersion = NCCL_NET_DEVICE_INVALID_VERSION; + props->vProps.ndevs = 1; + props->vProps.devs[0] = dev; + props->maxP2pBytes = ofi_properties.max_p2p_bytes; + props->maxCollBytes = ofi_properties.max_coll_bytes; + props->maxMultiRequestSize = 1; + props->railId = -1; + props->planeId = -1; + + return ncclSuccess; +} + +static ncclResult_t nccl_ofi_gin_gdaki_createContext(void *collComm, ncclGinConfig_v13_t *config, + void **ginCtx, + ncclNetDeviceHandle_v11_t **devHandle) +{ + NCCL_OFI_WARN("gin GDAKI: createContext not yet implemented (nSignals=%d, nCounters=%d)", + config->nSignals, config->nCounters); + return ncclInternalError; +} + +static ncclResult_t nccl_ofi_gin_gdaki_destroyContext(void *ginCtx) +{ + NCCL_OFI_WARN("gin GDAKI: destroyContext not yet implemented"); + return ncclSuccess; +} + +static ncclResult_t nccl_ofi_gin_gdaki_regMrSym(void *collComm, void *data, size_t size, int type, + uint64_t mrFlags, void **mhandle, void **ginHandle) +{ + NCCL_OFI_WARN("gin GDAKI: regMrSym not yet implemented"); + return ncclInternalError; +} + +static ncclResult_t nccl_ofi_gin_gdaki_regMrSymDmaBuf(void *collComm, void *data, size_t size, + int type, uint64_t offset, int fd, + uint64_t mrFlags, void **mhandle, + void **ginHandle) +{ + NCCL_OFI_WARN("gin GDAKI: regMrSymDmaBuf not yet implemented"); + return ncclInternalError; +} + +static ncclResult_t nccl_ofi_gin_gdaki_deregMrSym(void *collComm, void *mhandle) +{ + NCCL_OFI_WARN("gin GDAKI: deregMrSym not yet implemented"); + return ncclSuccess; +} + +static ncclResult_t nccl_ofi_gin_gdaki_closeColl(void *collComm) +{ + NCCL_OFI_WARN("gin GDAKI: closeColl not yet implemented"); + return ncclSuccess; +} + +static ncclResult_t nccl_ofi_gin_gdaki_closeListen(void *listenComm) +{ + NCCL_OFI_WARN("gin GDAKI: closeListen not yet implemented"); + return ncclSuccess; +} + +static ncclResult_t nccl_ofi_gin_gdaki_ginProgress(void *ginCtx) +{ + NCCL_OFI_WARN("gin GDAKI: ginProgress not yet implemented"); + return ncclSuccess; +} + +static ncclResult_t nccl_ofi_gin_gdaki_queryLastError(void *ginCtx, bool *hasError) +{ + NCCL_OFI_WARN("gin GDAKI: queryLastError not yet implemented"); + *hasError = false; + return ncclSuccess; +} + +static ncclResult_t nccl_ofi_gin_gdaki_finalize(void *ctx) +{ + NCCL_OFI_WARN("gin GDAKI: finalize not yet implemented"); + return ncclSuccess; +} + +/* + * GDAKI plugin. Function pointers for the ncclGin_v13_t interface: + * - iput, iputSignal, iget, iflush, test are nullptr (no CPU involvement + * in GDAKI mode) + * - init, devices, listen, connect are copied from the proxy plugin at + * init time. + */ +ncclGin_v13_t nccl_ofi_gin_gdaki_plugin = { + .name = "Libfabric_GDAKI", + .init = nullptr, /* Copied from proxy plugin at init time */ + .devices = nullptr, /* Copied from proxy plugin at init time */ + .getProperties = nccl_ofi_gin_gdaki_get_properties, + .listen = nullptr, /* Copied from proxy plugin at init time */ + .connect = nullptr, /* Copied from proxy plugin at init time */ + .createContext = nccl_ofi_gin_gdaki_createContext, + .regMrSym = nccl_ofi_gin_gdaki_regMrSym, + .regMrSymDmaBuf = nccl_ofi_gin_gdaki_regMrSymDmaBuf, + .deregMrSym = nccl_ofi_gin_gdaki_deregMrSym, + .destroyContext = nccl_ofi_gin_gdaki_destroyContext, + .closeColl = nccl_ofi_gin_gdaki_closeColl, + .closeListen = nccl_ofi_gin_gdaki_closeListen, + .iput = nullptr, /* No CPU involvement in GDAKI mode */ + .iputSignal = nullptr, /* No CPU involvement in GDAKI mode */ + .iget = nullptr, /* No CPU involvement in GDAKI mode */ + .iflush = nullptr, /* No CPU involvement in GDAKI mode */ + .test = nullptr, /* No CPU involvement in GDAKI mode */ + .ginProgress = nccl_ofi_gin_gdaki_ginProgress, + .queryLastError = nccl_ofi_gin_gdaki_queryLastError, + .finalize = nccl_ofi_gin_gdaki_finalize +};