Add RDMA / NVIDIA GPU Direct Storage support (#2233)

Author: harshavardhana

.github/workflows/go-rdma.yml

@@ -0,0 +1,73 @@
+name: Build (RDMA)
+
+on:
+  pull_request:
+    branches:
+    - master
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build with -tags=rdma against minio-cpp (${{ matrix.config.arch }})
+    runs-on: ${{ matrix.config.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+        - { os: ubuntu-latest,    arch: "amd64" }
+        - { os: ubuntu-24.04-arm, arch: "arm64" }
+
+    steps:
+    - name: Checkout minio-go
+      uses: actions/checkout@v4
+      with:
+        path: "minio-go"
+
+    - name: Checkout minio-cpp
+      uses: actions/checkout@v4
+      with:
+        repository: minio/minio-cpp
+        path: "minio-cpp"
+
+    - name: Checkout vcpkg
+      uses: actions/checkout@v4
+      with:
+        repository: microsoft/vcpkg
+        path: "vcpkg"
+
+    - name: Install system dependencies
+      run: |
+        sudo apt-get -qy update
+        sudo apt-get -qy install cmake libibverbs-dev librdmacm-dev libnuma-dev
+
+    - name: Build and install libminiocpp.so with RDMA
+      shell: bash
+      run: |
+        ./vcpkg/bootstrap-vcpkg.sh
+        cd minio-cpp
+        ../vcpkg/vcpkg install
+        cmake . -B ./build \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DBUILD_SHARED_LIBS=ON \
+          -DCMAKE_INSTALL_PREFIX=/usr/local \
+          -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake \
+          -DMINIO_CPP_ENABLE_RDMA:BOOL=ON
+        cmake --build ./build --config Release -j 4
+        sudo cmake --install ./build
+        cuobj_arch=$([ "${{ matrix.config.arch }}" = "amd64" ] && echo x86_64 || echo aarch64)
+        sudo cp -P vendor/cuobj/lib/${cuobj_arch}/* /usr/local/lib/
+        sudo ldconfig
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: "1.25.x"
+
+    - name: Build minio-go with -tags=rdma
+      working-directory: minio-go
+      run: |
+        go build -tags=rdma ./...
+        go build -tags=rdma -o /tmp/rdma-test ./cmd/rdma-test

api-get-object.go

@@ -46,6 +46,22 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o
 		}
 	}
 
+	if opts.RDMABuffer != nil && c.rdmaEnabled {
+		n, err := c.getObjectRDMA(ctx, bucketName, objectName, opts)
+		if err != nil {
+			return nil, err
+		}
+		return &Object{
+			mutex:    &sync.Mutex{},
+			isClosed: true,
+			objectInfo: ObjectInfo{
+				Key:  objectName,
+				Size: n,
+			},
+			objectInfoSet: true,
+		}, nil
+	}
+
 	gctx, cancel := context.WithCancel(ctx)
 
 	// Detect if snowball is server location we are talking to.

api-get-options.go

@@ -23,6 +23,7 @@ import (
 	"net/url"
 	"strconv"
 	"time"
+	"unsafe"
 
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 )
@@ -48,6 +49,12 @@ type GetObjectOptions struct {
 	// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
 	Checksum bool
 
+	// RDMABuffer, when non-nil and Options.EnableRDMA=true, downloads directly
+	// into a contiguous buffer via libminiocpp.so. The returned *Object's
+	// Read() returns EOF immediately; bytes-transferred is in Stat().Size.
+	RDMABuffer     unsafe.Pointer
+	RDMABufferSize int
+
 	// To be not used by external applications
 	Internal AdvancedGetOptions
 }

api-put-object.go

@@ -28,6 +28,7 @@ import (
 	"sort"
 	"strings"
 	"time"
+	"unsafe"
 
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/s3utils"
@@ -111,6 +112,12 @@ type PutObjectOptions struct {
 	ConcurrentStreamParts bool
 	Internal              AdvancedPutOptions
 
+	// RDMABuffer, when non-nil and Options.EnableRDMA=true, selects the RDMA
+	// path via libminiocpp.so. Must reference RDMABufferSize contiguous bytes.
+	// When set, the reader / size args to PutObject are ignored.
+	RDMABuffer     unsafe.Pointer
+	RDMABufferSize int
+
 	customHeaders http.Header
 }
 
@@ -322,6 +329,9 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part
 func (c *Client) PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64,
 	opts PutObjectOptions,
 ) (info UploadInfo, err error) {
+	if opts.RDMABuffer != nil && c.rdmaEnabled {
+		return c.putObjectRDMA(ctx, bucketName, objectName, opts)
+	}
 	if size < 0 && opts.DisableMultipart {
 		return UploadInfo{}, errors.New("object size must be provided with disable multipart upload")
 	}

api.go

@@ -109,6 +109,15 @@ type Client struct {
 
 	trailingHeaderSupport bool
 	maxRetries            int
+
+	// RDMA dispatch state. rdmaEnabled mirrors Options.EnableRDMA;
+	// the rest are only touched by rdma.go (built with -tags=rdma) but
+	// have to live on the struct so the stub and the tagged build share
+	// one shape.
+	rdmaEnabled bool
+	rdmaOnce    sync.Once         //nolint:unused
+	rdmaHandle  *rdmaClientHandle //nolint:unused
+	rdmaInitErr error             //nolint:unused
 }
 
 // Options for New method
@@ -156,6 +165,11 @@ type Options struct {
 	// Number of times a request is retried. Defaults to 10 retries if this option is not configured.
 	// Set to 1 to disable retries.
 	MaxRetries int
+
+	// EnableRDMA causes PutObject / GetObject to dispatch to libminiocpp.so
+	// when the caller supplies PutObjectOptions.RDMABuffer / GetObjectOptions.RDMABuffer.
+	// No-op unless built with -tags=rdma.
+	EnableRDMA bool
 }
 
 // Global constants.
@@ -311,6 +325,7 @@ func privateNew(endpoint string, opts *Options) (*Client, error) {
 	}
 
 	clnt.trailingHeaderSupport = opts.TrailingHeaders && clnt.overrideSignerType.IsV4()
+	clnt.rdmaEnabled = opts.EnableRDMA
 
 	// Sets bucket lookup style, whether server accepts DNS or Path lookup. Default is Auto - determined
 	// by the SDK. When Auto is specified, DNS lookup is used for Amazon/Google cloud endpoints and Path for all other endpoints.

cmd/rdma-test/main.go

@@ -0,0 +1,118 @@
+// Copyright 2024-2026 - MinIO, Inc. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// rdma-test exercises the unified minio-go RDMA path against a running
+// MinIO server. Requires -tags=rdma at build time and libminiocpp.so on
+// the host's library search path.
+//
+//   go build -tags=rdma -o rdma-test ./cmd/rdma-test
+//   MINIO_ENDPOINT=coe01:9000 MINIO_ACCESS_KEY=... MINIO_SECRET_KEY=... ./rdma-test
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"unsafe"
+
+	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+const (
+	testBucket = "rdma-test"
+	testObject = "test-object-cpu"
+	testSize   = 1 << 20 // 1 MiB
+)
+
+func envOr(k, d string) string {
+	if v := os.Getenv(k); v != "" {
+		return v
+	}
+	return d
+}
+
+func main() {
+	if err := run(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func run() error {
+	endpoint := envOr("MINIO_ENDPOINT", "coe01:9000")
+	accessKey := envOr("MINIO_ACCESS_KEY", "minioadmin")
+	secretKey := envOr("MINIO_SECRET_KEY", "minioadmin")
+
+	fmt.Printf("endpoint=%s rdma_available=%v\n", endpoint, minio.IsRDMAAvailable())
+
+	client, err := minio.New(endpoint, &minio.Options{
+		Creds:      credentials.NewStaticV4(accessKey, secretKey, ""),
+		Secure:     false,
+		EnableRDMA: true,
+	})
+	if err != nil {
+		return fmt.Errorf("New: %w", err)
+	}
+
+	ctx := context.Background()
+
+	exists, err := client.BucketExists(ctx, testBucket)
+	if err != nil {
+		return fmt.Errorf("BucketExists: %w", err)
+	}
+	if !exists {
+		if err := client.MakeBucket(ctx, testBucket, minio.MakeBucketOptions{}); err != nil {
+			return fmt.Errorf("MakeBucket: %w", err)
+		}
+	}
+
+	src := minio.AlignedBuffer(testSize)
+	if src == nil {
+		return fmt.Errorf("AlignedBuffer(%d) returned nil", testSize)
+	}
+	defer minio.FreeAlignedBuffer(src)
+	srcSlice := unsafe.Slice((*byte)(src), testSize)
+	for i := range srcSlice {
+		srcSlice[i] = byte(i)
+	}
+
+	fmt.Print("PutObject (RDMA)... ")
+	info, err := client.PutObject(ctx, testBucket, testObject, nil, 0, minio.PutObjectOptions{
+		RDMABuffer:     src,
+		RDMABufferSize: testSize,
+	})
+	if err != nil {
+		return fmt.Errorf("PutObject: %w", err)
+	}
+	fmt.Printf("ok etag=%s size=%d checksum=%s\n", info.ETag, info.Size, info.ChecksumCRC64NVME)
+
+	dst := minio.AlignedBuffer(testSize)
+	if dst == nil {
+		return fmt.Errorf("AlignedBuffer(%d) returned nil", testSize)
+	}
+	defer minio.FreeAlignedBuffer(dst)
+	dstSlice := unsafe.Slice((*byte)(dst), testSize)
+
+	fmt.Print("GetObject (RDMA)... ")
+	obj, err := client.GetObject(ctx, testBucket, testObject, minio.GetObjectOptions{
+		RDMABuffer:     dst,
+		RDMABufferSize: testSize,
+	})
+	if err != nil {
+		return fmt.Errorf("GetObject: %w", err)
+	}
+	stat, err := obj.Stat()
+	if err != nil {
+		return fmt.Errorf("Stat: %w", err)
+	}
+	fmt.Printf("ok size=%d\n", stat.Size)
+
+	if !bytes.Equal(srcSlice, dstSlice) {
+		return fmt.Errorf("FAIL: roundtrip data mismatch")
+	}
+	fmt.Println("PASS: roundtrip verified")
+	return nil
+}