12. Usage Patterns#

12.1. Basic Client Usage With Explicit Descriptor Acquisition#

// 1. Define callback operations
CUObjOps_t ops = {
    .get = my_get_callback,
    .put = my_put_callback
};

// 2. Create client instance
cuObjClient client(ops, CUOBJ_PROTO_RDMA_DC_V1);
if (!client.isConnected()) {
    // Handle connection error
    return;
}

// 3. Allocate and register memory
void* buffer = malloc(1024 * 1024);  // 1MB buffer
if (buffer == nullptr) {
    // Handle allocation error
    return;
}

if (client.cuMemObjGetDescriptor(buffer, 1024 * 1024) != CU_OBJ_SUCCESS) {
    // Handle registration error
    free(buffer);
    return;
}

// 4. Perform GET operation
MyContext ctx = { /* user data */ };
ssize_t result = client.cuObjGet(&ctx, buffer, 1024, 0, 0);
if (result < 0) {
    // Handle error
    client.cuMemObjPutDescriptor(buffer);
    free(buffer);
    return;
}

// 5. Perform PUT operation
result = client.cuObjPut(&ctx, buffer, 1024, 0, 0);
if (result < 0) {
    // Handle error
    client.cuMemObjPutDescriptor(buffer);
    free(buffer);
    return;
}

// 6. Cleanup
client.cuMemObjPutDescriptor(buffer);
free(buffer);

12.2. Callback Implementation Example#

ssize_t my_get_callback(const void* handle, char* ptr, size_t size,
                        loff_t offset, const cufileRDMAInfo_t* rdma_info) {
    // Extract user context
    MyContext* ctx = (MyContext*)cuObjClient::getCtx(handle);
    if (ctx == nullptr || rdma_info == nullptr || rdma_info->desc_str == nullptr) {
        return -EINVAL;
    }

    // Get RDMA descriptor string
    const char* rdma_desc = rdma_info->desc_str;

    // Communicate with the remote endpoint via control path
    // Send: operation=GET, size, offset, rdma_desc
    ssize_t bytes_read = send_request_on_control_path(ctx->connection,
                                                      "GET", size, offset, rdma_desc);

    // Return the amount of data transferred
    return bytes_read;
}

ssize_t my_put_callback(const void* handle, const char* ptr, size_t size,
                        loff_t offset, const cufileRDMAInfo_t* rdma_info) {
    // Extract user context
    MyContext* ctx = (MyContext*)cuObjClient::getCtx(handle);
    if (ctx == nullptr || rdma_info == nullptr || rdma_info->desc_str == nullptr) {
        return -EINVAL;
    }

    // Get RDMA descriptor string
    const char* rdma_desc = rdma_info->desc_str;

    // Communicate with the remote endpoint via control path
    // Send: operation=PUT, size, offset, rdma_desc
    ssize_t bytes_written = send_request_on_control_path(ctx->connection,
                                                         "PUT", size, offset, rdma_desc);

    // Return the amount of data transferred
    return bytes_written;
}

12.3. CUDA Device Memory Usage#

// Allocate CUDA device memory
void* d_buffer = nullptr;
if (cudaMalloc(&d_buffer, 1024 * 1024) != cudaSuccess) {
    // Handle allocation error
    return;
}

// Check memory type
cuObjMemoryType_t mem_type = cuObjClient::getMemoryType(d_buffer);
assert(mem_type == CUOBJ_MEMORY_CUDA_DEVICE);

// Register for RDMA
if (client.cuMemObjGetDescriptor(d_buffer, 1024 * 1024) != CU_OBJ_SUCCESS) {
    // Handle registration error
    cudaFree(d_buffer);
    return;
}

// Perform I/O operations
ssize_t result = client.cuObjGet(&ctx, d_buffer, 1024, 0, 0);
if (result < 0) {
    // Handle error
}

// Cleanup
client.cuMemObjPutDescriptor(d_buffer);
cudaFree(d_buffer);

12.4. Manual RDMA Token Management#

// Register memory
void* buffer = malloc(1024 * 1024);
if (buffer == nullptr) {
    // Handle allocation error
    return;
}

if (client.cuMemObjGetDescriptor(buffer, 1024 * 1024) != CU_OBJ_SUCCESS) {
    // Handle registration error
    free(buffer);
    return;
}

// Get RDMA token for a specific region
char* rdma_token = nullptr;
size_t region_size = 4096;
size_t region_offset = 1024;
cuObjErr_t err = client.cuMemObjGetRDMAToken(buffer, region_size,
                                             region_offset, CUOBJ_GET,
                                             &rdma_token);
if (err == CU_OBJ_SUCCESS) {
    // Pass rdma_token on the application control path
    // ...

    // Free the token
    client.cuMemObjPutRDMAToken(rdma_token);
}

// Cleanup
client.cuMemObjPutDescriptor(buffer);
free(buffer);

12.5. Telemetry Configuration#

// Setup telemetry with custom output stream
std::ofstream log_file("cuobj_client.log");
cuObjClient::setupTelemetry(false, &log_file);
cuObjClient::setTelemFlags(CUOBJ_LOG_PATH_ERROR |
                           CUOBJ_LOG_PATH_INFO |
                           CUOBJ_LOG_PATH_DEBUG);

{
    // Create and use client
    cuObjClient client(ops);
    // ... perform operations ...
}

// Shutdown telemetry before closing file
cuObjClient::shutdownTelemetry();
log_file.close();