12. Usage Patterns#
12.1. Basic Client Usage With Explicit Descriptor Acquisition#
// 1. Define callback operations
CUObjOps_t ops = {
.get = my_get_callback,
.put = my_put_callback
};
// 2. Create client instance
cuObjClient client(ops, CUOBJ_PROTO_RDMA_DC_V1);
if (!client.isConnected()) {
// Handle connection error
return;
}
// 3. Allocate and register memory
void* buffer = malloc(1024 * 1024); // 1MB buffer
if (buffer == nullptr) {
// Handle allocation error
return;
}
if (client.cuMemObjGetDescriptor(buffer, 1024 * 1024) != CU_OBJ_SUCCESS) {
// Handle registration error
free(buffer);
return;
}
// 4. Perform GET operation
MyContext ctx = { /* user data */ };
ssize_t result = client.cuObjGet(&ctx, buffer, 1024, 0, 0);
if (result < 0) {
// Handle error
client.cuMemObjPutDescriptor(buffer);
free(buffer);
return;
}
// 5. Perform PUT operation
result = client.cuObjPut(&ctx, buffer, 1024, 0, 0);
if (result < 0) {
// Handle error
client.cuMemObjPutDescriptor(buffer);
free(buffer);
return;
}
// 6. Cleanup
client.cuMemObjPutDescriptor(buffer);
free(buffer);
12.2. Callback Implementation Example#
ssize_t my_get_callback(const void* handle, char* ptr, size_t size,
loff_t offset, const cufileRDMAInfo_t* rdma_info) {
// Extract user context
MyContext* ctx = (MyContext*)cuObjClient::getCtx(handle);
if (ctx == nullptr || rdma_info == nullptr || rdma_info->desc_str == nullptr) {
return -EINVAL;
}
// Get RDMA descriptor string
const char* rdma_desc = rdma_info->desc_str;
// Communicate with the remote endpoint via control path
// Send: operation=GET, size, offset, rdma_desc
ssize_t bytes_read = send_request_on_control_path(ctx->connection,
"GET", size, offset, rdma_desc);
// Return the amount of data transferred
return bytes_read;
}
ssize_t my_put_callback(const void* handle, const char* ptr, size_t size,
loff_t offset, const cufileRDMAInfo_t* rdma_info) {
// Extract user context
MyContext* ctx = (MyContext*)cuObjClient::getCtx(handle);
if (ctx == nullptr || rdma_info == nullptr || rdma_info->desc_str == nullptr) {
return -EINVAL;
}
// Get RDMA descriptor string
const char* rdma_desc = rdma_info->desc_str;
// Communicate with the remote endpoint via control path
// Send: operation=PUT, size, offset, rdma_desc
ssize_t bytes_written = send_request_on_control_path(ctx->connection,
"PUT", size, offset, rdma_desc);
// Return the amount of data transferred
return bytes_written;
}
12.3. CUDA Device Memory Usage#
// Allocate CUDA device memory
void* d_buffer = nullptr;
if (cudaMalloc(&d_buffer, 1024 * 1024) != cudaSuccess) {
// Handle allocation error
return;
}
// Check memory type
cuObjMemoryType_t mem_type = cuObjClient::getMemoryType(d_buffer);
assert(mem_type == CUOBJ_MEMORY_CUDA_DEVICE);
// Register for RDMA
if (client.cuMemObjGetDescriptor(d_buffer, 1024 * 1024) != CU_OBJ_SUCCESS) {
// Handle registration error
cudaFree(d_buffer);
return;
}
// Perform I/O operations
ssize_t result = client.cuObjGet(&ctx, d_buffer, 1024, 0, 0);
if (result < 0) {
// Handle error
}
// Cleanup
client.cuMemObjPutDescriptor(d_buffer);
cudaFree(d_buffer);
12.4. Manual RDMA Token Management#
// Register memory
void* buffer = malloc(1024 * 1024);
if (buffer == nullptr) {
// Handle allocation error
return;
}
if (client.cuMemObjGetDescriptor(buffer, 1024 * 1024) != CU_OBJ_SUCCESS) {
// Handle registration error
free(buffer);
return;
}
// Get RDMA token for a specific region
char* rdma_token = nullptr;
size_t region_size = 4096;
size_t region_offset = 1024;
cuObjErr_t err = client.cuMemObjGetRDMAToken(buffer, region_size,
region_offset, CUOBJ_GET,
&rdma_token);
if (err == CU_OBJ_SUCCESS) {
// Pass rdma_token on the application control path
// ...
// Free the token
client.cuMemObjPutRDMAToken(rdma_token);
}
// Cleanup
client.cuMemObjPutDescriptor(buffer);
free(buffer);
12.5. Telemetry Configuration#
// Setup telemetry with custom output stream
std::ofstream log_file("cuobj_client.log");
cuObjClient::setupTelemetry(false, &log_file);
cuObjClient::setTelemFlags(CUOBJ_LOG_PATH_ERROR |
CUOBJ_LOG_PATH_INFO |
CUOBJ_LOG_PATH_DEBUG);
{
// Create and use client
cuObjClient client(ops);
// ... perform operations ...
}
// Shutdown telemetry before closing file
cuObjClient::shutdownTelemetry();
log_file.close();