8. Usage Patterns

8.1. Basic Server Usage

cuObjServer server("192.168.1.100", 18515, CUOBJ_PROTO_RDMA_DC_V1);
if (!server.isConnected()) {
    return;
}

uint16_t channel = server.allocateChannelId();
if (channel == INVALID_CHANNEL_ID) {
    return;
}

void* buffer = server.allocHostBuffer(1024 * 1024);
if (buffer == nullptr) {
    server.freeChannelId(channel);
    return;
}

struct rdma_buffer* rdma_buf = server.registerBuffer(buffer, 1024 * 1024);
if (rdma_buf == nullptr) {
    free(buffer);
    server.freeChannelId(channel);
    return;
}

ssize_t result = server.handleGetObject("request-1",
                                        rdma_buf,
                                        remote_addr,
                                        request_size,
                                        rdma_descriptor,
                                        channel);
if (result < 0) {
    // Handle error.
}

server.deRegisterBuffer(rdma_buf);
free(buffer);
server.freeChannelId(channel);

8.2. Synchronous PUT Usage

ssize_t result = server.handlePutObject("request-2",
                                        rdma_buf,
                                        remote_addr,
                                        request_size,
                                        rdma_descriptor,
                                        channel);
if (result > 0) {
    // result bytes are available in rdma_buf-backed local memory.
}

8.3. Asynchronous Usage

void* async_handle = reinterpret_cast<void*>(0x1);

ssize_t submit = server.handleGetObject("async-get",
                                        rdma_buf,
                                        remote_addr,
                                        request_size,
                                        rdma_descriptor,
                                        channel,
                                        0,
                                        nullptr,
                                        async_handle);
if (submit == 0) {
    cuObjAsyncEvent_t events[16];
    int count = 0;
    do {
        count = server.poll(events, 16, channel);
    } while (count == 0);

    if (count > 0) {
        for (int i = 0; i < count; ++i) {
            if (events[i].status == IBV_WC_SUCCESS) {
                // events[i].async_handle identifies the completed request.
            }
        }
    }
}

8.4. Scatter-Gather Registration

void* buf1 = server.allocHostBuffer(4096);
void* buf2 = server.allocHostBuffer(8192);

std::vector<cuObjScatterGatherEntry_t> sglist = {
    { buf1, 4096 },
    { buf2, 8192 }
};

struct rdma_buffer* rdma_buf = server.registerBuffer(sglist);
if (rdma_buf != nullptr) {
    ssize_t result = server.handleGetObject("sg-get",
                                            rdma_buf,
                                            remote_addr,
                                            12288,
                                            rdma_descriptor,
                                            channel);
    server.deRegisterBuffer(rdma_buf);
}

free(buf1);
free(buf2);

8.5. Registered Buffer View

void* base = server.allocHostBuffer(1024 * 1024);
struct rdma_buffer* base_buf = server.registerBuffer(base, 1024 * 1024);

std::vector<cuObjScatterGatherEntryWithOffset_t> view_entries = {
    { 0, 4096 },
    { 16384, 8192 }
};

struct rdma_buffer* view =
    server.getRDMABufferFromSgList(base_buf, view_entries);
if (view != nullptr) {
    server.handleGetObject("view-get",
                           view,
                           remote_addr,
                           12288,
                           rdma_descriptor,
                           channel);
    server.putRDMABufferFromSgList(view);
}

server.deRegisterBuffer(base_buf);
free(base);

8.6. Custom RDMA Tuning

cuObjRDMATunable params;
params.setNumDcis(256);
params.setCqDepth(1024);
params.setTimeout(20);
params.setRetryCount(7);
params.setDelayInterval(10000);
params.setQPResetOnFailure(true);
params.setMaxRdAtomic(16);

cuObjServer server("fd00::10", 18515, CUOBJ_PROTO_RDMA_DC_V1, params);

8.7. Telemetry Configuration

std::ofstream log_file("cuobj_server.log");

cuObjServer::setupTelemetry(false, &log_file);
cuObjServer::setTelemFlags(CUOBJ_LOG_PATH_ERROR |
                           CUOBJ_LOG_PATH_INFO |
                           CUOBJ_LOG_PATH_DEBUG);

{
    cuObjServer server("192.168.1.100", 18515, CUOBJ_PROTO_RDMA_DC_V1);
    // Use server.
}

cuObjServer::shutdownTelemetry();
log_file.close();