Template: int ibv_exp_query_device(struct ibv_context *context, struct ibv_exp_device_attr *attr)
Input Parameters:
context
Output Parameters:
attr
Return Value: returns 0 on success, or the value of errno on failure (which indicates the failure reason).
Description: ibv_exp_query_device returns the attributes of the device with context context. The argument attr is a pointer to an ibv_exp_device_attr struct, as defined in <infiniband/verbs_exp.h>.
struct ibv_exp_device_attr {
char fw_ver[64];
uint64_t node_guid;
uint64_t sys_image_guid;
uint64_t max_mr_size;
uint64_t page_size_cap;
uint32_t vendor_id;
uint32_t vendor_part_id;
uint32_t hw_ver;
int max_qp;
int max_qp_wr;
int reserved; /* place holder to align with ibv_device_attr */
int max_sge;
int max_sge_rd;
int max_cq;
int max_cqe;
int max_mr;
int max_pd;
int max_qp_rd_atom;
int max_ee_rd_atom;
int max_res_rd_atom;
int max_qp_init_rd_atom;
int max_ee_init_rd_atom;
enum ibv_exp_atomic_cap exp_atomic_cap;
int max_ee;
int max_rdd;
int max_mw;
int max_raw_ipv6_qp;
int max_raw_ethy_qp;
int max_mcast_grp;
int max_mcast_qp_attach;
int max_total_mcast_qp_attach;
int max_ah;
int max_fmr;
int max_map_per_fmr;
int max_srq;
int max_srq_wr;
int max_srq_sge;
uint16_t max_pkeys;
uint8_t local_ca_ack_delay;
uint8_t phys_port_cnt;
uint32_t comp_mask;
struct ibv_exp_device_calc_cap calc_cap;
uint64_t timestamp_mask;
uint64_t hca_core_clock;
uint64_t exp_device_cap_flags; /* use ibv_exp_device_cap_flags */
int max_dc_req_rd_atom;
int max_dc_res_rd_atom;
int inline_recv_sz;
uint32_t max_rss_tbl_sz;
struct ibv_exp_ext_atomics_params ext_atom;
uint32_t max_mkey_klm_list_size;
uint32_t max_send_wqe_inline_klms;
uint32_t max_umr_recursion_depth;
uint32_t max_umr_stride_dimension;
};
Template: ibv_exp_create_qp(struct ibv_context *context, struct ibv_exp_qp_init_attr *qp_init_attr)
Input Parameters:
Output Parameters:
Return Value: Returns a pointer to the created QP, or NULL if the request fails. Check the QP number (qp_num) in the returned QP.
Description: ibv_exp_create_qp creates a queue pair (QP) associated with the protection domain pd. The argument init_attr is an ibv_exp_qp_init_attr struct, as defined in <infiniband/verbs_exp.h>.
struct ibv_exp_qp_init_attr {
void *qp_context;
struct ibv_cq *send_cq;
struct ibv_cq *recv_cq;
struct ibv_srq *srq;
struct ibv_qp_cap cap;
enum ibv_qp_type qp_type;
int sq_sig_all;
uint32_t comp_mask; /* use ibv_exp_qp_init_attr_comp_mask */
struct ibv_pd *pd;
struct ibv_xrcd *xrcd;
uint32_t exp_create_flags; /* use ibv_exp_qp_create_flags */
uint32_t max_inl_recv;
struct ibv_exp_qpg qpg;
uint32_t max_atomic_arg;
uint32_t max_inl_send_klms;
};
Template: static inline int ibv_exp_post_send(struct ibv_qp *qp, struct ibv_exp_send_wr *wr, struct ibv_- exp_send_wr **bad_wr)
Input Parameters:
Output Parameters:
Return Value: returns 0 on success, or the value of errno on failure (which indicates the failure reason).
Description: ibv_exp_post_send posts the linked list of work requests (WRs) starting with wr to the send queue of the queue pair qp. It stops processing WRs from this list at the first failure (that can be detected immediately while requests are being posted), and returns this failing WR through bad_wr.
struct ibv_exp_send_wr {
uint64_t wr_id;
struct ibv_exp_send_wr *next;
struct ibv_sge *sg_list;
int num_sge;
enum ibv_exp_wr_opcode exp_opcode; /* use ibv_exp_wr_opcode */
int reserved; /* place holder to align with ibv_send_wr */
union {
uint32_t imm_data; /* in network byte order */
uint32_t invalidate_rkey;
} ex;
union {
struct {
uint64_t remote_addr;
uint32_t rkey;
} rdma;
struct {
uint64_t remote_addr;
uint64_t compare_add;
uint64_t swap;
uint32_t rkey;
} atomic;
struct {
struct ibv_ah *ah;
uint32_t remote_qpn;
uint32_t remote_qkey;
} ud;
} wr;
union {
union {
struct {
uint32_t remote_srqn;
} xrc;
} qp_type;
uint32_t xrc_remote_srq_num;
};
union {
struct {
uint64_t remote_addr;
uint32_t rkey;
} rdma;
struct {
uint64_t remote_addr;
uint64_t compare_add;
uint64_t swap;
uint32_t rkey;
} atomic;
struct {
struct ibv_cq *cq;
int32_t cq_count;
} cqe_wait;
struct {
struct ibv_qp *qp;
int32_t wqe_count;
} wqe_enable;
} task;
union {
struct {
enum ibv_exp_calc_op calc_op;
enum ibv_exp_calc_data_type data_type;
enum ibv_exp_calc_data_size data_size;
} calc;
} op;
struct {
struct ibv_ah *ah;
uint64_t dct_access_key;
uint32_t dct_number;
} dc;
struct {
struct ibv_mw *mw;
uint32_t rkey;
struct ibv_exp_mw_bind_info bind_info;
} bind_mw;
uint64_t exp_send_flags; /* use ibv_exp_send_flags */
uint32_t comp_mask; /* reserved for future growth (must be 0) */
union {
struct {
struct {
enum mem_layout_type mkey_type;
union {
struct ibv_exp_mem_region *mem_reg_list; /* array, size corresponds to wr->num_sge */
struct {
struct ibv_exp_mem_repeat_block *mem_repeat_block_list; /* array, size corresponds to wr->num_sge */
size_t *repeat_count; /* array size corresponds to ndim */
uint32_t ndim;
} rb;
} mem_list;
struct non_inline_data *memory_objects; /* used when IBV_EXP_SEND_INLINE is not set */
int access;
struct ibv_mr *modified_mr;
void *region_base_addr;
} memory_key;
} umr;
struct {
uint32_t log_arg_sz;
uint64_t remote_addr;
uint32_t rkey;
union {
struct {
/* For the next four fields:
* If operand_size <= 8 then inline data is immediate
* from the corresponding field; for small opernands,
* ls bits are used.
* Else the fields are pointers in the process's address space
* where arguments are stored
*/
union {
struct ibv_exp_cmp_swap cmp_swap;
struct ibv_exp_fetch_add fetch_add;
} op;
} inline_data; /* IBV_EXP_SEND_EXT_ATOMIC_INLINE is set */
/* in the future add support for non-inline argument provisioning */
} wr_data;
} masked_atomics;
} ext_op;
};
For atomic operations, to support atomic responses in big-endian format (the only way to use atomics on Connect-IB® on little-endian machines) is:
Use experimental verbs.
Check to see if the atomics capabilities flag IBV_EXP_ATOMIC_HCA_REPLY_BE in the exp_atomic_cap field of the struct ibv_exp_device_attr returned by ibv_exp_query_device().
Set the flag IBV_EXP_QP_CREATE_ATOMIC_BE_REPLY when opening the QP. This is what enables the use of atomic ops on Connect-IB.
Use the experimental post send verb.