image image image image image

On This Page

ibv_exp_query_device

Template: int ibv_exp_query_device(struct ibv_context *context, struct ibv_exp_device_attr *attr)

Input Parameters:

context

Output Parameters:

attr

Return Value: returns 0 on success, or the value of errno on failure (which indicates the failure reason).

Description: ibv_exp_query_device returns the attributes of the device with context context. The argument attr is a pointer to an ibv_exp_device_attr struct, as defined in <infiniband/verbs_exp.h>.

struct ibv_exp_device_attr {
  char      fw_ver[64];
  uint64_t    node_guid;
  uint64_t    sys_image_guid;
  uint64_t    max_mr_size;
  uint64_t    page_size_cap;
  uint32_t    vendor_id;
  uint32_t    vendor_part_id;
  uint32_t    hw_ver;
  int     max_qp;
  int     max_qp_wr;
  int     reserved; /* place holder to align with ibv_device_attr */
  int     max_sge;
  int     max_sge_rd;
 int     max_cq;
  int     max_cqe;
  int     max_mr;
  int     max_pd;
  int     max_qp_rd_atom;
  int     max_ee_rd_atom;
  int     max_res_rd_atom;
  int     max_qp_init_rd_atom;
  int     max_ee_init_rd_atom;
  enum ibv_exp_atomic_cap exp_atomic_cap;
 int     max_ee;
  int     max_rdd;
  int     max_mw;
  int     max_raw_ipv6_qp;
  int     max_raw_ethy_qp;
  int     max_mcast_grp;
  int     max_mcast_qp_attach;
  int     max_total_mcast_qp_attach;
  int     max_ah;
  int     max_fmr;
  int     max_map_per_fmr;
  int     max_srq;
  int     max_srq_wr;
  int     max_srq_sge;
  uint16_t    max_pkeys;
  uint8_t     local_ca_ack_delay;
  uint8_t     phys_port_cnt;
  uint32_t                comp_mask;
  struct ibv_exp_device_calc_cap calc_cap;
  uint64_t    timestamp_mask;
  uint64_t    hca_core_clock;
  uint64_t    exp_device_cap_flags; /* use ibv_exp_device_cap_flags */
  int     max_dc_req_rd_atom;
  int     max_dc_res_rd_atom;
  int     inline_recv_sz;
  uint32_t    max_rss_tbl_sz;
  struct ibv_exp_ext_atomics_params ext_atom;
  uint32_t                max_mkey_klm_list_size;
  uint32_t                max_send_wqe_inline_klms;
  uint32_t                max_umr_recursion_depth;
  uint32_t                max_umr_stride_dimension;
};

ibv_exp_create_qp

Template: ibv_exp_create_qp(struct ibv_context *context, struct ibv_exp_qp_init_attr *qp_init_attr)

Input Parameters:

Output Parameters:

Return Value: Returns a pointer to the created QP, or NULL if the request fails. Check the QP number (qp_num) in the returned QP.

Description: ibv_exp_create_qp creates a queue pair (QP) associated with the protection domain pd. The argument init_attr is an ibv_exp_qp_init_attr struct, as defined in <infiniband/verbs_exp.h>.

struct ibv_exp_qp_init_attr {
  void           *qp_context;
  struct ibv_cq        *send_cq;
  struct ibv_cq        *recv_cq;
  struct ibv_srq         *srq;
  struct ibv_qp_cap cap;
  enum ibv_qp_type  qp_type;
  int     sq_sig_all;

  uint32_t    comp_mask; /* use ibv_exp_qp_init_attr_comp_mask */
  struct ibv_pd        *pd;
  struct ibv_xrcd        *xrcd;
  uint32_t    exp_create_flags; /* use ibv_exp_qp_create_flags */

  uint32_t    max_inl_recv;
  struct ibv_exp_qpg  qpg;
  uint32_t    max_atomic_arg;
  uint32_t                max_inl_send_klms;
};

ibv_exp_post_send

Template: static inline int ibv_exp_post_send(struct ibv_qp *qp, struct ibv_exp_send_wr *wr, struct ibv_- exp_send_wr **bad_wr)

Input Parameters:

Output Parameters:

Return Value: returns 0 on success, or the value of errno on failure (which indicates the failure reason).

Description: ibv_exp_post_send posts the linked list of work requests (WRs) starting with wr to the send queue of the queue pair qp. It stops processing WRs from this list at the first failure (that can be detected immediately while requests are being posted), and returns this failing WR through bad_wr.

struct ibv_exp_send_wr {
  uint64_t    wr_id;
  struct ibv_exp_send_wr *next;
  struct ibv_sge         *sg_list;
  int     num_sge;
  enum ibv_exp_wr_opcode  exp_opcode; /* use ibv_exp_wr_opcode */
  int     reserved; /* place holder to align with ibv_send_wr */
  union {
    uint32_t  imm_data; /* in network byte order */
    uint32_t  invalidate_rkey;
  } ex;
  union {
    struct {
      uint64_t  remote_addr;
      uint32_t  rkey;
    } rdma;
    struct {
      uint64_t  remote_addr;
      uint64_t  compare_add;
      uint64_t  swap;
      uint32_t  rkey;
    } atomic;
    struct {
      struct ibv_ah  *ah;
      uint32_t  remote_qpn;
      uint32_t  remote_qkey;
    } ud;
  } wr;
  union {
    union {
      struct {
        uint32_t    remote_srqn;
      } xrc;
    } qp_type;

    uint32_t    xrc_remote_srq_num;
  };
  union {
    struct {
      uint64_t    remote_addr;
      uint32_t    rkey;
    } rdma;
    struct {
      uint64_t    remote_addr;
      uint64_t    compare_add;
      uint64_t    swap;
      uint32_t    rkey;
    } atomic;
    struct {
      struct ibv_cq *cq;
      int32_t  cq_count;
    } cqe_wait;
    struct {
      struct ibv_qp *qp;
      int32_t  wqe_count;
    } wqe_enable;
  } task;
  union {
    struct {
      enum ibv_exp_calc_op        calc_op;
      enum ibv_exp_calc_data_type data_type;
      enum ibv_exp_calc_data_size data_size;
    } calc;
  } op;
  struct {
    struct ibv_ah   *ah;
    uint64_t        dct_access_key;
    uint32_t        dct_number;
  } dc;
  struct {
    struct ibv_mw     *mw;
    uint32_t      rkey;
    struct ibv_exp_mw_bind_info bind_info;
  } bind_mw;
  uint64_t  exp_send_flags; /* use ibv_exp_send_flags */
  uint32_t  comp_mask; /* reserved for future growth (must be 0) */
  union {
    struct {
      struct {
        enum mem_layout_type mkey_type;
        union {
          struct ibv_exp_mem_region *mem_reg_list; /* array, size corresponds to wr->num_sge */
          struct {
            struct ibv_exp_mem_repeat_block *mem_repeat_block_list; /* array,  size corresponds to wr->num_sge */
            size_t *repeat_count; /* array size corresponds to ndim */
            uint32_t ndim;
          } rb;
        } mem_list;
        struct non_inline_data *memory_objects; /* used when IBV_EXP_SEND_INLINE is not set */
        int access;
        struct ibv_mr *modified_mr;
        void *region_base_addr;
      } memory_key;
    } umr;
    struct {
      uint32_t        log_arg_sz;
      uint64_t  remote_addr;
      uint32_t  rkey;
      union {
        struct {
          /* For the next four fields:
          * If operand_size <= 8 then inline data is immediate
           * from the corresponding field; for small opernands,
           * ls bits are used.
           * Else the fields are pointers in the process's address space
           * where arguments are stored
           */
          union {
            struct ibv_exp_cmp_swap cmp_swap;
            struct ibv_exp_fetch_add fetch_add;
          } op;
        } inline_data;       /* IBV_EXP_SEND_EXT_ATOMIC_INLINE is set */
        /* in the future add support for non-inline argument provisioning */
      } wr_data;
    } masked_atomics;
  } ext_op;
};

For atomic operations, to support atomic responses in big-endian format (the only way to use atomics on Connect-IB® on little-endian machines) is:

  1. Use experimental verbs.
  2. Check to see if the atomics capabilities flag IBV_EXP_ATOMIC_HCA_REPLY_BE in the exp_atomic_cap field of the struct ibv_exp_device_attr returned by ibv_exp_query_device().
  3. Set the flag IBV_EXP_QP_CREATE_ATOMIC_BE_REPLY when opening the QP. This is what enables the use of atomic ops on Connect-IB.
  4. Use the experimental post send verb.