1 /* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
2 /* Copyright (c) 2015 - 2020 Intel Corporation */
6 #define irdma_handle void *
7 #define irdma_adapter_handle irdma_handle
8 #define irdma_qp_handle irdma_handle
9 #define irdma_cq_handle irdma_handle
10 #define irdma_pd_id irdma_handle
11 #define irdma_stag_handle irdma_handle
12 #define irdma_stag_index u32
13 #define irdma_stag u32
14 #define irdma_stag_key u8
15 #define irdma_tagged_offset u64
16 #define irdma_access_privileges u32
17 #define irdma_physical_fragment u64
18 #define irdma_address_list u64 *
19 #define irdma_sgl struct irdma_sge *
21 #define IRDMA_MAX_MR_SIZE 0x200000000000ULL
23 #define IRDMA_ACCESS_FLAGS_LOCALREAD 0x01
24 #define IRDMA_ACCESS_FLAGS_LOCALWRITE 0x02
25 #define IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY 0x04
26 #define IRDMA_ACCESS_FLAGS_REMOTEREAD 0x05
27 #define IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY 0x08
28 #define IRDMA_ACCESS_FLAGS_REMOTEWRITE 0x0a
29 #define IRDMA_ACCESS_FLAGS_BIND_WINDOW 0x10
30 #define IRDMA_ACCESS_FLAGS_ZERO_BASED 0x20
31 #define IRDMA_ACCESS_FLAGS_ALL 0x3f
33 #define IRDMA_OP_TYPE_RDMA_WRITE 0x00
34 #define IRDMA_OP_TYPE_RDMA_READ 0x01
35 #define IRDMA_OP_TYPE_SEND 0x03
36 #define IRDMA_OP_TYPE_SEND_INV 0x04
37 #define IRDMA_OP_TYPE_SEND_SOL 0x05
38 #define IRDMA_OP_TYPE_SEND_SOL_INV 0x06
39 #define IRDMA_OP_TYPE_RDMA_WRITE_SOL 0x0d
40 #define IRDMA_OP_TYPE_BIND_MW 0x08
41 #define IRDMA_OP_TYPE_FAST_REG_NSMR 0x09
42 #define IRDMA_OP_TYPE_INV_STAG 0x0a
43 #define IRDMA_OP_TYPE_RDMA_READ_INV_STAG 0x0b
44 #define IRDMA_OP_TYPE_NOP 0x0c
45 #define IRDMA_OP_TYPE_REC 0x3e
46 #define IRDMA_OP_TYPE_REC_IMM 0x3f
48 #define IRDMA_FLUSH_MAJOR_ERR 1
50 enum irdma_device_caps_const {
52 IRDMA_CQP_WQE_SIZE = 8,
54 IRDMA_EXTENDED_CQE_SIZE = 8,
57 IRDMA_CQP_CTX_SIZE = 8,
58 IRDMA_SHADOW_AREA_SIZE = 8,
59 IRDMA_QUERY_FPM_BUF_SIZE = 176,
60 IRDMA_COMMIT_FPM_BUF_SIZE = 176,
61 IRDMA_GATHER_STATS_BUF_SIZE = 1024,
62 IRDMA_MIN_IW_QP_ID = 0,
63 IRDMA_MAX_IW_QP_ID = 262143,
65 IRDMA_MAX_CEQID = 1023,
66 IRDMA_CEQ_MAX_COUNT = IRDMA_MAX_CEQID + 1,
68 IRDMA_MAX_CQID = 524287,
69 IRDMA_MIN_AEQ_ENTRIES = 1,
70 IRDMA_MAX_AEQ_ENTRIES = 524287,
71 IRDMA_MIN_CEQ_ENTRIES = 1,
72 IRDMA_MAX_CEQ_ENTRIES = 262143,
73 IRDMA_MIN_CQ_SIZE = 1,
74 IRDMA_MAX_CQ_SIZE = 1048575,
76 IRDMA_MAX_WQ_FRAGMENT_COUNT = 13,
77 IRDMA_MAX_SGE_RD = 13,
78 IRDMA_MAX_OUTBOUND_MSG_SIZE = 2147483647,
79 IRDMA_MAX_INBOUND_MSG_SIZE = 2147483647,
80 IRDMA_MAX_PUSH_PAGE_COUNT = 1024,
81 IRDMA_MAX_PE_ENA_VF_COUNT = 32,
82 IRDMA_MAX_VF_FPM_ID = 47,
83 IRDMA_MAX_SQ_PAYLOAD_SIZE = 2145386496,
84 IRDMA_MAX_INLINE_DATA_SIZE = 101,
85 IRDMA_MAX_WQ_ENTRIES = 32768,
86 IRDMA_Q2_BUF_SIZE = 256,
87 IRDMA_QP_CTX_SIZE = 256,
88 IRDMA_MAX_PDS = 262144,
91 enum irdma_addressing_type {
92 IRDMA_ADDR_TYPE_ZERO_BASED = 0,
93 IRDMA_ADDR_TYPE_VA_BASED = 1,
96 enum irdma_flush_opcode {
100 FLUSH_REM_ACCESS_ERR,
107 enum irdma_cmpl_status {
108 IRDMA_COMPL_STATUS_SUCCESS = 0,
109 IRDMA_COMPL_STATUS_FLUSHED,
110 IRDMA_COMPL_STATUS_INVALID_WQE,
111 IRDMA_COMPL_STATUS_QP_CATASTROPHIC,
112 IRDMA_COMPL_STATUS_REMOTE_TERMINATION,
113 IRDMA_COMPL_STATUS_INVALID_STAG,
114 IRDMA_COMPL_STATUS_BASE_BOUND_VIOLATION,
115 IRDMA_COMPL_STATUS_ACCESS_VIOLATION,
116 IRDMA_COMPL_STATUS_INVALID_PD_ID,
117 IRDMA_COMPL_STATUS_WRAP_ERROR,
118 IRDMA_COMPL_STATUS_STAG_INVALID_PDID,
119 IRDMA_COMPL_STATUS_RDMA_READ_ZERO_ORD,
120 IRDMA_COMPL_STATUS_QP_NOT_PRIVLEDGED,
121 IRDMA_COMPL_STATUS_STAG_NOT_INVALID,
122 IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_SIZE,
123 IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_ENTRY,
124 IRDMA_COMPL_STATUS_INVALID_FBO,
125 IRDMA_COMPL_STATUS_INVALID_LEN,
126 IRDMA_COMPL_STATUS_INVALID_ACCESS,
127 IRDMA_COMPL_STATUS_PHYS_BUF_LIST_TOO_LONG,
128 IRDMA_COMPL_STATUS_INVALID_VIRT_ADDRESS,
129 IRDMA_COMPL_STATUS_INVALID_REGION,
130 IRDMA_COMPL_STATUS_INVALID_WINDOW,
131 IRDMA_COMPL_STATUS_INVALID_TOTAL_LEN,
132 IRDMA_COMPL_STATUS_UNKNOWN,
135 enum irdma_cmpl_notify {
136 IRDMA_CQ_COMPL_EVENT = 0,
137 IRDMA_CQ_COMPL_SOLICITED = 1,
141 IRDMA_WRITE_WITH_IMM = 1,
142 IRDMA_SEND_WITH_IMM = 2,
149 struct irdma_qp_uk_init_info;
150 struct irdma_cq_uk_init_info;
153 irdma_tagged_offset tag_off;
165 __le64 buf[IRDMA_CQE_SIZE];
168 struct irdma_extended_cqe {
169 __le64 buf[IRDMA_EXTENDED_CQE_SIZE];
172 struct irdma_post_send {
180 struct irdma_post_inline_send {
188 struct irdma_post_rq_info {
194 struct irdma_rdma_write {
195 irdma_sgl lo_sg_list;
197 struct irdma_sge rem_addr;
200 struct irdma_inline_rdma_write {
203 struct irdma_sge rem_addr;
206 struct irdma_rdma_read {
207 irdma_sgl lo_sg_list;
209 struct irdma_sge rem_addr;
212 struct irdma_bind_window {
216 enum irdma_addressing_type addressing_type;
220 bool mem_window_type_1:1;
223 struct irdma_inv_local_stag {
224 irdma_stag target_stag;
227 struct irdma_post_sq_info {
235 bool imm_data_valid:1;
243 struct irdma_post_send send;
244 struct irdma_rdma_write rdma_write;
245 struct irdma_rdma_read rdma_read;
246 struct irdma_bind_window bind_window;
247 struct irdma_inv_local_stag inv_local_stag;
248 struct irdma_inline_rdma_write inline_rdma_write;
249 struct irdma_post_inline_send inline_send;
253 struct irdma_cq_poll_info {
255 irdma_qp_handle qp_handle;
261 irdma_stag inv_stag; /* or L_R_Key */
262 enum irdma_cmpl_status comp_status;
268 bool stag_invalid_set:1; /* or L_R_Key set */
271 bool solicited_event:1;
273 bool ud_vlan_valid:1;
274 bool ud_smac_valid:1;
278 enum irdma_status_code irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
279 struct irdma_post_sq_info *info,
281 enum irdma_status_code irdma_uk_inline_send(struct irdma_qp_uk *qp,
282 struct irdma_post_sq_info *info,
284 enum irdma_status_code irdma_uk_mw_bind(struct irdma_qp_uk *qp,
285 struct irdma_post_sq_info *info,
287 enum irdma_status_code irdma_uk_post_nop(struct irdma_qp_uk *qp, u64 wr_id,
288 bool signaled, bool post_sq);
289 enum irdma_status_code irdma_uk_post_receive(struct irdma_qp_uk *qp,
290 struct irdma_post_rq_info *info);
291 void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp);
292 enum irdma_status_code irdma_uk_rdma_read(struct irdma_qp_uk *qp,
293 struct irdma_post_sq_info *info,
294 bool inv_stag, bool post_sq);
295 enum irdma_status_code irdma_uk_rdma_write(struct irdma_qp_uk *qp,
296 struct irdma_post_sq_info *info,
298 enum irdma_status_code irdma_uk_send(struct irdma_qp_uk *qp,
299 struct irdma_post_sq_info *info, bool post_sq);
300 enum irdma_status_code irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
301 struct irdma_post_sq_info *info,
304 struct irdma_wqe_uk_ops {
305 void (*iw_copy_inline_data)(u8 *dest, u8 *src, u32 len, u8 polarity);
306 u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
307 void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct irdma_sge *sge,
309 void (*iw_set_mw_bind_wqe)(__le64 *wqe,
310 struct irdma_bind_window *op_info);
313 enum irdma_status_code irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
314 struct irdma_cq_poll_info *info);
315 void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq,
316 enum irdma_cmpl_notify cq_notify);
317 void irdma_uk_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int size);
318 void irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk *qp, u16 cnt);
319 enum irdma_status_code irdma_uk_cq_init(struct irdma_cq_uk *cq,
320 struct irdma_cq_uk_init_info *info);
321 enum irdma_status_code irdma_uk_qp_init(struct irdma_qp_uk *qp,
322 struct irdma_qp_uk_init_info *info);
323 struct irdma_sq_uk_wr_trk_info {
330 struct irdma_qp_quanta {
331 __le64 elem[IRDMA_WQE_SIZE];
335 struct irdma_qp_quanta *sq_base;
336 struct irdma_qp_quanta *rq_base;
337 struct irdma_uk_attrs *uk_attrs;
338 u32 __iomem *wqe_alloc_db;
339 struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
344 struct irdma_ring sq_ring;
345 struct irdma_ring rq_ring;
346 struct irdma_ring initial_ring;
354 struct irdma_wqe_uk_ops wqe_ops;
358 u8 swqe_polarity_deferred;
361 u8 rq_wqe_size_multiplier;
362 bool deferred_flag:1;
363 bool push_mode:1; /* whether the last post wqe was pushed */
366 bool sq_flush_complete:1; /* Indicates flush was seen and SQ was empty after the flush */
367 bool rq_flush_complete:1; /* Indicates flush was seen and RQ was empty after the flush */
368 bool destroy_pending:1; /* Indicates the QP is being destroyed */
377 struct irdma_cqe *cq_base;
378 u32 __iomem *cqe_alloc_db;
379 u32 __iomem *cq_ack_db;
383 struct irdma_ring cq_ring;
385 bool avoid_mem_cflct:1;
388 struct irdma_qp_uk_init_info {
389 struct irdma_qp_quanta *sq;
390 struct irdma_qp_quanta *rq;
391 struct irdma_uk_attrs *uk_attrs;
392 u32 __iomem *wqe_alloc_db;
394 struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
409 struct irdma_cq_uk_init_info {
410 u32 __iomem *cqe_alloc_db;
411 u32 __iomem *cq_ack_db;
412 struct irdma_cqe *cq_base;
416 bool avoid_mem_cflct;
419 __le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
420 u16 quanta, u32 total_size,
421 struct irdma_post_sq_info *info);
422 __le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx);
423 void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq);
424 enum irdma_status_code irdma_nop(struct irdma_qp_uk *qp, u64 wr_id,
425 bool signaled, bool post_sq);
426 enum irdma_status_code irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta);
427 enum irdma_status_code irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size);
428 void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
429 u32 inline_data, u8 *shift);
430 enum irdma_status_code irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs,
431 u32 sq_size, u8 shift, u32 *wqdepth);
432 enum irdma_status_code irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs,
433 u32 rq_size, u8 shift, u32 *wqdepth);
434 void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta,
435 u32 wqe_idx, bool post_sq);
436 void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
437 #endif /* IRDMA_USER_H */