1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2 /* Copyright (c) 2021, Microsoft Corporation. */
10 #include "hw_channel.h"
12 /* Microsoft Azure Network Adapter (MANA)'s definitions
14 * Structures labeled with "HW DATA" are exchanged with the hardware. All of
15 * them are naturally aligned and hence don't need __packed.
18 /* MANA protocol version */
19 #define MANA_MAJOR_VERSION 0
20 #define MANA_MINOR_VERSION 1
21 #define MANA_MICRO_VERSION 1
23 typedef u64 mana_handle_t;
24 #define INVALID_MANA_HANDLE ((mana_handle_t)-1)
27 TRI_STATE_UNKNOWN = -1,
32 /* Number of entries for hardware indirection table must be in power of 2 */
33 #define MANA_INDIRECT_TABLE_SIZE 64
34 #define MANA_INDIRECT_TABLE_MASK (MANA_INDIRECT_TABLE_SIZE - 1)
36 /* The Toeplitz hash key's length in bytes: should be multiple of 8 */
37 #define MANA_HASH_KEY_SIZE 40
39 #define COMP_ENTRY_SIZE 64
41 #define RX_BUFFERS_PER_QUEUE 512
42 #define MANA_RX_DATA_ALIGN 64
44 #define MAX_SEND_BUFFERS_PER_QUEUE 256
46 #define EQ_SIZE (8 * PAGE_SIZE)
47 #define LOG2_EQ_THROTTLE 3
49 #define MAX_PORTS_IN_MANA_DEV 256
51 /* Update this count whenever the respective structures are changed */
52 #define MANA_STATS_RX_COUNT 5
53 #define MANA_STATS_TX_COUNT 11
55 struct mana_stats_rx {
61 struct u64_stats_sync syncp;
64 struct mana_stats_tx {
70 u64 tso_inner_packets;
76 struct u64_stats_sync syncp;
80 struct gdma_queue *gdma_sq;
93 struct net_device *ndev;
95 /* The SKBs are sent to the HW and we are waiting for the CQEs. */
96 struct sk_buff_head pending_skbs;
97 struct netdev_queue *net_txq;
99 atomic_t pending_sends;
101 struct mana_stats_tx stats;
104 /* skb data and frags dma mappings */
105 struct mana_skb_head {
106 dma_addr_t dma_handle[MAX_SKB_FRAGS + 1];
108 u32 size[MAX_SKB_FRAGS + 1];
111 #define MANA_HEADROOM sizeof(struct mana_skb_head)
113 enum mana_tx_pkt_format {
114 MANA_SHORT_PKT_FMT = 0,
115 MANA_LONG_PKT_FMT = 1,
118 struct mana_tx_short_oob {
120 u32 is_outer_ipv4 : 1;
121 u32 is_outer_ipv6 : 1;
122 u32 comp_iphdr_csum : 1;
123 u32 comp_tcp_csum : 1;
124 u32 comp_udp_csum : 1;
125 u32 supress_txcqe_gen : 1;
128 u32 trans_off : 10; /* Transport header offset */
130 u32 short_vp_offset : 8;
133 struct mana_tx_long_oob {
135 u32 inner_is_ipv6 : 1;
136 u32 inner_tcp_opt : 1;
137 u32 inject_vlan_pri_tag : 1;
139 u32 pcp : 3; /* 802.1Q */
140 u32 dei : 1; /* 802.1Q */
141 u32 vlan_id : 12; /* 802.1Q */
143 u32 inner_frame_offset : 10;
144 u32 inner_ip_rel_offset : 6;
145 u32 long_vp_offset : 12;
153 struct mana_tx_short_oob s_oob;
154 struct mana_tx_long_oob l_oob;
165 CQE_RX_COALESCED_4 = 2,
166 CQE_RX_OBJECT_FENCE = 3,
167 CQE_RX_TRUNCATED = 4,
171 CQE_TX_MTU_DROP = 34,
172 CQE_TX_INVALID_OOB = 35,
173 CQE_TX_INVALID_ETH_TYPE = 36,
174 CQE_TX_HDR_PROCESSING_ERROR = 37,
175 CQE_TX_VF_DISABLED = 38,
176 CQE_TX_VPORT_IDX_OUT_OF_RANGE = 39,
177 CQE_TX_VPORT_DISABLED = 40,
178 CQE_TX_VLAN_TAGGING_VIOLATION = 41,
181 #define MANA_CQE_COMPLETION 1
183 struct mana_cqe_header {
189 /* NDIS HASH Types */
190 #define NDIS_HASH_IPV4 BIT(0)
191 #define NDIS_HASH_TCP_IPV4 BIT(1)
192 #define NDIS_HASH_UDP_IPV4 BIT(2)
193 #define NDIS_HASH_IPV6 BIT(3)
194 #define NDIS_HASH_TCP_IPV6 BIT(4)
195 #define NDIS_HASH_UDP_IPV6 BIT(5)
196 #define NDIS_HASH_IPV6_EX BIT(6)
197 #define NDIS_HASH_TCP_IPV6_EX BIT(7)
198 #define NDIS_HASH_UDP_IPV6_EX BIT(8)
200 #define MANA_HASH_L3 (NDIS_HASH_IPV4 | NDIS_HASH_IPV6 | NDIS_HASH_IPV6_EX)
201 #define MANA_HASH_L4 \
202 (NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4 | NDIS_HASH_TCP_IPV6 | \
203 NDIS_HASH_UDP_IPV6 | NDIS_HASH_TCP_IPV6_EX | NDIS_HASH_UDP_IPV6_EX)
205 struct mana_rxcomp_perpkt_info {
212 #define MANA_RXCOMP_OOB_NUM_PPI 4
214 /* Receive completion OOB */
215 struct mana_rxcomp_oob {
216 struct mana_cqe_header cqe_hdr;
219 u32 rx_vlantag_present : 1;
220 u32 rx_outer_iphdr_csum_succeed : 1;
221 u32 rx_outer_iphdr_csum_fail : 1;
224 u32 rx_iphdr_csum_succeed : 1;
225 u32 rx_iphdr_csum_fail : 1;
226 u32 rx_tcp_csum_succeed : 1;
227 u32 rx_tcp_csum_fail : 1;
228 u32 rx_udp_csum_succeed : 1;
229 u32 rx_udp_csum_fail : 1;
232 struct mana_rxcomp_perpkt_info ppi[MANA_RXCOMP_OOB_NUM_PPI];
237 struct mana_tx_comp_oob {
238 struct mana_cqe_header cqe_hdr;
242 u32 tx_sgl_offset : 5;
243 u32 tx_wqe_offset : 27;
250 #define CQE_POLLING_BUFFER 512
253 struct gdma_queue *gdma_cq;
255 /* Cache the CQ id (used to verify if each CQE comes to the right CQ. */
258 /* Type of the CQ: TX or RX */
259 enum mana_cq_type type;
261 /* Pointer to the mana_rxq that is pushing RX CQEs to the queue.
262 * Only and must be non-NULL if type is MANA_CQ_TYPE_RX.
264 struct mana_rxq *rxq;
266 /* Pointer to the mana_txq that is pushing TX CQEs to the queue.
267 * Only and must be non-NULL if type is MANA_CQ_TYPE_TX.
269 struct mana_txq *txq;
271 /* Buffer which the CQ handler can copy the CQE's into. */
272 struct gdma_comp gdma_comp_buf[CQE_POLLING_BUFFER];
275 struct napi_struct napi;
280 struct mana_recv_buf_oob {
281 /* A valid GDMA work request representing the data buffer. */
282 struct gdma_wqe_request wqe_req;
285 bool from_pool; /* allocated from a page pool */
287 /* SGL of the buffer going to be sent has part of the work request. */
289 struct gdma_sge sgl[MAX_RX_WQE_SGL_ENTRIES];
291 /* Required to store the result of mana_gd_post_work_request.
292 * gdma_posted_wqe_info.wqe_size_in_bu is required for progressing the
293 * work queue when the WQE is consumed.
295 struct gdma_posted_wqe_info wqe_inf;
298 #define MANA_RXBUF_PAD (SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) \
301 #define MANA_XDP_MTU_MAX (PAGE_SIZE - MANA_RXBUF_PAD - XDP_PACKET_HEADROOM)
304 struct gdma_queue *gdma_rq;
305 /* Cache the gdma receive queue id */
308 /* Index of RQ in the vPort, not gdma receive queue id */
317 struct mana_cq rx_cq;
319 struct completion fence_event;
321 struct net_device *ndev;
323 /* Total number of receive buffers to be allocated */
328 struct mana_stats_rx stats;
330 struct bpf_prog __rcu *bpf_prog;
331 struct xdp_rxq_info xdp_rxq;
332 void *xdp_save_va; /* for reusing */
334 int xdp_rc; /* XDP redirect return code */
336 struct page_pool *page_pool;
338 /* MUST BE THE LAST MEMBER:
339 * Each receive buffer has an associated mana_recv_buf_oob.
341 struct mana_recv_buf_oob rx_oobs[];
347 struct mana_cq tx_cq;
349 mana_handle_t tx_object;
352 struct mana_ethtool_stats {
356 u64 tx_cqe_unknown_type;
357 u64 rx_coalesced_err;
358 u64 rx_cqe_unknown_type;
361 struct mana_context {
362 struct gdma_dev *gdma_dev;
368 struct net_device *ports[MAX_PORTS_IN_MANA_DEV];
371 struct mana_port_context {
372 struct mana_context *ac;
373 struct net_device *ndev;
375 u8 mac_addr[ETH_ALEN];
377 enum TRI_STATE rss_state;
379 mana_handle_t default_rxobj;
380 bool tx_shortform_allowed;
383 struct mana_tx_qp *tx_qp;
385 /* Indirection Table for RX & TX. The values are queue indexes */
386 u32 indir_table[MANA_INDIRECT_TABLE_SIZE];
388 /* Indirection table containing RxObject Handles */
389 mana_handle_t rxobj_table[MANA_INDIRECT_TABLE_SIZE];
391 /* Hash key used by the NIC */
392 u8 hashkey[MANA_HASH_KEY_SIZE];
394 /* This points to an array of num_queues of RQ pointers. */
395 struct mana_rxq **rxqs;
397 /* pre-allocated rx buffer array */
402 u32 rxbpre_alloc_size;
405 struct bpf_prog *bpf_prog;
407 /* Create num_queues EQs, SQs, SQ-CQs, RQs and RQ-CQs, respectively. */
408 unsigned int max_queues;
409 unsigned int num_queues;
411 mana_handle_t port_handle;
412 mana_handle_t pf_filter_handle;
414 /* Mutex for sharing access to vport_use_count */
415 struct mutex vport_mutex;
421 bool port_st_save; /* Saved port state */
423 struct mana_ethtool_stats eth_stats;
426 netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev);
427 int mana_config_rss(struct mana_port_context *ac, enum TRI_STATE rx,
428 bool update_hash, bool update_tab);
430 int mana_alloc_queues(struct net_device *ndev);
431 int mana_attach(struct net_device *ndev);
432 int mana_detach(struct net_device *ndev, bool from_close);
434 int mana_probe(struct gdma_dev *gd, bool resuming);
435 void mana_remove(struct gdma_dev *gd, bool suspending);
437 void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev);
438 int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frames,
440 u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq,
441 struct xdp_buff *xdp, void *buf_va, uint pkt_len);
442 struct bpf_prog *mana_xdp_get(struct mana_port_context *apc);
443 void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog);
444 int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf);
446 extern const struct ethtool_ops mana_ethtool_ops;
448 /* A CQ can be created not associated with any EQ */
449 #define GDMA_CQ_NO_EQ 0xffff
451 struct mana_obj_spec {
459 enum mana_command_code {
460 MANA_QUERY_DEV_CONFIG = 0x20001,
461 MANA_QUERY_GF_STAT = 0x20002,
462 MANA_CONFIG_VPORT_TX = 0x20003,
463 MANA_CREATE_WQ_OBJ = 0x20004,
464 MANA_DESTROY_WQ_OBJ = 0x20005,
465 MANA_FENCE_RQ = 0x20006,
466 MANA_CONFIG_VPORT_RX = 0x20007,
467 MANA_QUERY_VPORT_CONFIG = 0x20008,
469 /* Privileged commands for the PF mode */
470 MANA_REGISTER_FILTER = 0x28000,
471 MANA_DEREGISTER_FILTER = 0x28001,
472 MANA_REGISTER_HW_PORT = 0x28003,
473 MANA_DEREGISTER_HW_PORT = 0x28004,
476 /* Query Device Configuration */
477 struct mana_query_device_cfg_req {
478 struct gdma_req_hdr hdr;
480 /* MANA Nic Driver Capability flags */
481 u64 mn_drv_cap_flags1;
482 u64 mn_drv_cap_flags2;
483 u64 mn_drv_cap_flags3;
484 u64 mn_drv_cap_flags4;
493 struct mana_query_device_cfg_resp {
494 struct gdma_resp_hdr hdr;
511 /* Query vPort Configuration */
512 struct mana_query_vport_cfg_req {
513 struct gdma_req_hdr hdr;
517 struct mana_query_vport_cfg_resp {
518 struct gdma_resp_hdr hdr;
521 u32 num_indirection_ent;
528 /* Configure vPort */
529 struct mana_config_vport_req {
530 struct gdma_req_hdr hdr;
536 struct mana_config_vport_resp {
537 struct gdma_resp_hdr hdr;
539 u8 short_form_allowed;
543 /* Create WQ Object */
544 struct mana_create_wqobj_req {
545 struct gdma_req_hdr hdr;
553 u32 cq_moderation_ctx_id;
557 struct mana_create_wqobj_resp {
558 struct gdma_resp_hdr hdr;
561 mana_handle_t wq_obj;
564 /* Destroy WQ Object */
565 struct mana_destroy_wqobj_req {
566 struct gdma_req_hdr hdr;
569 mana_handle_t wq_obj_handle;
572 struct mana_destroy_wqobj_resp {
573 struct gdma_resp_hdr hdr;
577 struct mana_fence_rq_req {
578 struct gdma_req_hdr hdr;
579 mana_handle_t wq_obj_handle;
582 struct mana_fence_rq_resp {
583 struct gdma_resp_hdr hdr;
586 /* Configure vPort Rx Steering */
587 struct mana_cfg_rx_steer_req_v2 {
588 struct gdma_req_hdr hdr;
590 u16 num_indir_entries;
591 u16 indir_tab_offset;
594 u8 update_default_rxobj;
598 mana_handle_t default_rxobj;
599 u8 hashkey[MANA_HASH_KEY_SIZE];
600 u8 cqe_coalescing_enable;
604 struct mana_cfg_rx_steer_resp {
605 struct gdma_resp_hdr hdr;
608 /* Register HW vPort */
609 struct mana_register_hw_vport_req {
610 struct gdma_req_hdr hdr;
612 u8 is_pf_default_vport;
614 u8 allow_all_ether_types;
620 struct mana_register_hw_vport_resp {
621 struct gdma_resp_hdr hdr;
622 mana_handle_t hw_vport_handle;
625 /* Deregister HW vPort */
626 struct mana_deregister_hw_vport_req {
627 struct gdma_req_hdr hdr;
628 mana_handle_t hw_vport_handle;
631 struct mana_deregister_hw_vport_resp {
632 struct gdma_resp_hdr hdr;
635 /* Register filter */
636 struct mana_register_filter_req {
637 struct gdma_req_hdr hdr;
650 struct mana_register_filter_resp {
651 struct gdma_resp_hdr hdr;
652 mana_handle_t filter_handle;
655 /* Deregister filter */
656 struct mana_deregister_filter_req {
657 struct gdma_req_hdr hdr;
658 mana_handle_t filter_handle;
661 struct mana_deregister_filter_resp {
662 struct gdma_resp_hdr hdr;
665 #define MANA_MAX_NUM_QUEUES 64
667 #define MANA_SHORT_VPORT_OFFSET_MAX ((1U << 8) - 1)
669 struct mana_tx_package {
670 struct gdma_wqe_request wqe_req;
671 struct gdma_sge sgl_array[5];
672 struct gdma_sge *sgl_ptr;
674 struct mana_tx_oob tx_oob;
676 struct gdma_posted_wqe_info wqe_info;
679 int mana_create_wq_obj(struct mana_port_context *apc,
681 u32 wq_type, struct mana_obj_spec *wq_spec,
682 struct mana_obj_spec *cq_spec,
683 mana_handle_t *wq_obj);
685 void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type,
686 mana_handle_t wq_obj);
688 int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id,
690 void mana_uncfg_vport(struct mana_port_context *apc);