drivers/net/ethernet/amazon/ena/ena_xdp.h

   1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
   2 /*
   3  * Copyright 2015-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
   4  */
   5
   6 #ifndef ENA_XDP_H
   7 #define ENA_XDP_H
   8
   9 #include "ena_netdev.h"
  10 #include <linux/bpf_trace.h>
  11
  12 /* The max MTU size is configured to be the ethernet frame size without
  13  * the overhead of the ethernet header, which can have a VLAN header, and
  14  * a frame check sequence (FCS).
  15  * The buffer size we share with the device is defined to be ENA_PAGE_SIZE
  16  */
  17 #define ENA_XDP_MAX_MTU (ENA_PAGE_SIZE - ETH_HLEN - ETH_FCS_LEN -       \
  18                          VLAN_HLEN - XDP_PACKET_HEADROOM -              \
  19                          SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
  20
  21 #define ENA_IS_XDP_INDEX(adapter, index) (((index) >= (adapter)->xdp_first_ring) && \
  22         ((index) < (adapter)->xdp_first_ring + (adapter)->xdp_num_queues))
  23
  24 enum ENA_XDP_ACTIONS {
  25         ENA_XDP_PASS            = 0,
  26         ENA_XDP_TX              = BIT(0),
  27         ENA_XDP_REDIRECT        = BIT(1),
  28         ENA_XDP_DROP            = BIT(2)
  29 };
  30
  31 #define ENA_XDP_FORWARDED (ENA_XDP_TX | ENA_XDP_REDIRECT)
  32
  33 int ena_setup_and_create_all_xdp_queues(struct ena_adapter *adapter);
  34 void ena_xdp_exchange_program_rx_in_range(struct ena_adapter *adapter,
  35                                           struct bpf_prog *prog,
  36                                           int first, int count);
  37 int ena_xdp_io_poll(struct napi_struct *napi, int budget);
  38 int ena_xdp_xmit_frame(struct ena_ring *tx_ring,
  39                        struct ena_adapter *adapter,
  40                        struct xdp_frame *xdpf,
  41                        int flags);
  42 int ena_xdp_xmit(struct net_device *dev, int n,
  43                  struct xdp_frame **frames, u32 flags);
  44 int ena_xdp(struct net_device *netdev, struct netdev_bpf *bpf);
  45 int ena_xdp_register_rxq_info(struct ena_ring *rx_ring);
  46 void ena_xdp_unregister_rxq_info(struct ena_ring *rx_ring);
  47
  48 enum ena_xdp_errors_t {
  49         ENA_XDP_ALLOWED = 0,
  50         ENA_XDP_CURRENT_MTU_TOO_LARGE,
  51         ENA_XDP_NO_ENOUGH_QUEUES,
  52 };
  53
  54 static inline bool ena_xdp_present(struct ena_adapter *adapter)
  55 {
  56         return !!adapter->xdp_bpf_prog;
  57 }
  58
  59 static inline bool ena_xdp_present_ring(struct ena_ring *ring)
  60 {
  61         return !!ring->xdp_bpf_prog;
  62 }
  63
  64 static inline bool ena_xdp_legal_queue_count(struct ena_adapter *adapter,
  65                                              u32 queues)
  66 {
  67         return 2 * queues <= adapter->max_num_io_queues;
  68 }
  69
  70 static inline enum ena_xdp_errors_t ena_xdp_allowed(struct ena_adapter *adapter)
  71 {
  72         enum ena_xdp_errors_t rc = ENA_XDP_ALLOWED;
  73
  74         if (adapter->netdev->mtu > ENA_XDP_MAX_MTU)
  75                 rc = ENA_XDP_CURRENT_MTU_TOO_LARGE;
  76         else if (!ena_xdp_legal_queue_count(adapter, adapter->num_io_queues))
  77                 rc = ENA_XDP_NO_ENOUGH_QUEUES;
  78
  79         return rc;
  80 }
  81
  82 static inline int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp)
  83 {
  84         u32 verdict = ENA_XDP_PASS;
  85         struct bpf_prog *xdp_prog;
  86         struct ena_ring *xdp_ring;
  87         struct xdp_frame *xdpf;
  88         u64 *xdp_stat;
  89
  90         xdp_prog = READ_ONCE(rx_ring->xdp_bpf_prog);
  91
  92         verdict = bpf_prog_run_xdp(xdp_prog, xdp);
  93
  94         switch (verdict) {
  95         case XDP_TX:
  96                 xdpf = xdp_convert_buff_to_frame(xdp);
  97                 if (unlikely(!xdpf)) {
  98                         trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict);
  99                         xdp_stat = &rx_ring->rx_stats.xdp_aborted;
 100                         verdict = ENA_XDP_DROP;
 101                         break;
 102                 }
 103
 104                 /* Find xmit queue */
 105                 xdp_ring = rx_ring->xdp_ring;
 106
 107                 /* The XDP queues are shared between XDP_TX and XDP_REDIRECT */
 108                 spin_lock(&xdp_ring->xdp_tx_lock);
 109
 110                 if (ena_xdp_xmit_frame(xdp_ring, rx_ring->adapter, xdpf,
 111                                        XDP_XMIT_FLUSH))
 112                         xdp_return_frame(xdpf);
 113
 114                 spin_unlock(&xdp_ring->xdp_tx_lock);
 115                 xdp_stat = &rx_ring->rx_stats.xdp_tx;
 116                 verdict = ENA_XDP_TX;
 117                 break;
 118         case XDP_REDIRECT:
 119                 if (likely(!xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog))) {
 120                         xdp_stat = &rx_ring->rx_stats.xdp_redirect;
 121                         verdict = ENA_XDP_REDIRECT;
 122                         break;
 123                 }
 124                 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict);
 125                 xdp_stat = &rx_ring->rx_stats.xdp_aborted;
 126                 verdict = ENA_XDP_DROP;
 127                 break;
 128         case XDP_ABORTED:
 129                 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict);
 130                 xdp_stat = &rx_ring->rx_stats.xdp_aborted;
 131                 verdict = ENA_XDP_DROP;
 132                 break;
 133         case XDP_DROP:
 134                 xdp_stat = &rx_ring->rx_stats.xdp_drop;
 135                 verdict = ENA_XDP_DROP;
 136                 break;
 137         case XDP_PASS:
 138                 xdp_stat = &rx_ring->rx_stats.xdp_pass;
 139                 verdict = ENA_XDP_PASS;
 140                 break;
 141         default:
 142                 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, verdict);
 143                 xdp_stat = &rx_ring->rx_stats.xdp_invalid;
 144                 verdict = ENA_XDP_DROP;
 145         }
 146
 147         ena_increase_stat(xdp_stat, 1, &rx_ring->syncp);
 148
 149         return verdict;
 150 }
 151 #endif /* ENA_XDP_H */