drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */
   3
   4 #include "rqt.h"
   5 #include <linux/mlx5/transobj.h>
   6
   7 static bool verify_num_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids,
   8                                 unsigned int size)
   9 {
  10         unsigned int max_num_vhca_id = MLX5_CAP_GEN_2(mdev, max_rqt_vhca_id);
  11         int i;
  12
  13         /* Verify that all vhca_ids are in range [0, max_num_vhca_ids - 1] */
  14         for (i = 0; i < size; i++)
  15                 if (vhca_ids[i] >= max_num_vhca_id)
  16                         return false;
  17         return true;
  18 }
  19
  20 static bool rqt_verify_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids,
  21                                 unsigned int size)
  22 {
  23         if (!vhca_ids)
  24                 return true;
  25
  26         if (!MLX5_CAP_GEN(mdev, cross_vhca_rqt))
  27                 return false;
  28         if (!verify_num_vhca_ids(mdev, vhca_ids, size))
  29                 return false;
  30
  31         return true;
  32 }
  33
  34 void mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir *indir,
  35                                          unsigned int num_channels)
  36 {
  37         unsigned int i;
  38
  39         for (i = 0; i < indir->actual_table_size; i++)
  40                 indir->table[i] = i % num_channels;
  41 }
  42
  43 static void fill_rqn_list(void *rqtc, u32 *rqns, u32 *vhca_ids, unsigned int size)
  44 {
  45         unsigned int i;
  46
  47         if (vhca_ids) {
  48                 MLX5_SET(rqtc, rqtc, rq_vhca_id_format, 1);
  49                 for (i = 0; i < size; i++) {
  50                         MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_num, rqns[i]);
  51                         MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_vhca_id, vhca_ids[i]);
  52                 }
  53         } else {
  54                 for (i = 0; i < size; i++)
  55                         MLX5_SET(rqtc, rqtc, rq_num[i], rqns[i]);
  56         }
  57 }
  58 static int mlx5e_rqt_init(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,
  59                           u16 max_size, u32 *init_rqns, u32 *init_vhca_ids, u16 init_size)
  60 {
  61         int entry_sz;
  62         void *rqtc;
  63         int inlen;
  64         int err;
  65         u32 *in;
  66
  67         if (!rqt_verify_vhca_ids(mdev, init_vhca_ids, init_size))
  68                 return -EOPNOTSUPP;
  69
  70         rqt->mdev = mdev;
  71         rqt->size = max_size;
  72
  73         entry_sz = init_vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num);
  74         inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + entry_sz * init_size;
  75         in = kvzalloc(inlen, GFP_KERNEL);
  76         if (!in)
  77                 return -ENOMEM;
  78
  79         rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
  80
  81         MLX5_SET(rqtc, rqtc, rqt_max_size, rqt->size);
  82         MLX5_SET(rqtc, rqtc, rqt_actual_size, init_size);
  83
  84         fill_rqn_list(rqtc, init_rqns, init_vhca_ids, init_size);
  85
  86         err = mlx5_core_create_rqt(rqt->mdev, in, inlen, &rqt->rqtn);
  87
  88         kvfree(in);
  89         return err;
  90 }
  91
  92 int mlx5e_rqt_init_direct(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,
  93                           bool indir_enabled, u32 init_rqn, u32 indir_table_size)
  94 {
  95         u16 max_size = indir_enabled ? indir_table_size : 1;
  96
  97         return mlx5e_rqt_init(rqt, mdev, max_size, &init_rqn, NULL, 1);
  98 }
  99
 100 static int mlx5e_bits_invert(unsigned long a, int size)
 101 {
 102         int inv = 0;
 103         int i;
 104
 105         for (i = 0; i < size; i++)
 106                 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
 107
 108         return inv;
 109 }
 110
 111 static int mlx5e_calc_indir_rqns(u32 *rss_rqns, u32 *rqns, u32 *rss_vhca_ids, u32 *vhca_ids,
 112                                  unsigned int num_rqns,
 113                                  u8 hfunc, struct mlx5e_rss_params_indir *indir)
 114 {
 115         unsigned int i;
 116
 117         for (i = 0; i < indir->actual_table_size; i++) {
 118                 unsigned int ix = i;
 119
 120                 if (hfunc == ETH_RSS_HASH_XOR)
 121                         ix = mlx5e_bits_invert(ix, ilog2(indir->actual_table_size));
 122
 123                 ix = indir->table[ix];
 124
 125                 if (WARN_ON(ix >= num_rqns))
 126                         /* Could be a bug in the driver or in the kernel part of
 127                          * ethtool: indir table refers to non-existent RQs.
 128                          */
 129                         return -EINVAL;
 130                 rss_rqns[i] = rqns[ix];
 131                 if (vhca_ids)
 132                         rss_vhca_ids[i] = vhca_ids[ix];
 133         }
 134
 135         return 0;
 136 }
 137
 138 int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,
 139                          u32 *rqns, u32 *vhca_ids, unsigned int num_rqns,
 140                          u8 hfunc, struct mlx5e_rss_params_indir *indir)
 141 {
 142         u32 *rss_rqns, *rss_vhca_ids = NULL;
 143         int err;
 144
 145         rss_rqns = kvmalloc_array(indir->actual_table_size, sizeof(*rss_rqns), GFP_KERNEL);
 146         if (!rss_rqns)
 147                 return -ENOMEM;
 148
 149         if (vhca_ids) {
 150                 rss_vhca_ids = kvmalloc_array(indir->actual_table_size, sizeof(*rss_vhca_ids),
 151                                               GFP_KERNEL);
 152                 if (!rss_vhca_ids) {
 153                         kvfree(rss_rqns);
 154                         return -ENOMEM;
 155                 }
 156         }
 157
 158         err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir);
 159         if (err)
 160                 goto out;
 161
 162         err = mlx5e_rqt_init(rqt, mdev, indir->max_table_size, rss_rqns, rss_vhca_ids,
 163                              indir->actual_table_size);
 164
 165 out:
 166         kvfree(rss_vhca_ids);
 167         kvfree(rss_rqns);
 168         return err;
 169 }
 170
 171 #define MLX5E_UNIFORM_SPREAD_RQT_FACTOR 2
 172
 173 u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels)
 174 {
 175         u32 rqt_size = max_t(u32, MLX5E_INDIR_MIN_RQT_SIZE,
 176                              roundup_pow_of_two(num_channels * MLX5E_UNIFORM_SPREAD_RQT_FACTOR));
 177         u32 max_cap_rqt_size = 1 << MLX5_CAP_GEN(mdev, log_max_rqt_size);
 178
 179         return min_t(u32, rqt_size, max_cap_rqt_size);
 180 }
 181
 182 #define MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH 256
 183
 184 unsigned int mlx5e_rqt_max_num_channels_allowed_for_xor8(void)
 185 {
 186         return MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH / MLX5E_UNIFORM_SPREAD_RQT_FACTOR;
 187 }
 188
 189 void mlx5e_rqt_destroy(struct mlx5e_rqt *rqt)
 190 {
 191         mlx5_core_destroy_rqt(rqt->mdev, rqt->rqtn);
 192 }
 193
 194 static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids,
 195                               unsigned int size)
 196 {
 197         int entry_sz;
 198         void *rqtc;
 199         int inlen;
 200         u32 *in;
 201         int err;
 202
 203         if (!rqt_verify_vhca_ids(rqt->mdev, vhca_ids, size))
 204                 return -EINVAL;
 205
 206         entry_sz = vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num);
 207         inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + entry_sz * size;
 208         in = kvzalloc(inlen, GFP_KERNEL);
 209         if (!in)
 210                 return -ENOMEM;
 211
 212         rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
 213
 214         MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
 215         MLX5_SET(rqtc, rqtc, rqt_actual_size, size);
 216
 217         fill_rqn_list(rqtc, rqns, vhca_ids, size);
 218
 219         err = mlx5_core_modify_rqt(rqt->mdev, rqt->rqtn, in, inlen);
 220
 221         kvfree(in);
 222         return err;
 223 }
 224
 225 int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn, u32 *vhca_id)
 226 {
 227         return mlx5e_rqt_redirect(rqt, &rqn, vhca_id, 1);
 228 }
 229
 230 int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids,
 231                              unsigned int num_rqns,
 232                              u8 hfunc, struct mlx5e_rss_params_indir *indir)
 233 {
 234         u32 *rss_rqns, *rss_vhca_ids = NULL;
 235         int err;
 236
 237         if (!rqt_verify_vhca_ids(rqt->mdev, vhca_ids, num_rqns))
 238                 return -EINVAL;
 239
 240         if (WARN_ON(rqt->size != indir->max_table_size))
 241                 return -EINVAL;
 242
 243         rss_rqns = kvmalloc_array(indir->actual_table_size, sizeof(*rss_rqns), GFP_KERNEL);
 244         if (!rss_rqns)
 245                 return -ENOMEM;
 246
 247         if (vhca_ids) {
 248                 rss_vhca_ids = kvmalloc_array(indir->actual_table_size, sizeof(*rss_vhca_ids),
 249                                               GFP_KERNEL);
 250                 if (!rss_vhca_ids) {
 251                         kvfree(rss_rqns);
 252                         return -ENOMEM;
 253                 }
 254         }
 255
 256         err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir);
 257         if (err)
 258                 goto out;
 259
 260         err = mlx5e_rqt_redirect(rqt, rss_rqns, rss_vhca_ids, indir->actual_table_size);
 261
 262 out:
 263         kvfree(rss_vhca_ids);
 264         kvfree(rss_rqns);
 265         return err;
 266 }