net/core/devlink.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * net/core/devlink.c - Network physical/parent device Netlink interface
   4  *
   5  * Heavily inspired by net/wireless/
   6  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   7  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/module.h>
  12 #include <linux/types.h>
  13 #include <linux/slab.h>
  14 #include <linux/gfp.h>
  15 #include <linux/device.h>
  16 #include <linux/list.h>
  17 #include <linux/netdevice.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/refcount.h>
  20 #include <linux/workqueue.h>
  21 #include <linux/u64_stats_sync.h>
  22 #include <linux/timekeeping.h>
  23 #include <rdma/ib_verbs.h>
  24 #include <net/netlink.h>
  25 #include <net/genetlink.h>
  26 #include <net/rtnetlink.h>
  27 #include <net/net_namespace.h>
  28 #include <net/sock.h>
  29 #include <net/devlink.h>
  30 #define CREATE_TRACE_POINTS
  31 #include <trace/events/devlink.h>
  32
  33 #define DEVLINK_RELOAD_STATS_ARRAY_SIZE \
  34         (__DEVLINK_RELOAD_LIMIT_MAX * __DEVLINK_RELOAD_ACTION_MAX)
  35
  36 struct devlink_dev_stats {
  37         u32 reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
  38         u32 remote_reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
  39 };
  40
  41 struct devlink {
  42         u32 index;
  43         struct list_head port_list;
  44         struct list_head rate_list;
  45         struct list_head sb_list;
  46         struct list_head dpipe_table_list;
  47         struct list_head resource_list;
  48         struct list_head param_list;
  49         struct list_head region_list;
  50         struct list_head reporter_list;
  51         struct mutex reporters_lock; /* protects reporter_list */
  52         struct devlink_dpipe_headers *dpipe_headers;
  53         struct list_head trap_list;
  54         struct list_head trap_group_list;
  55         struct list_head trap_policer_list;
  56         const struct devlink_ops *ops;
  57         u64 features;
  58         struct xarray snapshot_ids;
  59         struct devlink_dev_stats stats;
  60         struct device *dev;
  61         possible_net_t _net;
  62         /* Serializes access to devlink instance specific objects such as
  63          * port, sb, dpipe, resource, params, region, traps and more.
  64          */
  65         struct mutex lock;
  66         u8 reload_failed:1;
  67         refcount_t refcount;
  68         struct completion comp;
  69         char priv[] __aligned(NETDEV_ALIGN);
  70 };
  71
  72 void *devlink_priv(struct devlink *devlink)
  73 {
  74         return &devlink->priv;
  75 }
  76 EXPORT_SYMBOL_GPL(devlink_priv);
  77
  78 struct devlink *priv_to_devlink(void *priv)
  79 {
  80         return container_of(priv, struct devlink, priv);
  81 }
  82 EXPORT_SYMBOL_GPL(priv_to_devlink);
  83
  84 struct device *devlink_to_dev(const struct devlink *devlink)
  85 {
  86         return devlink->dev;
  87 }
  88 EXPORT_SYMBOL_GPL(devlink_to_dev);
  89
  90 static struct devlink_dpipe_field devlink_dpipe_fields_ethernet[] = {
  91         {
  92                 .name = "destination mac",
  93                 .id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC,
  94                 .bitwidth = 48,
  95         },
  96 };
  97
  98 struct devlink_dpipe_header devlink_dpipe_header_ethernet = {
  99         .name = "ethernet",
 100         .id = DEVLINK_DPIPE_HEADER_ETHERNET,
 101         .fields = devlink_dpipe_fields_ethernet,
 102         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ethernet),
 103         .global = true,
 104 };
 105 EXPORT_SYMBOL_GPL(devlink_dpipe_header_ethernet);
 106
 107 static struct devlink_dpipe_field devlink_dpipe_fields_ipv4[] = {
 108         {
 109                 .name = "destination ip",
 110                 .id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP,
 111                 .bitwidth = 32,
 112         },
 113 };
 114
 115 struct devlink_dpipe_header devlink_dpipe_header_ipv4 = {
 116         .name = "ipv4",
 117         .id = DEVLINK_DPIPE_HEADER_IPV4,
 118         .fields = devlink_dpipe_fields_ipv4,
 119         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv4),
 120         .global = true,
 121 };
 122 EXPORT_SYMBOL_GPL(devlink_dpipe_header_ipv4);
 123
 124 static struct devlink_dpipe_field devlink_dpipe_fields_ipv6[] = {
 125         {
 126                 .name = "destination ip",
 127                 .id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP,
 128                 .bitwidth = 128,
 129         },
 130 };
 131
 132 struct devlink_dpipe_header devlink_dpipe_header_ipv6 = {
 133         .name = "ipv6",
 134         .id = DEVLINK_DPIPE_HEADER_IPV6,
 135         .fields = devlink_dpipe_fields_ipv6,
 136         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv6),
 137         .global = true,
 138 };
 139 EXPORT_SYMBOL_GPL(devlink_dpipe_header_ipv6);
 140
 141 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
 142 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr);
 143 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report);
 144
 145 static const struct nla_policy devlink_function_nl_policy[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1] = {
 146         [DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] = { .type = NLA_BINARY },
 147         [DEVLINK_PORT_FN_ATTR_STATE] =
 148                 NLA_POLICY_RANGE(NLA_U8, DEVLINK_PORT_FN_STATE_INACTIVE,
 149                                  DEVLINK_PORT_FN_STATE_ACTIVE),
 150 };
 151
 152 static DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC);
 153 #define DEVLINK_REGISTERED XA_MARK_1
 154
 155 /* devlink instances are open to the access from the user space after
 156  * devlink_register() call. Such logical barrier allows us to have certain
 157  * expectations related to locking.
 158  *
 159  * Before *_register() - we are in initialization stage and no parallel
 160  * access possible to the devlink instance. All drivers perform that phase
 161  * by implicitly holding device_lock.
 162  *
 163  * After *_register() - users and driver can access devlink instance at
 164  * the same time.
 165  */
 166 #define ASSERT_DEVLINK_REGISTERED(d)                                           \
 167         WARN_ON_ONCE(!xa_get_mark(&devlinks, (d)->index, DEVLINK_REGISTERED))
 168 #define ASSERT_DEVLINK_NOT_REGISTERED(d)                                       \
 169         WARN_ON_ONCE(xa_get_mark(&devlinks, (d)->index, DEVLINK_REGISTERED))
 170
 171 /* devlink_mutex
 172  *
 173  * An overall lock guarding every operation coming from userspace.
 174  * It also guards devlink devices list and it is taken when
 175  * driver registers/unregisters it.
 176  */
 177 static DEFINE_MUTEX(devlink_mutex);
 178
 179 struct net *devlink_net(const struct devlink *devlink)
 180 {
 181         return read_pnet(&devlink->_net);
 182 }
 183 EXPORT_SYMBOL_GPL(devlink_net);
 184
 185 void devlink_put(struct devlink *devlink)
 186 {
 187         if (refcount_dec_and_test(&devlink->refcount))
 188                 complete(&devlink->comp);
 189 }
 190
 191 struct devlink *__must_check devlink_try_get(struct devlink *devlink)
 192 {
 193         if (refcount_inc_not_zero(&devlink->refcount))
 194                 return devlink;
 195         return NULL;
 196 }
 197
 198 static struct devlink *devlink_get_from_attrs(struct net *net,
 199                                               struct nlattr **attrs)
 200 {
 201         struct devlink *devlink;
 202         unsigned long index;
 203         bool found = false;
 204         char *busname;
 205         char *devname;
 206
 207         if (!attrs[DEVLINK_ATTR_BUS_NAME] || !attrs[DEVLINK_ATTR_DEV_NAME])
 208                 return ERR_PTR(-EINVAL);
 209
 210         busname = nla_data(attrs[DEVLINK_ATTR_BUS_NAME]);
 211         devname = nla_data(attrs[DEVLINK_ATTR_DEV_NAME]);
 212
 213         lockdep_assert_held(&devlink_mutex);
 214
 215         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
 216                 if (strcmp(devlink->dev->bus->name, busname) == 0 &&
 217                     strcmp(dev_name(devlink->dev), devname) == 0 &&
 218                     net_eq(devlink_net(devlink), net)) {
 219                         found = true;
 220                         break;
 221                 }
 222         }
 223
 224         if (!found || !devlink_try_get(devlink))
 225                 devlink = ERR_PTR(-ENODEV);
 226
 227         return devlink;
 228 }
 229
 230 static struct devlink_port *devlink_port_get_by_index(struct devlink *devlink,
 231                                                       unsigned int port_index)
 232 {
 233         struct devlink_port *devlink_port;
 234
 235         list_for_each_entry(devlink_port, &devlink->port_list, list) {
 236                 if (devlink_port->index == port_index)
 237                         return devlink_port;
 238         }
 239         return NULL;
 240 }
 241
 242 static bool devlink_port_index_exists(struct devlink *devlink,
 243                                       unsigned int port_index)
 244 {
 245         return devlink_port_get_by_index(devlink, port_index);
 246 }
 247
 248 static struct devlink_port *devlink_port_get_from_attrs(struct devlink *devlink,
 249                                                         struct nlattr **attrs)
 250 {
 251         if (attrs[DEVLINK_ATTR_PORT_INDEX]) {
 252                 u32 port_index = nla_get_u32(attrs[DEVLINK_ATTR_PORT_INDEX]);
 253                 struct devlink_port *devlink_port;
 254
 255                 devlink_port = devlink_port_get_by_index(devlink, port_index);
 256                 if (!devlink_port)
 257                         return ERR_PTR(-ENODEV);
 258                 return devlink_port;
 259         }
 260         return ERR_PTR(-EINVAL);
 261 }
 262
 263 static struct devlink_port *devlink_port_get_from_info(struct devlink *devlink,
 264                                                        struct genl_info *info)
 265 {
 266         return devlink_port_get_from_attrs(devlink, info->attrs);
 267 }
 268
 269 static inline bool
 270 devlink_rate_is_leaf(struct devlink_rate *devlink_rate)
 271 {
 272         return devlink_rate->type == DEVLINK_RATE_TYPE_LEAF;
 273 }
 274
 275 static inline bool
 276 devlink_rate_is_node(struct devlink_rate *devlink_rate)
 277 {
 278         return devlink_rate->type == DEVLINK_RATE_TYPE_NODE;
 279 }
 280
 281 static struct devlink_rate *
 282 devlink_rate_leaf_get_from_info(struct devlink *devlink, struct genl_info *info)
 283 {
 284         struct devlink_rate *devlink_rate;
 285         struct devlink_port *devlink_port;
 286
 287         devlink_port = devlink_port_get_from_attrs(devlink, info->attrs);
 288         if (IS_ERR(devlink_port))
 289                 return ERR_CAST(devlink_port);
 290         devlink_rate = devlink_port->devlink_rate;
 291         return devlink_rate ?: ERR_PTR(-ENODEV);
 292 }
 293
 294 static struct devlink_rate *
 295 devlink_rate_node_get_by_name(struct devlink *devlink, const char *node_name)
 296 {
 297         static struct devlink_rate *devlink_rate;
 298
 299         list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
 300                 if (devlink_rate_is_node(devlink_rate) &&
 301                     !strcmp(node_name, devlink_rate->name))
 302                         return devlink_rate;
 303         }
 304         return ERR_PTR(-ENODEV);
 305 }
 306
 307 static struct devlink_rate *
 308 devlink_rate_node_get_from_attrs(struct devlink *devlink, struct nlattr **attrs)
 309 {
 310         const char *rate_node_name;
 311         size_t len;
 312
 313         if (!attrs[DEVLINK_ATTR_RATE_NODE_NAME])
 314                 return ERR_PTR(-EINVAL);
 315         rate_node_name = nla_data(attrs[DEVLINK_ATTR_RATE_NODE_NAME]);
 316         len = strlen(rate_node_name);
 317         /* Name cannot be empty or decimal number */
 318         if (!len || strspn(rate_node_name, "0123456789") == len)
 319                 return ERR_PTR(-EINVAL);
 320
 321         return devlink_rate_node_get_by_name(devlink, rate_node_name);
 322 }
 323
 324 static struct devlink_rate *
 325 devlink_rate_node_get_from_info(struct devlink *devlink, struct genl_info *info)
 326 {
 327         return devlink_rate_node_get_from_attrs(devlink, info->attrs);
 328 }
 329
 330 static struct devlink_rate *
 331 devlink_rate_get_from_info(struct devlink *devlink, struct genl_info *info)
 332 {
 333         struct nlattr **attrs = info->attrs;
 334
 335         if (attrs[DEVLINK_ATTR_PORT_INDEX])
 336                 return devlink_rate_leaf_get_from_info(devlink, info);
 337         else if (attrs[DEVLINK_ATTR_RATE_NODE_NAME])
 338                 return devlink_rate_node_get_from_info(devlink, info);
 339         else
 340                 return ERR_PTR(-EINVAL);
 341 }
 342
 343 struct devlink_sb {
 344         struct list_head list;
 345         unsigned int index;
 346         u32 size;
 347         u16 ingress_pools_count;
 348         u16 egress_pools_count;
 349         u16 ingress_tc_count;
 350         u16 egress_tc_count;
 351 };
 352
 353 static u16 devlink_sb_pool_count(struct devlink_sb *devlink_sb)
 354 {
 355         return devlink_sb->ingress_pools_count + devlink_sb->egress_pools_count;
 356 }
 357
 358 static struct devlink_sb *devlink_sb_get_by_index(struct devlink *devlink,
 359                                                   unsigned int sb_index)
 360 {
 361         struct devlink_sb *devlink_sb;
 362
 363         list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
 364                 if (devlink_sb->index == sb_index)
 365                         return devlink_sb;
 366         }
 367         return NULL;
 368 }
 369
 370 static bool devlink_sb_index_exists(struct devlink *devlink,
 371                                     unsigned int sb_index)
 372 {
 373         return devlink_sb_get_by_index(devlink, sb_index);
 374 }
 375
 376 static struct devlink_sb *devlink_sb_get_from_attrs(struct devlink *devlink,
 377                                                     struct nlattr **attrs)
 378 {
 379         if (attrs[DEVLINK_ATTR_SB_INDEX]) {
 380                 u32 sb_index = nla_get_u32(attrs[DEVLINK_ATTR_SB_INDEX]);
 381                 struct devlink_sb *devlink_sb;
 382
 383                 devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
 384                 if (!devlink_sb)
 385                         return ERR_PTR(-ENODEV);
 386                 return devlink_sb;
 387         }
 388         return ERR_PTR(-EINVAL);
 389 }
 390
 391 static struct devlink_sb *devlink_sb_get_from_info(struct devlink *devlink,
 392                                                    struct genl_info *info)
 393 {
 394         return devlink_sb_get_from_attrs(devlink, info->attrs);
 395 }
 396
 397 static int devlink_sb_pool_index_get_from_attrs(struct devlink_sb *devlink_sb,
 398                                                 struct nlattr **attrs,
 399                                                 u16 *p_pool_index)
 400 {
 401         u16 val;
 402
 403         if (!attrs[DEVLINK_ATTR_SB_POOL_INDEX])
 404                 return -EINVAL;
 405
 406         val = nla_get_u16(attrs[DEVLINK_ATTR_SB_POOL_INDEX]);
 407         if (val >= devlink_sb_pool_count(devlink_sb))
 408                 return -EINVAL;
 409         *p_pool_index = val;
 410         return 0;
 411 }
 412
 413 static int devlink_sb_pool_index_get_from_info(struct devlink_sb *devlink_sb,
 414                                                struct genl_info *info,
 415                                                u16 *p_pool_index)
 416 {
 417         return devlink_sb_pool_index_get_from_attrs(devlink_sb, info->attrs,
 418                                                     p_pool_index);
 419 }
 420
 421 static int
 422 devlink_sb_pool_type_get_from_attrs(struct nlattr **attrs,
 423                                     enum devlink_sb_pool_type *p_pool_type)
 424 {
 425         u8 val;
 426
 427         if (!attrs[DEVLINK_ATTR_SB_POOL_TYPE])
 428                 return -EINVAL;
 429
 430         val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_TYPE]);
 431         if (val != DEVLINK_SB_POOL_TYPE_INGRESS &&
 432             val != DEVLINK_SB_POOL_TYPE_EGRESS)
 433                 return -EINVAL;
 434         *p_pool_type = val;
 435         return 0;
 436 }
 437
 438 static int
 439 devlink_sb_pool_type_get_from_info(struct genl_info *info,
 440                                    enum devlink_sb_pool_type *p_pool_type)
 441 {
 442         return devlink_sb_pool_type_get_from_attrs(info->attrs, p_pool_type);
 443 }
 444
 445 static int
 446 devlink_sb_th_type_get_from_attrs(struct nlattr **attrs,
 447                                   enum devlink_sb_threshold_type *p_th_type)
 448 {
 449         u8 val;
 450
 451         if (!attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE])
 452                 return -EINVAL;
 453
 454         val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE]);
 455         if (val != DEVLINK_SB_THRESHOLD_TYPE_STATIC &&
 456             val != DEVLINK_SB_THRESHOLD_TYPE_DYNAMIC)
 457                 return -EINVAL;
 458         *p_th_type = val;
 459         return 0;
 460 }
 461
 462 static int
 463 devlink_sb_th_type_get_from_info(struct genl_info *info,
 464                                  enum devlink_sb_threshold_type *p_th_type)
 465 {
 466         return devlink_sb_th_type_get_from_attrs(info->attrs, p_th_type);
 467 }
 468
 469 static int
 470 devlink_sb_tc_index_get_from_attrs(struct devlink_sb *devlink_sb,
 471                                    struct nlattr **attrs,
 472                                    enum devlink_sb_pool_type pool_type,
 473                                    u16 *p_tc_index)
 474 {
 475         u16 val;
 476
 477         if (!attrs[DEVLINK_ATTR_SB_TC_INDEX])
 478                 return -EINVAL;
 479
 480         val = nla_get_u16(attrs[DEVLINK_ATTR_SB_TC_INDEX]);
 481         if (pool_type == DEVLINK_SB_POOL_TYPE_INGRESS &&
 482             val >= devlink_sb->ingress_tc_count)
 483                 return -EINVAL;
 484         if (pool_type == DEVLINK_SB_POOL_TYPE_EGRESS &&
 485             val >= devlink_sb->egress_tc_count)
 486                 return -EINVAL;
 487         *p_tc_index = val;
 488         return 0;
 489 }
 490
 491 static int
 492 devlink_sb_tc_index_get_from_info(struct devlink_sb *devlink_sb,
 493                                   struct genl_info *info,
 494                                   enum devlink_sb_pool_type pool_type,
 495                                   u16 *p_tc_index)
 496 {
 497         return devlink_sb_tc_index_get_from_attrs(devlink_sb, info->attrs,
 498                                                   pool_type, p_tc_index);
 499 }
 500
 501 struct devlink_region {
 502         struct devlink *devlink;
 503         struct devlink_port *port;
 504         struct list_head list;
 505         union {
 506                 const struct devlink_region_ops *ops;
 507                 const struct devlink_port_region_ops *port_ops;
 508         };
 509         struct list_head snapshot_list;
 510         u32 max_snapshots;
 511         u32 cur_snapshots;
 512         u64 size;
 513 };
 514
 515 struct devlink_snapshot {
 516         struct list_head list;
 517         struct devlink_region *region;
 518         u8 *data;
 519         u32 id;
 520 };
 521
 522 static struct devlink_region *
 523 devlink_region_get_by_name(struct devlink *devlink, const char *region_name)
 524 {
 525         struct devlink_region *region;
 526
 527         list_for_each_entry(region, &devlink->region_list, list)
 528                 if (!strcmp(region->ops->name, region_name))
 529                         return region;
 530
 531         return NULL;
 532 }
 533
 534 static struct devlink_region *
 535 devlink_port_region_get_by_name(struct devlink_port *port,
 536                                 const char *region_name)
 537 {
 538         struct devlink_region *region;
 539
 540         list_for_each_entry(region, &port->region_list, list)
 541                 if (!strcmp(region->ops->name, region_name))
 542                         return region;
 543
 544         return NULL;
 545 }
 546
 547 static struct devlink_snapshot *
 548 devlink_region_snapshot_get_by_id(struct devlink_region *region, u32 id)
 549 {
 550         struct devlink_snapshot *snapshot;
 551
 552         list_for_each_entry(snapshot, &region->snapshot_list, list)
 553                 if (snapshot->id == id)
 554                         return snapshot;
 555
 556         return NULL;
 557 }
 558
 559 #define DEVLINK_NL_FLAG_NEED_PORT               BIT(0)
 560 #define DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT    BIT(1)
 561 #define DEVLINK_NL_FLAG_NEED_RATE               BIT(2)
 562 #define DEVLINK_NL_FLAG_NEED_RATE_NODE          BIT(3)
 563
 564 /* The per devlink instance lock is taken by default in the pre-doit
 565  * operation, yet several commands do not require this. The global
 566  * devlink lock is taken and protects from disruption by user-calls.
 567  */
 568 #define DEVLINK_NL_FLAG_NO_LOCK                 BIT(4)
 569
 570 static int devlink_nl_pre_doit(const struct genl_ops *ops,
 571                                struct sk_buff *skb, struct genl_info *info)
 572 {
 573         struct devlink_port *devlink_port;
 574         struct devlink *devlink;
 575         int err;
 576
 577         mutex_lock(&devlink_mutex);
 578         devlink = devlink_get_from_attrs(genl_info_net(info), info->attrs);
 579         if (IS_ERR(devlink)) {
 580                 mutex_unlock(&devlink_mutex);
 581                 return PTR_ERR(devlink);
 582         }
 583         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 584                 mutex_lock(&devlink->lock);
 585         info->user_ptr[0] = devlink;
 586         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_PORT) {
 587                 devlink_port = devlink_port_get_from_info(devlink, info);
 588                 if (IS_ERR(devlink_port)) {
 589                         err = PTR_ERR(devlink_port);
 590                         goto unlock;
 591                 }
 592                 info->user_ptr[1] = devlink_port;
 593         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT) {
 594                 devlink_port = devlink_port_get_from_info(devlink, info);
 595                 if (!IS_ERR(devlink_port))
 596                         info->user_ptr[1] = devlink_port;
 597         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE) {
 598                 struct devlink_rate *devlink_rate;
 599
 600                 devlink_rate = devlink_rate_get_from_info(devlink, info);
 601                 if (IS_ERR(devlink_rate)) {
 602                         err = PTR_ERR(devlink_rate);
 603                         goto unlock;
 604                 }
 605                 info->user_ptr[1] = devlink_rate;
 606         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE_NODE) {
 607                 struct devlink_rate *rate_node;
 608
 609                 rate_node = devlink_rate_node_get_from_info(devlink, info);
 610                 if (IS_ERR(rate_node)) {
 611                         err = PTR_ERR(rate_node);
 612                         goto unlock;
 613                 }
 614                 info->user_ptr[1] = rate_node;
 615         }
 616         return 0;
 617
 618 unlock:
 619         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 620                 mutex_unlock(&devlink->lock);
 621         devlink_put(devlink);
 622         mutex_unlock(&devlink_mutex);
 623         return err;
 624 }
 625
 626 static void devlink_nl_post_doit(const struct genl_ops *ops,
 627                                  struct sk_buff *skb, struct genl_info *info)
 628 {
 629         struct devlink *devlink;
 630
 631         devlink = info->user_ptr[0];
 632         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 633                 mutex_unlock(&devlink->lock);
 634         devlink_put(devlink);
 635         mutex_unlock(&devlink_mutex);
 636 }
 637
 638 static struct genl_family devlink_nl_family;
 639
 640 enum devlink_multicast_groups {
 641         DEVLINK_MCGRP_CONFIG,
 642 };
 643
 644 static const struct genl_multicast_group devlink_nl_mcgrps[] = {
 645         [DEVLINK_MCGRP_CONFIG] = { .name = DEVLINK_GENL_MCGRP_CONFIG_NAME },
 646 };
 647
 648 static int devlink_nl_put_handle(struct sk_buff *msg, struct devlink *devlink)
 649 {
 650         if (nla_put_string(msg, DEVLINK_ATTR_BUS_NAME, devlink->dev->bus->name))
 651                 return -EMSGSIZE;
 652         if (nla_put_string(msg, DEVLINK_ATTR_DEV_NAME, dev_name(devlink->dev)))
 653                 return -EMSGSIZE;
 654         return 0;
 655 }
 656
 657 struct devlink_reload_combination {
 658         enum devlink_reload_action action;
 659         enum devlink_reload_limit limit;
 660 };
 661
 662 static const struct devlink_reload_combination devlink_reload_invalid_combinations[] = {
 663         {
 664                 /* can't reinitialize driver with no down time */
 665                 .action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
 666                 .limit = DEVLINK_RELOAD_LIMIT_NO_RESET,
 667         },
 668 };
 669
 670 static bool
 671 devlink_reload_combination_is_invalid(enum devlink_reload_action action,
 672                                       enum devlink_reload_limit limit)
 673 {
 674         int i;
 675
 676         for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)
 677                 if (devlink_reload_invalid_combinations[i].action == action &&
 678                     devlink_reload_invalid_combinations[i].limit == limit)
 679                         return true;
 680         return false;
 681 }
 682
 683 static bool
 684 devlink_reload_action_is_supported(struct devlink *devlink, enum devlink_reload_action action)
 685 {
 686         return test_bit(action, &devlink->ops->reload_actions);
 687 }
 688
 689 static bool
 690 devlink_reload_limit_is_supported(struct devlink *devlink, enum devlink_reload_limit limit)
 691 {
 692         return test_bit(limit, &devlink->ops->reload_limits);
 693 }
 694
 695 static int devlink_reload_stat_put(struct sk_buff *msg,
 696                                    enum devlink_reload_limit limit, u32 value)
 697 {
 698         struct nlattr *reload_stats_entry;
 699
 700         reload_stats_entry = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS_ENTRY);
 701         if (!reload_stats_entry)
 702                 return -EMSGSIZE;
 703
 704         if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_STATS_LIMIT, limit) ||
 705             nla_put_u32(msg, DEVLINK_ATTR_RELOAD_STATS_VALUE, value))
 706                 goto nla_put_failure;
 707         nla_nest_end(msg, reload_stats_entry);
 708         return 0;
 709
 710 nla_put_failure:
 711         nla_nest_cancel(msg, reload_stats_entry);
 712         return -EMSGSIZE;
 713 }
 714
 715 static int devlink_reload_stats_put(struct sk_buff *msg, struct devlink *devlink, bool is_remote)
 716 {
 717         struct nlattr *reload_stats_attr, *act_info, *act_stats;
 718         int i, j, stat_idx;
 719         u32 value;
 720
 721         if (!is_remote)
 722                 reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS);
 723         else
 724                 reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_REMOTE_RELOAD_STATS);
 725
 726         if (!reload_stats_attr)
 727                 return -EMSGSIZE;
 728
 729         for (i = 0; i <= DEVLINK_RELOAD_ACTION_MAX; i++) {
 730                 if ((!is_remote &&
 731                      !devlink_reload_action_is_supported(devlink, i)) ||
 732                     i == DEVLINK_RELOAD_ACTION_UNSPEC)
 733                         continue;
 734                 act_info = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_ACTION_INFO);
 735                 if (!act_info)
 736                         goto nla_put_failure;
 737
 738                 if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_ACTION, i))
 739                         goto action_info_nest_cancel;
 740                 act_stats = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_ACTION_STATS);
 741                 if (!act_stats)
 742                         goto action_info_nest_cancel;
 743
 744                 for (j = 0; j <= DEVLINK_RELOAD_LIMIT_MAX; j++) {
 745                         /* Remote stats are shown even if not locally supported.
 746                          * Stats of actions with unspecified limit are shown
 747                          * though drivers don't need to register unspecified
 748                          * limit.
 749                          */
 750                         if ((!is_remote && j != DEVLINK_RELOAD_LIMIT_UNSPEC &&
 751                              !devlink_reload_limit_is_supported(devlink, j)) ||
 752                             devlink_reload_combination_is_invalid(i, j))
 753                                 continue;
 754
 755                         stat_idx = j * __DEVLINK_RELOAD_ACTION_MAX + i;
 756                         if (!is_remote)
 757                                 value = devlink->stats.reload_stats[stat_idx];
 758                         else
 759                                 value = devlink->stats.remote_reload_stats[stat_idx];
 760                         if (devlink_reload_stat_put(msg, j, value))
 761                                 goto action_stats_nest_cancel;
 762                 }
 763                 nla_nest_end(msg, act_stats);
 764                 nla_nest_end(msg, act_info);
 765         }
 766         nla_nest_end(msg, reload_stats_attr);
 767         return 0;
 768
 769 action_stats_nest_cancel:
 770         nla_nest_cancel(msg, act_stats);
 771 action_info_nest_cancel:
 772         nla_nest_cancel(msg, act_info);
 773 nla_put_failure:
 774         nla_nest_cancel(msg, reload_stats_attr);
 775         return -EMSGSIZE;
 776 }
 777
 778 static int devlink_nl_fill(struct sk_buff *msg, struct devlink *devlink,
 779                            enum devlink_command cmd, u32 portid,
 780                            u32 seq, int flags)
 781 {
 782         struct nlattr *dev_stats;
 783         void *hdr;
 784
 785         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 786         if (!hdr)
 787                 return -EMSGSIZE;
 788
 789         if (devlink_nl_put_handle(msg, devlink))
 790                 goto nla_put_failure;
 791         if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_FAILED, devlink->reload_failed))
 792                 goto nla_put_failure;
 793
 794         dev_stats = nla_nest_start(msg, DEVLINK_ATTR_DEV_STATS);
 795         if (!dev_stats)
 796                 goto nla_put_failure;
 797
 798         if (devlink_reload_stats_put(msg, devlink, false))
 799                 goto dev_stats_nest_cancel;
 800         if (devlink_reload_stats_put(msg, devlink, true))
 801                 goto dev_stats_nest_cancel;
 802
 803         nla_nest_end(msg, dev_stats);
 804         genlmsg_end(msg, hdr);
 805         return 0;
 806
 807 dev_stats_nest_cancel:
 808         nla_nest_cancel(msg, dev_stats);
 809 nla_put_failure:
 810         genlmsg_cancel(msg, hdr);
 811         return -EMSGSIZE;
 812 }
 813
 814 static void devlink_notify(struct devlink *devlink, enum devlink_command cmd)
 815 {
 816         struct sk_buff *msg;
 817         int err;
 818
 819         WARN_ON(cmd != DEVLINK_CMD_NEW && cmd != DEVLINK_CMD_DEL);
 820         WARN_ON(!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED));
 821
 822         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 823         if (!msg)
 824                 return;
 825
 826         err = devlink_nl_fill(msg, devlink, cmd, 0, 0, 0);
 827         if (err) {
 828                 nlmsg_free(msg);
 829                 return;
 830         }
 831
 832         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
 833                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
 834 }
 835
 836 static int devlink_nl_port_attrs_put(struct sk_buff *msg,
 837                                      struct devlink_port *devlink_port)
 838 {
 839         struct devlink_port_attrs *attrs = &devlink_port->attrs;
 840
 841         if (!devlink_port->attrs_set)
 842                 return 0;
 843         if (attrs->lanes) {
 844                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_LANES, attrs->lanes))
 845                         return -EMSGSIZE;
 846         }
 847         if (nla_put_u8(msg, DEVLINK_ATTR_PORT_SPLITTABLE, attrs->splittable))
 848                 return -EMSGSIZE;
 849         if (nla_put_u16(msg, DEVLINK_ATTR_PORT_FLAVOUR, attrs->flavour))
 850                 return -EMSGSIZE;
 851         switch (devlink_port->attrs.flavour) {
 852         case DEVLINK_PORT_FLAVOUR_PCI_PF:
 853                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
 854                                 attrs->pci_pf.controller) ||
 855                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_pf.pf))
 856                         return -EMSGSIZE;
 857                 if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_pf.external))
 858                         return -EMSGSIZE;
 859                 break;
 860         case DEVLINK_PORT_FLAVOUR_PCI_VF:
 861                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
 862                                 attrs->pci_vf.controller) ||
 863                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_vf.pf) ||
 864                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_VF_NUMBER, attrs->pci_vf.vf))
 865                         return -EMSGSIZE;
 866                 if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_vf.external))
 867                         return -EMSGSIZE;
 868                 break;
 869         case DEVLINK_PORT_FLAVOUR_PCI_SF:
 870                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
 871                                 attrs->pci_sf.controller) ||
 872                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER,
 873                                 attrs->pci_sf.pf) ||
 874                     nla_put_u32(msg, DEVLINK_ATTR_PORT_PCI_SF_NUMBER,
 875                                 attrs->pci_sf.sf))
 876                         return -EMSGSIZE;
 877                 break;
 878         case DEVLINK_PORT_FLAVOUR_PHYSICAL:
 879         case DEVLINK_PORT_FLAVOUR_CPU:
 880         case DEVLINK_PORT_FLAVOUR_DSA:
 881                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_NUMBER,
 882                                 attrs->phys.port_number))
 883                         return -EMSGSIZE;
 884                 if (!attrs->split)
 885                         return 0;
 886                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_GROUP,
 887                                 attrs->phys.port_number))
 888                         return -EMSGSIZE;
 889                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_SUBPORT_NUMBER,
 890                                 attrs->phys.split_subport_number))
 891                         return -EMSGSIZE;
 892                 break;
 893         default:
 894                 break;
 895         }
 896         return 0;
 897 }
 898
 899 static int devlink_port_fn_hw_addr_fill(const struct devlink_ops *ops,
 900                                         struct devlink_port *port,
 901                                         struct sk_buff *msg,
 902                                         struct netlink_ext_ack *extack,
 903                                         bool *msg_updated)
 904 {
 905         u8 hw_addr[MAX_ADDR_LEN];
 906         int hw_addr_len;
 907         int err;
 908
 909         if (!ops->port_function_hw_addr_get)
 910                 return 0;
 911
 912         err = ops->port_function_hw_addr_get(port, hw_addr, &hw_addr_len,
 913                                              extack);
 914         if (err) {
 915                 if (err == -EOPNOTSUPP)
 916                         return 0;
 917                 return err;
 918         }
 919         err = nla_put(msg, DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR, hw_addr_len, hw_addr);
 920         if (err)
 921                 return err;
 922         *msg_updated = true;
 923         return 0;
 924 }
 925
 926 static int devlink_nl_rate_fill(struct sk_buff *msg,
 927                                 struct devlink_rate *devlink_rate,
 928                                 enum devlink_command cmd, u32 portid, u32 seq,
 929                                 int flags, struct netlink_ext_ack *extack)
 930 {
 931         struct devlink *devlink = devlink_rate->devlink;
 932         void *hdr;
 933
 934         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 935         if (!hdr)
 936                 return -EMSGSIZE;
 937
 938         if (devlink_nl_put_handle(msg, devlink))
 939                 goto nla_put_failure;
 940
 941         if (nla_put_u16(msg, DEVLINK_ATTR_RATE_TYPE, devlink_rate->type))
 942                 goto nla_put_failure;
 943
 944         if (devlink_rate_is_leaf(devlink_rate)) {
 945                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
 946                                 devlink_rate->devlink_port->index))
 947                         goto nla_put_failure;
 948         } else if (devlink_rate_is_node(devlink_rate)) {
 949                 if (nla_put_string(msg, DEVLINK_ATTR_RATE_NODE_NAME,
 950                                    devlink_rate->name))
 951                         goto nla_put_failure;
 952         }
 953
 954         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_SHARE,
 955                               devlink_rate->tx_share, DEVLINK_ATTR_PAD))
 956                 goto nla_put_failure;
 957
 958         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_MAX,
 959                               devlink_rate->tx_max, DEVLINK_ATTR_PAD))
 960                 goto nla_put_failure;
 961
 962         if (devlink_rate->parent)
 963                 if (nla_put_string(msg, DEVLINK_ATTR_RATE_PARENT_NODE_NAME,
 964                                    devlink_rate->parent->name))
 965                         goto nla_put_failure;
 966
 967         genlmsg_end(msg, hdr);
 968         return 0;
 969
 970 nla_put_failure:
 971         genlmsg_cancel(msg, hdr);
 972         return -EMSGSIZE;
 973 }
 974
 975 static bool
 976 devlink_port_fn_state_valid(enum devlink_port_fn_state state)
 977 {
 978         return state == DEVLINK_PORT_FN_STATE_INACTIVE ||
 979                state == DEVLINK_PORT_FN_STATE_ACTIVE;
 980 }
 981
 982 static bool
 983 devlink_port_fn_opstate_valid(enum devlink_port_fn_opstate opstate)
 984 {
 985         return opstate == DEVLINK_PORT_FN_OPSTATE_DETACHED ||
 986                opstate == DEVLINK_PORT_FN_OPSTATE_ATTACHED;
 987 }
 988
 989 static int devlink_port_fn_state_fill(const struct devlink_ops *ops,
 990                                       struct devlink_port *port,
 991                                       struct sk_buff *msg,
 992                                       struct netlink_ext_ack *extack,
 993                                       bool *msg_updated)
 994 {
 995         enum devlink_port_fn_opstate opstate;
 996         enum devlink_port_fn_state state;
 997         int err;
 998
 999         if (!ops->port_fn_state_get)
1000                 return 0;
1001
1002         err = ops->port_fn_state_get(port, &state, &opstate, extack);
1003         if (err) {
1004                 if (err == -EOPNOTSUPP)
1005                         return 0;
1006                 return err;
1007         }
1008         if (!devlink_port_fn_state_valid(state)) {
1009                 WARN_ON_ONCE(1);
1010                 NL_SET_ERR_MSG_MOD(extack, "Invalid state read from driver");
1011                 return -EINVAL;
1012         }
1013         if (!devlink_port_fn_opstate_valid(opstate)) {
1014                 WARN_ON_ONCE(1);
1015                 NL_SET_ERR_MSG_MOD(extack,
1016                                    "Invalid operational state read from driver");
1017                 return -EINVAL;
1018         }
1019         if (nla_put_u8(msg, DEVLINK_PORT_FN_ATTR_STATE, state) ||
1020             nla_put_u8(msg, DEVLINK_PORT_FN_ATTR_OPSTATE, opstate))
1021                 return -EMSGSIZE;
1022         *msg_updated = true;
1023         return 0;
1024 }
1025
1026 static int
1027 devlink_nl_port_function_attrs_put(struct sk_buff *msg, struct devlink_port *port,
1028                                    struct netlink_ext_ack *extack)
1029 {
1030         const struct devlink_ops *ops;
1031         struct nlattr *function_attr;
1032         bool msg_updated = false;
1033         int err;
1034
1035         function_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PORT_FUNCTION);
1036         if (!function_attr)
1037                 return -EMSGSIZE;
1038
1039         ops = port->devlink->ops;
1040         err = devlink_port_fn_hw_addr_fill(ops, port, msg, extack,
1041                                            &msg_updated);
1042         if (err)
1043                 goto out;
1044         err = devlink_port_fn_state_fill(ops, port, msg, extack, &msg_updated);
1045 out:
1046         if (err || !msg_updated)
1047                 nla_nest_cancel(msg, function_attr);
1048         else
1049                 nla_nest_end(msg, function_attr);
1050         return err;
1051 }
1052
1053 static int devlink_nl_port_fill(struct sk_buff *msg,
1054                                 struct devlink_port *devlink_port,
1055                                 enum devlink_command cmd, u32 portid, u32 seq,
1056                                 int flags, struct netlink_ext_ack *extack)
1057 {
1058         struct devlink *devlink = devlink_port->devlink;
1059         void *hdr;
1060
1061         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1062         if (!hdr)
1063                 return -EMSGSIZE;
1064
1065         if (devlink_nl_put_handle(msg, devlink))
1066                 goto nla_put_failure;
1067         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
1068                 goto nla_put_failure;
1069
1070         /* Hold rtnl lock while accessing port's netdev attributes. */
1071         rtnl_lock();
1072         spin_lock_bh(&devlink_port->type_lock);
1073         if (nla_put_u16(msg, DEVLINK_ATTR_PORT_TYPE, devlink_port->type))
1074                 goto nla_put_failure_type_locked;
1075         if (devlink_port->desired_type != DEVLINK_PORT_TYPE_NOTSET &&
1076             nla_put_u16(msg, DEVLINK_ATTR_PORT_DESIRED_TYPE,
1077                         devlink_port->desired_type))
1078                 goto nla_put_failure_type_locked;
1079         if (devlink_port->type == DEVLINK_PORT_TYPE_ETH) {
1080                 struct net *net = devlink_net(devlink_port->devlink);
1081                 struct net_device *netdev = devlink_port->type_dev;
1082
1083                 if (netdev && net_eq(net, dev_net(netdev)) &&
1084                     (nla_put_u32(msg, DEVLINK_ATTR_PORT_NETDEV_IFINDEX,
1085                                  netdev->ifindex) ||
1086                      nla_put_string(msg, DEVLINK_ATTR_PORT_NETDEV_NAME,
1087                                     netdev->name)))
1088                         goto nla_put_failure_type_locked;
1089         }
1090         if (devlink_port->type == DEVLINK_PORT_TYPE_IB) {
1091                 struct ib_device *ibdev = devlink_port->type_dev;
1092
1093                 if (ibdev &&
1094                     nla_put_string(msg, DEVLINK_ATTR_PORT_IBDEV_NAME,
1095                                    ibdev->name))
1096                         goto nla_put_failure_type_locked;
1097         }
1098         spin_unlock_bh(&devlink_port->type_lock);
1099         rtnl_unlock();
1100         if (devlink_nl_port_attrs_put(msg, devlink_port))
1101                 goto nla_put_failure;
1102         if (devlink_nl_port_function_attrs_put(msg, devlink_port, extack))
1103                 goto nla_put_failure;
1104
1105         genlmsg_end(msg, hdr);
1106         return 0;
1107
1108 nla_put_failure_type_locked:
1109         spin_unlock_bh(&devlink_port->type_lock);
1110         rtnl_unlock();
1111 nla_put_failure:
1112         genlmsg_cancel(msg, hdr);
1113         return -EMSGSIZE;
1114 }
1115
1116 static void devlink_port_notify(struct devlink_port *devlink_port,
1117                                 enum devlink_command cmd)
1118 {
1119         struct devlink *devlink = devlink_port->devlink;
1120         struct sk_buff *msg;
1121         int err;
1122
1123         WARN_ON(cmd != DEVLINK_CMD_PORT_NEW && cmd != DEVLINK_CMD_PORT_DEL);
1124
1125         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
1126                 return;
1127
1128         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1129         if (!msg)
1130                 return;
1131
1132         err = devlink_nl_port_fill(msg, devlink_port, cmd, 0, 0, 0, NULL);
1133         if (err) {
1134                 nlmsg_free(msg);
1135                 return;
1136         }
1137
1138         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
1139                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
1140 }
1141
1142 static void devlink_rate_notify(struct devlink_rate *devlink_rate,
1143                                 enum devlink_command cmd)
1144 {
1145         struct devlink *devlink = devlink_rate->devlink;
1146         struct sk_buff *msg;
1147         int err;
1148
1149         WARN_ON(cmd != DEVLINK_CMD_RATE_NEW && cmd != DEVLINK_CMD_RATE_DEL);
1150
1151         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
1152                 return;
1153
1154         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1155         if (!msg)
1156                 return;
1157
1158         err = devlink_nl_rate_fill(msg, devlink_rate, cmd, 0, 0, 0, NULL);
1159         if (err) {
1160                 nlmsg_free(msg);
1161                 return;
1162         }
1163
1164         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
1165                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
1166 }
1167
1168 static int devlink_nl_cmd_rate_get_dumpit(struct sk_buff *msg,
1169                                           struct netlink_callback *cb)
1170 {
1171         struct devlink_rate *devlink_rate;
1172         struct devlink *devlink;
1173         int start = cb->args[0];
1174         unsigned long index;
1175         int idx = 0;
1176         int err = 0;
1177
1178         mutex_lock(&devlink_mutex);
1179         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
1180                 if (!devlink_try_get(devlink))
1181                         continue;
1182
1183                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
1184                         goto retry;
1185
1186                 mutex_lock(&devlink->lock);
1187                 list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
1188                         enum devlink_command cmd = DEVLINK_CMD_RATE_NEW;
1189                         u32 id = NETLINK_CB(cb->skb).portid;
1190
1191                         if (idx < start) {
1192                                 idx++;
1193                                 continue;
1194                         }
1195                         err = devlink_nl_rate_fill(msg, devlink_rate, cmd, id,
1196                                                    cb->nlh->nlmsg_seq,
1197                                                    NLM_F_MULTI, NULL);
1198                         if (err) {
1199                                 mutex_unlock(&devlink->lock);
1200                                 devlink_put(devlink);
1201                                 goto out;
1202                         }
1203                         idx++;
1204                 }
1205                 mutex_unlock(&devlink->lock);
1206 retry:
1207                 devlink_put(devlink);
1208         }
1209 out:
1210         mutex_unlock(&devlink_mutex);
1211         if (err != -EMSGSIZE)
1212                 return err;
1213
1214         cb->args[0] = idx;
1215         return msg->len;
1216 }
1217
1218 static int devlink_nl_cmd_rate_get_doit(struct sk_buff *skb,
1219                                         struct genl_info *info)
1220 {
1221         struct devlink_rate *devlink_rate = info->user_ptr[1];
1222         struct sk_buff *msg;
1223         int err;
1224
1225         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1226         if (!msg)
1227                 return -ENOMEM;
1228
1229         err = devlink_nl_rate_fill(msg, devlink_rate, DEVLINK_CMD_RATE_NEW,
1230                                    info->snd_portid, info->snd_seq, 0,
1231                                    info->extack);
1232         if (err) {
1233                 nlmsg_free(msg);
1234                 return err;
1235         }
1236
1237         return genlmsg_reply(msg, info);
1238 }
1239
1240 static bool
1241 devlink_rate_is_parent_node(struct devlink_rate *devlink_rate,
1242                             struct devlink_rate *parent)
1243 {
1244         while (parent) {
1245                 if (parent == devlink_rate)
1246                         return true;
1247                 parent = parent->parent;
1248         }
1249         return false;
1250 }
1251
1252 static int devlink_nl_cmd_get_doit(struct sk_buff *skb, struct genl_info *info)
1253 {
1254         struct devlink *devlink = info->user_ptr[0];
1255         struct sk_buff *msg;
1256         int err;
1257
1258         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1259         if (!msg)
1260                 return -ENOMEM;
1261
1262         err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
1263                               info->snd_portid, info->snd_seq, 0);
1264         if (err) {
1265                 nlmsg_free(msg);
1266                 return err;
1267         }
1268
1269         return genlmsg_reply(msg, info);
1270 }
1271
1272 static int devlink_nl_cmd_get_dumpit(struct sk_buff *msg,
1273                                      struct netlink_callback *cb)
1274 {
1275         struct devlink *devlink;
1276         int start = cb->args[0];
1277         unsigned long index;
1278         int idx = 0;
1279         int err;
1280
1281         mutex_lock(&devlink_mutex);
1282         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
1283                 if (!devlink_try_get(devlink))
1284                         continue;
1285
1286                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk))) {
1287                         devlink_put(devlink);
1288                         continue;
1289                 }
1290
1291                 if (idx < start) {
1292                         idx++;
1293                         devlink_put(devlink);
1294                         continue;
1295                 }
1296
1297                 err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
1298                                       NETLINK_CB(cb->skb).portid,
1299                                       cb->nlh->nlmsg_seq, NLM_F_MULTI);
1300                 devlink_put(devlink);
1301                 if (err)
1302                         goto out;
1303                 idx++;
1304         }
1305 out:
1306         mutex_unlock(&devlink_mutex);
1307
1308         cb->args[0] = idx;
1309         return msg->len;
1310 }
1311
1312 static int devlink_nl_cmd_port_get_doit(struct sk_buff *skb,
1313                                         struct genl_info *info)
1314 {
1315         struct devlink_port *devlink_port = info->user_ptr[1];
1316         struct sk_buff *msg;
1317         int err;
1318
1319         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1320         if (!msg)
1321                 return -ENOMEM;
1322
1323         err = devlink_nl_port_fill(msg, devlink_port, DEVLINK_CMD_PORT_NEW,
1324                                    info->snd_portid, info->snd_seq, 0,
1325                                    info->extack);
1326         if (err) {
1327                 nlmsg_free(msg);
1328                 return err;
1329         }
1330
1331         return genlmsg_reply(msg, info);
1332 }
1333
1334 static int devlink_nl_cmd_port_get_dumpit(struct sk_buff *msg,
1335                                           struct netlink_callback *cb)
1336 {
1337         struct devlink *devlink;
1338         struct devlink_port *devlink_port;
1339         int start = cb->args[0];
1340         unsigned long index;
1341         int idx = 0;
1342         int err;
1343
1344         mutex_lock(&devlink_mutex);
1345         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
1346                 if (!devlink_try_get(devlink))
1347                         continue;
1348
1349                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
1350                         goto retry;
1351
1352                 mutex_lock(&devlink->lock);
1353                 list_for_each_entry(devlink_port, &devlink->port_list, list) {
1354                         if (idx < start) {
1355                                 idx++;
1356                                 continue;
1357                         }
1358                         err = devlink_nl_port_fill(msg, devlink_port,
1359                                                    DEVLINK_CMD_NEW,
1360                                                    NETLINK_CB(cb->skb).portid,
1361                                                    cb->nlh->nlmsg_seq,
1362                                                    NLM_F_MULTI, cb->extack);
1363                         if (err) {
1364                                 mutex_unlock(&devlink->lock);
1365                                 devlink_put(devlink);
1366                                 goto out;
1367                         }
1368                         idx++;
1369                 }
1370                 mutex_unlock(&devlink->lock);
1371 retry:
1372                 devlink_put(devlink);
1373         }
1374 out:
1375         mutex_unlock(&devlink_mutex);
1376
1377         cb->args[0] = idx;
1378         return msg->len;
1379 }
1380
1381 static int devlink_port_type_set(struct devlink_port *devlink_port,
1382                                  enum devlink_port_type port_type)
1383
1384 {
1385         int err;
1386
1387         if (!devlink_port->devlink->ops->port_type_set)
1388                 return -EOPNOTSUPP;
1389
1390         if (port_type == devlink_port->type)
1391                 return 0;
1392
1393         err = devlink_port->devlink->ops->port_type_set(devlink_port,
1394                                                         port_type);
1395         if (err)
1396                 return err;
1397
1398         devlink_port->desired_type = port_type;
1399         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
1400         return 0;
1401 }
1402
1403 static int devlink_port_function_hw_addr_set(struct devlink_port *port,
1404                                              const struct nlattr *attr,
1405                                              struct netlink_ext_ack *extack)
1406 {
1407         const struct devlink_ops *ops = port->devlink->ops;
1408         const u8 *hw_addr;
1409         int hw_addr_len;
1410
1411         hw_addr = nla_data(attr);
1412         hw_addr_len = nla_len(attr);
1413         if (hw_addr_len > MAX_ADDR_LEN) {
1414                 NL_SET_ERR_MSG_MOD(extack, "Port function hardware address too long");
1415                 return -EINVAL;
1416         }
1417         if (port->type == DEVLINK_PORT_TYPE_ETH) {
1418                 if (hw_addr_len != ETH_ALEN) {
1419                         NL_SET_ERR_MSG_MOD(extack, "Address must be 6 bytes for Ethernet device");
1420                         return -EINVAL;
1421                 }
1422                 if (!is_unicast_ether_addr(hw_addr)) {
1423                         NL_SET_ERR_MSG_MOD(extack, "Non-unicast hardware address unsupported");
1424                         return -EINVAL;
1425                 }
1426         }
1427
1428         if (!ops->port_function_hw_addr_set) {
1429                 NL_SET_ERR_MSG_MOD(extack, "Port doesn't support function attributes");
1430                 return -EOPNOTSUPP;
1431         }
1432
1433         return ops->port_function_hw_addr_set(port, hw_addr, hw_addr_len,
1434                                               extack);
1435 }
1436
1437 static int devlink_port_fn_state_set(struct devlink_port *port,
1438                                      const struct nlattr *attr,
1439                                      struct netlink_ext_ack *extack)
1440 {
1441         enum devlink_port_fn_state state;
1442         const struct devlink_ops *ops;
1443
1444         state = nla_get_u8(attr);
1445         ops = port->devlink->ops;
1446         if (!ops->port_fn_state_set) {
1447                 NL_SET_ERR_MSG_MOD(extack,
1448                                    "Function does not support state setting");
1449                 return -EOPNOTSUPP;
1450         }
1451         return ops->port_fn_state_set(port, state, extack);
1452 }
1453
1454 static int devlink_port_function_set(struct devlink_port *port,
1455                                      const struct nlattr *attr,
1456                                      struct netlink_ext_ack *extack)
1457 {
1458         struct nlattr *tb[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1];
1459         int err;
1460
1461         err = nla_parse_nested(tb, DEVLINK_PORT_FUNCTION_ATTR_MAX, attr,
1462                                devlink_function_nl_policy, extack);
1463         if (err < 0) {
1464                 NL_SET_ERR_MSG_MOD(extack, "Fail to parse port function attributes");
1465                 return err;
1466         }
1467
1468         attr = tb[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR];
1469         if (attr) {
1470                 err = devlink_port_function_hw_addr_set(port, attr, extack);
1471                 if (err)
1472                         return err;
1473         }
1474         /* Keep this as the last function attribute set, so that when
1475          * multiple port function attributes are set along with state,
1476          * Those can be applied first before activating the state.
1477          */
1478         attr = tb[DEVLINK_PORT_FN_ATTR_STATE];
1479         if (attr)
1480                 err = devlink_port_fn_state_set(port, attr, extack);
1481
1482         if (!err)
1483                 devlink_port_notify(port, DEVLINK_CMD_PORT_NEW);
1484         return err;
1485 }
1486
1487 static int devlink_nl_cmd_port_set_doit(struct sk_buff *skb,
1488                                         struct genl_info *info)
1489 {
1490         struct devlink_port *devlink_port = info->user_ptr[1];
1491         int err;
1492
1493         if (info->attrs[DEVLINK_ATTR_PORT_TYPE]) {
1494                 enum devlink_port_type port_type;
1495
1496                 port_type = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_TYPE]);
1497                 err = devlink_port_type_set(devlink_port, port_type);
1498                 if (err)
1499                         return err;
1500         }
1501
1502         if (info->attrs[DEVLINK_ATTR_PORT_FUNCTION]) {
1503                 struct nlattr *attr = info->attrs[DEVLINK_ATTR_PORT_FUNCTION];
1504                 struct netlink_ext_ack *extack = info->extack;
1505
1506                 err = devlink_port_function_set(devlink_port, attr, extack);
1507                 if (err)
1508                         return err;
1509         }
1510
1511         return 0;
1512 }
1513
1514 static int devlink_port_split(struct devlink *devlink, u32 port_index,
1515                               u32 count, struct netlink_ext_ack *extack)
1516
1517 {
1518         if (devlink->ops->port_split)
1519                 return devlink->ops->port_split(devlink, port_index, count,
1520                                                 extack);
1521         return -EOPNOTSUPP;
1522 }
1523
1524 static int devlink_nl_cmd_port_split_doit(struct sk_buff *skb,
1525                                           struct genl_info *info)
1526 {
1527         struct devlink *devlink = info->user_ptr[0];
1528         struct devlink_port *devlink_port;
1529         u32 port_index;
1530         u32 count;
1531
1532         if (!info->attrs[DEVLINK_ATTR_PORT_INDEX] ||
1533             !info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT])
1534                 return -EINVAL;
1535
1536         devlink_port = devlink_port_get_from_info(devlink, info);
1537         port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
1538         count = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT]);
1539
1540         if (IS_ERR(devlink_port))
1541                 return -EINVAL;
1542
1543         if (!devlink_port->attrs.splittable) {
1544                 /* Split ports cannot be split. */
1545                 if (devlink_port->attrs.split)
1546                         NL_SET_ERR_MSG_MOD(info->extack, "Port cannot be split further");
1547                 else
1548                         NL_SET_ERR_MSG_MOD(info->extack, "Port cannot be split");
1549                 return -EINVAL;
1550         }
1551
1552         if (count < 2 || !is_power_of_2(count) || count > devlink_port->attrs.lanes) {
1553                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid split count");
1554                 return -EINVAL;
1555         }
1556
1557         return devlink_port_split(devlink, port_index, count, info->extack);
1558 }
1559
1560 static int devlink_port_unsplit(struct devlink *devlink, u32 port_index,
1561                                 struct netlink_ext_ack *extack)
1562
1563 {
1564         if (devlink->ops->port_unsplit)
1565                 return devlink->ops->port_unsplit(devlink, port_index, extack);
1566         return -EOPNOTSUPP;
1567 }
1568
1569 static int devlink_nl_cmd_port_unsplit_doit(struct sk_buff *skb,
1570                                             struct genl_info *info)
1571 {
1572         struct devlink *devlink = info->user_ptr[0];
1573         u32 port_index;
1574
1575         if (!info->attrs[DEVLINK_ATTR_PORT_INDEX])
1576                 return -EINVAL;
1577
1578         port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
1579         return devlink_port_unsplit(devlink, port_index, info->extack);
1580 }
1581
1582 static int devlink_port_new_notifiy(struct devlink *devlink,
1583                                     unsigned int port_index,
1584                                     struct genl_info *info)
1585 {
1586         struct devlink_port *devlink_port;
1587         struct sk_buff *msg;
1588         int err;
1589
1590         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1591         if (!msg)
1592                 return -ENOMEM;
1593
1594         mutex_lock(&devlink->lock);
1595         devlink_port = devlink_port_get_by_index(devlink, port_index);
1596         if (!devlink_port) {
1597                 err = -ENODEV;
1598                 goto out;
1599         }
1600
1601         err = devlink_nl_port_fill(msg, devlink_port, DEVLINK_CMD_NEW,
1602                                    info->snd_portid, info->snd_seq, 0, NULL);
1603         if (err)
1604                 goto out;
1605
1606         err = genlmsg_reply(msg, info);
1607         mutex_unlock(&devlink->lock);
1608         return err;
1609
1610 out:
1611         mutex_unlock(&devlink->lock);
1612         nlmsg_free(msg);
1613         return err;
1614 }
1615
1616 static int devlink_nl_cmd_port_new_doit(struct sk_buff *skb,
1617                                         struct genl_info *info)
1618 {
1619         struct netlink_ext_ack *extack = info->extack;
1620         struct devlink_port_new_attrs new_attrs = {};
1621         struct devlink *devlink = info->user_ptr[0];
1622         unsigned int new_port_index;
1623         int err;
1624
1625         if (!devlink->ops->port_new || !devlink->ops->port_del)
1626                 return -EOPNOTSUPP;
1627
1628         if (!info->attrs[DEVLINK_ATTR_PORT_FLAVOUR] ||
1629             !info->attrs[DEVLINK_ATTR_PORT_PCI_PF_NUMBER]) {
1630                 NL_SET_ERR_MSG_MOD(extack, "Port flavour or PCI PF are not specified");
1631                 return -EINVAL;
1632         }
1633         new_attrs.flavour = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_FLAVOUR]);
1634         new_attrs.pfnum =
1635                 nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_PCI_PF_NUMBER]);
1636
1637         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
1638                 /* Port index of the new port being created by driver. */
1639                 new_attrs.port_index =
1640                         nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
1641                 new_attrs.port_index_valid = true;
1642         }
1643         if (info->attrs[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER]) {
1644                 new_attrs.controller =
1645                         nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER]);
1646                 new_attrs.controller_valid = true;
1647         }
1648         if (new_attrs.flavour == DEVLINK_PORT_FLAVOUR_PCI_SF &&
1649             info->attrs[DEVLINK_ATTR_PORT_PCI_SF_NUMBER]) {
1650                 new_attrs.sfnum = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_PCI_SF_NUMBER]);
1651                 new_attrs.sfnum_valid = true;
1652         }
1653
1654         err = devlink->ops->port_new(devlink, &new_attrs, extack,
1655                                      &new_port_index);
1656         if (err)
1657                 return err;
1658
1659         err = devlink_port_new_notifiy(devlink, new_port_index, info);
1660         if (err && err != -ENODEV) {
1661                 /* Fail to send the response; destroy newly created port. */
1662                 devlink->ops->port_del(devlink, new_port_index, extack);
1663         }
1664         return err;
1665 }
1666
1667 static int devlink_nl_cmd_port_del_doit(struct sk_buff *skb,
1668                                         struct genl_info *info)
1669 {
1670         struct netlink_ext_ack *extack = info->extack;
1671         struct devlink *devlink = info->user_ptr[0];
1672         unsigned int port_index;
1673
1674         if (!devlink->ops->port_del)
1675                 return -EOPNOTSUPP;
1676
1677         if (!info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
1678                 NL_SET_ERR_MSG_MOD(extack, "Port index is not specified");
1679                 return -EINVAL;
1680         }
1681         port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
1682
1683         return devlink->ops->port_del(devlink, port_index, extack);
1684 }
1685
1686 static int
1687 devlink_nl_rate_parent_node_set(struct devlink_rate *devlink_rate,
1688                                 struct genl_info *info,
1689                                 struct nlattr *nla_parent)
1690 {
1691         struct devlink *devlink = devlink_rate->devlink;
1692         const char *parent_name = nla_data(nla_parent);
1693         const struct devlink_ops *ops = devlink->ops;
1694         size_t len = strlen(parent_name);
1695         struct devlink_rate *parent;
1696         int err = -EOPNOTSUPP;
1697
1698         parent = devlink_rate->parent;
1699         if (parent && len) {
1700                 NL_SET_ERR_MSG_MOD(info->extack, "Rate object already has parent.");
1701                 return -EBUSY;
1702         } else if (parent && !len) {
1703                 if (devlink_rate_is_leaf(devlink_rate))
1704                         err = ops->rate_leaf_parent_set(devlink_rate, NULL,
1705                                                         devlink_rate->priv, NULL,
1706                                                         info->extack);
1707                 else if (devlink_rate_is_node(devlink_rate))
1708                         err = ops->rate_node_parent_set(devlink_rate, NULL,
1709                                                         devlink_rate->priv, NULL,
1710                                                         info->extack);
1711                 if (err)
1712                         return err;
1713
1714                 refcount_dec(&parent->refcnt);
1715                 devlink_rate->parent = NULL;
1716         } else if (!parent && len) {
1717                 parent = devlink_rate_node_get_by_name(devlink, parent_name);
1718                 if (IS_ERR(parent))
1719                         return -ENODEV;
1720
1721                 if (parent == devlink_rate) {
1722                         NL_SET_ERR_MSG_MOD(info->extack, "Parent to self is not allowed");
1723                         return -EINVAL;
1724                 }
1725
1726                 if (devlink_rate_is_node(devlink_rate) &&
1727                     devlink_rate_is_parent_node(devlink_rate, parent->parent)) {
1728                         NL_SET_ERR_MSG_MOD(info->extack, "Node is already a parent of parent node.");
1729                         return -EEXIST;
1730                 }
1731
1732                 if (devlink_rate_is_leaf(devlink_rate))
1733                         err = ops->rate_leaf_parent_set(devlink_rate, parent,
1734                                                         devlink_rate->priv, parent->priv,
1735                                                         info->extack);
1736                 else if (devlink_rate_is_node(devlink_rate))
1737                         err = ops->rate_node_parent_set(devlink_rate, parent,
1738                                                         devlink_rate->priv, parent->priv,
1739                                                         info->extack);
1740                 if (err)
1741                         return err;
1742
1743                 refcount_inc(&parent->refcnt);
1744                 devlink_rate->parent = parent;
1745         }
1746
1747         return 0;
1748 }
1749
1750 static int devlink_nl_rate_set(struct devlink_rate *devlink_rate,
1751                                const struct devlink_ops *ops,
1752                                struct genl_info *info)
1753 {
1754         struct nlattr *nla_parent, **attrs = info->attrs;
1755         int err = -EOPNOTSUPP;
1756         u64 rate;
1757
1758         if (attrs[DEVLINK_ATTR_RATE_TX_SHARE]) {
1759                 rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_SHARE]);
1760                 if (devlink_rate_is_leaf(devlink_rate))
1761                         err = ops->rate_leaf_tx_share_set(devlink_rate, devlink_rate->priv,
1762                                                           rate, info->extack);
1763                 else if (devlink_rate_is_node(devlink_rate))
1764                         err = ops->rate_node_tx_share_set(devlink_rate, devlink_rate->priv,
1765                                                           rate, info->extack);
1766                 if (err)
1767                         return err;
1768                 devlink_rate->tx_share = rate;
1769         }
1770
1771         if (attrs[DEVLINK_ATTR_RATE_TX_MAX]) {
1772                 rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_MAX]);
1773                 if (devlink_rate_is_leaf(devlink_rate))
1774                         err = ops->rate_leaf_tx_max_set(devlink_rate, devlink_rate->priv,
1775                                                         rate, info->extack);
1776                 else if (devlink_rate_is_node(devlink_rate))
1777                         err = ops->rate_node_tx_max_set(devlink_rate, devlink_rate->priv,
1778                                                         rate, info->extack);
1779                 if (err)
1780                         return err;
1781                 devlink_rate->tx_max = rate;
1782         }
1783
1784         nla_parent = attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME];
1785         if (nla_parent) {
1786                 err = devlink_nl_rate_parent_node_set(devlink_rate, info,
1787                                                       nla_parent);
1788                 if (err)
1789                         return err;
1790         }
1791
1792         return 0;
1793 }
1794
1795 static bool devlink_rate_set_ops_supported(const struct devlink_ops *ops,
1796                                            struct genl_info *info,
1797                                            enum devlink_rate_type type)
1798 {
1799         struct nlattr **attrs = info->attrs;
1800
1801         if (type == DEVLINK_RATE_TYPE_LEAF) {
1802                 if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_leaf_tx_share_set) {
1803                         NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the leafs");
1804                         return false;
1805                 }
1806                 if (attrs[DEVLINK_ATTR_RATE_TX_MAX] && !ops->rate_leaf_tx_max_set) {
1807                         NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the leafs");
1808                         return false;
1809                 }
1810                 if (attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] &&
1811                     !ops->rate_leaf_parent_set) {
1812                         NL_SET_ERR_MSG_MOD(info->extack, "Parent set isn't supported for the leafs");
1813                         return false;
1814                 }
1815         } else if (type == DEVLINK_RATE_TYPE_NODE) {
1816                 if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_node_tx_share_set) {
1817                         NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the nodes");
1818                         return false;
1819                 }
1820                 if (attrs[DEVLINK_ATTR_RATE_TX_MAX] && !ops->rate_node_tx_max_set) {
1821                         NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the nodes");
1822                         return false;
1823                 }
1824                 if (attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] &&
1825                     !ops->rate_node_parent_set) {
1826                         NL_SET_ERR_MSG_MOD(info->extack, "Parent set isn't supported for the nodes");
1827                         return false;
1828                 }
1829         } else {
1830                 WARN(1, "Unknown type of rate object");
1831                 return false;
1832         }
1833
1834         return true;
1835 }
1836
1837 static int devlink_nl_cmd_rate_set_doit(struct sk_buff *skb,
1838                                         struct genl_info *info)
1839 {
1840         struct devlink_rate *devlink_rate = info->user_ptr[1];
1841         struct devlink *devlink = devlink_rate->devlink;
1842         const struct devlink_ops *ops = devlink->ops;
1843         int err;
1844
1845         if (!ops || !devlink_rate_set_ops_supported(ops, info, devlink_rate->type))
1846                 return -EOPNOTSUPP;
1847
1848         err = devlink_nl_rate_set(devlink_rate, ops, info);
1849
1850         if (!err)
1851                 devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW);
1852         return err;
1853 }
1854
1855 static int devlink_nl_cmd_rate_new_doit(struct sk_buff *skb,
1856                                         struct genl_info *info)
1857 {
1858         struct devlink *devlink = info->user_ptr[0];
1859         struct devlink_rate *rate_node;
1860         const struct devlink_ops *ops;
1861         int err;
1862
1863         ops = devlink->ops;
1864         if (!ops || !ops->rate_node_new || !ops->rate_node_del) {
1865                 NL_SET_ERR_MSG_MOD(info->extack, "Rate nodes aren't supported");
1866                 return -EOPNOTSUPP;
1867         }
1868
1869         if (!devlink_rate_set_ops_supported(ops, info, DEVLINK_RATE_TYPE_NODE))
1870                 return -EOPNOTSUPP;
1871
1872         rate_node = devlink_rate_node_get_from_attrs(devlink, info->attrs);
1873         if (!IS_ERR(rate_node))
1874                 return -EEXIST;
1875         else if (rate_node == ERR_PTR(-EINVAL))
1876                 return -EINVAL;
1877
1878         rate_node = kzalloc(sizeof(*rate_node), GFP_KERNEL);
1879         if (!rate_node)
1880                 return -ENOMEM;
1881
1882         rate_node->devlink = devlink;
1883         rate_node->type = DEVLINK_RATE_TYPE_NODE;
1884         rate_node->name = nla_strdup(info->attrs[DEVLINK_ATTR_RATE_NODE_NAME], GFP_KERNEL);
1885         if (!rate_node->name) {
1886                 err = -ENOMEM;
1887                 goto err_strdup;
1888         }
1889
1890         err = ops->rate_node_new(rate_node, &rate_node->priv, info->extack);
1891         if (err)
1892                 goto err_node_new;
1893
1894         err = devlink_nl_rate_set(rate_node, ops, info);
1895         if (err)
1896                 goto err_rate_set;
1897
1898         refcount_set(&rate_node->refcnt, 1);
1899         list_add(&rate_node->list, &devlink->rate_list);
1900         devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
1901         return 0;
1902
1903 err_rate_set:
1904         ops->rate_node_del(rate_node, rate_node->priv, info->extack);
1905 err_node_new:
1906         kfree(rate_node->name);
1907 err_strdup:
1908         kfree(rate_node);
1909         return err;
1910 }
1911
1912 static int devlink_nl_cmd_rate_del_doit(struct sk_buff *skb,
1913                                         struct genl_info *info)
1914 {
1915         struct devlink_rate *rate_node = info->user_ptr[1];
1916         struct devlink *devlink = rate_node->devlink;
1917         const struct devlink_ops *ops = devlink->ops;
1918         int err;
1919
1920         if (refcount_read(&rate_node->refcnt) > 1) {
1921                 NL_SET_ERR_MSG_MOD(info->extack, "Node has children. Cannot delete node.");
1922                 return -EBUSY;
1923         }
1924
1925         devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
1926         err = ops->rate_node_del(rate_node, rate_node->priv, info->extack);
1927         if (rate_node->parent)
1928                 refcount_dec(&rate_node->parent->refcnt);
1929         list_del(&rate_node->list);
1930         kfree(rate_node->name);
1931         kfree(rate_node);
1932         return err;
1933 }
1934
1935 static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink,
1936                               struct devlink_sb *devlink_sb,
1937                               enum devlink_command cmd, u32 portid,
1938                               u32 seq, int flags)
1939 {
1940         void *hdr;
1941
1942         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1943         if (!hdr)
1944                 return -EMSGSIZE;
1945
1946         if (devlink_nl_put_handle(msg, devlink))
1947                 goto nla_put_failure;
1948         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
1949                 goto nla_put_failure;
1950         if (nla_put_u32(msg, DEVLINK_ATTR_SB_SIZE, devlink_sb->size))
1951                 goto nla_put_failure;
1952         if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_POOL_COUNT,
1953                         devlink_sb->ingress_pools_count))
1954                 goto nla_put_failure;
1955         if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_POOL_COUNT,
1956                         devlink_sb->egress_pools_count))
1957                 goto nla_put_failure;
1958         if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_TC_COUNT,
1959                         devlink_sb->ingress_tc_count))
1960                 goto nla_put_failure;
1961         if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_TC_COUNT,
1962                         devlink_sb->egress_tc_count))
1963                 goto nla_put_failure;
1964
1965         genlmsg_end(msg, hdr);
1966         return 0;
1967
1968 nla_put_failure:
1969         genlmsg_cancel(msg, hdr);
1970         return -EMSGSIZE;
1971 }
1972
1973 static int devlink_nl_cmd_sb_get_doit(struct sk_buff *skb,
1974                                       struct genl_info *info)
1975 {
1976         struct devlink *devlink = info->user_ptr[0];
1977         struct devlink_sb *devlink_sb;
1978         struct sk_buff *msg;
1979         int err;
1980
1981         devlink_sb = devlink_sb_get_from_info(devlink, info);
1982         if (IS_ERR(devlink_sb))
1983                 return PTR_ERR(devlink_sb);
1984
1985         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1986         if (!msg)
1987                 return -ENOMEM;
1988
1989         err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
1990                                  DEVLINK_CMD_SB_NEW,
1991                                  info->snd_portid, info->snd_seq, 0);
1992         if (err) {
1993                 nlmsg_free(msg);
1994                 return err;
1995         }
1996
1997         return genlmsg_reply(msg, info);
1998 }
1999
2000 static int devlink_nl_cmd_sb_get_dumpit(struct sk_buff *msg,
2001                                         struct netlink_callback *cb)
2002 {
2003         struct devlink *devlink;
2004         struct devlink_sb *devlink_sb;
2005         int start = cb->args[0];
2006         unsigned long index;
2007         int idx = 0;
2008         int err;
2009
2010         mutex_lock(&devlink_mutex);
2011         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2012                 if (!devlink_try_get(devlink))
2013                         continue;
2014
2015                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
2016                         goto retry;
2017
2018                 mutex_lock(&devlink->lock);
2019                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
2020                         if (idx < start) {
2021                                 idx++;
2022                                 continue;
2023                         }
2024                         err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
2025                                                  DEVLINK_CMD_SB_NEW,
2026                                                  NETLINK_CB(cb->skb).portid,
2027                                                  cb->nlh->nlmsg_seq,
2028                                                  NLM_F_MULTI);
2029                         if (err) {
2030                                 mutex_unlock(&devlink->lock);
2031                                 devlink_put(devlink);
2032                                 goto out;
2033                         }
2034                         idx++;
2035                 }
2036                 mutex_unlock(&devlink->lock);
2037 retry:
2038                 devlink_put(devlink);
2039         }
2040 out:
2041         mutex_unlock(&devlink_mutex);
2042
2043         cb->args[0] = idx;
2044         return msg->len;
2045 }
2046
2047 static int devlink_nl_sb_pool_fill(struct sk_buff *msg, struct devlink *devlink,
2048                                    struct devlink_sb *devlink_sb,
2049                                    u16 pool_index, enum devlink_command cmd,
2050                                    u32 portid, u32 seq, int flags)
2051 {
2052         struct devlink_sb_pool_info pool_info;
2053         void *hdr;
2054         int err;
2055
2056         err = devlink->ops->sb_pool_get(devlink, devlink_sb->index,
2057                                         pool_index, &pool_info);
2058         if (err)
2059                 return err;
2060
2061         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2062         if (!hdr)
2063                 return -EMSGSIZE;
2064
2065         if (devlink_nl_put_handle(msg, devlink))
2066                 goto nla_put_failure;
2067         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
2068                 goto nla_put_failure;
2069         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
2070                 goto nla_put_failure;
2071         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_info.pool_type))
2072                 goto nla_put_failure;
2073         if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_SIZE, pool_info.size))
2074                 goto nla_put_failure;
2075         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE,
2076                        pool_info.threshold_type))
2077                 goto nla_put_failure;
2078         if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_CELL_SIZE,
2079                         pool_info.cell_size))
2080                 goto nla_put_failure;
2081
2082         genlmsg_end(msg, hdr);
2083         return 0;
2084
2085 nla_put_failure:
2086         genlmsg_cancel(msg, hdr);
2087         return -EMSGSIZE;
2088 }
2089
2090 static int devlink_nl_cmd_sb_pool_get_doit(struct sk_buff *skb,
2091                                            struct genl_info *info)
2092 {
2093         struct devlink *devlink = info->user_ptr[0];
2094         struct devlink_sb *devlink_sb;
2095         struct sk_buff *msg;
2096         u16 pool_index;
2097         int err;
2098
2099         devlink_sb = devlink_sb_get_from_info(devlink, info);
2100         if (IS_ERR(devlink_sb))
2101                 return PTR_ERR(devlink_sb);
2102
2103         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2104                                                   &pool_index);
2105         if (err)
2106                 return err;
2107
2108         if (!devlink->ops->sb_pool_get)
2109                 return -EOPNOTSUPP;
2110
2111         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2112         if (!msg)
2113                 return -ENOMEM;
2114
2115         err = devlink_nl_sb_pool_fill(msg, devlink, devlink_sb, pool_index,
2116                                       DEVLINK_CMD_SB_POOL_NEW,
2117                                       info->snd_portid, info->snd_seq, 0);
2118         if (err) {
2119                 nlmsg_free(msg);
2120                 return err;
2121         }
2122
2123         return genlmsg_reply(msg, info);
2124 }
2125
2126 static int __sb_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
2127                                 struct devlink *devlink,
2128                                 struct devlink_sb *devlink_sb,
2129                                 u32 portid, u32 seq)
2130 {
2131         u16 pool_count = devlink_sb_pool_count(devlink_sb);
2132         u16 pool_index;
2133         int err;
2134
2135         for (pool_index = 0; pool_index < pool_count; pool_index++) {
2136                 if (*p_idx < start) {
2137                         (*p_idx)++;
2138                         continue;
2139                 }
2140                 err = devlink_nl_sb_pool_fill(msg, devlink,
2141                                               devlink_sb,
2142                                               pool_index,
2143                                               DEVLINK_CMD_SB_POOL_NEW,
2144                                               portid, seq, NLM_F_MULTI);
2145                 if (err)
2146                         return err;
2147                 (*p_idx)++;
2148         }
2149         return 0;
2150 }
2151
2152 static int devlink_nl_cmd_sb_pool_get_dumpit(struct sk_buff *msg,
2153                                              struct netlink_callback *cb)
2154 {
2155         struct devlink *devlink;
2156         struct devlink_sb *devlink_sb;
2157         int start = cb->args[0];
2158         unsigned long index;
2159         int idx = 0;
2160         int err = 0;
2161
2162         mutex_lock(&devlink_mutex);
2163         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2164                 if (!devlink_try_get(devlink))
2165                         continue;
2166
2167                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
2168                     !devlink->ops->sb_pool_get)
2169                         goto retry;
2170
2171                 mutex_lock(&devlink->lock);
2172                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
2173                         err = __sb_pool_get_dumpit(msg, start, &idx, devlink,
2174                                                    devlink_sb,
2175                                                    NETLINK_CB(cb->skb).portid,
2176                                                    cb->nlh->nlmsg_seq);
2177                         if (err == -EOPNOTSUPP) {
2178                                 err = 0;
2179                         } else if (err) {
2180                                 mutex_unlock(&devlink->lock);
2181                                 devlink_put(devlink);
2182                                 goto out;
2183                         }
2184                 }
2185                 mutex_unlock(&devlink->lock);
2186 retry:
2187                 devlink_put(devlink);
2188         }
2189 out:
2190         mutex_unlock(&devlink_mutex);
2191
2192         if (err != -EMSGSIZE)
2193                 return err;
2194
2195         cb->args[0] = idx;
2196         return msg->len;
2197 }
2198
2199 static int devlink_sb_pool_set(struct devlink *devlink, unsigned int sb_index,
2200                                u16 pool_index, u32 size,
2201                                enum devlink_sb_threshold_type threshold_type,
2202                                struct netlink_ext_ack *extack)
2203
2204 {
2205         const struct devlink_ops *ops = devlink->ops;
2206
2207         if (ops->sb_pool_set)
2208                 return ops->sb_pool_set(devlink, sb_index, pool_index,
2209                                         size, threshold_type, extack);
2210         return -EOPNOTSUPP;
2211 }
2212
2213 static int devlink_nl_cmd_sb_pool_set_doit(struct sk_buff *skb,
2214                                            struct genl_info *info)
2215 {
2216         struct devlink *devlink = info->user_ptr[0];
2217         enum devlink_sb_threshold_type threshold_type;
2218         struct devlink_sb *devlink_sb;
2219         u16 pool_index;
2220         u32 size;
2221         int err;
2222
2223         devlink_sb = devlink_sb_get_from_info(devlink, info);
2224         if (IS_ERR(devlink_sb))
2225                 return PTR_ERR(devlink_sb);
2226
2227         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2228                                                   &pool_index);
2229         if (err)
2230                 return err;
2231
2232         err = devlink_sb_th_type_get_from_info(info, &threshold_type);
2233         if (err)
2234                 return err;
2235
2236         if (!info->attrs[DEVLINK_ATTR_SB_POOL_SIZE])
2237                 return -EINVAL;
2238
2239         size = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_POOL_SIZE]);
2240         return devlink_sb_pool_set(devlink, devlink_sb->index,
2241                                    pool_index, size, threshold_type,
2242                                    info->extack);
2243 }
2244
2245 static int devlink_nl_sb_port_pool_fill(struct sk_buff *msg,
2246                                         struct devlink *devlink,
2247                                         struct devlink_port *devlink_port,
2248                                         struct devlink_sb *devlink_sb,
2249                                         u16 pool_index,
2250                                         enum devlink_command cmd,
2251                                         u32 portid, u32 seq, int flags)
2252 {
2253         const struct devlink_ops *ops = devlink->ops;
2254         u32 threshold;
2255         void *hdr;
2256         int err;
2257
2258         err = ops->sb_port_pool_get(devlink_port, devlink_sb->index,
2259                                     pool_index, &threshold);
2260         if (err)
2261                 return err;
2262
2263         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2264         if (!hdr)
2265                 return -EMSGSIZE;
2266
2267         if (devlink_nl_put_handle(msg, devlink))
2268                 goto nla_put_failure;
2269         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
2270                 goto nla_put_failure;
2271         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
2272                 goto nla_put_failure;
2273         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
2274                 goto nla_put_failure;
2275         if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
2276                 goto nla_put_failure;
2277
2278         if (ops->sb_occ_port_pool_get) {
2279                 u32 cur;
2280                 u32 max;
2281
2282                 err = ops->sb_occ_port_pool_get(devlink_port, devlink_sb->index,
2283                                                 pool_index, &cur, &max);
2284                 if (err && err != -EOPNOTSUPP)
2285                         goto sb_occ_get_failure;
2286                 if (!err) {
2287                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
2288                                 goto nla_put_failure;
2289                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
2290                                 goto nla_put_failure;
2291                 }
2292         }
2293
2294         genlmsg_end(msg, hdr);
2295         return 0;
2296
2297 nla_put_failure:
2298         err = -EMSGSIZE;
2299 sb_occ_get_failure:
2300         genlmsg_cancel(msg, hdr);
2301         return err;
2302 }
2303
2304 static int devlink_nl_cmd_sb_port_pool_get_doit(struct sk_buff *skb,
2305                                                 struct genl_info *info)
2306 {
2307         struct devlink_port *devlink_port = info->user_ptr[1];
2308         struct devlink *devlink = devlink_port->devlink;
2309         struct devlink_sb *devlink_sb;
2310         struct sk_buff *msg;
2311         u16 pool_index;
2312         int err;
2313
2314         devlink_sb = devlink_sb_get_from_info(devlink, info);
2315         if (IS_ERR(devlink_sb))
2316                 return PTR_ERR(devlink_sb);
2317
2318         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2319                                                   &pool_index);
2320         if (err)
2321                 return err;
2322
2323         if (!devlink->ops->sb_port_pool_get)
2324                 return -EOPNOTSUPP;
2325
2326         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2327         if (!msg)
2328                 return -ENOMEM;
2329
2330         err = devlink_nl_sb_port_pool_fill(msg, devlink, devlink_port,
2331                                            devlink_sb, pool_index,
2332                                            DEVLINK_CMD_SB_PORT_POOL_NEW,
2333                                            info->snd_portid, info->snd_seq, 0);
2334         if (err) {
2335                 nlmsg_free(msg);
2336                 return err;
2337         }
2338
2339         return genlmsg_reply(msg, info);
2340 }
2341
2342 static int __sb_port_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
2343                                      struct devlink *devlink,
2344                                      struct devlink_sb *devlink_sb,
2345                                      u32 portid, u32 seq)
2346 {
2347         struct devlink_port *devlink_port;
2348         u16 pool_count = devlink_sb_pool_count(devlink_sb);
2349         u16 pool_index;
2350         int err;
2351
2352         list_for_each_entry(devlink_port, &devlink->port_list, list) {
2353                 for (pool_index = 0; pool_index < pool_count; pool_index++) {
2354                         if (*p_idx < start) {
2355                                 (*p_idx)++;
2356                                 continue;
2357                         }
2358                         err = devlink_nl_sb_port_pool_fill(msg, devlink,
2359                                                            devlink_port,
2360                                                            devlink_sb,
2361                                                            pool_index,
2362                                                            DEVLINK_CMD_SB_PORT_POOL_NEW,
2363                                                            portid, seq,
2364                                                            NLM_F_MULTI);
2365                         if (err)
2366                                 return err;
2367                         (*p_idx)++;
2368                 }
2369         }
2370         return 0;
2371 }
2372
2373 static int devlink_nl_cmd_sb_port_pool_get_dumpit(struct sk_buff *msg,
2374                                                   struct netlink_callback *cb)
2375 {
2376         struct devlink *devlink;
2377         struct devlink_sb *devlink_sb;
2378         int start = cb->args[0];
2379         unsigned long index;
2380         int idx = 0;
2381         int err = 0;
2382
2383         mutex_lock(&devlink_mutex);
2384         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2385                 if (!devlink_try_get(devlink))
2386                         continue;
2387
2388                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
2389                     !devlink->ops->sb_port_pool_get)
2390                         goto retry;
2391
2392                 mutex_lock(&devlink->lock);
2393                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
2394                         err = __sb_port_pool_get_dumpit(msg, start, &idx,
2395                                                         devlink, devlink_sb,
2396                                                         NETLINK_CB(cb->skb).portid,
2397                                                         cb->nlh->nlmsg_seq);
2398                         if (err == -EOPNOTSUPP) {
2399                                 err = 0;
2400                         } else if (err) {
2401                                 mutex_unlock(&devlink->lock);
2402                                 devlink_put(devlink);
2403                                 goto out;
2404                         }
2405                 }
2406                 mutex_unlock(&devlink->lock);
2407 retry:
2408                 devlink_put(devlink);
2409         }
2410 out:
2411         mutex_unlock(&devlink_mutex);
2412
2413         if (err != -EMSGSIZE)
2414                 return err;
2415
2416         cb->args[0] = idx;
2417         return msg->len;
2418 }
2419
2420 static int devlink_sb_port_pool_set(struct devlink_port *devlink_port,
2421                                     unsigned int sb_index, u16 pool_index,
2422                                     u32 threshold,
2423                                     struct netlink_ext_ack *extack)
2424
2425 {
2426         const struct devlink_ops *ops = devlink_port->devlink->ops;
2427
2428         if (ops->sb_port_pool_set)
2429                 return ops->sb_port_pool_set(devlink_port, sb_index,
2430                                              pool_index, threshold, extack);
2431         return -EOPNOTSUPP;
2432 }
2433
2434 static int devlink_nl_cmd_sb_port_pool_set_doit(struct sk_buff *skb,
2435                                                 struct genl_info *info)
2436 {
2437         struct devlink_port *devlink_port = info->user_ptr[1];
2438         struct devlink *devlink = info->user_ptr[0];
2439         struct devlink_sb *devlink_sb;
2440         u16 pool_index;
2441         u32 threshold;
2442         int err;
2443
2444         devlink_sb = devlink_sb_get_from_info(devlink, info);
2445         if (IS_ERR(devlink_sb))
2446                 return PTR_ERR(devlink_sb);
2447
2448         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2449                                                   &pool_index);
2450         if (err)
2451                 return err;
2452
2453         if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
2454                 return -EINVAL;
2455
2456         threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
2457         return devlink_sb_port_pool_set(devlink_port, devlink_sb->index,
2458                                         pool_index, threshold, info->extack);
2459 }
2460
2461 static int
2462 devlink_nl_sb_tc_pool_bind_fill(struct sk_buff *msg, struct devlink *devlink,
2463                                 struct devlink_port *devlink_port,
2464                                 struct devlink_sb *devlink_sb, u16 tc_index,
2465                                 enum devlink_sb_pool_type pool_type,
2466                                 enum devlink_command cmd,
2467                                 u32 portid, u32 seq, int flags)
2468 {
2469         const struct devlink_ops *ops = devlink->ops;
2470         u16 pool_index;
2471         u32 threshold;
2472         void *hdr;
2473         int err;
2474
2475         err = ops->sb_tc_pool_bind_get(devlink_port, devlink_sb->index,
2476                                        tc_index, pool_type,
2477                                        &pool_index, &threshold);
2478         if (err)
2479                 return err;
2480
2481         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2482         if (!hdr)
2483                 return -EMSGSIZE;
2484
2485         if (devlink_nl_put_handle(msg, devlink))
2486                 goto nla_put_failure;
2487         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
2488                 goto nla_put_failure;
2489         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
2490                 goto nla_put_failure;
2491         if (nla_put_u16(msg, DEVLINK_ATTR_SB_TC_INDEX, tc_index))
2492                 goto nla_put_failure;
2493         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_type))
2494                 goto nla_put_failure;
2495         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
2496                 goto nla_put_failure;
2497         if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
2498                 goto nla_put_failure;
2499
2500         if (ops->sb_occ_tc_port_bind_get) {
2501                 u32 cur;
2502                 u32 max;
2503
2504                 err = ops->sb_occ_tc_port_bind_get(devlink_port,
2505                                                    devlink_sb->index,
2506                                                    tc_index, pool_type,
2507                                                    &cur, &max);
2508                 if (err && err != -EOPNOTSUPP)
2509                         return err;
2510                 if (!err) {
2511                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
2512                                 goto nla_put_failure;
2513                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
2514                                 goto nla_put_failure;
2515                 }
2516         }
2517
2518         genlmsg_end(msg, hdr);
2519         return 0;
2520
2521 nla_put_failure:
2522         genlmsg_cancel(msg, hdr);
2523         return -EMSGSIZE;
2524 }
2525
2526 static int devlink_nl_cmd_sb_tc_pool_bind_get_doit(struct sk_buff *skb,
2527                                                    struct genl_info *info)
2528 {
2529         struct devlink_port *devlink_port = info->user_ptr[1];
2530         struct devlink *devlink = devlink_port->devlink;
2531         struct devlink_sb *devlink_sb;
2532         struct sk_buff *msg;
2533         enum devlink_sb_pool_type pool_type;
2534         u16 tc_index;
2535         int err;
2536
2537         devlink_sb = devlink_sb_get_from_info(devlink, info);
2538         if (IS_ERR(devlink_sb))
2539                 return PTR_ERR(devlink_sb);
2540
2541         err = devlink_sb_pool_type_get_from_info(info, &pool_type);
2542         if (err)
2543                 return err;
2544
2545         err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
2546                                                 pool_type, &tc_index);
2547         if (err)
2548                 return err;
2549
2550         if (!devlink->ops->sb_tc_pool_bind_get)
2551                 return -EOPNOTSUPP;
2552
2553         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2554         if (!msg)
2555                 return -ENOMEM;
2556
2557         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink, devlink_port,
2558                                               devlink_sb, tc_index, pool_type,
2559                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
2560                                               info->snd_portid,
2561                                               info->snd_seq, 0);
2562         if (err) {
2563                 nlmsg_free(msg);
2564                 return err;
2565         }
2566
2567         return genlmsg_reply(msg, info);
2568 }
2569
2570 static int __sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
2571                                         int start, int *p_idx,
2572                                         struct devlink *devlink,
2573                                         struct devlink_sb *devlink_sb,
2574                                         u32 portid, u32 seq)
2575 {
2576         struct devlink_port *devlink_port;
2577         u16 tc_index;
2578         int err;
2579
2580         list_for_each_entry(devlink_port, &devlink->port_list, list) {
2581                 for (tc_index = 0;
2582                      tc_index < devlink_sb->ingress_tc_count; tc_index++) {
2583                         if (*p_idx < start) {
2584                                 (*p_idx)++;
2585                                 continue;
2586                         }
2587                         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
2588                                                               devlink_port,
2589                                                               devlink_sb,
2590                                                               tc_index,
2591                                                               DEVLINK_SB_POOL_TYPE_INGRESS,
2592                                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
2593                                                               portid, seq,
2594                                                               NLM_F_MULTI);
2595                         if (err)
2596                                 return err;
2597                         (*p_idx)++;
2598                 }
2599                 for (tc_index = 0;
2600                      tc_index < devlink_sb->egress_tc_count; tc_index++) {
2601                         if (*p_idx < start) {
2602                                 (*p_idx)++;
2603                                 continue;
2604                         }
2605                         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
2606                                                               devlink_port,
2607                                                               devlink_sb,
2608                                                               tc_index,
2609                                                               DEVLINK_SB_POOL_TYPE_EGRESS,
2610                                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
2611                                                               portid, seq,
2612                                                               NLM_F_MULTI);
2613                         if (err)
2614                                 return err;
2615                         (*p_idx)++;
2616                 }
2617         }
2618         return 0;
2619 }
2620
2621 static int
2622 devlink_nl_cmd_sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
2623                                           struct netlink_callback *cb)
2624 {
2625         struct devlink *devlink;
2626         struct devlink_sb *devlink_sb;
2627         int start = cb->args[0];
2628         unsigned long index;
2629         int idx = 0;
2630         int err = 0;
2631
2632         mutex_lock(&devlink_mutex);
2633         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2634                 if (!devlink_try_get(devlink))
2635                         continue;
2636
2637                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
2638                     !devlink->ops->sb_tc_pool_bind_get)
2639                         goto retry;
2640
2641                 mutex_lock(&devlink->lock);
2642                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
2643                         err = __sb_tc_pool_bind_get_dumpit(msg, start, &idx,
2644                                                            devlink,
2645                                                            devlink_sb,
2646                                                            NETLINK_CB(cb->skb).portid,
2647                                                            cb->nlh->nlmsg_seq);
2648                         if (err == -EOPNOTSUPP) {
2649                                 err = 0;
2650                         } else if (err) {
2651                                 mutex_unlock(&devlink->lock);
2652                                 devlink_put(devlink);
2653                                 goto out;
2654                         }
2655                 }
2656                 mutex_unlock(&devlink->lock);
2657 retry:
2658                 devlink_put(devlink);
2659         }
2660 out:
2661         mutex_unlock(&devlink_mutex);
2662
2663         if (err != -EMSGSIZE)
2664                 return err;
2665
2666         cb->args[0] = idx;
2667         return msg->len;
2668 }
2669
2670 static int devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
2671                                        unsigned int sb_index, u16 tc_index,
2672                                        enum devlink_sb_pool_type pool_type,
2673                                        u16 pool_index, u32 threshold,
2674                                        struct netlink_ext_ack *extack)
2675
2676 {
2677         const struct devlink_ops *ops = devlink_port->devlink->ops;
2678
2679         if (ops->sb_tc_pool_bind_set)
2680                 return ops->sb_tc_pool_bind_set(devlink_port, sb_index,
2681                                                 tc_index, pool_type,
2682                                                 pool_index, threshold, extack);
2683         return -EOPNOTSUPP;
2684 }
2685
2686 static int devlink_nl_cmd_sb_tc_pool_bind_set_doit(struct sk_buff *skb,
2687                                                    struct genl_info *info)
2688 {
2689         struct devlink_port *devlink_port = info->user_ptr[1];
2690         struct devlink *devlink = info->user_ptr[0];
2691         enum devlink_sb_pool_type pool_type;
2692         struct devlink_sb *devlink_sb;
2693         u16 tc_index;
2694         u16 pool_index;
2695         u32 threshold;
2696         int err;
2697
2698         devlink_sb = devlink_sb_get_from_info(devlink, info);
2699         if (IS_ERR(devlink_sb))
2700                 return PTR_ERR(devlink_sb);
2701
2702         err = devlink_sb_pool_type_get_from_info(info, &pool_type);
2703         if (err)
2704                 return err;
2705
2706         err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
2707                                                 pool_type, &tc_index);
2708         if (err)
2709                 return err;
2710
2711         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2712                                                   &pool_index);
2713         if (err)
2714                 return err;
2715
2716         if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
2717                 return -EINVAL;
2718
2719         threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
2720         return devlink_sb_tc_pool_bind_set(devlink_port, devlink_sb->index,
2721                                            tc_index, pool_type,
2722                                            pool_index, threshold, info->extack);
2723 }
2724
2725 static int devlink_nl_cmd_sb_occ_snapshot_doit(struct sk_buff *skb,
2726                                                struct genl_info *info)
2727 {
2728         struct devlink *devlink = info->user_ptr[0];
2729         const struct devlink_ops *ops = devlink->ops;
2730         struct devlink_sb *devlink_sb;
2731
2732         devlink_sb = devlink_sb_get_from_info(devlink, info);
2733         if (IS_ERR(devlink_sb))
2734                 return PTR_ERR(devlink_sb);
2735
2736         if (ops->sb_occ_snapshot)
2737                 return ops->sb_occ_snapshot(devlink, devlink_sb->index);
2738         return -EOPNOTSUPP;
2739 }
2740
2741 static int devlink_nl_cmd_sb_occ_max_clear_doit(struct sk_buff *skb,
2742                                                 struct genl_info *info)
2743 {
2744         struct devlink *devlink = info->user_ptr[0];
2745         const struct devlink_ops *ops = devlink->ops;
2746         struct devlink_sb *devlink_sb;
2747
2748         devlink_sb = devlink_sb_get_from_info(devlink, info);
2749         if (IS_ERR(devlink_sb))
2750                 return PTR_ERR(devlink_sb);
2751
2752         if (ops->sb_occ_max_clear)
2753                 return ops->sb_occ_max_clear(devlink, devlink_sb->index);
2754         return -EOPNOTSUPP;
2755 }
2756
2757 static int devlink_nl_eswitch_fill(struct sk_buff *msg, struct devlink *devlink,
2758                                    enum devlink_command cmd, u32 portid,
2759                                    u32 seq, int flags)
2760 {
2761         const struct devlink_ops *ops = devlink->ops;
2762         enum devlink_eswitch_encap_mode encap_mode;
2763         u8 inline_mode;
2764         void *hdr;
2765         int err = 0;
2766         u16 mode;
2767
2768         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2769         if (!hdr)
2770                 return -EMSGSIZE;
2771
2772         err = devlink_nl_put_handle(msg, devlink);
2773         if (err)
2774                 goto nla_put_failure;
2775
2776         if (ops->eswitch_mode_get) {
2777                 err = ops->eswitch_mode_get(devlink, &mode);
2778                 if (err)
2779                         goto nla_put_failure;
2780                 err = nla_put_u16(msg, DEVLINK_ATTR_ESWITCH_MODE, mode);
2781                 if (err)
2782                         goto nla_put_failure;
2783         }
2784
2785         if (ops->eswitch_inline_mode_get) {
2786                 err = ops->eswitch_inline_mode_get(devlink, &inline_mode);
2787                 if (err)
2788                         goto nla_put_failure;
2789                 err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_INLINE_MODE,
2790                                  inline_mode);
2791                 if (err)
2792                         goto nla_put_failure;
2793         }
2794
2795         if (ops->eswitch_encap_mode_get) {
2796                 err = ops->eswitch_encap_mode_get(devlink, &encap_mode);
2797                 if (err)
2798                         goto nla_put_failure;
2799                 err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_ENCAP_MODE, encap_mode);
2800                 if (err)
2801                         goto nla_put_failure;
2802         }
2803
2804         genlmsg_end(msg, hdr);
2805         return 0;
2806
2807 nla_put_failure:
2808         genlmsg_cancel(msg, hdr);
2809         return err;
2810 }
2811
2812 static int devlink_nl_cmd_eswitch_get_doit(struct sk_buff *skb,
2813                                            struct genl_info *info)
2814 {
2815         struct devlink *devlink = info->user_ptr[0];
2816         struct sk_buff *msg;
2817         int err;
2818
2819         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2820         if (!msg)
2821                 return -ENOMEM;
2822
2823         err = devlink_nl_eswitch_fill(msg, devlink, DEVLINK_CMD_ESWITCH_GET,
2824                                       info->snd_portid, info->snd_seq, 0);
2825
2826         if (err) {
2827                 nlmsg_free(msg);
2828                 return err;
2829         }
2830
2831         return genlmsg_reply(msg, info);
2832 }
2833
2834 static int devlink_rate_nodes_check(struct devlink *devlink, u16 mode,
2835                                     struct netlink_ext_ack *extack)
2836 {
2837         struct devlink_rate *devlink_rate;
2838
2839         /* Take the lock to sync with devlink_rate_nodes_destroy() */
2840         mutex_lock(&devlink->lock);
2841         list_for_each_entry(devlink_rate, &devlink->rate_list, list)
2842                 if (devlink_rate_is_node(devlink_rate)) {
2843                         mutex_unlock(&devlink->lock);
2844                         NL_SET_ERR_MSG_MOD(extack, "Rate node(s) exists.");
2845                         return -EBUSY;
2846                 }
2847         mutex_unlock(&devlink->lock);
2848         return 0;
2849 }
2850
2851 static int devlink_nl_cmd_eswitch_set_doit(struct sk_buff *skb,
2852                                            struct genl_info *info)
2853 {
2854         struct devlink *devlink = info->user_ptr[0];
2855         const struct devlink_ops *ops = devlink->ops;
2856         enum devlink_eswitch_encap_mode encap_mode;
2857         u8 inline_mode;
2858         int err = 0;
2859         u16 mode;
2860
2861         if (info->attrs[DEVLINK_ATTR_ESWITCH_MODE]) {
2862                 if (!ops->eswitch_mode_set)
2863                         return -EOPNOTSUPP;
2864                 mode = nla_get_u16(info->attrs[DEVLINK_ATTR_ESWITCH_MODE]);
2865                 err = devlink_rate_nodes_check(devlink, mode, info->extack);
2866                 if (err)
2867                         return err;
2868                 err = ops->eswitch_mode_set(devlink, mode, info->extack);
2869                 if (err)
2870                         return err;
2871         }
2872
2873         if (info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]) {
2874                 if (!ops->eswitch_inline_mode_set)
2875                         return -EOPNOTSUPP;
2876                 inline_mode = nla_get_u8(
2877                                 info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]);
2878                 err = ops->eswitch_inline_mode_set(devlink, inline_mode,
2879                                                    info->extack);
2880                 if (err)
2881                         return err;
2882         }
2883
2884         if (info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]) {
2885                 if (!ops->eswitch_encap_mode_set)
2886                         return -EOPNOTSUPP;
2887                 encap_mode = nla_get_u8(info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]);
2888                 err = ops->eswitch_encap_mode_set(devlink, encap_mode,
2889                                                   info->extack);
2890                 if (err)
2891                         return err;
2892         }
2893
2894         return 0;
2895 }
2896
2897 int devlink_dpipe_match_put(struct sk_buff *skb,
2898                             struct devlink_dpipe_match *match)
2899 {
2900         struct devlink_dpipe_header *header = match->header;
2901         struct devlink_dpipe_field *field = &header->fields[match->field_id];
2902         struct nlattr *match_attr;
2903
2904         match_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_MATCH);
2905         if (!match_attr)
2906                 return -EMSGSIZE;
2907
2908         if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_MATCH_TYPE, match->type) ||
2909             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, match->header_index) ||
2910             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
2911             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
2912             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
2913                 goto nla_put_failure;
2914
2915         nla_nest_end(skb, match_attr);
2916         return 0;
2917
2918 nla_put_failure:
2919         nla_nest_cancel(skb, match_attr);
2920         return -EMSGSIZE;
2921 }
2922 EXPORT_SYMBOL_GPL(devlink_dpipe_match_put);
2923
2924 static int devlink_dpipe_matches_put(struct devlink_dpipe_table *table,
2925                                      struct sk_buff *skb)
2926 {
2927         struct nlattr *matches_attr;
2928
2929         matches_attr = nla_nest_start_noflag(skb,
2930                                              DEVLINK_ATTR_DPIPE_TABLE_MATCHES);
2931         if (!matches_attr)
2932                 return -EMSGSIZE;
2933
2934         if (table->table_ops->matches_dump(table->priv, skb))
2935                 goto nla_put_failure;
2936
2937         nla_nest_end(skb, matches_attr);
2938         return 0;
2939
2940 nla_put_failure:
2941         nla_nest_cancel(skb, matches_attr);
2942         return -EMSGSIZE;
2943 }
2944
2945 int devlink_dpipe_action_put(struct sk_buff *skb,
2946                              struct devlink_dpipe_action *action)
2947 {
2948         struct devlink_dpipe_header *header = action->header;
2949         struct devlink_dpipe_field *field = &header->fields[action->field_id];
2950         struct nlattr *action_attr;
2951
2952         action_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_ACTION);
2953         if (!action_attr)
2954                 return -EMSGSIZE;
2955
2956         if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_ACTION_TYPE, action->type) ||
2957             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, action->header_index) ||
2958             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
2959             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
2960             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
2961                 goto nla_put_failure;
2962
2963         nla_nest_end(skb, action_attr);
2964         return 0;
2965
2966 nla_put_failure:
2967         nla_nest_cancel(skb, action_attr);
2968         return -EMSGSIZE;
2969 }
2970 EXPORT_SYMBOL_GPL(devlink_dpipe_action_put);
2971
2972 static int devlink_dpipe_actions_put(struct devlink_dpipe_table *table,
2973                                      struct sk_buff *skb)
2974 {
2975         struct nlattr *actions_attr;
2976
2977         actions_attr = nla_nest_start_noflag(skb,
2978                                              DEVLINK_ATTR_DPIPE_TABLE_ACTIONS);
2979         if (!actions_attr)
2980                 return -EMSGSIZE;
2981
2982         if (table->table_ops->actions_dump(table->priv, skb))
2983                 goto nla_put_failure;
2984
2985         nla_nest_end(skb, actions_attr);
2986         return 0;
2987
2988 nla_put_failure:
2989         nla_nest_cancel(skb, actions_attr);
2990         return -EMSGSIZE;
2991 }
2992
2993 static int devlink_dpipe_table_put(struct sk_buff *skb,
2994                                    struct devlink_dpipe_table *table)
2995 {
2996         struct nlattr *table_attr;
2997         u64 table_size;
2998
2999         table_size = table->table_ops->size_get(table->priv);
3000         table_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_TABLE);
3001         if (!table_attr)
3002                 return -EMSGSIZE;
3003
3004         if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_TABLE_NAME, table->name) ||
3005             nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_SIZE, table_size,
3006                               DEVLINK_ATTR_PAD))
3007                 goto nla_put_failure;
3008         if (nla_put_u8(skb, DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED,
3009                        table->counters_enabled))
3010                 goto nla_put_failure;
3011
3012         if (table->resource_valid) {
3013                 if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_ID,
3014                                       table->resource_id, DEVLINK_ATTR_PAD) ||
3015                     nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_UNITS,
3016                                       table->resource_units, DEVLINK_ATTR_PAD))
3017                         goto nla_put_failure;
3018         }
3019         if (devlink_dpipe_matches_put(table, skb))
3020                 goto nla_put_failure;
3021
3022         if (devlink_dpipe_actions_put(table, skb))
3023                 goto nla_put_failure;
3024
3025         nla_nest_end(skb, table_attr);
3026         return 0;
3027
3028 nla_put_failure:
3029         nla_nest_cancel(skb, table_attr);
3030         return -EMSGSIZE;
3031 }
3032
3033 static int devlink_dpipe_send_and_alloc_skb(struct sk_buff **pskb,
3034                                             struct genl_info *info)
3035 {
3036         int err;
3037
3038         if (*pskb) {
3039                 err = genlmsg_reply(*pskb, info);
3040                 if (err)
3041                         return err;
3042         }
3043         *pskb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
3044         if (!*pskb)
3045                 return -ENOMEM;
3046         return 0;
3047 }
3048
3049 static int devlink_dpipe_tables_fill(struct genl_info *info,
3050                                      enum devlink_command cmd, int flags,
3051                                      struct list_head *dpipe_tables,
3052                                      const char *table_name)
3053 {
3054         struct devlink *devlink = info->user_ptr[0];
3055         struct devlink_dpipe_table *table;
3056         struct nlattr *tables_attr;
3057         struct sk_buff *skb = NULL;
3058         struct nlmsghdr *nlh;
3059         bool incomplete;
3060         void *hdr;
3061         int i;
3062         int err;
3063
3064         table = list_first_entry(dpipe_tables,
3065                                  struct devlink_dpipe_table, list);
3066 start_again:
3067         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3068         if (err)
3069                 return err;
3070
3071         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
3072                           &devlink_nl_family, NLM_F_MULTI, cmd);
3073         if (!hdr) {
3074                 nlmsg_free(skb);
3075                 return -EMSGSIZE;
3076         }
3077
3078         if (devlink_nl_put_handle(skb, devlink))
3079                 goto nla_put_failure;
3080         tables_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_TABLES);
3081         if (!tables_attr)
3082                 goto nla_put_failure;
3083
3084         i = 0;
3085         incomplete = false;
3086         list_for_each_entry_from(table, dpipe_tables, list) {
3087                 if (!table_name) {
3088                         err = devlink_dpipe_table_put(skb, table);
3089                         if (err) {
3090                                 if (!i)
3091                                         goto err_table_put;
3092                                 incomplete = true;
3093                                 break;
3094                         }
3095                 } else {
3096                         if (!strcmp(table->name, table_name)) {
3097                                 err = devlink_dpipe_table_put(skb, table);
3098                                 if (err)
3099                                         break;
3100                         }
3101                 }
3102                 i++;
3103         }
3104
3105         nla_nest_end(skb, tables_attr);
3106         genlmsg_end(skb, hdr);
3107         if (incomplete)
3108                 goto start_again;
3109
3110 send_done:
3111         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
3112                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
3113         if (!nlh) {
3114                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3115                 if (err)
3116                         return err;
3117                 goto send_done;
3118         }
3119
3120         return genlmsg_reply(skb, info);
3121
3122 nla_put_failure:
3123         err = -EMSGSIZE;
3124 err_table_put:
3125         nlmsg_free(skb);
3126         return err;
3127 }
3128
3129 static int devlink_nl_cmd_dpipe_table_get(struct sk_buff *skb,
3130                                           struct genl_info *info)
3131 {
3132         struct devlink *devlink = info->user_ptr[0];
3133         const char *table_name =  NULL;
3134
3135         if (info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
3136                 table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
3137
3138         return devlink_dpipe_tables_fill(info, DEVLINK_CMD_DPIPE_TABLE_GET, 0,
3139                                          &devlink->dpipe_table_list,
3140                                          table_name);
3141 }
3142
3143 static int devlink_dpipe_value_put(struct sk_buff *skb,
3144                                    struct devlink_dpipe_value *value)
3145 {
3146         if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE,
3147                     value->value_size, value->value))
3148                 return -EMSGSIZE;
3149         if (value->mask)
3150                 if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE_MASK,
3151                             value->value_size, value->mask))
3152                         return -EMSGSIZE;
3153         if (value->mapping_valid)
3154                 if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_VALUE_MAPPING,
3155                                 value->mapping_value))
3156                         return -EMSGSIZE;
3157         return 0;
3158 }
3159
3160 static int devlink_dpipe_action_value_put(struct sk_buff *skb,
3161                                           struct devlink_dpipe_value *value)
3162 {
3163         if (!value->action)
3164                 return -EINVAL;
3165         if (devlink_dpipe_action_put(skb, value->action))
3166                 return -EMSGSIZE;
3167         if (devlink_dpipe_value_put(skb, value))
3168                 return -EMSGSIZE;
3169         return 0;
3170 }
3171
3172 static int devlink_dpipe_action_values_put(struct sk_buff *skb,
3173                                            struct devlink_dpipe_value *values,
3174                                            unsigned int values_count)
3175 {
3176         struct nlattr *action_attr;
3177         int i;
3178         int err;
3179
3180         for (i = 0; i < values_count; i++) {
3181                 action_attr = nla_nest_start_noflag(skb,
3182                                                     DEVLINK_ATTR_DPIPE_ACTION_VALUE);
3183                 if (!action_attr)
3184                         return -EMSGSIZE;
3185                 err = devlink_dpipe_action_value_put(skb, &values[i]);
3186                 if (err)
3187                         goto err_action_value_put;
3188                 nla_nest_end(skb, action_attr);
3189         }
3190         return 0;
3191
3192 err_action_value_put:
3193         nla_nest_cancel(skb, action_attr);
3194         return err;
3195 }
3196
3197 static int devlink_dpipe_match_value_put(struct sk_buff *skb,
3198                                          struct devlink_dpipe_value *value)
3199 {
3200         if (!value->match)
3201                 return -EINVAL;
3202         if (devlink_dpipe_match_put(skb, value->match))
3203                 return -EMSGSIZE;
3204         if (devlink_dpipe_value_put(skb, value))
3205                 return -EMSGSIZE;
3206         return 0;
3207 }
3208
3209 static int devlink_dpipe_match_values_put(struct sk_buff *skb,
3210                                           struct devlink_dpipe_value *values,
3211                                           unsigned int values_count)
3212 {
3213         struct nlattr *match_attr;
3214         int i;
3215         int err;
3216
3217         for (i = 0; i < values_count; i++) {
3218                 match_attr = nla_nest_start_noflag(skb,
3219                                                    DEVLINK_ATTR_DPIPE_MATCH_VALUE);
3220                 if (!match_attr)
3221                         return -EMSGSIZE;
3222                 err = devlink_dpipe_match_value_put(skb, &values[i]);
3223                 if (err)
3224                         goto err_match_value_put;
3225                 nla_nest_end(skb, match_attr);
3226         }
3227         return 0;
3228
3229 err_match_value_put:
3230         nla_nest_cancel(skb, match_attr);
3231         return err;
3232 }
3233
3234 static int devlink_dpipe_entry_put(struct sk_buff *skb,
3235                                    struct devlink_dpipe_entry *entry)
3236 {
3237         struct nlattr *entry_attr, *matches_attr, *actions_attr;
3238         int err;
3239
3240         entry_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_ENTRY);
3241         if (!entry_attr)
3242                 return  -EMSGSIZE;
3243
3244         if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_INDEX, entry->index,
3245                               DEVLINK_ATTR_PAD))
3246                 goto nla_put_failure;
3247         if (entry->counter_valid)
3248                 if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_COUNTER,
3249                                       entry->counter, DEVLINK_ATTR_PAD))
3250                         goto nla_put_failure;
3251
3252         matches_attr = nla_nest_start_noflag(skb,
3253                                              DEVLINK_ATTR_DPIPE_ENTRY_MATCH_VALUES);
3254         if (!matches_attr)
3255                 goto nla_put_failure;
3256
3257         err = devlink_dpipe_match_values_put(skb, entry->match_values,
3258                                              entry->match_values_count);
3259         if (err) {
3260                 nla_nest_cancel(skb, matches_attr);
3261                 goto err_match_values_put;
3262         }
3263         nla_nest_end(skb, matches_attr);
3264
3265         actions_attr = nla_nest_start_noflag(skb,
3266                                              DEVLINK_ATTR_DPIPE_ENTRY_ACTION_VALUES);
3267         if (!actions_attr)
3268                 goto nla_put_failure;
3269
3270         err = devlink_dpipe_action_values_put(skb, entry->action_values,
3271                                               entry->action_values_count);
3272         if (err) {
3273                 nla_nest_cancel(skb, actions_attr);
3274                 goto err_action_values_put;
3275         }
3276         nla_nest_end(skb, actions_attr);
3277
3278         nla_nest_end(skb, entry_attr);
3279         return 0;
3280
3281 nla_put_failure:
3282         err = -EMSGSIZE;
3283 err_match_values_put:
3284 err_action_values_put:
3285         nla_nest_cancel(skb, entry_attr);
3286         return err;
3287 }
3288
3289 static struct devlink_dpipe_table *
3290 devlink_dpipe_table_find(struct list_head *dpipe_tables,
3291                          const char *table_name, struct devlink *devlink)
3292 {
3293         struct devlink_dpipe_table *table;
3294         list_for_each_entry_rcu(table, dpipe_tables, list,
3295                                 lockdep_is_held(&devlink->lock)) {
3296                 if (!strcmp(table->name, table_name))
3297                         return table;
3298         }
3299         return NULL;
3300 }
3301
3302 int devlink_dpipe_entry_ctx_prepare(struct devlink_dpipe_dump_ctx *dump_ctx)
3303 {
3304         struct devlink *devlink;
3305         int err;
3306
3307         err = devlink_dpipe_send_and_alloc_skb(&dump_ctx->skb,
3308                                                dump_ctx->info);
3309         if (err)
3310                 return err;
3311
3312         dump_ctx->hdr = genlmsg_put(dump_ctx->skb,
3313                                     dump_ctx->info->snd_portid,
3314                                     dump_ctx->info->snd_seq,
3315                                     &devlink_nl_family, NLM_F_MULTI,
3316                                     dump_ctx->cmd);
3317         if (!dump_ctx->hdr)
3318                 goto nla_put_failure;
3319
3320         devlink = dump_ctx->info->user_ptr[0];
3321         if (devlink_nl_put_handle(dump_ctx->skb, devlink))
3322                 goto nla_put_failure;
3323         dump_ctx->nest = nla_nest_start_noflag(dump_ctx->skb,
3324                                                DEVLINK_ATTR_DPIPE_ENTRIES);
3325         if (!dump_ctx->nest)
3326                 goto nla_put_failure;
3327         return 0;
3328
3329 nla_put_failure:
3330         nlmsg_free(dump_ctx->skb);
3331         return -EMSGSIZE;
3332 }
3333 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_prepare);
3334
3335 int devlink_dpipe_entry_ctx_append(struct devlink_dpipe_dump_ctx *dump_ctx,
3336                                    struct devlink_dpipe_entry *entry)
3337 {
3338         return devlink_dpipe_entry_put(dump_ctx->skb, entry);
3339 }
3340 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_append);
3341
3342 int devlink_dpipe_entry_ctx_close(struct devlink_dpipe_dump_ctx *dump_ctx)
3343 {
3344         nla_nest_end(dump_ctx->skb, dump_ctx->nest);
3345         genlmsg_end(dump_ctx->skb, dump_ctx->hdr);
3346         return 0;
3347 }
3348 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_close);
3349
3350 void devlink_dpipe_entry_clear(struct devlink_dpipe_entry *entry)
3351
3352 {
3353         unsigned int value_count, value_index;
3354         struct devlink_dpipe_value *value;
3355
3356         value = entry->action_values;
3357         value_count = entry->action_values_count;
3358         for (value_index = 0; value_index < value_count; value_index++) {
3359                 kfree(value[value_index].value);
3360                 kfree(value[value_index].mask);
3361         }
3362
3363         value = entry->match_values;
3364         value_count = entry->match_values_count;
3365         for (value_index = 0; value_index < value_count; value_index++) {
3366                 kfree(value[value_index].value);
3367                 kfree(value[value_index].mask);
3368         }
3369 }
3370 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_clear);
3371
3372 static int devlink_dpipe_entries_fill(struct genl_info *info,
3373                                       enum devlink_command cmd, int flags,
3374                                       struct devlink_dpipe_table *table)
3375 {
3376         struct devlink_dpipe_dump_ctx dump_ctx;
3377         struct nlmsghdr *nlh;
3378         int err;
3379
3380         dump_ctx.skb = NULL;
3381         dump_ctx.cmd = cmd;
3382         dump_ctx.info = info;
3383
3384         err = table->table_ops->entries_dump(table->priv,
3385                                              table->counters_enabled,
3386                                              &dump_ctx);
3387         if (err)
3388                 return err;
3389
3390 send_done:
3391         nlh = nlmsg_put(dump_ctx.skb, info->snd_portid, info->snd_seq,
3392                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
3393         if (!nlh) {
3394                 err = devlink_dpipe_send_and_alloc_skb(&dump_ctx.skb, info);
3395                 if (err)
3396                         return err;
3397                 goto send_done;
3398         }
3399         return genlmsg_reply(dump_ctx.skb, info);
3400 }
3401
3402 static int devlink_nl_cmd_dpipe_entries_get(struct sk_buff *skb,
3403                                             struct genl_info *info)
3404 {
3405         struct devlink *devlink = info->user_ptr[0];
3406         struct devlink_dpipe_table *table;
3407         const char *table_name;
3408
3409         if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
3410                 return -EINVAL;
3411
3412         table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
3413         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
3414                                          table_name, devlink);
3415         if (!table)
3416                 return -EINVAL;
3417
3418         if (!table->table_ops->entries_dump)
3419                 return -EINVAL;
3420
3421         return devlink_dpipe_entries_fill(info, DEVLINK_CMD_DPIPE_ENTRIES_GET,
3422                                           0, table);
3423 }
3424
3425 static int devlink_dpipe_fields_put(struct sk_buff *skb,
3426                                     const struct devlink_dpipe_header *header)
3427 {
3428         struct devlink_dpipe_field *field;
3429         struct nlattr *field_attr;
3430         int i;
3431
3432         for (i = 0; i < header->fields_count; i++) {
3433                 field = &header->fields[i];
3434                 field_attr = nla_nest_start_noflag(skb,
3435                                                    DEVLINK_ATTR_DPIPE_FIELD);
3436                 if (!field_attr)
3437                         return -EMSGSIZE;
3438                 if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_FIELD_NAME, field->name) ||
3439                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
3440                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_BITWIDTH, field->bitwidth) ||
3441                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE, field->mapping_type))
3442                         goto nla_put_failure;
3443                 nla_nest_end(skb, field_attr);
3444         }
3445         return 0;
3446
3447 nla_put_failure:
3448         nla_nest_cancel(skb, field_attr);
3449         return -EMSGSIZE;
3450 }
3451
3452 static int devlink_dpipe_header_put(struct sk_buff *skb,
3453                                     struct devlink_dpipe_header *header)
3454 {
3455         struct nlattr *fields_attr, *header_attr;
3456         int err;
3457
3458         header_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_HEADER);
3459         if (!header_attr)
3460                 return -EMSGSIZE;
3461
3462         if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_HEADER_NAME, header->name) ||
3463             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
3464             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
3465                 goto nla_put_failure;
3466
3467         fields_attr = nla_nest_start_noflag(skb,
3468                                             DEVLINK_ATTR_DPIPE_HEADER_FIELDS);
3469         if (!fields_attr)
3470                 goto nla_put_failure;
3471
3472         err = devlink_dpipe_fields_put(skb, header);
3473         if (err) {
3474                 nla_nest_cancel(skb, fields_attr);
3475                 goto nla_put_failure;
3476         }
3477         nla_nest_end(skb, fields_attr);
3478         nla_nest_end(skb, header_attr);
3479         return 0;
3480
3481 nla_put_failure:
3482         err = -EMSGSIZE;
3483         nla_nest_cancel(skb, header_attr);
3484         return err;
3485 }
3486
3487 static int devlink_dpipe_headers_fill(struct genl_info *info,
3488                                       enum devlink_command cmd, int flags,
3489                                       struct devlink_dpipe_headers *
3490                                       dpipe_headers)
3491 {
3492         struct devlink *devlink = info->user_ptr[0];
3493         struct nlattr *headers_attr;
3494         struct sk_buff *skb = NULL;
3495         struct nlmsghdr *nlh;
3496         void *hdr;
3497         int i, j;
3498         int err;
3499
3500         i = 0;
3501 start_again:
3502         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3503         if (err)
3504                 return err;
3505
3506         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
3507                           &devlink_nl_family, NLM_F_MULTI, cmd);
3508         if (!hdr) {
3509                 nlmsg_free(skb);
3510                 return -EMSGSIZE;
3511         }
3512
3513         if (devlink_nl_put_handle(skb, devlink))
3514                 goto nla_put_failure;
3515         headers_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_HEADERS);
3516         if (!headers_attr)
3517                 goto nla_put_failure;
3518
3519         j = 0;
3520         for (; i < dpipe_headers->headers_count; i++) {
3521                 err = devlink_dpipe_header_put(skb, dpipe_headers->headers[i]);
3522                 if (err) {
3523                         if (!j)
3524                                 goto err_table_put;
3525                         break;
3526                 }
3527                 j++;
3528         }
3529         nla_nest_end(skb, headers_attr);
3530         genlmsg_end(skb, hdr);
3531         if (i != dpipe_headers->headers_count)
3532                 goto start_again;
3533
3534 send_done:
3535         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
3536                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
3537         if (!nlh) {
3538                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3539                 if (err)
3540                         return err;
3541                 goto send_done;
3542         }
3543         return genlmsg_reply(skb, info);
3544
3545 nla_put_failure:
3546         err = -EMSGSIZE;
3547 err_table_put:
3548         nlmsg_free(skb);
3549         return err;
3550 }
3551
3552 static int devlink_nl_cmd_dpipe_headers_get(struct sk_buff *skb,
3553                                             struct genl_info *info)
3554 {
3555         struct devlink *devlink = info->user_ptr[0];
3556
3557         if (!devlink->dpipe_headers)
3558                 return -EOPNOTSUPP;
3559         return devlink_dpipe_headers_fill(info, DEVLINK_CMD_DPIPE_HEADERS_GET,
3560                                           0, devlink->dpipe_headers);
3561 }
3562
3563 static int devlink_dpipe_table_counters_set(struct devlink *devlink,
3564                                             const char *table_name,
3565                                             bool enable)
3566 {
3567         struct devlink_dpipe_table *table;
3568
3569         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
3570                                          table_name, devlink);
3571         if (!table)
3572                 return -EINVAL;
3573
3574         if (table->counter_control_extern)
3575                 return -EOPNOTSUPP;
3576
3577         if (!(table->counters_enabled ^ enable))
3578                 return 0;
3579
3580         table->counters_enabled = enable;
3581         if (table->table_ops->counters_set_update)
3582                 table->table_ops->counters_set_update(table->priv, enable);
3583         return 0;
3584 }
3585
3586 static int devlink_nl_cmd_dpipe_table_counters_set(struct sk_buff *skb,
3587                                                    struct genl_info *info)
3588 {
3589         struct devlink *devlink = info->user_ptr[0];
3590         const char *table_name;
3591         bool counters_enable;
3592
3593         if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME] ||
3594             !info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED])
3595                 return -EINVAL;
3596
3597         table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
3598         counters_enable = !!nla_get_u8(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED]);
3599
3600         return devlink_dpipe_table_counters_set(devlink, table_name,
3601                                                 counters_enable);
3602 }
3603
3604 static struct devlink_resource *
3605 devlink_resource_find(struct devlink *devlink,
3606                       struct devlink_resource *resource, u64 resource_id)
3607 {
3608         struct list_head *resource_list;
3609
3610         if (resource)
3611                 resource_list = &resource->resource_list;
3612         else
3613                 resource_list = &devlink->resource_list;
3614
3615         list_for_each_entry(resource, resource_list, list) {
3616                 struct devlink_resource *child_resource;
3617
3618                 if (resource->id == resource_id)
3619                         return resource;
3620
3621                 child_resource = devlink_resource_find(devlink, resource,
3622                                                        resource_id);
3623                 if (child_resource)
3624                         return child_resource;
3625         }
3626         return NULL;
3627 }
3628
3629 static void
3630 devlink_resource_validate_children(struct devlink_resource *resource)
3631 {
3632         struct devlink_resource *child_resource;
3633         bool size_valid = true;
3634         u64 parts_size = 0;
3635
3636         if (list_empty(&resource->resource_list))
3637                 goto out;
3638
3639         list_for_each_entry(child_resource, &resource->resource_list, list)
3640                 parts_size += child_resource->size_new;
3641
3642         if (parts_size > resource->size_new)
3643                 size_valid = false;
3644 out:
3645         resource->size_valid = size_valid;
3646 }
3647
3648 static int
3649 devlink_resource_validate_size(struct devlink_resource *resource, u64 size,
3650                                struct netlink_ext_ack *extack)
3651 {
3652         u64 reminder;
3653         int err = 0;
3654
3655         if (size > resource->size_params.size_max) {
3656                 NL_SET_ERR_MSG_MOD(extack, "Size larger than maximum");
3657                 err = -EINVAL;
3658         }
3659
3660         if (size < resource->size_params.size_min) {
3661                 NL_SET_ERR_MSG_MOD(extack, "Size smaller than minimum");
3662                 err = -EINVAL;
3663         }
3664
3665         div64_u64_rem(size, resource->size_params.size_granularity, &reminder);
3666         if (reminder) {
3667                 NL_SET_ERR_MSG_MOD(extack, "Wrong granularity");
3668                 err = -EINVAL;
3669         }
3670
3671         return err;
3672 }
3673
3674 static int devlink_nl_cmd_resource_set(struct sk_buff *skb,
3675                                        struct genl_info *info)
3676 {
3677         struct devlink *devlink = info->user_ptr[0];
3678         struct devlink_resource *resource;
3679         u64 resource_id;
3680         u64 size;
3681         int err;
3682
3683         if (!info->attrs[DEVLINK_ATTR_RESOURCE_ID] ||
3684             !info->attrs[DEVLINK_ATTR_RESOURCE_SIZE])
3685                 return -EINVAL;
3686         resource_id = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_ID]);
3687
3688         resource = devlink_resource_find(devlink, NULL, resource_id);
3689         if (!resource)
3690                 return -EINVAL;
3691
3692         size = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_SIZE]);
3693         err = devlink_resource_validate_size(resource, size, info->extack);
3694         if (err)
3695                 return err;
3696
3697         resource->size_new = size;
3698         devlink_resource_validate_children(resource);
3699         if (resource->parent)
3700                 devlink_resource_validate_children(resource->parent);
3701         return 0;
3702 }
3703
3704 static int
3705 devlink_resource_size_params_put(struct devlink_resource *resource,
3706                                  struct sk_buff *skb)
3707 {
3708         struct devlink_resource_size_params *size_params;
3709
3710         size_params = &resource->size_params;
3711         if (nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_GRAN,
3712                               size_params->size_granularity, DEVLINK_ATTR_PAD) ||
3713             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_MAX,
3714                               size_params->size_max, DEVLINK_ATTR_PAD) ||
3715             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_MIN,
3716                               size_params->size_min, DEVLINK_ATTR_PAD) ||
3717             nla_put_u8(skb, DEVLINK_ATTR_RESOURCE_UNIT, size_params->unit))
3718                 return -EMSGSIZE;
3719         return 0;
3720 }
3721
3722 static int devlink_resource_occ_put(struct devlink_resource *resource,
3723                                     struct sk_buff *skb)
3724 {
3725         if (!resource->occ_get)
3726                 return 0;
3727         return nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_OCC,
3728                                  resource->occ_get(resource->occ_get_priv),
3729                                  DEVLINK_ATTR_PAD);
3730 }
3731
3732 static int devlink_resource_put(struct devlink *devlink, struct sk_buff *skb,
3733                                 struct devlink_resource *resource)
3734 {
3735         struct devlink_resource *child_resource;
3736         struct nlattr *child_resource_attr;
3737         struct nlattr *resource_attr;
3738
3739         resource_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_RESOURCE);
3740         if (!resource_attr)
3741                 return -EMSGSIZE;
3742
3743         if (nla_put_string(skb, DEVLINK_ATTR_RESOURCE_NAME, resource->name) ||
3744             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE, resource->size,
3745                               DEVLINK_ATTR_PAD) ||
3746             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_ID, resource->id,
3747                               DEVLINK_ATTR_PAD))
3748                 goto nla_put_failure;
3749         if (resource->size != resource->size_new)
3750                 nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_NEW,
3751                                   resource->size_new, DEVLINK_ATTR_PAD);
3752         if (devlink_resource_occ_put(resource, skb))
3753                 goto nla_put_failure;
3754         if (devlink_resource_size_params_put(resource, skb))
3755                 goto nla_put_failure;
3756         if (list_empty(&resource->resource_list))
3757                 goto out;
3758
3759         if (nla_put_u8(skb, DEVLINK_ATTR_RESOURCE_SIZE_VALID,
3760                        resource->size_valid))
3761                 goto nla_put_failure;
3762
3763         child_resource_attr = nla_nest_start_noflag(skb,
3764                                                     DEVLINK_ATTR_RESOURCE_LIST);
3765         if (!child_resource_attr)
3766                 goto nla_put_failure;
3767
3768         list_for_each_entry(child_resource, &resource->resource_list, list) {
3769                 if (devlink_resource_put(devlink, skb, child_resource))
3770                         goto resource_put_failure;
3771         }
3772
3773         nla_nest_end(skb, child_resource_attr);
3774 out:
3775         nla_nest_end(skb, resource_attr);
3776         return 0;
3777
3778 resource_put_failure:
3779         nla_nest_cancel(skb, child_resource_attr);
3780 nla_put_failure:
3781         nla_nest_cancel(skb, resource_attr);
3782         return -EMSGSIZE;
3783 }
3784
3785 static int devlink_resource_fill(struct genl_info *info,
3786                                  enum devlink_command cmd, int flags)
3787 {
3788         struct devlink *devlink = info->user_ptr[0];
3789         struct devlink_resource *resource;
3790         struct nlattr *resources_attr;
3791         struct sk_buff *skb = NULL;
3792         struct nlmsghdr *nlh;
3793         bool incomplete;
3794         void *hdr;
3795         int i;
3796         int err;
3797
3798         resource = list_first_entry(&devlink->resource_list,
3799                                     struct devlink_resource, list);
3800 start_again:
3801         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3802         if (err)
3803                 return err;
3804
3805         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
3806                           &devlink_nl_family, NLM_F_MULTI, cmd);
3807         if (!hdr) {
3808                 nlmsg_free(skb);
3809                 return -EMSGSIZE;
3810         }
3811
3812         if (devlink_nl_put_handle(skb, devlink))
3813                 goto nla_put_failure;
3814
3815         resources_attr = nla_nest_start_noflag(skb,
3816                                                DEVLINK_ATTR_RESOURCE_LIST);
3817         if (!resources_attr)
3818                 goto nla_put_failure;
3819
3820         incomplete = false;
3821         i = 0;
3822         list_for_each_entry_from(resource, &devlink->resource_list, list) {
3823                 err = devlink_resource_put(devlink, skb, resource);
3824                 if (err) {
3825                         if (!i)
3826                                 goto err_resource_put;
3827                         incomplete = true;
3828                         break;
3829                 }
3830                 i++;
3831         }
3832         nla_nest_end(skb, resources_attr);
3833         genlmsg_end(skb, hdr);
3834         if (incomplete)
3835                 goto start_again;
3836 send_done:
3837         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
3838                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
3839         if (!nlh) {
3840                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3841                 if (err)
3842                         return err;
3843                 goto send_done;
3844         }
3845         return genlmsg_reply(skb, info);
3846
3847 nla_put_failure:
3848         err = -EMSGSIZE;
3849 err_resource_put:
3850         nlmsg_free(skb);
3851         return err;
3852 }
3853
3854 static int devlink_nl_cmd_resource_dump(struct sk_buff *skb,
3855                                         struct genl_info *info)
3856 {
3857         struct devlink *devlink = info->user_ptr[0];
3858
3859         if (list_empty(&devlink->resource_list))
3860                 return -EOPNOTSUPP;
3861
3862         return devlink_resource_fill(info, DEVLINK_CMD_RESOURCE_DUMP, 0);
3863 }
3864
3865 static int
3866 devlink_resources_validate(struct devlink *devlink,
3867                            struct devlink_resource *resource,
3868                            struct genl_info *info)
3869 {
3870         struct list_head *resource_list;
3871         int err = 0;
3872
3873         if (resource)
3874                 resource_list = &resource->resource_list;
3875         else
3876                 resource_list = &devlink->resource_list;
3877
3878         list_for_each_entry(resource, resource_list, list) {
3879                 if (!resource->size_valid)
3880                         return -EINVAL;
3881                 err = devlink_resources_validate(devlink, resource, info);
3882                 if (err)
3883                         return err;
3884         }
3885         return err;
3886 }
3887
3888 static struct net *devlink_netns_get(struct sk_buff *skb,
3889                                      struct genl_info *info)
3890 {
3891         struct nlattr *netns_pid_attr = info->attrs[DEVLINK_ATTR_NETNS_PID];
3892         struct nlattr *netns_fd_attr = info->attrs[DEVLINK_ATTR_NETNS_FD];
3893         struct nlattr *netns_id_attr = info->attrs[DEVLINK_ATTR_NETNS_ID];
3894         struct net *net;
3895
3896         if (!!netns_pid_attr + !!netns_fd_attr + !!netns_id_attr > 1) {
3897                 NL_SET_ERR_MSG_MOD(info->extack, "multiple netns identifying attributes specified");
3898                 return ERR_PTR(-EINVAL);
3899         }
3900
3901         if (netns_pid_attr) {
3902                 net = get_net_ns_by_pid(nla_get_u32(netns_pid_attr));
3903         } else if (netns_fd_attr) {
3904                 net = get_net_ns_by_fd(nla_get_u32(netns_fd_attr));
3905         } else if (netns_id_attr) {
3906                 net = get_net_ns_by_id(sock_net(skb->sk),
3907                                        nla_get_u32(netns_id_attr));
3908                 if (!net)
3909                         net = ERR_PTR(-EINVAL);
3910         } else {
3911                 WARN_ON(1);
3912                 net = ERR_PTR(-EINVAL);
3913         }
3914         if (IS_ERR(net)) {
3915                 NL_SET_ERR_MSG_MOD(info->extack, "Unknown network namespace");
3916                 return ERR_PTR(-EINVAL);
3917         }
3918         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) {
3919                 put_net(net);
3920                 return ERR_PTR(-EPERM);
3921         }
3922         return net;
3923 }
3924
3925 static void devlink_param_notify(struct devlink *devlink,
3926                                  unsigned int port_index,
3927                                  struct devlink_param_item *param_item,
3928                                  enum devlink_command cmd);
3929
3930 static void devlink_ns_change_notify(struct devlink *devlink,
3931                                      struct net *dest_net, struct net *curr_net,
3932                                      bool new)
3933 {
3934         struct devlink_param_item *param_item;
3935         enum devlink_command cmd;
3936
3937         /* Userspace needs to be notified about devlink objects
3938          * removed from original and entering new network namespace.
3939          * The rest of the devlink objects are re-created during
3940          * reload process so the notifications are generated separatelly.
3941          */
3942
3943         if (!dest_net || net_eq(dest_net, curr_net))
3944                 return;
3945
3946         if (new)
3947                 devlink_notify(devlink, DEVLINK_CMD_NEW);
3948
3949         cmd = new ? DEVLINK_CMD_PARAM_NEW : DEVLINK_CMD_PARAM_DEL;
3950         list_for_each_entry(param_item, &devlink->param_list, list)
3951                 devlink_param_notify(devlink, 0, param_item, cmd);
3952
3953         if (!new)
3954                 devlink_notify(devlink, DEVLINK_CMD_DEL);
3955 }
3956
3957 static bool devlink_reload_supported(const struct devlink_ops *ops)
3958 {
3959         return ops->reload_down && ops->reload_up;
3960 }
3961
3962 static void devlink_reload_failed_set(struct devlink *devlink,
3963                                       bool reload_failed)
3964 {
3965         if (devlink->reload_failed == reload_failed)
3966                 return;
3967         devlink->reload_failed = reload_failed;
3968         devlink_notify(devlink, DEVLINK_CMD_NEW);
3969 }
3970
3971 bool devlink_is_reload_failed(const struct devlink *devlink)
3972 {
3973         return devlink->reload_failed;
3974 }
3975 EXPORT_SYMBOL_GPL(devlink_is_reload_failed);
3976
3977 static void
3978 __devlink_reload_stats_update(struct devlink *devlink, u32 *reload_stats,
3979                               enum devlink_reload_limit limit, u32 actions_performed)
3980 {
3981         unsigned long actions = actions_performed;
3982         int stat_idx;
3983         int action;
3984
3985         for_each_set_bit(action, &actions, __DEVLINK_RELOAD_ACTION_MAX) {
3986                 stat_idx = limit * __DEVLINK_RELOAD_ACTION_MAX + action;
3987                 reload_stats[stat_idx]++;
3988         }
3989         devlink_notify(devlink, DEVLINK_CMD_NEW);
3990 }
3991
3992 static void
3993 devlink_reload_stats_update(struct devlink *devlink, enum devlink_reload_limit limit,
3994                             u32 actions_performed)
3995 {
3996         __devlink_reload_stats_update(devlink, devlink->stats.reload_stats, limit,
3997                                       actions_performed);
3998 }
3999
4000 /**
4001  *      devlink_remote_reload_actions_performed - Update devlink on reload actions
4002  *        performed which are not a direct result of devlink reload call.
4003  *
4004  *      This should be called by a driver after performing reload actions in case it was not
4005  *      a result of devlink reload call. For example fw_activate was performed as a result
4006  *      of devlink reload triggered fw_activate on another host.
4007  *      The motivation for this function is to keep data on reload actions performed on this
4008  *      function whether it was done due to direct devlink reload call or not.
4009  *
4010  *      @devlink: devlink
4011  *      @limit: reload limit
4012  *      @actions_performed: bitmask of actions performed
4013  */
4014 void devlink_remote_reload_actions_performed(struct devlink *devlink,
4015                                              enum devlink_reload_limit limit,
4016                                              u32 actions_performed)
4017 {
4018         if (WARN_ON(!actions_performed ||
4019                     actions_performed & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
4020                     actions_performed >= BIT(__DEVLINK_RELOAD_ACTION_MAX) ||
4021                     limit > DEVLINK_RELOAD_LIMIT_MAX))
4022                 return;
4023
4024         __devlink_reload_stats_update(devlink, devlink->stats.remote_reload_stats, limit,
4025                                       actions_performed);
4026 }
4027 EXPORT_SYMBOL_GPL(devlink_remote_reload_actions_performed);
4028
4029 static int devlink_reload(struct devlink *devlink, struct net *dest_net,
4030                           enum devlink_reload_action action, enum devlink_reload_limit limit,
4031                           u32 *actions_performed, struct netlink_ext_ack *extack)
4032 {
4033         u32 remote_reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
4034         struct net *curr_net;
4035         int err;
4036
4037         memcpy(remote_reload_stats, devlink->stats.remote_reload_stats,
4038                sizeof(remote_reload_stats));
4039
4040         curr_net = devlink_net(devlink);
4041         devlink_ns_change_notify(devlink, dest_net, curr_net, false);
4042         err = devlink->ops->reload_down(devlink, !!dest_net, action, limit, extack);
4043         if (err)
4044                 return err;
4045
4046         if (dest_net && !net_eq(dest_net, curr_net))
4047                 write_pnet(&devlink->_net, dest_net);
4048
4049         err = devlink->ops->reload_up(devlink, action, limit, actions_performed, extack);
4050         devlink_reload_failed_set(devlink, !!err);
4051         if (err)
4052                 return err;
4053
4054         devlink_ns_change_notify(devlink, dest_net, curr_net, true);
4055         WARN_ON(!(*actions_performed & BIT(action)));
4056         /* Catch driver on updating the remote action within devlink reload */
4057         WARN_ON(memcmp(remote_reload_stats, devlink->stats.remote_reload_stats,
4058                        sizeof(remote_reload_stats)));
4059         devlink_reload_stats_update(devlink, limit, *actions_performed);
4060         return 0;
4061 }
4062
4063 static int
4064 devlink_nl_reload_actions_performed_snd(struct devlink *devlink, u32 actions_performed,
4065                                         enum devlink_command cmd, struct genl_info *info)
4066 {
4067         struct sk_buff *msg;
4068         void *hdr;
4069
4070         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4071         if (!msg)
4072                 return -ENOMEM;
4073
4074         hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq, &devlink_nl_family, 0, cmd);
4075         if (!hdr)
4076                 goto free_msg;
4077
4078         if (devlink_nl_put_handle(msg, devlink))
4079                 goto nla_put_failure;
4080
4081         if (nla_put_bitfield32(msg, DEVLINK_ATTR_RELOAD_ACTIONS_PERFORMED, actions_performed,
4082                                actions_performed))
4083                 goto nla_put_failure;
4084         genlmsg_end(msg, hdr);
4085
4086         return genlmsg_reply(msg, info);
4087
4088 nla_put_failure:
4089         genlmsg_cancel(msg, hdr);
4090 free_msg:
4091         nlmsg_free(msg);
4092         return -EMSGSIZE;
4093 }
4094
4095 static int devlink_nl_cmd_reload(struct sk_buff *skb, struct genl_info *info)
4096 {
4097         struct devlink *devlink = info->user_ptr[0];
4098         enum devlink_reload_action action;
4099         enum devlink_reload_limit limit;
4100         struct net *dest_net = NULL;
4101         u32 actions_performed;
4102         int err;
4103
4104         if (!(devlink->features & DEVLINK_F_RELOAD))
4105                 return -EOPNOTSUPP;
4106
4107         err = devlink_resources_validate(devlink, NULL, info);
4108         if (err) {
4109                 NL_SET_ERR_MSG_MOD(info->extack, "resources size validation failed");
4110                 return err;
4111         }
4112
4113         if (info->attrs[DEVLINK_ATTR_NETNS_PID] ||
4114             info->attrs[DEVLINK_ATTR_NETNS_FD] ||
4115             info->attrs[DEVLINK_ATTR_NETNS_ID]) {
4116                 dest_net = devlink_netns_get(skb, info);
4117                 if (IS_ERR(dest_net))
4118                         return PTR_ERR(dest_net);
4119         }
4120
4121         if (info->attrs[DEVLINK_ATTR_RELOAD_ACTION])
4122                 action = nla_get_u8(info->attrs[DEVLINK_ATTR_RELOAD_ACTION]);
4123         else
4124                 action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT;
4125
4126         if (!devlink_reload_action_is_supported(devlink, action)) {
4127                 NL_SET_ERR_MSG_MOD(info->extack,
4128                                    "Requested reload action is not supported by the driver");
4129                 return -EOPNOTSUPP;
4130         }
4131
4132         limit = DEVLINK_RELOAD_LIMIT_UNSPEC;
4133         if (info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]) {
4134                 struct nla_bitfield32 limits;
4135                 u32 limits_selected;
4136
4137                 limits = nla_get_bitfield32(info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]);
4138                 limits_selected = limits.value & limits.selector;
4139                 if (!limits_selected) {
4140                         NL_SET_ERR_MSG_MOD(info->extack, "Invalid limit selected");
4141                         return -EINVAL;
4142                 }
4143                 for (limit = 0 ; limit <= DEVLINK_RELOAD_LIMIT_MAX ; limit++)
4144                         if (limits_selected & BIT(limit))
4145                                 break;
4146                 /* UAPI enables multiselection, but currently it is not used */
4147                 if (limits_selected != BIT(limit)) {
4148                         NL_SET_ERR_MSG_MOD(info->extack,
4149                                            "Multiselection of limit is not supported");
4150                         return -EOPNOTSUPP;
4151                 }
4152                 if (!devlink_reload_limit_is_supported(devlink, limit)) {
4153                         NL_SET_ERR_MSG_MOD(info->extack,
4154                                            "Requested limit is not supported by the driver");
4155                         return -EOPNOTSUPP;
4156                 }
4157                 if (devlink_reload_combination_is_invalid(action, limit)) {
4158                         NL_SET_ERR_MSG_MOD(info->extack,
4159                                            "Requested limit is invalid for this action");
4160                         return -EINVAL;
4161                 }
4162         }
4163         err = devlink_reload(devlink, dest_net, action, limit, &actions_performed, info->extack);
4164
4165         if (dest_net)
4166                 put_net(dest_net);
4167
4168         if (err)
4169                 return err;
4170         /* For backward compatibility generate reply only if attributes used by user */
4171         if (!info->attrs[DEVLINK_ATTR_RELOAD_ACTION] && !info->attrs[DEVLINK_ATTR_RELOAD_LIMITS])
4172                 return 0;
4173
4174         return devlink_nl_reload_actions_performed_snd(devlink, actions_performed,
4175                                                        DEVLINK_CMD_RELOAD, info);
4176 }
4177
4178 static int devlink_nl_flash_update_fill(struct sk_buff *msg,
4179                                         struct devlink *devlink,
4180                                         enum devlink_command cmd,
4181                                         struct devlink_flash_notify *params)
4182 {
4183         void *hdr;
4184
4185         hdr = genlmsg_put(msg, 0, 0, &devlink_nl_family, 0, cmd);
4186         if (!hdr)
4187                 return -EMSGSIZE;
4188
4189         if (devlink_nl_put_handle(msg, devlink))
4190                 goto nla_put_failure;
4191
4192         if (cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS)
4193                 goto out;
4194
4195         if (params->status_msg &&
4196             nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_MSG,
4197                            params->status_msg))
4198                 goto nla_put_failure;
4199         if (params->component &&
4200             nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_COMPONENT,
4201                            params->component))
4202                 goto nla_put_failure;
4203         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_DONE,
4204                               params->done, DEVLINK_ATTR_PAD))
4205                 goto nla_put_failure;
4206         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TOTAL,
4207                               params->total, DEVLINK_ATTR_PAD))
4208                 goto nla_put_failure;
4209         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TIMEOUT,
4210                               params->timeout, DEVLINK_ATTR_PAD))
4211                 goto nla_put_failure;
4212
4213 out:
4214         genlmsg_end(msg, hdr);
4215         return 0;
4216
4217 nla_put_failure:
4218         genlmsg_cancel(msg, hdr);
4219         return -EMSGSIZE;
4220 }
4221
4222 static void __devlink_flash_update_notify(struct devlink *devlink,
4223                                           enum devlink_command cmd,
4224                                           struct devlink_flash_notify *params)
4225 {
4226         struct sk_buff *msg;
4227         int err;
4228
4229         WARN_ON(cmd != DEVLINK_CMD_FLASH_UPDATE &&
4230                 cmd != DEVLINK_CMD_FLASH_UPDATE_END &&
4231                 cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS);
4232         WARN_ON(!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED));
4233
4234         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4235         if (!msg)
4236                 return;
4237
4238         err = devlink_nl_flash_update_fill(msg, devlink, cmd, params);
4239         if (err)
4240                 goto out_free_msg;
4241
4242         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
4243                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
4244         return;
4245
4246 out_free_msg:
4247         nlmsg_free(msg);
4248 }
4249
4250 static void devlink_flash_update_begin_notify(struct devlink *devlink)
4251 {
4252         struct devlink_flash_notify params = {};
4253
4254         __devlink_flash_update_notify(devlink,
4255                                       DEVLINK_CMD_FLASH_UPDATE,
4256                                       &params);
4257 }
4258
4259 static void devlink_flash_update_end_notify(struct devlink *devlink)
4260 {
4261         struct devlink_flash_notify params = {};
4262
4263         __devlink_flash_update_notify(devlink,
4264                                       DEVLINK_CMD_FLASH_UPDATE_END,
4265                                       &params);
4266 }
4267
4268 void devlink_flash_update_status_notify(struct devlink *devlink,
4269                                         const char *status_msg,
4270                                         const char *component,
4271                                         unsigned long done,
4272                                         unsigned long total)
4273 {
4274         struct devlink_flash_notify params = {
4275                 .status_msg = status_msg,
4276                 .component = component,
4277                 .done = done,
4278                 .total = total,
4279         };
4280
4281         __devlink_flash_update_notify(devlink,
4282                                       DEVLINK_CMD_FLASH_UPDATE_STATUS,
4283                                       &params);
4284 }
4285 EXPORT_SYMBOL_GPL(devlink_flash_update_status_notify);
4286
4287 void devlink_flash_update_timeout_notify(struct devlink *devlink,
4288                                          const char *status_msg,
4289                                          const char *component,
4290                                          unsigned long timeout)
4291 {
4292         struct devlink_flash_notify params = {
4293                 .status_msg = status_msg,
4294                 .component = component,
4295                 .timeout = timeout,
4296         };
4297
4298         __devlink_flash_update_notify(devlink,
4299                                       DEVLINK_CMD_FLASH_UPDATE_STATUS,
4300                                       &params);
4301 }
4302 EXPORT_SYMBOL_GPL(devlink_flash_update_timeout_notify);
4303
4304 static int devlink_nl_cmd_flash_update(struct sk_buff *skb,
4305                                        struct genl_info *info)
4306 {
4307         struct nlattr *nla_component, *nla_overwrite_mask, *nla_file_name;
4308         struct devlink_flash_update_params params = {};
4309         struct devlink *devlink = info->user_ptr[0];
4310         const char *file_name;
4311         u32 supported_params;
4312         int ret;
4313
4314         if (!devlink->ops->flash_update)
4315                 return -EOPNOTSUPP;
4316
4317         if (!info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME])
4318                 return -EINVAL;
4319
4320         supported_params = devlink->ops->supported_flash_update_params;
4321
4322         nla_component = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_COMPONENT];
4323         if (nla_component) {
4324                 if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_COMPONENT)) {
4325                         NL_SET_ERR_MSG_ATTR(info->extack, nla_component,
4326                                             "component update is not supported by this device");
4327                         return -EOPNOTSUPP;
4328                 }
4329                 params.component = nla_data(nla_component);
4330         }
4331
4332         nla_overwrite_mask = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK];
4333         if (nla_overwrite_mask) {
4334                 struct nla_bitfield32 sections;
4335
4336                 if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK)) {
4337                         NL_SET_ERR_MSG_ATTR(info->extack, nla_overwrite_mask,
4338                                             "overwrite settings are not supported by this device");
4339                         return -EOPNOTSUPP;
4340                 }
4341                 sections = nla_get_bitfield32(nla_overwrite_mask);
4342                 params.overwrite_mask = sections.value & sections.selector;
4343         }
4344
4345         nla_file_name = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME];
4346         file_name = nla_data(nla_file_name);
4347         ret = request_firmware(&params.fw, file_name, devlink->dev);
4348         if (ret) {
4349                 NL_SET_ERR_MSG_ATTR(info->extack, nla_file_name, "failed to locate the requested firmware file");
4350                 return ret;
4351         }
4352
4353         devlink_flash_update_begin_notify(devlink);
4354         ret = devlink->ops->flash_update(devlink, &params, info->extack);
4355         devlink_flash_update_end_notify(devlink);
4356
4357         release_firmware(params.fw);
4358
4359         return ret;
4360 }
4361
4362 static const struct devlink_param devlink_param_generic[] = {
4363         {
4364                 .id = DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
4365                 .name = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_NAME,
4366                 .type = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_TYPE,
4367         },
4368         {
4369                 .id = DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
4370                 .name = DEVLINK_PARAM_GENERIC_MAX_MACS_NAME,
4371                 .type = DEVLINK_PARAM_GENERIC_MAX_MACS_TYPE,
4372         },
4373         {
4374                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_SRIOV,
4375                 .name = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_NAME,
4376                 .type = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_TYPE,
4377         },
4378         {
4379                 .id = DEVLINK_PARAM_GENERIC_ID_REGION_SNAPSHOT,
4380                 .name = DEVLINK_PARAM_GENERIC_REGION_SNAPSHOT_NAME,
4381                 .type = DEVLINK_PARAM_GENERIC_REGION_SNAPSHOT_TYPE,
4382         },
4383         {
4384                 .id = DEVLINK_PARAM_GENERIC_ID_IGNORE_ARI,
4385                 .name = DEVLINK_PARAM_GENERIC_IGNORE_ARI_NAME,
4386                 .type = DEVLINK_PARAM_GENERIC_IGNORE_ARI_TYPE,
4387         },
4388         {
4389                 .id = DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MAX,
4390                 .name = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MAX_NAME,
4391                 .type = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MAX_TYPE,
4392         },
4393         {
4394                 .id = DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MIN,
4395                 .name = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MIN_NAME,
4396                 .type = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MIN_TYPE,
4397         },
4398         {
4399                 .id = DEVLINK_PARAM_GENERIC_ID_FW_LOAD_POLICY,
4400                 .name = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_NAME,
4401                 .type = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_TYPE,
4402         },
4403         {
4404                 .id = DEVLINK_PARAM_GENERIC_ID_RESET_DEV_ON_DRV_PROBE,
4405                 .name = DEVLINK_PARAM_GENERIC_RESET_DEV_ON_DRV_PROBE_NAME,
4406                 .type = DEVLINK_PARAM_GENERIC_RESET_DEV_ON_DRV_PROBE_TYPE,
4407         },
4408         {
4409                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE,
4410                 .name = DEVLINK_PARAM_GENERIC_ENABLE_ROCE_NAME,
4411                 .type = DEVLINK_PARAM_GENERIC_ENABLE_ROCE_TYPE,
4412         },
4413         {
4414                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_REMOTE_DEV_RESET,
4415                 .name = DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_NAME,
4416                 .type = DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_TYPE,
4417         },
4418         {
4419                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,
4420                 .name = DEVLINK_PARAM_GENERIC_ENABLE_ETH_NAME,
4421                 .type = DEVLINK_PARAM_GENERIC_ENABLE_ETH_TYPE,
4422         },
4423         {
4424                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
4425                 .name = DEVLINK_PARAM_GENERIC_ENABLE_RDMA_NAME,
4426                 .type = DEVLINK_PARAM_GENERIC_ENABLE_RDMA_TYPE,
4427         },
4428         {
4429                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET,
4430                 .name = DEVLINK_PARAM_GENERIC_ENABLE_VNET_NAME,
4431                 .type = DEVLINK_PARAM_GENERIC_ENABLE_VNET_TYPE,
4432         },
4433 };
4434
4435 static int devlink_param_generic_verify(const struct devlink_param *param)
4436 {
4437         /* verify it match generic parameter by id and name */
4438         if (param->id > DEVLINK_PARAM_GENERIC_ID_MAX)
4439                 return -EINVAL;
4440         if (strcmp(param->name, devlink_param_generic[param->id].name))
4441                 return -ENOENT;
4442
4443         WARN_ON(param->type != devlink_param_generic[param->id].type);
4444
4445         return 0;
4446 }
4447
4448 static int devlink_param_driver_verify(const struct devlink_param *param)
4449 {
4450         int i;
4451
4452         if (param->id <= DEVLINK_PARAM_GENERIC_ID_MAX)
4453                 return -EINVAL;
4454         /* verify no such name in generic params */
4455         for (i = 0; i <= DEVLINK_PARAM_GENERIC_ID_MAX; i++)
4456                 if (!strcmp(param->name, devlink_param_generic[i].name))
4457                         return -EEXIST;
4458
4459         return 0;
4460 }
4461
4462 static struct devlink_param_item *
4463 devlink_param_find_by_name(struct list_head *param_list,
4464                            const char *param_name)
4465 {
4466         struct devlink_param_item *param_item;
4467
4468         list_for_each_entry(param_item, param_list, list)
4469                 if (!strcmp(param_item->param->name, param_name))
4470                         return param_item;
4471         return NULL;
4472 }
4473
4474 static struct devlink_param_item *
4475 devlink_param_find_by_id(struct list_head *param_list, u32 param_id)
4476 {
4477         struct devlink_param_item *param_item;
4478
4479         list_for_each_entry(param_item, param_list, list)
4480                 if (param_item->param->id == param_id)
4481                         return param_item;
4482         return NULL;
4483 }
4484
4485 static bool
4486 devlink_param_cmode_is_supported(const struct devlink_param *param,
4487                                  enum devlink_param_cmode cmode)
4488 {
4489         return test_bit(cmode, &param->supported_cmodes);
4490 }
4491
4492 static int devlink_param_get(struct devlink *devlink,
4493                              const struct devlink_param *param,
4494                              struct devlink_param_gset_ctx *ctx)
4495 {
4496         if (!param->get)
4497                 return -EOPNOTSUPP;
4498         return param->get(devlink, param->id, ctx);
4499 }
4500
4501 static int devlink_param_set(struct devlink *devlink,
4502                              const struct devlink_param *param,
4503                              struct devlink_param_gset_ctx *ctx)
4504 {
4505         if (!param->set)
4506                 return -EOPNOTSUPP;
4507         return param->set(devlink, param->id, ctx);
4508 }
4509
4510 static int
4511 devlink_param_type_to_nla_type(enum devlink_param_type param_type)
4512 {
4513         switch (param_type) {
4514         case DEVLINK_PARAM_TYPE_U8:
4515                 return NLA_U8;
4516         case DEVLINK_PARAM_TYPE_U16:
4517                 return NLA_U16;
4518         case DEVLINK_PARAM_TYPE_U32:
4519                 return NLA_U32;
4520         case DEVLINK_PARAM_TYPE_STRING:
4521                 return NLA_STRING;
4522         case DEVLINK_PARAM_TYPE_BOOL:
4523                 return NLA_FLAG;
4524         default:
4525                 return -EINVAL;
4526         }
4527 }
4528
4529 static int
4530 devlink_nl_param_value_fill_one(struct sk_buff *msg,
4531                                 enum devlink_param_type type,
4532                                 enum devlink_param_cmode cmode,
4533                                 union devlink_param_value val)
4534 {
4535         struct nlattr *param_value_attr;
4536
4537         param_value_attr = nla_nest_start_noflag(msg,
4538                                                  DEVLINK_ATTR_PARAM_VALUE);
4539         if (!param_value_attr)
4540                 goto nla_put_failure;
4541
4542         if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_CMODE, cmode))
4543                 goto value_nest_cancel;
4544
4545         switch (type) {
4546         case DEVLINK_PARAM_TYPE_U8:
4547                 if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu8))
4548                         goto value_nest_cancel;
4549                 break;
4550         case DEVLINK_PARAM_TYPE_U16:
4551                 if (nla_put_u16(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu16))
4552                         goto value_nest_cancel;
4553                 break;
4554         case DEVLINK_PARAM_TYPE_U32:
4555                 if (nla_put_u32(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu32))
4556                         goto value_nest_cancel;
4557                 break;
4558         case DEVLINK_PARAM_TYPE_STRING:
4559                 if (nla_put_string(msg, DEVLINK_ATTR_PARAM_VALUE_DATA,
4560                                    val.vstr))
4561                         goto value_nest_cancel;
4562                 break;
4563         case DEVLINK_PARAM_TYPE_BOOL:
4564                 if (val.vbool &&
4565                     nla_put_flag(msg, DEVLINK_ATTR_PARAM_VALUE_DATA))
4566                         goto value_nest_cancel;
4567                 break;
4568         }
4569
4570         nla_nest_end(msg, param_value_attr);
4571         return 0;
4572
4573 value_nest_cancel:
4574         nla_nest_cancel(msg, param_value_attr);
4575 nla_put_failure:
4576         return -EMSGSIZE;
4577 }
4578
4579 static int devlink_nl_param_fill(struct sk_buff *msg, struct devlink *devlink,
4580                                  unsigned int port_index,
4581                                  struct devlink_param_item *param_item,
4582                                  enum devlink_command cmd,
4583                                  u32 portid, u32 seq, int flags)
4584 {
4585         union devlink_param_value param_value[DEVLINK_PARAM_CMODE_MAX + 1];
4586         bool param_value_set[DEVLINK_PARAM_CMODE_MAX + 1] = {};
4587         const struct devlink_param *param = param_item->param;
4588         struct devlink_param_gset_ctx ctx;
4589         struct nlattr *param_values_list;
4590         struct nlattr *param_attr;
4591         int nla_type;
4592         void *hdr;
4593         int err;
4594         int i;
4595
4596         /* Get value from driver part to driverinit configuration mode */
4597         for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
4598                 if (!devlink_param_cmode_is_supported(param, i))
4599                         continue;
4600                 if (i == DEVLINK_PARAM_CMODE_DRIVERINIT) {
4601                         if (!param_item->driverinit_value_valid)
4602                                 return -EOPNOTSUPP;
4603                         param_value[i] = param_item->driverinit_value;
4604                 } else {
4605                         ctx.cmode = i;
4606                         err = devlink_param_get(devlink, param, &ctx);
4607                         if (err)
4608                                 return err;
4609                         param_value[i] = ctx.val;
4610                 }
4611                 param_value_set[i] = true;
4612         }
4613
4614         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
4615         if (!hdr)
4616                 return -EMSGSIZE;
4617
4618         if (devlink_nl_put_handle(msg, devlink))
4619                 goto genlmsg_cancel;
4620
4621         if (cmd == DEVLINK_CMD_PORT_PARAM_GET ||
4622             cmd == DEVLINK_CMD_PORT_PARAM_NEW ||
4623             cmd == DEVLINK_CMD_PORT_PARAM_DEL)
4624                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, port_index))
4625                         goto genlmsg_cancel;
4626
4627         param_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PARAM);
4628         if (!param_attr)
4629                 goto genlmsg_cancel;
4630         if (nla_put_string(msg, DEVLINK_ATTR_PARAM_NAME, param->name))
4631                 goto param_nest_cancel;
4632         if (param->generic && nla_put_flag(msg, DEVLINK_ATTR_PARAM_GENERIC))
4633                 goto param_nest_cancel;
4634
4635         nla_type = devlink_param_type_to_nla_type(param->type);
4636         if (nla_type < 0)
4637                 goto param_nest_cancel;
4638         if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_TYPE, nla_type))
4639                 goto param_nest_cancel;
4640
4641         param_values_list = nla_nest_start_noflag(msg,
4642                                                   DEVLINK_ATTR_PARAM_VALUES_LIST);
4643         if (!param_values_list)
4644                 goto param_nest_cancel;
4645
4646         for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
4647                 if (!param_value_set[i])
4648                         continue;
4649                 err = devlink_nl_param_value_fill_one(msg, param->type,
4650                                                       i, param_value[i]);
4651                 if (err)
4652                         goto values_list_nest_cancel;
4653         }
4654
4655         nla_nest_end(msg, param_values_list);
4656         nla_nest_end(msg, param_attr);
4657         genlmsg_end(msg, hdr);
4658         return 0;
4659
4660 values_list_nest_cancel:
4661         nla_nest_end(msg, param_values_list);
4662 param_nest_cancel:
4663         nla_nest_cancel(msg, param_attr);
4664 genlmsg_cancel:
4665         genlmsg_cancel(msg, hdr);
4666         return -EMSGSIZE;
4667 }
4668
4669 static void devlink_param_notify(struct devlink *devlink,
4670                                  unsigned int port_index,
4671                                  struct devlink_param_item *param_item,
4672                                  enum devlink_command cmd)
4673 {
4674         struct sk_buff *msg;
4675         int err;
4676
4677         WARN_ON(cmd != DEVLINK_CMD_PARAM_NEW && cmd != DEVLINK_CMD_PARAM_DEL &&
4678                 cmd != DEVLINK_CMD_PORT_PARAM_NEW &&
4679                 cmd != DEVLINK_CMD_PORT_PARAM_DEL);
4680         ASSERT_DEVLINK_REGISTERED(devlink);
4681
4682         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4683         if (!msg)
4684                 return;
4685         err = devlink_nl_param_fill(msg, devlink, port_index, param_item, cmd,
4686                                     0, 0, 0);
4687         if (err) {
4688                 nlmsg_free(msg);
4689                 return;
4690         }
4691
4692         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
4693                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
4694 }
4695
4696 static int devlink_nl_cmd_param_get_dumpit(struct sk_buff *msg,
4697                                            struct netlink_callback *cb)
4698 {
4699         struct devlink_param_item *param_item;
4700         struct devlink *devlink;
4701         int start = cb->args[0];
4702         unsigned long index;
4703         int idx = 0;
4704         int err = 0;
4705
4706         mutex_lock(&devlink_mutex);
4707         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
4708                 if (!devlink_try_get(devlink))
4709                         continue;
4710
4711                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
4712                         goto retry;
4713
4714                 mutex_lock(&devlink->lock);
4715                 list_for_each_entry(param_item, &devlink->param_list, list) {
4716                         if (idx < start) {
4717                                 idx++;
4718                                 continue;
4719                         }
4720                         err = devlink_nl_param_fill(msg, devlink, 0, param_item,
4721                                                     DEVLINK_CMD_PARAM_GET,
4722                                                     NETLINK_CB(cb->skb).portid,
4723                                                     cb->nlh->nlmsg_seq,
4724                                                     NLM_F_MULTI);
4725                         if (err == -EOPNOTSUPP) {
4726                                 err = 0;
4727                         } else if (err) {
4728                                 mutex_unlock(&devlink->lock);
4729                                 devlink_put(devlink);
4730                                 goto out;
4731                         }
4732                         idx++;
4733                 }
4734                 mutex_unlock(&devlink->lock);
4735 retry:
4736                 devlink_put(devlink);
4737         }
4738 out:
4739         mutex_unlock(&devlink_mutex);
4740
4741         if (err != -EMSGSIZE)
4742                 return err;
4743
4744         cb->args[0] = idx;
4745         return msg->len;
4746 }
4747
4748 static int
4749 devlink_param_type_get_from_info(struct genl_info *info,
4750                                  enum devlink_param_type *param_type)
4751 {
4752         if (!info->attrs[DEVLINK_ATTR_PARAM_TYPE])
4753                 return -EINVAL;
4754
4755         switch (nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_TYPE])) {
4756         case NLA_U8:
4757                 *param_type = DEVLINK_PARAM_TYPE_U8;
4758                 break;
4759         case NLA_U16:
4760                 *param_type = DEVLINK_PARAM_TYPE_U16;
4761                 break;
4762         case NLA_U32:
4763                 *param_type = DEVLINK_PARAM_TYPE_U32;
4764                 break;
4765         case NLA_STRING:
4766                 *param_type = DEVLINK_PARAM_TYPE_STRING;
4767                 break;
4768         case NLA_FLAG:
4769                 *param_type = DEVLINK_PARAM_TYPE_BOOL;
4770                 break;
4771         default:
4772                 return -EINVAL;
4773         }
4774
4775         return 0;
4776 }
4777
4778 static int
4779 devlink_param_value_get_from_info(const struct devlink_param *param,
4780                                   struct genl_info *info,
4781                                   union devlink_param_value *value)
4782 {
4783         struct nlattr *param_data;
4784         int len;
4785
4786         param_data = info->attrs[DEVLINK_ATTR_PARAM_VALUE_DATA];
4787
4788         if (param->type != DEVLINK_PARAM_TYPE_BOOL && !param_data)
4789                 return -EINVAL;
4790
4791         switch (param->type) {
4792         case DEVLINK_PARAM_TYPE_U8:
4793                 if (nla_len(param_data) != sizeof(u8))
4794                         return -EINVAL;
4795                 value->vu8 = nla_get_u8(param_data);
4796                 break;
4797         case DEVLINK_PARAM_TYPE_U16:
4798                 if (nla_len(param_data) != sizeof(u16))
4799                         return -EINVAL;
4800                 value->vu16 = nla_get_u16(param_data);
4801                 break;
4802         case DEVLINK_PARAM_TYPE_U32:
4803                 if (nla_len(param_data) != sizeof(u32))
4804                         return -EINVAL;
4805                 value->vu32 = nla_get_u32(param_data);
4806                 break;
4807         case DEVLINK_PARAM_TYPE_STRING:
4808                 len = strnlen(nla_data(param_data), nla_len(param_data));
4809                 if (len == nla_len(param_data) ||
4810                     len >= __DEVLINK_PARAM_MAX_STRING_VALUE)
4811                         return -EINVAL;
4812                 strcpy(value->vstr, nla_data(param_data));
4813                 break;
4814         case DEVLINK_PARAM_TYPE_BOOL:
4815                 if (param_data && nla_len(param_data))
4816                         return -EINVAL;
4817                 value->vbool = nla_get_flag(param_data);
4818                 break;
4819         }
4820         return 0;
4821 }
4822
4823 static struct devlink_param_item *
4824 devlink_param_get_from_info(struct list_head *param_list,
4825                             struct genl_info *info)
4826 {
4827         char *param_name;
4828
4829         if (!info->attrs[DEVLINK_ATTR_PARAM_NAME])
4830                 return NULL;
4831
4832         param_name = nla_data(info->attrs[DEVLINK_ATTR_PARAM_NAME]);
4833         return devlink_param_find_by_name(param_list, param_name);
4834 }
4835
4836 static int devlink_nl_cmd_param_get_doit(struct sk_buff *skb,
4837                                          struct genl_info *info)
4838 {
4839         struct devlink *devlink = info->user_ptr[0];
4840         struct devlink_param_item *param_item;
4841         struct sk_buff *msg;
4842         int err;
4843
4844         param_item = devlink_param_get_from_info(&devlink->param_list, info);
4845         if (!param_item)
4846                 return -EINVAL;
4847
4848         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4849         if (!msg)
4850                 return -ENOMEM;
4851
4852         err = devlink_nl_param_fill(msg, devlink, 0, param_item,
4853                                     DEVLINK_CMD_PARAM_GET,
4854                                     info->snd_portid, info->snd_seq, 0);
4855         if (err) {
4856                 nlmsg_free(msg);
4857                 return err;
4858         }
4859
4860         return genlmsg_reply(msg, info);
4861 }
4862
4863 static int __devlink_nl_cmd_param_set_doit(struct devlink *devlink,
4864                                            unsigned int port_index,
4865                                            struct list_head *param_list,
4866                                            struct genl_info *info,
4867                                            enum devlink_command cmd)
4868 {
4869         enum devlink_param_type param_type;
4870         struct devlink_param_gset_ctx ctx;
4871         enum devlink_param_cmode cmode;
4872         struct devlink_param_item *param_item;
4873         const struct devlink_param *param;
4874         union devlink_param_value value;
4875         int err = 0;
4876
4877         param_item = devlink_param_get_from_info(param_list, info);
4878         if (!param_item)
4879                 return -EINVAL;
4880         param = param_item->param;
4881         err = devlink_param_type_get_from_info(info, &param_type);
4882         if (err)
4883                 return err;
4884         if (param_type != param->type)
4885                 return -EINVAL;
4886         err = devlink_param_value_get_from_info(param, info, &value);
4887         if (err)
4888                 return err;
4889         if (param->validate) {
4890                 err = param->validate(devlink, param->id, value, info->extack);
4891                 if (err)
4892                         return err;
4893         }
4894
4895         if (!info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE])
4896                 return -EINVAL;
4897         cmode = nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE]);
4898         if (!devlink_param_cmode_is_supported(param, cmode))
4899                 return -EOPNOTSUPP;
4900
4901         if (cmode == DEVLINK_PARAM_CMODE_DRIVERINIT) {
4902                 if (param->type == DEVLINK_PARAM_TYPE_STRING)
4903                         strcpy(param_item->driverinit_value.vstr, value.vstr);
4904                 else
4905                         param_item->driverinit_value = value;
4906                 param_item->driverinit_value_valid = true;
4907         } else {
4908                 if (!param->set)
4909                         return -EOPNOTSUPP;
4910                 ctx.val = value;
4911                 ctx.cmode = cmode;
4912                 err = devlink_param_set(devlink, param, &ctx);
4913                 if (err)
4914                         return err;
4915         }
4916
4917         devlink_param_notify(devlink, port_index, param_item, cmd);
4918         return 0;
4919 }
4920
4921 static int devlink_nl_cmd_param_set_doit(struct sk_buff *skb,
4922                                          struct genl_info *info)
4923 {
4924         struct devlink *devlink = info->user_ptr[0];
4925
4926         return __devlink_nl_cmd_param_set_doit(devlink, 0, &devlink->param_list,
4927                                                info, DEVLINK_CMD_PARAM_NEW);
4928 }
4929
4930 static int devlink_nl_cmd_port_param_get_dumpit(struct sk_buff *msg,
4931                                                 struct netlink_callback *cb)
4932 {
4933         struct devlink_param_item *param_item;
4934         struct devlink_port *devlink_port;
4935         struct devlink *devlink;
4936         int start = cb->args[0];
4937         unsigned long index;
4938         int idx = 0;
4939         int err = 0;
4940
4941         mutex_lock(&devlink_mutex);
4942         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
4943                 if (!devlink_try_get(devlink))
4944                         continue;
4945
4946                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
4947                         goto retry;
4948
4949                 mutex_lock(&devlink->lock);
4950                 list_for_each_entry(devlink_port, &devlink->port_list, list) {
4951                         list_for_each_entry(param_item,
4952                                             &devlink_port->param_list, list) {
4953                                 if (idx < start) {
4954                                         idx++;
4955                                         continue;
4956                                 }
4957                                 err = devlink_nl_param_fill(msg,
4958                                                 devlink_port->devlink,
4959                                                 devlink_port->index, param_item,
4960                                                 DEVLINK_CMD_PORT_PARAM_GET,
4961                                                 NETLINK_CB(cb->skb).portid,
4962                                                 cb->nlh->nlmsg_seq,
4963                                                 NLM_F_MULTI);
4964                                 if (err == -EOPNOTSUPP) {
4965                                         err = 0;
4966                                 } else if (err) {
4967                                         mutex_unlock(&devlink->lock);
4968                                         devlink_put(devlink);
4969                                         goto out;
4970                                 }
4971                                 idx++;
4972                         }
4973                 }
4974                 mutex_unlock(&devlink->lock);
4975 retry:
4976                 devlink_put(devlink);
4977         }
4978 out:
4979         mutex_unlock(&devlink_mutex);
4980
4981         if (err != -EMSGSIZE)
4982                 return err;
4983
4984         cb->args[0] = idx;
4985         return msg->len;
4986 }
4987
4988 static int devlink_nl_cmd_port_param_get_doit(struct sk_buff *skb,
4989                                               struct genl_info *info)
4990 {
4991         struct devlink_port *devlink_port = info->user_ptr[1];
4992         struct devlink_param_item *param_item;
4993         struct sk_buff *msg;
4994         int err;
4995
4996         param_item = devlink_param_get_from_info(&devlink_port->param_list,
4997                                                  info);
4998         if (!param_item)
4999                 return -EINVAL;
5000
5001         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5002         if (!msg)
5003                 return -ENOMEM;
5004
5005         err = devlink_nl_param_fill(msg, devlink_port->devlink,
5006                                     devlink_port->index, param_item,
5007                                     DEVLINK_CMD_PORT_PARAM_GET,
5008                                     info->snd_portid, info->snd_seq, 0);
5009         if (err) {
5010                 nlmsg_free(msg);
5011                 return err;
5012         }
5013
5014         return genlmsg_reply(msg, info);
5015 }
5016
5017 static int devlink_nl_cmd_port_param_set_doit(struct sk_buff *skb,
5018                                               struct genl_info *info)
5019 {
5020         struct devlink_port *devlink_port = info->user_ptr[1];
5021
5022         return __devlink_nl_cmd_param_set_doit(devlink_port->devlink,
5023                                                devlink_port->index,
5024                                                &devlink_port->param_list, info,
5025                                                DEVLINK_CMD_PORT_PARAM_NEW);
5026 }
5027
5028 static int devlink_nl_region_snapshot_id_put(struct sk_buff *msg,
5029                                              struct devlink *devlink,
5030                                              struct devlink_snapshot *snapshot)
5031 {
5032         struct nlattr *snap_attr;
5033         int err;
5034
5035         snap_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_REGION_SNAPSHOT);
5036         if (!snap_attr)
5037                 return -EINVAL;
5038
5039         err = nla_put_u32(msg, DEVLINK_ATTR_REGION_SNAPSHOT_ID, snapshot->id);
5040         if (err)
5041                 goto nla_put_failure;
5042
5043         nla_nest_end(msg, snap_attr);
5044         return 0;
5045
5046 nla_put_failure:
5047         nla_nest_cancel(msg, snap_attr);
5048         return err;
5049 }
5050
5051 static int devlink_nl_region_snapshots_id_put(struct sk_buff *msg,
5052                                               struct devlink *devlink,
5053                                               struct devlink_region *region)
5054 {
5055         struct devlink_snapshot *snapshot;
5056         struct nlattr *snapshots_attr;
5057         int err;
5058
5059         snapshots_attr = nla_nest_start_noflag(msg,
5060                                                DEVLINK_ATTR_REGION_SNAPSHOTS);
5061         if (!snapshots_attr)
5062                 return -EINVAL;
5063
5064         list_for_each_entry(snapshot, &region->snapshot_list, list) {
5065                 err = devlink_nl_region_snapshot_id_put(msg, devlink, snapshot);
5066                 if (err)
5067                         goto nla_put_failure;
5068         }
5069
5070         nla_nest_end(msg, snapshots_attr);
5071         return 0;
5072
5073 nla_put_failure:
5074         nla_nest_cancel(msg, snapshots_attr);
5075         return err;
5076 }
5077
5078 static int devlink_nl_region_fill(struct sk_buff *msg, struct devlink *devlink,
5079                                   enum devlink_command cmd, u32 portid,
5080                                   u32 seq, int flags,
5081                                   struct devlink_region *region)
5082 {
5083         void *hdr;
5084         int err;
5085
5086         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
5087         if (!hdr)
5088                 return -EMSGSIZE;
5089
5090         err = devlink_nl_put_handle(msg, devlink);
5091         if (err)
5092                 goto nla_put_failure;
5093
5094         if (region->port) {
5095                 err = nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
5096                                   region->port->index);
5097                 if (err)
5098                         goto nla_put_failure;
5099         }
5100
5101         err = nla_put_string(msg, DEVLINK_ATTR_REGION_NAME, region->ops->name);
5102         if (err)
5103                 goto nla_put_failure;
5104
5105         err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_SIZE,
5106                                 region->size,
5107                                 DEVLINK_ATTR_PAD);
5108         if (err)
5109                 goto nla_put_failure;
5110
5111         err = nla_put_u32(msg, DEVLINK_ATTR_REGION_MAX_SNAPSHOTS,
5112                           region->max_snapshots);
5113         if (err)
5114                 goto nla_put_failure;
5115
5116         err = devlink_nl_region_snapshots_id_put(msg, devlink, region);
5117         if (err)
5118                 goto nla_put_failure;
5119
5120         genlmsg_end(msg, hdr);
5121         return 0;
5122
5123 nla_put_failure:
5124         genlmsg_cancel(msg, hdr);
5125         return err;
5126 }
5127
5128 static struct sk_buff *
5129 devlink_nl_region_notify_build(struct devlink_region *region,
5130                                struct devlink_snapshot *snapshot,
5131                                enum devlink_command cmd, u32 portid, u32 seq)
5132 {
5133         struct devlink *devlink = region->devlink;
5134         struct sk_buff *msg;
5135         void *hdr;
5136         int err;
5137
5138
5139         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5140         if (!msg)
5141                 return ERR_PTR(-ENOMEM);
5142
5143         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, 0, cmd);
5144         if (!hdr) {
5145                 err = -EMSGSIZE;
5146                 goto out_free_msg;
5147         }
5148
5149         err = devlink_nl_put_handle(msg, devlink);
5150         if (err)
5151                 goto out_cancel_msg;
5152
5153         if (region->port) {
5154                 err = nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
5155                                   region->port->index);
5156                 if (err)
5157                         goto out_cancel_msg;
5158         }
5159
5160         err = nla_put_string(msg, DEVLINK_ATTR_REGION_NAME,
5161                              region->ops->name);
5162         if (err)
5163                 goto out_cancel_msg;
5164
5165         if (snapshot) {
5166                 err = nla_put_u32(msg, DEVLINK_ATTR_REGION_SNAPSHOT_ID,
5167                                   snapshot->id);
5168                 if (err)
5169                         goto out_cancel_msg;
5170         } else {
5171                 err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_SIZE,
5172                                         region->size, DEVLINK_ATTR_PAD);
5173                 if (err)
5174                         goto out_cancel_msg;
5175         }
5176         genlmsg_end(msg, hdr);
5177
5178         return msg;
5179
5180 out_cancel_msg:
5181         genlmsg_cancel(msg, hdr);
5182 out_free_msg:
5183         nlmsg_free(msg);
5184         return ERR_PTR(err);
5185 }
5186
5187 static void devlink_nl_region_notify(struct devlink_region *region,
5188                                      struct devlink_snapshot *snapshot,
5189                                      enum devlink_command cmd)
5190 {
5191         struct devlink *devlink = region->devlink;
5192         struct sk_buff *msg;
5193
5194         WARN_ON(cmd != DEVLINK_CMD_REGION_NEW && cmd != DEVLINK_CMD_REGION_DEL);
5195         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
5196                 return;
5197
5198         msg = devlink_nl_region_notify_build(region, snapshot, cmd, 0, 0);
5199         if (IS_ERR(msg))
5200                 return;
5201
5202         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
5203                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
5204 }
5205
5206 /**
5207  * __devlink_snapshot_id_increment - Increment number of snapshots using an id
5208  *      @devlink: devlink instance
5209  *      @id: the snapshot id
5210  *
5211  *      Track when a new snapshot begins using an id. Load the count for the
5212  *      given id from the snapshot xarray, increment it, and store it back.
5213  *
5214  *      Called when a new snapshot is created with the given id.
5215  *
5216  *      The id *must* have been previously allocated by
5217  *      devlink_region_snapshot_id_get().
5218  *
5219  *      Returns 0 on success, or an error on failure.
5220  */
5221 static int __devlink_snapshot_id_increment(struct devlink *devlink, u32 id)
5222 {
5223         unsigned long count;
5224         void *p;
5225
5226         lockdep_assert_held(&devlink->lock);
5227
5228         p = xa_load(&devlink->snapshot_ids, id);
5229         if (WARN_ON(!p))
5230                 return -EINVAL;
5231
5232         if (WARN_ON(!xa_is_value(p)))
5233                 return -EINVAL;
5234
5235         count = xa_to_value(p);
5236         count++;
5237
5238         return xa_err(xa_store(&devlink->snapshot_ids, id, xa_mk_value(count),
5239                                GFP_KERNEL));
5240 }
5241
5242 /**
5243  * __devlink_snapshot_id_decrement - Decrease number of snapshots using an id
5244  *      @devlink: devlink instance
5245  *      @id: the snapshot id
5246  *
5247  *      Track when a snapshot is deleted and stops using an id. Load the count
5248  *      for the given id from the snapshot xarray, decrement it, and store it
5249  *      back.
5250  *
5251  *      If the count reaches zero, erase this id from the xarray, freeing it
5252  *      up for future re-use by devlink_region_snapshot_id_get().
5253  *
5254  *      Called when a snapshot using the given id is deleted, and when the
5255  *      initial allocator of the id is finished using it.
5256  */
5257 static void __devlink_snapshot_id_decrement(struct devlink *devlink, u32 id)
5258 {
5259         unsigned long count;
5260         void *p;
5261
5262         lockdep_assert_held(&devlink->lock);
5263
5264         p = xa_load(&devlink->snapshot_ids, id);
5265         if (WARN_ON(!p))
5266                 return;
5267
5268         if (WARN_ON(!xa_is_value(p)))
5269                 return;
5270
5271         count = xa_to_value(p);
5272
5273         if (count > 1) {
5274                 count--;
5275                 xa_store(&devlink->snapshot_ids, id, xa_mk_value(count),
5276                          GFP_KERNEL);
5277         } else {
5278                 /* If this was the last user, we can erase this id */
5279                 xa_erase(&devlink->snapshot_ids, id);
5280         }
5281 }
5282
5283 /**
5284  *      __devlink_snapshot_id_insert - Insert a specific snapshot ID
5285  *      @devlink: devlink instance
5286  *      @id: the snapshot id
5287  *
5288  *      Mark the given snapshot id as used by inserting a zero value into the
5289  *      snapshot xarray.
5290  *
5291  *      This must be called while holding the devlink instance lock. Unlike
5292  *      devlink_snapshot_id_get, the initial reference count is zero, not one.
5293  *      It is expected that the id will immediately be used before
5294  *      releasing the devlink instance lock.
5295  *
5296  *      Returns zero on success, or an error code if the snapshot id could not
5297  *      be inserted.
5298  */
5299 static int __devlink_snapshot_id_insert(struct devlink *devlink, u32 id)
5300 {
5301         lockdep_assert_held(&devlink->lock);
5302
5303         if (xa_load(&devlink->snapshot_ids, id))
5304                 return -EEXIST;
5305
5306         return xa_err(xa_store(&devlink->snapshot_ids, id, xa_mk_value(0),
5307                                GFP_KERNEL));
5308 }
5309
5310 /**
5311  *      __devlink_region_snapshot_id_get - get snapshot ID
5312  *      @devlink: devlink instance
5313  *      @id: storage to return snapshot id
5314  *
5315  *      Allocates a new snapshot id. Returns zero on success, or a negative
5316  *      error on failure. Must be called while holding the devlink instance
5317  *      lock.
5318  *
5319  *      Snapshot IDs are tracked using an xarray which stores the number of
5320  *      users of the snapshot id.
5321  *
5322  *      Note that the caller of this function counts as a 'user', in order to
5323  *      avoid race conditions. The caller must release its hold on the
5324  *      snapshot by using devlink_region_snapshot_id_put.
5325  */
5326 static int __devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id)
5327 {
5328         lockdep_assert_held(&devlink->lock);
5329
5330         return xa_alloc(&devlink->snapshot_ids, id, xa_mk_value(1),
5331                         xa_limit_32b, GFP_KERNEL);
5332 }
5333
5334 /**
5335  *      __devlink_region_snapshot_create - create a new snapshot
5336  *      This will add a new snapshot of a region. The snapshot
5337  *      will be stored on the region struct and can be accessed
5338  *      from devlink. This is useful for future analyses of snapshots.
5339  *      Multiple snapshots can be created on a region.
5340  *      The @snapshot_id should be obtained using the getter function.
5341  *
5342  *      Must be called only while holding the devlink instance lock.
5343  *
5344  *      @region: devlink region of the snapshot
5345  *      @data: snapshot data
5346  *      @snapshot_id: snapshot id to be created
5347  */
5348 static int
5349 __devlink_region_snapshot_create(struct devlink_region *region,
5350                                  u8 *data, u32 snapshot_id)
5351 {
5352         struct devlink *devlink = region->devlink;
5353         struct devlink_snapshot *snapshot;
5354         int err;
5355
5356         lockdep_assert_held(&devlink->lock);
5357
5358         /* check if region can hold one more snapshot */
5359         if (region->cur_snapshots == region->max_snapshots)
5360                 return -ENOSPC;
5361
5362         if (devlink_region_snapshot_get_by_id(region, snapshot_id))
5363                 return -EEXIST;
5364
5365         snapshot = kzalloc(sizeof(*snapshot), GFP_KERNEL);
5366         if (!snapshot)
5367                 return -ENOMEM;
5368
5369         err = __devlink_snapshot_id_increment(devlink, snapshot_id);
5370         if (err)
5371                 goto err_snapshot_id_increment;
5372
5373         snapshot->id = snapshot_id;
5374         snapshot->region = region;
5375         snapshot->data = data;
5376
5377         list_add_tail(&snapshot->list, &region->snapshot_list);
5378
5379         region->cur_snapshots++;
5380
5381         devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_NEW);
5382         return 0;
5383
5384 err_snapshot_id_increment:
5385         kfree(snapshot);
5386         return err;
5387 }
5388
5389 static void devlink_region_snapshot_del(struct devlink_region *region,
5390                                         struct devlink_snapshot *snapshot)
5391 {
5392         struct devlink *devlink = region->devlink;
5393
5394         lockdep_assert_held(&devlink->lock);
5395
5396         devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_DEL);
5397         region->cur_snapshots--;
5398         list_del(&snapshot->list);
5399         region->ops->destructor(snapshot->data);
5400         __devlink_snapshot_id_decrement(devlink, snapshot->id);
5401         kfree(snapshot);
5402 }
5403
5404 static int devlink_nl_cmd_region_get_doit(struct sk_buff *skb,
5405                                           struct genl_info *info)
5406 {
5407         struct devlink *devlink = info->user_ptr[0];
5408         struct devlink_port *port = NULL;
5409         struct devlink_region *region;
5410         const char *region_name;
5411         struct sk_buff *msg;
5412         unsigned int index;
5413         int err;
5414
5415         if (!info->attrs[DEVLINK_ATTR_REGION_NAME])
5416                 return -EINVAL;
5417
5418         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
5419                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
5420
5421                 port = devlink_port_get_by_index(devlink, index);
5422                 if (!port)
5423                         return -ENODEV;
5424         }
5425
5426         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
5427         if (port)
5428                 region = devlink_port_region_get_by_name(port, region_name);
5429         else
5430                 region = devlink_region_get_by_name(devlink, region_name);
5431
5432         if (!region)
5433                 return -EINVAL;
5434
5435         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5436         if (!msg)
5437                 return -ENOMEM;
5438
5439         err = devlink_nl_region_fill(msg, devlink, DEVLINK_CMD_REGION_GET,
5440                                      info->snd_portid, info->snd_seq, 0,
5441                                      region);
5442         if (err) {
5443                 nlmsg_free(msg);
5444                 return err;
5445         }
5446
5447         return genlmsg_reply(msg, info);
5448 }
5449
5450 static int devlink_nl_cmd_region_get_port_dumpit(struct sk_buff *msg,
5451                                                  struct netlink_callback *cb,
5452                                                  struct devlink_port *port,
5453                                                  int *idx,
5454                                                  int start)
5455 {
5456         struct devlink_region *region;
5457         int err = 0;
5458
5459         list_for_each_entry(region, &port->region_list, list) {
5460                 if (*idx < start) {
5461                         (*idx)++;
5462                         continue;
5463                 }
5464                 err = devlink_nl_region_fill(msg, port->devlink,
5465                                              DEVLINK_CMD_REGION_GET,
5466                                              NETLINK_CB(cb->skb).portid,
5467                                              cb->nlh->nlmsg_seq,
5468                                              NLM_F_MULTI, region);
5469                 if (err)
5470                         goto out;
5471                 (*idx)++;
5472         }
5473
5474 out:
5475         return err;
5476 }
5477
5478 static int devlink_nl_cmd_region_get_devlink_dumpit(struct sk_buff *msg,
5479                                                     struct netlink_callback *cb,
5480                                                     struct devlink *devlink,
5481                                                     int *idx,
5482                                                     int start)
5483 {
5484         struct devlink_region *region;
5485         struct devlink_port *port;
5486         int err = 0;
5487
5488         mutex_lock(&devlink->lock);
5489         list_for_each_entry(region, &devlink->region_list, list) {
5490                 if (*idx < start) {
5491                         (*idx)++;
5492                         continue;
5493                 }
5494                 err = devlink_nl_region_fill(msg, devlink,
5495                                              DEVLINK_CMD_REGION_GET,
5496                                              NETLINK_CB(cb->skb).portid,
5497                                              cb->nlh->nlmsg_seq,
5498                                              NLM_F_MULTI, region);
5499                 if (err)
5500                         goto out;
5501                 (*idx)++;
5502         }
5503
5504         list_for_each_entry(port, &devlink->port_list, list) {
5505                 err = devlink_nl_cmd_region_get_port_dumpit(msg, cb, port, idx,
5506                                                             start);
5507                 if (err)
5508                         goto out;
5509         }
5510
5511 out:
5512         mutex_unlock(&devlink->lock);
5513         return err;
5514 }
5515
5516 static int devlink_nl_cmd_region_get_dumpit(struct sk_buff *msg,
5517                                             struct netlink_callback *cb)
5518 {
5519         struct devlink *devlink;
5520         int start = cb->args[0];
5521         unsigned long index;
5522         int idx = 0;
5523         int err = 0;
5524
5525         mutex_lock(&devlink_mutex);
5526         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
5527                 if (!devlink_try_get(devlink))
5528                         continue;
5529
5530                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
5531                         goto retry;
5532
5533                 err = devlink_nl_cmd_region_get_devlink_dumpit(msg, cb, devlink,
5534                                                                &idx, start);
5535 retry:
5536                 devlink_put(devlink);
5537                 if (err)
5538                         goto out;
5539         }
5540 out:
5541         mutex_unlock(&devlink_mutex);
5542         cb->args[0] = idx;
5543         return msg->len;
5544 }
5545
5546 static int devlink_nl_cmd_region_del(struct sk_buff *skb,
5547                                      struct genl_info *info)
5548 {
5549         struct devlink *devlink = info->user_ptr[0];
5550         struct devlink_snapshot *snapshot;
5551         struct devlink_port *port = NULL;
5552         struct devlink_region *region;
5553         const char *region_name;
5554         unsigned int index;
5555         u32 snapshot_id;
5556
5557         if (!info->attrs[DEVLINK_ATTR_REGION_NAME] ||
5558             !info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID])
5559                 return -EINVAL;
5560
5561         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
5562         snapshot_id = nla_get_u32(info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]);
5563
5564         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
5565                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
5566
5567                 port = devlink_port_get_by_index(devlink, index);
5568                 if (!port)
5569                         return -ENODEV;
5570         }
5571
5572         if (port)
5573                 region = devlink_port_region_get_by_name(port, region_name);
5574         else
5575                 region = devlink_region_get_by_name(devlink, region_name);
5576
5577         if (!region)
5578                 return -EINVAL;
5579
5580         snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id);
5581         if (!snapshot)
5582                 return -EINVAL;
5583
5584         devlink_region_snapshot_del(region, snapshot);
5585         return 0;
5586 }
5587
5588 static int
5589 devlink_nl_cmd_region_new(struct sk_buff *skb, struct genl_info *info)
5590 {
5591         struct devlink *devlink = info->user_ptr[0];
5592         struct devlink_snapshot *snapshot;
5593         struct devlink_port *port = NULL;
5594         struct nlattr *snapshot_id_attr;
5595         struct devlink_region *region;
5596         const char *region_name;
5597         unsigned int index;
5598         u32 snapshot_id;
5599         u8 *data;
5600         int err;
5601
5602         if (!info->attrs[DEVLINK_ATTR_REGION_NAME]) {
5603                 NL_SET_ERR_MSG_MOD(info->extack, "No region name provided");
5604                 return -EINVAL;
5605         }
5606
5607         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
5608
5609         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
5610                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
5611
5612                 port = devlink_port_get_by_index(devlink, index);
5613                 if (!port)
5614                         return -ENODEV;
5615         }
5616
5617         if (port)
5618                 region = devlink_port_region_get_by_name(port, region_name);
5619         else
5620                 region = devlink_region_get_by_name(devlink, region_name);
5621
5622         if (!region) {
5623                 NL_SET_ERR_MSG_MOD(info->extack, "The requested region does not exist");
5624                 return -EINVAL;
5625         }
5626
5627         if (!region->ops->snapshot) {
5628                 NL_SET_ERR_MSG_MOD(info->extack, "The requested region does not support taking an immediate snapshot");
5629                 return -EOPNOTSUPP;
5630         }
5631
5632         if (region->cur_snapshots == region->max_snapshots) {
5633                 NL_SET_ERR_MSG_MOD(info->extack, "The region has reached the maximum number of stored snapshots");
5634                 return -ENOSPC;
5635         }
5636
5637         snapshot_id_attr = info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID];
5638         if (snapshot_id_attr) {
5639                 snapshot_id = nla_get_u32(snapshot_id_attr);
5640
5641                 if (devlink_region_snapshot_get_by_id(region, snapshot_id)) {
5642                         NL_SET_ERR_MSG_MOD(info->extack, "The requested snapshot id is already in use");
5643                         return -EEXIST;
5644                 }
5645
5646                 err = __devlink_snapshot_id_insert(devlink, snapshot_id);
5647                 if (err)
5648                         return err;
5649         } else {
5650                 err = __devlink_region_snapshot_id_get(devlink, &snapshot_id);
5651                 if (err) {
5652                         NL_SET_ERR_MSG_MOD(info->extack, "Failed to allocate a new snapshot id");
5653                         return err;
5654                 }
5655         }
5656
5657         if (port)
5658                 err = region->port_ops->snapshot(port, region->port_ops,
5659                                                  info->extack, &data);
5660         else
5661                 err = region->ops->snapshot(devlink, region->ops,
5662                                             info->extack, &data);
5663         if (err)
5664                 goto err_snapshot_capture;
5665
5666         err = __devlink_region_snapshot_create(region, data, snapshot_id);
5667         if (err)
5668                 goto err_snapshot_create;
5669
5670         if (!snapshot_id_attr) {
5671                 struct sk_buff *msg;
5672
5673                 snapshot = devlink_region_snapshot_get_by_id(region,
5674                                                              snapshot_id);
5675                 if (WARN_ON(!snapshot))
5676                         return -EINVAL;
5677
5678                 msg = devlink_nl_region_notify_build(region, snapshot,
5679                                                      DEVLINK_CMD_REGION_NEW,
5680                                                      info->snd_portid,
5681                                                      info->snd_seq);
5682                 err = PTR_ERR_OR_ZERO(msg);
5683                 if (err)
5684                         goto err_notify;
5685
5686                 err = genlmsg_reply(msg, info);
5687                 if (err)
5688                         goto err_notify;
5689         }
5690
5691         return 0;
5692
5693 err_snapshot_create:
5694         region->ops->destructor(data);
5695 err_snapshot_capture:
5696         __devlink_snapshot_id_decrement(devlink, snapshot_id);
5697         return err;
5698
5699 err_notify:
5700         devlink_region_snapshot_del(region, snapshot);
5701         return err;
5702 }
5703
5704 static int devlink_nl_cmd_region_read_chunk_fill(struct sk_buff *msg,
5705                                                  struct devlink *devlink,
5706                                                  u8 *chunk, u32 chunk_size,
5707                                                  u64 addr)
5708 {
5709         struct nlattr *chunk_attr;
5710         int err;
5711
5712         chunk_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_REGION_CHUNK);
5713         if (!chunk_attr)
5714                 return -EINVAL;
5715
5716         err = nla_put(msg, DEVLINK_ATTR_REGION_CHUNK_DATA, chunk_size, chunk);
5717         if (err)
5718                 goto nla_put_failure;
5719
5720         err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_CHUNK_ADDR, addr,
5721                                 DEVLINK_ATTR_PAD);
5722         if (err)
5723                 goto nla_put_failure;
5724
5725         nla_nest_end(msg, chunk_attr);
5726         return 0;
5727
5728 nla_put_failure:
5729         nla_nest_cancel(msg, chunk_attr);
5730         return err;
5731 }
5732
5733 #define DEVLINK_REGION_READ_CHUNK_SIZE 256
5734
5735 static int devlink_nl_region_read_snapshot_fill(struct sk_buff *skb,
5736                                                 struct devlink *devlink,
5737                                                 struct devlink_region *region,
5738                                                 struct nlattr **attrs,
5739                                                 u64 start_offset,
5740                                                 u64 end_offset,
5741                                                 u64 *new_offset)
5742 {
5743         struct devlink_snapshot *snapshot;
5744         u64 curr_offset = start_offset;
5745         u32 snapshot_id;
5746         int err = 0;
5747
5748         *new_offset = start_offset;
5749
5750         snapshot_id = nla_get_u32(attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]);
5751         snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id);
5752         if (!snapshot)
5753                 return -EINVAL;
5754
5755         while (curr_offset < end_offset) {
5756                 u32 data_size;
5757                 u8 *data;
5758
5759                 if (end_offset - curr_offset < DEVLINK_REGION_READ_CHUNK_SIZE)
5760                         data_size = end_offset - curr_offset;
5761                 else
5762                         data_size = DEVLINK_REGION_READ_CHUNK_SIZE;
5763
5764                 data = &snapshot->data[curr_offset];
5765                 err = devlink_nl_cmd_region_read_chunk_fill(skb, devlink,
5766                                                             data, data_size,
5767                                                             curr_offset);
5768                 if (err)
5769                         break;
5770
5771                 curr_offset += data_size;
5772         }
5773         *new_offset = curr_offset;
5774
5775         return err;
5776 }
5777
5778 static int devlink_nl_cmd_region_read_dumpit(struct sk_buff *skb,
5779                                              struct netlink_callback *cb)
5780 {
5781         const struct genl_dumpit_info *info = genl_dumpit_info(cb);
5782         u64 ret_offset, start_offset, end_offset = U64_MAX;
5783         struct nlattr **attrs = info->attrs;
5784         struct devlink_port *port = NULL;
5785         struct devlink_region *region;
5786         struct nlattr *chunks_attr;
5787         const char *region_name;
5788         struct devlink *devlink;
5789         unsigned int index;
5790         void *hdr;
5791         int err;
5792
5793         start_offset = *((u64 *)&cb->args[0]);
5794
5795         mutex_lock(&devlink_mutex);
5796         devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
5797         if (IS_ERR(devlink)) {
5798                 err = PTR_ERR(devlink);
5799                 goto out_dev;
5800         }
5801
5802         mutex_lock(&devlink->lock);
5803
5804         if (!attrs[DEVLINK_ATTR_REGION_NAME] ||
5805             !attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]) {
5806                 err = -EINVAL;
5807                 goto out_unlock;
5808         }
5809
5810         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
5811                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
5812
5813                 port = devlink_port_get_by_index(devlink, index);
5814                 if (!port) {
5815                         err = -ENODEV;
5816                         goto out_unlock;
5817                 }
5818         }
5819
5820         region_name = nla_data(attrs[DEVLINK_ATTR_REGION_NAME]);
5821
5822         if (port)
5823                 region = devlink_port_region_get_by_name(port, region_name);
5824         else
5825                 region = devlink_region_get_by_name(devlink, region_name);
5826
5827         if (!region) {
5828                 err = -EINVAL;
5829                 goto out_unlock;
5830         }
5831
5832         if (attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR] &&
5833             attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]) {
5834                 if (!start_offset)
5835                         start_offset =
5836                                 nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR]);
5837
5838                 end_offset = nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR]);
5839                 end_offset += nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]);
5840         }
5841
5842         if (end_offset > region->size)
5843                 end_offset = region->size;
5844
5845         /* return 0 if there is no further data to read */
5846         if (start_offset == end_offset) {
5847                 err = 0;
5848                 goto out_unlock;
5849         }
5850
5851         hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
5852                           &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI,
5853                           DEVLINK_CMD_REGION_READ);
5854         if (!hdr) {
5855                 err = -EMSGSIZE;
5856                 goto out_unlock;
5857         }
5858
5859         err = devlink_nl_put_handle(skb, devlink);
5860         if (err)
5861                 goto nla_put_failure;
5862
5863         if (region->port) {
5864                 err = nla_put_u32(skb, DEVLINK_ATTR_PORT_INDEX,
5865                                   region->port->index);
5866                 if (err)
5867                         goto nla_put_failure;
5868         }
5869
5870         err = nla_put_string(skb, DEVLINK_ATTR_REGION_NAME, region_name);
5871         if (err)
5872                 goto nla_put_failure;
5873
5874         chunks_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_REGION_CHUNKS);
5875         if (!chunks_attr) {
5876                 err = -EMSGSIZE;
5877                 goto nla_put_failure;
5878         }
5879
5880         err = devlink_nl_region_read_snapshot_fill(skb, devlink,
5881                                                    region, attrs,
5882                                                    start_offset,
5883                                                    end_offset, &ret_offset);
5884
5885         if (err && err != -EMSGSIZE)
5886                 goto nla_put_failure;
5887
5888         /* Check if there was any progress done to prevent infinite loop */
5889         if (ret_offset == start_offset) {
5890                 err = -EINVAL;
5891                 goto nla_put_failure;
5892         }
5893
5894         *((u64 *)&cb->args[0]) = ret_offset;
5895
5896         nla_nest_end(skb, chunks_attr);
5897         genlmsg_end(skb, hdr);
5898         mutex_unlock(&devlink->lock);
5899         devlink_put(devlink);
5900         mutex_unlock(&devlink_mutex);
5901
5902         return skb->len;
5903
5904 nla_put_failure:
5905         genlmsg_cancel(skb, hdr);
5906 out_unlock:
5907         mutex_unlock(&devlink->lock);
5908         devlink_put(devlink);
5909 out_dev:
5910         mutex_unlock(&devlink_mutex);
5911         return err;
5912 }
5913
5914 struct devlink_info_req {
5915         struct sk_buff *msg;
5916 };
5917
5918 int devlink_info_driver_name_put(struct devlink_info_req *req, const char *name)
5919 {
5920         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_DRIVER_NAME, name);
5921 }
5922 EXPORT_SYMBOL_GPL(devlink_info_driver_name_put);
5923
5924 int devlink_info_serial_number_put(struct devlink_info_req *req, const char *sn)
5925 {
5926         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_SERIAL_NUMBER, sn);
5927 }
5928 EXPORT_SYMBOL_GPL(devlink_info_serial_number_put);
5929
5930 int devlink_info_board_serial_number_put(struct devlink_info_req *req,
5931                                          const char *bsn)
5932 {
5933         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_BOARD_SERIAL_NUMBER,
5934                               bsn);
5935 }
5936 EXPORT_SYMBOL_GPL(devlink_info_board_serial_number_put);
5937
5938 static int devlink_info_version_put(struct devlink_info_req *req, int attr,
5939                                     const char *version_name,
5940                                     const char *version_value)
5941 {
5942         struct nlattr *nest;
5943         int err;
5944
5945         nest = nla_nest_start_noflag(req->msg, attr);
5946         if (!nest)
5947                 return -EMSGSIZE;
5948
5949         err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_NAME,
5950                              version_name);
5951         if (err)
5952                 goto nla_put_failure;
5953
5954         err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_VALUE,
5955                              version_value);
5956         if (err)
5957                 goto nla_put_failure;
5958
5959         nla_nest_end(req->msg, nest);
5960
5961         return 0;
5962
5963 nla_put_failure:
5964         nla_nest_cancel(req->msg, nest);
5965         return err;
5966 }
5967
5968 int devlink_info_version_fixed_put(struct devlink_info_req *req,
5969                                    const char *version_name,
5970                                    const char *version_value)
5971 {
5972         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_FIXED,
5973                                         version_name, version_value);
5974 }
5975 EXPORT_SYMBOL_GPL(devlink_info_version_fixed_put);
5976
5977 int devlink_info_version_stored_put(struct devlink_info_req *req,
5978                                     const char *version_name,
5979                                     const char *version_value)
5980 {
5981         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED,
5982                                         version_name, version_value);
5983 }
5984 EXPORT_SYMBOL_GPL(devlink_info_version_stored_put);
5985
5986 int devlink_info_version_running_put(struct devlink_info_req *req,
5987                                      const char *version_name,
5988                                      const char *version_value)
5989 {
5990         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING,
5991                                         version_name, version_value);
5992 }
5993 EXPORT_SYMBOL_GPL(devlink_info_version_running_put);
5994
5995 static int
5996 devlink_nl_info_fill(struct sk_buff *msg, struct devlink *devlink,
5997                      enum devlink_command cmd, u32 portid,
5998                      u32 seq, int flags, struct netlink_ext_ack *extack)
5999 {
6000         struct devlink_info_req req;
6001         void *hdr;
6002         int err;
6003
6004         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
6005         if (!hdr)
6006                 return -EMSGSIZE;
6007
6008         err = -EMSGSIZE;
6009         if (devlink_nl_put_handle(msg, devlink))
6010                 goto err_cancel_msg;
6011
6012         req.msg = msg;
6013         err = devlink->ops->info_get(devlink, &req, extack);
6014         if (err)
6015                 goto err_cancel_msg;
6016
6017         genlmsg_end(msg, hdr);
6018         return 0;
6019
6020 err_cancel_msg:
6021         genlmsg_cancel(msg, hdr);
6022         return err;
6023 }
6024
6025 static int devlink_nl_cmd_info_get_doit(struct sk_buff *skb,
6026                                         struct genl_info *info)
6027 {
6028         struct devlink *devlink = info->user_ptr[0];
6029         struct sk_buff *msg;
6030         int err;
6031
6032         if (!devlink->ops->info_get)
6033                 return -EOPNOTSUPP;
6034
6035         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
6036         if (!msg)
6037                 return -ENOMEM;
6038
6039         err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
6040                                    info->snd_portid, info->snd_seq, 0,
6041                                    info->extack);
6042         if (err) {
6043                 nlmsg_free(msg);
6044                 return err;
6045         }
6046
6047         return genlmsg_reply(msg, info);
6048 }
6049
6050 static int devlink_nl_cmd_info_get_dumpit(struct sk_buff *msg,
6051                                           struct netlink_callback *cb)
6052 {
6053         struct devlink *devlink;
6054         int start = cb->args[0];
6055         unsigned long index;
6056         int idx = 0;
6057         int err = 0;
6058
6059         mutex_lock(&devlink_mutex);
6060         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
6061                 if (!devlink_try_get(devlink))
6062                         continue;
6063
6064                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
6065                         goto retry;
6066
6067                 if (idx < start || !devlink->ops->info_get)
6068                         goto inc;
6069
6070                 mutex_lock(&devlink->lock);
6071                 err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
6072                                            NETLINK_CB(cb->skb).portid,
6073                                            cb->nlh->nlmsg_seq, NLM_F_MULTI,
6074                                            cb->extack);
6075                 mutex_unlock(&devlink->lock);
6076                 if (err == -EOPNOTSUPP)
6077                         err = 0;
6078                 else if (err) {
6079                         devlink_put(devlink);
6080                         break;
6081                 }
6082 inc:
6083                 idx++;
6084 retry:
6085                 devlink_put(devlink);
6086         }
6087         mutex_unlock(&devlink_mutex);
6088
6089         if (err != -EMSGSIZE)
6090                 return err;
6091
6092         cb->args[0] = idx;
6093         return msg->len;
6094 }
6095
6096 struct devlink_fmsg_item {
6097         struct list_head list;
6098         int attrtype;
6099         u8 nla_type;
6100         u16 len;
6101         int value[];
6102 };
6103
6104 struct devlink_fmsg {
6105         struct list_head item_list;
6106         bool putting_binary; /* This flag forces enclosing of binary data
6107                               * in an array brackets. It forces using
6108                               * of designated API:
6109                               * devlink_fmsg_binary_pair_nest_start()
6110                               * devlink_fmsg_binary_pair_nest_end()
6111                               */
6112 };
6113
6114 static struct devlink_fmsg *devlink_fmsg_alloc(void)
6115 {
6116         struct devlink_fmsg *fmsg;
6117
6118         fmsg = kzalloc(sizeof(*fmsg), GFP_KERNEL);
6119         if (!fmsg)
6120                 return NULL;
6121
6122         INIT_LIST_HEAD(&fmsg->item_list);
6123
6124         return fmsg;
6125 }
6126
6127 static void devlink_fmsg_free(struct devlink_fmsg *fmsg)
6128 {
6129         struct devlink_fmsg_item *item, *tmp;
6130
6131         list_for_each_entry_safe(item, tmp, &fmsg->item_list, list) {
6132                 list_del(&item->list);
6133                 kfree(item);
6134         }
6135         kfree(fmsg);
6136 }
6137
6138 static int devlink_fmsg_nest_common(struct devlink_fmsg *fmsg,
6139                                     int attrtype)
6140 {
6141         struct devlink_fmsg_item *item;
6142
6143         item = kzalloc(sizeof(*item), GFP_KERNEL);
6144         if (!item)
6145                 return -ENOMEM;
6146
6147         item->attrtype = attrtype;
6148         list_add_tail(&item->list, &fmsg->item_list);
6149
6150         return 0;
6151 }
6152
6153 int devlink_fmsg_obj_nest_start(struct devlink_fmsg *fmsg)
6154 {
6155         if (fmsg->putting_binary)
6156                 return -EINVAL;
6157
6158         return devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_OBJ_NEST_START);
6159 }
6160 EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_start);
6161
6162 static int devlink_fmsg_nest_end(struct devlink_fmsg *fmsg)
6163 {
6164         if (fmsg->putting_binary)
6165                 return -EINVAL;
6166
6167         return devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_NEST_END);
6168 }
6169
6170 int devlink_fmsg_obj_nest_end(struct devlink_fmsg *fmsg)
6171 {
6172         if (fmsg->putting_binary)
6173                 return -EINVAL;
6174
6175         return devlink_fmsg_nest_end(fmsg);
6176 }
6177 EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_end);
6178
6179 #define DEVLINK_FMSG_MAX_SIZE (GENLMSG_DEFAULT_SIZE - GENL_HDRLEN - NLA_HDRLEN)
6180
6181 static int devlink_fmsg_put_name(struct devlink_fmsg *fmsg, const char *name)
6182 {
6183         struct devlink_fmsg_item *item;
6184
6185         if (fmsg->putting_binary)
6186                 return -EINVAL;
6187
6188         if (strlen(name) + 1 > DEVLINK_FMSG_MAX_SIZE)
6189                 return -EMSGSIZE;
6190
6191         item = kzalloc(sizeof(*item) + strlen(name) + 1, GFP_KERNEL);
6192         if (!item)
6193                 return -ENOMEM;
6194
6195         item->nla_type = NLA_NUL_STRING;
6196         item->len = strlen(name) + 1;
6197         item->attrtype = DEVLINK_ATTR_FMSG_OBJ_NAME;
6198         memcpy(&item->value, name, item->len);
6199         list_add_tail(&item->list, &fmsg->item_list);
6200
6201         return 0;
6202 }
6203
6204 int devlink_fmsg_pair_nest_start(struct devlink_fmsg *fmsg, const char *name)
6205 {
6206         int err;
6207
6208         if (fmsg->putting_binary)
6209                 return -EINVAL;
6210
6211         err = devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_PAIR_NEST_START);
6212         if (err)
6213                 return err;
6214
6215         err = devlink_fmsg_put_name(fmsg, name);
6216         if (err)
6217                 return err;
6218
6219         return 0;
6220 }
6221 EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_start);
6222
6223 int devlink_fmsg_pair_nest_end(struct devlink_fmsg *fmsg)
6224 {
6225         if (fmsg->putting_binary)
6226                 return -EINVAL;
6227
6228         return devlink_fmsg_nest_end(fmsg);
6229 }
6230 EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_end);
6231
6232 int devlink_fmsg_arr_pair_nest_start(struct devlink_fmsg *fmsg,
6233                                      const char *name)
6234 {
6235         int err;
6236
6237         if (fmsg->putting_binary)
6238                 return -EINVAL;
6239
6240         err = devlink_fmsg_pair_nest_start(fmsg, name);
6241         if (err)
6242                 return err;
6243
6244         err = devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_ARR_NEST_START);
6245         if (err)
6246                 return err;
6247
6248         return 0;
6249 }
6250 EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_start);
6251
6252 int devlink_fmsg_arr_pair_nest_end(struct devlink_fmsg *fmsg)
6253 {
6254         int err;
6255
6256         if (fmsg->putting_binary)
6257                 return -EINVAL;
6258
6259         err = devlink_fmsg_nest_end(fmsg);
6260         if (err)
6261                 return err;
6262
6263         err = devlink_fmsg_nest_end(fmsg);
6264         if (err)
6265                 return err;
6266
6267         return 0;
6268 }
6269 EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_end);
6270
6271 int devlink_fmsg_binary_pair_nest_start(struct devlink_fmsg *fmsg,
6272                                         const char *name)
6273 {
6274         int err;
6275
6276         err = devlink_fmsg_arr_pair_nest_start(fmsg, name);
6277         if (err)
6278                 return err;
6279
6280         fmsg->putting_binary = true;
6281         return err;
6282 }
6283 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_start);
6284
6285 int devlink_fmsg_binary_pair_nest_end(struct devlink_fmsg *fmsg)
6286 {
6287         if (!fmsg->putting_binary)
6288                 return -EINVAL;
6289
6290         fmsg->putting_binary = false;
6291         return devlink_fmsg_arr_pair_nest_end(fmsg);
6292 }
6293 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_end);
6294
6295 static int devlink_fmsg_put_value(struct devlink_fmsg *fmsg,
6296                                   const void *value, u16 value_len,
6297                                   u8 value_nla_type)
6298 {
6299         struct devlink_fmsg_item *item;
6300
6301         if (value_len > DEVLINK_FMSG_MAX_SIZE)
6302                 return -EMSGSIZE;
6303
6304         item = kzalloc(sizeof(*item) + value_len, GFP_KERNEL);
6305         if (!item)
6306                 return -ENOMEM;
6307
6308         item->nla_type = value_nla_type;
6309         item->len = value_len;
6310         item->attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
6311         memcpy(&item->value, value, item->len);
6312         list_add_tail(&item->list, &fmsg->item_list);
6313
6314         return 0;
6315 }
6316
6317 static int devlink_fmsg_bool_put(struct devlink_fmsg *fmsg, bool value)
6318 {
6319         if (fmsg->putting_binary)
6320                 return -EINVAL;
6321
6322         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_FLAG);
6323 }
6324
6325 static int devlink_fmsg_u8_put(struct devlink_fmsg *fmsg, u8 value)
6326 {
6327         if (fmsg->putting_binary)
6328                 return -EINVAL;
6329
6330         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U8);
6331 }
6332
6333 int devlink_fmsg_u32_put(struct devlink_fmsg *fmsg, u32 value)
6334 {
6335         if (fmsg->putting_binary)
6336                 return -EINVAL;
6337
6338         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U32);
6339 }
6340 EXPORT_SYMBOL_GPL(devlink_fmsg_u32_put);
6341
6342 static int devlink_fmsg_u64_put(struct devlink_fmsg *fmsg, u64 value)
6343 {
6344         if (fmsg->putting_binary)
6345                 return -EINVAL;
6346
6347         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U64);
6348 }
6349
6350 int devlink_fmsg_string_put(struct devlink_fmsg *fmsg, const char *value)
6351 {
6352         if (fmsg->putting_binary)
6353                 return -EINVAL;
6354
6355         return devlink_fmsg_put_value(fmsg, value, strlen(value) + 1,
6356                                       NLA_NUL_STRING);
6357 }
6358 EXPORT_SYMBOL_GPL(devlink_fmsg_string_put);
6359
6360 int devlink_fmsg_binary_put(struct devlink_fmsg *fmsg, const void *value,
6361                             u16 value_len)
6362 {
6363         if (!fmsg->putting_binary)
6364                 return -EINVAL;
6365
6366         return devlink_fmsg_put_value(fmsg, value, value_len, NLA_BINARY);
6367 }
6368 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_put);
6369
6370 int devlink_fmsg_bool_pair_put(struct devlink_fmsg *fmsg, const char *name,
6371                                bool value)
6372 {
6373         int err;
6374
6375         err = devlink_fmsg_pair_nest_start(fmsg, name);
6376         if (err)
6377                 return err;
6378
6379         err = devlink_fmsg_bool_put(fmsg, value);
6380         if (err)
6381                 return err;
6382
6383         err = devlink_fmsg_pair_nest_end(fmsg);
6384         if (err)
6385                 return err;
6386
6387         return 0;
6388 }
6389 EXPORT_SYMBOL_GPL(devlink_fmsg_bool_pair_put);
6390
6391 int devlink_fmsg_u8_pair_put(struct devlink_fmsg *fmsg, const char *name,
6392                              u8 value)
6393 {
6394         int err;
6395
6396         err = devlink_fmsg_pair_nest_start(fmsg, name);
6397         if (err)
6398                 return err;
6399
6400         err = devlink_fmsg_u8_put(fmsg, value);
6401         if (err)
6402                 return err;
6403
6404         err = devlink_fmsg_pair_nest_end(fmsg);
6405         if (err)
6406                 return err;
6407
6408         return 0;
6409 }
6410 EXPORT_SYMBOL_GPL(devlink_fmsg_u8_pair_put);
6411
6412 int devlink_fmsg_u32_pair_put(struct devlink_fmsg *fmsg, const char *name,
6413                               u32 value)
6414 {
6415         int err;
6416
6417         err = devlink_fmsg_pair_nest_start(fmsg, name);
6418         if (err)
6419                 return err;
6420
6421         err = devlink_fmsg_u32_put(fmsg, value);
6422         if (err)
6423                 return err;
6424
6425         err = devlink_fmsg_pair_nest_end(fmsg);
6426         if (err)
6427                 return err;
6428
6429         return 0;
6430 }
6431 EXPORT_SYMBOL_GPL(devlink_fmsg_u32_pair_put);
6432
6433 int devlink_fmsg_u64_pair_put(struct devlink_fmsg *fmsg, const char *name,
6434                               u64 value)
6435 {
6436         int err;
6437
6438         err = devlink_fmsg_pair_nest_start(fmsg, name);
6439         if (err)
6440                 return err;
6441
6442         err = devlink_fmsg_u64_put(fmsg, value);
6443         if (err)
6444                 return err;
6445
6446         err = devlink_fmsg_pair_nest_end(fmsg);
6447         if (err)
6448                 return err;
6449
6450         return 0;
6451 }
6452 EXPORT_SYMBOL_GPL(devlink_fmsg_u64_pair_put);
6453
6454 int devlink_fmsg_string_pair_put(struct devlink_fmsg *fmsg, const char *name,
6455                                  const char *value)
6456 {
6457         int err;
6458
6459         err = devlink_fmsg_pair_nest_start(fmsg, name);
6460         if (err)
6461                 return err;
6462
6463         err = devlink_fmsg_string_put(fmsg, value);
6464         if (err)
6465                 return err;
6466
6467         err = devlink_fmsg_pair_nest_end(fmsg);
6468         if (err)
6469                 return err;
6470
6471         return 0;
6472 }
6473 EXPORT_SYMBOL_GPL(devlink_fmsg_string_pair_put);
6474
6475 int devlink_fmsg_binary_pair_put(struct devlink_fmsg *fmsg, const char *name,
6476                                  const void *value, u32 value_len)
6477 {
6478         u32 data_size;
6479         int end_err;
6480         u32 offset;
6481         int err;
6482
6483         err = devlink_fmsg_binary_pair_nest_start(fmsg, name);
6484         if (err)
6485                 return err;
6486
6487         for (offset = 0; offset < value_len; offset += data_size) {
6488                 data_size = value_len - offset;
6489                 if (data_size > DEVLINK_FMSG_MAX_SIZE)
6490                         data_size = DEVLINK_FMSG_MAX_SIZE;
6491                 err = devlink_fmsg_binary_put(fmsg, value + offset, data_size);
6492                 if (err)
6493                         break;
6494                 /* Exit from loop with a break (instead of
6495                  * return) to make sure putting_binary is turned off in
6496                  * devlink_fmsg_binary_pair_nest_end
6497                  */
6498         }
6499
6500         end_err = devlink_fmsg_binary_pair_nest_end(fmsg);
6501         if (end_err)
6502                 err = end_err;
6503
6504         return err;
6505 }
6506 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_put);
6507
6508 static int
6509 devlink_fmsg_item_fill_type(struct devlink_fmsg_item *msg, struct sk_buff *skb)
6510 {
6511         switch (msg->nla_type) {
6512         case NLA_FLAG:
6513         case NLA_U8:
6514         case NLA_U32:
6515         case NLA_U64:
6516         case NLA_NUL_STRING:
6517         case NLA_BINARY:
6518                 return nla_put_u8(skb, DEVLINK_ATTR_FMSG_OBJ_VALUE_TYPE,
6519                                   msg->nla_type);
6520         default:
6521                 return -EINVAL;
6522         }
6523 }
6524
6525 static int
6526 devlink_fmsg_item_fill_data(struct devlink_fmsg_item *msg, struct sk_buff *skb)
6527 {
6528         int attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
6529         u8 tmp;
6530
6531         switch (msg->nla_type) {
6532         case NLA_FLAG:
6533                 /* Always provide flag data, regardless of its value */
6534                 tmp = *(bool *) msg->value;
6535
6536                 return nla_put_u8(skb, attrtype, tmp);
6537         case NLA_U8:
6538                 return nla_put_u8(skb, attrtype, *(u8 *) msg->value);
6539         case NLA_U32:
6540                 return nla_put_u32(skb, attrtype, *(u32 *) msg->value);
6541         case NLA_U64:
6542                 return nla_put_u64_64bit(skb, attrtype, *(u64 *) msg->value,
6543                                          DEVLINK_ATTR_PAD);
6544         case NLA_NUL_STRING:
6545                 return nla_put_string(skb, attrtype, (char *) &msg->value);
6546         case NLA_BINARY:
6547                 return nla_put(skb, attrtype, msg->len, (void *) &msg->value);
6548         default:
6549                 return -EINVAL;
6550         }
6551 }
6552
6553 static int
6554 devlink_fmsg_prepare_skb(struct devlink_fmsg *fmsg, struct sk_buff *skb,
6555                          int *start)
6556 {
6557         struct devlink_fmsg_item *item;
6558         struct nlattr *fmsg_nlattr;
6559         int i = 0;
6560         int err;
6561
6562         fmsg_nlattr = nla_nest_start_noflag(skb, DEVLINK_ATTR_FMSG);
6563         if (!fmsg_nlattr)
6564                 return -EMSGSIZE;
6565
6566         list_for_each_entry(item, &fmsg->item_list, list) {
6567                 if (i < *start) {
6568                         i++;
6569                         continue;
6570                 }
6571
6572                 switch (item->attrtype) {
6573                 case DEVLINK_ATTR_FMSG_OBJ_NEST_START:
6574                 case DEVLINK_ATTR_FMSG_PAIR_NEST_START:
6575                 case DEVLINK_ATTR_FMSG_ARR_NEST_START:
6576                 case DEVLINK_ATTR_FMSG_NEST_END:
6577                         err = nla_put_flag(skb, item->attrtype);
6578                         break;
6579                 case DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA:
6580                         err = devlink_fmsg_item_fill_type(item, skb);
6581                         if (err)
6582                                 break;
6583                         err = devlink_fmsg_item_fill_data(item, skb);
6584                         break;
6585                 case DEVLINK_ATTR_FMSG_OBJ_NAME:
6586                         err = nla_put_string(skb, item->attrtype,
6587                                              (char *) &item->value);
6588                         break;
6589                 default:
6590                         err = -EINVAL;
6591                         break;
6592                 }
6593                 if (!err)
6594                         *start = ++i;
6595                 else
6596                         break;
6597         }
6598
6599         nla_nest_end(skb, fmsg_nlattr);
6600         return err;
6601 }
6602
6603 static int devlink_fmsg_snd(struct devlink_fmsg *fmsg,
6604                             struct genl_info *info,
6605                             enum devlink_command cmd, int flags)
6606 {
6607         struct nlmsghdr *nlh;
6608         struct sk_buff *skb;
6609         bool last = false;
6610         int index = 0;
6611         void *hdr;
6612         int err;
6613
6614         while (!last) {
6615                 int tmp_index = index;
6616
6617                 skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
6618                 if (!skb)
6619                         return -ENOMEM;
6620
6621                 hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
6622                                   &devlink_nl_family, flags | NLM_F_MULTI, cmd);
6623                 if (!hdr) {
6624                         err = -EMSGSIZE;
6625                         goto nla_put_failure;
6626                 }
6627
6628                 err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
6629                 if (!err)
6630                         last = true;
6631                 else if (err != -EMSGSIZE || tmp_index == index)
6632                         goto nla_put_failure;
6633
6634                 genlmsg_end(skb, hdr);
6635                 err = genlmsg_reply(skb, info);
6636                 if (err)
6637                         return err;
6638         }
6639
6640         skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
6641         if (!skb)
6642                 return -ENOMEM;
6643         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
6644                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
6645         if (!nlh) {
6646                 err = -EMSGSIZE;
6647                 goto nla_put_failure;
6648         }
6649
6650         return genlmsg_reply(skb, info);
6651
6652 nla_put_failure:
6653         nlmsg_free(skb);
6654         return err;
6655 }
6656
6657 static int devlink_fmsg_dumpit(struct devlink_fmsg *fmsg, struct sk_buff *skb,
6658                                struct netlink_callback *cb,
6659                                enum devlink_command cmd)
6660 {
6661         int index = cb->args[0];
6662         int tmp_index = index;
6663         void *hdr;
6664         int err;
6665
6666         hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
6667                           &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI, cmd);
6668         if (!hdr) {
6669                 err = -EMSGSIZE;
6670                 goto nla_put_failure;
6671         }
6672
6673         err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
6674         if ((err && err != -EMSGSIZE) || tmp_index == index)
6675                 goto nla_put_failure;
6676
6677         cb->args[0] = index;
6678         genlmsg_end(skb, hdr);
6679         return skb->len;
6680
6681 nla_put_failure:
6682         genlmsg_cancel(skb, hdr);
6683         return err;
6684 }
6685
6686 struct devlink_health_reporter {
6687         struct list_head list;
6688         void *priv;
6689         const struct devlink_health_reporter_ops *ops;
6690         struct devlink *devlink;
6691         struct devlink_port *devlink_port;
6692         struct devlink_fmsg *dump_fmsg;
6693         struct mutex dump_lock; /* lock parallel read/write from dump buffers */
6694         u64 graceful_period;
6695         bool auto_recover;
6696         bool auto_dump;
6697         u8 health_state;
6698         u64 dump_ts;
6699         u64 dump_real_ts;
6700         u64 error_count;
6701         u64 recovery_count;
6702         u64 last_recovery_ts;
6703         refcount_t refcount;
6704 };
6705
6706 void *
6707 devlink_health_reporter_priv(struct devlink_health_reporter *reporter)
6708 {
6709         return reporter->priv;
6710 }
6711 EXPORT_SYMBOL_GPL(devlink_health_reporter_priv);
6712
6713 static struct devlink_health_reporter *
6714 __devlink_health_reporter_find_by_name(struct list_head *reporter_list,
6715                                        struct mutex *list_lock,
6716                                        const char *reporter_name)
6717 {
6718         struct devlink_health_reporter *reporter;
6719
6720         lockdep_assert_held(list_lock);
6721         list_for_each_entry(reporter, reporter_list, list)
6722                 if (!strcmp(reporter->ops->name, reporter_name))
6723                         return reporter;
6724         return NULL;
6725 }
6726
6727 static struct devlink_health_reporter *
6728 devlink_health_reporter_find_by_name(struct devlink *devlink,
6729                                      const char *reporter_name)
6730 {
6731         return __devlink_health_reporter_find_by_name(&devlink->reporter_list,
6732                                                       &devlink->reporters_lock,
6733                                                       reporter_name);
6734 }
6735
6736 static struct devlink_health_reporter *
6737 devlink_port_health_reporter_find_by_name(struct devlink_port *devlink_port,
6738                                           const char *reporter_name)
6739 {
6740         return __devlink_health_reporter_find_by_name(&devlink_port->reporter_list,
6741                                                       &devlink_port->reporters_lock,
6742                                                       reporter_name);
6743 }
6744
6745 static struct devlink_health_reporter *
6746 __devlink_health_reporter_create(struct devlink *devlink,
6747                                  const struct devlink_health_reporter_ops *ops,
6748                                  u64 graceful_period, void *priv)
6749 {
6750         struct devlink_health_reporter *reporter;
6751
6752         if (WARN_ON(graceful_period && !ops->recover))
6753                 return ERR_PTR(-EINVAL);
6754
6755         reporter = kzalloc(sizeof(*reporter), GFP_KERNEL);
6756         if (!reporter)
6757                 return ERR_PTR(-ENOMEM);
6758
6759         reporter->priv = priv;
6760         reporter->ops = ops;
6761         reporter->devlink = devlink;
6762         reporter->graceful_period = graceful_period;
6763         reporter->auto_recover = !!ops->recover;
6764         reporter->auto_dump = !!ops->dump;
6765         mutex_init(&reporter->dump_lock);
6766         refcount_set(&reporter->refcount, 1);
6767         return reporter;
6768 }
6769
6770 /**
6771  *      devlink_port_health_reporter_create - create devlink health reporter for
6772  *                                            specified port instance
6773  *
6774  *      @port: devlink_port which should contain the new reporter
6775  *      @ops: ops
6776  *      @graceful_period: to avoid recovery loops, in msecs
6777  *      @priv: priv
6778  */
6779 struct devlink_health_reporter *
6780 devlink_port_health_reporter_create(struct devlink_port *port,
6781                                     const struct devlink_health_reporter_ops *ops,
6782                                     u64 graceful_period, void *priv)
6783 {
6784         struct devlink_health_reporter *reporter;
6785
6786         mutex_lock(&port->reporters_lock);
6787         if (__devlink_health_reporter_find_by_name(&port->reporter_list,
6788                                                    &port->reporters_lock, ops->name)) {
6789                 reporter = ERR_PTR(-EEXIST);
6790                 goto unlock;
6791         }
6792
6793         reporter = __devlink_health_reporter_create(port->devlink, ops,
6794                                                     graceful_period, priv);
6795         if (IS_ERR(reporter))
6796                 goto unlock;
6797
6798         reporter->devlink_port = port;
6799         list_add_tail(&reporter->list, &port->reporter_list);
6800 unlock:
6801         mutex_unlock(&port->reporters_lock);
6802         return reporter;
6803 }
6804 EXPORT_SYMBOL_GPL(devlink_port_health_reporter_create);
6805
6806 /**
6807  *      devlink_health_reporter_create - create devlink health reporter
6808  *
6809  *      @devlink: devlink
6810  *      @ops: ops
6811  *      @graceful_period: to avoid recovery loops, in msecs
6812  *      @priv: priv
6813  */
6814 struct devlink_health_reporter *
6815 devlink_health_reporter_create(struct devlink *devlink,
6816                                const struct devlink_health_reporter_ops *ops,
6817                                u64 graceful_period, void *priv)
6818 {
6819         struct devlink_health_reporter *reporter;
6820
6821         mutex_lock(&devlink->reporters_lock);
6822         if (devlink_health_reporter_find_by_name(devlink, ops->name)) {
6823                 reporter = ERR_PTR(-EEXIST);
6824                 goto unlock;
6825         }
6826
6827         reporter = __devlink_health_reporter_create(devlink, ops,
6828                                                     graceful_period, priv);
6829         if (IS_ERR(reporter))
6830                 goto unlock;
6831
6832         list_add_tail(&reporter->list, &devlink->reporter_list);
6833 unlock:
6834         mutex_unlock(&devlink->reporters_lock);
6835         return reporter;
6836 }
6837 EXPORT_SYMBOL_GPL(devlink_health_reporter_create);
6838
6839 static void
6840 devlink_health_reporter_free(struct devlink_health_reporter *reporter)
6841 {
6842         mutex_destroy(&reporter->dump_lock);
6843         if (reporter->dump_fmsg)
6844                 devlink_fmsg_free(reporter->dump_fmsg);
6845         kfree(reporter);
6846 }
6847
6848 static void
6849 devlink_health_reporter_put(struct devlink_health_reporter *reporter)
6850 {
6851         if (refcount_dec_and_test(&reporter->refcount))
6852                 devlink_health_reporter_free(reporter);
6853 }
6854
6855 static void
6856 __devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
6857 {
6858         list_del(&reporter->list);
6859         devlink_health_reporter_put(reporter);
6860 }
6861
6862 /**
6863  *      devlink_health_reporter_destroy - destroy devlink health reporter
6864  *
6865  *      @reporter: devlink health reporter to destroy
6866  */
6867 void
6868 devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
6869 {
6870         struct mutex *lock = &reporter->devlink->reporters_lock;
6871
6872         mutex_lock(lock);
6873         __devlink_health_reporter_destroy(reporter);
6874         mutex_unlock(lock);
6875 }
6876 EXPORT_SYMBOL_GPL(devlink_health_reporter_destroy);
6877
6878 /**
6879  *      devlink_port_health_reporter_destroy - destroy devlink port health reporter
6880  *
6881  *      @reporter: devlink health reporter to destroy
6882  */
6883 void
6884 devlink_port_health_reporter_destroy(struct devlink_health_reporter *reporter)
6885 {
6886         struct mutex *lock = &reporter->devlink_port->reporters_lock;
6887
6888         mutex_lock(lock);
6889         __devlink_health_reporter_destroy(reporter);
6890         mutex_unlock(lock);
6891 }
6892 EXPORT_SYMBOL_GPL(devlink_port_health_reporter_destroy);
6893
6894 static int
6895 devlink_nl_health_reporter_fill(struct sk_buff *msg,
6896                                 struct devlink_health_reporter *reporter,
6897                                 enum devlink_command cmd, u32 portid,
6898                                 u32 seq, int flags)
6899 {
6900         struct devlink *devlink = reporter->devlink;
6901         struct nlattr *reporter_attr;
6902         void *hdr;
6903
6904         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
6905         if (!hdr)
6906                 return -EMSGSIZE;
6907
6908         if (devlink_nl_put_handle(msg, devlink))
6909                 goto genlmsg_cancel;
6910
6911         if (reporter->devlink_port) {
6912                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, reporter->devlink_port->index))
6913                         goto genlmsg_cancel;
6914         }
6915         reporter_attr = nla_nest_start_noflag(msg,
6916                                               DEVLINK_ATTR_HEALTH_REPORTER);
6917         if (!reporter_attr)
6918                 goto genlmsg_cancel;
6919         if (nla_put_string(msg, DEVLINK_ATTR_HEALTH_REPORTER_NAME,
6920                            reporter->ops->name))
6921                 goto reporter_nest_cancel;
6922         if (nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_STATE,
6923                        reporter->health_state))
6924                 goto reporter_nest_cancel;
6925         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_ERR_COUNT,
6926                               reporter->error_count, DEVLINK_ATTR_PAD))
6927                 goto reporter_nest_cancel;
6928         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_RECOVER_COUNT,
6929                               reporter->recovery_count, DEVLINK_ATTR_PAD))
6930                 goto reporter_nest_cancel;
6931         if (reporter->ops->recover &&
6932             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD,
6933                               reporter->graceful_period,
6934                               DEVLINK_ATTR_PAD))
6935                 goto reporter_nest_cancel;
6936         if (reporter->ops->recover &&
6937             nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER,
6938                        reporter->auto_recover))
6939                 goto reporter_nest_cancel;
6940         if (reporter->dump_fmsg &&
6941             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS,
6942                               jiffies_to_msecs(reporter->dump_ts),
6943                               DEVLINK_ATTR_PAD))
6944                 goto reporter_nest_cancel;
6945         if (reporter->dump_fmsg &&
6946             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS_NS,
6947                               reporter->dump_real_ts, DEVLINK_ATTR_PAD))
6948                 goto reporter_nest_cancel;
6949         if (reporter->ops->dump &&
6950             nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP,
6951                        reporter->auto_dump))
6952                 goto reporter_nest_cancel;
6953
6954         nla_nest_end(msg, reporter_attr);
6955         genlmsg_end(msg, hdr);
6956         return 0;
6957
6958 reporter_nest_cancel:
6959         nla_nest_end(msg, reporter_attr);
6960 genlmsg_cancel:
6961         genlmsg_cancel(msg, hdr);
6962         return -EMSGSIZE;
6963 }
6964
6965 static void devlink_recover_notify(struct devlink_health_reporter *reporter,
6966                                    enum devlink_command cmd)
6967 {
6968         struct devlink *devlink = reporter->devlink;
6969         struct sk_buff *msg;
6970         int err;
6971
6972         WARN_ON(cmd != DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
6973         WARN_ON(!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED));
6974
6975         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
6976         if (!msg)
6977                 return;
6978
6979         err = devlink_nl_health_reporter_fill(msg, reporter, cmd, 0, 0, 0);
6980         if (err) {
6981                 nlmsg_free(msg);
6982                 return;
6983         }
6984
6985         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
6986                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
6987 }
6988
6989 void
6990 devlink_health_reporter_recovery_done(struct devlink_health_reporter *reporter)
6991 {
6992         reporter->recovery_count++;
6993         reporter->last_recovery_ts = jiffies;
6994 }
6995 EXPORT_SYMBOL_GPL(devlink_health_reporter_recovery_done);
6996
6997 static int
6998 devlink_health_reporter_recover(struct devlink_health_reporter *reporter,
6999                                 void *priv_ctx, struct netlink_ext_ack *extack)
7000 {
7001         int err;
7002
7003         if (reporter->health_state == DEVLINK_HEALTH_REPORTER_STATE_HEALTHY)
7004                 return 0;
7005
7006         if (!reporter->ops->recover)
7007                 return -EOPNOTSUPP;
7008
7009         err = reporter->ops->recover(reporter, priv_ctx, extack);
7010         if (err)
7011                 return err;
7012
7013         devlink_health_reporter_recovery_done(reporter);
7014         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_HEALTHY;
7015         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7016
7017         return 0;
7018 }
7019
7020 static void
7021 devlink_health_dump_clear(struct devlink_health_reporter *reporter)
7022 {
7023         if (!reporter->dump_fmsg)
7024                 return;
7025         devlink_fmsg_free(reporter->dump_fmsg);
7026         reporter->dump_fmsg = NULL;
7027 }
7028
7029 static int devlink_health_do_dump(struct devlink_health_reporter *reporter,
7030                                   void *priv_ctx,
7031                                   struct netlink_ext_ack *extack)
7032 {
7033         int err;
7034
7035         if (!reporter->ops->dump)
7036                 return 0;
7037
7038         if (reporter->dump_fmsg)
7039                 return 0;
7040
7041         reporter->dump_fmsg = devlink_fmsg_alloc();
7042         if (!reporter->dump_fmsg) {
7043                 err = -ENOMEM;
7044                 return err;
7045         }
7046
7047         err = devlink_fmsg_obj_nest_start(reporter->dump_fmsg);
7048         if (err)
7049                 goto dump_err;
7050
7051         err = reporter->ops->dump(reporter, reporter->dump_fmsg,
7052                                   priv_ctx, extack);
7053         if (err)
7054                 goto dump_err;
7055
7056         err = devlink_fmsg_obj_nest_end(reporter->dump_fmsg);
7057         if (err)
7058                 goto dump_err;
7059
7060         reporter->dump_ts = jiffies;
7061         reporter->dump_real_ts = ktime_get_real_ns();
7062
7063         return 0;
7064
7065 dump_err:
7066         devlink_health_dump_clear(reporter);
7067         return err;
7068 }
7069
7070 int devlink_health_report(struct devlink_health_reporter *reporter,
7071                           const char *msg, void *priv_ctx)
7072 {
7073         enum devlink_health_reporter_state prev_health_state;
7074         struct devlink *devlink = reporter->devlink;
7075         unsigned long recover_ts_threshold;
7076
7077         /* write a log message of the current error */
7078         WARN_ON(!msg);
7079         trace_devlink_health_report(devlink, reporter->ops->name, msg);
7080         reporter->error_count++;
7081         prev_health_state = reporter->health_state;
7082         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
7083         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7084
7085         /* abort if the previous error wasn't recovered */
7086         recover_ts_threshold = reporter->last_recovery_ts +
7087                                msecs_to_jiffies(reporter->graceful_period);
7088         if (reporter->auto_recover &&
7089             (prev_health_state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY ||
7090              (reporter->last_recovery_ts && reporter->recovery_count &&
7091               time_is_after_jiffies(recover_ts_threshold)))) {
7092                 trace_devlink_health_recover_aborted(devlink,
7093                                                      reporter->ops->name,
7094                                                      reporter->health_state,
7095                                                      jiffies -
7096                                                      reporter->last_recovery_ts);
7097                 return -ECANCELED;
7098         }
7099
7100         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
7101
7102         if (reporter->auto_dump) {
7103                 mutex_lock(&reporter->dump_lock);
7104                 /* store current dump of current error, for later analysis */
7105                 devlink_health_do_dump(reporter, priv_ctx, NULL);
7106                 mutex_unlock(&reporter->dump_lock);
7107         }
7108
7109         if (reporter->auto_recover)
7110                 return devlink_health_reporter_recover(reporter,
7111                                                        priv_ctx, NULL);
7112
7113         return 0;
7114 }
7115 EXPORT_SYMBOL_GPL(devlink_health_report);
7116
7117 static struct devlink_health_reporter *
7118 devlink_health_reporter_get_from_attrs(struct devlink *devlink,
7119                                        struct nlattr **attrs)
7120 {
7121         struct devlink_health_reporter *reporter;
7122         struct devlink_port *devlink_port;
7123         char *reporter_name;
7124
7125         if (!attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME])
7126                 return NULL;
7127
7128         reporter_name = nla_data(attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME]);
7129         devlink_port = devlink_port_get_from_attrs(devlink, attrs);
7130         if (IS_ERR(devlink_port)) {
7131                 mutex_lock(&devlink->reporters_lock);
7132                 reporter = devlink_health_reporter_find_by_name(devlink, reporter_name);
7133                 if (reporter)
7134                         refcount_inc(&reporter->refcount);
7135                 mutex_unlock(&devlink->reporters_lock);
7136         } else {
7137                 mutex_lock(&devlink_port->reporters_lock);
7138                 reporter = devlink_port_health_reporter_find_by_name(devlink_port, reporter_name);
7139                 if (reporter)
7140                         refcount_inc(&reporter->refcount);
7141                 mutex_unlock(&devlink_port->reporters_lock);
7142         }
7143
7144         return reporter;
7145 }
7146
7147 static struct devlink_health_reporter *
7148 devlink_health_reporter_get_from_info(struct devlink *devlink,
7149                                       struct genl_info *info)
7150 {
7151         return devlink_health_reporter_get_from_attrs(devlink, info->attrs);
7152 }
7153
7154 static struct devlink_health_reporter *
7155 devlink_health_reporter_get_from_cb(struct netlink_callback *cb)
7156 {
7157         const struct genl_dumpit_info *info = genl_dumpit_info(cb);
7158         struct devlink_health_reporter *reporter;
7159         struct nlattr **attrs = info->attrs;
7160         struct devlink *devlink;
7161
7162         mutex_lock(&devlink_mutex);
7163         devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
7164         if (IS_ERR(devlink))
7165                 goto unlock;
7166
7167         reporter = devlink_health_reporter_get_from_attrs(devlink, attrs);
7168         devlink_put(devlink);
7169         mutex_unlock(&devlink_mutex);
7170         return reporter;
7171 unlock:
7172         mutex_unlock(&devlink_mutex);
7173         return NULL;
7174 }
7175
7176 void
7177 devlink_health_reporter_state_update(struct devlink_health_reporter *reporter,
7178                                      enum devlink_health_reporter_state state)
7179 {
7180         if (WARN_ON(state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY &&
7181                     state != DEVLINK_HEALTH_REPORTER_STATE_ERROR))
7182                 return;
7183
7184         if (reporter->health_state == state)
7185                 return;
7186
7187         reporter->health_state = state;
7188         trace_devlink_health_reporter_state_update(reporter->devlink,
7189                                                    reporter->ops->name, state);
7190         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7191 }
7192 EXPORT_SYMBOL_GPL(devlink_health_reporter_state_update);
7193
7194 static int devlink_nl_cmd_health_reporter_get_doit(struct sk_buff *skb,
7195                                                    struct genl_info *info)
7196 {
7197         struct devlink *devlink = info->user_ptr[0];
7198         struct devlink_health_reporter *reporter;
7199         struct sk_buff *msg;
7200         int err;
7201
7202         reporter = devlink_health_reporter_get_from_info(devlink, info);
7203         if (!reporter)
7204                 return -EINVAL;
7205
7206         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
7207         if (!msg) {
7208                 err = -ENOMEM;
7209                 goto out;
7210         }
7211
7212         err = devlink_nl_health_reporter_fill(msg, reporter,
7213                                               DEVLINK_CMD_HEALTH_REPORTER_GET,
7214                                               info->snd_portid, info->snd_seq,
7215                                               0);
7216         if (err) {
7217                 nlmsg_free(msg);
7218                 goto out;
7219         }
7220
7221         err = genlmsg_reply(msg, info);
7222 out:
7223         devlink_health_reporter_put(reporter);
7224         return err;
7225 }
7226
7227 static int
7228 devlink_nl_cmd_health_reporter_get_dumpit(struct sk_buff *msg,
7229                                           struct netlink_callback *cb)
7230 {
7231         struct devlink_health_reporter *reporter;
7232         struct devlink_port *port;
7233         struct devlink *devlink;
7234         int start = cb->args[0];
7235         unsigned long index;
7236         int idx = 0;
7237         int err;
7238
7239         mutex_lock(&devlink_mutex);
7240         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
7241                 if (!devlink_try_get(devlink))
7242                         continue;
7243
7244                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
7245                         goto retry_rep;
7246
7247                 mutex_lock(&devlink->reporters_lock);
7248                 list_for_each_entry(reporter, &devlink->reporter_list,
7249                                     list) {
7250                         if (idx < start) {
7251                                 idx++;
7252                                 continue;
7253                         }
7254                         err = devlink_nl_health_reporter_fill(
7255                                 msg, reporter, DEVLINK_CMD_HEALTH_REPORTER_GET,
7256                                 NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
7257                                 NLM_F_MULTI);
7258                         if (err) {
7259                                 mutex_unlock(&devlink->reporters_lock);
7260                                 devlink_put(devlink);
7261                                 goto out;
7262                         }
7263                         idx++;
7264                 }
7265                 mutex_unlock(&devlink->reporters_lock);
7266 retry_rep:
7267                 devlink_put(devlink);
7268         }
7269
7270         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
7271                 if (!devlink_try_get(devlink))
7272                         continue;
7273
7274                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
7275                         goto retry_port;
7276
7277                 mutex_lock(&devlink->lock);
7278                 list_for_each_entry(port, &devlink->port_list, list) {
7279                         mutex_lock(&port->reporters_lock);
7280                         list_for_each_entry(reporter, &port->reporter_list, list) {
7281                                 if (idx < start) {
7282                                         idx++;
7283                                         continue;
7284                                 }
7285                                 err = devlink_nl_health_reporter_fill(
7286                                         msg, reporter,
7287                                         DEVLINK_CMD_HEALTH_REPORTER_GET,
7288                                         NETLINK_CB(cb->skb).portid,
7289                                         cb->nlh->nlmsg_seq, NLM_F_MULTI);
7290                                 if (err) {
7291                                         mutex_unlock(&port->reporters_lock);
7292                                         mutex_unlock(&devlink->lock);
7293                                         devlink_put(devlink);
7294                                         goto out;
7295                                 }
7296                                 idx++;
7297                         }
7298                         mutex_unlock(&port->reporters_lock);
7299                 }
7300                 mutex_unlock(&devlink->lock);
7301 retry_port:
7302                 devlink_put(devlink);
7303         }
7304 out:
7305         mutex_unlock(&devlink_mutex);
7306
7307         cb->args[0] = idx;
7308         return msg->len;
7309 }
7310
7311 static int
7312 devlink_nl_cmd_health_reporter_set_doit(struct sk_buff *skb,
7313                                         struct genl_info *info)
7314 {
7315         struct devlink *devlink = info->user_ptr[0];
7316         struct devlink_health_reporter *reporter;
7317         int err;
7318
7319         reporter = devlink_health_reporter_get_from_info(devlink, info);
7320         if (!reporter)
7321                 return -EINVAL;
7322
7323         if (!reporter->ops->recover &&
7324             (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] ||
7325              info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])) {
7326                 err = -EOPNOTSUPP;
7327                 goto out;
7328         }
7329         if (!reporter->ops->dump &&
7330             info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]) {
7331                 err = -EOPNOTSUPP;
7332                 goto out;
7333         }
7334
7335         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD])
7336                 reporter->graceful_period =
7337                         nla_get_u64(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD]);
7338
7339         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])
7340                 reporter->auto_recover =
7341                         nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER]);
7342
7343         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP])
7344                 reporter->auto_dump =
7345                 nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]);
7346
7347         devlink_health_reporter_put(reporter);
7348         return 0;
7349 out:
7350         devlink_health_reporter_put(reporter);
7351         return err;
7352 }
7353
7354 static int devlink_nl_cmd_health_reporter_recover_doit(struct sk_buff *skb,
7355                                                        struct genl_info *info)
7356 {
7357         struct devlink *devlink = info->user_ptr[0];
7358         struct devlink_health_reporter *reporter;
7359         int err;
7360
7361         reporter = devlink_health_reporter_get_from_info(devlink, info);
7362         if (!reporter)
7363                 return -EINVAL;
7364
7365         err = devlink_health_reporter_recover(reporter, NULL, info->extack);
7366
7367         devlink_health_reporter_put(reporter);
7368         return err;
7369 }
7370
7371 static int devlink_nl_cmd_health_reporter_diagnose_doit(struct sk_buff *skb,
7372                                                         struct genl_info *info)
7373 {
7374         struct devlink *devlink = info->user_ptr[0];
7375         struct devlink_health_reporter *reporter;
7376         struct devlink_fmsg *fmsg;
7377         int err;
7378
7379         reporter = devlink_health_reporter_get_from_info(devlink, info);
7380         if (!reporter)
7381                 return -EINVAL;
7382
7383         if (!reporter->ops->diagnose) {
7384                 devlink_health_reporter_put(reporter);
7385                 return -EOPNOTSUPP;
7386         }
7387
7388         fmsg = devlink_fmsg_alloc();
7389         if (!fmsg) {
7390                 devlink_health_reporter_put(reporter);
7391                 return -ENOMEM;
7392         }
7393
7394         err = devlink_fmsg_obj_nest_start(fmsg);
7395         if (err)
7396                 goto out;
7397
7398         err = reporter->ops->diagnose(reporter, fmsg, info->extack);
7399         if (err)
7400                 goto out;
7401
7402         err = devlink_fmsg_obj_nest_end(fmsg);
7403         if (err)
7404                 goto out;
7405
7406         err = devlink_fmsg_snd(fmsg, info,
7407                                DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE, 0);
7408
7409 out:
7410         devlink_fmsg_free(fmsg);
7411         devlink_health_reporter_put(reporter);
7412         return err;
7413 }
7414
7415 static int
7416 devlink_nl_cmd_health_reporter_dump_get_dumpit(struct sk_buff *skb,
7417                                                struct netlink_callback *cb)
7418 {
7419         struct devlink_health_reporter *reporter;
7420         u64 start = cb->args[0];
7421         int err;
7422
7423         reporter = devlink_health_reporter_get_from_cb(cb);
7424         if (!reporter)
7425                 return -EINVAL;
7426
7427         if (!reporter->ops->dump) {
7428                 err = -EOPNOTSUPP;
7429                 goto out;
7430         }
7431         mutex_lock(&reporter->dump_lock);
7432         if (!start) {
7433                 err = devlink_health_do_dump(reporter, NULL, cb->extack);
7434                 if (err)
7435                         goto unlock;
7436                 cb->args[1] = reporter->dump_ts;
7437         }
7438         if (!reporter->dump_fmsg || cb->args[1] != reporter->dump_ts) {
7439                 NL_SET_ERR_MSG_MOD(cb->extack, "Dump trampled, please retry");
7440                 err = -EAGAIN;
7441                 goto unlock;
7442         }
7443
7444         err = devlink_fmsg_dumpit(reporter->dump_fmsg, skb, cb,
7445                                   DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET);
7446 unlock:
7447         mutex_unlock(&reporter->dump_lock);
7448 out:
7449         devlink_health_reporter_put(reporter);
7450         return err;
7451 }
7452
7453 static int
7454 devlink_nl_cmd_health_reporter_dump_clear_doit(struct sk_buff *skb,
7455                                                struct genl_info *info)
7456 {
7457         struct devlink *devlink = info->user_ptr[0];
7458         struct devlink_health_reporter *reporter;
7459
7460         reporter = devlink_health_reporter_get_from_info(devlink, info);
7461         if (!reporter)
7462                 return -EINVAL;
7463
7464         if (!reporter->ops->dump) {
7465                 devlink_health_reporter_put(reporter);
7466                 return -EOPNOTSUPP;
7467         }
7468
7469         mutex_lock(&reporter->dump_lock);
7470         devlink_health_dump_clear(reporter);
7471         mutex_unlock(&reporter->dump_lock);
7472         devlink_health_reporter_put(reporter);
7473         return 0;
7474 }
7475
7476 static int devlink_nl_cmd_health_reporter_test_doit(struct sk_buff *skb,
7477                                                     struct genl_info *info)
7478 {
7479         struct devlink *devlink = info->user_ptr[0];
7480         struct devlink_health_reporter *reporter;
7481         int err;
7482
7483         reporter = devlink_health_reporter_get_from_info(devlink, info);
7484         if (!reporter)
7485                 return -EINVAL;
7486
7487         if (!reporter->ops->test) {
7488                 devlink_health_reporter_put(reporter);
7489                 return -EOPNOTSUPP;
7490         }
7491
7492         err = reporter->ops->test(reporter, info->extack);
7493
7494         devlink_health_reporter_put(reporter);
7495         return err;
7496 }
7497
7498 struct devlink_stats {
7499         u64 rx_bytes;
7500         u64 rx_packets;
7501         struct u64_stats_sync syncp;
7502 };
7503
7504 /**
7505  * struct devlink_trap_policer_item - Packet trap policer attributes.
7506  * @policer: Immutable packet trap policer attributes.
7507  * @rate: Rate in packets / sec.
7508  * @burst: Burst size in packets.
7509  * @list: trap_policer_list member.
7510  *
7511  * Describes packet trap policer attributes. Created by devlink during trap
7512  * policer registration.
7513  */
7514 struct devlink_trap_policer_item {
7515         const struct devlink_trap_policer *policer;
7516         u64 rate;
7517         u64 burst;
7518         struct list_head list;
7519 };
7520
7521 /**
7522  * struct devlink_trap_group_item - Packet trap group attributes.
7523  * @group: Immutable packet trap group attributes.
7524  * @policer_item: Associated policer item. Can be NULL.
7525  * @list: trap_group_list member.
7526  * @stats: Trap group statistics.
7527  *
7528  * Describes packet trap group attributes. Created by devlink during trap
7529  * group registration.
7530  */
7531 struct devlink_trap_group_item {
7532         const struct devlink_trap_group *group;
7533         struct devlink_trap_policer_item *policer_item;
7534         struct list_head list;
7535         struct devlink_stats __percpu *stats;
7536 };
7537
7538 /**
7539  * struct devlink_trap_item - Packet trap attributes.
7540  * @trap: Immutable packet trap attributes.
7541  * @group_item: Associated group item.
7542  * @list: trap_list member.
7543  * @action: Trap action.
7544  * @stats: Trap statistics.
7545  * @priv: Driver private information.
7546  *
7547  * Describes both mutable and immutable packet trap attributes. Created by
7548  * devlink during trap registration and used for all trap related operations.
7549  */
7550 struct devlink_trap_item {
7551         const struct devlink_trap *trap;
7552         struct devlink_trap_group_item *group_item;
7553         struct list_head list;
7554         enum devlink_trap_action action;
7555         struct devlink_stats __percpu *stats;
7556         void *priv;
7557 };
7558
7559 static struct devlink_trap_policer_item *
7560 devlink_trap_policer_item_lookup(struct devlink *devlink, u32 id)
7561 {
7562         struct devlink_trap_policer_item *policer_item;
7563
7564         list_for_each_entry(policer_item, &devlink->trap_policer_list, list) {
7565                 if (policer_item->policer->id == id)
7566                         return policer_item;
7567         }
7568
7569         return NULL;
7570 }
7571
7572 static struct devlink_trap_item *
7573 devlink_trap_item_lookup(struct devlink *devlink, const char *name)
7574 {
7575         struct devlink_trap_item *trap_item;
7576
7577         list_for_each_entry(trap_item, &devlink->trap_list, list) {
7578                 if (!strcmp(trap_item->trap->name, name))
7579                         return trap_item;
7580         }
7581
7582         return NULL;
7583 }
7584
7585 static struct devlink_trap_item *
7586 devlink_trap_item_get_from_info(struct devlink *devlink,
7587                                 struct genl_info *info)
7588 {
7589         struct nlattr *attr;
7590
7591         if (!info->attrs[DEVLINK_ATTR_TRAP_NAME])
7592                 return NULL;
7593         attr = info->attrs[DEVLINK_ATTR_TRAP_NAME];
7594
7595         return devlink_trap_item_lookup(devlink, nla_data(attr));
7596 }
7597
7598 static int
7599 devlink_trap_action_get_from_info(struct genl_info *info,
7600                                   enum devlink_trap_action *p_trap_action)
7601 {
7602         u8 val;
7603
7604         val = nla_get_u8(info->attrs[DEVLINK_ATTR_TRAP_ACTION]);
7605         switch (val) {
7606         case DEVLINK_TRAP_ACTION_DROP:
7607         case DEVLINK_TRAP_ACTION_TRAP:
7608         case DEVLINK_TRAP_ACTION_MIRROR:
7609                 *p_trap_action = val;
7610                 break;
7611         default:
7612                 return -EINVAL;
7613         }
7614
7615         return 0;
7616 }
7617
7618 static int devlink_trap_metadata_put(struct sk_buff *msg,
7619                                      const struct devlink_trap *trap)
7620 {
7621         struct nlattr *attr;
7622
7623         attr = nla_nest_start(msg, DEVLINK_ATTR_TRAP_METADATA);
7624         if (!attr)
7625                 return -EMSGSIZE;
7626
7627         if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_IN_PORT) &&
7628             nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_IN_PORT))
7629                 goto nla_put_failure;
7630         if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_FA_COOKIE) &&
7631             nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_FA_COOKIE))
7632                 goto nla_put_failure;
7633
7634         nla_nest_end(msg, attr);
7635
7636         return 0;
7637
7638 nla_put_failure:
7639         nla_nest_cancel(msg, attr);
7640         return -EMSGSIZE;
7641 }
7642
7643 static void devlink_trap_stats_read(struct devlink_stats __percpu *trap_stats,
7644                                     struct devlink_stats *stats)
7645 {
7646         int i;
7647
7648         memset(stats, 0, sizeof(*stats));
7649         for_each_possible_cpu(i) {
7650                 struct devlink_stats *cpu_stats;
7651                 u64 rx_packets, rx_bytes;
7652                 unsigned int start;
7653
7654                 cpu_stats = per_cpu_ptr(trap_stats, i);
7655                 do {
7656                         start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
7657                         rx_packets = cpu_stats->rx_packets;
7658                         rx_bytes = cpu_stats->rx_bytes;
7659                 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
7660
7661                 stats->rx_packets += rx_packets;
7662                 stats->rx_bytes += rx_bytes;
7663         }
7664 }
7665
7666 static int
7667 devlink_trap_group_stats_put(struct sk_buff *msg,
7668                              struct devlink_stats __percpu *trap_stats)
7669 {
7670         struct devlink_stats stats;
7671         struct nlattr *attr;
7672
7673         devlink_trap_stats_read(trap_stats, &stats);
7674
7675         attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
7676         if (!attr)
7677                 return -EMSGSIZE;
7678
7679         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_PACKETS,
7680                               stats.rx_packets, DEVLINK_ATTR_PAD))
7681                 goto nla_put_failure;
7682
7683         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_BYTES,
7684                               stats.rx_bytes, DEVLINK_ATTR_PAD))
7685                 goto nla_put_failure;
7686
7687         nla_nest_end(msg, attr);
7688
7689         return 0;
7690
7691 nla_put_failure:
7692         nla_nest_cancel(msg, attr);
7693         return -EMSGSIZE;
7694 }
7695
7696 static int devlink_trap_stats_put(struct sk_buff *msg, struct devlink *devlink,
7697                                   const struct devlink_trap_item *trap_item)
7698 {
7699         struct devlink_stats stats;
7700         struct nlattr *attr;
7701         u64 drops = 0;
7702         int err;
7703
7704         if (devlink->ops->trap_drop_counter_get) {
7705                 err = devlink->ops->trap_drop_counter_get(devlink,
7706                                                           trap_item->trap,
7707                                                           &drops);
7708                 if (err)
7709                         return err;
7710         }
7711
7712         devlink_trap_stats_read(trap_item->stats, &stats);
7713
7714         attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
7715         if (!attr)
7716                 return -EMSGSIZE;
7717
7718         if (devlink->ops->trap_drop_counter_get &&
7719             nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_DROPPED, drops,
7720                               DEVLINK_ATTR_PAD))
7721                 goto nla_put_failure;
7722
7723         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_PACKETS,
7724                               stats.rx_packets, DEVLINK_ATTR_PAD))
7725                 goto nla_put_failure;
7726
7727         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_BYTES,
7728                               stats.rx_bytes, DEVLINK_ATTR_PAD))
7729                 goto nla_put_failure;
7730
7731         nla_nest_end(msg, attr);
7732
7733         return 0;
7734
7735 nla_put_failure:
7736         nla_nest_cancel(msg, attr);
7737         return -EMSGSIZE;
7738 }
7739
7740 static int devlink_nl_trap_fill(struct sk_buff *msg, struct devlink *devlink,
7741                                 const struct devlink_trap_item *trap_item,
7742                                 enum devlink_command cmd, u32 portid, u32 seq,
7743                                 int flags)
7744 {
7745         struct devlink_trap_group_item *group_item = trap_item->group_item;
7746         void *hdr;
7747         int err;
7748
7749         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
7750         if (!hdr)
7751                 return -EMSGSIZE;
7752
7753         if (devlink_nl_put_handle(msg, devlink))
7754                 goto nla_put_failure;
7755
7756         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
7757                            group_item->group->name))
7758                 goto nla_put_failure;
7759
7760         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_NAME, trap_item->trap->name))
7761                 goto nla_put_failure;
7762
7763         if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_TYPE, trap_item->trap->type))
7764                 goto nla_put_failure;
7765
7766         if (trap_item->trap->generic &&
7767             nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
7768                 goto nla_put_failure;
7769
7770         if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_ACTION, trap_item->action))
7771                 goto nla_put_failure;
7772
7773         err = devlink_trap_metadata_put(msg, trap_item->trap);
7774         if (err)
7775                 goto nla_put_failure;
7776
7777         err = devlink_trap_stats_put(msg, devlink, trap_item);
7778         if (err)
7779                 goto nla_put_failure;
7780
7781         genlmsg_end(msg, hdr);
7782
7783         return 0;
7784
7785 nla_put_failure:
7786         genlmsg_cancel(msg, hdr);
7787         return -EMSGSIZE;
7788 }
7789
7790 static int devlink_nl_cmd_trap_get_doit(struct sk_buff *skb,
7791                                         struct genl_info *info)
7792 {
7793         struct netlink_ext_ack *extack = info->extack;
7794         struct devlink *devlink = info->user_ptr[0];
7795         struct devlink_trap_item *trap_item;
7796         struct sk_buff *msg;
7797         int err;
7798
7799         if (list_empty(&devlink->trap_list))
7800                 return -EOPNOTSUPP;
7801
7802         trap_item = devlink_trap_item_get_from_info(devlink, info);
7803         if (!trap_item) {
7804                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
7805                 return -ENOENT;
7806         }
7807
7808         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
7809         if (!msg)
7810                 return -ENOMEM;
7811
7812         err = devlink_nl_trap_fill(msg, devlink, trap_item,
7813                                    DEVLINK_CMD_TRAP_NEW, info->snd_portid,
7814                                    info->snd_seq, 0);
7815         if (err)
7816                 goto err_trap_fill;
7817
7818         return genlmsg_reply(msg, info);
7819
7820 err_trap_fill:
7821         nlmsg_free(msg);
7822         return err;
7823 }
7824
7825 static int devlink_nl_cmd_trap_get_dumpit(struct sk_buff *msg,
7826                                           struct netlink_callback *cb)
7827 {
7828         struct devlink_trap_item *trap_item;
7829         struct devlink *devlink;
7830         int start = cb->args[0];
7831         unsigned long index;
7832         int idx = 0;
7833         int err;
7834
7835         mutex_lock(&devlink_mutex);
7836         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
7837                 if (!devlink_try_get(devlink))
7838                         continue;
7839
7840                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
7841                         goto retry;
7842
7843                 mutex_lock(&devlink->lock);
7844                 list_for_each_entry(trap_item, &devlink->trap_list, list) {
7845                         if (idx < start) {
7846                                 idx++;
7847                                 continue;
7848                         }
7849                         err = devlink_nl_trap_fill(msg, devlink, trap_item,
7850                                                    DEVLINK_CMD_TRAP_NEW,
7851                                                    NETLINK_CB(cb->skb).portid,
7852                                                    cb->nlh->nlmsg_seq,
7853                                                    NLM_F_MULTI);
7854                         if (err) {
7855                                 mutex_unlock(&devlink->lock);
7856                                 devlink_put(devlink);
7857                                 goto out;
7858                         }
7859                         idx++;
7860                 }
7861                 mutex_unlock(&devlink->lock);
7862 retry:
7863                 devlink_put(devlink);
7864         }
7865 out:
7866         mutex_unlock(&devlink_mutex);
7867
7868         cb->args[0] = idx;
7869         return msg->len;
7870 }
7871
7872 static int __devlink_trap_action_set(struct devlink *devlink,
7873                                      struct devlink_trap_item *trap_item,
7874                                      enum devlink_trap_action trap_action,
7875                                      struct netlink_ext_ack *extack)
7876 {
7877         int err;
7878
7879         if (trap_item->action != trap_action &&
7880             trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP) {
7881                 NL_SET_ERR_MSG_MOD(extack, "Cannot change action of non-drop traps. Skipping");
7882                 return 0;
7883         }
7884
7885         err = devlink->ops->trap_action_set(devlink, trap_item->trap,
7886                                             trap_action, extack);
7887         if (err)
7888                 return err;
7889
7890         trap_item->action = trap_action;
7891
7892         return 0;
7893 }
7894
7895 static int devlink_trap_action_set(struct devlink *devlink,
7896                                    struct devlink_trap_item *trap_item,
7897                                    struct genl_info *info)
7898 {
7899         enum devlink_trap_action trap_action;
7900         int err;
7901
7902         if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
7903                 return 0;
7904
7905         err = devlink_trap_action_get_from_info(info, &trap_action);
7906         if (err) {
7907                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
7908                 return -EINVAL;
7909         }
7910
7911         return __devlink_trap_action_set(devlink, trap_item, trap_action,
7912                                          info->extack);
7913 }
7914
7915 static int devlink_nl_cmd_trap_set_doit(struct sk_buff *skb,
7916                                         struct genl_info *info)
7917 {
7918         struct netlink_ext_ack *extack = info->extack;
7919         struct devlink *devlink = info->user_ptr[0];
7920         struct devlink_trap_item *trap_item;
7921
7922         if (list_empty(&devlink->trap_list))
7923                 return -EOPNOTSUPP;
7924
7925         trap_item = devlink_trap_item_get_from_info(devlink, info);
7926         if (!trap_item) {
7927                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
7928                 return -ENOENT;
7929         }
7930
7931         return devlink_trap_action_set(devlink, trap_item, info);
7932 }
7933
7934 static struct devlink_trap_group_item *
7935 devlink_trap_group_item_lookup(struct devlink *devlink, const char *name)
7936 {
7937         struct devlink_trap_group_item *group_item;
7938
7939         list_for_each_entry(group_item, &devlink->trap_group_list, list) {
7940                 if (!strcmp(group_item->group->name, name))
7941                         return group_item;
7942         }
7943
7944         return NULL;
7945 }
7946
7947 static struct devlink_trap_group_item *
7948 devlink_trap_group_item_lookup_by_id(struct devlink *devlink, u16 id)
7949 {
7950         struct devlink_trap_group_item *group_item;
7951
7952         list_for_each_entry(group_item, &devlink->trap_group_list, list) {
7953                 if (group_item->group->id == id)
7954                         return group_item;
7955         }
7956
7957         return NULL;
7958 }
7959
7960 static struct devlink_trap_group_item *
7961 devlink_trap_group_item_get_from_info(struct devlink *devlink,
7962                                       struct genl_info *info)
7963 {
7964         char *name;
7965
7966         if (!info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME])
7967                 return NULL;
7968         name = nla_data(info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME]);
7969
7970         return devlink_trap_group_item_lookup(devlink, name);
7971 }
7972
7973 static int
7974 devlink_nl_trap_group_fill(struct sk_buff *msg, struct devlink *devlink,
7975                            const struct devlink_trap_group_item *group_item,
7976                            enum devlink_command cmd, u32 portid, u32 seq,
7977                            int flags)
7978 {
7979         void *hdr;
7980         int err;
7981
7982         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
7983         if (!hdr)
7984                 return -EMSGSIZE;
7985
7986         if (devlink_nl_put_handle(msg, devlink))
7987                 goto nla_put_failure;
7988
7989         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
7990                            group_item->group->name))
7991                 goto nla_put_failure;
7992
7993         if (group_item->group->generic &&
7994             nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
7995                 goto nla_put_failure;
7996
7997         if (group_item->policer_item &&
7998             nla_put_u32(msg, DEVLINK_ATTR_TRAP_POLICER_ID,
7999                         group_item->policer_item->policer->id))
8000                 goto nla_put_failure;
8001
8002         err = devlink_trap_group_stats_put(msg, group_item->stats);
8003         if (err)
8004                 goto nla_put_failure;
8005
8006         genlmsg_end(msg, hdr);
8007
8008         return 0;
8009
8010 nla_put_failure:
8011         genlmsg_cancel(msg, hdr);
8012         return -EMSGSIZE;
8013 }
8014
8015 static int devlink_nl_cmd_trap_group_get_doit(struct sk_buff *skb,
8016                                               struct genl_info *info)
8017 {
8018         struct netlink_ext_ack *extack = info->extack;
8019         struct devlink *devlink = info->user_ptr[0];
8020         struct devlink_trap_group_item *group_item;
8021         struct sk_buff *msg;
8022         int err;
8023
8024         if (list_empty(&devlink->trap_group_list))
8025                 return -EOPNOTSUPP;
8026
8027         group_item = devlink_trap_group_item_get_from_info(devlink, info);
8028         if (!group_item) {
8029                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
8030                 return -ENOENT;
8031         }
8032
8033         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
8034         if (!msg)
8035                 return -ENOMEM;
8036
8037         err = devlink_nl_trap_group_fill(msg, devlink, group_item,
8038                                          DEVLINK_CMD_TRAP_GROUP_NEW,
8039                                          info->snd_portid, info->snd_seq, 0);
8040         if (err)
8041                 goto err_trap_group_fill;
8042
8043         return genlmsg_reply(msg, info);
8044
8045 err_trap_group_fill:
8046         nlmsg_free(msg);
8047         return err;
8048 }
8049
8050 static int devlink_nl_cmd_trap_group_get_dumpit(struct sk_buff *msg,
8051                                                 struct netlink_callback *cb)
8052 {
8053         enum devlink_command cmd = DEVLINK_CMD_TRAP_GROUP_NEW;
8054         struct devlink_trap_group_item *group_item;
8055         u32 portid = NETLINK_CB(cb->skb).portid;
8056         struct devlink *devlink;
8057         int start = cb->args[0];
8058         unsigned long index;
8059         int idx = 0;
8060         int err;
8061
8062         mutex_lock(&devlink_mutex);
8063         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
8064                 if (!devlink_try_get(devlink))
8065                         continue;
8066
8067                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
8068                         goto retry;
8069
8070                 mutex_lock(&devlink->lock);
8071                 list_for_each_entry(group_item, &devlink->trap_group_list,
8072                                     list) {
8073                         if (idx < start) {
8074                                 idx++;
8075                                 continue;
8076                         }
8077                         err = devlink_nl_trap_group_fill(msg, devlink,
8078                                                          group_item, cmd,
8079                                                          portid,
8080                                                          cb->nlh->nlmsg_seq,
8081                                                          NLM_F_MULTI);
8082                         if (err) {
8083                                 mutex_unlock(&devlink->lock);
8084                                 devlink_put(devlink);
8085                                 goto out;
8086                         }
8087                         idx++;
8088                 }
8089                 mutex_unlock(&devlink->lock);
8090 retry:
8091                 devlink_put(devlink);
8092         }
8093 out:
8094         mutex_unlock(&devlink_mutex);
8095
8096         cb->args[0] = idx;
8097         return msg->len;
8098 }
8099
8100 static int
8101 __devlink_trap_group_action_set(struct devlink *devlink,
8102                                 struct devlink_trap_group_item *group_item,
8103                                 enum devlink_trap_action trap_action,
8104                                 struct netlink_ext_ack *extack)
8105 {
8106         const char *group_name = group_item->group->name;
8107         struct devlink_trap_item *trap_item;
8108         int err;
8109
8110         if (devlink->ops->trap_group_action_set) {
8111                 err = devlink->ops->trap_group_action_set(devlink, group_item->group,
8112                                                           trap_action, extack);
8113                 if (err)
8114                         return err;
8115
8116                 list_for_each_entry(trap_item, &devlink->trap_list, list) {
8117                         if (strcmp(trap_item->group_item->group->name, group_name))
8118                                 continue;
8119                         if (trap_item->action != trap_action &&
8120                             trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP)
8121                                 continue;
8122                         trap_item->action = trap_action;
8123                 }
8124
8125                 return 0;
8126         }
8127
8128         list_for_each_entry(trap_item, &devlink->trap_list, list) {
8129                 if (strcmp(trap_item->group_item->group->name, group_name))
8130                         continue;
8131                 err = __devlink_trap_action_set(devlink, trap_item,
8132                                                 trap_action, extack);
8133                 if (err)
8134                         return err;
8135         }
8136
8137         return 0;
8138 }
8139
8140 static int
8141 devlink_trap_group_action_set(struct devlink *devlink,
8142                               struct devlink_trap_group_item *group_item,
8143                               struct genl_info *info, bool *p_modified)
8144 {
8145         enum devlink_trap_action trap_action;
8146         int err;
8147
8148         if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
8149                 return 0;
8150
8151         err = devlink_trap_action_get_from_info(info, &trap_action);
8152         if (err) {
8153                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
8154                 return -EINVAL;
8155         }
8156
8157         err = __devlink_trap_group_action_set(devlink, group_item, trap_action,
8158                                               info->extack);
8159         if (err)
8160                 return err;
8161
8162         *p_modified = true;
8163
8164         return 0;
8165 }
8166
8167 static int devlink_trap_group_set(struct devlink *devlink,
8168                                   struct devlink_trap_group_item *group_item,
8169                                   struct genl_info *info)
8170 {
8171         struct devlink_trap_policer_item *policer_item;
8172         struct netlink_ext_ack *extack = info->extack;
8173         const struct devlink_trap_policer *policer;
8174         struct nlattr **attrs = info->attrs;
8175         int err;
8176
8177         if (!attrs[DEVLINK_ATTR_TRAP_POLICER_ID])
8178                 return 0;
8179
8180         if (!devlink->ops->trap_group_set)
8181                 return -EOPNOTSUPP;
8182
8183         policer_item = group_item->policer_item;
8184         if (attrs[DEVLINK_ATTR_TRAP_POLICER_ID]) {
8185                 u32 policer_id;
8186
8187                 policer_id = nla_get_u32(attrs[DEVLINK_ATTR_TRAP_POLICER_ID]);
8188                 policer_item = devlink_trap_policer_item_lookup(devlink,
8189                                                                 policer_id);
8190                 if (policer_id && !policer_item) {
8191                         NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
8192                         return -ENOENT;
8193                 }
8194         }
8195         policer = policer_item ? policer_item->policer : NULL;
8196
8197         err = devlink->ops->trap_group_set(devlink, group_item->group, policer,
8198                                            extack);
8199         if (err)
8200                 return err;
8201
8202         group_item->policer_item = policer_item;
8203
8204         return 0;
8205 }
8206
8207 static int devlink_nl_cmd_trap_group_set_doit(struct sk_buff *skb,
8208                                               struct genl_info *info)
8209 {
8210         struct netlink_ext_ack *extack = info->extack;
8211         struct devlink *devlink = info->user_ptr[0];
8212         struct devlink_trap_group_item *group_item;
8213         bool modified = false;
8214         int err;
8215
8216         if (list_empty(&devlink->trap_group_list))
8217                 return -EOPNOTSUPP;
8218
8219         group_item = devlink_trap_group_item_get_from_info(devlink, info);
8220         if (!group_item) {
8221                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
8222                 return -ENOENT;
8223         }
8224
8225         err = devlink_trap_group_action_set(devlink, group_item, info,
8226                                             &modified);
8227         if (err)
8228                 return err;
8229
8230         err = devlink_trap_group_set(devlink, group_item, info);
8231         if (err)
8232                 goto err_trap_group_set;
8233
8234         return 0;
8235
8236 err_trap_group_set:
8237         if (modified)
8238                 NL_SET_ERR_MSG_MOD(extack, "Trap group set failed, but some changes were committed already");
8239         return err;
8240 }
8241
8242 static struct devlink_trap_policer_item *
8243 devlink_trap_policer_item_get_from_info(struct devlink *devlink,
8244                                         struct genl_info *info)
8245 {
8246         u32 id;
8247
8248         if (!info->attrs[DEVLINK_ATTR_TRAP_POLICER_ID])
8249                 return NULL;
8250         id = nla_get_u32(info->attrs[DEVLINK_ATTR_TRAP_POLICER_ID]);
8251
8252         return devlink_trap_policer_item_lookup(devlink, id);
8253 }
8254
8255 static int
8256 devlink_trap_policer_stats_put(struct sk_buff *msg, struct devlink *devlink,
8257                                const struct devlink_trap_policer *policer)
8258 {
8259         struct nlattr *attr;
8260         u64 drops;
8261         int err;
8262
8263         if (!devlink->ops->trap_policer_counter_get)
8264                 return 0;
8265
8266         err = devlink->ops->trap_policer_counter_get(devlink, policer, &drops);
8267         if (err)
8268                 return err;
8269
8270         attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
8271         if (!attr)
8272                 return -EMSGSIZE;
8273
8274         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_DROPPED, drops,
8275                               DEVLINK_ATTR_PAD))
8276                 goto nla_put_failure;
8277
8278         nla_nest_end(msg, attr);
8279
8280         return 0;
8281
8282 nla_put_failure:
8283         nla_nest_cancel(msg, attr);
8284         return -EMSGSIZE;
8285 }
8286
8287 static int
8288 devlink_nl_trap_policer_fill(struct sk_buff *msg, struct devlink *devlink,
8289                              const struct devlink_trap_policer_item *policer_item,
8290                              enum devlink_command cmd, u32 portid, u32 seq,
8291                              int flags)
8292 {
8293         void *hdr;
8294         int err;
8295
8296         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
8297         if (!hdr)
8298                 return -EMSGSIZE;
8299
8300         if (devlink_nl_put_handle(msg, devlink))
8301                 goto nla_put_failure;
8302
8303         if (nla_put_u32(msg, DEVLINK_ATTR_TRAP_POLICER_ID,
8304                         policer_item->policer->id))
8305                 goto nla_put_failure;
8306
8307         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_TRAP_POLICER_RATE,
8308                               policer_item->rate, DEVLINK_ATTR_PAD))
8309                 goto nla_put_failure;
8310
8311         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_TRAP_POLICER_BURST,
8312                               policer_item->burst, DEVLINK_ATTR_PAD))
8313                 goto nla_put_failure;
8314
8315         err = devlink_trap_policer_stats_put(msg, devlink,
8316                                              policer_item->policer);
8317         if (err)
8318                 goto nla_put_failure;
8319
8320         genlmsg_end(msg, hdr);
8321
8322         return 0;
8323
8324 nla_put_failure:
8325         genlmsg_cancel(msg, hdr);
8326         return -EMSGSIZE;
8327 }
8328
8329 static int devlink_nl_cmd_trap_policer_get_doit(struct sk_buff *skb,
8330                                                 struct genl_info *info)
8331 {
8332         struct devlink_trap_policer_item *policer_item;
8333         struct netlink_ext_ack *extack = info->extack;
8334         struct devlink *devlink = info->user_ptr[0];
8335         struct sk_buff *msg;
8336         int err;
8337
8338         if (list_empty(&devlink->trap_policer_list))
8339                 return -EOPNOTSUPP;
8340
8341         policer_item = devlink_trap_policer_item_get_from_info(devlink, info);
8342         if (!policer_item) {
8343                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
8344                 return -ENOENT;
8345         }
8346
8347         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
8348         if (!msg)
8349                 return -ENOMEM;
8350
8351         err = devlink_nl_trap_policer_fill(msg, devlink, policer_item,
8352                                            DEVLINK_CMD_TRAP_POLICER_NEW,
8353                                            info->snd_portid, info->snd_seq, 0);
8354         if (err)
8355                 goto err_trap_policer_fill;
8356
8357         return genlmsg_reply(msg, info);
8358
8359 err_trap_policer_fill:
8360         nlmsg_free(msg);
8361         return err;
8362 }
8363
8364 static int devlink_nl_cmd_trap_policer_get_dumpit(struct sk_buff *msg,
8365                                                   struct netlink_callback *cb)
8366 {
8367         enum devlink_command cmd = DEVLINK_CMD_TRAP_POLICER_NEW;
8368         struct devlink_trap_policer_item *policer_item;
8369         u32 portid = NETLINK_CB(cb->skb).portid;
8370         struct devlink *devlink;
8371         int start = cb->args[0];
8372         unsigned long index;
8373         int idx = 0;
8374         int err;
8375
8376         mutex_lock(&devlink_mutex);
8377         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
8378                 if (!devlink_try_get(devlink))
8379                         continue;
8380
8381                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
8382                         goto retry;
8383
8384                 mutex_lock(&devlink->lock);
8385                 list_for_each_entry(policer_item, &devlink->trap_policer_list,
8386                                     list) {
8387                         if (idx < start) {
8388                                 idx++;
8389                                 continue;
8390                         }
8391                         err = devlink_nl_trap_policer_fill(msg, devlink,
8392                                                            policer_item, cmd,
8393                                                            portid,
8394                                                            cb->nlh->nlmsg_seq,
8395                                                            NLM_F_MULTI);
8396                         if (err) {
8397                                 mutex_unlock(&devlink->lock);
8398                                 devlink_put(devlink);
8399                                 goto out;
8400                         }
8401                         idx++;
8402                 }
8403                 mutex_unlock(&devlink->lock);
8404 retry:
8405                 devlink_put(devlink);
8406         }
8407 out:
8408         mutex_unlock(&devlink_mutex);
8409
8410         cb->args[0] = idx;
8411         return msg->len;
8412 }
8413
8414 static int
8415 devlink_trap_policer_set(struct devlink *devlink,
8416                          struct devlink_trap_policer_item *policer_item,
8417                          struct genl_info *info)
8418 {
8419         struct netlink_ext_ack *extack = info->extack;
8420         struct nlattr **attrs = info->attrs;
8421         u64 rate, burst;
8422         int err;
8423
8424         rate = policer_item->rate;
8425         burst = policer_item->burst;
8426
8427         if (attrs[DEVLINK_ATTR_TRAP_POLICER_RATE])
8428                 rate = nla_get_u64(attrs[DEVLINK_ATTR_TRAP_POLICER_RATE]);
8429
8430         if (attrs[DEVLINK_ATTR_TRAP_POLICER_BURST])
8431                 burst = nla_get_u64(attrs[DEVLINK_ATTR_TRAP_POLICER_BURST]);
8432
8433         if (rate < policer_item->policer->min_rate) {
8434                 NL_SET_ERR_MSG_MOD(extack, "Policer rate lower than limit");
8435                 return -EINVAL;
8436         }
8437
8438         if (rate > policer_item->policer->max_rate) {
8439                 NL_SET_ERR_MSG_MOD(extack, "Policer rate higher than limit");
8440                 return -EINVAL;
8441         }
8442
8443         if (burst < policer_item->policer->min_burst) {
8444                 NL_SET_ERR_MSG_MOD(extack, "Policer burst size lower than limit");
8445                 return -EINVAL;
8446         }
8447
8448         if (burst > policer_item->policer->max_burst) {
8449                 NL_SET_ERR_MSG_MOD(extack, "Policer burst size higher than limit");
8450                 return -EINVAL;
8451         }
8452
8453         err = devlink->ops->trap_policer_set(devlink, policer_item->policer,
8454                                              rate, burst, info->extack);
8455         if (err)
8456                 return err;
8457
8458         policer_item->rate = rate;
8459         policer_item->burst = burst;
8460
8461         return 0;
8462 }
8463
8464 static int devlink_nl_cmd_trap_policer_set_doit(struct sk_buff *skb,
8465                                                 struct genl_info *info)
8466 {
8467         struct devlink_trap_policer_item *policer_item;
8468         struct netlink_ext_ack *extack = info->extack;
8469         struct devlink *devlink = info->user_ptr[0];
8470
8471         if (list_empty(&devlink->trap_policer_list))
8472                 return -EOPNOTSUPP;
8473
8474         if (!devlink->ops->trap_policer_set)
8475                 return -EOPNOTSUPP;
8476
8477         policer_item = devlink_trap_policer_item_get_from_info(devlink, info);
8478         if (!policer_item) {
8479                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
8480                 return -ENOENT;
8481         }
8482
8483         return devlink_trap_policer_set(devlink, policer_item, info);
8484 }
8485
8486 static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = {
8487         [DEVLINK_ATTR_UNSPEC] = { .strict_start_type =
8488                 DEVLINK_ATTR_TRAP_POLICER_ID },
8489         [DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING },
8490         [DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING },
8491         [DEVLINK_ATTR_PORT_INDEX] = { .type = NLA_U32 },
8492         [DEVLINK_ATTR_PORT_TYPE] = NLA_POLICY_RANGE(NLA_U16, DEVLINK_PORT_TYPE_AUTO,
8493                                                     DEVLINK_PORT_TYPE_IB),
8494         [DEVLINK_ATTR_PORT_SPLIT_COUNT] = { .type = NLA_U32 },
8495         [DEVLINK_ATTR_SB_INDEX] = { .type = NLA_U32 },
8496         [DEVLINK_ATTR_SB_POOL_INDEX] = { .type = NLA_U16 },
8497         [DEVLINK_ATTR_SB_POOL_TYPE] = { .type = NLA_U8 },
8498         [DEVLINK_ATTR_SB_POOL_SIZE] = { .type = NLA_U32 },
8499         [DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE] = { .type = NLA_U8 },
8500         [DEVLINK_ATTR_SB_THRESHOLD] = { .type = NLA_U32 },
8501         [DEVLINK_ATTR_SB_TC_INDEX] = { .type = NLA_U16 },
8502         [DEVLINK_ATTR_ESWITCH_MODE] = NLA_POLICY_RANGE(NLA_U16, DEVLINK_ESWITCH_MODE_LEGACY,
8503                                                        DEVLINK_ESWITCH_MODE_SWITCHDEV),
8504         [DEVLINK_ATTR_ESWITCH_INLINE_MODE] = { .type = NLA_U8 },
8505         [DEVLINK_ATTR_ESWITCH_ENCAP_MODE] = { .type = NLA_U8 },
8506         [DEVLINK_ATTR_DPIPE_TABLE_NAME] = { .type = NLA_NUL_STRING },
8507         [DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED] = { .type = NLA_U8 },
8508         [DEVLINK_ATTR_RESOURCE_ID] = { .type = NLA_U64},
8509         [DEVLINK_ATTR_RESOURCE_SIZE] = { .type = NLA_U64},
8510         [DEVLINK_ATTR_PARAM_NAME] = { .type = NLA_NUL_STRING },
8511         [DEVLINK_ATTR_PARAM_TYPE] = { .type = NLA_U8 },
8512         [DEVLINK_ATTR_PARAM_VALUE_CMODE] = { .type = NLA_U8 },
8513         [DEVLINK_ATTR_REGION_NAME] = { .type = NLA_NUL_STRING },
8514         [DEVLINK_ATTR_REGION_SNAPSHOT_ID] = { .type = NLA_U32 },
8515         [DEVLINK_ATTR_REGION_CHUNK_ADDR] = { .type = NLA_U64 },
8516         [DEVLINK_ATTR_REGION_CHUNK_LEN] = { .type = NLA_U64 },
8517         [DEVLINK_ATTR_HEALTH_REPORTER_NAME] = { .type = NLA_NUL_STRING },
8518         [DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] = { .type = NLA_U64 },
8519         [DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER] = { .type = NLA_U8 },
8520         [DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME] = { .type = NLA_NUL_STRING },
8521         [DEVLINK_ATTR_FLASH_UPDATE_COMPONENT] = { .type = NLA_NUL_STRING },
8522         [DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK] =
8523                 NLA_POLICY_BITFIELD32(DEVLINK_SUPPORTED_FLASH_OVERWRITE_SECTIONS),
8524         [DEVLINK_ATTR_TRAP_NAME] = { .type = NLA_NUL_STRING },
8525         [DEVLINK_ATTR_TRAP_ACTION] = { .type = NLA_U8 },
8526         [DEVLINK_ATTR_TRAP_GROUP_NAME] = { .type = NLA_NUL_STRING },
8527         [DEVLINK_ATTR_NETNS_PID] = { .type = NLA_U32 },
8528         [DEVLINK_ATTR_NETNS_FD] = { .type = NLA_U32 },
8529         [DEVLINK_ATTR_NETNS_ID] = { .type = NLA_U32 },
8530         [DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP] = { .type = NLA_U8 },
8531         [DEVLINK_ATTR_TRAP_POLICER_ID] = { .type = NLA_U32 },
8532         [DEVLINK_ATTR_TRAP_POLICER_RATE] = { .type = NLA_U64 },
8533         [DEVLINK_ATTR_TRAP_POLICER_BURST] = { .type = NLA_U64 },
8534         [DEVLINK_ATTR_PORT_FUNCTION] = { .type = NLA_NESTED },
8535         [DEVLINK_ATTR_RELOAD_ACTION] = NLA_POLICY_RANGE(NLA_U8, DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
8536                                                         DEVLINK_RELOAD_ACTION_MAX),
8537         [DEVLINK_ATTR_RELOAD_LIMITS] = NLA_POLICY_BITFIELD32(DEVLINK_RELOAD_LIMITS_VALID_MASK),
8538         [DEVLINK_ATTR_PORT_FLAVOUR] = { .type = NLA_U16 },
8539         [DEVLINK_ATTR_PORT_PCI_PF_NUMBER] = { .type = NLA_U16 },
8540         [DEVLINK_ATTR_PORT_PCI_SF_NUMBER] = { .type = NLA_U32 },
8541         [DEVLINK_ATTR_PORT_CONTROLLER_NUMBER] = { .type = NLA_U32 },
8542         [DEVLINK_ATTR_RATE_TYPE] = { .type = NLA_U16 },
8543         [DEVLINK_ATTR_RATE_TX_SHARE] = { .type = NLA_U64 },
8544         [DEVLINK_ATTR_RATE_TX_MAX] = { .type = NLA_U64 },
8545         [DEVLINK_ATTR_RATE_NODE_NAME] = { .type = NLA_NUL_STRING },
8546         [DEVLINK_ATTR_RATE_PARENT_NODE_NAME] = { .type = NLA_NUL_STRING },
8547 };
8548
8549 static const struct genl_small_ops devlink_nl_ops[] = {
8550         {
8551                 .cmd = DEVLINK_CMD_GET,
8552                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8553                 .doit = devlink_nl_cmd_get_doit,
8554                 .dumpit = devlink_nl_cmd_get_dumpit,
8555                 /* can be retrieved by unprivileged users */
8556         },
8557         {
8558                 .cmd = DEVLINK_CMD_PORT_GET,
8559                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8560                 .doit = devlink_nl_cmd_port_get_doit,
8561                 .dumpit = devlink_nl_cmd_port_get_dumpit,
8562                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8563                 /* can be retrieved by unprivileged users */
8564         },
8565         {
8566                 .cmd = DEVLINK_CMD_PORT_SET,
8567                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8568                 .doit = devlink_nl_cmd_port_set_doit,
8569                 .flags = GENL_ADMIN_PERM,
8570                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8571         },
8572         {
8573                 .cmd = DEVLINK_CMD_RATE_GET,
8574                 .doit = devlink_nl_cmd_rate_get_doit,
8575                 .dumpit = devlink_nl_cmd_rate_get_dumpit,
8576                 .internal_flags = DEVLINK_NL_FLAG_NEED_RATE,
8577                 /* can be retrieved by unprivileged users */
8578         },
8579         {
8580                 .cmd = DEVLINK_CMD_RATE_SET,
8581                 .doit = devlink_nl_cmd_rate_set_doit,
8582                 .flags = GENL_ADMIN_PERM,
8583                 .internal_flags = DEVLINK_NL_FLAG_NEED_RATE,
8584         },
8585         {
8586                 .cmd = DEVLINK_CMD_RATE_NEW,
8587                 .doit = devlink_nl_cmd_rate_new_doit,
8588                 .flags = GENL_ADMIN_PERM,
8589         },
8590         {
8591                 .cmd = DEVLINK_CMD_RATE_DEL,
8592                 .doit = devlink_nl_cmd_rate_del_doit,
8593                 .flags = GENL_ADMIN_PERM,
8594                 .internal_flags = DEVLINK_NL_FLAG_NEED_RATE_NODE,
8595         },
8596         {
8597                 .cmd = DEVLINK_CMD_PORT_SPLIT,
8598                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8599                 .doit = devlink_nl_cmd_port_split_doit,
8600                 .flags = GENL_ADMIN_PERM,
8601                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
8602         },
8603         {
8604                 .cmd = DEVLINK_CMD_PORT_UNSPLIT,
8605                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8606                 .doit = devlink_nl_cmd_port_unsplit_doit,
8607                 .flags = GENL_ADMIN_PERM,
8608                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
8609         },
8610         {
8611                 .cmd = DEVLINK_CMD_PORT_NEW,
8612                 .doit = devlink_nl_cmd_port_new_doit,
8613                 .flags = GENL_ADMIN_PERM,
8614                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
8615         },
8616         {
8617                 .cmd = DEVLINK_CMD_PORT_DEL,
8618                 .doit = devlink_nl_cmd_port_del_doit,
8619                 .flags = GENL_ADMIN_PERM,
8620                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
8621         },
8622         {
8623                 .cmd = DEVLINK_CMD_SB_GET,
8624                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8625                 .doit = devlink_nl_cmd_sb_get_doit,
8626                 .dumpit = devlink_nl_cmd_sb_get_dumpit,
8627                 /* can be retrieved by unprivileged users */
8628         },
8629         {
8630                 .cmd = DEVLINK_CMD_SB_POOL_GET,
8631                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8632                 .doit = devlink_nl_cmd_sb_pool_get_doit,
8633                 .dumpit = devlink_nl_cmd_sb_pool_get_dumpit,
8634                 /* can be retrieved by unprivileged users */
8635         },
8636         {
8637                 .cmd = DEVLINK_CMD_SB_POOL_SET,
8638                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8639                 .doit = devlink_nl_cmd_sb_pool_set_doit,
8640                 .flags = GENL_ADMIN_PERM,
8641         },
8642         {
8643                 .cmd = DEVLINK_CMD_SB_PORT_POOL_GET,
8644                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8645                 .doit = devlink_nl_cmd_sb_port_pool_get_doit,
8646                 .dumpit = devlink_nl_cmd_sb_port_pool_get_dumpit,
8647                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8648                 /* can be retrieved by unprivileged users */
8649         },
8650         {
8651                 .cmd = DEVLINK_CMD_SB_PORT_POOL_SET,
8652                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8653                 .doit = devlink_nl_cmd_sb_port_pool_set_doit,
8654                 .flags = GENL_ADMIN_PERM,
8655                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8656         },
8657         {
8658                 .cmd = DEVLINK_CMD_SB_TC_POOL_BIND_GET,
8659                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8660                 .doit = devlink_nl_cmd_sb_tc_pool_bind_get_doit,
8661                 .dumpit = devlink_nl_cmd_sb_tc_pool_bind_get_dumpit,
8662                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8663                 /* can be retrieved by unprivileged users */
8664         },
8665         {
8666                 .cmd = DEVLINK_CMD_SB_TC_POOL_BIND_SET,
8667                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8668                 .doit = devlink_nl_cmd_sb_tc_pool_bind_set_doit,
8669                 .flags = GENL_ADMIN_PERM,
8670                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8671         },
8672         {
8673                 .cmd = DEVLINK_CMD_SB_OCC_SNAPSHOT,
8674                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8675                 .doit = devlink_nl_cmd_sb_occ_snapshot_doit,
8676                 .flags = GENL_ADMIN_PERM,
8677         },
8678         {
8679                 .cmd = DEVLINK_CMD_SB_OCC_MAX_CLEAR,
8680                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8681                 .doit = devlink_nl_cmd_sb_occ_max_clear_doit,
8682                 .flags = GENL_ADMIN_PERM,
8683         },
8684         {
8685                 .cmd = DEVLINK_CMD_ESWITCH_GET,
8686                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8687                 .doit = devlink_nl_cmd_eswitch_get_doit,
8688                 .flags = GENL_ADMIN_PERM,
8689                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
8690         },
8691         {
8692                 .cmd = DEVLINK_CMD_ESWITCH_SET,
8693                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8694                 .doit = devlink_nl_cmd_eswitch_set_doit,
8695                 .flags = GENL_ADMIN_PERM,
8696                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
8697         },
8698         {
8699                 .cmd = DEVLINK_CMD_DPIPE_TABLE_GET,
8700                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8701                 .doit = devlink_nl_cmd_dpipe_table_get,
8702                 /* can be retrieved by unprivileged users */
8703         },
8704         {
8705                 .cmd = DEVLINK_CMD_DPIPE_ENTRIES_GET,
8706                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8707                 .doit = devlink_nl_cmd_dpipe_entries_get,
8708                 /* can be retrieved by unprivileged users */
8709         },
8710         {
8711                 .cmd = DEVLINK_CMD_DPIPE_HEADERS_GET,
8712                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8713                 .doit = devlink_nl_cmd_dpipe_headers_get,
8714                 /* can be retrieved by unprivileged users */
8715         },
8716         {
8717                 .cmd = DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET,
8718                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8719                 .doit = devlink_nl_cmd_dpipe_table_counters_set,
8720                 .flags = GENL_ADMIN_PERM,
8721         },
8722         {
8723                 .cmd = DEVLINK_CMD_RESOURCE_SET,
8724                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8725                 .doit = devlink_nl_cmd_resource_set,
8726                 .flags = GENL_ADMIN_PERM,
8727         },
8728         {
8729                 .cmd = DEVLINK_CMD_RESOURCE_DUMP,
8730                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8731                 .doit = devlink_nl_cmd_resource_dump,
8732                 /* can be retrieved by unprivileged users */
8733         },
8734         {
8735                 .cmd = DEVLINK_CMD_RELOAD,
8736                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8737                 .doit = devlink_nl_cmd_reload,
8738                 .flags = GENL_ADMIN_PERM,
8739                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
8740         },
8741         {
8742                 .cmd = DEVLINK_CMD_PARAM_GET,
8743                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8744                 .doit = devlink_nl_cmd_param_get_doit,
8745                 .dumpit = devlink_nl_cmd_param_get_dumpit,
8746                 /* can be retrieved by unprivileged users */
8747         },
8748         {
8749                 .cmd = DEVLINK_CMD_PARAM_SET,
8750                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8751                 .doit = devlink_nl_cmd_param_set_doit,
8752                 .flags = GENL_ADMIN_PERM,
8753         },
8754         {
8755                 .cmd = DEVLINK_CMD_PORT_PARAM_GET,
8756                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8757                 .doit = devlink_nl_cmd_port_param_get_doit,
8758                 .dumpit = devlink_nl_cmd_port_param_get_dumpit,
8759                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8760                 /* can be retrieved by unprivileged users */
8761         },
8762         {
8763                 .cmd = DEVLINK_CMD_PORT_PARAM_SET,
8764                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8765                 .doit = devlink_nl_cmd_port_param_set_doit,
8766                 .flags = GENL_ADMIN_PERM,
8767                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8768         },
8769         {
8770                 .cmd = DEVLINK_CMD_REGION_GET,
8771                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8772                 .doit = devlink_nl_cmd_region_get_doit,
8773                 .dumpit = devlink_nl_cmd_region_get_dumpit,
8774                 .flags = GENL_ADMIN_PERM,
8775         },
8776         {
8777                 .cmd = DEVLINK_CMD_REGION_NEW,
8778                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8779                 .doit = devlink_nl_cmd_region_new,
8780                 .flags = GENL_ADMIN_PERM,
8781         },
8782         {
8783                 .cmd = DEVLINK_CMD_REGION_DEL,
8784                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8785                 .doit = devlink_nl_cmd_region_del,
8786                 .flags = GENL_ADMIN_PERM,
8787         },
8788         {
8789                 .cmd = DEVLINK_CMD_REGION_READ,
8790                 .validate = GENL_DONT_VALIDATE_STRICT |
8791                             GENL_DONT_VALIDATE_DUMP_STRICT,
8792                 .dumpit = devlink_nl_cmd_region_read_dumpit,
8793                 .flags = GENL_ADMIN_PERM,
8794         },
8795         {
8796                 .cmd = DEVLINK_CMD_INFO_GET,
8797                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8798                 .doit = devlink_nl_cmd_info_get_doit,
8799                 .dumpit = devlink_nl_cmd_info_get_dumpit,
8800                 /* can be retrieved by unprivileged users */
8801         },
8802         {
8803                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_GET,
8804                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8805                 .doit = devlink_nl_cmd_health_reporter_get_doit,
8806                 .dumpit = devlink_nl_cmd_health_reporter_get_dumpit,
8807                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
8808                                   DEVLINK_NL_FLAG_NO_LOCK,
8809                 /* can be retrieved by unprivileged users */
8810         },
8811         {
8812                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_SET,
8813                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8814                 .doit = devlink_nl_cmd_health_reporter_set_doit,
8815                 .flags = GENL_ADMIN_PERM,
8816                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
8817                                   DEVLINK_NL_FLAG_NO_LOCK,
8818         },
8819         {
8820                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_RECOVER,
8821                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8822                 .doit = devlink_nl_cmd_health_reporter_recover_doit,
8823                 .flags = GENL_ADMIN_PERM,
8824                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
8825                                   DEVLINK_NL_FLAG_NO_LOCK,
8826         },
8827         {
8828                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE,
8829                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8830                 .doit = devlink_nl_cmd_health_reporter_diagnose_doit,
8831                 .flags = GENL_ADMIN_PERM,
8832                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
8833                                   DEVLINK_NL_FLAG_NO_LOCK,
8834         },
8835         {
8836                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET,
8837                 .validate = GENL_DONT_VALIDATE_STRICT |
8838                             GENL_DONT_VALIDATE_DUMP_STRICT,
8839                 .dumpit = devlink_nl_cmd_health_reporter_dump_get_dumpit,
8840                 .flags = GENL_ADMIN_PERM,
8841                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
8842                                   DEVLINK_NL_FLAG_NO_LOCK,
8843         },
8844         {
8845                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_CLEAR,
8846                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8847                 .doit = devlink_nl_cmd_health_reporter_dump_clear_doit,
8848                 .flags = GENL_ADMIN_PERM,
8849                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
8850                                   DEVLINK_NL_FLAG_NO_LOCK,
8851         },
8852         {
8853                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_TEST,
8854                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8855                 .doit = devlink_nl_cmd_health_reporter_test_doit,
8856                 .flags = GENL_ADMIN_PERM,
8857                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
8858                                   DEVLINK_NL_FLAG_NO_LOCK,
8859         },
8860         {
8861                 .cmd = DEVLINK_CMD_FLASH_UPDATE,
8862                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8863                 .doit = devlink_nl_cmd_flash_update,
8864                 .flags = GENL_ADMIN_PERM,
8865         },
8866         {
8867                 .cmd = DEVLINK_CMD_TRAP_GET,
8868                 .doit = devlink_nl_cmd_trap_get_doit,
8869                 .dumpit = devlink_nl_cmd_trap_get_dumpit,
8870                 /* can be retrieved by unprivileged users */
8871         },
8872         {
8873                 .cmd = DEVLINK_CMD_TRAP_SET,
8874                 .doit = devlink_nl_cmd_trap_set_doit,
8875                 .flags = GENL_ADMIN_PERM,
8876         },
8877         {
8878                 .cmd = DEVLINK_CMD_TRAP_GROUP_GET,
8879                 .doit = devlink_nl_cmd_trap_group_get_doit,
8880                 .dumpit = devlink_nl_cmd_trap_group_get_dumpit,
8881                 /* can be retrieved by unprivileged users */
8882         },
8883         {
8884                 .cmd = DEVLINK_CMD_TRAP_GROUP_SET,
8885                 .doit = devlink_nl_cmd_trap_group_set_doit,
8886                 .flags = GENL_ADMIN_PERM,
8887         },
8888         {
8889                 .cmd = DEVLINK_CMD_TRAP_POLICER_GET,
8890                 .doit = devlink_nl_cmd_trap_policer_get_doit,
8891                 .dumpit = devlink_nl_cmd_trap_policer_get_dumpit,
8892                 /* can be retrieved by unprivileged users */
8893         },
8894         {
8895                 .cmd = DEVLINK_CMD_TRAP_POLICER_SET,
8896                 .doit = devlink_nl_cmd_trap_policer_set_doit,
8897                 .flags = GENL_ADMIN_PERM,
8898         },
8899 };
8900
8901 static struct genl_family devlink_nl_family __ro_after_init = {
8902         .name           = DEVLINK_GENL_NAME,
8903         .version        = DEVLINK_GENL_VERSION,
8904         .maxattr        = DEVLINK_ATTR_MAX,
8905         .policy = devlink_nl_policy,
8906         .netnsok        = true,
8907         .pre_doit       = devlink_nl_pre_doit,
8908         .post_doit      = devlink_nl_post_doit,
8909         .module         = THIS_MODULE,
8910         .small_ops      = devlink_nl_ops,
8911         .n_small_ops    = ARRAY_SIZE(devlink_nl_ops),
8912         .mcgrps         = devlink_nl_mcgrps,
8913         .n_mcgrps       = ARRAY_SIZE(devlink_nl_mcgrps),
8914 };
8915
8916 static bool devlink_reload_actions_valid(const struct devlink_ops *ops)
8917 {
8918         const struct devlink_reload_combination *comb;
8919         int i;
8920
8921         if (!devlink_reload_supported(ops)) {
8922                 if (WARN_ON(ops->reload_actions))
8923                         return false;
8924                 return true;
8925         }
8926
8927         if (WARN_ON(!ops->reload_actions ||
8928                     ops->reload_actions & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
8929                     ops->reload_actions >= BIT(__DEVLINK_RELOAD_ACTION_MAX)))
8930                 return false;
8931
8932         if (WARN_ON(ops->reload_limits & BIT(DEVLINK_RELOAD_LIMIT_UNSPEC) ||
8933                     ops->reload_limits >= BIT(__DEVLINK_RELOAD_LIMIT_MAX)))
8934                 return false;
8935
8936         for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)  {
8937                 comb = &devlink_reload_invalid_combinations[i];
8938                 if (ops->reload_actions == BIT(comb->action) &&
8939                     ops->reload_limits == BIT(comb->limit))
8940                         return false;
8941         }
8942         return true;
8943 }
8944
8945 /**
8946  *      devlink_set_features - Set devlink supported features
8947  *
8948  *      @devlink: devlink
8949  *      @features: devlink support features
8950  *
8951  *      This interface allows us to set reload ops separatelly from
8952  *      the devlink_alloc.
8953  */
8954 void devlink_set_features(struct devlink *devlink, u64 features)
8955 {
8956         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
8957
8958         WARN_ON(features & DEVLINK_F_RELOAD &&
8959                 !devlink_reload_supported(devlink->ops));
8960         devlink->features = features;
8961 }
8962 EXPORT_SYMBOL_GPL(devlink_set_features);
8963
8964 /**
8965  *      devlink_alloc_ns - Allocate new devlink instance resources
8966  *      in specific namespace
8967  *
8968  *      @ops: ops
8969  *      @priv_size: size of user private data
8970  *      @net: net namespace
8971  *      @dev: parent device
8972  *
8973  *      Allocate new devlink instance resources, including devlink index
8974  *      and name.
8975  */
8976 struct devlink *devlink_alloc_ns(const struct devlink_ops *ops,
8977                                  size_t priv_size, struct net *net,
8978                                  struct device *dev)
8979 {
8980         struct devlink *devlink;
8981         static u32 last_id;
8982         int ret;
8983
8984         WARN_ON(!ops || !dev);
8985         if (!devlink_reload_actions_valid(ops))
8986                 return NULL;
8987
8988         devlink = kzalloc(sizeof(*devlink) + priv_size, GFP_KERNEL);
8989         if (!devlink)
8990                 return NULL;
8991
8992         ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b,
8993                               &last_id, GFP_KERNEL);
8994         if (ret < 0) {
8995                 kfree(devlink);
8996                 return NULL;
8997         }
8998
8999         devlink->dev = dev;
9000         devlink->ops = ops;
9001         xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC);
9002         write_pnet(&devlink->_net, net);
9003         INIT_LIST_HEAD(&devlink->port_list);
9004         INIT_LIST_HEAD(&devlink->rate_list);
9005         INIT_LIST_HEAD(&devlink->sb_list);
9006         INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
9007         INIT_LIST_HEAD(&devlink->resource_list);
9008         INIT_LIST_HEAD(&devlink->param_list);
9009         INIT_LIST_HEAD(&devlink->region_list);
9010         INIT_LIST_HEAD(&devlink->reporter_list);
9011         INIT_LIST_HEAD(&devlink->trap_list);
9012         INIT_LIST_HEAD(&devlink->trap_group_list);
9013         INIT_LIST_HEAD(&devlink->trap_policer_list);
9014         mutex_init(&devlink->lock);
9015         mutex_init(&devlink->reporters_lock);
9016         refcount_set(&devlink->refcount, 1);
9017         init_completion(&devlink->comp);
9018
9019         return devlink;
9020 }
9021 EXPORT_SYMBOL_GPL(devlink_alloc_ns);
9022
9023 static void
9024 devlink_trap_policer_notify(struct devlink *devlink,
9025                             const struct devlink_trap_policer_item *policer_item,
9026                             enum devlink_command cmd);
9027 static void
9028 devlink_trap_group_notify(struct devlink *devlink,
9029                           const struct devlink_trap_group_item *group_item,
9030                           enum devlink_command cmd);
9031 static void devlink_trap_notify(struct devlink *devlink,
9032                                 const struct devlink_trap_item *trap_item,
9033                                 enum devlink_command cmd);
9034
9035 static void devlink_notify_register(struct devlink *devlink)
9036 {
9037         struct devlink_trap_policer_item *policer_item;
9038         struct devlink_trap_group_item *group_item;
9039         struct devlink_param_item *param_item;
9040         struct devlink_trap_item *trap_item;
9041         struct devlink_port *devlink_port;
9042         struct devlink_rate *rate_node;
9043         struct devlink_region *region;
9044
9045         devlink_notify(devlink, DEVLINK_CMD_NEW);
9046         list_for_each_entry(devlink_port, &devlink->port_list, list)
9047                 devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
9048
9049         list_for_each_entry(policer_item, &devlink->trap_policer_list, list)
9050                 devlink_trap_policer_notify(devlink, policer_item,
9051                                             DEVLINK_CMD_TRAP_POLICER_NEW);
9052
9053         list_for_each_entry(group_item, &devlink->trap_group_list, list)
9054                 devlink_trap_group_notify(devlink, group_item,
9055                                           DEVLINK_CMD_TRAP_GROUP_NEW);
9056
9057         list_for_each_entry(trap_item, &devlink->trap_list, list)
9058                 devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_NEW);
9059
9060         list_for_each_entry(rate_node, &devlink->rate_list, list)
9061                 devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
9062
9063         list_for_each_entry(region, &devlink->region_list, list)
9064                 devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
9065
9066         list_for_each_entry(param_item, &devlink->param_list, list)
9067                 devlink_param_notify(devlink, 0, param_item,
9068                                      DEVLINK_CMD_PARAM_NEW);
9069 }
9070
9071 static void devlink_notify_unregister(struct devlink *devlink)
9072 {
9073         struct devlink_trap_policer_item *policer_item;
9074         struct devlink_trap_group_item *group_item;
9075         struct devlink_param_item *param_item;
9076         struct devlink_trap_item *trap_item;
9077         struct devlink_port *devlink_port;
9078         struct devlink_rate *rate_node;
9079         struct devlink_region *region;
9080
9081         list_for_each_entry_reverse(param_item, &devlink->param_list, list)
9082                 devlink_param_notify(devlink, 0, param_item,
9083                                      DEVLINK_CMD_PARAM_DEL);
9084
9085         list_for_each_entry_reverse(region, &devlink->region_list, list)
9086                 devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_DEL);
9087
9088         list_for_each_entry_reverse(rate_node, &devlink->rate_list, list)
9089                 devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
9090
9091         list_for_each_entry_reverse(trap_item, &devlink->trap_list, list)
9092                 devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_DEL);
9093
9094         list_for_each_entry_reverse(group_item, &devlink->trap_group_list, list)
9095                 devlink_trap_group_notify(devlink, group_item,
9096                                           DEVLINK_CMD_TRAP_GROUP_DEL);
9097         list_for_each_entry_reverse(policer_item, &devlink->trap_policer_list,
9098                                     list)
9099                 devlink_trap_policer_notify(devlink, policer_item,
9100                                             DEVLINK_CMD_TRAP_POLICER_DEL);
9101
9102         list_for_each_entry_reverse(devlink_port, &devlink->port_list, list)
9103                 devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
9104         devlink_notify(devlink, DEVLINK_CMD_DEL);
9105 }
9106
9107 /**
9108  *      devlink_register - Register devlink instance
9109  *
9110  *      @devlink: devlink
9111  */
9112 void devlink_register(struct devlink *devlink)
9113 {
9114         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
9115         /* Make sure that we are in .probe() routine */
9116
9117         mutex_lock(&devlink_mutex);
9118         xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
9119         devlink_notify_register(devlink);
9120         mutex_unlock(&devlink_mutex);
9121 }
9122 EXPORT_SYMBOL_GPL(devlink_register);
9123
9124 /**
9125  *      devlink_unregister - Unregister devlink instance
9126  *
9127  *      @devlink: devlink
9128  */
9129 void devlink_unregister(struct devlink *devlink)
9130 {
9131         ASSERT_DEVLINK_REGISTERED(devlink);
9132         /* Make sure that we are in .remove() routine */
9133
9134         devlink_put(devlink);
9135         wait_for_completion(&devlink->comp);
9136
9137         mutex_lock(&devlink_mutex);
9138         devlink_notify_unregister(devlink);
9139         xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
9140         mutex_unlock(&devlink_mutex);
9141 }
9142 EXPORT_SYMBOL_GPL(devlink_unregister);
9143
9144 /**
9145  *      devlink_free - Free devlink instance resources
9146  *
9147  *      @devlink: devlink
9148  */
9149 void devlink_free(struct devlink *devlink)
9150 {
9151         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
9152
9153         mutex_destroy(&devlink->reporters_lock);
9154         mutex_destroy(&devlink->lock);
9155         WARN_ON(!list_empty(&devlink->trap_policer_list));
9156         WARN_ON(!list_empty(&devlink->trap_group_list));
9157         WARN_ON(!list_empty(&devlink->trap_list));
9158         WARN_ON(!list_empty(&devlink->reporter_list));
9159         WARN_ON(!list_empty(&devlink->region_list));
9160         WARN_ON(!list_empty(&devlink->param_list));
9161         WARN_ON(!list_empty(&devlink->resource_list));
9162         WARN_ON(!list_empty(&devlink->dpipe_table_list));
9163         WARN_ON(!list_empty(&devlink->sb_list));
9164         WARN_ON(!list_empty(&devlink->rate_list));
9165         WARN_ON(!list_empty(&devlink->port_list));
9166
9167         xa_destroy(&devlink->snapshot_ids);
9168         xa_erase(&devlinks, devlink->index);
9169
9170         kfree(devlink);
9171 }
9172 EXPORT_SYMBOL_GPL(devlink_free);
9173
9174 static void devlink_port_type_warn(struct work_struct *work)
9175 {
9176         WARN(true, "Type was not set for devlink port.");
9177 }
9178
9179 static bool devlink_port_type_should_warn(struct devlink_port *devlink_port)
9180 {
9181         /* Ignore CPU and DSA flavours. */
9182         return devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_CPU &&
9183                devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_DSA &&
9184                devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_UNUSED;
9185 }
9186
9187 #define DEVLINK_PORT_TYPE_WARN_TIMEOUT (HZ * 3600)
9188
9189 static void devlink_port_type_warn_schedule(struct devlink_port *devlink_port)
9190 {
9191         if (!devlink_port_type_should_warn(devlink_port))
9192                 return;
9193         /* Schedule a work to WARN in case driver does not set port
9194          * type within timeout.
9195          */
9196         schedule_delayed_work(&devlink_port->type_warn_dw,
9197                               DEVLINK_PORT_TYPE_WARN_TIMEOUT);
9198 }
9199
9200 static void devlink_port_type_warn_cancel(struct devlink_port *devlink_port)
9201 {
9202         if (!devlink_port_type_should_warn(devlink_port))
9203                 return;
9204         cancel_delayed_work_sync(&devlink_port->type_warn_dw);
9205 }
9206
9207 /**
9208  *      devlink_port_register - Register devlink port
9209  *
9210  *      @devlink: devlink
9211  *      @devlink_port: devlink port
9212  *      @port_index: driver-specific numerical identifier of the port
9213  *
9214  *      Register devlink port with provided port index. User can use
9215  *      any indexing, even hw-related one. devlink_port structure
9216  *      is convenient to be embedded inside user driver private structure.
9217  *      Note that the caller should take care of zeroing the devlink_port
9218  *      structure.
9219  */
9220 int devlink_port_register(struct devlink *devlink,
9221                           struct devlink_port *devlink_port,
9222                           unsigned int port_index)
9223 {
9224         mutex_lock(&devlink->lock);
9225         if (devlink_port_index_exists(devlink, port_index)) {
9226                 mutex_unlock(&devlink->lock);
9227                 return -EEXIST;
9228         }
9229
9230         WARN_ON(devlink_port->devlink);
9231         devlink_port->devlink = devlink;
9232         devlink_port->index = port_index;
9233         spin_lock_init(&devlink_port->type_lock);
9234         INIT_LIST_HEAD(&devlink_port->reporter_list);
9235         mutex_init(&devlink_port->reporters_lock);
9236         list_add_tail(&devlink_port->list, &devlink->port_list);
9237         INIT_LIST_HEAD(&devlink_port->param_list);
9238         INIT_LIST_HEAD(&devlink_port->region_list);
9239         mutex_unlock(&devlink->lock);
9240         INIT_DELAYED_WORK(&devlink_port->type_warn_dw, &devlink_port_type_warn);
9241         devlink_port_type_warn_schedule(devlink_port);
9242         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
9243         return 0;
9244 }
9245 EXPORT_SYMBOL_GPL(devlink_port_register);
9246
9247 /**
9248  *      devlink_port_unregister - Unregister devlink port
9249  *
9250  *      @devlink_port: devlink port
9251  */
9252 void devlink_port_unregister(struct devlink_port *devlink_port)
9253 {
9254         struct devlink *devlink = devlink_port->devlink;
9255
9256         devlink_port_type_warn_cancel(devlink_port);
9257         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
9258         mutex_lock(&devlink->lock);
9259         list_del(&devlink_port->list);
9260         mutex_unlock(&devlink->lock);
9261         WARN_ON(!list_empty(&devlink_port->reporter_list));
9262         WARN_ON(!list_empty(&devlink_port->region_list));
9263         mutex_destroy(&devlink_port->reporters_lock);
9264 }
9265 EXPORT_SYMBOL_GPL(devlink_port_unregister);
9266
9267 static void __devlink_port_type_set(struct devlink_port *devlink_port,
9268                                     enum devlink_port_type type,
9269                                     void *type_dev)
9270 {
9271         if (WARN_ON(!devlink_port->devlink))
9272                 return;
9273         devlink_port_type_warn_cancel(devlink_port);
9274         spin_lock_bh(&devlink_port->type_lock);
9275         devlink_port->type = type;
9276         devlink_port->type_dev = type_dev;
9277         spin_unlock_bh(&devlink_port->type_lock);
9278         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
9279 }
9280
9281 static void devlink_port_type_netdev_checks(struct devlink_port *devlink_port,
9282                                             struct net_device *netdev)
9283 {
9284         const struct net_device_ops *ops = netdev->netdev_ops;
9285
9286         /* If driver registers devlink port, it should set devlink port
9287          * attributes accordingly so the compat functions are called
9288          * and the original ops are not used.
9289          */
9290         if (ops->ndo_get_phys_port_name) {
9291                 /* Some drivers use the same set of ndos for netdevs
9292                  * that have devlink_port registered and also for
9293                  * those who don't. Make sure that ndo_get_phys_port_name
9294                  * returns -EOPNOTSUPP here in case it is defined.
9295                  * Warn if not.
9296                  */
9297                 char name[IFNAMSIZ];
9298                 int err;
9299
9300                 err = ops->ndo_get_phys_port_name(netdev, name, sizeof(name));
9301                 WARN_ON(err != -EOPNOTSUPP);
9302         }
9303         if (ops->ndo_get_port_parent_id) {
9304                 /* Some drivers use the same set of ndos for netdevs
9305                  * that have devlink_port registered and also for
9306                  * those who don't. Make sure that ndo_get_port_parent_id
9307                  * returns -EOPNOTSUPP here in case it is defined.
9308                  * Warn if not.
9309                  */
9310                 struct netdev_phys_item_id ppid;
9311                 int err;
9312
9313                 err = ops->ndo_get_port_parent_id(netdev, &ppid);
9314                 WARN_ON(err != -EOPNOTSUPP);
9315         }
9316 }
9317
9318 /**
9319  *      devlink_port_type_eth_set - Set port type to Ethernet
9320  *
9321  *      @devlink_port: devlink port
9322  *      @netdev: related netdevice
9323  */
9324 void devlink_port_type_eth_set(struct devlink_port *devlink_port,
9325                                struct net_device *netdev)
9326 {
9327         if (netdev)
9328                 devlink_port_type_netdev_checks(devlink_port, netdev);
9329         else
9330                 dev_warn(devlink_port->devlink->dev,
9331                          "devlink port type for port %d set to Ethernet without a software interface reference, device type not supported by the kernel?\n",
9332                          devlink_port->index);
9333
9334         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_ETH, netdev);
9335 }
9336 EXPORT_SYMBOL_GPL(devlink_port_type_eth_set);
9337
9338 /**
9339  *      devlink_port_type_ib_set - Set port type to InfiniBand
9340  *
9341  *      @devlink_port: devlink port
9342  *      @ibdev: related IB device
9343  */
9344 void devlink_port_type_ib_set(struct devlink_port *devlink_port,
9345                               struct ib_device *ibdev)
9346 {
9347         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_IB, ibdev);
9348 }
9349 EXPORT_SYMBOL_GPL(devlink_port_type_ib_set);
9350
9351 /**
9352  *      devlink_port_type_clear - Clear port type
9353  *
9354  *      @devlink_port: devlink port
9355  */
9356 void devlink_port_type_clear(struct devlink_port *devlink_port)
9357 {
9358         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_NOTSET, NULL);
9359         devlink_port_type_warn_schedule(devlink_port);
9360 }
9361 EXPORT_SYMBOL_GPL(devlink_port_type_clear);
9362
9363 static int __devlink_port_attrs_set(struct devlink_port *devlink_port,
9364                                     enum devlink_port_flavour flavour)
9365 {
9366         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9367
9368         devlink_port->attrs_set = true;
9369         attrs->flavour = flavour;
9370         if (attrs->switch_id.id_len) {
9371                 devlink_port->switch_port = true;
9372                 if (WARN_ON(attrs->switch_id.id_len > MAX_PHYS_ITEM_ID_LEN))
9373                         attrs->switch_id.id_len = MAX_PHYS_ITEM_ID_LEN;
9374         } else {
9375                 devlink_port->switch_port = false;
9376         }
9377         return 0;
9378 }
9379
9380 /**
9381  *      devlink_port_attrs_set - Set port attributes
9382  *
9383  *      @devlink_port: devlink port
9384  *      @attrs: devlink port attrs
9385  */
9386 void devlink_port_attrs_set(struct devlink_port *devlink_port,
9387                             struct devlink_port_attrs *attrs)
9388 {
9389         int ret;
9390
9391         if (WARN_ON(devlink_port->devlink))
9392                 return;
9393         devlink_port->attrs = *attrs;
9394         ret = __devlink_port_attrs_set(devlink_port, attrs->flavour);
9395         if (ret)
9396                 return;
9397         WARN_ON(attrs->splittable && attrs->split);
9398 }
9399 EXPORT_SYMBOL_GPL(devlink_port_attrs_set);
9400
9401 /**
9402  *      devlink_port_attrs_pci_pf_set - Set PCI PF port attributes
9403  *
9404  *      @devlink_port: devlink port
9405  *      @controller: associated controller number for the devlink port instance
9406  *      @pf: associated PF for the devlink port instance
9407  *      @external: indicates if the port is for an external controller
9408  */
9409 void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 controller,
9410                                    u16 pf, bool external)
9411 {
9412         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9413         int ret;
9414
9415         if (WARN_ON(devlink_port->devlink))
9416                 return;
9417         ret = __devlink_port_attrs_set(devlink_port,
9418                                        DEVLINK_PORT_FLAVOUR_PCI_PF);
9419         if (ret)
9420                 return;
9421         attrs->pci_pf.controller = controller;
9422         attrs->pci_pf.pf = pf;
9423         attrs->pci_pf.external = external;
9424 }
9425 EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_pf_set);
9426
9427 /**
9428  *      devlink_port_attrs_pci_vf_set - Set PCI VF port attributes
9429  *
9430  *      @devlink_port: devlink port
9431  *      @controller: associated controller number for the devlink port instance
9432  *      @pf: associated PF for the devlink port instance
9433  *      @vf: associated VF of a PF for the devlink port instance
9434  *      @external: indicates if the port is for an external controller
9435  */
9436 void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller,
9437                                    u16 pf, u16 vf, bool external)
9438 {
9439         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9440         int ret;
9441
9442         if (WARN_ON(devlink_port->devlink))
9443                 return;
9444         ret = __devlink_port_attrs_set(devlink_port,
9445                                        DEVLINK_PORT_FLAVOUR_PCI_VF);
9446         if (ret)
9447                 return;
9448         attrs->pci_vf.controller = controller;
9449         attrs->pci_vf.pf = pf;
9450         attrs->pci_vf.vf = vf;
9451         attrs->pci_vf.external = external;
9452 }
9453 EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_vf_set);
9454
9455 /**
9456  *      devlink_port_attrs_pci_sf_set - Set PCI SF port attributes
9457  *
9458  *      @devlink_port: devlink port
9459  *      @controller: associated controller number for the devlink port instance
9460  *      @pf: associated PF for the devlink port instance
9461  *      @sf: associated SF of a PF for the devlink port instance
9462  *      @external: indicates if the port is for an external controller
9463  */
9464 void devlink_port_attrs_pci_sf_set(struct devlink_port *devlink_port, u32 controller,
9465                                    u16 pf, u32 sf, bool external)
9466 {
9467         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9468         int ret;
9469
9470         if (WARN_ON(devlink_port->devlink))
9471                 return;
9472         ret = __devlink_port_attrs_set(devlink_port,
9473                                        DEVLINK_PORT_FLAVOUR_PCI_SF);
9474         if (ret)
9475                 return;
9476         attrs->pci_sf.controller = controller;
9477         attrs->pci_sf.pf = pf;
9478         attrs->pci_sf.sf = sf;
9479         attrs->pci_sf.external = external;
9480 }
9481 EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_sf_set);
9482
9483 /**
9484  * devlink_rate_leaf_create - create devlink rate leaf
9485  *
9486  * @devlink_port: devlink port object to create rate object on
9487  * @priv: driver private data
9488  *
9489  * Create devlink rate object of type leaf on provided @devlink_port.
9490  * Throws call trace if @devlink_port already has a devlink rate object.
9491  *
9492  * Context: Takes and release devlink->lock <mutex>.
9493  *
9494  * Return: -ENOMEM if failed to allocate rate object, 0 otherwise.
9495  */
9496 int
9497 devlink_rate_leaf_create(struct devlink_port *devlink_port, void *priv)
9498 {
9499         struct devlink *devlink = devlink_port->devlink;
9500         struct devlink_rate *devlink_rate;
9501
9502         devlink_rate = kzalloc(sizeof(*devlink_rate), GFP_KERNEL);
9503         if (!devlink_rate)
9504                 return -ENOMEM;
9505
9506         mutex_lock(&devlink->lock);
9507         WARN_ON(devlink_port->devlink_rate);
9508         devlink_rate->type = DEVLINK_RATE_TYPE_LEAF;
9509         devlink_rate->devlink = devlink;
9510         devlink_rate->devlink_port = devlink_port;
9511         devlink_rate->priv = priv;
9512         list_add_tail(&devlink_rate->list, &devlink->rate_list);
9513         devlink_port->devlink_rate = devlink_rate;
9514         devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW);
9515         mutex_unlock(&devlink->lock);
9516
9517         return 0;
9518 }
9519 EXPORT_SYMBOL_GPL(devlink_rate_leaf_create);
9520
9521 /**
9522  * devlink_rate_leaf_destroy - destroy devlink rate leaf
9523  *
9524  * @devlink_port: devlink port linked to the rate object
9525  *
9526  * Context: Takes and release devlink->lock <mutex>.
9527  */
9528 void devlink_rate_leaf_destroy(struct devlink_port *devlink_port)
9529 {
9530         struct devlink_rate *devlink_rate = devlink_port->devlink_rate;
9531         struct devlink *devlink = devlink_port->devlink;
9532
9533         if (!devlink_rate)
9534                 return;
9535
9536         mutex_lock(&devlink->lock);
9537         devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_DEL);
9538         if (devlink_rate->parent)
9539                 refcount_dec(&devlink_rate->parent->refcnt);
9540         list_del(&devlink_rate->list);
9541         devlink_port->devlink_rate = NULL;
9542         mutex_unlock(&devlink->lock);
9543         kfree(devlink_rate);
9544 }
9545 EXPORT_SYMBOL_GPL(devlink_rate_leaf_destroy);
9546
9547 /**
9548  * devlink_rate_nodes_destroy - destroy all devlink rate nodes on device
9549  *
9550  * @devlink: devlink instance
9551  *
9552  * Unset parent for all rate objects and destroy all rate nodes
9553  * on specified device.
9554  *
9555  * Context: Takes and release devlink->lock <mutex>.
9556  */
9557 void devlink_rate_nodes_destroy(struct devlink *devlink)
9558 {
9559         static struct devlink_rate *devlink_rate, *tmp;
9560         const struct devlink_ops *ops = devlink->ops;
9561
9562         mutex_lock(&devlink->lock);
9563         list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
9564                 if (!devlink_rate->parent)
9565                         continue;
9566
9567                 refcount_dec(&devlink_rate->parent->refcnt);
9568                 if (devlink_rate_is_leaf(devlink_rate))
9569                         ops->rate_leaf_parent_set(devlink_rate, NULL, devlink_rate->priv,
9570                                                   NULL, NULL);
9571                 else if (devlink_rate_is_node(devlink_rate))
9572                         ops->rate_node_parent_set(devlink_rate, NULL, devlink_rate->priv,
9573                                                   NULL, NULL);
9574         }
9575         list_for_each_entry_safe(devlink_rate, tmp, &devlink->rate_list, list) {
9576                 if (devlink_rate_is_node(devlink_rate)) {
9577                         ops->rate_node_del(devlink_rate, devlink_rate->priv, NULL);
9578                         list_del(&devlink_rate->list);
9579                         kfree(devlink_rate->name);
9580                         kfree(devlink_rate);
9581                 }
9582         }
9583         mutex_unlock(&devlink->lock);
9584 }
9585 EXPORT_SYMBOL_GPL(devlink_rate_nodes_destroy);
9586
9587 static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port,
9588                                              char *name, size_t len)
9589 {
9590         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9591         int n = 0;
9592
9593         if (!devlink_port->attrs_set)
9594                 return -EOPNOTSUPP;
9595
9596         switch (attrs->flavour) {
9597         case DEVLINK_PORT_FLAVOUR_PHYSICAL:
9598                 n = snprintf(name, len, "p%u", attrs->phys.port_number);
9599                 if (n < len && attrs->split)
9600                         n += snprintf(name + n, len - n, "s%u",
9601                                       attrs->phys.split_subport_number);
9602                 break;
9603         case DEVLINK_PORT_FLAVOUR_CPU:
9604         case DEVLINK_PORT_FLAVOUR_DSA:
9605         case DEVLINK_PORT_FLAVOUR_UNUSED:
9606                 /* As CPU and DSA ports do not have a netdevice associated
9607                  * case should not ever happen.
9608                  */
9609                 WARN_ON(1);
9610                 return -EINVAL;
9611         case DEVLINK_PORT_FLAVOUR_PCI_PF:
9612                 if (attrs->pci_pf.external) {
9613                         n = snprintf(name, len, "c%u", attrs->pci_pf.controller);
9614                         if (n >= len)
9615                                 return -EINVAL;
9616                         len -= n;
9617                         name += n;
9618                 }
9619                 n = snprintf(name, len, "pf%u", attrs->pci_pf.pf);
9620                 break;
9621         case DEVLINK_PORT_FLAVOUR_PCI_VF:
9622                 if (attrs->pci_vf.external) {
9623                         n = snprintf(name, len, "c%u", attrs->pci_vf.controller);
9624                         if (n >= len)
9625                                 return -EINVAL;
9626                         len -= n;
9627                         name += n;
9628                 }
9629                 n = snprintf(name, len, "pf%uvf%u",
9630                              attrs->pci_vf.pf, attrs->pci_vf.vf);
9631                 break;
9632         case DEVLINK_PORT_FLAVOUR_PCI_SF:
9633                 if (attrs->pci_sf.external) {
9634                         n = snprintf(name, len, "c%u", attrs->pci_sf.controller);
9635                         if (n >= len)
9636                                 return -EINVAL;
9637                         len -= n;
9638                         name += n;
9639                 }
9640                 n = snprintf(name, len, "pf%usf%u", attrs->pci_sf.pf,
9641                              attrs->pci_sf.sf);
9642                 break;
9643         case DEVLINK_PORT_FLAVOUR_VIRTUAL:
9644                 return -EOPNOTSUPP;
9645         }
9646
9647         if (n >= len)
9648                 return -EINVAL;
9649
9650         return 0;
9651 }
9652
9653 int devlink_sb_register(struct devlink *devlink, unsigned int sb_index,
9654                         u32 size, u16 ingress_pools_count,
9655                         u16 egress_pools_count, u16 ingress_tc_count,
9656                         u16 egress_tc_count)
9657 {
9658         struct devlink_sb *devlink_sb;
9659         int err = 0;
9660
9661         mutex_lock(&devlink->lock);
9662         if (devlink_sb_index_exists(devlink, sb_index)) {
9663                 err = -EEXIST;
9664                 goto unlock;
9665         }
9666
9667         devlink_sb = kzalloc(sizeof(*devlink_sb), GFP_KERNEL);
9668         if (!devlink_sb) {
9669                 err = -ENOMEM;
9670                 goto unlock;
9671         }
9672         devlink_sb->index = sb_index;
9673         devlink_sb->size = size;
9674         devlink_sb->ingress_pools_count = ingress_pools_count;
9675         devlink_sb->egress_pools_count = egress_pools_count;
9676         devlink_sb->ingress_tc_count = ingress_tc_count;
9677         devlink_sb->egress_tc_count = egress_tc_count;
9678         list_add_tail(&devlink_sb->list, &devlink->sb_list);
9679 unlock:
9680         mutex_unlock(&devlink->lock);
9681         return err;
9682 }
9683 EXPORT_SYMBOL_GPL(devlink_sb_register);
9684
9685 void devlink_sb_unregister(struct devlink *devlink, unsigned int sb_index)
9686 {
9687         struct devlink_sb *devlink_sb;
9688
9689         mutex_lock(&devlink->lock);
9690         devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
9691         WARN_ON(!devlink_sb);
9692         list_del(&devlink_sb->list);
9693         mutex_unlock(&devlink->lock);
9694         kfree(devlink_sb);
9695 }
9696 EXPORT_SYMBOL_GPL(devlink_sb_unregister);
9697
9698 /**
9699  *      devlink_dpipe_headers_register - register dpipe headers
9700  *
9701  *      @devlink: devlink
9702  *      @dpipe_headers: dpipe header array
9703  *
9704  *      Register the headers supported by hardware.
9705  */
9706 int devlink_dpipe_headers_register(struct devlink *devlink,
9707                                    struct devlink_dpipe_headers *dpipe_headers)
9708 {
9709         mutex_lock(&devlink->lock);
9710         devlink->dpipe_headers = dpipe_headers;
9711         mutex_unlock(&devlink->lock);
9712         return 0;
9713 }
9714 EXPORT_SYMBOL_GPL(devlink_dpipe_headers_register);
9715
9716 /**
9717  *      devlink_dpipe_headers_unregister - unregister dpipe headers
9718  *
9719  *      @devlink: devlink
9720  *
9721  *      Unregister the headers supported by hardware.
9722  */
9723 void devlink_dpipe_headers_unregister(struct devlink *devlink)
9724 {
9725         mutex_lock(&devlink->lock);
9726         devlink->dpipe_headers = NULL;
9727         mutex_unlock(&devlink->lock);
9728 }
9729 EXPORT_SYMBOL_GPL(devlink_dpipe_headers_unregister);
9730
9731 /**
9732  *      devlink_dpipe_table_counter_enabled - check if counter allocation
9733  *                                            required
9734  *      @devlink: devlink
9735  *      @table_name: tables name
9736  *
9737  *      Used by driver to check if counter allocation is required.
9738  *      After counter allocation is turned on the table entries
9739  *      are updated to include counter statistics.
9740  *
9741  *      After that point on the driver must respect the counter
9742  *      state so that each entry added to the table is added
9743  *      with a counter.
9744  */
9745 bool devlink_dpipe_table_counter_enabled(struct devlink *devlink,
9746                                          const char *table_name)
9747 {
9748         struct devlink_dpipe_table *table;
9749         bool enabled;
9750
9751         rcu_read_lock();
9752         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
9753                                          table_name, devlink);
9754         enabled = false;
9755         if (table)
9756                 enabled = table->counters_enabled;
9757         rcu_read_unlock();
9758         return enabled;
9759 }
9760 EXPORT_SYMBOL_GPL(devlink_dpipe_table_counter_enabled);
9761
9762 /**
9763  *      devlink_dpipe_table_register - register dpipe table
9764  *
9765  *      @devlink: devlink
9766  *      @table_name: table name
9767  *      @table_ops: table ops
9768  *      @priv: priv
9769  *      @counter_control_extern: external control for counters
9770  */
9771 int devlink_dpipe_table_register(struct devlink *devlink,
9772                                  const char *table_name,
9773                                  struct devlink_dpipe_table_ops *table_ops,
9774                                  void *priv, bool counter_control_extern)
9775 {
9776         struct devlink_dpipe_table *table;
9777         int err = 0;
9778
9779         if (WARN_ON(!table_ops->size_get))
9780                 return -EINVAL;
9781
9782         mutex_lock(&devlink->lock);
9783
9784         if (devlink_dpipe_table_find(&devlink->dpipe_table_list, table_name,
9785                                      devlink)) {
9786                 err = -EEXIST;
9787                 goto unlock;
9788         }
9789
9790         table = kzalloc(sizeof(*table), GFP_KERNEL);
9791         if (!table) {
9792                 err = -ENOMEM;
9793                 goto unlock;
9794         }
9795
9796         table->name = table_name;
9797         table->table_ops = table_ops;
9798         table->priv = priv;
9799         table->counter_control_extern = counter_control_extern;
9800
9801         list_add_tail_rcu(&table->list, &devlink->dpipe_table_list);
9802 unlock:
9803         mutex_unlock(&devlink->lock);
9804         return err;
9805 }
9806 EXPORT_SYMBOL_GPL(devlink_dpipe_table_register);
9807
9808 /**
9809  *      devlink_dpipe_table_unregister - unregister dpipe table
9810  *
9811  *      @devlink: devlink
9812  *      @table_name: table name
9813  */
9814 void devlink_dpipe_table_unregister(struct devlink *devlink,
9815                                     const char *table_name)
9816 {
9817         struct devlink_dpipe_table *table;
9818
9819         mutex_lock(&devlink->lock);
9820         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
9821                                          table_name, devlink);
9822         if (!table)
9823                 goto unlock;
9824         list_del_rcu(&table->list);
9825         mutex_unlock(&devlink->lock);
9826         kfree_rcu(table, rcu);
9827         return;
9828 unlock:
9829         mutex_unlock(&devlink->lock);
9830 }
9831 EXPORT_SYMBOL_GPL(devlink_dpipe_table_unregister);
9832
9833 /**
9834  *      devlink_resource_register - devlink resource register
9835  *
9836  *      @devlink: devlink
9837  *      @resource_name: resource's name
9838  *      @resource_size: resource's size
9839  *      @resource_id: resource's id
9840  *      @parent_resource_id: resource's parent id
9841  *      @size_params: size parameters
9842  *
9843  *      Generic resources should reuse the same names across drivers.
9844  *      Please see the generic resources list at:
9845  *      Documentation/networking/devlink/devlink-resource.rst
9846  */
9847 int devlink_resource_register(struct devlink *devlink,
9848                               const char *resource_name,
9849                               u64 resource_size,
9850                               u64 resource_id,
9851                               u64 parent_resource_id,
9852                               const struct devlink_resource_size_params *size_params)
9853 {
9854         struct devlink_resource *resource;
9855         struct list_head *resource_list;
9856         bool top_hierarchy;
9857         int err = 0;
9858
9859         top_hierarchy = parent_resource_id == DEVLINK_RESOURCE_ID_PARENT_TOP;
9860
9861         mutex_lock(&devlink->lock);
9862         resource = devlink_resource_find(devlink, NULL, resource_id);
9863         if (resource) {
9864                 err = -EINVAL;
9865                 goto out;
9866         }
9867
9868         resource = kzalloc(sizeof(*resource), GFP_KERNEL);
9869         if (!resource) {
9870                 err = -ENOMEM;
9871                 goto out;
9872         }
9873
9874         if (top_hierarchy) {
9875                 resource_list = &devlink->resource_list;
9876         } else {
9877                 struct devlink_resource *parent_resource;
9878
9879                 parent_resource = devlink_resource_find(devlink, NULL,
9880                                                         parent_resource_id);
9881                 if (parent_resource) {
9882                         resource_list = &parent_resource->resource_list;
9883                         resource->parent = parent_resource;
9884                 } else {
9885                         kfree(resource);
9886                         err = -EINVAL;
9887                         goto out;
9888                 }
9889         }
9890
9891         resource->name = resource_name;
9892         resource->size = resource_size;
9893         resource->size_new = resource_size;
9894         resource->id = resource_id;
9895         resource->size_valid = true;
9896         memcpy(&resource->size_params, size_params,
9897                sizeof(resource->size_params));
9898         INIT_LIST_HEAD(&resource->resource_list);
9899         list_add_tail(&resource->list, resource_list);
9900 out:
9901         mutex_unlock(&devlink->lock);
9902         return err;
9903 }
9904 EXPORT_SYMBOL_GPL(devlink_resource_register);
9905
9906 /**
9907  *      devlink_resources_unregister - free all resources
9908  *
9909  *      @devlink: devlink
9910  *      @resource: resource
9911  */
9912 void devlink_resources_unregister(struct devlink *devlink,
9913                                   struct devlink_resource *resource)
9914 {
9915         struct devlink_resource *tmp, *child_resource;
9916         struct list_head *resource_list;
9917
9918         if (resource)
9919                 resource_list = &resource->resource_list;
9920         else
9921                 resource_list = &devlink->resource_list;
9922
9923         if (!resource)
9924                 mutex_lock(&devlink->lock);
9925
9926         list_for_each_entry_safe(child_resource, tmp, resource_list, list) {
9927                 devlink_resources_unregister(devlink, child_resource);
9928                 list_del(&child_resource->list);
9929                 kfree(child_resource);
9930         }
9931
9932         if (!resource)
9933                 mutex_unlock(&devlink->lock);
9934 }
9935 EXPORT_SYMBOL_GPL(devlink_resources_unregister);
9936
9937 /**
9938  *      devlink_resource_size_get - get and update size
9939  *
9940  *      @devlink: devlink
9941  *      @resource_id: the requested resource id
9942  *      @p_resource_size: ptr to update
9943  */
9944 int devlink_resource_size_get(struct devlink *devlink,
9945                               u64 resource_id,
9946                               u64 *p_resource_size)
9947 {
9948         struct devlink_resource *resource;
9949         int err = 0;
9950
9951         mutex_lock(&devlink->lock);
9952         resource = devlink_resource_find(devlink, NULL, resource_id);
9953         if (!resource) {
9954                 err = -EINVAL;
9955                 goto out;
9956         }
9957         *p_resource_size = resource->size_new;
9958         resource->size = resource->size_new;
9959 out:
9960         mutex_unlock(&devlink->lock);
9961         return err;
9962 }
9963 EXPORT_SYMBOL_GPL(devlink_resource_size_get);
9964
9965 /**
9966  *      devlink_dpipe_table_resource_set - set the resource id
9967  *
9968  *      @devlink: devlink
9969  *      @table_name: table name
9970  *      @resource_id: resource id
9971  *      @resource_units: number of resource's units consumed per table's entry
9972  */
9973 int devlink_dpipe_table_resource_set(struct devlink *devlink,
9974                                      const char *table_name, u64 resource_id,
9975                                      u64 resource_units)
9976 {
9977         struct devlink_dpipe_table *table;
9978         int err = 0;
9979
9980         mutex_lock(&devlink->lock);
9981         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
9982                                          table_name, devlink);
9983         if (!table) {
9984                 err = -EINVAL;
9985                 goto out;
9986         }
9987         table->resource_id = resource_id;
9988         table->resource_units = resource_units;
9989         table->resource_valid = true;
9990 out:
9991         mutex_unlock(&devlink->lock);
9992         return err;
9993 }
9994 EXPORT_SYMBOL_GPL(devlink_dpipe_table_resource_set);
9995
9996 /**
9997  *      devlink_resource_occ_get_register - register occupancy getter
9998  *
9999  *      @devlink: devlink
10000  *      @resource_id: resource id
10001  *      @occ_get: occupancy getter callback
10002  *      @occ_get_priv: occupancy getter callback priv
10003  */
10004 void devlink_resource_occ_get_register(struct devlink *devlink,
10005                                        u64 resource_id,
10006                                        devlink_resource_occ_get_t *occ_get,
10007                                        void *occ_get_priv)
10008 {
10009         struct devlink_resource *resource;
10010
10011         mutex_lock(&devlink->lock);
10012         resource = devlink_resource_find(devlink, NULL, resource_id);
10013         if (WARN_ON(!resource))
10014                 goto out;
10015         WARN_ON(resource->occ_get);
10016
10017         resource->occ_get = occ_get;
10018         resource->occ_get_priv = occ_get_priv;
10019 out:
10020         mutex_unlock(&devlink->lock);
10021 }
10022 EXPORT_SYMBOL_GPL(devlink_resource_occ_get_register);
10023
10024 /**
10025  *      devlink_resource_occ_get_unregister - unregister occupancy getter
10026  *
10027  *      @devlink: devlink
10028  *      @resource_id: resource id
10029  */
10030 void devlink_resource_occ_get_unregister(struct devlink *devlink,
10031                                          u64 resource_id)
10032 {
10033         struct devlink_resource *resource;
10034
10035         mutex_lock(&devlink->lock);
10036         resource = devlink_resource_find(devlink, NULL, resource_id);
10037         if (WARN_ON(!resource))
10038                 goto out;
10039         WARN_ON(!resource->occ_get);
10040
10041         resource->occ_get = NULL;
10042         resource->occ_get_priv = NULL;
10043 out:
10044         mutex_unlock(&devlink->lock);
10045 }
10046 EXPORT_SYMBOL_GPL(devlink_resource_occ_get_unregister);
10047
10048 static int devlink_param_verify(const struct devlink_param *param)
10049 {
10050         if (!param || !param->name || !param->supported_cmodes)
10051                 return -EINVAL;
10052         if (param->generic)
10053                 return devlink_param_generic_verify(param);
10054         else
10055                 return devlink_param_driver_verify(param);
10056 }
10057
10058 /**
10059  *      devlink_params_register - register configuration parameters
10060  *
10061  *      @devlink: devlink
10062  *      @params: configuration parameters array
10063  *      @params_count: number of parameters provided
10064  *
10065  *      Register the configuration parameters supported by the driver.
10066  */
10067 int devlink_params_register(struct devlink *devlink,
10068                             const struct devlink_param *params,
10069                             size_t params_count)
10070 {
10071         const struct devlink_param *param = params;
10072         int i, err;
10073
10074         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10075
10076         for (i = 0; i < params_count; i++, param++) {
10077                 err = devlink_param_register(devlink, param);
10078                 if (err)
10079                         goto rollback;
10080         }
10081         return 0;
10082
10083 rollback:
10084         if (!i)
10085                 return err;
10086
10087         for (param--; i > 0; i--, param--)
10088                 devlink_param_unregister(devlink, param);
10089         return err;
10090 }
10091 EXPORT_SYMBOL_GPL(devlink_params_register);
10092
10093 /**
10094  *      devlink_params_unregister - unregister configuration parameters
10095  *      @devlink: devlink
10096  *      @params: configuration parameters to unregister
10097  *      @params_count: number of parameters provided
10098  */
10099 void devlink_params_unregister(struct devlink *devlink,
10100                                const struct devlink_param *params,
10101                                size_t params_count)
10102 {
10103         const struct devlink_param *param = params;
10104         int i;
10105
10106         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10107
10108         for (i = 0; i < params_count; i++, param++)
10109                 devlink_param_unregister(devlink, param);
10110 }
10111 EXPORT_SYMBOL_GPL(devlink_params_unregister);
10112
10113 /**
10114  * devlink_param_register - register one configuration parameter
10115  *
10116  * @devlink: devlink
10117  * @param: one configuration parameter
10118  *
10119  * Register the configuration parameter supported by the driver.
10120  * Return: returns 0 on successful registration or error code otherwise.
10121  */
10122 int devlink_param_register(struct devlink *devlink,
10123                            const struct devlink_param *param)
10124 {
10125         struct devlink_param_item *param_item;
10126
10127         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10128
10129         WARN_ON(devlink_param_verify(param));
10130         WARN_ON(devlink_param_find_by_name(&devlink->param_list, param->name));
10131
10132         if (param->supported_cmodes == BIT(DEVLINK_PARAM_CMODE_DRIVERINIT))
10133                 WARN_ON(param->get || param->set);
10134         else
10135                 WARN_ON(!param->get || !param->set);
10136
10137         param_item = kzalloc(sizeof(*param_item), GFP_KERNEL);
10138         if (!param_item)
10139                 return -ENOMEM;
10140
10141         param_item->param = param;
10142
10143         list_add_tail(&param_item->list, &devlink->param_list);
10144         return 0;
10145 }
10146 EXPORT_SYMBOL_GPL(devlink_param_register);
10147
10148 /**
10149  * devlink_param_unregister - unregister one configuration parameter
10150  * @devlink: devlink
10151  * @param: configuration parameter to unregister
10152  */
10153 void devlink_param_unregister(struct devlink *devlink,
10154                               const struct devlink_param *param)
10155 {
10156         struct devlink_param_item *param_item;
10157
10158         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10159
10160         param_item =
10161                 devlink_param_find_by_name(&devlink->param_list, param->name);
10162         WARN_ON(!param_item);
10163         list_del(&param_item->list);
10164         kfree(param_item);
10165 }
10166 EXPORT_SYMBOL_GPL(devlink_param_unregister);
10167
10168 /**
10169  *      devlink_param_driverinit_value_get - get configuration parameter
10170  *                                           value for driver initializing
10171  *
10172  *      @devlink: devlink
10173  *      @param_id: parameter ID
10174  *      @init_val: value of parameter in driverinit configuration mode
10175  *
10176  *      This function should be used by the driver to get driverinit
10177  *      configuration for initialization after reload command.
10178  */
10179 int devlink_param_driverinit_value_get(struct devlink *devlink, u32 param_id,
10180                                        union devlink_param_value *init_val)
10181 {
10182         struct devlink_param_item *param_item;
10183
10184         if (!devlink_reload_supported(devlink->ops))
10185                 return -EOPNOTSUPP;
10186
10187         param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
10188         if (!param_item)
10189                 return -EINVAL;
10190
10191         if (!param_item->driverinit_value_valid ||
10192             !devlink_param_cmode_is_supported(param_item->param,
10193                                               DEVLINK_PARAM_CMODE_DRIVERINIT))
10194                 return -EOPNOTSUPP;
10195
10196         if (param_item->param->type == DEVLINK_PARAM_TYPE_STRING)
10197                 strcpy(init_val->vstr, param_item->driverinit_value.vstr);
10198         else
10199                 *init_val = param_item->driverinit_value;
10200
10201         return 0;
10202 }
10203 EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_get);
10204
10205 /**
10206  *      devlink_param_driverinit_value_set - set value of configuration
10207  *                                           parameter for driverinit
10208  *                                           configuration mode
10209  *
10210  *      @devlink: devlink
10211  *      @param_id: parameter ID
10212  *      @init_val: value of parameter to set for driverinit configuration mode
10213  *
10214  *      This function should be used by the driver to set driverinit
10215  *      configuration mode default value.
10216  */
10217 int devlink_param_driverinit_value_set(struct devlink *devlink, u32 param_id,
10218                                        union devlink_param_value init_val)
10219 {
10220         struct devlink_param_item *param_item;
10221
10222         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10223
10224         param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
10225         if (!param_item)
10226                 return -EINVAL;
10227
10228         if (!devlink_param_cmode_is_supported(param_item->param,
10229                                               DEVLINK_PARAM_CMODE_DRIVERINIT))
10230                 return -EOPNOTSUPP;
10231
10232         if (param_item->param->type == DEVLINK_PARAM_TYPE_STRING)
10233                 strcpy(param_item->driverinit_value.vstr, init_val.vstr);
10234         else
10235                 param_item->driverinit_value = init_val;
10236         param_item->driverinit_value_valid = true;
10237         return 0;
10238 }
10239 EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_set);
10240
10241 /**
10242  *      devlink_param_value_changed - notify devlink on a parameter's value
10243  *                                    change. Should be called by the driver
10244  *                                    right after the change.
10245  *
10246  *      @devlink: devlink
10247  *      @param_id: parameter ID
10248  *
10249  *      This function should be used by the driver to notify devlink on value
10250  *      change, excluding driverinit configuration mode.
10251  *      For driverinit configuration mode driver should use the function
10252  */
10253 void devlink_param_value_changed(struct devlink *devlink, u32 param_id)
10254 {
10255         struct devlink_param_item *param_item;
10256
10257         param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
10258         WARN_ON(!param_item);
10259
10260         devlink_param_notify(devlink, 0, param_item, DEVLINK_CMD_PARAM_NEW);
10261 }
10262 EXPORT_SYMBOL_GPL(devlink_param_value_changed);
10263
10264 /**
10265  *      devlink_region_create - create a new address region
10266  *
10267  *      @devlink: devlink
10268  *      @ops: region operations and name
10269  *      @region_max_snapshots: Maximum supported number of snapshots for region
10270  *      @region_size: size of region
10271  */
10272 struct devlink_region *
10273 devlink_region_create(struct devlink *devlink,
10274                       const struct devlink_region_ops *ops,
10275                       u32 region_max_snapshots, u64 region_size)
10276 {
10277         struct devlink_region *region;
10278         int err = 0;
10279
10280         if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
10281                 return ERR_PTR(-EINVAL);
10282
10283         mutex_lock(&devlink->lock);
10284
10285         if (devlink_region_get_by_name(devlink, ops->name)) {
10286                 err = -EEXIST;
10287                 goto unlock;
10288         }
10289
10290         region = kzalloc(sizeof(*region), GFP_KERNEL);
10291         if (!region) {
10292                 err = -ENOMEM;
10293                 goto unlock;
10294         }
10295
10296         region->devlink = devlink;
10297         region->max_snapshots = region_max_snapshots;
10298         region->ops = ops;
10299         region->size = region_size;
10300         INIT_LIST_HEAD(&region->snapshot_list);
10301         list_add_tail(&region->list, &devlink->region_list);
10302         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
10303
10304         mutex_unlock(&devlink->lock);
10305         return region;
10306
10307 unlock:
10308         mutex_unlock(&devlink->lock);
10309         return ERR_PTR(err);
10310 }
10311 EXPORT_SYMBOL_GPL(devlink_region_create);
10312
10313 /**
10314  *      devlink_port_region_create - create a new address region for a port
10315  *
10316  *      @port: devlink port
10317  *      @ops: region operations and name
10318  *      @region_max_snapshots: Maximum supported number of snapshots for region
10319  *      @region_size: size of region
10320  */
10321 struct devlink_region *
10322 devlink_port_region_create(struct devlink_port *port,
10323                            const struct devlink_port_region_ops *ops,
10324                            u32 region_max_snapshots, u64 region_size)
10325 {
10326         struct devlink *devlink = port->devlink;
10327         struct devlink_region *region;
10328         int err = 0;
10329
10330         if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
10331                 return ERR_PTR(-EINVAL);
10332
10333         mutex_lock(&devlink->lock);
10334
10335         if (devlink_port_region_get_by_name(port, ops->name)) {
10336                 err = -EEXIST;
10337                 goto unlock;
10338         }
10339
10340         region = kzalloc(sizeof(*region), GFP_KERNEL);
10341         if (!region) {
10342                 err = -ENOMEM;
10343                 goto unlock;
10344         }
10345
10346         region->devlink = devlink;
10347         region->port = port;
10348         region->max_snapshots = region_max_snapshots;
10349         region->port_ops = ops;
10350         region->size = region_size;
10351         INIT_LIST_HEAD(&region->snapshot_list);
10352         list_add_tail(&region->list, &port->region_list);
10353         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
10354
10355         mutex_unlock(&devlink->lock);
10356         return region;
10357
10358 unlock:
10359         mutex_unlock(&devlink->lock);
10360         return ERR_PTR(err);
10361 }
10362 EXPORT_SYMBOL_GPL(devlink_port_region_create);
10363
10364 /**
10365  *      devlink_region_destroy - destroy address region
10366  *
10367  *      @region: devlink region to destroy
10368  */
10369 void devlink_region_destroy(struct devlink_region *region)
10370 {
10371         struct devlink *devlink = region->devlink;
10372         struct devlink_snapshot *snapshot, *ts;
10373
10374         mutex_lock(&devlink->lock);
10375
10376         /* Free all snapshots of region */
10377         list_for_each_entry_safe(snapshot, ts, &region->snapshot_list, list)
10378                 devlink_region_snapshot_del(region, snapshot);
10379
10380         list_del(&region->list);
10381
10382         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_DEL);
10383         mutex_unlock(&devlink->lock);
10384         kfree(region);
10385 }
10386 EXPORT_SYMBOL_GPL(devlink_region_destroy);
10387
10388 /**
10389  *      devlink_region_snapshot_id_get - get snapshot ID
10390  *
10391  *      This callback should be called when adding a new snapshot,
10392  *      Driver should use the same id for multiple snapshots taken
10393  *      on multiple regions at the same time/by the same trigger.
10394  *
10395  *      The caller of this function must use devlink_region_snapshot_id_put
10396  *      when finished creating regions using this id.
10397  *
10398  *      Returns zero on success, or a negative error code on failure.
10399  *
10400  *      @devlink: devlink
10401  *      @id: storage to return id
10402  */
10403 int devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id)
10404 {
10405         int err;
10406
10407         mutex_lock(&devlink->lock);
10408         err = __devlink_region_snapshot_id_get(devlink, id);
10409         mutex_unlock(&devlink->lock);
10410
10411         return err;
10412 }
10413 EXPORT_SYMBOL_GPL(devlink_region_snapshot_id_get);
10414
10415 /**
10416  *      devlink_region_snapshot_id_put - put snapshot ID reference
10417  *
10418  *      This should be called by a driver after finishing creating snapshots
10419  *      with an id. Doing so ensures that the ID can later be released in the
10420  *      event that all snapshots using it have been destroyed.
10421  *
10422  *      @devlink: devlink
10423  *      @id: id to release reference on
10424  */
10425 void devlink_region_snapshot_id_put(struct devlink *devlink, u32 id)
10426 {
10427         mutex_lock(&devlink->lock);
10428         __devlink_snapshot_id_decrement(devlink, id);
10429         mutex_unlock(&devlink->lock);
10430 }
10431 EXPORT_SYMBOL_GPL(devlink_region_snapshot_id_put);
10432
10433 /**
10434  *      devlink_region_snapshot_create - create a new snapshot
10435  *      This will add a new snapshot of a region. The snapshot
10436  *      will be stored on the region struct and can be accessed
10437  *      from devlink. This is useful for future analyses of snapshots.
10438  *      Multiple snapshots can be created on a region.
10439  *      The @snapshot_id should be obtained using the getter function.
10440  *
10441  *      @region: devlink region of the snapshot
10442  *      @data: snapshot data
10443  *      @snapshot_id: snapshot id to be created
10444  */
10445 int devlink_region_snapshot_create(struct devlink_region *region,
10446                                    u8 *data, u32 snapshot_id)
10447 {
10448         struct devlink *devlink = region->devlink;
10449         int err;
10450
10451         mutex_lock(&devlink->lock);
10452         err = __devlink_region_snapshot_create(region, data, snapshot_id);
10453         mutex_unlock(&devlink->lock);
10454
10455         return err;
10456 }
10457 EXPORT_SYMBOL_GPL(devlink_region_snapshot_create);
10458
10459 #define DEVLINK_TRAP(_id, _type)                                              \
10460         {                                                                     \
10461                 .type = DEVLINK_TRAP_TYPE_##_type,                            \
10462                 .id = DEVLINK_TRAP_GENERIC_ID_##_id,                          \
10463                 .name = DEVLINK_TRAP_GENERIC_NAME_##_id,                      \
10464         }
10465
10466 static const struct devlink_trap devlink_trap_generic[] = {
10467         DEVLINK_TRAP(SMAC_MC, DROP),
10468         DEVLINK_TRAP(VLAN_TAG_MISMATCH, DROP),
10469         DEVLINK_TRAP(INGRESS_VLAN_FILTER, DROP),
10470         DEVLINK_TRAP(INGRESS_STP_FILTER, DROP),
10471         DEVLINK_TRAP(EMPTY_TX_LIST, DROP),
10472         DEVLINK_TRAP(PORT_LOOPBACK_FILTER, DROP),
10473         DEVLINK_TRAP(BLACKHOLE_ROUTE, DROP),
10474         DEVLINK_TRAP(TTL_ERROR, EXCEPTION),
10475         DEVLINK_TRAP(TAIL_DROP, DROP),
10476         DEVLINK_TRAP(NON_IP_PACKET, DROP),
10477         DEVLINK_TRAP(UC_DIP_MC_DMAC, DROP),
10478         DEVLINK_TRAP(DIP_LB, DROP),
10479         DEVLINK_TRAP(SIP_MC, DROP),
10480         DEVLINK_TRAP(SIP_LB, DROP),
10481         DEVLINK_TRAP(CORRUPTED_IP_HDR, DROP),
10482         DEVLINK_TRAP(IPV4_SIP_BC, DROP),
10483         DEVLINK_TRAP(IPV6_MC_DIP_RESERVED_SCOPE, DROP),
10484         DEVLINK_TRAP(IPV6_MC_DIP_INTERFACE_LOCAL_SCOPE, DROP),
10485         DEVLINK_TRAP(MTU_ERROR, EXCEPTION),
10486         DEVLINK_TRAP(UNRESOLVED_NEIGH, EXCEPTION),
10487         DEVLINK_TRAP(RPF, EXCEPTION),
10488         DEVLINK_TRAP(REJECT_ROUTE, EXCEPTION),
10489         DEVLINK_TRAP(IPV4_LPM_UNICAST_MISS, EXCEPTION),
10490         DEVLINK_TRAP(IPV6_LPM_UNICAST_MISS, EXCEPTION),
10491         DEVLINK_TRAP(NON_ROUTABLE, DROP),
10492         DEVLINK_TRAP(DECAP_ERROR, EXCEPTION),
10493         DEVLINK_TRAP(OVERLAY_SMAC_MC, DROP),
10494         DEVLINK_TRAP(INGRESS_FLOW_ACTION_DROP, DROP),
10495         DEVLINK_TRAP(EGRESS_FLOW_ACTION_DROP, DROP),
10496         DEVLINK_TRAP(STP, CONTROL),
10497         DEVLINK_TRAP(LACP, CONTROL),
10498         DEVLINK_TRAP(LLDP, CONTROL),
10499         DEVLINK_TRAP(IGMP_QUERY, CONTROL),
10500         DEVLINK_TRAP(IGMP_V1_REPORT, CONTROL),
10501         DEVLINK_TRAP(IGMP_V2_REPORT, CONTROL),
10502         DEVLINK_TRAP(IGMP_V3_REPORT, CONTROL),
10503         DEVLINK_TRAP(IGMP_V2_LEAVE, CONTROL),
10504         DEVLINK_TRAP(MLD_QUERY, CONTROL),
10505         DEVLINK_TRAP(MLD_V1_REPORT, CONTROL),
10506         DEVLINK_TRAP(MLD_V2_REPORT, CONTROL),
10507         DEVLINK_TRAP(MLD_V1_DONE, CONTROL),
10508         DEVLINK_TRAP(IPV4_DHCP, CONTROL),
10509         DEVLINK_TRAP(IPV6_DHCP, CONTROL),
10510         DEVLINK_TRAP(ARP_REQUEST, CONTROL),
10511         DEVLINK_TRAP(ARP_RESPONSE, CONTROL),
10512         DEVLINK_TRAP(ARP_OVERLAY, CONTROL),
10513         DEVLINK_TRAP(IPV6_NEIGH_SOLICIT, CONTROL),
10514         DEVLINK_TRAP(IPV6_NEIGH_ADVERT, CONTROL),
10515         DEVLINK_TRAP(IPV4_BFD, CONTROL),
10516         DEVLINK_TRAP(IPV6_BFD, CONTROL),
10517         DEVLINK_TRAP(IPV4_OSPF, CONTROL),
10518         DEVLINK_TRAP(IPV6_OSPF, CONTROL),
10519         DEVLINK_TRAP(IPV4_BGP, CONTROL),
10520         DEVLINK_TRAP(IPV6_BGP, CONTROL),
10521         DEVLINK_TRAP(IPV4_VRRP, CONTROL),
10522         DEVLINK_TRAP(IPV6_VRRP, CONTROL),
10523         DEVLINK_TRAP(IPV4_PIM, CONTROL),
10524         DEVLINK_TRAP(IPV6_PIM, CONTROL),
10525         DEVLINK_TRAP(UC_LB, CONTROL),
10526         DEVLINK_TRAP(LOCAL_ROUTE, CONTROL),
10527         DEVLINK_TRAP(EXTERNAL_ROUTE, CONTROL),
10528         DEVLINK_TRAP(IPV6_UC_DIP_LINK_LOCAL_SCOPE, CONTROL),
10529         DEVLINK_TRAP(IPV6_DIP_ALL_NODES, CONTROL),
10530         DEVLINK_TRAP(IPV6_DIP_ALL_ROUTERS, CONTROL),
10531         DEVLINK_TRAP(IPV6_ROUTER_SOLICIT, CONTROL),
10532         DEVLINK_TRAP(IPV6_ROUTER_ADVERT, CONTROL),
10533         DEVLINK_TRAP(IPV6_REDIRECT, CONTROL),
10534         DEVLINK_TRAP(IPV4_ROUTER_ALERT, CONTROL),
10535         DEVLINK_TRAP(IPV6_ROUTER_ALERT, CONTROL),
10536         DEVLINK_TRAP(PTP_EVENT, CONTROL),
10537         DEVLINK_TRAP(PTP_GENERAL, CONTROL),
10538         DEVLINK_TRAP(FLOW_ACTION_SAMPLE, CONTROL),
10539         DEVLINK_TRAP(FLOW_ACTION_TRAP, CONTROL),
10540         DEVLINK_TRAP(EARLY_DROP, DROP),
10541         DEVLINK_TRAP(VXLAN_PARSING, DROP),
10542         DEVLINK_TRAP(LLC_SNAP_PARSING, DROP),
10543         DEVLINK_TRAP(VLAN_PARSING, DROP),
10544         DEVLINK_TRAP(PPPOE_PPP_PARSING, DROP),
10545         DEVLINK_TRAP(MPLS_PARSING, DROP),
10546         DEVLINK_TRAP(ARP_PARSING, DROP),
10547         DEVLINK_TRAP(IP_1_PARSING, DROP),
10548         DEVLINK_TRAP(IP_N_PARSING, DROP),
10549         DEVLINK_TRAP(GRE_PARSING, DROP),
10550         DEVLINK_TRAP(UDP_PARSING, DROP),
10551         DEVLINK_TRAP(TCP_PARSING, DROP),
10552         DEVLINK_TRAP(IPSEC_PARSING, DROP),
10553         DEVLINK_TRAP(SCTP_PARSING, DROP),
10554         DEVLINK_TRAP(DCCP_PARSING, DROP),
10555         DEVLINK_TRAP(GTP_PARSING, DROP),
10556         DEVLINK_TRAP(ESP_PARSING, DROP),
10557         DEVLINK_TRAP(BLACKHOLE_NEXTHOP, DROP),
10558         DEVLINK_TRAP(DMAC_FILTER, DROP),
10559 };
10560
10561 #define DEVLINK_TRAP_GROUP(_id)                                               \
10562         {                                                                     \
10563                 .id = DEVLINK_TRAP_GROUP_GENERIC_ID_##_id,                    \
10564                 .name = DEVLINK_TRAP_GROUP_GENERIC_NAME_##_id,                \
10565         }
10566
10567 static const struct devlink_trap_group devlink_trap_group_generic[] = {
10568         DEVLINK_TRAP_GROUP(L2_DROPS),
10569         DEVLINK_TRAP_GROUP(L3_DROPS),
10570         DEVLINK_TRAP_GROUP(L3_EXCEPTIONS),
10571         DEVLINK_TRAP_GROUP(BUFFER_DROPS),
10572         DEVLINK_TRAP_GROUP(TUNNEL_DROPS),
10573         DEVLINK_TRAP_GROUP(ACL_DROPS),
10574         DEVLINK_TRAP_GROUP(STP),
10575         DEVLINK_TRAP_GROUP(LACP),
10576         DEVLINK_TRAP_GROUP(LLDP),
10577         DEVLINK_TRAP_GROUP(MC_SNOOPING),
10578         DEVLINK_TRAP_GROUP(DHCP),
10579         DEVLINK_TRAP_GROUP(NEIGH_DISCOVERY),
10580         DEVLINK_TRAP_GROUP(BFD),
10581         DEVLINK_TRAP_GROUP(OSPF),
10582         DEVLINK_TRAP_GROUP(BGP),
10583         DEVLINK_TRAP_GROUP(VRRP),
10584         DEVLINK_TRAP_GROUP(PIM),
10585         DEVLINK_TRAP_GROUP(UC_LB),
10586         DEVLINK_TRAP_GROUP(LOCAL_DELIVERY),
10587         DEVLINK_TRAP_GROUP(EXTERNAL_DELIVERY),
10588         DEVLINK_TRAP_GROUP(IPV6),
10589         DEVLINK_TRAP_GROUP(PTP_EVENT),
10590         DEVLINK_TRAP_GROUP(PTP_GENERAL),
10591         DEVLINK_TRAP_GROUP(ACL_SAMPLE),
10592         DEVLINK_TRAP_GROUP(ACL_TRAP),
10593         DEVLINK_TRAP_GROUP(PARSER_ERROR_DROPS),
10594 };
10595
10596 static int devlink_trap_generic_verify(const struct devlink_trap *trap)
10597 {
10598         if (trap->id > DEVLINK_TRAP_GENERIC_ID_MAX)
10599                 return -EINVAL;
10600
10601         if (strcmp(trap->name, devlink_trap_generic[trap->id].name))
10602                 return -EINVAL;
10603
10604         if (trap->type != devlink_trap_generic[trap->id].type)
10605                 return -EINVAL;
10606
10607         return 0;
10608 }
10609
10610 static int devlink_trap_driver_verify(const struct devlink_trap *trap)
10611 {
10612         int i;
10613
10614         if (trap->id <= DEVLINK_TRAP_GENERIC_ID_MAX)
10615                 return -EINVAL;
10616
10617         for (i = 0; i < ARRAY_SIZE(devlink_trap_generic); i++) {
10618                 if (!strcmp(trap->name, devlink_trap_generic[i].name))
10619                         return -EEXIST;
10620         }
10621
10622         return 0;
10623 }
10624
10625 static int devlink_trap_verify(const struct devlink_trap *trap)
10626 {
10627         if (!trap || !trap->name)
10628                 return -EINVAL;
10629
10630         if (trap->generic)
10631                 return devlink_trap_generic_verify(trap);
10632         else
10633                 return devlink_trap_driver_verify(trap);
10634 }
10635
10636 static int
10637 devlink_trap_group_generic_verify(const struct devlink_trap_group *group)
10638 {
10639         if (group->id > DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
10640                 return -EINVAL;
10641
10642         if (strcmp(group->name, devlink_trap_group_generic[group->id].name))
10643                 return -EINVAL;
10644
10645         return 0;
10646 }
10647
10648 static int
10649 devlink_trap_group_driver_verify(const struct devlink_trap_group *group)
10650 {
10651         int i;
10652
10653         if (group->id <= DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
10654                 return -EINVAL;
10655
10656         for (i = 0; i < ARRAY_SIZE(devlink_trap_group_generic); i++) {
10657                 if (!strcmp(group->name, devlink_trap_group_generic[i].name))
10658                         return -EEXIST;
10659         }
10660
10661         return 0;
10662 }
10663
10664 static int devlink_trap_group_verify(const struct devlink_trap_group *group)
10665 {
10666         if (group->generic)
10667                 return devlink_trap_group_generic_verify(group);
10668         else
10669                 return devlink_trap_group_driver_verify(group);
10670 }
10671
10672 static void
10673 devlink_trap_group_notify(struct devlink *devlink,
10674                           const struct devlink_trap_group_item *group_item,
10675                           enum devlink_command cmd)
10676 {
10677         struct sk_buff *msg;
10678         int err;
10679
10680         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_GROUP_NEW &&
10681                      cmd != DEVLINK_CMD_TRAP_GROUP_DEL);
10682         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
10683                 return;
10684
10685         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
10686         if (!msg)
10687                 return;
10688
10689         err = devlink_nl_trap_group_fill(msg, devlink, group_item, cmd, 0, 0,
10690                                          0);
10691         if (err) {
10692                 nlmsg_free(msg);
10693                 return;
10694         }
10695
10696         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
10697                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
10698 }
10699
10700 static int
10701 devlink_trap_item_group_link(struct devlink *devlink,
10702                              struct devlink_trap_item *trap_item)
10703 {
10704         u16 group_id = trap_item->trap->init_group_id;
10705         struct devlink_trap_group_item *group_item;
10706
10707         group_item = devlink_trap_group_item_lookup_by_id(devlink, group_id);
10708         if (WARN_ON_ONCE(!group_item))
10709                 return -EINVAL;
10710
10711         trap_item->group_item = group_item;
10712
10713         return 0;
10714 }
10715
10716 static void devlink_trap_notify(struct devlink *devlink,
10717                                 const struct devlink_trap_item *trap_item,
10718                                 enum devlink_command cmd)
10719 {
10720         struct sk_buff *msg;
10721         int err;
10722
10723         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_NEW &&
10724                      cmd != DEVLINK_CMD_TRAP_DEL);
10725         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
10726                 return;
10727
10728         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
10729         if (!msg)
10730                 return;
10731
10732         err = devlink_nl_trap_fill(msg, devlink, trap_item, cmd, 0, 0, 0);
10733         if (err) {
10734                 nlmsg_free(msg);
10735                 return;
10736         }
10737
10738         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
10739                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
10740 }
10741
10742 static int
10743 devlink_trap_register(struct devlink *devlink,
10744                       const struct devlink_trap *trap, void *priv)
10745 {
10746         struct devlink_trap_item *trap_item;
10747         int err;
10748
10749         if (devlink_trap_item_lookup(devlink, trap->name))
10750                 return -EEXIST;
10751
10752         trap_item = kzalloc(sizeof(*trap_item), GFP_KERNEL);
10753         if (!trap_item)
10754                 return -ENOMEM;
10755
10756         trap_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
10757         if (!trap_item->stats) {
10758                 err = -ENOMEM;
10759                 goto err_stats_alloc;
10760         }
10761
10762         trap_item->trap = trap;
10763         trap_item->action = trap->init_action;
10764         trap_item->priv = priv;
10765
10766         err = devlink_trap_item_group_link(devlink, trap_item);
10767         if (err)
10768                 goto err_group_link;
10769
10770         err = devlink->ops->trap_init(devlink, trap, trap_item);
10771         if (err)
10772                 goto err_trap_init;
10773
10774         list_add_tail(&trap_item->list, &devlink->trap_list);
10775         devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_NEW);
10776
10777         return 0;
10778
10779 err_trap_init:
10780 err_group_link:
10781         free_percpu(trap_item->stats);
10782 err_stats_alloc:
10783         kfree(trap_item);
10784         return err;
10785 }
10786
10787 static void devlink_trap_unregister(struct devlink *devlink,
10788                                     const struct devlink_trap *trap)
10789 {
10790         struct devlink_trap_item *trap_item;
10791
10792         trap_item = devlink_trap_item_lookup(devlink, trap->name);
10793         if (WARN_ON_ONCE(!trap_item))
10794                 return;
10795
10796         devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_DEL);
10797         list_del(&trap_item->list);
10798         if (devlink->ops->trap_fini)
10799                 devlink->ops->trap_fini(devlink, trap, trap_item);
10800         free_percpu(trap_item->stats);
10801         kfree(trap_item);
10802 }
10803
10804 static void devlink_trap_disable(struct devlink *devlink,
10805                                  const struct devlink_trap *trap)
10806 {
10807         struct devlink_trap_item *trap_item;
10808
10809         trap_item = devlink_trap_item_lookup(devlink, trap->name);
10810         if (WARN_ON_ONCE(!trap_item))
10811                 return;
10812
10813         devlink->ops->trap_action_set(devlink, trap, DEVLINK_TRAP_ACTION_DROP,
10814                                       NULL);
10815         trap_item->action = DEVLINK_TRAP_ACTION_DROP;
10816 }
10817
10818 /**
10819  * devlink_traps_register - Register packet traps with devlink.
10820  * @devlink: devlink.
10821  * @traps: Packet traps.
10822  * @traps_count: Count of provided packet traps.
10823  * @priv: Driver private information.
10824  *
10825  * Return: Non-zero value on failure.
10826  */
10827 int devlink_traps_register(struct devlink *devlink,
10828                            const struct devlink_trap *traps,
10829                            size_t traps_count, void *priv)
10830 {
10831         int i, err;
10832
10833         if (!devlink->ops->trap_init || !devlink->ops->trap_action_set)
10834                 return -EINVAL;
10835
10836         mutex_lock(&devlink->lock);
10837         for (i = 0; i < traps_count; i++) {
10838                 const struct devlink_trap *trap = &traps[i];
10839
10840                 err = devlink_trap_verify(trap);
10841                 if (err)
10842                         goto err_trap_verify;
10843
10844                 err = devlink_trap_register(devlink, trap, priv);
10845                 if (err)
10846                         goto err_trap_register;
10847         }
10848         mutex_unlock(&devlink->lock);
10849
10850         return 0;
10851
10852 err_trap_register:
10853 err_trap_verify:
10854         for (i--; i >= 0; i--)
10855                 devlink_trap_unregister(devlink, &traps[i]);
10856         mutex_unlock(&devlink->lock);
10857         return err;
10858 }
10859 EXPORT_SYMBOL_GPL(devlink_traps_register);
10860
10861 /**
10862  * devlink_traps_unregister - Unregister packet traps from devlink.
10863  * @devlink: devlink.
10864  * @traps: Packet traps.
10865  * @traps_count: Count of provided packet traps.
10866  */
10867 void devlink_traps_unregister(struct devlink *devlink,
10868                               const struct devlink_trap *traps,
10869                               size_t traps_count)
10870 {
10871         int i;
10872
10873         mutex_lock(&devlink->lock);
10874         /* Make sure we do not have any packets in-flight while unregistering
10875          * traps by disabling all of them and waiting for a grace period.
10876          */
10877         for (i = traps_count - 1; i >= 0; i--)
10878                 devlink_trap_disable(devlink, &traps[i]);
10879         synchronize_rcu();
10880         for (i = traps_count - 1; i >= 0; i--)
10881                 devlink_trap_unregister(devlink, &traps[i]);
10882         mutex_unlock(&devlink->lock);
10883 }
10884 EXPORT_SYMBOL_GPL(devlink_traps_unregister);
10885
10886 static void
10887 devlink_trap_stats_update(struct devlink_stats __percpu *trap_stats,
10888                           size_t skb_len)
10889 {
10890         struct devlink_stats *stats;
10891
10892         stats = this_cpu_ptr(trap_stats);
10893         u64_stats_update_begin(&stats->syncp);
10894         stats->rx_bytes += skb_len;
10895         stats->rx_packets++;
10896         u64_stats_update_end(&stats->syncp);
10897 }
10898
10899 static void
10900 devlink_trap_report_metadata_set(struct devlink_trap_metadata *metadata,
10901                                  const struct devlink_trap_item *trap_item,
10902                                  struct devlink_port *in_devlink_port,
10903                                  const struct flow_action_cookie *fa_cookie)
10904 {
10905         metadata->trap_name = trap_item->trap->name;
10906         metadata->trap_group_name = trap_item->group_item->group->name;
10907         metadata->fa_cookie = fa_cookie;
10908         metadata->trap_type = trap_item->trap->type;
10909
10910         spin_lock(&in_devlink_port->type_lock);
10911         if (in_devlink_port->type == DEVLINK_PORT_TYPE_ETH)
10912                 metadata->input_dev = in_devlink_port->type_dev;
10913         spin_unlock(&in_devlink_port->type_lock);
10914 }
10915
10916 /**
10917  * devlink_trap_report - Report trapped packet to drop monitor.
10918  * @devlink: devlink.
10919  * @skb: Trapped packet.
10920  * @trap_ctx: Trap context.
10921  * @in_devlink_port: Input devlink port.
10922  * @fa_cookie: Flow action cookie. Could be NULL.
10923  */
10924 void devlink_trap_report(struct devlink *devlink, struct sk_buff *skb,
10925                          void *trap_ctx, struct devlink_port *in_devlink_port,
10926                          const struct flow_action_cookie *fa_cookie)
10927
10928 {
10929         struct devlink_trap_item *trap_item = trap_ctx;
10930
10931         devlink_trap_stats_update(trap_item->stats, skb->len);
10932         devlink_trap_stats_update(trap_item->group_item->stats, skb->len);
10933
10934         if (trace_devlink_trap_report_enabled()) {
10935                 struct devlink_trap_metadata metadata = {};
10936
10937                 devlink_trap_report_metadata_set(&metadata, trap_item,
10938                                                  in_devlink_port, fa_cookie);
10939                 trace_devlink_trap_report(devlink, skb, &metadata);
10940         }
10941 }
10942 EXPORT_SYMBOL_GPL(devlink_trap_report);
10943
10944 /**
10945  * devlink_trap_ctx_priv - Trap context to driver private information.
10946  * @trap_ctx: Trap context.
10947  *
10948  * Return: Driver private information passed during registration.
10949  */
10950 void *devlink_trap_ctx_priv(void *trap_ctx)
10951 {
10952         struct devlink_trap_item *trap_item = trap_ctx;
10953
10954         return trap_item->priv;
10955 }
10956 EXPORT_SYMBOL_GPL(devlink_trap_ctx_priv);
10957
10958 static int
10959 devlink_trap_group_item_policer_link(struct devlink *devlink,
10960                                      struct devlink_trap_group_item *group_item)
10961 {
10962         u32 policer_id = group_item->group->init_policer_id;
10963         struct devlink_trap_policer_item *policer_item;
10964
10965         if (policer_id == 0)
10966                 return 0;
10967
10968         policer_item = devlink_trap_policer_item_lookup(devlink, policer_id);
10969         if (WARN_ON_ONCE(!policer_item))
10970                 return -EINVAL;
10971
10972         group_item->policer_item = policer_item;
10973
10974         return 0;
10975 }
10976
10977 static int
10978 devlink_trap_group_register(struct devlink *devlink,
10979                             const struct devlink_trap_group *group)
10980 {
10981         struct devlink_trap_group_item *group_item;
10982         int err;
10983
10984         if (devlink_trap_group_item_lookup(devlink, group->name))
10985                 return -EEXIST;
10986
10987         group_item = kzalloc(sizeof(*group_item), GFP_KERNEL);
10988         if (!group_item)
10989                 return -ENOMEM;
10990
10991         group_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
10992         if (!group_item->stats) {
10993                 err = -ENOMEM;
10994                 goto err_stats_alloc;
10995         }
10996
10997         group_item->group = group;
10998
10999         err = devlink_trap_group_item_policer_link(devlink, group_item);
11000         if (err)
11001                 goto err_policer_link;
11002
11003         if (devlink->ops->trap_group_init) {
11004                 err = devlink->ops->trap_group_init(devlink, group);
11005                 if (err)
11006                         goto err_group_init;
11007         }
11008
11009         list_add_tail(&group_item->list, &devlink->trap_group_list);
11010         devlink_trap_group_notify(devlink, group_item,
11011                                   DEVLINK_CMD_TRAP_GROUP_NEW);
11012
11013         return 0;
11014
11015 err_group_init:
11016 err_policer_link:
11017         free_percpu(group_item->stats);
11018 err_stats_alloc:
11019         kfree(group_item);
11020         return err;
11021 }
11022
11023 static void
11024 devlink_trap_group_unregister(struct devlink *devlink,
11025                               const struct devlink_trap_group *group)
11026 {
11027         struct devlink_trap_group_item *group_item;
11028
11029         group_item = devlink_trap_group_item_lookup(devlink, group->name);
11030         if (WARN_ON_ONCE(!group_item))
11031                 return;
11032
11033         devlink_trap_group_notify(devlink, group_item,
11034                                   DEVLINK_CMD_TRAP_GROUP_DEL);
11035         list_del(&group_item->list);
11036         free_percpu(group_item->stats);
11037         kfree(group_item);
11038 }
11039
11040 /**
11041  * devlink_trap_groups_register - Register packet trap groups with devlink.
11042  * @devlink: devlink.
11043  * @groups: Packet trap groups.
11044  * @groups_count: Count of provided packet trap groups.
11045  *
11046  * Return: Non-zero value on failure.
11047  */
11048 int devlink_trap_groups_register(struct devlink *devlink,
11049                                  const struct devlink_trap_group *groups,
11050                                  size_t groups_count)
11051 {
11052         int i, err;
11053
11054         mutex_lock(&devlink->lock);
11055         for (i = 0; i < groups_count; i++) {
11056                 const struct devlink_trap_group *group = &groups[i];
11057
11058                 err = devlink_trap_group_verify(group);
11059                 if (err)
11060                         goto err_trap_group_verify;
11061
11062                 err = devlink_trap_group_register(devlink, group);
11063                 if (err)
11064                         goto err_trap_group_register;
11065         }
11066         mutex_unlock(&devlink->lock);
11067
11068         return 0;
11069
11070 err_trap_group_register:
11071 err_trap_group_verify:
11072         for (i--; i >= 0; i--)
11073                 devlink_trap_group_unregister(devlink, &groups[i]);
11074         mutex_unlock(&devlink->lock);
11075         return err;
11076 }
11077 EXPORT_SYMBOL_GPL(devlink_trap_groups_register);
11078
11079 /**
11080  * devlink_trap_groups_unregister - Unregister packet trap groups from devlink.
11081  * @devlink: devlink.
11082  * @groups: Packet trap groups.
11083  * @groups_count: Count of provided packet trap groups.
11084  */
11085 void devlink_trap_groups_unregister(struct devlink *devlink,
11086                                     const struct devlink_trap_group *groups,
11087                                     size_t groups_count)
11088 {
11089         int i;
11090
11091         mutex_lock(&devlink->lock);
11092         for (i = groups_count - 1; i >= 0; i--)
11093                 devlink_trap_group_unregister(devlink, &groups[i]);
11094         mutex_unlock(&devlink->lock);
11095 }
11096 EXPORT_SYMBOL_GPL(devlink_trap_groups_unregister);
11097
11098 static void
11099 devlink_trap_policer_notify(struct devlink *devlink,
11100                             const struct devlink_trap_policer_item *policer_item,
11101                             enum devlink_command cmd)
11102 {
11103         struct sk_buff *msg;
11104         int err;
11105
11106         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_POLICER_NEW &&
11107                      cmd != DEVLINK_CMD_TRAP_POLICER_DEL);
11108         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
11109                 return;
11110
11111         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
11112         if (!msg)
11113                 return;
11114
11115         err = devlink_nl_trap_policer_fill(msg, devlink, policer_item, cmd, 0,
11116                                            0, 0);
11117         if (err) {
11118                 nlmsg_free(msg);
11119                 return;
11120         }
11121
11122         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
11123                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
11124 }
11125
11126 static int
11127 devlink_trap_policer_register(struct devlink *devlink,
11128                               const struct devlink_trap_policer *policer)
11129 {
11130         struct devlink_trap_policer_item *policer_item;
11131         int err;
11132
11133         if (devlink_trap_policer_item_lookup(devlink, policer->id))
11134                 return -EEXIST;
11135
11136         policer_item = kzalloc(sizeof(*policer_item), GFP_KERNEL);
11137         if (!policer_item)
11138                 return -ENOMEM;
11139
11140         policer_item->policer = policer;
11141         policer_item->rate = policer->init_rate;
11142         policer_item->burst = policer->init_burst;
11143
11144         if (devlink->ops->trap_policer_init) {
11145                 err = devlink->ops->trap_policer_init(devlink, policer);
11146                 if (err)
11147                         goto err_policer_init;
11148         }
11149
11150         list_add_tail(&policer_item->list, &devlink->trap_policer_list);
11151         devlink_trap_policer_notify(devlink, policer_item,
11152                                     DEVLINK_CMD_TRAP_POLICER_NEW);
11153
11154         return 0;
11155
11156 err_policer_init:
11157         kfree(policer_item);
11158         return err;
11159 }
11160
11161 static void
11162 devlink_trap_policer_unregister(struct devlink *devlink,
11163                                 const struct devlink_trap_policer *policer)
11164 {
11165         struct devlink_trap_policer_item *policer_item;
11166
11167         policer_item = devlink_trap_policer_item_lookup(devlink, policer->id);
11168         if (WARN_ON_ONCE(!policer_item))
11169                 return;
11170
11171         devlink_trap_policer_notify(devlink, policer_item,
11172                                     DEVLINK_CMD_TRAP_POLICER_DEL);
11173         list_del(&policer_item->list);
11174         if (devlink->ops->trap_policer_fini)
11175                 devlink->ops->trap_policer_fini(devlink, policer);
11176         kfree(policer_item);
11177 }
11178
11179 /**
11180  * devlink_trap_policers_register - Register packet trap policers with devlink.
11181  * @devlink: devlink.
11182  * @policers: Packet trap policers.
11183  * @policers_count: Count of provided packet trap policers.
11184  *
11185  * Return: Non-zero value on failure.
11186  */
11187 int
11188 devlink_trap_policers_register(struct devlink *devlink,
11189                                const struct devlink_trap_policer *policers,
11190                                size_t policers_count)
11191 {
11192         int i, err;
11193
11194         mutex_lock(&devlink->lock);
11195         for (i = 0; i < policers_count; i++) {
11196                 const struct devlink_trap_policer *policer = &policers[i];
11197
11198                 if (WARN_ON(policer->id == 0 ||
11199                             policer->max_rate < policer->min_rate ||
11200                             policer->max_burst < policer->min_burst)) {
11201                         err = -EINVAL;
11202                         goto err_trap_policer_verify;
11203                 }
11204
11205                 err = devlink_trap_policer_register(devlink, policer);
11206                 if (err)
11207                         goto err_trap_policer_register;
11208         }
11209         mutex_unlock(&devlink->lock);
11210
11211         return 0;
11212
11213 err_trap_policer_register:
11214 err_trap_policer_verify:
11215         for (i--; i >= 0; i--)
11216                 devlink_trap_policer_unregister(devlink, &policers[i]);
11217         mutex_unlock(&devlink->lock);
11218         return err;
11219 }
11220 EXPORT_SYMBOL_GPL(devlink_trap_policers_register);
11221
11222 /**
11223  * devlink_trap_policers_unregister - Unregister packet trap policers from devlink.
11224  * @devlink: devlink.
11225  * @policers: Packet trap policers.
11226  * @policers_count: Count of provided packet trap policers.
11227  */
11228 void
11229 devlink_trap_policers_unregister(struct devlink *devlink,
11230                                  const struct devlink_trap_policer *policers,
11231                                  size_t policers_count)
11232 {
11233         int i;
11234
11235         mutex_lock(&devlink->lock);
11236         for (i = policers_count - 1; i >= 0; i--)
11237                 devlink_trap_policer_unregister(devlink, &policers[i]);
11238         mutex_unlock(&devlink->lock);
11239 }
11240 EXPORT_SYMBOL_GPL(devlink_trap_policers_unregister);
11241
11242 static void __devlink_compat_running_version(struct devlink *devlink,
11243                                              char *buf, size_t len)
11244 {
11245         const struct nlattr *nlattr;
11246         struct devlink_info_req req;
11247         struct sk_buff *msg;
11248         int rem, err;
11249
11250         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
11251         if (!msg)
11252                 return;
11253
11254         req.msg = msg;
11255         err = devlink->ops->info_get(devlink, &req, NULL);
11256         if (err)
11257                 goto free_msg;
11258
11259         nla_for_each_attr(nlattr, (void *)msg->data, msg->len, rem) {
11260                 const struct nlattr *kv;
11261                 int rem_kv;
11262
11263                 if (nla_type(nlattr) != DEVLINK_ATTR_INFO_VERSION_RUNNING)
11264                         continue;
11265
11266                 nla_for_each_nested(kv, nlattr, rem_kv) {
11267                         if (nla_type(kv) != DEVLINK_ATTR_INFO_VERSION_VALUE)
11268                                 continue;
11269
11270                         strlcat(buf, nla_data(kv), len);
11271                         strlcat(buf, " ", len);
11272                 }
11273         }
11274 free_msg:
11275         nlmsg_free(msg);
11276 }
11277
11278 static struct devlink_port *netdev_to_devlink_port(struct net_device *dev)
11279 {
11280         if (!dev->netdev_ops->ndo_get_devlink_port)
11281                 return NULL;
11282
11283         return dev->netdev_ops->ndo_get_devlink_port(dev);
11284 }
11285
11286 void devlink_compat_running_version(struct devlink *devlink,
11287                                     char *buf, size_t len)
11288 {
11289         if (!devlink->ops->info_get)
11290                 return;
11291
11292         mutex_lock(&devlink->lock);
11293         __devlink_compat_running_version(devlink, buf, len);
11294         mutex_unlock(&devlink->lock);
11295 }
11296
11297 int devlink_compat_flash_update(struct devlink *devlink, const char *file_name)
11298 {
11299         struct devlink_flash_update_params params = {};
11300         int ret;
11301
11302         if (!devlink->ops->flash_update)
11303                 return -EOPNOTSUPP;
11304
11305         ret = request_firmware(&params.fw, file_name, devlink->dev);
11306         if (ret)
11307                 return ret;
11308
11309         mutex_lock(&devlink->lock);
11310         devlink_flash_update_begin_notify(devlink);
11311         ret = devlink->ops->flash_update(devlink, &params, NULL);
11312         devlink_flash_update_end_notify(devlink);
11313         mutex_unlock(&devlink->lock);
11314
11315         release_firmware(params.fw);
11316
11317         return ret;
11318 }
11319
11320 int devlink_compat_phys_port_name_get(struct net_device *dev,
11321                                       char *name, size_t len)
11322 {
11323         struct devlink_port *devlink_port;
11324
11325         /* RTNL mutex is held here which ensures that devlink_port
11326          * instance cannot disappear in the middle. No need to take
11327          * any devlink lock as only permanent values are accessed.
11328          */
11329         ASSERT_RTNL();
11330
11331         devlink_port = netdev_to_devlink_port(dev);
11332         if (!devlink_port)
11333                 return -EOPNOTSUPP;
11334
11335         return __devlink_port_phys_port_name_get(devlink_port, name, len);
11336 }
11337
11338 int devlink_compat_switch_id_get(struct net_device *dev,
11339                                  struct netdev_phys_item_id *ppid)
11340 {
11341         struct devlink_port *devlink_port;
11342
11343         /* Caller must hold RTNL mutex or reference to dev, which ensures that
11344          * devlink_port instance cannot disappear in the middle. No need to take
11345          * any devlink lock as only permanent values are accessed.
11346          */
11347         devlink_port = netdev_to_devlink_port(dev);
11348         if (!devlink_port || !devlink_port->switch_port)
11349                 return -EOPNOTSUPP;
11350
11351         memcpy(ppid, &devlink_port->attrs.switch_id, sizeof(*ppid));
11352
11353         return 0;
11354 }
11355
11356 static void __net_exit devlink_pernet_pre_exit(struct net *net)
11357 {
11358         struct devlink *devlink;
11359         u32 actions_performed;
11360         unsigned long index;
11361         int err;
11362
11363         /* In case network namespace is getting destroyed, reload
11364          * all devlink instances from this namespace into init_net.
11365          */
11366         mutex_lock(&devlink_mutex);
11367         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
11368                 if (!devlink_try_get(devlink))
11369                         continue;
11370
11371                 if (!net_eq(devlink_net(devlink), net))
11372                         goto retry;
11373
11374                 WARN_ON(!(devlink->features & DEVLINK_F_RELOAD));
11375                 err = devlink_reload(devlink, &init_net,
11376                                      DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
11377                                      DEVLINK_RELOAD_LIMIT_UNSPEC,
11378                                      &actions_performed, NULL);
11379                 if (err && err != -EOPNOTSUPP)
11380                         pr_warn("Failed to reload devlink instance into init_net\n");
11381 retry:
11382                 devlink_put(devlink);
11383         }
11384         mutex_unlock(&devlink_mutex);
11385 }
11386
11387 static struct pernet_operations devlink_pernet_ops __net_initdata = {
11388         .pre_exit = devlink_pernet_pre_exit,
11389 };
11390
11391 static int __init devlink_init(void)
11392 {
11393         int err;
11394
11395         err = genl_register_family(&devlink_nl_family);
11396         if (err)
11397                 goto out;
11398         err = register_pernet_subsys(&devlink_pernet_ops);
11399
11400 out:
11401         WARN_ON(err);
11402         return err;
11403 }
11404
11405 subsys_initcall(devlink_init);