[linux.git] / net / devlink / core.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
 * Copyright (c) 2016 Jiri Pirko <[email protected]>
 */

#include <net/genetlink.h>
#define CREATE_TRACE_POINTS
#include <trace/events/devlink.h>

#include "devl_internal.h"

EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr);
EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report);

DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC);

static struct devlink *devlinks_xa_get(unsigned long index)
{
	struct devlink *devlink;

	rcu_read_lock();
	devlink = xa_find(&devlinks, &index, index, DEVLINK_REGISTERED);
	if (!devlink || !devlink_try_get(devlink))
		devlink = NULL;
	rcu_read_unlock();
	return devlink;
}

/* devlink_rels xarray contains 1:1 relationships between
 * devlink object and related nested devlink instance.
 * The xarray index is used to get the nested object from
 * the nested-in object code.
 */
static DEFINE_XARRAY_FLAGS(devlink_rels, XA_FLAGS_ALLOC1);

#define DEVLINK_REL_IN_USE XA_MARK_0

struct devlink_rel {
	u32 index;
	refcount_t refcount;
	u32 devlink_index;
	struct {
		u32 devlink_index;
		u32 obj_index;
		devlink_rel_notify_cb_t *notify_cb;
		devlink_rel_cleanup_cb_t *cleanup_cb;
		struct delayed_work notify_work;
	} nested_in;
};

static void devlink_rel_free(struct devlink_rel *rel)
{
	xa_erase(&devlink_rels, rel->index);
	kfree(rel);
}

static void __devlink_rel_get(struct devlink_rel *rel)
{
	refcount_inc(&rel->refcount);
}

static void __devlink_rel_put(struct devlink_rel *rel)
{
	if (refcount_dec_and_test(&rel->refcount))
		devlink_rel_free(rel);
}

static void devlink_rel_nested_in_notify_work(struct work_struct *work)
{
	struct devlink_rel *rel = container_of(work, struct devlink_rel,
					       nested_in.notify_work.work);
	struct devlink *devlink;

	devlink = devlinks_xa_get(rel->nested_in.devlink_index);
	if (!devlink)
		goto rel_put;
	if (!devl_trylock(devlink)) {
		devlink_put(devlink);
		goto reschedule_work;
	}
	if (!devl_is_registered(devlink)) {
		devl_unlock(devlink);
		devlink_put(devlink);
		goto rel_put;
	}
	if (!xa_get_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE))
		rel->nested_in.cleanup_cb(devlink, rel->nested_in.obj_index, rel->index);
	rel->nested_in.notify_cb(devlink, rel->nested_in.obj_index);
	devl_unlock(devlink);
	devlink_put(devlink);

rel_put:
	__devlink_rel_put(rel);
	return;

reschedule_work:
	schedule_delayed_work(&rel->nested_in.notify_work, 1);
}

static void devlink_rel_nested_in_notify_work_schedule(struct devlink_rel *rel)
{
	__devlink_rel_get(rel);
	schedule_delayed_work(&rel->nested_in.notify_work, 0);
}

static struct devlink_rel *devlink_rel_alloc(void)
{
	struct devlink_rel *rel;
	static u32 next;
	int err;

	rel = kzalloc(sizeof(*rel), GFP_KERNEL);
	if (!rel)
		return ERR_PTR(-ENOMEM);

	err = xa_alloc_cyclic(&devlink_rels, &rel->index, rel,
			      xa_limit_32b, &next, GFP_KERNEL);
	if (err) {
		kfree(rel);
		return ERR_PTR(err);
	}

	refcount_set(&rel->refcount, 1);
	INIT_DELAYED_WORK(&rel->nested_in.notify_work,
			  &devlink_rel_nested_in_notify_work);
	return rel;
}

static void devlink_rel_put(struct devlink *devlink)
{
	struct devlink_rel *rel = devlink->rel;

	if (!rel)
		return;
	xa_clear_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE);
	devlink_rel_nested_in_notify_work_schedule(rel);
	__devlink_rel_put(rel);
	devlink->rel = NULL;
}

void devlink_rel_nested_in_clear(u32 rel_index)
{
	xa_clear_mark(&devlink_rels, rel_index, DEVLINK_REL_IN_USE);
}

int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index,
			      u32 obj_index, devlink_rel_notify_cb_t *notify_cb,
			      devlink_rel_cleanup_cb_t *cleanup_cb,
			      struct devlink *devlink)
{
	struct devlink_rel *rel = devlink_rel_alloc();

	ASSERT_DEVLINK_NOT_REGISTERED(devlink);

	if (IS_ERR(rel))
		return PTR_ERR(rel);

	rel->devlink_index = devlink->index;
	rel->nested_in.devlink_index = devlink_index;
	rel->nested_in.obj_index = obj_index;
	rel->nested_in.notify_cb = notify_cb;
	rel->nested_in.cleanup_cb = cleanup_cb;
	*rel_index = rel->index;
	xa_set_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE);
	devlink->rel = rel;
	return 0;
}

/**
 * devlink_rel_nested_in_notify - Notify the object this devlink
 *				  instance is nested in.
 * @devlink: devlink
 *
 * This is called upon network namespace change of devlink instance.
 * In case this devlink instance is nested in another devlink object,
 * a notification of a change of this object should be sent
 * over netlink. The parent devlink instance lock needs to be
 * taken during the notification preparation.
 * However, since the devlink lock of nested instance is held here,
 * we would end with wrong devlink instance lock ordering and
 * deadlock. Therefore the work is utilized to avoid that.
 */
void devlink_rel_nested_in_notify(struct devlink *devlink)
{
	struct devlink_rel *rel = devlink->rel;

	if (!rel)
		return;
	devlink_rel_nested_in_notify_work_schedule(rel);
}

static struct devlink_rel *devlink_rel_find(unsigned long rel_index)
{
	return xa_find(&devlink_rels, &rel_index, rel_index,
		       DEVLINK_REL_IN_USE);
}

static struct devlink *devlink_rel_devlink_get(u32 rel_index)
{
	struct devlink_rel *rel;
	u32 devlink_index;

	if (!rel_index)
		return NULL;
	xa_lock(&devlink_rels);
	rel = devlink_rel_find(rel_index);
	if (rel)
		devlink_index = rel->devlink_index;
	xa_unlock(&devlink_rels);
	if (!rel)
		return NULL;
	return devlinks_xa_get(devlink_index);
}

int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink,
				   u32 rel_index, int attrtype,
				   bool *msg_updated)
{
	struct net *net = devlink_net(devlink);
	struct devlink *rel_devlink;
	int err;

	rel_devlink = devlink_rel_devlink_get(rel_index);
	if (!rel_devlink)
		return 0;
	err = devlink_nl_put_nested_handle(msg, net, rel_devlink, attrtype);
	devlink_put(rel_devlink);
	if (!err && msg_updated)
		*msg_updated = true;
	return err;
}

void *devlink_priv(struct devlink *devlink)
{
	return &devlink->priv;
}
EXPORT_SYMBOL_GPL(devlink_priv);

struct devlink *priv_to_devlink(void *priv)
{
	return container_of(priv, struct devlink, priv);
}
EXPORT_SYMBOL_GPL(priv_to_devlink);

struct device *devlink_to_dev(const struct devlink *devlink)
{
	return devlink->dev;
}
EXPORT_SYMBOL_GPL(devlink_to_dev);

struct net *devlink_net(const struct devlink *devlink)
{
	return read_pnet(&devlink->_net);
}
EXPORT_SYMBOL_GPL(devlink_net);

void devl_assert_locked(struct devlink *devlink)
{
	lockdep_assert_held(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_assert_locked);

#ifdef CONFIG_LOCKDEP
/* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */
bool devl_lock_is_held(struct devlink *devlink)
{
	return lockdep_is_held(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_lock_is_held);
#endif

void devl_lock(struct devlink *devlink)
{
	mutex_lock(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_lock);

int devl_trylock(struct devlink *devlink)
{
	return mutex_trylock(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_trylock);

void devl_unlock(struct devlink *devlink)
{
	mutex_unlock(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_unlock);

/**
 * devlink_try_get() - try to obtain a reference on a devlink instance
 * @devlink: instance to reference
 *
 * Obtain a reference on a devlink instance. A reference on a devlink instance
 * only implies that it's safe to take the instance lock. It does not imply
 * that the instance is registered, use devl_is_registered() after taking
 * the instance lock to check registration status.
 */
struct devlink *__must_check devlink_try_get(struct devlink *devlink)
{
	if (refcount_inc_not_zero(&devlink->refcount))
		return devlink;
	return NULL;
}

static void devlink_release(struct work_struct *work)
{
	struct devlink *devlink;

	devlink = container_of(to_rcu_work(work), struct devlink, rwork);

	mutex_destroy(&devlink->lock);
	lockdep_unregister_key(&devlink->lock_key);
	put_device(devlink->dev);
	kvfree(devlink);
}

void devlink_put(struct devlink *devlink)
{
	if (refcount_dec_and_test(&devlink->refcount))
		queue_rcu_work(system_wq, &devlink->rwork);
}

struct devlink *devlinks_xa_find_get(struct net *net, unsigned long *indexp)
{
	struct devlink *devlink = NULL;

	rcu_read_lock();
retry:
	devlink = xa_find(&devlinks, indexp, ULONG_MAX, DEVLINK_REGISTERED);
	if (!devlink)
		goto unlock;

	if (!devlink_try_get(devlink))
		goto next;
	if (!net_eq(devlink_net(devlink), net)) {
		devlink_put(devlink);
		goto next;
	}
unlock:
	rcu_read_unlock();
	return devlink;

next:
	(*indexp)++;
	goto retry;
}

/**
 * devl_register - Register devlink instance
 * @devlink: devlink
 */
int devl_register(struct devlink *devlink)
{
	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
	devl_assert_locked(devlink);

	xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
	devlink_notify_register(devlink);
	devlink_rel_nested_in_notify(devlink);

	return 0;
}
EXPORT_SYMBOL_GPL(devl_register);

void devlink_register(struct devlink *devlink)
{
	devl_lock(devlink);
	devl_register(devlink);
	devl_unlock(devlink);
}
EXPORT_SYMBOL_GPL(devlink_register);

/**
 * devl_unregister - Unregister devlink instance
 * @devlink: devlink
 */
void devl_unregister(struct devlink *devlink)
{
	ASSERT_DEVLINK_REGISTERED(devlink);
	devl_assert_locked(devlink);

	devlink_notify_unregister(devlink);
	xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
	devlink_rel_put(devlink);
}
EXPORT_SYMBOL_GPL(devl_unregister);

void devlink_unregister(struct devlink *devlink)
{
	devl_lock(devlink);
	devl_unregister(devlink);
	devl_unlock(devlink);
}
EXPORT_SYMBOL_GPL(devlink_unregister);

/**
 *	devlink_alloc_ns - Allocate new devlink instance resources
 *	in specific namespace
 *
 *	@ops: ops
 *	@priv_size: size of user private data
 *	@net: net namespace
 *	@dev: parent device
 *
 *	Allocate new devlink instance resources, including devlink index
 *	and name.
 */
struct devlink *devlink_alloc_ns(const struct devlink_ops *ops,
				 size_t priv_size, struct net *net,
				 struct device *dev)
{
	struct devlink *devlink;
	static u32 last_id;
	int ret;

	WARN_ON(!ops || !dev);
	if (!devlink_reload_actions_valid(ops))
		return NULL;

	devlink = kvzalloc(struct_size(devlink, priv, priv_size), GFP_KERNEL);
	if (!devlink)
		return NULL;

	ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b,
			      &last_id, GFP_KERNEL);
	if (ret < 0)
		goto err_xa_alloc;

	devlink->dev = get_device(dev);
	devlink->ops = ops;
	xa_init_flags(&devlink->ports, XA_FLAGS_ALLOC);
	xa_init_flags(&devlink->params, XA_FLAGS_ALLOC);
	xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC);
	xa_init_flags(&devlink->nested_rels, XA_FLAGS_ALLOC);
	write_pnet(&devlink->_net, net);
	INIT_LIST_HEAD(&devlink->rate_list);
	INIT_LIST_HEAD(&devlink->linecard_list);
	INIT_LIST_HEAD(&devlink->sb_list);
	INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
	INIT_LIST_HEAD(&devlink->resource_list);
	INIT_LIST_HEAD(&devlink->region_list);
	INIT_LIST_HEAD(&devlink->reporter_list);
	INIT_LIST_HEAD(&devlink->trap_list);
	INIT_LIST_HEAD(&devlink->trap_group_list);
	INIT_LIST_HEAD(&devlink->trap_policer_list);
	INIT_RCU_WORK(&devlink->rwork, devlink_release);
	lockdep_register_key(&devlink->lock_key);
	mutex_init(&devlink->lock);
	lockdep_set_class(&devlink->lock, &devlink->lock_key);
	refcount_set(&devlink->refcount, 1);

	return devlink;

err_xa_alloc:
	kvfree(devlink);
	return NULL;
}
EXPORT_SYMBOL_GPL(devlink_alloc_ns);

/**
 *	devlink_free - Free devlink instance resources
 *
 *	@devlink: devlink
 */
void devlink_free(struct devlink *devlink)
{
	ASSERT_DEVLINK_NOT_REGISTERED(devlink);

	WARN_ON(!list_empty(&devlink->trap_policer_list));
	WARN_ON(!list_empty(&devlink->trap_group_list));
	WARN_ON(!list_empty(&devlink->trap_list));
	WARN_ON(!list_empty(&devlink->reporter_list));
	WARN_ON(!list_empty(&devlink->region_list));
	WARN_ON(!list_empty(&devlink->resource_list));
	WARN_ON(!list_empty(&devlink->dpipe_table_list));
	WARN_ON(!list_empty(&devlink->sb_list));
	WARN_ON(!list_empty(&devlink->rate_list));
	WARN_ON(!list_empty(&devlink->linecard_list));
	WARN_ON(!xa_empty(&devlink->ports));

	xa_destroy(&devlink->nested_rels);
	xa_destroy(&devlink->snapshot_ids);
	xa_destroy(&devlink->params);
	xa_destroy(&devlink->ports);

	xa_erase(&devlinks, devlink->index);

	devlink_put(devlink);
}
EXPORT_SYMBOL_GPL(devlink_free);

static void __net_exit devlink_pernet_pre_exit(struct net *net)
{
	struct devlink *devlink;
	u32 actions_performed;
	unsigned long index;
	int err;

	/* In case network namespace is getting destroyed, reload
	 * all devlink instances from this namespace into init_net.
	 */
	devlinks_xa_for_each_registered_get(net, index, devlink) {
		devl_dev_lock(devlink, true);
		err = 0;
		if (devl_is_registered(devlink))
			err = devlink_reload(devlink, &init_net,
					     DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
					     DEVLINK_RELOAD_LIMIT_UNSPEC,
					     &actions_performed, NULL);
		devl_dev_unlock(devlink, true);
		devlink_put(devlink);
		if (err && err != -EOPNOTSUPP)
			pr_warn("Failed to reload devlink instance into init_net\n");
	}
}

static struct pernet_operations devlink_pernet_ops __net_initdata = {
	.pre_exit = devlink_pernet_pre_exit,
};

static struct notifier_block devlink_port_netdevice_nb = {
	.notifier_call = devlink_port_netdevice_event,
};

static int __init devlink_init(void)
{
	int err;

	err = register_pernet_subsys(&devlink_pernet_ops);
	if (err)
		goto out;
	err = genl_register_family(&devlink_nl_family);
	if (err)
		goto out_unreg_pernet_subsys;
	err = register_netdevice_notifier(&devlink_port_netdevice_nb);
	if (!err)
		return 0;

	genl_unregister_family(&devlink_nl_family);

out_unreg_pernet_subsys:
	unregister_pernet_subsys(&devlink_pernet_ops);
out:
	WARN_ON(err);
	return err;
}

subsys_initcall(devlink_init);
Commit	Line	Data
687125b5 JK	1	// SPDX-License-Identifier: GPL-2.0-or-later
	2	/*
	3	* Copyright (c) 2016 Mellanox Technologies. All rights reserved.
	4	* Copyright (c) 2016 Jiri Pirko <[email protected]>
	5	*/
	6
	7	#include <net/genetlink.h>
890c5566 JP	8	#define CREATE_TRACE_POINTS
890c5566 JP	9	#include <trace/events/devlink.h>
687125b5 JK	10
	11	#include "devl_internal.h"
	12
890c5566 JP	13	EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
	14	EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr);
	15	EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report);
	16
687125b5 JK	17	DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC);
687125b5 JK	18
c137743b JP	19	static struct devlink *devlinks_xa_get(unsigned long index)
	20	{
	21	struct devlink *devlink;
	22
	23	rcu_read_lock();
	24	devlink = xa_find(&devlinks, &index, index, DEVLINK_REGISTERED);
	25	if (!devlink \|\| !devlink_try_get(devlink))
	26	devlink = NULL;
	27	rcu_read_unlock();
	28	return devlink;
	29	}
	30
	31	/* devlink_rels xarray contains 1:1 relationships between
	32	* devlink object and related nested devlink instance.
	33	* The xarray index is used to get the nested object from
	34	* the nested-in object code.
	35	*/
	36	static DEFINE_XARRAY_FLAGS(devlink_rels, XA_FLAGS_ALLOC1);
	37
	38	#define DEVLINK_REL_IN_USE XA_MARK_0
	39
	40	struct devlink_rel {
	41	u32 index;
	42	refcount_t refcount;
	43	u32 devlink_index;
	44	struct {
	45	u32 devlink_index;
	46	u32 obj_index;
	47	devlink_rel_notify_cb_t *notify_cb;
	48	devlink_rel_cleanup_cb_t *cleanup_cb;
58086721	49	struct delayed_work notify_work;
c137743b JP	50	} nested_in;
	51	};
	52
	53	static void devlink_rel_free(struct devlink_rel *rel)
	54	{
	55	xa_erase(&devlink_rels, rel->index);
	56	kfree(rel);
	57	}
	58
	59	static void __devlink_rel_get(struct devlink_rel *rel)
	60	{
	61	refcount_inc(&rel->refcount);
	62	}
	63
	64	static void __devlink_rel_put(struct devlink_rel *rel)
	65	{
	66	if (refcount_dec_and_test(&rel->refcount))
	67	devlink_rel_free(rel);
	68	}
	69
	70	static void devlink_rel_nested_in_notify_work(struct work_struct *work)
	71	{
	72	struct devlink_rel *rel = container_of(work, struct devlink_rel,
58086721	73	nested_in.notify_work.work);
c137743b JP	74	struct devlink *devlink;
	75
	76	devlink = devlinks_xa_get(rel->nested_in.devlink_index);
	77	if (!devlink)
	78	goto rel_put;
	79	if (!devl_trylock(devlink)) {
	80	devlink_put(devlink);
	81	goto reschedule_work;
	82	}
	83	if (!devl_is_registered(devlink)) {
	84	devl_unlock(devlink);
	85	devlink_put(devlink);
	86	goto rel_put;
	87	}
	88	if (!xa_get_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE))
	89	rel->nested_in.cleanup_cb(devlink, rel->nested_in.obj_index, rel->index);
	90	rel->nested_in.notify_cb(devlink, rel->nested_in.obj_index);
	91	devl_unlock(devlink);
	92	devlink_put(devlink);
	93
	94	rel_put:
	95	__devlink_rel_put(rel);
	96	return;
	97
	98	reschedule_work:
58086721	99	schedule_delayed_work(&rel->nested_in.notify_work, 1);
c137743b JP	100	}
	101
	102	static void devlink_rel_nested_in_notify_work_schedule(struct devlink_rel *rel)
	103	{
	104	__devlink_rel_get(rel);
58086721	105	schedule_delayed_work(&rel->nested_in.notify_work, 0);
c137743b JP	106	}
	107
	108	static struct devlink_rel *devlink_rel_alloc(void)
	109	{
	110	struct devlink_rel *rel;
	111	static u32 next;
	112	int err;
	113
	114	rel = kzalloc(sizeof(*rel), GFP_KERNEL);
	115	if (!rel)
	116	return ERR_PTR(-ENOMEM);
	117
	118	err = xa_alloc_cyclic(&devlink_rels, &rel->index, rel,
	119	xa_limit_32b, &next, GFP_KERNEL);
	120	if (err) {
	121	kfree(rel);
	122	return ERR_PTR(err);
	123	}
	124
	125	refcount_set(&rel->refcount, 1);
58086721 JP	126	INIT_DELAYED_WORK(&rel->nested_in.notify_work,
58086721 JP	127	&devlink_rel_nested_in_notify_work);
c137743b JP	128	return rel;
	129	}
	130
	131	static void devlink_rel_put(struct devlink *devlink)
	132	{
	133	struct devlink_rel *rel = devlink->rel;
	134
	135	if (!rel)
	136	return;
	137	xa_clear_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE);
	138	devlink_rel_nested_in_notify_work_schedule(rel);
	139	__devlink_rel_put(rel);
	140	devlink->rel = NULL;
	141	}
	142
	143	void devlink_rel_nested_in_clear(u32 rel_index)
	144	{
	145	xa_clear_mark(&devlink_rels, rel_index, DEVLINK_REL_IN_USE);
	146	}
	147
	148	int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index,
	149	u32 obj_index, devlink_rel_notify_cb_t *notify_cb,
	150	devlink_rel_cleanup_cb_t *cleanup_cb,
	151	struct devlink *devlink)
	152	{
	153	struct devlink_rel *rel = devlink_rel_alloc();
	154
	155	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
	156
	157	if (IS_ERR(rel))
	158	return PTR_ERR(rel);
	159
	160	rel->devlink_index = devlink->index;
	161	rel->nested_in.devlink_index = devlink_index;
	162	rel->nested_in.obj_index = obj_index;
	163	rel->nested_in.notify_cb = notify_cb;
	164	rel->nested_in.cleanup_cb = cleanup_cb;
	165	*rel_index = rel->index;
	166	xa_set_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE);
	167	devlink->rel = rel;
	168	return 0;
	169	}
	170
5d77371e JP	171	/**
	172	* devlink_rel_nested_in_notify - Notify the object this devlink
	173	* instance is nested in.
	174	* @devlink: devlink
	175	*
	176	* This is called upon network namespace change of devlink instance.
	177	* In case this devlink instance is nested in another devlink object,
	178	* a notification of a change of this object should be sent
	179	* over netlink. The parent devlink instance lock needs to be
	180	* taken during the notification preparation.
	181	* However, since the devlink lock of nested instance is held here,
	182	* we would end with wrong devlink instance lock ordering and
	183	* deadlock. Therefore the work is utilized to avoid that.
	184	*/
c137743b JP	185	void devlink_rel_nested_in_notify(struct devlink *devlink)
	186	{
	187	struct devlink_rel *rel = devlink->rel;
	188
	189	if (!rel)
	190	return;
	191	devlink_rel_nested_in_notify_work_schedule(rel);
	192	}
	193
	194	static struct devlink_rel *devlink_rel_find(unsigned long rel_index)
	195	{
	196	return xa_find(&devlink_rels, &rel_index, rel_index,
	197	DEVLINK_REL_IN_USE);
	198	}
	199
b5f4e371	200	static struct devlink *devlink_rel_devlink_get(u32 rel_index)
c137743b	201	{
c137743b JP	202	struct devlink_rel *rel;
	203	u32 devlink_index;
	204
	205	if (!rel_index)
	206	return NULL;
	207	xa_lock(&devlink_rels);
	208	rel = devlink_rel_find(rel_index);
	209	if (rel)
	210	devlink_index = rel->devlink_index;
	211	xa_unlock(&devlink_rels);
	212	if (!rel)
	213	return NULL;
b5f4e371	214	return devlinks_xa_get(devlink_index);
c137743b JP	215	}
	216
	217	int devlink_rel_devlink_handle_put(struct sk_buff msg, struct devlink devlink,
	218	u32 rel_index, int attrtype,
	219	bool *msg_updated)
	220	{
	221	struct net *net = devlink_net(devlink);
	222	struct devlink *rel_devlink;
	223	int err;
	224
b5f4e371	225	rel_devlink = devlink_rel_devlink_get(rel_index);
c137743b JP	226	if (!rel_devlink)
	227	return 0;
	228	err = devlink_nl_put_nested_handle(msg, net, rel_devlink, attrtype);
c137743b JP	229	devlink_put(rel_devlink);
	230	if (!err && msg_updated)
	231	*msg_updated = true;
	232	return err;
	233	}
	234
687125b5 JK	235	void devlink_priv(struct devlink devlink)
	236	{
	237	return &devlink->priv;
	238	}
	239	EXPORT_SYMBOL_GPL(devlink_priv);
	240
	241	struct devlink priv_to_devlink(void priv)
	242	{
	243	return container_of(priv, struct devlink, priv);
	244	}
	245	EXPORT_SYMBOL_GPL(priv_to_devlink);
	246
	247	struct device devlink_to_dev(const struct devlink devlink)
	248	{
	249	return devlink->dev;
	250	}
	251	EXPORT_SYMBOL_GPL(devlink_to_dev);
	252
	253	struct net devlink_net(const struct devlink devlink)
	254	{
	255	return read_pnet(&devlink->_net);
	256	}
	257	EXPORT_SYMBOL_GPL(devlink_net);
	258
	259	void devl_assert_locked(struct devlink *devlink)
	260	{
	261	lockdep_assert_held(&devlink->lock);
	262	}
	263	EXPORT_SYMBOL_GPL(devl_assert_locked);
	264
	265	#ifdef CONFIG_LOCKDEP
	266	/* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */
	267	bool devl_lock_is_held(struct devlink *devlink)
	268	{
	269	return lockdep_is_held(&devlink->lock);
	270	}
	271	EXPORT_SYMBOL_GPL(devl_lock_is_held);
	272	#endif
	273
	274	void devl_lock(struct devlink *devlink)
	275	{
	276	mutex_lock(&devlink->lock);
	277	}
	278	EXPORT_SYMBOL_GPL(devl_lock);
	279
	280	int devl_trylock(struct devlink *devlink)
	281	{
	282	return mutex_trylock(&devlink->lock);
	283	}
	284	EXPORT_SYMBOL_GPL(devl_trylock);
	285
	286	void devl_unlock(struct devlink *devlink)
	287	{
	288	mutex_unlock(&devlink->lock);
	289	}
	290	EXPORT_SYMBOL_GPL(devl_unlock);
	291
ed539ba6 JK	292	/**
	293	* devlink_try_get() - try to obtain a reference on a devlink instance
	294	* @devlink: instance to reference
	295	*
	296	* Obtain a reference on a devlink instance. A reference on a devlink instance
	297	* only implies that it's safe to take the instance lock. It does not imply
	298	* that the instance is registered, use devl_is_registered() after taking
	299	* the instance lock to check registration status.
	300	*/
687125b5 JK	301	struct devlink __must_check devlink_try_get(struct devlink devlink)
	302	{
	303	if (refcount_inc_not_zero(&devlink->refcount))
	304	return devlink;
	305	return NULL;
	306	}
	307
93e71edf JK	308	static void devlink_release(struct work_struct *work)
	309	{
	310	struct devlink *devlink;
	311
	312	devlink = container_of(to_rcu_work(work), struct devlink, rwork);
	313
	314	mutex_destroy(&devlink->lock);
	315	lockdep_unregister_key(&devlink->lock_key);
a3806872	316	put_device(devlink->dev);
730fffce	317	kvfree(devlink);
93e71edf JK	318	}
93e71edf JK	319
687125b5 JK	320	void devlink_put(struct devlink *devlink)
	321	{
	322	if (refcount_dec_and_test(&devlink->refcount))
93e71edf	323	queue_rcu_work(system_wq, &devlink->rwork);
687125b5 JK	324	}
687125b5 JK	325
d7727819	326	struct devlink devlinks_xa_find_get(struct net net, unsigned long *indexp)
687125b5	327	{
d7727819	328	struct devlink *devlink = NULL;
687125b5 JK	329
	330	rcu_read_lock();
	331	retry:
d7727819	332	devlink = xa_find(&devlinks, indexp, ULONG_MAX, DEVLINK_REGISTERED);
687125b5 JK	333	if (!devlink)
	334	goto unlock;
	335
687125b5	336	if (!devlink_try_get(devlink))
d7727819	337	goto next;
687125b5 JK	338	if (!net_eq(devlink_net(devlink), net)) {
687125b5 JK	339	devlink_put(devlink);
d7727819	340	goto next;
687125b5 JK	341	}
	342	unlock:
	343	rcu_read_unlock();
	344	return devlink;
687125b5	345
d7727819 JK	346	next:
	347	(*indexp)++;
	348	goto retry;
687125b5 JK	349	}
687125b5 JK	350
687125b5	351	/**
9053637e JK	352	* devl_register - Register devlink instance
9053637e JK	353	* @devlink: devlink
687125b5	354	*/
9053637e	355	int devl_register(struct devlink *devlink)
687125b5 JK	356	{
687125b5 JK	357	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
9053637e	358	devl_assert_locked(devlink);
687125b5 JK	359
	360	xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
	361	devlink_notify_register(devlink);
c137743b	362	devlink_rel_nested_in_notify(devlink);
9053637e JK	363
	364	return 0;
	365	}
	366	EXPORT_SYMBOL_GPL(devl_register);
	367
	368	void devlink_register(struct devlink *devlink)
	369	{
	370	devl_lock(devlink);
	371	devl_register(devlink);
	372	devl_unlock(devlink);
687125b5 JK	373	}
	374	EXPORT_SYMBOL_GPL(devlink_register);
	375
	376	/**
9053637e JK	377	* devl_unregister - Unregister devlink instance
9053637e JK	378	* @devlink: devlink
687125b5	379	*/
9053637e	380	void devl_unregister(struct devlink *devlink)
687125b5 JK	381	{
687125b5 JK	382	ASSERT_DEVLINK_REGISTERED(devlink);
9053637e	383	devl_assert_locked(devlink);
687125b5 JK	384
	385	devlink_notify_unregister(devlink);
	386	xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
c137743b	387	devlink_rel_put(devlink);
9053637e JK	388	}
	389	EXPORT_SYMBOL_GPL(devl_unregister);
	390
	391	void devlink_unregister(struct devlink *devlink)
	392	{
	393	devl_lock(devlink);
	394	devl_unregister(devlink);
	395	devl_unlock(devlink);
687125b5 JK	396	}
	397	EXPORT_SYMBOL_GPL(devlink_unregister);
	398
	399	/**
	400	* devlink_alloc_ns - Allocate new devlink instance resources
	401	* in specific namespace
	402	*
	403	* @ops: ops
	404	* @priv_size: size of user private data
	405	* @net: net namespace
	406	* @dev: parent device
	407	*
	408	* Allocate new devlink instance resources, including devlink index
	409	* and name.
	410	*/
	411	struct devlink devlink_alloc_ns(const struct devlink_ops ops,
	412	size_t priv_size, struct net *net,
	413	struct device *dev)
	414	{
	415	struct devlink *devlink;
	416	static u32 last_id;
	417	int ret;
	418
	419	WARN_ON(!ops \|\| !dev);
	420	if (!devlink_reload_actions_valid(ops))
	421	return NULL;
	422
730fffce	423	devlink = kvzalloc(struct_size(devlink, priv, priv_size), GFP_KERNEL);
687125b5 JK	424	if (!devlink)
	425	return NULL;
	426
	427	ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b,
	428	&last_id, GFP_KERNEL);
	429	if (ret < 0)
	430	goto err_xa_alloc;
	431
a3806872	432	devlink->dev = get_device(dev);
687125b5 JK	433	devlink->ops = ops;
687125b5 JK	434	xa_init_flags(&devlink->ports, XA_FLAGS_ALLOC);
a72e17b4	435	xa_init_flags(&devlink->params, XA_FLAGS_ALLOC);
687125b5	436	xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC);
c5e1bf8a	437	xa_init_flags(&devlink->nested_rels, XA_FLAGS_ALLOC);
687125b5 JK	438	write_pnet(&devlink->_net, net);
	439	INIT_LIST_HEAD(&devlink->rate_list);
	440	INIT_LIST_HEAD(&devlink->linecard_list);
	441	INIT_LIST_HEAD(&devlink->sb_list);
	442	INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
	443	INIT_LIST_HEAD(&devlink->resource_list);
687125b5 JK	444	INIT_LIST_HEAD(&devlink->region_list);
	445	INIT_LIST_HEAD(&devlink->reporter_list);
	446	INIT_LIST_HEAD(&devlink->trap_list);
	447	INIT_LIST_HEAD(&devlink->trap_group_list);
	448	INIT_LIST_HEAD(&devlink->trap_policer_list);
93e71edf	449	INIT_RCU_WORK(&devlink->rwork, devlink_release);
687125b5 JK	450	lockdep_register_key(&devlink->lock_key);
	451	mutex_init(&devlink->lock);
	452	lockdep_set_class(&devlink->lock, &devlink->lock_key);
687125b5	453	refcount_set(&devlink->refcount, 1);
687125b5 JK	454
	455	return devlink;
	456
687125b5	457	err_xa_alloc:
730fffce	458	kvfree(devlink);
687125b5 JK	459	return NULL;
	460	}
	461	EXPORT_SYMBOL_GPL(devlink_alloc_ns);
	462
	463	/**
	464	* devlink_free - Free devlink instance resources
	465	*
	466	* @devlink: devlink
	467	*/
	468	void devlink_free(struct devlink *devlink)
	469	{
	470	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
	471
687125b5 JK	472	WARN_ON(!list_empty(&devlink->trap_policer_list));
	473	WARN_ON(!list_empty(&devlink->trap_group_list));
	474	WARN_ON(!list_empty(&devlink->trap_list));
	475	WARN_ON(!list_empty(&devlink->reporter_list));
	476	WARN_ON(!list_empty(&devlink->region_list));
687125b5 JK	477	WARN_ON(!list_empty(&devlink->resource_list));
	478	WARN_ON(!list_empty(&devlink->dpipe_table_list));
	479	WARN_ON(!list_empty(&devlink->sb_list));
	480	WARN_ON(!list_empty(&devlink->rate_list));
	481	WARN_ON(!list_empty(&devlink->linecard_list));
	482	WARN_ON(!xa_empty(&devlink->ports));
	483
c5e1bf8a	484	xa_destroy(&devlink->nested_rels);
687125b5	485	xa_destroy(&devlink->snapshot_ids);
a72e17b4	486	xa_destroy(&devlink->params);
687125b5 JK	487	xa_destroy(&devlink->ports);
687125b5 JK	488
687125b5 JK	489	xa_erase(&devlinks, devlink->index);
687125b5 JK	490
9053637e	491	devlink_put(devlink);
687125b5 JK	492	}
	493	EXPORT_SYMBOL_GPL(devlink_free);
	494
	495	static void __net_exit devlink_pernet_pre_exit(struct net *net)
	496	{
	497	struct devlink *devlink;
	498	u32 actions_performed;
	499	unsigned long index;
	500	int err;
	501
	502	/* In case network namespace is getting destroyed, reload
	503	* all devlink instances from this namespace into init_net.
	504	*/
	505	devlinks_xa_for_each_registered_get(net, index, devlink) {
e21c52d7	506	devl_dev_lock(devlink, true);
ed539ba6 JK	507	err = 0;
	508	if (devl_is_registered(devlink))
	509	err = devlink_reload(devlink, &init_net,
	510	DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
	511	DEVLINK_RELOAD_LIMIT_UNSPEC,
	512	&actions_performed, NULL);
e21c52d7	513	devl_dev_unlock(devlink, true);
7a54a519	514	devlink_put(devlink);
687125b5 JK	515	if (err && err != -EOPNOTSUPP)
687125b5 JK	516	pr_warn("Failed to reload devlink instance into init_net\n");
687125b5 JK	517	}
	518	}
	519
	520	static struct pernet_operations devlink_pernet_ops __net_initdata = {
	521	.pre_exit = devlink_pernet_pre_exit,
	522	};
	523
d6352dae	524	static struct notifier_block devlink_port_netdevice_nb = {
e93c9378 JP	525	.notifier_call = devlink_port_netdevice_event,
	526	};
	527
687125b5 JK	528	static int __init devlink_init(void)
	529	{
	530	int err;
	531
687125b5	532	err = register_pernet_subsys(&devlink_pernet_ops);
e93c9378 JP	533	if (err)
e93c9378 JP	534	goto out;
def689fc VK	535	err = genl_register_family(&devlink_nl_family);
	536	if (err)
	537	goto out_unreg_pernet_subsys;
e93c9378	538	err = register_netdevice_notifier(&devlink_port_netdevice_nb);
def689fc VK	539	if (!err)
	540	return 0;
	541
	542	genl_unregister_family(&devlink_nl_family);
687125b5	543
def689fc VK	544	out_unreg_pernet_subsys:
def689fc VK	545	unregister_pernet_subsys(&devlink_pernet_ops);
687125b5 JK	546	out:
	547	WARN_ON(err);
	548	return err;
	549	}
	550
	551	subsys_initcall(devlink_init);