[linux.git] / drivers / net / netdevsim / bpf.c

/*
 * Copyright (C) 2017 Netronome Systems, Inc.
 *
 * This software is licensed under the GNU General License Version 2,
 * June 1991 as shown in the file COPYING in the top-level directory of this
 * source tree.
 *
 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
 */

#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/debugfs.h>
#include <linux/kernel.h>
#include <linux/mutex.h>
#include <linux/rtnetlink.h>
#include <net/pkt_cls.h>

#include "netdevsim.h"

#define pr_vlog(env, fmt, ...)	\
	bpf_verifier_log_write(env, "[netdevsim] " fmt, ##__VA_ARGS__)

struct nsim_bpf_bound_prog {
	struct netdevsim *ns;
	struct bpf_prog *prog;
	struct dentry *ddir;
	const char *state;
	bool is_loaded;
	struct list_head l;
};

#define NSIM_BPF_MAX_KEYS		2

struct nsim_bpf_bound_map {
	struct netdevsim *ns;
	struct bpf_offloaded_map *map;
	struct mutex mutex;
	struct nsim_map_entry {
		void *key;
		void *value;
	} entry[NSIM_BPF_MAX_KEYS];
	struct list_head l;
};

static int nsim_debugfs_bpf_string_read(struct seq_file *file, void *data)
{
	const char **str = file->private;

	if (*str)
		seq_printf(file, "%s\n", *str);

	return 0;
}

static int nsim_debugfs_bpf_string_open(struct inode *inode, struct file *f)
{
	return single_open(f, nsim_debugfs_bpf_string_read, inode->i_private);
}

static const struct file_operations nsim_bpf_string_fops = {
	.owner = THIS_MODULE,
	.open = nsim_debugfs_bpf_string_open,
	.release = single_release,
	.read = seq_read,
	.llseek = seq_lseek
};

static int
nsim_bpf_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn)
{
	struct nsim_bpf_bound_prog *state;

	state = env->prog->aux->offload->dev_priv;
	if (state->ns->bpf_bind_verifier_delay && !insn_idx)
		msleep(state->ns->bpf_bind_verifier_delay);

	if (insn_idx == env->prog->len - 1)
		pr_vlog(env, "Hello from netdevsim!\n");

	return 0;
}

static int nsim_bpf_finalize(struct bpf_verifier_env *env)
{
	return 0;
}

static const struct bpf_prog_offload_ops nsim_bpf_analyzer_ops = {
	.insn_hook	= nsim_bpf_verify_insn,
	.finalize	= nsim_bpf_finalize,
};

static bool nsim_xdp_offload_active(struct netdevsim *ns)
{
	return ns->xdp_hw.prog;
}

static void nsim_prog_set_loaded(struct bpf_prog *prog, bool loaded)
{
	struct nsim_bpf_bound_prog *state;

	if (!prog || !prog->aux->offload)
		return;

	state = prog->aux->offload->dev_priv;
	state->is_loaded = loaded;
}

static int
nsim_bpf_offload(struct netdevsim *ns, struct bpf_prog *prog, bool oldprog)
{
	nsim_prog_set_loaded(ns->bpf_offloaded, false);

	WARN(!!ns->bpf_offloaded != oldprog,
	     "bad offload state, expected offload %sto be active",
	     oldprog ? "" : "not ");
	ns->bpf_offloaded = prog;
	ns->bpf_offloaded_id = prog ? prog->aux->id : 0;
	nsim_prog_set_loaded(prog, true);

	return 0;
}

int nsim_bpf_setup_tc_block_cb(enum tc_setup_type type,
			       void *type_data, void *cb_priv)
{
	struct tc_cls_bpf_offload *cls_bpf = type_data;
	struct bpf_prog *prog = cls_bpf->prog;
	struct netdevsim *ns = cb_priv;
	struct bpf_prog *oldprog;

	if (type != TC_SETUP_CLSBPF) {
		NSIM_EA(cls_bpf->common.extack,
			"only offload of BPF classifiers supported");
		return -EOPNOTSUPP;
	}

	if (!tc_cls_can_offload_and_chain0(ns->netdev, &cls_bpf->common))
		return -EOPNOTSUPP;

	if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
		NSIM_EA(cls_bpf->common.extack,
			"only ETH_P_ALL supported as filter protocol");
		return -EOPNOTSUPP;
	}

	if (!ns->bpf_tc_accept) {
		NSIM_EA(cls_bpf->common.extack,
			"netdevsim configured to reject BPF TC offload");
		return -EOPNOTSUPP;
	}
	/* Note: progs without skip_sw will probably not be dev bound */
	if (prog && !prog->aux->offload && !ns->bpf_tc_non_bound_accept) {
		NSIM_EA(cls_bpf->common.extack,
			"netdevsim configured to reject unbound programs");
		return -EOPNOTSUPP;
	}

	if (cls_bpf->command != TC_CLSBPF_OFFLOAD)
		return -EOPNOTSUPP;

	oldprog = cls_bpf->oldprog;

	/* Don't remove if oldprog doesn't match driver's state */
	if (ns->bpf_offloaded != oldprog) {
		oldprog = NULL;
		if (!cls_bpf->prog)
			return 0;
		if (ns->bpf_offloaded) {
			NSIM_EA(cls_bpf->common.extack,
				"driver and netdev offload states mismatch");
			return -EBUSY;
		}
	}

	return nsim_bpf_offload(ns, cls_bpf->prog, oldprog);
}

int nsim_bpf_disable_tc(struct netdevsim *ns)
{
	if (ns->bpf_offloaded && !nsim_xdp_offload_active(ns))
		return -EBUSY;
	return 0;
}

static int nsim_xdp_offload_prog(struct netdevsim *ns, struct netdev_bpf *bpf)
{
	if (!nsim_xdp_offload_active(ns) && !bpf->prog)
		return 0;
	if (!nsim_xdp_offload_active(ns) && bpf->prog && ns->bpf_offloaded) {
		NSIM_EA(bpf->extack, "TC program is already loaded");
		return -EBUSY;
	}

	return nsim_bpf_offload(ns, bpf->prog, nsim_xdp_offload_active(ns));
}

static int
nsim_xdp_set_prog(struct netdevsim *ns, struct netdev_bpf *bpf,
		  struct xdp_attachment_info *xdp)
{
	int err;

	if (!xdp_attachment_flags_ok(xdp, bpf))
		return -EBUSY;

	if (bpf->command == XDP_SETUP_PROG && !ns->bpf_xdpdrv_accept) {
		NSIM_EA(bpf->extack, "driver XDP disabled in DebugFS");
		return -EOPNOTSUPP;
	}
	if (bpf->command == XDP_SETUP_PROG_HW && !ns->bpf_xdpoffload_accept) {
		NSIM_EA(bpf->extack, "XDP offload disabled in DebugFS");
		return -EOPNOTSUPP;
	}

	if (bpf->command == XDP_SETUP_PROG_HW) {
		err = nsim_xdp_offload_prog(ns, bpf);
		if (err)
			return err;
	}

	xdp_attachment_setup(xdp, bpf);

	return 0;
}

static int nsim_bpf_create_prog(struct netdevsim *ns, struct bpf_prog *prog)
{
	struct nsim_bpf_bound_prog *state;
	char name[16];

	state = kzalloc(sizeof(*state), GFP_KERNEL);
	if (!state)
		return -ENOMEM;

	state->ns = ns;
	state->prog = prog;
	state->state = "verify";

	/* Program id is not populated yet when we create the state. */
	sprintf(name, "%u", ns->sdev->prog_id_gen++);
	state->ddir = debugfs_create_dir(name, ns->sdev->ddir_bpf_bound_progs);
	if (IS_ERR_OR_NULL(state->ddir)) {
		kfree(state);
		return -ENOMEM;
	}

	debugfs_create_u32("id", 0400, state->ddir, &prog->aux->id);
	debugfs_create_file("state", 0400, state->ddir,
			    &state->state, &nsim_bpf_string_fops);
	debugfs_create_bool("loaded", 0400, state->ddir, &state->is_loaded);

	list_add_tail(&state->l, &ns->sdev->bpf_bound_progs);

	prog->aux->offload->dev_priv = state;

	return 0;
}

static void nsim_bpf_destroy_prog(struct bpf_prog *prog)
{
	struct nsim_bpf_bound_prog *state;

	state = prog->aux->offload->dev_priv;
	WARN(state->is_loaded,
	     "offload state destroyed while program still bound");
	debugfs_remove_recursive(state->ddir);
	list_del(&state->l);
	kfree(state);
}

static int nsim_setup_prog_checks(struct netdevsim *ns, struct netdev_bpf *bpf)
{
	if (bpf->prog && bpf->prog->aux->offload) {
		NSIM_EA(bpf->extack, "attempt to load offloaded prog to drv");
		return -EINVAL;
	}
	if (ns->netdev->mtu > NSIM_XDP_MAX_MTU) {
		NSIM_EA(bpf->extack, "MTU too large w/ XDP enabled");
		return -EINVAL;
	}
	return 0;
}

static int
nsim_setup_prog_hw_checks(struct netdevsim *ns, struct netdev_bpf *bpf)
{
	struct nsim_bpf_bound_prog *state;

	if (!bpf->prog)
		return 0;

	if (!bpf->prog->aux->offload) {
		NSIM_EA(bpf->extack, "xdpoffload of non-bound program");
		return -EINVAL;
	}
	if (!bpf_offload_dev_match(bpf->prog, ns->netdev)) {
		NSIM_EA(bpf->extack, "program bound to different dev");
		return -EINVAL;
	}

	state = bpf->prog->aux->offload->dev_priv;
	if (WARN_ON(strcmp(state->state, "xlated"))) {
		NSIM_EA(bpf->extack, "offloading program in bad state");
		return -EINVAL;
	}
	return 0;
}

static bool
nsim_map_key_match(struct bpf_map *map, struct nsim_map_entry *e, void *key)
{
	return e->key && !memcmp(key, e->key, map->key_size);
}

static int nsim_map_key_find(struct bpf_offloaded_map *offmap, void *key)
{
	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
	unsigned int i;

	for (i = 0; i < ARRAY_SIZE(nmap->entry); i++)
		if (nsim_map_key_match(&offmap->map, &nmap->entry[i], key))
			return i;

	return -ENOENT;
}

static int
nsim_map_alloc_elem(struct bpf_offloaded_map *offmap, unsigned int idx)
{
	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;

	nmap->entry[idx].key = kmalloc(offmap->map.key_size, GFP_USER);
	if (!nmap->entry[idx].key)
		return -ENOMEM;
	nmap->entry[idx].value = kmalloc(offmap->map.value_size, GFP_USER);
	if (!nmap->entry[idx].value) {
		kfree(nmap->entry[idx].key);
		nmap->entry[idx].key = NULL;
		return -ENOMEM;
	}

	return 0;
}

static int
nsim_map_get_next_key(struct bpf_offloaded_map *offmap,
		      void *key, void *next_key)
{
	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
	int idx = -ENOENT;

	mutex_lock(&nmap->mutex);

	if (key)
		idx = nsim_map_key_find(offmap, key);
	if (idx == -ENOENT)
		idx = 0;
	else
		idx++;

	for (; idx < ARRAY_SIZE(nmap->entry); idx++) {
		if (nmap->entry[idx].key) {
			memcpy(next_key, nmap->entry[idx].key,
			       offmap->map.key_size);
			break;
		}
	}

	mutex_unlock(&nmap->mutex);

	if (idx == ARRAY_SIZE(nmap->entry))
		return -ENOENT;
	return 0;
}

static int
nsim_map_lookup_elem(struct bpf_offloaded_map *offmap, void *key, void *value)
{
	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
	int idx;

	mutex_lock(&nmap->mutex);

	idx = nsim_map_key_find(offmap, key);
	if (idx >= 0)
		memcpy(value, nmap->entry[idx].value, offmap->map.value_size);

	mutex_unlock(&nmap->mutex);

	return idx < 0 ? idx : 0;
}

static int
nsim_map_update_elem(struct bpf_offloaded_map *offmap,
		     void *key, void *value, u64 flags)
{
	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
	int idx, err = 0;

	mutex_lock(&nmap->mutex);

	idx = nsim_map_key_find(offmap, key);
	if (idx < 0 && flags == BPF_EXIST) {
		err = idx;
		goto exit_unlock;
	}
	if (idx >= 0 && flags == BPF_NOEXIST) {
		err = -EEXIST;
		goto exit_unlock;
	}

	if (idx < 0) {
		for (idx = 0; idx < ARRAY_SIZE(nmap->entry); idx++)
			if (!nmap->entry[idx].key)
				break;
		if (idx == ARRAY_SIZE(nmap->entry)) {
			err = -E2BIG;
			goto exit_unlock;
		}

		err = nsim_map_alloc_elem(offmap, idx);
		if (err)
			goto exit_unlock;
	}

	memcpy(nmap->entry[idx].key, key, offmap->map.key_size);
	memcpy(nmap->entry[idx].value, value, offmap->map.value_size);
exit_unlock:
	mutex_unlock(&nmap->mutex);

	return err;
}

static int nsim_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
{
	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
	int idx;

	if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
		return -EINVAL;

	mutex_lock(&nmap->mutex);

	idx = nsim_map_key_find(offmap, key);
	if (idx >= 0) {
		kfree(nmap->entry[idx].key);
		kfree(nmap->entry[idx].value);
		memset(&nmap->entry[idx], 0, sizeof(nmap->entry[idx]));
	}

	mutex_unlock(&nmap->mutex);

	return idx < 0 ? idx : 0;
}

static const struct bpf_map_dev_ops nsim_bpf_map_ops = {
	.map_get_next_key	= nsim_map_get_next_key,
	.map_lookup_elem	= nsim_map_lookup_elem,
	.map_update_elem	= nsim_map_update_elem,
	.map_delete_elem	= nsim_map_delete_elem,
};

static int
nsim_bpf_map_alloc(struct netdevsim *ns, struct bpf_offloaded_map *offmap)
{
	struct nsim_bpf_bound_map *nmap;
	int i, err;

	if (WARN_ON(offmap->map.map_type != BPF_MAP_TYPE_ARRAY &&
		    offmap->map.map_type != BPF_MAP_TYPE_HASH))
		return -EINVAL;
	if (offmap->map.max_entries > NSIM_BPF_MAX_KEYS)
		return -ENOMEM;
	if (offmap->map.map_flags)
		return -EINVAL;

	nmap = kzalloc(sizeof(*nmap), GFP_USER);
	if (!nmap)
		return -ENOMEM;

	offmap->dev_priv = nmap;
	nmap->ns = ns;
	nmap->map = offmap;
	mutex_init(&nmap->mutex);

	if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY) {
		for (i = 0; i < ARRAY_SIZE(nmap->entry); i++) {
			u32 *key;

			err = nsim_map_alloc_elem(offmap, i);
			if (err)
				goto err_free;
			key = nmap->entry[i].key;
			*key = i;
		}
	}

	offmap->dev_ops = &nsim_bpf_map_ops;
	list_add_tail(&nmap->l, &ns->sdev->bpf_bound_maps);

	return 0;

err_free:
	while (--i >= 0) {
		kfree(nmap->entry[i].key);
		kfree(nmap->entry[i].value);
	}
	kfree(nmap);
	return err;
}

static void nsim_bpf_map_free(struct bpf_offloaded_map *offmap)
{
	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
	unsigned int i;

	for (i = 0; i < ARRAY_SIZE(nmap->entry); i++) {
		kfree(nmap->entry[i].key);
		kfree(nmap->entry[i].value);
	}
	list_del_init(&nmap->l);
	mutex_destroy(&nmap->mutex);
	kfree(nmap);
}

int nsim_bpf(struct net_device *dev, struct netdev_bpf *bpf)
{
	struct netdevsim *ns = netdev_priv(dev);
	struct nsim_bpf_bound_prog *state;
	int err;

	ASSERT_RTNL();

	switch (bpf->command) {
	case BPF_OFFLOAD_VERIFIER_PREP:
		if (!ns->bpf_bind_accept)
			return -EOPNOTSUPP;

		err = nsim_bpf_create_prog(ns, bpf->verifier.prog);
		if (err)
			return err;

		bpf->verifier.ops = &nsim_bpf_analyzer_ops;
		return 0;
	case BPF_OFFLOAD_TRANSLATE:
		state = bpf->offload.prog->aux->offload->dev_priv;

		state->state = "xlated";
		return 0;
	case BPF_OFFLOAD_DESTROY:
		nsim_bpf_destroy_prog(bpf->offload.prog);
		return 0;
	case XDP_QUERY_PROG:
		return xdp_attachment_query(&ns->xdp, bpf);
	case XDP_QUERY_PROG_HW:
		return xdp_attachment_query(&ns->xdp_hw, bpf);
	case XDP_SETUP_PROG:
		err = nsim_setup_prog_checks(ns, bpf);
		if (err)
			return err;

		return nsim_xdp_set_prog(ns, bpf, &ns->xdp);
	case XDP_SETUP_PROG_HW:
		err = nsim_setup_prog_hw_checks(ns, bpf);
		if (err)
			return err;

		return nsim_xdp_set_prog(ns, bpf, &ns->xdp_hw);
	case BPF_OFFLOAD_MAP_ALLOC:
		if (!ns->bpf_map_accept)
			return -EOPNOTSUPP;

		return nsim_bpf_map_alloc(ns, bpf->offmap);
	case BPF_OFFLOAD_MAP_FREE:
		nsim_bpf_map_free(bpf->offmap);
		return 0;
	default:
		return -EINVAL;
	}
}

int nsim_bpf_init(struct netdevsim *ns)
{
	int err;

	if (ns->sdev->refcnt == 1) {
		INIT_LIST_HEAD(&ns->sdev->bpf_bound_progs);
		INIT_LIST_HEAD(&ns->sdev->bpf_bound_maps);

		ns->sdev->ddir_bpf_bound_progs =
			debugfs_create_dir("bpf_bound_progs", ns->sdev->ddir);
		if (IS_ERR_OR_NULL(ns->sdev->ddir_bpf_bound_progs))
			return -ENOMEM;

		ns->sdev->bpf_dev = bpf_offload_dev_create();
		err = PTR_ERR_OR_ZERO(ns->sdev->bpf_dev);
		if (err)
			return err;
	}

	err = bpf_offload_dev_netdev_register(ns->sdev->bpf_dev, ns->netdev);
	if (err)
		goto err_destroy_bdev;

	debugfs_create_u32("bpf_offloaded_id", 0400, ns->ddir,
			   &ns->bpf_offloaded_id);

	ns->bpf_bind_accept = true;
	debugfs_create_bool("bpf_bind_accept", 0600, ns->ddir,
			    &ns->bpf_bind_accept);
	debugfs_create_u32("bpf_bind_verifier_delay", 0600, ns->ddir,
			   &ns->bpf_bind_verifier_delay);

	ns->bpf_tc_accept = true;
	debugfs_create_bool("bpf_tc_accept", 0600, ns->ddir,
			    &ns->bpf_tc_accept);
	debugfs_create_bool("bpf_tc_non_bound_accept", 0600, ns->ddir,
			    &ns->bpf_tc_non_bound_accept);
	ns->bpf_xdpdrv_accept = true;
	debugfs_create_bool("bpf_xdpdrv_accept", 0600, ns->ddir,
			    &ns->bpf_xdpdrv_accept);
	ns->bpf_xdpoffload_accept = true;
	debugfs_create_bool("bpf_xdpoffload_accept", 0600, ns->ddir,
			    &ns->bpf_xdpoffload_accept);

	ns->bpf_map_accept = true;
	debugfs_create_bool("bpf_map_accept", 0600, ns->ddir,
			    &ns->bpf_map_accept);

	return 0;

err_destroy_bdev:
	if (ns->sdev->refcnt == 1)
		bpf_offload_dev_destroy(ns->sdev->bpf_dev);
	return err;
}

void nsim_bpf_uninit(struct netdevsim *ns)
{
	WARN_ON(ns->xdp.prog);
	WARN_ON(ns->xdp_hw.prog);
	WARN_ON(ns->bpf_offloaded);
	bpf_offload_dev_netdev_unregister(ns->sdev->bpf_dev, ns->netdev);

	if (ns->sdev->refcnt == 1) {
		WARN_ON(!list_empty(&ns->sdev->bpf_bound_progs));
		WARN_ON(!list_empty(&ns->sdev->bpf_bound_maps));
		bpf_offload_dev_destroy(ns->sdev->bpf_dev);
	}
}
Commit	Line	Data
31d3ad83 JK	1	/*
	2	* Copyright (C) 2017 Netronome Systems, Inc.
	3	*
	4	* This software is licensed under the GNU General License Version 2,
	5	* June 1991 as shown in the file COPYING in the top-level directory of this
	6	* source tree.
	7	*
	8	* THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
	9	* WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
	10	* BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
	11	* FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
	12	* OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
	13	* THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
	14	*/
	15
	16	#include <linux/bpf.h>
	17	#include <linux/bpf_verifier.h>
	18	#include <linux/debugfs.h>
	19	#include <linux/kernel.h>
395cacb5	20	#include <linux/mutex.h>
31d3ad83 JK	21	#include <linux/rtnetlink.h>
	22	#include <net/pkt_cls.h>
	23
	24	#include "netdevsim.h"
	25
9045bdc8 QM	26	#define pr_vlog(env, fmt, ...) \
	27	bpf_verifier_log_write(env, "[netdevsim] " fmt, ##__VA_ARGS__)
	28
31d3ad83 JK	29	struct nsim_bpf_bound_prog {
	30	struct netdevsim *ns;
	31	struct bpf_prog *prog;
	32	struct dentry *ddir;
	33	const char *state;
	34	bool is_loaded;
	35	struct list_head l;
	36	};
	37
395cacb5 JK	38	#define NSIM_BPF_MAX_KEYS 2
	39
	40	struct nsim_bpf_bound_map {
	41	struct netdevsim *ns;
	42	struct bpf_offloaded_map *map;
	43	struct mutex mutex;
	44	struct nsim_map_entry {
	45	void *key;
	46	void *value;
	47	} entry[NSIM_BPF_MAX_KEYS];
	48	struct list_head l;
	49	};
	50
31d3ad83 JK	51	static int nsim_debugfs_bpf_string_read(struct seq_file file, void data)
	52	{
	53	const char **str = file->private;
	54
	55	if (*str)
	56	seq_printf(file, "%s\n", *str);
	57
	58	return 0;
	59	}
	60
	61	static int nsim_debugfs_bpf_string_open(struct inode inode, struct file f)
	62	{
	63	return single_open(f, nsim_debugfs_bpf_string_read, inode->i_private);
	64	}
	65
	66	static const struct file_operations nsim_bpf_string_fops = {
	67	.owner = THIS_MODULE,
	68	.open = nsim_debugfs_bpf_string_open,
	69	.release = single_release,
	70	.read = seq_read,
	71	.llseek = seq_lseek
	72	};
	73
	74	static int
	75	nsim_bpf_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn)
	76	{
	77	struct nsim_bpf_bound_prog *state;
	78
	79	state = env->prog->aux->offload->dev_priv;
	80	if (state->ns->bpf_bind_verifier_delay && !insn_idx)
	81	msleep(state->ns->bpf_bind_verifier_delay);
	82
9045bdc8 QM	83	if (insn_idx == env->prog->len - 1)
	84	pr_vlog(env, "Hello from netdevsim!\n");
	85
31d3ad83 JK	86	return 0;
	87	}
	88
c941ce9c QM	89	static int nsim_bpf_finalize(struct bpf_verifier_env *env)
	90	{
	91	return 0;
	92	}
	93
cae1927c	94	static const struct bpf_prog_offload_ops nsim_bpf_analyzer_ops = {
c941ce9c QM	95	.insn_hook = nsim_bpf_verify_insn,
c941ce9c QM	96	.finalize = nsim_bpf_finalize,
31d3ad83 JK	97	};
	98
	99	static bool nsim_xdp_offload_active(struct netdevsim *ns)
	100	{
799e173d	101	return ns->xdp_hw.prog;
31d3ad83 JK	102	}
	103
	104	static void nsim_prog_set_loaded(struct bpf_prog *prog, bool loaded)
	105	{
	106	struct nsim_bpf_bound_prog *state;
	107
	108	if (!prog \|\| !prog->aux->offload)
	109	return;
	110
	111	state = prog->aux->offload->dev_priv;
	112	state->is_loaded = loaded;
	113	}
	114
	115	static int
	116	nsim_bpf_offload(struct netdevsim ns, struct bpf_prog prog, bool oldprog)
	117	{
	118	nsim_prog_set_loaded(ns->bpf_offloaded, false);
	119
	120	WARN(!!ns->bpf_offloaded != oldprog,
	121	"bad offload state, expected offload %sto be active",
	122	oldprog ? "" : "not ");
	123	ns->bpf_offloaded = prog;
	124	ns->bpf_offloaded_id = prog ? prog->aux->id : 0;
	125	nsim_prog_set_loaded(prog, true);
	126
	127	return 0;
	128	}
	129
	130	int nsim_bpf_setup_tc_block_cb(enum tc_setup_type type,
	131	void type_data, void cb_priv)
	132	{
	133	struct tc_cls_bpf_offload *cls_bpf = type_data;
	134	struct bpf_prog *prog = cls_bpf->prog;
	135	struct netdevsim *ns = cb_priv;
fba961ab	136	struct bpf_prog *oldprog;
31d3ad83	137
728461f2 QM	138	if (type != TC_SETUP_CLSBPF) {
	139	NSIM_EA(cls_bpf->common.extack,
	140	"only offload of BPF classifiers supported");
	141	return -EOPNOTSUPP;
	142	}
	143
a2b212a5	144	if (!tc_cls_can_offload_and_chain0(ns->netdev, &cls_bpf->common))
728461f2 QM	145	return -EOPNOTSUPP;
	146
	147	if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
	148	NSIM_EA(cls_bpf->common.extack,
	149	"only ETH_P_ALL supported as filter protocol");
	150	return -EOPNOTSUPP;
	151	}
	152
728461f2 QM	153	if (!ns->bpf_tc_accept) {
	154	NSIM_EA(cls_bpf->common.extack,
	155	"netdevsim configured to reject BPF TC offload");
31d3ad83	156	return -EOPNOTSUPP;
728461f2	157	}
31d3ad83	158	/* Note: progs without skip_sw will probably not be dev bound */
728461f2 QM	159	if (prog && !prog->aux->offload && !ns->bpf_tc_non_bound_accept) {
	160	NSIM_EA(cls_bpf->common.extack,
	161	"netdevsim configured to reject unbound programs");
31d3ad83	162	return -EOPNOTSUPP;
728461f2	163	}
31d3ad83	164
fba961ab	165	if (cls_bpf->command != TC_CLSBPF_OFFLOAD)
31d3ad83	166	return -EOPNOTSUPP;
fba961ab DM	167
	168	oldprog = cls_bpf->oldprog;
	169
	170	/* Don't remove if oldprog doesn't match driver's state */
	171	if (ns->bpf_offloaded != oldprog) {
	172	oldprog = NULL;
	173	if (!cls_bpf->prog)
	174	return 0;
728461f2 QM	175	if (ns->bpf_offloaded) {
	176	NSIM_EA(cls_bpf->common.extack,
	177	"driver and netdev offload states mismatch");
fba961ab	178	return -EBUSY;
728461f2	179	}
31d3ad83	180	}
fba961ab DM	181
fba961ab DM	182	return nsim_bpf_offload(ns, cls_bpf->prog, oldprog);
31d3ad83 JK	183	}
	184
	185	int nsim_bpf_disable_tc(struct netdevsim *ns)
	186	{
	187	if (ns->bpf_offloaded && !nsim_xdp_offload_active(ns))
	188	return -EBUSY;
	189	return 0;
	190	}
	191
	192	static int nsim_xdp_offload_prog(struct netdevsim ns, struct netdev_bpf bpf)
	193	{
	194	if (!nsim_xdp_offload_active(ns) && !bpf->prog)
	195	return 0;
	196	if (!nsim_xdp_offload_active(ns) && bpf->prog && ns->bpf_offloaded) {
	197	NSIM_EA(bpf->extack, "TC program is already loaded");
	198	return -EBUSY;
	199	}
	200
	201	return nsim_bpf_offload(ns, bpf->prog, nsim_xdp_offload_active(ns));
	202	}
	203
799e173d JK	204	static int
	205	nsim_xdp_set_prog(struct netdevsim ns, struct netdev_bpf bpf,
	206	struct xdp_attachment_info *xdp)
31d3ad83 JK	207	{
	208	int err;
	209
799e173d	210	if (!xdp_attachment_flags_ok(xdp, bpf))
31d3ad83	211	return -EBUSY;
31d3ad83 JK	212
	213	if (bpf->command == XDP_SETUP_PROG && !ns->bpf_xdpdrv_accept) {
	214	NSIM_EA(bpf->extack, "driver XDP disabled in DebugFS");
	215	return -EOPNOTSUPP;
	216	}
	217	if (bpf->command == XDP_SETUP_PROG_HW && !ns->bpf_xdpoffload_accept) {
	218	NSIM_EA(bpf->extack, "XDP offload disabled in DebugFS");
	219	return -EOPNOTSUPP;
	220	}
	221
	222	if (bpf->command == XDP_SETUP_PROG_HW) {
	223	err = nsim_xdp_offload_prog(ns, bpf);
	224	if (err)
	225	return err;
	226	}
	227
799e173d	228	xdp_attachment_setup(xdp, bpf);
31d3ad83 JK	229
	230	return 0;
	231	}
	232
1dfc2663	233	static int nsim_bpf_create_prog(struct netdevsim ns, struct bpf_prog prog)
31d3ad83 JK	234	{
	235	struct nsim_bpf_bound_prog *state;
	236	char name[16];
31d3ad83 JK	237
	238	state = kzalloc(sizeof(*state), GFP_KERNEL);
	239	if (!state)
	240	return -ENOMEM;
	241
	242	state->ns = ns;
	243	state->prog = prog;
	244	state->state = "verify";
	245
	246	/* Program id is not populated yet when we create the state. */
d6d60713 JK	247	sprintf(name, "%u", ns->sdev->prog_id_gen++);
d6d60713 JK	248	state->ddir = debugfs_create_dir(name, ns->sdev->ddir_bpf_bound_progs);
9ee1942c	249	if (IS_ERR_OR_NULL(state->ddir)) {
31d3ad83	250	kfree(state);
9ee1942c	251	return -ENOMEM;
31d3ad83 JK	252	}
	253
	254	debugfs_create_u32("id", 0400, state->ddir, &prog->aux->id);
	255	debugfs_create_file("state", 0400, state->ddir,
	256	&state->state, &nsim_bpf_string_fops);
	257	debugfs_create_bool("loaded", 0400, state->ddir, &state->is_loaded);
	258
d6d60713	259	list_add_tail(&state->l, &ns->sdev->bpf_bound_progs);
31d3ad83 JK	260
	261	prog->aux->offload->dev_priv = state;
	262
	263	return 0;
	264	}
	265
1dfc2663	266	static void nsim_bpf_destroy_prog(struct bpf_prog *prog)
31d3ad83 JK	267	{
	268	struct nsim_bpf_bound_prog *state;
	269
	270	state = prog->aux->offload->dev_priv;
	271	WARN(state->is_loaded,
	272	"offload state destroyed while program still bound");
	273	debugfs_remove_recursive(state->ddir);
	274	list_del(&state->l);
	275	kfree(state);
	276	}
	277
	278	static int nsim_setup_prog_checks(struct netdevsim ns, struct netdev_bpf bpf)
	279	{
	280	if (bpf->prog && bpf->prog->aux->offload) {
	281	NSIM_EA(bpf->extack, "attempt to load offloaded prog to drv");
	282	return -EINVAL;
	283	}
	284	if (ns->netdev->mtu > NSIM_XDP_MAX_MTU) {
	285	NSIM_EA(bpf->extack, "MTU too large w/ XDP enabled");
	286	return -EINVAL;
	287	}
31d3ad83 JK	288	return 0;
	289	}
	290
	291	static int
	292	nsim_setup_prog_hw_checks(struct netdevsim ns, struct netdev_bpf bpf)
	293	{
	294	struct nsim_bpf_bound_prog *state;
	295
	296	if (!bpf->prog)
	297	return 0;
	298
	299	if (!bpf->prog->aux->offload) {
	300	NSIM_EA(bpf->extack, "xdpoffload of non-bound program");
	301	return -EINVAL;
	302	}
9d1b66b8	303	if (!bpf_offload_dev_match(bpf->prog, ns->netdev)) {
31d3ad83 JK	304	NSIM_EA(bpf->extack, "program bound to different dev");
	305	return -EINVAL;
	306	}
	307
	308	state = bpf->prog->aux->offload->dev_priv;
	309	if (WARN_ON(strcmp(state->state, "xlated"))) {
	310	NSIM_EA(bpf->extack, "offloading program in bad state");
	311	return -EINVAL;
	312	}
	313	return 0;
	314	}
	315
395cacb5 JK	316	static bool
	317	nsim_map_key_match(struct bpf_map map, struct nsim_map_entry e, void *key)
	318	{
	319	return e->key && !memcmp(key, e->key, map->key_size);
	320	}
	321
	322	static int nsim_map_key_find(struct bpf_offloaded_map offmap, void key)
	323	{
	324	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
	325	unsigned int i;
	326
	327	for (i = 0; i < ARRAY_SIZE(nmap->entry); i++)
	328	if (nsim_map_key_match(&offmap->map, &nmap->entry[i], key))
	329	return i;
	330
	331	return -ENOENT;
	332	}
	333
	334	static int
	335	nsim_map_alloc_elem(struct bpf_offloaded_map *offmap, unsigned int idx)
	336	{
	337	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
	338
	339	nmap->entry[idx].key = kmalloc(offmap->map.key_size, GFP_USER);
	340	if (!nmap->entry[idx].key)
	341	return -ENOMEM;
	342	nmap->entry[idx].value = kmalloc(offmap->map.value_size, GFP_USER);
	343	if (!nmap->entry[idx].value) {
	344	kfree(nmap->entry[idx].key);
	345	nmap->entry[idx].key = NULL;
	346	return -ENOMEM;
	347	}
	348
	349	return 0;
	350	}
	351
	352	static int
	353	nsim_map_get_next_key(struct bpf_offloaded_map *offmap,
	354	void key, void next_key)
	355	{
	356	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
	357	int idx = -ENOENT;
	358
	359	mutex_lock(&nmap->mutex);
	360
	361	if (key)
	362	idx = nsim_map_key_find(offmap, key);
	363	if (idx == -ENOENT)
	364	idx = 0;
	365	else
	366	idx++;
	367
	368	for (; idx < ARRAY_SIZE(nmap->entry); idx++) {
	369	if (nmap->entry[idx].key) {
	370	memcpy(next_key, nmap->entry[idx].key,
	371	offmap->map.key_size);
	372	break;
	373	}
	374	}
	375
	376	mutex_unlock(&nmap->mutex);
	377
	378	if (idx == ARRAY_SIZE(nmap->entry))
	379	return -ENOENT;
380	return 0;
381	}
382
383	static int
384	nsim_map_lookup_elem(struct bpf_offloaded_map offmap, void key, void *value)
385	{
386	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
387	int idx;
388
389	mutex_lock(&nmap->mutex);
390
391	idx = nsim_map_key_find(offmap, key);
392	if (idx >= 0)
393	memcpy(value, nmap->entry[idx].value, offmap->map.value_size);
394
395	mutex_unlock(&nmap->mutex);
396
397	return idx < 0 ? idx : 0;
398	}
399
400	static int
401	nsim_map_update_elem(struct bpf_offloaded_map *offmap,
402	void key, void value, u64 flags)
403	{
404	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
405	int idx, err = 0;
406
407	mutex_lock(&nmap->mutex);
408
409	idx = nsim_map_key_find(offmap, key);
410	if (idx < 0 && flags == BPF_EXIST) {
411	err = idx;
412	goto exit_unlock;
413	}
414	if (idx >= 0 && flags == BPF_NOEXIST) {
415	err = -EEXIST;
416	goto exit_unlock;
417	}
418
419	if (idx < 0) {
420	for (idx = 0; idx < ARRAY_SIZE(nmap->entry); idx++)
421	if (!nmap->entry[idx].key)
422	break;
423	if (idx == ARRAY_SIZE(nmap->entry)) {
424	err = -E2BIG;
425	goto exit_unlock;
426	}
427
428	err = nsim_map_alloc_elem(offmap, idx);
429	if (err)
430	goto exit_unlock;
431	}
432
433	memcpy(nmap->entry[idx].key, key, offmap->map.key_size);
434	memcpy(nmap->entry[idx].value, value, offmap->map.value_size);
435	exit_unlock:
436	mutex_unlock(&nmap->mutex);
437
438	return err;
439	}
440
441	static int nsim_map_delete_elem(struct bpf_offloaded_map offmap, void key)
442	{
443	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
444	int idx;
445
446	if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
447	return -EINVAL;
448
449	mutex_lock(&nmap->mutex);
450
451	idx = nsim_map_key_find(offmap, key);
452	if (idx >= 0) {
453	kfree(nmap->entry[idx].key);
454	kfree(nmap->entry[idx].value);
455	memset(&nmap->entry[idx], 0, sizeof(nmap->entry[idx]));
456	}
457
458	mutex_unlock(&nmap->mutex);
459
460	return idx < 0 ? idx : 0;
461	}
462
463	static const struct bpf_map_dev_ops nsim_bpf_map_ops = {
464	.map_get_next_key = nsim_map_get_next_key,
465	.map_lookup_elem = nsim_map_lookup_elem,
466	.map_update_elem = nsim_map_update_elem,
467	.map_delete_elem = nsim_map_delete_elem,
468	};
469
470	static int
471	nsim_bpf_map_alloc(struct netdevsim ns, struct bpf_offloaded_map offmap)
472	{
473	struct nsim_bpf_bound_map *nmap;
e029f541	474	int i, err;
395cacb5 JK	475
	476	if (WARN_ON(offmap->map.map_type != BPF_MAP_TYPE_ARRAY &&
	477	offmap->map.map_type != BPF_MAP_TYPE_HASH))
	478	return -EINVAL;
	479	if (offmap->map.max_entries > NSIM_BPF_MAX_KEYS)
	480	return -ENOMEM;
	481	if (offmap->map.map_flags)
	482	return -EINVAL;
	483
	484	nmap = kzalloc(sizeof(*nmap), GFP_USER);
	485	if (!nmap)
	486	return -ENOMEM;
	487
	488	offmap->dev_priv = nmap;
	489	nmap->ns = ns;
	490	nmap->map = offmap;
	491	mutex_init(&nmap->mutex);
	492
	493	if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY) {
	494	for (i = 0; i < ARRAY_SIZE(nmap->entry); i++) {
	495	u32 *key;
	496
	497	err = nsim_map_alloc_elem(offmap, i);
	498	if (err)
	499	goto err_free;
	500	key = nmap->entry[i].key;
	501	*key = i;
	502	}
	503	}
	504
	505	offmap->dev_ops = &nsim_bpf_map_ops;
d6d60713	506	list_add_tail(&nmap->l, &ns->sdev->bpf_bound_maps);
395cacb5 JK	507
	508	return 0;
	509
	510	err_free:
e029f541	511	while (--i >= 0) {
395cacb5 JK	512	kfree(nmap->entry[i].key);
	513	kfree(nmap->entry[i].value);
	514	}
	515	kfree(nmap);
	516	return err;
	517	}
	518
	519	static void nsim_bpf_map_free(struct bpf_offloaded_map *offmap)
	520	{
	521	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
	522	unsigned int i;
	523
	524	for (i = 0; i < ARRAY_SIZE(nmap->entry); i++) {
	525	kfree(nmap->entry[i].key);
	526	kfree(nmap->entry[i].value);
	527	}
	528	list_del_init(&nmap->l);
	529	mutex_destroy(&nmap->mutex);
	530	kfree(nmap);
	531	}
	532
31d3ad83 JK	533	int nsim_bpf(struct net_device dev, struct netdev_bpf bpf)
	534	{
	535	struct netdevsim *ns = netdev_priv(dev);
	536	struct nsim_bpf_bound_prog *state;
	537	int err;
	538
	539	ASSERT_RTNL();
	540
	541	switch (bpf->command) {
	542	case BPF_OFFLOAD_VERIFIER_PREP:
	543	if (!ns->bpf_bind_accept)
	544	return -EOPNOTSUPP;
	545
	546	err = nsim_bpf_create_prog(ns, bpf->verifier.prog);
	547	if (err)
	548	return err;
	549
	550	bpf->verifier.ops = &nsim_bpf_analyzer_ops;
	551	return 0;
	552	case BPF_OFFLOAD_TRANSLATE:
	553	state = bpf->offload.prog->aux->offload->dev_priv;
	554
	555	state->state = "xlated";
	556	return 0;
	557	case BPF_OFFLOAD_DESTROY:
	558	nsim_bpf_destroy_prog(bpf->offload.prog);
	559	return 0;
	560	case XDP_QUERY_PROG:
a25717d2 JK	561	return xdp_attachment_query(&ns->xdp, bpf);
a25717d2 JK	562	case XDP_QUERY_PROG_HW:
799e173d	563	return xdp_attachment_query(&ns->xdp_hw, bpf);
31d3ad83 JK	564	case XDP_SETUP_PROG:
	565	err = nsim_setup_prog_checks(ns, bpf);
	566	if (err)
	567	return err;
	568
799e173d	569	return nsim_xdp_set_prog(ns, bpf, &ns->xdp);
31d3ad83 JK	570	case XDP_SETUP_PROG_HW:
	571	err = nsim_setup_prog_hw_checks(ns, bpf);
	572	if (err)
	573	return err;
	574
799e173d	575	return nsim_xdp_set_prog(ns, bpf, &ns->xdp_hw);
395cacb5 JK	576	case BPF_OFFLOAD_MAP_ALLOC:
	577	if (!ns->bpf_map_accept)
	578	return -EOPNOTSUPP;
	579
	580	return nsim_bpf_map_alloc(ns, bpf->offmap);
	581	case BPF_OFFLOAD_MAP_FREE:
	582	nsim_bpf_map_free(bpf->offmap);
	583	return 0;
31d3ad83 JK	584	default:
	585	return -EINVAL;
	586	}
	587	}
	588
	589	int nsim_bpf_init(struct netdevsim *ns)
	590	{
9fd7c555 JK	591	int err;
9fd7c555 JK	592
d6d60713 JK	593	if (ns->sdev->refcnt == 1) {
	594	INIT_LIST_HEAD(&ns->sdev->bpf_bound_progs);
	595	INIT_LIST_HEAD(&ns->sdev->bpf_bound_maps);
	596
	597	ns->sdev->ddir_bpf_bound_progs =
	598	debugfs_create_dir("bpf_bound_progs", ns->sdev->ddir);
	599	if (IS_ERR_OR_NULL(ns->sdev->ddir_bpf_bound_progs))
	600	return -ENOMEM;
602144c2 JK	601
	602	ns->sdev->bpf_dev = bpf_offload_dev_create();
	603	err = PTR_ERR_OR_ZERO(ns->sdev->bpf_dev);
	604	if (err)
	605	return err;
d6d60713	606	}
31d3ad83	607
602144c2	608	err = bpf_offload_dev_netdev_register(ns->sdev->bpf_dev, ns->netdev);
9fd7c555	609	if (err)
602144c2	610	goto err_destroy_bdev;
9fd7c555	611
31d3ad83 JK	612	debugfs_create_u32("bpf_offloaded_id", 0400, ns->ddir,
	613	&ns->bpf_offloaded_id);
	614
	615	ns->bpf_bind_accept = true;
	616	debugfs_create_bool("bpf_bind_accept", 0600, ns->ddir,
	617	&ns->bpf_bind_accept);
	618	debugfs_create_u32("bpf_bind_verifier_delay", 0600, ns->ddir,
	619	&ns->bpf_bind_verifier_delay);
31d3ad83 JK	620
	621	ns->bpf_tc_accept = true;
	622	debugfs_create_bool("bpf_tc_accept", 0600, ns->ddir,
	623	&ns->bpf_tc_accept);
	624	debugfs_create_bool("bpf_tc_non_bound_accept", 0600, ns->ddir,
	625	&ns->bpf_tc_non_bound_accept);
	626	ns->bpf_xdpdrv_accept = true;
	627	debugfs_create_bool("bpf_xdpdrv_accept", 0600, ns->ddir,
	628	&ns->bpf_xdpdrv_accept);
	629	ns->bpf_xdpoffload_accept = true;
	630	debugfs_create_bool("bpf_xdpoffload_accept", 0600, ns->ddir,
	631	&ns->bpf_xdpoffload_accept);
	632
395cacb5 JK	633	ns->bpf_map_accept = true;
	634	debugfs_create_bool("bpf_map_accept", 0600, ns->ddir,
	635	&ns->bpf_map_accept);
	636
31d3ad83	637	return 0;
602144c2 JK	638
	639	err_destroy_bdev:
	640	if (ns->sdev->refcnt == 1)
	641	bpf_offload_dev_destroy(ns->sdev->bpf_dev);
	642	return err;
31d3ad83 JK	643	}
	644
	645	void nsim_bpf_uninit(struct netdevsim *ns)
	646	{
05296620	647	WARN_ON(ns->xdp.prog);
799e173d	648	WARN_ON(ns->xdp_hw.prog);
31d3ad83	649	WARN_ON(ns->bpf_offloaded);
602144c2	650	bpf_offload_dev_netdev_unregister(ns->sdev->bpf_dev, ns->netdev);
d6d60713 JK	651
	652	if (ns->sdev->refcnt == 1) {
	653	WARN_ON(!list_empty(&ns->sdev->bpf_bound_progs));
	654	WARN_ON(!list_empty(&ns->sdev->bpf_bound_maps));
602144c2	655	bpf_offload_dev_destroy(ns->sdev->bpf_dev);
d6d60713	656	}
31d3ad83	657	}