1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright (c) 2022, SUSE.
7 #define pr_fmt(fmt) "MPTCP: " fmt
9 #include <linux/kernel.h>
10 #include <linux/module.h>
11 #include <linux/list.h>
12 #include <linux/rculist.h>
13 #include <linux/spinlock.h>
16 static DEFINE_SPINLOCK(mptcp_sched_list_lock);
17 static LIST_HEAD(mptcp_sched_list);
19 static int mptcp_sched_default_get_subflow(struct mptcp_sock *msk,
20 struct mptcp_sched_data *data)
24 ssk = data->reinject ? mptcp_subflow_get_retrans(msk) :
25 mptcp_subflow_get_send(msk);
29 mptcp_subflow_set_scheduled(mptcp_subflow_ctx(ssk), true);
33 static struct mptcp_sched_ops mptcp_sched_default = {
34 .get_subflow = mptcp_sched_default_get_subflow,
39 /* Must be called with rcu read lock held */
40 struct mptcp_sched_ops *mptcp_sched_find(const char *name)
42 struct mptcp_sched_ops *sched, *ret = NULL;
44 list_for_each_entry_rcu(sched, &mptcp_sched_list, list) {
45 if (!strcmp(sched->name, name)) {
54 /* Build string with list of available scheduler values.
55 * Similar to tcp_get_available_congestion_control()
57 void mptcp_get_available_schedulers(char *buf, size_t maxlen)
59 struct mptcp_sched_ops *sched;
63 spin_lock(&mptcp_sched_list_lock);
64 list_for_each_entry_rcu(sched, &mptcp_sched_list, list) {
65 offs += snprintf(buf + offs, maxlen - offs,
67 offs == 0 ? "" : " ", sched->name);
69 if (WARN_ON_ONCE(offs >= maxlen))
72 spin_unlock(&mptcp_sched_list_lock);
76 int mptcp_register_scheduler(struct mptcp_sched_ops *sched)
78 if (!sched->get_subflow)
81 spin_lock(&mptcp_sched_list_lock);
82 if (mptcp_sched_find(sched->name)) {
83 spin_unlock(&mptcp_sched_list_lock);
86 list_add_tail_rcu(&sched->list, &mptcp_sched_list);
87 spin_unlock(&mptcp_sched_list_lock);
89 pr_debug("%s registered", sched->name);
93 void mptcp_unregister_scheduler(struct mptcp_sched_ops *sched)
95 if (sched == &mptcp_sched_default)
98 spin_lock(&mptcp_sched_list_lock);
99 list_del_rcu(&sched->list);
100 spin_unlock(&mptcp_sched_list_lock);
103 void mptcp_sched_init(void)
105 mptcp_register_scheduler(&mptcp_sched_default);
108 int mptcp_init_sched(struct mptcp_sock *msk,
109 struct mptcp_sched_ops *sched)
112 sched = &mptcp_sched_default;
114 if (!bpf_try_module_get(sched, sched->owner))
118 if (msk->sched->init)
119 msk->sched->init(msk);
121 pr_debug("sched=%s", msk->sched->name);
126 void mptcp_release_sched(struct mptcp_sock *msk)
128 struct mptcp_sched_ops *sched = msk->sched;
137 bpf_module_put(sched, sched->owner);
140 void mptcp_subflow_set_scheduled(struct mptcp_subflow_context *subflow,
143 WRITE_ONCE(subflow->scheduled, scheduled);
146 int mptcp_sched_get_send(struct mptcp_sock *msk)
148 struct mptcp_subflow_context *subflow;
149 struct mptcp_sched_data data;
151 msk_owned_by_me(msk);
153 /* the following check is moved out of mptcp_subflow_get_send */
154 if (__mptcp_check_fallback(msk)) {
156 __tcp_can_send(msk->first) &&
157 sk_stream_memory_free(msk->first)) {
158 mptcp_subflow_set_scheduled(mptcp_subflow_ctx(msk->first), true);
164 mptcp_for_each_subflow(msk, subflow) {
165 if (READ_ONCE(subflow->scheduled))
169 data.reinject = false;
170 if (msk->sched == &mptcp_sched_default || !msk->sched)
171 return mptcp_sched_default_get_subflow(msk, &data);
172 return msk->sched->get_subflow(msk, &data);
175 int mptcp_sched_get_retrans(struct mptcp_sock *msk)
177 struct mptcp_subflow_context *subflow;
178 struct mptcp_sched_data data;
180 msk_owned_by_me(msk);
182 /* the following check is moved out of mptcp_subflow_get_retrans */
183 if (__mptcp_check_fallback(msk))
186 mptcp_for_each_subflow(msk, subflow) {
187 if (READ_ONCE(subflow->scheduled))
191 data.reinject = true;
192 if (msk->sched == &mptcp_sched_default || !msk->sched)
193 return mptcp_sched_default_get_subflow(msk, &data);
194 return msk->sched->get_subflow(msk, &data);