kernel/sched/core_sched.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 #include <linux/prctl.h>
   4 #include "sched.h"
   5
   6 /*
   7  * A simple wrapper around refcount. An allocated sched_core_cookie's
   8  * address is used to compute the cookie of the task.
   9  */
  10 struct sched_core_cookie {
  11         refcount_t refcnt;
  12 };
  13
  14 unsigned long sched_core_alloc_cookie(void)
  15 {
  16         struct sched_core_cookie *ck = kmalloc(sizeof(*ck), GFP_KERNEL);
  17         if (!ck)
  18                 return 0;
  19
  20         refcount_set(&ck->refcnt, 1);
  21         sched_core_get();
  22
  23         return (unsigned long)ck;
  24 }
  25
  26 void sched_core_put_cookie(unsigned long cookie)
  27 {
  28         struct sched_core_cookie *ptr = (void *)cookie;
  29
  30         if (ptr && refcount_dec_and_test(&ptr->refcnt)) {
  31                 kfree(ptr);
  32                 sched_core_put();
  33         }
  34 }
  35
  36 unsigned long sched_core_get_cookie(unsigned long cookie)
  37 {
  38         struct sched_core_cookie *ptr = (void *)cookie;
  39
  40         if (ptr)
  41                 refcount_inc(&ptr->refcnt);
  42
  43         return cookie;
  44 }
  45
  46 /*
  47  * sched_core_update_cookie - replace the cookie on a task
  48  * @p: the task to update
  49  * @cookie: the new cookie
  50  *
  51  * Effectively exchange the task cookie; caller is responsible for lifetimes on
  52  * both ends.
  53  *
  54  * Returns: the old cookie
  55  */
  56 unsigned long sched_core_update_cookie(struct task_struct *p, unsigned long cookie)
  57 {
  58         unsigned long old_cookie;
  59         struct rq_flags rf;
  60         struct rq *rq;
  61         bool enqueued;
  62
  63         rq = task_rq_lock(p, &rf);
  64
  65         /*
  66          * Since creating a cookie implies sched_core_get(), and we cannot set
  67          * a cookie until after we've created it, similarly, we cannot destroy
  68          * a cookie until after we've removed it, we must have core scheduling
  69          * enabled here.
  70          */
  71         SCHED_WARN_ON((p->core_cookie || cookie) && !sched_core_enabled(rq));
  72
  73         enqueued = sched_core_enqueued(p);
  74         if (enqueued)
  75                 sched_core_dequeue(rq, p);
  76
  77         old_cookie = p->core_cookie;
  78         p->core_cookie = cookie;
  79
  80         if (enqueued)
  81                 sched_core_enqueue(rq, p);
  82
  83         /*
  84          * If task is currently running, it may not be compatible anymore after
  85          * the cookie change, so enter the scheduler on its CPU to schedule it
  86          * away.
  87          */
  88         if (task_running(rq, p))
  89                 resched_curr(rq);
  90
  91         task_rq_unlock(rq, p, &rf);
  92
  93         return old_cookie;
  94 }
  95
  96 static unsigned long sched_core_clone_cookie(struct task_struct *p)
  97 {
  98         unsigned long cookie, flags;
  99
 100         raw_spin_lock_irqsave(&p->pi_lock, flags);
 101         cookie = sched_core_get_cookie(p->core_cookie);
 102         raw_spin_unlock_irqrestore(&p->pi_lock, flags);
 103
 104         return cookie;
 105 }
 106
 107 void sched_core_fork(struct task_struct *p)
 108 {
 109         RB_CLEAR_NODE(&p->core_node);
 110         p->core_cookie = sched_core_clone_cookie(current);
 111 }
 112
 113 void sched_core_free(struct task_struct *p)
 114 {
 115         sched_core_put_cookie(p->core_cookie);
 116 }
 117
 118 static void __sched_core_set(struct task_struct *p, unsigned long cookie)
 119 {
 120         cookie = sched_core_get_cookie(cookie);
 121         cookie = sched_core_update_cookie(p, cookie);
 122         sched_core_put_cookie(cookie);
 123 }
 124
 125 /* Called from prctl interface: PR_SCHED_CORE */
 126 int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
 127                          unsigned long uaddr)
 128 {
 129         unsigned long cookie = 0, id = 0;
 130         struct task_struct *task, *p;
 131         struct pid *grp;
 132         int err = 0;
 133
 134         if (!static_branch_likely(&sched_smt_present))
 135                 return -ENODEV;
 136
 137         BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD != PIDTYPE_PID);
 138         BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD_GROUP != PIDTYPE_TGID);
 139         BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_PROCESS_GROUP != PIDTYPE_PGID);
 140
 141         if (type > PIDTYPE_PGID || cmd >= PR_SCHED_CORE_MAX || pid < 0 ||
 142             (cmd != PR_SCHED_CORE_GET && uaddr))
 143                 return -EINVAL;
 144
 145         rcu_read_lock();
 146         if (pid == 0) {
 147                 task = current;
 148         } else {
 149                 task = find_task_by_vpid(pid);
 150                 if (!task) {
 151                         rcu_read_unlock();
 152                         return -ESRCH;
 153                 }
 154         }
 155         get_task_struct(task);
 156         rcu_read_unlock();
 157
 158         /*
 159          * Check if this process has the right to modify the specified
 160          * process. Use the regular "ptrace_may_access()" checks.
 161          */
 162         if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
 163                 err = -EPERM;
 164                 goto out;
 165         }
 166
 167         switch (cmd) {
 168         case PR_SCHED_CORE_GET:
 169                 if (type != PIDTYPE_PID || uaddr & 7) {
 170                         err = -EINVAL;
 171                         goto out;
 172                 }
 173                 cookie = sched_core_clone_cookie(task);
 174                 if (cookie) {
 175                         /* XXX improve ? */
 176                         ptr_to_hashval((void *)cookie, &id);
 177                 }
 178                 err = put_user(id, (u64 __user *)uaddr);
 179                 goto out;
 180
 181         case PR_SCHED_CORE_CREATE:
 182                 cookie = sched_core_alloc_cookie();
 183                 if (!cookie) {
 184                         err = -ENOMEM;
 185                         goto out;
 186                 }
 187                 break;
 188
 189         case PR_SCHED_CORE_SHARE_TO:
 190                 cookie = sched_core_clone_cookie(current);
 191                 break;
 192
 193         case PR_SCHED_CORE_SHARE_FROM:
 194                 if (type != PIDTYPE_PID) {
 195                         err = -EINVAL;
 196                         goto out;
 197                 }
 198                 cookie = sched_core_clone_cookie(task);
 199                 __sched_core_set(current, cookie);
 200                 goto out;
 201
 202         default:
 203                 err = -EINVAL;
 204                 goto out;
 205         };
 206
 207         if (type == PIDTYPE_PID) {
 208                 __sched_core_set(task, cookie);
 209                 goto out;
 210         }
 211
 212         read_lock(&tasklist_lock);
 213         grp = task_pid_type(task, type);
 214
 215         do_each_pid_thread(grp, type, p) {
 216                 if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS)) {
 217                         err = -EPERM;
 218                         goto out_tasklist;
 219                 }
 220         } while_each_pid_thread(grp, type, p);
 221
 222         do_each_pid_thread(grp, type, p) {
 223                 __sched_core_set(p, cookie);
 224         } while_each_pid_thread(grp, type, p);
 225 out_tasklist:
 226         read_unlock(&tasklist_lock);
 227
 228 out:
 229         sched_core_put_cookie(cookie);
 230         put_task_struct(task);
 231         return err;
 232 }
 233