drivers/md/dm-ps-io-affinity.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020 Oracle Corporation
   4  *
   5  * Module Author: Mike Christie
   6  */
   7 #include "dm-path-selector.h"
   8
   9 #include <linux/device-mapper.h>
  10 #include <linux/module.h>
  11
  12 #define DM_MSG_PREFIX "multipath io-affinity"
  13
  14 struct path_info {
  15         struct dm_path *path;
  16         cpumask_var_t cpumask;
  17         refcount_t refcount;
  18         bool failed;
  19 };
  20
  21 struct selector {
  22         struct path_info **path_map;
  23         cpumask_var_t path_mask;
  24         atomic_t map_misses;
  25 };
  26
  27 static void ioa_free_path(struct selector *s, unsigned int cpu)
  28 {
  29         struct path_info *pi = s->path_map[cpu];
  30
  31         if (!pi)
  32                 return;
  33
  34         if (refcount_dec_and_test(&pi->refcount)) {
  35                 cpumask_clear_cpu(cpu, s->path_mask);
  36                 free_cpumask_var(pi->cpumask);
  37                 kfree(pi);
  38
  39                 s->path_map[cpu] = NULL;
  40         }
  41 }
  42
  43 static int ioa_add_path(struct path_selector *ps, struct dm_path *path,
  44                         int argc, char **argv, char **error)
  45 {
  46         struct selector *s = ps->context;
  47         struct path_info *pi = NULL;
  48         unsigned int cpu;
  49         int ret;
  50
  51         if (argc != 1) {
  52                 *error = "io-affinity ps: invalid number of arguments";
  53                 return -EINVAL;
  54         }
  55
  56         pi = kzalloc(sizeof(*pi), GFP_KERNEL);
  57         if (!pi) {
  58                 *error = "io-affinity ps: Error allocating path context";
  59                 return -ENOMEM;
  60         }
  61
  62         pi->path = path;
  63         path->pscontext = pi;
  64         refcount_set(&pi->refcount, 1);
  65
  66         if (!zalloc_cpumask_var(&pi->cpumask, GFP_KERNEL)) {
  67                 *error = "io-affinity ps: Error allocating cpumask context";
  68                 ret = -ENOMEM;
  69                 goto free_pi;
  70         }
  71
  72         ret = cpumask_parse(argv[0], pi->cpumask);
  73         if (ret) {
  74                 *error = "io-affinity ps: invalid cpumask";
  75                 ret = -EINVAL;
  76                 goto free_mask;
  77         }
  78
  79         for_each_cpu(cpu, pi->cpumask) {
  80                 if (cpu >= nr_cpu_ids) {
  81                         DMWARN_LIMIT("Ignoring mapping for CPU %u. Max CPU is %u",
  82                                      cpu, nr_cpu_ids);
  83                         break;
  84                 }
  85
  86                 if (s->path_map[cpu]) {
  87                         DMWARN("CPU mapping for %u exists. Ignoring.", cpu);
  88                         continue;
  89                 }
  90
  91                 cpumask_set_cpu(cpu, s->path_mask);
  92                 s->path_map[cpu] = pi;
  93                 refcount_inc(&pi->refcount);
  94                 continue;
  95         }
  96
  97         if (refcount_dec_and_test(&pi->refcount)) {
  98                 *error = "io-affinity ps: No new/valid CPU mapping found";
  99                 ret = -EINVAL;
 100                 goto free_mask;
 101         }
 102
 103         return 0;
 104
 105 free_mask:
 106         free_cpumask_var(pi->cpumask);
 107 free_pi:
 108         kfree(pi);
 109         return ret;
 110 }
 111
 112 static int ioa_create(struct path_selector *ps, unsigned argc, char **argv)
 113 {
 114         struct selector *s;
 115
 116         s = kmalloc(sizeof(*s), GFP_KERNEL);
 117         if (!s)
 118                 return -ENOMEM;
 119
 120         s->path_map = kzalloc(nr_cpu_ids * sizeof(struct path_info *),
 121                               GFP_KERNEL);
 122         if (!s->path_map)
 123                 goto free_selector;
 124
 125         if (!zalloc_cpumask_var(&s->path_mask, GFP_KERNEL))
 126                 goto free_map;
 127
 128         atomic_set(&s->map_misses, 0);
 129         ps->context = s;
 130         return 0;
 131
 132 free_map:
 133         kfree(s->path_map);
 134 free_selector:
 135         kfree(s);
 136         return -ENOMEM;
 137 }
 138
 139 static void ioa_destroy(struct path_selector *ps)
 140 {
 141         struct selector *s = ps->context;
 142         unsigned cpu;
 143
 144         for_each_cpu(cpu, s->path_mask)
 145                 ioa_free_path(s, cpu);
 146
 147         free_cpumask_var(s->path_mask);
 148         kfree(s->path_map);
 149         kfree(s);
 150
 151         ps->context = NULL;
 152 }
 153
 154 static int ioa_status(struct path_selector *ps, struct dm_path *path,
 155                       status_type_t type, char *result, unsigned int maxlen)
 156 {
 157         struct selector *s = ps->context;
 158         struct path_info *pi;
 159         int sz = 0;
 160
 161         if (!path) {
 162                 DMEMIT("0 ");
 163                 return sz;
 164         }
 165
 166         switch(type) {
 167         case STATUSTYPE_INFO:
 168                 DMEMIT("%d ", atomic_read(&s->map_misses));
 169                 break;
 170         case STATUSTYPE_TABLE:
 171                 pi = path->pscontext;
 172                 DMEMIT("%*pb ", cpumask_pr_args(pi->cpumask));
 173                 break;
 174         }
 175
 176         return sz;
 177 }
 178
 179 static void ioa_fail_path(struct path_selector *ps, struct dm_path *p)
 180 {
 181         struct path_info *pi = p->pscontext;
 182
 183         pi->failed = true;
 184 }
 185
 186 static int ioa_reinstate_path(struct path_selector *ps, struct dm_path *p)
 187 {
 188         struct path_info *pi = p->pscontext;
 189
 190         pi->failed = false;
 191         return 0;
 192 }
 193
 194 static struct dm_path *ioa_select_path(struct path_selector *ps,
 195                                        size_t nr_bytes)
 196 {
 197         unsigned int cpu, node;
 198         struct selector *s = ps->context;
 199         const struct cpumask *cpumask;
 200         struct path_info *pi;
 201         int i;
 202
 203         cpu = get_cpu();
 204
 205         pi = s->path_map[cpu];
 206         if (pi && !pi->failed)
 207                 goto done;
 208
 209         /*
 210          * Perf is not optimal, but we at least try the local node then just
 211          * try not to fail.
 212          */
 213         if (!pi)
 214                 atomic_inc(&s->map_misses);
 215
 216         node = cpu_to_node(cpu);
 217         cpumask = cpumask_of_node(node);
 218         for_each_cpu(i, cpumask) {
 219                 pi = s->path_map[i];
 220                 if (pi && !pi->failed)
 221                         goto done;
 222         }
 223
 224         for_each_cpu(i, s->path_mask) {
 225                 pi = s->path_map[i];
 226                 if (pi && !pi->failed)
 227                         goto done;
 228         }
 229         pi = NULL;
 230
 231 done:
 232         put_cpu();
 233         return pi ? pi->path : NULL;
 234 }
 235
 236 static struct path_selector_type ioa_ps = {
 237         .name           = "io-affinity",
 238         .module         = THIS_MODULE,
 239         .table_args     = 1,
 240         .info_args      = 1,
 241         .create         = ioa_create,
 242         .destroy        = ioa_destroy,
 243         .status         = ioa_status,
 244         .add_path       = ioa_add_path,
 245         .fail_path      = ioa_fail_path,
 246         .reinstate_path = ioa_reinstate_path,
 247         .select_path    = ioa_select_path,
 248 };
 249
 250 static int __init dm_ioa_init(void)
 251 {
 252         int ret = dm_register_path_selector(&ioa_ps);
 253
 254         if (ret < 0)
 255                 DMERR("register failed %d", ret);
 256         return ret;
 257 }
 258
 259 static void __exit dm_ioa_exit(void)
 260 {
 261         int ret = dm_unregister_path_selector(&ioa_ps);
 262
 263         if (ret < 0)
 264                 DMERR("unregister failed %d", ret);
 265 }
 266
 267 module_init(dm_ioa_init);
 268 module_exit(dm_ioa_exit);
 269
 270 MODULE_DESCRIPTION(DM_NAME " multipath path selector that selects paths based on the CPU IO is being executed on");
 271 MODULE_AUTHOR("Mike Christie <[email protected]>");
 272 MODULE_LICENSE("GPL");