]> Git Repo - J-linux.git/blob - arch/s390/kernel/topology.c
Merge tag 'vfs-6.13-rc7.fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs
[J-linux.git] / arch / s390 / kernel / topology.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *    Copyright IBM Corp. 2007, 2011
4  */
5
6 #define KMSG_COMPONENT "cpu"
7 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
8
9 #include <linux/workqueue.h>
10 #include <linux/memblock.h>
11 #include <linux/uaccess.h>
12 #include <linux/sysctl.h>
13 #include <linux/cpuset.h>
14 #include <linux/device.h>
15 #include <linux/export.h>
16 #include <linux/kernel.h>
17 #include <linux/sched.h>
18 #include <linux/sched/topology.h>
19 #include <linux/delay.h>
20 #include <linux/init.h>
21 #include <linux/slab.h>
22 #include <linux/cpu.h>
23 #include <linux/smp.h>
24 #include <linux/mm.h>
25 #include <linux/nodemask.h>
26 #include <linux/node.h>
27 #include <asm/hiperdispatch.h>
28 #include <asm/sysinfo.h>
29 #include <asm/asm.h>
30
31 #define PTF_HORIZONTAL  (0UL)
32 #define PTF_VERTICAL    (1UL)
33 #define PTF_CHECK       (2UL)
34
35 enum {
36         TOPOLOGY_MODE_HW,
37         TOPOLOGY_MODE_SINGLE,
38         TOPOLOGY_MODE_PACKAGE,
39         TOPOLOGY_MODE_UNINITIALIZED
40 };
41
42 struct mask_info {
43         struct mask_info *next;
44         unsigned char id;
45         cpumask_t mask;
46 };
47
48 static int topology_mode = TOPOLOGY_MODE_UNINITIALIZED;
49 static void set_topology_timer(void);
50 static void topology_work_fn(struct work_struct *work);
51 static struct sysinfo_15_1_x *tl_info;
52 static int cpu_management;
53
54 static DECLARE_WORK(topology_work, topology_work_fn);
55
56 /*
57  * Socket/Book linked lists and cpu_topology updates are
58  * protected by "sched_domains_mutex".
59  */
60 static struct mask_info socket_info;
61 static struct mask_info book_info;
62 static struct mask_info drawer_info;
63
64 struct cpu_topology_s390 cpu_topology[NR_CPUS];
65 EXPORT_SYMBOL_GPL(cpu_topology);
66
67 static void cpu_group_map(cpumask_t *dst, struct mask_info *info, unsigned int cpu)
68 {
69         static cpumask_t mask;
70
71         cpumask_clear(&mask);
72         if (!cpumask_test_cpu(cpu, &cpu_setup_mask))
73                 goto out;
74         cpumask_set_cpu(cpu, &mask);
75         switch (topology_mode) {
76         case TOPOLOGY_MODE_HW:
77                 while (info) {
78                         if (cpumask_test_cpu(cpu, &info->mask)) {
79                                 cpumask_copy(&mask, &info->mask);
80                                 break;
81                         }
82                         info = info->next;
83                 }
84                 break;
85         case TOPOLOGY_MODE_PACKAGE:
86                 cpumask_copy(&mask, cpu_present_mask);
87                 break;
88         default:
89                 fallthrough;
90         case TOPOLOGY_MODE_SINGLE:
91                 break;
92         }
93         cpumask_and(&mask, &mask, &cpu_setup_mask);
94 out:
95         cpumask_copy(dst, &mask);
96 }
97
98 static void cpu_thread_map(cpumask_t *dst, unsigned int cpu)
99 {
100         static cpumask_t mask;
101         unsigned int max_cpu;
102
103         cpumask_clear(&mask);
104         if (!cpumask_test_cpu(cpu, &cpu_setup_mask))
105                 goto out;
106         cpumask_set_cpu(cpu, &mask);
107         if (topology_mode != TOPOLOGY_MODE_HW)
108                 goto out;
109         cpu -= cpu % (smp_cpu_mtid + 1);
110         max_cpu = min(cpu + smp_cpu_mtid, nr_cpu_ids - 1);
111         for (; cpu <= max_cpu; cpu++) {
112                 if (cpumask_test_cpu(cpu, &cpu_setup_mask))
113                         cpumask_set_cpu(cpu, &mask);
114         }
115 out:
116         cpumask_copy(dst, &mask);
117 }
118
119 #define TOPOLOGY_CORE_BITS      64
120
121 static void add_cpus_to_mask(struct topology_core *tl_core,
122                              struct mask_info *drawer,
123                              struct mask_info *book,
124                              struct mask_info *socket)
125 {
126         struct cpu_topology_s390 *topo;
127         unsigned int core;
128
129         for_each_set_bit(core, &tl_core->mask, TOPOLOGY_CORE_BITS) {
130                 unsigned int max_cpu, rcore;
131                 int cpu;
132
133                 rcore = TOPOLOGY_CORE_BITS - 1 - core + tl_core->origin;
134                 cpu = smp_find_processor_id(rcore << smp_cpu_mt_shift);
135                 if (cpu < 0)
136                         continue;
137                 max_cpu = min(cpu + smp_cpu_mtid, nr_cpu_ids - 1);
138                 for (; cpu <= max_cpu; cpu++) {
139                         topo = &cpu_topology[cpu];
140                         topo->drawer_id = drawer->id;
141                         topo->book_id = book->id;
142                         topo->socket_id = socket->id;
143                         topo->core_id = rcore;
144                         topo->thread_id = cpu;
145                         topo->dedicated = tl_core->d;
146                         cpumask_set_cpu(cpu, &drawer->mask);
147                         cpumask_set_cpu(cpu, &book->mask);
148                         cpumask_set_cpu(cpu, &socket->mask);
149                         smp_cpu_set_polarization(cpu, tl_core->pp);
150                         smp_cpu_set_capacity(cpu, CPU_CAPACITY_HIGH);
151                 }
152         }
153 }
154
155 static void clear_masks(void)
156 {
157         struct mask_info *info;
158
159         info = &socket_info;
160         while (info) {
161                 cpumask_clear(&info->mask);
162                 info = info->next;
163         }
164         info = &book_info;
165         while (info) {
166                 cpumask_clear(&info->mask);
167                 info = info->next;
168         }
169         info = &drawer_info;
170         while (info) {
171                 cpumask_clear(&info->mask);
172                 info = info->next;
173         }
174 }
175
176 static union topology_entry *next_tle(union topology_entry *tle)
177 {
178         if (!tle->nl)
179                 return (union topology_entry *)((struct topology_core *)tle + 1);
180         return (union topology_entry *)((struct topology_container *)tle + 1);
181 }
182
183 static void tl_to_masks(struct sysinfo_15_1_x *info)
184 {
185         struct mask_info *socket = &socket_info;
186         struct mask_info *book = &book_info;
187         struct mask_info *drawer = &drawer_info;
188         union topology_entry *tle, *end;
189
190         clear_masks();
191         tle = info->tle;
192         end = (union topology_entry *)((unsigned long)info + info->length);
193         while (tle < end) {
194                 switch (tle->nl) {
195                 case 3:
196                         drawer = drawer->next;
197                         drawer->id = tle->container.id;
198                         break;
199                 case 2:
200                         book = book->next;
201                         book->id = tle->container.id;
202                         break;
203                 case 1:
204                         socket = socket->next;
205                         socket->id = tle->container.id;
206                         break;
207                 case 0:
208                         add_cpus_to_mask(&tle->cpu, drawer, book, socket);
209                         break;
210                 default:
211                         clear_masks();
212                         return;
213                 }
214                 tle = next_tle(tle);
215         }
216 }
217
218 static void topology_update_polarization_simple(void)
219 {
220         int cpu;
221
222         for_each_possible_cpu(cpu)
223                 smp_cpu_set_polarization(cpu, POLARIZATION_HRZ);
224 }
225
226 static int ptf(unsigned long fc)
227 {
228         int cc;
229
230         asm volatile(
231                 "       .insn   rre,0xb9a20000,%[fc],%[fc]\n"
232                 CC_IPM(cc)
233                 : CC_OUT(cc, cc)
234                 : [fc] "d" (fc)
235                 : CC_CLOBBER);
236         return CC_TRANSFORM(cc);
237 }
238
239 int topology_set_cpu_management(int fc)
240 {
241         int cpu, rc;
242
243         if (!MACHINE_HAS_TOPOLOGY)
244                 return -EOPNOTSUPP;
245         if (fc)
246                 rc = ptf(PTF_VERTICAL);
247         else
248                 rc = ptf(PTF_HORIZONTAL);
249         if (rc)
250                 return -EBUSY;
251         for_each_possible_cpu(cpu)
252                 smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
253         return rc;
254 }
255
256 void update_cpu_masks(void)
257 {
258         struct cpu_topology_s390 *topo, *topo_package, *topo_sibling;
259         int cpu, sibling, pkg_first, smt_first, id;
260
261         for_each_possible_cpu(cpu) {
262                 topo = &cpu_topology[cpu];
263                 cpu_thread_map(&topo->thread_mask, cpu);
264                 cpu_group_map(&topo->core_mask, &socket_info, cpu);
265                 cpu_group_map(&topo->book_mask, &book_info, cpu);
266                 cpu_group_map(&topo->drawer_mask, &drawer_info, cpu);
267                 topo->booted_cores = 0;
268                 if (topology_mode != TOPOLOGY_MODE_HW) {
269                         id = topology_mode == TOPOLOGY_MODE_PACKAGE ? 0 : cpu;
270                         topo->thread_id = cpu;
271                         topo->core_id = cpu;
272                         topo->socket_id = id;
273                         topo->book_id = id;
274                         topo->drawer_id = id;
275                 }
276         }
277         hd_reset_state();
278         for_each_online_cpu(cpu) {
279                 topo = &cpu_topology[cpu];
280                 pkg_first = cpumask_first(&topo->core_mask);
281                 topo_package = &cpu_topology[pkg_first];
282                 if (cpu == pkg_first) {
283                         for_each_cpu(sibling, &topo->core_mask) {
284                                 topo_sibling = &cpu_topology[sibling];
285                                 smt_first = cpumask_first(&topo_sibling->thread_mask);
286                                 if (sibling == smt_first) {
287                                         topo_package->booted_cores++;
288                                         hd_add_core(sibling);
289                                 }
290                         }
291                 } else {
292                         topo->booted_cores = topo_package->booted_cores;
293                 }
294         }
295 }
296
297 void store_topology(struct sysinfo_15_1_x *info)
298 {
299         stsi(info, 15, 1, topology_mnest_limit());
300 }
301
302 static void __arch_update_dedicated_flag(void *arg)
303 {
304         if (topology_cpu_dedicated(smp_processor_id()))
305                 set_cpu_flag(CIF_DEDICATED_CPU);
306         else
307                 clear_cpu_flag(CIF_DEDICATED_CPU);
308 }
309
310 static int __arch_update_cpu_topology(void)
311 {
312         struct sysinfo_15_1_x *info = tl_info;
313         int rc, hd_status;
314
315         hd_status = 0;
316         rc = 0;
317         mutex_lock(&smp_cpu_state_mutex);
318         if (MACHINE_HAS_TOPOLOGY) {
319                 rc = 1;
320                 store_topology(info);
321                 tl_to_masks(info);
322         }
323         update_cpu_masks();
324         if (!MACHINE_HAS_TOPOLOGY)
325                 topology_update_polarization_simple();
326         if (cpu_management == 1)
327                 hd_status = hd_enable_hiperdispatch();
328         mutex_unlock(&smp_cpu_state_mutex);
329         if (hd_status == 0)
330                 hd_disable_hiperdispatch();
331         return rc;
332 }
333
334 int arch_update_cpu_topology(void)
335 {
336         int rc;
337
338         rc = __arch_update_cpu_topology();
339         on_each_cpu(__arch_update_dedicated_flag, NULL, 0);
340         return rc;
341 }
342
343 static void topology_work_fn(struct work_struct *work)
344 {
345         rebuild_sched_domains();
346 }
347
348 void topology_schedule_update(void)
349 {
350         schedule_work(&topology_work);
351 }
352
353 static void topology_flush_work(void)
354 {
355         flush_work(&topology_work);
356 }
357
358 static void topology_timer_fn(struct timer_list *unused)
359 {
360         if (ptf(PTF_CHECK))
361                 topology_schedule_update();
362         set_topology_timer();
363 }
364
365 static struct timer_list topology_timer;
366
367 static atomic_t topology_poll = ATOMIC_INIT(0);
368
369 static void set_topology_timer(void)
370 {
371         if (atomic_add_unless(&topology_poll, -1, 0))
372                 mod_timer(&topology_timer, jiffies + msecs_to_jiffies(100));
373         else
374                 mod_timer(&topology_timer, jiffies + msecs_to_jiffies(60 * MSEC_PER_SEC));
375 }
376
377 void topology_expect_change(void)
378 {
379         if (!MACHINE_HAS_TOPOLOGY)
380                 return;
381         /* This is racy, but it doesn't matter since it is just a heuristic.
382          * Worst case is that we poll in a higher frequency for a bit longer.
383          */
384         if (atomic_read(&topology_poll) > 60)
385                 return;
386         atomic_add(60, &topology_poll);
387         set_topology_timer();
388 }
389
390 static int set_polarization(int polarization)
391 {
392         int rc = 0;
393
394         cpus_read_lock();
395         mutex_lock(&smp_cpu_state_mutex);
396         if (cpu_management == polarization)
397                 goto out;
398         rc = topology_set_cpu_management(polarization);
399         if (rc)
400                 goto out;
401         cpu_management = polarization;
402         topology_expect_change();
403 out:
404         mutex_unlock(&smp_cpu_state_mutex);
405         cpus_read_unlock();
406         return rc;
407 }
408
409 static ssize_t dispatching_show(struct device *dev,
410                                 struct device_attribute *attr,
411                                 char *buf)
412 {
413         ssize_t count;
414
415         mutex_lock(&smp_cpu_state_mutex);
416         count = sysfs_emit(buf, "%d\n", cpu_management);
417         mutex_unlock(&smp_cpu_state_mutex);
418         return count;
419 }
420
421 static ssize_t dispatching_store(struct device *dev,
422                                  struct device_attribute *attr,
423                                  const char *buf,
424                                  size_t count)
425 {
426         int val, rc;
427         char delim;
428
429         if (sscanf(buf, "%d %c", &val, &delim) != 1)
430                 return -EINVAL;
431         if (val != 0 && val != 1)
432                 return -EINVAL;
433         rc = set_polarization(val);
434         return rc ? rc : count;
435 }
436 static DEVICE_ATTR_RW(dispatching);
437
438 static ssize_t cpu_polarization_show(struct device *dev,
439                                      struct device_attribute *attr, char *buf)
440 {
441         int cpu = dev->id;
442         ssize_t count;
443
444         mutex_lock(&smp_cpu_state_mutex);
445         switch (smp_cpu_get_polarization(cpu)) {
446         case POLARIZATION_HRZ:
447                 count = sysfs_emit(buf, "horizontal\n");
448                 break;
449         case POLARIZATION_VL:
450                 count = sysfs_emit(buf, "vertical:low\n");
451                 break;
452         case POLARIZATION_VM:
453                 count = sysfs_emit(buf, "vertical:medium\n");
454                 break;
455         case POLARIZATION_VH:
456                 count = sysfs_emit(buf, "vertical:high\n");
457                 break;
458         default:
459                 count = sysfs_emit(buf, "unknown\n");
460                 break;
461         }
462         mutex_unlock(&smp_cpu_state_mutex);
463         return count;
464 }
465 static DEVICE_ATTR(polarization, 0444, cpu_polarization_show, NULL);
466
467 static struct attribute *topology_cpu_attrs[] = {
468         &dev_attr_polarization.attr,
469         NULL,
470 };
471
472 static struct attribute_group topology_cpu_attr_group = {
473         .attrs = topology_cpu_attrs,
474 };
475
476 static ssize_t cpu_dedicated_show(struct device *dev,
477                                   struct device_attribute *attr, char *buf)
478 {
479         int cpu = dev->id;
480         ssize_t count;
481
482         mutex_lock(&smp_cpu_state_mutex);
483         count = sysfs_emit(buf, "%d\n", topology_cpu_dedicated(cpu));
484         mutex_unlock(&smp_cpu_state_mutex);
485         return count;
486 }
487 static DEVICE_ATTR(dedicated, 0444, cpu_dedicated_show, NULL);
488
489 static struct attribute *topology_extra_cpu_attrs[] = {
490         &dev_attr_dedicated.attr,
491         NULL,
492 };
493
494 static struct attribute_group topology_extra_cpu_attr_group = {
495         .attrs = topology_extra_cpu_attrs,
496 };
497
498 int topology_cpu_init(struct cpu *cpu)
499 {
500         int rc;
501
502         rc = sysfs_create_group(&cpu->dev.kobj, &topology_cpu_attr_group);
503         if (rc || !MACHINE_HAS_TOPOLOGY)
504                 return rc;
505         rc = sysfs_create_group(&cpu->dev.kobj, &topology_extra_cpu_attr_group);
506         if (rc)
507                 sysfs_remove_group(&cpu->dev.kobj, &topology_cpu_attr_group);
508         return rc;
509 }
510
511 static const struct cpumask *cpu_thread_mask(int cpu)
512 {
513         return &cpu_topology[cpu].thread_mask;
514 }
515
516
517 const struct cpumask *cpu_coregroup_mask(int cpu)
518 {
519         return &cpu_topology[cpu].core_mask;
520 }
521
522 static const struct cpumask *cpu_book_mask(int cpu)
523 {
524         return &cpu_topology[cpu].book_mask;
525 }
526
527 static const struct cpumask *cpu_drawer_mask(int cpu)
528 {
529         return &cpu_topology[cpu].drawer_mask;
530 }
531
532 static struct sched_domain_topology_level s390_topology[] = {
533         { cpu_thread_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
534         { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
535         { cpu_book_mask, SD_INIT_NAME(BOOK) },
536         { cpu_drawer_mask, SD_INIT_NAME(DRAWER) },
537         { cpu_cpu_mask, SD_INIT_NAME(PKG) },
538         { NULL, },
539 };
540
541 static void __init alloc_masks(struct sysinfo_15_1_x *info,
542                                struct mask_info *mask, int offset)
543 {
544         int i, nr_masks;
545
546         nr_masks = info->mag[TOPOLOGY_NR_MAG - offset];
547         for (i = 0; i < info->mnest - offset; i++)
548                 nr_masks *= info->mag[TOPOLOGY_NR_MAG - offset - 1 - i];
549         nr_masks = max(nr_masks, 1);
550         for (i = 0; i < nr_masks; i++) {
551                 mask->next = memblock_alloc(sizeof(*mask->next), 8);
552                 if (!mask->next)
553                         panic("%s: Failed to allocate %zu bytes align=0x%x\n",
554                               __func__, sizeof(*mask->next), 8);
555                 mask = mask->next;
556         }
557 }
558
559 void __init topology_init_early(void)
560 {
561         struct sysinfo_15_1_x *info;
562
563         set_sched_topology(s390_topology);
564         if (topology_mode == TOPOLOGY_MODE_UNINITIALIZED) {
565                 if (MACHINE_HAS_TOPOLOGY)
566                         topology_mode = TOPOLOGY_MODE_HW;
567                 else
568                         topology_mode = TOPOLOGY_MODE_SINGLE;
569         }
570         if (!MACHINE_HAS_TOPOLOGY)
571                 goto out;
572         tl_info = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
573         if (!tl_info)
574                 panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
575                       __func__, PAGE_SIZE, PAGE_SIZE);
576         info = tl_info;
577         store_topology(info);
578         pr_info("The CPU configuration topology of the machine is: %d %d %d %d %d %d / %d\n",
579                 info->mag[0], info->mag[1], info->mag[2], info->mag[3],
580                 info->mag[4], info->mag[5], info->mnest);
581         alloc_masks(info, &socket_info, 1);
582         alloc_masks(info, &book_info, 2);
583         alloc_masks(info, &drawer_info, 3);
584 out:
585         cpumask_set_cpu(0, &cpu_setup_mask);
586         __arch_update_cpu_topology();
587         __arch_update_dedicated_flag(NULL);
588 }
589
590 static inline int topology_get_mode(int enabled)
591 {
592         if (!enabled)
593                 return TOPOLOGY_MODE_SINGLE;
594         return MACHINE_HAS_TOPOLOGY ? TOPOLOGY_MODE_HW : TOPOLOGY_MODE_PACKAGE;
595 }
596
597 static inline int topology_is_enabled(void)
598 {
599         return topology_mode != TOPOLOGY_MODE_SINGLE;
600 }
601
602 static int __init topology_setup(char *str)
603 {
604         bool enabled;
605         int rc;
606
607         rc = kstrtobool(str, &enabled);
608         if (rc)
609                 return rc;
610         topology_mode = topology_get_mode(enabled);
611         return 0;
612 }
613 early_param("topology", topology_setup);
614
615 static int topology_ctl_handler(const struct ctl_table *ctl, int write,
616                                 void *buffer, size_t *lenp, loff_t *ppos)
617 {
618         int enabled = topology_is_enabled();
619         int new_mode;
620         int rc;
621         struct ctl_table ctl_entry = {
622                 .procname       = ctl->procname,
623                 .data           = &enabled,
624                 .maxlen         = sizeof(int),
625                 .extra1         = SYSCTL_ZERO,
626                 .extra2         = SYSCTL_ONE,
627         };
628
629         rc = proc_douintvec_minmax(&ctl_entry, write, buffer, lenp, ppos);
630         if (rc < 0 || !write)
631                 return rc;
632
633         mutex_lock(&smp_cpu_state_mutex);
634         new_mode = topology_get_mode(enabled);
635         if (topology_mode != new_mode) {
636                 topology_mode = new_mode;
637                 topology_schedule_update();
638         }
639         mutex_unlock(&smp_cpu_state_mutex);
640         topology_flush_work();
641
642         return rc;
643 }
644
645 static int polarization_ctl_handler(const struct ctl_table *ctl, int write,
646                                     void *buffer, size_t *lenp, loff_t *ppos)
647 {
648         int polarization;
649         int rc;
650         struct ctl_table ctl_entry = {
651                 .procname       = ctl->procname,
652                 .data           = &polarization,
653                 .maxlen         = sizeof(int),
654                 .extra1         = SYSCTL_ZERO,
655                 .extra2         = SYSCTL_ONE,
656         };
657
658         polarization = cpu_management;
659         rc = proc_douintvec_minmax(&ctl_entry, write, buffer, lenp, ppos);
660         if (rc < 0 || !write)
661                 return rc;
662         return set_polarization(polarization);
663 }
664
665 static struct ctl_table topology_ctl_table[] = {
666         {
667                 .procname       = "topology",
668                 .mode           = 0644,
669                 .proc_handler   = topology_ctl_handler,
670         },
671         {
672                 .procname       = "polarization",
673                 .mode           = 0644,
674                 .proc_handler   = polarization_ctl_handler,
675         },
676 };
677
678 static int __init topology_init(void)
679 {
680         struct device *dev_root;
681         int rc = 0;
682
683         timer_setup(&topology_timer, topology_timer_fn, TIMER_DEFERRABLE);
684         if (MACHINE_HAS_TOPOLOGY)
685                 set_topology_timer();
686         else
687                 topology_update_polarization_simple();
688         if (IS_ENABLED(CONFIG_SCHED_TOPOLOGY_VERTICAL))
689                 set_polarization(1);
690         register_sysctl("s390", topology_ctl_table);
691
692         dev_root = bus_get_dev_root(&cpu_subsys);
693         if (dev_root) {
694                 rc = device_create_file(dev_root, &dev_attr_dispatching);
695                 put_device(dev_root);
696         }
697         return rc;
698 }
699 device_initcall(topology_init);
This page took 0.063672 seconds and 4 git commands to generate.