drivers/acpi/numa/srat.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  *  acpi_numa.c - ACPI NUMA support
   4  *
   5  *  Copyright (C) 2002 Takayoshi Kochi <[email protected]>
   6  */
   7
   8 #define pr_fmt(fmt) "ACPI: " fmt
   9
  10 #include <linux/module.h>
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/types.h>
  14 #include <linux/errno.h>
  15 #include <linux/acpi.h>
  16 #include <linux/memblock.h>
  17 #include <linux/numa.h>
  18 #include <linux/nodemask.h>
  19 #include <linux/topology.h>
  20 #include <linux/numa_memblks.h>
  21
  22 static nodemask_t nodes_found_map = NODE_MASK_NONE;
  23
  24 /* maps to convert between proximity domain and logical node ID */
  25 static int pxm_to_node_map[MAX_PXM_DOMAINS]
  26                         = { [0 ... MAX_PXM_DOMAINS - 1] = NUMA_NO_NODE };
  27 static int node_to_pxm_map[MAX_NUMNODES]
  28                         = { [0 ... MAX_NUMNODES - 1] = PXM_INVAL };
  29
  30 unsigned char acpi_srat_revision __initdata;
  31 static int acpi_numa __initdata;
  32
  33 static int last_real_pxm;
  34
  35 void __init disable_srat(void)
  36 {
  37         acpi_numa = -1;
  38 }
  39
  40 int pxm_to_node(int pxm)
  41 {
  42         if (pxm < 0 || pxm >= MAX_PXM_DOMAINS || numa_off)
  43                 return NUMA_NO_NODE;
  44         return pxm_to_node_map[pxm];
  45 }
  46 EXPORT_SYMBOL(pxm_to_node);
  47
  48 int node_to_pxm(int node)
  49 {
  50         if (node < 0)
  51                 return PXM_INVAL;
  52         return node_to_pxm_map[node];
  53 }
  54
  55 static void __acpi_map_pxm_to_node(int pxm, int node)
  56 {
  57         if (pxm_to_node_map[pxm] == NUMA_NO_NODE || node < pxm_to_node_map[pxm])
  58                 pxm_to_node_map[pxm] = node;
  59         if (node_to_pxm_map[node] == PXM_INVAL || pxm < node_to_pxm_map[node])
  60                 node_to_pxm_map[node] = pxm;
  61 }
  62
  63 int acpi_map_pxm_to_node(int pxm)
  64 {
  65         int node;
  66
  67         if (pxm < 0 || pxm >= MAX_PXM_DOMAINS || numa_off)
  68                 return NUMA_NO_NODE;
  69
  70         node = pxm_to_node_map[pxm];
  71
  72         if (node == NUMA_NO_NODE) {
  73                 node = first_unset_node(nodes_found_map);
  74                 if (node >= MAX_NUMNODES)
  75                         return NUMA_NO_NODE;
  76                 __acpi_map_pxm_to_node(pxm, node);
  77                 node_set(node, nodes_found_map);
  78         }
  79
  80         return node;
  81 }
  82 EXPORT_SYMBOL(acpi_map_pxm_to_node);
  83
  84 static void __init
  85 acpi_table_print_srat_entry(struct acpi_subtable_header *header)
  86 {
  87         switch (header->type) {
  88         case ACPI_SRAT_TYPE_CPU_AFFINITY:
  89                 {
  90                         struct acpi_srat_cpu_affinity *p =
  91                             (struct acpi_srat_cpu_affinity *)header;
  92                         pr_debug("SRAT Processor (id[0x%02x] eid[0x%02x]) in proximity domain %d %s\n",
  93                                  p->apic_id, p->local_sapic_eid,
  94                                  p->proximity_domain_lo,
  95                                  (p->flags & ACPI_SRAT_CPU_ENABLED) ?
  96                                  "enabled" : "disabled");
  97                 }
  98                 break;
  99
 100         case ACPI_SRAT_TYPE_MEMORY_AFFINITY:
 101                 {
 102                         struct acpi_srat_mem_affinity *p =
 103                             (struct acpi_srat_mem_affinity *)header;
 104                         pr_debug("SRAT Memory (0x%llx length 0x%llx) in proximity domain %d %s%s%s\n",
 105                                  (unsigned long long)p->base_address,
 106                                  (unsigned long long)p->length,
 107                                  p->proximity_domain,
 108                                  (p->flags & ACPI_SRAT_MEM_ENABLED) ?
 109                                  "enabled" : "disabled",
 110                                  (p->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) ?
 111                                  " hot-pluggable" : "",
 112                                  (p->flags & ACPI_SRAT_MEM_NON_VOLATILE) ?
 113                                  " non-volatile" : "");
 114                 }
 115                 break;
 116
 117         case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY:
 118                 {
 119                         struct acpi_srat_x2apic_cpu_affinity *p =
 120                             (struct acpi_srat_x2apic_cpu_affinity *)header;
 121                         pr_debug("SRAT Processor (x2apicid[0x%08x]) in proximity domain %d %s\n",
 122                                  p->apic_id,
 123                                  p->proximity_domain,
 124                                  (p->flags & ACPI_SRAT_CPU_ENABLED) ?
 125                                  "enabled" : "disabled");
 126                 }
 127                 break;
 128
 129         case ACPI_SRAT_TYPE_GICC_AFFINITY:
 130                 {
 131                         struct acpi_srat_gicc_affinity *p =
 132                             (struct acpi_srat_gicc_affinity *)header;
 133                         pr_debug("SRAT Processor (acpi id[0x%04x]) in proximity domain %d %s\n",
 134                                  p->acpi_processor_uid,
 135                                  p->proximity_domain,
 136                                  (p->flags & ACPI_SRAT_GICC_ENABLED) ?
 137                                  "enabled" : "disabled");
 138                 }
 139                 break;
 140
 141         case ACPI_SRAT_TYPE_GENERIC_AFFINITY:
 142         {
 143                 struct acpi_srat_generic_affinity *p =
 144                         (struct acpi_srat_generic_affinity *)header;
 145
 146                 if (p->device_handle_type == 0) {
 147                         /*
 148                          * For pci devices this may be the only place they
 149                          * are assigned a proximity domain
 150                          */
 151                         pr_debug("SRAT Generic Initiator(Seg:%u BDF:%u) in proximity domain %d %s\n",
 152                                  *(u16 *)(&p->device_handle[0]),
 153                                  *(u16 *)(&p->device_handle[2]),
 154                                  p->proximity_domain,
 155                                  (p->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED) ?
 156                                 "enabled" : "disabled");
 157                 } else {
 158                         /*
 159                          * In this case we can rely on the device having a
 160                          * proximity domain reference
 161                          */
 162                         pr_debug("SRAT Generic Initiator(HID=%.8s UID=%.4s) in proximity domain %d %s\n",
 163                                 (char *)(&p->device_handle[0]),
 164                                 (char *)(&p->device_handle[8]),
 165                                 p->proximity_domain,
 166                                 (p->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED) ?
 167                                 "enabled" : "disabled");
 168                 }
 169         }
 170         break;
 171
 172         case ACPI_SRAT_TYPE_RINTC_AFFINITY:
 173                 {
 174                         struct acpi_srat_rintc_affinity *p =
 175                             (struct acpi_srat_rintc_affinity *)header;
 176                         pr_debug("SRAT Processor (acpi id[0x%04x]) in proximity domain %d %s\n",
 177                                  p->acpi_processor_uid,
 178                                  p->proximity_domain,
 179                                  (p->flags & ACPI_SRAT_RINTC_ENABLED) ?
 180                                  "enabled" : "disabled");
 181                 }
 182                 break;
 183
 184         default:
 185                 pr_warn("Found unsupported SRAT entry (type = 0x%x)\n",
 186                         header->type);
 187                 break;
 188         }
 189 }
 190
 191 /*
 192  * A lot of BIOS fill in 10 (= no distance) everywhere. This messes
 193  * up the NUMA heuristics which wants the local node to have a smaller
 194  * distance than the others.
 195  * Do some quick checks here and only use the SLIT if it passes.
 196  */
 197 static int __init slit_valid(struct acpi_table_slit *slit)
 198 {
 199         int i, j;
 200         int d = slit->locality_count;
 201         for (i = 0; i < d; i++) {
 202                 for (j = 0; j < d; j++) {
 203                         u8 val = slit->entry[d*i + j];
 204                         if (i == j) {
 205                                 if (val != LOCAL_DISTANCE)
 206                                         return 0;
 207                         } else if (val <= LOCAL_DISTANCE)
 208                                 return 0;
 209                 }
 210         }
 211         return 1;
 212 }
 213
 214 void __init bad_srat(void)
 215 {
 216         pr_err("SRAT: SRAT not used.\n");
 217         disable_srat();
 218 }
 219
 220 int __init srat_disabled(void)
 221 {
 222         return acpi_numa < 0;
 223 }
 224
 225 __weak int __init numa_fill_memblks(u64 start, u64 end)
 226 {
 227         return NUMA_NO_MEMBLK;
 228 }
 229
 230 /*
 231  * Callback for SLIT parsing.  pxm_to_node() returns NUMA_NO_NODE for
 232  * I/O localities since SRAT does not list them.  I/O localities are
 233  * not supported at this point.
 234  */
 235 static int __init acpi_parse_slit(struct acpi_table_header *table)
 236 {
 237         struct acpi_table_slit *slit = (struct acpi_table_slit *)table;
 238         int i, j;
 239
 240         if (!slit_valid(slit)) {
 241                 pr_info("SLIT table looks invalid. Not used.\n");
 242                 return -EINVAL;
 243         }
 244
 245         for (i = 0; i < slit->locality_count; i++) {
 246                 const int from_node = pxm_to_node(i);
 247
 248                 if (from_node == NUMA_NO_NODE)
 249                         continue;
 250
 251                 for (j = 0; j < slit->locality_count; j++) {
 252                         const int to_node = pxm_to_node(j);
 253
 254                         if (to_node == NUMA_NO_NODE)
 255                                 continue;
 256
 257                         numa_set_distance(from_node, to_node,
 258                                 slit->entry[slit->locality_count * i + j]);
 259                 }
 260         }
 261
 262         return 0;
 263 }
 264
 265 static int parsed_numa_memblks __initdata;
 266
 267 static int __init
 268 acpi_parse_memory_affinity(union acpi_subtable_headers *header,
 269                            const unsigned long table_end)
 270 {
 271         struct acpi_srat_mem_affinity *ma;
 272         u64 start, end;
 273         u32 hotpluggable;
 274         int node, pxm;
 275
 276         ma = (struct acpi_srat_mem_affinity *)header;
 277
 278         acpi_table_print_srat_entry(&header->common);
 279
 280         if (srat_disabled())
 281                 return 0;
 282         if (ma->header.length < sizeof(struct acpi_srat_mem_affinity)) {
 283                 pr_err("SRAT: Unexpected header length: %d\n",
 284                        ma->header.length);
 285                 goto out_err_bad_srat;
 286         }
 287         if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
 288                 return 0;
 289         hotpluggable = IS_ENABLED(CONFIG_MEMORY_HOTPLUG) &&
 290                 (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE);
 291
 292         start = ma->base_address;
 293         end = start + ma->length;
 294         pxm = ma->proximity_domain;
 295         if (acpi_srat_revision <= 1)
 296                 pxm &= 0xff;
 297
 298         node = acpi_map_pxm_to_node(pxm);
 299         if (node == NUMA_NO_NODE) {
 300                 pr_err("SRAT: Too many proximity domains.\n");
 301                 goto out_err_bad_srat;
 302         }
 303
 304         if (numa_add_memblk(node, start, end) < 0) {
 305                 pr_err("SRAT: Failed to add memblk to node %u [mem %#010Lx-%#010Lx]\n",
 306                        node, (unsigned long long) start,
 307                        (unsigned long long) end - 1);
 308                 goto out_err_bad_srat;
 309         }
 310
 311         node_set(node, numa_nodes_parsed);
 312
 313         pr_info("SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]%s%s\n",
 314                 node, pxm,
 315                 (unsigned long long) start, (unsigned long long) end - 1,
 316                 hotpluggable ? " hotplug" : "",
 317                 ma->flags & ACPI_SRAT_MEM_NON_VOLATILE ? " non-volatile" : "");
 318
 319         /* Mark hotplug range in memblock. */
 320         if (hotpluggable && memblock_mark_hotplug(start, ma->length))
 321                 pr_warn("SRAT: Failed to mark hotplug range [mem %#010Lx-%#010Lx] in memblock\n",
 322                         (unsigned long long)start, (unsigned long long)end - 1);
 323
 324         max_possible_pfn = max(max_possible_pfn, PFN_UP(end - 1));
 325
 326         parsed_numa_memblks++;
 327
 328         return 0;
 329
 330 out_err_bad_srat:
 331         /* Just disable SRAT, but do not fail and ignore errors. */
 332         bad_srat();
 333
 334         return 0;
 335 }
 336
 337 static int __init acpi_parse_cfmws(union acpi_subtable_headers *header,
 338                                    void *arg, const unsigned long table_end)
 339 {
 340         struct acpi_cedt_cfmws *cfmws;
 341         int *fake_pxm = arg;
 342         u64 start, end;
 343         int node;
 344
 345         cfmws = (struct acpi_cedt_cfmws *)header;
 346         start = cfmws->base_hpa;
 347         end = cfmws->base_hpa + cfmws->window_size;
 348
 349         /*
 350          * The SRAT may have already described NUMA details for all,
 351          * or a portion of, this CFMWS HPA range. Extend the memblks
 352          * found for any portion of the window to cover the entire
 353          * window.
 354          */
 355         if (!numa_fill_memblks(start, end))
 356                 return 0;
 357
 358         /* No SRAT description. Create a new node. */
 359         node = acpi_map_pxm_to_node(*fake_pxm);
 360
 361         if (node == NUMA_NO_NODE) {
 362                 pr_err("ACPI NUMA: Too many proximity domains while processing CFMWS.\n");
 363                 return -EINVAL;
 364         }
 365
 366         if (numa_add_memblk(node, start, end) < 0) {
 367                 /* CXL driver must handle the NUMA_NO_NODE case */
 368                 pr_warn("ACPI NUMA: Failed to add memblk for CFMWS node %d [mem %#llx-%#llx]\n",
 369                         node, start, end);
 370         }
 371         node_set(node, numa_nodes_parsed);
 372
 373         /* Set the next available fake_pxm value */
 374         (*fake_pxm)++;
 375         return 0;
 376 }
 377
 378 void __init __weak
 379 acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
 380 {
 381         pr_warn("Found unsupported x2apic [0x%08x] SRAT entry\n", pa->apic_id);
 382 }
 383
 384 static int __init
 385 acpi_parse_x2apic_affinity(union acpi_subtable_headers *header,
 386                            const unsigned long end)
 387 {
 388         struct acpi_srat_x2apic_cpu_affinity *processor_affinity;
 389
 390         processor_affinity = (struct acpi_srat_x2apic_cpu_affinity *)header;
 391
 392         acpi_table_print_srat_entry(&header->common);
 393
 394         /* let architecture-dependent part to do it */
 395         acpi_numa_x2apic_affinity_init(processor_affinity);
 396
 397         return 0;
 398 }
 399
 400 static int __init
 401 acpi_parse_processor_affinity(union acpi_subtable_headers *header,
 402                               const unsigned long end)
 403 {
 404         struct acpi_srat_cpu_affinity *processor_affinity;
 405
 406         processor_affinity = (struct acpi_srat_cpu_affinity *)header;
 407
 408         acpi_table_print_srat_entry(&header->common);
 409
 410         /* let architecture-dependent part to do it */
 411         acpi_numa_processor_affinity_init(processor_affinity);
 412
 413         return 0;
 414 }
 415
 416 static int __init
 417 acpi_parse_gicc_affinity(union acpi_subtable_headers *header,
 418                          const unsigned long end)
 419 {
 420         struct acpi_srat_gicc_affinity *processor_affinity;
 421
 422         processor_affinity = (struct acpi_srat_gicc_affinity *)header;
 423
 424         acpi_table_print_srat_entry(&header->common);
 425
 426         /* let architecture-dependent part to do it */
 427         acpi_numa_gicc_affinity_init(processor_affinity);
 428
 429         return 0;
 430 }
 431
 432 #if defined(CONFIG_X86) || defined(CONFIG_ARM64)
 433 static int __init
 434 acpi_parse_gi_affinity(union acpi_subtable_headers *header,
 435                        const unsigned long end)
 436 {
 437         struct acpi_srat_generic_affinity *gi_affinity;
 438         int node;
 439
 440         gi_affinity = (struct acpi_srat_generic_affinity *)header;
 441         if (!gi_affinity)
 442                 return -EINVAL;
 443         acpi_table_print_srat_entry(&header->common);
 444
 445         if (!(gi_affinity->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED))
 446                 return -EINVAL;
 447
 448         node = acpi_map_pxm_to_node(gi_affinity->proximity_domain);
 449         if (node == NUMA_NO_NODE) {
 450                 pr_err("SRAT: Too many proximity domains.\n");
 451                 return -EINVAL;
 452         }
 453         node_set(node, numa_nodes_parsed);
 454         node_set_state(node, N_GENERIC_INITIATOR);
 455
 456         return 0;
 457 }
 458 #else
 459 static int __init
 460 acpi_parse_gi_affinity(union acpi_subtable_headers *header,
 461                        const unsigned long end)
 462 {
 463         return 0;
 464 }
 465 #endif /* defined(CONFIG_X86) || defined (CONFIG_ARM64) */
 466
 467 static int __init
 468 acpi_parse_rintc_affinity(union acpi_subtable_headers *header,
 469                           const unsigned long end)
 470 {
 471         struct acpi_srat_rintc_affinity *rintc_affinity;
 472
 473         rintc_affinity = (struct acpi_srat_rintc_affinity *)header;
 474         acpi_table_print_srat_entry(&header->common);
 475
 476         /* let architecture-dependent part to do it */
 477         acpi_numa_rintc_affinity_init(rintc_affinity);
 478
 479         return 0;
 480 }
 481
 482 static int __init acpi_parse_srat(struct acpi_table_header *table)
 483 {
 484         struct acpi_table_srat *srat = (struct acpi_table_srat *)table;
 485
 486         acpi_srat_revision = srat->header.revision;
 487
 488         /* Real work done in acpi_table_parse_srat below. */
 489
 490         return 0;
 491 }
 492
 493 static int __init
 494 acpi_table_parse_srat(enum acpi_srat_type id,
 495                       acpi_tbl_entry_handler handler, unsigned int max_entries)
 496 {
 497         return acpi_table_parse_entries(ACPI_SIG_SRAT,
 498                                             sizeof(struct acpi_table_srat), id,
 499                                             handler, max_entries);
 500 }
 501
 502 int __init acpi_numa_init(void)
 503 {
 504         int i, fake_pxm, cnt = 0;
 505
 506         if (acpi_disabled)
 507                 return -EINVAL;
 508
 509         /*
 510          * Should not limit number with cpu num that is from NR_CPUS or nr_cpus=
 511          * SRAT cpu entries could have different order with that in MADT.
 512          * So go over all cpu entries in SRAT to get apicid to node mapping.
 513          */
 514
 515         /* SRAT: System Resource Affinity Table */
 516         if (!acpi_table_parse(ACPI_SIG_SRAT, acpi_parse_srat)) {
 517                 struct acpi_subtable_proc srat_proc[5];
 518
 519                 memset(srat_proc, 0, sizeof(srat_proc));
 520                 srat_proc[0].id = ACPI_SRAT_TYPE_CPU_AFFINITY;
 521                 srat_proc[0].handler = acpi_parse_processor_affinity;
 522                 srat_proc[1].id = ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY;
 523                 srat_proc[1].handler = acpi_parse_x2apic_affinity;
 524                 srat_proc[2].id = ACPI_SRAT_TYPE_GICC_AFFINITY;
 525                 srat_proc[2].handler = acpi_parse_gicc_affinity;
 526                 srat_proc[3].id = ACPI_SRAT_TYPE_GENERIC_AFFINITY;
 527                 srat_proc[3].handler = acpi_parse_gi_affinity;
 528                 srat_proc[4].id = ACPI_SRAT_TYPE_RINTC_AFFINITY;
 529                 srat_proc[4].handler = acpi_parse_rintc_affinity;
 530
 531                 acpi_table_parse_entries_array(ACPI_SIG_SRAT,
 532                                         sizeof(struct acpi_table_srat),
 533                                         srat_proc, ARRAY_SIZE(srat_proc), 0);
 534
 535                 cnt = acpi_table_parse_srat(ACPI_SRAT_TYPE_MEMORY_AFFINITY,
 536                                             acpi_parse_memory_affinity, 0);
 537         }
 538
 539         /* SLIT: System Locality Information Table */
 540         acpi_table_parse(ACPI_SIG_SLIT, acpi_parse_slit);
 541
 542         /*
 543          * CXL Fixed Memory Window Structures (CFMWS) must be parsed
 544          * after the SRAT. Create NUMA Nodes for CXL memory ranges that
 545          * are defined in the CFMWS and not already defined in the SRAT.
 546          * Initialize a fake_pxm as the first available PXM to emulate.
 547          */
 548
 549         /* fake_pxm is the next unused PXM value after SRAT parsing */
 550         for (i = 0, fake_pxm = -1; i < MAX_NUMNODES; i++) {
 551                 if (node_to_pxm_map[i] > fake_pxm)
 552                         fake_pxm = node_to_pxm_map[i];
 553         }
 554         last_real_pxm = fake_pxm;
 555         fake_pxm++;
 556         acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, acpi_parse_cfmws,
 557                               &fake_pxm);
 558
 559         if (cnt < 0)
 560                 return cnt;
 561         else if (!parsed_numa_memblks)
 562                 return -ENOENT;
 563         return 0;
 564 }
 565
 566 bool acpi_node_backed_by_real_pxm(int nid)
 567 {
 568         int pxm = node_to_pxm(nid);
 569
 570         return pxm <= last_real_pxm;
 571 }
 572 EXPORT_SYMBOL_GPL(acpi_node_backed_by_real_pxm);
 573
 574 static int acpi_get_pxm(acpi_handle h)
 575 {
 576         unsigned long long pxm;
 577         acpi_status status;
 578         acpi_handle handle;
 579         acpi_handle phandle = h;
 580
 581         do {
 582                 handle = phandle;
 583                 status = acpi_evaluate_integer(handle, "_PXM", NULL, &pxm);
 584                 if (ACPI_SUCCESS(status))
 585                         return pxm;
 586                 status = acpi_get_parent(handle, &phandle);
 587         } while (ACPI_SUCCESS(status));
 588         return -1;
 589 }
 590
 591 int acpi_get_node(acpi_handle handle)
 592 {
 593         int pxm;
 594
 595         pxm = acpi_get_pxm(handle);
 596
 597         return pxm_to_node(pxm);
 598 }
 599 EXPORT_SYMBOL(acpi_get_node);