drivers/acpi/numa/hmat.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2019, Intel Corporation.
   4  *
   5  * Heterogeneous Memory Attributes Table (HMAT) representation
   6  *
   7  * This program parses and reports the platform's HMAT tables, and registers
   8  * the applicable attributes with the node's interfaces.
   9  */
  10
  11 #define pr_fmt(fmt) "acpi/hmat: " fmt
  12 #define dev_fmt(fmt) "acpi/hmat: " fmt
  13
  14 #include <linux/acpi.h>
  15 #include <linux/bitops.h>
  16 #include <linux/device.h>
  17 #include <linux/init.h>
  18 #include <linux/list.h>
  19 #include <linux/mm.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/list_sort.h>
  22 #include <linux/memregion.h>
  23 #include <linux/memory.h>
  24 #include <linux/mutex.h>
  25 #include <linux/node.h>
  26 #include <linux/sysfs.h>
  27 #include <linux/dax.h>
  28
  29 static u8 hmat_revision;
  30 static int hmat_disable __initdata;
  31
  32 void __init disable_hmat(void)
  33 {
  34         hmat_disable = 1;
  35 }
  36
  37 static LIST_HEAD(targets);
  38 static LIST_HEAD(initiators);
  39 static LIST_HEAD(localities);
  40
  41 static DEFINE_MUTEX(target_lock);
  42
  43 /*
  44  * The defined enum order is used to prioritize attributes to break ties when
  45  * selecting the best performing node.
  46  */
  47 enum locality_types {
  48         WRITE_LATENCY,
  49         READ_LATENCY,
  50         WRITE_BANDWIDTH,
  51         READ_BANDWIDTH,
  52 };
  53
  54 static struct memory_locality *localities_types[4];
  55
  56 struct target_cache {
  57         struct list_head node;
  58         struct node_cache_attrs cache_attrs;
  59 };
  60
  61 struct memory_target {
  62         struct list_head node;
  63         unsigned int memory_pxm;
  64         unsigned int processor_pxm;
  65         struct resource memregions;
  66         struct node_hmem_attrs hmem_attrs[2];
  67         struct list_head caches;
  68         struct node_cache_attrs cache_attrs;
  69         bool registered;
  70 };
  71
  72 struct memory_initiator {
  73         struct list_head node;
  74         unsigned int processor_pxm;
  75         bool has_cpu;
  76 };
  77
  78 struct memory_locality {
  79         struct list_head node;
  80         struct acpi_hmat_locality *hmat_loc;
  81 };
  82
  83 static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
  84 {
  85         struct memory_initiator *initiator;
  86
  87         list_for_each_entry(initiator, &initiators, node)
  88                 if (initiator->processor_pxm == cpu_pxm)
  89                         return initiator;
  90         return NULL;
  91 }
  92
  93 static struct memory_target *find_mem_target(unsigned int mem_pxm)
  94 {
  95         struct memory_target *target;
  96
  97         list_for_each_entry(target, &targets, node)
  98                 if (target->memory_pxm == mem_pxm)
  99                         return target;
 100         return NULL;
 101 }
 102
 103 static __init void alloc_memory_initiator(unsigned int cpu_pxm)
 104 {
 105         struct memory_initiator *initiator;
 106
 107         if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE)
 108                 return;
 109
 110         initiator = find_mem_initiator(cpu_pxm);
 111         if (initiator)
 112                 return;
 113
 114         initiator = kzalloc(sizeof(*initiator), GFP_KERNEL);
 115         if (!initiator)
 116                 return;
 117
 118         initiator->processor_pxm = cpu_pxm;
 119         initiator->has_cpu = node_state(pxm_to_node(cpu_pxm), N_CPU);
 120         list_add_tail(&initiator->node, &initiators);
 121 }
 122
 123 static __init void alloc_memory_target(unsigned int mem_pxm,
 124                 resource_size_t start, resource_size_t len)
 125 {
 126         struct memory_target *target;
 127
 128         target = find_mem_target(mem_pxm);
 129         if (!target) {
 130                 target = kzalloc(sizeof(*target), GFP_KERNEL);
 131                 if (!target)
 132                         return;
 133                 target->memory_pxm = mem_pxm;
 134                 target->processor_pxm = PXM_INVAL;
 135                 target->memregions = (struct resource) {
 136                         .name   = "ACPI mem",
 137                         .start  = 0,
 138                         .end    = -1,
 139                         .flags  = IORESOURCE_MEM,
 140                 };
 141                 list_add_tail(&target->node, &targets);
 142                 INIT_LIST_HEAD(&target->caches);
 143         }
 144
 145         /*
 146          * There are potentially multiple ranges per PXM, so record each
 147          * in the per-target memregions resource tree.
 148          */
 149         if (!__request_region(&target->memregions, start, len, "memory target",
 150                                 IORESOURCE_MEM))
 151                 pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n",
 152                                 start, start + len, mem_pxm);
 153 }
 154
 155 static __init const char *hmat_data_type(u8 type)
 156 {
 157         switch (type) {
 158         case ACPI_HMAT_ACCESS_LATENCY:
 159                 return "Access Latency";
 160         case ACPI_HMAT_READ_LATENCY:
 161                 return "Read Latency";
 162         case ACPI_HMAT_WRITE_LATENCY:
 163                 return "Write Latency";
 164         case ACPI_HMAT_ACCESS_BANDWIDTH:
 165                 return "Access Bandwidth";
 166         case ACPI_HMAT_READ_BANDWIDTH:
 167                 return "Read Bandwidth";
 168         case ACPI_HMAT_WRITE_BANDWIDTH:
 169                 return "Write Bandwidth";
 170         default:
 171                 return "Reserved";
 172         }
 173 }
 174
 175 static __init const char *hmat_data_type_suffix(u8 type)
 176 {
 177         switch (type) {
 178         case ACPI_HMAT_ACCESS_LATENCY:
 179         case ACPI_HMAT_READ_LATENCY:
 180         case ACPI_HMAT_WRITE_LATENCY:
 181                 return " nsec";
 182         case ACPI_HMAT_ACCESS_BANDWIDTH:
 183         case ACPI_HMAT_READ_BANDWIDTH:
 184         case ACPI_HMAT_WRITE_BANDWIDTH:
 185                 return " MB/s";
 186         default:
 187                 return "";
 188         }
 189 }
 190
 191 static u32 hmat_normalize(u16 entry, u64 base, u8 type)
 192 {
 193         u32 value;
 194
 195         /*
 196          * Check for invalid and overflow values
 197          */
 198         if (entry == 0xffff || !entry)
 199                 return 0;
 200         else if (base > (UINT_MAX / (entry)))
 201                 return 0;
 202
 203         /*
 204          * Divide by the base unit for version 1, convert latency from
 205          * picosenonds to nanoseconds if revision 2.
 206          */
 207         value = entry * base;
 208         if (hmat_revision == 1) {
 209                 if (value < 10)
 210                         return 0;
 211                 value = DIV_ROUND_UP(value, 10);
 212         } else if (hmat_revision == 2) {
 213                 switch (type) {
 214                 case ACPI_HMAT_ACCESS_LATENCY:
 215                 case ACPI_HMAT_READ_LATENCY:
 216                 case ACPI_HMAT_WRITE_LATENCY:
 217                         value = DIV_ROUND_UP(value, 1000);
 218                         break;
 219                 default:
 220                         break;
 221                 }
 222         }
 223         return value;
 224 }
 225
 226 static void hmat_update_target_access(struct memory_target *target,
 227                                       u8 type, u32 value, int access)
 228 {
 229         switch (type) {
 230         case ACPI_HMAT_ACCESS_LATENCY:
 231                 target->hmem_attrs[access].read_latency = value;
 232                 target->hmem_attrs[access].write_latency = value;
 233                 break;
 234         case ACPI_HMAT_READ_LATENCY:
 235                 target->hmem_attrs[access].read_latency = value;
 236                 break;
 237         case ACPI_HMAT_WRITE_LATENCY:
 238                 target->hmem_attrs[access].write_latency = value;
 239                 break;
 240         case ACPI_HMAT_ACCESS_BANDWIDTH:
 241                 target->hmem_attrs[access].read_bandwidth = value;
 242                 target->hmem_attrs[access].write_bandwidth = value;
 243                 break;
 244         case ACPI_HMAT_READ_BANDWIDTH:
 245                 target->hmem_attrs[access].read_bandwidth = value;
 246                 break;
 247         case ACPI_HMAT_WRITE_BANDWIDTH:
 248                 target->hmem_attrs[access].write_bandwidth = value;
 249                 break;
 250         default:
 251                 break;
 252         }
 253 }
 254
 255 static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc)
 256 {
 257         struct memory_locality *loc;
 258
 259         loc = kzalloc(sizeof(*loc), GFP_KERNEL);
 260         if (!loc) {
 261                 pr_notice_once("Failed to allocate HMAT locality\n");
 262                 return;
 263         }
 264
 265         loc->hmat_loc = hmat_loc;
 266         list_add_tail(&loc->node, &localities);
 267
 268         switch (hmat_loc->data_type) {
 269         case ACPI_HMAT_ACCESS_LATENCY:
 270                 localities_types[READ_LATENCY] = loc;
 271                 localities_types[WRITE_LATENCY] = loc;
 272                 break;
 273         case ACPI_HMAT_READ_LATENCY:
 274                 localities_types[READ_LATENCY] = loc;
 275                 break;
 276         case ACPI_HMAT_WRITE_LATENCY:
 277                 localities_types[WRITE_LATENCY] = loc;
 278                 break;
 279         case ACPI_HMAT_ACCESS_BANDWIDTH:
 280                 localities_types[READ_BANDWIDTH] = loc;
 281                 localities_types[WRITE_BANDWIDTH] = loc;
 282                 break;
 283         case ACPI_HMAT_READ_BANDWIDTH:
 284                 localities_types[READ_BANDWIDTH] = loc;
 285                 break;
 286         case ACPI_HMAT_WRITE_BANDWIDTH:
 287                 localities_types[WRITE_BANDWIDTH] = loc;
 288                 break;
 289         default:
 290                 break;
 291         }
 292 }
 293
 294 static __init int hmat_parse_locality(union acpi_subtable_headers *header,
 295                                       const unsigned long end)
 296 {
 297         struct acpi_hmat_locality *hmat_loc = (void *)header;
 298         struct memory_target *target;
 299         unsigned int init, targ, total_size, ipds, tpds;
 300         u32 *inits, *targs, value;
 301         u16 *entries;
 302         u8 type, mem_hier;
 303
 304         if (hmat_loc->header.length < sizeof(*hmat_loc)) {
 305                 pr_notice("HMAT: Unexpected locality header length: %u\n",
 306                          hmat_loc->header.length);
 307                 return -EINVAL;
 308         }
 309
 310         type = hmat_loc->data_type;
 311         mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY;
 312         ipds = hmat_loc->number_of_initiator_Pds;
 313         tpds = hmat_loc->number_of_target_Pds;
 314         total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds +
 315                      sizeof(*inits) * ipds + sizeof(*targs) * tpds;
 316         if (hmat_loc->header.length < total_size) {
 317                 pr_notice("HMAT: Unexpected locality header length:%u, minimum required:%u\n",
 318                          hmat_loc->header.length, total_size);
 319                 return -EINVAL;
 320         }
 321
 322         pr_info("HMAT: Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n",
 323                 hmat_loc->flags, hmat_data_type(type), ipds, tpds,
 324                 hmat_loc->entry_base_unit);
 325
 326         inits = (u32 *)(hmat_loc + 1);
 327         targs = inits + ipds;
 328         entries = (u16 *)(targs + tpds);
 329         for (init = 0; init < ipds; init++) {
 330                 alloc_memory_initiator(inits[init]);
 331                 for (targ = 0; targ < tpds; targ++) {
 332                         value = hmat_normalize(entries[init * tpds + targ],
 333                                                hmat_loc->entry_base_unit,
 334                                                type);
 335                         pr_info("  Initiator-Target[%u-%u]:%u%s\n",
 336                                 inits[init], targs[targ], value,
 337                                 hmat_data_type_suffix(type));
 338
 339                         if (mem_hier == ACPI_HMAT_MEMORY) {
 340                                 target = find_mem_target(targs[targ]);
 341                                 if (target && target->processor_pxm == inits[init]) {
 342                                         hmat_update_target_access(target, type, value, 0);
 343                                         /* If the node has a CPU, update access 1 */
 344                                         if (node_state(pxm_to_node(inits[init]), N_CPU))
 345                                                 hmat_update_target_access(target, type, value, 1);
 346                                 }
 347                         }
 348                 }
 349         }
 350
 351         if (mem_hier == ACPI_HMAT_MEMORY)
 352                 hmat_add_locality(hmat_loc);
 353
 354         return 0;
 355 }
 356
 357 static __init int hmat_parse_cache(union acpi_subtable_headers *header,
 358                                    const unsigned long end)
 359 {
 360         struct acpi_hmat_cache *cache = (void *)header;
 361         struct memory_target *target;
 362         struct target_cache *tcache;
 363         u32 attrs;
 364
 365         if (cache->header.length < sizeof(*cache)) {
 366                 pr_notice("HMAT: Unexpected cache header length: %u\n",
 367                          cache->header.length);
 368                 return -EINVAL;
 369         }
 370
 371         attrs = cache->cache_attributes;
 372         pr_info("HMAT: Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n",
 373                 cache->memory_PD, cache->cache_size, attrs,
 374                 cache->number_of_SMBIOShandles);
 375
 376         target = find_mem_target(cache->memory_PD);
 377         if (!target)
 378                 return 0;
 379
 380         tcache = kzalloc(sizeof(*tcache), GFP_KERNEL);
 381         if (!tcache) {
 382                 pr_notice_once("Failed to allocate HMAT cache info\n");
 383                 return 0;
 384         }
 385
 386         tcache->cache_attrs.size = cache->cache_size;
 387         tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;
 388         tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;
 389
 390         switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) {
 391         case ACPI_HMAT_CA_DIRECT_MAPPED:
 392                 tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;
 393                 break;
 394         case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING:
 395                 tcache->cache_attrs.indexing = NODE_CACHE_INDEXED;
 396                 break;
 397         case ACPI_HMAT_CA_NONE:
 398         default:
 399                 tcache->cache_attrs.indexing = NODE_CACHE_OTHER;
 400                 break;
 401         }
 402
 403         switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) {
 404         case ACPI_HMAT_CP_WB:
 405                 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;
 406                 break;
 407         case ACPI_HMAT_CP_WT:
 408                 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;
 409                 break;
 410         case ACPI_HMAT_CP_NONE:
 411         default:
 412                 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;
 413                 break;
 414         }
 415         list_add_tail(&tcache->node, &target->caches);
 416
 417         return 0;
 418 }
 419
 420 static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header,
 421                                               const unsigned long end)
 422 {
 423         struct acpi_hmat_proximity_domain *p = (void *)header;
 424         struct memory_target *target = NULL;
 425
 426         if (p->header.length != sizeof(*p)) {
 427                 pr_notice("HMAT: Unexpected address range header length: %u\n",
 428                          p->header.length);
 429                 return -EINVAL;
 430         }
 431
 432         if (hmat_revision == 1)
 433                 pr_info("HMAT: Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n",
 434                         p->reserved3, p->reserved4, p->flags, p->processor_PD,
 435                         p->memory_PD);
 436         else
 437                 pr_info("HMAT: Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n",
 438                         p->flags, p->processor_PD, p->memory_PD);
 439
 440         if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) ||
 441             hmat_revision > 1) {
 442                 target = find_mem_target(p->memory_PD);
 443                 if (!target) {
 444                         pr_debug("HMAT: Memory Domain missing from SRAT\n");
 445                         return -EINVAL;
 446                 }
 447         }
 448         if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) {
 449                 int p_node = pxm_to_node(p->processor_PD);
 450
 451                 if (p_node == NUMA_NO_NODE) {
 452                         pr_debug("HMAT: Invalid Processor Domain\n");
 453                         return -EINVAL;
 454                 }
 455                 target->processor_pxm = p->processor_PD;
 456         }
 457
 458         return 0;
 459 }
 460
 461 static int __init hmat_parse_subtable(union acpi_subtable_headers *header,
 462                                       const unsigned long end)
 463 {
 464         struct acpi_hmat_structure *hdr = (void *)header;
 465
 466         if (!hdr)
 467                 return -EINVAL;
 468
 469         switch (hdr->type) {
 470         case ACPI_HMAT_TYPE_PROXIMITY:
 471                 return hmat_parse_proximity_domain(header, end);
 472         case ACPI_HMAT_TYPE_LOCALITY:
 473                 return hmat_parse_locality(header, end);
 474         case ACPI_HMAT_TYPE_CACHE:
 475                 return hmat_parse_cache(header, end);
 476         default:
 477                 return -EINVAL;
 478         }
 479 }
 480
 481 static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header,
 482                                           const unsigned long end)
 483 {
 484         struct acpi_srat_mem_affinity *ma = (void *)header;
 485
 486         if (!ma)
 487                 return -EINVAL;
 488         if (!(ma->flags & ACPI_SRAT_MEM_ENABLED))
 489                 return 0;
 490         alloc_memory_target(ma->proximity_domain, ma->base_address, ma->length);
 491         return 0;
 492 }
 493
 494 static u32 hmat_initiator_perf(struct memory_target *target,
 495                                struct memory_initiator *initiator,
 496                                struct acpi_hmat_locality *hmat_loc)
 497 {
 498         unsigned int ipds, tpds, i, idx = 0, tdx = 0;
 499         u32 *inits, *targs;
 500         u16 *entries;
 501
 502         ipds = hmat_loc->number_of_initiator_Pds;
 503         tpds = hmat_loc->number_of_target_Pds;
 504         inits = (u32 *)(hmat_loc + 1);
 505         targs = inits + ipds;
 506         entries = (u16 *)(targs + tpds);
 507
 508         for (i = 0; i < ipds; i++) {
 509                 if (inits[i] == initiator->processor_pxm) {
 510                         idx = i;
 511                         break;
 512                 }
 513         }
 514
 515         if (i == ipds)
 516                 return 0;
 517
 518         for (i = 0; i < tpds; i++) {
 519                 if (targs[i] == target->memory_pxm) {
 520                         tdx = i;
 521                         break;
 522                 }
 523         }
 524         if (i == tpds)
 525                 return 0;
 526
 527         return hmat_normalize(entries[idx * tpds + tdx],
 528                               hmat_loc->entry_base_unit,
 529                               hmat_loc->data_type);
 530 }
 531
 532 static bool hmat_update_best(u8 type, u32 value, u32 *best)
 533 {
 534         bool updated = false;
 535
 536         if (!value)
 537                 return false;
 538
 539         switch (type) {
 540         case ACPI_HMAT_ACCESS_LATENCY:
 541         case ACPI_HMAT_READ_LATENCY:
 542         case ACPI_HMAT_WRITE_LATENCY:
 543                 if (!*best || *best > value) {
 544                         *best = value;
 545                         updated = true;
 546                 }
 547                 break;
 548         case ACPI_HMAT_ACCESS_BANDWIDTH:
 549         case ACPI_HMAT_READ_BANDWIDTH:
 550         case ACPI_HMAT_WRITE_BANDWIDTH:
 551                 if (!*best || *best < value) {
 552                         *best = value;
 553                         updated = true;
 554                 }
 555                 break;
 556         }
 557
 558         return updated;
 559 }
 560
 561 static int initiator_cmp(void *priv, const struct list_head *a,
 562                          const struct list_head *b)
 563 {
 564         struct memory_initiator *ia;
 565         struct memory_initiator *ib;
 566         unsigned long *p_nodes = priv;
 567
 568         ia = list_entry(a, struct memory_initiator, node);
 569         ib = list_entry(b, struct memory_initiator, node);
 570
 571         set_bit(ia->processor_pxm, p_nodes);
 572         set_bit(ib->processor_pxm, p_nodes);
 573
 574         return ia->processor_pxm - ib->processor_pxm;
 575 }
 576
 577 static void hmat_register_target_initiators(struct memory_target *target)
 578 {
 579         static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
 580         struct memory_initiator *initiator;
 581         unsigned int mem_nid, cpu_nid;
 582         struct memory_locality *loc = NULL;
 583         u32 best = 0;
 584         bool access0done = false;
 585         int i;
 586
 587         mem_nid = pxm_to_node(target->memory_pxm);
 588         /*
 589          * If the Address Range Structure provides a local processor pxm, link
 590          * only that one. Otherwise, find the best performance attributes and
 591          * register all initiators that match.
 592          */
 593         if (target->processor_pxm != PXM_INVAL) {
 594                 cpu_nid = pxm_to_node(target->processor_pxm);
 595                 register_memory_node_under_compute_node(mem_nid, cpu_nid, 0);
 596                 access0done = true;
 597                 if (node_state(cpu_nid, N_CPU)) {
 598                         register_memory_node_under_compute_node(mem_nid, cpu_nid, 1);
 599                         return;
 600                 }
 601         }
 602
 603         if (list_empty(&localities))
 604                 return;
 605
 606         /*
 607          * We need the initiator list sorted so we can use bitmap_clear for
 608          * previously set initiators when we find a better memory accessor.
 609          * We'll also use the sorting to prime the candidate nodes with known
 610          * initiators.
 611          */
 612         bitmap_zero(p_nodes, MAX_NUMNODES);
 613         list_sort(p_nodes, &initiators, initiator_cmp);
 614         if (!access0done) {
 615                 for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
 616                         loc = localities_types[i];
 617                         if (!loc)
 618                                 continue;
 619
 620                         best = 0;
 621                         list_for_each_entry(initiator, &initiators, node) {
 622                                 u32 value;
 623
 624                                 if (!test_bit(initiator->processor_pxm, p_nodes))
 625                                         continue;
 626
 627                                 value = hmat_initiator_perf(target, initiator,
 628                                                             loc->hmat_loc);
 629                                 if (hmat_update_best(loc->hmat_loc->data_type, value, &best))
 630                                         bitmap_clear(p_nodes, 0, initiator->processor_pxm);
 631                                 if (value != best)
 632                                         clear_bit(initiator->processor_pxm, p_nodes);
 633                         }
 634                         if (best)
 635                                 hmat_update_target_access(target, loc->hmat_loc->data_type,
 636                                                           best, 0);
 637                 }
 638
 639                 for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
 640                         cpu_nid = pxm_to_node(i);
 641                         register_memory_node_under_compute_node(mem_nid, cpu_nid, 0);
 642                 }
 643         }
 644
 645         /* Access 1 ignores Generic Initiators */
 646         bitmap_zero(p_nodes, MAX_NUMNODES);
 647         list_sort(p_nodes, &initiators, initiator_cmp);
 648         best = 0;
 649         for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
 650                 loc = localities_types[i];
 651                 if (!loc)
 652                         continue;
 653
 654                 best = 0;
 655                 list_for_each_entry(initiator, &initiators, node) {
 656                         u32 value;
 657
 658                         if (!initiator->has_cpu) {
 659                                 clear_bit(initiator->processor_pxm, p_nodes);
 660                                 continue;
 661                         }
 662                         if (!test_bit(initiator->processor_pxm, p_nodes))
 663                                 continue;
 664
 665                         value = hmat_initiator_perf(target, initiator, loc->hmat_loc);
 666                         if (hmat_update_best(loc->hmat_loc->data_type, value, &best))
 667                                 bitmap_clear(p_nodes, 0, initiator->processor_pxm);
 668                         if (value != best)
 669                                 clear_bit(initiator->processor_pxm, p_nodes);
 670                 }
 671                 if (best)
 672                         hmat_update_target_access(target, loc->hmat_loc->data_type, best, 1);
 673         }
 674         for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
 675                 cpu_nid = pxm_to_node(i);
 676                 register_memory_node_under_compute_node(mem_nid, cpu_nid, 1);
 677         }
 678 }
 679
 680 static void hmat_register_target_cache(struct memory_target *target)
 681 {
 682         unsigned mem_nid = pxm_to_node(target->memory_pxm);
 683         struct target_cache *tcache;
 684
 685         list_for_each_entry(tcache, &target->caches, node)
 686                 node_add_cache(mem_nid, &tcache->cache_attrs);
 687 }
 688
 689 static void hmat_register_target_perf(struct memory_target *target, int access)
 690 {
 691         unsigned mem_nid = pxm_to_node(target->memory_pxm);
 692         node_set_perf_attrs(mem_nid, &target->hmem_attrs[access], access);
 693 }
 694
 695 static void hmat_register_target_devices(struct memory_target *target)
 696 {
 697         struct resource *res;
 698
 699         /*
 700          * Do not bother creating devices if no driver is available to
 701          * consume them.
 702          */
 703         if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM))
 704                 return;
 705
 706         for (res = target->memregions.child; res; res = res->sibling) {
 707                 int target_nid = pxm_to_node(target->memory_pxm);
 708
 709                 hmem_register_device(target_nid, res);
 710         }
 711 }
 712
 713 static void hmat_register_target(struct memory_target *target)
 714 {
 715         int nid = pxm_to_node(target->memory_pxm);
 716
 717         /*
 718          * Devices may belong to either an offline or online
 719          * node, so unconditionally add them.
 720          */
 721         hmat_register_target_devices(target);
 722
 723         /*
 724          * Skip offline nodes. This can happen when memory
 725          * marked EFI_MEMORY_SP, "specific purpose", is applied
 726          * to all the memory in a proximity domain leading to
 727          * the node being marked offline / unplugged, or if
 728          * memory-only "hotplug" node is offline.
 729          */
 730         if (nid == NUMA_NO_NODE || !node_online(nid))
 731                 return;
 732
 733         mutex_lock(&target_lock);
 734         if (!target->registered) {
 735                 hmat_register_target_initiators(target);
 736                 hmat_register_target_cache(target);
 737                 hmat_register_target_perf(target, 0);
 738                 hmat_register_target_perf(target, 1);
 739                 target->registered = true;
 740         }
 741         mutex_unlock(&target_lock);
 742 }
 743
 744 static void hmat_register_targets(void)
 745 {
 746         struct memory_target *target;
 747
 748         list_for_each_entry(target, &targets, node)
 749                 hmat_register_target(target);
 750 }
 751
 752 static int hmat_callback(struct notifier_block *self,
 753                          unsigned long action, void *arg)
 754 {
 755         struct memory_target *target;
 756         struct memory_notify *mnb = arg;
 757         int pxm, nid = mnb->status_change_nid;
 758
 759         if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
 760                 return NOTIFY_OK;
 761
 762         pxm = node_to_pxm(nid);
 763         target = find_mem_target(pxm);
 764         if (!target)
 765                 return NOTIFY_OK;
 766
 767         hmat_register_target(target);
 768         return NOTIFY_OK;
 769 }
 770
 771 static struct notifier_block hmat_callback_nb = {
 772         .notifier_call = hmat_callback,
 773         .priority = 2,
 774 };
 775
 776 static __init void hmat_free_structures(void)
 777 {
 778         struct memory_target *target, *tnext;
 779         struct memory_locality *loc, *lnext;
 780         struct memory_initiator *initiator, *inext;
 781         struct target_cache *tcache, *cnext;
 782
 783         list_for_each_entry_safe(target, tnext, &targets, node) {
 784                 struct resource *res, *res_next;
 785
 786                 list_for_each_entry_safe(tcache, cnext, &target->caches, node) {
 787                         list_del(&tcache->node);
 788                         kfree(tcache);
 789                 }
 790
 791                 list_del(&target->node);
 792                 res = target->memregions.child;
 793                 while (res) {
 794                         res_next = res->sibling;
 795                         __release_region(&target->memregions, res->start,
 796                                         resource_size(res));
 797                         res = res_next;
 798                 }
 799                 kfree(target);
 800         }
 801
 802         list_for_each_entry_safe(initiator, inext, &initiators, node) {
 803                 list_del(&initiator->node);
 804                 kfree(initiator);
 805         }
 806
 807         list_for_each_entry_safe(loc, lnext, &localities, node) {
 808                 list_del(&loc->node);
 809                 kfree(loc);
 810         }
 811 }
 812
 813 static __init int hmat_init(void)
 814 {
 815         struct acpi_table_header *tbl;
 816         enum acpi_hmat_type i;
 817         acpi_status status;
 818
 819         if (srat_disabled() || hmat_disable)
 820                 return 0;
 821
 822         status = acpi_get_table(ACPI_SIG_SRAT, 0, &tbl);
 823         if (ACPI_FAILURE(status))
 824                 return 0;
 825
 826         if (acpi_table_parse_entries(ACPI_SIG_SRAT,
 827                                 sizeof(struct acpi_table_srat),
 828                                 ACPI_SRAT_TYPE_MEMORY_AFFINITY,
 829                                 srat_parse_mem_affinity, 0) < 0)
 830                 goto out_put;
 831         acpi_put_table(tbl);
 832
 833         status = acpi_get_table(ACPI_SIG_HMAT, 0, &tbl);
 834         if (ACPI_FAILURE(status))
 835                 goto out_put;
 836
 837         hmat_revision = tbl->revision;
 838         switch (hmat_revision) {
 839         case 1:
 840         case 2:
 841                 break;
 842         default:
 843                 pr_notice("Ignoring HMAT: Unknown revision:%d\n", hmat_revision);
 844                 goto out_put;
 845         }
 846
 847         for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) {
 848                 if (acpi_table_parse_entries(ACPI_SIG_HMAT,
 849                                              sizeof(struct acpi_table_hmat), i,
 850                                              hmat_parse_subtable, 0) < 0) {
 851                         pr_notice("Ignoring HMAT: Invalid table");
 852                         goto out_put;
 853                 }
 854         }
 855         hmat_register_targets();
 856
 857         /* Keep the table and structures if the notifier may use them */
 858         if (!register_hotmemory_notifier(&hmat_callback_nb))
 859                 return 0;
 860 out_put:
 861         hmat_free_structures();
 862         acpi_put_table(tbl);
 863         return 0;
 864 }
 865 device_initcall(hmat_init);