drivers/cxl/core/region.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
   3 #include <linux/memregion.h>
   4 #include <linux/genalloc.h>
   5 #include <linux/device.h>
   6 #include <linux/module.h>
   7 #include <linux/slab.h>
   8 #include <linux/uuid.h>
   9 #include <linux/sort.h>
  10 #include <linux/idr.h>
  11 #include <cxlmem.h>
  12 #include <cxl.h>
  13 #include "core.h"
  14
  15 /**
  16  * DOC: cxl core region
  17  *
  18  * CXL Regions represent mapped memory capacity in system physical address
  19  * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
  20  * Memory ranges, Regions represent the active mapped capacity by the HDM
  21  * Decoder Capability structures throughout the Host Bridges, Switches, and
  22  * Endpoints in the topology.
  23  *
  24  * Region configuration has ordering constraints. UUID may be set at any time
  25  * but is only visible for persistent regions.
  26  * 1. Interleave granularity
  27  * 2. Interleave size
  28  * 3. Decoder targets
  29  */
  30
  31 /*
  32  * All changes to the interleave configuration occur with this lock held
  33  * for write.
  34  */
  35 static DECLARE_RWSEM(cxl_region_rwsem);
  36
  37 static struct cxl_region *to_cxl_region(struct device *dev);
  38
  39 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
  40                          char *buf)
  41 {
  42         struct cxl_region *cxlr = to_cxl_region(dev);
  43         struct cxl_region_params *p = &cxlr->params;
  44         ssize_t rc;
  45
  46         rc = down_read_interruptible(&cxl_region_rwsem);
  47         if (rc)
  48                 return rc;
  49         if (cxlr->mode != CXL_DECODER_PMEM)
  50                 rc = sysfs_emit(buf, "\n");
  51         else
  52                 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
  53         up_read(&cxl_region_rwsem);
  54
  55         return rc;
  56 }
  57
  58 static int is_dup(struct device *match, void *data)
  59 {
  60         struct cxl_region_params *p;
  61         struct cxl_region *cxlr;
  62         uuid_t *uuid = data;
  63
  64         if (!is_cxl_region(match))
  65                 return 0;
  66
  67         lockdep_assert_held(&cxl_region_rwsem);
  68         cxlr = to_cxl_region(match);
  69         p = &cxlr->params;
  70
  71         if (uuid_equal(&p->uuid, uuid)) {
  72                 dev_dbg(match, "already has uuid: %pUb\n", uuid);
  73                 return -EBUSY;
  74         }
  75
  76         return 0;
  77 }
  78
  79 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
  80                           const char *buf, size_t len)
  81 {
  82         struct cxl_region *cxlr = to_cxl_region(dev);
  83         struct cxl_region_params *p = &cxlr->params;
  84         uuid_t temp;
  85         ssize_t rc;
  86
  87         if (len != UUID_STRING_LEN + 1)
  88                 return -EINVAL;
  89
  90         rc = uuid_parse(buf, &temp);
  91         if (rc)
  92                 return rc;
  93
  94         if (uuid_is_null(&temp))
  95                 return -EINVAL;
  96
  97         rc = down_write_killable(&cxl_region_rwsem);
  98         if (rc)
  99                 return rc;
 100
 101         if (uuid_equal(&p->uuid, &temp))
 102                 goto out;
 103
 104         rc = -EBUSY;
 105         if (p->state >= CXL_CONFIG_ACTIVE)
 106                 goto out;
 107
 108         rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
 109         if (rc < 0)
 110                 goto out;
 111
 112         uuid_copy(&p->uuid, &temp);
 113 out:
 114         up_write(&cxl_region_rwsem);
 115
 116         if (rc)
 117                 return rc;
 118         return len;
 119 }
 120 static DEVICE_ATTR_RW(uuid);
 121
 122 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
 123                                           struct cxl_region *cxlr)
 124 {
 125         return xa_load(&port->regions, (unsigned long)cxlr);
 126 }
 127
 128 static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
 129 {
 130         struct cxl_region_params *p = &cxlr->params;
 131         int i;
 132
 133         for (i = count - 1; i >= 0; i--) {
 134                 struct cxl_endpoint_decoder *cxled = p->targets[i];
 135                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 136                 struct cxl_port *iter = cxled_to_port(cxled);
 137                 struct cxl_dev_state *cxlds = cxlmd->cxlds;
 138                 struct cxl_ep *ep;
 139                 int rc = 0;
 140
 141                 if (cxlds->rcd)
 142                         goto endpoint_reset;
 143
 144                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
 145                         iter = to_cxl_port(iter->dev.parent);
 146
 147                 for (ep = cxl_ep_load(iter, cxlmd); iter;
 148                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
 149                         struct cxl_region_ref *cxl_rr;
 150                         struct cxl_decoder *cxld;
 151
 152                         cxl_rr = cxl_rr_load(iter, cxlr);
 153                         cxld = cxl_rr->decoder;
 154                         if (cxld->reset)
 155                                 rc = cxld->reset(cxld);
 156                         if (rc)
 157                                 return rc;
 158                 }
 159
 160 endpoint_reset:
 161                 rc = cxled->cxld.reset(&cxled->cxld);
 162                 if (rc)
 163                         return rc;
 164         }
 165
 166         return 0;
 167 }
 168
 169 static int commit_decoder(struct cxl_decoder *cxld)
 170 {
 171         struct cxl_switch_decoder *cxlsd = NULL;
 172
 173         if (cxld->commit)
 174                 return cxld->commit(cxld);
 175
 176         if (is_switch_decoder(&cxld->dev))
 177                 cxlsd = to_cxl_switch_decoder(&cxld->dev);
 178
 179         if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
 180                           "->commit() is required\n"))
 181                 return -ENXIO;
 182         return 0;
 183 }
 184
 185 static int cxl_region_decode_commit(struct cxl_region *cxlr)
 186 {
 187         struct cxl_region_params *p = &cxlr->params;
 188         int i, rc = 0;
 189
 190         for (i = 0; i < p->nr_targets; i++) {
 191                 struct cxl_endpoint_decoder *cxled = p->targets[i];
 192                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 193                 struct cxl_region_ref *cxl_rr;
 194                 struct cxl_decoder *cxld;
 195                 struct cxl_port *iter;
 196                 struct cxl_ep *ep;
 197
 198                 /* commit bottom up */
 199                 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
 200                      iter = to_cxl_port(iter->dev.parent)) {
 201                         cxl_rr = cxl_rr_load(iter, cxlr);
 202                         cxld = cxl_rr->decoder;
 203                         rc = commit_decoder(cxld);
 204                         if (rc)
 205                                 break;
 206                 }
 207
 208                 if (rc) {
 209                         /* programming @iter failed, teardown */
 210                         for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
 211                              iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
 212                                 cxl_rr = cxl_rr_load(iter, cxlr);
 213                                 cxld = cxl_rr->decoder;
 214                                 if (cxld->reset)
 215                                         cxld->reset(cxld);
 216                         }
 217
 218                         cxled->cxld.reset(&cxled->cxld);
 219                         goto err;
 220                 }
 221         }
 222
 223         return 0;
 224
 225 err:
 226         /* undo the targets that were successfully committed */
 227         cxl_region_decode_reset(cxlr, i);
 228         return rc;
 229 }
 230
 231 static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
 232                             const char *buf, size_t len)
 233 {
 234         struct cxl_region *cxlr = to_cxl_region(dev);
 235         struct cxl_region_params *p = &cxlr->params;
 236         bool commit;
 237         ssize_t rc;
 238
 239         rc = kstrtobool(buf, &commit);
 240         if (rc)
 241                 return rc;
 242
 243         rc = down_write_killable(&cxl_region_rwsem);
 244         if (rc)
 245                 return rc;
 246
 247         /* Already in the requested state? */
 248         if (commit && p->state >= CXL_CONFIG_COMMIT)
 249                 goto out;
 250         if (!commit && p->state < CXL_CONFIG_COMMIT)
 251                 goto out;
 252
 253         /* Not ready to commit? */
 254         if (commit && p->state < CXL_CONFIG_ACTIVE) {
 255                 rc = -ENXIO;
 256                 goto out;
 257         }
 258
 259         if (commit)
 260                 rc = cxl_region_decode_commit(cxlr);
 261         else {
 262                 p->state = CXL_CONFIG_RESET_PENDING;
 263                 up_write(&cxl_region_rwsem);
 264                 device_release_driver(&cxlr->dev);
 265                 down_write(&cxl_region_rwsem);
 266
 267                 /*
 268                  * The lock was dropped, so need to revalidate that the reset is
 269                  * still pending.
 270                  */
 271                 if (p->state == CXL_CONFIG_RESET_PENDING)
 272                         rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
 273         }
 274
 275         if (rc)
 276                 goto out;
 277
 278         if (commit)
 279                 p->state = CXL_CONFIG_COMMIT;
 280         else if (p->state == CXL_CONFIG_RESET_PENDING)
 281                 p->state = CXL_CONFIG_ACTIVE;
 282
 283 out:
 284         up_write(&cxl_region_rwsem);
 285
 286         if (rc)
 287                 return rc;
 288         return len;
 289 }
 290
 291 static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
 292                            char *buf)
 293 {
 294         struct cxl_region *cxlr = to_cxl_region(dev);
 295         struct cxl_region_params *p = &cxlr->params;
 296         ssize_t rc;
 297
 298         rc = down_read_interruptible(&cxl_region_rwsem);
 299         if (rc)
 300                 return rc;
 301         rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
 302         up_read(&cxl_region_rwsem);
 303
 304         return rc;
 305 }
 306 static DEVICE_ATTR_RW(commit);
 307
 308 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
 309                                   int n)
 310 {
 311         struct device *dev = kobj_to_dev(kobj);
 312         struct cxl_region *cxlr = to_cxl_region(dev);
 313
 314         /*
 315          * Support tooling that expects to find a 'uuid' attribute for all
 316          * regions regardless of mode.
 317          */
 318         if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
 319                 return 0444;
 320         return a->mode;
 321 }
 322
 323 static ssize_t interleave_ways_show(struct device *dev,
 324                                     struct device_attribute *attr, char *buf)
 325 {
 326         struct cxl_region *cxlr = to_cxl_region(dev);
 327         struct cxl_region_params *p = &cxlr->params;
 328         ssize_t rc;
 329
 330         rc = down_read_interruptible(&cxl_region_rwsem);
 331         if (rc)
 332                 return rc;
 333         rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
 334         up_read(&cxl_region_rwsem);
 335
 336         return rc;
 337 }
 338
 339 static const struct attribute_group *get_cxl_region_target_group(void);
 340
 341 static ssize_t interleave_ways_store(struct device *dev,
 342                                      struct device_attribute *attr,
 343                                      const char *buf, size_t len)
 344 {
 345         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
 346         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
 347         struct cxl_region *cxlr = to_cxl_region(dev);
 348         struct cxl_region_params *p = &cxlr->params;
 349         unsigned int val, save;
 350         int rc;
 351         u8 iw;
 352
 353         rc = kstrtouint(buf, 0, &val);
 354         if (rc)
 355                 return rc;
 356
 357         rc = ways_to_eiw(val, &iw);
 358         if (rc)
 359                 return rc;
 360
 361         /*
 362          * Even for x3, x9, and x12 interleaves the region interleave must be a
 363          * power of 2 multiple of the host bridge interleave.
 364          */
 365         if (!is_power_of_2(val / cxld->interleave_ways) ||
 366             (val % cxld->interleave_ways)) {
 367                 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
 368                 return -EINVAL;
 369         }
 370
 371         rc = down_write_killable(&cxl_region_rwsem);
 372         if (rc)
 373                 return rc;
 374         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
 375                 rc = -EBUSY;
 376                 goto out;
 377         }
 378
 379         save = p->interleave_ways;
 380         p->interleave_ways = val;
 381         rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
 382         if (rc)
 383                 p->interleave_ways = save;
 384 out:
 385         up_write(&cxl_region_rwsem);
 386         if (rc)
 387                 return rc;
 388         return len;
 389 }
 390 static DEVICE_ATTR_RW(interleave_ways);
 391
 392 static ssize_t interleave_granularity_show(struct device *dev,
 393                                            struct device_attribute *attr,
 394                                            char *buf)
 395 {
 396         struct cxl_region *cxlr = to_cxl_region(dev);
 397         struct cxl_region_params *p = &cxlr->params;
 398         ssize_t rc;
 399
 400         rc = down_read_interruptible(&cxl_region_rwsem);
 401         if (rc)
 402                 return rc;
 403         rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
 404         up_read(&cxl_region_rwsem);
 405
 406         return rc;
 407 }
 408
 409 static ssize_t interleave_granularity_store(struct device *dev,
 410                                             struct device_attribute *attr,
 411                                             const char *buf, size_t len)
 412 {
 413         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
 414         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
 415         struct cxl_region *cxlr = to_cxl_region(dev);
 416         struct cxl_region_params *p = &cxlr->params;
 417         int rc, val;
 418         u16 ig;
 419
 420         rc = kstrtoint(buf, 0, &val);
 421         if (rc)
 422                 return rc;
 423
 424         rc = granularity_to_eig(val, &ig);
 425         if (rc)
 426                 return rc;
 427
 428         /*
 429          * When the host-bridge is interleaved, disallow region granularity !=
 430          * root granularity. Regions with a granularity less than the root
 431          * interleave result in needing multiple endpoints to support a single
 432          * slot in the interleave (possible to support in the future). Regions
 433          * with a granularity greater than the root interleave result in invalid
 434          * DPA translations (invalid to support).
 435          */
 436         if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
 437                 return -EINVAL;
 438
 439         rc = down_write_killable(&cxl_region_rwsem);
 440         if (rc)
 441                 return rc;
 442         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
 443                 rc = -EBUSY;
 444                 goto out;
 445         }
 446
 447         p->interleave_granularity = val;
 448 out:
 449         up_write(&cxl_region_rwsem);
 450         if (rc)
 451                 return rc;
 452         return len;
 453 }
 454 static DEVICE_ATTR_RW(interleave_granularity);
 455
 456 static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
 457                              char *buf)
 458 {
 459         struct cxl_region *cxlr = to_cxl_region(dev);
 460         struct cxl_region_params *p = &cxlr->params;
 461         u64 resource = -1ULL;
 462         ssize_t rc;
 463
 464         rc = down_read_interruptible(&cxl_region_rwsem);
 465         if (rc)
 466                 return rc;
 467         if (p->res)
 468                 resource = p->res->start;
 469         rc = sysfs_emit(buf, "%#llx\n", resource);
 470         up_read(&cxl_region_rwsem);
 471
 472         return rc;
 473 }
 474 static DEVICE_ATTR_RO(resource);
 475
 476 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
 477                          char *buf)
 478 {
 479         struct cxl_region *cxlr = to_cxl_region(dev);
 480
 481         return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
 482 }
 483 static DEVICE_ATTR_RO(mode);
 484
 485 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
 486 {
 487         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
 488         struct cxl_region_params *p = &cxlr->params;
 489         struct resource *res;
 490         u32 remainder = 0;
 491
 492         lockdep_assert_held_write(&cxl_region_rwsem);
 493
 494         /* Nothing to do... */
 495         if (p->res && resource_size(p->res) == size)
 496                 return 0;
 497
 498         /* To change size the old size must be freed first */
 499         if (p->res)
 500                 return -EBUSY;
 501
 502         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
 503                 return -EBUSY;
 504
 505         /* ways, granularity and uuid (if PMEM) need to be set before HPA */
 506         if (!p->interleave_ways || !p->interleave_granularity ||
 507             (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
 508                 return -ENXIO;
 509
 510         div_u64_rem(size, SZ_256M * p->interleave_ways, &remainder);
 511         if (remainder)
 512                 return -EINVAL;
 513
 514         res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
 515                                     dev_name(&cxlr->dev));
 516         if (IS_ERR(res)) {
 517                 dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n",
 518                         PTR_ERR(res));
 519                 return PTR_ERR(res);
 520         }
 521
 522         p->res = res;
 523         p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
 524
 525         return 0;
 526 }
 527
 528 static void cxl_region_iomem_release(struct cxl_region *cxlr)
 529 {
 530         struct cxl_region_params *p = &cxlr->params;
 531
 532         if (device_is_registered(&cxlr->dev))
 533                 lockdep_assert_held_write(&cxl_region_rwsem);
 534         if (p->res) {
 535                 /*
 536                  * Autodiscovered regions may not have been able to insert their
 537                  * resource.
 538                  */
 539                 if (p->res->parent)
 540                         remove_resource(p->res);
 541                 kfree(p->res);
 542                 p->res = NULL;
 543         }
 544 }
 545
 546 static int free_hpa(struct cxl_region *cxlr)
 547 {
 548         struct cxl_region_params *p = &cxlr->params;
 549
 550         lockdep_assert_held_write(&cxl_region_rwsem);
 551
 552         if (!p->res)
 553                 return 0;
 554
 555         if (p->state >= CXL_CONFIG_ACTIVE)
 556                 return -EBUSY;
 557
 558         cxl_region_iomem_release(cxlr);
 559         p->state = CXL_CONFIG_IDLE;
 560         return 0;
 561 }
 562
 563 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
 564                           const char *buf, size_t len)
 565 {
 566         struct cxl_region *cxlr = to_cxl_region(dev);
 567         u64 val;
 568         int rc;
 569
 570         rc = kstrtou64(buf, 0, &val);
 571         if (rc)
 572                 return rc;
 573
 574         rc = down_write_killable(&cxl_region_rwsem);
 575         if (rc)
 576                 return rc;
 577
 578         if (val)
 579                 rc = alloc_hpa(cxlr, val);
 580         else
 581                 rc = free_hpa(cxlr);
 582         up_write(&cxl_region_rwsem);
 583
 584         if (rc)
 585                 return rc;
 586
 587         return len;
 588 }
 589
 590 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
 591                          char *buf)
 592 {
 593         struct cxl_region *cxlr = to_cxl_region(dev);
 594         struct cxl_region_params *p = &cxlr->params;
 595         u64 size = 0;
 596         ssize_t rc;
 597
 598         rc = down_read_interruptible(&cxl_region_rwsem);
 599         if (rc)
 600                 return rc;
 601         if (p->res)
 602                 size = resource_size(p->res);
 603         rc = sysfs_emit(buf, "%#llx\n", size);
 604         up_read(&cxl_region_rwsem);
 605
 606         return rc;
 607 }
 608 static DEVICE_ATTR_RW(size);
 609
 610 static struct attribute *cxl_region_attrs[] = {
 611         &dev_attr_uuid.attr,
 612         &dev_attr_commit.attr,
 613         &dev_attr_interleave_ways.attr,
 614         &dev_attr_interleave_granularity.attr,
 615         &dev_attr_resource.attr,
 616         &dev_attr_size.attr,
 617         &dev_attr_mode.attr,
 618         NULL,
 619 };
 620
 621 static const struct attribute_group cxl_region_group = {
 622         .attrs = cxl_region_attrs,
 623         .is_visible = cxl_region_visible,
 624 };
 625
 626 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
 627 {
 628         struct cxl_region_params *p = &cxlr->params;
 629         struct cxl_endpoint_decoder *cxled;
 630         int rc;
 631
 632         rc = down_read_interruptible(&cxl_region_rwsem);
 633         if (rc)
 634                 return rc;
 635
 636         if (pos >= p->interleave_ways) {
 637                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
 638                         p->interleave_ways);
 639                 rc = -ENXIO;
 640                 goto out;
 641         }
 642
 643         cxled = p->targets[pos];
 644         if (!cxled)
 645                 rc = sysfs_emit(buf, "\n");
 646         else
 647                 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
 648 out:
 649         up_read(&cxl_region_rwsem);
 650
 651         return rc;
 652 }
 653
 654 static int match_free_decoder(struct device *dev, void *data)
 655 {
 656         struct cxl_decoder *cxld;
 657         int *id = data;
 658
 659         if (!is_switch_decoder(dev))
 660                 return 0;
 661
 662         cxld = to_cxl_decoder(dev);
 663
 664         /* enforce ordered allocation */
 665         if (cxld->id != *id)
 666                 return 0;
 667
 668         if (!cxld->region)
 669                 return 1;
 670
 671         (*id)++;
 672
 673         return 0;
 674 }
 675
 676 static struct cxl_decoder *cxl_region_find_decoder(struct cxl_port *port,
 677                                                    struct cxl_region *cxlr)
 678 {
 679         struct device *dev;
 680         int id = 0;
 681
 682         dev = device_find_child(&port->dev, &id, match_free_decoder);
 683         if (!dev)
 684                 return NULL;
 685         /*
 686          * This decoder is pinned registered as long as the endpoint decoder is
 687          * registered, and endpoint decoder unregistration holds the
 688          * cxl_region_rwsem over unregister events, so no need to hold on to
 689          * this extra reference.
 690          */
 691         put_device(dev);
 692         return to_cxl_decoder(dev);
 693 }
 694
 695 static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port,
 696                                                struct cxl_region *cxlr)
 697 {
 698         struct cxl_region_params *p = &cxlr->params;
 699         struct cxl_region_ref *cxl_rr, *iter;
 700         unsigned long index;
 701         int rc;
 702
 703         xa_for_each(&port->regions, index, iter) {
 704                 struct cxl_region_params *ip = &iter->region->params;
 705
 706                 if (!ip->res)
 707                         continue;
 708
 709                 if (ip->res->start > p->res->start) {
 710                         dev_dbg(&cxlr->dev,
 711                                 "%s: HPA order violation %s:%pr vs %pr\n",
 712                                 dev_name(&port->dev),
 713                                 dev_name(&iter->region->dev), ip->res, p->res);
 714                         return ERR_PTR(-EBUSY);
 715                 }
 716         }
 717
 718         cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
 719         if (!cxl_rr)
 720                 return ERR_PTR(-ENOMEM);
 721         cxl_rr->port = port;
 722         cxl_rr->region = cxlr;
 723         cxl_rr->nr_targets = 1;
 724         xa_init(&cxl_rr->endpoints);
 725
 726         rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
 727         if (rc) {
 728                 dev_dbg(&cxlr->dev,
 729                         "%s: failed to track region reference: %d\n",
 730                         dev_name(&port->dev), rc);
 731                 kfree(cxl_rr);
 732                 return ERR_PTR(rc);
 733         }
 734
 735         return cxl_rr;
 736 }
 737
 738 static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
 739 {
 740         struct cxl_region *cxlr = cxl_rr->region;
 741         struct cxl_decoder *cxld = cxl_rr->decoder;
 742
 743         if (!cxld)
 744                 return;
 745
 746         dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
 747         if (cxld->region == cxlr) {
 748                 cxld->region = NULL;
 749                 put_device(&cxlr->dev);
 750         }
 751 }
 752
 753 static void free_region_ref(struct cxl_region_ref *cxl_rr)
 754 {
 755         struct cxl_port *port = cxl_rr->port;
 756         struct cxl_region *cxlr = cxl_rr->region;
 757
 758         cxl_rr_free_decoder(cxl_rr);
 759         xa_erase(&port->regions, (unsigned long)cxlr);
 760         xa_destroy(&cxl_rr->endpoints);
 761         kfree(cxl_rr);
 762 }
 763
 764 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
 765                          struct cxl_endpoint_decoder *cxled)
 766 {
 767         int rc;
 768         struct cxl_port *port = cxl_rr->port;
 769         struct cxl_region *cxlr = cxl_rr->region;
 770         struct cxl_decoder *cxld = cxl_rr->decoder;
 771         struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
 772
 773         if (ep) {
 774                 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
 775                                GFP_KERNEL);
 776                 if (rc)
 777                         return rc;
 778         }
 779         cxl_rr->nr_eps++;
 780
 781         if (!cxld->region) {
 782                 cxld->region = cxlr;
 783                 get_device(&cxlr->dev);
 784         }
 785
 786         return 0;
 787 }
 788
 789 static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
 790                                 struct cxl_endpoint_decoder *cxled,
 791                                 struct cxl_region_ref *cxl_rr)
 792 {
 793         struct cxl_decoder *cxld;
 794
 795         if (port == cxled_to_port(cxled))
 796                 cxld = &cxled->cxld;
 797         else
 798                 cxld = cxl_region_find_decoder(port, cxlr);
 799         if (!cxld) {
 800                 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
 801                         dev_name(&port->dev));
 802                 return -EBUSY;
 803         }
 804
 805         if (cxld->region) {
 806                 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
 807                         dev_name(&port->dev), dev_name(&cxld->dev),
 808                         dev_name(&cxld->region->dev));
 809                 return -EBUSY;
 810         }
 811
 812         cxl_rr->decoder = cxld;
 813         return 0;
 814 }
 815
 816 /**
 817  * cxl_port_attach_region() - track a region's interest in a port by endpoint
 818  * @port: port to add a new region reference 'struct cxl_region_ref'
 819  * @cxlr: region to attach to @port
 820  * @cxled: endpoint decoder used to create or further pin a region reference
 821  * @pos: interleave position of @cxled in @cxlr
 822  *
 823  * The attach event is an opportunity to validate CXL decode setup
 824  * constraints and record metadata needed for programming HDM decoders,
 825  * in particular decoder target lists.
 826  *
 827  * The steps are:
 828  *
 829  * - validate that there are no other regions with a higher HPA already
 830  *   associated with @port
 831  * - establish a region reference if one is not already present
 832  *
 833  *   - additionally allocate a decoder instance that will host @cxlr on
 834  *     @port
 835  *
 836  * - pin the region reference by the endpoint
 837  * - account for how many entries in @port's target list are needed to
 838  *   cover all of the added endpoints.
 839  */
 840 static int cxl_port_attach_region(struct cxl_port *port,
 841                                   struct cxl_region *cxlr,
 842                                   struct cxl_endpoint_decoder *cxled, int pos)
 843 {
 844         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 845         struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
 846         struct cxl_region_ref *cxl_rr;
 847         bool nr_targets_inc = false;
 848         struct cxl_decoder *cxld;
 849         unsigned long index;
 850         int rc = -EBUSY;
 851
 852         lockdep_assert_held_write(&cxl_region_rwsem);
 853
 854         cxl_rr = cxl_rr_load(port, cxlr);
 855         if (cxl_rr) {
 856                 struct cxl_ep *ep_iter;
 857                 int found = 0;
 858
 859                 /*
 860                  * Walk the existing endpoints that have been attached to
 861                  * @cxlr at @port and see if they share the same 'next' port
 862                  * in the downstream direction. I.e. endpoints that share common
 863                  * upstream switch.
 864                  */
 865                 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
 866                         if (ep_iter == ep)
 867                                 continue;
 868                         if (ep_iter->next == ep->next) {
 869                                 found++;
 870                                 break;
 871                         }
 872                 }
 873
 874                 /*
 875                  * New target port, or @port is an endpoint port that always
 876                  * accounts its own local decode as a target.
 877                  */
 878                 if (!found || !ep->next) {
 879                         cxl_rr->nr_targets++;
 880                         nr_targets_inc = true;
 881                 }
 882         } else {
 883                 cxl_rr = alloc_region_ref(port, cxlr);
 884                 if (IS_ERR(cxl_rr)) {
 885                         dev_dbg(&cxlr->dev,
 886                                 "%s: failed to allocate region reference\n",
 887                                 dev_name(&port->dev));
 888                         return PTR_ERR(cxl_rr);
 889                 }
 890                 nr_targets_inc = true;
 891
 892                 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
 893                 if (rc)
 894                         goto out_erase;
 895         }
 896         cxld = cxl_rr->decoder;
 897
 898         rc = cxl_rr_ep_add(cxl_rr, cxled);
 899         if (rc) {
 900                 dev_dbg(&cxlr->dev,
 901                         "%s: failed to track endpoint %s:%s reference\n",
 902                         dev_name(&port->dev), dev_name(&cxlmd->dev),
 903                         dev_name(&cxld->dev));
 904                 goto out_erase;
 905         }
 906
 907         dev_dbg(&cxlr->dev,
 908                 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
 909                 dev_name(port->uport), dev_name(&port->dev),
 910                 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
 911                 dev_name(&cxled->cxld.dev), pos,
 912                 ep ? ep->next ? dev_name(ep->next->uport) :
 913                                       dev_name(&cxlmd->dev) :
 914                            "none",
 915                 cxl_rr->nr_eps, cxl_rr->nr_targets);
 916
 917         return 0;
 918 out_erase:
 919         if (nr_targets_inc)
 920                 cxl_rr->nr_targets--;
 921         if (cxl_rr->nr_eps == 0)
 922                 free_region_ref(cxl_rr);
 923         return rc;
 924 }
 925
 926 static void cxl_port_detach_region(struct cxl_port *port,
 927                                    struct cxl_region *cxlr,
 928                                    struct cxl_endpoint_decoder *cxled)
 929 {
 930         struct cxl_region_ref *cxl_rr;
 931         struct cxl_ep *ep = NULL;
 932
 933         lockdep_assert_held_write(&cxl_region_rwsem);
 934
 935         cxl_rr = cxl_rr_load(port, cxlr);
 936         if (!cxl_rr)
 937                 return;
 938
 939         /*
 940          * Endpoint ports do not carry cxl_ep references, and they
 941          * never target more than one endpoint by definition
 942          */
 943         if (cxl_rr->decoder == &cxled->cxld)
 944                 cxl_rr->nr_eps--;
 945         else
 946                 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
 947         if (ep) {
 948                 struct cxl_ep *ep_iter;
 949                 unsigned long index;
 950                 int found = 0;
 951
 952                 cxl_rr->nr_eps--;
 953                 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
 954                         if (ep_iter->next == ep->next) {
 955                                 found++;
 956                                 break;
 957                         }
 958                 }
 959                 if (!found)
 960                         cxl_rr->nr_targets--;
 961         }
 962
 963         if (cxl_rr->nr_eps == 0)
 964                 free_region_ref(cxl_rr);
 965 }
 966
 967 static int check_last_peer(struct cxl_endpoint_decoder *cxled,
 968                            struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
 969                            int distance)
 970 {
 971         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 972         struct cxl_region *cxlr = cxl_rr->region;
 973         struct cxl_region_params *p = &cxlr->params;
 974         struct cxl_endpoint_decoder *cxled_peer;
 975         struct cxl_port *port = cxl_rr->port;
 976         struct cxl_memdev *cxlmd_peer;
 977         struct cxl_ep *ep_peer;
 978         int pos = cxled->pos;
 979
 980         /*
 981          * If this position wants to share a dport with the last endpoint mapped
 982          * then that endpoint, at index 'position - distance', must also be
 983          * mapped by this dport.
 984          */
 985         if (pos < distance) {
 986                 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
 987                         dev_name(port->uport), dev_name(&port->dev),
 988                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
 989                 return -ENXIO;
 990         }
 991         cxled_peer = p->targets[pos - distance];
 992         cxlmd_peer = cxled_to_memdev(cxled_peer);
 993         ep_peer = cxl_ep_load(port, cxlmd_peer);
 994         if (ep->dport != ep_peer->dport) {
 995                 dev_dbg(&cxlr->dev,
 996                         "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
 997                         dev_name(port->uport), dev_name(&port->dev),
 998                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
 999                         dev_name(&cxlmd_peer->dev),
1000                         dev_name(&cxled_peer->cxld.dev));
1001                 return -ENXIO;
1002         }
1003
1004         return 0;
1005 }
1006
1007 static int cxl_port_setup_targets(struct cxl_port *port,
1008                                   struct cxl_region *cxlr,
1009                                   struct cxl_endpoint_decoder *cxled)
1010 {
1011         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1012         int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1013         struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1014         struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1015         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1016         struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1017         struct cxl_region_params *p = &cxlr->params;
1018         struct cxl_decoder *cxld = cxl_rr->decoder;
1019         struct cxl_switch_decoder *cxlsd;
1020         u16 eig, peig;
1021         u8 eiw, peiw;
1022
1023         /*
1024          * While root level decoders support x3, x6, x12, switch level
1025          * decoders only support powers of 2 up to x16.
1026          */
1027         if (!is_power_of_2(cxl_rr->nr_targets)) {
1028                 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1029                         dev_name(port->uport), dev_name(&port->dev),
1030                         cxl_rr->nr_targets);
1031                 return -EINVAL;
1032         }
1033
1034         cxlsd = to_cxl_switch_decoder(&cxld->dev);
1035         if (cxl_rr->nr_targets_set) {
1036                 int i, distance;
1037
1038                 /*
1039                  * Passthrough decoders impose no distance requirements between
1040                  * peers
1041                  */
1042                 if (cxl_rr->nr_targets == 1)
1043                         distance = 0;
1044                 else
1045                         distance = p->nr_targets / cxl_rr->nr_targets;
1046                 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1047                         if (ep->dport == cxlsd->target[i]) {
1048                                 rc = check_last_peer(cxled, ep, cxl_rr,
1049                                                      distance);
1050                                 if (rc)
1051                                         return rc;
1052                                 goto out_target_set;
1053                         }
1054                 goto add_target;
1055         }
1056
1057         if (is_cxl_root(parent_port)) {
1058                 parent_ig = cxlrd->cxlsd.cxld.interleave_granularity;
1059                 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1060                 /*
1061                  * For purposes of address bit routing, use power-of-2 math for
1062                  * switch ports.
1063                  */
1064                 if (!is_power_of_2(parent_iw))
1065                         parent_iw /= 3;
1066         } else {
1067                 struct cxl_region_ref *parent_rr;
1068                 struct cxl_decoder *parent_cxld;
1069
1070                 parent_rr = cxl_rr_load(parent_port, cxlr);
1071                 parent_cxld = parent_rr->decoder;
1072                 parent_ig = parent_cxld->interleave_granularity;
1073                 parent_iw = parent_cxld->interleave_ways;
1074         }
1075
1076         rc = granularity_to_eig(parent_ig, &peig);
1077         if (rc) {
1078                 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1079                         dev_name(parent_port->uport),
1080                         dev_name(&parent_port->dev), parent_ig);
1081                 return rc;
1082         }
1083
1084         rc = ways_to_eiw(parent_iw, &peiw);
1085         if (rc) {
1086                 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1087                         dev_name(parent_port->uport),
1088                         dev_name(&parent_port->dev), parent_iw);
1089                 return rc;
1090         }
1091
1092         iw = cxl_rr->nr_targets;
1093         rc = ways_to_eiw(iw, &eiw);
1094         if (rc) {
1095                 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1096                         dev_name(port->uport), dev_name(&port->dev), iw);
1097                 return rc;
1098         }
1099
1100         /*
1101          * If @parent_port is masking address bits, pick the next unused address
1102          * bit to route @port's targets.
1103          */
1104         if (parent_iw > 1 && cxl_rr->nr_targets > 1) {
1105                 u32 address_bit = max(peig + peiw, eiw + peig);
1106
1107                 eig = address_bit - eiw + 1;
1108         } else {
1109                 eiw = peiw;
1110                 eig = peig;
1111         }
1112
1113         rc = eig_to_granularity(eig, &ig);
1114         if (rc) {
1115                 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1116                         dev_name(port->uport), dev_name(&port->dev),
1117                         256 << eig);
1118                 return rc;
1119         }
1120
1121         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1122                 if (cxld->interleave_ways != iw ||
1123                     cxld->interleave_granularity != ig ||
1124                     cxld->hpa_range.start != p->res->start ||
1125                     cxld->hpa_range.end != p->res->end ||
1126                     ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1127                         dev_err(&cxlr->dev,
1128                                 "%s:%s %s expected iw: %d ig: %d %pr\n",
1129                                 dev_name(port->uport), dev_name(&port->dev),
1130                                 __func__, iw, ig, p->res);
1131                         dev_err(&cxlr->dev,
1132                                 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1133                                 dev_name(port->uport), dev_name(&port->dev),
1134                                 __func__, cxld->interleave_ways,
1135                                 cxld->interleave_granularity,
1136                                 (cxld->flags & CXL_DECODER_F_ENABLE) ?
1137                                         "enabled" :
1138                                         "disabled",
1139                                 cxld->hpa_range.start, cxld->hpa_range.end);
1140                         return -ENXIO;
1141                 }
1142         } else {
1143                 cxld->interleave_ways = iw;
1144                 cxld->interleave_granularity = ig;
1145                 cxld->hpa_range = (struct range) {
1146                         .start = p->res->start,
1147                         .end = p->res->end,
1148                 };
1149         }
1150         dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport),
1151                 dev_name(&port->dev), iw, ig);
1152 add_target:
1153         if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1154                 dev_dbg(&cxlr->dev,
1155                         "%s:%s: targets full trying to add %s:%s at %d\n",
1156                         dev_name(port->uport), dev_name(&port->dev),
1157                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1158                 return -ENXIO;
1159         }
1160         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1161                 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1162                         dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1163                                 dev_name(port->uport), dev_name(&port->dev),
1164                                 dev_name(&cxlsd->cxld.dev),
1165                                 dev_name(ep->dport->dport),
1166                                 cxl_rr->nr_targets_set);
1167                         return -ENXIO;
1168                 }
1169         } else
1170                 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1171         inc = 1;
1172 out_target_set:
1173         cxl_rr->nr_targets_set += inc;
1174         dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1175                 dev_name(port->uport), dev_name(&port->dev),
1176                 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport),
1177                 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1178
1179         return 0;
1180 }
1181
1182 static void cxl_port_reset_targets(struct cxl_port *port,
1183                                    struct cxl_region *cxlr)
1184 {
1185         struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1186         struct cxl_decoder *cxld;
1187
1188         /*
1189          * After the last endpoint has been detached the entire cxl_rr may now
1190          * be gone.
1191          */
1192         if (!cxl_rr)
1193                 return;
1194         cxl_rr->nr_targets_set = 0;
1195
1196         cxld = cxl_rr->decoder;
1197         cxld->hpa_range = (struct range) {
1198                 .start = 0,
1199                 .end = -1,
1200         };
1201 }
1202
1203 static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1204 {
1205         struct cxl_region_params *p = &cxlr->params;
1206         struct cxl_endpoint_decoder *cxled;
1207         struct cxl_dev_state *cxlds;
1208         struct cxl_memdev *cxlmd;
1209         struct cxl_port *iter;
1210         struct cxl_ep *ep;
1211         int i;
1212
1213         /*
1214          * In the auto-discovery case skip automatic teardown since the
1215          * address space is already active
1216          */
1217         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1218                 return;
1219
1220         for (i = 0; i < p->nr_targets; i++) {
1221                 cxled = p->targets[i];
1222                 cxlmd = cxled_to_memdev(cxled);
1223                 cxlds = cxlmd->cxlds;
1224
1225                 if (cxlds->rcd)
1226                         continue;
1227
1228                 iter = cxled_to_port(cxled);
1229                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1230                         iter = to_cxl_port(iter->dev.parent);
1231
1232                 for (ep = cxl_ep_load(iter, cxlmd); iter;
1233                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1234                         cxl_port_reset_targets(iter, cxlr);
1235         }
1236 }
1237
1238 static int cxl_region_setup_targets(struct cxl_region *cxlr)
1239 {
1240         struct cxl_region_params *p = &cxlr->params;
1241         struct cxl_endpoint_decoder *cxled;
1242         struct cxl_dev_state *cxlds;
1243         int i, rc, rch = 0, vh = 0;
1244         struct cxl_memdev *cxlmd;
1245         struct cxl_port *iter;
1246         struct cxl_ep *ep;
1247
1248         for (i = 0; i < p->nr_targets; i++) {
1249                 cxled = p->targets[i];
1250                 cxlmd = cxled_to_memdev(cxled);
1251                 cxlds = cxlmd->cxlds;
1252
1253                 /* validate that all targets agree on topology */
1254                 if (!cxlds->rcd) {
1255                         vh++;
1256                 } else {
1257                         rch++;
1258                         continue;
1259                 }
1260
1261                 iter = cxled_to_port(cxled);
1262                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1263                         iter = to_cxl_port(iter->dev.parent);
1264
1265                 /*
1266                  * Descend the topology tree programming / validating
1267                  * targets while looking for conflicts.
1268                  */
1269                 for (ep = cxl_ep_load(iter, cxlmd); iter;
1270                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1271                         rc = cxl_port_setup_targets(iter, cxlr, cxled);
1272                         if (rc) {
1273                                 cxl_region_teardown_targets(cxlr);
1274                                 return rc;
1275                         }
1276                 }
1277         }
1278
1279         if (rch && vh) {
1280                 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1281                 cxl_region_teardown_targets(cxlr);
1282                 return -ENXIO;
1283         }
1284
1285         return 0;
1286 }
1287
1288 static int cxl_region_validate_position(struct cxl_region *cxlr,
1289                                         struct cxl_endpoint_decoder *cxled,
1290                                         int pos)
1291 {
1292         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1293         struct cxl_region_params *p = &cxlr->params;
1294         int i;
1295
1296         if (pos < 0 || pos >= p->interleave_ways) {
1297                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1298                         p->interleave_ways);
1299                 return -ENXIO;
1300         }
1301
1302         if (p->targets[pos] == cxled)
1303                 return 0;
1304
1305         if (p->targets[pos]) {
1306                 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1307                 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1308
1309                 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1310                         pos, dev_name(&cxlmd_target->dev),
1311                         dev_name(&cxled_target->cxld.dev));
1312                 return -EBUSY;
1313         }
1314
1315         for (i = 0; i < p->interleave_ways; i++) {
1316                 struct cxl_endpoint_decoder *cxled_target;
1317                 struct cxl_memdev *cxlmd_target;
1318
1319                 cxled_target = p->targets[i];
1320                 if (!cxled_target)
1321                         continue;
1322
1323                 cxlmd_target = cxled_to_memdev(cxled_target);
1324                 if (cxlmd_target == cxlmd) {
1325                         dev_dbg(&cxlr->dev,
1326                                 "%s already specified at position %d via: %s\n",
1327                                 dev_name(&cxlmd->dev), pos,
1328                                 dev_name(&cxled_target->cxld.dev));
1329                         return -EBUSY;
1330                 }
1331         }
1332
1333         return 0;
1334 }
1335
1336 static int cxl_region_attach_position(struct cxl_region *cxlr,
1337                                       struct cxl_root_decoder *cxlrd,
1338                                       struct cxl_endpoint_decoder *cxled,
1339                                       const struct cxl_dport *dport, int pos)
1340 {
1341         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1342         struct cxl_port *iter;
1343         int rc;
1344
1345         if (cxlrd->calc_hb(cxlrd, pos) != dport) {
1346                 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1347                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1348                         dev_name(&cxlrd->cxlsd.cxld.dev));
1349                 return -ENXIO;
1350         }
1351
1352         for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1353              iter = to_cxl_port(iter->dev.parent)) {
1354                 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1355                 if (rc)
1356                         goto err;
1357         }
1358
1359         return 0;
1360
1361 err:
1362         for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1363              iter = to_cxl_port(iter->dev.parent))
1364                 cxl_port_detach_region(iter, cxlr, cxled);
1365         return rc;
1366 }
1367
1368 static int cxl_region_attach_auto(struct cxl_region *cxlr,
1369                                   struct cxl_endpoint_decoder *cxled, int pos)
1370 {
1371         struct cxl_region_params *p = &cxlr->params;
1372
1373         if (cxled->state != CXL_DECODER_STATE_AUTO) {
1374                 dev_err(&cxlr->dev,
1375                         "%s: unable to add decoder to autodetected region\n",
1376                         dev_name(&cxled->cxld.dev));
1377                 return -EINVAL;
1378         }
1379
1380         if (pos >= 0) {
1381                 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1382                         dev_name(&cxled->cxld.dev), pos);
1383                 return -EINVAL;
1384         }
1385
1386         if (p->nr_targets >= p->interleave_ways) {
1387                 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1388                         dev_name(&cxled->cxld.dev));
1389                 return -ENXIO;
1390         }
1391
1392         /*
1393          * Temporarily record the endpoint decoder into the target array. Yes,
1394          * this means that userspace can view devices in the wrong position
1395          * before the region activates, and must be careful to understand when
1396          * it might be racing region autodiscovery.
1397          */
1398         pos = p->nr_targets;
1399         p->targets[pos] = cxled;
1400         cxled->pos = pos;
1401         p->nr_targets++;
1402
1403         return 0;
1404 }
1405
1406 static struct cxl_port *next_port(struct cxl_port *port)
1407 {
1408         if (!port->parent_dport)
1409                 return NULL;
1410         return port->parent_dport->port;
1411 }
1412
1413 static int decoder_match_range(struct device *dev, void *data)
1414 {
1415         struct cxl_endpoint_decoder *cxled = data;
1416         struct cxl_switch_decoder *cxlsd;
1417
1418         if (!is_switch_decoder(dev))
1419                 return 0;
1420
1421         cxlsd = to_cxl_switch_decoder(dev);
1422         return range_contains(&cxlsd->cxld.hpa_range, &cxled->cxld.hpa_range);
1423 }
1424
1425 static void find_positions(const struct cxl_switch_decoder *cxlsd,
1426                            const struct cxl_port *iter_a,
1427                            const struct cxl_port *iter_b, int *a_pos,
1428                            int *b_pos)
1429 {
1430         int i;
1431
1432         for (i = 0, *a_pos = -1, *b_pos = -1; i < cxlsd->nr_targets; i++) {
1433                 if (cxlsd->target[i] == iter_a->parent_dport)
1434                         *a_pos = i;
1435                 else if (cxlsd->target[i] == iter_b->parent_dport)
1436                         *b_pos = i;
1437                 if (*a_pos >= 0 && *b_pos >= 0)
1438                         break;
1439         }
1440 }
1441
1442 static int cmp_decode_pos(const void *a, const void *b)
1443 {
1444         struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1445         struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1446         struct cxl_memdev *cxlmd_a = cxled_to_memdev(cxled_a);
1447         struct cxl_memdev *cxlmd_b = cxled_to_memdev(cxled_b);
1448         struct cxl_port *port_a = cxled_to_port(cxled_a);
1449         struct cxl_port *port_b = cxled_to_port(cxled_b);
1450         struct cxl_port *iter_a, *iter_b, *port = NULL;
1451         struct cxl_switch_decoder *cxlsd;
1452         struct device *dev;
1453         int a_pos, b_pos;
1454         unsigned int seq;
1455
1456         /* Exit early if any prior sorting failed */
1457         if (cxled_a->pos < 0 || cxled_b->pos < 0)
1458                 return 0;
1459
1460         /*
1461          * Walk up the hierarchy to find a shared port, find the decoder that
1462          * maps the range, compare the relative position of those dport
1463          * mappings.
1464          */
1465         for (iter_a = port_a; iter_a; iter_a = next_port(iter_a)) {
1466                 struct cxl_port *next_a, *next_b;
1467
1468                 next_a = next_port(iter_a);
1469                 if (!next_a)
1470                         break;
1471
1472                 for (iter_b = port_b; iter_b; iter_b = next_port(iter_b)) {
1473                         next_b = next_port(iter_b);
1474                         if (next_a != next_b)
1475                                 continue;
1476                         port = next_a;
1477                         break;
1478                 }
1479
1480                 if (port)
1481                         break;
1482         }
1483
1484         if (!port) {
1485                 dev_err(cxlmd_a->dev.parent,
1486                         "failed to find shared port with %s\n",
1487                         dev_name(cxlmd_b->dev.parent));
1488                 goto err;
1489         }
1490
1491         dev = device_find_child(&port->dev, cxled_a, decoder_match_range);
1492         if (!dev) {
1493                 struct range *range = &cxled_a->cxld.hpa_range;
1494
1495                 dev_err(port->uport,
1496                         "failed to find decoder that maps %#llx-%#llx\n",
1497                         range->start, range->end);
1498                 goto err;
1499         }
1500
1501         cxlsd = to_cxl_switch_decoder(dev);
1502         do {
1503                 seq = read_seqbegin(&cxlsd->target_lock);
1504                 find_positions(cxlsd, iter_a, iter_b, &a_pos, &b_pos);
1505         } while (read_seqretry(&cxlsd->target_lock, seq));
1506
1507         put_device(dev);
1508
1509         if (a_pos < 0 || b_pos < 0) {
1510                 dev_err(port->uport,
1511                         "failed to find shared decoder for %s and %s\n",
1512                         dev_name(cxlmd_a->dev.parent),
1513                         dev_name(cxlmd_b->dev.parent));
1514                 goto err;
1515         }
1516
1517         dev_dbg(port->uport, "%s comes %s %s\n", dev_name(cxlmd_a->dev.parent),
1518                 a_pos - b_pos < 0 ? "before" : "after",
1519                 dev_name(cxlmd_b->dev.parent));
1520
1521         return a_pos - b_pos;
1522 err:
1523         cxled_a->pos = -1;
1524         return 0;
1525 }
1526
1527 static int cxl_region_sort_targets(struct cxl_region *cxlr)
1528 {
1529         struct cxl_region_params *p = &cxlr->params;
1530         int i, rc = 0;
1531
1532         sort(p->targets, p->nr_targets, sizeof(p->targets[0]), cmp_decode_pos,
1533              NULL);
1534
1535         for (i = 0; i < p->nr_targets; i++) {
1536                 struct cxl_endpoint_decoder *cxled = p->targets[i];
1537
1538                 /*
1539                  * Record that sorting failed, but still continue to restore
1540                  * cxled->pos with its ->targets[] position so that follow-on
1541                  * code paths can reliably do p->targets[cxled->pos] to
1542                  * self-reference their entry.
1543                  */
1544                 if (cxled->pos < 0)
1545                         rc = -ENXIO;
1546                 cxled->pos = i;
1547         }
1548
1549         dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1550         return rc;
1551 }
1552
1553 static int cxl_region_attach(struct cxl_region *cxlr,
1554                              struct cxl_endpoint_decoder *cxled, int pos)
1555 {
1556         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1557         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1558         struct cxl_region_params *p = &cxlr->params;
1559         struct cxl_port *ep_port, *root_port;
1560         struct cxl_dport *dport;
1561         int rc = -ENXIO;
1562
1563         if (cxled->mode != cxlr->mode) {
1564                 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1565                         dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1566                 return -EINVAL;
1567         }
1568
1569         if (cxled->mode == CXL_DECODER_DEAD) {
1570                 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1571                 return -ENODEV;
1572         }
1573
1574         /* all full of members, or interleave config not established? */
1575         if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1576                 dev_dbg(&cxlr->dev, "region already active\n");
1577                 return -EBUSY;
1578         } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1579                 dev_dbg(&cxlr->dev, "interleave config missing\n");
1580                 return -ENXIO;
1581         }
1582
1583         ep_port = cxled_to_port(cxled);
1584         root_port = cxlrd_to_port(cxlrd);
1585         dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1586         if (!dport) {
1587                 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1588                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1589                         dev_name(cxlr->dev.parent));
1590                 return -ENXIO;
1591         }
1592
1593         if (cxled->cxld.target_type != cxlr->type) {
1594                 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1595                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1596                         cxled->cxld.target_type, cxlr->type);
1597                 return -ENXIO;
1598         }
1599
1600         if (!cxled->dpa_res) {
1601                 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1602                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1603                 return -ENXIO;
1604         }
1605
1606         if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1607             resource_size(p->res)) {
1608                 dev_dbg(&cxlr->dev,
1609                         "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1610                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1611                         (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1612                         (u64)resource_size(p->res));
1613                 return -EINVAL;
1614         }
1615
1616         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1617                 int i;
1618
1619                 rc = cxl_region_attach_auto(cxlr, cxled, pos);
1620                 if (rc)
1621                         return rc;
1622
1623                 /* await more targets to arrive... */
1624                 if (p->nr_targets < p->interleave_ways)
1625                         return 0;
1626
1627                 /*
1628                  * All targets are here, which implies all PCI enumeration that
1629                  * affects this region has been completed. Walk the topology to
1630                  * sort the devices into their relative region decode position.
1631                  */
1632                 rc = cxl_region_sort_targets(cxlr);
1633                 if (rc)
1634                         return rc;
1635
1636                 for (i = 0; i < p->nr_targets; i++) {
1637                         cxled = p->targets[i];
1638                         ep_port = cxled_to_port(cxled);
1639                         dport = cxl_find_dport_by_dev(root_port,
1640                                                       ep_port->host_bridge);
1641                         rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1642                                                         dport, i);
1643                         if (rc)
1644                                 return rc;
1645                 }
1646
1647                 rc = cxl_region_setup_targets(cxlr);
1648                 if (rc)
1649                         return rc;
1650
1651                 /*
1652                  * If target setup succeeds in the autodiscovery case
1653                  * then the region is already committed.
1654                  */
1655                 p->state = CXL_CONFIG_COMMIT;
1656
1657                 return 0;
1658         }
1659
1660         rc = cxl_region_validate_position(cxlr, cxled, pos);
1661         if (rc)
1662                 return rc;
1663
1664         rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
1665         if (rc)
1666                 return rc;
1667
1668         p->targets[pos] = cxled;
1669         cxled->pos = pos;
1670         p->nr_targets++;
1671
1672         if (p->nr_targets == p->interleave_ways) {
1673                 rc = cxl_region_setup_targets(cxlr);
1674                 if (rc)
1675                         goto err_decrement;
1676                 p->state = CXL_CONFIG_ACTIVE;
1677                 set_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags);
1678         }
1679
1680         cxled->cxld.interleave_ways = p->interleave_ways;
1681         cxled->cxld.interleave_granularity = p->interleave_granularity;
1682         cxled->cxld.hpa_range = (struct range) {
1683                 .start = p->res->start,
1684                 .end = p->res->end,
1685         };
1686
1687         return 0;
1688
1689 err_decrement:
1690         p->nr_targets--;
1691         cxled->pos = -1;
1692         p->targets[pos] = NULL;
1693         return rc;
1694 }
1695
1696 static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
1697 {
1698         struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
1699         struct cxl_region *cxlr = cxled->cxld.region;
1700         struct cxl_region_params *p;
1701         int rc = 0;
1702
1703         lockdep_assert_held_write(&cxl_region_rwsem);
1704
1705         if (!cxlr)
1706                 return 0;
1707
1708         p = &cxlr->params;
1709         get_device(&cxlr->dev);
1710
1711         if (p->state > CXL_CONFIG_ACTIVE) {
1712                 /*
1713                  * TODO: tear down all impacted regions if a device is
1714                  * removed out of order
1715                  */
1716                 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
1717                 if (rc)
1718                         goto out;
1719                 p->state = CXL_CONFIG_ACTIVE;
1720         }
1721
1722         for (iter = ep_port; !is_cxl_root(iter);
1723              iter = to_cxl_port(iter->dev.parent))
1724                 cxl_port_detach_region(iter, cxlr, cxled);
1725
1726         if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
1727             p->targets[cxled->pos] != cxled) {
1728                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1729
1730                 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
1731                               dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1732                               cxled->pos);
1733                 goto out;
1734         }
1735
1736         if (p->state == CXL_CONFIG_ACTIVE) {
1737                 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
1738                 cxl_region_teardown_targets(cxlr);
1739         }
1740         p->targets[cxled->pos] = NULL;
1741         p->nr_targets--;
1742         cxled->cxld.hpa_range = (struct range) {
1743                 .start = 0,
1744                 .end = -1,
1745         };
1746
1747         /* notify the region driver that one of its targets has departed */
1748         up_write(&cxl_region_rwsem);
1749         device_release_driver(&cxlr->dev);
1750         down_write(&cxl_region_rwsem);
1751 out:
1752         put_device(&cxlr->dev);
1753         return rc;
1754 }
1755
1756 void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
1757 {
1758         down_write(&cxl_region_rwsem);
1759         cxled->mode = CXL_DECODER_DEAD;
1760         cxl_region_detach(cxled);
1761         up_write(&cxl_region_rwsem);
1762 }
1763
1764 static int attach_target(struct cxl_region *cxlr,
1765                          struct cxl_endpoint_decoder *cxled, int pos,
1766                          unsigned int state)
1767 {
1768         int rc = 0;
1769
1770         if (state == TASK_INTERRUPTIBLE)
1771                 rc = down_write_killable(&cxl_region_rwsem);
1772         else
1773                 down_write(&cxl_region_rwsem);
1774         if (rc)
1775                 return rc;
1776
1777         down_read(&cxl_dpa_rwsem);
1778         rc = cxl_region_attach(cxlr, cxled, pos);
1779         up_read(&cxl_dpa_rwsem);
1780         up_write(&cxl_region_rwsem);
1781         return rc;
1782 }
1783
1784 static int detach_target(struct cxl_region *cxlr, int pos)
1785 {
1786         struct cxl_region_params *p = &cxlr->params;
1787         int rc;
1788
1789         rc = down_write_killable(&cxl_region_rwsem);
1790         if (rc)
1791                 return rc;
1792
1793         if (pos >= p->interleave_ways) {
1794                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1795                         p->interleave_ways);
1796                 rc = -ENXIO;
1797                 goto out;
1798         }
1799
1800         if (!p->targets[pos]) {
1801                 rc = 0;
1802                 goto out;
1803         }
1804
1805         rc = cxl_region_detach(p->targets[pos]);
1806 out:
1807         up_write(&cxl_region_rwsem);
1808         return rc;
1809 }
1810
1811 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
1812                             size_t len)
1813 {
1814         int rc;
1815
1816         if (sysfs_streq(buf, "\n"))
1817                 rc = detach_target(cxlr, pos);
1818         else {
1819                 struct device *dev;
1820
1821                 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
1822                 if (!dev)
1823                         return -ENODEV;
1824
1825                 if (!is_endpoint_decoder(dev)) {
1826                         rc = -EINVAL;
1827                         goto out;
1828                 }
1829
1830                 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
1831                                    TASK_INTERRUPTIBLE);
1832 out:
1833                 put_device(dev);
1834         }
1835
1836         if (rc < 0)
1837                 return rc;
1838         return len;
1839 }
1840
1841 #define TARGET_ATTR_RW(n)                                              \
1842 static ssize_t target##n##_show(                                       \
1843         struct device *dev, struct device_attribute *attr, char *buf)  \
1844 {                                                                      \
1845         return show_targetN(to_cxl_region(dev), buf, (n));             \
1846 }                                                                      \
1847 static ssize_t target##n##_store(struct device *dev,                   \
1848                                  struct device_attribute *attr,        \
1849                                  const char *buf, size_t len)          \
1850 {                                                                      \
1851         return store_targetN(to_cxl_region(dev), buf, (n), len);       \
1852 }                                                                      \
1853 static DEVICE_ATTR_RW(target##n)
1854
1855 TARGET_ATTR_RW(0);
1856 TARGET_ATTR_RW(1);
1857 TARGET_ATTR_RW(2);
1858 TARGET_ATTR_RW(3);
1859 TARGET_ATTR_RW(4);
1860 TARGET_ATTR_RW(5);
1861 TARGET_ATTR_RW(6);
1862 TARGET_ATTR_RW(7);
1863 TARGET_ATTR_RW(8);
1864 TARGET_ATTR_RW(9);
1865 TARGET_ATTR_RW(10);
1866 TARGET_ATTR_RW(11);
1867 TARGET_ATTR_RW(12);
1868 TARGET_ATTR_RW(13);
1869 TARGET_ATTR_RW(14);
1870 TARGET_ATTR_RW(15);
1871
1872 static struct attribute *target_attrs[] = {
1873         &dev_attr_target0.attr,
1874         &dev_attr_target1.attr,
1875         &dev_attr_target2.attr,
1876         &dev_attr_target3.attr,
1877         &dev_attr_target4.attr,
1878         &dev_attr_target5.attr,
1879         &dev_attr_target6.attr,
1880         &dev_attr_target7.attr,
1881         &dev_attr_target8.attr,
1882         &dev_attr_target9.attr,
1883         &dev_attr_target10.attr,
1884         &dev_attr_target11.attr,
1885         &dev_attr_target12.attr,
1886         &dev_attr_target13.attr,
1887         &dev_attr_target14.attr,
1888         &dev_attr_target15.attr,
1889         NULL,
1890 };
1891
1892 static umode_t cxl_region_target_visible(struct kobject *kobj,
1893                                          struct attribute *a, int n)
1894 {
1895         struct device *dev = kobj_to_dev(kobj);
1896         struct cxl_region *cxlr = to_cxl_region(dev);
1897         struct cxl_region_params *p = &cxlr->params;
1898
1899         if (n < p->interleave_ways)
1900                 return a->mode;
1901         return 0;
1902 }
1903
1904 static const struct attribute_group cxl_region_target_group = {
1905         .attrs = target_attrs,
1906         .is_visible = cxl_region_target_visible,
1907 };
1908
1909 static const struct attribute_group *get_cxl_region_target_group(void)
1910 {
1911         return &cxl_region_target_group;
1912 }
1913
1914 static const struct attribute_group *region_groups[] = {
1915         &cxl_base_attribute_group,
1916         &cxl_region_group,
1917         &cxl_region_target_group,
1918         NULL,
1919 };
1920
1921 static void cxl_region_release(struct device *dev)
1922 {
1923         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
1924         struct cxl_region *cxlr = to_cxl_region(dev);
1925         int id = atomic_read(&cxlrd->region_id);
1926
1927         /*
1928          * Try to reuse the recently idled id rather than the cached
1929          * next id to prevent the region id space from increasing
1930          * unnecessarily.
1931          */
1932         if (cxlr->id < id)
1933                 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
1934                         memregion_free(id);
1935                         goto out;
1936                 }
1937
1938         memregion_free(cxlr->id);
1939 out:
1940         put_device(dev->parent);
1941         kfree(cxlr);
1942 }
1943
1944 const struct device_type cxl_region_type = {
1945         .name = "cxl_region",
1946         .release = cxl_region_release,
1947         .groups = region_groups
1948 };
1949
1950 bool is_cxl_region(struct device *dev)
1951 {
1952         return dev->type == &cxl_region_type;
1953 }
1954 EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
1955
1956 static struct cxl_region *to_cxl_region(struct device *dev)
1957 {
1958         if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
1959                           "not a cxl_region device\n"))
1960                 return NULL;
1961
1962         return container_of(dev, struct cxl_region, dev);
1963 }
1964
1965 static void unregister_region(void *dev)
1966 {
1967         struct cxl_region *cxlr = to_cxl_region(dev);
1968         struct cxl_region_params *p = &cxlr->params;
1969         int i;
1970
1971         device_del(dev);
1972
1973         /*
1974          * Now that region sysfs is shutdown, the parameter block is now
1975          * read-only, so no need to hold the region rwsem to access the
1976          * region parameters.
1977          */
1978         for (i = 0; i < p->interleave_ways; i++)
1979                 detach_target(cxlr, i);
1980
1981         cxl_region_iomem_release(cxlr);
1982         put_device(dev);
1983 }
1984
1985 static struct lock_class_key cxl_region_key;
1986
1987 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
1988 {
1989         struct cxl_region *cxlr;
1990         struct device *dev;
1991
1992         cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
1993         if (!cxlr) {
1994                 memregion_free(id);
1995                 return ERR_PTR(-ENOMEM);
1996         }
1997
1998         dev = &cxlr->dev;
1999         device_initialize(dev);
2000         lockdep_set_class(&dev->mutex, &cxl_region_key);
2001         dev->parent = &cxlrd->cxlsd.cxld.dev;
2002         /*
2003          * Keep root decoder pinned through cxl_region_release to fixup
2004          * region id allocations
2005          */
2006         get_device(dev->parent);
2007         device_set_pm_not_required(dev);
2008         dev->bus = &cxl_bus_type;
2009         dev->type = &cxl_region_type;
2010         cxlr->id = id;
2011
2012         return cxlr;
2013 }
2014
2015 /**
2016  * devm_cxl_add_region - Adds a region to a decoder
2017  * @cxlrd: root decoder
2018  * @id: memregion id to create, or memregion_free() on failure
2019  * @mode: mode for the endpoint decoders of this region
2020  * @type: select whether this is an expander or accelerator (type-2 or type-3)
2021  *
2022  * This is the second step of region initialization. Regions exist within an
2023  * address space which is mapped by a @cxlrd.
2024  *
2025  * Return: 0 if the region was added to the @cxlrd, else returns negative error
2026  * code. The region will be named "regionZ" where Z is the unique region number.
2027  */
2028 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2029                                               int id,
2030                                               enum cxl_decoder_mode mode,
2031                                               enum cxl_decoder_type type)
2032 {
2033         struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2034         struct cxl_region *cxlr;
2035         struct device *dev;
2036         int rc;
2037
2038         switch (mode) {
2039         case CXL_DECODER_RAM:
2040         case CXL_DECODER_PMEM:
2041                 break;
2042         default:
2043                 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2044                 return ERR_PTR(-EINVAL);
2045         }
2046
2047         cxlr = cxl_region_alloc(cxlrd, id);
2048         if (IS_ERR(cxlr))
2049                 return cxlr;
2050         cxlr->mode = mode;
2051         cxlr->type = type;
2052
2053         dev = &cxlr->dev;
2054         rc = dev_set_name(dev, "region%d", id);
2055         if (rc)
2056                 goto err;
2057
2058         rc = device_add(dev);
2059         if (rc)
2060                 goto err;
2061
2062         rc = devm_add_action_or_reset(port->uport, unregister_region, cxlr);
2063         if (rc)
2064                 return ERR_PTR(rc);
2065
2066         dev_dbg(port->uport, "%s: created %s\n",
2067                 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2068         return cxlr;
2069
2070 err:
2071         put_device(dev);
2072         return ERR_PTR(rc);
2073 }
2074
2075 static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2076 {
2077         return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2078 }
2079
2080 static ssize_t create_pmem_region_show(struct device *dev,
2081                                        struct device_attribute *attr, char *buf)
2082 {
2083         return __create_region_show(to_cxl_root_decoder(dev), buf);
2084 }
2085
2086 static ssize_t create_ram_region_show(struct device *dev,
2087                                       struct device_attribute *attr, char *buf)
2088 {
2089         return __create_region_show(to_cxl_root_decoder(dev), buf);
2090 }
2091
2092 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2093                                           enum cxl_decoder_mode mode, int id)
2094 {
2095         int rc;
2096
2097         rc = memregion_alloc(GFP_KERNEL);
2098         if (rc < 0)
2099                 return ERR_PTR(rc);
2100
2101         if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2102                 memregion_free(rc);
2103                 return ERR_PTR(-EBUSY);
2104         }
2105
2106         return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_EXPANDER);
2107 }
2108
2109 static ssize_t create_pmem_region_store(struct device *dev,
2110                                         struct device_attribute *attr,
2111                                         const char *buf, size_t len)
2112 {
2113         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2114         struct cxl_region *cxlr;
2115         int rc, id;
2116
2117         rc = sscanf(buf, "region%d\n", &id);
2118         if (rc != 1)
2119                 return -EINVAL;
2120
2121         cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id);
2122         if (IS_ERR(cxlr))
2123                 return PTR_ERR(cxlr);
2124
2125         return len;
2126 }
2127 DEVICE_ATTR_RW(create_pmem_region);
2128
2129 static ssize_t create_ram_region_store(struct device *dev,
2130                                        struct device_attribute *attr,
2131                                        const char *buf, size_t len)
2132 {
2133         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2134         struct cxl_region *cxlr;
2135         int rc, id;
2136
2137         rc = sscanf(buf, "region%d\n", &id);
2138         if (rc != 1)
2139                 return -EINVAL;
2140
2141         cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id);
2142         if (IS_ERR(cxlr))
2143                 return PTR_ERR(cxlr);
2144
2145         return len;
2146 }
2147 DEVICE_ATTR_RW(create_ram_region);
2148
2149 static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2150                            char *buf)
2151 {
2152         struct cxl_decoder *cxld = to_cxl_decoder(dev);
2153         ssize_t rc;
2154
2155         rc = down_read_interruptible(&cxl_region_rwsem);
2156         if (rc)
2157                 return rc;
2158
2159         if (cxld->region)
2160                 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2161         else
2162                 rc = sysfs_emit(buf, "\n");
2163         up_read(&cxl_region_rwsem);
2164
2165         return rc;
2166 }
2167 DEVICE_ATTR_RO(region);
2168
2169 static struct cxl_region *
2170 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2171 {
2172         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2173         struct device *region_dev;
2174
2175         region_dev = device_find_child_by_name(&cxld->dev, name);
2176         if (!region_dev)
2177                 return ERR_PTR(-ENODEV);
2178
2179         return to_cxl_region(region_dev);
2180 }
2181
2182 static ssize_t delete_region_store(struct device *dev,
2183                                    struct device_attribute *attr,
2184                                    const char *buf, size_t len)
2185 {
2186         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2187         struct cxl_port *port = to_cxl_port(dev->parent);
2188         struct cxl_region *cxlr;
2189
2190         cxlr = cxl_find_region_by_name(cxlrd, buf);
2191         if (IS_ERR(cxlr))
2192                 return PTR_ERR(cxlr);
2193
2194         devm_release_action(port->uport, unregister_region, cxlr);
2195         put_device(&cxlr->dev);
2196
2197         return len;
2198 }
2199 DEVICE_ATTR_WO(delete_region);
2200
2201 static void cxl_pmem_region_release(struct device *dev)
2202 {
2203         struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2204         int i;
2205
2206         for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2207                 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2208
2209                 put_device(&cxlmd->dev);
2210         }
2211
2212         kfree(cxlr_pmem);
2213 }
2214
2215 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2216         &cxl_base_attribute_group,
2217         NULL,
2218 };
2219
2220 const struct device_type cxl_pmem_region_type = {
2221         .name = "cxl_pmem_region",
2222         .release = cxl_pmem_region_release,
2223         .groups = cxl_pmem_region_attribute_groups,
2224 };
2225
2226 bool is_cxl_pmem_region(struct device *dev)
2227 {
2228         return dev->type == &cxl_pmem_region_type;
2229 }
2230 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
2231
2232 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2233 {
2234         if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2235                           "not a cxl_pmem_region device\n"))
2236                 return NULL;
2237         return container_of(dev, struct cxl_pmem_region, dev);
2238 }
2239 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
2240
2241 struct cxl_poison_context {
2242         struct cxl_port *port;
2243         enum cxl_decoder_mode mode;
2244         u64 offset;
2245 };
2246
2247 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2248                                    struct cxl_poison_context *ctx)
2249 {
2250         struct cxl_dev_state *cxlds = cxlmd->cxlds;
2251         u64 offset, length;
2252         int rc = 0;
2253
2254         /*
2255          * Collect poison for the remaining unmapped resources
2256          * after poison is collected by committed endpoints.
2257          *
2258          * Knowing that PMEM must always follow RAM, get poison
2259          * for unmapped resources based on the last decoder's mode:
2260          *      ram: scan remains of ram range, then any pmem range
2261          *      pmem: scan remains of pmem range
2262          */
2263
2264         if (ctx->mode == CXL_DECODER_RAM) {
2265                 offset = ctx->offset;
2266                 length = resource_size(&cxlds->ram_res) - offset;
2267                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2268                 if (rc == -EFAULT)
2269                         rc = 0;
2270                 if (rc)
2271                         return rc;
2272         }
2273         if (ctx->mode == CXL_DECODER_PMEM) {
2274                 offset = ctx->offset;
2275                 length = resource_size(&cxlds->dpa_res) - offset;
2276                 if (!length)
2277                         return 0;
2278         } else if (resource_size(&cxlds->pmem_res)) {
2279                 offset = cxlds->pmem_res.start;
2280                 length = resource_size(&cxlds->pmem_res);
2281         } else {
2282                 return 0;
2283         }
2284
2285         return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2286 }
2287
2288 static int poison_by_decoder(struct device *dev, void *arg)
2289 {
2290         struct cxl_poison_context *ctx = arg;
2291         struct cxl_endpoint_decoder *cxled;
2292         struct cxl_memdev *cxlmd;
2293         u64 offset, length;
2294         int rc = 0;
2295
2296         if (!is_endpoint_decoder(dev))
2297                 return rc;
2298
2299         cxled = to_cxl_endpoint_decoder(dev);
2300         if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2301                 return rc;
2302
2303         /*
2304          * Regions are only created with single mode decoders: pmem or ram.
2305          * Linux does not support mixed mode decoders. This means that
2306          * reading poison per endpoint decoder adheres to the requirement
2307          * that poison reads of pmem and ram must be separated.
2308          * CXL 3.0 Spec 8.2.9.8.4.1
2309          */
2310         if (cxled->mode == CXL_DECODER_MIXED) {
2311                 dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2312                 return rc;
2313         }
2314
2315         cxlmd = cxled_to_memdev(cxled);
2316         if (cxled->skip) {
2317                 offset = cxled->dpa_res->start - cxled->skip;
2318                 length = cxled->skip;
2319                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2320                 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2321                         rc = 0;
2322                 if (rc)
2323                         return rc;
2324         }
2325
2326         offset = cxled->dpa_res->start;
2327         length = cxled->dpa_res->end - offset + 1;
2328         rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2329         if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2330                 rc = 0;
2331         if (rc)
2332                 return rc;
2333
2334         /* Iterate until commit_end is reached */
2335         if (cxled->cxld.id == ctx->port->commit_end) {
2336                 ctx->offset = cxled->dpa_res->end + 1;
2337                 ctx->mode = cxled->mode;
2338                 return 1;
2339         }
2340
2341         return 0;
2342 }
2343
2344 int cxl_get_poison_by_endpoint(struct cxl_port *port)
2345 {
2346         struct cxl_poison_context ctx;
2347         int rc = 0;
2348
2349         rc = down_read_interruptible(&cxl_region_rwsem);
2350         if (rc)
2351                 return rc;
2352
2353         ctx = (struct cxl_poison_context) {
2354                 .port = port
2355         };
2356
2357         rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2358         if (rc == 1)
2359                 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport), &ctx);
2360
2361         up_read(&cxl_region_rwsem);
2362         return rc;
2363 }
2364
2365 static struct lock_class_key cxl_pmem_region_key;
2366
2367 static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr)
2368 {
2369         struct cxl_region_params *p = &cxlr->params;
2370         struct cxl_nvdimm_bridge *cxl_nvb;
2371         struct cxl_pmem_region *cxlr_pmem;
2372         struct device *dev;
2373         int i;
2374
2375         down_read(&cxl_region_rwsem);
2376         if (p->state != CXL_CONFIG_COMMIT) {
2377                 cxlr_pmem = ERR_PTR(-ENXIO);
2378                 goto out;
2379         }
2380
2381         cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets),
2382                             GFP_KERNEL);
2383         if (!cxlr_pmem) {
2384                 cxlr_pmem = ERR_PTR(-ENOMEM);
2385                 goto out;
2386         }
2387
2388         cxlr_pmem->hpa_range.start = p->res->start;
2389         cxlr_pmem->hpa_range.end = p->res->end;
2390
2391         /* Snapshot the region configuration underneath the cxl_region_rwsem */
2392         cxlr_pmem->nr_mappings = p->nr_targets;
2393         for (i = 0; i < p->nr_targets; i++) {
2394                 struct cxl_endpoint_decoder *cxled = p->targets[i];
2395                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2396                 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2397
2398                 /*
2399                  * Regions never span CXL root devices, so by definition the
2400                  * bridge for one device is the same for all.
2401                  */
2402                 if (i == 0) {
2403                         cxl_nvb = cxl_find_nvdimm_bridge(cxlmd);
2404                         if (!cxl_nvb) {
2405                                 cxlr_pmem = ERR_PTR(-ENODEV);
2406                                 goto out;
2407                         }
2408                         cxlr->cxl_nvb = cxl_nvb;
2409                 }
2410                 m->cxlmd = cxlmd;
2411                 get_device(&cxlmd->dev);
2412                 m->start = cxled->dpa_res->start;
2413                 m->size = resource_size(cxled->dpa_res);
2414                 m->position = i;
2415         }
2416
2417         dev = &cxlr_pmem->dev;
2418         cxlr_pmem->cxlr = cxlr;
2419         cxlr->cxlr_pmem = cxlr_pmem;
2420         device_initialize(dev);
2421         lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2422         device_set_pm_not_required(dev);
2423         dev->parent = &cxlr->dev;
2424         dev->bus = &cxl_bus_type;
2425         dev->type = &cxl_pmem_region_type;
2426 out:
2427         up_read(&cxl_region_rwsem);
2428
2429         return cxlr_pmem;
2430 }
2431
2432 static void cxl_dax_region_release(struct device *dev)
2433 {
2434         struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
2435
2436         kfree(cxlr_dax);
2437 }
2438
2439 static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
2440         &cxl_base_attribute_group,
2441         NULL,
2442 };
2443
2444 const struct device_type cxl_dax_region_type = {
2445         .name = "cxl_dax_region",
2446         .release = cxl_dax_region_release,
2447         .groups = cxl_dax_region_attribute_groups,
2448 };
2449
2450 static bool is_cxl_dax_region(struct device *dev)
2451 {
2452         return dev->type == &cxl_dax_region_type;
2453 }
2454
2455 struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
2456 {
2457         if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
2458                           "not a cxl_dax_region device\n"))
2459                 return NULL;
2460         return container_of(dev, struct cxl_dax_region, dev);
2461 }
2462 EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL);
2463
2464 static struct lock_class_key cxl_dax_region_key;
2465
2466 static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
2467 {
2468         struct cxl_region_params *p = &cxlr->params;
2469         struct cxl_dax_region *cxlr_dax;
2470         struct device *dev;
2471
2472         down_read(&cxl_region_rwsem);
2473         if (p->state != CXL_CONFIG_COMMIT) {
2474                 cxlr_dax = ERR_PTR(-ENXIO);
2475                 goto out;
2476         }
2477
2478         cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
2479         if (!cxlr_dax) {
2480                 cxlr_dax = ERR_PTR(-ENOMEM);
2481                 goto out;
2482         }
2483
2484         cxlr_dax->hpa_range.start = p->res->start;
2485         cxlr_dax->hpa_range.end = p->res->end;
2486
2487         dev = &cxlr_dax->dev;
2488         cxlr_dax->cxlr = cxlr;
2489         device_initialize(dev);
2490         lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
2491         device_set_pm_not_required(dev);
2492         dev->parent = &cxlr->dev;
2493         dev->bus = &cxl_bus_type;
2494         dev->type = &cxl_dax_region_type;
2495 out:
2496         up_read(&cxl_region_rwsem);
2497
2498         return cxlr_dax;
2499 }
2500
2501 static void cxlr_pmem_unregister(void *_cxlr_pmem)
2502 {
2503         struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
2504         struct cxl_region *cxlr = cxlr_pmem->cxlr;
2505         struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2506
2507         /*
2508          * Either the bridge is in ->remove() context under the device_lock(),
2509          * or cxlr_release_nvdimm() is cancelling the bridge's release action
2510          * for @cxlr_pmem and doing it itself (while manually holding the bridge
2511          * lock).
2512          */
2513         device_lock_assert(&cxl_nvb->dev);
2514         cxlr->cxlr_pmem = NULL;
2515         cxlr_pmem->cxlr = NULL;
2516         device_unregister(&cxlr_pmem->dev);
2517 }
2518
2519 static void cxlr_release_nvdimm(void *_cxlr)
2520 {
2521         struct cxl_region *cxlr = _cxlr;
2522         struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2523
2524         device_lock(&cxl_nvb->dev);
2525         if (cxlr->cxlr_pmem)
2526                 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
2527                                     cxlr->cxlr_pmem);
2528         device_unlock(&cxl_nvb->dev);
2529         cxlr->cxl_nvb = NULL;
2530         put_device(&cxl_nvb->dev);
2531 }
2532
2533 /**
2534  * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
2535  * @cxlr: parent CXL region for this pmem region bridge device
2536  *
2537  * Return: 0 on success negative error code on failure.
2538  */
2539 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
2540 {
2541         struct cxl_pmem_region *cxlr_pmem;
2542         struct cxl_nvdimm_bridge *cxl_nvb;
2543         struct device *dev;
2544         int rc;
2545
2546         cxlr_pmem = cxl_pmem_region_alloc(cxlr);
2547         if (IS_ERR(cxlr_pmem))
2548                 return PTR_ERR(cxlr_pmem);
2549         cxl_nvb = cxlr->cxl_nvb;
2550
2551         dev = &cxlr_pmem->dev;
2552         rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
2553         if (rc)
2554                 goto err;
2555
2556         rc = device_add(dev);
2557         if (rc)
2558                 goto err;
2559
2560         dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2561                 dev_name(dev));
2562
2563         device_lock(&cxl_nvb->dev);
2564         if (cxl_nvb->dev.driver)
2565                 rc = devm_add_action_or_reset(&cxl_nvb->dev,
2566                                               cxlr_pmem_unregister, cxlr_pmem);
2567         else
2568                 rc = -ENXIO;
2569         device_unlock(&cxl_nvb->dev);
2570
2571         if (rc)
2572                 goto err_bridge;
2573
2574         /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
2575         return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
2576
2577 err:
2578         put_device(dev);
2579 err_bridge:
2580         put_device(&cxl_nvb->dev);
2581         cxlr->cxl_nvb = NULL;
2582         return rc;
2583 }
2584
2585 static void cxlr_dax_unregister(void *_cxlr_dax)
2586 {
2587         struct cxl_dax_region *cxlr_dax = _cxlr_dax;
2588
2589         device_unregister(&cxlr_dax->dev);
2590 }
2591
2592 static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
2593 {
2594         struct cxl_dax_region *cxlr_dax;
2595         struct device *dev;
2596         int rc;
2597
2598         cxlr_dax = cxl_dax_region_alloc(cxlr);
2599         if (IS_ERR(cxlr_dax))
2600                 return PTR_ERR(cxlr_dax);
2601
2602         dev = &cxlr_dax->dev;
2603         rc = dev_set_name(dev, "dax_region%d", cxlr->id);
2604         if (rc)
2605                 goto err;
2606
2607         rc = device_add(dev);
2608         if (rc)
2609                 goto err;
2610
2611         dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2612                 dev_name(dev));
2613
2614         return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
2615                                         cxlr_dax);
2616 err:
2617         put_device(dev);
2618         return rc;
2619 }
2620
2621 static int match_decoder_by_range(struct device *dev, void *data)
2622 {
2623         struct range *r1, *r2 = data;
2624         struct cxl_root_decoder *cxlrd;
2625
2626         if (!is_root_decoder(dev))
2627                 return 0;
2628
2629         cxlrd = to_cxl_root_decoder(dev);
2630         r1 = &cxlrd->cxlsd.cxld.hpa_range;
2631         return range_contains(r1, r2);
2632 }
2633
2634 static int match_region_by_range(struct device *dev, void *data)
2635 {
2636         struct cxl_region_params *p;
2637         struct cxl_region *cxlr;
2638         struct range *r = data;
2639         int rc = 0;
2640
2641         if (!is_cxl_region(dev))
2642                 return 0;
2643
2644         cxlr = to_cxl_region(dev);
2645         p = &cxlr->params;
2646
2647         down_read(&cxl_region_rwsem);
2648         if (p->res && p->res->start == r->start && p->res->end == r->end)
2649                 rc = 1;
2650         up_read(&cxl_region_rwsem);
2651
2652         return rc;
2653 }
2654
2655 /* Establish an empty region covering the given HPA range */
2656 static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
2657                                            struct cxl_endpoint_decoder *cxled)
2658 {
2659         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2660         struct cxl_port *port = cxlrd_to_port(cxlrd);
2661         struct range *hpa = &cxled->cxld.hpa_range;
2662         struct cxl_region_params *p;
2663         struct cxl_region *cxlr;
2664         struct resource *res;
2665         int rc;
2666
2667         do {
2668                 cxlr = __create_region(cxlrd, cxled->mode,
2669                                        atomic_read(&cxlrd->region_id));
2670         } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
2671
2672         if (IS_ERR(cxlr)) {
2673                 dev_err(cxlmd->dev.parent,
2674                         "%s:%s: %s failed assign region: %ld\n",
2675                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2676                         __func__, PTR_ERR(cxlr));
2677                 return cxlr;
2678         }
2679
2680         down_write(&cxl_region_rwsem);
2681         p = &cxlr->params;
2682         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
2683                 dev_err(cxlmd->dev.parent,
2684                         "%s:%s: %s autodiscovery interrupted\n",
2685                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2686                         __func__);
2687                 rc = -EBUSY;
2688                 goto err;
2689         }
2690
2691         set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
2692
2693         res = kmalloc(sizeof(*res), GFP_KERNEL);
2694         if (!res) {
2695                 rc = -ENOMEM;
2696                 goto err;
2697         }
2698
2699         *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
2700                                     dev_name(&cxlr->dev));
2701         rc = insert_resource(cxlrd->res, res);
2702         if (rc) {
2703                 /*
2704                  * Platform-firmware may not have split resources like "System
2705                  * RAM" on CXL window boundaries see cxl_region_iomem_release()
2706                  */
2707                 dev_warn(cxlmd->dev.parent,
2708                          "%s:%s: %s %s cannot insert resource\n",
2709                          dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2710                          __func__, dev_name(&cxlr->dev));
2711         }
2712
2713         p->res = res;
2714         p->interleave_ways = cxled->cxld.interleave_ways;
2715         p->interleave_granularity = cxled->cxld.interleave_granularity;
2716         p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2717
2718         rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
2719         if (rc)
2720                 goto err;
2721
2722         dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
2723                 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
2724                 dev_name(&cxlr->dev), p->res, p->interleave_ways,
2725                 p->interleave_granularity);
2726
2727         /* ...to match put_device() in cxl_add_to_region() */
2728         get_device(&cxlr->dev);
2729         up_write(&cxl_region_rwsem);
2730
2731         return cxlr;
2732
2733 err:
2734         up_write(&cxl_region_rwsem);
2735         devm_release_action(port->uport, unregister_region, cxlr);
2736         return ERR_PTR(rc);
2737 }
2738
2739 int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
2740 {
2741         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2742         struct range *hpa = &cxled->cxld.hpa_range;
2743         struct cxl_decoder *cxld = &cxled->cxld;
2744         struct device *cxlrd_dev, *region_dev;
2745         struct cxl_root_decoder *cxlrd;
2746         struct cxl_region_params *p;
2747         struct cxl_region *cxlr;
2748         bool attach = false;
2749         int rc;
2750
2751         cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
2752                                       match_decoder_by_range);
2753         if (!cxlrd_dev) {
2754                 dev_err(cxlmd->dev.parent,
2755                         "%s:%s no CXL window for range %#llx:%#llx\n",
2756                         dev_name(&cxlmd->dev), dev_name(&cxld->dev),
2757                         cxld->hpa_range.start, cxld->hpa_range.end);
2758                 return -ENXIO;
2759         }
2760
2761         cxlrd = to_cxl_root_decoder(cxlrd_dev);
2762
2763         /*
2764          * Ensure that if multiple threads race to construct_region() for @hpa
2765          * one does the construction and the others add to that.
2766          */
2767         mutex_lock(&cxlrd->range_lock);
2768         region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
2769                                        match_region_by_range);
2770         if (!region_dev) {
2771                 cxlr = construct_region(cxlrd, cxled);
2772                 region_dev = &cxlr->dev;
2773         } else
2774                 cxlr = to_cxl_region(region_dev);
2775         mutex_unlock(&cxlrd->range_lock);
2776
2777         rc = PTR_ERR_OR_ZERO(cxlr);
2778         if (rc)
2779                 goto out;
2780
2781         attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
2782
2783         down_read(&cxl_region_rwsem);
2784         p = &cxlr->params;
2785         attach = p->state == CXL_CONFIG_COMMIT;
2786         up_read(&cxl_region_rwsem);
2787
2788         if (attach) {
2789                 /*
2790                  * If device_attach() fails the range may still be active via
2791                  * the platform-firmware memory map, otherwise the driver for
2792                  * regions is local to this file, so driver matching can't fail.
2793                  */
2794                 if (device_attach(&cxlr->dev) < 0)
2795                         dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
2796                                 p->res);
2797         }
2798
2799         put_device(region_dev);
2800 out:
2801         put_device(cxlrd_dev);
2802         return rc;
2803 }
2804 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL);
2805
2806 static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
2807 {
2808         if (!test_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags))
2809                 return 0;
2810
2811         if (!cpu_cache_has_invalidate_memregion()) {
2812                 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
2813                         dev_warn_once(
2814                                 &cxlr->dev,
2815                                 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
2816                         clear_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags);
2817                         return 0;
2818                 } else {
2819                         dev_err(&cxlr->dev,
2820                                 "Failed to synchronize CPU cache state\n");
2821                         return -ENXIO;
2822                 }
2823         }
2824
2825         cpu_cache_invalidate_memregion(IORES_DESC_CXL);
2826         clear_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags);
2827         return 0;
2828 }
2829
2830 static int is_system_ram(struct resource *res, void *arg)
2831 {
2832         struct cxl_region *cxlr = arg;
2833         struct cxl_region_params *p = &cxlr->params;
2834
2835         dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
2836         return 1;
2837 }
2838
2839 static int cxl_region_probe(struct device *dev)
2840 {
2841         struct cxl_region *cxlr = to_cxl_region(dev);
2842         struct cxl_region_params *p = &cxlr->params;
2843         int rc;
2844
2845         rc = down_read_interruptible(&cxl_region_rwsem);
2846         if (rc) {
2847                 dev_dbg(&cxlr->dev, "probe interrupted\n");
2848                 return rc;
2849         }
2850
2851         if (p->state < CXL_CONFIG_COMMIT) {
2852                 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
2853                 rc = -ENXIO;
2854                 goto out;
2855         }
2856
2857         rc = cxl_region_invalidate_memregion(cxlr);
2858
2859         /*
2860          * From this point on any path that changes the region's state away from
2861          * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
2862          */
2863 out:
2864         up_read(&cxl_region_rwsem);
2865
2866         if (rc)
2867                 return rc;
2868
2869         switch (cxlr->mode) {
2870         case CXL_DECODER_PMEM:
2871                 return devm_cxl_add_pmem_region(cxlr);
2872         case CXL_DECODER_RAM:
2873                 /*
2874                  * The region can not be manged by CXL if any portion of
2875                  * it is already online as 'System RAM'
2876                  */
2877                 if (walk_iomem_res_desc(IORES_DESC_NONE,
2878                                         IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
2879                                         p->res->start, p->res->end, cxlr,
2880                                         is_system_ram) > 0)
2881                         return 0;
2882                 return devm_cxl_add_dax_region(cxlr);
2883         default:
2884                 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
2885                         cxlr->mode);
2886                 return -ENXIO;
2887         }
2888 }
2889
2890 static struct cxl_driver cxl_region_driver = {
2891         .name = "cxl_region",
2892         .probe = cxl_region_probe,
2893         .id = CXL_DEVICE_REGION,
2894 };
2895
2896 int cxl_region_init(void)
2897 {
2898         return cxl_driver_register(&cxl_region_driver);
2899 }
2900
2901 void cxl_region_exit(void)
2902 {
2903         cxl_driver_unregister(&cxl_region_driver);
2904 }
2905
2906 MODULE_IMPORT_NS(CXL);
2907 MODULE_IMPORT_NS(DEVMEM);
2908 MODULE_ALIAS_CXL(CXL_DEVICE_REGION);