drivers/cxl/core/memdev.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2020 Intel Corporation. */
   3
   4 #include <linux/io-64-nonatomic-lo-hi.h>
   5 #include <linux/firmware.h>
   6 #include <linux/device.h>
   7 #include <linux/slab.h>
   8 #include <linux/idr.h>
   9 #include <linux/pci.h>
  10 #include <cxlmem.h>
  11 #include "trace.h"
  12 #include "core.h"
  13
  14 static DECLARE_RWSEM(cxl_memdev_rwsem);
  15
  16 /*
  17  * An entire PCI topology full of devices should be enough for any
  18  * config
  19  */
  20 #define CXL_MEM_MAX_DEVS 65536
  21
  22 static int cxl_mem_major;
  23 static DEFINE_IDA(cxl_memdev_ida);
  24
  25 static void cxl_memdev_release(struct device *dev)
  26 {
  27         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  28
  29         ida_free(&cxl_memdev_ida, cxlmd->id);
  30         kfree(cxlmd);
  31 }
  32
  33 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid,
  34                                 kgid_t *gid)
  35 {
  36         return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev));
  37 }
  38
  39 static ssize_t firmware_version_show(struct device *dev,
  40                                      struct device_attribute *attr, char *buf)
  41 {
  42         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  43         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  44         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
  45
  46         if (!mds)
  47                 return sysfs_emit(buf, "\n");
  48         return sysfs_emit(buf, "%.16s\n", mds->firmware_version);
  49 }
  50 static DEVICE_ATTR_RO(firmware_version);
  51
  52 static ssize_t payload_max_show(struct device *dev,
  53                                 struct device_attribute *attr, char *buf)
  54 {
  55         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  56         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  57         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
  58
  59         if (!mds)
  60                 return sysfs_emit(buf, "\n");
  61         return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size);
  62 }
  63 static DEVICE_ATTR_RO(payload_max);
  64
  65 static ssize_t label_storage_size_show(struct device *dev,
  66                                        struct device_attribute *attr, char *buf)
  67 {
  68         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  69         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  70         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
  71
  72         if (!mds)
  73                 return sysfs_emit(buf, "\n");
  74         return sysfs_emit(buf, "%zu\n", mds->lsa_size);
  75 }
  76 static DEVICE_ATTR_RO(label_storage_size);
  77
  78 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr,
  79                              char *buf)
  80 {
  81         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  82         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  83         unsigned long long len = resource_size(&cxlds->ram_res);
  84
  85         return sysfs_emit(buf, "%#llx\n", len);
  86 }
  87
  88 static struct device_attribute dev_attr_ram_size =
  89         __ATTR(size, 0444, ram_size_show, NULL);
  90
  91 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr,
  92                               char *buf)
  93 {
  94         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  95         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  96         unsigned long long len = resource_size(&cxlds->pmem_res);
  97
  98         return sysfs_emit(buf, "%#llx\n", len);
  99 }
 100
 101 static struct device_attribute dev_attr_pmem_size =
 102         __ATTR(size, 0444, pmem_size_show, NULL);
 103
 104 static ssize_t serial_show(struct device *dev, struct device_attribute *attr,
 105                            char *buf)
 106 {
 107         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 108         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 109
 110         return sysfs_emit(buf, "%#llx\n", cxlds->serial);
 111 }
 112 static DEVICE_ATTR_RO(serial);
 113
 114 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr,
 115                               char *buf)
 116 {
 117         return sysfs_emit(buf, "%d\n", dev_to_node(dev));
 118 }
 119 static DEVICE_ATTR_RO(numa_node);
 120
 121 static ssize_t security_state_show(struct device *dev,
 122                                    struct device_attribute *attr,
 123                                    char *buf)
 124 {
 125         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 126         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 127         struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
 128         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
 129         unsigned long state = mds->security.state;
 130         int rc = 0;
 131
 132         /* sync with latest submission state */
 133         mutex_lock(&cxl_mbox->mbox_mutex);
 134         if (mds->security.sanitize_active)
 135                 rc = sysfs_emit(buf, "sanitize\n");
 136         mutex_unlock(&cxl_mbox->mbox_mutex);
 137         if (rc)
 138                 return rc;
 139
 140         if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET))
 141                 return sysfs_emit(buf, "disabled\n");
 142         if (state & CXL_PMEM_SEC_STATE_FROZEN ||
 143             state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT ||
 144             state & CXL_PMEM_SEC_STATE_USER_PLIMIT)
 145                 return sysfs_emit(buf, "frozen\n");
 146         if (state & CXL_PMEM_SEC_STATE_LOCKED)
 147                 return sysfs_emit(buf, "locked\n");
 148         else
 149                 return sysfs_emit(buf, "unlocked\n");
 150 }
 151 static struct device_attribute dev_attr_security_state =
 152         __ATTR(state, 0444, security_state_show, NULL);
 153
 154 static ssize_t security_sanitize_store(struct device *dev,
 155                                        struct device_attribute *attr,
 156                                        const char *buf, size_t len)
 157 {
 158         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 159         bool sanitize;
 160         ssize_t rc;
 161
 162         if (kstrtobool(buf, &sanitize) || !sanitize)
 163                 return -EINVAL;
 164
 165         rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE);
 166         if (rc)
 167                 return rc;
 168
 169         return len;
 170 }
 171 static struct device_attribute dev_attr_security_sanitize =
 172         __ATTR(sanitize, 0200, NULL, security_sanitize_store);
 173
 174 static ssize_t security_erase_store(struct device *dev,
 175                                     struct device_attribute *attr,
 176                                     const char *buf, size_t len)
 177 {
 178         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 179         ssize_t rc;
 180         bool erase;
 181
 182         if (kstrtobool(buf, &erase) || !erase)
 183                 return -EINVAL;
 184
 185         rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE);
 186         if (rc)
 187                 return rc;
 188
 189         return len;
 190 }
 191 static struct device_attribute dev_attr_security_erase =
 192         __ATTR(erase, 0200, NULL, security_erase_store);
 193
 194 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd)
 195 {
 196         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 197         u64 offset, length;
 198         int rc = 0;
 199
 200         /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */
 201         if (resource_size(&cxlds->pmem_res)) {
 202                 offset = cxlds->pmem_res.start;
 203                 length = resource_size(&cxlds->pmem_res);
 204                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
 205                 if (rc)
 206                         return rc;
 207         }
 208         if (resource_size(&cxlds->ram_res)) {
 209                 offset = cxlds->ram_res.start;
 210                 length = resource_size(&cxlds->ram_res);
 211                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
 212                 /*
 213                  * Invalid Physical Address is not an error for
 214                  * volatile addresses. Device support is optional.
 215                  */
 216                 if (rc == -EFAULT)
 217                         rc = 0;
 218         }
 219         return rc;
 220 }
 221
 222 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)
 223 {
 224         struct cxl_port *port;
 225         int rc;
 226
 227         port = cxlmd->endpoint;
 228         if (!port || !is_cxl_endpoint(port))
 229                 return -EINVAL;
 230
 231         rc = down_read_interruptible(&cxl_region_rwsem);
 232         if (rc)
 233                 return rc;
 234
 235         rc = down_read_interruptible(&cxl_dpa_rwsem);
 236         if (rc) {
 237                 up_read(&cxl_region_rwsem);
 238                 return rc;
 239         }
 240
 241         if (cxl_num_decoders_committed(port) == 0) {
 242                 /* No regions mapped to this memdev */
 243                 rc = cxl_get_poison_by_memdev(cxlmd);
 244         } else {
 245                 /* Regions mapped, collect poison by endpoint */
 246                 rc =  cxl_get_poison_by_endpoint(port);
 247         }
 248         up_read(&cxl_dpa_rwsem);
 249         up_read(&cxl_region_rwsem);
 250
 251         return rc;
 252 }
 253 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, CXL);
 254
 255 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa)
 256 {
 257         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 258
 259         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 260                 return 0;
 261
 262         if (!resource_size(&cxlds->dpa_res)) {
 263                 dev_dbg(cxlds->dev, "device has no dpa resource\n");
 264                 return -EINVAL;
 265         }
 266         if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) {
 267                 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n",
 268                         dpa, &cxlds->dpa_res);
 269                 return -EINVAL;
 270         }
 271         if (!IS_ALIGNED(dpa, 64)) {
 272                 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa);
 273                 return -EINVAL;
 274         }
 275
 276         return 0;
 277 }
 278
 279 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
 280 {
 281         struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
 282         struct cxl_mbox_inject_poison inject;
 283         struct cxl_poison_record record;
 284         struct cxl_mbox_cmd mbox_cmd;
 285         struct cxl_region *cxlr;
 286         int rc;
 287
 288         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 289                 return 0;
 290
 291         rc = down_read_interruptible(&cxl_region_rwsem);
 292         if (rc)
 293                 return rc;
 294
 295         rc = down_read_interruptible(&cxl_dpa_rwsem);
 296         if (rc) {
 297                 up_read(&cxl_region_rwsem);
 298                 return rc;
 299         }
 300
 301         rc = cxl_validate_poison_dpa(cxlmd, dpa);
 302         if (rc)
 303                 goto out;
 304
 305         inject.address = cpu_to_le64(dpa);
 306         mbox_cmd = (struct cxl_mbox_cmd) {
 307                 .opcode = CXL_MBOX_OP_INJECT_POISON,
 308                 .size_in = sizeof(inject),
 309                 .payload_in = &inject,
 310         };
 311         rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
 312         if (rc)
 313                 goto out;
 314
 315         cxlr = cxl_dpa_to_region(cxlmd, dpa);
 316         if (cxlr)
 317                 dev_warn_once(cxl_mbox->host,
 318                               "poison inject dpa:%#llx region: %s\n", dpa,
 319                               dev_name(&cxlr->dev));
 320
 321         record = (struct cxl_poison_record) {
 322                 .address = cpu_to_le64(dpa),
 323                 .length = cpu_to_le32(1),
 324         };
 325         trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT);
 326 out:
 327         up_read(&cxl_dpa_rwsem);
 328         up_read(&cxl_region_rwsem);
 329
 330         return rc;
 331 }
 332 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL);
 333
 334 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
 335 {
 336         struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
 337         struct cxl_mbox_clear_poison clear;
 338         struct cxl_poison_record record;
 339         struct cxl_mbox_cmd mbox_cmd;
 340         struct cxl_region *cxlr;
 341         int rc;
 342
 343         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 344                 return 0;
 345
 346         rc = down_read_interruptible(&cxl_region_rwsem);
 347         if (rc)
 348                 return rc;
 349
 350         rc = down_read_interruptible(&cxl_dpa_rwsem);
 351         if (rc) {
 352                 up_read(&cxl_region_rwsem);
 353                 return rc;
 354         }
 355
 356         rc = cxl_validate_poison_dpa(cxlmd, dpa);
 357         if (rc)
 358                 goto out;
 359
 360         /*
 361          * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command
 362          * is defined to accept 64 bytes of write-data, along with the
 363          * address to clear. This driver uses zeroes as write-data.
 364          */
 365         clear = (struct cxl_mbox_clear_poison) {
 366                 .address = cpu_to_le64(dpa)
 367         };
 368
 369         mbox_cmd = (struct cxl_mbox_cmd) {
 370                 .opcode = CXL_MBOX_OP_CLEAR_POISON,
 371                 .size_in = sizeof(clear),
 372                 .payload_in = &clear,
 373         };
 374
 375         rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
 376         if (rc)
 377                 goto out;
 378
 379         cxlr = cxl_dpa_to_region(cxlmd, dpa);
 380         if (cxlr)
 381                 dev_warn_once(cxl_mbox->host,
 382                               "poison clear dpa:%#llx region: %s\n", dpa,
 383                               dev_name(&cxlr->dev));
 384
 385         record = (struct cxl_poison_record) {
 386                 .address = cpu_to_le64(dpa),
 387                 .length = cpu_to_le32(1),
 388         };
 389         trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR);
 390 out:
 391         up_read(&cxl_dpa_rwsem);
 392         up_read(&cxl_region_rwsem);
 393
 394         return rc;
 395 }
 396 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, CXL);
 397
 398 static struct attribute *cxl_memdev_attributes[] = {
 399         &dev_attr_serial.attr,
 400         &dev_attr_firmware_version.attr,
 401         &dev_attr_payload_max.attr,
 402         &dev_attr_label_storage_size.attr,
 403         &dev_attr_numa_node.attr,
 404         NULL,
 405 };
 406
 407 static ssize_t pmem_qos_class_show(struct device *dev,
 408                                    struct device_attribute *attr, char *buf)
 409 {
 410         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 411         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 412         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
 413
 414         return sysfs_emit(buf, "%d\n", mds->pmem_perf.qos_class);
 415 }
 416
 417 static struct device_attribute dev_attr_pmem_qos_class =
 418         __ATTR(qos_class, 0444, pmem_qos_class_show, NULL);
 419
 420 static struct attribute *cxl_memdev_pmem_attributes[] = {
 421         &dev_attr_pmem_size.attr,
 422         &dev_attr_pmem_qos_class.attr,
 423         NULL,
 424 };
 425
 426 static ssize_t ram_qos_class_show(struct device *dev,
 427                                   struct device_attribute *attr, char *buf)
 428 {
 429         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 430         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 431         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
 432
 433         return sysfs_emit(buf, "%d\n", mds->ram_perf.qos_class);
 434 }
 435
 436 static struct device_attribute dev_attr_ram_qos_class =
 437         __ATTR(qos_class, 0444, ram_qos_class_show, NULL);
 438
 439 static struct attribute *cxl_memdev_ram_attributes[] = {
 440         &dev_attr_ram_size.attr,
 441         &dev_attr_ram_qos_class.attr,
 442         NULL,
 443 };
 444
 445 static struct attribute *cxl_memdev_security_attributes[] = {
 446         &dev_attr_security_state.attr,
 447         &dev_attr_security_sanitize.attr,
 448         &dev_attr_security_erase.attr,
 449         NULL,
 450 };
 451
 452 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a,
 453                                   int n)
 454 {
 455         if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr)
 456                 return 0;
 457         return a->mode;
 458 }
 459
 460 static struct attribute_group cxl_memdev_attribute_group = {
 461         .attrs = cxl_memdev_attributes,
 462         .is_visible = cxl_memdev_visible,
 463 };
 464
 465 static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n)
 466 {
 467         struct device *dev = kobj_to_dev(kobj);
 468         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 469         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 470
 471         if (a == &dev_attr_ram_qos_class.attr)
 472                 if (mds->ram_perf.qos_class == CXL_QOS_CLASS_INVALID)
 473                         return 0;
 474
 475         return a->mode;
 476 }
 477
 478 static struct attribute_group cxl_memdev_ram_attribute_group = {
 479         .name = "ram",
 480         .attrs = cxl_memdev_ram_attributes,
 481         .is_visible = cxl_ram_visible,
 482 };
 483
 484 static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n)
 485 {
 486         struct device *dev = kobj_to_dev(kobj);
 487         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 488         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 489
 490         if (a == &dev_attr_pmem_qos_class.attr)
 491                 if (mds->pmem_perf.qos_class == CXL_QOS_CLASS_INVALID)
 492                         return 0;
 493
 494         return a->mode;
 495 }
 496
 497 static struct attribute_group cxl_memdev_pmem_attribute_group = {
 498         .name = "pmem",
 499         .attrs = cxl_memdev_pmem_attributes,
 500         .is_visible = cxl_pmem_visible,
 501 };
 502
 503 static umode_t cxl_memdev_security_visible(struct kobject *kobj,
 504                                            struct attribute *a, int n)
 505 {
 506         struct device *dev = kobj_to_dev(kobj);
 507         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 508         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 509
 510         if (a == &dev_attr_security_sanitize.attr &&
 511             !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
 512                 return 0;
 513
 514         if (a == &dev_attr_security_erase.attr &&
 515             !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds))
 516                 return 0;
 517
 518         return a->mode;
 519 }
 520
 521 static struct attribute_group cxl_memdev_security_attribute_group = {
 522         .name = "security",
 523         .attrs = cxl_memdev_security_attributes,
 524         .is_visible = cxl_memdev_security_visible,
 525 };
 526
 527 static const struct attribute_group *cxl_memdev_attribute_groups[] = {
 528         &cxl_memdev_attribute_group,
 529         &cxl_memdev_ram_attribute_group,
 530         &cxl_memdev_pmem_attribute_group,
 531         &cxl_memdev_security_attribute_group,
 532         NULL,
 533 };
 534
 535 void cxl_memdev_update_perf(struct cxl_memdev *cxlmd)
 536 {
 537         sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group);
 538         sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group);
 539 }
 540 EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, CXL);
 541
 542 static const struct device_type cxl_memdev_type = {
 543         .name = "cxl_memdev",
 544         .release = cxl_memdev_release,
 545         .devnode = cxl_memdev_devnode,
 546         .groups = cxl_memdev_attribute_groups,
 547 };
 548
 549 bool is_cxl_memdev(const struct device *dev)
 550 {
 551         return dev->type == &cxl_memdev_type;
 552 }
 553 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL);
 554
 555 /**
 556  * set_exclusive_cxl_commands() - atomically disable user cxl commands
 557  * @mds: The device state to operate on
 558  * @cmds: bitmap of commands to mark exclusive
 559  *
 560  * Grab the cxl_memdev_rwsem in write mode to flush in-flight
 561  * invocations of the ioctl path and then disable future execution of
 562  * commands with the command ids set in @cmds.
 563  */
 564 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
 565                                 unsigned long *cmds)
 566 {
 567         down_write(&cxl_memdev_rwsem);
 568         bitmap_or(mds->exclusive_cmds, mds->exclusive_cmds, cmds,
 569                   CXL_MEM_COMMAND_ID_MAX);
 570         up_write(&cxl_memdev_rwsem);
 571 }
 572 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL);
 573
 574 /**
 575  * clear_exclusive_cxl_commands() - atomically enable user cxl commands
 576  * @mds: The device state to modify
 577  * @cmds: bitmap of commands to mark available for userspace
 578  */
 579 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,
 580                                   unsigned long *cmds)
 581 {
 582         down_write(&cxl_memdev_rwsem);
 583         bitmap_andnot(mds->exclusive_cmds, mds->exclusive_cmds, cmds,
 584                       CXL_MEM_COMMAND_ID_MAX);
 585         up_write(&cxl_memdev_rwsem);
 586 }
 587 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL);
 588
 589 static void cxl_memdev_shutdown(struct device *dev)
 590 {
 591         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 592
 593         down_write(&cxl_memdev_rwsem);
 594         cxlmd->cxlds = NULL;
 595         up_write(&cxl_memdev_rwsem);
 596 }
 597
 598 static void cxl_memdev_unregister(void *_cxlmd)
 599 {
 600         struct cxl_memdev *cxlmd = _cxlmd;
 601         struct device *dev = &cxlmd->dev;
 602
 603         cdev_device_del(&cxlmd->cdev, dev);
 604         cxl_memdev_shutdown(dev);
 605         put_device(dev);
 606 }
 607
 608 static void detach_memdev(struct work_struct *work)
 609 {
 610         struct cxl_memdev *cxlmd;
 611
 612         cxlmd = container_of(work, typeof(*cxlmd), detach_work);
 613         device_release_driver(&cxlmd->dev);
 614         put_device(&cxlmd->dev);
 615 }
 616
 617 static struct lock_class_key cxl_memdev_key;
 618
 619 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
 620                                            const struct file_operations *fops)
 621 {
 622         struct cxl_memdev *cxlmd;
 623         struct device *dev;
 624         struct cdev *cdev;
 625         int rc;
 626
 627         cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL);
 628         if (!cxlmd)
 629                 return ERR_PTR(-ENOMEM);
 630
 631         rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL);
 632         if (rc < 0)
 633                 goto err;
 634         cxlmd->id = rc;
 635         cxlmd->depth = -1;
 636
 637         dev = &cxlmd->dev;
 638         device_initialize(dev);
 639         lockdep_set_class(&dev->mutex, &cxl_memdev_key);
 640         dev->parent = cxlds->dev;
 641         dev->bus = &cxl_bus_type;
 642         dev->devt = MKDEV(cxl_mem_major, cxlmd->id);
 643         dev->type = &cxl_memdev_type;
 644         device_set_pm_not_required(dev);
 645         INIT_WORK(&cxlmd->detach_work, detach_memdev);
 646
 647         cdev = &cxlmd->cdev;
 648         cdev_init(cdev, fops);
 649         return cxlmd;
 650
 651 err:
 652         kfree(cxlmd);
 653         return ERR_PTR(rc);
 654 }
 655
 656 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
 657                                unsigned long arg)
 658 {
 659         switch (cmd) {
 660         case CXL_MEM_QUERY_COMMANDS:
 661                 return cxl_query_cmd(cxlmd, (void __user *)arg);
 662         case CXL_MEM_SEND_COMMAND:
 663                 return cxl_send_cmd(cxlmd, (void __user *)arg);
 664         default:
 665                 return -ENOTTY;
 666         }
 667 }
 668
 669 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,
 670                              unsigned long arg)
 671 {
 672         struct cxl_memdev *cxlmd = file->private_data;
 673         struct cxl_dev_state *cxlds;
 674         int rc = -ENXIO;
 675
 676         down_read(&cxl_memdev_rwsem);
 677         cxlds = cxlmd->cxlds;
 678         if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM)
 679                 rc = __cxl_memdev_ioctl(cxlmd, cmd, arg);
 680         up_read(&cxl_memdev_rwsem);
 681
 682         return rc;
 683 }
 684
 685 static int cxl_memdev_open(struct inode *inode, struct file *file)
 686 {
 687         struct cxl_memdev *cxlmd =
 688                 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
 689
 690         get_device(&cxlmd->dev);
 691         file->private_data = cxlmd;
 692
 693         return 0;
 694 }
 695
 696 static int cxl_memdev_release_file(struct inode *inode, struct file *file)
 697 {
 698         struct cxl_memdev *cxlmd =
 699                 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
 700
 701         put_device(&cxlmd->dev);
 702
 703         return 0;
 704 }
 705
 706 /**
 707  * cxl_mem_get_fw_info - Get Firmware info
 708  * @mds: The device data for the operation
 709  *
 710  * Retrieve firmware info for the device specified.
 711  *
 712  * Return: 0 if no error: or the result of the mailbox command.
 713  *
 714  * See CXL-3.0 8.2.9.3.1 Get FW Info
 715  */
 716 static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)
 717 {
 718         struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
 719         struct cxl_mbox_get_fw_info info;
 720         struct cxl_mbox_cmd mbox_cmd;
 721         int rc;
 722
 723         mbox_cmd = (struct cxl_mbox_cmd) {
 724                 .opcode = CXL_MBOX_OP_GET_FW_INFO,
 725                 .size_out = sizeof(info),
 726                 .payload_out = &info,
 727         };
 728
 729         rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
 730         if (rc < 0)
 731                 return rc;
 732
 733         mds->fw.num_slots = info.num_slots;
 734         mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK,
 735                                        info.slot_info);
 736
 737         return 0;
 738 }
 739
 740 /**
 741  * cxl_mem_activate_fw - Activate Firmware
 742  * @mds: The device data for the operation
 743  * @slot: slot number to activate
 744  *
 745  * Activate firmware in a given slot for the device specified.
 746  *
 747  * Return: 0 if no error: or the result of the mailbox command.
 748  *
 749  * See CXL-3.0 8.2.9.3.3 Activate FW
 750  */
 751 static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)
 752 {
 753         struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
 754         struct cxl_mbox_activate_fw activate;
 755         struct cxl_mbox_cmd mbox_cmd;
 756
 757         if (slot == 0 || slot > mds->fw.num_slots)
 758                 return -EINVAL;
 759
 760         mbox_cmd = (struct cxl_mbox_cmd) {
 761                 .opcode = CXL_MBOX_OP_ACTIVATE_FW,
 762                 .size_in = sizeof(activate),
 763                 .payload_in = &activate,
 764         };
 765
 766         /* Only offline activation supported for now */
 767         activate.action = CXL_FW_ACTIVATE_OFFLINE;
 768         activate.slot = slot;
 769
 770         return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
 771 }
 772
 773 /**
 774  * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer
 775  * @mds: The device data for the operation
 776  *
 777  * Abort an in-progress firmware transfer for the device specified.
 778  *
 779  * Return: 0 if no error: or the result of the mailbox command.
 780  *
 781  * See CXL-3.0 8.2.9.3.2 Transfer FW
 782  */
 783 static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds)
 784 {
 785         struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
 786         struct cxl_mbox_transfer_fw *transfer;
 787         struct cxl_mbox_cmd mbox_cmd;
 788         int rc;
 789
 790         transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL);
 791         if (!transfer)
 792                 return -ENOMEM;
 793
 794         /* Set a 1s poll interval and a total wait time of 30s */
 795         mbox_cmd = (struct cxl_mbox_cmd) {
 796                 .opcode = CXL_MBOX_OP_TRANSFER_FW,
 797                 .size_in = sizeof(*transfer),
 798                 .payload_in = transfer,
 799                 .poll_interval_ms = 1000,
 800                 .poll_count = 30,
 801         };
 802
 803         transfer->action = CXL_FW_TRANSFER_ACTION_ABORT;
 804
 805         rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
 806         kfree(transfer);
 807         return rc;
 808 }
 809
 810 static void cxl_fw_cleanup(struct fw_upload *fwl)
 811 {
 812         struct cxl_memdev_state *mds = fwl->dd_handle;
 813
 814         mds->fw.next_slot = 0;
 815 }
 816
 817 static int cxl_fw_do_cancel(struct fw_upload *fwl)
 818 {
 819         struct cxl_memdev_state *mds = fwl->dd_handle;
 820         struct cxl_dev_state *cxlds = &mds->cxlds;
 821         struct cxl_memdev *cxlmd = cxlds->cxlmd;
 822         int rc;
 823
 824         rc = cxl_mem_abort_fw_xfer(mds);
 825         if (rc < 0)
 826                 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc);
 827
 828         return FW_UPLOAD_ERR_CANCELED;
 829 }
 830
 831 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,
 832                                          u32 size)
 833 {
 834         struct cxl_memdev_state *mds = fwl->dd_handle;
 835         struct cxl_mbox_transfer_fw *transfer;
 836         struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
 837
 838         if (!size)
 839                 return FW_UPLOAD_ERR_INVALID_SIZE;
 840
 841         mds->fw.oneshot = struct_size(transfer, data, size) <
 842                             cxl_mbox->payload_size;
 843
 844         if (cxl_mem_get_fw_info(mds))
 845                 return FW_UPLOAD_ERR_HW_ERROR;
 846
 847         /*
 848          * So far no state has been changed, hence no other cleanup is
 849          * necessary. Simply return the cancelled status.
 850          */
 851         if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
 852                 return FW_UPLOAD_ERR_CANCELED;
 853
 854         return FW_UPLOAD_ERR_NONE;
 855 }
 856
 857 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
 858                                        u32 offset, u32 size, u32 *written)
 859 {
 860         struct cxl_memdev_state *mds = fwl->dd_handle;
 861         struct cxl_dev_state *cxlds = &mds->cxlds;
 862         struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
 863         struct cxl_memdev *cxlmd = cxlds->cxlmd;
 864         struct cxl_mbox_transfer_fw *transfer;
 865         struct cxl_mbox_cmd mbox_cmd;
 866         u32 cur_size, remaining;
 867         size_t size_in;
 868         int rc;
 869
 870         *written = 0;
 871
 872         /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */
 873         if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) {
 874                 dev_err(&cxlmd->dev,
 875                         "misaligned offset for FW transfer slice (%u)\n",
 876                         offset);
 877                 return FW_UPLOAD_ERR_RW_ERROR;
 878         }
 879
 880         /*
 881          * Pick transfer size based on mds->payload_size @size must bw 128-byte
 882          * aligned, ->payload_size is a power of 2 starting at 256 bytes, and
 883          * sizeof(*transfer) is 128.  These constraints imply that @cur_size
 884          * will always be 128b aligned.
 885          */
 886         cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer));
 887
 888         remaining = size - cur_size;
 889         size_in = struct_size(transfer, data, cur_size);
 890
 891         if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
 892                 return cxl_fw_do_cancel(fwl);
 893
 894         /*
 895          * Slot numbers are 1-indexed
 896          * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1')
 897          * Check for rollover using modulo, and 1-index it by adding 1
 898          */
 899         mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1;
 900
 901         /* Do the transfer via mailbox cmd */
 902         transfer = kzalloc(size_in, GFP_KERNEL);
 903         if (!transfer)
 904                 return FW_UPLOAD_ERR_RW_ERROR;
 905
 906         transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT);
 907         memcpy(transfer->data, data + offset, cur_size);
 908         if (mds->fw.oneshot) {
 909                 transfer->action = CXL_FW_TRANSFER_ACTION_FULL;
 910                 transfer->slot = mds->fw.next_slot;
 911         } else {
 912                 if (offset == 0) {
 913                         transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE;
 914                 } else if (remaining == 0) {
 915                         transfer->action = CXL_FW_TRANSFER_ACTION_END;
 916                         transfer->slot = mds->fw.next_slot;
 917                 } else {
 918                         transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE;
 919                 }
 920         }
 921
 922         mbox_cmd = (struct cxl_mbox_cmd) {
 923                 .opcode = CXL_MBOX_OP_TRANSFER_FW,
 924                 .size_in = size_in,
 925                 .payload_in = transfer,
 926                 .poll_interval_ms = 1000,
 927                 .poll_count = 30,
 928         };
 929
 930         rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
 931         if (rc < 0) {
 932                 rc = FW_UPLOAD_ERR_RW_ERROR;
 933                 goto out_free;
 934         }
 935
 936         *written = cur_size;
 937
 938         /* Activate FW if oneshot or if the last slice was written */
 939         if (mds->fw.oneshot || remaining == 0) {
 940                 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n",
 941                         mds->fw.next_slot);
 942                 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot);
 943                 if (rc < 0) {
 944                         dev_err(&cxlmd->dev, "Error activating firmware: %d\n",
 945                                 rc);
 946                         rc = FW_UPLOAD_ERR_HW_ERROR;
 947                         goto out_free;
 948                 }
 949         }
 950
 951         rc = FW_UPLOAD_ERR_NONE;
 952
 953 out_free:
 954         kfree(transfer);
 955         return rc;
 956 }
 957
 958 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl)
 959 {
 960         struct cxl_memdev_state *mds = fwl->dd_handle;
 961
 962         /*
 963          * cxl_internal_send_cmd() handles background operations synchronously.
 964          * No need to wait for completions here - any errors would've been
 965          * reported and handled during the ->write() call(s).
 966          * Just check if a cancel request was received, and return success.
 967          */
 968         if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
 969                 return cxl_fw_do_cancel(fwl);
 970
 971         return FW_UPLOAD_ERR_NONE;
 972 }
 973
 974 static void cxl_fw_cancel(struct fw_upload *fwl)
 975 {
 976         struct cxl_memdev_state *mds = fwl->dd_handle;
 977
 978         set_bit(CXL_FW_CANCEL, mds->fw.state);
 979 }
 980
 981 static const struct fw_upload_ops cxl_memdev_fw_ops = {
 982         .prepare = cxl_fw_prepare,
 983         .write = cxl_fw_write,
 984         .poll_complete = cxl_fw_poll_complete,
 985         .cancel = cxl_fw_cancel,
 986         .cleanup = cxl_fw_cleanup,
 987 };
 988
 989 static void cxl_remove_fw_upload(void *fwl)
 990 {
 991         firmware_upload_unregister(fwl);
 992 }
 993
 994 int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds)
 995 {
 996         struct cxl_dev_state *cxlds = &mds->cxlds;
 997         struct device *dev = &cxlds->cxlmd->dev;
 998         struct fw_upload *fwl;
 999
1000         if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, mds->enabled_cmds))
1001                 return 0;
1002
1003         fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev),
1004                                        &cxl_memdev_fw_ops, mds);
1005         if (IS_ERR(fwl))
1006                 return PTR_ERR(fwl);
1007         return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl);
1008 }
1009 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, CXL);
1010
1011 static const struct file_operations cxl_memdev_fops = {
1012         .owner = THIS_MODULE,
1013         .unlocked_ioctl = cxl_memdev_ioctl,
1014         .open = cxl_memdev_open,
1015         .release = cxl_memdev_release_file,
1016         .compat_ioctl = compat_ptr_ioctl,
1017         .llseek = noop_llseek,
1018 };
1019
1020 struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
1021                                        struct cxl_dev_state *cxlds)
1022 {
1023         struct cxl_memdev *cxlmd;
1024         struct device *dev;
1025         struct cdev *cdev;
1026         int rc;
1027
1028         cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops);
1029         if (IS_ERR(cxlmd))
1030                 return cxlmd;
1031
1032         dev = &cxlmd->dev;
1033         rc = dev_set_name(dev, "mem%d", cxlmd->id);
1034         if (rc)
1035                 goto err;
1036
1037         /*
1038          * Activate ioctl operations, no cxl_memdev_rwsem manipulation
1039          * needed as this is ordered with cdev_add() publishing the device.
1040          */
1041         cxlmd->cxlds = cxlds;
1042         cxlds->cxlmd = cxlmd;
1043
1044         cdev = &cxlmd->cdev;
1045         rc = cdev_device_add(cdev, dev);
1046         if (rc)
1047                 goto err;
1048
1049         rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd);
1050         if (rc)
1051                 return ERR_PTR(rc);
1052         return cxlmd;
1053
1054 err:
1055         /*
1056          * The cdev was briefly live, shutdown any ioctl operations that
1057          * saw that state.
1058          */
1059         cxl_memdev_shutdown(dev);
1060         put_device(dev);
1061         return ERR_PTR(rc);
1062 }
1063 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
1064
1065 static void sanitize_teardown_notifier(void *data)
1066 {
1067         struct cxl_memdev_state *mds = data;
1068         struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1069         struct kernfs_node *state;
1070
1071         /*
1072          * Prevent new irq triggered invocations of the workqueue and
1073          * flush inflight invocations.
1074          */
1075         mutex_lock(&cxl_mbox->mbox_mutex);
1076         state = mds->security.sanitize_node;
1077         mds->security.sanitize_node = NULL;
1078         mutex_unlock(&cxl_mbox->mbox_mutex);
1079
1080         cancel_delayed_work_sync(&mds->security.poll_dwork);
1081         sysfs_put(state);
1082 }
1083
1084 int devm_cxl_sanitize_setup_notifier(struct device *host,
1085                                      struct cxl_memdev *cxlmd)
1086 {
1087         struct cxl_dev_state *cxlds = cxlmd->cxlds;
1088         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
1089         struct kernfs_node *sec;
1090
1091         if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
1092                 return 0;
1093
1094         /*
1095          * Note, the expectation is that @cxlmd would have failed to be
1096          * created if these sysfs_get_dirent calls fail.
1097          */
1098         sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security");
1099         if (!sec)
1100                 return -ENOENT;
1101         mds->security.sanitize_node = sysfs_get_dirent(sec, "state");
1102         sysfs_put(sec);
1103         if (!mds->security.sanitize_node)
1104                 return -ENOENT;
1105
1106         return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds);
1107 }
1108 EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, CXL);
1109
1110 __init int cxl_memdev_init(void)
1111 {
1112         dev_t devt;
1113         int rc;
1114
1115         rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl");
1116         if (rc)
1117                 return rc;
1118
1119         cxl_mem_major = MAJOR(devt);
1120
1121         return 0;
1122 }
1123
1124 void cxl_memdev_exit(void)
1125 {
1126         unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS);
1127 }