drivers/cxl/core/memdev.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2020 Intel Corporation. */
   3
   4 #include <linux/io-64-nonatomic-lo-hi.h>
   5 #include <linux/firmware.h>
   6 #include <linux/device.h>
   7 #include <linux/slab.h>
   8 #include <linux/idr.h>
   9 #include <linux/pci.h>
  10 #include <cxlmem.h>
  11 #include "trace.h"
  12 #include "core.h"
  13
  14 static DECLARE_RWSEM(cxl_memdev_rwsem);
  15
  16 /*
  17  * An entire PCI topology full of devices should be enough for any
  18  * config
  19  */
  20 #define CXL_MEM_MAX_DEVS 65536
  21
  22 static int cxl_mem_major;
  23 static DEFINE_IDA(cxl_memdev_ida);
  24
  25 static void cxl_memdev_release(struct device *dev)
  26 {
  27         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  28
  29         ida_free(&cxl_memdev_ida, cxlmd->id);
  30         kfree(cxlmd);
  31 }
  32
  33 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid,
  34                                 kgid_t *gid)
  35 {
  36         return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev));
  37 }
  38
  39 static ssize_t firmware_version_show(struct device *dev,
  40                                      struct device_attribute *attr, char *buf)
  41 {
  42         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  43         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  44         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
  45
  46         if (!mds)
  47                 return sysfs_emit(buf, "\n");
  48         return sysfs_emit(buf, "%.16s\n", mds->firmware_version);
  49 }
  50 static DEVICE_ATTR_RO(firmware_version);
  51
  52 static ssize_t payload_max_show(struct device *dev,
  53                                 struct device_attribute *attr, char *buf)
  54 {
  55         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  56         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  57         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
  58
  59         if (!mds)
  60                 return sysfs_emit(buf, "\n");
  61         return sysfs_emit(buf, "%zu\n", mds->payload_size);
  62 }
  63 static DEVICE_ATTR_RO(payload_max);
  64
  65 static ssize_t label_storage_size_show(struct device *dev,
  66                                        struct device_attribute *attr, char *buf)
  67 {
  68         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  69         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  70         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
  71
  72         if (!mds)
  73                 return sysfs_emit(buf, "\n");
  74         return sysfs_emit(buf, "%zu\n", mds->lsa_size);
  75 }
  76 static DEVICE_ATTR_RO(label_storage_size);
  77
  78 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr,
  79                              char *buf)
  80 {
  81         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  82         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  83         unsigned long long len = resource_size(&cxlds->ram_res);
  84
  85         return sysfs_emit(buf, "%#llx\n", len);
  86 }
  87
  88 static struct device_attribute dev_attr_ram_size =
  89         __ATTR(size, 0444, ram_size_show, NULL);
  90
  91 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr,
  92                               char *buf)
  93 {
  94         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  95         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  96         unsigned long long len = resource_size(&cxlds->pmem_res);
  97
  98         return sysfs_emit(buf, "%#llx\n", len);
  99 }
 100
 101 static struct device_attribute dev_attr_pmem_size =
 102         __ATTR(size, 0444, pmem_size_show, NULL);
 103
 104 static ssize_t serial_show(struct device *dev, struct device_attribute *attr,
 105                            char *buf)
 106 {
 107         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 108         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 109
 110         return sysfs_emit(buf, "%#llx\n", cxlds->serial);
 111 }
 112 static DEVICE_ATTR_RO(serial);
 113
 114 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr,
 115                               char *buf)
 116 {
 117         return sprintf(buf, "%d\n", dev_to_node(dev));
 118 }
 119 static DEVICE_ATTR_RO(numa_node);
 120
 121 static ssize_t security_state_show(struct device *dev,
 122                                    struct device_attribute *attr,
 123                                    char *buf)
 124 {
 125         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 126         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 127         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
 128         unsigned long state = mds->security.state;
 129         int rc = 0;
 130
 131         /* sync with latest submission state */
 132         mutex_lock(&mds->mbox_mutex);
 133         if (mds->security.sanitize_active)
 134                 rc = sysfs_emit(buf, "sanitize\n");
 135         mutex_unlock(&mds->mbox_mutex);
 136         if (rc)
 137                 return rc;
 138
 139         if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET))
 140                 return sysfs_emit(buf, "disabled\n");
 141         if (state & CXL_PMEM_SEC_STATE_FROZEN ||
 142             state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT ||
 143             state & CXL_PMEM_SEC_STATE_USER_PLIMIT)
 144                 return sysfs_emit(buf, "frozen\n");
 145         if (state & CXL_PMEM_SEC_STATE_LOCKED)
 146                 return sysfs_emit(buf, "locked\n");
 147         else
 148                 return sysfs_emit(buf, "unlocked\n");
 149 }
 150 static struct device_attribute dev_attr_security_state =
 151         __ATTR(state, 0444, security_state_show, NULL);
 152
 153 static ssize_t security_sanitize_store(struct device *dev,
 154                                        struct device_attribute *attr,
 155                                        const char *buf, size_t len)
 156 {
 157         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 158         bool sanitize;
 159         ssize_t rc;
 160
 161         if (kstrtobool(buf, &sanitize) || !sanitize)
 162                 return -EINVAL;
 163
 164         rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE);
 165         if (rc)
 166                 return rc;
 167
 168         return len;
 169 }
 170 static struct device_attribute dev_attr_security_sanitize =
 171         __ATTR(sanitize, 0200, NULL, security_sanitize_store);
 172
 173 static ssize_t security_erase_store(struct device *dev,
 174                                     struct device_attribute *attr,
 175                                     const char *buf, size_t len)
 176 {
 177         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 178         ssize_t rc;
 179         bool erase;
 180
 181         if (kstrtobool(buf, &erase) || !erase)
 182                 return -EINVAL;
 183
 184         rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE);
 185         if (rc)
 186                 return rc;
 187
 188         return len;
 189 }
 190 static struct device_attribute dev_attr_security_erase =
 191         __ATTR(erase, 0200, NULL, security_erase_store);
 192
 193 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd)
 194 {
 195         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 196         u64 offset, length;
 197         int rc = 0;
 198
 199         /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */
 200         if (resource_size(&cxlds->pmem_res)) {
 201                 offset = cxlds->pmem_res.start;
 202                 length = resource_size(&cxlds->pmem_res);
 203                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
 204                 if (rc)
 205                         return rc;
 206         }
 207         if (resource_size(&cxlds->ram_res)) {
 208                 offset = cxlds->ram_res.start;
 209                 length = resource_size(&cxlds->ram_res);
 210                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
 211                 /*
 212                  * Invalid Physical Address is not an error for
 213                  * volatile addresses. Device support is optional.
 214                  */
 215                 if (rc == -EFAULT)
 216                         rc = 0;
 217         }
 218         return rc;
 219 }
 220
 221 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)
 222 {
 223         struct cxl_port *port;
 224         int rc;
 225
 226         port = cxlmd->endpoint;
 227         if (!port || !is_cxl_endpoint(port))
 228                 return -EINVAL;
 229
 230         rc = down_read_interruptible(&cxl_dpa_rwsem);
 231         if (rc)
 232                 return rc;
 233
 234         if (cxl_num_decoders_committed(port) == 0) {
 235                 /* No regions mapped to this memdev */
 236                 rc = cxl_get_poison_by_memdev(cxlmd);
 237         } else {
 238                 /* Regions mapped, collect poison by endpoint */
 239                 rc =  cxl_get_poison_by_endpoint(port);
 240         }
 241         up_read(&cxl_dpa_rwsem);
 242
 243         return rc;
 244 }
 245 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, CXL);
 246
 247 struct cxl_dpa_to_region_context {
 248         struct cxl_region *cxlr;
 249         u64 dpa;
 250 };
 251
 252 static int __cxl_dpa_to_region(struct device *dev, void *arg)
 253 {
 254         struct cxl_dpa_to_region_context *ctx = arg;
 255         struct cxl_endpoint_decoder *cxled;
 256         u64 dpa = ctx->dpa;
 257
 258         if (!is_endpoint_decoder(dev))
 259                 return 0;
 260
 261         cxled = to_cxl_endpoint_decoder(dev);
 262         if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
 263                 return 0;
 264
 265         if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start)
 266                 return 0;
 267
 268         dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa,
 269                 dev_name(&cxled->cxld.region->dev));
 270
 271         ctx->cxlr = cxled->cxld.region;
 272
 273         return 1;
 274 }
 275
 276 static struct cxl_region *cxl_dpa_to_region(struct cxl_memdev *cxlmd, u64 dpa)
 277 {
 278         struct cxl_dpa_to_region_context ctx;
 279         struct cxl_port *port;
 280
 281         ctx = (struct cxl_dpa_to_region_context) {
 282                 .dpa = dpa,
 283         };
 284         port = cxlmd->endpoint;
 285         if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port))
 286                 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region);
 287
 288         return ctx.cxlr;
 289 }
 290
 291 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa)
 292 {
 293         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 294
 295         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 296                 return 0;
 297
 298         if (!resource_size(&cxlds->dpa_res)) {
 299                 dev_dbg(cxlds->dev, "device has no dpa resource\n");
 300                 return -EINVAL;
 301         }
 302         if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) {
 303                 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n",
 304                         dpa, &cxlds->dpa_res);
 305                 return -EINVAL;
 306         }
 307         if (!IS_ALIGNED(dpa, 64)) {
 308                 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa);
 309                 return -EINVAL;
 310         }
 311
 312         return 0;
 313 }
 314
 315 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
 316 {
 317         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 318         struct cxl_mbox_inject_poison inject;
 319         struct cxl_poison_record record;
 320         struct cxl_mbox_cmd mbox_cmd;
 321         struct cxl_region *cxlr;
 322         int rc;
 323
 324         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 325                 return 0;
 326
 327         rc = down_read_interruptible(&cxl_dpa_rwsem);
 328         if (rc)
 329                 return rc;
 330
 331         rc = cxl_validate_poison_dpa(cxlmd, dpa);
 332         if (rc)
 333                 goto out;
 334
 335         inject.address = cpu_to_le64(dpa);
 336         mbox_cmd = (struct cxl_mbox_cmd) {
 337                 .opcode = CXL_MBOX_OP_INJECT_POISON,
 338                 .size_in = sizeof(inject),
 339                 .payload_in = &inject,
 340         };
 341         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 342         if (rc)
 343                 goto out;
 344
 345         cxlr = cxl_dpa_to_region(cxlmd, dpa);
 346         if (cxlr)
 347                 dev_warn_once(mds->cxlds.dev,
 348                               "poison inject dpa:%#llx region: %s\n", dpa,
 349                               dev_name(&cxlr->dev));
 350
 351         record = (struct cxl_poison_record) {
 352                 .address = cpu_to_le64(dpa),
 353                 .length = cpu_to_le32(1),
 354         };
 355         trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT);
 356 out:
 357         up_read(&cxl_dpa_rwsem);
 358
 359         return rc;
 360 }
 361 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL);
 362
 363 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
 364 {
 365         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 366         struct cxl_mbox_clear_poison clear;
 367         struct cxl_poison_record record;
 368         struct cxl_mbox_cmd mbox_cmd;
 369         struct cxl_region *cxlr;
 370         int rc;
 371
 372         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 373                 return 0;
 374
 375         rc = down_read_interruptible(&cxl_dpa_rwsem);
 376         if (rc)
 377                 return rc;
 378
 379         rc = cxl_validate_poison_dpa(cxlmd, dpa);
 380         if (rc)
 381                 goto out;
 382
 383         /*
 384          * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command
 385          * is defined to accept 64 bytes of write-data, along with the
 386          * address to clear. This driver uses zeroes as write-data.
 387          */
 388         clear = (struct cxl_mbox_clear_poison) {
 389                 .address = cpu_to_le64(dpa)
 390         };
 391
 392         mbox_cmd = (struct cxl_mbox_cmd) {
 393                 .opcode = CXL_MBOX_OP_CLEAR_POISON,
 394                 .size_in = sizeof(clear),
 395                 .payload_in = &clear,
 396         };
 397
 398         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 399         if (rc)
 400                 goto out;
 401
 402         cxlr = cxl_dpa_to_region(cxlmd, dpa);
 403         if (cxlr)
 404                 dev_warn_once(mds->cxlds.dev,
 405                               "poison clear dpa:%#llx region: %s\n", dpa,
 406                               dev_name(&cxlr->dev));
 407
 408         record = (struct cxl_poison_record) {
 409                 .address = cpu_to_le64(dpa),
 410                 .length = cpu_to_le32(1),
 411         };
 412         trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR);
 413 out:
 414         up_read(&cxl_dpa_rwsem);
 415
 416         return rc;
 417 }
 418 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, CXL);
 419
 420 static struct attribute *cxl_memdev_attributes[] = {
 421         &dev_attr_serial.attr,
 422         &dev_attr_firmware_version.attr,
 423         &dev_attr_payload_max.attr,
 424         &dev_attr_label_storage_size.attr,
 425         &dev_attr_numa_node.attr,
 426         NULL,
 427 };
 428
 429 static struct attribute *cxl_memdev_pmem_attributes[] = {
 430         &dev_attr_pmem_size.attr,
 431         NULL,
 432 };
 433
 434 static struct attribute *cxl_memdev_ram_attributes[] = {
 435         &dev_attr_ram_size.attr,
 436         NULL,
 437 };
 438
 439 static struct attribute *cxl_memdev_security_attributes[] = {
 440         &dev_attr_security_state.attr,
 441         &dev_attr_security_sanitize.attr,
 442         &dev_attr_security_erase.attr,
 443         NULL,
 444 };
 445
 446 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a,
 447                                   int n)
 448 {
 449         if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr)
 450                 return 0;
 451         return a->mode;
 452 }
 453
 454 static struct attribute_group cxl_memdev_attribute_group = {
 455         .attrs = cxl_memdev_attributes,
 456         .is_visible = cxl_memdev_visible,
 457 };
 458
 459 static struct attribute_group cxl_memdev_ram_attribute_group = {
 460         .name = "ram",
 461         .attrs = cxl_memdev_ram_attributes,
 462 };
 463
 464 static struct attribute_group cxl_memdev_pmem_attribute_group = {
 465         .name = "pmem",
 466         .attrs = cxl_memdev_pmem_attributes,
 467 };
 468
 469 static umode_t cxl_memdev_security_visible(struct kobject *kobj,
 470                                            struct attribute *a, int n)
 471 {
 472         struct device *dev = kobj_to_dev(kobj);
 473         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 474         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 475
 476         if (a == &dev_attr_security_sanitize.attr &&
 477             !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
 478                 return 0;
 479
 480         if (a == &dev_attr_security_erase.attr &&
 481             !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds))
 482                 return 0;
 483
 484         return a->mode;
 485 }
 486
 487 static struct attribute_group cxl_memdev_security_attribute_group = {
 488         .name = "security",
 489         .attrs = cxl_memdev_security_attributes,
 490         .is_visible = cxl_memdev_security_visible,
 491 };
 492
 493 static const struct attribute_group *cxl_memdev_attribute_groups[] = {
 494         &cxl_memdev_attribute_group,
 495         &cxl_memdev_ram_attribute_group,
 496         &cxl_memdev_pmem_attribute_group,
 497         &cxl_memdev_security_attribute_group,
 498         NULL,
 499 };
 500
 501 static const struct device_type cxl_memdev_type = {
 502         .name = "cxl_memdev",
 503         .release = cxl_memdev_release,
 504         .devnode = cxl_memdev_devnode,
 505         .groups = cxl_memdev_attribute_groups,
 506 };
 507
 508 bool is_cxl_memdev(const struct device *dev)
 509 {
 510         return dev->type == &cxl_memdev_type;
 511 }
 512 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL);
 513
 514 /**
 515  * set_exclusive_cxl_commands() - atomically disable user cxl commands
 516  * @mds: The device state to operate on
 517  * @cmds: bitmap of commands to mark exclusive
 518  *
 519  * Grab the cxl_memdev_rwsem in write mode to flush in-flight
 520  * invocations of the ioctl path and then disable future execution of
 521  * commands with the command ids set in @cmds.
 522  */
 523 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
 524                                 unsigned long *cmds)
 525 {
 526         down_write(&cxl_memdev_rwsem);
 527         bitmap_or(mds->exclusive_cmds, mds->exclusive_cmds, cmds,
 528                   CXL_MEM_COMMAND_ID_MAX);
 529         up_write(&cxl_memdev_rwsem);
 530 }
 531 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL);
 532
 533 /**
 534  * clear_exclusive_cxl_commands() - atomically enable user cxl commands
 535  * @mds: The device state to modify
 536  * @cmds: bitmap of commands to mark available for userspace
 537  */
 538 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,
 539                                   unsigned long *cmds)
 540 {
 541         down_write(&cxl_memdev_rwsem);
 542         bitmap_andnot(mds->exclusive_cmds, mds->exclusive_cmds, cmds,
 543                       CXL_MEM_COMMAND_ID_MAX);
 544         up_write(&cxl_memdev_rwsem);
 545 }
 546 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL);
 547
 548 static void cxl_memdev_shutdown(struct device *dev)
 549 {
 550         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 551
 552         down_write(&cxl_memdev_rwsem);
 553         cxlmd->cxlds = NULL;
 554         up_write(&cxl_memdev_rwsem);
 555 }
 556
 557 static void cxl_memdev_unregister(void *_cxlmd)
 558 {
 559         struct cxl_memdev *cxlmd = _cxlmd;
 560         struct device *dev = &cxlmd->dev;
 561
 562         cdev_device_del(&cxlmd->cdev, dev);
 563         cxl_memdev_shutdown(dev);
 564         put_device(dev);
 565 }
 566
 567 static void detach_memdev(struct work_struct *work)
 568 {
 569         struct cxl_memdev *cxlmd;
 570
 571         cxlmd = container_of(work, typeof(*cxlmd), detach_work);
 572         device_release_driver(&cxlmd->dev);
 573         put_device(&cxlmd->dev);
 574 }
 575
 576 static struct lock_class_key cxl_memdev_key;
 577
 578 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
 579                                            const struct file_operations *fops)
 580 {
 581         struct cxl_memdev *cxlmd;
 582         struct device *dev;
 583         struct cdev *cdev;
 584         int rc;
 585
 586         cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL);
 587         if (!cxlmd)
 588                 return ERR_PTR(-ENOMEM);
 589
 590         rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL);
 591         if (rc < 0)
 592                 goto err;
 593         cxlmd->id = rc;
 594         cxlmd->depth = -1;
 595
 596         dev = &cxlmd->dev;
 597         device_initialize(dev);
 598         lockdep_set_class(&dev->mutex, &cxl_memdev_key);
 599         dev->parent = cxlds->dev;
 600         dev->bus = &cxl_bus_type;
 601         dev->devt = MKDEV(cxl_mem_major, cxlmd->id);
 602         dev->type = &cxl_memdev_type;
 603         device_set_pm_not_required(dev);
 604         INIT_WORK(&cxlmd->detach_work, detach_memdev);
 605
 606         cdev = &cxlmd->cdev;
 607         cdev_init(cdev, fops);
 608         return cxlmd;
 609
 610 err:
 611         kfree(cxlmd);
 612         return ERR_PTR(rc);
 613 }
 614
 615 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
 616                                unsigned long arg)
 617 {
 618         switch (cmd) {
 619         case CXL_MEM_QUERY_COMMANDS:
 620                 return cxl_query_cmd(cxlmd, (void __user *)arg);
 621         case CXL_MEM_SEND_COMMAND:
 622                 return cxl_send_cmd(cxlmd, (void __user *)arg);
 623         default:
 624                 return -ENOTTY;
 625         }
 626 }
 627
 628 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,
 629                              unsigned long arg)
 630 {
 631         struct cxl_memdev *cxlmd = file->private_data;
 632         struct cxl_dev_state *cxlds;
 633         int rc = -ENXIO;
 634
 635         down_read(&cxl_memdev_rwsem);
 636         cxlds = cxlmd->cxlds;
 637         if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM)
 638                 rc = __cxl_memdev_ioctl(cxlmd, cmd, arg);
 639         up_read(&cxl_memdev_rwsem);
 640
 641         return rc;
 642 }
 643
 644 static int cxl_memdev_open(struct inode *inode, struct file *file)
 645 {
 646         struct cxl_memdev *cxlmd =
 647                 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
 648
 649         get_device(&cxlmd->dev);
 650         file->private_data = cxlmd;
 651
 652         return 0;
 653 }
 654
 655 static int cxl_memdev_release_file(struct inode *inode, struct file *file)
 656 {
 657         struct cxl_memdev *cxlmd =
 658                 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
 659
 660         put_device(&cxlmd->dev);
 661
 662         return 0;
 663 }
 664
 665 /**
 666  * cxl_mem_get_fw_info - Get Firmware info
 667  * @mds: The device data for the operation
 668  *
 669  * Retrieve firmware info for the device specified.
 670  *
 671  * Return: 0 if no error: or the result of the mailbox command.
 672  *
 673  * See CXL-3.0 8.2.9.3.1 Get FW Info
 674  */
 675 static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)
 676 {
 677         struct cxl_mbox_get_fw_info info;
 678         struct cxl_mbox_cmd mbox_cmd;
 679         int rc;
 680
 681         mbox_cmd = (struct cxl_mbox_cmd) {
 682                 .opcode = CXL_MBOX_OP_GET_FW_INFO,
 683                 .size_out = sizeof(info),
 684                 .payload_out = &info,
 685         };
 686
 687         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 688         if (rc < 0)
 689                 return rc;
 690
 691         mds->fw.num_slots = info.num_slots;
 692         mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK,
 693                                        info.slot_info);
 694
 695         return 0;
 696 }
 697
 698 /**
 699  * cxl_mem_activate_fw - Activate Firmware
 700  * @mds: The device data for the operation
 701  * @slot: slot number to activate
 702  *
 703  * Activate firmware in a given slot for the device specified.
 704  *
 705  * Return: 0 if no error: or the result of the mailbox command.
 706  *
 707  * See CXL-3.0 8.2.9.3.3 Activate FW
 708  */
 709 static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)
 710 {
 711         struct cxl_mbox_activate_fw activate;
 712         struct cxl_mbox_cmd mbox_cmd;
 713
 714         if (slot == 0 || slot > mds->fw.num_slots)
 715                 return -EINVAL;
 716
 717         mbox_cmd = (struct cxl_mbox_cmd) {
 718                 .opcode = CXL_MBOX_OP_ACTIVATE_FW,
 719                 .size_in = sizeof(activate),
 720                 .payload_in = &activate,
 721         };
 722
 723         /* Only offline activation supported for now */
 724         activate.action = CXL_FW_ACTIVATE_OFFLINE;
 725         activate.slot = slot;
 726
 727         return cxl_internal_send_cmd(mds, &mbox_cmd);
 728 }
 729
 730 /**
 731  * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer
 732  * @mds: The device data for the operation
 733  *
 734  * Abort an in-progress firmware transfer for the device specified.
 735  *
 736  * Return: 0 if no error: or the result of the mailbox command.
 737  *
 738  * See CXL-3.0 8.2.9.3.2 Transfer FW
 739  */
 740 static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds)
 741 {
 742         struct cxl_mbox_transfer_fw *transfer;
 743         struct cxl_mbox_cmd mbox_cmd;
 744         int rc;
 745
 746         transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL);
 747         if (!transfer)
 748                 return -ENOMEM;
 749
 750         /* Set a 1s poll interval and a total wait time of 30s */
 751         mbox_cmd = (struct cxl_mbox_cmd) {
 752                 .opcode = CXL_MBOX_OP_TRANSFER_FW,
 753                 .size_in = sizeof(*transfer),
 754                 .payload_in = transfer,
 755                 .poll_interval_ms = 1000,
 756                 .poll_count = 30,
 757         };
 758
 759         transfer->action = CXL_FW_TRANSFER_ACTION_ABORT;
 760
 761         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 762         kfree(transfer);
 763         return rc;
 764 }
 765
 766 static void cxl_fw_cleanup(struct fw_upload *fwl)
 767 {
 768         struct cxl_memdev_state *mds = fwl->dd_handle;
 769
 770         mds->fw.next_slot = 0;
 771 }
 772
 773 static int cxl_fw_do_cancel(struct fw_upload *fwl)
 774 {
 775         struct cxl_memdev_state *mds = fwl->dd_handle;
 776         struct cxl_dev_state *cxlds = &mds->cxlds;
 777         struct cxl_memdev *cxlmd = cxlds->cxlmd;
 778         int rc;
 779
 780         rc = cxl_mem_abort_fw_xfer(mds);
 781         if (rc < 0)
 782                 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc);
 783
 784         return FW_UPLOAD_ERR_CANCELED;
 785 }
 786
 787 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,
 788                                          u32 size)
 789 {
 790         struct cxl_memdev_state *mds = fwl->dd_handle;
 791         struct cxl_mbox_transfer_fw *transfer;
 792
 793         if (!size)
 794                 return FW_UPLOAD_ERR_INVALID_SIZE;
 795
 796         mds->fw.oneshot = struct_size(transfer, data, size) <
 797                             mds->payload_size;
 798
 799         if (cxl_mem_get_fw_info(mds))
 800                 return FW_UPLOAD_ERR_HW_ERROR;
 801
 802         /*
 803          * So far no state has been changed, hence no other cleanup is
 804          * necessary. Simply return the cancelled status.
 805          */
 806         if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
 807                 return FW_UPLOAD_ERR_CANCELED;
 808
 809         return FW_UPLOAD_ERR_NONE;
 810 }
 811
 812 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
 813                                        u32 offset, u32 size, u32 *written)
 814 {
 815         struct cxl_memdev_state *mds = fwl->dd_handle;
 816         struct cxl_dev_state *cxlds = &mds->cxlds;
 817         struct cxl_memdev *cxlmd = cxlds->cxlmd;
 818         struct cxl_mbox_transfer_fw *transfer;
 819         struct cxl_mbox_cmd mbox_cmd;
 820         u32 cur_size, remaining;
 821         size_t size_in;
 822         int rc;
 823
 824         *written = 0;
 825
 826         /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */
 827         if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) {
 828                 dev_err(&cxlmd->dev,
 829                         "misaligned offset for FW transfer slice (%u)\n",
 830                         offset);
 831                 return FW_UPLOAD_ERR_RW_ERROR;
 832         }
 833
 834         /*
 835          * Pick transfer size based on mds->payload_size @size must bw 128-byte
 836          * aligned, ->payload_size is a power of 2 starting at 256 bytes, and
 837          * sizeof(*transfer) is 128.  These constraints imply that @cur_size
 838          * will always be 128b aligned.
 839          */
 840         cur_size = min_t(size_t, size, mds->payload_size - sizeof(*transfer));
 841
 842         remaining = size - cur_size;
 843         size_in = struct_size(transfer, data, cur_size);
 844
 845         if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
 846                 return cxl_fw_do_cancel(fwl);
 847
 848         /*
 849          * Slot numbers are 1-indexed
 850          * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1')
 851          * Check for rollover using modulo, and 1-index it by adding 1
 852          */
 853         mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1;
 854
 855         /* Do the transfer via mailbox cmd */
 856         transfer = kzalloc(size_in, GFP_KERNEL);
 857         if (!transfer)
 858                 return FW_UPLOAD_ERR_RW_ERROR;
 859
 860         transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT);
 861         memcpy(transfer->data, data + offset, cur_size);
 862         if (mds->fw.oneshot) {
 863                 transfer->action = CXL_FW_TRANSFER_ACTION_FULL;
 864                 transfer->slot = mds->fw.next_slot;
 865         } else {
 866                 if (offset == 0) {
 867                         transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE;
 868                 } else if (remaining == 0) {
 869                         transfer->action = CXL_FW_TRANSFER_ACTION_END;
 870                         transfer->slot = mds->fw.next_slot;
 871                 } else {
 872                         transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE;
 873                 }
 874         }
 875
 876         mbox_cmd = (struct cxl_mbox_cmd) {
 877                 .opcode = CXL_MBOX_OP_TRANSFER_FW,
 878                 .size_in = size_in,
 879                 .payload_in = transfer,
 880                 .poll_interval_ms = 1000,
 881                 .poll_count = 30,
 882         };
 883
 884         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 885         if (rc < 0) {
 886                 rc = FW_UPLOAD_ERR_RW_ERROR;
 887                 goto out_free;
 888         }
 889
 890         *written = cur_size;
 891
 892         /* Activate FW if oneshot or if the last slice was written */
 893         if (mds->fw.oneshot || remaining == 0) {
 894                 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n",
 895                         mds->fw.next_slot);
 896                 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot);
 897                 if (rc < 0) {
 898                         dev_err(&cxlmd->dev, "Error activating firmware: %d\n",
 899                                 rc);
 900                         rc = FW_UPLOAD_ERR_HW_ERROR;
 901                         goto out_free;
 902                 }
 903         }
 904
 905         rc = FW_UPLOAD_ERR_NONE;
 906
 907 out_free:
 908         kfree(transfer);
 909         return rc;
 910 }
 911
 912 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl)
 913 {
 914         struct cxl_memdev_state *mds = fwl->dd_handle;
 915
 916         /*
 917          * cxl_internal_send_cmd() handles background operations synchronously.
 918          * No need to wait for completions here - any errors would've been
 919          * reported and handled during the ->write() call(s).
 920          * Just check if a cancel request was received, and return success.
 921          */
 922         if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
 923                 return cxl_fw_do_cancel(fwl);
 924
 925         return FW_UPLOAD_ERR_NONE;
 926 }
 927
 928 static void cxl_fw_cancel(struct fw_upload *fwl)
 929 {
 930         struct cxl_memdev_state *mds = fwl->dd_handle;
 931
 932         set_bit(CXL_FW_CANCEL, mds->fw.state);
 933 }
 934
 935 static const struct fw_upload_ops cxl_memdev_fw_ops = {
 936         .prepare = cxl_fw_prepare,
 937         .write = cxl_fw_write,
 938         .poll_complete = cxl_fw_poll_complete,
 939         .cancel = cxl_fw_cancel,
 940         .cleanup = cxl_fw_cleanup,
 941 };
 942
 943 static void cxl_remove_fw_upload(void *fwl)
 944 {
 945         firmware_upload_unregister(fwl);
 946 }
 947
 948 int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds)
 949 {
 950         struct cxl_dev_state *cxlds = &mds->cxlds;
 951         struct device *dev = &cxlds->cxlmd->dev;
 952         struct fw_upload *fwl;
 953
 954         if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, mds->enabled_cmds))
 955                 return 0;
 956
 957         fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev),
 958                                        &cxl_memdev_fw_ops, mds);
 959         if (IS_ERR(fwl))
 960                 return PTR_ERR(fwl);
 961         return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl);
 962 }
 963 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, CXL);
 964
 965 static const struct file_operations cxl_memdev_fops = {
 966         .owner = THIS_MODULE,
 967         .unlocked_ioctl = cxl_memdev_ioctl,
 968         .open = cxl_memdev_open,
 969         .release = cxl_memdev_release_file,
 970         .compat_ioctl = compat_ptr_ioctl,
 971         .llseek = noop_llseek,
 972 };
 973
 974 struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
 975                                        struct cxl_dev_state *cxlds)
 976 {
 977         struct cxl_memdev *cxlmd;
 978         struct device *dev;
 979         struct cdev *cdev;
 980         int rc;
 981
 982         cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops);
 983         if (IS_ERR(cxlmd))
 984                 return cxlmd;
 985
 986         dev = &cxlmd->dev;
 987         rc = dev_set_name(dev, "mem%d", cxlmd->id);
 988         if (rc)
 989                 goto err;
 990
 991         /*
 992          * Activate ioctl operations, no cxl_memdev_rwsem manipulation
 993          * needed as this is ordered with cdev_add() publishing the device.
 994          */
 995         cxlmd->cxlds = cxlds;
 996         cxlds->cxlmd = cxlmd;
 997
 998         cdev = &cxlmd->cdev;
 999         rc = cdev_device_add(cdev, dev);
1000         if (rc)
1001                 goto err;
1002
1003         rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd);
1004         if (rc)
1005                 return ERR_PTR(rc);
1006         return cxlmd;
1007
1008 err:
1009         /*
1010          * The cdev was briefly live, shutdown any ioctl operations that
1011          * saw that state.
1012          */
1013         cxl_memdev_shutdown(dev);
1014         put_device(dev);
1015         return ERR_PTR(rc);
1016 }
1017 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
1018
1019 static void sanitize_teardown_notifier(void *data)
1020 {
1021         struct cxl_memdev_state *mds = data;
1022         struct kernfs_node *state;
1023
1024         /*
1025          * Prevent new irq triggered invocations of the workqueue and
1026          * flush inflight invocations.
1027          */
1028         mutex_lock(&mds->mbox_mutex);
1029         state = mds->security.sanitize_node;
1030         mds->security.sanitize_node = NULL;
1031         mutex_unlock(&mds->mbox_mutex);
1032
1033         cancel_delayed_work_sync(&mds->security.poll_dwork);
1034         sysfs_put(state);
1035 }
1036
1037 int devm_cxl_sanitize_setup_notifier(struct device *host,
1038                                      struct cxl_memdev *cxlmd)
1039 {
1040         struct cxl_dev_state *cxlds = cxlmd->cxlds;
1041         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
1042         struct kernfs_node *sec;
1043
1044         if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
1045                 return 0;
1046
1047         /*
1048          * Note, the expectation is that @cxlmd would have failed to be
1049          * created if these sysfs_get_dirent calls fail.
1050          */
1051         sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security");
1052         if (!sec)
1053                 return -ENOENT;
1054         mds->security.sanitize_node = sysfs_get_dirent(sec, "state");
1055         sysfs_put(sec);
1056         if (!mds->security.sanitize_node)
1057                 return -ENOENT;
1058
1059         return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds);
1060 }
1061 EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, CXL);
1062
1063 __init int cxl_memdev_init(void)
1064 {
1065         dev_t devt;
1066         int rc;
1067
1068         rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl");
1069         if (rc)
1070                 return rc;
1071
1072         cxl_mem_major = MAJOR(devt);
1073
1074         return 0;
1075 }
1076
1077 void cxl_memdev_exit(void)
1078 {
1079         unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS);
1080 }