drivers/cxl/core/memdev.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2020 Intel Corporation. */
   3
   4 #include <linux/io-64-nonatomic-lo-hi.h>
   5 #include <linux/firmware.h>
   6 #include <linux/device.h>
   7 #include <linux/slab.h>
   8 #include <linux/idr.h>
   9 #include <linux/pci.h>
  10 #include <cxlmem.h>
  11 #include "trace.h"
  12 #include "core.h"
  13
  14 static DECLARE_RWSEM(cxl_memdev_rwsem);
  15
  16 /*
  17  * An entire PCI topology full of devices should be enough for any
  18  * config
  19  */
  20 #define CXL_MEM_MAX_DEVS 65536
  21
  22 static int cxl_mem_major;
  23 static DEFINE_IDA(cxl_memdev_ida);
  24
  25 static void cxl_memdev_release(struct device *dev)
  26 {
  27         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  28
  29         ida_free(&cxl_memdev_ida, cxlmd->id);
  30         kfree(cxlmd);
  31 }
  32
  33 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid,
  34                                 kgid_t *gid)
  35 {
  36         return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev));
  37 }
  38
  39 static ssize_t firmware_version_show(struct device *dev,
  40                                      struct device_attribute *attr, char *buf)
  41 {
  42         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  43         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  44         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
  45
  46         if (!mds)
  47                 return sysfs_emit(buf, "\n");
  48         return sysfs_emit(buf, "%.16s\n", mds->firmware_version);
  49 }
  50 static DEVICE_ATTR_RO(firmware_version);
  51
  52 static ssize_t payload_max_show(struct device *dev,
  53                                 struct device_attribute *attr, char *buf)
  54 {
  55         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  56         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  57         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
  58
  59         if (!mds)
  60                 return sysfs_emit(buf, "\n");
  61         return sysfs_emit(buf, "%zu\n", mds->payload_size);
  62 }
  63 static DEVICE_ATTR_RO(payload_max);
  64
  65 static ssize_t label_storage_size_show(struct device *dev,
  66                                        struct device_attribute *attr, char *buf)
  67 {
  68         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  69         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  70         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
  71
  72         if (!mds)
  73                 return sysfs_emit(buf, "\n");
  74         return sysfs_emit(buf, "%zu\n", mds->lsa_size);
  75 }
  76 static DEVICE_ATTR_RO(label_storage_size);
  77
  78 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr,
  79                              char *buf)
  80 {
  81         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  82         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  83         unsigned long long len = resource_size(&cxlds->ram_res);
  84
  85         return sysfs_emit(buf, "%#llx\n", len);
  86 }
  87
  88 static struct device_attribute dev_attr_ram_size =
  89         __ATTR(size, 0444, ram_size_show, NULL);
  90
  91 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr,
  92                               char *buf)
  93 {
  94         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  95         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  96         unsigned long long len = resource_size(&cxlds->pmem_res);
  97
  98         return sysfs_emit(buf, "%#llx\n", len);
  99 }
 100
 101 static struct device_attribute dev_attr_pmem_size =
 102         __ATTR(size, 0444, pmem_size_show, NULL);
 103
 104 static ssize_t serial_show(struct device *dev, struct device_attribute *attr,
 105                            char *buf)
 106 {
 107         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 108         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 109
 110         return sysfs_emit(buf, "%#llx\n", cxlds->serial);
 111 }
 112 static DEVICE_ATTR_RO(serial);
 113
 114 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr,
 115                               char *buf)
 116 {
 117         return sprintf(buf, "%d\n", dev_to_node(dev));
 118 }
 119 static DEVICE_ATTR_RO(numa_node);
 120
 121 static ssize_t security_state_show(struct device *dev,
 122                                    struct device_attribute *attr,
 123                                    char *buf)
 124 {
 125         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 126         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 127         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
 128         u64 reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET);
 129         u32 pct = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK, reg);
 130         u16 cmd = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK, reg);
 131         unsigned long state = mds->security.state;
 132
 133         if (cmd == CXL_MBOX_OP_SANITIZE && pct != 100)
 134                 return sysfs_emit(buf, "sanitize\n");
 135
 136         if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET))
 137                 return sysfs_emit(buf, "disabled\n");
 138         if (state & CXL_PMEM_SEC_STATE_FROZEN ||
 139             state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT ||
 140             state & CXL_PMEM_SEC_STATE_USER_PLIMIT)
 141                 return sysfs_emit(buf, "frozen\n");
 142         if (state & CXL_PMEM_SEC_STATE_LOCKED)
 143                 return sysfs_emit(buf, "locked\n");
 144         else
 145                 return sysfs_emit(buf, "unlocked\n");
 146 }
 147 static struct device_attribute dev_attr_security_state =
 148         __ATTR(state, 0444, security_state_show, NULL);
 149
 150 static ssize_t security_sanitize_store(struct device *dev,
 151                                        struct device_attribute *attr,
 152                                        const char *buf, size_t len)
 153 {
 154         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 155         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 156         struct cxl_port *port = cxlmd->endpoint;
 157         bool sanitize;
 158         ssize_t rc;
 159
 160         if (kstrtobool(buf, &sanitize) || !sanitize)
 161                 return -EINVAL;
 162
 163         if (!port || !is_cxl_endpoint(port))
 164                 return -EINVAL;
 165
 166         /* ensure no regions are mapped to this memdev */
 167         if (port->commit_end != -1)
 168                 return -EBUSY;
 169
 170         rc = cxl_mem_sanitize(mds, CXL_MBOX_OP_SANITIZE);
 171
 172         return rc ? rc : len;
 173 }
 174 static struct device_attribute dev_attr_security_sanitize =
 175         __ATTR(sanitize, 0200, NULL, security_sanitize_store);
 176
 177 static ssize_t security_erase_store(struct device *dev,
 178                                     struct device_attribute *attr,
 179                                     const char *buf, size_t len)
 180 {
 181         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 182         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 183         struct cxl_port *port = cxlmd->endpoint;
 184         ssize_t rc;
 185         bool erase;
 186
 187         if (kstrtobool(buf, &erase) || !erase)
 188                 return -EINVAL;
 189
 190         if (!port || !is_cxl_endpoint(port))
 191                 return -EINVAL;
 192
 193         /* ensure no regions are mapped to this memdev */
 194         if (port->commit_end != -1)
 195                 return -EBUSY;
 196
 197         rc = cxl_mem_sanitize(mds, CXL_MBOX_OP_SECURE_ERASE);
 198
 199         return rc ? rc : len;
 200 }
 201 static struct device_attribute dev_attr_security_erase =
 202         __ATTR(erase, 0200, NULL, security_erase_store);
 203
 204 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd)
 205 {
 206         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 207         u64 offset, length;
 208         int rc = 0;
 209
 210         /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */
 211         if (resource_size(&cxlds->pmem_res)) {
 212                 offset = cxlds->pmem_res.start;
 213                 length = resource_size(&cxlds->pmem_res);
 214                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
 215                 if (rc)
 216                         return rc;
 217         }
 218         if (resource_size(&cxlds->ram_res)) {
 219                 offset = cxlds->ram_res.start;
 220                 length = resource_size(&cxlds->ram_res);
 221                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
 222                 /*
 223                  * Invalid Physical Address is not an error for
 224                  * volatile addresses. Device support is optional.
 225                  */
 226                 if (rc == -EFAULT)
 227                         rc = 0;
 228         }
 229         return rc;
 230 }
 231
 232 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)
 233 {
 234         struct cxl_port *port;
 235         int rc;
 236
 237         port = cxlmd->endpoint;
 238         if (!port || !is_cxl_endpoint(port))
 239                 return -EINVAL;
 240
 241         rc = down_read_interruptible(&cxl_dpa_rwsem);
 242         if (rc)
 243                 return rc;
 244
 245         if (port->commit_end == -1) {
 246                 /* No regions mapped to this memdev */
 247                 rc = cxl_get_poison_by_memdev(cxlmd);
 248         } else {
 249                 /* Regions mapped, collect poison by endpoint */
 250                 rc =  cxl_get_poison_by_endpoint(port);
 251         }
 252         up_read(&cxl_dpa_rwsem);
 253
 254         return rc;
 255 }
 256 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, CXL);
 257
 258 struct cxl_dpa_to_region_context {
 259         struct cxl_region *cxlr;
 260         u64 dpa;
 261 };
 262
 263 static int __cxl_dpa_to_region(struct device *dev, void *arg)
 264 {
 265         struct cxl_dpa_to_region_context *ctx = arg;
 266         struct cxl_endpoint_decoder *cxled;
 267         u64 dpa = ctx->dpa;
 268
 269         if (!is_endpoint_decoder(dev))
 270                 return 0;
 271
 272         cxled = to_cxl_endpoint_decoder(dev);
 273         if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
 274                 return 0;
 275
 276         if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start)
 277                 return 0;
 278
 279         dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa,
 280                 dev_name(&cxled->cxld.region->dev));
 281
 282         ctx->cxlr = cxled->cxld.region;
 283
 284         return 1;
 285 }
 286
 287 static struct cxl_region *cxl_dpa_to_region(struct cxl_memdev *cxlmd, u64 dpa)
 288 {
 289         struct cxl_dpa_to_region_context ctx;
 290         struct cxl_port *port;
 291
 292         ctx = (struct cxl_dpa_to_region_context) {
 293                 .dpa = dpa,
 294         };
 295         port = cxlmd->endpoint;
 296         if (port && is_cxl_endpoint(port) && port->commit_end != -1)
 297                 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region);
 298
 299         return ctx.cxlr;
 300 }
 301
 302 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa)
 303 {
 304         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 305
 306         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 307                 return 0;
 308
 309         if (!resource_size(&cxlds->dpa_res)) {
 310                 dev_dbg(cxlds->dev, "device has no dpa resource\n");
 311                 return -EINVAL;
 312         }
 313         if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) {
 314                 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n",
 315                         dpa, &cxlds->dpa_res);
 316                 return -EINVAL;
 317         }
 318         if (!IS_ALIGNED(dpa, 64)) {
 319                 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa);
 320                 return -EINVAL;
 321         }
 322
 323         return 0;
 324 }
 325
 326 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
 327 {
 328         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 329         struct cxl_mbox_inject_poison inject;
 330         struct cxl_poison_record record;
 331         struct cxl_mbox_cmd mbox_cmd;
 332         struct cxl_region *cxlr;
 333         int rc;
 334
 335         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 336                 return 0;
 337
 338         rc = down_read_interruptible(&cxl_dpa_rwsem);
 339         if (rc)
 340                 return rc;
 341
 342         rc = cxl_validate_poison_dpa(cxlmd, dpa);
 343         if (rc)
 344                 goto out;
 345
 346         inject.address = cpu_to_le64(dpa);
 347         mbox_cmd = (struct cxl_mbox_cmd) {
 348                 .opcode = CXL_MBOX_OP_INJECT_POISON,
 349                 .size_in = sizeof(inject),
 350                 .payload_in = &inject,
 351         };
 352         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 353         if (rc)
 354                 goto out;
 355
 356         cxlr = cxl_dpa_to_region(cxlmd, dpa);
 357         if (cxlr)
 358                 dev_warn_once(mds->cxlds.dev,
 359                               "poison inject dpa:%#llx region: %s\n", dpa,
 360                               dev_name(&cxlr->dev));
 361
 362         record = (struct cxl_poison_record) {
 363                 .address = cpu_to_le64(dpa),
 364                 .length = cpu_to_le32(1),
 365         };
 366         trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT);
 367 out:
 368         up_read(&cxl_dpa_rwsem);
 369
 370         return rc;
 371 }
 372 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL);
 373
 374 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
 375 {
 376         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 377         struct cxl_mbox_clear_poison clear;
 378         struct cxl_poison_record record;
 379         struct cxl_mbox_cmd mbox_cmd;
 380         struct cxl_region *cxlr;
 381         int rc;
 382
 383         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 384                 return 0;
 385
 386         rc = down_read_interruptible(&cxl_dpa_rwsem);
 387         if (rc)
 388                 return rc;
 389
 390         rc = cxl_validate_poison_dpa(cxlmd, dpa);
 391         if (rc)
 392                 goto out;
 393
 394         /*
 395          * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command
 396          * is defined to accept 64 bytes of write-data, along with the
 397          * address to clear. This driver uses zeroes as write-data.
 398          */
 399         clear = (struct cxl_mbox_clear_poison) {
 400                 .address = cpu_to_le64(dpa)
 401         };
 402
 403         mbox_cmd = (struct cxl_mbox_cmd) {
 404                 .opcode = CXL_MBOX_OP_CLEAR_POISON,
 405                 .size_in = sizeof(clear),
 406                 .payload_in = &clear,
 407         };
 408
 409         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 410         if (rc)
 411                 goto out;
 412
 413         cxlr = cxl_dpa_to_region(cxlmd, dpa);
 414         if (cxlr)
 415                 dev_warn_once(mds->cxlds.dev,
 416                               "poison clear dpa:%#llx region: %s\n", dpa,
 417                               dev_name(&cxlr->dev));
 418
 419         record = (struct cxl_poison_record) {
 420                 .address = cpu_to_le64(dpa),
 421                 .length = cpu_to_le32(1),
 422         };
 423         trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR);
 424 out:
 425         up_read(&cxl_dpa_rwsem);
 426
 427         return rc;
 428 }
 429 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, CXL);
 430
 431 static struct attribute *cxl_memdev_attributes[] = {
 432         &dev_attr_serial.attr,
 433         &dev_attr_firmware_version.attr,
 434         &dev_attr_payload_max.attr,
 435         &dev_attr_label_storage_size.attr,
 436         &dev_attr_numa_node.attr,
 437         NULL,
 438 };
 439
 440 static struct attribute *cxl_memdev_pmem_attributes[] = {
 441         &dev_attr_pmem_size.attr,
 442         NULL,
 443 };
 444
 445 static struct attribute *cxl_memdev_ram_attributes[] = {
 446         &dev_attr_ram_size.attr,
 447         NULL,
 448 };
 449
 450 static struct attribute *cxl_memdev_security_attributes[] = {
 451         &dev_attr_security_state.attr,
 452         &dev_attr_security_sanitize.attr,
 453         &dev_attr_security_erase.attr,
 454         NULL,
 455 };
 456
 457 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a,
 458                                   int n)
 459 {
 460         if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr)
 461                 return 0;
 462         return a->mode;
 463 }
 464
 465 static struct attribute_group cxl_memdev_attribute_group = {
 466         .attrs = cxl_memdev_attributes,
 467         .is_visible = cxl_memdev_visible,
 468 };
 469
 470 static struct attribute_group cxl_memdev_ram_attribute_group = {
 471         .name = "ram",
 472         .attrs = cxl_memdev_ram_attributes,
 473 };
 474
 475 static struct attribute_group cxl_memdev_pmem_attribute_group = {
 476         .name = "pmem",
 477         .attrs = cxl_memdev_pmem_attributes,
 478 };
 479
 480 static umode_t cxl_memdev_security_visible(struct kobject *kobj,
 481                                            struct attribute *a, int n)
 482 {
 483         struct device *dev = kobj_to_dev(kobj);
 484         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 485         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 486
 487         if (a == &dev_attr_security_sanitize.attr &&
 488             !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
 489                 return 0;
 490
 491         if (a == &dev_attr_security_erase.attr &&
 492             !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds))
 493                 return 0;
 494
 495         return a->mode;
 496 }
 497
 498 static struct attribute_group cxl_memdev_security_attribute_group = {
 499         .name = "security",
 500         .attrs = cxl_memdev_security_attributes,
 501         .is_visible = cxl_memdev_security_visible,
 502 };
 503
 504 static const struct attribute_group *cxl_memdev_attribute_groups[] = {
 505         &cxl_memdev_attribute_group,
 506         &cxl_memdev_ram_attribute_group,
 507         &cxl_memdev_pmem_attribute_group,
 508         &cxl_memdev_security_attribute_group,
 509         NULL,
 510 };
 511
 512 static const struct device_type cxl_memdev_type = {
 513         .name = "cxl_memdev",
 514         .release = cxl_memdev_release,
 515         .devnode = cxl_memdev_devnode,
 516         .groups = cxl_memdev_attribute_groups,
 517 };
 518
 519 bool is_cxl_memdev(const struct device *dev)
 520 {
 521         return dev->type == &cxl_memdev_type;
 522 }
 523 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL);
 524
 525 /**
 526  * set_exclusive_cxl_commands() - atomically disable user cxl commands
 527  * @mds: The device state to operate on
 528  * @cmds: bitmap of commands to mark exclusive
 529  *
 530  * Grab the cxl_memdev_rwsem in write mode to flush in-flight
 531  * invocations of the ioctl path and then disable future execution of
 532  * commands with the command ids set in @cmds.
 533  */
 534 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
 535                                 unsigned long *cmds)
 536 {
 537         down_write(&cxl_memdev_rwsem);
 538         bitmap_or(mds->exclusive_cmds, mds->exclusive_cmds, cmds,
 539                   CXL_MEM_COMMAND_ID_MAX);
 540         up_write(&cxl_memdev_rwsem);
 541 }
 542 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL);
 543
 544 /**
 545  * clear_exclusive_cxl_commands() - atomically enable user cxl commands
 546  * @mds: The device state to modify
 547  * @cmds: bitmap of commands to mark available for userspace
 548  */
 549 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,
 550                                   unsigned long *cmds)
 551 {
 552         down_write(&cxl_memdev_rwsem);
 553         bitmap_andnot(mds->exclusive_cmds, mds->exclusive_cmds, cmds,
 554                       CXL_MEM_COMMAND_ID_MAX);
 555         up_write(&cxl_memdev_rwsem);
 556 }
 557 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL);
 558
 559 static void cxl_memdev_security_shutdown(struct device *dev)
 560 {
 561         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 562         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 563
 564         if (mds->security.poll)
 565                 cancel_delayed_work_sync(&mds->security.poll_dwork);
 566 }
 567
 568 static void cxl_memdev_shutdown(struct device *dev)
 569 {
 570         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 571
 572         down_write(&cxl_memdev_rwsem);
 573         cxl_memdev_security_shutdown(dev);
 574         cxlmd->cxlds = NULL;
 575         up_write(&cxl_memdev_rwsem);
 576 }
 577
 578 static void cxl_memdev_unregister(void *_cxlmd)
 579 {
 580         struct cxl_memdev *cxlmd = _cxlmd;
 581         struct device *dev = &cxlmd->dev;
 582
 583         cxl_memdev_shutdown(dev);
 584         cdev_device_del(&cxlmd->cdev, dev);
 585         put_device(dev);
 586 }
 587
 588 static void detach_memdev(struct work_struct *work)
 589 {
 590         struct cxl_memdev *cxlmd;
 591
 592         cxlmd = container_of(work, typeof(*cxlmd), detach_work);
 593         device_release_driver(&cxlmd->dev);
 594         put_device(&cxlmd->dev);
 595 }
 596
 597 static struct lock_class_key cxl_memdev_key;
 598
 599 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
 600                                            const struct file_operations *fops)
 601 {
 602         struct cxl_memdev *cxlmd;
 603         struct device *dev;
 604         struct cdev *cdev;
 605         int rc;
 606
 607         cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL);
 608         if (!cxlmd)
 609                 return ERR_PTR(-ENOMEM);
 610
 611         rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL);
 612         if (rc < 0)
 613                 goto err;
 614         cxlmd->id = rc;
 615         cxlmd->depth = -1;
 616
 617         dev = &cxlmd->dev;
 618         device_initialize(dev);
 619         lockdep_set_class(&dev->mutex, &cxl_memdev_key);
 620         dev->parent = cxlds->dev;
 621         dev->bus = &cxl_bus_type;
 622         dev->devt = MKDEV(cxl_mem_major, cxlmd->id);
 623         dev->type = &cxl_memdev_type;
 624         device_set_pm_not_required(dev);
 625         INIT_WORK(&cxlmd->detach_work, detach_memdev);
 626
 627         cdev = &cxlmd->cdev;
 628         cdev_init(cdev, fops);
 629         return cxlmd;
 630
 631 err:
 632         kfree(cxlmd);
 633         return ERR_PTR(rc);
 634 }
 635
 636 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
 637                                unsigned long arg)
 638 {
 639         switch (cmd) {
 640         case CXL_MEM_QUERY_COMMANDS:
 641                 return cxl_query_cmd(cxlmd, (void __user *)arg);
 642         case CXL_MEM_SEND_COMMAND:
 643                 return cxl_send_cmd(cxlmd, (void __user *)arg);
 644         default:
 645                 return -ENOTTY;
 646         }
 647 }
 648
 649 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,
 650                              unsigned long arg)
 651 {
 652         struct cxl_memdev *cxlmd = file->private_data;
 653         struct cxl_dev_state *cxlds;
 654         int rc = -ENXIO;
 655
 656         down_read(&cxl_memdev_rwsem);
 657         cxlds = cxlmd->cxlds;
 658         if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM)
 659                 rc = __cxl_memdev_ioctl(cxlmd, cmd, arg);
 660         up_read(&cxl_memdev_rwsem);
 661
 662         return rc;
 663 }
 664
 665 static int cxl_memdev_open(struct inode *inode, struct file *file)
 666 {
 667         struct cxl_memdev *cxlmd =
 668                 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
 669
 670         get_device(&cxlmd->dev);
 671         file->private_data = cxlmd;
 672
 673         return 0;
 674 }
 675
 676 static int cxl_memdev_release_file(struct inode *inode, struct file *file)
 677 {
 678         struct cxl_memdev *cxlmd =
 679                 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
 680
 681         put_device(&cxlmd->dev);
 682
 683         return 0;
 684 }
 685
 686 /**
 687  * cxl_mem_get_fw_info - Get Firmware info
 688  * @mds: The device data for the operation
 689  *
 690  * Retrieve firmware info for the device specified.
 691  *
 692  * Return: 0 if no error: or the result of the mailbox command.
 693  *
 694  * See CXL-3.0 8.2.9.3.1 Get FW Info
 695  */
 696 static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)
 697 {
 698         struct cxl_mbox_get_fw_info info;
 699         struct cxl_mbox_cmd mbox_cmd;
 700         int rc;
 701
 702         mbox_cmd = (struct cxl_mbox_cmd) {
 703                 .opcode = CXL_MBOX_OP_GET_FW_INFO,
 704                 .size_out = sizeof(info),
 705                 .payload_out = &info,
 706         };
 707
 708         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 709         if (rc < 0)
 710                 return rc;
 711
 712         mds->fw.num_slots = info.num_slots;
 713         mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK,
 714                                        info.slot_info);
 715
 716         return 0;
 717 }
 718
 719 /**
 720  * cxl_mem_activate_fw - Activate Firmware
 721  * @mds: The device data for the operation
 722  * @slot: slot number to activate
 723  *
 724  * Activate firmware in a given slot for the device specified.
 725  *
 726  * Return: 0 if no error: or the result of the mailbox command.
 727  *
 728  * See CXL-3.0 8.2.9.3.3 Activate FW
 729  */
 730 static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)
 731 {
 732         struct cxl_mbox_activate_fw activate;
 733         struct cxl_mbox_cmd mbox_cmd;
 734
 735         if (slot == 0 || slot > mds->fw.num_slots)
 736                 return -EINVAL;
 737
 738         mbox_cmd = (struct cxl_mbox_cmd) {
 739                 .opcode = CXL_MBOX_OP_ACTIVATE_FW,
 740                 .size_in = sizeof(activate),
 741                 .payload_in = &activate,
 742         };
 743
 744         /* Only offline activation supported for now */
 745         activate.action = CXL_FW_ACTIVATE_OFFLINE;
 746         activate.slot = slot;
 747
 748         return cxl_internal_send_cmd(mds, &mbox_cmd);
 749 }
 750
 751 /**
 752  * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer
 753  * @mds: The device data for the operation
 754  *
 755  * Abort an in-progress firmware transfer for the device specified.
 756  *
 757  * Return: 0 if no error: or the result of the mailbox command.
 758  *
 759  * See CXL-3.0 8.2.9.3.2 Transfer FW
 760  */
 761 static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds)
 762 {
 763         struct cxl_mbox_transfer_fw *transfer;
 764         struct cxl_mbox_cmd mbox_cmd;
 765         int rc;
 766
 767         transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL);
 768         if (!transfer)
 769                 return -ENOMEM;
 770
 771         /* Set a 1s poll interval and a total wait time of 30s */
 772         mbox_cmd = (struct cxl_mbox_cmd) {
 773                 .opcode = CXL_MBOX_OP_TRANSFER_FW,
 774                 .size_in = sizeof(*transfer),
 775                 .payload_in = transfer,
 776                 .poll_interval_ms = 1000,
 777                 .poll_count = 30,
 778         };
 779
 780         transfer->action = CXL_FW_TRANSFER_ACTION_ABORT;
 781
 782         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 783         kfree(transfer);
 784         return rc;
 785 }
 786
 787 static void cxl_fw_cleanup(struct fw_upload *fwl)
 788 {
 789         struct cxl_memdev_state *mds = fwl->dd_handle;
 790
 791         mds->fw.next_slot = 0;
 792 }
 793
 794 static int cxl_fw_do_cancel(struct fw_upload *fwl)
 795 {
 796         struct cxl_memdev_state *mds = fwl->dd_handle;
 797         struct cxl_dev_state *cxlds = &mds->cxlds;
 798         struct cxl_memdev *cxlmd = cxlds->cxlmd;
 799         int rc;
 800
 801         rc = cxl_mem_abort_fw_xfer(mds);
 802         if (rc < 0)
 803                 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc);
 804
 805         return FW_UPLOAD_ERR_CANCELED;
 806 }
 807
 808 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,
 809                                          u32 size)
 810 {
 811         struct cxl_memdev_state *mds = fwl->dd_handle;
 812         struct cxl_mbox_transfer_fw *transfer;
 813
 814         if (!size)
 815                 return FW_UPLOAD_ERR_INVALID_SIZE;
 816
 817         mds->fw.oneshot = struct_size(transfer, data, size) <
 818                             mds->payload_size;
 819
 820         if (cxl_mem_get_fw_info(mds))
 821                 return FW_UPLOAD_ERR_HW_ERROR;
 822
 823         /*
 824          * So far no state has been changed, hence no other cleanup is
 825          * necessary. Simply return the cancelled status.
 826          */
 827         if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
 828                 return FW_UPLOAD_ERR_CANCELED;
 829
 830         return FW_UPLOAD_ERR_NONE;
 831 }
 832
 833 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
 834                                        u32 offset, u32 size, u32 *written)
 835 {
 836         struct cxl_memdev_state *mds = fwl->dd_handle;
 837         struct cxl_dev_state *cxlds = &mds->cxlds;
 838         struct cxl_memdev *cxlmd = cxlds->cxlmd;
 839         struct cxl_mbox_transfer_fw *transfer;
 840         struct cxl_mbox_cmd mbox_cmd;
 841         u32 cur_size, remaining;
 842         size_t size_in;
 843         int rc;
 844
 845         *written = 0;
 846
 847         /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */
 848         if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) {
 849                 dev_err(&cxlmd->dev,
 850                         "misaligned offset for FW transfer slice (%u)\n",
 851                         offset);
 852                 return FW_UPLOAD_ERR_RW_ERROR;
 853         }
 854
 855         /*
 856          * Pick transfer size based on mds->payload_size @size must bw 128-byte
 857          * aligned, ->payload_size is a power of 2 starting at 256 bytes, and
 858          * sizeof(*transfer) is 128.  These constraints imply that @cur_size
 859          * will always be 128b aligned.
 860          */
 861         cur_size = min_t(size_t, size, mds->payload_size - sizeof(*transfer));
 862
 863         remaining = size - cur_size;
 864         size_in = struct_size(transfer, data, cur_size);
 865
 866         if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
 867                 return cxl_fw_do_cancel(fwl);
 868
 869         /*
 870          * Slot numbers are 1-indexed
 871          * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1')
 872          * Check for rollover using modulo, and 1-index it by adding 1
 873          */
 874         mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1;
 875
 876         /* Do the transfer via mailbox cmd */
 877         transfer = kzalloc(size_in, GFP_KERNEL);
 878         if (!transfer)
 879                 return FW_UPLOAD_ERR_RW_ERROR;
 880
 881         transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT);
 882         memcpy(transfer->data, data + offset, cur_size);
 883         if (mds->fw.oneshot) {
 884                 transfer->action = CXL_FW_TRANSFER_ACTION_FULL;
 885                 transfer->slot = mds->fw.next_slot;
 886         } else {
 887                 if (offset == 0) {
 888                         transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE;
 889                 } else if (remaining == 0) {
 890                         transfer->action = CXL_FW_TRANSFER_ACTION_END;
 891                         transfer->slot = mds->fw.next_slot;
 892                 } else {
 893                         transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE;
 894                 }
 895         }
 896
 897         mbox_cmd = (struct cxl_mbox_cmd) {
 898                 .opcode = CXL_MBOX_OP_TRANSFER_FW,
 899                 .size_in = size_in,
 900                 .payload_in = transfer,
 901                 .poll_interval_ms = 1000,
 902                 .poll_count = 30,
 903         };
 904
 905         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 906         if (rc < 0) {
 907                 rc = FW_UPLOAD_ERR_RW_ERROR;
 908                 goto out_free;
 909         }
 910
 911         *written = cur_size;
 912
 913         /* Activate FW if oneshot or if the last slice was written */
 914         if (mds->fw.oneshot || remaining == 0) {
 915                 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n",
 916                         mds->fw.next_slot);
 917                 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot);
 918                 if (rc < 0) {
 919                         dev_err(&cxlmd->dev, "Error activating firmware: %d\n",
 920                                 rc);
 921                         rc = FW_UPLOAD_ERR_HW_ERROR;
 922                         goto out_free;
 923                 }
 924         }
 925
 926         rc = FW_UPLOAD_ERR_NONE;
 927
 928 out_free:
 929         kfree(transfer);
 930         return rc;
 931 }
 932
 933 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl)
 934 {
 935         struct cxl_memdev_state *mds = fwl->dd_handle;
 936
 937         /*
 938          * cxl_internal_send_cmd() handles background operations synchronously.
 939          * No need to wait for completions here - any errors would've been
 940          * reported and handled during the ->write() call(s).
 941          * Just check if a cancel request was received, and return success.
 942          */
 943         if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
 944                 return cxl_fw_do_cancel(fwl);
 945
 946         return FW_UPLOAD_ERR_NONE;
 947 }
 948
 949 static void cxl_fw_cancel(struct fw_upload *fwl)
 950 {
 951         struct cxl_memdev_state *mds = fwl->dd_handle;
 952
 953         set_bit(CXL_FW_CANCEL, mds->fw.state);
 954 }
 955
 956 static const struct fw_upload_ops cxl_memdev_fw_ops = {
 957         .prepare = cxl_fw_prepare,
 958         .write = cxl_fw_write,
 959         .poll_complete = cxl_fw_poll_complete,
 960         .cancel = cxl_fw_cancel,
 961         .cleanup = cxl_fw_cleanup,
 962 };
 963
 964 static void devm_cxl_remove_fw_upload(void *fwl)
 965 {
 966         firmware_upload_unregister(fwl);
 967 }
 968
 969 int cxl_memdev_setup_fw_upload(struct cxl_memdev_state *mds)
 970 {
 971         struct cxl_dev_state *cxlds = &mds->cxlds;
 972         struct device *dev = &cxlds->cxlmd->dev;
 973         struct fw_upload *fwl;
 974         int rc;
 975
 976         if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, mds->enabled_cmds))
 977                 return 0;
 978
 979         fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev),
 980                                        &cxl_memdev_fw_ops, mds);
 981         if (IS_ERR(fwl))
 982                 return dev_err_probe(dev, PTR_ERR(fwl),
 983                                      "Failed to register firmware loader\n");
 984
 985         rc = devm_add_action_or_reset(cxlds->dev, devm_cxl_remove_fw_upload,
 986                                       fwl);
 987         if (rc)
 988                 dev_err(dev,
 989                         "Failed to add firmware loader remove action: %d\n",
 990                         rc);
 991
 992         return rc;
 993 }
 994 EXPORT_SYMBOL_NS_GPL(cxl_memdev_setup_fw_upload, CXL);
 995
 996 static const struct file_operations cxl_memdev_fops = {
 997         .owner = THIS_MODULE,
 998         .unlocked_ioctl = cxl_memdev_ioctl,
 999         .open = cxl_memdev_open,
1000         .release = cxl_memdev_release_file,
1001         .compat_ioctl = compat_ptr_ioctl,
1002         .llseek = noop_llseek,
1003 };
1004
1005 static void put_sanitize(void *data)
1006 {
1007         struct cxl_memdev_state *mds = data;
1008
1009         sysfs_put(mds->security.sanitize_node);
1010 }
1011
1012 static int cxl_memdev_security_init(struct cxl_memdev *cxlmd)
1013 {
1014         struct cxl_dev_state *cxlds = cxlmd->cxlds;
1015         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
1016         struct device *dev = &cxlmd->dev;
1017         struct kernfs_node *sec;
1018
1019         sec = sysfs_get_dirent(dev->kobj.sd, "security");
1020         if (!sec) {
1021                 dev_err(dev, "sysfs_get_dirent 'security' failed\n");
1022                 return -ENODEV;
1023         }
1024         mds->security.sanitize_node = sysfs_get_dirent(sec, "state");
1025         sysfs_put(sec);
1026         if (!mds->security.sanitize_node) {
1027                 dev_err(dev, "sysfs_get_dirent 'state' failed\n");
1028                 return -ENODEV;
1029         }
1030
1031         return devm_add_action_or_reset(cxlds->dev, put_sanitize, mds);
1032  }
1033
1034 struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds)
1035 {
1036         struct cxl_memdev *cxlmd;
1037         struct device *dev;
1038         struct cdev *cdev;
1039         int rc;
1040
1041         cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops);
1042         if (IS_ERR(cxlmd))
1043                 return cxlmd;
1044
1045         dev = &cxlmd->dev;
1046         rc = dev_set_name(dev, "mem%d", cxlmd->id);
1047         if (rc)
1048                 goto err;
1049
1050         /*
1051          * Activate ioctl operations, no cxl_memdev_rwsem manipulation
1052          * needed as this is ordered with cdev_add() publishing the device.
1053          */
1054         cxlmd->cxlds = cxlds;
1055         cxlds->cxlmd = cxlmd;
1056
1057         cdev = &cxlmd->cdev;
1058         rc = cdev_device_add(cdev, dev);
1059         if (rc)
1060                 goto err;
1061
1062         rc = cxl_memdev_security_init(cxlmd);
1063         if (rc)
1064                 goto err;
1065
1066         rc = devm_add_action_or_reset(cxlds->dev, cxl_memdev_unregister, cxlmd);
1067         if (rc)
1068                 return ERR_PTR(rc);
1069         return cxlmd;
1070
1071 err:
1072         /*
1073          * The cdev was briefly live, shutdown any ioctl operations that
1074          * saw that state.
1075          */
1076         cxl_memdev_shutdown(dev);
1077         put_device(dev);
1078         return ERR_PTR(rc);
1079 }
1080 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
1081
1082 __init int cxl_memdev_init(void)
1083 {
1084         dev_t devt;
1085         int rc;
1086
1087         rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl");
1088         if (rc)
1089                 return rc;
1090
1091         cxl_mem_major = MAJOR(devt);
1092
1093         return 0;
1094 }
1095
1096 void cxl_memdev_exit(void)
1097 {
1098         unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS);
1099 }