drivers/cxl/core/memdev.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2020 Intel Corporation. */
   3
   4 #include <linux/io-64-nonatomic-lo-hi.h>
   5 #include <linux/device.h>
   6 #include <linux/slab.h>
   7 #include <linux/idr.h>
   8 #include <linux/pci.h>
   9 #include <cxlmem.h>
  10 #include "trace.h"
  11 #include "core.h"
  12
  13 static DECLARE_RWSEM(cxl_memdev_rwsem);
  14
  15 /*
  16  * An entire PCI topology full of devices should be enough for any
  17  * config
  18  */
  19 #define CXL_MEM_MAX_DEVS 65536
  20
  21 static int cxl_mem_major;
  22 static DEFINE_IDA(cxl_memdev_ida);
  23
  24 static void cxl_memdev_release(struct device *dev)
  25 {
  26         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  27
  28         ida_free(&cxl_memdev_ida, cxlmd->id);
  29         kfree(cxlmd);
  30 }
  31
  32 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid,
  33                                 kgid_t *gid)
  34 {
  35         return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev));
  36 }
  37
  38 static ssize_t firmware_version_show(struct device *dev,
  39                                      struct device_attribute *attr, char *buf)
  40 {
  41         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  42         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  43
  44         return sysfs_emit(buf, "%.16s\n", cxlds->firmware_version);
  45 }
  46 static DEVICE_ATTR_RO(firmware_version);
  47
  48 static ssize_t payload_max_show(struct device *dev,
  49                                 struct device_attribute *attr, char *buf)
  50 {
  51         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  52         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  53
  54         return sysfs_emit(buf, "%zu\n", cxlds->payload_size);
  55 }
  56 static DEVICE_ATTR_RO(payload_max);
  57
  58 static ssize_t label_storage_size_show(struct device *dev,
  59                                        struct device_attribute *attr, char *buf)
  60 {
  61         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  62         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  63
  64         return sysfs_emit(buf, "%zu\n", cxlds->lsa_size);
  65 }
  66 static DEVICE_ATTR_RO(label_storage_size);
  67
  68 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr,
  69                              char *buf)
  70 {
  71         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  72         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  73         unsigned long long len = resource_size(&cxlds->ram_res);
  74
  75         return sysfs_emit(buf, "%#llx\n", len);
  76 }
  77
  78 static struct device_attribute dev_attr_ram_size =
  79         __ATTR(size, 0444, ram_size_show, NULL);
  80
  81 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr,
  82                               char *buf)
  83 {
  84         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  85         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  86         unsigned long long len = resource_size(&cxlds->pmem_res);
  87
  88         return sysfs_emit(buf, "%#llx\n", len);
  89 }
  90
  91 static struct device_attribute dev_attr_pmem_size =
  92         __ATTR(size, 0444, pmem_size_show, NULL);
  93
  94 static ssize_t serial_show(struct device *dev, struct device_attribute *attr,
  95                            char *buf)
  96 {
  97         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  98         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  99
 100         return sysfs_emit(buf, "%#llx\n", cxlds->serial);
 101 }
 102 static DEVICE_ATTR_RO(serial);
 103
 104 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr,
 105                               char *buf)
 106 {
 107         return sprintf(buf, "%d\n", dev_to_node(dev));
 108 }
 109 static DEVICE_ATTR_RO(numa_node);
 110
 111 static ssize_t security_state_show(struct device *dev,
 112                                    struct device_attribute *attr,
 113                                    char *buf)
 114 {
 115         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 116         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 117         unsigned long state = cxlds->security.state;
 118         u64 reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET);
 119         u32 pct = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK, reg);
 120         u16 cmd = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK, reg);
 121
 122         if (cmd == CXL_MBOX_OP_SANITIZE && pct != 100)
 123                 return sysfs_emit(buf, "sanitize\n");
 124
 125         if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET))
 126                 return sysfs_emit(buf, "disabled\n");
 127         if (state & CXL_PMEM_SEC_STATE_FROZEN ||
 128             state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT ||
 129             state & CXL_PMEM_SEC_STATE_USER_PLIMIT)
 130                 return sysfs_emit(buf, "frozen\n");
 131         if (state & CXL_PMEM_SEC_STATE_LOCKED)
 132                 return sysfs_emit(buf, "locked\n");
 133         else
 134                 return sysfs_emit(buf, "unlocked\n");
 135 }
 136 static struct device_attribute dev_attr_security_state =
 137         __ATTR(state, 0444, security_state_show, NULL);
 138
 139 static ssize_t security_sanitize_store(struct device *dev,
 140                                        struct device_attribute *attr,
 141                                        const char *buf, size_t len)
 142 {
 143         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 144         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 145         struct cxl_port *port = dev_get_drvdata(&cxlmd->dev);
 146         ssize_t rc;
 147         bool sanitize;
 148
 149         if (kstrtobool(buf, &sanitize) || !sanitize)
 150                 return -EINVAL;
 151
 152         if (!port || !is_cxl_endpoint(port))
 153                 return -EINVAL;
 154
 155         /* ensure no regions are mapped to this memdev */
 156         if (port->commit_end != -1)
 157                 return -EBUSY;
 158
 159         rc = cxl_mem_sanitize(cxlds, CXL_MBOX_OP_SANITIZE);
 160
 161         return rc ? rc : len;
 162 }
 163 static struct device_attribute dev_attr_security_sanitize =
 164         __ATTR(sanitize, 0200, NULL, security_sanitize_store);
 165
 166 static ssize_t security_erase_store(struct device *dev,
 167                                     struct device_attribute *attr,
 168                                     const char *buf, size_t len)
 169 {
 170         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 171         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 172         struct cxl_port *port = dev_get_drvdata(&cxlmd->dev);
 173         ssize_t rc;
 174         bool erase;
 175
 176         if (kstrtobool(buf, &erase) || !erase)
 177                 return -EINVAL;
 178
 179         if (!port || !is_cxl_endpoint(port))
 180                 return -EINVAL;
 181
 182         /* ensure no regions are mapped to this memdev */
 183         if (port->commit_end != -1)
 184                 return -EBUSY;
 185
 186         rc = cxl_mem_sanitize(cxlds, CXL_MBOX_OP_SECURE_ERASE);
 187
 188         return rc ? rc : len;
 189 }
 190 static struct device_attribute dev_attr_security_erase =
 191         __ATTR(erase, 0200, NULL, security_erase_store);
 192
 193 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd)
 194 {
 195         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 196         u64 offset, length;
 197         int rc = 0;
 198
 199         /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */
 200         if (resource_size(&cxlds->pmem_res)) {
 201                 offset = cxlds->pmem_res.start;
 202                 length = resource_size(&cxlds->pmem_res);
 203                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
 204                 if (rc)
 205                         return rc;
 206         }
 207         if (resource_size(&cxlds->ram_res)) {
 208                 offset = cxlds->ram_res.start;
 209                 length = resource_size(&cxlds->ram_res);
 210                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
 211                 /*
 212                  * Invalid Physical Address is not an error for
 213                  * volatile addresses. Device support is optional.
 214                  */
 215                 if (rc == -EFAULT)
 216                         rc = 0;
 217         }
 218         return rc;
 219 }
 220
 221 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)
 222 {
 223         struct cxl_port *port;
 224         int rc;
 225
 226         port = dev_get_drvdata(&cxlmd->dev);
 227         if (!port || !is_cxl_endpoint(port))
 228                 return -EINVAL;
 229
 230         rc = down_read_interruptible(&cxl_dpa_rwsem);
 231         if (rc)
 232                 return rc;
 233
 234         if (port->commit_end == -1) {
 235                 /* No regions mapped to this memdev */
 236                 rc = cxl_get_poison_by_memdev(cxlmd);
 237         } else {
 238                 /* Regions mapped, collect poison by endpoint */
 239                 rc =  cxl_get_poison_by_endpoint(port);
 240         }
 241         up_read(&cxl_dpa_rwsem);
 242
 243         return rc;
 244 }
 245 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, CXL);
 246
 247 struct cxl_dpa_to_region_context {
 248         struct cxl_region *cxlr;
 249         u64 dpa;
 250 };
 251
 252 static int __cxl_dpa_to_region(struct device *dev, void *arg)
 253 {
 254         struct cxl_dpa_to_region_context *ctx = arg;
 255         struct cxl_endpoint_decoder *cxled;
 256         u64 dpa = ctx->dpa;
 257
 258         if (!is_endpoint_decoder(dev))
 259                 return 0;
 260
 261         cxled = to_cxl_endpoint_decoder(dev);
 262         if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
 263                 return 0;
 264
 265         if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start)
 266                 return 0;
 267
 268         dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa,
 269                 dev_name(&cxled->cxld.region->dev));
 270
 271         ctx->cxlr = cxled->cxld.region;
 272
 273         return 1;
 274 }
 275
 276 static struct cxl_region *cxl_dpa_to_region(struct cxl_memdev *cxlmd, u64 dpa)
 277 {
 278         struct cxl_dpa_to_region_context ctx;
 279         struct cxl_port *port;
 280
 281         ctx = (struct cxl_dpa_to_region_context) {
 282                 .dpa = dpa,
 283         };
 284         port = dev_get_drvdata(&cxlmd->dev);
 285         if (port && is_cxl_endpoint(port) && port->commit_end != -1)
 286                 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region);
 287
 288         return ctx.cxlr;
 289 }
 290
 291 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa)
 292 {
 293         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 294
 295         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 296                 return 0;
 297
 298         if (!resource_size(&cxlds->dpa_res)) {
 299                 dev_dbg(cxlds->dev, "device has no dpa resource\n");
 300                 return -EINVAL;
 301         }
 302         if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) {
 303                 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n",
 304                         dpa, &cxlds->dpa_res);
 305                 return -EINVAL;
 306         }
 307         if (!IS_ALIGNED(dpa, 64)) {
 308                 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa);
 309                 return -EINVAL;
 310         }
 311
 312         return 0;
 313 }
 314
 315 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
 316 {
 317         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 318         struct cxl_mbox_inject_poison inject;
 319         struct cxl_poison_record record;
 320         struct cxl_mbox_cmd mbox_cmd;
 321         struct cxl_region *cxlr;
 322         int rc;
 323
 324         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 325                 return 0;
 326
 327         rc = down_read_interruptible(&cxl_dpa_rwsem);
 328         if (rc)
 329                 return rc;
 330
 331         rc = cxl_validate_poison_dpa(cxlmd, dpa);
 332         if (rc)
 333                 goto out;
 334
 335         inject.address = cpu_to_le64(dpa);
 336         mbox_cmd = (struct cxl_mbox_cmd) {
 337                 .opcode = CXL_MBOX_OP_INJECT_POISON,
 338                 .size_in = sizeof(inject),
 339                 .payload_in = &inject,
 340         };
 341         rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
 342         if (rc)
 343                 goto out;
 344
 345         cxlr = cxl_dpa_to_region(cxlmd, dpa);
 346         if (cxlr)
 347                 dev_warn_once(cxlds->dev,
 348                               "poison inject dpa:%#llx region: %s\n", dpa,
 349                               dev_name(&cxlr->dev));
 350
 351         record = (struct cxl_poison_record) {
 352                 .address = cpu_to_le64(dpa),
 353                 .length = cpu_to_le32(1),
 354         };
 355         trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT);
 356 out:
 357         up_read(&cxl_dpa_rwsem);
 358
 359         return rc;
 360 }
 361 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL);
 362
 363 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
 364 {
 365         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 366         struct cxl_mbox_clear_poison clear;
 367         struct cxl_poison_record record;
 368         struct cxl_mbox_cmd mbox_cmd;
 369         struct cxl_region *cxlr;
 370         int rc;
 371
 372         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 373                 return 0;
 374
 375         rc = down_read_interruptible(&cxl_dpa_rwsem);
 376         if (rc)
 377                 return rc;
 378
 379         rc = cxl_validate_poison_dpa(cxlmd, dpa);
 380         if (rc)
 381                 goto out;
 382
 383         /*
 384          * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command
 385          * is defined to accept 64 bytes of write-data, along with the
 386          * address to clear. This driver uses zeroes as write-data.
 387          */
 388         clear = (struct cxl_mbox_clear_poison) {
 389                 .address = cpu_to_le64(dpa)
 390         };
 391
 392         mbox_cmd = (struct cxl_mbox_cmd) {
 393                 .opcode = CXL_MBOX_OP_CLEAR_POISON,
 394                 .size_in = sizeof(clear),
 395                 .payload_in = &clear,
 396         };
 397
 398         rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
 399         if (rc)
 400                 goto out;
 401
 402         cxlr = cxl_dpa_to_region(cxlmd, dpa);
 403         if (cxlr)
 404                 dev_warn_once(cxlds->dev, "poison clear dpa:%#llx region: %s\n",
 405                               dpa, dev_name(&cxlr->dev));
 406
 407         record = (struct cxl_poison_record) {
 408                 .address = cpu_to_le64(dpa),
 409                 .length = cpu_to_le32(1),
 410         };
 411         trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR);
 412 out:
 413         up_read(&cxl_dpa_rwsem);
 414
 415         return rc;
 416 }
 417 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, CXL);
 418
 419 static struct attribute *cxl_memdev_attributes[] = {
 420         &dev_attr_serial.attr,
 421         &dev_attr_firmware_version.attr,
 422         &dev_attr_payload_max.attr,
 423         &dev_attr_label_storage_size.attr,
 424         &dev_attr_numa_node.attr,
 425         NULL,
 426 };
 427
 428 static struct attribute *cxl_memdev_pmem_attributes[] = {
 429         &dev_attr_pmem_size.attr,
 430         NULL,
 431 };
 432
 433 static struct attribute *cxl_memdev_ram_attributes[] = {
 434         &dev_attr_ram_size.attr,
 435         NULL,
 436 };
 437
 438 static struct attribute *cxl_memdev_security_attributes[] = {
 439         &dev_attr_security_state.attr,
 440         &dev_attr_security_sanitize.attr,
 441         &dev_attr_security_erase.attr,
 442         NULL,
 443 };
 444
 445 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a,
 446                                   int n)
 447 {
 448         if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr)
 449                 return 0;
 450         return a->mode;
 451 }
 452
 453 static struct attribute_group cxl_memdev_attribute_group = {
 454         .attrs = cxl_memdev_attributes,
 455         .is_visible = cxl_memdev_visible,
 456 };
 457
 458 static struct attribute_group cxl_memdev_ram_attribute_group = {
 459         .name = "ram",
 460         .attrs = cxl_memdev_ram_attributes,
 461 };
 462
 463 static struct attribute_group cxl_memdev_pmem_attribute_group = {
 464         .name = "pmem",
 465         .attrs = cxl_memdev_pmem_attributes,
 466 };
 467
 468 static struct attribute_group cxl_memdev_security_attribute_group = {
 469         .name = "security",
 470         .attrs = cxl_memdev_security_attributes,
 471 };
 472
 473 static const struct attribute_group *cxl_memdev_attribute_groups[] = {
 474         &cxl_memdev_attribute_group,
 475         &cxl_memdev_ram_attribute_group,
 476         &cxl_memdev_pmem_attribute_group,
 477         &cxl_memdev_security_attribute_group,
 478         NULL,
 479 };
 480
 481 static const struct device_type cxl_memdev_type = {
 482         .name = "cxl_memdev",
 483         .release = cxl_memdev_release,
 484         .devnode = cxl_memdev_devnode,
 485         .groups = cxl_memdev_attribute_groups,
 486 };
 487
 488 bool is_cxl_memdev(const struct device *dev)
 489 {
 490         return dev->type == &cxl_memdev_type;
 491 }
 492 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL);
 493
 494 /**
 495  * set_exclusive_cxl_commands() - atomically disable user cxl commands
 496  * @cxlds: The device state to operate on
 497  * @cmds: bitmap of commands to mark exclusive
 498  *
 499  * Grab the cxl_memdev_rwsem in write mode to flush in-flight
 500  * invocations of the ioctl path and then disable future execution of
 501  * commands with the command ids set in @cmds.
 502  */
 503 void set_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds)
 504 {
 505         down_write(&cxl_memdev_rwsem);
 506         bitmap_or(cxlds->exclusive_cmds, cxlds->exclusive_cmds, cmds,
 507                   CXL_MEM_COMMAND_ID_MAX);
 508         up_write(&cxl_memdev_rwsem);
 509 }
 510 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL);
 511
 512 /**
 513  * clear_exclusive_cxl_commands() - atomically enable user cxl commands
 514  * @cxlds: The device state to modify
 515  * @cmds: bitmap of commands to mark available for userspace
 516  */
 517 void clear_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds)
 518 {
 519         down_write(&cxl_memdev_rwsem);
 520         bitmap_andnot(cxlds->exclusive_cmds, cxlds->exclusive_cmds, cmds,
 521                       CXL_MEM_COMMAND_ID_MAX);
 522         up_write(&cxl_memdev_rwsem);
 523 }
 524 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL);
 525
 526 static void cxl_memdev_security_shutdown(struct device *dev)
 527 {
 528         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 529         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 530
 531         if (cxlds->security.poll)
 532                 cancel_delayed_work_sync(&cxlds->security.poll_dwork);
 533 }
 534
 535 static void cxl_memdev_shutdown(struct device *dev)
 536 {
 537         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 538
 539         down_write(&cxl_memdev_rwsem);
 540         cxl_memdev_security_shutdown(dev);
 541         cxlmd->cxlds = NULL;
 542         up_write(&cxl_memdev_rwsem);
 543 }
 544
 545 static void cxl_memdev_unregister(void *_cxlmd)
 546 {
 547         struct cxl_memdev *cxlmd = _cxlmd;
 548         struct device *dev = &cxlmd->dev;
 549
 550         cxl_memdev_shutdown(dev);
 551         cdev_device_del(&cxlmd->cdev, dev);
 552         put_device(dev);
 553 }
 554
 555 static void detach_memdev(struct work_struct *work)
 556 {
 557         struct cxl_memdev *cxlmd;
 558
 559         cxlmd = container_of(work, typeof(*cxlmd), detach_work);
 560         device_release_driver(&cxlmd->dev);
 561         put_device(&cxlmd->dev);
 562 }
 563
 564 static struct lock_class_key cxl_memdev_key;
 565
 566 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
 567                                            const struct file_operations *fops)
 568 {
 569         struct cxl_memdev *cxlmd;
 570         struct device *dev;
 571         struct cdev *cdev;
 572         int rc;
 573
 574         cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL);
 575         if (!cxlmd)
 576                 return ERR_PTR(-ENOMEM);
 577
 578         rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL);
 579         if (rc < 0)
 580                 goto err;
 581         cxlmd->id = rc;
 582         cxlmd->depth = -1;
 583
 584         dev = &cxlmd->dev;
 585         device_initialize(dev);
 586         lockdep_set_class(&dev->mutex, &cxl_memdev_key);
 587         dev->parent = cxlds->dev;
 588         dev->bus = &cxl_bus_type;
 589         dev->devt = MKDEV(cxl_mem_major, cxlmd->id);
 590         dev->type = &cxl_memdev_type;
 591         device_set_pm_not_required(dev);
 592         INIT_WORK(&cxlmd->detach_work, detach_memdev);
 593
 594         cdev = &cxlmd->cdev;
 595         cdev_init(cdev, fops);
 596         return cxlmd;
 597
 598 err:
 599         kfree(cxlmd);
 600         return ERR_PTR(rc);
 601 }
 602
 603 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
 604                                unsigned long arg)
 605 {
 606         switch (cmd) {
 607         case CXL_MEM_QUERY_COMMANDS:
 608                 return cxl_query_cmd(cxlmd, (void __user *)arg);
 609         case CXL_MEM_SEND_COMMAND:
 610                 return cxl_send_cmd(cxlmd, (void __user *)arg);
 611         default:
 612                 return -ENOTTY;
 613         }
 614 }
 615
 616 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,
 617                              unsigned long arg)
 618 {
 619         struct cxl_memdev *cxlmd = file->private_data;
 620         int rc = -ENXIO;
 621
 622         down_read(&cxl_memdev_rwsem);
 623         if (cxlmd->cxlds)
 624                 rc = __cxl_memdev_ioctl(cxlmd, cmd, arg);
 625         up_read(&cxl_memdev_rwsem);
 626
 627         return rc;
 628 }
 629
 630 static int cxl_memdev_open(struct inode *inode, struct file *file)
 631 {
 632         struct cxl_memdev *cxlmd =
 633                 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
 634
 635         get_device(&cxlmd->dev);
 636         file->private_data = cxlmd;
 637
 638         return 0;
 639 }
 640
 641 static int cxl_memdev_release_file(struct inode *inode, struct file *file)
 642 {
 643         struct cxl_memdev *cxlmd =
 644                 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
 645
 646         put_device(&cxlmd->dev);
 647
 648         return 0;
 649 }
 650
 651 static const struct file_operations cxl_memdev_fops = {
 652         .owner = THIS_MODULE,
 653         .unlocked_ioctl = cxl_memdev_ioctl,
 654         .open = cxl_memdev_open,
 655         .release = cxl_memdev_release_file,
 656         .compat_ioctl = compat_ptr_ioctl,
 657         .llseek = noop_llseek,
 658 };
 659
 660 static void put_sanitize(void *data)
 661 {
 662         struct cxl_dev_state *cxlds = data;
 663
 664         sysfs_put(cxlds->security.sanitize_node);
 665 }
 666
 667 static int cxl_memdev_security_init(struct cxl_memdev *cxlmd)
 668 {
 669         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 670         struct device *dev = &cxlmd->dev;
 671         struct kernfs_node *sec;
 672
 673         sec = sysfs_get_dirent(dev->kobj.sd, "security");
 674         if (!sec) {
 675                 dev_err(dev, "sysfs_get_dirent 'security' failed\n");
 676                 return -ENODEV;
 677         }
 678         cxlds->security.sanitize_node = sysfs_get_dirent(sec, "state");
 679         sysfs_put(sec);
 680         if (!cxlds->security.sanitize_node) {
 681                 dev_err(dev, "sysfs_get_dirent 'state' failed\n");
 682                 return -ENODEV;
 683         }
 684
 685         return devm_add_action_or_reset(cxlds->dev, put_sanitize, cxlds);
 686  }
 687
 688 struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds)
 689 {
 690         struct cxl_memdev *cxlmd;
 691         struct device *dev;
 692         struct cdev *cdev;
 693         int rc;
 694
 695         cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops);
 696         if (IS_ERR(cxlmd))
 697                 return cxlmd;
 698
 699         dev = &cxlmd->dev;
 700         rc = dev_set_name(dev, "mem%d", cxlmd->id);
 701         if (rc)
 702                 goto err;
 703
 704         /*
 705          * Activate ioctl operations, no cxl_memdev_rwsem manipulation
 706          * needed as this is ordered with cdev_add() publishing the device.
 707          */
 708         cxlmd->cxlds = cxlds;
 709         cxlds->cxlmd = cxlmd;
 710
 711         cdev = &cxlmd->cdev;
 712         rc = cdev_device_add(cdev, dev);
 713         if (rc)
 714                 goto err;
 715
 716         rc = cxl_memdev_security_init(cxlmd);
 717         if (rc)
 718                 goto err;
 719
 720         rc = devm_add_action_or_reset(cxlds->dev, cxl_memdev_unregister, cxlmd);
 721         if (rc)
 722                 return ERR_PTR(rc);
 723         return cxlmd;
 724
 725 err:
 726         /*
 727          * The cdev was briefly live, shutdown any ioctl operations that
 728          * saw that state.
 729          */
 730         cxl_memdev_shutdown(dev);
 731         put_device(dev);
 732         return ERR_PTR(rc);
 733 }
 734 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
 735
 736 __init int cxl_memdev_init(void)
 737 {
 738         dev_t devt;
 739         int rc;
 740
 741         rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl");
 742         if (rc)
 743                 return rc;
 744
 745         cxl_mem_major = MAJOR(devt);
 746
 747         return 0;
 748 }
 749
 750 void cxl_memdev_exit(void)
 751 {
 752         unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS);
 753 }