drivers/cxl/pmem.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2021 Intel Corporation. All rights reserved. */
   3 #include <linux/libnvdimm.h>
   4 #include <linux/unaligned.h>
   5 #include <linux/device.h>
   6 #include <linux/module.h>
   7 #include <linux/ndctl.h>
   8 #include <linux/async.h>
   9 #include <linux/slab.h>
  10 #include <linux/nd.h>
  11 #include "cxlmem.h"
  12 #include "cxl.h"
  13
  14 static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
  15
  16 static void clear_exclusive(void *mds)
  17 {
  18         clear_exclusive_cxl_commands(mds, exclusive_cmds);
  19 }
  20
  21 static void unregister_nvdimm(void *nvdimm)
  22 {
  23         nvdimm_delete(nvdimm);
  24 }
  25
  26 static ssize_t provider_show(struct device *dev, struct device_attribute *attr, char *buf)
  27 {
  28         struct nvdimm *nvdimm = to_nvdimm(dev);
  29         struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
  30
  31         return sysfs_emit(buf, "%s\n", dev_name(&cxl_nvd->dev));
  32 }
  33 static DEVICE_ATTR_RO(provider);
  34
  35 static ssize_t id_show(struct device *dev, struct device_attribute *attr, char *buf)
  36 {
  37         struct nvdimm *nvdimm = to_nvdimm(dev);
  38         struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
  39         struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds;
  40
  41         return sysfs_emit(buf, "%lld\n", cxlds->serial);
  42 }
  43 static DEVICE_ATTR_RO(id);
  44
  45 static struct attribute *cxl_dimm_attributes[] = {
  46         &dev_attr_id.attr,
  47         &dev_attr_provider.attr,
  48         NULL
  49 };
  50
  51 static const struct attribute_group cxl_dimm_attribute_group = {
  52         .name = "cxl",
  53         .attrs = cxl_dimm_attributes,
  54 };
  55
  56 static const struct attribute_group *cxl_dimm_attribute_groups[] = {
  57         &cxl_dimm_attribute_group,
  58         NULL
  59 };
  60
  61 static int cxl_nvdimm_probe(struct device *dev)
  62 {
  63         struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
  64         struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
  65         struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb;
  66         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
  67         unsigned long flags = 0, cmd_mask = 0;
  68         struct nvdimm *nvdimm;
  69         int rc;
  70
  71         set_exclusive_cxl_commands(mds, exclusive_cmds);
  72         rc = devm_add_action_or_reset(dev, clear_exclusive, mds);
  73         if (rc)
  74                 return rc;
  75
  76         set_bit(NDD_LABELING, &flags);
  77         set_bit(NDD_REGISTER_SYNC, &flags);
  78         set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
  79         set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
  80         set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
  81         nvdimm = __nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd,
  82                                  cxl_dimm_attribute_groups, flags,
  83                                  cmd_mask, 0, NULL, cxl_nvd->dev_id,
  84                                  cxl_security_ops, NULL);
  85         if (!nvdimm)
  86                 return -ENOMEM;
  87
  88         dev_set_drvdata(dev, nvdimm);
  89         return devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
  90 }
  91
  92 static struct cxl_driver cxl_nvdimm_driver = {
  93         .name = "cxl_nvdimm",
  94         .probe = cxl_nvdimm_probe,
  95         .id = CXL_DEVICE_NVDIMM,
  96         .drv = {
  97                 .suppress_bind_attrs = true,
  98         },
  99 };
 100
 101 static int cxl_pmem_get_config_size(struct cxl_memdev_state *mds,
 102                                     struct nd_cmd_get_config_size *cmd,
 103                                     unsigned int buf_len)
 104 {
 105         struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
 106
 107         if (sizeof(*cmd) > buf_len)
 108                 return -EINVAL;
 109
 110         *cmd = (struct nd_cmd_get_config_size){
 111                 .config_size = mds->lsa_size,
 112                 .max_xfer =
 113                         cxl_mbox->payload_size - sizeof(struct cxl_mbox_set_lsa),
 114         };
 115
 116         return 0;
 117 }
 118
 119 static int cxl_pmem_get_config_data(struct cxl_memdev_state *mds,
 120                                     struct nd_cmd_get_config_data_hdr *cmd,
 121                                     unsigned int buf_len)
 122 {
 123         struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
 124         struct cxl_mbox_get_lsa get_lsa;
 125         struct cxl_mbox_cmd mbox_cmd;
 126         int rc;
 127
 128         if (sizeof(*cmd) > buf_len)
 129                 return -EINVAL;
 130         if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
 131                 return -EINVAL;
 132
 133         get_lsa = (struct cxl_mbox_get_lsa) {
 134                 .offset = cpu_to_le32(cmd->in_offset),
 135                 .length = cpu_to_le32(cmd->in_length),
 136         };
 137         mbox_cmd = (struct cxl_mbox_cmd) {
 138                 .opcode = CXL_MBOX_OP_GET_LSA,
 139                 .payload_in = &get_lsa,
 140                 .size_in = sizeof(get_lsa),
 141                 .size_out = cmd->in_length,
 142                 .payload_out = cmd->out_buf,
 143         };
 144
 145         rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
 146         cmd->status = 0;
 147
 148         return rc;
 149 }
 150
 151 static int cxl_pmem_set_config_data(struct cxl_memdev_state *mds,
 152                                     struct nd_cmd_set_config_hdr *cmd,
 153                                     unsigned int buf_len)
 154 {
 155         struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
 156         struct cxl_mbox_set_lsa *set_lsa;
 157         struct cxl_mbox_cmd mbox_cmd;
 158         int rc;
 159
 160         if (sizeof(*cmd) > buf_len)
 161                 return -EINVAL;
 162
 163         /* 4-byte status follows the input data in the payload */
 164         if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len)
 165                 return -EINVAL;
 166
 167         set_lsa =
 168                 kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);
 169         if (!set_lsa)
 170                 return -ENOMEM;
 171
 172         *set_lsa = (struct cxl_mbox_set_lsa) {
 173                 .offset = cpu_to_le32(cmd->in_offset),
 174         };
 175         memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
 176         mbox_cmd = (struct cxl_mbox_cmd) {
 177                 .opcode = CXL_MBOX_OP_SET_LSA,
 178                 .payload_in = set_lsa,
 179                 .size_in = struct_size(set_lsa, data, cmd->in_length),
 180         };
 181
 182         rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
 183
 184         /*
 185          * Set "firmware" status (4-packed bytes at the end of the input
 186          * payload.
 187          */
 188         put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
 189         kvfree(set_lsa);
 190
 191         return rc;
 192 }
 193
 194 static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
 195                                void *buf, unsigned int buf_len)
 196 {
 197         struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
 198         unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
 199         struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
 200         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 201
 202         if (!test_bit(cmd, &cmd_mask))
 203                 return -ENOTTY;
 204
 205         switch (cmd) {
 206         case ND_CMD_GET_CONFIG_SIZE:
 207                 return cxl_pmem_get_config_size(mds, buf, buf_len);
 208         case ND_CMD_GET_CONFIG_DATA:
 209                 return cxl_pmem_get_config_data(mds, buf, buf_len);
 210         case ND_CMD_SET_CONFIG_DATA:
 211                 return cxl_pmem_set_config_data(mds, buf, buf_len);
 212         default:
 213                 return -ENOTTY;
 214         }
 215 }
 216
 217 static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
 218                         struct nvdimm *nvdimm, unsigned int cmd, void *buf,
 219                         unsigned int buf_len, int *cmd_rc)
 220 {
 221         /*
 222          * No firmware response to translate, let the transport error
 223          * code take precedence.
 224          */
 225         *cmd_rc = 0;
 226
 227         if (!nvdimm)
 228                 return -ENOTTY;
 229         return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
 230 }
 231
 232 static int detach_nvdimm(struct device *dev, void *data)
 233 {
 234         struct cxl_nvdimm *cxl_nvd;
 235         bool release = false;
 236
 237         if (!is_cxl_nvdimm(dev))
 238                 return 0;
 239
 240         scoped_guard(device, dev) {
 241                 if (dev->driver) {
 242                         cxl_nvd = to_cxl_nvdimm(dev);
 243                         if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data)
 244                                 release = true;
 245                 }
 246         }
 247         if (release)
 248                 device_release_driver(dev);
 249         return 0;
 250 }
 251
 252 static void unregister_nvdimm_bus(void *_cxl_nvb)
 253 {
 254         struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb;
 255         struct nvdimm_bus *nvdimm_bus = cxl_nvb->nvdimm_bus;
 256
 257         bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb, detach_nvdimm);
 258
 259         cxl_nvb->nvdimm_bus = NULL;
 260         nvdimm_bus_unregister(nvdimm_bus);
 261 }
 262
 263 static int cxl_nvdimm_bridge_probe(struct device *dev)
 264 {
 265         struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
 266
 267         cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {
 268                 .provider_name = "CXL",
 269                 .module = THIS_MODULE,
 270                 .ndctl = cxl_pmem_ctl,
 271         };
 272
 273         cxl_nvb->nvdimm_bus =
 274                 nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);
 275
 276         if (!cxl_nvb->nvdimm_bus)
 277                 return -ENOMEM;
 278
 279         return devm_add_action_or_reset(dev, unregister_nvdimm_bus, cxl_nvb);
 280 }
 281
 282 static struct cxl_driver cxl_nvdimm_bridge_driver = {
 283         .name = "cxl_nvdimm_bridge",
 284         .probe = cxl_nvdimm_bridge_probe,
 285         .id = CXL_DEVICE_NVDIMM_BRIDGE,
 286         .drv = {
 287                 .suppress_bind_attrs = true,
 288         },
 289 };
 290
 291 static void unregister_nvdimm_region(void *nd_region)
 292 {
 293         nvdimm_region_delete(nd_region);
 294 }
 295
 296 static void cxlr_pmem_remove_resource(void *res)
 297 {
 298         remove_resource(res);
 299 }
 300
 301 struct cxl_pmem_region_info {
 302         u64 offset;
 303         u64 serial;
 304 };
 305
 306 static int cxl_pmem_region_probe(struct device *dev)
 307 {
 308         struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE];
 309         struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
 310         struct cxl_region *cxlr = cxlr_pmem->cxlr;
 311         struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
 312         struct cxl_pmem_region_info *info = NULL;
 313         struct nd_interleave_set *nd_set;
 314         struct nd_region_desc ndr_desc;
 315         struct cxl_nvdimm *cxl_nvd;
 316         struct nvdimm *nvdimm;
 317         struct resource *res;
 318         int rc, i = 0;
 319
 320         memset(&mappings, 0, sizeof(mappings));
 321         memset(&ndr_desc, 0, sizeof(ndr_desc));
 322
 323         res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
 324         if (!res)
 325                 return -ENOMEM;
 326
 327         res->name = "Persistent Memory";
 328         res->start = cxlr_pmem->hpa_range.start;
 329         res->end = cxlr_pmem->hpa_range.end;
 330         res->flags = IORESOURCE_MEM;
 331         res->desc = IORES_DESC_PERSISTENT_MEMORY;
 332
 333         rc = insert_resource(&iomem_resource, res);
 334         if (rc)
 335                 return rc;
 336
 337         rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res);
 338         if (rc)
 339                 return rc;
 340
 341         ndr_desc.res = res;
 342         ndr_desc.provider_data = cxlr_pmem;
 343
 344         ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start);
 345         ndr_desc.target_node = phys_to_target_node(res->start);
 346         if (ndr_desc.target_node == NUMA_NO_NODE) {
 347                 ndr_desc.target_node = ndr_desc.numa_node;
 348                 dev_dbg(&cxlr->dev, "changing target node from %d to %d",
 349                         NUMA_NO_NODE, ndr_desc.target_node);
 350         }
 351
 352         nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
 353         if (!nd_set)
 354                 return -ENOMEM;
 355
 356         ndr_desc.memregion = cxlr->id;
 357         set_bit(ND_REGION_CXL, &ndr_desc.flags);
 358         set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags);
 359
 360         info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL);
 361         if (!info)
 362                 return -ENOMEM;
 363
 364         for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
 365                 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
 366                 struct cxl_memdev *cxlmd = m->cxlmd;
 367                 struct cxl_dev_state *cxlds = cxlmd->cxlds;
 368
 369                 cxl_nvd = cxlmd->cxl_nvd;
 370                 nvdimm = dev_get_drvdata(&cxl_nvd->dev);
 371                 if (!nvdimm) {
 372                         dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i,
 373                                 dev_name(&cxlmd->dev));
 374                         rc = -ENODEV;
 375                         goto out_nvd;
 376                 }
 377
 378                 m->cxl_nvd = cxl_nvd;
 379                 mappings[i] = (struct nd_mapping_desc) {
 380                         .nvdimm = nvdimm,
 381                         .start = m->start,
 382                         .size = m->size,
 383                         .position = i,
 384                 };
 385                 info[i].offset = m->start;
 386                 info[i].serial = cxlds->serial;
 387         }
 388         ndr_desc.num_mappings = cxlr_pmem->nr_mappings;
 389         ndr_desc.mapping = mappings;
 390
 391         /*
 392          * TODO enable CXL labels which skip the need for 'interleave-set cookie'
 393          */
 394         nd_set->cookie1 =
 395                 nd_fletcher64(info, sizeof(*info) * cxlr_pmem->nr_mappings, 0);
 396         nd_set->cookie2 = nd_set->cookie1;
 397         ndr_desc.nd_set = nd_set;
 398
 399         cxlr_pmem->nd_region =
 400                 nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc);
 401         if (!cxlr_pmem->nd_region) {
 402                 rc = -ENOMEM;
 403                 goto out_nvd;
 404         }
 405
 406         rc = devm_add_action_or_reset(dev, unregister_nvdimm_region,
 407                                       cxlr_pmem->nd_region);
 408 out_nvd:
 409         kfree(info);
 410
 411         return rc;
 412 }
 413
 414 static struct cxl_driver cxl_pmem_region_driver = {
 415         .name = "cxl_pmem_region",
 416         .probe = cxl_pmem_region_probe,
 417         .id = CXL_DEVICE_PMEM_REGION,
 418         .drv = {
 419                 .suppress_bind_attrs = true,
 420         },
 421 };
 422
 423 static __init int cxl_pmem_init(void)
 424 {
 425         int rc;
 426
 427         set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
 428         set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);
 429
 430         rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);
 431         if (rc)
 432                 return rc;
 433
 434         rc = cxl_driver_register(&cxl_nvdimm_driver);
 435         if (rc)
 436                 goto err_nvdimm;
 437
 438         rc = cxl_driver_register(&cxl_pmem_region_driver);
 439         if (rc)
 440                 goto err_region;
 441
 442         return 0;
 443
 444 err_region:
 445         cxl_driver_unregister(&cxl_nvdimm_driver);
 446 err_nvdimm:
 447         cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
 448         return rc;
 449 }
 450
 451 static __exit void cxl_pmem_exit(void)
 452 {
 453         cxl_driver_unregister(&cxl_pmem_region_driver);
 454         cxl_driver_unregister(&cxl_nvdimm_driver);
 455         cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
 456 }
 457
 458 MODULE_DESCRIPTION("CXL PMEM: Persistent Memory Support");
 459 MODULE_LICENSE("GPL v2");
 460 module_init(cxl_pmem_init);
 461 module_exit(cxl_pmem_exit);
 462 MODULE_IMPORT_NS("CXL");
 463 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
 464 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
 465 MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION);