drivers/nvme/host/core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVM Express device driver
   4  * Copyright (c) 2011-2014, Intel Corporation.
   5  */
   6
   7 #include <linux/blkdev.h>
   8 #include <linux/blk-mq.h>
   9 #include <linux/blk-integrity.h>
  10 #include <linux/compat.h>
  11 #include <linux/delay.h>
  12 #include <linux/errno.h>
  13 #include <linux/hdreg.h>
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/backing-dev.h>
  17 #include <linux/slab.h>
  18 #include <linux/types.h>
  19 #include <linux/pr.h>
  20 #include <linux/ptrace.h>
  21 #include <linux/nvme_ioctl.h>
  22 #include <linux/pm_qos.h>
  23 #include <linux/ratelimit.h>
  24 #include <asm/unaligned.h>
  25
  26 #include "nvme.h"
  27 #include "fabrics.h"
  28 #include <linux/nvme-auth.h>
  29
  30 #define CREATE_TRACE_POINTS
  31 #include "trace.h"
  32
  33 #define NVME_MINORS             (1U << MINORBITS)
  34
  35 struct nvme_ns_info {
  36         struct nvme_ns_ids ids;
  37         u32 nsid;
  38         __le32 anagrpid;
  39         u8 pi_offset;
  40         bool is_shared;
  41         bool is_readonly;
  42         bool is_ready;
  43         bool is_removed;
  44 };
  45
  46 unsigned int admin_timeout = 60;
  47 module_param(admin_timeout, uint, 0644);
  48 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
  49 EXPORT_SYMBOL_GPL(admin_timeout);
  50
  51 unsigned int nvme_io_timeout = 30;
  52 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
  53 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
  54 EXPORT_SYMBOL_GPL(nvme_io_timeout);
  55
  56 static unsigned char shutdown_timeout = 5;
  57 module_param(shutdown_timeout, byte, 0644);
  58 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
  59
  60 static u8 nvme_max_retries = 5;
  61 module_param_named(max_retries, nvme_max_retries, byte, 0644);
  62 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
  63
  64 static unsigned long default_ps_max_latency_us = 100000;
  65 module_param(default_ps_max_latency_us, ulong, 0644);
  66 MODULE_PARM_DESC(default_ps_max_latency_us,
  67                  "max power saving latency for new devices; use PM QOS to change per device");
  68
  69 static bool force_apst;
  70 module_param(force_apst, bool, 0644);
  71 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
  72
  73 static unsigned long apst_primary_timeout_ms = 100;
  74 module_param(apst_primary_timeout_ms, ulong, 0644);
  75 MODULE_PARM_DESC(apst_primary_timeout_ms,
  76         "primary APST timeout in ms");
  77
  78 static unsigned long apst_secondary_timeout_ms = 2000;
  79 module_param(apst_secondary_timeout_ms, ulong, 0644);
  80 MODULE_PARM_DESC(apst_secondary_timeout_ms,
  81         "secondary APST timeout in ms");
  82
  83 static unsigned long apst_primary_latency_tol_us = 15000;
  84 module_param(apst_primary_latency_tol_us, ulong, 0644);
  85 MODULE_PARM_DESC(apst_primary_latency_tol_us,
  86         "primary APST latency tolerance in us");
  87
  88 static unsigned long apst_secondary_latency_tol_us = 100000;
  89 module_param(apst_secondary_latency_tol_us, ulong, 0644);
  90 MODULE_PARM_DESC(apst_secondary_latency_tol_us,
  91         "secondary APST latency tolerance in us");
  92
  93 /*
  94  * nvme_wq - hosts nvme related works that are not reset or delete
  95  * nvme_reset_wq - hosts nvme reset works
  96  * nvme_delete_wq - hosts nvme delete works
  97  *
  98  * nvme_wq will host works such as scan, aen handling, fw activation,
  99  * keep-alive, periodic reconnects etc. nvme_reset_wq
 100  * runs reset works which also flush works hosted on nvme_wq for
 101  * serialization purposes. nvme_delete_wq host controller deletion
 102  * works which flush reset works for serialization.
 103  */
 104 struct workqueue_struct *nvme_wq;
 105 EXPORT_SYMBOL_GPL(nvme_wq);
 106
 107 struct workqueue_struct *nvme_reset_wq;
 108 EXPORT_SYMBOL_GPL(nvme_reset_wq);
 109
 110 struct workqueue_struct *nvme_delete_wq;
 111 EXPORT_SYMBOL_GPL(nvme_delete_wq);
 112
 113 static LIST_HEAD(nvme_subsystems);
 114 DEFINE_MUTEX(nvme_subsystems_lock);
 115
 116 static DEFINE_IDA(nvme_instance_ida);
 117 static dev_t nvme_ctrl_base_chr_devt;
 118 static int nvme_class_uevent(const struct device *dev, struct kobj_uevent_env *env);
 119 static const struct class nvme_class = {
 120         .name = "nvme",
 121         .dev_uevent = nvme_class_uevent,
 122 };
 123
 124 static const struct class nvme_subsys_class = {
 125         .name = "nvme-subsystem",
 126 };
 127
 128 static DEFINE_IDA(nvme_ns_chr_minor_ida);
 129 static dev_t nvme_ns_chr_devt;
 130 static const struct class nvme_ns_chr_class = {
 131         .name = "nvme-generic",
 132 };
 133
 134 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
 135 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
 136                                            unsigned nsid);
 137 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
 138                                    struct nvme_command *cmd);
 139
 140 void nvme_queue_scan(struct nvme_ctrl *ctrl)
 141 {
 142         /*
 143          * Only new queue scan work when admin and IO queues are both alive
 144          */
 145         if (nvme_ctrl_state(ctrl) == NVME_CTRL_LIVE && ctrl->tagset)
 146                 queue_work(nvme_wq, &ctrl->scan_work);
 147 }
 148
 149 /*
 150  * Use this function to proceed with scheduling reset_work for a controller
 151  * that had previously been set to the resetting state. This is intended for
 152  * code paths that can't be interrupted by other reset attempts. A hot removal
 153  * may prevent this from succeeding.
 154  */
 155 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
 156 {
 157         if (nvme_ctrl_state(ctrl) != NVME_CTRL_RESETTING)
 158                 return -EBUSY;
 159         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 160                 return -EBUSY;
 161         return 0;
 162 }
 163 EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
 164
 165 static void nvme_failfast_work(struct work_struct *work)
 166 {
 167         struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
 168                         struct nvme_ctrl, failfast_work);
 169
 170         if (nvme_ctrl_state(ctrl) != NVME_CTRL_CONNECTING)
 171                 return;
 172
 173         set_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
 174         dev_info(ctrl->device, "failfast expired\n");
 175         nvme_kick_requeue_lists(ctrl);
 176 }
 177
 178 static inline void nvme_start_failfast_work(struct nvme_ctrl *ctrl)
 179 {
 180         if (!ctrl->opts || ctrl->opts->fast_io_fail_tmo == -1)
 181                 return;
 182
 183         schedule_delayed_work(&ctrl->failfast_work,
 184                               ctrl->opts->fast_io_fail_tmo * HZ);
 185 }
 186
 187 static inline void nvme_stop_failfast_work(struct nvme_ctrl *ctrl)
 188 {
 189         if (!ctrl->opts)
 190                 return;
 191
 192         cancel_delayed_work_sync(&ctrl->failfast_work);
 193         clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
 194 }
 195
 196
 197 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
 198 {
 199         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 200                 return -EBUSY;
 201         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 202                 return -EBUSY;
 203         return 0;
 204 }
 205 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
 206
 207 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
 208 {
 209         int ret;
 210
 211         ret = nvme_reset_ctrl(ctrl);
 212         if (!ret) {
 213                 flush_work(&ctrl->reset_work);
 214                 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
 215                         ret = -ENETRESET;
 216         }
 217
 218         return ret;
 219 }
 220
 221 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
 222 {
 223         dev_info(ctrl->device,
 224                  "Removing ctrl: NQN \"%s\"\n", nvmf_ctrl_subsysnqn(ctrl));
 225
 226         flush_work(&ctrl->reset_work);
 227         nvme_stop_ctrl(ctrl);
 228         nvme_remove_namespaces(ctrl);
 229         ctrl->ops->delete_ctrl(ctrl);
 230         nvme_uninit_ctrl(ctrl);
 231 }
 232
 233 static void nvme_delete_ctrl_work(struct work_struct *work)
 234 {
 235         struct nvme_ctrl *ctrl =
 236                 container_of(work, struct nvme_ctrl, delete_work);
 237
 238         nvme_do_delete_ctrl(ctrl);
 239 }
 240
 241 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
 242 {
 243         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 244                 return -EBUSY;
 245         if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
 246                 return -EBUSY;
 247         return 0;
 248 }
 249 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
 250
 251 void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
 252 {
 253         /*
 254          * Keep a reference until nvme_do_delete_ctrl() complete,
 255          * since ->delete_ctrl can free the controller.
 256          */
 257         nvme_get_ctrl(ctrl);
 258         if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 259                 nvme_do_delete_ctrl(ctrl);
 260         nvme_put_ctrl(ctrl);
 261 }
 262
 263 static blk_status_t nvme_error_status(u16 status)
 264 {
 265         switch (status & NVME_SCT_SC_MASK) {
 266         case NVME_SC_SUCCESS:
 267                 return BLK_STS_OK;
 268         case NVME_SC_CAP_EXCEEDED:
 269                 return BLK_STS_NOSPC;
 270         case NVME_SC_LBA_RANGE:
 271         case NVME_SC_CMD_INTERRUPTED:
 272         case NVME_SC_NS_NOT_READY:
 273                 return BLK_STS_TARGET;
 274         case NVME_SC_BAD_ATTRIBUTES:
 275         case NVME_SC_ONCS_NOT_SUPPORTED:
 276         case NVME_SC_INVALID_OPCODE:
 277         case NVME_SC_INVALID_FIELD:
 278         case NVME_SC_INVALID_NS:
 279                 return BLK_STS_NOTSUPP;
 280         case NVME_SC_WRITE_FAULT:
 281         case NVME_SC_READ_ERROR:
 282         case NVME_SC_UNWRITTEN_BLOCK:
 283         case NVME_SC_ACCESS_DENIED:
 284         case NVME_SC_READ_ONLY:
 285         case NVME_SC_COMPARE_FAILED:
 286                 return BLK_STS_MEDIUM;
 287         case NVME_SC_GUARD_CHECK:
 288         case NVME_SC_APPTAG_CHECK:
 289         case NVME_SC_REFTAG_CHECK:
 290         case NVME_SC_INVALID_PI:
 291                 return BLK_STS_PROTECTION;
 292         case NVME_SC_RESERVATION_CONFLICT:
 293                 return BLK_STS_RESV_CONFLICT;
 294         case NVME_SC_HOST_PATH_ERROR:
 295                 return BLK_STS_TRANSPORT;
 296         case NVME_SC_ZONE_TOO_MANY_ACTIVE:
 297                 return BLK_STS_ZONE_ACTIVE_RESOURCE;
 298         case NVME_SC_ZONE_TOO_MANY_OPEN:
 299                 return BLK_STS_ZONE_OPEN_RESOURCE;
 300         default:
 301                 return BLK_STS_IOERR;
 302         }
 303 }
 304
 305 static void nvme_retry_req(struct request *req)
 306 {
 307         unsigned long delay = 0;
 308         u16 crd;
 309
 310         /* The mask and shift result must be <= 3 */
 311         crd = (nvme_req(req)->status & NVME_STATUS_CRD) >> 11;
 312         if (crd)
 313                 delay = nvme_req(req)->ctrl->crdt[crd - 1] * 100;
 314
 315         nvme_req(req)->retries++;
 316         blk_mq_requeue_request(req, false);
 317         blk_mq_delay_kick_requeue_list(req->q, delay);
 318 }
 319
 320 static void nvme_log_error(struct request *req)
 321 {
 322         struct nvme_ns *ns = req->q->queuedata;
 323         struct nvme_request *nr = nvme_req(req);
 324
 325         if (ns) {
 326                 pr_err_ratelimited("%s: %s(0x%x) @ LBA %llu, %u blocks, %s (sct 0x%x / sc 0x%x) %s%s\n",
 327                        ns->disk ? ns->disk->disk_name : "?",
 328                        nvme_get_opcode_str(nr->cmd->common.opcode),
 329                        nr->cmd->common.opcode,
 330                        nvme_sect_to_lba(ns->head, blk_rq_pos(req)),
 331                        blk_rq_bytes(req) >> ns->head->lba_shift,
 332                        nvme_get_error_status_str(nr->status),
 333                        NVME_SCT(nr->status),            /* Status Code Type */
 334                        nr->status & NVME_SC_MASK,       /* Status Code */
 335                        nr->status & NVME_STATUS_MORE ? "MORE " : "",
 336                        nr->status & NVME_STATUS_DNR  ? "DNR "  : "");
 337                 return;
 338         }
 339
 340         pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s\n",
 341                            dev_name(nr->ctrl->device),
 342                            nvme_get_admin_opcode_str(nr->cmd->common.opcode),
 343                            nr->cmd->common.opcode,
 344                            nvme_get_error_status_str(nr->status),
 345                            NVME_SCT(nr->status),        /* Status Code Type */
 346                            nr->status & NVME_SC_MASK,   /* Status Code */
 347                            nr->status & NVME_STATUS_MORE ? "MORE " : "",
 348                            nr->status & NVME_STATUS_DNR  ? "DNR "  : "");
 349 }
 350
 351 static void nvme_log_err_passthru(struct request *req)
 352 {
 353         struct nvme_ns *ns = req->q->queuedata;
 354         struct nvme_request *nr = nvme_req(req);
 355
 356         pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s"
 357                 "cdw10=0x%x cdw11=0x%x cdw12=0x%x cdw13=0x%x cdw14=0x%x cdw15=0x%x\n",
 358                 ns ? ns->disk->disk_name : dev_name(nr->ctrl->device),
 359                 ns ? nvme_get_opcode_str(nr->cmd->common.opcode) :
 360                      nvme_get_admin_opcode_str(nr->cmd->common.opcode),
 361                 nr->cmd->common.opcode,
 362                 nvme_get_error_status_str(nr->status),
 363                 NVME_SCT(nr->status),           /* Status Code Type */
 364                 nr->status & NVME_SC_MASK,      /* Status Code */
 365                 nr->status & NVME_STATUS_MORE ? "MORE " : "",
 366                 nr->status & NVME_STATUS_DNR  ? "DNR "  : "",
 367                 nr->cmd->common.cdw10,
 368                 nr->cmd->common.cdw11,
 369                 nr->cmd->common.cdw12,
 370                 nr->cmd->common.cdw13,
 371                 nr->cmd->common.cdw14,
 372                 nr->cmd->common.cdw14);
 373 }
 374
 375 enum nvme_disposition {
 376         COMPLETE,
 377         RETRY,
 378         FAILOVER,
 379         AUTHENTICATE,
 380 };
 381
 382 static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
 383 {
 384         if (likely(nvme_req(req)->status == 0))
 385                 return COMPLETE;
 386
 387         if (blk_noretry_request(req) ||
 388             (nvme_req(req)->status & NVME_STATUS_DNR) ||
 389             nvme_req(req)->retries >= nvme_max_retries)
 390                 return COMPLETE;
 391
 392         if ((nvme_req(req)->status & NVME_SCT_SC_MASK) == NVME_SC_AUTH_REQUIRED)
 393                 return AUTHENTICATE;
 394
 395         if (req->cmd_flags & REQ_NVME_MPATH) {
 396                 if (nvme_is_path_error(nvme_req(req)->status) ||
 397                     blk_queue_dying(req->q))
 398                         return FAILOVER;
 399         } else {
 400                 if (blk_queue_dying(req->q))
 401                         return COMPLETE;
 402         }
 403
 404         return RETRY;
 405 }
 406
 407 static inline void nvme_end_req_zoned(struct request *req)
 408 {
 409         if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
 410             req_op(req) == REQ_OP_ZONE_APPEND) {
 411                 struct nvme_ns *ns = req->q->queuedata;
 412
 413                 req->__sector = nvme_lba_to_sect(ns->head,
 414                         le64_to_cpu(nvme_req(req)->result.u64));
 415         }
 416 }
 417
 418 static inline void __nvme_end_req(struct request *req)
 419 {
 420         nvme_end_req_zoned(req);
 421         nvme_trace_bio_complete(req);
 422         if (req->cmd_flags & REQ_NVME_MPATH)
 423                 nvme_mpath_end_request(req);
 424 }
 425
 426 void nvme_end_req(struct request *req)
 427 {
 428         blk_status_t status = nvme_error_status(nvme_req(req)->status);
 429
 430         if (unlikely(nvme_req(req)->status && !(req->rq_flags & RQF_QUIET))) {
 431                 if (blk_rq_is_passthrough(req))
 432                         nvme_log_err_passthru(req);
 433                 else
 434                         nvme_log_error(req);
 435         }
 436         __nvme_end_req(req);
 437         blk_mq_end_request(req, status);
 438 }
 439
 440 void nvme_complete_rq(struct request *req)
 441 {
 442         struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
 443
 444         trace_nvme_complete_rq(req);
 445         nvme_cleanup_cmd(req);
 446
 447         /*
 448          * Completions of long-running commands should not be able to
 449          * defer sending of periodic keep alives, since the controller
 450          * may have completed processing such commands a long time ago
 451          * (arbitrarily close to command submission time).
 452          * req->deadline - req->timeout is the command submission time
 453          * in jiffies.
 454          */
 455         if (ctrl->kas &&
 456             req->deadline - req->timeout >= ctrl->ka_last_check_time)
 457                 ctrl->comp_seen = true;
 458
 459         switch (nvme_decide_disposition(req)) {
 460         case COMPLETE:
 461                 nvme_end_req(req);
 462                 return;
 463         case RETRY:
 464                 nvme_retry_req(req);
 465                 return;
 466         case FAILOVER:
 467                 nvme_failover_req(req);
 468                 return;
 469         case AUTHENTICATE:
 470 #ifdef CONFIG_NVME_HOST_AUTH
 471                 queue_work(nvme_wq, &ctrl->dhchap_auth_work);
 472                 nvme_retry_req(req);
 473 #else
 474                 nvme_end_req(req);
 475 #endif
 476                 return;
 477         }
 478 }
 479 EXPORT_SYMBOL_GPL(nvme_complete_rq);
 480
 481 void nvme_complete_batch_req(struct request *req)
 482 {
 483         trace_nvme_complete_rq(req);
 484         nvme_cleanup_cmd(req);
 485         __nvme_end_req(req);
 486 }
 487 EXPORT_SYMBOL_GPL(nvme_complete_batch_req);
 488
 489 /*
 490  * Called to unwind from ->queue_rq on a failed command submission so that the
 491  * multipathing code gets called to potentially failover to another path.
 492  * The caller needs to unwind all transport specific resource allocations and
 493  * must return propagate the return value.
 494  */
 495 blk_status_t nvme_host_path_error(struct request *req)
 496 {
 497         nvme_req(req)->status = NVME_SC_HOST_PATH_ERROR;
 498         blk_mq_set_request_complete(req);
 499         nvme_complete_rq(req);
 500         return BLK_STS_OK;
 501 }
 502 EXPORT_SYMBOL_GPL(nvme_host_path_error);
 503
 504 bool nvme_cancel_request(struct request *req, void *data)
 505 {
 506         dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
 507                                 "Cancelling I/O %d", req->tag);
 508
 509         /* don't abort one completed or idle request */
 510         if (blk_mq_rq_state(req) != MQ_RQ_IN_FLIGHT)
 511                 return true;
 512
 513         nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
 514         nvme_req(req)->flags |= NVME_REQ_CANCELLED;
 515         blk_mq_complete_request(req);
 516         return true;
 517 }
 518 EXPORT_SYMBOL_GPL(nvme_cancel_request);
 519
 520 void nvme_cancel_tagset(struct nvme_ctrl *ctrl)
 521 {
 522         if (ctrl->tagset) {
 523                 blk_mq_tagset_busy_iter(ctrl->tagset,
 524                                 nvme_cancel_request, ctrl);
 525                 blk_mq_tagset_wait_completed_request(ctrl->tagset);
 526         }
 527 }
 528 EXPORT_SYMBOL_GPL(nvme_cancel_tagset);
 529
 530 void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
 531 {
 532         if (ctrl->admin_tagset) {
 533                 blk_mq_tagset_busy_iter(ctrl->admin_tagset,
 534                                 nvme_cancel_request, ctrl);
 535                 blk_mq_tagset_wait_completed_request(ctrl->admin_tagset);
 536         }
 537 }
 538 EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
 539
 540 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 541                 enum nvme_ctrl_state new_state)
 542 {
 543         enum nvme_ctrl_state old_state;
 544         unsigned long flags;
 545         bool changed = false;
 546
 547         spin_lock_irqsave(&ctrl->lock, flags);
 548
 549         old_state = nvme_ctrl_state(ctrl);
 550         switch (new_state) {
 551         case NVME_CTRL_LIVE:
 552                 switch (old_state) {
 553                 case NVME_CTRL_NEW:
 554                 case NVME_CTRL_RESETTING:
 555                 case NVME_CTRL_CONNECTING:
 556                         changed = true;
 557                         fallthrough;
 558                 default:
 559                         break;
 560                 }
 561                 break;
 562         case NVME_CTRL_RESETTING:
 563                 switch (old_state) {
 564                 case NVME_CTRL_NEW:
 565                 case NVME_CTRL_LIVE:
 566                         changed = true;
 567                         fallthrough;
 568                 default:
 569                         break;
 570                 }
 571                 break;
 572         case NVME_CTRL_CONNECTING:
 573                 switch (old_state) {
 574                 case NVME_CTRL_NEW:
 575                 case NVME_CTRL_RESETTING:
 576                         changed = true;
 577                         fallthrough;
 578                 default:
 579                         break;
 580                 }
 581                 break;
 582         case NVME_CTRL_DELETING:
 583                 switch (old_state) {
 584                 case NVME_CTRL_LIVE:
 585                 case NVME_CTRL_RESETTING:
 586                 case NVME_CTRL_CONNECTING:
 587                         changed = true;
 588                         fallthrough;
 589                 default:
 590                         break;
 591                 }
 592                 break;
 593         case NVME_CTRL_DELETING_NOIO:
 594                 switch (old_state) {
 595                 case NVME_CTRL_DELETING:
 596                 case NVME_CTRL_DEAD:
 597                         changed = true;
 598                         fallthrough;
 599                 default:
 600                         break;
 601                 }
 602                 break;
 603         case NVME_CTRL_DEAD:
 604                 switch (old_state) {
 605                 case NVME_CTRL_DELETING:
 606                         changed = true;
 607                         fallthrough;
 608                 default:
 609                         break;
 610                 }
 611                 break;
 612         default:
 613                 break;
 614         }
 615
 616         if (changed) {
 617                 WRITE_ONCE(ctrl->state, new_state);
 618                 wake_up_all(&ctrl->state_wq);
 619         }
 620
 621         spin_unlock_irqrestore(&ctrl->lock, flags);
 622         if (!changed)
 623                 return false;
 624
 625         if (new_state == NVME_CTRL_LIVE) {
 626                 if (old_state == NVME_CTRL_CONNECTING)
 627                         nvme_stop_failfast_work(ctrl);
 628                 nvme_kick_requeue_lists(ctrl);
 629         } else if (new_state == NVME_CTRL_CONNECTING &&
 630                 old_state == NVME_CTRL_RESETTING) {
 631                 nvme_start_failfast_work(ctrl);
 632         }
 633         return changed;
 634 }
 635 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
 636
 637 /*
 638  * Waits for the controller state to be resetting, or returns false if it is
 639  * not possible to ever transition to that state.
 640  */
 641 bool nvme_wait_reset(struct nvme_ctrl *ctrl)
 642 {
 643         wait_event(ctrl->state_wq,
 644                    nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
 645                    nvme_state_terminal(ctrl));
 646         return nvme_ctrl_state(ctrl) == NVME_CTRL_RESETTING;
 647 }
 648 EXPORT_SYMBOL_GPL(nvme_wait_reset);
 649
 650 static void nvme_free_ns_head(struct kref *ref)
 651 {
 652         struct nvme_ns_head *head =
 653                 container_of(ref, struct nvme_ns_head, ref);
 654
 655         nvme_mpath_remove_disk(head);
 656         ida_free(&head->subsys->ns_ida, head->instance);
 657         cleanup_srcu_struct(&head->srcu);
 658         nvme_put_subsystem(head->subsys);
 659         kfree(head);
 660 }
 661
 662 bool nvme_tryget_ns_head(struct nvme_ns_head *head)
 663 {
 664         return kref_get_unless_zero(&head->ref);
 665 }
 666
 667 void nvme_put_ns_head(struct nvme_ns_head *head)
 668 {
 669         kref_put(&head->ref, nvme_free_ns_head);
 670 }
 671
 672 static void nvme_free_ns(struct kref *kref)
 673 {
 674         struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
 675
 676         put_disk(ns->disk);
 677         nvme_put_ns_head(ns->head);
 678         nvme_put_ctrl(ns->ctrl);
 679         kfree(ns);
 680 }
 681
 682 bool nvme_get_ns(struct nvme_ns *ns)
 683 {
 684         return kref_get_unless_zero(&ns->kref);
 685 }
 686
 687 void nvme_put_ns(struct nvme_ns *ns)
 688 {
 689         kref_put(&ns->kref, nvme_free_ns);
 690 }
 691 EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
 692
 693 static inline void nvme_clear_nvme_request(struct request *req)
 694 {
 695         nvme_req(req)->status = 0;
 696         nvme_req(req)->retries = 0;
 697         nvme_req(req)->flags = 0;
 698         req->rq_flags |= RQF_DONTPREP;
 699 }
 700
 701 /* initialize a passthrough request */
 702 void nvme_init_request(struct request *req, struct nvme_command *cmd)
 703 {
 704         struct nvme_request *nr = nvme_req(req);
 705         bool logging_enabled;
 706
 707         if (req->q->queuedata) {
 708                 struct nvme_ns *ns = req->q->disk->private_data;
 709
 710                 logging_enabled = ns->head->passthru_err_log_enabled;
 711                 req->timeout = NVME_IO_TIMEOUT;
 712         } else { /* no queuedata implies admin queue */
 713                 logging_enabled = nr->ctrl->passthru_err_log_enabled;
 714                 req->timeout = NVME_ADMIN_TIMEOUT;
 715         }
 716
 717         if (!logging_enabled)
 718                 req->rq_flags |= RQF_QUIET;
 719
 720         /* passthru commands should let the driver set the SGL flags */
 721         cmd->common.flags &= ~NVME_CMD_SGL_ALL;
 722
 723         req->cmd_flags |= REQ_FAILFAST_DRIVER;
 724         if (req->mq_hctx->type == HCTX_TYPE_POLL)
 725                 req->cmd_flags |= REQ_POLLED;
 726         nvme_clear_nvme_request(req);
 727         memcpy(nr->cmd, cmd, sizeof(*cmd));
 728 }
 729 EXPORT_SYMBOL_GPL(nvme_init_request);
 730
 731 /*
 732  * For something we're not in a state to send to the device the default action
 733  * is to busy it and retry it after the controller state is recovered.  However,
 734  * if the controller is deleting or if anything is marked for failfast or
 735  * nvme multipath it is immediately failed.
 736  *
 737  * Note: commands used to initialize the controller will be marked for failfast.
 738  * Note: nvme cli/ioctl commands are marked for failfast.
 739  */
 740 blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl,
 741                 struct request *rq)
 742 {
 743         enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
 744
 745         if (state != NVME_CTRL_DELETING_NOIO &&
 746             state != NVME_CTRL_DELETING &&
 747             state != NVME_CTRL_DEAD &&
 748             !test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags) &&
 749             !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH))
 750                 return BLK_STS_RESOURCE;
 751         return nvme_host_path_error(rq);
 752 }
 753 EXPORT_SYMBOL_GPL(nvme_fail_nonready_command);
 754
 755 bool __nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq,
 756                 bool queue_live, enum nvme_ctrl_state state)
 757 {
 758         struct nvme_request *req = nvme_req(rq);
 759
 760         /*
 761          * currently we have a problem sending passthru commands
 762          * on the admin_q if the controller is not LIVE because we can't
 763          * make sure that they are going out after the admin connect,
 764          * controller enable and/or other commands in the initialization
 765          * sequence. until the controller will be LIVE, fail with
 766          * BLK_STS_RESOURCE so that they will be rescheduled.
 767          */
 768         if (rq->q == ctrl->admin_q && (req->flags & NVME_REQ_USERCMD))
 769                 return false;
 770
 771         if (ctrl->ops->flags & NVME_F_FABRICS) {
 772                 /*
 773                  * Only allow commands on a live queue, except for the connect
 774                  * command, which is require to set the queue live in the
 775                  * appropinquate states.
 776                  */
 777                 switch (state) {
 778                 case NVME_CTRL_CONNECTING:
 779                         if (blk_rq_is_passthrough(rq) && nvme_is_fabrics(req->cmd) &&
 780                             (req->cmd->fabrics.fctype == nvme_fabrics_type_connect ||
 781                              req->cmd->fabrics.fctype == nvme_fabrics_type_auth_send ||
 782                              req->cmd->fabrics.fctype == nvme_fabrics_type_auth_receive))
 783                                 return true;
 784                         break;
 785                 default:
 786                         break;
 787                 case NVME_CTRL_DEAD:
 788                         return false;
 789                 }
 790         }
 791
 792         return queue_live;
 793 }
 794 EXPORT_SYMBOL_GPL(__nvme_check_ready);
 795
 796 static inline void nvme_setup_flush(struct nvme_ns *ns,
 797                 struct nvme_command *cmnd)
 798 {
 799         memset(cmnd, 0, sizeof(*cmnd));
 800         cmnd->common.opcode = nvme_cmd_flush;
 801         cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
 802 }
 803
 804 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
 805                 struct nvme_command *cmnd)
 806 {
 807         unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
 808         struct nvme_dsm_range *range;
 809         struct bio *bio;
 810
 811         /*
 812          * Some devices do not consider the DSM 'Number of Ranges' field when
 813          * determining how much data to DMA. Always allocate memory for maximum
 814          * number of segments to prevent device reading beyond end of buffer.
 815          */
 816         static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
 817
 818         range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
 819         if (!range) {
 820                 /*
 821                  * If we fail allocation our range, fallback to the controller
 822                  * discard page. If that's also busy, it's safe to return
 823                  * busy, as we know we can make progress once that's freed.
 824                  */
 825                 if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
 826                         return BLK_STS_RESOURCE;
 827
 828                 range = page_address(ns->ctrl->discard_page);
 829         }
 830
 831         if (queue_max_discard_segments(req->q) == 1) {
 832                 u64 slba = nvme_sect_to_lba(ns->head, blk_rq_pos(req));
 833                 u32 nlb = blk_rq_sectors(req) >> (ns->head->lba_shift - 9);
 834
 835                 range[0].cattr = cpu_to_le32(0);
 836                 range[0].nlb = cpu_to_le32(nlb);
 837                 range[0].slba = cpu_to_le64(slba);
 838                 n = 1;
 839         } else {
 840                 __rq_for_each_bio(bio, req) {
 841                         u64 slba = nvme_sect_to_lba(ns->head,
 842                                                     bio->bi_iter.bi_sector);
 843                         u32 nlb = bio->bi_iter.bi_size >> ns->head->lba_shift;
 844
 845                         if (n < segments) {
 846                                 range[n].cattr = cpu_to_le32(0);
 847                                 range[n].nlb = cpu_to_le32(nlb);
 848                                 range[n].slba = cpu_to_le64(slba);
 849                         }
 850                         n++;
 851                 }
 852         }
 853
 854         if (WARN_ON_ONCE(n != segments)) {
 855                 if (virt_to_page(range) == ns->ctrl->discard_page)
 856                         clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
 857                 else
 858                         kfree(range);
 859                 return BLK_STS_IOERR;
 860         }
 861
 862         memset(cmnd, 0, sizeof(*cmnd));
 863         cmnd->dsm.opcode = nvme_cmd_dsm;
 864         cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
 865         cmnd->dsm.nr = cpu_to_le32(segments - 1);
 866         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
 867
 868         bvec_set_virt(&req->special_vec, range, alloc_size);
 869         req->rq_flags |= RQF_SPECIAL_PAYLOAD;
 870
 871         return BLK_STS_OK;
 872 }
 873
 874 static void nvme_set_ref_tag(struct nvme_ns *ns, struct nvme_command *cmnd,
 875                               struct request *req)
 876 {
 877         u32 upper, lower;
 878         u64 ref48;
 879
 880         /* both rw and write zeroes share the same reftag format */
 881         switch (ns->head->guard_type) {
 882         case NVME_NVM_NS_16B_GUARD:
 883                 cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
 884                 break;
 885         case NVME_NVM_NS_64B_GUARD:
 886                 ref48 = ext_pi_ref_tag(req);
 887                 lower = lower_32_bits(ref48);
 888                 upper = upper_32_bits(ref48);
 889
 890                 cmnd->rw.reftag = cpu_to_le32(lower);
 891                 cmnd->rw.cdw3 = cpu_to_le32(upper);
 892                 break;
 893         default:
 894                 break;
 895         }
 896 }
 897
 898 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
 899                 struct request *req, struct nvme_command *cmnd)
 900 {
 901         memset(cmnd, 0, sizeof(*cmnd));
 902
 903         if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
 904                 return nvme_setup_discard(ns, req, cmnd);
 905
 906         cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
 907         cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
 908         cmnd->write_zeroes.slba =
 909                 cpu_to_le64(nvme_sect_to_lba(ns->head, blk_rq_pos(req)));
 910         cmnd->write_zeroes.length =
 911                 cpu_to_le16((blk_rq_bytes(req) >> ns->head->lba_shift) - 1);
 912
 913         if (!(req->cmd_flags & REQ_NOUNMAP) &&
 914             (ns->head->features & NVME_NS_DEAC))
 915                 cmnd->write_zeroes.control |= cpu_to_le16(NVME_WZ_DEAC);
 916
 917         if (nvme_ns_has_pi(ns->head)) {
 918                 cmnd->write_zeroes.control |= cpu_to_le16(NVME_RW_PRINFO_PRACT);
 919
 920                 switch (ns->head->pi_type) {
 921                 case NVME_NS_DPS_PI_TYPE1:
 922                 case NVME_NS_DPS_PI_TYPE2:
 923                         nvme_set_ref_tag(ns, cmnd, req);
 924                         break;
 925                 }
 926         }
 927
 928         return BLK_STS_OK;
 929 }
 930
 931 /*
 932  * NVMe does not support a dedicated command to issue an atomic write. A write
 933  * which does adhere to the device atomic limits will silently be executed
 934  * non-atomically. The request issuer should ensure that the write is within
 935  * the queue atomic writes limits, but just validate this in case it is not.
 936  */
 937 static bool nvme_valid_atomic_write(struct request *req)
 938 {
 939         struct request_queue *q = req->q;
 940         u32 boundary_bytes = queue_atomic_write_boundary_bytes(q);
 941
 942         if (blk_rq_bytes(req) > queue_atomic_write_unit_max_bytes(q))
 943                 return false;
 944
 945         if (boundary_bytes) {
 946                 u64 mask = boundary_bytes - 1, imask = ~mask;
 947                 u64 start = blk_rq_pos(req) << SECTOR_SHIFT;
 948                 u64 end = start + blk_rq_bytes(req) - 1;
 949
 950                 /* If greater then must be crossing a boundary */
 951                 if (blk_rq_bytes(req) > boundary_bytes)
 952                         return false;
 953
 954                 if ((start & imask) != (end & imask))
 955                         return false;
 956         }
 957
 958         return true;
 959 }
 960
 961 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
 962                 struct request *req, struct nvme_command *cmnd,
 963                 enum nvme_opcode op)
 964 {
 965         u16 control = 0;
 966         u32 dsmgmt = 0;
 967
 968         if (req->cmd_flags & REQ_FUA)
 969                 control |= NVME_RW_FUA;
 970         if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
 971                 control |= NVME_RW_LR;
 972
 973         if (req->cmd_flags & REQ_RAHEAD)
 974                 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
 975
 976         if (req->cmd_flags & REQ_ATOMIC && !nvme_valid_atomic_write(req))
 977                 return BLK_STS_INVAL;
 978
 979         cmnd->rw.opcode = op;
 980         cmnd->rw.flags = 0;
 981         cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
 982         cmnd->rw.cdw2 = 0;
 983         cmnd->rw.cdw3 = 0;
 984         cmnd->rw.metadata = 0;
 985         cmnd->rw.slba =
 986                 cpu_to_le64(nvme_sect_to_lba(ns->head, blk_rq_pos(req)));
 987         cmnd->rw.length =
 988                 cpu_to_le16((blk_rq_bytes(req) >> ns->head->lba_shift) - 1);
 989         cmnd->rw.reftag = 0;
 990         cmnd->rw.apptag = 0;
 991         cmnd->rw.appmask = 0;
 992
 993         if (ns->head->ms) {
 994                 /*
 995                  * If formated with metadata, the block layer always provides a
 996                  * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled.  Else
 997                  * we enable the PRACT bit for protection information or set the
 998                  * namespace capacity to zero to prevent any I/O.
 999                  */
1000                 if (!blk_integrity_rq(req)) {
1001                         if (WARN_ON_ONCE(!nvme_ns_has_pi(ns->head)))
1002                                 return BLK_STS_NOTSUPP;
1003                         control |= NVME_RW_PRINFO_PRACT;
1004                 }
1005
1006                 switch (ns->head->pi_type) {
1007                 case NVME_NS_DPS_PI_TYPE3:
1008                         control |= NVME_RW_PRINFO_PRCHK_GUARD;
1009                         break;
1010                 case NVME_NS_DPS_PI_TYPE1:
1011                 case NVME_NS_DPS_PI_TYPE2:
1012                         control |= NVME_RW_PRINFO_PRCHK_GUARD |
1013                                         NVME_RW_PRINFO_PRCHK_REF;
1014                         if (op == nvme_cmd_zone_append)
1015                                 control |= NVME_RW_APPEND_PIREMAP;
1016                         nvme_set_ref_tag(ns, cmnd, req);
1017                         break;
1018                 }
1019         }
1020
1021         cmnd->rw.control = cpu_to_le16(control);
1022         cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
1023         return 0;
1024 }
1025
1026 void nvme_cleanup_cmd(struct request *req)
1027 {
1028         if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
1029                 struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
1030
1031                 if (req->special_vec.bv_page == ctrl->discard_page)
1032                         clear_bit_unlock(0, &ctrl->discard_page_busy);
1033                 else
1034                         kfree(bvec_virt(&req->special_vec));
1035                 req->rq_flags &= ~RQF_SPECIAL_PAYLOAD;
1036         }
1037 }
1038 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
1039
1040 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req)
1041 {
1042         struct nvme_command *cmd = nvme_req(req)->cmd;
1043         blk_status_t ret = BLK_STS_OK;
1044
1045         if (!(req->rq_flags & RQF_DONTPREP))
1046                 nvme_clear_nvme_request(req);
1047
1048         switch (req_op(req)) {
1049         case REQ_OP_DRV_IN:
1050         case REQ_OP_DRV_OUT:
1051                 /* these are setup prior to execution in nvme_init_request() */
1052                 break;
1053         case REQ_OP_FLUSH:
1054                 nvme_setup_flush(ns, cmd);
1055                 break;
1056         case REQ_OP_ZONE_RESET_ALL:
1057         case REQ_OP_ZONE_RESET:
1058                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
1059                 break;
1060         case REQ_OP_ZONE_OPEN:
1061                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
1062                 break;
1063         case REQ_OP_ZONE_CLOSE:
1064                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
1065                 break;
1066         case REQ_OP_ZONE_FINISH:
1067                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
1068                 break;
1069         case REQ_OP_WRITE_ZEROES:
1070                 ret = nvme_setup_write_zeroes(ns, req, cmd);
1071                 break;
1072         case REQ_OP_DISCARD:
1073                 ret = nvme_setup_discard(ns, req, cmd);
1074                 break;
1075         case REQ_OP_READ:
1076                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
1077                 break;
1078         case REQ_OP_WRITE:
1079                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
1080                 break;
1081         case REQ_OP_ZONE_APPEND:
1082                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
1083                 break;
1084         default:
1085                 WARN_ON_ONCE(1);
1086                 return BLK_STS_IOERR;
1087         }
1088
1089         cmd->common.command_id = nvme_cid(req);
1090         trace_nvme_setup_cmd(req, cmd);
1091         return ret;
1092 }
1093 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
1094
1095 /*
1096  * Return values:
1097  * 0:  success
1098  * >0: nvme controller's cqe status response
1099  * <0: kernel error in lieu of controller response
1100  */
1101 int nvme_execute_rq(struct request *rq, bool at_head)
1102 {
1103         blk_status_t status;
1104
1105         status = blk_execute_rq(rq, at_head);
1106         if (nvme_req(rq)->flags & NVME_REQ_CANCELLED)
1107                 return -EINTR;
1108         if (nvme_req(rq)->status)
1109                 return nvme_req(rq)->status;
1110         return blk_status_to_errno(status);
1111 }
1112 EXPORT_SYMBOL_NS_GPL(nvme_execute_rq, NVME_TARGET_PASSTHRU);
1113
1114 /*
1115  * Returns 0 on success.  If the result is negative, it's a Linux error code;
1116  * if the result is positive, it's an NVM Express status code
1117  */
1118 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1119                 union nvme_result *result, void *buffer, unsigned bufflen,
1120                 int qid, nvme_submit_flags_t flags)
1121 {
1122         struct request *req;
1123         int ret;
1124         blk_mq_req_flags_t blk_flags = 0;
1125
1126         if (flags & NVME_SUBMIT_NOWAIT)
1127                 blk_flags |= BLK_MQ_REQ_NOWAIT;
1128         if (flags & NVME_SUBMIT_RESERVED)
1129                 blk_flags |= BLK_MQ_REQ_RESERVED;
1130         if (qid == NVME_QID_ANY)
1131                 req = blk_mq_alloc_request(q, nvme_req_op(cmd), blk_flags);
1132         else
1133                 req = blk_mq_alloc_request_hctx(q, nvme_req_op(cmd), blk_flags,
1134                                                 qid - 1);
1135
1136         if (IS_ERR(req))
1137                 return PTR_ERR(req);
1138         nvme_init_request(req, cmd);
1139         if (flags & NVME_SUBMIT_RETRY)
1140                 req->cmd_flags &= ~REQ_FAILFAST_DRIVER;
1141
1142         if (buffer && bufflen) {
1143                 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
1144                 if (ret)
1145                         goto out;
1146         }
1147
1148         ret = nvme_execute_rq(req, flags & NVME_SUBMIT_AT_HEAD);
1149         if (result && ret >= 0)
1150                 *result = nvme_req(req)->result;
1151  out:
1152         blk_mq_free_request(req);
1153         return ret;
1154 }
1155 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
1156
1157 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1158                 void *buffer, unsigned bufflen)
1159 {
1160         return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen,
1161                         NVME_QID_ANY, 0);
1162 }
1163 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
1164
1165 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
1166 {
1167         u32 effects = 0;
1168
1169         if (ns) {
1170                 effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
1171                 if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
1172                         dev_warn_once(ctrl->device,
1173                                 "IO command:%02x has unusual effects:%08x\n",
1174                                 opcode, effects);
1175
1176                 /*
1177                  * NVME_CMD_EFFECTS_CSE_MASK causes a freeze all I/O queues,
1178                  * which would deadlock when done on an I/O command.  Note that
1179                  * We already warn about an unusual effect above.
1180                  */
1181                 effects &= ~NVME_CMD_EFFECTS_CSE_MASK;
1182         } else {
1183                 effects = le32_to_cpu(ctrl->effects->acs[opcode]);
1184
1185                 /* Ignore execution restrictions if any relaxation bits are set */
1186                 if (effects & NVME_CMD_EFFECTS_CSER_MASK)
1187                         effects &= ~NVME_CMD_EFFECTS_CSE_MASK;
1188         }
1189
1190         return effects;
1191 }
1192 EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
1193
1194 u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
1195 {
1196         u32 effects = nvme_command_effects(ctrl, ns, opcode);
1197
1198         /*
1199          * For simplicity, IO to all namespaces is quiesced even if the command
1200          * effects say only one namespace is affected.
1201          */
1202         if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1203                 mutex_lock(&ctrl->scan_lock);
1204                 mutex_lock(&ctrl->subsys->lock);
1205                 nvme_mpath_start_freeze(ctrl->subsys);
1206                 nvme_mpath_wait_freeze(ctrl->subsys);
1207                 nvme_start_freeze(ctrl);
1208                 nvme_wait_freeze(ctrl);
1209         }
1210         return effects;
1211 }
1212 EXPORT_SYMBOL_NS_GPL(nvme_passthru_start, NVME_TARGET_PASSTHRU);
1213
1214 void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects,
1215                        struct nvme_command *cmd, int status)
1216 {
1217         if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1218                 nvme_unfreeze(ctrl);
1219                 nvme_mpath_unfreeze(ctrl->subsys);
1220                 mutex_unlock(&ctrl->subsys->lock);
1221                 mutex_unlock(&ctrl->scan_lock);
1222         }
1223         if (effects & NVME_CMD_EFFECTS_CCC) {
1224                 if (!test_and_set_bit(NVME_CTRL_DIRTY_CAPABILITY,
1225                                       &ctrl->flags)) {
1226                         dev_info(ctrl->device,
1227 "controller capabilities changed, reset may be required to take effect.\n");
1228                 }
1229         }
1230         if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {
1231                 nvme_queue_scan(ctrl);
1232                 flush_work(&ctrl->scan_work);
1233         }
1234         if (ns)
1235                 return;
1236
1237         switch (cmd->common.opcode) {
1238         case nvme_admin_set_features:
1239                 switch (le32_to_cpu(cmd->common.cdw10) & 0xFF) {
1240                 case NVME_FEAT_KATO:
1241                         /*
1242                          * Keep alive commands interval on the host should be
1243                          * updated when KATO is modified by Set Features
1244                          * commands.
1245                          */
1246                         if (!status)
1247                                 nvme_update_keep_alive(ctrl, cmd);
1248                         break;
1249                 default:
1250                         break;
1251                 }
1252                 break;
1253         default:
1254                 break;
1255         }
1256 }
1257 EXPORT_SYMBOL_NS_GPL(nvme_passthru_end, NVME_TARGET_PASSTHRU);
1258
1259 /*
1260  * Recommended frequency for KATO commands per NVMe 1.4 section 7.12.1:
1261  *
1262  *   The host should send Keep Alive commands at half of the Keep Alive Timeout
1263  *   accounting for transport roundtrip times [..].
1264  */
1265 static unsigned long nvme_keep_alive_work_period(struct nvme_ctrl *ctrl)
1266 {
1267         unsigned long delay = ctrl->kato * HZ / 2;
1268
1269         /*
1270          * When using Traffic Based Keep Alive, we need to run
1271          * nvme_keep_alive_work at twice the normal frequency, as one
1272          * command completion can postpone sending a keep alive command
1273          * by up to twice the delay between runs.
1274          */
1275         if (ctrl->ctratt & NVME_CTRL_ATTR_TBKAS)
1276                 delay /= 2;
1277         return delay;
1278 }
1279
1280 static void nvme_queue_keep_alive_work(struct nvme_ctrl *ctrl)
1281 {
1282         unsigned long now = jiffies;
1283         unsigned long delay = nvme_keep_alive_work_period(ctrl);
1284         unsigned long ka_next_check_tm = ctrl->ka_last_check_time + delay;
1285
1286         if (time_after(now, ka_next_check_tm))
1287                 delay = 0;
1288         else
1289                 delay = ka_next_check_tm - now;
1290
1291         queue_delayed_work(nvme_wq, &ctrl->ka_work, delay);
1292 }
1293
1294 static enum rq_end_io_ret nvme_keep_alive_end_io(struct request *rq,
1295                                                  blk_status_t status)
1296 {
1297         struct nvme_ctrl *ctrl = rq->end_io_data;
1298         unsigned long flags;
1299         bool startka = false;
1300         unsigned long rtt = jiffies - (rq->deadline - rq->timeout);
1301         unsigned long delay = nvme_keep_alive_work_period(ctrl);
1302
1303         /*
1304          * Subtract off the keepalive RTT so nvme_keep_alive_work runs
1305          * at the desired frequency.
1306          */
1307         if (rtt <= delay) {
1308                 delay -= rtt;
1309         } else {
1310                 dev_warn(ctrl->device, "long keepalive RTT (%u ms)\n",
1311                          jiffies_to_msecs(rtt));
1312                 delay = 0;
1313         }
1314
1315         blk_mq_free_request(rq);
1316
1317         if (status) {
1318                 dev_err(ctrl->device,
1319                         "failed nvme_keep_alive_end_io error=%d\n",
1320                                 status);
1321                 return RQ_END_IO_NONE;
1322         }
1323
1324         ctrl->ka_last_check_time = jiffies;
1325         ctrl->comp_seen = false;
1326         spin_lock_irqsave(&ctrl->lock, flags);
1327         if (ctrl->state == NVME_CTRL_LIVE ||
1328             ctrl->state == NVME_CTRL_CONNECTING)
1329                 startka = true;
1330         spin_unlock_irqrestore(&ctrl->lock, flags);
1331         if (startka)
1332                 queue_delayed_work(nvme_wq, &ctrl->ka_work, delay);
1333         return RQ_END_IO_NONE;
1334 }
1335
1336 static void nvme_keep_alive_work(struct work_struct *work)
1337 {
1338         struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1339                         struct nvme_ctrl, ka_work);
1340         bool comp_seen = ctrl->comp_seen;
1341         struct request *rq;
1342
1343         ctrl->ka_last_check_time = jiffies;
1344
1345         if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1346                 dev_dbg(ctrl->device,
1347                         "reschedule traffic based keep-alive timer\n");
1348                 ctrl->comp_seen = false;
1349                 nvme_queue_keep_alive_work(ctrl);
1350                 return;
1351         }
1352
1353         rq = blk_mq_alloc_request(ctrl->admin_q, nvme_req_op(&ctrl->ka_cmd),
1354                                   BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
1355         if (IS_ERR(rq)) {
1356                 /* allocation failure, reset the controller */
1357                 dev_err(ctrl->device, "keep-alive failed: %ld\n", PTR_ERR(rq));
1358                 nvme_reset_ctrl(ctrl);
1359                 return;
1360         }
1361         nvme_init_request(rq, &ctrl->ka_cmd);
1362
1363         rq->timeout = ctrl->kato * HZ;
1364         rq->end_io = nvme_keep_alive_end_io;
1365         rq->end_io_data = ctrl;
1366         blk_execute_rq_nowait(rq, false);
1367 }
1368
1369 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
1370 {
1371         if (unlikely(ctrl->kato == 0))
1372                 return;
1373
1374         nvme_queue_keep_alive_work(ctrl);
1375 }
1376
1377 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1378 {
1379         if (unlikely(ctrl->kato == 0))
1380                 return;
1381
1382         cancel_delayed_work_sync(&ctrl->ka_work);
1383 }
1384 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1385
1386 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
1387                                    struct nvme_command *cmd)
1388 {
1389         unsigned int new_kato =
1390                 DIV_ROUND_UP(le32_to_cpu(cmd->common.cdw11), 1000);
1391
1392         dev_info(ctrl->device,
1393                  "keep alive interval updated from %u ms to %u ms\n",
1394                  ctrl->kato * 1000 / 2, new_kato * 1000 / 2);
1395
1396         nvme_stop_keep_alive(ctrl);
1397         ctrl->kato = new_kato;
1398         nvme_start_keep_alive(ctrl);
1399 }
1400
1401 /*
1402  * In NVMe 1.0 the CNS field was just a binary controller or namespace
1403  * flag, thus sending any new CNS opcodes has a big chance of not working.
1404  * Qemu unfortunately had that bug after reporting a 1.1 version compliance
1405  * (but not for any later version).
1406  */
1407 static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)
1408 {
1409         if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)
1410                 return ctrl->vs < NVME_VS(1, 2, 0);
1411         return ctrl->vs < NVME_VS(1, 1, 0);
1412 }
1413
1414 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
1415 {
1416         struct nvme_command c = { };
1417         int error;
1418
1419         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1420         c.identify.opcode = nvme_admin_identify;
1421         c.identify.cns = NVME_ID_CNS_CTRL;
1422
1423         *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1424         if (!*id)
1425                 return -ENOMEM;
1426
1427         error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1428                         sizeof(struct nvme_id_ctrl));
1429         if (error) {
1430                 kfree(*id);
1431                 *id = NULL;
1432         }
1433         return error;
1434 }
1435
1436 static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
1437                 struct nvme_ns_id_desc *cur, bool *csi_seen)
1438 {
1439         const char *warn_str = "ctrl returned bogus length:";
1440         void *data = cur;
1441
1442         switch (cur->nidt) {
1443         case NVME_NIDT_EUI64:
1444                 if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1445                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
1446                                  warn_str, cur->nidl);
1447                         return -1;
1448                 }
1449                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1450                         return NVME_NIDT_EUI64_LEN;
1451                 memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
1452                 return NVME_NIDT_EUI64_LEN;
1453         case NVME_NIDT_NGUID:
1454                 if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1455                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
1456                                  warn_str, cur->nidl);
1457                         return -1;
1458                 }
1459                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1460                         return NVME_NIDT_NGUID_LEN;
1461                 memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
1462                 return NVME_NIDT_NGUID_LEN;
1463         case NVME_NIDT_UUID:
1464                 if (cur->nidl != NVME_NIDT_UUID_LEN) {
1465                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
1466                                  warn_str, cur->nidl);
1467                         return -1;
1468                 }
1469                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1470                         return NVME_NIDT_UUID_LEN;
1471                 uuid_copy(&ids->uuid, data + sizeof(*cur));
1472                 return NVME_NIDT_UUID_LEN;
1473         case NVME_NIDT_CSI:
1474                 if (cur->nidl != NVME_NIDT_CSI_LEN) {
1475                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
1476                                  warn_str, cur->nidl);
1477                         return -1;
1478                 }
1479                 memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
1480                 *csi_seen = true;
1481                 return NVME_NIDT_CSI_LEN;
1482         default:
1483                 /* Skip unknown types */
1484                 return cur->nidl;
1485         }
1486 }
1487
1488 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl,
1489                 struct nvme_ns_info *info)
1490 {
1491         struct nvme_command c = { };
1492         bool csi_seen = false;
1493         int status, pos, len;
1494         void *data;
1495
1496         if (ctrl->vs < NVME_VS(1, 3, 0) && !nvme_multi_css(ctrl))
1497                 return 0;
1498         if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
1499                 return 0;
1500
1501         c.identify.opcode = nvme_admin_identify;
1502         c.identify.nsid = cpu_to_le32(info->nsid);
1503         c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1504
1505         data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1506         if (!data)
1507                 return -ENOMEM;
1508
1509         status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
1510                                       NVME_IDENTIFY_DATA_SIZE);
1511         if (status) {
1512                 dev_warn(ctrl->device,
1513                         "Identify Descriptors failed (nsid=%u, status=0x%x)\n",
1514                         info->nsid, status);
1515                 goto free_data;
1516         }
1517
1518         for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1519                 struct nvme_ns_id_desc *cur = data + pos;
1520
1521                 if (cur->nidl == 0)
1522                         break;
1523
1524                 len = nvme_process_ns_desc(ctrl, &info->ids, cur, &csi_seen);
1525                 if (len < 0)
1526                         break;
1527
1528                 len += sizeof(*cur);
1529         }
1530
1531         if (nvme_multi_css(ctrl) && !csi_seen) {
1532                 dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
1533                          info->nsid);
1534                 status = -EINVAL;
1535         }
1536
1537 free_data:
1538         kfree(data);
1539         return status;
1540 }
1541
1542 int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
1543                         struct nvme_id_ns **id)
1544 {
1545         struct nvme_command c = { };
1546         int error;
1547
1548         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1549         c.identify.opcode = nvme_admin_identify;
1550         c.identify.nsid = cpu_to_le32(nsid);
1551         c.identify.cns = NVME_ID_CNS_NS;
1552
1553         *id = kmalloc(sizeof(**id), GFP_KERNEL);
1554         if (!*id)
1555                 return -ENOMEM;
1556
1557         error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1558         if (error) {
1559                 dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
1560                 kfree(*id);
1561                 *id = NULL;
1562         }
1563         return error;
1564 }
1565
1566 static int nvme_ns_info_from_identify(struct nvme_ctrl *ctrl,
1567                 struct nvme_ns_info *info)
1568 {
1569         struct nvme_ns_ids *ids = &info->ids;
1570         struct nvme_id_ns *id;
1571         int ret;
1572
1573         ret = nvme_identify_ns(ctrl, info->nsid, &id);
1574         if (ret)
1575                 return ret;
1576
1577         if (id->ncap == 0) {
1578                 /* namespace not allocated or attached */
1579                 info->is_removed = true;
1580                 ret = -ENODEV;
1581                 goto error;
1582         }
1583
1584         info->anagrpid = id->anagrpid;
1585         info->is_shared = id->nmic & NVME_NS_NMIC_SHARED;
1586         info->is_readonly = id->nsattr & NVME_NS_ATTR_RO;
1587         info->is_ready = true;
1588         if (ctrl->quirks & NVME_QUIRK_BOGUS_NID) {
1589                 dev_info(ctrl->device,
1590                          "Ignoring bogus Namespace Identifiers\n");
1591         } else {
1592                 if (ctrl->vs >= NVME_VS(1, 1, 0) &&
1593                     !memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
1594                         memcpy(ids->eui64, id->eui64, sizeof(ids->eui64));
1595                 if (ctrl->vs >= NVME_VS(1, 2, 0) &&
1596                     !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
1597                         memcpy(ids->nguid, id->nguid, sizeof(ids->nguid));
1598         }
1599
1600 error:
1601         kfree(id);
1602         return ret;
1603 }
1604
1605 static int nvme_ns_info_from_id_cs_indep(struct nvme_ctrl *ctrl,
1606                 struct nvme_ns_info *info)
1607 {
1608         struct nvme_id_ns_cs_indep *id;
1609         struct nvme_command c = {
1610                 .identify.opcode        = nvme_admin_identify,
1611                 .identify.nsid          = cpu_to_le32(info->nsid),
1612                 .identify.cns           = NVME_ID_CNS_NS_CS_INDEP,
1613         };
1614         int ret;
1615
1616         id = kmalloc(sizeof(*id), GFP_KERNEL);
1617         if (!id)
1618                 return -ENOMEM;
1619
1620         ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
1621         if (!ret) {
1622                 info->anagrpid = id->anagrpid;
1623                 info->is_shared = id->nmic & NVME_NS_NMIC_SHARED;
1624                 info->is_readonly = id->nsattr & NVME_NS_ATTR_RO;
1625                 info->is_ready = id->nstat & NVME_NSTAT_NRDY;
1626         }
1627         kfree(id);
1628         return ret;
1629 }
1630
1631 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1632                 unsigned int dword11, void *buffer, size_t buflen, u32 *result)
1633 {
1634         union nvme_result res = { 0 };
1635         struct nvme_command c = { };
1636         int ret;
1637
1638         c.features.opcode = op;
1639         c.features.fid = cpu_to_le32(fid);
1640         c.features.dword11 = cpu_to_le32(dword11);
1641
1642         ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1643                         buffer, buflen, NVME_QID_ANY, 0);
1644         if (ret >= 0 && result)
1645                 *result = le32_to_cpu(res.u32);
1646         return ret;
1647 }
1648
1649 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1650                       unsigned int dword11, void *buffer, size_t buflen,
1651                       u32 *result)
1652 {
1653         return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1654                              buflen, result);
1655 }
1656 EXPORT_SYMBOL_GPL(nvme_set_features);
1657
1658 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1659                       unsigned int dword11, void *buffer, size_t buflen,
1660                       u32 *result)
1661 {
1662         return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1663                              buflen, result);
1664 }
1665 EXPORT_SYMBOL_GPL(nvme_get_features);
1666
1667 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1668 {
1669         u32 q_count = (*count - 1) | ((*count - 1) << 16);
1670         u32 result;
1671         int status, nr_io_queues;
1672
1673         status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
1674                         &result);
1675         if (status < 0)
1676                 return status;
1677
1678         /*
1679          * Degraded controllers might return an error when setting the queue
1680          * count.  We still want to be able to bring them online and offer
1681          * access to the admin queue, as that might be only way to fix them up.
1682          */
1683         if (status > 0) {
1684                 dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1685                 *count = 0;
1686         } else {
1687                 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1688                 *count = min(*count, nr_io_queues);
1689         }
1690
1691         return 0;
1692 }
1693 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1694
1695 #define NVME_AEN_SUPPORTED \
1696         (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1697          NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
1698
1699 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1700 {
1701         u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
1702         int status;
1703
1704         if (!supported_aens)
1705                 return;
1706
1707         status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1708                         NULL, 0, &result);
1709         if (status)
1710                 dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1711                          supported_aens);
1712
1713         queue_work(nvme_wq, &ctrl->async_event_work);
1714 }
1715
1716 static int nvme_ns_open(struct nvme_ns *ns)
1717 {
1718
1719         /* should never be called due to GENHD_FL_HIDDEN */
1720         if (WARN_ON_ONCE(nvme_ns_head_multipath(ns->head)))
1721                 goto fail;
1722         if (!nvme_get_ns(ns))
1723                 goto fail;
1724         if (!try_module_get(ns->ctrl->ops->module))
1725                 goto fail_put_ns;
1726
1727         return 0;
1728
1729 fail_put_ns:
1730         nvme_put_ns(ns);
1731 fail:
1732         return -ENXIO;
1733 }
1734
1735 static void nvme_ns_release(struct nvme_ns *ns)
1736 {
1737
1738         module_put(ns->ctrl->ops->module);
1739         nvme_put_ns(ns);
1740 }
1741
1742 static int nvme_open(struct gendisk *disk, blk_mode_t mode)
1743 {
1744         return nvme_ns_open(disk->private_data);
1745 }
1746
1747 static void nvme_release(struct gendisk *disk)
1748 {
1749         nvme_ns_release(disk->private_data);
1750 }
1751
1752 int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1753 {
1754         /* some standard values */
1755         geo->heads = 1 << 6;
1756         geo->sectors = 1 << 5;
1757         geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1758         return 0;
1759 }
1760
1761 static bool nvme_init_integrity(struct nvme_ns_head *head,
1762                 struct queue_limits *lim, struct nvme_ns_info *info)
1763 {
1764         struct blk_integrity *bi = &lim->integrity;
1765
1766         memset(bi, 0, sizeof(*bi));
1767
1768         if (!head->ms)
1769                 return true;
1770
1771         /*
1772          * PI can always be supported as we can ask the controller to simply
1773          * insert/strip it, which is not possible for other kinds of metadata.
1774          */
1775         if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) ||
1776             !(head->features & NVME_NS_METADATA_SUPPORTED))
1777                 return nvme_ns_has_pi(head);
1778
1779         switch (head->pi_type) {
1780         case NVME_NS_DPS_PI_TYPE3:
1781                 switch (head->guard_type) {
1782                 case NVME_NVM_NS_16B_GUARD:
1783                         bi->csum_type = BLK_INTEGRITY_CSUM_CRC;
1784                         bi->tag_size = sizeof(u16) + sizeof(u32);
1785                         bi->flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1786                         break;
1787                 case NVME_NVM_NS_64B_GUARD:
1788                         bi->csum_type = BLK_INTEGRITY_CSUM_CRC64;
1789                         bi->tag_size = sizeof(u16) + 6;
1790                         bi->flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1791                         break;
1792                 default:
1793                         break;
1794                 }
1795                 break;
1796         case NVME_NS_DPS_PI_TYPE1:
1797         case NVME_NS_DPS_PI_TYPE2:
1798                 switch (head->guard_type) {
1799                 case NVME_NVM_NS_16B_GUARD:
1800                         bi->csum_type = BLK_INTEGRITY_CSUM_CRC;
1801                         bi->tag_size = sizeof(u16);
1802                         bi->flags |= BLK_INTEGRITY_DEVICE_CAPABLE |
1803                                      BLK_INTEGRITY_REF_TAG;
1804                         break;
1805                 case NVME_NVM_NS_64B_GUARD:
1806                         bi->csum_type = BLK_INTEGRITY_CSUM_CRC64;
1807                         bi->tag_size = sizeof(u16);
1808                         bi->flags |= BLK_INTEGRITY_DEVICE_CAPABLE |
1809                                      BLK_INTEGRITY_REF_TAG;
1810                         break;
1811                 default:
1812                         break;
1813                 }
1814                 break;
1815         default:
1816                 break;
1817         }
1818
1819         bi->tuple_size = head->ms;
1820         bi->pi_offset = info->pi_offset;
1821         return true;
1822 }
1823
1824 static void nvme_config_discard(struct nvme_ns *ns, struct queue_limits *lim)
1825 {
1826         struct nvme_ctrl *ctrl = ns->ctrl;
1827
1828         if (ctrl->dmrsl && ctrl->dmrsl <= nvme_sect_to_lba(ns->head, UINT_MAX))
1829                 lim->max_hw_discard_sectors =
1830                         nvme_lba_to_sect(ns->head, ctrl->dmrsl);
1831         else if (ctrl->oncs & NVME_CTRL_ONCS_DSM)
1832                 lim->max_hw_discard_sectors = UINT_MAX;
1833         else
1834                 lim->max_hw_discard_sectors = 0;
1835
1836         lim->discard_granularity = lim->logical_block_size;
1837
1838         if (ctrl->dmrl)
1839                 lim->max_discard_segments = ctrl->dmrl;
1840         else
1841                 lim->max_discard_segments = NVME_DSM_MAX_RANGES;
1842 }
1843
1844 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1845 {
1846         return uuid_equal(&a->uuid, &b->uuid) &&
1847                 memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1848                 memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
1849                 a->csi == b->csi;
1850 }
1851
1852 static int nvme_identify_ns_nvm(struct nvme_ctrl *ctrl, unsigned int nsid,
1853                 struct nvme_id_ns_nvm **nvmp)
1854 {
1855         struct nvme_command c = {
1856                 .identify.opcode        = nvme_admin_identify,
1857                 .identify.nsid          = cpu_to_le32(nsid),
1858                 .identify.cns           = NVME_ID_CNS_CS_NS,
1859                 .identify.csi           = NVME_CSI_NVM,
1860         };
1861         struct nvme_id_ns_nvm *nvm;
1862         int ret;
1863
1864         nvm = kzalloc(sizeof(*nvm), GFP_KERNEL);
1865         if (!nvm)
1866                 return -ENOMEM;
1867
1868         ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, nvm, sizeof(*nvm));
1869         if (ret)
1870                 kfree(nvm);
1871         else
1872                 *nvmp = nvm;
1873         return ret;
1874 }
1875
1876 static void nvme_configure_pi_elbas(struct nvme_ns_head *head,
1877                 struct nvme_id_ns *id, struct nvme_id_ns_nvm *nvm)
1878 {
1879         u32 elbaf = le32_to_cpu(nvm->elbaf[nvme_lbaf_index(id->flbas)]);
1880         u8 guard_type;
1881
1882         /* no support for storage tag formats right now */
1883         if (nvme_elbaf_sts(elbaf))
1884                 return;
1885
1886         guard_type = nvme_elbaf_guard_type(elbaf);
1887         if ((nvm->pic & NVME_ID_NS_NVM_QPIFS) &&
1888              guard_type == NVME_NVM_NS_QTYPE_GUARD)
1889                 guard_type = nvme_elbaf_qualified_guard_type(elbaf);
1890
1891         head->guard_type = guard_type;
1892         switch (head->guard_type) {
1893         case NVME_NVM_NS_64B_GUARD:
1894                 head->pi_size = sizeof(struct crc64_pi_tuple);
1895                 break;
1896         case NVME_NVM_NS_16B_GUARD:
1897                 head->pi_size = sizeof(struct t10_pi_tuple);
1898                 break;
1899         default:
1900                 break;
1901         }
1902 }
1903
1904 static void nvme_configure_metadata(struct nvme_ctrl *ctrl,
1905                 struct nvme_ns_head *head, struct nvme_id_ns *id,
1906                 struct nvme_id_ns_nvm *nvm, struct nvme_ns_info *info)
1907 {
1908         head->features &= ~(NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS);
1909         head->pi_type = 0;
1910         head->pi_size = 0;
1911         head->ms = le16_to_cpu(id->lbaf[nvme_lbaf_index(id->flbas)].ms);
1912         if (!head->ms || !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
1913                 return;
1914
1915         if (nvm && (ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)) {
1916                 nvme_configure_pi_elbas(head, id, nvm);
1917         } else {
1918                 head->pi_size = sizeof(struct t10_pi_tuple);
1919                 head->guard_type = NVME_NVM_NS_16B_GUARD;
1920         }
1921
1922         if (head->pi_size && head->ms >= head->pi_size)
1923                 head->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1924         if (!(id->dps & NVME_NS_DPS_PI_FIRST))
1925                 info->pi_offset = head->ms - head->pi_size;
1926
1927         if (ctrl->ops->flags & NVME_F_FABRICS) {
1928                 /*
1929                  * The NVMe over Fabrics specification only supports metadata as
1930                  * part of the extended data LBA.  We rely on HCA/HBA support to
1931                  * remap the separate metadata buffer from the block layer.
1932                  */
1933                 if (WARN_ON_ONCE(!(id->flbas & NVME_NS_FLBAS_META_EXT)))
1934                         return;
1935
1936                 head->features |= NVME_NS_EXT_LBAS;
1937
1938                 /*
1939                  * The current fabrics transport drivers support namespace
1940                  * metadata formats only if nvme_ns_has_pi() returns true.
1941                  * Suppress support for all other formats so the namespace will
1942                  * have a 0 capacity and not be usable through the block stack.
1943                  *
1944                  * Note, this check will need to be modified if any drivers
1945                  * gain the ability to use other metadata formats.
1946                  */
1947                 if (ctrl->max_integrity_segments && nvme_ns_has_pi(head))
1948                         head->features |= NVME_NS_METADATA_SUPPORTED;
1949         } else {
1950                 /*
1951                  * For PCIe controllers, we can't easily remap the separate
1952                  * metadata buffer from the block layer and thus require a
1953                  * separate metadata buffer for block layer metadata/PI support.
1954                  * We allow extended LBAs for the passthrough interface, though.
1955                  */
1956                 if (id->flbas & NVME_NS_FLBAS_META_EXT)
1957                         head->features |= NVME_NS_EXT_LBAS;
1958                 else
1959                         head->features |= NVME_NS_METADATA_SUPPORTED;
1960         }
1961 }
1962
1963
1964 static void nvme_update_atomic_write_disk_info(struct nvme_ns *ns,
1965                         struct nvme_id_ns *id, struct queue_limits *lim,
1966                         u32 bs, u32 atomic_bs)
1967 {
1968         unsigned int boundary = 0;
1969
1970         if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf) {
1971                 if (le16_to_cpu(id->nabspf))
1972                         boundary = (le16_to_cpu(id->nabspf) + 1) * bs;
1973         }
1974         lim->atomic_write_hw_max = atomic_bs;
1975         lim->atomic_write_hw_boundary = boundary;
1976         lim->atomic_write_hw_unit_min = bs;
1977         lim->atomic_write_hw_unit_max = rounddown_pow_of_two(atomic_bs);
1978 }
1979
1980 static u32 nvme_max_drv_segments(struct nvme_ctrl *ctrl)
1981 {
1982         return ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> SECTOR_SHIFT) + 1;
1983 }
1984
1985 static void nvme_set_ctrl_limits(struct nvme_ctrl *ctrl,
1986                 struct queue_limits *lim)
1987 {
1988         lim->max_hw_sectors = ctrl->max_hw_sectors;
1989         lim->max_segments = min_t(u32, USHRT_MAX,
1990                 min_not_zero(nvme_max_drv_segments(ctrl), ctrl->max_segments));
1991         lim->max_integrity_segments = ctrl->max_integrity_segments;
1992         lim->virt_boundary_mask = NVME_CTRL_PAGE_SIZE - 1;
1993         lim->max_segment_size = UINT_MAX;
1994         lim->dma_alignment = 3;
1995 }
1996
1997 static bool nvme_update_disk_info(struct nvme_ns *ns, struct nvme_id_ns *id,
1998                 struct queue_limits *lim)
1999 {
2000         struct nvme_ns_head *head = ns->head;
2001         u32 bs = 1U << head->lba_shift;
2002         u32 atomic_bs, phys_bs, io_opt = 0;
2003         bool valid = true;
2004
2005         /*
2006          * The block layer can't support LBA sizes larger than the page size
2007          * or smaller than a sector size yet, so catch this early and don't
2008          * allow block I/O.
2009          */
2010         if (head->lba_shift > PAGE_SHIFT || head->lba_shift < SECTOR_SHIFT) {
2011                 bs = (1 << 9);
2012                 valid = false;
2013         }
2014
2015         atomic_bs = phys_bs = bs;
2016         if (id->nabo == 0) {
2017                 /*
2018                  * Bit 1 indicates whether NAWUPF is defined for this namespace
2019                  * and whether it should be used instead of AWUPF. If NAWUPF ==
2020                  * 0 then AWUPF must be used instead.
2021                  */
2022                 if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
2023                         atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
2024                 else
2025                         atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
2026
2027                 nvme_update_atomic_write_disk_info(ns, id, lim, bs, atomic_bs);
2028         }
2029
2030         if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
2031                 /* NPWG = Namespace Preferred Write Granularity */
2032                 phys_bs = bs * (1 + le16_to_cpu(id->npwg));
2033                 /* NOWS = Namespace Optimal Write Size */
2034                 if (id->nows)
2035                         io_opt = bs * (1 + le16_to_cpu(id->nows));
2036         }
2037
2038         /*
2039          * Linux filesystems assume writing a single physical block is
2040          * an atomic operation. Hence limit the physical block size to the
2041          * value of the Atomic Write Unit Power Fail parameter.
2042          */
2043         lim->logical_block_size = bs;
2044         lim->physical_block_size = min(phys_bs, atomic_bs);
2045         lim->io_min = phys_bs;
2046         lim->io_opt = io_opt;
2047         if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
2048                 lim->max_write_zeroes_sectors = UINT_MAX;
2049         else
2050                 lim->max_write_zeroes_sectors = ns->ctrl->max_zeroes_sectors;
2051         return valid;
2052 }
2053
2054 static bool nvme_ns_is_readonly(struct nvme_ns *ns, struct nvme_ns_info *info)
2055 {
2056         return info->is_readonly || test_bit(NVME_NS_FORCE_RO, &ns->flags);
2057 }
2058
2059 static inline bool nvme_first_scan(struct gendisk *disk)
2060 {
2061         /* nvme_alloc_ns() scans the disk prior to adding it */
2062         return !disk_live(disk);
2063 }
2064
2065 static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id,
2066                 struct queue_limits *lim)
2067 {
2068         struct nvme_ctrl *ctrl = ns->ctrl;
2069         u32 iob;
2070
2071         if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
2072             is_power_of_2(ctrl->max_hw_sectors))
2073                 iob = ctrl->max_hw_sectors;
2074         else
2075                 iob = nvme_lba_to_sect(ns->head, le16_to_cpu(id->noiob));
2076
2077         if (!iob)
2078                 return;
2079
2080         if (!is_power_of_2(iob)) {
2081                 if (nvme_first_scan(ns->disk))
2082                         pr_warn("%s: ignoring unaligned IO boundary:%u\n",
2083                                 ns->disk->disk_name, iob);
2084                 return;
2085         }
2086
2087         if (blk_queue_is_zoned(ns->disk->queue)) {
2088                 if (nvme_first_scan(ns->disk))
2089                         pr_warn("%s: ignoring zoned namespace IO boundary\n",
2090                                 ns->disk->disk_name);
2091                 return;
2092         }
2093
2094         lim->chunk_sectors = iob;
2095 }
2096
2097 static int nvme_update_ns_info_generic(struct nvme_ns *ns,
2098                 struct nvme_ns_info *info)
2099 {
2100         struct queue_limits lim;
2101         int ret;
2102
2103         blk_mq_freeze_queue(ns->disk->queue);
2104         lim = queue_limits_start_update(ns->disk->queue);
2105         nvme_set_ctrl_limits(ns->ctrl, &lim);
2106         ret = queue_limits_commit_update(ns->disk->queue, &lim);
2107         set_disk_ro(ns->disk, nvme_ns_is_readonly(ns, info));
2108         blk_mq_unfreeze_queue(ns->disk->queue);
2109
2110         /* Hide the block-interface for these devices */
2111         if (!ret)
2112                 ret = -ENODEV;
2113         return ret;
2114 }
2115
2116 static int nvme_update_ns_info_block(struct nvme_ns *ns,
2117                 struct nvme_ns_info *info)
2118 {
2119         struct queue_limits lim;
2120         struct nvme_id_ns_nvm *nvm = NULL;
2121         struct nvme_zone_info zi = {};
2122         struct nvme_id_ns *id;
2123         sector_t capacity;
2124         unsigned lbaf;
2125         int ret;
2126
2127         ret = nvme_identify_ns(ns->ctrl, info->nsid, &id);
2128         if (ret)
2129                 return ret;
2130
2131         if (id->ncap == 0) {
2132                 /* namespace not allocated or attached */
2133                 info->is_removed = true;
2134                 ret = -ENXIO;
2135                 goto out;
2136         }
2137         lbaf = nvme_lbaf_index(id->flbas);
2138
2139         if (ns->ctrl->ctratt & NVME_CTRL_ATTR_ELBAS) {
2140                 ret = nvme_identify_ns_nvm(ns->ctrl, info->nsid, &nvm);
2141                 if (ret < 0)
2142                         goto out;
2143         }
2144
2145         if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
2146             ns->head->ids.csi == NVME_CSI_ZNS) {
2147                 ret = nvme_query_zone_info(ns, lbaf, &zi);
2148                 if (ret < 0)
2149                         goto out;
2150         }
2151
2152         blk_mq_freeze_queue(ns->disk->queue);
2153         ns->head->lba_shift = id->lbaf[lbaf].ds;
2154         ns->head->nuse = le64_to_cpu(id->nuse);
2155         capacity = nvme_lba_to_sect(ns->head, le64_to_cpu(id->nsze));
2156
2157         lim = queue_limits_start_update(ns->disk->queue);
2158         nvme_set_ctrl_limits(ns->ctrl, &lim);
2159         nvme_configure_metadata(ns->ctrl, ns->head, id, nvm, info);
2160         nvme_set_chunk_sectors(ns, id, &lim);
2161         if (!nvme_update_disk_info(ns, id, &lim))
2162                 capacity = 0;
2163         nvme_config_discard(ns, &lim);
2164         if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
2165             ns->head->ids.csi == NVME_CSI_ZNS)
2166                 nvme_update_zone_info(ns, &lim, &zi);
2167
2168         if (ns->ctrl->vwc & NVME_CTRL_VWC_PRESENT)
2169                 lim.features |= BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA;
2170         else
2171                 lim.features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA);
2172
2173         /*
2174          * Register a metadata profile for PI, or the plain non-integrity NVMe
2175          * metadata masquerading as Type 0 if supported, otherwise reject block
2176          * I/O to namespaces with metadata except when the namespace supports
2177          * PI, as it can strip/insert in that case.
2178          */
2179         if (!nvme_init_integrity(ns->head, &lim, info))
2180                 capacity = 0;
2181
2182         ret = queue_limits_commit_update(ns->disk->queue, &lim);
2183         if (ret) {
2184                 blk_mq_unfreeze_queue(ns->disk->queue);
2185                 goto out;
2186         }
2187
2188         set_capacity_and_notify(ns->disk, capacity);
2189
2190         /*
2191          * Only set the DEAC bit if the device guarantees that reads from
2192          * deallocated data return zeroes.  While the DEAC bit does not
2193          * require that, it must be a no-op if reads from deallocated data
2194          * do not return zeroes.
2195          */
2196         if ((id->dlfeat & 0x7) == 0x1 && (id->dlfeat & (1 << 3)))
2197                 ns->head->features |= NVME_NS_DEAC;
2198         set_disk_ro(ns->disk, nvme_ns_is_readonly(ns, info));
2199         set_bit(NVME_NS_READY, &ns->flags);
2200         blk_mq_unfreeze_queue(ns->disk->queue);
2201
2202         if (blk_queue_is_zoned(ns->queue)) {
2203                 ret = blk_revalidate_disk_zones(ns->disk);
2204                 if (ret && !nvme_first_scan(ns->disk))
2205                         goto out;
2206         }
2207
2208         ret = 0;
2209 out:
2210         kfree(nvm);
2211         kfree(id);
2212         return ret;
2213 }
2214
2215 static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_ns_info *info)
2216 {
2217         bool unsupported = false;
2218         int ret;
2219
2220         switch (info->ids.csi) {
2221         case NVME_CSI_ZNS:
2222                 if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
2223                         dev_info(ns->ctrl->device,
2224         "block device for nsid %u not supported without CONFIG_BLK_DEV_ZONED\n",
2225                                 info->nsid);
2226                         ret = nvme_update_ns_info_generic(ns, info);
2227                         break;
2228                 }
2229                 ret = nvme_update_ns_info_block(ns, info);
2230                 break;
2231         case NVME_CSI_NVM:
2232                 ret = nvme_update_ns_info_block(ns, info);
2233                 break;
2234         default:
2235                 dev_info(ns->ctrl->device,
2236                         "block device for nsid %u not supported (csi %u)\n",
2237                         info->nsid, info->ids.csi);
2238                 ret = nvme_update_ns_info_generic(ns, info);
2239                 break;
2240         }
2241
2242         /*
2243          * If probing fails due an unsupported feature, hide the block device,
2244          * but still allow other access.
2245          */
2246         if (ret == -ENODEV) {
2247                 ns->disk->flags |= GENHD_FL_HIDDEN;
2248                 set_bit(NVME_NS_READY, &ns->flags);
2249                 unsupported = true;
2250                 ret = 0;
2251         }
2252
2253         if (!ret && nvme_ns_head_multipath(ns->head)) {
2254                 struct queue_limits *ns_lim = &ns->disk->queue->limits;
2255                 struct queue_limits lim;
2256
2257                 blk_mq_freeze_queue(ns->head->disk->queue);
2258                 /*
2259                  * queue_limits mixes values that are the hardware limitations
2260                  * for bio splitting with what is the device configuration.
2261                  *
2262                  * For NVMe the device configuration can change after e.g. a
2263                  * Format command, and we really want to pick up the new format
2264                  * value here.  But we must still stack the queue limits to the
2265                  * least common denominator for multipathing to split the bios
2266                  * properly.
2267                  *
2268                  * To work around this, we explicitly set the device
2269                  * configuration to those that we just queried, but only stack
2270                  * the splitting limits in to make sure we still obey possibly
2271                  * lower limitations of other controllers.
2272                  */
2273                 lim = queue_limits_start_update(ns->head->disk->queue);
2274                 lim.logical_block_size = ns_lim->logical_block_size;
2275                 lim.physical_block_size = ns_lim->physical_block_size;
2276                 lim.io_min = ns_lim->io_min;
2277                 lim.io_opt = ns_lim->io_opt;
2278                 queue_limits_stack_bdev(&lim, ns->disk->part0, 0,
2279                                         ns->head->disk->disk_name);
2280                 if (unsupported)
2281                         ns->head->disk->flags |= GENHD_FL_HIDDEN;
2282                 else
2283                         nvme_init_integrity(ns->head, &lim, info);
2284                 ret = queue_limits_commit_update(ns->head->disk->queue, &lim);
2285
2286                 set_capacity_and_notify(ns->head->disk, get_capacity(ns->disk));
2287                 set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info));
2288                 nvme_mpath_revalidate_paths(ns);
2289
2290                 blk_mq_unfreeze_queue(ns->head->disk->queue);
2291         }
2292
2293         return ret;
2294 }
2295
2296 int nvme_ns_get_unique_id(struct nvme_ns *ns, u8 id[16],
2297                 enum blk_unique_id type)
2298 {
2299         struct nvme_ns_ids *ids = &ns->head->ids;
2300
2301         if (type != BLK_UID_EUI64)
2302                 return -EINVAL;
2303
2304         if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid))) {
2305                 memcpy(id, &ids->nguid, sizeof(ids->nguid));
2306                 return sizeof(ids->nguid);
2307         }
2308         if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64))) {
2309                 memcpy(id, &ids->eui64, sizeof(ids->eui64));
2310                 return sizeof(ids->eui64);
2311         }
2312
2313         return -EINVAL;
2314 }
2315
2316 static int nvme_get_unique_id(struct gendisk *disk, u8 id[16],
2317                 enum blk_unique_id type)
2318 {
2319         return nvme_ns_get_unique_id(disk->private_data, id, type);
2320 }
2321
2322 #ifdef CONFIG_BLK_SED_OPAL
2323 static int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2324                 bool send)
2325 {
2326         struct nvme_ctrl *ctrl = data;
2327         struct nvme_command cmd = { };
2328
2329         if (send)
2330                 cmd.common.opcode = nvme_admin_security_send;
2331         else
2332                 cmd.common.opcode = nvme_admin_security_recv;
2333         cmd.common.nsid = 0;
2334         cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2335         cmd.common.cdw11 = cpu_to_le32(len);
2336
2337         return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
2338                         NVME_QID_ANY, NVME_SUBMIT_AT_HEAD);
2339 }
2340
2341 static void nvme_configure_opal(struct nvme_ctrl *ctrl, bool was_suspended)
2342 {
2343         if (ctrl->oacs & NVME_CTRL_OACS_SEC_SUPP) {
2344                 if (!ctrl->opal_dev)
2345                         ctrl->opal_dev = init_opal_dev(ctrl, &nvme_sec_submit);
2346                 else if (was_suspended)
2347                         opal_unlock_from_suspend(ctrl->opal_dev);
2348         } else {
2349                 free_opal_dev(ctrl->opal_dev);
2350                 ctrl->opal_dev = NULL;
2351         }
2352 }
2353 #else
2354 static void nvme_configure_opal(struct nvme_ctrl *ctrl, bool was_suspended)
2355 {
2356 }
2357 #endif /* CONFIG_BLK_SED_OPAL */
2358
2359 #ifdef CONFIG_BLK_DEV_ZONED
2360 static int nvme_report_zones(struct gendisk *disk, sector_t sector,
2361                 unsigned int nr_zones, report_zones_cb cb, void *data)
2362 {
2363         return nvme_ns_report_zones(disk->private_data, sector, nr_zones, cb,
2364                         data);
2365 }
2366 #else
2367 #define nvme_report_zones       NULL
2368 #endif /* CONFIG_BLK_DEV_ZONED */
2369
2370 const struct block_device_operations nvme_bdev_ops = {
2371         .owner          = THIS_MODULE,
2372         .ioctl          = nvme_ioctl,
2373         .compat_ioctl   = blkdev_compat_ptr_ioctl,
2374         .open           = nvme_open,
2375         .release        = nvme_release,
2376         .getgeo         = nvme_getgeo,
2377         .get_unique_id  = nvme_get_unique_id,
2378         .report_zones   = nvme_report_zones,
2379         .pr_ops         = &nvme_pr_ops,
2380 };
2381
2382 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 mask, u32 val,
2383                 u32 timeout, const char *op)
2384 {
2385         unsigned long timeout_jiffies = jiffies + timeout * HZ;
2386         u32 csts;
2387         int ret;
2388
2389         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2390                 if (csts == ~0)
2391                         return -ENODEV;
2392                 if ((csts & mask) == val)
2393                         break;
2394
2395                 usleep_range(1000, 2000);
2396                 if (fatal_signal_pending(current))
2397                         return -EINTR;
2398                 if (time_after(jiffies, timeout_jiffies)) {
2399                         dev_err(ctrl->device,
2400                                 "Device not ready; aborting %s, CSTS=0x%x\n",
2401                                 op, csts);
2402                         return -ENODEV;
2403                 }
2404         }
2405
2406         return ret;
2407 }
2408
2409 int nvme_disable_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2410 {
2411         int ret;
2412
2413         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2414         if (shutdown)
2415                 ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2416         else
2417                 ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2418
2419         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2420         if (ret)
2421                 return ret;
2422
2423         if (shutdown) {
2424                 return nvme_wait_ready(ctrl, NVME_CSTS_SHST_MASK,
2425                                        NVME_CSTS_SHST_CMPLT,
2426                                        ctrl->shutdown_timeout, "shutdown");
2427         }
2428         if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
2429                 msleep(NVME_QUIRK_DELAY_AMOUNT);
2430         return nvme_wait_ready(ctrl, NVME_CSTS_RDY, 0,
2431                                (NVME_CAP_TIMEOUT(ctrl->cap) + 1) / 2, "reset");
2432 }
2433 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
2434
2435 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
2436 {
2437         unsigned dev_page_min;
2438         u32 timeout;
2439         int ret;
2440
2441         ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2442         if (ret) {
2443                 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2444                 return ret;
2445         }
2446         dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2447
2448         if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
2449                 dev_err(ctrl->device,
2450                         "Minimum device page size %u too large for host (%u)\n",
2451                         1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
2452                 return -ENODEV;
2453         }
2454
2455         if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
2456                 ctrl->ctrl_config = NVME_CC_CSS_CSI;
2457         else
2458                 ctrl->ctrl_config = NVME_CC_CSS_NVM;
2459
2460         if (ctrl->cap & NVME_CAP_CRMS_CRWMS && ctrl->cap & NVME_CAP_CRMS_CRIMS)
2461                 ctrl->ctrl_config |= NVME_CC_CRIME;
2462
2463         ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
2464         ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
2465         ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2466         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2467         if (ret)
2468                 return ret;
2469
2470         /* Flush write to device (required if transport is PCI) */
2471         ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CC, &ctrl->ctrl_config);
2472         if (ret)
2473                 return ret;
2474
2475         /* CAP value may change after initial CC write */
2476         ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2477         if (ret)
2478                 return ret;
2479
2480         timeout = NVME_CAP_TIMEOUT(ctrl->cap);
2481         if (ctrl->cap & NVME_CAP_CRMS_CRWMS) {
2482                 u32 crto, ready_timeout;
2483
2484                 ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CRTO, &crto);
2485                 if (ret) {
2486                         dev_err(ctrl->device, "Reading CRTO failed (%d)\n",
2487                                 ret);
2488                         return ret;
2489                 }
2490
2491                 /*
2492                  * CRTO should always be greater or equal to CAP.TO, but some
2493                  * devices are known to get this wrong. Use the larger of the
2494                  * two values.
2495                  */
2496                 if (ctrl->ctrl_config & NVME_CC_CRIME)
2497                         ready_timeout = NVME_CRTO_CRIMT(crto);
2498                 else
2499                         ready_timeout = NVME_CRTO_CRWMT(crto);
2500
2501                 if (ready_timeout < timeout)
2502                         dev_warn_once(ctrl->device, "bad crto:%x cap:%llx\n",
2503                                       crto, ctrl->cap);
2504                 else
2505                         timeout = ready_timeout;
2506         }
2507
2508         ctrl->ctrl_config |= NVME_CC_ENABLE;
2509         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2510         if (ret)
2511                 return ret;
2512         return nvme_wait_ready(ctrl, NVME_CSTS_RDY, NVME_CSTS_RDY,
2513                                (timeout + 1) / 2, "initialisation");
2514 }
2515 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
2516
2517 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2518 {
2519         __le64 ts;
2520         int ret;
2521
2522         if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2523                 return 0;
2524
2525         ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2526         ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2527                         NULL);
2528         if (ret)
2529                 dev_warn_once(ctrl->device,
2530                         "could not set timestamp (%d)\n", ret);
2531         return ret;
2532 }
2533
2534 static int nvme_configure_host_options(struct nvme_ctrl *ctrl)
2535 {
2536         struct nvme_feat_host_behavior *host;
2537         u8 acre = 0, lbafee = 0;
2538         int ret;
2539
2540         /* Don't bother enabling the feature if retry delay is not reported */
2541         if (ctrl->crdt[0])
2542                 acre = NVME_ENABLE_ACRE;
2543         if (ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)
2544                 lbafee = NVME_ENABLE_LBAFEE;
2545
2546         if (!acre && !lbafee)
2547                 return 0;
2548
2549         host = kzalloc(sizeof(*host), GFP_KERNEL);
2550         if (!host)
2551                 return 0;
2552
2553         host->acre = acre;
2554         host->lbafee = lbafee;
2555         ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2556                                 host, sizeof(*host), NULL);
2557         kfree(host);
2558         return ret;
2559 }
2560
2561 /*
2562  * The function checks whether the given total (exlat + enlat) latency of
2563  * a power state allows the latter to be used as an APST transition target.
2564  * It does so by comparing the latency to the primary and secondary latency
2565  * tolerances defined by module params. If there's a match, the corresponding
2566  * timeout value is returned and the matching tolerance index (1 or 2) is
2567  * reported.
2568  */
2569 static bool nvme_apst_get_transition_time(u64 total_latency,
2570                 u64 *transition_time, unsigned *last_index)
2571 {
2572         if (total_latency <= apst_primary_latency_tol_us) {
2573                 if (*last_index == 1)
2574                         return false;
2575                 *last_index = 1;
2576                 *transition_time = apst_primary_timeout_ms;
2577                 return true;
2578         }
2579         if (apst_secondary_timeout_ms &&
2580                 total_latency <= apst_secondary_latency_tol_us) {
2581                 if (*last_index <= 2)
2582                         return false;
2583                 *last_index = 2;
2584                 *transition_time = apst_secondary_timeout_ms;
2585                 return true;
2586         }
2587         return false;
2588 }
2589
2590 /*
2591  * APST (Autonomous Power State Transition) lets us program a table of power
2592  * state transitions that the controller will perform automatically.
2593  *
2594  * Depending on module params, one of the two supported techniques will be used:
2595  *
2596  * - If the parameters provide explicit timeouts and tolerances, they will be
2597  *   used to build a table with up to 2 non-operational states to transition to.
2598  *   The default parameter values were selected based on the values used by
2599  *   Microsoft's and Intel's NVMe drivers. Yet, since we don't implement dynamic
2600  *   regeneration of the APST table in the event of switching between external
2601  *   and battery power, the timeouts and tolerances reflect a compromise
2602  *   between values used by Microsoft for AC and battery scenarios.
2603  * - If not, we'll configure the table with a simple heuristic: we are willing
2604  *   to spend at most 2% of the time transitioning between power states.
2605  *   Therefore, when running in any given state, we will enter the next
2606  *   lower-power non-operational state after waiting 50 * (enlat + exlat)
2607  *   microseconds, as long as that state's exit latency is under the requested
2608  *   maximum latency.
2609  *
2610  * We will not autonomously enter any non-operational state for which the total
2611  * latency exceeds ps_max_latency_us.
2612  *
2613  * Users can set ps_max_latency_us to zero to turn off APST.
2614  */
2615 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
2616 {
2617         struct nvme_feat_auto_pst *table;
2618         unsigned apste = 0;
2619         u64 max_lat_us = 0;
2620         __le64 target = 0;
2621         int max_ps = -1;
2622         int state;
2623         int ret;
2624         unsigned last_lt_index = UINT_MAX;
2625
2626         /*
2627          * If APST isn't supported or if we haven't been initialized yet,
2628          * then don't do anything.
2629          */
2630         if (!ctrl->apsta)
2631                 return 0;
2632
2633         if (ctrl->npss > 31) {
2634                 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
2635                 return 0;
2636         }
2637
2638         table = kzalloc(sizeof(*table), GFP_KERNEL);
2639         if (!table)
2640                 return 0;
2641
2642         if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
2643                 /* Turn off APST. */
2644                 dev_dbg(ctrl->device, "APST disabled\n");
2645                 goto done;
2646         }
2647
2648         /*
2649          * Walk through all states from lowest- to highest-power.
2650          * According to the spec, lower-numbered states use more power.  NPSS,
2651          * despite the name, is the index of the lowest-power state, not the
2652          * number of states.
2653          */
2654         for (state = (int)ctrl->npss; state >= 0; state--) {
2655                 u64 total_latency_us, exit_latency_us, transition_ms;
2656
2657                 if (target)
2658                         table->entries[state] = target;
2659
2660                 /*
2661                  * Don't allow transitions to the deepest state if it's quirked
2662                  * off.
2663                  */
2664                 if (state == ctrl->npss &&
2665                     (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2666                         continue;
2667
2668                 /*
2669                  * Is this state a useful non-operational state for higher-power
2670                  * states to autonomously transition to?
2671                  */
2672                 if (!(ctrl->psd[state].flags & NVME_PS_FLAGS_NON_OP_STATE))
2673                         continue;
2674
2675                 exit_latency_us = (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2676                 if (exit_latency_us > ctrl->ps_max_latency_us)
2677                         continue;
2678
2679                 total_latency_us = exit_latency_us +
2680                         le32_to_cpu(ctrl->psd[state].entry_lat);
2681
2682                 /*
2683                  * This state is good. It can be used as the APST idle target
2684                  * for higher power states.
2685                  */
2686                 if (apst_primary_timeout_ms && apst_primary_latency_tol_us) {
2687                         if (!nvme_apst_get_transition_time(total_latency_us,
2688                                         &transition_ms, &last_lt_index))
2689                                 continue;
2690                 } else {
2691                         transition_ms = total_latency_us + 19;
2692                         do_div(transition_ms, 20);
2693                         if (transition_ms > (1 << 24) - 1)
2694                                 transition_ms = (1 << 24) - 1;
2695                 }
2696
2697                 target = cpu_to_le64((state << 3) | (transition_ms << 8));
2698                 if (max_ps == -1)
2699                         max_ps = state;
2700                 if (total_latency_us > max_lat_us)
2701                         max_lat_us = total_latency_us;
2702         }
2703
2704         if (max_ps == -1)
2705                 dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2706         else
2707                 dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2708                         max_ps, max_lat_us, (int)sizeof(*table), table);
2709         apste = 1;
2710
2711 done:
2712         ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2713                                 table, sizeof(*table), NULL);
2714         if (ret)
2715                 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2716         kfree(table);
2717         return ret;
2718 }
2719
2720 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2721 {
2722         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2723         u64 latency;
2724
2725         switch (val) {
2726         case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2727         case PM_QOS_LATENCY_ANY:
2728                 latency = U64_MAX;
2729                 break;
2730
2731         default:
2732                 latency = val;
2733         }
2734
2735         if (ctrl->ps_max_latency_us != latency) {
2736                 ctrl->ps_max_latency_us = latency;
2737                 if (nvme_ctrl_state(ctrl) == NVME_CTRL_LIVE)
2738                         nvme_configure_apst(ctrl);
2739         }
2740 }
2741
2742 struct nvme_core_quirk_entry {
2743         /*
2744          * NVMe model and firmware strings are padded with spaces.  For
2745          * simplicity, strings in the quirk table are padded with NULLs
2746          * instead.
2747          */
2748         u16 vid;
2749         const char *mn;
2750         const char *fr;
2751         unsigned long quirks;
2752 };
2753
2754 static const struct nvme_core_quirk_entry core_quirks[] = {
2755         {
2756                 /*
2757                  * This Toshiba device seems to die using any APST states.  See:
2758                  * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2759                  */
2760                 .vid = 0x1179,
2761                 .mn = "THNSF5256GPUK TOSHIBA",
2762                 .quirks = NVME_QUIRK_NO_APST,
2763         },
2764         {
2765                 /*
2766                  * This LiteON CL1-3D*-Q11 firmware version has a race
2767                  * condition associated with actions related to suspend to idle
2768                  * LiteON has resolved the problem in future firmware
2769                  */
2770                 .vid = 0x14a4,
2771                 .fr = "22301111",
2772                 .quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2773         },
2774         {
2775                 /*
2776                  * This Kioxia CD6-V Series / HPE PE8030 device times out and
2777                  * aborts I/O during any load, but more easily reproducible
2778                  * with discards (fstrim).
2779                  *
2780                  * The device is left in a state where it is also not possible
2781                  * to use "nvme set-feature" to disable APST, but booting with
2782                  * nvme_core.default_ps_max_latency=0 works.
2783                  */
2784                 .vid = 0x1e0f,
2785                 .mn = "KCD6XVUL6T40",
2786                 .quirks = NVME_QUIRK_NO_APST,
2787         },
2788         {
2789                 /*
2790                  * The external Samsung X5 SSD fails initialization without a
2791                  * delay before checking if it is ready and has a whole set of
2792                  * other problems.  To make this even more interesting, it
2793                  * shares the PCI ID with internal Samsung 970 Evo Plus that
2794                  * does not need or want these quirks.
2795                  */
2796                 .vid = 0x144d,
2797                 .mn = "Samsung Portable SSD X5",
2798                 .quirks = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
2799                           NVME_QUIRK_NO_DEEPEST_PS |
2800                           NVME_QUIRK_IGNORE_DEV_SUBNQN,
2801         }
2802 };
2803
2804 /* match is null-terminated but idstr is space-padded. */
2805 static bool string_matches(const char *idstr, const char *match, size_t len)
2806 {
2807         size_t matchlen;
2808
2809         if (!match)
2810                 return true;
2811
2812         matchlen = strlen(match);
2813         WARN_ON_ONCE(matchlen > len);
2814
2815         if (memcmp(idstr, match, matchlen))
2816                 return false;
2817
2818         for (; matchlen < len; matchlen++)
2819                 if (idstr[matchlen] != ' ')
2820                         return false;
2821
2822         return true;
2823 }
2824
2825 static bool quirk_matches(const struct nvme_id_ctrl *id,
2826                           const struct nvme_core_quirk_entry *q)
2827 {
2828         return q->vid == le16_to_cpu(id->vid) &&
2829                 string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2830                 string_matches(id->fr, q->fr, sizeof(id->fr));
2831 }
2832
2833 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2834                 struct nvme_id_ctrl *id)
2835 {
2836         size_t nqnlen;
2837         int off;
2838
2839         if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2840                 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2841                 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2842                         strscpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2843                         return;
2844                 }
2845
2846                 if (ctrl->vs >= NVME_VS(1, 2, 1))
2847                         dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2848         }
2849
2850         /*
2851          * Generate a "fake" NQN similar to the one in Section 4.5 of the NVMe
2852          * Base Specification 2.0.  It is slightly different from the format
2853          * specified there due to historic reasons, and we can't change it now.
2854          */
2855         off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2856                         "nqn.2014.08.org.nvmexpress:%04x%04x",
2857                         le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
2858         memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2859         off += sizeof(id->sn);
2860         memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2861         off += sizeof(id->mn);
2862         memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2863 }
2864
2865 static void nvme_release_subsystem(struct device *dev)
2866 {
2867         struct nvme_subsystem *subsys =
2868                 container_of(dev, struct nvme_subsystem, dev);
2869
2870         if (subsys->instance >= 0)
2871                 ida_free(&nvme_instance_ida, subsys->instance);
2872         kfree(subsys);
2873 }
2874
2875 static void nvme_destroy_subsystem(struct kref *ref)
2876 {
2877         struct nvme_subsystem *subsys =
2878                         container_of(ref, struct nvme_subsystem, ref);
2879
2880         mutex_lock(&nvme_subsystems_lock);
2881         list_del(&subsys->entry);
2882         mutex_unlock(&nvme_subsystems_lock);
2883
2884         ida_destroy(&subsys->ns_ida);
2885         device_del(&subsys->dev);
2886         put_device(&subsys->dev);
2887 }
2888
2889 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2890 {
2891         kref_put(&subsys->ref, nvme_destroy_subsystem);
2892 }
2893
2894 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2895 {
2896         struct nvme_subsystem *subsys;
2897
2898         lockdep_assert_held(&nvme_subsystems_lock);
2899
2900         /*
2901          * Fail matches for discovery subsystems. This results
2902          * in each discovery controller bound to a unique subsystem.
2903          * This avoids issues with validating controller values
2904          * that can only be true when there is a single unique subsystem.
2905          * There may be multiple and completely independent entities
2906          * that provide discovery controllers.
2907          */
2908         if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2909                 return NULL;
2910
2911         list_for_each_entry(subsys, &nvme_subsystems, entry) {
2912                 if (strcmp(subsys->subnqn, subsysnqn))
2913                         continue;
2914                 if (!kref_get_unless_zero(&subsys->ref))
2915                         continue;
2916                 return subsys;
2917         }
2918
2919         return NULL;
2920 }
2921
2922 static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
2923 {
2924         return ctrl->opts && ctrl->opts->discovery_nqn;
2925 }
2926
2927 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2928                 struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2929 {
2930         struct nvme_ctrl *tmp;
2931
2932         lockdep_assert_held(&nvme_subsystems_lock);
2933
2934         list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
2935                 if (nvme_state_terminal(tmp))
2936                         continue;
2937
2938                 if (tmp->cntlid == ctrl->cntlid) {
2939                         dev_err(ctrl->device,
2940                                 "Duplicate cntlid %u with %s, subsys %s, rejecting\n",
2941                                 ctrl->cntlid, dev_name(tmp->device),
2942                                 subsys->subnqn);
2943                         return false;
2944                 }
2945
2946                 if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
2947                     nvme_discovery_ctrl(ctrl))
2948                         continue;
2949
2950                 dev_err(ctrl->device,
2951                         "Subsystem does not support multiple controllers\n");
2952                 return false;
2953         }
2954
2955         return true;
2956 }
2957
2958 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2959 {
2960         struct nvme_subsystem *subsys, *found;
2961         int ret;
2962
2963         subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2964         if (!subsys)
2965                 return -ENOMEM;
2966
2967         subsys->instance = -1;
2968         mutex_init(&subsys->lock);
2969         kref_init(&subsys->ref);
2970         INIT_LIST_HEAD(&subsys->ctrls);
2971         INIT_LIST_HEAD(&subsys->nsheads);
2972         nvme_init_subnqn(subsys, ctrl, id);
2973         memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2974         memcpy(subsys->model, id->mn, sizeof(subsys->model));
2975         subsys->vendor_id = le16_to_cpu(id->vid);
2976         subsys->cmic = id->cmic;
2977
2978         /* Versions prior to 1.4 don't necessarily report a valid type */
2979         if (id->cntrltype == NVME_CTRL_DISC ||
2980             !strcmp(subsys->subnqn, NVME_DISC_SUBSYS_NAME))
2981                 subsys->subtype = NVME_NQN_DISC;
2982         else
2983                 subsys->subtype = NVME_NQN_NVME;
2984
2985         if (nvme_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) {
2986                 dev_err(ctrl->device,
2987                         "Subsystem %s is not a discovery controller",
2988                         subsys->subnqn);
2989                 kfree(subsys);
2990                 return -EINVAL;
2991         }
2992         subsys->awupf = le16_to_cpu(id->awupf);
2993         nvme_mpath_default_iopolicy(subsys);
2994
2995         subsys->dev.class = &nvme_subsys_class;
2996         subsys->dev.release = nvme_release_subsystem;
2997         subsys->dev.groups = nvme_subsys_attrs_groups;
2998         dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
2999         device_initialize(&subsys->dev);
3000
3001         mutex_lock(&nvme_subsystems_lock);
3002         found = __nvme_find_get_subsystem(subsys->subnqn);
3003         if (found) {
3004                 put_device(&subsys->dev);
3005                 subsys = found;
3006
3007                 if (!nvme_validate_cntlid(subsys, ctrl, id)) {
3008                         ret = -EINVAL;
3009                         goto out_put_subsystem;
3010                 }
3011         } else {
3012                 ret = device_add(&subsys->dev);
3013                 if (ret) {
3014                         dev_err(ctrl->device,
3015                                 "failed to register subsystem device.\n");
3016                         put_device(&subsys->dev);
3017                         goto out_unlock;
3018                 }
3019                 ida_init(&subsys->ns_ida);
3020                 list_add_tail(&subsys->entry, &nvme_subsystems);
3021         }
3022
3023         ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
3024                                 dev_name(ctrl->device));
3025         if (ret) {
3026                 dev_err(ctrl->device,
3027                         "failed to create sysfs link from subsystem.\n");
3028                 goto out_put_subsystem;
3029         }
3030
3031         if (!found)
3032                 subsys->instance = ctrl->instance;
3033         ctrl->subsys = subsys;
3034         list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
3035         mutex_unlock(&nvme_subsystems_lock);
3036         return 0;
3037
3038 out_put_subsystem:
3039         nvme_put_subsystem(subsys);
3040 out_unlock:
3041         mutex_unlock(&nvme_subsystems_lock);
3042         return ret;
3043 }
3044
3045 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
3046                 void *log, size_t size, u64 offset)
3047 {
3048         struct nvme_command c = { };
3049         u32 dwlen = nvme_bytes_to_numd(size);
3050
3051         c.get_log_page.opcode = nvme_admin_get_log_page;
3052         c.get_log_page.nsid = cpu_to_le32(nsid);
3053         c.get_log_page.lid = log_page;
3054         c.get_log_page.lsp = lsp;
3055         c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
3056         c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
3057         c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
3058         c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
3059         c.get_log_page.csi = csi;
3060
3061         return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
3062 }
3063
3064 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
3065                                 struct nvme_effects_log **log)
3066 {
3067         struct nvme_effects_log *cel = xa_load(&ctrl->cels, csi);
3068         int ret;
3069
3070         if (cel)
3071                 goto out;
3072
3073         cel = kzalloc(sizeof(*cel), GFP_KERNEL);
3074         if (!cel)
3075                 return -ENOMEM;
3076
3077         ret = nvme_get_log(ctrl, 0x00, NVME_LOG_CMD_EFFECTS, 0, csi,
3078                         cel, sizeof(*cel), 0);
3079         if (ret) {
3080                 kfree(cel);
3081                 return ret;
3082         }
3083
3084         xa_store(&ctrl->cels, csi, cel, GFP_KERNEL);
3085 out:
3086         *log = cel;
3087         return 0;
3088 }
3089
3090 static inline u32 nvme_mps_to_sectors(struct nvme_ctrl *ctrl, u32 units)
3091 {
3092         u32 page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12, val;
3093
3094         if (check_shl_overflow(1U, units + page_shift - 9, &val))
3095                 return UINT_MAX;
3096         return val;
3097 }
3098
3099 static int nvme_init_non_mdts_limits(struct nvme_ctrl *ctrl)
3100 {
3101         struct nvme_command c = { };
3102         struct nvme_id_ctrl_nvm *id;
3103         int ret;
3104
3105         /*
3106          * Even though NVMe spec explicitly states that MDTS is not applicable
3107          * to the write-zeroes, we are cautious and limit the size to the
3108          * controllers max_hw_sectors value, which is based on the MDTS field
3109          * and possibly other limiting factors.
3110          */
3111         if ((ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) &&
3112             !(ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
3113                 ctrl->max_zeroes_sectors = ctrl->max_hw_sectors;
3114         else
3115                 ctrl->max_zeroes_sectors = 0;
3116
3117         if (ctrl->subsys->subtype != NVME_NQN_NVME ||
3118             nvme_ctrl_limited_cns(ctrl) ||
3119             test_bit(NVME_CTRL_SKIP_ID_CNS_CS, &ctrl->flags))
3120                 return 0;
3121
3122         id = kzalloc(sizeof(*id), GFP_KERNEL);
3123         if (!id)
3124                 return -ENOMEM;
3125
3126         c.identify.opcode = nvme_admin_identify;
3127         c.identify.cns = NVME_ID_CNS_CS_CTRL;
3128         c.identify.csi = NVME_CSI_NVM;
3129
3130         ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
3131         if (ret)
3132                 goto free_data;
3133
3134         ctrl->dmrl = id->dmrl;
3135         ctrl->dmrsl = le32_to_cpu(id->dmrsl);
3136         if (id->wzsl)
3137                 ctrl->max_zeroes_sectors = nvme_mps_to_sectors(ctrl, id->wzsl);
3138
3139 free_data:
3140         if (ret > 0)
3141                 set_bit(NVME_CTRL_SKIP_ID_CNS_CS, &ctrl->flags);
3142         kfree(id);
3143         return ret;
3144 }
3145
3146 static void nvme_init_known_nvm_effects(struct nvme_ctrl *ctrl)
3147 {
3148         struct nvme_effects_log *log = ctrl->effects;
3149
3150         log->acs[nvme_admin_format_nvm] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC |
3151                                                 NVME_CMD_EFFECTS_NCC |
3152                                                 NVME_CMD_EFFECTS_CSE_MASK);
3153         log->acs[nvme_admin_sanitize_nvm] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC |
3154                                                 NVME_CMD_EFFECTS_CSE_MASK);
3155
3156         /*
3157          * The spec says the result of a security receive command depends on
3158          * the previous security send command. As such, many vendors log this
3159          * command as one to submitted only when no other commands to the same
3160          * namespace are outstanding. The intention is to tell the host to
3161          * prevent mixing security send and receive.
3162          *
3163          * This driver can only enforce such exclusive access against IO
3164          * queues, though. We are not readily able to enforce such a rule for
3165          * two commands to the admin queue, which is the only queue that
3166          * matters for this command.
3167          *
3168          * Rather than blindly freezing the IO queues for this effect that
3169          * doesn't even apply to IO, mask it off.
3170          */
3171         log->acs[nvme_admin_security_recv] &= cpu_to_le32(~NVME_CMD_EFFECTS_CSE_MASK);
3172
3173         log->iocs[nvme_cmd_write] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
3174         log->iocs[nvme_cmd_write_zeroes] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
3175         log->iocs[nvme_cmd_write_uncor] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
3176 }
3177
3178 static int nvme_init_effects(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
3179 {
3180         int ret = 0;
3181
3182         if (ctrl->effects)
3183                 return 0;
3184
3185         if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
3186                 ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
3187                 if (ret < 0)
3188                         return ret;
3189         }
3190
3191         if (!ctrl->effects) {
3192                 ctrl->effects = kzalloc(sizeof(*ctrl->effects), GFP_KERNEL);
3193                 if (!ctrl->effects)
3194                         return -ENOMEM;
3195                 xa_store(&ctrl->cels, NVME_CSI_NVM, ctrl->effects, GFP_KERNEL);
3196         }
3197
3198         nvme_init_known_nvm_effects(ctrl);
3199         return 0;
3200 }
3201
3202 static int nvme_check_ctrl_fabric_info(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
3203 {
3204         /*
3205          * In fabrics we need to verify the cntlid matches the
3206          * admin connect
3207          */
3208         if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
3209                 dev_err(ctrl->device,
3210                         "Mismatching cntlid: Connect %u vs Identify %u, rejecting\n",
3211                         ctrl->cntlid, le16_to_cpu(id->cntlid));
3212                 return -EINVAL;
3213         }
3214
3215         if (!nvme_discovery_ctrl(ctrl) && !ctrl->kas) {
3216                 dev_err(ctrl->device,
3217                         "keep-alive support is mandatory for fabrics\n");
3218                 return -EINVAL;
3219         }
3220
3221         if (!nvme_discovery_ctrl(ctrl) && ctrl->ioccsz < 4) {
3222                 dev_err(ctrl->device,
3223                         "I/O queue command capsule supported size %d < 4\n",
3224                         ctrl->ioccsz);
3225                 return -EINVAL;
3226         }
3227
3228         if (!nvme_discovery_ctrl(ctrl) && ctrl->iorcsz < 1) {
3229                 dev_err(ctrl->device,
3230                         "I/O queue response capsule supported size %d < 1\n",
3231                         ctrl->iorcsz);
3232                 return -EINVAL;
3233         }
3234
3235         if (!ctrl->maxcmd) {
3236                 dev_err(ctrl->device, "Maximum outstanding commands is 0\n");
3237                 return -EINVAL;
3238         }
3239
3240         return 0;
3241 }
3242
3243 static int nvme_init_identify(struct nvme_ctrl *ctrl)
3244 {
3245         struct queue_limits lim;
3246         struct nvme_id_ctrl *id;
3247         u32 max_hw_sectors;
3248         bool prev_apst_enabled;
3249         int ret;
3250
3251         ret = nvme_identify_ctrl(ctrl, &id);
3252         if (ret) {
3253                 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
3254                 return -EIO;
3255         }
3256
3257         if (!(ctrl->ops->flags & NVME_F_FABRICS))
3258                 ctrl->cntlid = le16_to_cpu(id->cntlid);
3259
3260         if (!ctrl->identified) {
3261                 unsigned int i;
3262
3263                 /*
3264                  * Check for quirks.  Quirk can depend on firmware version,
3265                  * so, in principle, the set of quirks present can change
3266                  * across a reset.  As a possible future enhancement, we
3267                  * could re-scan for quirks every time we reinitialize
3268                  * the device, but we'd have to make sure that the driver
3269                  * behaves intelligently if the quirks change.
3270                  */
3271                 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
3272                         if (quirk_matches(id, &core_quirks[i]))
3273                                 ctrl->quirks |= core_quirks[i].quirks;
3274                 }
3275
3276                 ret = nvme_init_subsystem(ctrl, id);
3277                 if (ret)
3278                         goto out_free;
3279
3280                 ret = nvme_init_effects(ctrl, id);
3281                 if (ret)
3282                         goto out_free;
3283         }
3284         memcpy(ctrl->subsys->firmware_rev, id->fr,
3285                sizeof(ctrl->subsys->firmware_rev));
3286
3287         if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
3288                 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
3289                 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
3290         }
3291
3292         ctrl->crdt[0] = le16_to_cpu(id->crdt1);
3293         ctrl->crdt[1] = le16_to_cpu(id->crdt2);
3294         ctrl->crdt[2] = le16_to_cpu(id->crdt3);
3295
3296         ctrl->oacs = le16_to_cpu(id->oacs);
3297         ctrl->oncs = le16_to_cpu(id->oncs);
3298         ctrl->mtfa = le16_to_cpu(id->mtfa);
3299         ctrl->oaes = le32_to_cpu(id->oaes);
3300         ctrl->wctemp = le16_to_cpu(id->wctemp);
3301         ctrl->cctemp = le16_to_cpu(id->cctemp);
3302
3303         atomic_set(&ctrl->abort_limit, id->acl + 1);
3304         ctrl->vwc = id->vwc;
3305         if (id->mdts)
3306                 max_hw_sectors = nvme_mps_to_sectors(ctrl, id->mdts);
3307         else
3308                 max_hw_sectors = UINT_MAX;
3309         ctrl->max_hw_sectors =
3310                 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
3311
3312         lim = queue_limits_start_update(ctrl->admin_q);
3313         nvme_set_ctrl_limits(ctrl, &lim);
3314         ret = queue_limits_commit_update(ctrl->admin_q, &lim);
3315         if (ret)
3316                 goto out_free;
3317
3318         ctrl->sgls = le32_to_cpu(id->sgls);
3319         ctrl->kas = le16_to_cpu(id->kas);
3320         ctrl->max_namespaces = le32_to_cpu(id->mnan);
3321         ctrl->ctratt = le32_to_cpu(id->ctratt);
3322
3323         ctrl->cntrltype = id->cntrltype;
3324         ctrl->dctype = id->dctype;
3325
3326         if (id->rtd3e) {
3327                 /* us -> s */
3328                 u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
3329
3330                 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
3331                                                  shutdown_timeout, 60);
3332
3333                 if (ctrl->shutdown_timeout != shutdown_timeout)
3334                         dev_info(ctrl->device,
3335                                  "D3 entry latency set to %u seconds\n",
3336                                  ctrl->shutdown_timeout);
3337         } else
3338                 ctrl->shutdown_timeout = shutdown_timeout;
3339
3340         ctrl->npss = id->npss;
3341         ctrl->apsta = id->apsta;
3342         prev_apst_enabled = ctrl->apst_enabled;
3343         if (ctrl->quirks & NVME_QUIRK_NO_APST) {
3344                 if (force_apst && id->apsta) {
3345                         dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
3346                         ctrl->apst_enabled = true;
3347                 } else {
3348                         ctrl->apst_enabled = false;
3349                 }
3350         } else {
3351                 ctrl->apst_enabled = id->apsta;
3352         }
3353         memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
3354
3355         if (ctrl->ops->flags & NVME_F_FABRICS) {
3356                 ctrl->icdoff = le16_to_cpu(id->icdoff);
3357                 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
3358                 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
3359                 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
3360
3361                 ret = nvme_check_ctrl_fabric_info(ctrl, id);
3362                 if (ret)
3363                         goto out_free;
3364         } else {
3365                 ctrl->hmpre = le32_to_cpu(id->hmpre);
3366                 ctrl->hmmin = le32_to_cpu(id->hmmin);
3367                 ctrl->hmminds = le32_to_cpu(id->hmminds);
3368                 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
3369         }
3370
3371         ret = nvme_mpath_init_identify(ctrl, id);
3372         if (ret < 0)
3373                 goto out_free;
3374
3375         if (ctrl->apst_enabled && !prev_apst_enabled)
3376                 dev_pm_qos_expose_latency_tolerance(ctrl->device);
3377         else if (!ctrl->apst_enabled && prev_apst_enabled)
3378                 dev_pm_qos_hide_latency_tolerance(ctrl->device);
3379
3380 out_free:
3381         kfree(id);
3382         return ret;
3383 }
3384
3385 /*
3386  * Initialize the cached copies of the Identify data and various controller
3387  * register in our nvme_ctrl structure.  This should be called as soon as
3388  * the admin queue is fully up and running.
3389  */
3390 int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl, bool was_suspended)
3391 {
3392         int ret;
3393
3394         ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
3395         if (ret) {
3396                 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
3397                 return ret;
3398         }
3399
3400         ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
3401
3402         if (ctrl->vs >= NVME_VS(1, 1, 0))
3403                 ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
3404
3405         ret = nvme_init_identify(ctrl);
3406         if (ret)
3407                 return ret;
3408
3409         ret = nvme_configure_apst(ctrl);
3410         if (ret < 0)
3411                 return ret;
3412
3413         ret = nvme_configure_timestamp(ctrl);
3414         if (ret < 0)
3415                 return ret;
3416
3417         ret = nvme_configure_host_options(ctrl);
3418         if (ret < 0)
3419                 return ret;
3420
3421         nvme_configure_opal(ctrl, was_suspended);
3422
3423         if (!ctrl->identified && !nvme_discovery_ctrl(ctrl)) {
3424                 /*
3425                  * Do not return errors unless we are in a controller reset,
3426                  * the controller works perfectly fine without hwmon.
3427                  */
3428                 ret = nvme_hwmon_init(ctrl);
3429                 if (ret == -EINTR)
3430                         return ret;
3431         }
3432
3433         clear_bit(NVME_CTRL_DIRTY_CAPABILITY, &ctrl->flags);
3434         ctrl->identified = true;
3435
3436         nvme_start_keep_alive(ctrl);
3437
3438         return 0;
3439 }
3440 EXPORT_SYMBOL_GPL(nvme_init_ctrl_finish);
3441
3442 static int nvme_dev_open(struct inode *inode, struct file *file)
3443 {
3444         struct nvme_ctrl *ctrl =
3445                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3446
3447         switch (nvme_ctrl_state(ctrl)) {
3448         case NVME_CTRL_LIVE:
3449                 break;
3450         default:
3451                 return -EWOULDBLOCK;
3452         }
3453
3454         nvme_get_ctrl(ctrl);
3455         if (!try_module_get(ctrl->ops->module)) {
3456                 nvme_put_ctrl(ctrl);
3457                 return -EINVAL;
3458         }
3459
3460         file->private_data = ctrl;
3461         return 0;
3462 }
3463
3464 static int nvme_dev_release(struct inode *inode, struct file *file)
3465 {
3466         struct nvme_ctrl *ctrl =
3467                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3468
3469         module_put(ctrl->ops->module);
3470         nvme_put_ctrl(ctrl);
3471         return 0;
3472 }
3473
3474 static const struct file_operations nvme_dev_fops = {
3475         .owner          = THIS_MODULE,
3476         .open           = nvme_dev_open,
3477         .release        = nvme_dev_release,
3478         .unlocked_ioctl = nvme_dev_ioctl,
3479         .compat_ioctl   = compat_ptr_ioctl,
3480         .uring_cmd      = nvme_dev_uring_cmd,
3481 };
3482
3483 static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl,
3484                 unsigned nsid)
3485 {
3486         struct nvme_ns_head *h;
3487
3488         lockdep_assert_held(&ctrl->subsys->lock);
3489
3490         list_for_each_entry(h, &ctrl->subsys->nsheads, entry) {
3491                 /*
3492                  * Private namespaces can share NSIDs under some conditions.
3493                  * In that case we can't use the same ns_head for namespaces
3494                  * with the same NSID.
3495                  */
3496                 if (h->ns_id != nsid || !nvme_is_unique_nsid(ctrl, h))
3497                         continue;
3498                 if (!list_empty(&h->list) && nvme_tryget_ns_head(h))
3499                         return h;
3500         }
3501
3502         return NULL;
3503 }
3504
3505 static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys,
3506                 struct nvme_ns_ids *ids)
3507 {
3508         bool has_uuid = !uuid_is_null(&ids->uuid);
3509         bool has_nguid = memchr_inv(ids->nguid, 0, sizeof(ids->nguid));
3510         bool has_eui64 = memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
3511         struct nvme_ns_head *h;
3512
3513         lockdep_assert_held(&subsys->lock);
3514
3515         list_for_each_entry(h, &subsys->nsheads, entry) {
3516                 if (has_uuid && uuid_equal(&ids->uuid, &h->ids.uuid))
3517                         return -EINVAL;
3518                 if (has_nguid &&
3519                     memcmp(&ids->nguid, &h->ids.nguid, sizeof(ids->nguid)) == 0)
3520                         return -EINVAL;
3521                 if (has_eui64 &&
3522                     memcmp(&ids->eui64, &h->ids.eui64, sizeof(ids->eui64)) == 0)
3523                         return -EINVAL;
3524         }
3525
3526         return 0;
3527 }
3528
3529 static void nvme_cdev_rel(struct device *dev)
3530 {
3531         ida_free(&nvme_ns_chr_minor_ida, MINOR(dev->devt));
3532 }
3533
3534 void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device)
3535 {
3536         cdev_device_del(cdev, cdev_device);
3537         put_device(cdev_device);
3538 }
3539
3540 int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device,
3541                 const struct file_operations *fops, struct module *owner)
3542 {
3543         int minor, ret;
3544
3545         minor = ida_alloc(&nvme_ns_chr_minor_ida, GFP_KERNEL);
3546         if (minor < 0)
3547                 return minor;
3548         cdev_device->devt = MKDEV(MAJOR(nvme_ns_chr_devt), minor);
3549         cdev_device->class = &nvme_ns_chr_class;
3550         cdev_device->release = nvme_cdev_rel;
3551         device_initialize(cdev_device);
3552         cdev_init(cdev, fops);
3553         cdev->owner = owner;
3554         ret = cdev_device_add(cdev, cdev_device);
3555         if (ret)
3556                 put_device(cdev_device);
3557
3558         return ret;
3559 }
3560
3561 static int nvme_ns_chr_open(struct inode *inode, struct file *file)
3562 {
3563         return nvme_ns_open(container_of(inode->i_cdev, struct nvme_ns, cdev));
3564 }
3565
3566 static int nvme_ns_chr_release(struct inode *inode, struct file *file)
3567 {
3568         nvme_ns_release(container_of(inode->i_cdev, struct nvme_ns, cdev));
3569         return 0;
3570 }
3571
3572 static const struct file_operations nvme_ns_chr_fops = {
3573         .owner          = THIS_MODULE,
3574         .open           = nvme_ns_chr_open,
3575         .release        = nvme_ns_chr_release,
3576         .unlocked_ioctl = nvme_ns_chr_ioctl,
3577         .compat_ioctl   = compat_ptr_ioctl,
3578         .uring_cmd      = nvme_ns_chr_uring_cmd,
3579         .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll,
3580 };
3581
3582 static int nvme_add_ns_cdev(struct nvme_ns *ns)
3583 {
3584         int ret;
3585
3586         ns->cdev_device.parent = ns->ctrl->device;
3587         ret = dev_set_name(&ns->cdev_device, "ng%dn%d",
3588                            ns->ctrl->instance, ns->head->instance);
3589         if (ret)
3590                 return ret;
3591
3592         return nvme_cdev_add(&ns->cdev, &ns->cdev_device, &nvme_ns_chr_fops,
3593                              ns->ctrl->ops->module);
3594 }
3595
3596 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
3597                 struct nvme_ns_info *info)
3598 {
3599         struct nvme_ns_head *head;
3600         size_t size = sizeof(*head);
3601         int ret = -ENOMEM;
3602
3603 #ifdef CONFIG_NVME_MULTIPATH
3604         size += num_possible_nodes() * sizeof(struct nvme_ns *);
3605 #endif
3606
3607         head = kzalloc(size, GFP_KERNEL);
3608         if (!head)
3609                 goto out;
3610         ret = ida_alloc_min(&ctrl->subsys->ns_ida, 1, GFP_KERNEL);
3611         if (ret < 0)
3612                 goto out_free_head;
3613         head->instance = ret;
3614         INIT_LIST_HEAD(&head->list);
3615         ret = init_srcu_struct(&head->srcu);
3616         if (ret)
3617                 goto out_ida_remove;
3618         head->subsys = ctrl->subsys;
3619         head->ns_id = info->nsid;
3620         head->ids = info->ids;
3621         head->shared = info->is_shared;
3622         ratelimit_state_init(&head->rs_nuse, 5 * HZ, 1);
3623         ratelimit_set_flags(&head->rs_nuse, RATELIMIT_MSG_ON_RELEASE);
3624         kref_init(&head->ref);
3625
3626         if (head->ids.csi) {
3627                 ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
3628                 if (ret)
3629                         goto out_cleanup_srcu;
3630         } else
3631                 head->effects = ctrl->effects;
3632
3633         ret = nvme_mpath_alloc_disk(ctrl, head);
3634         if (ret)
3635                 goto out_cleanup_srcu;
3636
3637         list_add_tail(&head->entry, &ctrl->subsys->nsheads);
3638
3639         kref_get(&ctrl->subsys->ref);
3640
3641         return head;
3642 out_cleanup_srcu:
3643         cleanup_srcu_struct(&head->srcu);
3644 out_ida_remove:
3645         ida_free(&ctrl->subsys->ns_ida, head->instance);
3646 out_free_head:
3647         kfree(head);
3648 out:
3649         if (ret > 0)
3650                 ret = blk_status_to_errno(nvme_error_status(ret));
3651         return ERR_PTR(ret);
3652 }
3653
3654 static int nvme_global_check_duplicate_ids(struct nvme_subsystem *this,
3655                 struct nvme_ns_ids *ids)
3656 {
3657         struct nvme_subsystem *s;
3658         int ret = 0;
3659
3660         /*
3661          * Note that this check is racy as we try to avoid holding the global
3662          * lock over the whole ns_head creation.  But it is only intended as
3663          * a sanity check anyway.
3664          */
3665         mutex_lock(&nvme_subsystems_lock);
3666         list_for_each_entry(s, &nvme_subsystems, entry) {
3667                 if (s == this)
3668                         continue;
3669                 mutex_lock(&s->lock);
3670                 ret = nvme_subsys_check_duplicate_ids(s, ids);
3671                 mutex_unlock(&s->lock);
3672                 if (ret)
3673                         break;
3674         }
3675         mutex_unlock(&nvme_subsystems_lock);
3676
3677         return ret;
3678 }
3679
3680 static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
3681 {
3682         struct nvme_ctrl *ctrl = ns->ctrl;
3683         struct nvme_ns_head *head = NULL;
3684         int ret;
3685
3686         ret = nvme_global_check_duplicate_ids(ctrl->subsys, &info->ids);
3687         if (ret) {
3688                 /*
3689                  * We've found two different namespaces on two different
3690                  * subsystems that report the same ID.  This is pretty nasty
3691                  * for anything that actually requires unique device
3692                  * identification.  In the kernel we need this for multipathing,
3693                  * and in user space the /dev/disk/by-id/ links rely on it.
3694                  *
3695                  * If the device also claims to be multi-path capable back off
3696                  * here now and refuse the probe the second device as this is a
3697                  * recipe for data corruption.  If not this is probably a
3698                  * cheap consumer device if on the PCIe bus, so let the user
3699                  * proceed and use the shiny toy, but warn that with changing
3700                  * probing order (which due to our async probing could just be
3701                  * device taking longer to startup) the other device could show
3702                  * up at any time.
3703                  */
3704                 nvme_print_device_info(ctrl);
3705                 if ((ns->ctrl->ops->flags & NVME_F_FABRICS) || /* !PCIe */
3706                     ((ns->ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) &&
3707                      info->is_shared)) {
3708                         dev_err(ctrl->device,
3709                                 "ignoring nsid %d because of duplicate IDs\n",
3710                                 info->nsid);
3711                         return ret;
3712                 }
3713
3714                 dev_err(ctrl->device,
3715                         "clearing duplicate IDs for nsid %d\n", info->nsid);
3716                 dev_err(ctrl->device,
3717                         "use of /dev/disk/by-id/ may cause data corruption\n");
3718                 memset(&info->ids.nguid, 0, sizeof(info->ids.nguid));
3719                 memset(&info->ids.uuid, 0, sizeof(info->ids.uuid));
3720                 memset(&info->ids.eui64, 0, sizeof(info->ids.eui64));
3721                 ctrl->quirks |= NVME_QUIRK_BOGUS_NID;
3722         }
3723
3724         mutex_lock(&ctrl->subsys->lock);
3725         head = nvme_find_ns_head(ctrl, info->nsid);
3726         if (!head) {
3727                 ret = nvme_subsys_check_duplicate_ids(ctrl->subsys, &info->ids);
3728                 if (ret) {
3729                         dev_err(ctrl->device,
3730                                 "duplicate IDs in subsystem for nsid %d\n",
3731                                 info->nsid);
3732                         goto out_unlock;
3733                 }
3734                 head = nvme_alloc_ns_head(ctrl, info);
3735                 if (IS_ERR(head)) {
3736                         ret = PTR_ERR(head);
3737                         goto out_unlock;
3738                 }
3739         } else {
3740                 ret = -EINVAL;
3741                 if (!info->is_shared || !head->shared) {
3742                         dev_err(ctrl->device,
3743                                 "Duplicate unshared namespace %d\n",
3744                                 info->nsid);
3745                         goto out_put_ns_head;
3746                 }
3747                 if (!nvme_ns_ids_equal(&head->ids, &info->ids)) {
3748                         dev_err(ctrl->device,
3749                                 "IDs don't match for shared namespace %d\n",
3750                                         info->nsid);
3751                         goto out_put_ns_head;
3752                 }
3753
3754                 if (!multipath) {
3755                         dev_warn(ctrl->device,
3756                                 "Found shared namespace %d, but multipathing not supported.\n",
3757                                 info->nsid);
3758                         dev_warn_once(ctrl->device,
3759                                 "Support for shared namespaces without CONFIG_NVME_MULTIPATH is deprecated and will be removed in Linux 6.0.\n");
3760                 }
3761         }
3762
3763         list_add_tail_rcu(&ns->siblings, &head->list);
3764         ns->head = head;
3765         mutex_unlock(&ctrl->subsys->lock);
3766         return 0;
3767
3768 out_put_ns_head:
3769         nvme_put_ns_head(head);
3770 out_unlock:
3771         mutex_unlock(&ctrl->subsys->lock);
3772         return ret;
3773 }
3774
3775 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3776 {
3777         struct nvme_ns *ns, *ret = NULL;
3778         int srcu_idx;
3779
3780         srcu_idx = srcu_read_lock(&ctrl->srcu);
3781         list_for_each_entry_rcu(ns, &ctrl->namespaces, list) {
3782                 if (ns->head->ns_id == nsid) {
3783                         if (!nvme_get_ns(ns))
3784                                 continue;
3785                         ret = ns;
3786                         break;
3787                 }
3788                 if (ns->head->ns_id > nsid)
3789                         break;
3790         }
3791         srcu_read_unlock(&ctrl->srcu, srcu_idx);
3792         return ret;
3793 }
3794 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
3795
3796 /*
3797  * Add the namespace to the controller list while keeping the list ordered.
3798  */
3799 static void nvme_ns_add_to_ctrl_list(struct nvme_ns *ns)
3800 {
3801         struct nvme_ns *tmp;
3802
3803         list_for_each_entry_reverse(tmp, &ns->ctrl->namespaces, list) {
3804                 if (tmp->head->ns_id < ns->head->ns_id) {
3805                         list_add_rcu(&ns->list, &tmp->list);
3806                         return;
3807                 }
3808         }
3809         list_add(&ns->list, &ns->ctrl->namespaces);
3810 }
3811
3812 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info)
3813 {
3814         struct queue_limits lim = { };
3815         struct nvme_ns *ns;
3816         struct gendisk *disk;
3817         int node = ctrl->numa_node;
3818
3819         ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
3820         if (!ns)
3821                 return;
3822
3823         if (ctrl->opts && ctrl->opts->data_digest)
3824                 lim.features |= BLK_FEAT_STABLE_WRITES;
3825         if (ctrl->ops->supports_pci_p2pdma &&
3826             ctrl->ops->supports_pci_p2pdma(ctrl))
3827                 lim.features |= BLK_FEAT_PCI_P2PDMA;
3828
3829         disk = blk_mq_alloc_disk(ctrl->tagset, &lim, ns);
3830         if (IS_ERR(disk))
3831                 goto out_free_ns;
3832         disk->fops = &nvme_bdev_ops;
3833         disk->private_data = ns;
3834
3835         ns->disk = disk;
3836         ns->queue = disk->queue;
3837         ns->ctrl = ctrl;
3838         kref_init(&ns->kref);
3839
3840         if (nvme_init_ns_head(ns, info))
3841                 goto out_cleanup_disk;
3842
3843         /*
3844          * If multipathing is enabled, the device name for all disks and not
3845          * just those that represent shared namespaces needs to be based on the
3846          * subsystem instance.  Using the controller instance for private
3847          * namespaces could lead to naming collisions between shared and private
3848          * namespaces if they don't use a common numbering scheme.
3849          *
3850          * If multipathing is not enabled, disk names must use the controller
3851          * instance as shared namespaces will show up as multiple block
3852          * devices.
3853          */
3854         if (nvme_ns_head_multipath(ns->head)) {
3855                 sprintf(disk->disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
3856                         ctrl->instance, ns->head->instance);
3857                 disk->flags |= GENHD_FL_HIDDEN;
3858         } else if (multipath) {
3859                 sprintf(disk->disk_name, "nvme%dn%d", ctrl->subsys->instance,
3860                         ns->head->instance);
3861         } else {
3862                 sprintf(disk->disk_name, "nvme%dn%d", ctrl->instance,
3863                         ns->head->instance);
3864         }
3865
3866         if (nvme_update_ns_info(ns, info))
3867                 goto out_unlink_ns;
3868
3869         mutex_lock(&ctrl->namespaces_lock);
3870         /*
3871          * Ensure that no namespaces are added to the ctrl list after the queues
3872          * are frozen, thereby avoiding a deadlock between scan and reset.
3873          */
3874         if (test_bit(NVME_CTRL_FROZEN, &ctrl->flags)) {
3875                 mutex_unlock(&ctrl->namespaces_lock);
3876                 goto out_unlink_ns;
3877         }
3878         nvme_ns_add_to_ctrl_list(ns);
3879         mutex_unlock(&ctrl->namespaces_lock);
3880         synchronize_srcu(&ctrl->srcu);
3881         nvme_get_ctrl(ctrl);
3882
3883         if (device_add_disk(ctrl->device, ns->disk, nvme_ns_attr_groups))
3884                 goto out_cleanup_ns_from_list;
3885
3886         if (!nvme_ns_head_multipath(ns->head))
3887                 nvme_add_ns_cdev(ns);
3888
3889         nvme_mpath_add_disk(ns, info->anagrpid);
3890         nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
3891
3892         /*
3893          * Set ns->disk->device->driver_data to ns so we can access
3894          * ns->head->passthru_err_log_enabled in
3895          * nvme_io_passthru_err_log_enabled_[store | show]().
3896          */
3897         dev_set_drvdata(disk_to_dev(ns->disk), ns);
3898
3899         return;
3900
3901  out_cleanup_ns_from_list:
3902         nvme_put_ctrl(ctrl);
3903         mutex_lock(&ctrl->namespaces_lock);
3904         list_del_rcu(&ns->list);
3905         mutex_unlock(&ctrl->namespaces_lock);
3906         synchronize_srcu(&ctrl->srcu);
3907  out_unlink_ns:
3908         mutex_lock(&ctrl->subsys->lock);
3909         list_del_rcu(&ns->siblings);
3910         if (list_empty(&ns->head->list))
3911                 list_del_init(&ns->head->entry);
3912         mutex_unlock(&ctrl->subsys->lock);
3913         nvme_put_ns_head(ns->head);
3914  out_cleanup_disk:
3915         put_disk(disk);
3916  out_free_ns:
3917         kfree(ns);
3918 }
3919
3920 static void nvme_ns_remove(struct nvme_ns *ns)
3921 {
3922         bool last_path = false;
3923
3924         if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
3925                 return;
3926
3927         clear_bit(NVME_NS_READY, &ns->flags);
3928         set_capacity(ns->disk, 0);
3929         nvme_fault_inject_fini(&ns->fault_inject);
3930
3931         /*
3932          * Ensure that !NVME_NS_READY is seen by other threads to prevent
3933          * this ns going back into current_path.
3934          */
3935         synchronize_srcu(&ns->head->srcu);
3936
3937         /* wait for concurrent submissions */
3938         if (nvme_mpath_clear_current_path(ns))
3939                 synchronize_srcu(&ns->head->srcu);
3940
3941         mutex_lock(&ns->ctrl->subsys->lock);
3942         list_del_rcu(&ns->siblings);
3943         if (list_empty(&ns->head->list)) {
3944                 list_del_init(&ns->head->entry);
3945                 last_path = true;
3946         }
3947         mutex_unlock(&ns->ctrl->subsys->lock);
3948
3949         /* guarantee not available in head->list */
3950         synchronize_srcu(&ns->head->srcu);
3951
3952         if (!nvme_ns_head_multipath(ns->head))
3953                 nvme_cdev_del(&ns->cdev, &ns->cdev_device);
3954         del_gendisk(ns->disk);
3955
3956         mutex_lock(&ns->ctrl->namespaces_lock);
3957         list_del_rcu(&ns->list);
3958         mutex_unlock(&ns->ctrl->namespaces_lock);
3959         synchronize_srcu(&ns->ctrl->srcu);
3960
3961         if (last_path)
3962                 nvme_mpath_shutdown_disk(ns->head);
3963         nvme_put_ns(ns);
3964 }
3965
3966 static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
3967 {
3968         struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
3969
3970         if (ns) {
3971                 nvme_ns_remove(ns);
3972                 nvme_put_ns(ns);
3973         }
3974 }
3975
3976 static void nvme_validate_ns(struct nvme_ns *ns, struct nvme_ns_info *info)
3977 {
3978         int ret = NVME_SC_INVALID_NS | NVME_STATUS_DNR;
3979
3980         if (!nvme_ns_ids_equal(&ns->head->ids, &info->ids)) {
3981                 dev_err(ns->ctrl->device,
3982                         "identifiers changed for nsid %d\n", ns->head->ns_id);
3983                 goto out;
3984         }
3985
3986         ret = nvme_update_ns_info(ns, info);
3987 out:
3988         /*
3989          * Only remove the namespace if we got a fatal error back from the
3990          * device, otherwise ignore the error and just move on.
3991          *
3992          * TODO: we should probably schedule a delayed retry here.
3993          */
3994         if (ret > 0 && (ret & NVME_STATUS_DNR))
3995                 nvme_ns_remove(ns);
3996 }
3997
3998 static void nvme_scan_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3999 {
4000         struct nvme_ns_info info = { .nsid = nsid };
4001         struct nvme_ns *ns;
4002         int ret;
4003
4004         if (nvme_identify_ns_descs(ctrl, &info))
4005                 return;
4006
4007         if (info.ids.csi != NVME_CSI_NVM && !nvme_multi_css(ctrl)) {
4008                 dev_warn(ctrl->device,
4009                         "command set not reported for nsid: %d\n", nsid);
4010                 return;
4011         }
4012
4013         /*
4014          * If available try to use the Command Set Idependent Identify Namespace
4015          * data structure to find all the generic information that is needed to
4016          * set up a namespace.  If not fall back to the legacy version.
4017          */
4018         if ((ctrl->cap & NVME_CAP_CRMS_CRIMS) ||
4019             (info.ids.csi != NVME_CSI_NVM && info.ids.csi != NVME_CSI_ZNS))
4020                 ret = nvme_ns_info_from_id_cs_indep(ctrl, &info);
4021         else
4022                 ret = nvme_ns_info_from_identify(ctrl, &info);
4023
4024         if (info.is_removed)
4025                 nvme_ns_remove_by_nsid(ctrl, nsid);
4026
4027         /*
4028          * Ignore the namespace if it is not ready. We will get an AEN once it
4029          * becomes ready and restart the scan.
4030          */
4031         if (ret || !info.is_ready)
4032                 return;
4033
4034         ns = nvme_find_get_ns(ctrl, nsid);
4035         if (ns) {
4036                 nvme_validate_ns(ns, &info);
4037                 nvme_put_ns(ns);
4038         } else {
4039                 nvme_alloc_ns(ctrl, &info);
4040         }
4041 }
4042
4043 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
4044                                         unsigned nsid)
4045 {
4046         struct nvme_ns *ns, *next;
4047         LIST_HEAD(rm_list);
4048
4049         mutex_lock(&ctrl->namespaces_lock);
4050         list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
4051                 if (ns->head->ns_id > nsid) {
4052                         list_del_rcu(&ns->list);
4053                         synchronize_srcu(&ctrl->srcu);
4054                         list_add_tail_rcu(&ns->list, &rm_list);
4055                 }
4056         }
4057         mutex_unlock(&ctrl->namespaces_lock);
4058
4059         list_for_each_entry_safe(ns, next, &rm_list, list)
4060                 nvme_ns_remove(ns);
4061 }
4062
4063 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
4064 {
4065         const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
4066         __le32 *ns_list;
4067         u32 prev = 0;
4068         int ret = 0, i;
4069
4070         ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
4071         if (!ns_list)
4072                 return -ENOMEM;
4073
4074         for (;;) {
4075                 struct nvme_command cmd = {
4076                         .identify.opcode        = nvme_admin_identify,
4077                         .identify.cns           = NVME_ID_CNS_NS_ACTIVE_LIST,
4078                         .identify.nsid          = cpu_to_le32(prev),
4079                 };
4080
4081                 ret = nvme_submit_sync_cmd(ctrl->admin_q, &cmd, ns_list,
4082                                             NVME_IDENTIFY_DATA_SIZE);
4083                 if (ret) {
4084                         dev_warn(ctrl->device,
4085                                 "Identify NS List failed (status=0x%x)\n", ret);
4086                         goto free;
4087                 }
4088
4089                 for (i = 0; i < nr_entries; i++) {
4090                         u32 nsid = le32_to_cpu(ns_list[i]);
4091
4092                         if (!nsid)      /* end of the list? */
4093                                 goto out;
4094                         nvme_scan_ns(ctrl, nsid);
4095                         while (++prev < nsid)
4096                                 nvme_ns_remove_by_nsid(ctrl, prev);
4097                 }
4098         }
4099  out:
4100         nvme_remove_invalid_namespaces(ctrl, prev);
4101  free:
4102         kfree(ns_list);
4103         return ret;
4104 }
4105
4106 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
4107 {
4108         struct nvme_id_ctrl *id;
4109         u32 nn, i;
4110
4111         if (nvme_identify_ctrl(ctrl, &id))
4112                 return;
4113         nn = le32_to_cpu(id->nn);
4114         kfree(id);
4115
4116         for (i = 1; i <= nn; i++)
4117                 nvme_scan_ns(ctrl, i);
4118
4119         nvme_remove_invalid_namespaces(ctrl, nn);
4120 }
4121
4122 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
4123 {
4124         size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
4125         __le32 *log;
4126         int error;
4127
4128         log = kzalloc(log_size, GFP_KERNEL);
4129         if (!log)
4130                 return;
4131
4132         /*
4133          * We need to read the log to clear the AEN, but we don't want to rely
4134          * on it for the changed namespace information as userspace could have
4135          * raced with us in reading the log page, which could cause us to miss
4136          * updates.
4137          */
4138         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
4139                         NVME_CSI_NVM, log, log_size, 0);
4140         if (error)
4141                 dev_warn(ctrl->device,
4142                         "reading changed ns log failed: %d\n", error);
4143
4144         kfree(log);
4145 }
4146
4147 static void nvme_scan_work(struct work_struct *work)
4148 {
4149         struct nvme_ctrl *ctrl =
4150                 container_of(work, struct nvme_ctrl, scan_work);
4151         int ret;
4152
4153         /* No tagset on a live ctrl means IO queues could not created */
4154         if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE || !ctrl->tagset)
4155                 return;
4156
4157         /*
4158          * Identify controller limits can change at controller reset due to
4159          * new firmware download, even though it is not common we cannot ignore
4160          * such scenario. Controller's non-mdts limits are reported in the unit
4161          * of logical blocks that is dependent on the format of attached
4162          * namespace. Hence re-read the limits at the time of ns allocation.
4163          */
4164         ret = nvme_init_non_mdts_limits(ctrl);
4165         if (ret < 0) {
4166                 dev_warn(ctrl->device,
4167                         "reading non-mdts-limits failed: %d\n", ret);
4168                 return;
4169         }
4170
4171         if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
4172                 dev_info(ctrl->device, "rescanning namespaces.\n");
4173                 nvme_clear_changed_ns_log(ctrl);
4174         }
4175
4176         mutex_lock(&ctrl->scan_lock);
4177         if (nvme_ctrl_limited_cns(ctrl)) {
4178                 nvme_scan_ns_sequential(ctrl);
4179         } else {
4180                 /*
4181                  * Fall back to sequential scan if DNR is set to handle broken
4182                  * devices which should support Identify NS List (as per the VS
4183                  * they report) but don't actually support it.
4184                  */
4185                 ret = nvme_scan_ns_list(ctrl);
4186                 if (ret > 0 && ret & NVME_STATUS_DNR)
4187                         nvme_scan_ns_sequential(ctrl);
4188         }
4189         mutex_unlock(&ctrl->scan_lock);
4190 }
4191
4192 /*
4193  * This function iterates the namespace list unlocked to allow recovery from
4194  * controller failure. It is up to the caller to ensure the namespace list is
4195  * not modified by scan work while this function is executing.
4196  */
4197 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
4198 {
4199         struct nvme_ns *ns, *next;
4200         LIST_HEAD(ns_list);
4201
4202         /*
4203          * make sure to requeue I/O to all namespaces as these
4204          * might result from the scan itself and must complete
4205          * for the scan_work to make progress
4206          */
4207         nvme_mpath_clear_ctrl_paths(ctrl);
4208
4209         /*
4210          * Unquiesce io queues so any pending IO won't hang, especially
4211          * those submitted from scan work
4212          */
4213         nvme_unquiesce_io_queues(ctrl);
4214
4215         /* prevent racing with ns scanning */
4216         flush_work(&ctrl->scan_work);
4217
4218         /*
4219          * The dead states indicates the controller was not gracefully
4220          * disconnected. In that case, we won't be able to flush any data while
4221          * removing the namespaces' disks; fail all the queues now to avoid
4222          * potentially having to clean up the failed sync later.
4223          */
4224         if (nvme_ctrl_state(ctrl) == NVME_CTRL_DEAD)
4225                 nvme_mark_namespaces_dead(ctrl);
4226
4227         /* this is a no-op when called from the controller reset handler */
4228         nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
4229
4230         mutex_lock(&ctrl->namespaces_lock);
4231         list_splice_init_rcu(&ctrl->namespaces, &ns_list, synchronize_rcu);
4232         mutex_unlock(&ctrl->namespaces_lock);
4233         synchronize_srcu(&ctrl->srcu);
4234
4235         list_for_each_entry_safe(ns, next, &ns_list, list)
4236                 nvme_ns_remove(ns);
4237 }
4238 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
4239
4240 static int nvme_class_uevent(const struct device *dev, struct kobj_uevent_env *env)
4241 {
4242         const struct nvme_ctrl *ctrl =
4243                 container_of(dev, struct nvme_ctrl, ctrl_device);
4244         struct nvmf_ctrl_options *opts = ctrl->opts;
4245         int ret;
4246
4247         ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
4248         if (ret)
4249                 return ret;
4250
4251         if (opts) {
4252                 ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
4253                 if (ret)
4254                         return ret;
4255
4256                 ret = add_uevent_var(env, "NVME_TRSVCID=%s",
4257                                 opts->trsvcid ?: "none");
4258                 if (ret)
4259                         return ret;
4260
4261                 ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
4262                                 opts->host_traddr ?: "none");
4263                 if (ret)
4264                         return ret;
4265
4266                 ret = add_uevent_var(env, "NVME_HOST_IFACE=%s",
4267                                 opts->host_iface ?: "none");
4268         }
4269         return ret;
4270 }
4271
4272 static void nvme_change_uevent(struct nvme_ctrl *ctrl, char *envdata)
4273 {
4274         char *envp[2] = { envdata, NULL };
4275
4276         kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4277 }
4278
4279 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
4280 {
4281         char *envp[2] = { NULL, NULL };
4282         u32 aen_result = ctrl->aen_result;
4283
4284         ctrl->aen_result = 0;
4285         if (!aen_result)
4286                 return;
4287
4288         envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
4289         if (!envp[0])
4290                 return;
4291         kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4292         kfree(envp[0]);
4293 }
4294
4295 static void nvme_async_event_work(struct work_struct *work)
4296 {
4297         struct nvme_ctrl *ctrl =
4298                 container_of(work, struct nvme_ctrl, async_event_work);
4299
4300         nvme_aen_uevent(ctrl);
4301
4302         /*
4303          * The transport drivers must guarantee AER submission here is safe by
4304          * flushing ctrl async_event_work after changing the controller state
4305          * from LIVE and before freeing the admin queue.
4306         */
4307         if (nvme_ctrl_state(ctrl) == NVME_CTRL_LIVE)
4308                 ctrl->ops->submit_async_event(ctrl);
4309 }
4310
4311 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
4312 {
4313
4314         u32 csts;
4315
4316         if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
4317                 return false;
4318
4319         if (csts == ~0)
4320                 return false;
4321
4322         return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
4323 }
4324
4325 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
4326 {
4327         struct nvme_fw_slot_info_log *log;
4328         u8 next_fw_slot, cur_fw_slot;
4329
4330         log = kmalloc(sizeof(*log), GFP_KERNEL);
4331         if (!log)
4332                 return;
4333
4334         if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
4335                          log, sizeof(*log), 0)) {
4336                 dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
4337                 goto out_free_log;
4338         }
4339
4340         cur_fw_slot = log->afi & 0x7;
4341         next_fw_slot = (log->afi & 0x70) >> 4;
4342         if (!cur_fw_slot || (next_fw_slot && (cur_fw_slot != next_fw_slot))) {
4343                 dev_info(ctrl->device,
4344                          "Firmware is activated after next Controller Level Reset\n");
4345                 goto out_free_log;
4346         }
4347
4348         memcpy(ctrl->subsys->firmware_rev, &log->frs[cur_fw_slot - 1],
4349                 sizeof(ctrl->subsys->firmware_rev));
4350
4351 out_free_log:
4352         kfree(log);
4353 }
4354
4355 static void nvme_fw_act_work(struct work_struct *work)
4356 {
4357         struct nvme_ctrl *ctrl = container_of(work,
4358                                 struct nvme_ctrl, fw_act_work);
4359         unsigned long fw_act_timeout;
4360
4361         nvme_auth_stop(ctrl);
4362
4363         if (ctrl->mtfa)
4364                 fw_act_timeout = jiffies +
4365                                 msecs_to_jiffies(ctrl->mtfa * 100);
4366         else
4367                 fw_act_timeout = jiffies +
4368                                 msecs_to_jiffies(admin_timeout * 1000);
4369
4370         nvme_quiesce_io_queues(ctrl);
4371         while (nvme_ctrl_pp_status(ctrl)) {
4372                 if (time_after(jiffies, fw_act_timeout)) {
4373                         dev_warn(ctrl->device,
4374                                 "Fw activation timeout, reset controller\n");
4375                         nvme_try_sched_reset(ctrl);
4376                         return;
4377                 }
4378                 msleep(100);
4379         }
4380
4381         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
4382                 return;
4383
4384         nvme_unquiesce_io_queues(ctrl);
4385         /* read FW slot information to clear the AER */
4386         nvme_get_fw_slot_info(ctrl);
4387
4388         queue_work(nvme_wq, &ctrl->async_event_work);
4389 }
4390
4391 static u32 nvme_aer_type(u32 result)
4392 {
4393         return result & 0x7;
4394 }
4395
4396 static u32 nvme_aer_subtype(u32 result)
4397 {
4398         return (result & 0xff00) >> 8;
4399 }
4400
4401 static bool nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
4402 {
4403         u32 aer_notice_type = nvme_aer_subtype(result);
4404         bool requeue = true;
4405
4406         switch (aer_notice_type) {
4407         case NVME_AER_NOTICE_NS_CHANGED:
4408                 set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
4409                 nvme_queue_scan(ctrl);
4410                 break;
4411         case NVME_AER_NOTICE_FW_ACT_STARTING:
4412                 /*
4413                  * We are (ab)using the RESETTING state to prevent subsequent
4414                  * recovery actions from interfering with the controller's
4415                  * firmware activation.
4416                  */
4417                 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) {
4418                         requeue = false;
4419                         queue_work(nvme_wq, &ctrl->fw_act_work);
4420                 }
4421                 break;
4422 #ifdef CONFIG_NVME_MULTIPATH
4423         case NVME_AER_NOTICE_ANA:
4424                 if (!ctrl->ana_log_buf)
4425                         break;
4426                 queue_work(nvme_wq, &ctrl->ana_work);
4427                 break;
4428 #endif
4429         case NVME_AER_NOTICE_DISC_CHANGED:
4430                 ctrl->aen_result = result;
4431                 break;
4432         default:
4433                 dev_warn(ctrl->device, "async event result %08x\n", result);
4434         }
4435         return requeue;
4436 }
4437
4438 static void nvme_handle_aer_persistent_error(struct nvme_ctrl *ctrl)
4439 {
4440         dev_warn(ctrl->device, "resetting controller due to AER\n");
4441         nvme_reset_ctrl(ctrl);
4442 }
4443
4444 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
4445                 volatile union nvme_result *res)
4446 {
4447         u32 result = le32_to_cpu(res->u32);
4448         u32 aer_type = nvme_aer_type(result);
4449         u32 aer_subtype = nvme_aer_subtype(result);
4450         bool requeue = true;
4451
4452         if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
4453                 return;
4454
4455         trace_nvme_async_event(ctrl, result);
4456         switch (aer_type) {
4457         case NVME_AER_NOTICE:
4458                 requeue = nvme_handle_aen_notice(ctrl, result);
4459                 break;
4460         case NVME_AER_ERROR:
4461                 /*
4462                  * For a persistent internal error, don't run async_event_work
4463                  * to submit a new AER. The controller reset will do it.
4464                  */
4465                 if (aer_subtype == NVME_AER_ERROR_PERSIST_INT_ERR) {
4466                         nvme_handle_aer_persistent_error(ctrl);
4467                         return;
4468                 }
4469                 fallthrough;
4470         case NVME_AER_SMART:
4471         case NVME_AER_CSS:
4472         case NVME_AER_VS:
4473                 ctrl->aen_result = result;
4474                 break;
4475         default:
4476                 break;
4477         }
4478
4479         if (requeue)
4480                 queue_work(nvme_wq, &ctrl->async_event_work);
4481 }
4482 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
4483
4484 int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
4485                 const struct blk_mq_ops *ops, unsigned int cmd_size)
4486 {
4487         struct queue_limits lim = {};
4488         int ret;
4489
4490         memset(set, 0, sizeof(*set));
4491         set->ops = ops;
4492         set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
4493         if (ctrl->ops->flags & NVME_F_FABRICS)
4494                 /* Reserved for fabric connect and keep alive */
4495                 set->reserved_tags = 2;
4496         set->numa_node = ctrl->numa_node;
4497         set->flags = BLK_MQ_F_NO_SCHED;
4498         if (ctrl->ops->flags & NVME_F_BLOCKING)
4499                 set->flags |= BLK_MQ_F_BLOCKING;
4500         set->cmd_size = cmd_size;
4501         set->driver_data = ctrl;
4502         set->nr_hw_queues = 1;
4503         set->timeout = NVME_ADMIN_TIMEOUT;
4504         ret = blk_mq_alloc_tag_set(set);
4505         if (ret)
4506                 return ret;
4507
4508         ctrl->admin_q = blk_mq_alloc_queue(set, &lim, NULL);
4509         if (IS_ERR(ctrl->admin_q)) {
4510                 ret = PTR_ERR(ctrl->admin_q);
4511                 goto out_free_tagset;
4512         }
4513
4514         if (ctrl->ops->flags & NVME_F_FABRICS) {
4515                 ctrl->fabrics_q = blk_mq_alloc_queue(set, NULL, NULL);
4516                 if (IS_ERR(ctrl->fabrics_q)) {
4517                         ret = PTR_ERR(ctrl->fabrics_q);
4518                         goto out_cleanup_admin_q;
4519                 }
4520         }
4521
4522         ctrl->admin_tagset = set;
4523         return 0;
4524
4525 out_cleanup_admin_q:
4526         blk_mq_destroy_queue(ctrl->admin_q);
4527         blk_put_queue(ctrl->admin_q);
4528 out_free_tagset:
4529         blk_mq_free_tag_set(set);
4530         ctrl->admin_q = NULL;
4531         ctrl->fabrics_q = NULL;
4532         return ret;
4533 }
4534 EXPORT_SYMBOL_GPL(nvme_alloc_admin_tag_set);
4535
4536 void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl)
4537 {
4538         blk_mq_destroy_queue(ctrl->admin_q);
4539         blk_put_queue(ctrl->admin_q);
4540         if (ctrl->ops->flags & NVME_F_FABRICS) {
4541                 blk_mq_destroy_queue(ctrl->fabrics_q);
4542                 blk_put_queue(ctrl->fabrics_q);
4543         }
4544         blk_mq_free_tag_set(ctrl->admin_tagset);
4545 }
4546 EXPORT_SYMBOL_GPL(nvme_remove_admin_tag_set);
4547
4548 int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
4549                 const struct blk_mq_ops *ops, unsigned int nr_maps,
4550                 unsigned int cmd_size)
4551 {
4552         int ret;
4553
4554         memset(set, 0, sizeof(*set));
4555         set->ops = ops;
4556         set->queue_depth = min_t(unsigned, ctrl->sqsize, BLK_MQ_MAX_DEPTH - 1);
4557         /*
4558          * Some Apple controllers requires tags to be unique across admin and
4559          * the (only) I/O queue, so reserve the first 32 tags of the I/O queue.
4560          */
4561         if (ctrl->quirks & NVME_QUIRK_SHARED_TAGS)
4562                 set->reserved_tags = NVME_AQ_DEPTH;
4563         else if (ctrl->ops->flags & NVME_F_FABRICS)
4564                 /* Reserved for fabric connect */
4565                 set->reserved_tags = 1;
4566         set->numa_node = ctrl->numa_node;
4567         set->flags = BLK_MQ_F_SHOULD_MERGE;
4568         if (ctrl->ops->flags & NVME_F_BLOCKING)
4569                 set->flags |= BLK_MQ_F_BLOCKING;
4570         set->cmd_size = cmd_size,
4571         set->driver_data = ctrl;
4572         set->nr_hw_queues = ctrl->queue_count - 1;
4573         set->timeout = NVME_IO_TIMEOUT;
4574         set->nr_maps = nr_maps;
4575         ret = blk_mq_alloc_tag_set(set);
4576         if (ret)
4577                 return ret;
4578
4579         if (ctrl->ops->flags & NVME_F_FABRICS) {
4580                 struct queue_limits lim = {
4581                         .features       = BLK_FEAT_SKIP_TAGSET_QUIESCE,
4582                 };
4583
4584                 ctrl->connect_q = blk_mq_alloc_queue(set, &lim, NULL);
4585                 if (IS_ERR(ctrl->connect_q)) {
4586                         ret = PTR_ERR(ctrl->connect_q);
4587                         goto out_free_tag_set;
4588                 }
4589         }
4590
4591         ctrl->tagset = set;
4592         return 0;
4593
4594 out_free_tag_set:
4595         blk_mq_free_tag_set(set);
4596         ctrl->connect_q = NULL;
4597         return ret;
4598 }
4599 EXPORT_SYMBOL_GPL(nvme_alloc_io_tag_set);
4600
4601 void nvme_remove_io_tag_set(struct nvme_ctrl *ctrl)
4602 {
4603         if (ctrl->ops->flags & NVME_F_FABRICS) {
4604                 blk_mq_destroy_queue(ctrl->connect_q);
4605                 blk_put_queue(ctrl->connect_q);
4606         }
4607         blk_mq_free_tag_set(ctrl->tagset);
4608 }
4609 EXPORT_SYMBOL_GPL(nvme_remove_io_tag_set);
4610
4611 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
4612 {
4613         nvme_mpath_stop(ctrl);
4614         nvme_auth_stop(ctrl);
4615         nvme_stop_keep_alive(ctrl);
4616         nvme_stop_failfast_work(ctrl);
4617         flush_work(&ctrl->async_event_work);
4618         cancel_work_sync(&ctrl->fw_act_work);
4619         if (ctrl->ops->stop_ctrl)
4620                 ctrl->ops->stop_ctrl(ctrl);
4621 }
4622 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
4623
4624 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
4625 {
4626         nvme_enable_aen(ctrl);
4627
4628         /*
4629          * persistent discovery controllers need to send indication to userspace
4630          * to re-read the discovery log page to learn about possible changes
4631          * that were missed. We identify persistent discovery controllers by
4632          * checking that they started once before, hence are reconnecting back.
4633          */
4634         if (test_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags) &&
4635             nvme_discovery_ctrl(ctrl))
4636                 nvme_change_uevent(ctrl, "NVME_EVENT=rediscover");
4637
4638         if (ctrl->queue_count > 1) {
4639                 nvme_queue_scan(ctrl);
4640                 nvme_unquiesce_io_queues(ctrl);
4641                 nvme_mpath_update(ctrl);
4642         }
4643
4644         nvme_change_uevent(ctrl, "NVME_EVENT=connected");
4645         set_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags);
4646 }
4647 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
4648
4649 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
4650 {
4651         nvme_hwmon_exit(ctrl);
4652         nvme_fault_inject_fini(&ctrl->fault_inject);
4653         dev_pm_qos_hide_latency_tolerance(ctrl->device);
4654         cdev_device_del(&ctrl->cdev, ctrl->device);
4655         nvme_put_ctrl(ctrl);
4656 }
4657 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
4658
4659 static void nvme_free_cels(struct nvme_ctrl *ctrl)
4660 {
4661         struct nvme_effects_log *cel;
4662         unsigned long i;
4663
4664         xa_for_each(&ctrl->cels, i, cel) {
4665                 xa_erase(&ctrl->cels, i);
4666                 kfree(cel);
4667         }
4668
4669         xa_destroy(&ctrl->cels);
4670 }
4671
4672 static void nvme_free_ctrl(struct device *dev)
4673 {
4674         struct nvme_ctrl *ctrl =
4675                 container_of(dev, struct nvme_ctrl, ctrl_device);
4676         struct nvme_subsystem *subsys = ctrl->subsys;
4677
4678         if (!subsys || ctrl->instance != subsys->instance)
4679                 ida_free(&nvme_instance_ida, ctrl->instance);
4680         key_put(ctrl->tls_key);
4681         nvme_free_cels(ctrl);
4682         nvme_mpath_uninit(ctrl);
4683         cleanup_srcu_struct(&ctrl->srcu);
4684         nvme_auth_stop(ctrl);
4685         nvme_auth_free(ctrl);
4686         __free_page(ctrl->discard_page);
4687         free_opal_dev(ctrl->opal_dev);
4688
4689         if (subsys) {
4690                 mutex_lock(&nvme_subsystems_lock);
4691                 list_del(&ctrl->subsys_entry);
4692                 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
4693                 mutex_unlock(&nvme_subsystems_lock);
4694         }
4695
4696         ctrl->ops->free_ctrl(ctrl);
4697
4698         if (subsys)
4699                 nvme_put_subsystem(subsys);
4700 }
4701
4702 /*
4703  * Initialize a NVMe controller structures.  This needs to be called during
4704  * earliest initialization so that we have the initialized structured around
4705  * during probing.
4706  *
4707  * On success, the caller must use the nvme_put_ctrl() to release this when
4708  * needed, which also invokes the ops->free_ctrl() callback.
4709  */
4710 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
4711                 const struct nvme_ctrl_ops *ops, unsigned long quirks)
4712 {
4713         int ret;
4714
4715         WRITE_ONCE(ctrl->state, NVME_CTRL_NEW);
4716         ctrl->passthru_err_log_enabled = false;
4717         clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
4718         spin_lock_init(&ctrl->lock);
4719         mutex_init(&ctrl->namespaces_lock);
4720
4721         ret = init_srcu_struct(&ctrl->srcu);
4722         if (ret)
4723                 return ret;
4724
4725         mutex_init(&ctrl->scan_lock);
4726         INIT_LIST_HEAD(&ctrl->namespaces);
4727         xa_init(&ctrl->cels);
4728         ctrl->dev = dev;
4729         ctrl->ops = ops;
4730         ctrl->quirks = quirks;
4731         ctrl->numa_node = NUMA_NO_NODE;
4732         INIT_WORK(&ctrl->scan_work, nvme_scan_work);
4733         INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
4734         INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
4735         INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
4736         init_waitqueue_head(&ctrl->state_wq);
4737
4738         INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
4739         INIT_DELAYED_WORK(&ctrl->failfast_work, nvme_failfast_work);
4740         memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
4741         ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
4742         ctrl->ka_last_check_time = jiffies;
4743
4744         BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
4745                         PAGE_SIZE);
4746         ctrl->discard_page = alloc_page(GFP_KERNEL);
4747         if (!ctrl->discard_page) {
4748                 ret = -ENOMEM;
4749                 goto out;
4750         }
4751
4752         ret = ida_alloc(&nvme_instance_ida, GFP_KERNEL);
4753         if (ret < 0)
4754                 goto out;
4755         ctrl->instance = ret;
4756
4757         ret = nvme_auth_init_ctrl(ctrl);
4758         if (ret)
4759                 goto out_release_instance;
4760
4761         nvme_mpath_init_ctrl(ctrl);
4762
4763         device_initialize(&ctrl->ctrl_device);
4764         ctrl->device = &ctrl->ctrl_device;
4765         ctrl->device->devt = MKDEV(MAJOR(nvme_ctrl_base_chr_devt),
4766                         ctrl->instance);
4767         ctrl->device->class = &nvme_class;
4768         ctrl->device->parent = ctrl->dev;
4769         if (ops->dev_attr_groups)
4770                 ctrl->device->groups = ops->dev_attr_groups;
4771         else
4772                 ctrl->device->groups = nvme_dev_attr_groups;
4773         ctrl->device->release = nvme_free_ctrl;
4774         dev_set_drvdata(ctrl->device, ctrl);
4775
4776         return ret;
4777
4778 out_release_instance:
4779         ida_free(&nvme_instance_ida, ctrl->instance);
4780 out:
4781         if (ctrl->discard_page)
4782                 __free_page(ctrl->discard_page);
4783         cleanup_srcu_struct(&ctrl->srcu);
4784         return ret;
4785 }
4786 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
4787
4788 /*
4789  * On success, returns with an elevated controller reference and caller must
4790  * use nvme_uninit_ctrl() to properly free resources associated with the ctrl.
4791  */
4792 int nvme_add_ctrl(struct nvme_ctrl *ctrl)
4793 {
4794         int ret;
4795
4796         ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
4797         if (ret)
4798                 return ret;
4799
4800         cdev_init(&ctrl->cdev, &nvme_dev_fops);
4801         ctrl->cdev.owner = ctrl->ops->module;
4802         ret = cdev_device_add(&ctrl->cdev, ctrl->device);
4803         if (ret)
4804                 return ret;
4805
4806         /*
4807          * Initialize latency tolerance controls.  The sysfs files won't
4808          * be visible to userspace unless the device actually supports APST.
4809          */
4810         ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
4811         dev_pm_qos_update_user_latency_tolerance(ctrl->device,
4812                 min(default_ps_max_latency_us, (unsigned long)S32_MAX));
4813
4814         nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
4815         nvme_get_ctrl(ctrl);
4816
4817         return 0;
4818 }
4819 EXPORT_SYMBOL_GPL(nvme_add_ctrl);
4820
4821 /* let I/O to all namespaces fail in preparation for surprise removal */
4822 void nvme_mark_namespaces_dead(struct nvme_ctrl *ctrl)
4823 {
4824         struct nvme_ns *ns;
4825         int srcu_idx;
4826
4827         srcu_idx = srcu_read_lock(&ctrl->srcu);
4828         list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
4829                 blk_mark_disk_dead(ns->disk);
4830         srcu_read_unlock(&ctrl->srcu, srcu_idx);
4831 }
4832 EXPORT_SYMBOL_GPL(nvme_mark_namespaces_dead);
4833
4834 void nvme_unfreeze(struct nvme_ctrl *ctrl)
4835 {
4836         struct nvme_ns *ns;
4837         int srcu_idx;
4838
4839         srcu_idx = srcu_read_lock(&ctrl->srcu);
4840         list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
4841                 blk_mq_unfreeze_queue(ns->queue);
4842         srcu_read_unlock(&ctrl->srcu, srcu_idx);
4843         clear_bit(NVME_CTRL_FROZEN, &ctrl->flags);
4844 }
4845 EXPORT_SYMBOL_GPL(nvme_unfreeze);
4846
4847 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
4848 {
4849         struct nvme_ns *ns;
4850         int srcu_idx;
4851
4852         srcu_idx = srcu_read_lock(&ctrl->srcu);
4853         list_for_each_entry_rcu(ns, &ctrl->namespaces, list) {
4854                 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
4855                 if (timeout <= 0)
4856                         break;
4857         }
4858         srcu_read_unlock(&ctrl->srcu, srcu_idx);
4859         return timeout;
4860 }
4861 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
4862
4863 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
4864 {
4865         struct nvme_ns *ns;
4866         int srcu_idx;
4867
4868         srcu_idx = srcu_read_lock(&ctrl->srcu);
4869         list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
4870                 blk_mq_freeze_queue_wait(ns->queue);
4871         srcu_read_unlock(&ctrl->srcu, srcu_idx);
4872 }
4873 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
4874
4875 void nvme_start_freeze(struct nvme_ctrl *ctrl)
4876 {
4877         struct nvme_ns *ns;
4878         int srcu_idx;
4879
4880         set_bit(NVME_CTRL_FROZEN, &ctrl->flags);
4881         srcu_idx = srcu_read_lock(&ctrl->srcu);
4882         list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
4883                 blk_freeze_queue_start(ns->queue);
4884         srcu_read_unlock(&ctrl->srcu, srcu_idx);
4885 }
4886 EXPORT_SYMBOL_GPL(nvme_start_freeze);
4887
4888 void nvme_quiesce_io_queues(struct nvme_ctrl *ctrl)
4889 {
4890         if (!ctrl->tagset)
4891                 return;
4892         if (!test_and_set_bit(NVME_CTRL_STOPPED, &ctrl->flags))
4893                 blk_mq_quiesce_tagset(ctrl->tagset);
4894         else
4895                 blk_mq_wait_quiesce_done(ctrl->tagset);
4896 }
4897 EXPORT_SYMBOL_GPL(nvme_quiesce_io_queues);
4898
4899 void nvme_unquiesce_io_queues(struct nvme_ctrl *ctrl)
4900 {
4901         if (!ctrl->tagset)
4902                 return;
4903         if (test_and_clear_bit(NVME_CTRL_STOPPED, &ctrl->flags))
4904                 blk_mq_unquiesce_tagset(ctrl->tagset);
4905 }
4906 EXPORT_SYMBOL_GPL(nvme_unquiesce_io_queues);
4907
4908 void nvme_quiesce_admin_queue(struct nvme_ctrl *ctrl)
4909 {
4910         if (!test_and_set_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
4911                 blk_mq_quiesce_queue(ctrl->admin_q);
4912         else
4913                 blk_mq_wait_quiesce_done(ctrl->admin_q->tag_set);
4914 }
4915 EXPORT_SYMBOL_GPL(nvme_quiesce_admin_queue);
4916
4917 void nvme_unquiesce_admin_queue(struct nvme_ctrl *ctrl)
4918 {
4919         if (test_and_clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
4920                 blk_mq_unquiesce_queue(ctrl->admin_q);
4921 }
4922 EXPORT_SYMBOL_GPL(nvme_unquiesce_admin_queue);
4923
4924 void nvme_sync_io_queues(struct nvme_ctrl *ctrl)
4925 {
4926         struct nvme_ns *ns;
4927         int srcu_idx;
4928
4929         srcu_idx = srcu_read_lock(&ctrl->srcu);
4930         list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
4931                 blk_sync_queue(ns->queue);
4932         srcu_read_unlock(&ctrl->srcu, srcu_idx);
4933 }
4934 EXPORT_SYMBOL_GPL(nvme_sync_io_queues);
4935
4936 void nvme_sync_queues(struct nvme_ctrl *ctrl)
4937 {
4938         nvme_sync_io_queues(ctrl);
4939         if (ctrl->admin_q)
4940                 blk_sync_queue(ctrl->admin_q);
4941 }
4942 EXPORT_SYMBOL_GPL(nvme_sync_queues);
4943
4944 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file)
4945 {
4946         if (file->f_op != &nvme_dev_fops)
4947                 return NULL;
4948         return file->private_data;
4949 }
4950 EXPORT_SYMBOL_NS_GPL(nvme_ctrl_from_file, NVME_TARGET_PASSTHRU);
4951
4952 /*
4953  * Check we didn't inadvertently grow the command structure sizes:
4954  */
4955 static inline void _nvme_check_size(void)
4956 {
4957         BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
4958         BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
4959         BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
4960         BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
4961         BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
4962         BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
4963         BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
4964         BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
4965         BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
4966         BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
4967         BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
4968         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
4969         BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
4970         BUILD_BUG_ON(sizeof(struct nvme_id_ns_cs_indep) !=
4971                         NVME_IDENTIFY_DATA_SIZE);
4972         BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
4973         BUILD_BUG_ON(sizeof(struct nvme_id_ns_nvm) != NVME_IDENTIFY_DATA_SIZE);
4974         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
4975         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_nvm) != NVME_IDENTIFY_DATA_SIZE);
4976         BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
4977         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
4978         BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
4979         BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
4980         BUILD_BUG_ON(sizeof(struct nvme_feat_host_behavior) != 512);
4981 }
4982
4983
4984 static int __init nvme_core_init(void)
4985 {
4986         int result = -ENOMEM;
4987
4988         _nvme_check_size();
4989
4990         nvme_wq = alloc_workqueue("nvme-wq",
4991                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4992         if (!nvme_wq)
4993                 goto out;
4994
4995         nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
4996                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4997         if (!nvme_reset_wq)
4998                 goto destroy_wq;
4999
5000         nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
5001                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5002         if (!nvme_delete_wq)
5003                 goto destroy_reset_wq;
5004
5005         result = alloc_chrdev_region(&nvme_ctrl_base_chr_devt, 0,
5006                         NVME_MINORS, "nvme");
5007         if (result < 0)
5008                 goto destroy_delete_wq;
5009
5010         result = class_register(&nvme_class);
5011         if (result)
5012                 goto unregister_chrdev;
5013
5014         result = class_register(&nvme_subsys_class);
5015         if (result)
5016                 goto destroy_class;
5017
5018         result = alloc_chrdev_region(&nvme_ns_chr_devt, 0, NVME_MINORS,
5019                                      "nvme-generic");
5020         if (result < 0)
5021                 goto destroy_subsys_class;
5022
5023         result = class_register(&nvme_ns_chr_class);
5024         if (result)
5025                 goto unregister_generic_ns;
5026
5027         result = nvme_init_auth();
5028         if (result)
5029                 goto destroy_ns_chr;
5030         return 0;
5031
5032 destroy_ns_chr:
5033         class_unregister(&nvme_ns_chr_class);
5034 unregister_generic_ns:
5035         unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
5036 destroy_subsys_class:
5037         class_unregister(&nvme_subsys_class);
5038 destroy_class:
5039         class_unregister(&nvme_class);
5040 unregister_chrdev:
5041         unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
5042 destroy_delete_wq:
5043         destroy_workqueue(nvme_delete_wq);
5044 destroy_reset_wq:
5045         destroy_workqueue(nvme_reset_wq);
5046 destroy_wq:
5047         destroy_workqueue(nvme_wq);
5048 out:
5049         return result;
5050 }
5051
5052 static void __exit nvme_core_exit(void)
5053 {
5054         nvme_exit_auth();
5055         class_unregister(&nvme_ns_chr_class);
5056         class_unregister(&nvme_subsys_class);
5057         class_unregister(&nvme_class);
5058         unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
5059         unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
5060         destroy_workqueue(nvme_delete_wq);
5061         destroy_workqueue(nvme_reset_wq);
5062         destroy_workqueue(nvme_wq);
5063         ida_destroy(&nvme_ns_chr_minor_ida);
5064         ida_destroy(&nvme_instance_ida);
5065 }
5066
5067 MODULE_LICENSE("GPL");
5068 MODULE_VERSION("1.0");
5069 MODULE_DESCRIPTION("NVMe host core framework");
5070 module_init(nvme_core_init);
5071 module_exit(nvme_core_exit);