drivers/nvme/target/pr.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVMe over Fabrics Persist Reservation.
   4  * Copyright (c) 2024 Guixin Liu, Alibaba Group.
   5  * All rights reserved.
   6  */
   7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   8 #include <linux/unaligned.h>
   9 #include "nvmet.h"
  10
  11 #define NVMET_PR_NOTIFI_MASK_ALL \
  12         (1 << NVME_PR_NOTIFY_BIT_REG_PREEMPTED | \
  13          1 << NVME_PR_NOTIFY_BIT_RESV_RELEASED | \
  14          1 << NVME_PR_NOTIFY_BIT_RESV_PREEMPTED)
  15
  16 static inline bool nvmet_pr_parse_ignore_key(u32 cdw10)
  17 {
  18         /* Ignore existing key, bit 03. */
  19         return (cdw10 >> 3) & 1;
  20 }
  21
  22 static inline struct nvmet_ns *nvmet_pr_to_ns(struct nvmet_pr *pr)
  23 {
  24         return container_of(pr, struct nvmet_ns, pr);
  25 }
  26
  27 static struct nvmet_pr_registrant *
  28 nvmet_pr_find_registrant(struct nvmet_pr *pr, uuid_t *hostid)
  29 {
  30         struct nvmet_pr_registrant *reg;
  31
  32         list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
  33                 if (uuid_equal(&reg->hostid, hostid))
  34                         return reg;
  35         }
  36         return NULL;
  37 }
  38
  39 u16 nvmet_set_feat_resv_notif_mask(struct nvmet_req *req, u32 mask)
  40 {
  41         u32 nsid = le32_to_cpu(req->cmd->common.nsid);
  42         struct nvmet_ctrl *ctrl = req->sq->ctrl;
  43         struct nvmet_ns *ns;
  44         unsigned long idx;
  45         u16 status;
  46
  47         if (mask & ~(NVMET_PR_NOTIFI_MASK_ALL)) {
  48                 req->error_loc = offsetof(struct nvme_common_command, cdw11);
  49                 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
  50         }
  51
  52         if (nsid != U32_MAX) {
  53                 status = nvmet_req_find_ns(req);
  54                 if (status)
  55                         return status;
  56                 if (!req->ns->pr.enable)
  57                         return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
  58
  59                 WRITE_ONCE(req->ns->pr.notify_mask, mask);
  60                 goto success;
  61         }
  62
  63         nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) {
  64                 if (ns->pr.enable)
  65                         WRITE_ONCE(ns->pr.notify_mask, mask);
  66         }
  67
  68 success:
  69         nvmet_set_result(req, mask);
  70         return NVME_SC_SUCCESS;
  71 }
  72
  73 u16 nvmet_get_feat_resv_notif_mask(struct nvmet_req *req)
  74 {
  75         u16 status;
  76
  77         status = nvmet_req_find_ns(req);
  78         if (status)
  79                 return status;
  80
  81         if (!req->ns->pr.enable)
  82                 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
  83
  84         nvmet_set_result(req, READ_ONCE(req->ns->pr.notify_mask));
  85         return status;
  86 }
  87
  88 void nvmet_execute_get_log_page_resv(struct nvmet_req *req)
  89 {
  90         struct nvmet_pr_log_mgr *log_mgr = &req->sq->ctrl->pr_log_mgr;
  91         struct nvme_pr_log next_log = {0};
  92         struct nvme_pr_log log = {0};
  93         u16 status = NVME_SC_SUCCESS;
  94         u64 lost_count;
  95         u64 cur_count;
  96         u64 next_count;
  97
  98         mutex_lock(&log_mgr->lock);
  99         if (!kfifo_get(&log_mgr->log_queue, &log))
 100                 goto out;
 101
 102         /*
 103          * We can't get the last in kfifo.
 104          * Utilize the current count and the count from the next log to
 105          * calculate the number of lost logs, while also addressing cases
 106          * of overflow. If there is no subsequent log, the number of lost
 107          * logs is equal to the lost_count within the nvmet_pr_log_mgr.
 108          */
 109         cur_count = le64_to_cpu(log.count);
 110         if (kfifo_peek(&log_mgr->log_queue, &next_log)) {
 111                 next_count = le64_to_cpu(next_log.count);
 112                 if (next_count > cur_count)
 113                         lost_count = next_count - cur_count - 1;
 114                 else
 115                         lost_count = U64_MAX - cur_count + next_count - 1;
 116         } else {
 117                 lost_count = log_mgr->lost_count;
 118         }
 119
 120         log.count = cpu_to_le64((cur_count + lost_count) == 0 ?
 121                                 1 : (cur_count + lost_count));
 122         log_mgr->lost_count -= lost_count;
 123
 124         log.nr_pages = kfifo_len(&log_mgr->log_queue);
 125
 126 out:
 127         status = nvmet_copy_to_sgl(req, 0, &log, sizeof(log));
 128         mutex_unlock(&log_mgr->lock);
 129         nvmet_req_complete(req, status);
 130 }
 131
 132 static void nvmet_pr_add_resv_log(struct nvmet_ctrl *ctrl, u8 log_type,
 133                                   u32 nsid)
 134 {
 135         struct nvmet_pr_log_mgr *log_mgr = &ctrl->pr_log_mgr;
 136         struct nvme_pr_log log = {0};
 137
 138         mutex_lock(&log_mgr->lock);
 139         log_mgr->counter++;
 140         if (log_mgr->counter == 0)
 141                 log_mgr->counter = 1;
 142
 143         log.count = cpu_to_le64(log_mgr->counter);
 144         log.type = log_type;
 145         log.nsid = cpu_to_le32(nsid);
 146
 147         if (!kfifo_put(&log_mgr->log_queue, log)) {
 148                 pr_info("a reservation log lost, cntlid:%d, log_type:%d, nsid:%d\n",
 149                         ctrl->cntlid, log_type, nsid);
 150                 log_mgr->lost_count++;
 151         }
 152
 153         mutex_unlock(&log_mgr->lock);
 154 }
 155
 156 static void nvmet_pr_resv_released(struct nvmet_pr *pr, uuid_t *hostid)
 157 {
 158         struct nvmet_ns *ns = nvmet_pr_to_ns(pr);
 159         struct nvmet_subsys *subsys = ns->subsys;
 160         struct nvmet_ctrl *ctrl;
 161
 162         if (test_bit(NVME_PR_NOTIFY_BIT_RESV_RELEASED, &pr->notify_mask))
 163                 return;
 164
 165         mutex_lock(&subsys->lock);
 166         list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
 167                 if (!uuid_equal(&ctrl->hostid, hostid) &&
 168                     nvmet_pr_find_registrant(pr, &ctrl->hostid)) {
 169                         nvmet_pr_add_resv_log(ctrl,
 170                                 NVME_PR_LOG_RESERVATION_RELEASED, ns->nsid);
 171                         nvmet_add_async_event(ctrl, NVME_AER_CSS,
 172                                 NVME_AEN_RESV_LOG_PAGE_AVALIABLE,
 173                                 NVME_LOG_RESERVATION);
 174                 }
 175         }
 176         mutex_unlock(&subsys->lock);
 177 }
 178
 179 static void nvmet_pr_send_event_to_host(struct nvmet_pr *pr, uuid_t *hostid,
 180                                           u8 log_type)
 181 {
 182         struct nvmet_ns *ns = nvmet_pr_to_ns(pr);
 183         struct nvmet_subsys *subsys = ns->subsys;
 184         struct nvmet_ctrl *ctrl;
 185
 186         mutex_lock(&subsys->lock);
 187         list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
 188                 if (uuid_equal(hostid, &ctrl->hostid)) {
 189                         nvmet_pr_add_resv_log(ctrl, log_type, ns->nsid);
 190                         nvmet_add_async_event(ctrl, NVME_AER_CSS,
 191                                 NVME_AEN_RESV_LOG_PAGE_AVALIABLE,
 192                                 NVME_LOG_RESERVATION);
 193                 }
 194         }
 195         mutex_unlock(&subsys->lock);
 196 }
 197
 198 static void nvmet_pr_resv_preempted(struct nvmet_pr *pr, uuid_t *hostid)
 199 {
 200         if (test_bit(NVME_PR_NOTIFY_BIT_RESV_PREEMPTED, &pr->notify_mask))
 201                 return;
 202
 203         nvmet_pr_send_event_to_host(pr, hostid,
 204                 NVME_PR_LOG_RESERVATOIN_PREEMPTED);
 205 }
 206
 207 static void nvmet_pr_registration_preempted(struct nvmet_pr *pr,
 208                                             uuid_t *hostid)
 209 {
 210         if (test_bit(NVME_PR_NOTIFY_BIT_REG_PREEMPTED, &pr->notify_mask))
 211                 return;
 212
 213         nvmet_pr_send_event_to_host(pr, hostid,
 214                 NVME_PR_LOG_REGISTRATION_PREEMPTED);
 215 }
 216
 217 static inline void nvmet_pr_set_new_holder(struct nvmet_pr *pr, u8 new_rtype,
 218                                            struct nvmet_pr_registrant *reg)
 219 {
 220         reg->rtype = new_rtype;
 221         rcu_assign_pointer(pr->holder, reg);
 222 }
 223
 224 static u16 nvmet_pr_register(struct nvmet_req *req,
 225                              struct nvmet_pr_register_data *d)
 226 {
 227         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 228         struct nvmet_pr_registrant *new, *reg;
 229         struct nvmet_pr *pr = &req->ns->pr;
 230         u16 status = NVME_SC_SUCCESS;
 231         u64 nrkey = le64_to_cpu(d->nrkey);
 232
 233         new = kmalloc(sizeof(*new), GFP_KERNEL);
 234         if (!new)
 235                 return NVME_SC_INTERNAL;
 236
 237         down(&pr->pr_sem);
 238         reg = nvmet_pr_find_registrant(pr, &ctrl->hostid);
 239         if (reg) {
 240                 if (reg->rkey != nrkey)
 241                         status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
 242                 kfree(new);
 243                 goto out;
 244         }
 245
 246         memset(new, 0, sizeof(*new));
 247         INIT_LIST_HEAD(&new->entry);
 248         new->rkey = nrkey;
 249         uuid_copy(&new->hostid, &ctrl->hostid);
 250         list_add_tail_rcu(&new->entry, &pr->registrant_list);
 251
 252 out:
 253         up(&pr->pr_sem);
 254         return status;
 255 }
 256
 257 static void nvmet_pr_unregister_one(struct nvmet_pr *pr,
 258                                     struct nvmet_pr_registrant *reg)
 259 {
 260         struct nvmet_pr_registrant *first_reg;
 261         struct nvmet_pr_registrant *holder;
 262         u8 original_rtype;
 263
 264         list_del_rcu(&reg->entry);
 265
 266         holder = rcu_dereference_protected(pr->holder, 1);
 267         if (reg != holder)
 268                 goto out;
 269
 270         original_rtype = holder->rtype;
 271         if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS ||
 272             original_rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) {
 273                 first_reg = list_first_or_null_rcu(&pr->registrant_list,
 274                                 struct nvmet_pr_registrant, entry);
 275                 if (first_reg)
 276                         first_reg->rtype = original_rtype;
 277                 rcu_assign_pointer(pr->holder, first_reg);
 278         } else {
 279                 rcu_assign_pointer(pr->holder, NULL);
 280
 281                 if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_REG_ONLY ||
 282                     original_rtype == NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY)
 283                         nvmet_pr_resv_released(pr, &reg->hostid);
 284         }
 285 out:
 286         kfree_rcu(reg, rcu);
 287 }
 288
 289 static u16 nvmet_pr_unregister(struct nvmet_req *req,
 290                                struct nvmet_pr_register_data *d,
 291                                bool ignore_key)
 292 {
 293         u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
 294         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 295         struct nvmet_pr *pr = &req->ns->pr;
 296         struct nvmet_pr_registrant *reg;
 297
 298         down(&pr->pr_sem);
 299         list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
 300                 if (uuid_equal(&reg->hostid, &ctrl->hostid)) {
 301                         if (ignore_key || reg->rkey == le64_to_cpu(d->crkey)) {
 302                                 status = NVME_SC_SUCCESS;
 303                                 nvmet_pr_unregister_one(pr, reg);
 304                         }
 305                         break;
 306                 }
 307         }
 308         up(&pr->pr_sem);
 309
 310         return status;
 311 }
 312
 313 static void nvmet_pr_update_reg_rkey(struct nvmet_pr_registrant *reg,
 314                                      void *attr)
 315 {
 316         reg->rkey = *(u64 *)attr;
 317 }
 318
 319 static u16 nvmet_pr_update_reg_attr(struct nvmet_pr *pr,
 320                         struct nvmet_pr_registrant *reg,
 321                         void (*change_attr)(struct nvmet_pr_registrant *reg,
 322                         void *attr),
 323                         void *attr)
 324 {
 325         struct nvmet_pr_registrant *holder;
 326         struct nvmet_pr_registrant *new;
 327
 328         holder = rcu_dereference_protected(pr->holder, 1);
 329         if (reg != holder) {
 330                 change_attr(reg, attr);
 331                 return NVME_SC_SUCCESS;
 332         }
 333
 334         new = kmalloc(sizeof(*new), GFP_ATOMIC);
 335         if (!new)
 336                 return NVME_SC_INTERNAL;
 337
 338         new->rkey = holder->rkey;
 339         new->rtype = holder->rtype;
 340         uuid_copy(&new->hostid, &holder->hostid);
 341         INIT_LIST_HEAD(&new->entry);
 342
 343         change_attr(new, attr);
 344         list_replace_rcu(&holder->entry, &new->entry);
 345         rcu_assign_pointer(pr->holder, new);
 346         kfree_rcu(holder, rcu);
 347
 348         return NVME_SC_SUCCESS;
 349 }
 350
 351 static u16 nvmet_pr_replace(struct nvmet_req *req,
 352                             struct nvmet_pr_register_data *d,
 353                             bool ignore_key)
 354 {
 355         u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
 356         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 357         struct nvmet_pr *pr = &req->ns->pr;
 358         struct nvmet_pr_registrant *reg;
 359         u64 nrkey = le64_to_cpu(d->nrkey);
 360
 361         down(&pr->pr_sem);
 362         list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
 363                 if (uuid_equal(&reg->hostid, &ctrl->hostid)) {
 364                         if (ignore_key || reg->rkey == le64_to_cpu(d->crkey))
 365                                 status = nvmet_pr_update_reg_attr(pr, reg,
 366                                                 nvmet_pr_update_reg_rkey,
 367                                                 &nrkey);
 368                         break;
 369                 }
 370         }
 371         up(&pr->pr_sem);
 372         return status;
 373 }
 374
 375 static void nvmet_execute_pr_register(struct nvmet_req *req)
 376 {
 377         u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
 378         bool ignore_key = nvmet_pr_parse_ignore_key(cdw10);
 379         struct nvmet_pr_register_data *d;
 380         u8 reg_act = cdw10 & 0x07; /* Reservation Register Action, bit 02:00 */
 381         u16 status;
 382
 383         d = kmalloc(sizeof(*d), GFP_KERNEL);
 384         if (!d) {
 385                 status = NVME_SC_INTERNAL;
 386                 goto out;
 387         }
 388
 389         status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d));
 390         if (status)
 391                 goto free_data;
 392
 393         switch (reg_act) {
 394         case NVME_PR_REGISTER_ACT_REG:
 395                 status = nvmet_pr_register(req, d);
 396                 break;
 397         case NVME_PR_REGISTER_ACT_UNREG:
 398                 status = nvmet_pr_unregister(req, d, ignore_key);
 399                 break;
 400         case NVME_PR_REGISTER_ACT_REPLACE:
 401                 status = nvmet_pr_replace(req, d, ignore_key);
 402                 break;
 403         default:
 404                 req->error_loc = offsetof(struct nvme_common_command, cdw10);
 405                 status = NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
 406                 break;
 407         }
 408 free_data:
 409         kfree(d);
 410 out:
 411         if (!status)
 412                 atomic_inc(&req->ns->pr.generation);
 413         nvmet_req_complete(req, status);
 414 }
 415
 416 static u16 nvmet_pr_acquire(struct nvmet_req *req,
 417                             struct nvmet_pr_registrant *reg,
 418                             u8 rtype)
 419 {
 420         struct nvmet_pr *pr = &req->ns->pr;
 421         struct nvmet_pr_registrant *holder;
 422
 423         holder = rcu_dereference_protected(pr->holder, 1);
 424         if (holder && reg != holder)
 425                 return  NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
 426         if (holder && reg == holder) {
 427                 if (holder->rtype == rtype)
 428                         return NVME_SC_SUCCESS;
 429                 return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
 430         }
 431
 432         nvmet_pr_set_new_holder(pr, rtype, reg);
 433         return NVME_SC_SUCCESS;
 434 }
 435
 436 static void nvmet_pr_confirm_ns_pc_ref(struct percpu_ref *ref)
 437 {
 438         struct nvmet_pr_per_ctrl_ref *pc_ref =
 439                 container_of(ref, struct nvmet_pr_per_ctrl_ref, ref);
 440
 441         complete(&pc_ref->confirm_done);
 442 }
 443
 444 static void nvmet_pr_set_ctrl_to_abort(struct nvmet_req *req, uuid_t *hostid)
 445 {
 446         struct nvmet_pr_per_ctrl_ref *pc_ref;
 447         struct nvmet_ns *ns = req->ns;
 448         unsigned long idx;
 449
 450         xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
 451                 if (uuid_equal(&pc_ref->hostid, hostid)) {
 452                         percpu_ref_kill_and_confirm(&pc_ref->ref,
 453                                                 nvmet_pr_confirm_ns_pc_ref);
 454                         wait_for_completion(&pc_ref->confirm_done);
 455                 }
 456         }
 457 }
 458
 459 static u16 nvmet_pr_unreg_all_host_by_prkey(struct nvmet_req *req, u64 prkey,
 460                                             uuid_t *send_hostid,
 461                                             bool abort)
 462 {
 463         u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
 464         struct nvmet_pr_registrant *reg, *tmp;
 465         struct nvmet_pr *pr = &req->ns->pr;
 466         uuid_t hostid;
 467
 468         list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
 469                 if (reg->rkey == prkey) {
 470                         status = NVME_SC_SUCCESS;
 471                         uuid_copy(&hostid, &reg->hostid);
 472                         if (abort)
 473                                 nvmet_pr_set_ctrl_to_abort(req, &hostid);
 474                         nvmet_pr_unregister_one(pr, reg);
 475                         if (!uuid_equal(&hostid, send_hostid))
 476                                 nvmet_pr_registration_preempted(pr, &hostid);
 477                 }
 478         }
 479         return status;
 480 }
 481
 482 static void nvmet_pr_unreg_all_others_by_prkey(struct nvmet_req *req,
 483                                                u64 prkey,
 484                                                uuid_t *send_hostid,
 485                                                bool abort)
 486 {
 487         struct nvmet_pr_registrant *reg, *tmp;
 488         struct nvmet_pr *pr = &req->ns->pr;
 489         uuid_t hostid;
 490
 491         list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
 492                 if (reg->rkey == prkey &&
 493                     !uuid_equal(&reg->hostid, send_hostid)) {
 494                         uuid_copy(&hostid, &reg->hostid);
 495                         if (abort)
 496                                 nvmet_pr_set_ctrl_to_abort(req, &hostid);
 497                         nvmet_pr_unregister_one(pr, reg);
 498                         nvmet_pr_registration_preempted(pr, &hostid);
 499                 }
 500         }
 501 }
 502
 503 static void nvmet_pr_unreg_all_others(struct nvmet_req *req,
 504                                       uuid_t *send_hostid,
 505                                       bool abort)
 506 {
 507         struct nvmet_pr_registrant *reg, *tmp;
 508         struct nvmet_pr *pr = &req->ns->pr;
 509         uuid_t hostid;
 510
 511         list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
 512                 if (!uuid_equal(&reg->hostid, send_hostid)) {
 513                         uuid_copy(&hostid, &reg->hostid);
 514                         if (abort)
 515                                 nvmet_pr_set_ctrl_to_abort(req, &hostid);
 516                         nvmet_pr_unregister_one(pr, reg);
 517                         nvmet_pr_registration_preempted(pr, &hostid);
 518                 }
 519         }
 520 }
 521
 522 static void nvmet_pr_update_holder_rtype(struct nvmet_pr_registrant *reg,
 523                                          void *attr)
 524 {
 525         u8 new_rtype = *(u8 *)attr;
 526
 527         reg->rtype = new_rtype;
 528 }
 529
 530 static u16 nvmet_pr_preempt(struct nvmet_req *req,
 531                             struct nvmet_pr_registrant *reg,
 532                             u8 rtype,
 533                             struct nvmet_pr_acquire_data *d,
 534                             bool abort)
 535 {
 536         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 537         struct nvmet_pr *pr = &req->ns->pr;
 538         struct nvmet_pr_registrant *holder;
 539         enum nvme_pr_type original_rtype;
 540         u64 prkey = le64_to_cpu(d->prkey);
 541         u16 status;
 542
 543         holder = rcu_dereference_protected(pr->holder, 1);
 544         if (!holder)
 545                 return nvmet_pr_unreg_all_host_by_prkey(req, prkey,
 546                                         &ctrl->hostid, abort);
 547
 548         original_rtype = holder->rtype;
 549         if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS ||
 550             original_rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) {
 551                 if (!prkey) {
 552                         /*
 553                          * To prevent possible access from other hosts, and
 554                          * avoid terminate the holder, set the new holder
 555                          * first before unregistering.
 556                          */
 557                         nvmet_pr_set_new_holder(pr, rtype, reg);
 558                         nvmet_pr_unreg_all_others(req, &ctrl->hostid, abort);
 559                         return NVME_SC_SUCCESS;
 560                 }
 561                 return nvmet_pr_unreg_all_host_by_prkey(req, prkey,
 562                                 &ctrl->hostid, abort);
 563         }
 564
 565         if (holder == reg) {
 566                 status = nvmet_pr_update_reg_attr(pr, holder,
 567                                 nvmet_pr_update_holder_rtype, &rtype);
 568                 if (!status && original_rtype != rtype)
 569                         nvmet_pr_resv_released(pr, &reg->hostid);
 570                 return status;
 571         }
 572
 573         if (prkey == holder->rkey) {
 574                 /*
 575                  * Same as before, set the new holder first.
 576                  */
 577                 nvmet_pr_set_new_holder(pr, rtype, reg);
 578                 nvmet_pr_unreg_all_others_by_prkey(req, prkey, &ctrl->hostid,
 579                                                 abort);
 580                 if (original_rtype != rtype)
 581                         nvmet_pr_resv_released(pr, &reg->hostid);
 582                 return NVME_SC_SUCCESS;
 583         }
 584
 585         if (prkey)
 586                 return nvmet_pr_unreg_all_host_by_prkey(req, prkey,
 587                                         &ctrl->hostid, abort);
 588         return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
 589 }
 590
 591 static void nvmet_pr_do_abort(struct work_struct *w)
 592 {
 593         struct nvmet_req *req = container_of(w, struct nvmet_req, r.abort_work);
 594         struct nvmet_pr_per_ctrl_ref *pc_ref;
 595         struct nvmet_ns *ns = req->ns;
 596         unsigned long idx;
 597
 598         /*
 599          * The target does not support abort, just wait per-controller ref to 0.
 600          */
 601         xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
 602                 if (percpu_ref_is_dying(&pc_ref->ref)) {
 603                         wait_for_completion(&pc_ref->free_done);
 604                         reinit_completion(&pc_ref->confirm_done);
 605                         reinit_completion(&pc_ref->free_done);
 606                         percpu_ref_resurrect(&pc_ref->ref);
 607                 }
 608         }
 609
 610         up(&ns->pr.pr_sem);
 611         nvmet_req_complete(req, NVME_SC_SUCCESS);
 612 }
 613
 614 static u16 __nvmet_execute_pr_acquire(struct nvmet_req *req,
 615                                       struct nvmet_pr_registrant *reg,
 616                                       u8 acquire_act,
 617                                       u8 rtype,
 618                                       struct nvmet_pr_acquire_data *d)
 619 {
 620         u16 status;
 621
 622         switch (acquire_act) {
 623         case NVME_PR_ACQUIRE_ACT_ACQUIRE:
 624                 status = nvmet_pr_acquire(req, reg, rtype);
 625                 goto out;
 626         case NVME_PR_ACQUIRE_ACT_PREEMPT:
 627                 status = nvmet_pr_preempt(req, reg, rtype, d, false);
 628                 goto inc_gen;
 629         case NVME_PR_ACQUIRE_ACT_PREEMPT_AND_ABORT:
 630                 status = nvmet_pr_preempt(req, reg, rtype, d, true);
 631                 goto inc_gen;
 632         default:
 633                 req->error_loc = offsetof(struct nvme_common_command, cdw10);
 634                 status = NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
 635                 goto out;
 636         }
 637 inc_gen:
 638         if (!status)
 639                 atomic_inc(&req->ns->pr.generation);
 640 out:
 641         return status;
 642 }
 643
 644 static void nvmet_execute_pr_acquire(struct nvmet_req *req)
 645 {
 646         u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
 647         bool ignore_key = nvmet_pr_parse_ignore_key(cdw10);
 648         /* Reservation type, bit 15:08 */
 649         u8 rtype = (u8)((cdw10 >> 8) & 0xff);
 650         /* Reservation acquire action, bit 02:00 */
 651         u8 acquire_act = cdw10 & 0x07;
 652         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 653         struct nvmet_pr_acquire_data *d = NULL;
 654         struct nvmet_pr *pr = &req->ns->pr;
 655         struct nvmet_pr_registrant *reg;
 656         u16 status = NVME_SC_SUCCESS;
 657
 658         if (ignore_key ||
 659             rtype < NVME_PR_WRITE_EXCLUSIVE ||
 660             rtype > NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) {
 661                 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
 662                 goto out;
 663         }
 664
 665         d = kmalloc(sizeof(*d), GFP_KERNEL);
 666         if (!d) {
 667                 status = NVME_SC_INTERNAL;
 668                 goto out;
 669         }
 670
 671         status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d));
 672         if (status)
 673                 goto free_data;
 674
 675         status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
 676         down(&pr->pr_sem);
 677         list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
 678                 if (uuid_equal(&reg->hostid, &ctrl->hostid) &&
 679                     reg->rkey == le64_to_cpu(d->crkey)) {
 680                         status = __nvmet_execute_pr_acquire(req, reg,
 681                                         acquire_act, rtype, d);
 682                         break;
 683                 }
 684         }
 685
 686         if (!status && acquire_act == NVME_PR_ACQUIRE_ACT_PREEMPT_AND_ABORT) {
 687                 kfree(d);
 688                 INIT_WORK(&req->r.abort_work, nvmet_pr_do_abort);
 689                 queue_work(nvmet_wq, &req->r.abort_work);
 690                 return;
 691         }
 692
 693         up(&pr->pr_sem);
 694
 695 free_data:
 696         kfree(d);
 697 out:
 698         nvmet_req_complete(req, status);
 699 }
 700
 701 static u16 nvmet_pr_release(struct nvmet_req *req,
 702                             struct nvmet_pr_registrant *reg,
 703                             u8 rtype)
 704 {
 705         struct nvmet_pr *pr = &req->ns->pr;
 706         struct nvmet_pr_registrant *holder;
 707         u8 original_rtype;
 708
 709         holder = rcu_dereference_protected(pr->holder, 1);
 710         if (!holder || reg != holder)
 711                 return NVME_SC_SUCCESS;
 712
 713         original_rtype = holder->rtype;
 714         if (original_rtype != rtype)
 715                 return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
 716
 717         rcu_assign_pointer(pr->holder, NULL);
 718
 719         if (original_rtype != NVME_PR_WRITE_EXCLUSIVE &&
 720             original_rtype != NVME_PR_EXCLUSIVE_ACCESS)
 721                 nvmet_pr_resv_released(pr, &reg->hostid);
 722
 723         return NVME_SC_SUCCESS;
 724 }
 725
 726 static void nvmet_pr_clear(struct nvmet_req *req)
 727 {
 728         struct nvmet_pr_registrant *reg, *tmp;
 729         struct nvmet_pr *pr = &req->ns->pr;
 730
 731         rcu_assign_pointer(pr->holder, NULL);
 732
 733         list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
 734                 list_del_rcu(&reg->entry);
 735                 if (!uuid_equal(&req->sq->ctrl->hostid, &reg->hostid))
 736                         nvmet_pr_resv_preempted(pr, &reg->hostid);
 737                 kfree_rcu(reg, rcu);
 738         }
 739
 740         atomic_inc(&pr->generation);
 741 }
 742
 743 static u16 __nvmet_execute_pr_release(struct nvmet_req *req,
 744                                       struct nvmet_pr_registrant *reg,
 745                                       u8 release_act, u8 rtype)
 746 {
 747         switch (release_act) {
 748         case NVME_PR_RELEASE_ACT_RELEASE:
 749                 return nvmet_pr_release(req, reg, rtype);
 750         case NVME_PR_RELEASE_ACT_CLEAR:
 751                 nvmet_pr_clear(req);
 752                 return NVME_SC_SUCCESS;
 753         default:
 754                 req->error_loc = offsetof(struct nvme_common_command, cdw10);
 755                 return NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
 756         }
 757 }
 758
 759 static void nvmet_execute_pr_release(struct nvmet_req *req)
 760 {
 761         u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
 762         bool ignore_key = nvmet_pr_parse_ignore_key(cdw10);
 763         u8 rtype = (u8)((cdw10 >> 8) & 0xff); /* Reservation type, bit 15:08 */
 764         u8 release_act = cdw10 & 0x07; /* Reservation release action, bit 02:00 */
 765         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 766         struct nvmet_pr *pr = &req->ns->pr;
 767         struct nvmet_pr_release_data *d;
 768         struct nvmet_pr_registrant *reg;
 769         u16 status;
 770
 771         if (ignore_key) {
 772                 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
 773                 goto out;
 774         }
 775
 776         d = kmalloc(sizeof(*d), GFP_KERNEL);
 777         if (!d) {
 778                 status = NVME_SC_INTERNAL;
 779                 goto out;
 780         }
 781
 782         status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d));
 783         if (status)
 784                 goto free_data;
 785
 786         status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
 787         down(&pr->pr_sem);
 788         list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
 789                 if (uuid_equal(&reg->hostid, &ctrl->hostid) &&
 790                     reg->rkey == le64_to_cpu(d->crkey)) {
 791                         status = __nvmet_execute_pr_release(req, reg,
 792                                         release_act, rtype);
 793                         break;
 794                 }
 795         }
 796         up(&pr->pr_sem);
 797 free_data:
 798         kfree(d);
 799 out:
 800         nvmet_req_complete(req, status);
 801 }
 802
 803 static void nvmet_execute_pr_report(struct nvmet_req *req)
 804 {
 805         u32 cdw11 = le32_to_cpu(req->cmd->common.cdw11);
 806         u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
 807         u32 num_bytes = 4 * (cdw10 + 1); /* cdw10 is number of dwords */
 808         u8 eds = cdw11 & 1; /* Extended data structure, bit 00 */
 809         struct nvme_registered_ctrl_ext *ctrl_eds;
 810         struct nvme_reservation_status_ext *data;
 811         struct nvmet_pr *pr = &req->ns->pr;
 812         struct nvmet_pr_registrant *holder;
 813         struct nvmet_pr_registrant *reg;
 814         u16 num_ctrls = 0;
 815         u16 status;
 816         u8 rtype;
 817
 818         /* nvmet hostid(uuid_t) is 128 bit. */
 819         if (!eds) {
 820                 req->error_loc = offsetof(struct nvme_common_command, cdw11);
 821                 status = NVME_SC_HOST_ID_INCONSIST | NVME_STATUS_DNR;
 822                 goto out;
 823         }
 824
 825         if (num_bytes < sizeof(struct nvme_reservation_status_ext)) {
 826                 req->error_loc = offsetof(struct nvme_common_command, cdw10);
 827                 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
 828                 goto out;
 829         }
 830
 831         data = kzalloc(num_bytes, GFP_KERNEL);
 832         if (!data) {
 833                 status = NVME_SC_INTERNAL;
 834                 goto out;
 835         }
 836         data->gen = cpu_to_le32(atomic_read(&pr->generation));
 837         data->ptpls = 0;
 838         ctrl_eds = data->regctl_eds;
 839
 840         rcu_read_lock();
 841         holder = rcu_dereference(pr->holder);
 842         rtype = holder ? holder->rtype : 0;
 843         data->rtype = rtype;
 844
 845         list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
 846                 num_ctrls++;
 847                 /*
 848                  * continue to get the number of all registrans.
 849                  */
 850                 if (((void *)ctrl_eds + sizeof(*ctrl_eds)) >
 851                     ((void *)data + num_bytes))
 852                         continue;
 853                 /*
 854                  * Dynamic controller, set cntlid to 0xffff.
 855                  */
 856                 ctrl_eds->cntlid = cpu_to_le16(NVME_CNTLID_DYNAMIC);
 857                 if (rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS ||
 858                     rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS)
 859                         ctrl_eds->rcsts = 1;
 860                 if (reg == holder)
 861                         ctrl_eds->rcsts = 1;
 862                 uuid_copy((uuid_t *)&ctrl_eds->hostid, &reg->hostid);
 863                 ctrl_eds->rkey = cpu_to_le64(reg->rkey);
 864                 ctrl_eds++;
 865         }
 866         rcu_read_unlock();
 867
 868         put_unaligned_le16(num_ctrls, data->regctl);
 869         status = nvmet_copy_to_sgl(req, 0, data, num_bytes);
 870         kfree(data);
 871 out:
 872         nvmet_req_complete(req, status);
 873 }
 874
 875 u16 nvmet_parse_pr_cmd(struct nvmet_req *req)
 876 {
 877         struct nvme_command *cmd = req->cmd;
 878
 879         switch (cmd->common.opcode) {
 880         case nvme_cmd_resv_register:
 881                 req->execute = nvmet_execute_pr_register;
 882                 break;
 883         case nvme_cmd_resv_acquire:
 884                 req->execute = nvmet_execute_pr_acquire;
 885                 break;
 886         case nvme_cmd_resv_release:
 887                 req->execute = nvmet_execute_pr_release;
 888                 break;
 889         case nvme_cmd_resv_report:
 890                 req->execute = nvmet_execute_pr_report;
 891                 break;
 892         default:
 893                 return 1;
 894         }
 895         return NVME_SC_SUCCESS;
 896 }
 897
 898 static bool nvmet_is_req_write_cmd_group(struct nvmet_req *req)
 899 {
 900         u8 opcode = req->cmd->common.opcode;
 901
 902         if (req->sq->qid) {
 903                 switch (opcode) {
 904                 case nvme_cmd_flush:
 905                 case nvme_cmd_write:
 906                 case nvme_cmd_write_zeroes:
 907                 case nvme_cmd_dsm:
 908                 case nvme_cmd_zone_append:
 909                 case nvme_cmd_zone_mgmt_send:
 910                         return true;
 911                 default:
 912                         return false;
 913                 }
 914         }
 915         return false;
 916 }
 917
 918 static bool nvmet_is_req_read_cmd_group(struct nvmet_req *req)
 919 {
 920         u8 opcode = req->cmd->common.opcode;
 921
 922         if (req->sq->qid) {
 923                 switch (opcode) {
 924                 case nvme_cmd_read:
 925                 case nvme_cmd_zone_mgmt_recv:
 926                         return true;
 927                 default:
 928                         return false;
 929                 }
 930         }
 931         return false;
 932 }
 933
 934 u16 nvmet_pr_check_cmd_access(struct nvmet_req *req)
 935 {
 936         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 937         struct nvmet_pr_registrant *holder;
 938         struct nvmet_ns *ns = req->ns;
 939         struct nvmet_pr *pr = &ns->pr;
 940         u16 status = NVME_SC_SUCCESS;
 941
 942         rcu_read_lock();
 943         holder = rcu_dereference(pr->holder);
 944         if (!holder)
 945                 goto unlock;
 946         if (uuid_equal(&ctrl->hostid, &holder->hostid))
 947                 goto unlock;
 948
 949         /*
 950          * The Reservation command group is checked in executing,
 951          * allow it here.
 952          */
 953         switch (holder->rtype) {
 954         case NVME_PR_WRITE_EXCLUSIVE:
 955                 if (nvmet_is_req_write_cmd_group(req))
 956                         status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
 957                 break;
 958         case NVME_PR_EXCLUSIVE_ACCESS:
 959                 if (nvmet_is_req_read_cmd_group(req) ||
 960                     nvmet_is_req_write_cmd_group(req))
 961                         status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
 962                 break;
 963         case NVME_PR_WRITE_EXCLUSIVE_REG_ONLY:
 964         case NVME_PR_WRITE_EXCLUSIVE_ALL_REGS:
 965                 if ((nvmet_is_req_write_cmd_group(req)) &&
 966                     !nvmet_pr_find_registrant(pr, &ctrl->hostid))
 967                         status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
 968                 break;
 969         case NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY:
 970         case NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS:
 971                 if ((nvmet_is_req_read_cmd_group(req) ||
 972                     nvmet_is_req_write_cmd_group(req)) &&
 973                     !nvmet_pr_find_registrant(pr, &ctrl->hostid))
 974                         status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
 975                 break;
 976         default:
 977                 pr_warn("the reservation type is set wrong, type:%d\n",
 978                         holder->rtype);
 979                 break;
 980         }
 981
 982 unlock:
 983         rcu_read_unlock();
 984         if (status)
 985                 req->error_loc = offsetof(struct nvme_common_command, opcode);
 986         return status;
 987 }
 988
 989 u16 nvmet_pr_get_ns_pc_ref(struct nvmet_req *req)
 990 {
 991         struct nvmet_pr_per_ctrl_ref *pc_ref;
 992
 993         pc_ref = xa_load(&req->ns->pr_per_ctrl_refs,
 994                         req->sq->ctrl->cntlid);
 995         if (unlikely(!percpu_ref_tryget_live(&pc_ref->ref)))
 996                 return NVME_SC_INTERNAL;
 997         req->pc_ref = pc_ref;
 998         return NVME_SC_SUCCESS;
 999 }
1000
1001 static void nvmet_pr_ctrl_ns_all_cmds_done(struct percpu_ref *ref)
1002 {
1003         struct nvmet_pr_per_ctrl_ref *pc_ref =
1004                 container_of(ref, struct nvmet_pr_per_ctrl_ref, ref);
1005
1006         complete(&pc_ref->free_done);
1007 }
1008
1009 static int nvmet_pr_alloc_and_insert_pc_ref(struct nvmet_ns *ns,
1010                                             unsigned long idx,
1011                                             uuid_t *hostid)
1012 {
1013         struct nvmet_pr_per_ctrl_ref *pc_ref;
1014         int ret;
1015
1016         pc_ref = kmalloc(sizeof(*pc_ref), GFP_ATOMIC);
1017         if (!pc_ref)
1018                 return  -ENOMEM;
1019
1020         ret = percpu_ref_init(&pc_ref->ref, nvmet_pr_ctrl_ns_all_cmds_done,
1021                         PERCPU_REF_ALLOW_REINIT, GFP_KERNEL);
1022         if (ret)
1023                 goto free;
1024
1025         init_completion(&pc_ref->free_done);
1026         init_completion(&pc_ref->confirm_done);
1027         uuid_copy(&pc_ref->hostid, hostid);
1028
1029         ret = xa_insert(&ns->pr_per_ctrl_refs, idx, pc_ref, GFP_KERNEL);
1030         if (ret)
1031                 goto exit;
1032         return ret;
1033 exit:
1034         percpu_ref_exit(&pc_ref->ref);
1035 free:
1036         kfree(pc_ref);
1037         return ret;
1038 }
1039
1040 int nvmet_ctrl_init_pr(struct nvmet_ctrl *ctrl)
1041 {
1042         struct nvmet_subsys *subsys = ctrl->subsys;
1043         struct nvmet_pr_per_ctrl_ref *pc_ref;
1044         struct nvmet_ns *ns = NULL;
1045         unsigned long idx;
1046         int ret;
1047
1048         ctrl->pr_log_mgr.counter = 0;
1049         ctrl->pr_log_mgr.lost_count = 0;
1050         mutex_init(&ctrl->pr_log_mgr.lock);
1051         INIT_KFIFO(ctrl->pr_log_mgr.log_queue);
1052
1053         /*
1054          * Here we are under subsys lock, if an ns not in subsys->namespaces,
1055          * we can make sure that ns is not enabled, and not call
1056          * nvmet_pr_init_ns(), see more details in nvmet_ns_enable().
1057          * So just check ns->pr.enable.
1058          */
1059         nvmet_for_each_enabled_ns(&subsys->namespaces, idx, ns) {
1060                 if (ns->pr.enable) {
1061                         ret = nvmet_pr_alloc_and_insert_pc_ref(ns, ctrl->cntlid,
1062                                                         &ctrl->hostid);
1063                         if (ret)
1064                                 goto free_per_ctrl_refs;
1065                 }
1066         }
1067         return 0;
1068
1069 free_per_ctrl_refs:
1070         nvmet_for_each_enabled_ns(&subsys->namespaces, idx, ns) {
1071                 if (ns->pr.enable) {
1072                         pc_ref = xa_erase(&ns->pr_per_ctrl_refs, ctrl->cntlid);
1073                         if (pc_ref)
1074                                 percpu_ref_exit(&pc_ref->ref);
1075                         kfree(pc_ref);
1076                 }
1077         }
1078         return ret;
1079 }
1080
1081 void nvmet_ctrl_destroy_pr(struct nvmet_ctrl *ctrl)
1082 {
1083         struct nvmet_pr_per_ctrl_ref *pc_ref;
1084         struct nvmet_ns *ns;
1085         unsigned long idx;
1086
1087         kfifo_free(&ctrl->pr_log_mgr.log_queue);
1088         mutex_destroy(&ctrl->pr_log_mgr.lock);
1089
1090         nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) {
1091                 if (ns->pr.enable) {
1092                         pc_ref = xa_erase(&ns->pr_per_ctrl_refs, ctrl->cntlid);
1093                         if (pc_ref)
1094                                 percpu_ref_exit(&pc_ref->ref);
1095                         kfree(pc_ref);
1096                 }
1097         }
1098 }
1099
1100 int nvmet_pr_init_ns(struct nvmet_ns *ns)
1101 {
1102         struct nvmet_subsys *subsys = ns->subsys;
1103         struct nvmet_pr_per_ctrl_ref *pc_ref;
1104         struct nvmet_ctrl *ctrl = NULL;
1105         unsigned long idx;
1106         int ret;
1107
1108         ns->pr.holder = NULL;
1109         atomic_set(&ns->pr.generation, 0);
1110         sema_init(&ns->pr.pr_sem, 1);
1111         INIT_LIST_HEAD(&ns->pr.registrant_list);
1112         ns->pr.notify_mask = 0;
1113
1114         xa_init(&ns->pr_per_ctrl_refs);
1115
1116         list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
1117                 ret = nvmet_pr_alloc_and_insert_pc_ref(ns, ctrl->cntlid,
1118                                                 &ctrl->hostid);
1119                 if (ret)
1120                         goto free_per_ctrl_refs;
1121         }
1122         return 0;
1123
1124 free_per_ctrl_refs:
1125         xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
1126                 xa_erase(&ns->pr_per_ctrl_refs, idx);
1127                 percpu_ref_exit(&pc_ref->ref);
1128                 kfree(pc_ref);
1129         }
1130         return ret;
1131 }
1132
1133 void nvmet_pr_exit_ns(struct nvmet_ns *ns)
1134 {
1135         struct nvmet_pr_registrant *reg, *tmp;
1136         struct nvmet_pr_per_ctrl_ref *pc_ref;
1137         struct nvmet_pr *pr = &ns->pr;
1138         unsigned long idx;
1139
1140         list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
1141                 list_del(&reg->entry);
1142                 kfree(reg);
1143         }
1144
1145         xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
1146                 /*
1147                  * No command on ns here, we can safely free pc_ref.
1148                  */
1149                 pc_ref = xa_erase(&ns->pr_per_ctrl_refs, idx);
1150                 percpu_ref_exit(&pc_ref->ref);
1151                 kfree(pc_ref);
1152         }
1153
1154         xa_destroy(&ns->pr_per_ctrl_refs);
1155 }