drivers/nvme/target/fabrics-cmd-auth.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVMe over Fabrics DH-HMAC-CHAP authentication command handling.
   4  * Copyright (c) 2020 Hannes Reinecke, SUSE Software Solutions.
   5  * All rights reserved.
   6  */
   7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   8 #include <linux/blkdev.h>
   9 #include <linux/random.h>
  10 #include <linux/nvme-auth.h>
  11 #include <crypto/hash.h>
  12 #include <crypto/kpp.h>
  13 #include "nvmet.h"
  14
  15 static void nvmet_auth_expired_work(struct work_struct *work)
  16 {
  17         struct nvmet_sq *sq = container_of(to_delayed_work(work),
  18                         struct nvmet_sq, auth_expired_work);
  19
  20         pr_debug("%s: ctrl %d qid %d transaction %u expired, resetting\n",
  21                  __func__, sq->ctrl->cntlid, sq->qid, sq->dhchap_tid);
  22         sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
  23         sq->dhchap_tid = -1;
  24 }
  25
  26 void nvmet_auth_sq_init(struct nvmet_sq *sq)
  27 {
  28         /* Initialize in-band authentication */
  29         INIT_DELAYED_WORK(&sq->auth_expired_work, nvmet_auth_expired_work);
  30         sq->authenticated = false;
  31         sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
  32 }
  33
  34 static u16 nvmet_auth_negotiate(struct nvmet_req *req, void *d)
  35 {
  36         struct nvmet_ctrl *ctrl = req->sq->ctrl;
  37         struct nvmf_auth_dhchap_negotiate_data *data = d;
  38         int i, hash_id = 0, fallback_hash_id = 0, dhgid, fallback_dhgid;
  39
  40         pr_debug("%s: ctrl %d qid %d: data sc_d %d napd %d authid %d halen %d dhlen %d\n",
  41                  __func__, ctrl->cntlid, req->sq->qid,
  42                  data->sc_c, data->napd, data->auth_protocol[0].dhchap.authid,
  43                  data->auth_protocol[0].dhchap.halen,
  44                  data->auth_protocol[0].dhchap.dhlen);
  45         req->sq->dhchap_tid = le16_to_cpu(data->t_id);
  46         if (data->sc_c)
  47                 return NVME_AUTH_DHCHAP_FAILURE_CONCAT_MISMATCH;
  48
  49         if (data->napd != 1)
  50                 return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
  51
  52         if (data->auth_protocol[0].dhchap.authid !=
  53             NVME_AUTH_DHCHAP_AUTH_ID)
  54                 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
  55
  56         for (i = 0; i < data->auth_protocol[0].dhchap.halen; i++) {
  57                 u8 host_hmac_id = data->auth_protocol[0].dhchap.idlist[i];
  58
  59                 if (!fallback_hash_id &&
  60                     crypto_has_shash(nvme_auth_hmac_name(host_hmac_id), 0, 0))
  61                         fallback_hash_id = host_hmac_id;
  62                 if (ctrl->shash_id != host_hmac_id)
  63                         continue;
  64                 hash_id = ctrl->shash_id;
  65                 break;
  66         }
  67         if (hash_id == 0) {
  68                 if (fallback_hash_id == 0) {
  69                         pr_debug("%s: ctrl %d qid %d: no usable hash found\n",
  70                                  __func__, ctrl->cntlid, req->sq->qid);
  71                         return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
  72                 }
  73                 pr_debug("%s: ctrl %d qid %d: no usable hash found, falling back to %s\n",
  74                          __func__, ctrl->cntlid, req->sq->qid,
  75                          nvme_auth_hmac_name(fallback_hash_id));
  76                 ctrl->shash_id = fallback_hash_id;
  77         }
  78
  79         dhgid = -1;
  80         fallback_dhgid = -1;
  81         for (i = 0; i < data->auth_protocol[0].dhchap.dhlen; i++) {
  82                 int tmp_dhgid = data->auth_protocol[0].dhchap.idlist[i + 30];
  83
  84                 if (tmp_dhgid != ctrl->dh_gid) {
  85                         dhgid = tmp_dhgid;
  86                         break;
  87                 }
  88                 if (fallback_dhgid < 0) {
  89                         const char *kpp = nvme_auth_dhgroup_kpp(tmp_dhgid);
  90
  91                         if (crypto_has_kpp(kpp, 0, 0))
  92                                 fallback_dhgid = tmp_dhgid;
  93                 }
  94         }
  95         if (dhgid < 0) {
  96                 if (fallback_dhgid < 0) {
  97                         pr_debug("%s: ctrl %d qid %d: no usable DH group found\n",
  98                                  __func__, ctrl->cntlid, req->sq->qid);
  99                         return NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
 100                 }
 101                 pr_debug("%s: ctrl %d qid %d: configured DH group %s not found\n",
 102                          __func__, ctrl->cntlid, req->sq->qid,
 103                          nvme_auth_dhgroup_name(fallback_dhgid));
 104                 ctrl->dh_gid = fallback_dhgid;
 105         }
 106         pr_debug("%s: ctrl %d qid %d: selected DH group %s (%d)\n",
 107                  __func__, ctrl->cntlid, req->sq->qid,
 108                  nvme_auth_dhgroup_name(ctrl->dh_gid), ctrl->dh_gid);
 109         return 0;
 110 }
 111
 112 static u16 nvmet_auth_reply(struct nvmet_req *req, void *d)
 113 {
 114         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 115         struct nvmf_auth_dhchap_reply_data *data = d;
 116         u16 dhvlen = le16_to_cpu(data->dhvlen);
 117         u8 *response;
 118
 119         pr_debug("%s: ctrl %d qid %d: data hl %d cvalid %d dhvlen %u\n",
 120                  __func__, ctrl->cntlid, req->sq->qid,
 121                  data->hl, data->cvalid, dhvlen);
 122
 123         if (dhvlen) {
 124                 if (!ctrl->dh_tfm)
 125                         return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 126                 if (nvmet_auth_ctrl_sesskey(req, data->rval + 2 * data->hl,
 127                                             dhvlen) < 0)
 128                         return NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
 129         }
 130
 131         response = kmalloc(data->hl, GFP_KERNEL);
 132         if (!response)
 133                 return NVME_AUTH_DHCHAP_FAILURE_FAILED;
 134
 135         if (!ctrl->host_key) {
 136                 pr_warn("ctrl %d qid %d no host key\n",
 137                         ctrl->cntlid, req->sq->qid);
 138                 kfree(response);
 139                 return NVME_AUTH_DHCHAP_FAILURE_FAILED;
 140         }
 141         if (nvmet_auth_host_hash(req, response, data->hl) < 0) {
 142                 pr_debug("ctrl %d qid %d host hash failed\n",
 143                          ctrl->cntlid, req->sq->qid);
 144                 kfree(response);
 145                 return NVME_AUTH_DHCHAP_FAILURE_FAILED;
 146         }
 147
 148         if (memcmp(data->rval, response, data->hl)) {
 149                 pr_info("ctrl %d qid %d host response mismatch\n",
 150                         ctrl->cntlid, req->sq->qid);
 151                 kfree(response);
 152                 return NVME_AUTH_DHCHAP_FAILURE_FAILED;
 153         }
 154         kfree(response);
 155         pr_debug("%s: ctrl %d qid %d host authenticated\n",
 156                  __func__, ctrl->cntlid, req->sq->qid);
 157         if (data->cvalid) {
 158                 req->sq->dhchap_c2 = kmemdup(data->rval + data->hl, data->hl,
 159                                              GFP_KERNEL);
 160                 if (!req->sq->dhchap_c2)
 161                         return NVME_AUTH_DHCHAP_FAILURE_FAILED;
 162
 163                 pr_debug("%s: ctrl %d qid %d challenge %*ph\n",
 164                          __func__, ctrl->cntlid, req->sq->qid, data->hl,
 165                          req->sq->dhchap_c2);
 166                 req->sq->dhchap_s2 = le32_to_cpu(data->seqnum);
 167         } else {
 168                 req->sq->authenticated = true;
 169                 req->sq->dhchap_c2 = NULL;
 170         }
 171
 172         return 0;
 173 }
 174
 175 static u16 nvmet_auth_failure2(void *d)
 176 {
 177         struct nvmf_auth_dhchap_failure_data *data = d;
 178
 179         return data->rescode_exp;
 180 }
 181
 182 void nvmet_execute_auth_send(struct nvmet_req *req)
 183 {
 184         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 185         struct nvmf_auth_dhchap_success2_data *data;
 186         void *d;
 187         u32 tl;
 188         u16 status = 0;
 189
 190         if (req->cmd->auth_send.secp != NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER) {
 191                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 192                 req->error_loc =
 193                         offsetof(struct nvmf_auth_send_command, secp);
 194                 goto done;
 195         }
 196         if (req->cmd->auth_send.spsp0 != 0x01) {
 197                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 198                 req->error_loc =
 199                         offsetof(struct nvmf_auth_send_command, spsp0);
 200                 goto done;
 201         }
 202         if (req->cmd->auth_send.spsp1 != 0x01) {
 203                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 204                 req->error_loc =
 205                         offsetof(struct nvmf_auth_send_command, spsp1);
 206                 goto done;
 207         }
 208         tl = le32_to_cpu(req->cmd->auth_send.tl);
 209         if (!tl) {
 210                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 211                 req->error_loc =
 212                         offsetof(struct nvmf_auth_send_command, tl);
 213                 goto done;
 214         }
 215         if (!nvmet_check_transfer_len(req, tl)) {
 216                 pr_debug("%s: transfer length mismatch (%u)\n", __func__, tl);
 217                 return;
 218         }
 219
 220         d = kmalloc(tl, GFP_KERNEL);
 221         if (!d) {
 222                 status = NVME_SC_INTERNAL;
 223                 goto done;
 224         }
 225
 226         status = nvmet_copy_from_sgl(req, 0, d, tl);
 227         if (status)
 228                 goto done_kfree;
 229
 230         data = d;
 231         pr_debug("%s: ctrl %d qid %d type %d id %d step %x\n", __func__,
 232                  ctrl->cntlid, req->sq->qid, data->auth_type, data->auth_id,
 233                  req->sq->dhchap_step);
 234         if (data->auth_type != NVME_AUTH_COMMON_MESSAGES &&
 235             data->auth_type != NVME_AUTH_DHCHAP_MESSAGES)
 236                 goto done_failure1;
 237         if (data->auth_type == NVME_AUTH_COMMON_MESSAGES) {
 238                 if (data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE) {
 239                         /* Restart negotiation */
 240                         pr_debug("%s: ctrl %d qid %d reset negotiation\n", __func__,
 241                                  ctrl->cntlid, req->sq->qid);
 242                         if (!req->sq->qid) {
 243                                 if (nvmet_setup_auth(ctrl) < 0) {
 244                                         status = NVME_SC_INTERNAL;
 245                                         pr_err("ctrl %d qid 0 failed to setup"
 246                                                "re-authentication",
 247                                                ctrl->cntlid);
 248                                         goto done_failure1;
 249                                 }
 250                         }
 251                         req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
 252                 } else if (data->auth_id != req->sq->dhchap_step)
 253                         goto done_failure1;
 254                 /* Validate negotiation parameters */
 255                 status = nvmet_auth_negotiate(req, d);
 256                 if (status == 0)
 257                         req->sq->dhchap_step =
 258                                 NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE;
 259                 else {
 260                         req->sq->dhchap_step =
 261                                 NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
 262                         req->sq->dhchap_status = status;
 263                         status = 0;
 264                 }
 265                 goto done_kfree;
 266         }
 267         if (data->auth_id != req->sq->dhchap_step) {
 268                 pr_debug("%s: ctrl %d qid %d step mismatch (%d != %d)\n",
 269                          __func__, ctrl->cntlid, req->sq->qid,
 270                          data->auth_id, req->sq->dhchap_step);
 271                 goto done_failure1;
 272         }
 273         if (le16_to_cpu(data->t_id) != req->sq->dhchap_tid) {
 274                 pr_debug("%s: ctrl %d qid %d invalid transaction %d (expected %d)\n",
 275                          __func__, ctrl->cntlid, req->sq->qid,
 276                          le16_to_cpu(data->t_id),
 277                          req->sq->dhchap_tid);
 278                 req->sq->dhchap_step =
 279                         NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
 280                 req->sq->dhchap_status =
 281                         NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 282                 goto done_kfree;
 283         }
 284
 285         switch (data->auth_id) {
 286         case NVME_AUTH_DHCHAP_MESSAGE_REPLY:
 287                 status = nvmet_auth_reply(req, d);
 288                 if (status == 0)
 289                         req->sq->dhchap_step =
 290                                 NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1;
 291                 else {
 292                         req->sq->dhchap_step =
 293                                 NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
 294                         req->sq->dhchap_status = status;
 295                         status = 0;
 296                 }
 297                 goto done_kfree;
 298                 break;
 299         case NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2:
 300                 req->sq->authenticated = true;
 301                 pr_debug("%s: ctrl %d qid %d ctrl authenticated\n",
 302                          __func__, ctrl->cntlid, req->sq->qid);
 303                 goto done_kfree;
 304                 break;
 305         case NVME_AUTH_DHCHAP_MESSAGE_FAILURE2:
 306                 status = nvmet_auth_failure2(d);
 307                 if (status) {
 308                         pr_warn("ctrl %d qid %d: authentication failed (%d)\n",
 309                                 ctrl->cntlid, req->sq->qid, status);
 310                         req->sq->dhchap_status = status;
 311                         req->sq->authenticated = false;
 312                         status = 0;
 313                 }
 314                 goto done_kfree;
 315                 break;
 316         default:
 317                 req->sq->dhchap_status =
 318                         NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
 319                 req->sq->dhchap_step =
 320                         NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
 321                 req->sq->authenticated = false;
 322                 goto done_kfree;
 323                 break;
 324         }
 325 done_failure1:
 326         req->sq->dhchap_status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
 327         req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
 328
 329 done_kfree:
 330         kfree(d);
 331 done:
 332         pr_debug("%s: ctrl %d qid %d dhchap status %x step %x\n", __func__,
 333                  ctrl->cntlid, req->sq->qid,
 334                  req->sq->dhchap_status, req->sq->dhchap_step);
 335         if (status)
 336                 pr_debug("%s: ctrl %d qid %d nvme status %x error loc %d\n",
 337                          __func__, ctrl->cntlid, req->sq->qid,
 338                          status, req->error_loc);
 339         req->cqe->result.u64 = 0;
 340         nvmet_req_complete(req, status);
 341         if (req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 &&
 342             req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_FAILURE2) {
 343                 unsigned long auth_expire_secs = ctrl->kato ? ctrl->kato : 120;
 344
 345                 mod_delayed_work(system_wq, &req->sq->auth_expired_work,
 346                                  auth_expire_secs * HZ);
 347                 return;
 348         }
 349         /* Final states, clear up variables */
 350         nvmet_auth_sq_free(req->sq);
 351         if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE2)
 352                 nvmet_ctrl_fatal_error(ctrl);
 353 }
 354
 355 static int nvmet_auth_challenge(struct nvmet_req *req, void *d, int al)
 356 {
 357         struct nvmf_auth_dhchap_challenge_data *data = d;
 358         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 359         int ret = 0;
 360         int hash_len = nvme_auth_hmac_hash_len(ctrl->shash_id);
 361         int data_size = sizeof(*d) + hash_len;
 362
 363         if (ctrl->dh_tfm)
 364                 data_size += ctrl->dh_keysize;
 365         if (al < data_size) {
 366                 pr_debug("%s: buffer too small (al %d need %d)\n", __func__,
 367                          al, data_size);
 368                 return -EINVAL;
 369         }
 370         memset(data, 0, data_size);
 371         req->sq->dhchap_s1 = nvme_auth_get_seqnum();
 372         data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
 373         data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE;
 374         data->t_id = cpu_to_le16(req->sq->dhchap_tid);
 375         data->hashid = ctrl->shash_id;
 376         data->hl = hash_len;
 377         data->seqnum = cpu_to_le32(req->sq->dhchap_s1);
 378         req->sq->dhchap_c1 = kmalloc(data->hl, GFP_KERNEL);
 379         if (!req->sq->dhchap_c1)
 380                 return -ENOMEM;
 381         get_random_bytes(req->sq->dhchap_c1, data->hl);
 382         memcpy(data->cval, req->sq->dhchap_c1, data->hl);
 383         if (ctrl->dh_tfm) {
 384                 data->dhgid = ctrl->dh_gid;
 385                 data->dhvlen = cpu_to_le16(ctrl->dh_keysize);
 386                 ret = nvmet_auth_ctrl_exponential(req, data->cval + data->hl,
 387                                                   ctrl->dh_keysize);
 388         }
 389         pr_debug("%s: ctrl %d qid %d seq %d transaction %d hl %d dhvlen %zu\n",
 390                  __func__, ctrl->cntlid, req->sq->qid, req->sq->dhchap_s1,
 391                  req->sq->dhchap_tid, data->hl, ctrl->dh_keysize);
 392         return ret;
 393 }
 394
 395 static int nvmet_auth_success1(struct nvmet_req *req, void *d, int al)
 396 {
 397         struct nvmf_auth_dhchap_success1_data *data = d;
 398         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 399         int hash_len = nvme_auth_hmac_hash_len(ctrl->shash_id);
 400
 401         WARN_ON(al < sizeof(*data));
 402         memset(data, 0, sizeof(*data));
 403         data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
 404         data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1;
 405         data->t_id = cpu_to_le16(req->sq->dhchap_tid);
 406         data->hl = hash_len;
 407         if (req->sq->dhchap_c2) {
 408                 if (!ctrl->ctrl_key) {
 409                         pr_warn("ctrl %d qid %d no ctrl key\n",
 410                                 ctrl->cntlid, req->sq->qid);
 411                         return NVME_AUTH_DHCHAP_FAILURE_FAILED;
 412                 }
 413                 if (nvmet_auth_ctrl_hash(req, data->rval, data->hl))
 414                         return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
 415                 data->rvalid = 1;
 416                 pr_debug("ctrl %d qid %d response %*ph\n",
 417                          ctrl->cntlid, req->sq->qid, data->hl, data->rval);
 418         }
 419         return 0;
 420 }
 421
 422 static void nvmet_auth_failure1(struct nvmet_req *req, void *d, int al)
 423 {
 424         struct nvmf_auth_dhchap_failure_data *data = d;
 425
 426         WARN_ON(al < sizeof(*data));
 427         data->auth_type = NVME_AUTH_COMMON_MESSAGES;
 428         data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
 429         data->t_id = cpu_to_le16(req->sq->dhchap_tid);
 430         data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED;
 431         data->rescode_exp = req->sq->dhchap_status;
 432 }
 433
 434 void nvmet_execute_auth_receive(struct nvmet_req *req)
 435 {
 436         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 437         void *d;
 438         u32 al;
 439         u16 status = 0;
 440
 441         if (req->cmd->auth_receive.secp != NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER) {
 442                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 443                 req->error_loc =
 444                         offsetof(struct nvmf_auth_receive_command, secp);
 445                 goto done;
 446         }
 447         if (req->cmd->auth_receive.spsp0 != 0x01) {
 448                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 449                 req->error_loc =
 450                         offsetof(struct nvmf_auth_receive_command, spsp0);
 451                 goto done;
 452         }
 453         if (req->cmd->auth_receive.spsp1 != 0x01) {
 454                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 455                 req->error_loc =
 456                         offsetof(struct nvmf_auth_receive_command, spsp1);
 457                 goto done;
 458         }
 459         al = le32_to_cpu(req->cmd->auth_receive.al);
 460         if (!al) {
 461                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 462                 req->error_loc =
 463                         offsetof(struct nvmf_auth_receive_command, al);
 464                 goto done;
 465         }
 466         if (!nvmet_check_transfer_len(req, al)) {
 467                 pr_debug("%s: transfer length mismatch (%u)\n", __func__, al);
 468                 return;
 469         }
 470
 471         d = kmalloc(al, GFP_KERNEL);
 472         if (!d) {
 473                 status = NVME_SC_INTERNAL;
 474                 goto done;
 475         }
 476         pr_debug("%s: ctrl %d qid %d step %x\n", __func__,
 477                  ctrl->cntlid, req->sq->qid, req->sq->dhchap_step);
 478         switch (req->sq->dhchap_step) {
 479         case NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE:
 480                 if (nvmet_auth_challenge(req, d, al) < 0) {
 481                         pr_warn("ctrl %d qid %d: challenge error (%d)\n",
 482                                 ctrl->cntlid, req->sq->qid, status);
 483                         status = NVME_SC_INTERNAL;
 484                         break;
 485                 }
 486                 if (status) {
 487                         req->sq->dhchap_status = status;
 488                         nvmet_auth_failure1(req, d, al);
 489                         pr_warn("ctrl %d qid %d: challenge status (%x)\n",
 490                                 ctrl->cntlid, req->sq->qid,
 491                                 req->sq->dhchap_status);
 492                         status = 0;
 493                         break;
 494                 }
 495                 req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_REPLY;
 496                 break;
 497         case NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1:
 498                 status = nvmet_auth_success1(req, d, al);
 499                 if (status) {
 500                         req->sq->dhchap_status = status;
 501                         req->sq->authenticated = false;
 502                         nvmet_auth_failure1(req, d, al);
 503                         pr_warn("ctrl %d qid %d: success1 status (%x)\n",
 504                                 ctrl->cntlid, req->sq->qid,
 505                                 req->sq->dhchap_status);
 506                         break;
 507                 }
 508                 req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2;
 509                 break;
 510         case NVME_AUTH_DHCHAP_MESSAGE_FAILURE1:
 511                 req->sq->authenticated = false;
 512                 nvmet_auth_failure1(req, d, al);
 513                 pr_warn("ctrl %d qid %d failure1 (%x)\n",
 514                         ctrl->cntlid, req->sq->qid, req->sq->dhchap_status);
 515                 break;
 516         default:
 517                 pr_warn("ctrl %d qid %d unhandled step (%d)\n",
 518                         ctrl->cntlid, req->sq->qid, req->sq->dhchap_step);
 519                 req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
 520                 req->sq->dhchap_status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 521                 nvmet_auth_failure1(req, d, al);
 522                 status = 0;
 523                 break;
 524         }
 525
 526         status = nvmet_copy_to_sgl(req, 0, d, al);
 527         kfree(d);
 528 done:
 529         req->cqe->result.u64 = 0;
 530         nvmet_req_complete(req, status);
 531         if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2)
 532                 nvmet_auth_sq_free(req->sq);
 533         else if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
 534                 nvmet_auth_sq_free(req->sq);
 535                 nvmet_ctrl_fatal_error(ctrl);
 536         }
 537 }