drivers/nvme/target/fabrics-cmd-auth.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVMe over Fabrics DH-HMAC-CHAP authentication command handling.
   4  * Copyright (c) 2020 Hannes Reinecke, SUSE Software Solutions.
   5  * All rights reserved.
   6  */
   7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   8 #include <linux/blkdev.h>
   9 #include <linux/random.h>
  10 #include <linux/nvme-auth.h>
  11 #include <crypto/hash.h>
  12 #include <crypto/kpp.h>
  13 #include "nvmet.h"
  14
  15 static void nvmet_auth_expired_work(struct work_struct *work)
  16 {
  17         struct nvmet_sq *sq = container_of(to_delayed_work(work),
  18                         struct nvmet_sq, auth_expired_work);
  19
  20         pr_debug("%s: ctrl %d qid %d transaction %u expired, resetting\n",
  21                  __func__, sq->ctrl->cntlid, sq->qid, sq->dhchap_tid);
  22         sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
  23         sq->dhchap_tid = -1;
  24 }
  25
  26 void nvmet_init_auth(struct nvmet_ctrl *ctrl, struct nvmet_req *req)
  27 {
  28         u32 result = le32_to_cpu(req->cqe->result.u32);
  29
  30         /* Initialize in-band authentication */
  31         INIT_DELAYED_WORK(&req->sq->auth_expired_work,
  32                           nvmet_auth_expired_work);
  33         req->sq->authenticated = false;
  34         req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
  35         result |= (u32)NVME_CONNECT_AUTHREQ_ATR << 16;
  36         req->cqe->result.u32 = cpu_to_le32(result);
  37 }
  38
  39 static u16 nvmet_auth_negotiate(struct nvmet_req *req, void *d)
  40 {
  41         struct nvmet_ctrl *ctrl = req->sq->ctrl;
  42         struct nvmf_auth_dhchap_negotiate_data *data = d;
  43         int i, hash_id = 0, fallback_hash_id = 0, dhgid, fallback_dhgid;
  44
  45         pr_debug("%s: ctrl %d qid %d: data sc_d %d napd %d authid %d halen %d dhlen %d\n",
  46                  __func__, ctrl->cntlid, req->sq->qid,
  47                  data->sc_c, data->napd, data->auth_protocol[0].dhchap.authid,
  48                  data->auth_protocol[0].dhchap.halen,
  49                  data->auth_protocol[0].dhchap.dhlen);
  50         req->sq->dhchap_tid = le16_to_cpu(data->t_id);
  51         if (data->sc_c)
  52                 return NVME_AUTH_DHCHAP_FAILURE_CONCAT_MISMATCH;
  53
  54         if (data->napd != 1)
  55                 return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
  56
  57         if (data->auth_protocol[0].dhchap.authid !=
  58             NVME_AUTH_DHCHAP_AUTH_ID)
  59                 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
  60
  61         for (i = 0; i < data->auth_protocol[0].dhchap.halen; i++) {
  62                 u8 host_hmac_id = data->auth_protocol[0].dhchap.idlist[i];
  63
  64                 if (!fallback_hash_id &&
  65                     crypto_has_shash(nvme_auth_hmac_name(host_hmac_id), 0, 0))
  66                         fallback_hash_id = host_hmac_id;
  67                 if (ctrl->shash_id != host_hmac_id)
  68                         continue;
  69                 hash_id = ctrl->shash_id;
  70                 break;
  71         }
  72         if (hash_id == 0) {
  73                 if (fallback_hash_id == 0) {
  74                         pr_debug("%s: ctrl %d qid %d: no usable hash found\n",
  75                                  __func__, ctrl->cntlid, req->sq->qid);
  76                         return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
  77                 }
  78                 pr_debug("%s: ctrl %d qid %d: no usable hash found, falling back to %s\n",
  79                          __func__, ctrl->cntlid, req->sq->qid,
  80                          nvme_auth_hmac_name(fallback_hash_id));
  81                 ctrl->shash_id = fallback_hash_id;
  82         }
  83
  84         dhgid = -1;
  85         fallback_dhgid = -1;
  86         for (i = 0; i < data->auth_protocol[0].dhchap.dhlen; i++) {
  87                 int tmp_dhgid = data->auth_protocol[0].dhchap.idlist[i + 30];
  88
  89                 if (tmp_dhgid != ctrl->dh_gid) {
  90                         dhgid = tmp_dhgid;
  91                         break;
  92                 }
  93                 if (fallback_dhgid < 0) {
  94                         const char *kpp = nvme_auth_dhgroup_kpp(tmp_dhgid);
  95
  96                         if (crypto_has_kpp(kpp, 0, 0))
  97                                 fallback_dhgid = tmp_dhgid;
  98                 }
  99         }
 100         if (dhgid < 0) {
 101                 if (fallback_dhgid < 0) {
 102                         pr_debug("%s: ctrl %d qid %d: no usable DH group found\n",
 103                                  __func__, ctrl->cntlid, req->sq->qid);
 104                         return NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
 105                 }
 106                 pr_debug("%s: ctrl %d qid %d: configured DH group %s not found\n",
 107                          __func__, ctrl->cntlid, req->sq->qid,
 108                          nvme_auth_dhgroup_name(fallback_dhgid));
 109                 ctrl->dh_gid = fallback_dhgid;
 110         }
 111         pr_debug("%s: ctrl %d qid %d: selected DH group %s (%d)\n",
 112                  __func__, ctrl->cntlid, req->sq->qid,
 113                  nvme_auth_dhgroup_name(ctrl->dh_gid), ctrl->dh_gid);
 114         return 0;
 115 }
 116
 117 static u16 nvmet_auth_reply(struct nvmet_req *req, void *d)
 118 {
 119         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 120         struct nvmf_auth_dhchap_reply_data *data = d;
 121         u16 dhvlen = le16_to_cpu(data->dhvlen);
 122         u8 *response;
 123
 124         pr_debug("%s: ctrl %d qid %d: data hl %d cvalid %d dhvlen %u\n",
 125                  __func__, ctrl->cntlid, req->sq->qid,
 126                  data->hl, data->cvalid, dhvlen);
 127
 128         if (dhvlen) {
 129                 if (!ctrl->dh_tfm)
 130                         return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 131                 if (nvmet_auth_ctrl_sesskey(req, data->rval + 2 * data->hl,
 132                                             dhvlen) < 0)
 133                         return NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
 134         }
 135
 136         response = kmalloc(data->hl, GFP_KERNEL);
 137         if (!response)
 138                 return NVME_AUTH_DHCHAP_FAILURE_FAILED;
 139
 140         if (!ctrl->host_key) {
 141                 pr_warn("ctrl %d qid %d no host key\n",
 142                         ctrl->cntlid, req->sq->qid);
 143                 kfree(response);
 144                 return NVME_AUTH_DHCHAP_FAILURE_FAILED;
 145         }
 146         if (nvmet_auth_host_hash(req, response, data->hl) < 0) {
 147                 pr_debug("ctrl %d qid %d host hash failed\n",
 148                          ctrl->cntlid, req->sq->qid);
 149                 kfree(response);
 150                 return NVME_AUTH_DHCHAP_FAILURE_FAILED;
 151         }
 152
 153         if (memcmp(data->rval, response, data->hl)) {
 154                 pr_info("ctrl %d qid %d host response mismatch\n",
 155                         ctrl->cntlid, req->sq->qid);
 156                 kfree(response);
 157                 return NVME_AUTH_DHCHAP_FAILURE_FAILED;
 158         }
 159         kfree(response);
 160         pr_debug("%s: ctrl %d qid %d host authenticated\n",
 161                  __func__, ctrl->cntlid, req->sq->qid);
 162         if (data->cvalid) {
 163                 req->sq->dhchap_c2 = kmalloc(data->hl, GFP_KERNEL);
 164                 if (!req->sq->dhchap_c2)
 165                         return NVME_AUTH_DHCHAP_FAILURE_FAILED;
 166                 memcpy(req->sq->dhchap_c2, data->rval + data->hl, data->hl);
 167
 168                 pr_debug("%s: ctrl %d qid %d challenge %*ph\n",
 169                          __func__, ctrl->cntlid, req->sq->qid, data->hl,
 170                          req->sq->dhchap_c2);
 171                 req->sq->dhchap_s2 = le32_to_cpu(data->seqnum);
 172         } else {
 173                 req->sq->authenticated = true;
 174                 req->sq->dhchap_c2 = NULL;
 175         }
 176
 177         return 0;
 178 }
 179
 180 static u16 nvmet_auth_failure2(struct nvmet_req *req, void *d)
 181 {
 182         struct nvmf_auth_dhchap_failure_data *data = d;
 183
 184         return data->rescode_exp;
 185 }
 186
 187 void nvmet_execute_auth_send(struct nvmet_req *req)
 188 {
 189         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 190         struct nvmf_auth_dhchap_success2_data *data;
 191         void *d;
 192         u32 tl;
 193         u16 status = 0;
 194
 195         if (req->cmd->auth_send.secp != NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER) {
 196                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 197                 req->error_loc =
 198                         offsetof(struct nvmf_auth_send_command, secp);
 199                 goto done;
 200         }
 201         if (req->cmd->auth_send.spsp0 != 0x01) {
 202                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 203                 req->error_loc =
 204                         offsetof(struct nvmf_auth_send_command, spsp0);
 205                 goto done;
 206         }
 207         if (req->cmd->auth_send.spsp1 != 0x01) {
 208                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 209                 req->error_loc =
 210                         offsetof(struct nvmf_auth_send_command, spsp1);
 211                 goto done;
 212         }
 213         tl = le32_to_cpu(req->cmd->auth_send.tl);
 214         if (!tl) {
 215                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 216                 req->error_loc =
 217                         offsetof(struct nvmf_auth_send_command, tl);
 218                 goto done;
 219         }
 220         if (!nvmet_check_transfer_len(req, tl)) {
 221                 pr_debug("%s: transfer length mismatch (%u)\n", __func__, tl);
 222                 return;
 223         }
 224
 225         d = kmalloc(tl, GFP_KERNEL);
 226         if (!d) {
 227                 status = NVME_SC_INTERNAL;
 228                 goto done;
 229         }
 230
 231         status = nvmet_copy_from_sgl(req, 0, d, tl);
 232         if (status) {
 233                 kfree(d);
 234                 goto done;
 235         }
 236
 237         data = d;
 238         pr_debug("%s: ctrl %d qid %d type %d id %d step %x\n", __func__,
 239                  ctrl->cntlid, req->sq->qid, data->auth_type, data->auth_id,
 240                  req->sq->dhchap_step);
 241         if (data->auth_type != NVME_AUTH_COMMON_MESSAGES &&
 242             data->auth_type != NVME_AUTH_DHCHAP_MESSAGES)
 243                 goto done_failure1;
 244         if (data->auth_type == NVME_AUTH_COMMON_MESSAGES) {
 245                 if (data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE) {
 246                         /* Restart negotiation */
 247                         pr_debug("%s: ctrl %d qid %d reset negotiation\n", __func__,
 248                                  ctrl->cntlid, req->sq->qid);
 249                         if (!req->sq->qid) {
 250                                 if (nvmet_setup_auth(ctrl) < 0) {
 251                                         status = NVME_SC_INTERNAL;
 252                                         pr_err("ctrl %d qid 0 failed to setup"
 253                                                "re-authentication",
 254                                                ctrl->cntlid);
 255                                         goto done_failure1;
 256                                 }
 257                         }
 258                         req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
 259                 } else if (data->auth_id != req->sq->dhchap_step)
 260                         goto done_failure1;
 261                 /* Validate negotiation parameters */
 262                 status = nvmet_auth_negotiate(req, d);
 263                 if (status == 0)
 264                         req->sq->dhchap_step =
 265                                 NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE;
 266                 else {
 267                         req->sq->dhchap_step =
 268                                 NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
 269                         req->sq->dhchap_status = status;
 270                         status = 0;
 271                 }
 272                 goto done_kfree;
 273         }
 274         if (data->auth_id != req->sq->dhchap_step) {
 275                 pr_debug("%s: ctrl %d qid %d step mismatch (%d != %d)\n",
 276                          __func__, ctrl->cntlid, req->sq->qid,
 277                          data->auth_id, req->sq->dhchap_step);
 278                 goto done_failure1;
 279         }
 280         if (le16_to_cpu(data->t_id) != req->sq->dhchap_tid) {
 281                 pr_debug("%s: ctrl %d qid %d invalid transaction %d (expected %d)\n",
 282                          __func__, ctrl->cntlid, req->sq->qid,
 283                          le16_to_cpu(data->t_id),
 284                          req->sq->dhchap_tid);
 285                 req->sq->dhchap_step =
 286                         NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
 287                 req->sq->dhchap_status =
 288                         NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 289                 goto done_kfree;
 290         }
 291
 292         switch (data->auth_id) {
 293         case NVME_AUTH_DHCHAP_MESSAGE_REPLY:
 294                 status = nvmet_auth_reply(req, d);
 295                 if (status == 0)
 296                         req->sq->dhchap_step =
 297                                 NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1;
 298                 else {
 299                         req->sq->dhchap_step =
 300                                 NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
 301                         req->sq->dhchap_status = status;
 302                         status = 0;
 303                 }
 304                 goto done_kfree;
 305                 break;
 306         case NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2:
 307                 req->sq->authenticated = true;
 308                 pr_debug("%s: ctrl %d qid %d ctrl authenticated\n",
 309                          __func__, ctrl->cntlid, req->sq->qid);
 310                 goto done_kfree;
 311                 break;
 312         case NVME_AUTH_DHCHAP_MESSAGE_FAILURE2:
 313                 status = nvmet_auth_failure2(req, d);
 314                 if (status) {
 315                         pr_warn("ctrl %d qid %d: authentication failed (%d)\n",
 316                                 ctrl->cntlid, req->sq->qid, status);
 317                         req->sq->dhchap_status = status;
 318                         req->sq->authenticated = false;
 319                         status = 0;
 320                 }
 321                 goto done_kfree;
 322                 break;
 323         default:
 324                 req->sq->dhchap_status =
 325                         NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
 326                 req->sq->dhchap_step =
 327                         NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
 328                 req->sq->authenticated = false;
 329                 goto done_kfree;
 330                 break;
 331         }
 332 done_failure1:
 333         req->sq->dhchap_status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
 334         req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
 335
 336 done_kfree:
 337         kfree(d);
 338 done:
 339         pr_debug("%s: ctrl %d qid %d dhchap status %x step %x\n", __func__,
 340                  ctrl->cntlid, req->sq->qid,
 341                  req->sq->dhchap_status, req->sq->dhchap_step);
 342         if (status)
 343                 pr_debug("%s: ctrl %d qid %d nvme status %x error loc %d\n",
 344                          __func__, ctrl->cntlid, req->sq->qid,
 345                          status, req->error_loc);
 346         req->cqe->result.u64 = 0;
 347         nvmet_req_complete(req, status);
 348         if (req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 &&
 349             req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_FAILURE2) {
 350                 unsigned long auth_expire_secs = ctrl->kato ? ctrl->kato : 120;
 351
 352                 mod_delayed_work(system_wq, &req->sq->auth_expired_work,
 353                                  auth_expire_secs * HZ);
 354                 return;
 355         }
 356         /* Final states, clear up variables */
 357         nvmet_auth_sq_free(req->sq);
 358         if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE2)
 359                 nvmet_ctrl_fatal_error(ctrl);
 360 }
 361
 362 static int nvmet_auth_challenge(struct nvmet_req *req, void *d, int al)
 363 {
 364         struct nvmf_auth_dhchap_challenge_data *data = d;
 365         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 366         int ret = 0;
 367         int hash_len = nvme_auth_hmac_hash_len(ctrl->shash_id);
 368         int data_size = sizeof(*d) + hash_len;
 369
 370         if (ctrl->dh_tfm)
 371                 data_size += ctrl->dh_keysize;
 372         if (al < data_size) {
 373                 pr_debug("%s: buffer too small (al %d need %d)\n", __func__,
 374                          al, data_size);
 375                 return -EINVAL;
 376         }
 377         memset(data, 0, data_size);
 378         req->sq->dhchap_s1 = nvme_auth_get_seqnum();
 379         data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
 380         data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE;
 381         data->t_id = cpu_to_le16(req->sq->dhchap_tid);
 382         data->hashid = ctrl->shash_id;
 383         data->hl = hash_len;
 384         data->seqnum = cpu_to_le32(req->sq->dhchap_s1);
 385         req->sq->dhchap_c1 = kmalloc(data->hl, GFP_KERNEL);
 386         if (!req->sq->dhchap_c1)
 387                 return -ENOMEM;
 388         get_random_bytes(req->sq->dhchap_c1, data->hl);
 389         memcpy(data->cval, req->sq->dhchap_c1, data->hl);
 390         if (ctrl->dh_tfm) {
 391                 data->dhgid = ctrl->dh_gid;
 392                 data->dhvlen = cpu_to_le16(ctrl->dh_keysize);
 393                 ret = nvmet_auth_ctrl_exponential(req, data->cval + data->hl,
 394                                                   ctrl->dh_keysize);
 395         }
 396         pr_debug("%s: ctrl %d qid %d seq %d transaction %d hl %d dhvlen %zu\n",
 397                  __func__, ctrl->cntlid, req->sq->qid, req->sq->dhchap_s1,
 398                  req->sq->dhchap_tid, data->hl, ctrl->dh_keysize);
 399         return ret;
 400 }
 401
 402 static int nvmet_auth_success1(struct nvmet_req *req, void *d, int al)
 403 {
 404         struct nvmf_auth_dhchap_success1_data *data = d;
 405         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 406         int hash_len = nvme_auth_hmac_hash_len(ctrl->shash_id);
 407
 408         WARN_ON(al < sizeof(*data));
 409         memset(data, 0, sizeof(*data));
 410         data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
 411         data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1;
 412         data->t_id = cpu_to_le16(req->sq->dhchap_tid);
 413         data->hl = hash_len;
 414         if (req->sq->dhchap_c2) {
 415                 if (!ctrl->ctrl_key) {
 416                         pr_warn("ctrl %d qid %d no ctrl key\n",
 417                                 ctrl->cntlid, req->sq->qid);
 418                         return NVME_AUTH_DHCHAP_FAILURE_FAILED;
 419                 }
 420                 if (nvmet_auth_ctrl_hash(req, data->rval, data->hl))
 421                         return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
 422                 data->rvalid = 1;
 423                 pr_debug("ctrl %d qid %d response %*ph\n",
 424                          ctrl->cntlid, req->sq->qid, data->hl, data->rval);
 425         }
 426         return 0;
 427 }
 428
 429 static void nvmet_auth_failure1(struct nvmet_req *req, void *d, int al)
 430 {
 431         struct nvmf_auth_dhchap_failure_data *data = d;
 432
 433         WARN_ON(al < sizeof(*data));
 434         data->auth_type = NVME_AUTH_COMMON_MESSAGES;
 435         data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
 436         data->t_id = cpu_to_le16(req->sq->dhchap_tid);
 437         data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED;
 438         data->rescode_exp = req->sq->dhchap_status;
 439 }
 440
 441 void nvmet_execute_auth_receive(struct nvmet_req *req)
 442 {
 443         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 444         void *d;
 445         u32 al;
 446         u16 status = 0;
 447
 448         if (req->cmd->auth_receive.secp != NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER) {
 449                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 450                 req->error_loc =
 451                         offsetof(struct nvmf_auth_receive_command, secp);
 452                 goto done;
 453         }
 454         if (req->cmd->auth_receive.spsp0 != 0x01) {
 455                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 456                 req->error_loc =
 457                         offsetof(struct nvmf_auth_receive_command, spsp0);
 458                 goto done;
 459         }
 460         if (req->cmd->auth_receive.spsp1 != 0x01) {
 461                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 462                 req->error_loc =
 463                         offsetof(struct nvmf_auth_receive_command, spsp1);
 464                 goto done;
 465         }
 466         al = le32_to_cpu(req->cmd->auth_receive.al);
 467         if (!al) {
 468                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 469                 req->error_loc =
 470                         offsetof(struct nvmf_auth_receive_command, al);
 471                 goto done;
 472         }
 473         if (!nvmet_check_transfer_len(req, al)) {
 474                 pr_debug("%s: transfer length mismatch (%u)\n", __func__, al);
 475                 return;
 476         }
 477
 478         d = kmalloc(al, GFP_KERNEL);
 479         if (!d) {
 480                 status = NVME_SC_INTERNAL;
 481                 goto done;
 482         }
 483         pr_debug("%s: ctrl %d qid %d step %x\n", __func__,
 484                  ctrl->cntlid, req->sq->qid, req->sq->dhchap_step);
 485         switch (req->sq->dhchap_step) {
 486         case NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE:
 487                 if (nvmet_auth_challenge(req, d, al) < 0) {
 488                         pr_warn("ctrl %d qid %d: challenge error (%d)\n",
 489                                 ctrl->cntlid, req->sq->qid, status);
 490                         status = NVME_SC_INTERNAL;
 491                         break;
 492                 }
 493                 if (status) {
 494                         req->sq->dhchap_status = status;
 495                         nvmet_auth_failure1(req, d, al);
 496                         pr_warn("ctrl %d qid %d: challenge status (%x)\n",
 497                                 ctrl->cntlid, req->sq->qid,
 498                                 req->sq->dhchap_status);
 499                         status = 0;
 500                         break;
 501                 }
 502                 req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_REPLY;
 503                 break;
 504         case NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1:
 505                 status = nvmet_auth_success1(req, d, al);
 506                 if (status) {
 507                         req->sq->dhchap_status = status;
 508                         req->sq->authenticated = false;
 509                         nvmet_auth_failure1(req, d, al);
 510                         pr_warn("ctrl %d qid %d: success1 status (%x)\n",
 511                                 ctrl->cntlid, req->sq->qid,
 512                                 req->sq->dhchap_status);
 513                         break;
 514                 }
 515                 req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2;
 516                 break;
 517         case NVME_AUTH_DHCHAP_MESSAGE_FAILURE1:
 518                 req->sq->authenticated = false;
 519                 nvmet_auth_failure1(req, d, al);
 520                 pr_warn("ctrl %d qid %d failure1 (%x)\n",
 521                         ctrl->cntlid, req->sq->qid, req->sq->dhchap_status);
 522                 break;
 523         default:
 524                 pr_warn("ctrl %d qid %d unhandled step (%d)\n",
 525                         ctrl->cntlid, req->sq->qid, req->sq->dhchap_step);
 526                 req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
 527                 req->sq->dhchap_status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 528                 nvmet_auth_failure1(req, d, al);
 529                 status = 0;
 530                 break;
 531         }
 532
 533         status = nvmet_copy_to_sgl(req, 0, d, al);
 534         kfree(d);
 535 done:
 536         req->cqe->result.u64 = 0;
 537         nvmet_req_complete(req, status);
 538         if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2)
 539                 nvmet_auth_sq_free(req->sq);
 540         else if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
 541                 nvmet_auth_sq_free(req->sq);
 542                 nvmet_ctrl_fatal_error(ctrl);
 543         }
 544 }