drivers/nvme/host/auth.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2020 Hannes Reinecke, SUSE Linux
   4  */
   5
   6 #include <linux/crc32.h>
   7 #include <linux/base64.h>
   8 #include <linux/prandom.h>
   9 #include <asm/unaligned.h>
  10 #include <crypto/hash.h>
  11 #include <crypto/dh.h>
  12 #include "nvme.h"
  13 #include "fabrics.h"
  14 #include <linux/nvme-auth.h>
  15
  16 #define CHAP_BUF_SIZE 4096
  17 static struct kmem_cache *nvme_chap_buf_cache;
  18 static mempool_t *nvme_chap_buf_pool;
  19
  20 struct nvme_dhchap_queue_context {
  21         struct list_head entry;
  22         struct work_struct auth_work;
  23         struct nvme_ctrl *ctrl;
  24         struct crypto_shash *shash_tfm;
  25         struct crypto_kpp *dh_tfm;
  26         struct nvme_dhchap_key *transformed_key;
  27         void *buf;
  28         int qid;
  29         int error;
  30         u32 s1;
  31         u32 s2;
  32         bool bi_directional;
  33         u16 transaction;
  34         u8 status;
  35         u8 dhgroup_id;
  36         u8 hash_id;
  37         size_t hash_len;
  38         u8 c1[64];
  39         u8 c2[64];
  40         u8 response[64];
  41         u8 *ctrl_key;
  42         u8 *host_key;
  43         u8 *sess_key;
  44         int ctrl_key_len;
  45         int host_key_len;
  46         int sess_key_len;
  47 };
  48
  49 static struct workqueue_struct *nvme_auth_wq;
  50
  51 static inline int ctrl_max_dhchaps(struct nvme_ctrl *ctrl)
  52 {
  53         return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues +
  54                         ctrl->opts->nr_poll_queues + 1;
  55 }
  56
  57 static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid,
  58                             void *data, size_t data_len, bool auth_send)
  59 {
  60         struct nvme_command cmd = {};
  61         nvme_submit_flags_t flags = NVME_SUBMIT_RETRY;
  62         struct request_queue *q = ctrl->fabrics_q;
  63         int ret;
  64
  65         if (qid != 0) {
  66                 flags |= NVME_SUBMIT_NOWAIT | NVME_SUBMIT_RESERVED;
  67                 q = ctrl->connect_q;
  68         }
  69
  70         cmd.auth_common.opcode = nvme_fabrics_command;
  71         cmd.auth_common.secp = NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER;
  72         cmd.auth_common.spsp0 = 0x01;
  73         cmd.auth_common.spsp1 = 0x01;
  74         if (auth_send) {
  75                 cmd.auth_send.fctype = nvme_fabrics_type_auth_send;
  76                 cmd.auth_send.tl = cpu_to_le32(data_len);
  77         } else {
  78                 cmd.auth_receive.fctype = nvme_fabrics_type_auth_receive;
  79                 cmd.auth_receive.al = cpu_to_le32(data_len);
  80         }
  81
  82         ret = __nvme_submit_sync_cmd(q, &cmd, NULL, data, data_len,
  83                                      qid == 0 ? NVME_QID_ANY : qid, flags);
  84         if (ret > 0)
  85                 dev_warn(ctrl->device,
  86                         "qid %d auth_send failed with status %d\n", qid, ret);
  87         else if (ret < 0)
  88                 dev_err(ctrl->device,
  89                         "qid %d auth_send failed with error %d\n", qid, ret);
  90         return ret;
  91 }
  92
  93 static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid,
  94                 struct nvmf_auth_dhchap_failure_data *data,
  95                 u16 transaction, u8 expected_msg)
  96 {
  97         dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n",
  98                 __func__, qid, data->auth_type, data->auth_id);
  99
 100         if (data->auth_type == NVME_AUTH_COMMON_MESSAGES &&
 101             data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
 102                 return data->rescode_exp;
 103         }
 104         if (data->auth_type != NVME_AUTH_DHCHAP_MESSAGES ||
 105             data->auth_id != expected_msg) {
 106                 dev_warn(ctrl->device,
 107                          "qid %d invalid message %02x/%02x\n",
 108                          qid, data->auth_type, data->auth_id);
 109                 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
 110         }
 111         if (le16_to_cpu(data->t_id) != transaction) {
 112                 dev_warn(ctrl->device,
 113                          "qid %d invalid transaction ID %d\n",
 114                          qid, le16_to_cpu(data->t_id));
 115                 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
 116         }
 117         return 0;
 118 }
 119
 120 static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl,
 121                 struct nvme_dhchap_queue_context *chap)
 122 {
 123         struct nvmf_auth_dhchap_negotiate_data *data = chap->buf;
 124         size_t size = sizeof(*data) + sizeof(union nvmf_auth_protocol);
 125
 126         if (size > CHAP_BUF_SIZE) {
 127                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 128                 return -EINVAL;
 129         }
 130         memset((u8 *)chap->buf, 0, size);
 131         data->auth_type = NVME_AUTH_COMMON_MESSAGES;
 132         data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
 133         data->t_id = cpu_to_le16(chap->transaction);
 134         data->sc_c = 0; /* No secure channel concatenation */
 135         data->napd = 1;
 136         data->auth_protocol[0].dhchap.authid = NVME_AUTH_DHCHAP_AUTH_ID;
 137         data->auth_protocol[0].dhchap.halen = 3;
 138         data->auth_protocol[0].dhchap.dhlen = 6;
 139         data->auth_protocol[0].dhchap.idlist[0] = NVME_AUTH_HASH_SHA256;
 140         data->auth_protocol[0].dhchap.idlist[1] = NVME_AUTH_HASH_SHA384;
 141         data->auth_protocol[0].dhchap.idlist[2] = NVME_AUTH_HASH_SHA512;
 142         data->auth_protocol[0].dhchap.idlist[30] = NVME_AUTH_DHGROUP_NULL;
 143         data->auth_protocol[0].dhchap.idlist[31] = NVME_AUTH_DHGROUP_2048;
 144         data->auth_protocol[0].dhchap.idlist[32] = NVME_AUTH_DHGROUP_3072;
 145         data->auth_protocol[0].dhchap.idlist[33] = NVME_AUTH_DHGROUP_4096;
 146         data->auth_protocol[0].dhchap.idlist[34] = NVME_AUTH_DHGROUP_6144;
 147         data->auth_protocol[0].dhchap.idlist[35] = NVME_AUTH_DHGROUP_8192;
 148
 149         return size;
 150 }
 151
 152 static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl,
 153                 struct nvme_dhchap_queue_context *chap)
 154 {
 155         struct nvmf_auth_dhchap_challenge_data *data = chap->buf;
 156         u16 dhvlen = le16_to_cpu(data->dhvlen);
 157         size_t size = sizeof(*data) + data->hl + dhvlen;
 158         const char *gid_name = nvme_auth_dhgroup_name(data->dhgid);
 159         const char *hmac_name, *kpp_name;
 160
 161         if (size > CHAP_BUF_SIZE) {
 162                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 163                 return -EINVAL;
 164         }
 165
 166         hmac_name = nvme_auth_hmac_name(data->hashid);
 167         if (!hmac_name) {
 168                 dev_warn(ctrl->device,
 169                          "qid %d: invalid HASH ID %d\n",
 170                          chap->qid, data->hashid);
 171                 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
 172                 return -EPROTO;
 173         }
 174
 175         if (chap->hash_id == data->hashid && chap->shash_tfm &&
 176             !strcmp(crypto_shash_alg_name(chap->shash_tfm), hmac_name) &&
 177             crypto_shash_digestsize(chap->shash_tfm) == data->hl) {
 178                 dev_dbg(ctrl->device,
 179                         "qid %d: reuse existing hash %s\n",
 180                         chap->qid, hmac_name);
 181                 goto select_kpp;
 182         }
 183
 184         /* Reset if hash cannot be reused */
 185         if (chap->shash_tfm) {
 186                 crypto_free_shash(chap->shash_tfm);
 187                 chap->hash_id = 0;
 188                 chap->hash_len = 0;
 189         }
 190         chap->shash_tfm = crypto_alloc_shash(hmac_name, 0,
 191                                              CRYPTO_ALG_ALLOCATES_MEMORY);
 192         if (IS_ERR(chap->shash_tfm)) {
 193                 dev_warn(ctrl->device,
 194                          "qid %d: failed to allocate hash %s, error %ld\n",
 195                          chap->qid, hmac_name, PTR_ERR(chap->shash_tfm));
 196                 chap->shash_tfm = NULL;
 197                 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 198                 return -ENOMEM;
 199         }
 200
 201         if (crypto_shash_digestsize(chap->shash_tfm) != data->hl) {
 202                 dev_warn(ctrl->device,
 203                          "qid %d: invalid hash length %d\n",
 204                          chap->qid, data->hl);
 205                 crypto_free_shash(chap->shash_tfm);
 206                 chap->shash_tfm = NULL;
 207                 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
 208                 return -EPROTO;
 209         }
 210
 211         chap->hash_id = data->hashid;
 212         chap->hash_len = data->hl;
 213         dev_dbg(ctrl->device, "qid %d: selected hash %s\n",
 214                 chap->qid, hmac_name);
 215
 216 select_kpp:
 217         kpp_name = nvme_auth_dhgroup_kpp(data->dhgid);
 218         if (!kpp_name) {
 219                 dev_warn(ctrl->device,
 220                          "qid %d: invalid DH group id %d\n",
 221                          chap->qid, data->dhgid);
 222                 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
 223                 /* Leave previous dh_tfm intact */
 224                 return -EPROTO;
 225         }
 226
 227         if (chap->dhgroup_id == data->dhgid &&
 228             (data->dhgid == NVME_AUTH_DHGROUP_NULL || chap->dh_tfm)) {
 229                 dev_dbg(ctrl->device,
 230                         "qid %d: reuse existing DH group %s\n",
 231                         chap->qid, gid_name);
 232                 goto skip_kpp;
 233         }
 234
 235         /* Reset dh_tfm if it can't be reused */
 236         if (chap->dh_tfm) {
 237                 crypto_free_kpp(chap->dh_tfm);
 238                 chap->dh_tfm = NULL;
 239         }
 240
 241         if (data->dhgid != NVME_AUTH_DHGROUP_NULL) {
 242                 if (dhvlen == 0) {
 243                         dev_warn(ctrl->device,
 244                                  "qid %d: empty DH value\n",
 245                                  chap->qid);
 246                         chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
 247                         return -EPROTO;
 248                 }
 249
 250                 chap->dh_tfm = crypto_alloc_kpp(kpp_name, 0, 0);
 251                 if (IS_ERR(chap->dh_tfm)) {
 252                         int ret = PTR_ERR(chap->dh_tfm);
 253
 254                         dev_warn(ctrl->device,
 255                                  "qid %d: error %d initializing DH group %s\n",
 256                                  chap->qid, ret, gid_name);
 257                         chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
 258                         chap->dh_tfm = NULL;
 259                         return ret;
 260                 }
 261                 dev_dbg(ctrl->device, "qid %d: selected DH group %s\n",
 262                         chap->qid, gid_name);
 263         } else if (dhvlen != 0) {
 264                 dev_warn(ctrl->device,
 265                          "qid %d: invalid DH value for NULL DH\n",
 266                          chap->qid);
 267                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 268                 return -EPROTO;
 269         }
 270         chap->dhgroup_id = data->dhgid;
 271
 272 skip_kpp:
 273         chap->s1 = le32_to_cpu(data->seqnum);
 274         memcpy(chap->c1, data->cval, chap->hash_len);
 275         if (dhvlen) {
 276                 chap->ctrl_key = kmalloc(dhvlen, GFP_KERNEL);
 277                 if (!chap->ctrl_key) {
 278                         chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 279                         return -ENOMEM;
 280                 }
 281                 chap->ctrl_key_len = dhvlen;
 282                 memcpy(chap->ctrl_key, data->cval + chap->hash_len,
 283                        dhvlen);
 284                 dev_dbg(ctrl->device, "ctrl public key %*ph\n",
 285                          (int)chap->ctrl_key_len, chap->ctrl_key);
 286         }
 287
 288         return 0;
 289 }
 290
 291 static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl,
 292                 struct nvme_dhchap_queue_context *chap)
 293 {
 294         struct nvmf_auth_dhchap_reply_data *data = chap->buf;
 295         size_t size = sizeof(*data);
 296
 297         size += 2 * chap->hash_len;
 298
 299         if (chap->host_key_len)
 300                 size += chap->host_key_len;
 301
 302         if (size > CHAP_BUF_SIZE) {
 303                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 304                 return -EINVAL;
 305         }
 306
 307         memset(chap->buf, 0, size);
 308         data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
 309         data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_REPLY;
 310         data->t_id = cpu_to_le16(chap->transaction);
 311         data->hl = chap->hash_len;
 312         data->dhvlen = cpu_to_le16(chap->host_key_len);
 313         memcpy(data->rval, chap->response, chap->hash_len);
 314         if (ctrl->ctrl_key) {
 315                 chap->bi_directional = true;
 316                 get_random_bytes(chap->c2, chap->hash_len);
 317                 data->cvalid = 1;
 318                 memcpy(data->rval + chap->hash_len, chap->c2,
 319                        chap->hash_len);
 320                 dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n",
 321                         __func__, chap->qid, (int)chap->hash_len, chap->c2);
 322         } else {
 323                 memset(chap->c2, 0, chap->hash_len);
 324         }
 325         chap->s2 = nvme_auth_get_seqnum();
 326         data->seqnum = cpu_to_le32(chap->s2);
 327         if (chap->host_key_len) {
 328                 dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n",
 329                         __func__, chap->qid,
 330                         chap->host_key_len, chap->host_key);
 331                 memcpy(data->rval + 2 * chap->hash_len, chap->host_key,
 332                        chap->host_key_len);
 333         }
 334
 335         return size;
 336 }
 337
 338 static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl,
 339                 struct nvme_dhchap_queue_context *chap)
 340 {
 341         struct nvmf_auth_dhchap_success1_data *data = chap->buf;
 342         size_t size = sizeof(*data) + chap->hash_len;
 343
 344         if (size > CHAP_BUF_SIZE) {
 345                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 346                 return -EINVAL;
 347         }
 348
 349         if (data->hl != chap->hash_len) {
 350                 dev_warn(ctrl->device,
 351                          "qid %d: invalid hash length %u\n",
 352                          chap->qid, data->hl);
 353                 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
 354                 return -EPROTO;
 355         }
 356
 357         /* Just print out information for the admin queue */
 358         if (chap->qid == 0)
 359                 dev_info(ctrl->device,
 360                          "qid 0: authenticated with hash %s dhgroup %s\n",
 361                          nvme_auth_hmac_name(chap->hash_id),
 362                          nvme_auth_dhgroup_name(chap->dhgroup_id));
 363
 364         if (!data->rvalid)
 365                 return 0;
 366
 367         /* Validate controller response */
 368         if (memcmp(chap->response, data->rval, data->hl)) {
 369                 dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n",
 370                         __func__, chap->qid, (int)chap->hash_len, data->rval);
 371                 dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n",
 372                         __func__, chap->qid, (int)chap->hash_len,
 373                         chap->response);
 374                 dev_warn(ctrl->device,
 375                          "qid %d: controller authentication failed\n",
 376                          chap->qid);
 377                 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 378                 return -ECONNREFUSED;
 379         }
 380
 381         /* Just print out information for the admin queue */
 382         if (chap->qid == 0)
 383                 dev_info(ctrl->device,
 384                          "qid 0: controller authenticated\n");
 385         return 0;
 386 }
 387
 388 static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl,
 389                 struct nvme_dhchap_queue_context *chap)
 390 {
 391         struct nvmf_auth_dhchap_success2_data *data = chap->buf;
 392         size_t size = sizeof(*data);
 393
 394         memset(chap->buf, 0, size);
 395         data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
 396         data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2;
 397         data->t_id = cpu_to_le16(chap->transaction);
 398
 399         return size;
 400 }
 401
 402 static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl,
 403                 struct nvme_dhchap_queue_context *chap)
 404 {
 405         struct nvmf_auth_dhchap_failure_data *data = chap->buf;
 406         size_t size = sizeof(*data);
 407
 408         memset(chap->buf, 0, size);
 409         data->auth_type = NVME_AUTH_COMMON_MESSAGES;
 410         data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
 411         data->t_id = cpu_to_le16(chap->transaction);
 412         data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED;
 413         data->rescode_exp = chap->status;
 414
 415         return size;
 416 }
 417
 418 static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl,
 419                 struct nvme_dhchap_queue_context *chap)
 420 {
 421         SHASH_DESC_ON_STACK(shash, chap->shash_tfm);
 422         u8 buf[4], *challenge = chap->c1;
 423         int ret;
 424
 425         dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n",
 426                 __func__, chap->qid, chap->s1, chap->transaction);
 427
 428         if (!chap->transformed_key) {
 429                 chap->transformed_key = nvme_auth_transform_key(ctrl->host_key,
 430                                                 ctrl->opts->host->nqn);
 431                 if (IS_ERR(chap->transformed_key)) {
 432                         ret = PTR_ERR(chap->transformed_key);
 433                         chap->transformed_key = NULL;
 434                         return ret;
 435                 }
 436         } else {
 437                 dev_dbg(ctrl->device, "%s: qid %d re-using host response\n",
 438                         __func__, chap->qid);
 439         }
 440
 441         ret = crypto_shash_setkey(chap->shash_tfm,
 442                         chap->transformed_key->key, chap->transformed_key->len);
 443         if (ret) {
 444                 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n",
 445                          chap->qid, ret);
 446                 goto out;
 447         }
 448
 449         if (chap->dh_tfm) {
 450                 challenge = kmalloc(chap->hash_len, GFP_KERNEL);
 451                 if (!challenge) {
 452                         ret = -ENOMEM;
 453                         goto out;
 454                 }
 455                 ret = nvme_auth_augmented_challenge(chap->hash_id,
 456                                                     chap->sess_key,
 457                                                     chap->sess_key_len,
 458                                                     chap->c1, challenge,
 459                                                     chap->hash_len);
 460                 if (ret)
 461                         goto out;
 462         }
 463
 464         shash->tfm = chap->shash_tfm;
 465         ret = crypto_shash_init(shash);
 466         if (ret)
 467                 goto out;
 468         ret = crypto_shash_update(shash, challenge, chap->hash_len);
 469         if (ret)
 470                 goto out;
 471         put_unaligned_le32(chap->s1, buf);
 472         ret = crypto_shash_update(shash, buf, 4);
 473         if (ret)
 474                 goto out;
 475         put_unaligned_le16(chap->transaction, buf);
 476         ret = crypto_shash_update(shash, buf, 2);
 477         if (ret)
 478                 goto out;
 479         memset(buf, 0, sizeof(buf));
 480         ret = crypto_shash_update(shash, buf, 1);
 481         if (ret)
 482                 goto out;
 483         ret = crypto_shash_update(shash, "HostHost", 8);
 484         if (ret)
 485                 goto out;
 486         ret = crypto_shash_update(shash, ctrl->opts->host->nqn,
 487                                   strlen(ctrl->opts->host->nqn));
 488         if (ret)
 489                 goto out;
 490         ret = crypto_shash_update(shash, buf, 1);
 491         if (ret)
 492                 goto out;
 493         ret = crypto_shash_update(shash, ctrl->opts->subsysnqn,
 494                             strlen(ctrl->opts->subsysnqn));
 495         if (ret)
 496                 goto out;
 497         ret = crypto_shash_final(shash, chap->response);
 498 out:
 499         if (challenge != chap->c1)
 500                 kfree(challenge);
 501         return ret;
 502 }
 503
 504 static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl,
 505                 struct nvme_dhchap_queue_context *chap)
 506 {
 507         SHASH_DESC_ON_STACK(shash, chap->shash_tfm);
 508         struct nvme_dhchap_key *transformed_key;
 509         u8 buf[4], *challenge = chap->c2;
 510         int ret;
 511
 512         transformed_key = nvme_auth_transform_key(ctrl->ctrl_key,
 513                                 ctrl->opts->subsysnqn);
 514         if (IS_ERR(transformed_key)) {
 515                 ret = PTR_ERR(transformed_key);
 516                 return ret;
 517         }
 518
 519         ret = crypto_shash_setkey(chap->shash_tfm,
 520                         transformed_key->key, transformed_key->len);
 521         if (ret) {
 522                 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n",
 523                          chap->qid, ret);
 524                 goto out;
 525         }
 526
 527         if (chap->dh_tfm) {
 528                 challenge = kmalloc(chap->hash_len, GFP_KERNEL);
 529                 if (!challenge) {
 530                         ret = -ENOMEM;
 531                         goto out;
 532                 }
 533                 ret = nvme_auth_augmented_challenge(chap->hash_id,
 534                                                     chap->sess_key,
 535                                                     chap->sess_key_len,
 536                                                     chap->c2, challenge,
 537                                                     chap->hash_len);
 538                 if (ret)
 539                         goto out;
 540         }
 541         dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n",
 542                 __func__, chap->qid, chap->s2, chap->transaction);
 543         dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n",
 544                 __func__, chap->qid, (int)chap->hash_len, challenge);
 545         dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n",
 546                 __func__, chap->qid, ctrl->opts->subsysnqn);
 547         dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n",
 548                 __func__, chap->qid, ctrl->opts->host->nqn);
 549         shash->tfm = chap->shash_tfm;
 550         ret = crypto_shash_init(shash);
 551         if (ret)
 552                 goto out;
 553         ret = crypto_shash_update(shash, challenge, chap->hash_len);
 554         if (ret)
 555                 goto out;
 556         put_unaligned_le32(chap->s2, buf);
 557         ret = crypto_shash_update(shash, buf, 4);
 558         if (ret)
 559                 goto out;
 560         put_unaligned_le16(chap->transaction, buf);
 561         ret = crypto_shash_update(shash, buf, 2);
 562         if (ret)
 563                 goto out;
 564         memset(buf, 0, 4);
 565         ret = crypto_shash_update(shash, buf, 1);
 566         if (ret)
 567                 goto out;
 568         ret = crypto_shash_update(shash, "Controller", 10);
 569         if (ret)
 570                 goto out;
 571         ret = crypto_shash_update(shash, ctrl->opts->subsysnqn,
 572                                   strlen(ctrl->opts->subsysnqn));
 573         if (ret)
 574                 goto out;
 575         ret = crypto_shash_update(shash, buf, 1);
 576         if (ret)
 577                 goto out;
 578         ret = crypto_shash_update(shash, ctrl->opts->host->nqn,
 579                                   strlen(ctrl->opts->host->nqn));
 580         if (ret)
 581                 goto out;
 582         ret = crypto_shash_final(shash, chap->response);
 583 out:
 584         if (challenge != chap->c2)
 585                 kfree(challenge);
 586         nvme_auth_free_key(transformed_key);
 587         return ret;
 588 }
 589
 590 static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl,
 591                 struct nvme_dhchap_queue_context *chap)
 592 {
 593         int ret;
 594
 595         if (chap->host_key && chap->host_key_len) {
 596                 dev_dbg(ctrl->device,
 597                         "qid %d: reusing host key\n", chap->qid);
 598                 goto gen_sesskey;
 599         }
 600         ret = nvme_auth_gen_privkey(chap->dh_tfm, chap->dhgroup_id);
 601         if (ret < 0) {
 602                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 603                 return ret;
 604         }
 605
 606         chap->host_key_len = crypto_kpp_maxsize(chap->dh_tfm);
 607
 608         chap->host_key = kzalloc(chap->host_key_len, GFP_KERNEL);
 609         if (!chap->host_key) {
 610                 chap->host_key_len = 0;
 611                 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 612                 return -ENOMEM;
 613         }
 614         ret = nvme_auth_gen_pubkey(chap->dh_tfm,
 615                                    chap->host_key, chap->host_key_len);
 616         if (ret) {
 617                 dev_dbg(ctrl->device,
 618                         "failed to generate public key, error %d\n", ret);
 619                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 620                 return ret;
 621         }
 622
 623 gen_sesskey:
 624         chap->sess_key_len = chap->host_key_len;
 625         chap->sess_key = kmalloc(chap->sess_key_len, GFP_KERNEL);
 626         if (!chap->sess_key) {
 627                 chap->sess_key_len = 0;
 628                 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 629                 return -ENOMEM;
 630         }
 631
 632         ret = nvme_auth_gen_shared_secret(chap->dh_tfm,
 633                                           chap->ctrl_key, chap->ctrl_key_len,
 634                                           chap->sess_key, chap->sess_key_len);
 635         if (ret) {
 636                 dev_dbg(ctrl->device,
 637                         "failed to generate shared secret, error %d\n", ret);
 638                 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
 639                 return ret;
 640         }
 641         dev_dbg(ctrl->device, "shared secret %*ph\n",
 642                 (int)chap->sess_key_len, chap->sess_key);
 643         return 0;
 644 }
 645
 646 static void nvme_auth_reset_dhchap(struct nvme_dhchap_queue_context *chap)
 647 {
 648         nvme_auth_free_key(chap->transformed_key);
 649         chap->transformed_key = NULL;
 650         kfree_sensitive(chap->host_key);
 651         chap->host_key = NULL;
 652         chap->host_key_len = 0;
 653         kfree_sensitive(chap->ctrl_key);
 654         chap->ctrl_key = NULL;
 655         chap->ctrl_key_len = 0;
 656         kfree_sensitive(chap->sess_key);
 657         chap->sess_key = NULL;
 658         chap->sess_key_len = 0;
 659         chap->status = 0;
 660         chap->error = 0;
 661         chap->s1 = 0;
 662         chap->s2 = 0;
 663         chap->bi_directional = false;
 664         chap->transaction = 0;
 665         memset(chap->c1, 0, sizeof(chap->c1));
 666         memset(chap->c2, 0, sizeof(chap->c2));
 667         mempool_free(chap->buf, nvme_chap_buf_pool);
 668         chap->buf = NULL;
 669 }
 670
 671 static void nvme_auth_free_dhchap(struct nvme_dhchap_queue_context *chap)
 672 {
 673         nvme_auth_reset_dhchap(chap);
 674         if (chap->shash_tfm)
 675                 crypto_free_shash(chap->shash_tfm);
 676         if (chap->dh_tfm)
 677                 crypto_free_kpp(chap->dh_tfm);
 678 }
 679
 680 static void nvme_queue_auth_work(struct work_struct *work)
 681 {
 682         struct nvme_dhchap_queue_context *chap =
 683                 container_of(work, struct nvme_dhchap_queue_context, auth_work);
 684         struct nvme_ctrl *ctrl = chap->ctrl;
 685         size_t tl;
 686         int ret = 0;
 687
 688         /*
 689          * Allocate a large enough buffer for the entire negotiation:
 690          * 4k is enough to ffdhe8192.
 691          */
 692         chap->buf = mempool_alloc(nvme_chap_buf_pool, GFP_KERNEL);
 693         if (!chap->buf) {
 694                 chap->error = -ENOMEM;
 695                 return;
 696         }
 697
 698         chap->transaction = ctrl->transaction++;
 699
 700         /* DH-HMAC-CHAP Step 1: send negotiate */
 701         dev_dbg(ctrl->device, "%s: qid %d send negotiate\n",
 702                 __func__, chap->qid);
 703         ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap);
 704         if (ret < 0) {
 705                 chap->error = ret;
 706                 return;
 707         }
 708         tl = ret;
 709         ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
 710         if (ret) {
 711                 chap->error = ret;
 712                 return;
 713         }
 714
 715         /* DH-HMAC-CHAP Step 2: receive challenge */
 716         dev_dbg(ctrl->device, "%s: qid %d receive challenge\n",
 717                 __func__, chap->qid);
 718
 719         memset(chap->buf, 0, CHAP_BUF_SIZE);
 720         ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,
 721                                false);
 722         if (ret) {
 723                 dev_warn(ctrl->device,
 724                          "qid %d failed to receive challenge, %s %d\n",
 725                          chap->qid, ret < 0 ? "error" : "nvme status", ret);
 726                 chap->error = ret;
 727                 return;
 728         }
 729         ret = nvme_auth_receive_validate(ctrl, chap->qid, chap->buf, chap->transaction,
 730                                          NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE);
 731         if (ret) {
 732                 chap->status = ret;
 733                 chap->error = -EKEYREJECTED;
 734                 return;
 735         }
 736
 737         ret = nvme_auth_process_dhchap_challenge(ctrl, chap);
 738         if (ret) {
 739                 /* Invalid challenge parameters */
 740                 chap->error = ret;
 741                 goto fail2;
 742         }
 743
 744         if (chap->ctrl_key_len) {
 745                 dev_dbg(ctrl->device,
 746                         "%s: qid %d DH exponential\n",
 747                         __func__, chap->qid);
 748                 ret = nvme_auth_dhchap_exponential(ctrl, chap);
 749                 if (ret) {
 750                         chap->error = ret;
 751                         goto fail2;
 752                 }
 753         }
 754
 755         dev_dbg(ctrl->device, "%s: qid %d host response\n",
 756                 __func__, chap->qid);
 757         mutex_lock(&ctrl->dhchap_auth_mutex);
 758         ret = nvme_auth_dhchap_setup_host_response(ctrl, chap);
 759         mutex_unlock(&ctrl->dhchap_auth_mutex);
 760         if (ret) {
 761                 chap->error = ret;
 762                 goto fail2;
 763         }
 764
 765         /* DH-HMAC-CHAP Step 3: send reply */
 766         dev_dbg(ctrl->device, "%s: qid %d send reply\n",
 767                 __func__, chap->qid);
 768         ret = nvme_auth_set_dhchap_reply_data(ctrl, chap);
 769         if (ret < 0) {
 770                 chap->error = ret;
 771                 goto fail2;
 772         }
 773
 774         tl = ret;
 775         ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
 776         if (ret) {
 777                 chap->error = ret;
 778                 goto fail2;
 779         }
 780
 781         /* DH-HMAC-CHAP Step 4: receive success1 */
 782         dev_dbg(ctrl->device, "%s: qid %d receive success1\n",
 783                 __func__, chap->qid);
 784
 785         memset(chap->buf, 0, CHAP_BUF_SIZE);
 786         ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,
 787                                false);
 788         if (ret) {
 789                 dev_warn(ctrl->device,
 790                          "qid %d failed to receive success1, %s %d\n",
 791                          chap->qid, ret < 0 ? "error" : "nvme status", ret);
 792                 chap->error = ret;
 793                 return;
 794         }
 795         ret = nvme_auth_receive_validate(ctrl, chap->qid,
 796                                          chap->buf, chap->transaction,
 797                                          NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1);
 798         if (ret) {
 799                 chap->status = ret;
 800                 chap->error = -EKEYREJECTED;
 801                 return;
 802         }
 803
 804         mutex_lock(&ctrl->dhchap_auth_mutex);
 805         if (ctrl->ctrl_key) {
 806                 dev_dbg(ctrl->device,
 807                         "%s: qid %d controller response\n",
 808                         __func__, chap->qid);
 809                 ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap);
 810                 if (ret) {
 811                         mutex_unlock(&ctrl->dhchap_auth_mutex);
 812                         chap->error = ret;
 813                         goto fail2;
 814                 }
 815         }
 816         mutex_unlock(&ctrl->dhchap_auth_mutex);
 817
 818         ret = nvme_auth_process_dhchap_success1(ctrl, chap);
 819         if (ret) {
 820                 /* Controller authentication failed */
 821                 chap->error = -EKEYREJECTED;
 822                 goto fail2;
 823         }
 824
 825         if (chap->bi_directional) {
 826                 /* DH-HMAC-CHAP Step 5: send success2 */
 827                 dev_dbg(ctrl->device, "%s: qid %d send success2\n",
 828                         __func__, chap->qid);
 829                 tl = nvme_auth_set_dhchap_success2_data(ctrl, chap);
 830                 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
 831                 if (ret)
 832                         chap->error = ret;
 833         }
 834         if (!ret) {
 835                 chap->error = 0;
 836                 return;
 837         }
 838
 839 fail2:
 840         if (chap->status == 0)
 841                 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 842         dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n",
 843                 __func__, chap->qid, chap->status);
 844         tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap);
 845         ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
 846         /*
 847          * only update error if send failure2 failed and no other
 848          * error had been set during authentication.
 849          */
 850         if (ret && !chap->error)
 851                 chap->error = ret;
 852 }
 853
 854 int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid)
 855 {
 856         struct nvme_dhchap_queue_context *chap;
 857
 858         if (!ctrl->host_key) {
 859                 dev_warn(ctrl->device, "qid %d: no key\n", qid);
 860                 return -ENOKEY;
 861         }
 862
 863         if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) {
 864                 dev_warn(ctrl->device, "qid %d: invalid ctrl key\n", qid);
 865                 return -ENOKEY;
 866         }
 867
 868         chap = &ctrl->dhchap_ctxs[qid];
 869         cancel_work_sync(&chap->auth_work);
 870         queue_work(nvme_auth_wq, &chap->auth_work);
 871         return 0;
 872 }
 873 EXPORT_SYMBOL_GPL(nvme_auth_negotiate);
 874
 875 int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid)
 876 {
 877         struct nvme_dhchap_queue_context *chap;
 878         int ret;
 879
 880         chap = &ctrl->dhchap_ctxs[qid];
 881         flush_work(&chap->auth_work);
 882         ret = chap->error;
 883         /* clear sensitive info */
 884         nvme_auth_reset_dhchap(chap);
 885         return ret;
 886 }
 887 EXPORT_SYMBOL_GPL(nvme_auth_wait);
 888
 889 static void nvme_ctrl_auth_work(struct work_struct *work)
 890 {
 891         struct nvme_ctrl *ctrl =
 892                 container_of(work, struct nvme_ctrl, dhchap_auth_work);
 893         int ret, q;
 894
 895         /*
 896          * If the ctrl is no connected, bail as reconnect will handle
 897          * authentication.
 898          */
 899         if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
 900                 return;
 901
 902         /* Authenticate admin queue first */
 903         ret = nvme_auth_negotiate(ctrl, 0);
 904         if (ret) {
 905                 dev_warn(ctrl->device,
 906                          "qid 0: error %d setting up authentication\n", ret);
 907                 return;
 908         }
 909         ret = nvme_auth_wait(ctrl, 0);
 910         if (ret) {
 911                 dev_warn(ctrl->device,
 912                          "qid 0: authentication failed\n");
 913                 return;
 914         }
 915
 916         for (q = 1; q < ctrl->queue_count; q++) {
 917                 ret = nvme_auth_negotiate(ctrl, q);
 918                 if (ret) {
 919                         dev_warn(ctrl->device,
 920                                  "qid %d: error %d setting up authentication\n",
 921                                  q, ret);
 922                         break;
 923                 }
 924         }
 925
 926         /*
 927          * Failure is a soft-state; credentials remain valid until
 928          * the controller terminates the connection.
 929          */
 930         for (q = 1; q < ctrl->queue_count; q++) {
 931                 ret = nvme_auth_wait(ctrl, q);
 932                 if (ret)
 933                         dev_warn(ctrl->device,
 934                                  "qid %d: authentication failed\n", q);
 935         }
 936 }
 937
 938 int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl)
 939 {
 940         struct nvme_dhchap_queue_context *chap;
 941         int i, ret;
 942
 943         mutex_init(&ctrl->dhchap_auth_mutex);
 944         INIT_WORK(&ctrl->dhchap_auth_work, nvme_ctrl_auth_work);
 945         if (!ctrl->opts)
 946                 return 0;
 947         ret = nvme_auth_generate_key(ctrl->opts->dhchap_secret,
 948                         &ctrl->host_key);
 949         if (ret)
 950                 return ret;
 951         ret = nvme_auth_generate_key(ctrl->opts->dhchap_ctrl_secret,
 952                         &ctrl->ctrl_key);
 953         if (ret)
 954                 goto err_free_dhchap_secret;
 955
 956         if (!ctrl->opts->dhchap_secret && !ctrl->opts->dhchap_ctrl_secret)
 957                 return 0;
 958
 959         ctrl->dhchap_ctxs = kvcalloc(ctrl_max_dhchaps(ctrl),
 960                                 sizeof(*chap), GFP_KERNEL);
 961         if (!ctrl->dhchap_ctxs) {
 962                 ret = -ENOMEM;
 963                 goto err_free_dhchap_ctrl_secret;
 964         }
 965
 966         for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) {
 967                 chap = &ctrl->dhchap_ctxs[i];
 968                 chap->qid = i;
 969                 chap->ctrl = ctrl;
 970                 INIT_WORK(&chap->auth_work, nvme_queue_auth_work);
 971         }
 972
 973         return 0;
 974 err_free_dhchap_ctrl_secret:
 975         nvme_auth_free_key(ctrl->ctrl_key);
 976         ctrl->ctrl_key = NULL;
 977 err_free_dhchap_secret:
 978         nvme_auth_free_key(ctrl->host_key);
 979         ctrl->host_key = NULL;
 980         return ret;
 981 }
 982 EXPORT_SYMBOL_GPL(nvme_auth_init_ctrl);
 983
 984 void nvme_auth_stop(struct nvme_ctrl *ctrl)
 985 {
 986         cancel_work_sync(&ctrl->dhchap_auth_work);
 987 }
 988 EXPORT_SYMBOL_GPL(nvme_auth_stop);
 989
 990 void nvme_auth_free(struct nvme_ctrl *ctrl)
 991 {
 992         int i;
 993
 994         if (ctrl->dhchap_ctxs) {
 995                 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++)
 996                         nvme_auth_free_dhchap(&ctrl->dhchap_ctxs[i]);
 997                 kfree(ctrl->dhchap_ctxs);
 998         }
 999         if (ctrl->host_key) {
1000                 nvme_auth_free_key(ctrl->host_key);
1001                 ctrl->host_key = NULL;
1002         }
1003         if (ctrl->ctrl_key) {
1004                 nvme_auth_free_key(ctrl->ctrl_key);
1005                 ctrl->ctrl_key = NULL;
1006         }
1007 }
1008 EXPORT_SYMBOL_GPL(nvme_auth_free);
1009
1010 int __init nvme_init_auth(void)
1011 {
1012         nvme_auth_wq = alloc_workqueue("nvme-auth-wq",
1013                                WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
1014         if (!nvme_auth_wq)
1015                 return -ENOMEM;
1016
1017         nvme_chap_buf_cache = kmem_cache_create("nvme-chap-buf-cache",
1018                                 CHAP_BUF_SIZE, 0, SLAB_HWCACHE_ALIGN, NULL);
1019         if (!nvme_chap_buf_cache)
1020                 goto err_destroy_workqueue;
1021
1022         nvme_chap_buf_pool = mempool_create(16, mempool_alloc_slab,
1023                         mempool_free_slab, nvme_chap_buf_cache);
1024         if (!nvme_chap_buf_pool)
1025                 goto err_destroy_chap_buf_cache;
1026
1027         return 0;
1028 err_destroy_chap_buf_cache:
1029         kmem_cache_destroy(nvme_chap_buf_cache);
1030 err_destroy_workqueue:
1031         destroy_workqueue(nvme_auth_wq);
1032         return -ENOMEM;
1033 }
1034
1035 void __exit nvme_exit_auth(void)
1036 {
1037         mempool_destroy(nvme_chap_buf_pool);
1038         kmem_cache_destroy(nvme_chap_buf_cache);
1039         destroy_workqueue(nvme_auth_wq);
1040 }