net/tls/tls_sw.c

   1 /*
   2  * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
   3  * Copyright (c) 2016-2017, Dave Watson <[email protected]>. All rights reserved.
   4  * Copyright (c) 2016-2017, Lance Chao <[email protected]>. All rights reserved.
   5  * Copyright (c) 2016, Fridolin Pokorny <[email protected]>. All rights reserved.
   6  * Copyright (c) 2016, Nikos Mavrogiannopoulos <[email protected]>. All rights reserved.
   7  *
   8  * This software is available to you under a choice of one of two
   9  * licenses.  You may choose to be licensed under the terms of the GNU
  10  * General Public License (GPL) Version 2, available from the file
  11  * COPYING in the main directory of this source tree, or the
  12  * OpenIB.org BSD license below:
  13  *
  14  *     Redistribution and use in source and binary forms, with or
  15  *     without modification, are permitted provided that the following
  16  *     conditions are met:
  17  *
  18  *      - Redistributions of source code must retain the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer.
  21  *
  22  *      - Redistributions in binary form must reproduce the above
  23  *        copyright notice, this list of conditions and the following
  24  *        disclaimer in the documentation and/or other materials
  25  *        provided with the distribution.
  26  *
  27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34  * SOFTWARE.
  35  */
  36
  37 #include <linux/module.h>
  38 #include <crypto/aead.h>
  39
  40 #include <net/tls.h>
  41
  42 static void trim_sg(struct sock *sk, struct scatterlist *sg,
  43                     int *sg_num_elem, unsigned int *sg_size, int target_size)
  44 {
  45         int i = *sg_num_elem - 1;
  46         int trim = *sg_size - target_size;
  47
  48         if (trim <= 0) {
  49                 WARN_ON(trim < 0);
  50                 return;
  51         }
  52
  53         *sg_size = target_size;
  54         while (trim >= sg[i].length) {
  55                 trim -= sg[i].length;
  56                 sk_mem_uncharge(sk, sg[i].length);
  57                 put_page(sg_page(&sg[i]));
  58                 i--;
  59
  60                 if (i < 0)
  61                         goto out;
  62         }
  63
  64         sg[i].length -= trim;
  65         sk_mem_uncharge(sk, trim);
  66
  67 out:
  68         *sg_num_elem = i + 1;
  69 }
  70
  71 static void trim_both_sgl(struct sock *sk, int target_size)
  72 {
  73         struct tls_context *tls_ctx = tls_get_ctx(sk);
  74         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
  75
  76         trim_sg(sk, ctx->sg_plaintext_data,
  77                 &ctx->sg_plaintext_num_elem,
  78                 &ctx->sg_plaintext_size,
  79                 target_size);
  80
  81         if (target_size > 0)
  82                 target_size += tls_ctx->overhead_size;
  83
  84         trim_sg(sk, ctx->sg_encrypted_data,
  85                 &ctx->sg_encrypted_num_elem,
  86                 &ctx->sg_encrypted_size,
  87                 target_size);
  88 }
  89
  90 static int alloc_sg(struct sock *sk, int len, struct scatterlist *sg,
  91                     int *sg_num_elem, unsigned int *sg_size,
  92                     int first_coalesce)
  93 {
  94         struct page_frag *pfrag;
  95         unsigned int size = *sg_size;
  96         int num_elem = *sg_num_elem, use = 0, rc = 0;
  97         struct scatterlist *sge;
  98         unsigned int orig_offset;
  99
 100         len -= size;
 101         pfrag = sk_page_frag(sk);
 102
 103         while (len > 0) {
 104                 if (!sk_page_frag_refill(sk, pfrag)) {
 105                         rc = -ENOMEM;
 106                         goto out;
 107                 }
 108
 109                 use = min_t(int, len, pfrag->size - pfrag->offset);
 110
 111                 if (!sk_wmem_schedule(sk, use)) {
 112                         rc = -ENOMEM;
 113                         goto out;
 114                 }
 115
 116                 sk_mem_charge(sk, use);
 117                 size += use;
 118                 orig_offset = pfrag->offset;
 119                 pfrag->offset += use;
 120
 121                 sge = sg + num_elem - 1;
 122                 if (num_elem > first_coalesce && sg_page(sg) == pfrag->page &&
 123                     sg->offset + sg->length == orig_offset) {
 124                         sg->length += use;
 125                 } else {
 126                         sge++;
 127                         sg_unmark_end(sge);
 128                         sg_set_page(sge, pfrag->page, use, orig_offset);
 129                         get_page(pfrag->page);
 130                         ++num_elem;
 131                         if (num_elem == MAX_SKB_FRAGS) {
 132                                 rc = -ENOSPC;
 133                                 break;
 134                         }
 135                 }
 136
 137                 len -= use;
 138         }
 139         goto out;
 140
 141 out:
 142         *sg_size = size;
 143         *sg_num_elem = num_elem;
 144         return rc;
 145 }
 146
 147 static int alloc_encrypted_sg(struct sock *sk, int len)
 148 {
 149         struct tls_context *tls_ctx = tls_get_ctx(sk);
 150         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 151         int rc = 0;
 152
 153         rc = alloc_sg(sk, len, ctx->sg_encrypted_data,
 154                       &ctx->sg_encrypted_num_elem, &ctx->sg_encrypted_size, 0);
 155
 156         return rc;
 157 }
 158
 159 static int alloc_plaintext_sg(struct sock *sk, int len)
 160 {
 161         struct tls_context *tls_ctx = tls_get_ctx(sk);
 162         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 163         int rc = 0;
 164
 165         rc = alloc_sg(sk, len, ctx->sg_plaintext_data,
 166                       &ctx->sg_plaintext_num_elem, &ctx->sg_plaintext_size,
 167                       tls_ctx->pending_open_record_frags);
 168
 169         return rc;
 170 }
 171
 172 static void free_sg(struct sock *sk, struct scatterlist *sg,
 173                     int *sg_num_elem, unsigned int *sg_size)
 174 {
 175         int i, n = *sg_num_elem;
 176
 177         for (i = 0; i < n; ++i) {
 178                 sk_mem_uncharge(sk, sg[i].length);
 179                 put_page(sg_page(&sg[i]));
 180         }
 181         *sg_num_elem = 0;
 182         *sg_size = 0;
 183 }
 184
 185 static void tls_free_both_sg(struct sock *sk)
 186 {
 187         struct tls_context *tls_ctx = tls_get_ctx(sk);
 188         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 189
 190         free_sg(sk, ctx->sg_encrypted_data, &ctx->sg_encrypted_num_elem,
 191                 &ctx->sg_encrypted_size);
 192
 193         free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem,
 194                 &ctx->sg_plaintext_size);
 195 }
 196
 197 static int tls_do_encryption(struct tls_context *tls_ctx,
 198                              struct tls_sw_context *ctx, size_t data_len,
 199                              gfp_t flags)
 200 {
 201         unsigned int req_size = sizeof(struct aead_request) +
 202                 crypto_aead_reqsize(ctx->aead_send);
 203         struct aead_request *aead_req;
 204         int rc;
 205
 206         aead_req = kzalloc(req_size, flags);
 207         if (!aead_req)
 208                 return -ENOMEM;
 209
 210         ctx->sg_encrypted_data[0].offset += tls_ctx->prepend_size;
 211         ctx->sg_encrypted_data[0].length -= tls_ctx->prepend_size;
 212
 213         aead_request_set_tfm(aead_req, ctx->aead_send);
 214         aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
 215         aead_request_set_crypt(aead_req, ctx->sg_aead_in, ctx->sg_aead_out,
 216                                data_len, tls_ctx->iv);
 217
 218         aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 219                                   crypto_req_done, &ctx->async_wait);
 220
 221         rc = crypto_wait_req(crypto_aead_encrypt(aead_req), &ctx->async_wait);
 222
 223         ctx->sg_encrypted_data[0].offset -= tls_ctx->prepend_size;
 224         ctx->sg_encrypted_data[0].length += tls_ctx->prepend_size;
 225
 226         kfree(aead_req);
 227         return rc;
 228 }
 229
 230 static int tls_push_record(struct sock *sk, int flags,
 231                            unsigned char record_type)
 232 {
 233         struct tls_context *tls_ctx = tls_get_ctx(sk);
 234         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 235         int rc;
 236
 237         sg_mark_end(ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem - 1);
 238         sg_mark_end(ctx->sg_encrypted_data + ctx->sg_encrypted_num_elem - 1);
 239
 240         tls_make_aad(ctx->aad_space, ctx->sg_plaintext_size,
 241                      tls_ctx->rec_seq, tls_ctx->rec_seq_size,
 242                      record_type);
 243
 244         tls_fill_prepend(tls_ctx,
 245                          page_address(sg_page(&ctx->sg_encrypted_data[0])) +
 246                          ctx->sg_encrypted_data[0].offset,
 247                          ctx->sg_plaintext_size, record_type);
 248
 249         tls_ctx->pending_open_record_frags = 0;
 250         set_bit(TLS_PENDING_CLOSED_RECORD, &tls_ctx->flags);
 251
 252         rc = tls_do_encryption(tls_ctx, ctx, ctx->sg_plaintext_size,
 253                                sk->sk_allocation);
 254         if (rc < 0) {
 255                 /* If we are called from write_space and
 256                  * we fail, we need to set this SOCK_NOSPACE
 257                  * to trigger another write_space in the future.
 258                  */
 259                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 260                 return rc;
 261         }
 262
 263         free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem,
 264                 &ctx->sg_plaintext_size);
 265
 266         ctx->sg_encrypted_num_elem = 0;
 267         ctx->sg_encrypted_size = 0;
 268
 269         /* Only pass through MSG_DONTWAIT and MSG_NOSIGNAL flags */
 270         rc = tls_push_sg(sk, tls_ctx, ctx->sg_encrypted_data, 0, flags);
 271         if (rc < 0 && rc != -EAGAIN)
 272                 tls_err_abort(sk);
 273
 274         tls_advance_record_sn(sk, tls_ctx);
 275         return rc;
 276 }
 277
 278 static int tls_sw_push_pending_record(struct sock *sk, int flags)
 279 {
 280         return tls_push_record(sk, flags, TLS_RECORD_TYPE_DATA);
 281 }
 282
 283 static int zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
 284                               int length)
 285 {
 286         struct tls_context *tls_ctx = tls_get_ctx(sk);
 287         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 288         struct page *pages[MAX_SKB_FRAGS];
 289
 290         size_t offset;
 291         ssize_t copied, use;
 292         int i = 0;
 293         unsigned int size = ctx->sg_plaintext_size;
 294         int num_elem = ctx->sg_plaintext_num_elem;
 295         int rc = 0;
 296         int maxpages;
 297
 298         while (length > 0) {
 299                 i = 0;
 300                 maxpages = ARRAY_SIZE(ctx->sg_plaintext_data) - num_elem;
 301                 if (maxpages == 0) {
 302                         rc = -EFAULT;
 303                         goto out;
 304                 }
 305                 copied = iov_iter_get_pages(from, pages,
 306                                             length,
 307                                             maxpages, &offset);
 308                 if (copied <= 0) {
 309                         rc = -EFAULT;
 310                         goto out;
 311                 }
 312
 313                 iov_iter_advance(from, copied);
 314
 315                 length -= copied;
 316                 size += copied;
 317                 while (copied) {
 318                         use = min_t(int, copied, PAGE_SIZE - offset);
 319
 320                         sg_set_page(&ctx->sg_plaintext_data[num_elem],
 321                                     pages[i], use, offset);
 322                         sg_unmark_end(&ctx->sg_plaintext_data[num_elem]);
 323                         sk_mem_charge(sk, use);
 324
 325                         offset = 0;
 326                         copied -= use;
 327
 328                         ++i;
 329                         ++num_elem;
 330                 }
 331         }
 332
 333 out:
 334         ctx->sg_plaintext_size = size;
 335         ctx->sg_plaintext_num_elem = num_elem;
 336         return rc;
 337 }
 338
 339 static int memcopy_from_iter(struct sock *sk, struct iov_iter *from,
 340                              int bytes)
 341 {
 342         struct tls_context *tls_ctx = tls_get_ctx(sk);
 343         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 344         struct scatterlist *sg = ctx->sg_plaintext_data;
 345         int copy, i, rc = 0;
 346
 347         for (i = tls_ctx->pending_open_record_frags;
 348              i < ctx->sg_plaintext_num_elem; ++i) {
 349                 copy = sg[i].length;
 350                 if (copy_from_iter(
 351                                 page_address(sg_page(&sg[i])) + sg[i].offset,
 352                                 copy, from) != copy) {
 353                         rc = -EFAULT;
 354                         goto out;
 355                 }
 356                 bytes -= copy;
 357
 358                 ++tls_ctx->pending_open_record_frags;
 359
 360                 if (!bytes)
 361                         break;
 362         }
 363
 364 out:
 365         return rc;
 366 }
 367
 368 int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
 369 {
 370         struct tls_context *tls_ctx = tls_get_ctx(sk);
 371         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 372         int ret = 0;
 373         int required_size;
 374         long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
 375         bool eor = !(msg->msg_flags & MSG_MORE);
 376         size_t try_to_copy, copied = 0;
 377         unsigned char record_type = TLS_RECORD_TYPE_DATA;
 378         int record_room;
 379         bool full_record;
 380         int orig_size;
 381
 382         if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
 383                 return -ENOTSUPP;
 384
 385         lock_sock(sk);
 386
 387         if (tls_complete_pending_work(sk, tls_ctx, msg->msg_flags, &timeo))
 388                 goto send_end;
 389
 390         if (unlikely(msg->msg_controllen)) {
 391                 ret = tls_proccess_cmsg(sk, msg, &record_type);
 392                 if (ret)
 393                         goto send_end;
 394         }
 395
 396         while (msg_data_left(msg)) {
 397                 if (sk->sk_err) {
 398                         ret = -sk->sk_err;
 399                         goto send_end;
 400                 }
 401
 402                 orig_size = ctx->sg_plaintext_size;
 403                 full_record = false;
 404                 try_to_copy = msg_data_left(msg);
 405                 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size;
 406                 if (try_to_copy >= record_room) {
 407                         try_to_copy = record_room;
 408                         full_record = true;
 409                 }
 410
 411                 required_size = ctx->sg_plaintext_size + try_to_copy +
 412                                 tls_ctx->overhead_size;
 413
 414                 if (!sk_stream_memory_free(sk))
 415                         goto wait_for_sndbuf;
 416 alloc_encrypted:
 417                 ret = alloc_encrypted_sg(sk, required_size);
 418                 if (ret) {
 419                         if (ret != -ENOSPC)
 420                                 goto wait_for_memory;
 421
 422                         /* Adjust try_to_copy according to the amount that was
 423                          * actually allocated. The difference is due
 424                          * to max sg elements limit
 425                          */
 426                         try_to_copy -= required_size - ctx->sg_encrypted_size;
 427                         full_record = true;
 428                 }
 429
 430                 if (full_record || eor) {
 431                         ret = zerocopy_from_iter(sk, &msg->msg_iter,
 432                                                  try_to_copy);
 433                         if (ret)
 434                                 goto fallback_to_reg_send;
 435
 436                         copied += try_to_copy;
 437                         ret = tls_push_record(sk, msg->msg_flags, record_type);
 438                         if (!ret)
 439                                 continue;
 440                         if (ret == -EAGAIN)
 441                                 goto send_end;
 442
 443                         copied -= try_to_copy;
 444 fallback_to_reg_send:
 445                         iov_iter_revert(&msg->msg_iter,
 446                                         ctx->sg_plaintext_size - orig_size);
 447                         trim_sg(sk, ctx->sg_plaintext_data,
 448                                 &ctx->sg_plaintext_num_elem,
 449                                 &ctx->sg_plaintext_size,
 450                                 orig_size);
 451                 }
 452
 453                 required_size = ctx->sg_plaintext_size + try_to_copy;
 454 alloc_plaintext:
 455                 ret = alloc_plaintext_sg(sk, required_size);
 456                 if (ret) {
 457                         if (ret != -ENOSPC)
 458                                 goto wait_for_memory;
 459
 460                         /* Adjust try_to_copy according to the amount that was
 461                          * actually allocated. The difference is due
 462                          * to max sg elements limit
 463                          */
 464                         try_to_copy -= required_size - ctx->sg_plaintext_size;
 465                         full_record = true;
 466
 467                         trim_sg(sk, ctx->sg_encrypted_data,
 468                                 &ctx->sg_encrypted_num_elem,
 469                                 &ctx->sg_encrypted_size,
 470                                 ctx->sg_plaintext_size +
 471                                 tls_ctx->overhead_size);
 472                 }
 473
 474                 ret = memcopy_from_iter(sk, &msg->msg_iter, try_to_copy);
 475                 if (ret)
 476                         goto trim_sgl;
 477
 478                 copied += try_to_copy;
 479                 if (full_record || eor) {
 480 push_record:
 481                         ret = tls_push_record(sk, msg->msg_flags, record_type);
 482                         if (ret) {
 483                                 if (ret == -ENOMEM)
 484                                         goto wait_for_memory;
 485
 486                                 goto send_end;
 487                         }
 488                 }
 489
 490                 continue;
 491
 492 wait_for_sndbuf:
 493                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 494 wait_for_memory:
 495                 ret = sk_stream_wait_memory(sk, &timeo);
 496                 if (ret) {
 497 trim_sgl:
 498                         trim_both_sgl(sk, orig_size);
 499                         goto send_end;
 500                 }
 501
 502                 if (tls_is_pending_closed_record(tls_ctx))
 503                         goto push_record;
 504
 505                 if (ctx->sg_encrypted_size < required_size)
 506                         goto alloc_encrypted;
 507
 508                 goto alloc_plaintext;
 509         }
 510
 511 send_end:
 512         ret = sk_stream_error(sk, msg->msg_flags, ret);
 513
 514         release_sock(sk);
 515         return copied ? copied : ret;
 516 }
 517
 518 int tls_sw_sendpage(struct sock *sk, struct page *page,
 519                     int offset, size_t size, int flags)
 520 {
 521         struct tls_context *tls_ctx = tls_get_ctx(sk);
 522         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 523         int ret = 0;
 524         long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 525         bool eor;
 526         size_t orig_size = size;
 527         unsigned char record_type = TLS_RECORD_TYPE_DATA;
 528         struct scatterlist *sg;
 529         bool full_record;
 530         int record_room;
 531
 532         if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
 533                       MSG_SENDPAGE_NOTLAST))
 534                 return -ENOTSUPP;
 535
 536         /* No MSG_EOR from splice, only look at MSG_MORE */
 537         eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST));
 538
 539         lock_sock(sk);
 540
 541         sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 542
 543         if (tls_complete_pending_work(sk, tls_ctx, flags, &timeo))
 544                 goto sendpage_end;
 545
 546         /* Call the sk_stream functions to manage the sndbuf mem. */
 547         while (size > 0) {
 548                 size_t copy, required_size;
 549
 550                 if (sk->sk_err) {
 551                         ret = -sk->sk_err;
 552                         goto sendpage_end;
 553                 }
 554
 555                 full_record = false;
 556                 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size;
 557                 copy = size;
 558                 if (copy >= record_room) {
 559                         copy = record_room;
 560                         full_record = true;
 561                 }
 562                 required_size = ctx->sg_plaintext_size + copy +
 563                               tls_ctx->overhead_size;
 564
 565                 if (!sk_stream_memory_free(sk))
 566                         goto wait_for_sndbuf;
 567 alloc_payload:
 568                 ret = alloc_encrypted_sg(sk, required_size);
 569                 if (ret) {
 570                         if (ret != -ENOSPC)
 571                                 goto wait_for_memory;
 572
 573                         /* Adjust copy according to the amount that was
 574                          * actually allocated. The difference is due
 575                          * to max sg elements limit
 576                          */
 577                         copy -= required_size - ctx->sg_plaintext_size;
 578                         full_record = true;
 579                 }
 580
 581                 get_page(page);
 582                 sg = ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem;
 583                 sg_set_page(sg, page, copy, offset);
 584                 sg_unmark_end(sg);
 585
 586                 ctx->sg_plaintext_num_elem++;
 587
 588                 sk_mem_charge(sk, copy);
 589                 offset += copy;
 590                 size -= copy;
 591                 ctx->sg_plaintext_size += copy;
 592                 tls_ctx->pending_open_record_frags = ctx->sg_plaintext_num_elem;
 593
 594                 if (full_record || eor ||
 595                     ctx->sg_plaintext_num_elem ==
 596                     ARRAY_SIZE(ctx->sg_plaintext_data)) {
 597 push_record:
 598                         ret = tls_push_record(sk, flags, record_type);
 599                         if (ret) {
 600                                 if (ret == -ENOMEM)
 601                                         goto wait_for_memory;
 602
 603                                 goto sendpage_end;
 604                         }
 605                 }
 606                 continue;
 607 wait_for_sndbuf:
 608                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 609 wait_for_memory:
 610                 ret = sk_stream_wait_memory(sk, &timeo);
 611                 if (ret) {
 612                         trim_both_sgl(sk, ctx->sg_plaintext_size);
 613                         goto sendpage_end;
 614                 }
 615
 616                 if (tls_is_pending_closed_record(tls_ctx))
 617                         goto push_record;
 618
 619                 goto alloc_payload;
 620         }
 621
 622 sendpage_end:
 623         if (orig_size > size)
 624                 ret = orig_size - size;
 625         else
 626                 ret = sk_stream_error(sk, flags, ret);
 627
 628         release_sock(sk);
 629         return ret;
 630 }
 631
 632 void tls_sw_free_tx_resources(struct sock *sk)
 633 {
 634         struct tls_context *tls_ctx = tls_get_ctx(sk);
 635         struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx);
 636
 637         if (ctx->aead_send)
 638                 crypto_free_aead(ctx->aead_send);
 639
 640         tls_free_both_sg(sk);
 641
 642         kfree(ctx);
 643         kfree(tls_ctx);
 644 }
 645
 646 int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx)
 647 {
 648         char keyval[TLS_CIPHER_AES_GCM_128_KEY_SIZE];
 649         struct tls_crypto_info *crypto_info;
 650         struct tls12_crypto_info_aes_gcm_128 *gcm_128_info;
 651         struct tls_sw_context *sw_ctx;
 652         u16 nonce_size, tag_size, iv_size, rec_seq_size;
 653         char *iv, *rec_seq;
 654         int rc = 0;
 655
 656         if (!ctx) {
 657                 rc = -EINVAL;
 658                 goto out;
 659         }
 660
 661         if (ctx->priv_ctx) {
 662                 rc = -EEXIST;
 663                 goto out;
 664         }
 665
 666         sw_ctx = kzalloc(sizeof(*sw_ctx), GFP_KERNEL);
 667         if (!sw_ctx) {
 668                 rc = -ENOMEM;
 669                 goto out;
 670         }
 671
 672         crypto_init_wait(&sw_ctx->async_wait);
 673
 674         ctx->priv_ctx = (struct tls_offload_context *)sw_ctx;
 675
 676         crypto_info = &ctx->crypto_send;
 677         switch (crypto_info->cipher_type) {
 678         case TLS_CIPHER_AES_GCM_128: {
 679                 nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
 680                 tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
 681                 iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
 682                 iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv;
 683                 rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
 684                 rec_seq =
 685                  ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq;
 686                 gcm_128_info =
 687                         (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
 688                 break;
 689         }
 690         default:
 691                 rc = -EINVAL;
 692                 goto free_priv;
 693         }
 694
 695         ctx->prepend_size = TLS_HEADER_SIZE + nonce_size;
 696         ctx->tag_size = tag_size;
 697         ctx->overhead_size = ctx->prepend_size + ctx->tag_size;
 698         ctx->iv_size = iv_size;
 699         ctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE, GFP_KERNEL);
 700         if (!ctx->iv) {
 701                 rc = -ENOMEM;
 702                 goto free_priv;
 703         }
 704         memcpy(ctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
 705         memcpy(ctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size);
 706         ctx->rec_seq_size = rec_seq_size;
 707         ctx->rec_seq = kmalloc(rec_seq_size, GFP_KERNEL);
 708         if (!ctx->rec_seq) {
 709                 rc = -ENOMEM;
 710                 goto free_iv;
 711         }
 712         memcpy(ctx->rec_seq, rec_seq, rec_seq_size);
 713
 714         sg_init_table(sw_ctx->sg_encrypted_data,
 715                       ARRAY_SIZE(sw_ctx->sg_encrypted_data));
 716         sg_init_table(sw_ctx->sg_plaintext_data,
 717                       ARRAY_SIZE(sw_ctx->sg_plaintext_data));
 718
 719         sg_init_table(sw_ctx->sg_aead_in, 2);
 720         sg_set_buf(&sw_ctx->sg_aead_in[0], sw_ctx->aad_space,
 721                    sizeof(sw_ctx->aad_space));
 722         sg_unmark_end(&sw_ctx->sg_aead_in[1]);
 723         sg_chain(sw_ctx->sg_aead_in, 2, sw_ctx->sg_plaintext_data);
 724         sg_init_table(sw_ctx->sg_aead_out, 2);
 725         sg_set_buf(&sw_ctx->sg_aead_out[0], sw_ctx->aad_space,
 726                    sizeof(sw_ctx->aad_space));
 727         sg_unmark_end(&sw_ctx->sg_aead_out[1]);
 728         sg_chain(sw_ctx->sg_aead_out, 2, sw_ctx->sg_encrypted_data);
 729
 730         if (!sw_ctx->aead_send) {
 731                 sw_ctx->aead_send = crypto_alloc_aead("gcm(aes)", 0, 0);
 732                 if (IS_ERR(sw_ctx->aead_send)) {
 733                         rc = PTR_ERR(sw_ctx->aead_send);
 734                         sw_ctx->aead_send = NULL;
 735                         goto free_rec_seq;
 736                 }
 737         }
 738
 739         ctx->push_pending_record = tls_sw_push_pending_record;
 740
 741         memcpy(keyval, gcm_128_info->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE);
 742
 743         rc = crypto_aead_setkey(sw_ctx->aead_send, keyval,
 744                                 TLS_CIPHER_AES_GCM_128_KEY_SIZE);
 745         if (rc)
 746                 goto free_aead;
 747
 748         rc = crypto_aead_setauthsize(sw_ctx->aead_send, ctx->tag_size);
 749         if (!rc)
 750                 return 0;
 751
 752 free_aead:
 753         crypto_free_aead(sw_ctx->aead_send);
 754         sw_ctx->aead_send = NULL;
 755 free_rec_seq:
 756         kfree(ctx->rec_seq);
 757         ctx->rec_seq = NULL;
 758 free_iv:
 759         kfree(ctx->iv);
 760         ctx->iv = NULL;
 761 free_priv:
 762         kfree(ctx->priv_ctx);
 763         ctx->priv_ctx = NULL;
 764 out:
 765         return rc;
 766 }