drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2018 Chelsio Communications, Inc.
   4  *
   5  * Written by: Atul Gupta ([email protected])
   6  */
   7 #include <linux/kernel.h>
   8 #include <linux/module.h>
   9 #include <linux/skbuff.h>
  10 #include <linux/socket.h>
  11 #include <linux/hash.h>
  12 #include <linux/in.h>
  13 #include <linux/net.h>
  14 #include <linux/ip.h>
  15 #include <linux/tcp.h>
  16 #include <net/ipv6.h>
  17 #include <net/transp_v6.h>
  18 #include <net/tcp.h>
  19 #include <net/tls.h>
  20
  21 #include "chtls.h"
  22 #include "chtls_cm.h"
  23
  24 #define DRV_NAME "chtls"
  25
  26 /*
  27  * chtls device management
  28  * maintains a list of the chtls devices
  29  */
  30 static LIST_HEAD(cdev_list);
  31 static DEFINE_MUTEX(cdev_mutex);
  32
  33 static DEFINE_MUTEX(notify_mutex);
  34 static RAW_NOTIFIER_HEAD(listen_notify_list);
  35 static struct proto chtls_cpl_prot, chtls_cpl_protv6;
  36 struct request_sock_ops chtls_rsk_ops, chtls_rsk_opsv6;
  37 static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT;
  38
  39 static void register_listen_notifier(struct notifier_block *nb)
  40 {
  41         mutex_lock(&notify_mutex);
  42         raw_notifier_chain_register(&listen_notify_list, nb);
  43         mutex_unlock(&notify_mutex);
  44 }
  45
  46 static void unregister_listen_notifier(struct notifier_block *nb)
  47 {
  48         mutex_lock(&notify_mutex);
  49         raw_notifier_chain_unregister(&listen_notify_list, nb);
  50         mutex_unlock(&notify_mutex);
  51 }
  52
  53 static int listen_notify_handler(struct notifier_block *this,
  54                                  unsigned long event, void *data)
  55 {
  56         struct chtls_listen *clisten;
  57         int ret = NOTIFY_DONE;
  58
  59         clisten = (struct chtls_listen *)data;
  60
  61         switch (event) {
  62         case CHTLS_LISTEN_START:
  63                 ret = chtls_listen_start(clisten->cdev, clisten->sk);
  64                 kfree(clisten);
  65                 break;
  66         case CHTLS_LISTEN_STOP:
  67                 chtls_listen_stop(clisten->cdev, clisten->sk);
  68                 kfree(clisten);
  69                 break;
  70         }
  71         return ret;
  72 }
  73
  74 static struct notifier_block listen_notifier = {
  75         .notifier_call = listen_notify_handler
  76 };
  77
  78 static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb)
  79 {
  80         if (likely(skb_transport_header(skb) != skb_network_header(skb)))
  81                 return tcp_v4_do_rcv(sk, skb);
  82         BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
  83         return 0;
  84 }
  85
  86 static int chtls_start_listen(struct chtls_dev *cdev, struct sock *sk)
  87 {
  88         struct chtls_listen *clisten;
  89
  90         if (sk->sk_protocol != IPPROTO_TCP)
  91                 return -EPROTONOSUPPORT;
  92
  93         if (sk->sk_family == PF_INET &&
  94             LOOPBACK(inet_sk(sk)->inet_rcv_saddr))
  95                 return -EADDRNOTAVAIL;
  96
  97         sk->sk_backlog_rcv = listen_backlog_rcv;
  98         clisten = kmalloc(sizeof(*clisten), GFP_KERNEL);
  99         if (!clisten)
 100                 return -ENOMEM;
 101         clisten->cdev = cdev;
 102         clisten->sk = sk;
 103         mutex_lock(&notify_mutex);
 104         raw_notifier_call_chain(&listen_notify_list,
 105                                       CHTLS_LISTEN_START, clisten);
 106         mutex_unlock(&notify_mutex);
 107         return 0;
 108 }
 109
 110 static void chtls_stop_listen(struct chtls_dev *cdev, struct sock *sk)
 111 {
 112         struct chtls_listen *clisten;
 113
 114         if (sk->sk_protocol != IPPROTO_TCP)
 115                 return;
 116
 117         clisten = kmalloc(sizeof(*clisten), GFP_KERNEL);
 118         if (!clisten)
 119                 return;
 120         clisten->cdev = cdev;
 121         clisten->sk = sk;
 122         mutex_lock(&notify_mutex);
 123         raw_notifier_call_chain(&listen_notify_list,
 124                                 CHTLS_LISTEN_STOP, clisten);
 125         mutex_unlock(&notify_mutex);
 126 }
 127
 128 static int chtls_inline_feature(struct tls_toe_device *dev)
 129 {
 130         struct net_device *netdev;
 131         struct chtls_dev *cdev;
 132         int i;
 133
 134         cdev = to_chtls_dev(dev);
 135
 136         for (i = 0; i < cdev->lldi->nports; i++) {
 137                 netdev = cdev->ports[i];
 138                 if (netdev->features & NETIF_F_HW_TLS_RECORD)
 139                         return 1;
 140         }
 141         return 0;
 142 }
 143
 144 static int chtls_create_hash(struct tls_toe_device *dev, struct sock *sk)
 145 {
 146         struct chtls_dev *cdev = to_chtls_dev(dev);
 147
 148         if (sk->sk_state == TCP_LISTEN)
 149                 return chtls_start_listen(cdev, sk);
 150         return 0;
 151 }
 152
 153 static void chtls_destroy_hash(struct tls_toe_device *dev, struct sock *sk)
 154 {
 155         struct chtls_dev *cdev = to_chtls_dev(dev);
 156
 157         if (sk->sk_state == TCP_LISTEN)
 158                 chtls_stop_listen(cdev, sk);
 159 }
 160
 161 static void chtls_free_uld(struct chtls_dev *cdev)
 162 {
 163         int i;
 164
 165         tls_toe_unregister_device(&cdev->tlsdev);
 166         kvfree(cdev->kmap.addr);
 167         idr_destroy(&cdev->hwtid_idr);
 168         for (i = 0; i < (1 << RSPQ_HASH_BITS); i++)
 169                 kfree_skb(cdev->rspq_skb_cache[i]);
 170         kfree(cdev->lldi);
 171         kfree_skb(cdev->askb);
 172         kfree(cdev);
 173 }
 174
 175 static inline void chtls_dev_release(struct kref *kref)
 176 {
 177         struct tls_toe_device *dev;
 178         struct chtls_dev *cdev;
 179         struct adapter *adap;
 180
 181         dev = container_of(kref, struct tls_toe_device, kref);
 182         cdev = to_chtls_dev(dev);
 183
 184         /* Reset tls rx/tx stats */
 185         adap = pci_get_drvdata(cdev->pdev);
 186         atomic_set(&adap->chcr_stats.tls_pdu_tx, 0);
 187         atomic_set(&adap->chcr_stats.tls_pdu_rx, 0);
 188
 189         chtls_free_uld(cdev);
 190 }
 191
 192 static void chtls_register_dev(struct chtls_dev *cdev)
 193 {
 194         struct tls_toe_device *tlsdev = &cdev->tlsdev;
 195
 196         strscpy(tlsdev->name, "chtls", TLS_TOE_DEVICE_NAME_MAX);
 197         strlcat(tlsdev->name, cdev->lldi->ports[0]->name,
 198                 TLS_TOE_DEVICE_NAME_MAX);
 199         tlsdev->feature = chtls_inline_feature;
 200         tlsdev->hash = chtls_create_hash;
 201         tlsdev->unhash = chtls_destroy_hash;
 202         tlsdev->release = chtls_dev_release;
 203         kref_init(&tlsdev->kref);
 204         tls_toe_register_device(tlsdev);
 205         cdev->cdev_state = CHTLS_CDEV_STATE_UP;
 206 }
 207
 208 static void process_deferq(struct work_struct *task_param)
 209 {
 210         struct chtls_dev *cdev = container_of(task_param,
 211                                 struct chtls_dev, deferq_task);
 212         struct sk_buff *skb;
 213
 214         spin_lock_bh(&cdev->deferq.lock);
 215         while ((skb = __skb_dequeue(&cdev->deferq)) != NULL) {
 216                 spin_unlock_bh(&cdev->deferq.lock);
 217                 DEFERRED_SKB_CB(skb)->handler(cdev, skb);
 218                 spin_lock_bh(&cdev->deferq.lock);
 219         }
 220         spin_unlock_bh(&cdev->deferq.lock);
 221 }
 222
 223 static int chtls_get_skb(struct chtls_dev *cdev)
 224 {
 225         cdev->askb = alloc_skb(sizeof(struct tcphdr), GFP_KERNEL);
 226         if (!cdev->askb)
 227                 return -ENOMEM;
 228
 229         skb_put(cdev->askb, sizeof(struct tcphdr));
 230         skb_reset_transport_header(cdev->askb);
 231         memset(cdev->askb->data, 0, cdev->askb->len);
 232         return 0;
 233 }
 234
 235 static void *chtls_uld_add(const struct cxgb4_lld_info *info)
 236 {
 237         struct cxgb4_lld_info *lldi;
 238         struct chtls_dev *cdev;
 239         int i, j;
 240
 241         cdev = kzalloc(sizeof(*cdev), GFP_KERNEL);
 242         if (!cdev)
 243                 goto out;
 244
 245         lldi = kzalloc(sizeof(*lldi), GFP_KERNEL);
 246         if (!lldi)
 247                 goto out_lldi;
 248
 249         if (chtls_get_skb(cdev))
 250                 goto out_skb;
 251
 252         *lldi = *info;
 253         cdev->lldi = lldi;
 254         cdev->pdev = lldi->pdev;
 255         cdev->tids = lldi->tids;
 256         cdev->ports = lldi->ports;
 257         cdev->mtus = lldi->mtus;
 258         cdev->tids = lldi->tids;
 259         cdev->pfvf = FW_VIID_PFN_G(cxgb4_port_viid(lldi->ports[0]))
 260                         << FW_VIID_PFN_S;
 261
 262         for (i = 0; i < (1 << RSPQ_HASH_BITS); i++) {
 263                 unsigned int size = 64 - sizeof(struct rsp_ctrl) - 8;
 264
 265                 cdev->rspq_skb_cache[i] = __alloc_skb(size,
 266                                                       gfp_any(), 0,
 267                                                       lldi->nodeid);
 268                 if (unlikely(!cdev->rspq_skb_cache[i]))
 269                         goto out_rspq_skb;
 270         }
 271
 272         idr_init(&cdev->hwtid_idr);
 273         INIT_WORK(&cdev->deferq_task, process_deferq);
 274         spin_lock_init(&cdev->listen_lock);
 275         spin_lock_init(&cdev->idr_lock);
 276         cdev->send_page_order = min_t(uint, get_order(32768),
 277                                       send_page_order);
 278         cdev->max_host_sndbuf = 48 * 1024;
 279
 280         if (lldi->vr->key.size)
 281                 if (chtls_init_kmap(cdev, lldi))
 282                         goto out_rspq_skb;
 283
 284         mutex_lock(&cdev_mutex);
 285         list_add_tail(&cdev->list, &cdev_list);
 286         mutex_unlock(&cdev_mutex);
 287
 288         return cdev;
 289 out_rspq_skb:
 290         for (j = 0; j < i; j++)
 291                 kfree_skb(cdev->rspq_skb_cache[j]);
 292         kfree_skb(cdev->askb);
 293 out_skb:
 294         kfree(lldi);
 295 out_lldi:
 296         kfree(cdev);
 297 out:
 298         return NULL;
 299 }
 300
 301 static void chtls_free_all_uld(void)
 302 {
 303         struct chtls_dev *cdev, *tmp;
 304
 305         mutex_lock(&cdev_mutex);
 306         list_for_each_entry_safe(cdev, tmp, &cdev_list, list) {
 307                 if (cdev->cdev_state == CHTLS_CDEV_STATE_UP) {
 308                         list_del(&cdev->list);
 309                         kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release);
 310                 }
 311         }
 312         mutex_unlock(&cdev_mutex);
 313 }
 314
 315 static int chtls_uld_state_change(void *handle, enum cxgb4_state new_state)
 316 {
 317         struct chtls_dev *cdev = handle;
 318
 319         switch (new_state) {
 320         case CXGB4_STATE_UP:
 321                 chtls_register_dev(cdev);
 322                 break;
 323         case CXGB4_STATE_DOWN:
 324                 break;
 325         case CXGB4_STATE_START_RECOVERY:
 326                 break;
 327         case CXGB4_STATE_DETACH:
 328                 mutex_lock(&cdev_mutex);
 329                 list_del(&cdev->list);
 330                 mutex_unlock(&cdev_mutex);
 331                 kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release);
 332                 break;
 333         default:
 334                 break;
 335         }
 336         return 0;
 337 }
 338
 339 static struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl,
 340                                           const __be64 *rsp,
 341                                           u32 pktshift)
 342 {
 343         struct sk_buff *skb;
 344
 345         /* Allocate space for cpl_pass_accept_req which will be synthesized by
 346          * driver. Once driver synthesizes cpl_pass_accept_req the skb will go
 347          * through the regular cpl_pass_accept_req processing in TOM.
 348          */
 349         skb = alloc_skb(size_add(gl->tot_len,
 350                                  sizeof(struct cpl_pass_accept_req)) -
 351                         pktshift, GFP_ATOMIC);
 352         if (unlikely(!skb))
 353                 return NULL;
 354         __skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req)
 355                    - pktshift);
 356         /* For now we will copy  cpl_rx_pkt in the skb */
 357         skb_copy_to_linear_data(skb, rsp, sizeof(struct cpl_rx_pkt));
 358         skb_copy_to_linear_data_offset(skb, sizeof(struct cpl_pass_accept_req)
 359                                        , gl->va + pktshift,
 360                                        gl->tot_len - pktshift);
 361
 362         return skb;
 363 }
 364
 365 static int chtls_recv_packet(struct chtls_dev *cdev,
 366                              const struct pkt_gl *gl, const __be64 *rsp)
 367 {
 368         unsigned int opcode = *(u8 *)rsp;
 369         struct sk_buff *skb;
 370         int ret;
 371
 372         skb = copy_gl_to_skb_pkt(gl, rsp, cdev->lldi->sge_pktshift);
 373         if (!skb)
 374                 return -ENOMEM;
 375
 376         ret = chtls_handlers[opcode](cdev, skb);
 377         if (ret & CPL_RET_BUF_DONE)
 378                 kfree_skb(skb);
 379
 380         return 0;
 381 }
 382
 383 static int chtls_recv_rsp(struct chtls_dev *cdev, const __be64 *rsp)
 384 {
 385         unsigned long rspq_bin;
 386         unsigned int opcode;
 387         struct sk_buff *skb;
 388         unsigned int len;
 389         int ret;
 390
 391         len = 64 - sizeof(struct rsp_ctrl) - 8;
 392         opcode = *(u8 *)rsp;
 393
 394         rspq_bin = hash_ptr((void *)rsp, RSPQ_HASH_BITS);
 395         skb = cdev->rspq_skb_cache[rspq_bin];
 396         if (skb && !skb_is_nonlinear(skb) &&
 397             !skb_shared(skb) && !skb_cloned(skb)) {
 398                 refcount_inc(&skb->users);
 399                 if (refcount_read(&skb->users) == 2) {
 400                         __skb_trim(skb, 0);
 401                         if (skb_tailroom(skb) >= len)
 402                                 goto copy_out;
 403                 }
 404                 refcount_dec(&skb->users);
 405         }
 406         skb = alloc_skb(len, GFP_ATOMIC);
 407         if (unlikely(!skb))
 408                 return -ENOMEM;
 409
 410 copy_out:
 411         __skb_put(skb, len);
 412         skb_copy_to_linear_data(skb, rsp, len);
 413         skb_reset_network_header(skb);
 414         skb_reset_transport_header(skb);
 415         ret = chtls_handlers[opcode](cdev, skb);
 416
 417         if (ret & CPL_RET_BUF_DONE)
 418                 kfree_skb(skb);
 419         return 0;
 420 }
 421
 422 static void chtls_recv(struct chtls_dev *cdev,
 423                        struct sk_buff **skbs, const __be64 *rsp)
 424 {
 425         struct sk_buff *skb = *skbs;
 426         unsigned int opcode;
 427         int ret;
 428
 429         opcode = *(u8 *)rsp;
 430
 431         __skb_push(skb, sizeof(struct rss_header));
 432         skb_copy_to_linear_data(skb, rsp, sizeof(struct rss_header));
 433
 434         ret = chtls_handlers[opcode](cdev, skb);
 435         if (ret & CPL_RET_BUF_DONE)
 436                 kfree_skb(skb);
 437 }
 438
 439 static int chtls_uld_rx_handler(void *handle, const __be64 *rsp,
 440                                 const struct pkt_gl *gl)
 441 {
 442         struct chtls_dev *cdev = handle;
 443         unsigned int opcode;
 444         struct sk_buff *skb;
 445
 446         opcode = *(u8 *)rsp;
 447
 448         if (unlikely(opcode == CPL_RX_PKT)) {
 449                 if (chtls_recv_packet(cdev, gl, rsp) < 0)
 450                         goto nomem;
 451                 return 0;
 452         }
 453
 454         if (!gl)
 455                 return chtls_recv_rsp(cdev, rsp);
 456
 457 #define RX_PULL_LEN 128
 458         skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN);
 459         if (unlikely(!skb))
 460                 goto nomem;
 461         chtls_recv(cdev, &skb, rsp);
 462         return 0;
 463
 464 nomem:
 465         return -ENOMEM;
 466 }
 467
 468 static int do_chtls_getsockopt(struct sock *sk, char __user *optval,
 469                                int __user *optlen)
 470 {
 471         struct tls_crypto_info crypto_info = { 0 };
 472
 473         crypto_info.version = TLS_1_2_VERSION;
 474         if (copy_to_user(optval, &crypto_info, sizeof(struct tls_crypto_info)))
 475                 return -EFAULT;
 476         return 0;
 477 }
 478
 479 static int chtls_getsockopt(struct sock *sk, int level, int optname,
 480                             char __user *optval, int __user *optlen)
 481 {
 482         struct tls_context *ctx = tls_get_ctx(sk);
 483
 484         if (level != SOL_TLS)
 485                 return ctx->sk_proto->getsockopt(sk, level,
 486                                                  optname, optval, optlen);
 487
 488         return do_chtls_getsockopt(sk, optval, optlen);
 489 }
 490
 491 static int do_chtls_setsockopt(struct sock *sk, int optname,
 492                                sockptr_t optval, unsigned int optlen)
 493 {
 494         struct tls_crypto_info *crypto_info, tmp_crypto_info;
 495         struct chtls_sock *csk;
 496         int keylen;
 497         int cipher_type;
 498         int rc = 0;
 499
 500         csk = rcu_dereference_sk_user_data(sk);
 501
 502         if (sockptr_is_null(optval) || optlen < sizeof(*crypto_info)) {
 503                 rc = -EINVAL;
 504                 goto out;
 505         }
 506
 507         rc = copy_from_sockptr(&tmp_crypto_info, optval, sizeof(*crypto_info));
 508         if (rc) {
 509                 rc = -EFAULT;
 510                 goto out;
 511         }
 512
 513         /* check version */
 514         if (tmp_crypto_info.version != TLS_1_2_VERSION) {
 515                 rc = -ENOTSUPP;
 516                 goto out;
 517         }
 518
 519         crypto_info = (struct tls_crypto_info *)&csk->tlshws.crypto_info;
 520
 521         /* GCM mode of AES supports 128 and 256 bit encryption, so
 522          * copy keys from user based on GCM cipher type.
 523          */
 524         switch (tmp_crypto_info.cipher_type) {
 525         case TLS_CIPHER_AES_GCM_128: {
 526                 /* Obtain version and type from previous copy */
 527                 crypto_info[0] = tmp_crypto_info;
 528                 /* Now copy the following data */
 529                 rc = copy_from_sockptr_offset((char *)crypto_info +
 530                                 sizeof(*crypto_info),
 531                                 optval, sizeof(*crypto_info),
 532                                 sizeof(struct tls12_crypto_info_aes_gcm_128)
 533                                 - sizeof(*crypto_info));
 534
 535                 if (rc) {
 536                         rc = -EFAULT;
 537                         goto out;
 538                 }
 539
 540                 keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
 541                 cipher_type = TLS_CIPHER_AES_GCM_128;
 542                 break;
 543         }
 544         case TLS_CIPHER_AES_GCM_256: {
 545                 crypto_info[0] = tmp_crypto_info;
 546                 rc = copy_from_sockptr_offset((char *)crypto_info +
 547                                 sizeof(*crypto_info),
 548                                 optval, sizeof(*crypto_info),
 549                                 sizeof(struct tls12_crypto_info_aes_gcm_256)
 550                                 - sizeof(*crypto_info));
 551
 552                 if (rc) {
 553                         rc = -EFAULT;
 554                         goto out;
 555                 }
 556
 557                 keylen = TLS_CIPHER_AES_GCM_256_KEY_SIZE;
 558                 cipher_type = TLS_CIPHER_AES_GCM_256;
 559                 break;
 560         }
 561         default:
 562                 rc = -EINVAL;
 563                 goto out;
 564         }
 565         rc = chtls_setkey(csk, keylen, optname, cipher_type);
 566 out:
 567         return rc;
 568 }
 569
 570 static int chtls_setsockopt(struct sock *sk, int level, int optname,
 571                             sockptr_t optval, unsigned int optlen)
 572 {
 573         struct tls_context *ctx = tls_get_ctx(sk);
 574
 575         if (level != SOL_TLS)
 576                 return ctx->sk_proto->setsockopt(sk, level,
 577                                                  optname, optval, optlen);
 578
 579         return do_chtls_setsockopt(sk, optname, optval, optlen);
 580 }
 581
 582 static struct cxgb4_uld_info chtls_uld_info = {
 583         .name = DRV_NAME,
 584         .nrxq = MAX_ULD_QSETS,
 585         .ntxq = MAX_ULD_QSETS,
 586         .rxq_size = 1024,
 587         .add = chtls_uld_add,
 588         .state_change = chtls_uld_state_change,
 589         .rx_handler = chtls_uld_rx_handler,
 590 };
 591
 592 void chtls_install_cpl_ops(struct sock *sk)
 593 {
 594         if (sk->sk_family == AF_INET)
 595                 sk->sk_prot = &chtls_cpl_prot;
 596         else
 597                 sk->sk_prot = &chtls_cpl_protv6;
 598 }
 599
 600 static void __init chtls_init_ulp_ops(void)
 601 {
 602         chtls_cpl_prot                  = tcp_prot;
 603         chtls_init_rsk_ops(&chtls_cpl_prot, &chtls_rsk_ops,
 604                            &tcp_prot, PF_INET);
 605         chtls_cpl_prot.close            = chtls_close;
 606         chtls_cpl_prot.disconnect       = chtls_disconnect;
 607         chtls_cpl_prot.destroy          = chtls_destroy_sock;
 608         chtls_cpl_prot.shutdown         = chtls_shutdown;
 609         chtls_cpl_prot.sendmsg          = chtls_sendmsg;
 610         chtls_cpl_prot.splice_eof       = chtls_splice_eof;
 611         chtls_cpl_prot.recvmsg          = chtls_recvmsg;
 612         chtls_cpl_prot.setsockopt       = chtls_setsockopt;
 613         chtls_cpl_prot.getsockopt       = chtls_getsockopt;
 614 #if IS_ENABLED(CONFIG_IPV6)
 615         chtls_cpl_protv6                = chtls_cpl_prot;
 616         chtls_init_rsk_ops(&chtls_cpl_protv6, &chtls_rsk_opsv6,
 617                            &tcpv6_prot, PF_INET6);
 618 #endif
 619 }
 620
 621 static int __init chtls_register(void)
 622 {
 623         chtls_init_ulp_ops();
 624         register_listen_notifier(&listen_notifier);
 625         cxgb4_register_uld(CXGB4_ULD_TLS, &chtls_uld_info);
 626         return 0;
 627 }
 628
 629 static void __exit chtls_unregister(void)
 630 {
 631         unregister_listen_notifier(&listen_notifier);
 632         chtls_free_all_uld();
 633         cxgb4_unregister_uld(CXGB4_ULD_TLS);
 634 }
 635
 636 module_init(chtls_register);
 637 module_exit(chtls_unregister);
 638
 639 MODULE_DESCRIPTION("Chelsio TLS Inline driver");
 640 MODULE_LICENSE("GPL");
 641 MODULE_AUTHOR("Chelsio Communications");
 642 MODULE_VERSION(CHTLS_DRV_VERSION);