drivers/crypto/intel/qat/qat_common/adf_transport.c

   1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
   2 /* Copyright(c) 2014 - 2020 Intel Corporation */
   3 #include <linux/delay.h>
   4 #include <linux/nospec.h>
   5 #include "adf_accel_devices.h"
   6 #include "adf_transport_internal.h"
   7 #include "adf_transport_access_macros.h"
   8 #include "adf_cfg.h"
   9 #include "adf_common_drv.h"
  10
  11 #define ADF_MAX_RING_THRESHOLD          80
  12 #define ADF_PERCENT(tot, percent)       (((tot) * (percent)) / 100)
  13
  14 static inline u32 adf_modulo(u32 data, u32 shift)
  15 {
  16         u32 div = data >> shift;
  17         u32 mult = div << shift;
  18
  19         return data - mult;
  20 }
  21
  22 static inline int adf_check_ring_alignment(u64 addr, u64 size)
  23 {
  24         if (((size - 1) & addr) != 0)
  25                 return -EFAULT;
  26         return 0;
  27 }
  28
  29 static int adf_verify_ring_size(u32 msg_size, u32 msg_num)
  30 {
  31         int i = ADF_MIN_RING_SIZE;
  32
  33         for (; i <= ADF_MAX_RING_SIZE; i++)
  34                 if ((msg_size * msg_num) == ADF_SIZE_TO_RING_SIZE_IN_BYTES(i))
  35                         return i;
  36
  37         return ADF_DEFAULT_RING_SIZE;
  38 }
  39
  40 static int adf_reserve_ring(struct adf_etr_bank_data *bank, u32 ring)
  41 {
  42         spin_lock(&bank->lock);
  43         if (bank->ring_mask & (1 << ring)) {
  44                 spin_unlock(&bank->lock);
  45                 return -EFAULT;
  46         }
  47         bank->ring_mask |= (1 << ring);
  48         spin_unlock(&bank->lock);
  49         return 0;
  50 }
  51
  52 static void adf_unreserve_ring(struct adf_etr_bank_data *bank, u32 ring)
  53 {
  54         spin_lock(&bank->lock);
  55         bank->ring_mask &= ~(1 << ring);
  56         spin_unlock(&bank->lock);
  57 }
  58
  59 static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)
  60 {
  61         struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
  62
  63         spin_lock_bh(&bank->lock);
  64         bank->irq_mask |= (1 << ring);
  65         spin_unlock_bh(&bank->lock);
  66         csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number,
  67                                       bank->irq_mask);
  68         csr_ops->write_csr_int_col_ctl(bank->csr_addr, bank->bank_number,
  69                                        bank->irq_coalesc_timer);
  70 }
  71
  72 static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)
  73 {
  74         struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
  75
  76         spin_lock_bh(&bank->lock);
  77         bank->irq_mask &= ~(1 << ring);
  78         spin_unlock_bh(&bank->lock);
  79         csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number,
  80                                       bank->irq_mask);
  81 }
  82
  83 bool adf_ring_nearly_full(struct adf_etr_ring_data *ring)
  84 {
  85         return atomic_read(ring->inflights) > ring->threshold;
  86 }
  87
  88 int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg)
  89 {
  90         struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
  91
  92         if (atomic_add_return(1, ring->inflights) >
  93             ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) {
  94                 atomic_dec(ring->inflights);
  95                 return -EAGAIN;
  96         }
  97         spin_lock_bh(&ring->lock);
  98         memcpy((void *)((uintptr_t)ring->base_addr + ring->tail), msg,
  99                ADF_MSG_SIZE_TO_BYTES(ring->msg_size));
 100
 101         ring->tail = adf_modulo(ring->tail +
 102                                 ADF_MSG_SIZE_TO_BYTES(ring->msg_size),
 103                                 ADF_RING_SIZE_MODULO(ring->ring_size));
 104         csr_ops->write_csr_ring_tail(ring->bank->csr_addr,
 105                                      ring->bank->bank_number, ring->ring_number,
 106                                      ring->tail);
 107         spin_unlock_bh(&ring->lock);
 108
 109         return 0;
 110 }
 111
 112 static int adf_handle_response(struct adf_etr_ring_data *ring)
 113 {
 114         struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
 115         u32 msg_counter = 0;
 116         u32 *msg = (u32 *)((uintptr_t)ring->base_addr + ring->head);
 117
 118         while (*msg != ADF_RING_EMPTY_SIG) {
 119                 ring->callback((u32 *)msg);
 120                 atomic_dec(ring->inflights);
 121                 *msg = ADF_RING_EMPTY_SIG;
 122                 ring->head = adf_modulo(ring->head +
 123                                         ADF_MSG_SIZE_TO_BYTES(ring->msg_size),
 124                                         ADF_RING_SIZE_MODULO(ring->ring_size));
 125                 msg_counter++;
 126                 msg = (u32 *)((uintptr_t)ring->base_addr + ring->head);
 127         }
 128         if (msg_counter > 0) {
 129                 csr_ops->write_csr_ring_head(ring->bank->csr_addr,
 130                                              ring->bank->bank_number,
 131                                              ring->ring_number, ring->head);
 132         }
 133         return 0;
 134 }
 135
 136 static void adf_configure_tx_ring(struct adf_etr_ring_data *ring)
 137 {
 138         struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
 139         u32 ring_config = BUILD_RING_CONFIG(ring->ring_size);
 140
 141         csr_ops->write_csr_ring_config(ring->bank->csr_addr,
 142                                        ring->bank->bank_number,
 143                                        ring->ring_number, ring_config);
 144
 145 }
 146
 147 static void adf_configure_rx_ring(struct adf_etr_ring_data *ring)
 148 {
 149         struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
 150         u32 ring_config =
 151                         BUILD_RESP_RING_CONFIG(ring->ring_size,
 152                                                ADF_RING_NEAR_WATERMARK_512,
 153                                                ADF_RING_NEAR_WATERMARK_0);
 154
 155         csr_ops->write_csr_ring_config(ring->bank->csr_addr,
 156                                        ring->bank->bank_number,
 157                                        ring->ring_number, ring_config);
 158 }
 159
 160 static int adf_init_ring(struct adf_etr_ring_data *ring)
 161 {
 162         struct adf_etr_bank_data *bank = ring->bank;
 163         struct adf_accel_dev *accel_dev = bank->accel_dev;
 164         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 165         struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
 166         u64 ring_base;
 167         u32 ring_size_bytes =
 168                         ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);
 169
 170         ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes);
 171         ring->base_addr = dma_alloc_coherent(&GET_DEV(accel_dev),
 172                                              ring_size_bytes, &ring->dma_addr,
 173                                              GFP_KERNEL);
 174         if (!ring->base_addr)
 175                 return -ENOMEM;
 176
 177         memset(ring->base_addr, 0x7F, ring_size_bytes);
 178         /* The base_addr has to be aligned to the size of the buffer */
 179         if (adf_check_ring_alignment(ring->dma_addr, ring_size_bytes)) {
 180                 dev_err(&GET_DEV(accel_dev), "Ring address not aligned\n");
 181                 dma_free_coherent(&GET_DEV(accel_dev), ring_size_bytes,
 182                                   ring->base_addr, ring->dma_addr);
 183                 ring->base_addr = NULL;
 184                 return -EFAULT;
 185         }
 186
 187         if (hw_data->tx_rings_mask & (1 << ring->ring_number))
 188                 adf_configure_tx_ring(ring);
 189
 190         else
 191                 adf_configure_rx_ring(ring);
 192
 193         ring_base = csr_ops->build_csr_ring_base_addr(ring->dma_addr,
 194                                                       ring->ring_size);
 195
 196         csr_ops->write_csr_ring_base(ring->bank->csr_addr,
 197                                      ring->bank->bank_number, ring->ring_number,
 198                                      ring_base);
 199         spin_lock_init(&ring->lock);
 200         return 0;
 201 }
 202
 203 static void adf_cleanup_ring(struct adf_etr_ring_data *ring)
 204 {
 205         u32 ring_size_bytes =
 206                         ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);
 207         ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes);
 208
 209         if (ring->base_addr) {
 210                 memset(ring->base_addr, 0x7F, ring_size_bytes);
 211                 dma_free_coherent(&GET_DEV(ring->bank->accel_dev),
 212                                   ring_size_bytes, ring->base_addr,
 213                                   ring->dma_addr);
 214         }
 215 }
 216
 217 int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section,
 218                     u32 bank_num, u32 num_msgs,
 219                     u32 msg_size, const char *ring_name,
 220                     adf_callback_fn callback, int poll_mode,
 221                     struct adf_etr_ring_data **ring_ptr)
 222 {
 223         struct adf_etr_data *transport_data = accel_dev->transport;
 224         u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev);
 225         struct adf_etr_bank_data *bank;
 226         struct adf_etr_ring_data *ring;
 227         char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
 228         int max_inflights;
 229         u32 ring_num;
 230         int ret;
 231
 232         if (bank_num >= GET_MAX_BANKS(accel_dev)) {
 233                 dev_err(&GET_DEV(accel_dev), "Invalid bank number\n");
 234                 return -EFAULT;
 235         }
 236         if (msg_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
 237                 dev_err(&GET_DEV(accel_dev), "Invalid msg size\n");
 238                 return -EFAULT;
 239         }
 240         if (ADF_MAX_INFLIGHTS(adf_verify_ring_size(msg_size, num_msgs),
 241                               ADF_BYTES_TO_MSG_SIZE(msg_size)) < 2) {
 242                 dev_err(&GET_DEV(accel_dev),
 243                         "Invalid ring size for given msg size\n");
 244                 return -EFAULT;
 245         }
 246         if (adf_cfg_get_param_value(accel_dev, section, ring_name, val)) {
 247                 dev_err(&GET_DEV(accel_dev), "Section %s, no such entry : %s\n",
 248                         section, ring_name);
 249                 return -EFAULT;
 250         }
 251         if (kstrtouint(val, 10, &ring_num)) {
 252                 dev_err(&GET_DEV(accel_dev), "Can't get ring number\n");
 253                 return -EFAULT;
 254         }
 255         if (ring_num >= num_rings_per_bank) {
 256                 dev_err(&GET_DEV(accel_dev), "Invalid ring number\n");
 257                 return -EFAULT;
 258         }
 259
 260         ring_num = array_index_nospec(ring_num, num_rings_per_bank);
 261         bank = &transport_data->banks[bank_num];
 262         if (adf_reserve_ring(bank, ring_num)) {
 263                 dev_err(&GET_DEV(accel_dev), "Ring %d, %s already exists.\n",
 264                         ring_num, ring_name);
 265                 return -EFAULT;
 266         }
 267         ring = &bank->rings[ring_num];
 268         ring->ring_number = ring_num;
 269         ring->bank = bank;
 270         ring->callback = callback;
 271         ring->msg_size = ADF_BYTES_TO_MSG_SIZE(msg_size);
 272         ring->ring_size = adf_verify_ring_size(msg_size, num_msgs);
 273         ring->head = 0;
 274         ring->tail = 0;
 275         max_inflights = ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size);
 276         ring->threshold = ADF_PERCENT(max_inflights, ADF_MAX_RING_THRESHOLD);
 277         atomic_set(ring->inflights, 0);
 278         ret = adf_init_ring(ring);
 279         if (ret)
 280                 goto err;
 281
 282         /* Enable HW arbitration for the given ring */
 283         adf_update_ring_arb(ring);
 284
 285         if (adf_ring_debugfs_add(ring, ring_name)) {
 286                 dev_err(&GET_DEV(accel_dev),
 287                         "Couldn't add ring debugfs entry\n");
 288                 ret = -EFAULT;
 289                 goto err;
 290         }
 291
 292         /* Enable interrupts if needed */
 293         if (callback && (!poll_mode))
 294                 adf_enable_ring_irq(bank, ring->ring_number);
 295         *ring_ptr = ring;
 296         return 0;
 297 err:
 298         adf_cleanup_ring(ring);
 299         adf_unreserve_ring(bank, ring_num);
 300         adf_update_ring_arb(ring);
 301         return ret;
 302 }
 303
 304 void adf_remove_ring(struct adf_etr_ring_data *ring)
 305 {
 306         struct adf_etr_bank_data *bank = ring->bank;
 307         struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
 308
 309         /* Disable interrupts for the given ring */
 310         adf_disable_ring_irq(bank, ring->ring_number);
 311
 312         /* Clear PCI config space */
 313
 314         csr_ops->write_csr_ring_config(bank->csr_addr, bank->bank_number,
 315                                        ring->ring_number, 0);
 316         csr_ops->write_csr_ring_base(bank->csr_addr, bank->bank_number,
 317                                      ring->ring_number, 0);
 318         adf_ring_debugfs_rm(ring);
 319         adf_unreserve_ring(bank, ring->ring_number);
 320         /* Disable HW arbitration for the given ring */
 321         adf_update_ring_arb(ring);
 322         adf_cleanup_ring(ring);
 323 }
 324
 325 static void adf_ring_response_handler(struct adf_etr_bank_data *bank)
 326 {
 327         struct adf_accel_dev *accel_dev = bank->accel_dev;
 328         u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev);
 329         struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
 330         unsigned long empty_rings;
 331         int i;
 332
 333         empty_rings = csr_ops->read_csr_e_stat(bank->csr_addr,
 334                                                bank->bank_number);
 335         empty_rings = ~empty_rings & bank->irq_mask;
 336
 337         for_each_set_bit(i, &empty_rings, num_rings_per_bank)
 338                 adf_handle_response(&bank->rings[i]);
 339 }
 340
 341 void adf_response_handler(uintptr_t bank_addr)
 342 {
 343         struct adf_etr_bank_data *bank = (void *)bank_addr;
 344         struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
 345
 346         /* Handle all the responses and reenable IRQs */
 347         adf_ring_response_handler(bank);
 348
 349         csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number,
 350                                             bank->irq_mask);
 351 }
 352
 353 static inline int adf_get_cfg_int(struct adf_accel_dev *accel_dev,
 354                                   const char *section, const char *format,
 355                                   u32 key, u32 *value)
 356 {
 357         char key_buf[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
 358         char val_buf[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
 359
 360         snprintf(key_buf, ADF_CFG_MAX_KEY_LEN_IN_BYTES, format, key);
 361
 362         if (adf_cfg_get_param_value(accel_dev, section, key_buf, val_buf))
 363                 return -EFAULT;
 364
 365         if (kstrtouint(val_buf, 10, value))
 366                 return -EFAULT;
 367         return 0;
 368 }
 369
 370 static void adf_get_coalesc_timer(struct adf_etr_bank_data *bank,
 371                                   const char *section,
 372                                   u32 bank_num_in_accel)
 373 {
 374         if (adf_get_cfg_int(bank->accel_dev, section,
 375                             ADF_ETRMGR_COALESCE_TIMER_FORMAT,
 376                             bank_num_in_accel, &bank->irq_coalesc_timer))
 377                 bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME;
 378
 379         if (ADF_COALESCING_MAX_TIME < bank->irq_coalesc_timer ||
 380             ADF_COALESCING_MIN_TIME > bank->irq_coalesc_timer)
 381                 bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME;
 382 }
 383
 384 static int adf_init_bank(struct adf_accel_dev *accel_dev,
 385                          struct adf_etr_bank_data *bank,
 386                          u32 bank_num, void __iomem *csr_addr)
 387 {
 388         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 389         u8 num_rings_per_bank = hw_data->num_rings_per_bank;
 390         struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops;
 391         u32 irq_mask = BIT(num_rings_per_bank) - 1;
 392         struct adf_etr_ring_data *ring;
 393         struct adf_etr_ring_data *tx_ring;
 394         u32 i, coalesc_enabled = 0;
 395         unsigned long ring_mask;
 396         int size;
 397
 398         memset(bank, 0, sizeof(*bank));
 399         bank->bank_number = bank_num;
 400         bank->csr_addr = csr_addr;
 401         bank->accel_dev = accel_dev;
 402         spin_lock_init(&bank->lock);
 403
 404         /* Allocate the rings in the bank */
 405         size = num_rings_per_bank * sizeof(struct adf_etr_ring_data);
 406         bank->rings = kzalloc_node(size, GFP_KERNEL,
 407                                    dev_to_node(&GET_DEV(accel_dev)));
 408         if (!bank->rings)
 409                 return -ENOMEM;
 410
 411         /* Enable IRQ coalescing always. This will allow to use
 412          * the optimised flag and coalesc register.
 413          * If it is disabled in the config file just use min time value */
 414         if ((adf_get_cfg_int(accel_dev, "Accelerator0",
 415                              ADF_ETRMGR_COALESCING_ENABLED_FORMAT, bank_num,
 416                              &coalesc_enabled) == 0) && coalesc_enabled)
 417                 adf_get_coalesc_timer(bank, "Accelerator0", bank_num);
 418         else
 419                 bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME;
 420
 421         for (i = 0; i < num_rings_per_bank; i++) {
 422                 csr_ops->write_csr_ring_config(csr_addr, bank_num, i, 0);
 423                 csr_ops->write_csr_ring_base(csr_addr, bank_num, i, 0);
 424
 425                 ring = &bank->rings[i];
 426                 if (hw_data->tx_rings_mask & (1 << i)) {
 427                         ring->inflights =
 428                                 kzalloc_node(sizeof(atomic_t),
 429                                              GFP_KERNEL,
 430                                              dev_to_node(&GET_DEV(accel_dev)));
 431                         if (!ring->inflights)
 432                                 goto err;
 433                 } else {
 434                         if (i < hw_data->tx_rx_gap) {
 435                                 dev_err(&GET_DEV(accel_dev),
 436                                         "Invalid tx rings mask config\n");
 437                                 goto err;
 438                         }
 439                         tx_ring = &bank->rings[i - hw_data->tx_rx_gap];
 440                         ring->inflights = tx_ring->inflights;
 441                 }
 442         }
 443         if (adf_bank_debugfs_add(bank)) {
 444                 dev_err(&GET_DEV(accel_dev),
 445                         "Failed to add bank debugfs entry\n");
 446                 goto err;
 447         }
 448
 449         csr_ops->write_csr_int_flag(csr_addr, bank_num, irq_mask);
 450         csr_ops->write_csr_int_srcsel(csr_addr, bank_num);
 451
 452         return 0;
 453 err:
 454         ring_mask = hw_data->tx_rings_mask;
 455         for_each_set_bit(i, &ring_mask, num_rings_per_bank) {
 456                 ring = &bank->rings[i];
 457                 kfree(ring->inflights);
 458                 ring->inflights = NULL;
 459         }
 460         kfree(bank->rings);
 461         return -ENOMEM;
 462 }
 463
 464 /**
 465  * adf_init_etr_data() - Initialize transport rings for acceleration device
 466  * @accel_dev:  Pointer to acceleration device.
 467  *
 468  * Function is the initializes the communications channels (rings) to the
 469  * acceleration device accel_dev.
 470  * To be used by QAT device specific drivers.
 471  *
 472  * Return: 0 on success, error code otherwise.
 473  */
 474 int adf_init_etr_data(struct adf_accel_dev *accel_dev)
 475 {
 476         struct adf_etr_data *etr_data;
 477         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 478         void __iomem *csr_addr;
 479         u32 size;
 480         u32 num_banks = 0;
 481         int i, ret;
 482
 483         etr_data = kzalloc_node(sizeof(*etr_data), GFP_KERNEL,
 484                                 dev_to_node(&GET_DEV(accel_dev)));
 485         if (!etr_data)
 486                 return -ENOMEM;
 487
 488         num_banks = GET_MAX_BANKS(accel_dev);
 489         size = num_banks * sizeof(struct adf_etr_bank_data);
 490         etr_data->banks = kzalloc_node(size, GFP_KERNEL,
 491                                        dev_to_node(&GET_DEV(accel_dev)));
 492         if (!etr_data->banks) {
 493                 ret = -ENOMEM;
 494                 goto err_bank;
 495         }
 496
 497         accel_dev->transport = etr_data;
 498         i = hw_data->get_etr_bar_id(hw_data);
 499         csr_addr = accel_dev->accel_pci_dev.pci_bars[i].virt_addr;
 500
 501         /* accel_dev->debugfs_dir should always be non-NULL here */
 502         etr_data->debug = debugfs_create_dir("transport",
 503                                              accel_dev->debugfs_dir);
 504
 505         for (i = 0; i < num_banks; i++) {
 506                 ret = adf_init_bank(accel_dev, &etr_data->banks[i], i,
 507                                     csr_addr);
 508                 if (ret)
 509                         goto err_bank_all;
 510         }
 511
 512         return 0;
 513
 514 err_bank_all:
 515         debugfs_remove(etr_data->debug);
 516         kfree(etr_data->banks);
 517 err_bank:
 518         kfree(etr_data);
 519         accel_dev->transport = NULL;
 520         return ret;
 521 }
 522 EXPORT_SYMBOL_GPL(adf_init_etr_data);
 523
 524 static void cleanup_bank(struct adf_etr_bank_data *bank)
 525 {
 526         struct adf_accel_dev *accel_dev = bank->accel_dev;
 527         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 528         u8 num_rings_per_bank = hw_data->num_rings_per_bank;
 529         u32 i;
 530
 531         for (i = 0; i < num_rings_per_bank; i++) {
 532                 struct adf_etr_ring_data *ring = &bank->rings[i];
 533
 534                 if (bank->ring_mask & (1 << i))
 535                         adf_cleanup_ring(ring);
 536
 537                 if (hw_data->tx_rings_mask & (1 << i))
 538                         kfree(ring->inflights);
 539         }
 540         kfree(bank->rings);
 541         adf_bank_debugfs_rm(bank);
 542         memset(bank, 0, sizeof(*bank));
 543 }
 544
 545 static void adf_cleanup_etr_handles(struct adf_accel_dev *accel_dev)
 546 {
 547         struct adf_etr_data *etr_data = accel_dev->transport;
 548         u32 i, num_banks = GET_MAX_BANKS(accel_dev);
 549
 550         for (i = 0; i < num_banks; i++)
 551                 cleanup_bank(&etr_data->banks[i]);
 552 }
 553
 554 /**
 555  * adf_cleanup_etr_data() - Clear transport rings for acceleration device
 556  * @accel_dev:  Pointer to acceleration device.
 557  *
 558  * Function is the clears the communications channels (rings) of the
 559  * acceleration device accel_dev.
 560  * To be used by QAT device specific drivers.
 561  *
 562  * Return: void
 563  */
 564 void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev)
 565 {
 566         struct adf_etr_data *etr_data = accel_dev->transport;
 567
 568         if (etr_data) {
 569                 adf_cleanup_etr_handles(accel_dev);
 570                 debugfs_remove(etr_data->debug);
 571                 kfree(etr_data->banks->rings);
 572                 kfree(etr_data->banks);
 573                 kfree(etr_data);
 574                 accel_dev->transport = NULL;
 575         }
 576 }
 577 EXPORT_SYMBOL_GPL(adf_cleanup_etr_data);