drivers/s390/net/ism_drv.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * ISM driver for s390.
   4  *
   5  * Copyright IBM Corp. 2018
   6  */
   7 #define KMSG_COMPONENT "ism"
   8 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
   9
  10 #include <linux/module.h>
  11 #include <linux/types.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/device.h>
  14 #include <linux/err.h>
  15 #include <linux/ctype.h>
  16 #include <linux/processor.h>
  17
  18 #include "ism.h"
  19
  20 MODULE_DESCRIPTION("ISM driver for s390");
  21 MODULE_LICENSE("GPL");
  22
  23 #define PCI_DEVICE_ID_IBM_ISM 0x04ED
  24 #define DRV_NAME "ism"
  25
  26 static const struct pci_device_id ism_device_table[] = {
  27         { PCI_VDEVICE(IBM, PCI_DEVICE_ID_IBM_ISM), 0 },
  28         { 0, }
  29 };
  30 MODULE_DEVICE_TABLE(pci, ism_device_table);
  31
  32 static debug_info_t *ism_debug_info;
  33
  34 #define NO_CLIENT               0xff            /* must be >= MAX_CLIENTS */
  35 static struct ism_client *clients[MAX_CLIENTS]; /* use an array rather than */
  36                                                 /* a list for fast mapping  */
  37 static u8 max_client;
  38 static DEFINE_MUTEX(clients_lock);
  39 static bool ism_v2_capable;
  40 struct ism_dev_list {
  41         struct list_head list;
  42         struct mutex mutex; /* protects ism device list */
  43 };
  44
  45 static struct ism_dev_list ism_dev_list = {
  46         .list = LIST_HEAD_INIT(ism_dev_list.list),
  47         .mutex = __MUTEX_INITIALIZER(ism_dev_list.mutex),
  48 };
  49
  50 static void ism_setup_forwarding(struct ism_client *client, struct ism_dev *ism)
  51 {
  52         unsigned long flags;
  53
  54         spin_lock_irqsave(&ism->lock, flags);
  55         ism->subs[client->id] = client;
  56         spin_unlock_irqrestore(&ism->lock, flags);
  57 }
  58
  59 int ism_register_client(struct ism_client *client)
  60 {
  61         struct ism_dev *ism;
  62         int i, rc = -ENOSPC;
  63
  64         mutex_lock(&ism_dev_list.mutex);
  65         mutex_lock(&clients_lock);
  66         for (i = 0; i < MAX_CLIENTS; ++i) {
  67                 if (!clients[i]) {
  68                         clients[i] = client;
  69                         client->id = i;
  70                         if (i == max_client)
  71                                 max_client++;
  72                         rc = 0;
  73                         break;
  74                 }
  75         }
  76         mutex_unlock(&clients_lock);
  77
  78         if (i < MAX_CLIENTS) {
  79                 /* initialize with all devices that we got so far */
  80                 list_for_each_entry(ism, &ism_dev_list.list, list) {
  81                         ism->priv[i] = NULL;
  82                         client->add(ism);
  83                         ism_setup_forwarding(client, ism);
  84                 }
  85         }
  86         mutex_unlock(&ism_dev_list.mutex);
  87
  88         return rc;
  89 }
  90 EXPORT_SYMBOL_GPL(ism_register_client);
  91
  92 int ism_unregister_client(struct ism_client *client)
  93 {
  94         struct ism_dev *ism;
  95         unsigned long flags;
  96         int rc = 0;
  97
  98         mutex_lock(&ism_dev_list.mutex);
  99         list_for_each_entry(ism, &ism_dev_list.list, list) {
 100                 spin_lock_irqsave(&ism->lock, flags);
 101                 /* Stop forwarding IRQs and events */
 102                 ism->subs[client->id] = NULL;
 103                 for (int i = 0; i < ISM_NR_DMBS; ++i) {
 104                         if (ism->sba_client_arr[i] == client->id) {
 105                                 WARN(1, "%s: attempt to unregister '%s' with registered dmb(s)\n",
 106                                      __func__, client->name);
 107                                 rc = -EBUSY;
 108                                 goto err_reg_dmb;
 109                         }
 110                 }
 111                 spin_unlock_irqrestore(&ism->lock, flags);
 112         }
 113         mutex_unlock(&ism_dev_list.mutex);
 114
 115         mutex_lock(&clients_lock);
 116         clients[client->id] = NULL;
 117         if (client->id + 1 == max_client)
 118                 max_client--;
 119         mutex_unlock(&clients_lock);
 120         return rc;
 121
 122 err_reg_dmb:
 123         spin_unlock_irqrestore(&ism->lock, flags);
 124         mutex_unlock(&ism_dev_list.mutex);
 125         return rc;
 126 }
 127 EXPORT_SYMBOL_GPL(ism_unregister_client);
 128
 129 static int ism_cmd(struct ism_dev *ism, void *cmd)
 130 {
 131         struct ism_req_hdr *req = cmd;
 132         struct ism_resp_hdr *resp = cmd;
 133
 134         __ism_write_cmd(ism, req + 1, sizeof(*req), req->len - sizeof(*req));
 135         __ism_write_cmd(ism, req, 0, sizeof(*req));
 136
 137         WRITE_ONCE(resp->ret, ISM_ERROR);
 138
 139         __ism_read_cmd(ism, resp, 0, sizeof(*resp));
 140         if (resp->ret) {
 141                 debug_text_event(ism_debug_info, 0, "cmd failure");
 142                 debug_event(ism_debug_info, 0, resp, sizeof(*resp));
 143                 goto out;
 144         }
 145         __ism_read_cmd(ism, resp + 1, sizeof(*resp), resp->len - sizeof(*resp));
 146 out:
 147         return resp->ret;
 148 }
 149
 150 static int ism_cmd_simple(struct ism_dev *ism, u32 cmd_code)
 151 {
 152         union ism_cmd_simple cmd;
 153
 154         memset(&cmd, 0, sizeof(cmd));
 155         cmd.request.hdr.cmd = cmd_code;
 156         cmd.request.hdr.len = sizeof(cmd.request);
 157
 158         return ism_cmd(ism, &cmd);
 159 }
 160
 161 static int query_info(struct ism_dev *ism)
 162 {
 163         union ism_qi cmd;
 164
 165         memset(&cmd, 0, sizeof(cmd));
 166         cmd.request.hdr.cmd = ISM_QUERY_INFO;
 167         cmd.request.hdr.len = sizeof(cmd.request);
 168
 169         if (ism_cmd(ism, &cmd))
 170                 goto out;
 171
 172         debug_text_event(ism_debug_info, 3, "query info");
 173         debug_event(ism_debug_info, 3, &cmd.response, sizeof(cmd.response));
 174 out:
 175         return 0;
 176 }
 177
 178 static int register_sba(struct ism_dev *ism)
 179 {
 180         union ism_reg_sba cmd;
 181         dma_addr_t dma_handle;
 182         struct ism_sba *sba;
 183
 184         sba = dma_alloc_coherent(&ism->pdev->dev, PAGE_SIZE, &dma_handle,
 185                                  GFP_KERNEL);
 186         if (!sba)
 187                 return -ENOMEM;
 188
 189         memset(&cmd, 0, sizeof(cmd));
 190         cmd.request.hdr.cmd = ISM_REG_SBA;
 191         cmd.request.hdr.len = sizeof(cmd.request);
 192         cmd.request.sba = dma_handle;
 193
 194         if (ism_cmd(ism, &cmd)) {
 195                 dma_free_coherent(&ism->pdev->dev, PAGE_SIZE, sba, dma_handle);
 196                 return -EIO;
 197         }
 198
 199         ism->sba = sba;
 200         ism->sba_dma_addr = dma_handle;
 201
 202         return 0;
 203 }
 204
 205 static int register_ieq(struct ism_dev *ism)
 206 {
 207         union ism_reg_ieq cmd;
 208         dma_addr_t dma_handle;
 209         struct ism_eq *ieq;
 210
 211         ieq = dma_alloc_coherent(&ism->pdev->dev, PAGE_SIZE, &dma_handle,
 212                                  GFP_KERNEL);
 213         if (!ieq)
 214                 return -ENOMEM;
 215
 216         memset(&cmd, 0, sizeof(cmd));
 217         cmd.request.hdr.cmd = ISM_REG_IEQ;
 218         cmd.request.hdr.len = sizeof(cmd.request);
 219         cmd.request.ieq = dma_handle;
 220         cmd.request.len = sizeof(*ieq);
 221
 222         if (ism_cmd(ism, &cmd)) {
 223                 dma_free_coherent(&ism->pdev->dev, PAGE_SIZE, ieq, dma_handle);
 224                 return -EIO;
 225         }
 226
 227         ism->ieq = ieq;
 228         ism->ieq_idx = -1;
 229         ism->ieq_dma_addr = dma_handle;
 230
 231         return 0;
 232 }
 233
 234 static int unregister_sba(struct ism_dev *ism)
 235 {
 236         int ret;
 237
 238         if (!ism->sba)
 239                 return 0;
 240
 241         ret = ism_cmd_simple(ism, ISM_UNREG_SBA);
 242         if (ret && ret != ISM_ERROR)
 243                 return -EIO;
 244
 245         dma_free_coherent(&ism->pdev->dev, PAGE_SIZE,
 246                           ism->sba, ism->sba_dma_addr);
 247
 248         ism->sba = NULL;
 249         ism->sba_dma_addr = 0;
 250
 251         return 0;
 252 }
 253
 254 static int unregister_ieq(struct ism_dev *ism)
 255 {
 256         int ret;
 257
 258         if (!ism->ieq)
 259                 return 0;
 260
 261         ret = ism_cmd_simple(ism, ISM_UNREG_IEQ);
 262         if (ret && ret != ISM_ERROR)
 263                 return -EIO;
 264
 265         dma_free_coherent(&ism->pdev->dev, PAGE_SIZE,
 266                           ism->ieq, ism->ieq_dma_addr);
 267
 268         ism->ieq = NULL;
 269         ism->ieq_dma_addr = 0;
 270
 271         return 0;
 272 }
 273
 274 static int ism_read_local_gid(struct ism_dev *ism)
 275 {
 276         union ism_read_gid cmd;
 277         int ret;
 278
 279         memset(&cmd, 0, sizeof(cmd));
 280         cmd.request.hdr.cmd = ISM_READ_GID;
 281         cmd.request.hdr.len = sizeof(cmd.request);
 282
 283         ret = ism_cmd(ism, &cmd);
 284         if (ret)
 285                 goto out;
 286
 287         ism->local_gid = cmd.response.gid;
 288 out:
 289         return ret;
 290 }
 291
 292 static void ism_free_dmb(struct ism_dev *ism, struct ism_dmb *dmb)
 293 {
 294         clear_bit(dmb->sba_idx, ism->sba_bitmap);
 295         dma_unmap_page(&ism->pdev->dev, dmb->dma_addr, dmb->dmb_len,
 296                        DMA_FROM_DEVICE);
 297         folio_put(virt_to_folio(dmb->cpu_addr));
 298 }
 299
 300 static int ism_alloc_dmb(struct ism_dev *ism, struct ism_dmb *dmb)
 301 {
 302         struct folio *folio;
 303         unsigned long bit;
 304         int rc;
 305
 306         if (PAGE_ALIGN(dmb->dmb_len) > dma_get_max_seg_size(&ism->pdev->dev))
 307                 return -EINVAL;
 308
 309         if (!dmb->sba_idx) {
 310                 bit = find_next_zero_bit(ism->sba_bitmap, ISM_NR_DMBS,
 311                                          ISM_DMB_BIT_OFFSET);
 312                 if (bit == ISM_NR_DMBS)
 313                         return -ENOSPC;
 314
 315                 dmb->sba_idx = bit;
 316         }
 317         if (dmb->sba_idx < ISM_DMB_BIT_OFFSET ||
 318             test_and_set_bit(dmb->sba_idx, ism->sba_bitmap))
 319                 return -EINVAL;
 320
 321         folio = folio_alloc(GFP_KERNEL | __GFP_NOWARN | __GFP_NOMEMALLOC |
 322                             __GFP_NORETRY, get_order(dmb->dmb_len));
 323
 324         if (!folio) {
 325                 rc = -ENOMEM;
 326                 goto out_bit;
 327         }
 328
 329         dmb->cpu_addr = folio_address(folio);
 330         dmb->dma_addr = dma_map_page(&ism->pdev->dev,
 331                                      virt_to_page(dmb->cpu_addr), 0,
 332                                      dmb->dmb_len, DMA_FROM_DEVICE);
 333         if (dma_mapping_error(&ism->pdev->dev, dmb->dma_addr)) {
 334                 rc = -ENOMEM;
 335                 goto out_free;
 336         }
 337
 338         return 0;
 339
 340 out_free:
 341         kfree(dmb->cpu_addr);
 342 out_bit:
 343         clear_bit(dmb->sba_idx, ism->sba_bitmap);
 344         return rc;
 345 }
 346
 347 int ism_register_dmb(struct ism_dev *ism, struct ism_dmb *dmb,
 348                      struct ism_client *client)
 349 {
 350         union ism_reg_dmb cmd;
 351         unsigned long flags;
 352         int ret;
 353
 354         ret = ism_alloc_dmb(ism, dmb);
 355         if (ret)
 356                 goto out;
 357
 358         memset(&cmd, 0, sizeof(cmd));
 359         cmd.request.hdr.cmd = ISM_REG_DMB;
 360         cmd.request.hdr.len = sizeof(cmd.request);
 361
 362         cmd.request.dmb = dmb->dma_addr;
 363         cmd.request.dmb_len = dmb->dmb_len;
 364         cmd.request.sba_idx = dmb->sba_idx;
 365         cmd.request.vlan_valid = dmb->vlan_valid;
 366         cmd.request.vlan_id = dmb->vlan_id;
 367         cmd.request.rgid = dmb->rgid;
 368
 369         ret = ism_cmd(ism, &cmd);
 370         if (ret) {
 371                 ism_free_dmb(ism, dmb);
 372                 goto out;
 373         }
 374         dmb->dmb_tok = cmd.response.dmb_tok;
 375         spin_lock_irqsave(&ism->lock, flags);
 376         ism->sba_client_arr[dmb->sba_idx - ISM_DMB_BIT_OFFSET] = client->id;
 377         spin_unlock_irqrestore(&ism->lock, flags);
 378 out:
 379         return ret;
 380 }
 381 EXPORT_SYMBOL_GPL(ism_register_dmb);
 382
 383 int ism_unregister_dmb(struct ism_dev *ism, struct ism_dmb *dmb)
 384 {
 385         union ism_unreg_dmb cmd;
 386         unsigned long flags;
 387         int ret;
 388
 389         memset(&cmd, 0, sizeof(cmd));
 390         cmd.request.hdr.cmd = ISM_UNREG_DMB;
 391         cmd.request.hdr.len = sizeof(cmd.request);
 392
 393         cmd.request.dmb_tok = dmb->dmb_tok;
 394
 395         spin_lock_irqsave(&ism->lock, flags);
 396         ism->sba_client_arr[dmb->sba_idx - ISM_DMB_BIT_OFFSET] = NO_CLIENT;
 397         spin_unlock_irqrestore(&ism->lock, flags);
 398
 399         ret = ism_cmd(ism, &cmd);
 400         if (ret && ret != ISM_ERROR)
 401                 goto out;
 402
 403         ism_free_dmb(ism, dmb);
 404 out:
 405         return ret;
 406 }
 407 EXPORT_SYMBOL_GPL(ism_unregister_dmb);
 408
 409 static int ism_add_vlan_id(struct ism_dev *ism, u64 vlan_id)
 410 {
 411         union ism_set_vlan_id cmd;
 412
 413         memset(&cmd, 0, sizeof(cmd));
 414         cmd.request.hdr.cmd = ISM_ADD_VLAN_ID;
 415         cmd.request.hdr.len = sizeof(cmd.request);
 416
 417         cmd.request.vlan_id = vlan_id;
 418
 419         return ism_cmd(ism, &cmd);
 420 }
 421
 422 static int ism_del_vlan_id(struct ism_dev *ism, u64 vlan_id)
 423 {
 424         union ism_set_vlan_id cmd;
 425
 426         memset(&cmd, 0, sizeof(cmd));
 427         cmd.request.hdr.cmd = ISM_DEL_VLAN_ID;
 428         cmd.request.hdr.len = sizeof(cmd.request);
 429
 430         cmd.request.vlan_id = vlan_id;
 431
 432         return ism_cmd(ism, &cmd);
 433 }
 434
 435 static unsigned int max_bytes(unsigned int start, unsigned int len,
 436                               unsigned int boundary)
 437 {
 438         return min(boundary - (start & (boundary - 1)), len);
 439 }
 440
 441 int ism_move(struct ism_dev *ism, u64 dmb_tok, unsigned int idx, bool sf,
 442              unsigned int offset, void *data, unsigned int size)
 443 {
 444         unsigned int bytes;
 445         u64 dmb_req;
 446         int ret;
 447
 448         while (size) {
 449                 bytes = max_bytes(offset, size, PAGE_SIZE);
 450                 dmb_req = ISM_CREATE_REQ(dmb_tok, idx, size == bytes ? sf : 0,
 451                                          offset);
 452
 453                 ret = __ism_move(ism, dmb_req, data, bytes);
 454                 if (ret)
 455                         return ret;
 456
 457                 size -= bytes;
 458                 data += bytes;
 459                 offset += bytes;
 460         }
 461
 462         return 0;
 463 }
 464 EXPORT_SYMBOL_GPL(ism_move);
 465
 466 static void ism_handle_event(struct ism_dev *ism)
 467 {
 468         struct ism_event *entry;
 469         struct ism_client *clt;
 470         int i;
 471
 472         while ((ism->ieq_idx + 1) != READ_ONCE(ism->ieq->header.idx)) {
 473                 if (++(ism->ieq_idx) == ARRAY_SIZE(ism->ieq->entry))
 474                         ism->ieq_idx = 0;
 475
 476                 entry = &ism->ieq->entry[ism->ieq_idx];
 477                 debug_event(ism_debug_info, 2, entry, sizeof(*entry));
 478                 for (i = 0; i < max_client; ++i) {
 479                         clt = ism->subs[i];
 480                         if (clt)
 481                                 clt->handle_event(ism, entry);
 482                 }
 483         }
 484 }
 485
 486 static irqreturn_t ism_handle_irq(int irq, void *data)
 487 {
 488         struct ism_dev *ism = data;
 489         unsigned long bit, end;
 490         unsigned long *bv;
 491         u16 dmbemask;
 492         u8 client_id;
 493
 494         bv = (void *) &ism->sba->dmb_bits[ISM_DMB_WORD_OFFSET];
 495         end = sizeof(ism->sba->dmb_bits) * BITS_PER_BYTE - ISM_DMB_BIT_OFFSET;
 496
 497         spin_lock(&ism->lock);
 498         ism->sba->s = 0;
 499         barrier();
 500         for (bit = 0;;) {
 501                 bit = find_next_bit_inv(bv, end, bit);
 502                 if (bit >= end)
 503                         break;
 504
 505                 clear_bit_inv(bit, bv);
 506                 dmbemask = ism->sba->dmbe_mask[bit + ISM_DMB_BIT_OFFSET];
 507                 ism->sba->dmbe_mask[bit + ISM_DMB_BIT_OFFSET] = 0;
 508                 barrier();
 509                 client_id = ism->sba_client_arr[bit];
 510                 if (unlikely(client_id == NO_CLIENT || !ism->subs[client_id]))
 511                         continue;
 512                 ism->subs[client_id]->handle_irq(ism, bit + ISM_DMB_BIT_OFFSET, dmbemask);
 513         }
 514
 515         if (ism->sba->e) {
 516                 ism->sba->e = 0;
 517                 barrier();
 518                 ism_handle_event(ism);
 519         }
 520         spin_unlock(&ism->lock);
 521         return IRQ_HANDLED;
 522 }
 523
 524 static int ism_dev_init(struct ism_dev *ism)
 525 {
 526         struct pci_dev *pdev = ism->pdev;
 527         int i, ret;
 528
 529         ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
 530         if (ret <= 0)
 531                 goto out;
 532
 533         ism->sba_client_arr = kzalloc(ISM_NR_DMBS, GFP_KERNEL);
 534         if (!ism->sba_client_arr)
 535                 goto free_vectors;
 536         memset(ism->sba_client_arr, NO_CLIENT, ISM_NR_DMBS);
 537
 538         ret = request_irq(pci_irq_vector(pdev, 0), ism_handle_irq, 0,
 539                           pci_name(pdev), ism);
 540         if (ret)
 541                 goto free_client_arr;
 542
 543         ret = register_sba(ism);
 544         if (ret)
 545                 goto free_irq;
 546
 547         ret = register_ieq(ism);
 548         if (ret)
 549                 goto unreg_sba;
 550
 551         ret = ism_read_local_gid(ism);
 552         if (ret)
 553                 goto unreg_ieq;
 554
 555         if (!ism_add_vlan_id(ism, ISM_RESERVED_VLANID))
 556                 /* hardware is V2 capable */
 557                 ism_v2_capable = true;
 558         else
 559                 ism_v2_capable = false;
 560
 561         mutex_lock(&ism_dev_list.mutex);
 562         mutex_lock(&clients_lock);
 563         for (i = 0; i < max_client; ++i) {
 564                 if (clients[i]) {
 565                         clients[i]->add(ism);
 566                         ism_setup_forwarding(clients[i], ism);
 567                 }
 568         }
 569         mutex_unlock(&clients_lock);
 570
 571         list_add(&ism->list, &ism_dev_list.list);
 572         mutex_unlock(&ism_dev_list.mutex);
 573
 574         query_info(ism);
 575         return 0;
 576
 577 unreg_ieq:
 578         unregister_ieq(ism);
 579 unreg_sba:
 580         unregister_sba(ism);
 581 free_irq:
 582         free_irq(pci_irq_vector(pdev, 0), ism);
 583 free_client_arr:
 584         kfree(ism->sba_client_arr);
 585 free_vectors:
 586         pci_free_irq_vectors(pdev);
 587 out:
 588         return ret;
 589 }
 590
 591 static int ism_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 592 {
 593         struct ism_dev *ism;
 594         int ret;
 595
 596         ism = kzalloc(sizeof(*ism), GFP_KERNEL);
 597         if (!ism)
 598                 return -ENOMEM;
 599
 600         spin_lock_init(&ism->lock);
 601         dev_set_drvdata(&pdev->dev, ism);
 602         ism->pdev = pdev;
 603         ism->dev.parent = &pdev->dev;
 604         device_initialize(&ism->dev);
 605         dev_set_name(&ism->dev, dev_name(&pdev->dev));
 606         ret = device_add(&ism->dev);
 607         if (ret)
 608                 goto err_dev;
 609
 610         ret = pci_enable_device_mem(pdev);
 611         if (ret)
 612                 goto err;
 613
 614         ret = pci_request_mem_regions(pdev, DRV_NAME);
 615         if (ret)
 616                 goto err_disable;
 617
 618         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 619         if (ret)
 620                 goto err_resource;
 621
 622         dma_set_seg_boundary(&pdev->dev, SZ_1M - 1);
 623         dma_set_max_seg_size(&pdev->dev, SZ_1M);
 624         pci_set_master(pdev);
 625
 626         ret = ism_dev_init(ism);
 627         if (ret)
 628                 goto err_resource;
 629
 630         return 0;
 631
 632 err_resource:
 633         pci_release_mem_regions(pdev);
 634 err_disable:
 635         pci_disable_device(pdev);
 636 err:
 637         device_del(&ism->dev);
 638 err_dev:
 639         dev_set_drvdata(&pdev->dev, NULL);
 640         kfree(ism);
 641
 642         return ret;
 643 }
 644
 645 static void ism_dev_exit(struct ism_dev *ism)
 646 {
 647         struct pci_dev *pdev = ism->pdev;
 648         unsigned long flags;
 649         int i;
 650
 651         spin_lock_irqsave(&ism->lock, flags);
 652         for (i = 0; i < max_client; ++i)
 653                 ism->subs[i] = NULL;
 654         spin_unlock_irqrestore(&ism->lock, flags);
 655
 656         mutex_lock(&ism_dev_list.mutex);
 657         mutex_lock(&clients_lock);
 658         for (i = 0; i < max_client; ++i) {
 659                 if (clients[i])
 660                         clients[i]->remove(ism);
 661         }
 662         mutex_unlock(&clients_lock);
 663
 664         if (ism_v2_capable)
 665                 ism_del_vlan_id(ism, ISM_RESERVED_VLANID);
 666         unregister_ieq(ism);
 667         unregister_sba(ism);
 668         free_irq(pci_irq_vector(pdev, 0), ism);
 669         kfree(ism->sba_client_arr);
 670         pci_free_irq_vectors(pdev);
 671         list_del_init(&ism->list);
 672         mutex_unlock(&ism_dev_list.mutex);
 673 }
 674
 675 static void ism_remove(struct pci_dev *pdev)
 676 {
 677         struct ism_dev *ism = dev_get_drvdata(&pdev->dev);
 678
 679         ism_dev_exit(ism);
 680
 681         pci_release_mem_regions(pdev);
 682         pci_disable_device(pdev);
 683         device_del(&ism->dev);
 684         dev_set_drvdata(&pdev->dev, NULL);
 685         kfree(ism);
 686 }
 687
 688 static struct pci_driver ism_driver = {
 689         .name     = DRV_NAME,
 690         .id_table = ism_device_table,
 691         .probe    = ism_probe,
 692         .remove   = ism_remove,
 693 };
 694
 695 static int __init ism_init(void)
 696 {
 697         int ret;
 698
 699         ism_debug_info = debug_register("ism", 2, 1, 16);
 700         if (!ism_debug_info)
 701                 return -ENODEV;
 702
 703         memset(clients, 0, sizeof(clients));
 704         max_client = 0;
 705         debug_register_view(ism_debug_info, &debug_hex_ascii_view);
 706         ret = pci_register_driver(&ism_driver);
 707         if (ret)
 708                 debug_unregister(ism_debug_info);
 709
 710         return ret;
 711 }
 712
 713 static void __exit ism_exit(void)
 714 {
 715         pci_unregister_driver(&ism_driver);
 716         debug_unregister(ism_debug_info);
 717 }
 718
 719 module_init(ism_init);
 720 module_exit(ism_exit);
 721
 722 /*************************** SMC-D Implementation *****************************/
 723
 724 #if IS_ENABLED(CONFIG_SMC)
 725 static int ism_query_rgid(struct ism_dev *ism, u64 rgid, u32 vid_valid,
 726                           u32 vid)
 727 {
 728         union ism_query_rgid cmd;
 729
 730         memset(&cmd, 0, sizeof(cmd));
 731         cmd.request.hdr.cmd = ISM_QUERY_RGID;
 732         cmd.request.hdr.len = sizeof(cmd.request);
 733
 734         cmd.request.rgid = rgid;
 735         cmd.request.vlan_valid = vid_valid;
 736         cmd.request.vlan_id = vid;
 737
 738         return ism_cmd(ism, &cmd);
 739 }
 740
 741 static int smcd_query_rgid(struct smcd_dev *smcd, struct smcd_gid *rgid,
 742                            u32 vid_valid, u32 vid)
 743 {
 744         return ism_query_rgid(smcd->priv, rgid->gid, vid_valid, vid);
 745 }
 746
 747 static int smcd_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
 748                              void *client)
 749 {
 750         return ism_register_dmb(smcd->priv, (struct ism_dmb *)dmb, client);
 751 }
 752
 753 static int smcd_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
 754 {
 755         return ism_unregister_dmb(smcd->priv, (struct ism_dmb *)dmb);
 756 }
 757
 758 static int smcd_add_vlan_id(struct smcd_dev *smcd, u64 vlan_id)
 759 {
 760         return ism_add_vlan_id(smcd->priv, vlan_id);
 761 }
 762
 763 static int smcd_del_vlan_id(struct smcd_dev *smcd, u64 vlan_id)
 764 {
 765         return ism_del_vlan_id(smcd->priv, vlan_id);
 766 }
 767
 768 static int smcd_set_vlan_required(struct smcd_dev *smcd)
 769 {
 770         return ism_cmd_simple(smcd->priv, ISM_SET_VLAN);
 771 }
 772
 773 static int smcd_reset_vlan_required(struct smcd_dev *smcd)
 774 {
 775         return ism_cmd_simple(smcd->priv, ISM_RESET_VLAN);
 776 }
 777
 778 static int ism_signal_ieq(struct ism_dev *ism, u64 rgid, u32 trigger_irq,
 779                           u32 event_code, u64 info)
 780 {
 781         union ism_sig_ieq cmd;
 782
 783         memset(&cmd, 0, sizeof(cmd));
 784         cmd.request.hdr.cmd = ISM_SIGNAL_IEQ;
 785         cmd.request.hdr.len = sizeof(cmd.request);
 786
 787         cmd.request.rgid = rgid;
 788         cmd.request.trigger_irq = trigger_irq;
 789         cmd.request.event_code = event_code;
 790         cmd.request.info = info;
 791
 792         return ism_cmd(ism, &cmd);
 793 }
 794
 795 static int smcd_signal_ieq(struct smcd_dev *smcd, struct smcd_gid *rgid,
 796                            u32 trigger_irq, u32 event_code, u64 info)
 797 {
 798         return ism_signal_ieq(smcd->priv, rgid->gid,
 799                               trigger_irq, event_code, info);
 800 }
 801
 802 static int smcd_move(struct smcd_dev *smcd, u64 dmb_tok, unsigned int idx,
 803                      bool sf, unsigned int offset, void *data,
 804                      unsigned int size)
 805 {
 806         return ism_move(smcd->priv, dmb_tok, idx, sf, offset, data, size);
 807 }
 808
 809 static int smcd_supports_v2(void)
 810 {
 811         return ism_v2_capable;
 812 }
 813
 814 static u64 ism_get_local_gid(struct ism_dev *ism)
 815 {
 816         return ism->local_gid;
 817 }
 818
 819 static void smcd_get_local_gid(struct smcd_dev *smcd,
 820                                struct smcd_gid *smcd_gid)
 821 {
 822         smcd_gid->gid = ism_get_local_gid(smcd->priv);
 823         smcd_gid->gid_ext = 0;
 824 }
 825
 826 static u16 ism_get_chid(struct ism_dev *ism)
 827 {
 828         if (!ism || !ism->pdev)
 829                 return 0;
 830
 831         return to_zpci(ism->pdev)->pchid;
 832 }
 833
 834 static u16 smcd_get_chid(struct smcd_dev *smcd)
 835 {
 836         return ism_get_chid(smcd->priv);
 837 }
 838
 839 static inline struct device *smcd_get_dev(struct smcd_dev *dev)
 840 {
 841         struct ism_dev *ism = dev->priv;
 842
 843         return &ism->dev;
 844 }
 845
 846 static const struct smcd_ops ism_ops = {
 847         .query_remote_gid = smcd_query_rgid,
 848         .register_dmb = smcd_register_dmb,
 849         .unregister_dmb = smcd_unregister_dmb,
 850         .add_vlan_id = smcd_add_vlan_id,
 851         .del_vlan_id = smcd_del_vlan_id,
 852         .set_vlan_required = smcd_set_vlan_required,
 853         .reset_vlan_required = smcd_reset_vlan_required,
 854         .signal_event = smcd_signal_ieq,
 855         .move_data = smcd_move,
 856         .supports_v2 = smcd_supports_v2,
 857         .get_local_gid = smcd_get_local_gid,
 858         .get_chid = smcd_get_chid,
 859         .get_dev = smcd_get_dev,
 860 };
 861
 862 const struct smcd_ops *ism_get_smcd_ops(void)
 863 {
 864         return &ism_ops;
 865 }
 866 EXPORT_SYMBOL_GPL(ism_get_smcd_ops);
 867 #endif