drivers/misc/genwqe/card_dev.c

   1 /**
   2  * IBM Accelerator Family 'GenWQE'
   3  *
   4  * (C) Copyright IBM Corp. 2013
   5  *
   6  * Author: Frank Haverkamp <[email protected]>
   7  * Author: Joerg-Stephan Vogt <[email protected]>
   8  * Author: Michael Jung <[email protected]>
   9  * Author: Michael Ruettger <[email protected]>
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License (version 2 only)
  13  * as published by the Free Software Foundation.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18  * GNU General Public License for more details.
  19  */
  20
  21 /*
  22  * Character device representation of the GenWQE device. This allows
  23  * user-space applications to communicate with the card.
  24  */
  25
  26 #include <linux/kernel.h>
  27 #include <linux/types.h>
  28 #include <linux/module.h>
  29 #include <linux/pci.h>
  30 #include <linux/string.h>
  31 #include <linux/fs.h>
  32 #include <linux/sched/signal.h>
  33 #include <linux/wait.h>
  34 #include <linux/delay.h>
  35 #include <linux/atomic.h>
  36
  37 #include "card_base.h"
  38 #include "card_ddcb.h"
  39
  40 static int genwqe_open_files(struct genwqe_dev *cd)
  41 {
  42         int rc;
  43         unsigned long flags;
  44
  45         spin_lock_irqsave(&cd->file_lock, flags);
  46         rc = list_empty(&cd->file_list);
  47         spin_unlock_irqrestore(&cd->file_lock, flags);
  48         return !rc;
  49 }
  50
  51 static void genwqe_add_file(struct genwqe_dev *cd, struct genwqe_file *cfile)
  52 {
  53         unsigned long flags;
  54
  55         cfile->owner = current;
  56         spin_lock_irqsave(&cd->file_lock, flags);
  57         list_add(&cfile->list, &cd->file_list);
  58         spin_unlock_irqrestore(&cd->file_lock, flags);
  59 }
  60
  61 static int genwqe_del_file(struct genwqe_dev *cd, struct genwqe_file *cfile)
  62 {
  63         unsigned long flags;
  64
  65         spin_lock_irqsave(&cd->file_lock, flags);
  66         list_del(&cfile->list);
  67         spin_unlock_irqrestore(&cd->file_lock, flags);
  68
  69         return 0;
  70 }
  71
  72 static void genwqe_add_pin(struct genwqe_file *cfile, struct dma_mapping *m)
  73 {
  74         unsigned long flags;
  75
  76         spin_lock_irqsave(&cfile->pin_lock, flags);
  77         list_add(&m->pin_list, &cfile->pin_list);
  78         spin_unlock_irqrestore(&cfile->pin_lock, flags);
  79 }
  80
  81 static int genwqe_del_pin(struct genwqe_file *cfile, struct dma_mapping *m)
  82 {
  83         unsigned long flags;
  84
  85         spin_lock_irqsave(&cfile->pin_lock, flags);
  86         list_del(&m->pin_list);
  87         spin_unlock_irqrestore(&cfile->pin_lock, flags);
  88
  89         return 0;
  90 }
  91
  92 /**
  93  * genwqe_search_pin() - Search for the mapping for a userspace address
  94  * @cfile:      Descriptor of opened file
  95  * @u_addr:     User virtual address
  96  * @size:       Size of buffer
  97  * @dma_addr:   DMA address to be updated
  98  *
  99  * Return: Pointer to the corresponding mapping NULL if not found
 100  */
 101 static struct dma_mapping *genwqe_search_pin(struct genwqe_file *cfile,
 102                                             unsigned long u_addr,
 103                                             unsigned int size,
 104                                             void **virt_addr)
 105 {
 106         unsigned long flags;
 107         struct dma_mapping *m;
 108
 109         spin_lock_irqsave(&cfile->pin_lock, flags);
 110
 111         list_for_each_entry(m, &cfile->pin_list, pin_list) {
 112                 if ((((u64)m->u_vaddr) <= (u_addr)) &&
 113                     (((u64)m->u_vaddr + m->size) >= (u_addr + size))) {
 114
 115                         if (virt_addr)
 116                                 *virt_addr = m->k_vaddr +
 117                                         (u_addr - (u64)m->u_vaddr);
 118
 119                         spin_unlock_irqrestore(&cfile->pin_lock, flags);
 120                         return m;
 121                 }
 122         }
 123         spin_unlock_irqrestore(&cfile->pin_lock, flags);
 124         return NULL;
 125 }
 126
 127 static void __genwqe_add_mapping(struct genwqe_file *cfile,
 128                               struct dma_mapping *dma_map)
 129 {
 130         unsigned long flags;
 131
 132         spin_lock_irqsave(&cfile->map_lock, flags);
 133         list_add(&dma_map->card_list, &cfile->map_list);
 134         spin_unlock_irqrestore(&cfile->map_lock, flags);
 135 }
 136
 137 static void __genwqe_del_mapping(struct genwqe_file *cfile,
 138                               struct dma_mapping *dma_map)
 139 {
 140         unsigned long flags;
 141
 142         spin_lock_irqsave(&cfile->map_lock, flags);
 143         list_del(&dma_map->card_list);
 144         spin_unlock_irqrestore(&cfile->map_lock, flags);
 145 }
 146
 147
 148 /**
 149  * __genwqe_search_mapping() - Search for the mapping for a userspace address
 150  * @cfile:      descriptor of opened file
 151  * @u_addr:     user virtual address
 152  * @size:       size of buffer
 153  * @dma_addr:   DMA address to be updated
 154  * Return: Pointer to the corresponding mapping NULL if not found
 155  */
 156 static struct dma_mapping *__genwqe_search_mapping(struct genwqe_file *cfile,
 157                                                    unsigned long u_addr,
 158                                                    unsigned int size,
 159                                                    dma_addr_t *dma_addr,
 160                                                    void **virt_addr)
 161 {
 162         unsigned long flags;
 163         struct dma_mapping *m;
 164         struct pci_dev *pci_dev = cfile->cd->pci_dev;
 165
 166         spin_lock_irqsave(&cfile->map_lock, flags);
 167         list_for_each_entry(m, &cfile->map_list, card_list) {
 168
 169                 if ((((u64)m->u_vaddr) <= (u_addr)) &&
 170                     (((u64)m->u_vaddr + m->size) >= (u_addr + size))) {
 171
 172                         /* match found: current is as expected and
 173                            addr is in range */
 174                         if (dma_addr)
 175                                 *dma_addr = m->dma_addr +
 176                                         (u_addr - (u64)m->u_vaddr);
 177
 178                         if (virt_addr)
 179                                 *virt_addr = m->k_vaddr +
 180                                         (u_addr - (u64)m->u_vaddr);
 181
 182                         spin_unlock_irqrestore(&cfile->map_lock, flags);
 183                         return m;
 184                 }
 185         }
 186         spin_unlock_irqrestore(&cfile->map_lock, flags);
 187
 188         dev_err(&pci_dev->dev,
 189                 "[%s] Entry not found: u_addr=%lx, size=%x\n",
 190                 __func__, u_addr, size);
 191
 192         return NULL;
 193 }
 194
 195 static void genwqe_remove_mappings(struct genwqe_file *cfile)
 196 {
 197         int i = 0;
 198         struct list_head *node, *next;
 199         struct dma_mapping *dma_map;
 200         struct genwqe_dev *cd = cfile->cd;
 201         struct pci_dev *pci_dev = cfile->cd->pci_dev;
 202
 203         list_for_each_safe(node, next, &cfile->map_list) {
 204                 dma_map = list_entry(node, struct dma_mapping, card_list);
 205
 206                 list_del_init(&dma_map->card_list);
 207
 208                 /*
 209                  * This is really a bug, because those things should
 210                  * have been already tidied up.
 211                  *
 212                  * GENWQE_MAPPING_RAW should have been removed via mmunmap().
 213                  * GENWQE_MAPPING_SGL_TEMP should be removed by tidy up code.
 214                  */
 215                 dev_err(&pci_dev->dev,
 216                         "[%s] %d. cleanup mapping: u_vaddr=%p u_kaddr=%016lx dma_addr=%lx\n",
 217                         __func__, i++, dma_map->u_vaddr,
 218                         (unsigned long)dma_map->k_vaddr,
 219                         (unsigned long)dma_map->dma_addr);
 220
 221                 if (dma_map->type == GENWQE_MAPPING_RAW) {
 222                         /* we allocated this dynamically */
 223                         __genwqe_free_consistent(cd, dma_map->size,
 224                                                 dma_map->k_vaddr,
 225                                                 dma_map->dma_addr);
 226                         kfree(dma_map);
 227                 } else if (dma_map->type == GENWQE_MAPPING_SGL_TEMP) {
 228                         /* we use dma_map statically from the request */
 229                         genwqe_user_vunmap(cd, dma_map, NULL);
 230                 }
 231         }
 232 }
 233
 234 static void genwqe_remove_pinnings(struct genwqe_file *cfile)
 235 {
 236         struct list_head *node, *next;
 237         struct dma_mapping *dma_map;
 238         struct genwqe_dev *cd = cfile->cd;
 239
 240         list_for_each_safe(node, next, &cfile->pin_list) {
 241                 dma_map = list_entry(node, struct dma_mapping, pin_list);
 242
 243                 /*
 244                  * This is not a bug, because a killed processed might
 245                  * not call the unpin ioctl, which is supposed to free
 246                  * the resources.
 247                  *
 248                  * Pinnings are dymically allocated and need to be
 249                  * deleted.
 250                  */
 251                 list_del_init(&dma_map->pin_list);
 252                 genwqe_user_vunmap(cd, dma_map, NULL);
 253                 kfree(dma_map);
 254         }
 255 }
 256
 257 /**
 258  * genwqe_kill_fasync() - Send signal to all processes with open GenWQE files
 259  *
 260  * E.g. genwqe_send_signal(cd, SIGIO);
 261  */
 262 static int genwqe_kill_fasync(struct genwqe_dev *cd, int sig)
 263 {
 264         unsigned int files = 0;
 265         unsigned long flags;
 266         struct genwqe_file *cfile;
 267
 268         spin_lock_irqsave(&cd->file_lock, flags);
 269         list_for_each_entry(cfile, &cd->file_list, list) {
 270                 if (cfile->async_queue)
 271                         kill_fasync(&cfile->async_queue, sig, POLL_HUP);
 272                 files++;
 273         }
 274         spin_unlock_irqrestore(&cd->file_lock, flags);
 275         return files;
 276 }
 277
 278 static int genwqe_force_sig(struct genwqe_dev *cd, int sig)
 279 {
 280         unsigned int files = 0;
 281         unsigned long flags;
 282         struct genwqe_file *cfile;
 283
 284         spin_lock_irqsave(&cd->file_lock, flags);
 285         list_for_each_entry(cfile, &cd->file_list, list) {
 286                 force_sig(sig, cfile->owner);
 287                 files++;
 288         }
 289         spin_unlock_irqrestore(&cd->file_lock, flags);
 290         return files;
 291 }
 292
 293 /**
 294  * genwqe_open() - file open
 295  * @inode:      file system information
 296  * @filp:       file handle
 297  *
 298  * This function is executed whenever an application calls
 299  * open("/dev/genwqe",..).
 300  *
 301  * Return: 0 if successful or <0 if errors
 302  */
 303 static int genwqe_open(struct inode *inode, struct file *filp)
 304 {
 305         struct genwqe_dev *cd;
 306         struct genwqe_file *cfile;
 307         struct pci_dev *pci_dev;
 308
 309         cfile = kzalloc(sizeof(*cfile), GFP_KERNEL);
 310         if (cfile == NULL)
 311                 return -ENOMEM;
 312
 313         cd = container_of(inode->i_cdev, struct genwqe_dev, cdev_genwqe);
 314         pci_dev = cd->pci_dev;
 315         cfile->cd = cd;
 316         cfile->filp = filp;
 317         cfile->client = NULL;
 318
 319         spin_lock_init(&cfile->map_lock);  /* list of raw memory allocations */
 320         INIT_LIST_HEAD(&cfile->map_list);
 321
 322         spin_lock_init(&cfile->pin_lock);  /* list of user pinned memory */
 323         INIT_LIST_HEAD(&cfile->pin_list);
 324
 325         filp->private_data = cfile;
 326
 327         genwqe_add_file(cd, cfile);
 328         return 0;
 329 }
 330
 331 /**
 332  * genwqe_fasync() - Setup process to receive SIGIO.
 333  * @fd:        file descriptor
 334  * @filp:      file handle
 335  * @mode:      file mode
 336  *
 337  * Sending a signal is working as following:
 338  *
 339  * if (cdev->async_queue)
 340  *         kill_fasync(&cdev->async_queue, SIGIO, POLL_IN);
 341  *
 342  * Some devices also implement asynchronous notification to indicate
 343  * when the device can be written; in this case, of course,
 344  * kill_fasync must be called with a mode of POLL_OUT.
 345  */
 346 static int genwqe_fasync(int fd, struct file *filp, int mode)
 347 {
 348         struct genwqe_file *cdev = (struct genwqe_file *)filp->private_data;
 349
 350         return fasync_helper(fd, filp, mode, &cdev->async_queue);
 351 }
 352
 353
 354 /**
 355  * genwqe_release() - file close
 356  * @inode:      file system information
 357  * @filp:       file handle
 358  *
 359  * This function is executed whenever an application calls 'close(fd_genwqe)'
 360  *
 361  * Return: always 0
 362  */
 363 static int genwqe_release(struct inode *inode, struct file *filp)
 364 {
 365         struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;
 366         struct genwqe_dev *cd = cfile->cd;
 367
 368         /* there must be no entries in these lists! */
 369         genwqe_remove_mappings(cfile);
 370         genwqe_remove_pinnings(cfile);
 371
 372         /* remove this filp from the asynchronously notified filp's */
 373         genwqe_fasync(-1, filp, 0);
 374
 375         /*
 376          * For this to work we must not release cd when this cfile is
 377          * not yet released, otherwise the list entry is invalid,
 378          * because the list itself gets reinstantiated!
 379          */
 380         genwqe_del_file(cd, cfile);
 381         kfree(cfile);
 382         return 0;
 383 }
 384
 385 static void genwqe_vma_open(struct vm_area_struct *vma)
 386 {
 387         /* nothing ... */
 388 }
 389
 390 /**
 391  * genwqe_vma_close() - Called each time when vma is unmapped
 392  *
 393  * Free memory which got allocated by GenWQE mmap().
 394  */
 395 static void genwqe_vma_close(struct vm_area_struct *vma)
 396 {
 397         unsigned long vsize = vma->vm_end - vma->vm_start;
 398         struct inode *inode = file_inode(vma->vm_file);
 399         struct dma_mapping *dma_map;
 400         struct genwqe_dev *cd = container_of(inode->i_cdev, struct genwqe_dev,
 401                                             cdev_genwqe);
 402         struct pci_dev *pci_dev = cd->pci_dev;
 403         dma_addr_t d_addr = 0;
 404         struct genwqe_file *cfile = vma->vm_private_data;
 405
 406         dma_map = __genwqe_search_mapping(cfile, vma->vm_start, vsize,
 407                                          &d_addr, NULL);
 408         if (dma_map == NULL) {
 409                 dev_err(&pci_dev->dev,
 410                         "  [%s] err: mapping not found: v=%lx, p=%lx s=%lx\n",
 411                         __func__, vma->vm_start, vma->vm_pgoff << PAGE_SHIFT,
 412                         vsize);
 413                 return;
 414         }
 415         __genwqe_del_mapping(cfile, dma_map);
 416         __genwqe_free_consistent(cd, dma_map->size, dma_map->k_vaddr,
 417                                  dma_map->dma_addr);
 418         kfree(dma_map);
 419 }
 420
 421 static const struct vm_operations_struct genwqe_vma_ops = {
 422         .open   = genwqe_vma_open,
 423         .close  = genwqe_vma_close,
 424 };
 425
 426 /**
 427  * genwqe_mmap() - Provide contignous buffers to userspace
 428  *
 429  * We use mmap() to allocate contignous buffers used for DMA
 430  * transfers. After the buffer is allocated we remap it to user-space
 431  * and remember a reference to our dma_mapping data structure, where
 432  * we store the associated DMA address and allocated size.
 433  *
 434  * When we receive a DDCB execution request with the ATS bits set to
 435  * plain buffer, we lookup our dma_mapping list to find the
 436  * corresponding DMA address for the associated user-space address.
 437  */
 438 static int genwqe_mmap(struct file *filp, struct vm_area_struct *vma)
 439 {
 440         int rc;
 441         unsigned long pfn, vsize = vma->vm_end - vma->vm_start;
 442         struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;
 443         struct genwqe_dev *cd = cfile->cd;
 444         struct dma_mapping *dma_map;
 445
 446         if (vsize == 0)
 447                 return -EINVAL;
 448
 449         if (get_order(vsize) > MAX_ORDER)
 450                 return -ENOMEM;
 451
 452         dma_map = kzalloc(sizeof(struct dma_mapping), GFP_KERNEL);
 453         if (dma_map == NULL)
 454                 return -ENOMEM;
 455
 456         genwqe_mapping_init(dma_map, GENWQE_MAPPING_RAW);
 457         dma_map->u_vaddr = (void *)vma->vm_start;
 458         dma_map->size = vsize;
 459         dma_map->nr_pages = DIV_ROUND_UP(vsize, PAGE_SIZE);
 460         dma_map->k_vaddr = __genwqe_alloc_consistent(cd, vsize,
 461                                                      &dma_map->dma_addr);
 462         if (dma_map->k_vaddr == NULL) {
 463                 rc = -ENOMEM;
 464                 goto free_dma_map;
 465         }
 466
 467         if (capable(CAP_SYS_ADMIN) && (vsize > sizeof(dma_addr_t)))
 468                 *(dma_addr_t *)dma_map->k_vaddr = dma_map->dma_addr;
 469
 470         pfn = virt_to_phys(dma_map->k_vaddr) >> PAGE_SHIFT;
 471         rc = remap_pfn_range(vma,
 472                              vma->vm_start,
 473                              pfn,
 474                              vsize,
 475                              vma->vm_page_prot);
 476         if (rc != 0) {
 477                 rc = -EFAULT;
 478                 goto free_dma_mem;
 479         }
 480
 481         vma->vm_private_data = cfile;
 482         vma->vm_ops = &genwqe_vma_ops;
 483         __genwqe_add_mapping(cfile, dma_map);
 484
 485         return 0;
 486
 487  free_dma_mem:
 488         __genwqe_free_consistent(cd, dma_map->size,
 489                                 dma_map->k_vaddr,
 490                                 dma_map->dma_addr);
 491  free_dma_map:
 492         kfree(dma_map);
 493         return rc;
 494 }
 495
 496 /**
 497  * do_flash_update() - Excute flash update (write image or CVPD)
 498  * @cd:        genwqe device
 499  * @load:      details about image load
 500  *
 501  * Return: 0 if successful
 502  */
 503
 504 #define FLASH_BLOCK     0x40000 /* we use 256k blocks */
 505
 506 static int do_flash_update(struct genwqe_file *cfile,
 507                            struct genwqe_bitstream *load)
 508 {
 509         int rc = 0;
 510         int blocks_to_flash;
 511         dma_addr_t dma_addr;
 512         u64 flash = 0;
 513         size_t tocopy = 0;
 514         u8 __user *buf;
 515         u8 *xbuf;
 516         u32 crc;
 517         u8 cmdopts;
 518         struct genwqe_dev *cd = cfile->cd;
 519         struct file *filp = cfile->filp;
 520         struct pci_dev *pci_dev = cd->pci_dev;
 521
 522         if ((load->size & 0x3) != 0)
 523                 return -EINVAL;
 524
 525         if (((unsigned long)(load->data_addr) & ~PAGE_MASK) != 0)
 526                 return -EINVAL;
 527
 528         /* FIXME Bits have changed for new service layer! */
 529         switch ((char)load->partition) {
 530         case '0':
 531                 cmdopts = 0x14;
 532                 break;          /* download/erase_first/part_0 */
 533         case '1':
 534                 cmdopts = 0x1C;
 535                 break;          /* download/erase_first/part_1 */
 536         case 'v':
 537                 cmdopts = 0x0C;
 538                 break;          /* download/erase_first/vpd */
 539         default:
 540                 return -EINVAL;
 541         }
 542
 543         buf = (u8 __user *)load->data_addr;
 544         xbuf = __genwqe_alloc_consistent(cd, FLASH_BLOCK, &dma_addr);
 545         if (xbuf == NULL)
 546                 return -ENOMEM;
 547
 548         blocks_to_flash = load->size / FLASH_BLOCK;
 549         while (load->size) {
 550                 struct genwqe_ddcb_cmd *req;
 551
 552                 /*
 553                  * We must be 4 byte aligned. Buffer must be 0 appened
 554                  * to have defined values when calculating CRC.
 555                  */
 556                 tocopy = min_t(size_t, load->size, FLASH_BLOCK);
 557
 558                 rc = copy_from_user(xbuf, buf, tocopy);
 559                 if (rc) {
 560                         rc = -EFAULT;
 561                         goto free_buffer;
 562                 }
 563                 crc = genwqe_crc32(xbuf, tocopy, 0xffffffff);
 564
 565                 dev_dbg(&pci_dev->dev,
 566                         "[%s] DMA: %lx CRC: %08x SZ: %ld %d\n",
 567                         __func__, (unsigned long)dma_addr, crc, tocopy,
 568                         blocks_to_flash);
 569
 570                 /* prepare DDCB for SLU process */
 571                 req = ddcb_requ_alloc();
 572                 if (req == NULL) {
 573                         rc = -ENOMEM;
 574                         goto free_buffer;
 575                 }
 576
 577                 req->cmd = SLCMD_MOVE_FLASH;
 578                 req->cmdopts = cmdopts;
 579
 580                 /* prepare invariant values */
 581                 if (genwqe_get_slu_id(cd) <= 0x2) {
 582                         *(__be64 *)&req->__asiv[0]  = cpu_to_be64(dma_addr);
 583                         *(__be64 *)&req->__asiv[8]  = cpu_to_be64(tocopy);
 584                         *(__be64 *)&req->__asiv[16] = cpu_to_be64(flash);
 585                         *(__be32 *)&req->__asiv[24] = cpu_to_be32(0);
 586                         req->__asiv[24]        = load->uid;
 587                         *(__be32 *)&req->__asiv[28] = cpu_to_be32(crc);
 588
 589                         /* for simulation only */
 590                         *(__be64 *)&req->__asiv[88] = cpu_to_be64(load->slu_id);
 591                         *(__be64 *)&req->__asiv[96] = cpu_to_be64(load->app_id);
 592                         req->asiv_length = 32; /* bytes included in crc calc */
 593                 } else {        /* setup DDCB for ATS architecture */
 594                         *(__be64 *)&req->asiv[0]  = cpu_to_be64(dma_addr);
 595                         *(__be32 *)&req->asiv[8]  = cpu_to_be32(tocopy);
 596                         *(__be32 *)&req->asiv[12] = cpu_to_be32(0); /* resvd */
 597                         *(__be64 *)&req->asiv[16] = cpu_to_be64(flash);
 598                         *(__be32 *)&req->asiv[24] = cpu_to_be32(load->uid<<24);
 599                         *(__be32 *)&req->asiv[28] = cpu_to_be32(crc);
 600
 601                         /* for simulation only */
 602                         *(__be64 *)&req->asiv[80] = cpu_to_be64(load->slu_id);
 603                         *(__be64 *)&req->asiv[88] = cpu_to_be64(load->app_id);
 604
 605                         /* Rd only */
 606                         req->ats = 0x4ULL << 44;
 607                         req->asiv_length = 40; /* bytes included in crc calc */
 608                 }
 609                 req->asv_length  = 8;
 610
 611                 /* For Genwqe5 we get back the calculated CRC */
 612                 *(u64 *)&req->asv[0] = 0ULL;                    /* 0x80 */
 613
 614                 rc = __genwqe_execute_raw_ddcb(cd, req, filp->f_flags);
 615
 616                 load->retc = req->retc;
 617                 load->attn = req->attn;
 618                 load->progress = req->progress;
 619
 620                 if (rc < 0) {
 621                         ddcb_requ_free(req);
 622                         goto free_buffer;
 623                 }
 624
 625                 if (req->retc != DDCB_RETC_COMPLETE) {
 626                         rc = -EIO;
 627                         ddcb_requ_free(req);
 628                         goto free_buffer;
 629                 }
 630
 631                 load->size  -= tocopy;
 632                 flash += tocopy;
 633                 buf += tocopy;
 634                 blocks_to_flash--;
 635                 ddcb_requ_free(req);
 636         }
 637
 638  free_buffer:
 639         __genwqe_free_consistent(cd, FLASH_BLOCK, xbuf, dma_addr);
 640         return rc;
 641 }
 642
 643 static int do_flash_read(struct genwqe_file *cfile,
 644                          struct genwqe_bitstream *load)
 645 {
 646         int rc, blocks_to_flash;
 647         dma_addr_t dma_addr;
 648         u64 flash = 0;
 649         size_t tocopy = 0;
 650         u8 __user *buf;
 651         u8 *xbuf;
 652         u8 cmdopts;
 653         struct genwqe_dev *cd = cfile->cd;
 654         struct file *filp = cfile->filp;
 655         struct pci_dev *pci_dev = cd->pci_dev;
 656         struct genwqe_ddcb_cmd *cmd;
 657
 658         if ((load->size & 0x3) != 0)
 659                 return -EINVAL;
 660
 661         if (((unsigned long)(load->data_addr) & ~PAGE_MASK) != 0)
 662                 return -EINVAL;
 663
 664         /* FIXME Bits have changed for new service layer! */
 665         switch ((char)load->partition) {
 666         case '0':
 667                 cmdopts = 0x12;
 668                 break;          /* upload/part_0 */
 669         case '1':
 670                 cmdopts = 0x1A;
 671                 break;          /* upload/part_1 */
 672         case 'v':
 673                 cmdopts = 0x0A;
 674                 break;          /* upload/vpd */
 675         default:
 676                 return -EINVAL;
 677         }
 678
 679         buf = (u8 __user *)load->data_addr;
 680         xbuf = __genwqe_alloc_consistent(cd, FLASH_BLOCK, &dma_addr);
 681         if (xbuf == NULL)
 682                 return -ENOMEM;
 683
 684         blocks_to_flash = load->size / FLASH_BLOCK;
 685         while (load->size) {
 686                 /*
 687                  * We must be 4 byte aligned. Buffer must be 0 appened
 688                  * to have defined values when calculating CRC.
 689                  */
 690                 tocopy = min_t(size_t, load->size, FLASH_BLOCK);
 691
 692                 dev_dbg(&pci_dev->dev,
 693                         "[%s] DMA: %lx SZ: %ld %d\n",
 694                         __func__, (unsigned long)dma_addr, tocopy,
 695                         blocks_to_flash);
 696
 697                 /* prepare DDCB for SLU process */
 698                 cmd = ddcb_requ_alloc();
 699                 if (cmd == NULL) {
 700                         rc = -ENOMEM;
 701                         goto free_buffer;
 702                 }
 703                 cmd->cmd = SLCMD_MOVE_FLASH;
 704                 cmd->cmdopts = cmdopts;
 705
 706                 /* prepare invariant values */
 707                 if (genwqe_get_slu_id(cd) <= 0x2) {
 708                         *(__be64 *)&cmd->__asiv[0]  = cpu_to_be64(dma_addr);
 709                         *(__be64 *)&cmd->__asiv[8]  = cpu_to_be64(tocopy);
 710                         *(__be64 *)&cmd->__asiv[16] = cpu_to_be64(flash);
 711                         *(__be32 *)&cmd->__asiv[24] = cpu_to_be32(0);
 712                         cmd->__asiv[24] = load->uid;
 713                         *(__be32 *)&cmd->__asiv[28] = cpu_to_be32(0) /* CRC */;
 714                         cmd->asiv_length = 32; /* bytes included in crc calc */
 715                 } else {        /* setup DDCB for ATS architecture */
 716                         *(__be64 *)&cmd->asiv[0]  = cpu_to_be64(dma_addr);
 717                         *(__be32 *)&cmd->asiv[8]  = cpu_to_be32(tocopy);
 718                         *(__be32 *)&cmd->asiv[12] = cpu_to_be32(0); /* resvd */
 719                         *(__be64 *)&cmd->asiv[16] = cpu_to_be64(flash);
 720                         *(__be32 *)&cmd->asiv[24] = cpu_to_be32(load->uid<<24);
 721                         *(__be32 *)&cmd->asiv[28] = cpu_to_be32(0); /* CRC */
 722
 723                         /* rd/wr */
 724                         cmd->ats = 0x5ULL << 44;
 725                         cmd->asiv_length = 40; /* bytes included in crc calc */
 726                 }
 727                 cmd->asv_length  = 8;
 728
 729                 /* we only get back the calculated CRC */
 730                 *(u64 *)&cmd->asv[0] = 0ULL;    /* 0x80 */
 731
 732                 rc = __genwqe_execute_raw_ddcb(cd, cmd, filp->f_flags);
 733
 734                 load->retc = cmd->retc;
 735                 load->attn = cmd->attn;
 736                 load->progress = cmd->progress;
 737
 738                 if ((rc < 0) && (rc != -EBADMSG)) {
 739                         ddcb_requ_free(cmd);
 740                         goto free_buffer;
 741                 }
 742
 743                 rc = copy_to_user(buf, xbuf, tocopy);
 744                 if (rc) {
 745                         rc = -EFAULT;
 746                         ddcb_requ_free(cmd);
 747                         goto free_buffer;
 748                 }
 749
 750                 /* We know that we can get retc 0x104 with CRC err */
 751                 if (((cmd->retc == DDCB_RETC_FAULT) &&
 752                      (cmd->attn != 0x02)) ||  /* Normally ignore CRC error */
 753                     ((cmd->retc == DDCB_RETC_COMPLETE) &&
 754                      (cmd->attn != 0x00))) {  /* Everything was fine */
 755                         rc = -EIO;
 756                         ddcb_requ_free(cmd);
 757                         goto free_buffer;
 758                 }
 759
 760                 load->size  -= tocopy;
 761                 flash += tocopy;
 762                 buf += tocopy;
 763                 blocks_to_flash--;
 764                 ddcb_requ_free(cmd);
 765         }
 766         rc = 0;
 767
 768  free_buffer:
 769         __genwqe_free_consistent(cd, FLASH_BLOCK, xbuf, dma_addr);
 770         return rc;
 771 }
 772
 773 static int genwqe_pin_mem(struct genwqe_file *cfile, struct genwqe_mem *m)
 774 {
 775         int rc;
 776         struct genwqe_dev *cd = cfile->cd;
 777         struct pci_dev *pci_dev = cfile->cd->pci_dev;
 778         struct dma_mapping *dma_map;
 779         unsigned long map_addr;
 780         unsigned long map_size;
 781
 782         if ((m->addr == 0x0) || (m->size == 0))
 783                 return -EINVAL;
 784
 785         map_addr = (m->addr & PAGE_MASK);
 786         map_size = round_up(m->size + (m->addr & ~PAGE_MASK), PAGE_SIZE);
 787
 788         dma_map = kzalloc(sizeof(struct dma_mapping), GFP_KERNEL);
 789         if (dma_map == NULL)
 790                 return -ENOMEM;
 791
 792         genwqe_mapping_init(dma_map, GENWQE_MAPPING_SGL_PINNED);
 793         rc = genwqe_user_vmap(cd, dma_map, (void *)map_addr, map_size, NULL);
 794         if (rc != 0) {
 795                 dev_err(&pci_dev->dev,
 796                         "[%s] genwqe_user_vmap rc=%d\n", __func__, rc);
 797                 kfree(dma_map);
 798                 return rc;
 799         }
 800
 801         genwqe_add_pin(cfile, dma_map);
 802         return 0;
 803 }
 804
 805 static int genwqe_unpin_mem(struct genwqe_file *cfile, struct genwqe_mem *m)
 806 {
 807         struct genwqe_dev *cd = cfile->cd;
 808         struct dma_mapping *dma_map;
 809         unsigned long map_addr;
 810         unsigned long map_size;
 811
 812         if (m->addr == 0x0)
 813                 return -EINVAL;
 814
 815         map_addr = (m->addr & PAGE_MASK);
 816         map_size = round_up(m->size + (m->addr & ~PAGE_MASK), PAGE_SIZE);
 817
 818         dma_map = genwqe_search_pin(cfile, map_addr, map_size, NULL);
 819         if (dma_map == NULL)
 820                 return -ENOENT;
 821
 822         genwqe_del_pin(cfile, dma_map);
 823         genwqe_user_vunmap(cd, dma_map, NULL);
 824         kfree(dma_map);
 825         return 0;
 826 }
 827
 828 /**
 829  * ddcb_cmd_cleanup() - Remove dynamically created fixup entries
 830  *
 831  * Only if there are any. Pinnings are not removed.
 832  */
 833 static int ddcb_cmd_cleanup(struct genwqe_file *cfile, struct ddcb_requ *req)
 834 {
 835         unsigned int i;
 836         struct dma_mapping *dma_map;
 837         struct genwqe_dev *cd = cfile->cd;
 838
 839         for (i = 0; i < DDCB_FIXUPS; i++) {
 840                 dma_map = &req->dma_mappings[i];
 841
 842                 if (dma_mapping_used(dma_map)) {
 843                         __genwqe_del_mapping(cfile, dma_map);
 844                         genwqe_user_vunmap(cd, dma_map, req);
 845                 }
 846                 if (req->sgls[i].sgl != NULL)
 847                         genwqe_free_sync_sgl(cd, &req->sgls[i]);
 848         }
 849         return 0;
 850 }
 851
 852 /**
 853  * ddcb_cmd_fixups() - Establish DMA fixups/sglists for user memory references
 854  *
 855  * Before the DDCB gets executed we need to handle the fixups. We
 856  * replace the user-space addresses with DMA addresses or do
 857  * additional setup work e.g. generating a scatter-gather list which
 858  * is used to describe the memory referred to in the fixup.
 859  */
 860 static int ddcb_cmd_fixups(struct genwqe_file *cfile, struct ddcb_requ *req)
 861 {
 862         int rc;
 863         unsigned int asiv_offs, i;
 864         struct genwqe_dev *cd = cfile->cd;
 865         struct genwqe_ddcb_cmd *cmd = &req->cmd;
 866         struct dma_mapping *m;
 867         const char *type = "UNKNOWN";
 868
 869         for (i = 0, asiv_offs = 0x00; asiv_offs <= 0x58;
 870              i++, asiv_offs += 0x08) {
 871
 872                 u64 u_addr;
 873                 dma_addr_t d_addr;
 874                 u32 u_size = 0;
 875                 u64 ats_flags;
 876
 877                 ats_flags = ATS_GET_FLAGS(cmd->ats, asiv_offs);
 878
 879                 switch (ats_flags) {
 880
 881                 case ATS_TYPE_DATA:
 882                         break;  /* nothing to do here */
 883
 884                 case ATS_TYPE_FLAT_RDWR:
 885                 case ATS_TYPE_FLAT_RD: {
 886                         u_addr = be64_to_cpu(*((__be64 *)&cmd->
 887                                                asiv[asiv_offs]));
 888                         u_size = be32_to_cpu(*((__be32 *)&cmd->
 889                                                asiv[asiv_offs + 0x08]));
 890
 891                         /*
 892                          * No data available. Ignore u_addr in this
 893                          * case and set addr to 0. Hardware must not
 894                          * fetch the buffer.
 895                          */
 896                         if (u_size == 0x0) {
 897                                 *((__be64 *)&cmd->asiv[asiv_offs]) =
 898                                         cpu_to_be64(0x0);
 899                                 break;
 900                         }
 901
 902                         m = __genwqe_search_mapping(cfile, u_addr, u_size,
 903                                                    &d_addr, NULL);
 904                         if (m == NULL) {
 905                                 rc = -EFAULT;
 906                                 goto err_out;
 907                         }
 908
 909                         *((__be64 *)&cmd->asiv[asiv_offs]) =
 910                                 cpu_to_be64(d_addr);
 911                         break;
 912                 }
 913
 914                 case ATS_TYPE_SGL_RDWR:
 915                 case ATS_TYPE_SGL_RD: {
 916                         int page_offs;
 917
 918                         u_addr = be64_to_cpu(*((__be64 *)
 919                                                &cmd->asiv[asiv_offs]));
 920                         u_size = be32_to_cpu(*((__be32 *)
 921                                                &cmd->asiv[asiv_offs + 0x08]));
 922
 923                         /*
 924                          * No data available. Ignore u_addr in this
 925                          * case and set addr to 0. Hardware must not
 926                          * fetch the empty sgl.
 927                          */
 928                         if (u_size == 0x0) {
 929                                 *((__be64 *)&cmd->asiv[asiv_offs]) =
 930                                         cpu_to_be64(0x0);
 931                                 break;
 932                         }
 933
 934                         m = genwqe_search_pin(cfile, u_addr, u_size, NULL);
 935                         if (m != NULL) {
 936                                 type = "PINNING";
 937                                 page_offs = (u_addr -
 938                                              (u64)m->u_vaddr)/PAGE_SIZE;
 939                         } else {
 940                                 type = "MAPPING";
 941                                 m = &req->dma_mappings[i];
 942
 943                                 genwqe_mapping_init(m,
 944                                                     GENWQE_MAPPING_SGL_TEMP);
 945
 946                                 if (ats_flags == ATS_TYPE_SGL_RD)
 947                                         m->write = 0;
 948
 949                                 rc = genwqe_user_vmap(cd, m, (void *)u_addr,
 950                                                       u_size, req);
 951                                 if (rc != 0)
 952                                         goto err_out;
 953
 954                                 __genwqe_add_mapping(cfile, m);
 955                                 page_offs = 0;
 956                         }
 957
 958                         /* create genwqe style scatter gather list */
 959                         rc = genwqe_alloc_sync_sgl(cd, &req->sgls[i],
 960                                                    (void __user *)u_addr,
 961                                                    u_size, m->write);
 962                         if (rc != 0)
 963                                 goto err_out;
 964
 965                         genwqe_setup_sgl(cd, &req->sgls[i],
 966                                          &m->dma_list[page_offs]);
 967
 968                         *((__be64 *)&cmd->asiv[asiv_offs]) =
 969                                 cpu_to_be64(req->sgls[i].sgl_dma_addr);
 970
 971                         break;
 972                 }
 973                 default:
 974                         rc = -EINVAL;
 975                         goto err_out;
 976                 }
 977         }
 978         return 0;
 979
 980  err_out:
 981         ddcb_cmd_cleanup(cfile, req);
 982         return rc;
 983 }
 984
 985 /**
 986  * genwqe_execute_ddcb() - Execute DDCB using userspace address fixups
 987  *
 988  * The code will build up the translation tables or lookup the
 989  * contignous memory allocation table to find the right translations
 990  * and DMA addresses.
 991  */
 992 static int genwqe_execute_ddcb(struct genwqe_file *cfile,
 993                                struct genwqe_ddcb_cmd *cmd)
 994 {
 995         int rc;
 996         struct genwqe_dev *cd = cfile->cd;
 997         struct file *filp = cfile->filp;
 998         struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
 999
1000         rc = ddcb_cmd_fixups(cfile, req);
1001         if (rc != 0)
1002                 return rc;
1003
1004         rc = __genwqe_execute_raw_ddcb(cd, cmd, filp->f_flags);
1005         ddcb_cmd_cleanup(cfile, req);
1006         return rc;
1007 }
1008
1009 static int do_execute_ddcb(struct genwqe_file *cfile,
1010                            unsigned long arg, int raw)
1011 {
1012         int rc;
1013         struct genwqe_ddcb_cmd *cmd;
1014         struct ddcb_requ *req;
1015         struct genwqe_dev *cd = cfile->cd;
1016         struct file *filp = cfile->filp;
1017
1018         cmd = ddcb_requ_alloc();
1019         if (cmd == NULL)
1020                 return -ENOMEM;
1021
1022         req = container_of(cmd, struct ddcb_requ, cmd);
1023
1024         if (copy_from_user(cmd, (void __user *)arg, sizeof(*cmd))) {
1025                 ddcb_requ_free(cmd);
1026                 return -EFAULT;
1027         }
1028
1029         if (!raw)
1030                 rc = genwqe_execute_ddcb(cfile, cmd);
1031         else
1032                 rc = __genwqe_execute_raw_ddcb(cd, cmd, filp->f_flags);
1033
1034         /* Copy back only the modifed fields. Do not copy ASIV
1035            back since the copy got modified by the driver. */
1036         if (copy_to_user((void __user *)arg, cmd,
1037                          sizeof(*cmd) - DDCB_ASIV_LENGTH)) {
1038                 ddcb_requ_free(cmd);
1039                 return -EFAULT;
1040         }
1041
1042         ddcb_requ_free(cmd);
1043         return rc;
1044 }
1045
1046 /**
1047  * genwqe_ioctl() - IO control
1048  * @filp:       file handle
1049  * @cmd:        command identifier (passed from user)
1050  * @arg:        argument (passed from user)
1051  *
1052  * Return: 0 success
1053  */
1054 static long genwqe_ioctl(struct file *filp, unsigned int cmd,
1055                          unsigned long arg)
1056 {
1057         int rc = 0;
1058         struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;
1059         struct genwqe_dev *cd = cfile->cd;
1060         struct pci_dev *pci_dev = cd->pci_dev;
1061         struct genwqe_reg_io __user *io;
1062         u64 val;
1063         u32 reg_offs;
1064
1065         /* Return -EIO if card hit EEH */
1066         if (pci_channel_offline(pci_dev))
1067                 return -EIO;
1068
1069         if (_IOC_TYPE(cmd) != GENWQE_IOC_CODE)
1070                 return -EINVAL;
1071
1072         switch (cmd) {
1073
1074         case GENWQE_GET_CARD_STATE:
1075                 put_user(cd->card_state, (enum genwqe_card_state __user *)arg);
1076                 return 0;
1077
1078                 /* Register access */
1079         case GENWQE_READ_REG64: {
1080                 io = (struct genwqe_reg_io __user *)arg;
1081
1082                 if (get_user(reg_offs, &io->num))
1083                         return -EFAULT;
1084
1085                 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x7))
1086                         return -EINVAL;
1087
1088                 val = __genwqe_readq(cd, reg_offs);
1089                 put_user(val, &io->val64);
1090                 return 0;
1091         }
1092
1093         case GENWQE_WRITE_REG64: {
1094                 io = (struct genwqe_reg_io __user *)arg;
1095
1096                 if (!capable(CAP_SYS_ADMIN))
1097                         return -EPERM;
1098
1099                 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)
1100                         return -EPERM;
1101
1102                 if (get_user(reg_offs, &io->num))
1103                         return -EFAULT;
1104
1105                 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x7))
1106                         return -EINVAL;
1107
1108                 if (get_user(val, &io->val64))
1109                         return -EFAULT;
1110
1111                 __genwqe_writeq(cd, reg_offs, val);
1112                 return 0;
1113         }
1114
1115         case GENWQE_READ_REG32: {
1116                 io = (struct genwqe_reg_io __user *)arg;
1117
1118                 if (get_user(reg_offs, &io->num))
1119                         return -EFAULT;
1120
1121                 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x3))
1122                         return -EINVAL;
1123
1124                 val = __genwqe_readl(cd, reg_offs);
1125                 put_user(val, &io->val64);
1126                 return 0;
1127         }
1128
1129         case GENWQE_WRITE_REG32: {
1130                 io = (struct genwqe_reg_io __user *)arg;
1131
1132                 if (!capable(CAP_SYS_ADMIN))
1133                         return -EPERM;
1134
1135                 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)
1136                         return -EPERM;
1137
1138                 if (get_user(reg_offs, &io->num))
1139                         return -EFAULT;
1140
1141                 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x3))
1142                         return -EINVAL;
1143
1144                 if (get_user(val, &io->val64))
1145                         return -EFAULT;
1146
1147                 __genwqe_writel(cd, reg_offs, val);
1148                 return 0;
1149         }
1150
1151                 /* Flash update/reading */
1152         case GENWQE_SLU_UPDATE: {
1153                 struct genwqe_bitstream load;
1154
1155                 if (!genwqe_is_privileged(cd))
1156                         return -EPERM;
1157
1158                 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)
1159                         return -EPERM;
1160
1161                 if (copy_from_user(&load, (void __user *)arg,
1162                                    sizeof(load)))
1163                         return -EFAULT;
1164
1165                 rc = do_flash_update(cfile, &load);
1166
1167                 if (copy_to_user((void __user *)arg, &load, sizeof(load)))
1168                         return -EFAULT;
1169
1170                 return rc;
1171         }
1172
1173         case GENWQE_SLU_READ: {
1174                 struct genwqe_bitstream load;
1175
1176                 if (!genwqe_is_privileged(cd))
1177                         return -EPERM;
1178
1179                 if (genwqe_flash_readback_fails(cd))
1180                         return -ENOSPC;  /* known to fail for old versions */
1181
1182                 if (copy_from_user(&load, (void __user *)arg, sizeof(load)))
1183                         return -EFAULT;
1184
1185                 rc = do_flash_read(cfile, &load);
1186
1187                 if (copy_to_user((void __user *)arg, &load, sizeof(load)))
1188                         return -EFAULT;
1189
1190                 return rc;
1191         }
1192
1193                 /* memory pinning and unpinning */
1194         case GENWQE_PIN_MEM: {
1195                 struct genwqe_mem m;
1196
1197                 if (copy_from_user(&m, (void __user *)arg, sizeof(m)))
1198                         return -EFAULT;
1199
1200                 return genwqe_pin_mem(cfile, &m);
1201         }
1202
1203         case GENWQE_UNPIN_MEM: {
1204                 struct genwqe_mem m;
1205
1206                 if (copy_from_user(&m, (void __user *)arg, sizeof(m)))
1207                         return -EFAULT;
1208
1209                 return genwqe_unpin_mem(cfile, &m);
1210         }
1211
1212                 /* launch an DDCB and wait for completion */
1213         case GENWQE_EXECUTE_DDCB:
1214                 return do_execute_ddcb(cfile, arg, 0);
1215
1216         case GENWQE_EXECUTE_RAW_DDCB: {
1217
1218                 if (!capable(CAP_SYS_ADMIN))
1219                         return -EPERM;
1220
1221                 return do_execute_ddcb(cfile, arg, 1);
1222         }
1223
1224         default:
1225                 return -EINVAL;
1226         }
1227
1228         return rc;
1229 }
1230
1231 #if defined(CONFIG_COMPAT)
1232 /**
1233  * genwqe_compat_ioctl() - Compatibility ioctl
1234  *
1235  * Called whenever a 32-bit process running under a 64-bit kernel
1236  * performs an ioctl on /dev/genwqe<n>_card.
1237  *
1238  * @filp:        file pointer.
1239  * @cmd:         command.
1240  * @arg:         user argument.
1241  * Return:       zero on success or negative number on failure.
1242  */
1243 static long genwqe_compat_ioctl(struct file *filp, unsigned int cmd,
1244                                 unsigned long arg)
1245 {
1246         return genwqe_ioctl(filp, cmd, arg);
1247 }
1248 #endif /* defined(CONFIG_COMPAT) */
1249
1250 static const struct file_operations genwqe_fops = {
1251         .owner          = THIS_MODULE,
1252         .open           = genwqe_open,
1253         .fasync         = genwqe_fasync,
1254         .mmap           = genwqe_mmap,
1255         .unlocked_ioctl = genwqe_ioctl,
1256 #if defined(CONFIG_COMPAT)
1257         .compat_ioctl   = genwqe_compat_ioctl,
1258 #endif
1259         .release        = genwqe_release,
1260 };
1261
1262 static int genwqe_device_initialized(struct genwqe_dev *cd)
1263 {
1264         return cd->dev != NULL;
1265 }
1266
1267 /**
1268  * genwqe_device_create() - Create and configure genwqe char device
1269  * @cd:      genwqe device descriptor
1270  *
1271  * This function must be called before we create any more genwqe
1272  * character devices, because it is allocating the major and minor
1273  * number which are supposed to be used by the client drivers.
1274  */
1275 int genwqe_device_create(struct genwqe_dev *cd)
1276 {
1277         int rc;
1278         struct pci_dev *pci_dev = cd->pci_dev;
1279
1280         /*
1281          * Here starts the individual setup per client. It must
1282          * initialize its own cdev data structure with its own fops.
1283          * The appropriate devnum needs to be created. The ranges must
1284          * not overlap.
1285          */
1286         rc = alloc_chrdev_region(&cd->devnum_genwqe, 0,
1287                                  GENWQE_MAX_MINOR, GENWQE_DEVNAME);
1288         if (rc < 0) {
1289                 dev_err(&pci_dev->dev, "err: alloc_chrdev_region failed\n");
1290                 goto err_dev;
1291         }
1292
1293         cdev_init(&cd->cdev_genwqe, &genwqe_fops);
1294         cd->cdev_genwqe.owner = THIS_MODULE;
1295
1296         rc = cdev_add(&cd->cdev_genwqe, cd->devnum_genwqe, 1);
1297         if (rc < 0) {
1298                 dev_err(&pci_dev->dev, "err: cdev_add failed\n");
1299                 goto err_add;
1300         }
1301
1302         /*
1303          * Finally the device in /dev/... must be created. The rule is
1304          * to use card%d_clientname for each created device.
1305          */
1306         cd->dev = device_create_with_groups(cd->class_genwqe,
1307                                             &cd->pci_dev->dev,
1308                                             cd->devnum_genwqe, cd,
1309                                             genwqe_attribute_groups,
1310                                             GENWQE_DEVNAME "%u_card",
1311                                             cd->card_idx);
1312         if (IS_ERR(cd->dev)) {
1313                 rc = PTR_ERR(cd->dev);
1314                 goto err_cdev;
1315         }
1316
1317         rc = genwqe_init_debugfs(cd);
1318         if (rc != 0)
1319                 goto err_debugfs;
1320
1321         return 0;
1322
1323  err_debugfs:
1324         device_destroy(cd->class_genwqe, cd->devnum_genwqe);
1325  err_cdev:
1326         cdev_del(&cd->cdev_genwqe);
1327  err_add:
1328         unregister_chrdev_region(cd->devnum_genwqe, GENWQE_MAX_MINOR);
1329  err_dev:
1330         cd->dev = NULL;
1331         return rc;
1332 }
1333
1334 static int genwqe_inform_and_stop_processes(struct genwqe_dev *cd)
1335 {
1336         int rc;
1337         unsigned int i;
1338         struct pci_dev *pci_dev = cd->pci_dev;
1339
1340         if (!genwqe_open_files(cd))
1341                 return 0;
1342
1343         dev_warn(&pci_dev->dev, "[%s] send SIGIO and wait ...\n", __func__);
1344
1345         rc = genwqe_kill_fasync(cd, SIGIO);
1346         if (rc > 0) {
1347                 /* give kill_timeout seconds to close file descriptors ... */
1348                 for (i = 0; (i < genwqe_kill_timeout) &&
1349                              genwqe_open_files(cd); i++) {
1350                         dev_info(&pci_dev->dev, "  %d sec ...", i);
1351
1352                         cond_resched();
1353                         msleep(1000);
1354                 }
1355
1356                 /* if no open files we can safely continue, else ... */
1357                 if (!genwqe_open_files(cd))
1358                         return 0;
1359
1360                 dev_warn(&pci_dev->dev,
1361                          "[%s] send SIGKILL and wait ...\n", __func__);
1362
1363                 rc = genwqe_force_sig(cd, SIGKILL); /* force terminate */
1364                 if (rc) {
1365                         /* Give kill_timout more seconds to end processes */
1366                         for (i = 0; (i < genwqe_kill_timeout) &&
1367                                      genwqe_open_files(cd); i++) {
1368                                 dev_warn(&pci_dev->dev, "  %d sec ...", i);
1369
1370                                 cond_resched();
1371                                 msleep(1000);
1372                         }
1373                 }
1374         }
1375         return 0;
1376 }
1377
1378 /**
1379  * genwqe_device_remove() - Remove genwqe's char device
1380  *
1381  * This function must be called after the client devices are removed
1382  * because it will free the major/minor number range for the genwqe
1383  * drivers.
1384  *
1385  * This function must be robust enough to be called twice.
1386  */
1387 int genwqe_device_remove(struct genwqe_dev *cd)
1388 {
1389         int rc;
1390         struct pci_dev *pci_dev = cd->pci_dev;
1391
1392         if (!genwqe_device_initialized(cd))
1393                 return 1;
1394
1395         genwqe_inform_and_stop_processes(cd);
1396
1397         /*
1398          * We currently do wait until all filedescriptors are
1399          * closed. This leads to a problem when we abort the
1400          * application which will decrease this reference from
1401          * 1/unused to 0/illegal and not from 2/used 1/empty.
1402          */
1403         rc = kref_read(&cd->cdev_genwqe.kobj.kref);
1404         if (rc != 1) {
1405                 dev_err(&pci_dev->dev,
1406                         "[%s] err: cdev_genwqe...refcount=%d\n", __func__, rc);
1407                 panic("Fatal err: cannot free resources with pending references!");
1408         }
1409
1410         genqwe_exit_debugfs(cd);
1411         device_destroy(cd->class_genwqe, cd->devnum_genwqe);
1412         cdev_del(&cd->cdev_genwqe);
1413         unregister_chrdev_region(cd->devnum_genwqe, GENWQE_MAX_MINOR);
1414         cd->dev = NULL;
1415
1416         return 0;
1417 }